code (written in several steps for readability):
import csv
from collections import defaultdict
iffrom __name__pprint ==import '__main__':pprint
import timeit
def method1():
rows = [row for row in csv.reader(open('raw_data.csv', 'r'), delimiter=',')]
cols = zip(*rows)
unik = set(cols[0])
positionsindexed = [cols[0].indexdefaultdict(xlist) for x in unik]
results
= [rows[p] for px in positions]unik:
indexes i = [rowscols[0].index(resultx)
for result in results]
indexedindexed[i] = defaultdict(list)rows[i]
for resultreturn inindexed
def resultsmethod2():
rows = [row for indexed[rowsrow in csv.indexreader(resultopen('raw_data.csv', 'r'), delimiter=',')]
cols = resultzip(*rows)
unik = set(cols[0])
print 'positions:'
pprintindexed = defaultdict(positionslist)
printfor '\nresultsx in unik:'
pprint(results)
printi '\nindexes:'
= next(index for index,fid pprintin enumerate(indexescols[0])
if fid == x)
print '\nindexed:'
indexed[i] pprint(dict(indexed))= rows[i]
Output (Python counts from 0, not 1):
positions:
[3, 7, 0, 6, 8]return indexed
resultsdef method3():
[['20110117559572', '4', '40', 'fii'],
['20110117559587',rows '8',= '80',[row 'bor'],
for ['20110117559515',row '1'in csv.reader(open('raw_data.csv', '10''r'), 'faa']delimiter=',')]
['20110117559574', '7', '70', 'foo'],
fIds ['20110117559588',= '9',[row[0] '90',for 'boz']]
indexes:row in rows]
[3, 7, 0, 6, 8]unik = set(fIds)
indexed: = defaultdict(list)
{0: ['20110117559515', '1', '10', 'faa'],
3: ['20110117559572', '4', '40',for 'fii'],
x 6in unik:
['20110117559574', '7', '70', 'foo'],
7: ['20110117559587', '8', '80',i 'bor'],
= 8:next(index ['20110117559588',for '9'index,fid '90',in 'boz']}
Simplified code:
importenumerate(fIds) csvif fid == x)
from collections import defaultdict indexed[i] = rows[i]
from
pprint import pprint return indexed
if __name__ == '__main__':
rowsresults = [rowmethod1() for row in csv.reader(open('raw_data.csv',
'r'), delimiter=', print 'indexed:'
pprint(dict(results)])
cols
= zip(*rows) print '-' * 80
unik
results = setmethod2(cols[0])
indexedprint ='indexed:'
defaultdict pprint(listdict(results))
forprint x'-' in* unik:80
iresults = cols[0].indexmethod3(x)
print 'indexed:'
indexed[i] =pprint(dict(results))
rows[i]
#--- Timeit ---
print 'indexed'method1:', timeit.timeit('method1()', setup="from __main__ import method1", number=10000)
pprintprint 'method2:', timeit.timeit(dict'method2(indexed)', setup="from __main__ import method2", number=10000)
print 'method3:', timeit.timeit('method3()', setup="from __main__ import method3", number=10000)
indexed:
{0: ['20110117559515', '1', '10', 'faa'],
3: ['20110117559572', '4', '40', 'fii'],
6: ['20110117559574', '7', '70', 'foo'],
7: ['20110117559587', '8', '80', 'bor'],
8: ['20110117559588', '9', '90', 'boz']}
Second method:
rows = [row for row--------------------------------------------------------------------------------
indexed:
{0: in['20110117559515', csv.reader(open('raw_data.csv''1', 'r')'10', delimiter=''faa'],')]
3: ['20110117559572', cols'4', ='40', zip(*rows)'fii'],
6: ['20110117559574', unik'7', ='70', set(cols[0])'foo'],
7: ['20110117559587', '8',
'80', 'bor'],
8: ['20110117559588', indexed'9', ='90', defaultdict(list)'boz']}
--------------------------------------------------------------------------------
for x in unikindexed:
{0: ['20110117559515', '1', '10', 'faa'],
3: ['20110117559572', i'4', ='40', next(index'fii'],
for6: index['20110117559574',fid in'7', enumerate(cols[0])'70', if'foo'],
fid7: ==['20110117559587', x)
'8', '80', 'bor'],
8: ['20110117559588', '9', '90', 'boz']}
method1: indexed[i]0.275833129883
method2: =0.367926120758
method3: rows[i]0.362459182739
Is this better in terms of memory usage or performance?