Is there better way to read from a file and connect all data in one big data than to use generators?
At the moment, I do the following:
- use generators to read data from files.
- use NumPy to pack all files in 3D array.
- use pandas to stack it to 2D array so readable for next operations (e.g. plotting).
In my example:
I split reading from file operations generators in 2 modules, and have a third module for reading names (file is not connected with others generators_module). In the last separate file I import that generators_modules, and make arrays with NumPy, then pandas.
In the code of modules I use os.walk
to read from files, regex to read only that data which is needed.
In the code of making arrays with NumPy, pandas:
- I enter variable parameters needed to get data with generator_modules
I pass from generators to array with code:
xdata=np.array( [(float(line['PRESSURE']), float(line['CURVE'])) for line in dt_cols] )
I have four python files, as below:
module1 (gen_enter
):
import re, matplotlib as mpl, matplotlib.pyplot as plt, os, fnmatch
def gen_find(filepat,top):
for path, dirlist, filelist in os.walk(top):
for name in fnmatch.filter(filelist,filepat): yield os.path.join(path,name)
def gen_open(filenames):
for name in filenames:
if name.endswith(".XL"): yield open(name)
else: pass
def gen_get(sources):
for s in sources:
for item in s: yield item
def gen_grep(pattern, fileparse):
for line in fileparse:
inputlines = line[:].strip().replace(';',' ')
if pattern.search(inputlines): yield inputlines
else: pass
def field_map(dictseq,name,func):
for d in dictseq:
d[name] = func(d[name])
yield d
module2 (gen_returnlines
):
from gen_enter import *
def lines_from_dir(filepat, dirname):
findnames = gen_find(filepat, dirname)
openfiles = gen_open(findnames)
getlines = gen_get(openfiles)
#patlines = gen_grep(pattern, getlines)
return getlines
module3 (gen_shownames
):
import re, matplotlib as mpl, matplotlib.pyplot as plt, os, fnmatch
def gen_shownames(filepat,top):
for path, dirlist, filelist in os.walk(top):
for name in fnmatch.filter(filelist,filepat): yield name
The main code:
import numpy as np, pandas as pd, os, fnmatch, re
from pylab import *
from gen_returnlines import *
from gen_shownames import *
def dict_cols(lines):
groups = (patlines.match(line) for line in getlines)
tuples = (group.groups() for group in groups if group)
colnames = ('PRESSURE','CURVE')
line = (dict(zip(colnames,t)) for t in tuples)
line = (field_map(line,"PRESSURE", lambda s: float(s)))
line = (field_map(line,"CURVE",float))
return line
dir='C:\\Users\\REDHOOD\\workspace\\Politechnika_python\\\silniki\\files' #note: all files from ../files/
pattern = re.compile(r'(\d{3}\.\d{1})\D*(\d{3}\.\d{1})\D*')
pats = '(\d{3}\.\d{1})\D*(\d{3}\.\d{1})\D*'
patlines = re.compile(pats)
if __name__ == '__main__':
names = gen_shownames('*', dir)
getlines = lines_from_dir('*',dir)
dt_cols = dict_cols(getlines)
xdata = np.array( [(float(line['PRESSURE']), float(line['CURVE'])) for line in dt_cols] )
xrdata = np.reshape(xdata,(17,360,2))
datapanel = pd.Panel(
xrdata,
items=[k for k in names],
)
datapaneldf = datapanel.to_frame() #pressure na curve -> 360 stopni
read_csv
which will probably be faster (especially in pandas 0.10). \$\endgroup\$