In my opinion, Vectorization operation with numpy should be much faster than use for in pure python. I write two function to get and process data in a csv file, one in numpy and another in pure python, but numpy one takes nearly four times time of the other. Why? Is this the "wrong" way to numpy? Any suggestion would be greatly appreciated!
The python code is below, while csv file in rather long, and I put it to enter link description here
The csv file includes some info about an engine, in which, the first column means crankshaft angle in degrees, and the 8th column means (header "PCYL_1") means the first cylinder pressure in bar.
what I want to do:
- get angle-pressure data pairs with only integer angle,
- group the data by angle, and get the max pressure of each angle
- get new angle-max_pressure data pairs
- shift angle range from -360~359 to 0~719
- sort data-pairs by angle
- because angle range must be 0~720, and first pressure equals last pressure, add a [720.0, first angle] to data pairs
- output data pairs to a dat file
my run eviroment is
- python3.6.4 MSC v.1900 32 bit (Intel)
- win8.1 64 bit
I i run ipython in the script file direcotry and input below:
from gen_cylinder_pressure_data_from_csv import *
In [5]: %timeit main_pure_python()
153 ms ± 1.11 ms per loop (mean ± std. dev. of 7 runs, 10 loops each
In [6]: %timeit main_with_numpy()
627 ms ± 3.51 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)
python code is below:
from glob import glob
import numpy
def get_data(filename):
with open(filename, 'r', encoding='utf-8-sig') as f:
headers = None
for line_number, line in enumerate(f.readlines()):
if line_number == 0:
headers = line.strip().split(',')
angle_index = headers.index('曲轴转角')
# cylinder_pressure_indexes = [i for i in range(len(headers)) if headers[i].startswith('PCYL_1')]
cylinder_pressure_indexes = [i for i in range(len(headers)) if headers[i].startswith('PCYL_1')]
elif line_number == 1:
continue
else:
data = line.strip()
if data != '':
datas = data.split(',')
angle = datas[angle_index]
if '.' not in angle:
# cylinder_pressure = max(datas[i] for i in cylinder_pressure_indexes)
cylinder_pressure = datas[cylinder_pressure_indexes[0]]
# if angle == '17':
# print(angle, cylinder_pressure)
yield angle, cylinder_pressure
def write_data(filename):
data_dic = {}
for angle, cylinder_pressure in get_data(filename):
k = int(angle)
v = float(cylinder_pressure)
if k in data_dic:
data_dic[k].append(v)
else:
data_dic[k] = [v]
for k, v in data_dic.items():
# datas_dic[k] = sum(v) / len(v)
data_dic[k] = max(v)
angles = sorted(data_dic.keys())
if angles[-1] - angles[0] != 720:
data_dic[angles[0] + 720] = data_dic[angles[0]]
angles.append(angles[0] + 720)
else:
print(angles[0], angles[-1])
with open('%srpm.dat' % filename[-8:-4], 'w', encoding='utf-8') as f:
for k in angles:
# print('%s,%s\n' % (k,datas_dic[k]))
f.write('%s,%s\n' % (k, data_dic[k]))
def main_with_numpy():
# rather slow than main_pure_python
for filename in glob('Ten*.csv'):
with open(filename, mode='r', encoding='utf-8-sig') as f:
data_array = numpy.loadtxt(f, delimiter=',', usecols=(0, 7), skiprows=2)[::10]
pressure_array = data_array[:, 1]
pressure_array = pressure_array.reshape(720, pressure_array.shape[0] // 720)
pressure_array = numpy.amax(pressure_array, axis=1, keepdims=True)
data_output = numpy.zeros((721, 2), )
data_output[:-1, 0] = data_array[:720, 0]
data_output[:-1, 1] = pressure_array.reshape(720)
data_output[:, 0] = (data_output[:, 0] + 720) % 720
data_output[-1, 0] = 721
data_output = data_output[data_output[:, 0].argsort()]
data_output[-1] = data_output[0]
data_output[-1, 0] = 720.0
with open('%srpm.dat' % filename[-8:-4], 'w', encoding='utf-8') as f:
numpy.savetxt(f, data_output, fmt='%f', delimiter=',')
pass
def main_pure_python():
for filename in glob('Ten*.csv'):
write_data(filename)
pass
if __name__ == '__main__':
main_pure_python()
loadtxtandsavetxtdon't make much use of compilednumpy. They use python file io. Their performance and code has been discussed on SO many times. \$\endgroup\$