Skip to main content
3 of 4
edited tags
Jamal
  • 34.9k
  • 13
  • 133
  • 237

Optimize the process program

The program below is about finding the values below the percentage and splitting them into five different output files. This works great for small input data files. But when I am trying to run it for a big data it takes ages to compute. Any suggestions on optimizing the program so that I can get fast computation?

#seperate by col 5
import numpy as np
import os
import re
import math
import csv
import sys
import time
import multiprocessing
from scipy import stats

curdir = 'Tools/'
inputdir = 'input/'
filename = 'input'
outputdir = 'output/'

def seperate(filename):
    print "Begin seperating File: %s" % filename
    dtype = np.dtype([('id1',str,12),('id2',str,12),('c1',int),('c2',int),('c3',float)])
    data = np.loadtxt(curdir + inputdir + filename,dtype=dtype)
    col = 'c3'
    col_data = np.array(data[:][col])
    cache = {}
    def get_per(per):
            if(cache.has_key(per)):
                return cache.get(per)
            else:
                r = 100 - stats.percentileofscore(col_data,per)
                cache[per] = r
                return r
    o1 = file(curdir + outputdir+ filename + '_0_20','w')
    o2 = file(curdir + outputdir+ filename + '_20_40','w')
    o3 = file(curdir + outputdir+ filename + '_40_60','w')
    o4 = file(curdir + outputdir+ filename + '_60_80','w')
    o5 = file(curdir + outputdir+ filename + '_80_100','w')
    for line in data:
        per = get_per(line[col])
        output_format = "%s %s %d %d %.1f\n"
        output_data = (line['id1'],line['id2'],line['c1'],line['c2'],line['c3'])
        if per>=0 and per < 20:
            o1.write(output_format % output_data)
        elif per>=20 and per<40:
            o2.write(output_format % output_data)
            pass
        elif per>=40 and per<60:
            o3.write(output_format % output_data)
            pass
        elif per>=60 and per<80:
            o4.write(output_format % output_data)
            pass
        elif per>=80 and per<=100:
            o5.write(output_format % output_data)
            pass
    o1.close()
    o2.close()
    o3.close()
    o4.close()
    o5.close()

    print "Finish seperating File: %s" % filename

 
ps = []
print "Reading Files"
for parent,dirNames,fileNames in os.walk(curdir+inputdir):
    if fileNames:
        #multi_process
        print fileNames
        for fileName in fileNames:
            m = re.match(r'^(?!\.)',fileName)
            if m:
                ps.append(multiprocessing.Process(target=seperate,args=(fileName,)))

#runing by multiple processes
for p in ps:
    p.start()
for p in ps:
    p.join()