1
\$\begingroup\$

I am trying to optimize/improve my coding skills in python

The rationale behind this code is to extract some data from files and store them in a dictionary.

I tried to comments lines to improve readability. I suppose I may use functions or (x for x in list) in place of [x for x in list] at some point to improve performance.

Country=['usa','germany','spain','brazil','russia']#country list
Demofiles=os.listdir('E:/CtryData/capital/')#files locations
Incfiles=os.listdir('E:/CtryData/income/')#files locations

Append MissDataLst list if Country file has empty data:

MissDataLst=[]

for indices,ticker in enumerate(Country):
    if ticker.lower() not in (Demofiles or Incfiles):
        #file does not exist
        continue
    IncDirLocation='C:/CtryData/income/'+ticker[0].lower()
    CapitalDirLocation='C:/CtryData/capital/'+ticker[0].lower()
    if 'There is currently no data for this country' in (open(IncDirLocation).read() or open(CapitalDirLocation).read()):
        #file exist but no data
        MissDataLst.append(indices)

Append a data dictionnary with extracted data from files in the form of:

'country':
    {
    'investments':
            {2009:100,2010:150,2011:200},
    'Debt':
            {2009:1000,2010:1500,2011:2000},
    'gdp':
            {2009:10000,2010:15000,2011:20000}
    }


#empty dictionary to append with data
DataDic={}

for indices,ticker in enumerate(country):
    if indices in MissDataLst:
        continue
    if ticker.lower() not in (Demofiles or Incfiles):
        continue
    IncDirLocation='C:/CtryData/income/'+ticker[0].lower()
    inc_read=BeautifulSoup(open(IncDirLocation).read())
    #find existing tables
    tbl=inc_read.findAll('td',{'class':'td_genTable'})
    #extract tables title
    inc_titles=[x.findPrevious().getText() for x in tbl]
    if 'GDP' not in inc_titles:
        continue
    #replace (456) digit format by -456 format and clean empty cells: re.sub(r',|\$','',x.getText())) and re.sub('^0$','',x)
Gdp=[re.sub('\((.*)\)','-\\1',re.sub(r',|\$','',x.getText())) for x in inc_read.findAll('td',{'class':'td_genTable'})[inc_titles.index('GDP')].findNextSiblings()]
    #convert to integer if not empty
    Gdp=[int(x) for x in gdp if x]
    #################same process as above for debt and investments data#########
    capitalDirLocation='C:/CtryData/capital/'+ticker.lower()
    bs_read=BeautifulSoup(open(capitalDirLocation).read())
    tblbs=bs_read.findAll('td',{'class':'td_genTable'})
    #if no debt or investments data
    if tblbs==[]:
            continue
    bs_titles=[x.findPrevious().getText() for x in tblbs]
    if 'Debt' not in bs_titles:
            continue
    Years=[x.getText().split('/')[-1] for x in inc_read.body.findAll('th') if '/20' in x.getText()]
    Debt=[re.sub('\((.*)\)','-\\1',re.sub(r',|\$','',x.getText())) for x in bs_read.findAll('td',{'class':'td_genTable'})[bs_titles.index('Debt')].findNextSiblings()]
    #convert to integer if not empty
Debt=[int(x) for x in Debt if x]
	#replace (456) format by -456 format and clean empty cells: re.sub(r',|\$','',x.getText())) and re.sub('^0$','',x)
    Investments=[re.sub('\((.*)\)','-\\1',re.sub(r',|\$','',x.getText())) for x in bs_read.findAll('td',{'class':'td_genTable'})[bs_titles.index('Total investments')].findNextSiblings()]
    Investments=[re.sub('^0$','',x) for x in investments ]
    #append DataDic in the form:{'country':{'investments':{2009:100,2010:150,2011:200},'Debt':{2009:1000,2010:1500,2011:2000},'Gdp':{2009:10000,2010:15000,2011:20000}}
    DataDic[ticker]={'investments': {y:Investments[x] for x, y in enumerate(Years)},'Debt':{y:Debt[x] for x, y in enumerate(Years)},'gdp':{y:Gdp[x] for x, y in enumerate(Years)}}
\$\endgroup\$

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Browse other questions tagged or ask your own question.