I wrote this code to parse an HTML file which is basically a report generated by a software. This file is 40+ MB in size. The code is 100% perfect. The only problem that I am facing is the parsing process is extremely slow. It took nearly 5 minutes just parsing the first report. The script has a "break" statement in it and so it only works on 1st report but it is very slow.
import os
from lxml import html
def main():
with open(os.getcwd()+'/dump.html', "r") as f:
page = f.read()
tree = html.fromstring(page)
reportCount = len(tree.xpath('//div[contains(@class,"onereport")]'))
sheet1, sheet2, sheet3=[], [], []
for i in range(reportCount):
print "Getting in "+str(i+1)+" report..."
device = tree.xpath('//div[@class="onereport report'+str(i+1)+'"]/table[1]//tr/td[contains(text(), "Device")]/following-sibling::td/text()')[0].split("> ")
deviceID = device[len(device)-1]
timespan = tree.xpath('//div[@class="onereport report'+str(i+1)+'"]/table[1]//tr/td[contains(text(), "Time Span")]/following-sibling::td/text()')[0]
uptime = tree.xpath('//div[@class="onereport report'+str(i+1)+'"]/table[1]//tr/td[contains(text(), "Uptime Stats:")]/following-sibling::td[text()="Up:"]/following-sibling::td/text()')[0]
uptimeDU = tree.xpath('//div[@class="onereport report'+str(i+1)+'"]/table[1]//tr/td[contains(text(), "Uptime Stats:")]/following-sibling::td[text()="Up:"]/following-sibling::td/following-sibling::td/span/text()')[0]
uptimeDo = tree.xpath('//div[@class="onereport report'+str(i+1)+'"]/table[1]//tr/td[contains(text(), "Uptime Stats:")]/following-sibling::td[text()="Down:"]/following-sibling::td/text()')[0]
uptimeDoDU = tree.xpath('//div[@class="onereport report'+str(i+1)+'"]/table[1]//tr/td[contains(text(), "Uptime Stats:")]/following-sibling::td[text()="Down:"]/following-sibling::td/following-sibling::td/span/text()')[0]
pingTime = tree.xpath('//div[@class="onereport report'+str(i+1)+'"]/table[2]//tr/td[text()="Ping Time"]/following-sibling::td/text()')[0]
minimim = tree.xpath('//div[@class="onereport report'+str(i+1)+'"]/table[2]//tr/td[text()="Minimum"]/following-sibling::td/text()')[0]
maximum = tree.xpath('//div[@class="onereport report'+str(i+1)+'"]/table[2]//tr/td[text()="Maximum"]/following-sibling::td/text()')[0]
packetLoss = tree.xpath('//div[@class="onereport report'+str(i+1)+'"]/table[2]//tr/td[text()="Packet Loss"]/following-sibling::td/text()')[0]
sheet1.append([deviceID, timespan, uptime, uptimeDU.replace("[", "").replace("]", ""), uptimeDo, uptimeDoDU.replace("[", "").replace("]", ""), pingTime, minimim, maximum, packetLoss])
print "Getting in "+str(i+1)+" report's table1..."
for j in range(len(tree.xpath('//div[@class="onereport report'+str(i+1)+'"]/div[1]/form/table/thead[@class="headerswithlinks"]/following-sibling::tbody/tr'))):
dateTime = tree.xpath('//div[@class="onereport report'+str(i+1)+'"]/div[1]/form/table/thead[@class="headerswithlinks"]/following-sibling::tbody/tr['+str(j+1)+']/td/nobr/text()')[0]
pTime = tree.xpath('//div[@class="onereport report'+str(i+1)+'"]/div[1]/form/table/thead[@class="headerswithlinks"]/following-sibling::tbody/tr['+str(j+1)+']/td/text()')[0]
mini = tree.xpath('//div[@class="onereport report'+str(i+1)+'"]/div[1]/form/table/thead[@class="headerswithlinks"]/following-sibling::tbody/tr['+str(j+1)+']/td/text()')[1]
maxi = tree.xpath('//div[@class="onereport report'+str(i+1)+'"]/div[1]/form/table/thead[@class="headerswithlinks"]/following-sibling::tbody/tr['+str(j+1)+']/td/text()')[2]
pLoss = tree.xpath('//div[@class="onereport report'+str(i+1)+'"]/div[1]/form/table/thead[@class="headerswithlinks"]/following-sibling::tbody/tr['+str(j+1)+']/td/text()')[3]
dTime = tree.xpath('//div[@class="onereport report'+str(i+1)+'"]/div[1]/form/table/thead[@class="headerswithlinks"]/following-sibling::tbody/tr['+str(j+1)+']/td/text()')[4]
coverage = tree.xpath('//div[@class="onereport report'+str(i+1)+'"]/div[1]/form/table/thead[@class="headerswithlinks"]/following-sibling::tbody/tr['+str(j+1)+']/td/text()')[5]
sheet2.append([deviceID, dateTime, pTime, mini, maxi, pLoss, dTime, coverage])
print "Getting in "+str(i+1)+" report's table2..."
for k in range(len(tree.xpath('//div[@class="onereport report'+str(i+1)+'"]/div[2]/form/table/tbody/tr'))):
status = tree.xpath('//div[@class="onereport report'+str(i+1)+'"]/div[2]/form/table/tbody/tr['+str(k+1)+']/td[1]/text()')[0]
dtime = tree.xpath('//div[@class="onereport report'+str(i+1)+'"]/div[2]/form/table/tbody/tr['+str(k+1)+']/td[2]/nobr/text()')[0]
duration = tree.xpath('//div[@class="onereport report'+str(i+1)+'"]/div[2]/form/table/tbody/tr['+str(k+1)+']/td[2]/nobr/span[2]/text()')[0]
sheet3.append([deviceID, status, dtime, duration.replace("=", "").replace("(", "").replace(")", "")])
break
if __name__ == '__main__':
main()