I have here a modified version of a web scraping code I wrote some weeks back. With some help from this forum, this modified version is faster (at 4secs per iteration) than the earlier version. However, I need to run many iterations (over 1million) which is so much time. Is there any way to further enhance its performance? Thank you.
sample data (data.csv)
Code Origin
1 Eisenstadt
2 Tirana
3 St Pölten Hbf
6 Wien Westbahnhof
7 Wien Hauptbahnhof
8 Klagenfurt Hbf
9 Villach Hbf
11 Graz Hbf
12 Liezen
Code:
import csv
from functools import wraps
from datetime import datetime, time
import urllib2
from mechanize import Browser
from bs4 import BeautifulSoup, SoupStrainer
# function to group elements of a list
def group(lst, n):
return zip(*[lst[i::n] for i in range(n)])
# function to convert time string to minutes
def get_min(time_str):
h, m = time_str.split(':')
return int(h) * 60 + int(m)
# Delay function incase of network disconnection
def retry(ExceptionToCheck, tries=1000, delay=3, backoff=2, logger=None):
def deco_retry(f):
@wraps(f)
def f_retry(*args, **kwargs):
mtries, mdelay = tries, delay
while mtries > 1:
try:
return f(*args, **kwargs)
except ExceptionToCheck, e:
msg = "%s, Retrying in %d seconds..." % (str(e), mdelay)
if logger:
logger.warning(msg)
else:
print msg
time.sleep(mdelay)
mtries -= 1
mdelay *= backoff
return f(*args, **kwargs)
return f_retry # true decorator
return deco_retry
def datareader(datafile):
""" This function reads the cities data from csv file and processes
them into an O-D for input into the web scrapper """
# Read the csv
with open(datafile, 'r') as f:
reader = csv.reader(f)
next(reader, None)
ListOfCities = [lines for lines in reader]
temp = ListOfCities[:]
city_num = []
city_orig_dest = []
for i in ListOfCities:
for j in temp:
ans1 = i[0], j[0]
if ans1[0] != ans1[1]:
city_num.append(ans1)
ans = (unicode(i[1], 'iso-8859-1'), unicode(j[1], 'iso-8859-1'), i[0], j[0])
if ans[0] != ans[1] and ans[2] != ans[3]:
city_orig_dest.append(ans)
yield city_orig_dest
input_data = datareader('data.csv')
def webscrapper(x):
""" This function scraped the required website and extracts the
quickest connection time within given time durations """
#Create a browser object
br = Browser()
# Ignore robots.txt
br.set_handle_robots(False)
# Google demands a user-agent that isn't a robot
br.addheaders = [('User-agent', 'Chrome')]
@retry(urllib2.URLError, tries=1000, delay=3, backoff=2)
def urlopen_with_retry():
try:
# Retrieve the website,
return br.open('http://fahrplan.sbb.ch/bin/query.exe/en')
except urllib2.HTTPError, e:
print e.code
except urllib2.URLError, e:
print e.args
# call the retry function
urlopen_with_retry()
# Select the 6th form on the webpage
br.select_form(nr=6)
# Assign origin and destination to the o d variables
o = i[0].encode('iso-8859-1')
d = i[1].encode('iso-8859-1')
print 'o-d:', i[0], i[1]
# Enter the text input (This section should be automated to read multiple text input as shown in the question)
br.form["REQ0JourneyStopsS0G"] = o # Origin train station (From)
br.form["REQ0JourneyStopsZ0G"] = d # Destination train station (To)
br.form["REQ0JourneyTime"] = x # Search Time
br.form["date"] = '10.05.17' # Search Date
# Get the search results
br.submit()
connections_times = []
ListOfSearchTimes = []
#Click the LATER link a given number of times times to get MORE trip times
for _ in xrange(3):
# Read the result of each click and convert to response for beautiful soup formatting
for l in br.links(text='Later'):
response = br.follow_link(l)
# get the response from mechanize Browser
parse_only = SoupStrainer("table", class_="hfs_overview")
soup = BeautifulSoup(br.response(), 'lxml', from_encoding="utf-8", parse_only=parse_only)
trs = soup.select('tr')
# Scrape the search results from the resulting table
for tr in trs:
locations = tr.select('td.location')
if locations:
time = tr.select('td.time')[0].contents[0].strip()
ListOfSearchTimes.append(time.encode('latin-1'))
durations = tr.select('td.duration')
# Check that the duration cell is not empty
if not durations:
duration = ''
else:
duration = durations[0].contents[0].strip()
# Convert duration time string to minutes
connections_times.append(get_min(duration))
arrivals_and_departure_pair = group(ListOfSearchTimes, 2)
#Check that the selected departures for one interval occurs before the departure of the next interval
fmt = '%H:%M'
finalDepartureList = []
for idx, res in arrivals_and_departure_pair:
t1 = datetime.strptime(idx, fmt)
if x == '05:30':
control = datetime.strptime('09:00', fmt)
elif x == '09:00':
control = datetime.strptime('12:00', fmt)
elif x == '12:00':
control = datetime.strptime('15:00', fmt)
elif x == '15:00':
control = datetime.strptime('18:00', fmt)
elif x == '18:00':
control = datetime.strptime('21:00', fmt)
else:
x == '21:00'
control = datetime.strptime('05:30', fmt)
if t1 < control:
finalDepartureList.append(idx)
# Get the the list of connection times for the departures above
fastest_connect = connections_times[:len(finalDepartureList)]
# Return the result of the search
if not fastest_connect:
return [i[2], i[3], NO_CONNECTION]
else:
return [i[2], i[3], str(min(fastest_connect))]
NO_CONNECTION = '999999'
# List of time intervals
times = ['05:30', '09:00', '12:00', '15:00', '18:00', '21:00']
# Write the heading of the output text file
headings = ["from_BAKCode", "to_BAKCode", "interval", "duration"]
with open("output.txt", "w+") as f:
f.write(','.join(headings))
f.write('\n')
if __name__ == "__main__":
for ind, i in enumerate(input_data.next()):
print 'index', ind
for ind, t in enumerate(times):
result = webscrapper(t)
result.insert(2, str(ind + 1))
print 'result:', result
print
with open("output.txt", "a") as f:
f.write(','.join(result[0:4]))
f.write('\n')