I have here a modified version of a web scraping code I wrote some weeks back. With some help from this forum, this modified version is faster (at 4secs per iteration) than the earlier version. However, I need to run many iterations (over 1million) which is so much time. Is there any way to further enhance its performance? Thank you.

sample data (data.csv)

Code    Origin
1       Eisenstadt
2       Tirana
3       St Pölten Hbf
6       Wien Westbahnhof
7       Wien Hauptbahnhof
8       Klagenfurt Hbf
9       Villach Hbf
11      Graz Hbf
12      Liezen


import csv
from functools import wraps
from datetime import datetime, time
import urllib2
from mechanize import Browser
from bs4 import BeautifulSoup, SoupStrainer

# function to group elements of a list
def group(lst, n):
    return zip(*[lst[i::n] for i in range(n)])

# function to convert time string to minutes
def get_min(time_str):
    h, m = time_str.split(':')
    return int(h) * 60 + int(m)

# Delay function incase of network disconnection
def retry(ExceptionToCheck, tries=1000, delay=3, backoff=2, logger=None):

    def deco_retry(f):

        def f_retry(*args, **kwargs):
            mtries, mdelay = tries, delay
            while mtries > 1:
                    return f(*args, **kwargs)
                except ExceptionToCheck, e:
                    msg = "%s, Retrying in %d seconds..." % (str(e), mdelay)
                    if logger:
                        print msg
                    mtries -= 1
                    mdelay *= backoff
            return f(*args, **kwargs)

        return f_retry  # true decorator

    return deco_retry

def datareader(datafile):
    """ This function reads the cities data from csv file and processes
        them into an O-D for input into the web scrapper """

    # Read the csv
    with open(datafile, 'r') as f:

        reader = csv.reader(f)
        next(reader, None)
        ListOfCities = [lines for lines in reader]
        temp = ListOfCities[:]

        city_num = []
        city_orig_dest = []
        for i in ListOfCities:
            for j in temp:
                ans1 = i[0], j[0]

                if ans1[0] != ans1[1]:

                ans = (unicode(i[1], 'iso-8859-1'), unicode(j[1], 'iso-8859-1'), i[0], j[0])
                if ans[0] != ans[1] and ans[2] != ans[3]:

    yield city_orig_dest

input_data = datareader('data.csv')

def webscrapper(x):

    """ This function scraped the required website and extracts the
        quickest connection time within given time durations """

    #Create a browser object
    br = Browser()

    # Ignore robots.txt

    # Google demands a user-agent that isn't a robot
    br.addheaders = [('User-agent', 'Chrome')]

    @retry(urllib2.URLError, tries=1000, delay=3, backoff=2)
    def urlopen_with_retry():
            # Retrieve the website,
            return br.open('http://fahrplan.sbb.ch/bin/query.exe/en')
        except urllib2.HTTPError, e:
            print e.code
        except urllib2.URLError, e:
            print e.args

    # call the retry function

    # Select the 6th form on the webpage

    # Assign origin and destination to the o d variables
    o = i[0].encode('iso-8859-1')
    d = i[1].encode('iso-8859-1')
    print 'o-d:', i[0], i[1]

    # Enter the text input (This section should be automated to read multiple text input as shown in the question)
    br.form["REQ0JourneyStopsS0G"] = o  # Origin train station (From)
    br.form["REQ0JourneyStopsZ0G"] = d  # Destination train station (To)
    br.form["REQ0JourneyTime"] = x  # Search Time
    br.form["date"] = '10.05.17'  # Search Date

    # Get the search results

    connections_times = []
    ListOfSearchTimes = []
    #Click the LATER link a given number of times times to get MORE trip times
    for _ in xrange(3):

        # Read the result of each click and convert to response for beautiful soup formatting
        for l in br.links(text='Later'):
            response = br.follow_link(l)

        # get the response from mechanize Browser
        parse_only = SoupStrainer("table", class_="hfs_overview")
        soup = BeautifulSoup(br.response(), 'lxml', from_encoding="utf-8", parse_only=parse_only)
        trs = soup.select('tr')

        # Scrape the search results from the resulting table
        for tr in trs:
            locations = tr.select('td.location')
            if locations:
                time = tr.select('td.time')[0].contents[0].strip()
                durations = tr.select('td.duration')

                # Check that the duration cell is not empty
                if not durations:
                    duration = ''
                    duration = durations[0].contents[0].strip()

                    # Convert duration time string to minutes

    arrivals_and_departure_pair = group(ListOfSearchTimes, 2)

    #Check that the selected departures for one interval occurs before the departure of the next interval
    fmt = '%H:%M'
    finalDepartureList = []

    for idx, res in arrivals_and_departure_pair:

        t1 = datetime.strptime(idx, fmt)

        if x == '05:30':

            control = datetime.strptime('09:00', fmt)

        elif x == '09:00':
            control = datetime.strptime('12:00', fmt)

        elif x == '12:00':
            control = datetime.strptime('15:00', fmt)

        elif x == '15:00':
            control = datetime.strptime('18:00', fmt)

        elif x == '18:00':
            control = datetime.strptime('21:00', fmt)

            x == '21:00'
            control = datetime.strptime('05:30', fmt)

        if t1 < control:


    # Get the the list of connection times for the departures above
    fastest_connect = connections_times[:len(finalDepartureList)]

    # Return the result of the search
    if not fastest_connect:
        return [i[2], i[3], NO_CONNECTION]
        return [i[2], i[3], str(min(fastest_connect))]

NO_CONNECTION = '999999'

# List of time intervals
times = ['05:30', '09:00', '12:00', '15:00', '18:00', '21:00']

# Write the heading of the output text file
headings = ["from_BAKCode", "to_BAKCode", "interval", "duration"]
with open("output.txt", "w+") as f:

if __name__ == "__main__":

    for ind, i in enumerate(input_data.next()):

        print 'index', ind

        for ind, t in enumerate(times):
            result = webscrapper(t)
            result.insert(2, str(ind + 1))
            print 'result:', result

            with open("output.txt", "a") as f:

There is a major limitation. Your code is of a blocking nature - you process timetable searches sequentially - one at a time.

I really think you should switch to Scrapy web-scraping framework - it is fast, pluggable and entirely asynchronous. As a bonus point, you will be able to scale your spider to multiple instances or multiple machines. For example, you may divide your input data evenly into N parts and run a spider instance for every part (check out scrapyd).

Here is a sample spider that works for a single timetable search:

import scrapy

TIMES = ['05:30', '09:00', '12:00', '15:00', '18:00', '21:00']
    "changeQueryInputData=yes&start": "Search connection",

    "REQ0Total_KissRideMotorClass": "404",
    "REQ0Total_KissRideCarClass": "5",
    "REQ0Total_KissRide_maxDist": "10000000",
    "REQ0Total_KissRide_minDist": "0",
    "REQComparisonCarload": "0",

    "REQ0JourneyStopsS0A": "255",
    "REQ0JourneyStopsZ0A": "255",
    "REQ0JourneyStops1.0G": "",
    "REQ0JourneyStops1.0A": "1",
    "REQ0JourneyStopover1": ""

def merge_two_dicts(x, y):
    """Given two dicts, merge them into a new dict as a shallow copy."""
    z = x.copy()
    return z

class FahrplanSpider(scrapy.Spider):
    name = "fahrplan"
    allowed_domains = ["fahrplan.sbb.ch"]

    def start_requests(self):
        params = {
            "REQ0JourneyStopsS0G": "Eisenstadt",
            "REQ0JourneyStopsZ0G": "Tirano, Stazione",
            "date": "27.02.17",
            "REQ0JourneyTime": "17:00"
        formdata = merge_two_dicts(DEFAULT_PARAMS, params)
        yield scrapy.FormRequest("http://fahrplan.sbb.ch/bin/query.exe/en", method="POST", formdata=formdata)

    def parse(self, response):
        for trip_time in response.css("table.hfs_overview tr td.time::text").extract():

If you want to take it further, you should do the following:

  • use the datareader() results in the start_requests() method and start a form request for every input item
  • define an Item class and yield/return it in the parse() callback
  • use an "Item Pipeline" to "pipe" your items into the output file

I understand that there is a lot of new information for you, but doing web-scraping for a long time, I can say that's really worth it, especially performance-wise.

  • \$\begingroup\$ @ on that site after the first search, I then have to click a link named LATER to reveal other timetables, is there a way to get that without sequential clicking of the link? I think it speaks to the blocking nature you highlight \$\endgroup\$ – Nobi Mar 13 '17 at 12:59
  • \$\begingroup\$ @Nobi you can still do that with scrapy, by issuing follow-up requests - yielding/returning a scrapy.Request with a different callback..thanks. \$\endgroup\$ – alecxe Mar 13 '17 at 13:05
  • 1
    \$\begingroup\$ instead of your merge_two_dicts function, you could use collections.ChainMap(params, DEFAULT_PARAMS). All keys not found in the first map will be looked-up in the second and so on, down the chain. \$\endgroup\$ – Graipher Mar 14 '17 at 13:44
  • \$\begingroup\$ @Graipher would be a great idea for Python 3! :) Thanks. \$\endgroup\$ – alecxe Mar 14 '17 at 15:45
  • \$\begingroup\$ @alecxe True, did not realize it was Python 3 only :-) \$\endgroup\$ – Graipher Mar 14 '17 at 15:47

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Not the answer you're looking for? Browse other questions tagged or ask your own question.