Improving the performance of a webscraper

Question

I have here a modified version of a web scraping code I wrote some weeks back. With some help from this forum, this modified version is faster (at 4secs per iteration) than the earlier version. However, I need to run many iterations (over 1million) which is so much time. Is there any way to further enhance its performance? Thank you.

sample data (data.csv)

Code    Origin
1       Eisenstadt
2       Tirana
3       St Pölten Hbf
6       Wien Westbahnhof
7       Wien Hauptbahnhof
8       Klagenfurt Hbf
9       Villach Hbf
11      Graz Hbf
12      Liezen

Code:

import csv
from functools import wraps
from datetime import datetime, time
import urllib2
from mechanize import Browser
from bs4 import BeautifulSoup, SoupStrainer

# function to group elements of a list
def group(lst, n):
    return zip(*[lst[i::n] for i in range(n)])


# function to convert time string to minutes
def get_min(time_str):
    h, m = time_str.split(':')
    return int(h) * 60 + int(m)


# Delay function incase of network disconnection
def retry(ExceptionToCheck, tries=1000, delay=3, backoff=2, logger=None):

    def deco_retry(f):

        @wraps(f)
        def f_retry(*args, **kwargs):
            mtries, mdelay = tries, delay
            while mtries > 1:
                try:
                    return f(*args, **kwargs)
                except ExceptionToCheck, e:
                    msg = "%s, Retrying in %d seconds..." % (str(e), mdelay)
                    if logger:
                        logger.warning(msg)
                    else:
                        print msg
                    time.sleep(mdelay)
                    mtries -= 1
                    mdelay *= backoff
            return f(*args, **kwargs)

        return f_retry  # true decorator

    return deco_retry


def datareader(datafile):
    """ This function reads the cities data from csv file and processes
        them into an O-D for input into the web scrapper """

    # Read the csv
    with open(datafile, 'r') as f:

        reader = csv.reader(f)
        next(reader, None)
        ListOfCities = [lines for lines in reader]
        temp = ListOfCities[:]

        city_num = []
        city_orig_dest = []
        for i in ListOfCities:
            for j in temp:
                ans1 = i[0], j[0]

                if ans1[0] != ans1[1]:
                    city_num.append(ans1)

                ans = (unicode(i[1], 'iso-8859-1'), unicode(j[1], 'iso-8859-1'), i[0], j[0])
                if ans[0] != ans[1] and ans[2] != ans[3]:
                    city_orig_dest.append(ans)

    yield city_orig_dest

input_data = datareader('data.csv')


def webscrapper(x):

    """ This function scraped the required website and extracts the
        quickest connection time within given time durations """

    #Create a browser object
    br = Browser()

    # Ignore robots.txt
    br.set_handle_robots(False)

    # Google demands a user-agent that isn't a robot
    br.addheaders = [('User-agent', 'Chrome')]

    @retry(urllib2.URLError, tries=1000, delay=3, backoff=2)
    def urlopen_with_retry():
        try:
            # Retrieve the website,
            return br.open('http://fahrplan.sbb.ch/bin/query.exe/en')
        except urllib2.HTTPError, e:
            print e.code
        except urllib2.URLError, e:
            print e.args

    # call the retry function
    urlopen_with_retry()

    # Select the 6th form on the webpage
    br.select_form(nr=6)

    # Assign origin and destination to the o d variables
    o = i[0].encode('iso-8859-1')
    d = i[1].encode('iso-8859-1')
    print 'o-d:', i[0], i[1]

    # Enter the text input (This section should be automated to read multiple text input as shown in the question)
    br.form["REQ0JourneyStopsS0G"] = o  # Origin train station (From)
    br.form["REQ0JourneyStopsZ0G"] = d  # Destination train station (To)
    br.form["REQ0JourneyTime"] = x  # Search Time
    br.form["date"] = '10.05.17'  # Search Date

    # Get the search results
    br.submit()

    connections_times = []
    ListOfSearchTimes = []
    #Click the LATER link a given number of times times to get MORE trip times
    for _ in xrange(3):

        # Read the result of each click and convert to response for beautiful soup formatting
        for l in br.links(text='Later'):
            response = br.follow_link(l)

        # get the response from mechanize Browser
        parse_only = SoupStrainer("table", class_="hfs_overview")
        soup = BeautifulSoup(br.response(), 'lxml', from_encoding="utf-8", parse_only=parse_only)
        trs = soup.select('tr')

        # Scrape the search results from the resulting table
        for tr in trs:
            locations = tr.select('td.location')
            if locations:
                time = tr.select('td.time')[0].contents[0].strip()
                ListOfSearchTimes.append(time.encode('latin-1'))
                durations = tr.select('td.duration')

                # Check that the duration cell is not empty
                if not durations:
                    duration = ''
                else:
                    duration = durations[0].contents[0].strip()

                    # Convert duration time string to minutes
                    connections_times.append(get_min(duration))

    arrivals_and_departure_pair = group(ListOfSearchTimes, 2)

    #Check that the selected departures for one interval occurs before the departure of the next interval
    fmt = '%H:%M'
    finalDepartureList = []

    for idx, res in arrivals_and_departure_pair:

        t1 = datetime.strptime(idx, fmt)

        if x == '05:30':

            control = datetime.strptime('09:00', fmt)

        elif x == '09:00':
            control = datetime.strptime('12:00', fmt)

        elif x == '12:00':
            control = datetime.strptime('15:00', fmt)

        elif x == '15:00':
            control = datetime.strptime('18:00', fmt)

        elif x == '18:00':
            control = datetime.strptime('21:00', fmt)

        else:
            x == '21:00'
            control = datetime.strptime('05:30', fmt)

        if t1 < control:

            finalDepartureList.append(idx)

    # Get the the list of connection times for the departures above
    fastest_connect = connections_times[:len(finalDepartureList)]

    # Return the result of the search
    if not fastest_connect:
        return [i[2], i[3], NO_CONNECTION]
    else:
        return [i[2], i[3], str(min(fastest_connect))]

NO_CONNECTION = '999999'

# List of time intervals
times = ['05:30', '09:00', '12:00', '15:00', '18:00', '21:00']

# Write the heading of the output text file
headings = ["from_BAKCode", "to_BAKCode", "interval", "duration"]
with open("output.txt", "w+") as f:
    f.write(','.join(headings))
    f.write('\n')


if __name__ == "__main__":

    for ind, i in enumerate(input_data.next()):

        print 'index', ind

        for ind, t in enumerate(times):
            result = webscrapper(t)
            result.insert(2, str(ind + 1))
            print 'result:', result
            print

            with open("output.txt", "a") as f:
                f.write(','.join(result[0:4]))
                f.write('\n')

Answer 1

There is a major limitation. Your code is of a blocking nature - you process timetable searches sequentially - one at a time.

I really think you should switch to Scrapy web-scraping framework - it is fast, pluggable and entirely asynchronous. As a bonus point, you will be able to scale your spider to multiple instances or multiple machines. For example, you may divide your input data evenly into N parts and run a spider instance for every part (check out scrapyd).

Here is a sample spider that works for a single timetable search:

import scrapy


TIMES = ['05:30', '09:00', '12:00', '15:00', '18:00', '21:00']
DEFAULT_PARAMS = {
    "changeQueryInputData=yes&start": "Search connection",

    "REQ0Total_KissRideMotorClass": "404",
    "REQ0Total_KissRideCarClass": "5",
    "REQ0Total_KissRide_maxDist": "10000000",
    "REQ0Total_KissRide_minDist": "0",
    "REQComparisonCarload": "0",

    "REQ0JourneyStopsS0A": "255",
    "REQ0JourneyStopsZ0A": "255",
    "REQ0JourneyStops1.0G": "",
    "REQ0JourneyStops1.0A": "1",
    "REQ0JourneyStopover1": ""
}


def merge_two_dicts(x, y):
    """Given two dicts, merge them into a new dict as a shallow copy."""
    z = x.copy()
    z.update(y)
    return z


class FahrplanSpider(scrapy.Spider):
    name = "fahrplan"
    allowed_domains = ["fahrplan.sbb.ch"]

    def start_requests(self):
        params = {
            "REQ0JourneyStopsS0G": "Eisenstadt",
            "REQ0JourneyStopsZ0G": "Tirano, Stazione",
            "date": "27.02.17",
            "REQ0JourneyTime": "17:00"
        }
        formdata = merge_two_dicts(DEFAULT_PARAMS, params)
        yield scrapy.FormRequest("http://fahrplan.sbb.ch/bin/query.exe/en", method="POST", formdata=formdata)

    def parse(self, response):
        for trip_time in response.css("table.hfs_overview tr td.time::text").extract():
            print(trip_time.strip())

If you want to take it further, you should do the following:

• use the datareader() results in the start_requests() method and start a form request for every input item
• define an Item class and yield/return it in the parse() callback
• use an "Item Pipeline" to "pipe" your items into the output file

I understand that there is a lot of new information for you, but doing web-scraping for a long time, I can say that's really worth it, especially performance-wise.