This is my first attempt at a sizeable amount of code in Python and I've made an attempt to standardize a script into a library so that it can be reused.

However, scraper.scraper (as well as scraper.Scraper?) appears in Idle when you import the module.

Some constructive criticism on the code style/layout would be appreciated.

import requests #lib to send requests to url
from bs4 import BeautifulSoup #parse HTML data into python data types
from user_agent import generate_user_agent #user agent spoof
from time import sleep #wait when failing to get url

#Custom Exception that can be caught top level
class ScrapeException(Exception):
    __doc__ = '''Exception will be thrown from this module if a standard error occurs
if __debig__ is true it will contain the BeautifulSoup object from the last scrape
    def __init__(self, scrape_error, soup = None):
        self.scrape_error = scrape_error
        self.soup = soup
    def __str__(self):
        return self.scrape_error

#Retrieve site specified by url, response on success otherwise will throw ScrapeException
def getpage(url, timeout=5, headers=None, retries=10, twait=1):
    __doc__ = 'Download a webpage'
    if not headers:
        headers = {'User-Agent': generate_user_agent(device_type="desktop", os=('mac', 'linux'))}

    attempt = 0
    while(attempt < retries):
            #download page using requests lib
            response = requests.get(url, timeout=timeout, headers=headers)

            if(response.ok): #If download successful return response
                return response
                error =  'Error fetching url"{}", "{}"'.format(url, response.reason)

        except (requests.ConnectTimeout, requests.ReadTimeout, requests.ConnectionError) as e:
            error = 'Timed out fetching url"{}"\n{}'.format(url, e) #Time out

        print('Attempt {} failed, retrying....'.format(attempt))

        attempt += 1

    raise ScrapeException('Failed after {} attempts\n{}'.format(retries, error))

#class for scraping a particular element from a webpage
#For any derived classes, scrapenext() must be defined
#For all instances, [instance of ScrapBase].nxt must point to the next scraper or be Null for last instance
class ScrapeBase:
    #virtual method, returns scraped soup for next stage or return
    def scrape(self, soup):

    def scrapenext(self, soup):
        #storing soup output at each level is wasteful, but useful for detecting when things have gone wrong!
        if __debug__: 
            self.soup = soup
        if self.nxt:
            return self.nxt.scrape(soup) #Scrape next stage
            return soup #No next stage, return output

#class for retrieveing web page using beautifulsoup
class ScrapePage(ScrapeBase):
    #input (url of website to scrape, timeout in seconds (default 5), HTML headers (default destkop linux))
    def __init__(self, url, timeout=5, headers=None, retries=10, twait=1):
        self.url = url
        self.timeout = timeout
        self.headers = headers
        self.twait = twait
        self.retries = retries

    #soup argument inherited ScrapBase but is not used, can set to Null
    def scrape(self, soup):
        print('Scraping page from url "{}"'.format(self.url))

        response = getpage(self.url, timeout=self.timeout, headers=self.headers, retries=self.retries, twait=self.twait)
        return self.scrapenext(BeautifulSoup(response.content, 'html5lib'))

#Find a tag of type [name], can specify attributes in [attrs] 
class ScrapeTag(ScrapeBase):
    def __init__(self, name, attrs={}, getAll=False):
        self.name = name
        self.attrs = attrs
        self.getAll = getAll

    def scrape(self, soup):
        print('Searching for tag type "{}" with attributes "{}"'.format(self.name, self.attrs))

        if self.getAll: #Find all tags with name and specified attributes
            f = soup.findAll
        else: #Find next tag with name and specified attributes
            f = soup.find

        nextsoup = f(self.name, attrs=self.attrs)

        if not nextsoup:
            #Error finding tag or tag not exist, throw error
            raise ScrapeException('Could not find "{}" with attributes {}'.format(self.name, self.attrs), soup)

        return self.scrapenext(nextsoup)

#Retrieve attribute from tag specified by [attr]
class ScrapeHTMLAttribute(ScrapeBase):
    def __init__(self, attr):
        self.attr = attr

    def scrape(self, soup):
        print('Retrieving HTML attribute "{}"'.format(self.attr))

        #Check if attribute exists
        if not soup.has_attr(self.attr):
            raise ScrapeException('Could not find "{}" attribute'.format(self.attr), soup)

        return self.scrapenext(soup.attrs[self.attr])

#Get array element, can either give [i] as an index or slice
class ScrapeArrayElement(ScrapeBase):
    def __init__(self, i):
        assert (type(i) == int) or (type(i) == slice)

        self.i = i
        self.topi = i if type(i)==int else i.stop #if i is int then array index max is i, if slice then i.top

    def scrape(self, soup):
        print('Getting element {}'.format(self.i))

        if len(soup) < self.topi:
            raise ScrapeException('List was big enough to index element\slice {}'.format(self.i), soup)

        return self.scrapenext(soup[self.i])

#Get an attribute from the python object itself, e.g. for tag.contents input 'contents'
class ScrapeSoupAttribute(ScrapeBase):
    def __init__(self, attr):
        self.attr = attr

    def scrape(self, soup):
        print('Retrieving python (soup) attribute "{}"'.format(self.attr))

        if not hasattr(soup, self.attr):
            raise ScrapeException('Python (soup) object has no attribute "{}"'.format(self.attr))

        return self.scrapenext(getattr(soup, self.attr))

#Scrape a full webpage. Pass iterator of derived ScrapeBase instances in order for scraping.
class Scraper:
    __doc__ = '''Class for scraping a web page\n
[scrapers]: Array of scraper types with their corresponding arguments passed as a dictionary
[soup]:     BeautifulSoup object passed to the first element of [scrapers] - not required if
            first element is of type 'page'

type            req     opt                         Description                         
'page'          'url'   timeout=5, headers=None     Fetch a webpage from a given url, [url]
                        retries=10, twait=1         
'tag'           'name'  attrs, getAll               Get first HTML tag <[name]> with
                                                    attributes [attrs]. If getAll is
                                                    true returns array of all matching tags
'HTML attr'     'attr'                              Get a HTML attribute value
'soup attr'     'attr'                              Get a python attribute [attr] of
                                                    BeautifulSoup object, normally
                                                    this is 'contents'
'array'         'i'                                 Get elements array, [i] can be either be
                                                    an integer or slice
    __author__ = 'the_noobie_programmer'

    def __init__(self, scrapes, soup=None):

        #assign soup input for passing to first call of scraper
        self.soup = soup

        #check array passed is not empty        
        if len(scrapes) < 1:
            raise ScrapeException('Need at least 1 scraper to begin!')

        #keeping __debug__ on uses more memory as it stores every scrape object which in turns stores the soup object for processing
        if __debug__:
            self.scrapers = []

        #last scraper has no next
        nxt = None

        #loop from top, i.e. last scraper first
        for args in reversed(scrapes):
            if len(args) < 1:
                raise ScrapeException('Cannot pass an empty scraper!')

            #create scraper object - first argument is name, proceeding args ar arguments passed to constructor
            s = self._createscraper(args)

            #set next scraper to that of previous cycle
            s.nxt = nxt

            #update next scraper to the current object (looping backwards remember)
            nxt = s

            #if __debug__ on keep an instance of each scraper so that any errors in scraping can be easily looked up
            if __debug__:
                self.scrapers.insert(0, s)

        self.firstscraper = s

    #entry point for user - go!
    def scrape(self):
        return self.firstscraper.scrape(self.soup)

    #Create scraper from string name and arguments
    def _createscraper(self, args):
        if 'type' not in args:
            #type is a required argument
            raise ScrapeException('Argument dictionary must contain "type"!')

        #get scraper type - remove type from dictionary
        name = args.pop('type')

        if name not in self._scrapers:
            #scraper type not valid
            valid_names = [nm for nm in self._scrapers.keys()]
            raise ScrapeException('"{}" is not recognised as a valid scraper type\n'
            'If you are only user one scraper be sure to put double square brackets!\n'
            'Valid types are: {}'.format(name, valid_names))

        #lookup in dictionary from name string and pass arguments to constructor 
        return self._scrapers[name](**args) 

    #dictionary from strings to scraper class types
    _scrapers = {
        'page': ScrapePage,
        'tag': ScrapeTag,
        'HTML attr': ScrapeHTMLAttribute,
        'soup attr': ScrapeSoupAttribute,
        'array': ScrapeArrayElement }

The init file import is:

from .scraper import Scraper, ScrapeException, getpage

An example of it in action:

import scraper
{'type':'page', 'url':'http://google.co.uk/search?q="What is my ip"'},
{'type':'tag', 'name':'w-answer-desktop'},
{'type':'tag', 'name':'div'},
{'type':'soup attr', 'attr':'contents'},
{'type':'array', 'i':0}]
google = scraper.Scraper(scrapes)
ip = google.scrape()
  • 1
    \$\begingroup\$ Hey Joel, thanks for your post. I actually watched this on the weekend: youtube.com/watch?v=o9pEzgHorH0 it was quite interesting. Looking at your code, I see many of the points raised. Just a question: def getpage is that meant to be part of class ScrapeException or not? [formatting prob?] \$\endgroup\$ – C. Harley Sep 25 '18 at 6:16

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Browse other questions tagged or ask your own question.