Web crawlers for three image sites

Question

I'm very new to python and only vaguely remember OOP from doing some Java a few years ago so I don't know what the best way to do this is.

I've build a bunch of classes that represent a crawler that scrapes images from a specific website. e.g: for the website stocksnap I have a class StocksnapCrawler

I have 9 of these crawler classes and it's awful, I know there can be a much better way of representing them, they share a lot in common.

here are three of these crawlers:

class MagdeleineCrawler:

    def __init__(self, crawler_db):
        self.current_page = crawler_db.current_page
        self.crawler_db = crawler_db

    def crawl(self):
        current_page = self.current_page
        print("Starting crawl on page " + str(current_page))
        while True:
            print("crawling page " + str(current_page))
            page_response = requests.get(
                'http://magdeleine.co/license/cc0/page/{}/'.format(current_page))
            page_soup = BeautifulSoup(page_response.text)
            image_links = [link["href"]
                           for link in page_soup.find_all('a', {'class': 'photo-link'})]

            for image_link in image_links:
                print("scraping image at " + image_link)
                response = requests.get(image_link)
                image_page_soup = BeautifulSoup(response.text)
                print('getting image source link')
                image_source_link = image_page_soup.find(
                    'a', {'class': 'download'})['href']

                # Get Tags
                print('getting tags')
                ul = image_page_soup.find('ul', {'class': 'tags'})
                if ul:
                    tag_links = ul.find_all('a', {'rel': 'tag'})
                    tag_names = [tag_link.string for tag_link in tag_links]
                    try:
                        tag_names.remove('editor\'s pick')
                    except:
                        pass

                thumbnail_url = image_page_soup.find(
                    'img', {'id': 'main-img'})['src']

                print("storing image in db")
                store_image(
                    image_source_link, image_link, thumbnail_url, 'MG', tag_names)
                self.crawler_db.images_scraped += 1
                self.crawler_db.save()

            current_page += 1
            self.crawler_db.current_page += 1
            self.crawler_db.save()


class FancycraveCrawler:

    def __init__(self, crawler_db):
        self.current_page = crawler_db.current_page
        self.crawler_db = crawler_db

    def crawl(self):
        current_page = self.current_page
        print("Starting crawl on page " + str(current_page))
        while True:
            print("crawling page " + str(current_page))
            page_response = requests.get(
                'http://fancycrave.com/page/{}'.format(current_page))
            page_soup = BeautifulSoup(page_response.text)
            image_articles = page_soup.find_all(
                'article', {'class': 'type-photo'})

            for image_article in image_articles:
                print("scraping image")
                image_source_link = image_article.find(
                    'a', text='Download')['href']
                image_link = image_article.find(
                    'input', {'class': 'short-url-field'})["value"]

                # Get Tags
                tag_links = image_article.find(
                    'div', {'class': 'tags'}).find_all('a')
                tag_names = [tag_link.string[1:] for tag_link in tag_links]

                thumbnail_url = image_article.find(
                    'div', {'class': 'photo-data'}).find('img')['src']

                print("storing image in db")
                store_image(
                    image_source_link, image_link, thumbnail_url, 'FC', tag_names)
                self.crawler_db.images_scraped += 1
                self.crawler_db.save()

            current_page += 1
            self.crawler_db.current_page += 1
            self.crawler_db.save()


class StocksnapCrawler:
    def __init__(self, crawler_db):
        self.current_page = crawler_db.current_page
        self.crawler_db = crawler_db

    def crawl(self):
        current_page = self.current_page
        print("Starting crawl on page " + str(current_page))
        while True:
            print("crawling page " + str(current_page))
            page_response = requests.get(
                'https://stocksnap.io/view-photos/sort/date/desc/page-{}'.format(current_page))
            page_soup = BeautifulSoup(page_response.text)
            image_links = ['https://stocksnap.io' + link['href']
                           for link in page_soup.find_all('a', {'class': 'photo-link'})]

            for image_link in image_links:
                print("scraping image at " + image_link)
                response = requests.get(image_link)
                image_page_soup = BeautifulSoup(response.text)
                print('getting image source link')
                image_source_link = image_page_soup.find(
                    'img', {'class': 'img-photo'})['src']

                # Get Tags
                print('getting tags')
                table = image_page_soup.find('table', {'class': 'img-details'})
                if table:
                    tag_links = table.find_all('a')
                    tag_names = [tag_link.string for tag_link in tag_links if tag_link.string is not None]

                thumbnail_url = image_source_link

                print("storing image in db")
                store_image(
                    image_source_link, image_link, thumbnail_url, 'SS', tag_names)
                self.crawler_db.images_scraped += 1
                self.crawler_db.save()

            current_page += 1
            self.crawler_db.current_page += 1
            self.crawler_db.save()

basically each class has the same two properties and a crawl method. the crawl method follows a pretty standard structure with some variations depending on the html layout for the website it's scraping.

pseudocode:

current_page - page number that it is currently scraping

image_links - a list of links each images unique page on the website

image_source_link - a direct url to the highest quality version of the image

thumbnail_url - a url to a smaller version of the image to be downloaded and turned into a thumbnail

tags - a list of strings, each string is a tag associated with the image

origin - a two letter uppercase string to represent the origin website of the image, e.g: "SS" for the website stocksnap

while True:
    response = get_page(current_page)
    page_soup = BeautifulSoup(response)

    image_links = get_image_links(page_soup)

    for link in image_links:
        response = response = requests.get(link)
        page_soup = BeautifulSoup(response)
        image_source_link = get_image_source_link(page_soup)
        thumbnail_url = get_thumbnail_url(page_soup)
        tags = get_tags(page_soup)
        store_image(image_source_link, image_link, thumbnail_url, origin , tags)

current_page+=1

Each website would have to implement these methods differently

Any ideas?

Answer 1

Your first problem is that the crawl function contains almost all the code. One large function is harder to reuse, read and make changes to. If you break it up things will get a lot easier.

Think about each function as a task. You want them to do one thing each. For example, you could get all the image links as one function:

def get_links(self, page):
    page_response = requests.get(
        'http://magdeleine.co/license/cc0/page/{}/'.format(current_page))
    page_soup = BeautifulSoup(page_response.text)
    return [link["href"] for link in
                page_soup.find_all('a', {'class': 'photo-link'})]

But notice, you have almost the same process for all three classes shown here. The difference is the URL base that you're requesting from and the 'class' attribute. But those should both be attributes of the class. Then you could rewrite the function like this:

def get_links(self, page):
    page_response = requests.get(self.base_url.format(current_page))
    page_soup = BeautifulSoup(page_response.text)
    return [link["href"] for link in
                page_soup.find_all('a', {'class': self.image_class})]

You understand the program and can likely form better names, but now this could be the same function in all three cases.

Similarly you could turn other parts into functions too:

def scrape_image(self, image_link, image_class, html_tag):
    print("scraping image at " + image_link)
    response = requests.get(image_link)
    image_page_soup = BeautifulSoup(response.text)
    print('getting image source link')
    image_source_link = image_page_soup.find(
        'a', {'class': image_class})[html_tag]
    return image_page_soup

You can then call this with individual Crawler's attributes, like this:

def crawl(self):

    ...

    self.scrape_image(image_link, self.scrape_image_class,
                      self.scrape_html_tag)

Your get_tags function is trickier, as there are entirely difference commands in different cases. But in this case you could overwrite the function instead. Did you learn about inheritance when you did OOP before? Inheritance is basically when one class takes the attributes of another and then adds to them. So in this case, perhaps you have a Crawler class that has the common attributes as well as common similar functions like scrape_image, but then you add on new features for that class, here's a quick template of how the Crawler might look:

class Crawler():
    def __init__():
    def crawl():
    def scrape_image():

And now you make the MagdeleineCrawler. To inherit from Crawler just put it in brackets with the class definition.

class MagdeleineCrawler(Crawler):
    def __init__():
        Crawler.__init__(self) # Pass parameters to Crawler in here
    def get_tags():

This way you can do a mix of common similar functions that get passed attributes as well as defining individual functions for each class.