A very small crawler created using class

Question

I've created a crawler using class. This crawler is able to scrape a certain webpage. Total data out there is 249 and the data are displayed there through different pages. I tried to make it accurately. Here is what I did.

import requests
from lxml import html

class wiseowl:

    def __init__(self, start_url):
        self.start_url = start_url
        self.links = [self.start_url]    # a list of links to crawl
        self.storage = []

    def crawl(self):           # calling get_link for every link in self.links 
        for link in self.links : 
            self.get_link(link)

    def get_link(self,link):
        print('Crawling: ' + link)
        url = "http://www.wiseowl.co.uk"
        response = requests.get(link)
        tree = html.fromstring(response.text)
        for items in tree.xpath("//p[@class='woVideoListDefaultSeriesTitle']"):
            name = items.xpath(".//a/text()")[0]
            urls = url + items.xpath(".//a/@href")[0]
            docs = name , urls
            self.storage.append(docs)
            next_page = tree.xpath("//div[contains(concat(' ', @class, ' '), ' woPaging ')]//*[@class='woPagingItem' or @class='woPagingNext']/@href")    # get links form 'woPagingItem' or 'woPagingNext' # 
            for npage in next_page:
                if npage and url + npage not in self.links :    # avoid getting the same link twice 
                    self.links += [url + npage]

    def __str__(self):
        return "{}".format(self.storage)

crawler=wiseowl("http://www.wiseowl.co.uk/videos/")
crawler.crawl()
for item in crawler.storage:
    print(item)

Answer 1

General

• Imports should be grouped, and groups should be separated by a single blank line.¹

• Class names should use CamelCase.²

• There shouldn't be a blank line following the class signature.

• Top-level function and class definitions should be separated by two blank lines.³ You already correctly separate method definitions by a single blank line. :)

• Assignment operators should be separated by whitespace.⁴

• In wiseowl, no method ever needs access to self.start_url (the only exception being __init__(), of course). You might as well get rid of it.

• If you want to cast an object to str, just pass the object to the str constructor:

  str_obj = str(obj)
  

Rewrite

I've removed redundant code and improved code style.

from lxml import html

import requests


class WiseOwl:
    def __init__(self, start_url):
        self.links = [start_url]
        self.storage = []

    def crawl(self):           
        # Calling get_link for every link in self.links 
        for link in self.links : 
            self.get_link(link)

    def get_link(self, link):
        print('Crawling: ' + link)
        url = "http://www.wiseowl.co.uk"
        response = requests.get(link)
        tree = html.fromstring(response.text)

        for items in tree.xpath("//p[@class='woVideoListDefaultSeriesTitle']"):
            name = items.xpath(".//a/text()")[0]
            urls = url + items.xpath(".//a/@href")[0]
            docs = name , urls
            self.storage.append(docs)
            next_page = tree.xpath("//div[contains(concat(' ', @class, ' '), ' 
woPaging ')]//*[@class='woPagingItem' or @class='woPagingNext']/@href")    

            for npage in next_page:
                if npage and url + npage not in self.links:   
                # Avoid getting the same link twice 
                    self.links += [url + npage]

    def __str__(self):
        return str(self.storage)

crawler = WiseOwl("http://www.wiseowl.co.uk/videos/")
crawler.crawl()
for item in crawler.storage:
    print(item)

References

1 PEP-8: Imports

2 PEP-8: Naming Conventions: Descriptive: Naming Styles

3 PEP-8: Blank Lines

4 PEP-8: Other Recommendations