Web scraper for parsing names and email addresses from Yellowpage

Question

I've written a script using python to parse the names and email addresses of different pizza shops in USA. I am very new in writing classes using python so I'm not very sure I didn't do anything wrong with it's design.

However, it serves the purpose scraping the required fields. Firstly, it scrapes all the links of different pizza shops then parses the links to the next page and finally tracking down the links of each pizza shops to it's main page to harvest the documents from there. I would be very happy if I could learn the way I could improve my class crawler.

Here is what I have tried with:

import requests

from lxml import html

class YellowPage:
    main_url = "https://www.yellowpages.com/search?search_terms=pizza&geo_location_terms=Los+Angeles%2C+CA"

    def __init__(self):
        self.links = [self.main_url]
        self.storage = []

    def crawl(self):           

        for link in self.links : 
            self.get_link(link)

    def get_link(self, link):

        print('Scraping Now: ' + link)
        url = "https://www.yellowpages.com"
        response = requests.get(link)
        tree = html.fromstring(response.text)

        #scraping links of each pizza shop
        for items in tree.xpath("//div[@class='info']"):
            link_page = items.xpath(".//a[@class='business-name'][not(@itemprop='name')]/@href")
            for page in link_page:
                if page and url + page not in self.links:   
                    self.links += [url + page]

        #parsing the links to the next page            
        next_page = tree.xpath("//div[@class='pagination']//li/a/@href")    
        for nepage in next_page:
            if nepage and url + nepage not in self.links:   
                self.links += [url + nepage]

        #going to the main page of each pizza shop and harvest the record       
        for posts in tree.xpath("//*[@id='main-header']"):
            name = posts.xpath(".//div[@class='sales-info']/h1/text()")[0] if posts.xpath(".//div[@class='sales-info']/h1/text()") else ""
            email = posts.xpath(".//a[@class='email-business']/@href")[0] if posts.xpath(".//a[@class='email-business']/@href") else ""
            records = name, email
            self.storage.append(records)


    def __str__(self):
        return self.storage


crawler = YellowPage()
crawler.crawl()
for item in crawler.storage:
    print(item)

Btw, when I write the above code using function then i find that very speedy.

Answer 1

Here is the summary of the applied improvements:

• keeping a visited set of links - \$O(1)\$ lookups
• using collections.deque() to keep the queue of the next links to scrape - popping left from it is \$O(1)\$
• reusing the same requests.Session() instance, keeping a session instance as an instance variable of your crawler class
• using urljoin() to join the base url and the relative url
• collecting both pizza shop links and the next page links into a single list and process that list once
• removing the loop when extracting a pizza shop profile page information
• put the main execution logic to under the if __name__ == '__main__':

Improved code:

from collections import deque
from urllib.parse import urljoin

import requests
from lxml import html


class YellowPage:
    base_url = "https://www.yellowpages.com"
    main_url = "https://www.yellowpages.com/search?search_terms=pizza&geo_location_terms=Los+Angeles%2C+CA"

    def __init__(self):
        self.session = requests.Session()
        self.session.headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36'}

        self.visited = set([])
        self.links = deque([self.main_url])

        self.storage = []

    def crawl(self):
        while self.links:
            link = self.links.popleft()
            self.scrape(link)

    def scrape(self, link):
        if link in self.visited:
            return

        self.visited.add(link)
        print('Scraping Now: ' + link)

        response = self.session.get(link)
        tree = html.fromstring(response.text)

        # extracting the follow up links
        pizza_shop_links = tree.xpath("//div[@class='info']//a[@class='business-name'][not(@itemprop='name')]/@href")
        next_page_links = tree.xpath("//div[@class='pagination']//li/a/@href")
        links = pizza_shop_links + next_page_links

        for link in links:
            link = urljoin(self.base_url, link)
            if link not in self.visited:
                self.links.append(link)

        # going to the main page of each pizza shop and harvest the record
        is_pizza_shop_page = tree.xpath("//*[@id='main-header']/article")
        if is_pizza_shop_page:
            name = tree.findtext(".//div[@class='sales-info']/h1")
            try:
                email = tree.xpath(".//a[@class='email-business']/@href")[0]
            except IndexError:
                email = ""
            self.storage.append((name, email))

    def __str__(self):
        return self.storage

if __name__ == '__main__':
    crawler = YellowPage()
    crawler.crawl()
    for item in crawler.storage:
        print(item)

(not tested)