Web scraper for harvesting a full-site

Question

I've written some code using python. My goal was to harvest name, phone, and web address from extracting all the links available in that webpage. The scraper is so far following the instructions nicely and harvesting data as it is supposed to.

The logic I applied here is crystal and easy to maintain. If any flaw is found in it's basic design, It is my incapability I could not shake that off. Hope I'll have any suggestion to improve the design. Thanks a lot for going it through.

The complete code:

import requests
from lxml import html

class HouzzParser:
    main_url = "https://www.houzz.com/professionals/"

    def __init__(self):
        self.keeper = []
        self.session = requests.Session()

    def Scrape(self):
        self.receiver(self.main_url)

    def receiver(self, link):
        # Collecting side-bar links.
        response = self.session.get(link)
        tree = html.fromstring(response.text)

        for item in tree.xpath("//a[@class='sidebar-item-label']"):
            itm_lnks = item.xpath(".//@href")
            for page in itm_lnks:
                self.profile_links(page)     # Sending links to create new hrefs for getting to each profile of first page
                self.pagination_links(page)  # Sending links to process hrefs to the next page

    def pagination_links(self, link):
        # Creating links for the next page
        response = self.session.get(link)
        tree = html.fromstring(response.text)

        for n_url in tree.xpath("//ul[@class='pagination']"):
            items = n_url.xpath(".//a[@class='pageNumber']/@href")
            for item in items:
                self.profile_links(item)

    def profile_links(self, links):
        # Creating links for the profile page
        response = self.session.get(links)
        tree = html.fromstring(response.text)
        for titles in tree.xpath("//div[@class='name-info']"):
            links = titles.xpath(".//a[@class='pro-title']/@href")
            for doc in links:
                self.target_page(doc)

    def target_page(self, urls):
        # Extracting necessary docs from here
        response = self.session.get(urls)
        tree = html.fromstring(response.text)

        for titles in tree.xpath("//div[@class='profile-cover']"):
            name = titles.findtext(".//a[@class='profile-full-name']")
            phone = titles.xpath(".//a[contains(concat(' ', @class, ' '), ' click-to-call-link ')]/@phone")[0] if titles.xpath(".//a[contains(concat(' ', @class, ' '), ' click-to-call-link ')]/@phone") else ""
            web = titles.xpath(".//a[@class='proWebsiteLink']/@href")[0] if titles.xpath(".//a[@class='proWebsiteLink']/@href") else ""
            print(name, phone, web)
            data = name, phone, web
            if data not in self.keeper:
                self.keeper.append(data)

    def __str__(self):
        return self.keeper


if __name__ == '__main__':
    scraper = HouzzParser()
    scraper.Scrape()
    for item in scraper.keeper:
        print(item)

Answer 1

The code is quite clean and understandable - even though I'd work on the following:

• Improve variable naming - for instance, Scrape() should become scrape() - in Python there is a lower_case_with_underscores notation for variable, function and method names.

• You can cut down on the nested loops. For example:

  for item in tree.xpath("//a[@class='sidebar-item-label']"):
      itm_lnks = item.xpath(".//@href")
      for page in itm_lnks:
          self.profile_links(page)     # Sending links to create new hrefs for getting to each profile of first page
          self.pagination_links(page)  # Sending links to process hrefs to the next page
  

  will become:

  for link in tree.xpath("//a[@class='sidebar-item-label']//@href"):
      self.profile_links(link)     # Sending links to create new hrefs for getting to each profile of first page
      self.pagination_links(link)  # Sending links to process hrefs to the next page
  

• I would use the EAFP approach when reading the name and web values - you would at least avoid repeating your XPath expressions this way.
• Convert your method comments into proper documentation strings.

Modified code:

import requests
from lxml import html


class HouzzParser:
    main_url = "https://www.houzz.com/professionals/"

    def __init__(self):
        self.keeper = []
        self.session = requests.Session()

    def scrape(self):
        self.receiver(self.main_url)

    def receiver(self, link):
        """Collecting side-bar links."""
        response = self.session.get(link)
        tree = html.fromstring(response.text)

        for link in tree.xpath("//a[@class='sidebar-item-label']//@href"):
            self.profile_links(link)     # Sending links to create new hrefs for getting to each profile of first page
            self.pagination_links(link)  # Sending links to process hrefs to the next page

    def pagination_links(self, link):
        """Creating links for the next page."""
        response = self.session.get(link)
        tree = html.fromstring(response.text)

        for pagination_link in tree.xpath("//ul[@class='pagination']//a[@class='pageNumber']/@href"):
            self.profile_links(pagination_link)

    def profile_links(self, links):
        """Creating links for the profile page."""
        response = self.session.get(links)
        tree = html.fromstring(response.text)

        for profile_link in tree.xpath("//div[@class='name-info']//a[@class='pro-title']/@href"):
            self.target_page(profile_link)

    def target_page(self, urls):
        """Extracting necessary docs from here."""
        response = self.session.get(urls)
        tree = html.fromstring(response.text)

        profile_info = tree.xpath("//div[@class='profile-cover']")[0]

        name = profile_info.findtext(".//a[@class='profile-full-name']")
        try:
            phone = profile_info.xpath(".//a[contains(concat(' ', @class, ' '), ' click-to-call-link ')]/@phone")[0]
        except IndexError:
            phone = ""

        try:
            web = profile_info.xpath(".//a[@class='proWebsiteLink']/@href")[0]
        except IndexError:
            web = ""

        print(name, phone, web)
        data = name, phone, web
        if data not in self.keeper:
            self.keeper.append(data)

    def __str__(self):
        return self.keeper


if __name__ == '__main__':
    scraper = HouzzParser()
    scraper.scrape()
    for item in scraper.keeper:
        print(item)