Web-crawler for yellowpage designed using python

Question

I've written some code to scrape name, address, and phone number from yellowpage using python. This scraper has got input parameter. If the input is properly filled in and the filled in url exists in yellowpage then the scraper will definitely parse the three categories I mentioned earlier. I tried to make it maintaining the guidelines of OOP. It is working specklessly at this moment. Hope there is something to do betterment of this crawler:

import requests
from lxml import html

class YellowPage:
    @classmethod
    def Crawl(self, name, state):   
        page = requests.get("https://www.yellowpages.com/search?search_terms=" + name + "&geo_location_terms=" + state).text
        tree = html.fromstring(page)
        for items in tree.xpath("//div[@class='info']"):
            name = items.findtext(".//span[@itemprop='name']")
            address = items.findtext(".//span[@class='street-address']")
            phone = items.findtext(".//div[@itemprop='telephone']")
            print(name, address, phone)

YellowPage.Crawl("pizza","florida")

Answer 1

• If you've got a single method, there is probably not much sense in having a class and just have a separate function instead. More info at Stop Writing Classes.

• If you do need a classmethod, the first argument to such a method is agreed to be named cls and not self.

• Crawl should be named crawl according to Python PEP8 naming practices.

• Use string formatting instead of string concatenation to make the final URL.

• Or, you can use params to pass GET parameters.

• Use if __name__ == '__main__':.

• I would return from the crawl() method instead of printing.

Modified code:

import requests
from lxml import html


class YellowPage:
    URL_TEMPLATE = "https://www.yellowpages.com/search?search_terms={name}&geo_location_terms={state}"

    @classmethod
    def crawl(cls, name, state):
        page = requests.get(cls.URL_TEMPLATE.format(name=name, state=state)).text
        tree = html.fromstring(page)
        for items in tree.xpath("//div[@class='info']"):
            name = items.findtext(".//span[@itemprop='name']")
            address = items.findtext(".//span[@class='street-address']")
            phone = items.findtext(".//div[@itemprop='telephone']")
            yield (name, address, phone)


if __name__ == '__main__':
    for result in YellowPage.crawl("pizza", "florida"):
        print(result)

Another version (with params):

import requests
from lxml import html


class YellowPage:
    SEARCH_URL = "https://www.yellowpages.com/search"

    @classmethod
    def crawl(cls, name, state):
        page = requests.get(cls.SEARCH_URL, params={'search_terms': name, 'geo_location_terms': state}).text
        tree = html.fromstring(page)
        for items in tree.xpath("//div[@class='info']"):
            name = items.findtext(".//span[@itemprop='name']")
            address = items.findtext(".//span[@class='street-address']")
            phone = items.findtext(".//div[@itemprop='telephone']")
            yield (name, address, phone)


if __name__ == '__main__':
    for result in YellowPage.crawl("pizza", "florida"):
        print(result)

Version without a class:

import requests
from lxml import html


SEARCH_URL = "https://www.yellowpages.com/search"


def crawl(name, state):
    page = requests.get(SEARCH_URL, params={'search_terms': name, 'geo_location_terms': state}).text
    tree = html.fromstring(page)
    for items in tree.xpath("//div[@class='info']"):
        name = items.findtext(".//span[@itemprop='name']")
        address = items.findtext(".//span[@class='street-address']")
        phone = items.findtext(".//div[@itemprop='telephone']")
        yield (name, address, phone)


if __name__ == '__main__':
    for result in crawl("pizza", "florida"):
        print(result)

Answer 2

Alternatively, since Yellow Pages follows the Schema.org item definition specifications, you can make things much more simple if you would use microdata library:

import microdata
import requests


def crawl(name, state):
    page = requests.get("https://www.yellowpages.com/search", params={'search_terms': name, 'geo_location_terms': state}).text

    for item in microdata.get_items(page):
        if item.itemtype[0].string == '//schema.org/Restaurant':
            yield (item.name, item.address, item.telephone)


if __name__ == '__main__':
    for result in crawl("pizza", "florida"):
        print(result)