I've written some code using python to scrape names and phone numbers from a webpage which is working nice at this moment. However, The way I've written my class crawler is unconventional. As I've hardly seen any crawler which have been written the way I did here, I'm little dubious as to whether this style I should keep up. It serves the purpose, though! Expecting reviews to do the betterment of this crawler. Thanks in advance. Here is the script:
import requests
from lxml import html
class YelPage:
base_url = "https://www.yellowpages.com/search?search_terms=pizza&geo_location_terms=Los+Angeles%2C+CA"
url = "https://www.yellowpages.com"
def __init__(self):
self.vault = []
def parse(self):
self.get_urls(self.base_url)
self.process_links(self.base_url)
# Going to the main page for collecting links leading to the target page
def get_urls(self, link):
response = requests.get(link)
tree = html.fromstring(response.text)
for items in tree.xpath("//div[@class='info']"):
item_link = items.xpath(".//a[@class='business-name'][not(@itemprop='name')]/@href")
for page in item_link:
self.target_page(self.url + page)
# Creating links leading to the next page
def process_links(self, link):
response = requests.get(link)
tree = html.fromstring(response.text)
next_page = tree.xpath("//div[@class='pagination']//li/a/@href")
for n_page in next_page:
self.get_urls(self.url + n_page)
# Scraping required data reaching this target page
def target_page(self, urls):
print(urls)
response = requests.get(urls)
tree = html.fromstring(response.text)
for posts in tree.xpath("//*[@id='main-header']"):
name = posts.xpath(".//div[@class='sales-info']/h1/text()")[0] if posts.xpath(".//div[@class='sales-info']/h1/text()") else ""
phone = posts.xpath('.//p[@class="phone"]/text()')[0] if posts.xpath('.//p[@class="phone"]/text()') else ""
records = name, phone
if records not in self.vault:
self.vault.append(records)
def __str__(self):
return self.vault
if __name__ == '__main__':
crawler = YelPage()
crawler.parse()
for item in crawler.vault:
print(item)