I've written some code using python having tried to comply with OOP design for the purpose of collecting name of the app, price and developer name from i-tune site by going deeper until there is a limit [which can be toggled within my script] to stop as it has millions of links to go. It is working impeccably at this moment. I tried my best to make this scraper pythonic. However, suggestions to bring about any change to make this crawler more robust will be highly appreciable. Thanks in advance.
from lxml import html import requests class app_scraper: start_url = "https://itunes.apple.com/us/app/candy-crush-saga/id553834731?mt=8" def __init__(self): self.links = [self.start_url] self.vault =  def crawler(self): for url in self.links: self.get_app(url) def get_app(self, url): print("Scraping now "+ url) page = requests.get(url) tree = html.fromstring(page.text) name = tree.xpath('//h1[@itemprop="name"]/text()') developer = tree.xpath('//div[@class="left"]/h2/text()') price = tree.xpath('//div[@itemprop="price"]/text()') item = processor(name, developer, price) self.vault.append(item) p_links = tree.xpath('//div[@class="lockup-info"]//li/a[@class="name"]/@href') for p_link in p_links: if not len(self.links)>=5: # It'll stop crawling until it meets the limit whatever it is self.links += [p_link] class processor: def __init__(self, name, developer, price): self.name = name self.developer = developer self.price = price def __str__(self): return (self.name+" "+self.price+" "+self.developer) crawl = app_scraper() crawl.crawler() for info in crawl.vault: print(info)