If you've got a single method, there is probably not much sense in having a class and just have a separate function instead. More info at Stop Writing Classes.
If you do need a classmethod
, the first argument to such a method is agreed to be named cls
and not self
.
Crawl
should be named crawl
according to Python PEP8 naming practices.
Use string formatting instead of string concatenation to make the final URL.
Or, you can use params
to pass GET parameters.
Use if __name__ == '__main__':
.
- I would return from the
crawl()
method instead of printing.
Modified code:
import requests
from lxml import html
class YellowPage:
URL_TEMPLATE = "https://www.yellowpages.com/search?search_terms={name}&geo_location_terms={state}"
@classmethod
def crawl(cls, name, state):
page = requests.get(cls.URL_TEMPLATE.format(name=name, state=state)).text
tree = html.fromstring(page)
for items in tree.xpath("//div[@class='info']"):
name = items.findtext(".//span[@itemprop='name']")
address = items.findtext(".//span[@class='street-address']")
phone = items.findtext(".//div[@itemprop='telephone']")
yield (name, address, phone)
if __name__ == '__main__':
for result in YellowPage.crawl("pizza", "florida"):
print(result)
Another version (with params
):
import requests
from lxml import html
class YellowPage:
SEARCH_URL = "https://www.yellowpages.com/search"
@classmethod
def crawl(cls, name, state):
page = requests.get(cls.SEARCH_URL, params={'search_terms': name, 'geo_location_terms': state}).text
tree = html.fromstring(page)
for items in tree.xpath("//div[@class='info']"):
name = items.findtext(".//span[@itemprop='name']")
address = items.findtext(".//span[@class='street-address']")
phone = items.findtext(".//div[@itemprop='telephone']")
yield (name, address, phone)
if __name__ == '__main__':
for result in YellowPage.crawl("pizza", "florida"):
print(result)
Version without a class:
import requests
from lxml import html
SEARCH_URL = "https://www.yellowpages.com/search"
def crawl(name, state):
page = requests.get(SEARCH_URL, params={'search_terms': name, 'geo_location_terms': state}).text
tree = html.fromstring(page)
for items in tree.xpath("//div[@class='info']"):
name = items.findtext(".//span[@itemprop='name']")
address = items.findtext(".//span[@class='street-address']")
phone = items.findtext(".//div[@itemprop='telephone']")
yield (name, address, phone)
if __name__ == '__main__':
for result in crawl("pizza", "florida"):
print(result)