I have written a crawler in python with the combination of class and function. Few days back I saw a scraper in a tutorial more or less similar to what I did here. I found it hard the necessity of using class here. However, I decided to create one. My scraper is able to traverse all the next pages and print the collected results errorlesly. If there is any suggestion or input to give this scraper a better look, I'm ready to comply with that. Thanks in advance.
Here is what I've written:
import requests
from urllib.request import urljoin
from lxml.html import fromstring
main_link = "https://www.yify-torrent.org/search/western/"
class movie_doc(object):
def __init__(self,name,genre,quality,size,rating):
self.name = name
self.genre = genre
self.quality = quality
self.size = size
self.rating = rating
film_storage = [] # global variable
def get_links(link):
root = fromstring(requests.get(link).text)
for item in root.cssselect(".mv"):
name = item.cssselect("h3 a")[0].text
genre = item.cssselect("li b:contains('Genre:')")[0].tail
quality = item.cssselect("li b:contains('Quality:')")[0].tail
size = item.cssselect("li b:contains('Size:')")[0].tail
rating = item.cssselect("li b:contains('Rating:')")[0].tail
film_details = movie_doc(name,genre,quality,size,rating)
film_storage.append(film_details)
next_page = root.cssselect(".pager a:contains('Next')")[0].attrib['href'] if root.cssselect(".pager a:contains('Next')") else ""
if next_page:
full_link = urljoin(link,next_page)
get_links(full_link)
if __name__ == '__main__':
get_links(main_link)
for item in film_storage: # as film_storage variable is declared global I called here directly
print(item.name,item.genre,item.quality,item.size,item.rating)