I've created a crawler using class. This crawler is able to scrape a certain webpage. Total data out there is 249 and the data are displayed there through different pages. I tried to make it accurately. Here is what I did.
import requests
from lxml import html
class wiseowl:
def __init__(self, start_url):
self.start_url = start_url
self.links = [self.start_url] # a list of links to crawl
self.storage = []
def crawl(self): # calling get_link for every link in self.links
for link in self.links :
self.get_link(link)
def get_link(self,link):
print('Crawling: ' + link)
url = "http://www.wiseowl.co.uk"
response = requests.get(link)
tree = html.fromstring(response.text)
for items in tree.xpath("//p[@class='woVideoListDefaultSeriesTitle']"):
name = items.xpath(".//a/text()")[0]
urls = url + items.xpath(".//a/@href")[0]
docs = name , urls
self.storage.append(docs)
next_page = tree.xpath("//div[contains(concat(' ', @class, ' '), ' woPaging ')]//*[@class='woPagingItem' or @class='woPagingNext']/@href") # get links form 'woPagingItem' or 'woPagingNext' #
for npage in next_page:
if npage and url + npage not in self.links : # avoid getting the same link twice
self.links += [url + npage]
def __str__(self):
return "{}".format(self.storage)
crawler=wiseowl("http://www.wiseowl.co.uk/videos/")
crawler.crawl()
for item in crawler.storage:
print(item)