I've written a script to crawl a website recursively until all the links connected to some tutorials are exhausted. It is working smoothly now. There is always room for improvement, though!
import requests
from lxml import html
Page_link="http://www.wiseowl.co.uk/videos/"
visited_links = []
def GrabbingData(url):
base="http://www.wiseowl.co.uk"
visited_links.append(url)
response = requests.get(url)
tree = html.fromstring(response.text)
title = tree.xpath('//p[@class="woVideoListDefaultSeriesTitle"]//a/text()')
link = tree.xpath('//p[@class="woVideoListDefaultSeriesTitle"]//a/@href')
for i,j in zip(title,link):
print(i,j)
pagination=tree.xpath("//div[contains(concat(' ', @class, ' '), ' woPaging ')]//a[@class='woPagingItem' or @class='woPagingNext']/@href")
for nextp in pagination:
url1 = str(base + nextp)
if url1 not in visited_links:
GrabbingData(url1)
GrabbingData(Page_link)