I recently made a webcrawler that I submitted here for a review:
Minimal webcrawler - bad structure and error handling?
With that help, I've made a much cleaner and better(?) webcrawler.
The only problem I have is that this crawler is extremely slow compared to the other one (13.2s for a depth of 5, compared to 1.3s on the old solution).
Why is it so slow? With cProfile I have gotten this output over the 10 most time consuming functions. With depth 5, 57 links are crawled or scheduled for crawling - while is_valid_link are called 243 times. Could there be some errors in that method?
from bs4 import BeautifulSoup
import httplib
import urlparse
import urllib2
import time
def sanitize(url, parent):
if is_internal_link(url):
if parent.endswith("/") or url.startswith("/"):
return parent + url
else:
return parent + "/" + url
return url
def is_internal_link(url):
if url.startswith("http"):
return False
return True
def hostname(link):
hostname = urlparse.urlparse(link).hostname
return hostname
def get_server_status_code(link):
# http://stackoverflow.com/questions/1140661
host, path = urlparse.urlparse(link)[1:3]
try:
conn = httplib.HTTPConnection(host)
conn.request('HEAD', path)
return conn.getresponse().status
except StandardError:
return None
def is_valid_link(link):
link = sanitize(link, hostname(urlsToCrawl_Parent[0]))
# http://stackoverflow.com/questions/2924422
good_codes = [httplib.OK, httplib.FOUND, httplib.MOVED_PERMANENTLY]
return get_server_status_code(link) in good_codes
def fetch_webpage(url):
webpage = urllib2.urlopen(url)
return webpage
def fetch_links(webpage):
html = webpage.read()
soup = BeautifulSoup(html)
links = soup.find_all("a", href=True)
return links
def schedule_link(link):
if is_valid_link(link):
if hostname(link) in urlsToCrawl_Parent[0]:
urlsToCrawl_Parent.append(link)
else:
urlsToCrawl_Child.append(link)
def havent_visited(link):
if link not in urls_Crawled and link not in urlsToCrawl_Parent and link not in urlsToCrawl_Child:
return True
else:
return False
def crawl(url):
webpage = fetch_webpage(url)
links = fetch_links(webpage)
for link in links:
link = link['href']
#This is not done on a good way at all, should clearly do this in fetch_links?
if havent_visited(link):
schedule_link(link)
if __name__ == "__main__":
urls_Crawled = []
urlsToCrawl_Parent = []
urlsToCrawl_Child = []
urlsToCrawl_Parent.append("STARTLINK")
while (len(urls_Crawled) < 5):
if (len(urlsToCrawl_Parent) > 0):
crawl(urlsToCrawl_Parent[0])
urls_Crawled.append(urlsToCrawl_Parent[0])
urlsToCrawl_Parent.pop(0)
elif (len(urlsToCrawl_Child) > 0):
urlsToCrawl_Parent.append(urlsToCrawl_Child[0])
crawl(urlsToCrawl_Parent[0])
urls_Crawled.append(urlsToCrawl_Parent[0])
urlsToCrawl_Parent.pop(0)