# Reaching the philosophy wiki page - Follow Up

This is a follow up to my original post:

I've written a class that will start from a random Wikipedia page, then choose the first link in the main body, and then navigate following the links until it finds the Philosophy page. When I run the testCrawler() method, it crawls starting from 20 pages and then plots the lengths of all of the paths. This generally works but I just want to confirm that the code looks clean/intuitive.

I implemented the vast majority of the suggestions and came up with the following code. I just wanted to make sure that everything looks fine stylistically.

import sys
from urlparse import urljoin

import requests
from lxml.html import fromstring
from bs4 import BeautifulSoup,NavigableString, Tag
import matplotlib.pyplot as plt
import scipy
import scipy.stats

sys.setdefaultencoding('utf-8')

class Crawler:
""" Class used to crawl wikipedia pages starting from a random article.
"""
def __init__(self):
self.base_url = "https://en.wikipedia.org"
self.NUM_PAGES_TO_CRAWL = 500

"""Takes an html response and returns the first link in the main body of the article.
"""
curr_root = BeautifulSoup(curr_response.text,"lxml")
first = curr_root.select_one("#mw-content-text") # locate main body
if not first:
return None
par = first.find_all("p",recursive = False,limit = 10)
first_paragraph_found = False

# Find which paragraph has the first link
i = 0
for i in range(len(par)):
if par[i].b is not None:
bold = ""
for string in par[i].find_all("b"):
bold += " " + string.text
bold = reformat_string('(', bold)
bold_tokens = tokenize(bold)
first_paragraph_found = True
break
if par[i].a is not None:
anchor = par[i].a.text
if anchor:
anchor = reformat_string('(', anchor)
a_tokens = tokenize(anchor)
break
if first_paragraph_found and par[i].a:
break
i += 1

# if none of the paragraphs have a link and article contains only a list
if i >= len(par)-1 and first_paragraph_found:
u_list = first.find_all('ul')
try:
return u_list[0].li.a['href']
except (IndexError, AttributeError,TypeError):
return None
elif i >= len(par)-1:# Reached article with no main body
return None

main_body_idx = i
stack = []
# Find the first link before or after parentheses
for child in par[main_body_idx].children:
if isinstance(child,NavigableString):
if "(" in child:
stack.append("(")
if ")" in child:
try:
stack.pop()
except IndexError: # html malformed
return None

if isinstance(child, Tag) and child.name == "a" and not stack:
try:
except KeyError: # Reached article with no main body
return None

def crawl_to_philosophy(self, start_url,session):
"""Follow the path of each url until the philosophy page is reached and return the path.
"""
try:
init_response = session.get(start_url)
return None

return None

i = 0
while True:
break
try:
return None

return None
return None
i += 1

def find_paths_to_philosophy(self,url):
"""Find paths starting from 500 links.
"""
i = 0
crawl_list = []
with requests.Session() as s:
while i < self.NUM_PAGES_TO_CRAWL:
path = self.crawl_to_philosophy(url,s)
if path is not None:
crawl_list.append(len(path))
i += 1
plot_lengths(crawl_list)

def plot_lengths(lens):
"""Plot the distribution of path lengths.
"""
freq = {}
max_len = 0

for length in lens:
max_len = max(length,max_len)
if length in freq:
freq[length] += 1
else:
freq[length] = 1

max_freq = max(freq, key=freq.get)
bins = range(0, max_len + 1, 2)
plt.hist(lens,bins,histtype = 'bar',rwidth = 0.8)
plt.xlabel('x')
plt.ylabel('Path Lengths')
plt.title('Distribution of path lengths')
dist_names = ['gamma', 'beta', 'rayleigh', 'norm', 'pareto']

for dist_name in dist_names:
dist = getattr(scipy.stats, dist_name)
param = dist.fit(lens)
pdf_fitted = dist.pdf(bins, *param[:-2], loc=param[-2], scale=param[-1]) * max_len
plt.plot(pdf_fitted, label=dist_name)

plt.xlim(0,max_len)
plt.ylim(0,max_freq)
plt.legend(loc='upper right')
plt.show()

# Utility functions used by Crawler class

def reformat_string(char, word):
"""Remove passed in char from a string and convert its characters to lowercase.
"""
word = word.lower()
char_idx = word.find(char)
if char_idx != -1:
return word[:char_idx]
return word

"""Determine whether or not any part of the article heading is in the string and vice versa.
"""
for i in range(len(string)):
return True
return False

def tokenize(word):
"""Split the passed in 'word' on space characters and return a list of tokens.
"""
tokens = []
curr_word = ""
for i in range(len(word)):
if word[i] == " " and i == len(word)-1:
tokens.append(word.strip(" "))
curr_word += word[i]
if word[i] == " " :
tokens.append(curr_word)
curr_word = ""
i+=1
if i == len(word)-1:
tokens.append(curr_word)

if __name__ == "__main__":
url = "https://en.wikipedia.org/wiki/Special:Random"
crawler = Crawler()
crawler.find_paths_to_philosophy(url)

• All crawlers should obey the robots.txt protocol. Wikipedia's version is here file. You are going to piss somebody off otherwise. – Martin York Apr 3 '17 at 4:25

Styling

Take an html response....

Takes an html response

Also, you should think about using one line dosctrings, I don't see much of reasons why do you do the multiline in your code.

Two spaces should be used for inline comments, so this:

except KeyError: # Reached article with no main body


should become:

except KeyError:  # Reached article with no main body

1. Catching exceptions:

If you are not going to work with an exception that you don't have to store in in the variable, e.g:

except requests.exceptions.RequestException as e:  # bad link
return None


So just catch it like this:

except requests.exceptions.RequestException:
return None


There are also other issues, like missing spaces, newlines, so I suggest you install flake8 and run it against your project to find out all the issues, I also suggest you install plugins such as flake8-import-order and flake8-docstrings

Code improvements

Class Crawler

def __init__(self):
self.base_url = "https://en.wikipedia.org"
self.NUM_PAGES_TO_CRAWL = 500


Those two attributes are not related to the instance, well at least, they don't depend on the instance, so it makes sense to move it out of __init__ and make them class variables/constants.

Removing those 2 variables from __init__, will also make __init__ itself not needed, which indicates that your class now is just a namespace and you don't need a class anymore, since there is no state in your instances.

The method get_valid_link is way too complex, you've already split it into smaller pieces by newlines and comments, but it's better if you define separate methods for that.

Helpers

The function check_name_match can be replaced by set.intersection. See the documentation for sets.

Function tokenize, as far as I understand it, can just be replaced by str.split, the only difference is that your tokenize will also add trailing space to each of your tokens, which I think is not needed.