It is very messy and I lack the experience to make it eloquent so I wanted some help with it. The process time is also very slow.
Currently it goes into the first page and goes through the links in the index looking for the word "symptoms." If its not found it will look for a link inside that link named "faqs" and go to that link and look for 'symptoms'.
import requests
import string
from bs4 import BeautifulSoup, Tag
alpha = 'a b c d e f g h i j k l m n o p q r s t u v w x y z'
alpha = alpha.split()
w = []
def disease_spider(maxpages):
i = 0
while i <= maxpages:
url = 'http://www.cdc.gov/DiseasesConditions/az/'+ alpha[i]+'.html'
source_code = requests.get(url)
plain_text = source_code.text
soup = BeautifulSoup(plain_text)
for l in soup.findAll('a', {'class':'noLinking'}):
if l.text != "USA.gov" and l.text != 'Inspector General' and l.text != '':
href = l.get('href')
diseasechecker(href)
i += 1
def diseasechecker(url2):
source_code = requests.get(url2)
plain_text = source_code.text
soup = BeautifulSoup(plain_text)
for y in soup.findAll('p'):
x = str(y)
txt = y.text
if 'symptoms' in x and url2 not in w:
w.append(url2)
elif 'symptoms' in txt and url2 not in w:
w.append(url2)
for y in soup.findAll('li'):
x = str(y)
txt = y.text
if 'symptoms' in x and url2 not in w:
w.append(url2)
elif 'symptoms' in txt and url2 not in w:
w.append(url2)
for y in soup.findAll('a'):
href2 = y.get('href')
try:
s = str(href2)
u = "http://www.cdc.gov"+s
except UnicodeEncodeError:
pass
l3 = 0
if 'faqs' in s:
l3 = level3(u)
if l3 == 1 and url2 not in w:
w.append(url2)
def level3(url3):
source_code = requests.get(url3)
plain_text = source_code.text
soup = BeautifulSoup(plain_text)
for y in soup.findAll('p'):
x = str(y)
txt = y.text
if 'symptoms' in x:
return 1
elif 'symptoms' in txt:
return 1
disease_spider(0)
for i in range(len(w)):
print w[i]