I am very new to python and I am just learning things from tutorials online and code snippets.
My code is below, it works in that it scrapes the data I want but I think I could make the code much more efficient.
Code:
import requests
import time
import csv
import sys
from bs4 import BeautifulSoup
hygiene = []
def deletelist():
hygiene.clear()
def savefile():
filename = input("Please input name of file to be saved")
with open (filename + '.csv','w') as file:
writer=csv.writer(file)
writer.writerow(['Establishment Name','Address','Inspection Date', 'Hygiene Rating'])
for row in hygiene:
writer.writerow(row)
print("File Saved Successfully")
def appendhygiene(scrape):
hygiene.append(scrape)
def makesoup(url):
time.sleep(3)
headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'}
page=requests.get(url, headers=headers)
print(url + " scraped successfully")
return BeautifulSoup(page.text,"lxml")
def hygienescrape(g_data):
for item in g_data:
try:
name = (item.find_all("div", {"class": "ResultsBusinessName"})[0].text.strip())
except:
pass
try:
address = (item.find_all("div", {"class": "ResultsBusinessAddress"})[0].text.strip())
except:
pass
try:
postcode = (item.find_all("div", {"class": "ResultsBusinessPostcode"})[0].text.strip())
except:
pass
try:
rating = (item.find_all("div", {"class": "ResultsRatingDate"})[0].text.strip())
except:
pass
try:
bleh = item.find_all('img', {'alt': True})[0]['alt']
appendhygiene(scrape=[name,address + "\n" + postcode,rating,bleh])
except:
pass
def loop():
search = input("Please Enter Postcode(First Part Only, e.g. BT36), Town, or Establishment Name")
if search == 'London' or search == 'Glasgow' or search == 'Manchester':
for i in range(1,10):
soup=makesoup(url = "http://ratings.food.gov.uk/authority-search/en-GB/%5E/"+ search +"/Relevance/0/%5E/%5E/0/"+ str(i) +"/100000")
hygienescrape(g_data = soup.findAll("div", {"class": "ResultRow"}))
elif search:
soup=makesoup(url = "http://ratings.food.gov.uk/authority-search/en-GB/%5E/"+ search +"/Relevance/0/%5E/%5E/0/0/100000")
hygienescrape(g_data = soup.findAll("div", {"class": "ResultRow"}))
def loopforequalto():
search = input("Please Enter Postcode(First Part Only, e.g. BT36), Town, or Establishment Name")
ratingno = input("Please Enter Hygiene Rating")
if search == 'London' or search == 'Glasgow' or search == 'Manchester':
for i in range(1,10):
soup=makesoup(url = "http://ratings.food.gov.uk/authority-search/en-GB/%5E/"+search+"/Relevance/0/%5E/Equal"+ratingno+"/0/"+i+"/100000")
hygienescrape(g_data = soup.findAll("div", {"class": "ResultRow"}))
elif search:
soup=makesoup(url = "http://ratings.food.gov.uk/authority-search/en-GB/%5E/"+search+"/Relevance/0/%5E/Equal"+ratingno+"/0/1/100000")
hygienescrape(g_data = soup.findAll("div", {"class": "ResultRow"}))
def loopforgreaterthanequalto():
search = input("Please Enter Postcode(First Part Only, e.g. BT36), Town, or Establishment Name")
ratingno = input("Please Enter Hygiene Rating")
if search == 'London' or search == 'Glasgow' or search == 'Manchester':
for i in range(1,10):
soup=makesoup(url = "http://ratings.food.gov.uk/authority-search/en-GB/%5E/"+search+"/Relevance/0/%5E/GreaterThanOrEqual"+ratingno+"/0/"+i+"/100000")
hygienescrape(g_data = soup.findAll("div", {"class": "ResultRow"}))
elif search:
soup=makesoup(url = "http://ratings.food.gov.uk/authority-search/en-GB/%5E/"+search+"/Relevance/0/%5E/GreaterThanOrEqual"+ratingno+"/0/1/100000")
hygienescrape(g_data = soup.findAll("div", {"class": "ResultRow"}))
def loopforlessthanequalto():
search = input("Please Enter Postcode(First Part Only, e.g. BT36), Town, or Establishment Name")
ratingno = input("Please Enter Hygiene Rating")
if search == 'London' or search == 'Glasgow' or search == 'Manchester':
for i in range(1,10):
soup=makesoup(url = "http://ratings.food.gov.uk/authority-search/en-GB/%5E/"+search+"/LessThanOrEqual/0/%5E/GreaterThanOrEqual"+ratingno+"/0/"+i+"/100000")
hygienescrape(g_data = soup.findAll("div", {"class": "ResultRow"}))
elif search:
soup=makesoup(url = "http://ratings.food.gov.uk/authority-search/en-GB/%5E/"+search+"/LessThanOrEqual/0/%5E/GreaterThanOrEqual"+ratingno+"/0/1/100000")
hygienescrape(g_data = soup.findAll("div", {"class": "ResultRow"}))
def hygieneratings():
loop()
def hygieneratingsbylessthanorequalrating():
loopforlessthanequalto()
def hygieneratingsbygreaterthanequalrating():
loopforgreaterthanequalto()
def hygieneratingsbyequalrating():
loopforequalto()
def menu():
strs = ('Enter 1 to search Food Hygiene ratings by Postcode(First Part Only e.g. BT36), Town, or Restaurant Name \n'
'Enter 2 to search by by Postcode(First Part Only), Town, or Establishment Name and Hygiene Rating Score\n'
'Enter 3 to search by by Postcode(First Part Only), Town, or Establishment Name and Greater Than or Equal to Hygiene Rating Score\n'
'Enter 4 to search by by Postcode(First Part Only), Town, or Establishment Name and Less than Or Equal to Hygiene Rating Score\n'
'Enter 5 to Exit\n' )
choice = input(strs)
return int(choice)
while True: #use while True
choice = menu()
if choice == 1:
hygieneratings()
savefile()
deletelist()
elif choice == 2:
hygieneratingsbyequalrating()
savefile()
deletelist()
elif choice == 3:
hygieneratingsbygreaterthanequalrating()
savefile()
deletelist()
elif choice == 4:
hygieneratingsbylessthanorequalrating()
savefile()
deletelist()
elif choice == 5:
break
The things that I don't like about the code is :
Loop functions e.g. Loopforequalto etc, contain the same repeated code except for the URL differing slightly. I am trying to find a way to put this into one function and call the function with a parameter for the URL, so I can just call the function containing the repeated code with the URL parameter as the URL for each corresponding loop. I have unsure how to go about this though.
I don't like how I am using the for I in range loops to scrape pages that have a lot of records, example London. I looked at the pagination and seen it was inside a Div container with no a href links for the "Next Button" in the pagination, and as I have only successfully scraped pagination with Next Buttons as a hrefs I am having an issue getting the Next Pages links, hence the use of the for loop.