Python 3 & BeautifulSoup4 - Food Hygiene Scraper to CSV file

Question

I am very new to python and I am just learning things from tutorials online and code snippets.

My code is below, it works in that it scrapes the data I want but I think I could make the code much more efficient.

Code:

import requests
import time
import csv
import sys
from bs4 import BeautifulSoup

hygiene = []

def deletelist():
    hygiene.clear()


def savefile():
    filename = input("Please input name of file to be saved")        
    with open (filename + '.csv','w') as file:
       writer=csv.writer(file)
       writer.writerow(['Establishment Name','Address','Inspection Date', 'Hygiene Rating'])
       for row in hygiene:
          writer.writerow(row)
    print("File Saved Successfully")


def appendhygiene(scrape):
    hygiene.append(scrape)

def makesoup(url):
    time.sleep(3)
    headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'}
    page=requests.get(url, headers=headers)
    print(url + "  scraped successfully")
    return BeautifulSoup(page.text,"lxml")

def hygienescrape(g_data):
     for item in g_data:
        try:
            name = (item.find_all("div", {"class": "ResultsBusinessName"})[0].text.strip())
        except:
            pass
        try:
            address = (item.find_all("div", {"class": "ResultsBusinessAddress"})[0].text.strip())
        except:
            pass
        try:
            postcode = (item.find_all("div", {"class": "ResultsBusinessPostcode"})[0].text.strip())
        except:
            pass
        try:
            rating = (item.find_all("div", {"class": "ResultsRatingDate"})[0].text.strip())
        except:
            pass
        try:
            bleh = item.find_all('img', {'alt': True})[0]['alt']
            appendhygiene(scrape=[name,address + "\n" + postcode,rating,bleh])
        except:
            pass


def loop():
     search = input("Please Enter Postcode(First Part Only, e.g. BT36), Town, or Establishment Name")
     if search == 'London' or search == 'Glasgow' or search == 'Manchester':
        for i in range(1,10):
            soup=makesoup(url = "http://ratings.food.gov.uk/authority-search/en-GB/%5E/"+ search +"/Relevance/0/%5E/%5E/0/"+ str(i) +"/100000")
            hygienescrape(g_data = soup.findAll("div", {"class": "ResultRow"}))
     elif search:
         soup=makesoup(url = "http://ratings.food.gov.uk/authority-search/en-GB/%5E/"+ search +"/Relevance/0/%5E/%5E/0/0/100000")
         hygienescrape(g_data = soup.findAll("div", {"class": "ResultRow"}))

def loopforequalto():
     search = input("Please Enter Postcode(First Part Only, e.g. BT36), Town, or Establishment Name")
     ratingno = input("Please Enter Hygiene Rating")
     if search == 'London' or search == 'Glasgow' or search == 'Manchester':
        for i in range(1,10):
            soup=makesoup(url = "http://ratings.food.gov.uk/authority-search/en-GB/%5E/"+search+"/Relevance/0/%5E/Equal"+ratingno+"/0/"+i+"/100000")
            hygienescrape(g_data = soup.findAll("div", {"class": "ResultRow"}))
     elif search:
         soup=makesoup(url = "http://ratings.food.gov.uk/authority-search/en-GB/%5E/"+search+"/Relevance/0/%5E/Equal"+ratingno+"/0/1/100000")
         hygienescrape(g_data = soup.findAll("div", {"class": "ResultRow"}))

def loopforgreaterthanequalto():
     search = input("Please Enter Postcode(First Part Only, e.g. BT36), Town, or Establishment Name")
     ratingno = input("Please Enter Hygiene Rating")
     if search == 'London' or search == 'Glasgow' or search == 'Manchester':
        for i in range(1,10):
            soup=makesoup(url = "http://ratings.food.gov.uk/authority-search/en-GB/%5E/"+search+"/Relevance/0/%5E/GreaterThanOrEqual"+ratingno+"/0/"+i+"/100000")
             hygienescrape(g_data = soup.findAll("div", {"class": "ResultRow"}))
     elif search:
         soup=makesoup(url = "http://ratings.food.gov.uk/authority-search/en-GB/%5E/"+search+"/Relevance/0/%5E/GreaterThanOrEqual"+ratingno+"/0/1/100000")
         hygienescrape(g_data = soup.findAll("div", {"class": "ResultRow"}))

def loopforlessthanequalto():
     search = input("Please Enter Postcode(First Part Only, e.g. BT36), Town, or Establishment Name")
     ratingno = input("Please Enter Hygiene Rating")
     if search == 'London' or search == 'Glasgow' or search == 'Manchester':
        for i in range(1,10):
            soup=makesoup(url = "http://ratings.food.gov.uk/authority-search/en-GB/%5E/"+search+"/LessThanOrEqual/0/%5E/GreaterThanOrEqual"+ratingno+"/0/"+i+"/100000")
            hygienescrape(g_data = soup.findAll("div", {"class": "ResultRow"}))
     elif search:
         soup=makesoup(url = "http://ratings.food.gov.uk/authority-search/en-GB/%5E/"+search+"/LessThanOrEqual/0/%5E/GreaterThanOrEqual"+ratingno+"/0/1/100000")
         hygienescrape(g_data = soup.findAll("div", {"class": "ResultRow"}))




def hygieneratings():
        loop()  


def hygieneratingsbylessthanorequalrating():
    loopforlessthanequalto()


def hygieneratingsbygreaterthanequalrating():
    loopforgreaterthanequalto()       


def hygieneratingsbyequalrating():
    loopforequalto()

def menu():
        strs = ('Enter 1 to search Food Hygiene ratings by Postcode(First Part Only e.g. BT36), Town, or Restaurant Name \n'
            'Enter 2 to search by by Postcode(First Part Only), Town, or Establishment Name and Hygiene Rating Score\n'
            'Enter 3 to search by by Postcode(First Part Only), Town, or Establishment Name and Greater Than or Equal to Hygiene Rating Score\n'
            'Enter 4 to search by by Postcode(First Part Only), Town, or Establishment Name and Less than Or Equal to Hygiene Rating Score\n'
            'Enter 5 to Exit\n' )
        choice = input(strs)
        return int(choice) 

while True:          #use while True
    choice = menu()
    if choice == 1:
        hygieneratings()
        savefile()
        deletelist()
    elif choice == 2:
        hygieneratingsbyequalrating()
        savefile()
        deletelist()
    elif choice == 3:
        hygieneratingsbygreaterthanequalrating()
        savefile()
        deletelist()
    elif choice == 4:
        hygieneratingsbylessthanorequalrating()
        savefile()
        deletelist()
    elif choice == 5:
        break

The things that I don't like about the code is :

1. Loop functions e.g. Loopforequalto etc, contain the same repeated code except for the URL differing slightly. I am trying to find a way to put this into one function and call the function with a parameter for the URL, so I can just call the function containing the repeated code with the URL parameter as the URL for each corresponding loop. I have unsure how to go about this though.

2. I don't like how I am using the for I in range loops to scrape pages that have a lot of records, example London. I looked at the pagination and seen it was inside a Div container with no a href links for the "Next Button" in the pagination, and as I have only successfully scraped pagination with Next Buttons as a hrefs I am having an issue getting the Next Pages links, hence the use of the for loop.

Answer 1

You're off to a good start. Keep reading and learning. :) Here are some nits and improvements for you:

1. What is time.sleep(3) for?

   def makesoup(url):
       time.sleep(3)
       ...
   

2. Using try/except/pass as a NoneType guard is not a good idea. Also, it seems like you only want the first item.

   try:
       ...
   except:
       pass
   

   Use something like this instead:

   name = item.find("div", {"class", .... })
   if name is not None:
       name = name.text.strip()
   

   You could also refactor your code by storing the classes you're selecting in a list:

   classes = [
       "ResultsBusinessName", "ResultsBusinessAddress", ...
   ]
   data = []
   for item in g_data:
       for class in classes:
           data.append(item.find("div", { "class": class }))
   

3. bleh is not a good variable name. How about imageAlts?

4. No need to specify as a kwarg.

   appendhygiene(scrape=[name, address + "\n" + postcode, rating, bleh])
   

   is the same as:

   appendhygiene([name, address + "\n" + postcode, rating, bleh])
   

   Concatenating the address and postcode seems arbitrary to me, if you did the refactoring as mentioned above, you could shorten it to:

   appendhygiene(data)
   

5. The only thing different between all your loopfor* functions is the url. Write a URL formatting function:

   def createUrl(search, equivalence, ratingno, i):
       return "http://ratings.food.gov.uk/authority-search/en-GB/%5E/{}/Relevance/0/%5E/{}{}/0/{}/100000".format(search, equivalence, ratingno, i)
   

   You can use this in all sorts of ways depending on how you want to refactor your functions.

6. hygieneratings(), hygieneratingsbylessthanorequalrating(), and the other like functions are just aliases. Why not just name the functions that in the first place?

7. Use an if __name__ == "__main__": guard for your code and wrap your main code in a main() method (just convention, not really necessary and varies since you're not importing these modules anywhere else).

   def main():
       while True:
           choice = menu():
           if choice == 1:
               hygieneratings()
           elif choice == 2:
               hygieneratingsbyequalrating()
           ...
           elif choice == 5:
               break
           else:
               print("Invalid choice!")
           if 1 <= choice <= 4:
               savefile()
               deletelist()
   
   if __name__ == "__main__":
       main()