I've created a script in python which is able to parse proxies (supposed to support "https") from a website. Then the script will use those proxies randomly to parse the title of different coffe shops from a website. With every new request, the script is supposed to use new proxies. I've tried my best to make it flawless. The scraper is doing fine at this moment.
I'll be happy to shake off any redundancy within my script (I meant DRY) or to bring about any change to make it better.
This is the complete approach:
import requests
from bs4 import BeautifulSoup
from fake_useragent import UserAgent
from random import choice
links = ['https://www.yellowpages.com/search?search_terms=Coffee%20Shops&geo_location_terms=Los%20Angeles%2C%20CA&page={}'.format(page) for page in range(1,6)]
def get_proxies():
link = 'https://www.sslproxies.org/'
response = requests.get(link)
soup = BeautifulSoup(response.text,"lxml")
proxies = [':'.join([item.select_one("td").text,item.select_one("td:nth-of-type(2)").text]) for item in soup.select("table.table tr") if "yes" in item.text]
return proxies #producing list of proxies that supports "https"
def check_proxy(session, proxy_list=get_proxies(), validated=False):
proxy = choice(proxy_list)
session.proxies = {'https': 'https://{}'.format(proxy)}
try:
print(session.get('https://httpbin.org/ip').json())
validated = True #try to make sure it is a working proxy
return
except Exception: pass
while True:
proxy = choice(proxy_list)
session.proxies = {'https': 'https://{}'.format(proxy)}
if not validated: #otherwise get back to ensure it does fetch a working proxy
print("-------go validate--------")
return
def parse_content(url):
ua = UserAgent()
session = requests.Session()
session.headers = {'User-Agent': ua.random}
check_proxy(session) #collect a working proxy to be used to fetch a valid response
while True:
try:
response = session.get(url)
break #as soon as it fetch a valid response, it will break out of the while loop to continue with the rest
except Exception as e:
session.headers = {'User-Agent': ua.random}
check_proxy(session) #if exception is raised, start over again
parse_content(url)
soup = BeautifulSoup(response.text, 'lxml')
for items in soup.select(".info span[itemprop='name']"):
print(items.text)
if __name__ == '__main__':
for link in links:
parse_content(link)