# Web-scraping through a rotating proxy script

I've created a script in python which is able to parse proxies (supposed to support "https") from a website. Then the script will use those proxies randomly to parse the title of different coffe shops from a website. With every new request, the script is supposed to use new proxies. I've tried my best to make it flawless. The scraper is doing fine at this moment.

I'll be happy to shake off any redundancy within my script (I meant DRY) or to bring about any change to make it better.

This is the complete approach:

import requests
from bs4 import BeautifulSoup
from fake_useragent import UserAgent
from random import choice

links = ['https://www.yellowpages.com/search?search_terms=Coffee%20Shops&geo_location_terms=Los%20Angeles%2C%20CA&page={}'.format(page) for page in range(1,6)]

def get_proxies():
soup = BeautifulSoup(response.text,"lxml")
proxies = [':'.join([item.select_one("td").text,item.select_one("td:nth-of-type(2)").text]) for item in soup.select("table.table tr") if "yes" in item.text]
return proxies  #producing list of proxies that supports "https"

def check_proxy(session, proxy_list=get_proxies(), validated=False):
proxy = choice(proxy_list)
session.proxies = {'https': 'https://{}'.format(proxy)}
try:
print(session.get('https://httpbin.org/ip').json())
validated = True  #try to make sure it is a working proxy
return
except Exception: pass

while True:
proxy = choice(proxy_list)
session.proxies = {'https': 'https://{}'.format(proxy)}
if not validated:  #otherwise get back to ensure it does fetch a working proxy
print("-------go validate--------")
return

def parse_content(url):
ua = UserAgent()
session = requests.Session()
check_proxy(session)  #collect a working proxy to be used to fetch a valid response

while True:
try:
response = session.get(url)
break     #as soon as it fetch a valid response, it will break out of the while loop to continue with the rest
except Exception as e:
check_proxy(session)  #if exception is raised, start over again
parse_content(url)

soup = BeautifulSoup(response.text, 'lxml')
for items in soup.select(".info span[itemprop='name']"):
print(items.text)

if __name__ == '__main__':

• except /pass is usually a bad idea. You'll want to at least know which exceptions to swallow and which ones to print. Jun 5, 2018 at 1:19
• Why are you reassigning session.proxies? Jun 5, 2018 at 1:20

1. Get a list of proxies
2. That support https
3. That are actually working

And you want that list to be randomized (from your description without repetition, from your code repetitions are fine).

I would use a couple of generators for that:

import requests
from bs4 import BeautifulSoup
from fake_useragent import UserAgent
from random import shuffle

soup = BeautifulSoup(response.text,"lxml")
https_proxies = filter(lambda item: "yes" in item.text,
soup.select("table.table tr"))
for item in https_proxies:
yield "{}:{}".format(item.select_one("td").text,
item.select_one("td:nth-of-type(2)").text)

def get_random_proxies_iter():
proxies = list(get_proxies('https://www.sslproxies.org/'))
shuffle(proxies)
return iter(proxies)  # iter so we can call next on it to get the next proxy

def get_proxy(session, proxies, validated=False):
session.proxies = {'https': 'https://{}'.format(next(proxies))}
if validated:
while True:
try:
return session.get('https://httpbin.org/ip').json()
except Exception:
session.proxies = {'https': 'https://{}'.format(next(proxies))}

def get_response(url):
session = requests.Session()
ua = UserAgent()
proxies = get_random_proxies_iter()
while True:
try:
print(get_proxy(session, proxies, validated=True))  #collect a working proxy to be used to fetch a valid response
return session.get(url) # as soon as it fetches a valid response, it will break out of the while loop
except StopIteration:
raise  # No more proxies left to try
except Exception:
pass  # Other errors: try again

def parse_content(url):
response = get_response(url)
soup = BeautifulSoup(response.text, 'lxml')
for items in soup.select(".info span[itemprop='name']"):
print(items.text)

if __name__ == '__main__':
url = 'https://www.yellowpages.com/search?search_terms=Coffee%20Shops&geo_location_terms=Los%20Angeles%2C%20CA&page={}'
links = [url.format(page) for page in range(1, 6)]

This actually makes sure that no proxy is reused for each site. The order in which the proxies are tried is different for each site. If you are not fine with trying the same proxies again for a new site, just call get_random_proxies_iter outside of parse_content and feed it all the way down to get_proxy.