I am making my first web scraper in Python. It works great but it runs extremely slow. The website loads in about 10ms but it only does like 1 every couple of seconds. There are about 4-6 million records I need to scrape through. Any ideas?
from bs4 import BeautifulSoup
import requests
import json
import re
import urllib
import threading
prox = {"http" : "127.0.0.1:8888", "https" : "127.0.0.1:8888"}
def GetVS(Soup):
return Soup.find('input', {'name' : '__VIEWSTATE'})['value']
def GetEV(Soup):
return Soup.find('input', {'name' : '__EVENTVALIDATION'})['value']
def GetSearch(Viewstate, Eventvalidation):
return requests.post('website',
data="__EVENTTARGET=ctl00%24cpMain%24ctl01%24rblSearchType%241&__EVENTARGUMENT=&__LASTFOCUS=&__VIEWSTATE="+urllib.quote(Viewstate, '')+"&__EVENTVALIDATION="+urllib.quote(Eventvalidation, '')+"&ctl00%24txtsearch=&ctl00%24rdoSearch=rdoSite&ctl00%24cpMain%24ctl01%24rblSearchType=PropertyID&ctl00%24cpMain%24ctl01%24txtOwner=",
verify=False,
headers={"User-Agent" : "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.125 Safari/537.36", "Referer" : "https://nevadatreasurer.gov/UPSearch/", "Content-Type" : "application/x-www-form-urlencoded"})
def PropertySearch(PropertyID, Viewstate, Eventvalidation):
return requests.post('website',
data="__EVENTTARGET=&__EVENTARGUMENT=&__LASTFOCUS=&__VIEWSTATE="+urllib.quote(Viewstate, '')+"&__EVENTVALIDATION="+urllib.quote(Eventvalidation, '')+"&ctl00%24txtsearch=&ctl00%24rdoSearch=rdoSite&ctl00%24cpMain%24ctl01%24rblSearchType=PropertyID&ctl00%24cpMain%24ctl01%24txtPropertyID="+urllib.quote(PropertyID, '')+"&ctl00%24cpMain%24ctl01%24btnSearch=Click+Here+to+Search+for+Property",
verify=False,
headers={"User-Agent" : "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.125 Safari/537.36", "Referer" : "https://nevadatreasurer.gov/UPSearch/", "Content-Type" : "application/x-www-form-urlencoded"})
def getAmmount(Selid, PropertyID, Viewstate, Eventvalidation):
return requests.post('website',
data="__LASTFOCUS=&__EVENTTARGET=ctl00%24cpMain%24ctl01%24grdResults&__EVENTARGUMENT=Select%24"+urllib.quote(Selid, '')+"&__VIEWSTATE="+urllib.quote(Viewstate, '')+"&__EVENTVALIDATION="+urllib.quote(Eventvalidation, '')+"&ctl00%24txtsearch=&ctl00%24rdoSearch=rdoSite&ctl00%24cpMain%24ctl01%24rblSearchType=PropertyID&ctl00%24cpMain%24ctl01%24txtPropertyID="+urllib.quote(PropertyID, ''),
verify=False,
headers={"User-Agent" : "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.125 Safari/537.36", "Referer" : "https://nevadatreasurer.gov/UPSearch/", "Content-Type" : "application/x-www-form-urlencoded"})
def GetData(Html):
Soup = BeautifulSoup(Html)
return Soup.find('span', {"id" : "cpMain_ctl01_lblAmountValue"}).text
def GetPropertyID(Html):
Soup = BeautifulSoup(Html)
return Soup.find('span', {"id" : "cpMain_ctl01_lblPropertyIDValue"}).text
def GetReportYear(Html):
Soup = BeautifulSoup(Html)
return Soup.find('span', {"id" : "cpMain_ctl01_lblReportYearValue"}).text
def GetName(Html):
Soup = BeautifulSoup(Html)
return Soup.find('span', {"id" : "cpMain_ctl01_lblNameValue"}).text
def GetAddress1(Html):
Soup = BeautifulSoup(Html)
return Soup.find('span', {"id" : "cpMain_ctl01_lblAddress1Value"}).text
def GetAddress2(Html):
Soup = BeautifulSoup(Html)
return Soup.find('span', {"id" : "cpMain_ctl01_lblAddress2Value"}).text
def GetCity(Html):
Soup = BeautifulSoup(Html)
return Soup.find('span', {"id" : "cpMain_ctl01_lblCityValue"}).text
def GetState(Html):
Soup = BeautifulSoup(Html)
return Soup.find('span', {"id" : "cpMain_ctl01_lblStateValue"}).text
def GetZip(Html):
Soup = BeautifulSoup(Html)
return Soup.find('span', {"id" : "cpMain_ctl01_lblZipValue"}).text
def GetInst(Html):
Soup = BeautifulSoup(Html)
return Soup.find('span', {"id" : "cpMain_ctl01_lblInstitutionValue"}).text
def GetDesc(Html):
Soup = BeautifulSoup(Html)
return Soup.find('span', {"id" : "cpMain_ctl01_lblDescriptionValue"}).text
Request_1 = requests.get('website', verify=False)
Soup_1 = BeautifulSoup(Request_1.text)
Viewstate_1 = GetVS(Soup_1)
Eventvalidation_1 = GetEV(Soup_1)
Request_2 = GetSearch(Viewstate_1, Eventvalidation_1)
Soup_2 = BeautifulSoup(Request_2.text)
Viewstate_2 = GetVS(Soup_2)
Eventvalidation_2 = GetEV(Soup_2)
def dowork(start):
while start < 4000000:
start = start + 1
Request = PropertySearch(str(start), Viewstate_2, Eventvalidation_2)
Soup = BeautifulSoup(Request.text)
Viewstate = GetVS(Soup)
Eventvalidation = GetEV(Soup)
regex = re.compile("Over \$100.*__doPostBack.*Select\$(.*)\&")
r = regex.findall(Request.text)
for i in r:
print GetData(getAmmount(i, str(start), Viewstate, Eventvalidation).text)
threads = []
for i in range(50):
t = threading.Thread(target=dowork, args=(i*100000+1000000,))
threads.append(t)
t.start()