I am currently trying to make a Walmart Scraper, where I have to scrape the title, link, image URL, price, category and subcategory of the product.
Now my approach is that I will collect seed pages for each department, like a link of USB drives from computer from electronics departments. Likewise that there are so many of them.
Now the problem is, I have kept all the seed links into one list (I use for loop to iterate over that list). I have to extract product information from each page (so am using a for
loop to request the URL for each page). The last for
loop extracts the information from the list and puts it into the XML file.
How can I optimize this?
from bs4 import BeautifulSoup
import urllib
import urllib2
import re
import time
import unicodedata
import xmlcreator
xml=xmlcreator
xml.rootC('database')
xml.sub1C('product')
xml.sub2('ptitle','price','purl','urlimage','categ','subcateg')
urllist=[["http://www.walmart.com/browse/computer-accessories/usb-flash-drives/3944_3951_132959_529295?tab_value=online&catNavId=%3Cc%3aout+value%3d%27%26lt%3bc%3aout+value%3d%27%26lt%3bc%3aout+value%3d%27132959%27%2f%26gt%3b%27%2f%26gt%3b%27%2f%3E&ss=false&ic=32_0",'Electronics','USB Flash Drives']]
for i in range(0,len(urllist)):
print 'Scrapping '+urllist[i][0]+' in '+urllist[i][1]+' Category and '+urllist[i][2]+' subcategory.......'
count=0
err=[]
url=urllist[i][0]
request = urllib2.Request(url)
request.add_header('UserAgent', 'Ruel.ME Walmart Scraper')
htmlFile = urllib2.urlopen(request)
data = htmlFile.read()
soup=BeautifulSoup(data)
y=soup.find_all("span",class_="floatLeft numResults mt5")
y=re.findall('<span class="floatLeft numResults mt5">(.*)</span>',str(y[0]))
y=y[0].strip('Results');
y=int(y)
for k in range(0,(y/32)):
print 'Scapping Page '+str(k)+' of '+str(y/32)+'......'
try:
request = urllib2.Request(url)
request.add_header('UserAgent', 'Ruel.ME Walmart Scraper')
htmlFile = urllib2.urlopen(request)
data = htmlFile.read()
soup=BeautifulSoup(data)
purl=[]
ptitle=[]
pimg=[]
price=[]
except urllib2.URLError:
print 'Error in Opening this url'
continue
for a in soup.find_all("a",class_="prodLink ListItemLink"):
var=unicodedata.normalize('NFKD', a.text).encode('ascii','ignore')
var=var.replace('&','')
ptitle.append(var)
var1=re.findall('<a class=.*? href="(.*)" onclick=.*?',str(a))
var1='http://www.walmart.com'+var1[0]
purl.append(var1)
for img in soup.find_all("img",class_="prodImg"):
temp=re.findall('src="([^"]+)"',str(img))
pimg.append(temp[0])
for it in soup.find_all("div",class_="ItemShelfAvail"):
var=re.findall('<span class="bigPriceText2">.*</span>',str(it))
if not var:
var=re.findall('<span class="bigPriceTextOutStock2">.*</span>',str(it))
try:
ne=BeautifulSoup(var[0])
z=ne.find_all("span",class_="bigPriceText2")
if not z:
z=ne.find_all("span",class_="bigPriceTextOutStock2")
temp=z[0]
z=re.findall('<span class="bigPriceText2">(.*)</span>',str(z[0]))
if not z:
z=re.findall('<span class="bigPriceTextOutStock2">(.*)</span>',str(temp))
z=z[0]
w=ne.find_all("span",class_="smallPriceText2")
if not w:
w=ne.find_all("span",class_="smallPriceTextOutStock2")
temp=w[0]
w=re.findall('<span class="smallPriceText2">(.*)</span>',str(w[0]))
if not w:
w=re.findall('<span class="smallPriceTextOutStock2">(.*)</span>',str(temp))
w=w[0]
prc=z+w
price.append(prc)
except IndexError:
err.append(count)
price.append('')
count=count+1
for j in range(0,len(ptitle)):
if j not in err:
xml.createOne(ptitle[j],price[j],purl[j],pimg[j],urllist[i][1],urllist[i][2])
ln=len(url)
url=url[0:(ln-1)]+str((k+1)*32)
xml.cls()
Is there a way that I can do this without collecting seed links, or do I need to collect them?
And how can I optimize?