I recently submitted a code sample for a web scraping project and was rejected without feedback as to what they didn't like. The prompt, while I cannot give it here verbatim, basically stated that I needed to write a spider to crawl a site for product items. They suggested using a generic spider to scrape the site in question while using URL rules for efficiency. They gave links to documentation in case you hadn't used scrapy before. I felt like this meant that they didn't mind hiring people unfamiliar with their toolset.
Speaking of which we could only use pyquery for dom traversal. I usually would have opted for pure lxml and xpaths.
I understood the concept of using rules to limit extraneous requests but after noticing that the site in question contained a sitemap I decided to start there instead.
I do know that they explicitly said not to use any outside libraries, so that is why I didn't use Pillow for image processing. However, I did cheat and use requests for some other things that the actual spider didn't utilize but again I wasn't told why my code wasn't good enough. So at this point I would like to learn why.
# -*- coding: utf-8 -*-
import scrapy
from scrapy.spiders.sitemap import *
from pyquery import PyQuery as pq
from oxygendemo.items import OxygendemoItem
import oxygendemo.utilities
from oxygendemo.utilities import *
class OxygenSpider(SitemapSpider):
print 'MY SPIDER, IS ALIVE'
name = "oxygen"
allowed_domains = ["oxygenboutique.com"]
sitemap_urls = ['http://www.oxygenboutique.com/sitemap.xml']
sitemap_rules = generate_sitemap_rules()
ex_rates = get_exchange_rates()
def parse_sitemap_url(self, response):
self.logger.info('Entered into parse_sitemap_url method')
self.logger.info('Received response from: {}'.format(response.url))
self.logger.debug('Respons status: {}'.format(response.status))
item = OxygendemoItem()
d = pq(response.body)
parsed_url = urlparse.urlparse(response.url)
base_url = get_base(parsed_url)
product_info = d('.right div#accordion').children()
image_links = d('div#product-images tr td a img')
description = product_info.eq(1).text()\
.encode('ascii', 'ignore')
item['code'] = str(parsed_url[2].lstrip('/')[:-5])
item['description'] = description
item['link'] = parsed_url.geturl()
item['name'] = d('.right h2').text()
gbp_price = {
'prices': d('.price').children(),
'discount': 0
}
item['gbp_price'], item['sale_discount'] = get_price_and_discount(
gbp_price
)
if 'error' not in self.ex_rates:
item['usd_price'] = "{0:.2f}".format(
item['gbp_price'] * self.ex_rates['USD']
)
item['eur_price'] = "{0:.2f}".format(
item['gbp_price'] * self.ex_rates['EUR']
)
else:
item['usd_price'], item['eur_price'] = ['N/A'] * 2
item['designer'] = d('.right').find('.brand_name a').text()
item['stock_status'] = json.dumps(determine_stock_status(d('select')
.children()))
item['gender'] = 'F' # Oxygen boutique carries Womens's clothing
item['image_urls'] = fetch_images(image_links, base_url)
item['raw_color'] = get_product_color_from_description(description)
yield item
This is the utilities module I used:
# -*- coding: utf-8 -*-
import requests
import json
import urlparse
from pyquery import PyQuery as pq
import re
def get_base(parsed_url):
base_url = parsed_url[0] + '://' + parsed_url[1]
base_url = base_url.encode('ascii', 'ignore')
return base_url
def get_exchange_rates():
''' return dictionary of exchange rates with british pound as base
currency '''
url = 'http://api.fixer.io/latest?base=GBP'
try:
response = requests.get(url)
er = json.loads(response.content)['rates']
return er
except:
return {'error': 'Could not contact server'}
def determine_stock_status(sizes):
result = {}
for i in xrange(1, len(sizes)):
option = sizes.eq(i).text()
if 'Sold Out' not in option:
result[option] = 'In Stock'
else:
size = option.split(' ')[0]
result[size] = 'Sold Out'
return result
def determine_type(short_summary):
short_summary = short_summary.upper()
S = {
'HEEL', 'SNEAKER', 'SNEAKERS',
'BOOT', 'FLATS', 'WEDGES',
'SANDALS'
}
J = {
'RING', 'NECKLACE', 'RING',
'BANGLE', 'CHOKER', 'COLLIER',
'BRACELET', 'TATTOO', 'EAR JACKET'
}
B = {
'BAG', 'PURSE', 'CLUTCH',
'TOTE'
}
A = {
'PINNI', 'BLOUSE', 'TOP',
'SKIRT', 'KNICKER', 'DRESS',
'DENIM', 'COAT', 'JACKET',
'SWEATER', 'JUMPER', 'SHIRT',
'SKINNY', 'SHORT', 'TEE',
'PANTS', 'JUMPSUIT', 'HIGH NECK',
'GOWN', 'TROUSER', 'ROBE',
'PLAYSUIT', 'CULOTTE', 'JODPHUR',
'PANTALON', 'FLARE', 'CARDIGAN',
'VEST', 'CAMI', 'BEDSHORT',
'PYJAMA', 'BRALET', 'TUNIC',
'HOODY', 'SATEEN', 'BIKER',
'JEAN', 'SWEAT', 'PULL',
'BIKINI', 'LE GRAND GARCON'
}
types = {
'B': B, 'S': S,
'J': J, 'A': A
}
for key, val in types.iteritems():
for t in val:
if t in short_summary:
return key
else:
return 'R' # Tag as accessory as failsafe
def fetch_images(image_links, base_url):
''' base_url will come as unicode change to python string '''
images = []
for image in image_links:
images.append(urlparse.urljoin(base_url, image.attrib['src']))
return images
def get_price_and_discount(gbp_price):
if gbp_price['prices']('.mark').text() == '': # No discount
gbp_price['discount'] = '0%'
orig_price = float(gbp_price['prices'].parent().text()
.encode('ascii', 'ignore'))
else: # Calculate discount
prices = gbp_price['prices']
orig_price = "{0:.2f}".format(float(prices('.mark').text()))
new_price = "{0:.2f}".format(float(gbp_price['prices'].eq(1).text()))
gbp_price['discount'] = "{0:.2f}"\
.format(float(orig_price) / float(new_price) * 100) + '%'
return float(orig_price), gbp_price['discount']
def get_raw_image_color(image):
''' Note that Pillow imaging library would be perfect
for this task. But external libraries are not
allowed via the constraints noted in the instructions.
Example: Image.get_color(image)
Could be used with Pillow.
'''
# only import Pillow image library if this is used
# Later
from PIL import Image
im = Image.open(image)
colors = im.getcolors()
if colors is None:
return None
else:
return colors[0] # Not functional at this point
def get_product_color_from_description(description):
''' Will go this route to avoid external imports '''
description = description.upper().split(' ')
colors = (
'BLACK', 'WHITE', 'BLUE',
'YELLOW', 'ORANGE', 'GREY',
'PINK', 'FUSCIA', 'RED',
'GREEN', 'PURPLE', 'INDIGO',
'VIOLET'
)
for word in description:
for color in colors:
if word == color:
return color.lower()
else:
return None
def generate_sitemap_rules():
d = pq(requests.get('http://www.oxygenboutique.com').content)
# Proof of concept regex can be found here --> http://regexr.com/3c0lc
designers = d('ul.tame').children()
re_front = r'(http:\/\/)(www\.)(.+\/)((?!'
re_back = r').+)'
re_middle = 'products|newin|product|lingerie|clothing'
for li in designers:
''' This removes 36 requests from the queue '''
link = pq(li.find('a')).attr('href').rstrip('.aspx')
re_middle += '|' + link
return [(re_front + re_middle.replace('-', r'\-') + re_back,
'parse_sitemap_url')]
OxygendemItem() declaration:
import scrapy
from scrapy import Field
class OxygendemoItem(scrapy.Item):
code = Field() # unique identifier (retailers perspective)
description = Field() # Detailed description
designer = Field() # manufacturer
eur_price = Field() # full (non_discounted) price
gender = Field() # F - Female, M - male
gbp_price = Field() # full (non_discounted) price
image_urls = Field() # list of urls representing the item
link = Field() # url of product page
name = Field() # short summary of the item
raw_color = Field() # best guess of color. Default = None
sale_discount = Field() # % discount for sale item where applicable
stock_status = Field() # dictionary of sizes to stock status
'''
size: quantity
Example: { 'L': 'In Stock',
'M': 'In Stock',
'S': 'In Stock',
'XS': 'In Stock'
}
'''
# 'A' = apparel, 'B' = bags, 'S' = shoes, 'J' = jewelry, 'R' = accessories
type = Field()
usd_price = Field() # full (non_discounted) price
