I've done this parser to scrape all footwear data, but I don't know if it is good to use OOP in this case. Can you please check this out and give me the strongest feedback? I'm working on improving my coding style and looking for an opportunity to do coding better. How can I make this parser more maintainable for the future?
#!/usr/bin/env python3
# coding=utf-8
import re
from datetime import datetime
from bs4 import BeautifulSoup
import http
import requests
import urllib.request
html_hdrs = {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 ('
'KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
'Accept-Encoding': 'none',
'Accept-Language': 'en-US,en;q=0.8',
'X-Requested-With': 'XMLHttpRequest',
'Connection': 'keep-alive'}
def make_soup(input_url):
try:
req = urllib.request.Request(input_url, headers=html_hdrs)
html = urllib.request.urlopen(req).read()
except http.client.IncompleteRead as e:
html = e.partial
return BeautifulSoup(html, "html.parser")
class Items(object):
"""Exporting all Matches Fashion website items
to a list of soups"""
def __init__(self):
self._ul_tag = 'ul'
self._li_tag = 'li'
self._href_attr = 'href'
self.main_url = 'https://www.matchesfashion.com'
self._gender_urls = ('https://www.matchesfashion.com/intl/mens/shop/shoes',
'https://www.matchesfashion.com/intl/womens/shop/shoes',
'https://www.matchesfashion.com/intl/mens/sale/shoes',
'https://www.matchesfashion.com/intl/womens/sale/shoes')
def _get_categories(self):
category_box_class = 'filter__box__list'
sale_category_box_class = 'innerFilterMobile catLevel3'
out_categories = set()
for gender in self._gender_urls:
if '/sale' in gender:
raw_page = make_soup(gender)
raw_categories = raw_page.findAll(name=self._ul_tag, class_=sale_category_box_class)
raw_categories = raw_categories[0].findAll(name=self._li_tag)
for raw_category in raw_categories:
out_categories.add('{}{}'.format(self.main_url, raw_category.a[self._href_attr]))
else:
raw_page = make_soup(gender)
raw_categories = raw_page.findAll(name=self._ul_tag, class_=category_box_class)
raw_categories = raw_categories[0].findAll(name=self._li_tag)
for raw_category in raw_categories:
out_categories.add('{}{}'.format(self.main_url, raw_category.a[self._href_attr]))
self._category_urls = out_categories
def _get_pages(self):
pagination_class = 'redefine__right__pager'
self._pages = list()
def _paginate(input_url, input_num):
out_url = '{}?page={}&noOfRecordsPerPage=240&sort='.format(input_url, input_num)
return out_url
for url in self._category_urls:
page_soup = make_soup(url)
last_page_num = page_soup.findAll(name=self._ul_tag, class_=pagination_class)
if len(last_page_num) != 0:
last_page_num = int(last_page_num[0].findAll(name=self._li_tag)[-3].text)
for page_num in range(1, last_page_num+1):
page_list_url = _paginate(url, page_num)
self._pages.append(page_list_url)
else:
self._pages.append(url)
def get(self):
def get_gender(input_url):
if '/womens/' in input_url:
return 'women'
elif '/mens/' in input_url:
return 'men'
def is_sale(input_url):
if '/sale' in input_url:
return True
else:
return False
def get_category(input_url):
if '?page=' in input_url:
out_category = re.search(r'/shoes/(.*)(\?)', input_url).group(1)
else:
out_category = re.search(r'/shoes/(.*)', input_url).group(1)
return out_category
out_items = list()
item_class = 'lister__item'
self._get_categories()
self._get_pages()
for url in self._pages:
gender = get_gender(url)
category = get_category(url)
sale = is_sale(url)
url_soup = make_soup(url)
url_soup = url_soup.findAll(name=self._li_tag, class_=item_class)
for item_soup in url_soup:
out_items.append((sale, gender, category, item_soup))
return out_items
class Info(object):
"""Getting item info out of soup to an Info object"""
def __init__(self, input_item):
self._div_tag = 'div'
self._strike_tag = 'strike'
self._span_tag = 'span'
self._li_tag = 'li'
self._alt_attr = 'alt'
self._pic_attr = 'data-original'
self._href_attr = 'href'
self._shop_name = 'Matches Fashion'
self.sale, self.gender, self.category, self.item_soup = input_item
def _get_raw_info(self):
vendor_class = 'lister__item__title'
self.title = self.item_soup.img[self._alt_attr]
self.description = self.title
self.model = self.title
self.time = datetime.now(timezone('Europe/Moscow')).strftime('%Y-%m-%d %H:%M:%S')
self.picture = 'https:{}'.format(self.item_soup.img[self._pic_attr])
self.url = 'https://www.matchesfashion.com{}'.format(self.item_soup.a[self._href_attr])
self.vendor = self.item_soup.findAll(name=self._div_tag, class_=vendor_class)[0].text
self.shop = self._shop_name
self.freeship = False
def _get_prices(self):
full_price_class = 'lister__item__price-full'
new_price_class = 'lister__item__price-down'
discount_class = 'lister__item__price-save'
try:
self.price = self.item_soup.findAll(name=self._span_tag, class_=full_price_class)[0].text
self.oldprice = self.price
self.discount = 0
except IndexError:
self.price = self.item_soup.findAll(name=self._span_tag, class_=new_price_class)[0].text
self.oldprice = self.item_soup.findAll(name=self._strike_tag)[0].text
self.discount = self.item_soup.findAll(name=self._span_tag, class_=discount_class)[0].text
def _get_sizes(self):
no_stock_class = 'noStock'
raw_sizes = self.item_soup.select("{}[class!={}]".format(self._li_tag, no_stock_class))
self.sizes = []
for raw_size in raw_sizes:
size = raw_size.text
self.sizes.append(size)
if len(raw_sizes) != 0:
size = raw_sizes[0].text
if 'eu' in size.lower():
self.sizetype = 'eu'
if 'us' in size.lower():
self.sizetype = 'us'
if 'uk' in size.lower():
self.sizetype = 'uk'
def get(self):
self._get_raw_info()
self._get_prices()
self._get_sizes()
raw_items = Items()
items = raw_items.get()
for item in items:
info = Info(item)
info.get()
This code is a bit raw, but the only thing that is left is the one more class Process(object)
to process all data left.