We’re rewarding the question askers & reputations are being recalculated! Read more.
    Tweeted twitter.com/StackCodeReview/status/1156218006298660865
5 On Code Review, Python questions get tagged with both their version and the main Python tag.
| link
4 deleted 44 characters in body; edited tags
source | link
#!/usr/bin/env python3
# coding=utf-8

import re
from datetime import datetime
from bs4 import BeautifulSoup
import http
import requests
import urllib.request


html_hdrs = {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 ('
         'KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11',
         'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
         'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
         'Accept-Encoding': 'none',
         'Accept-Language': 'en-US,en;q=0.8',
         'X-Requested-With': 'XMLHttpRequest',
         'Connection': 'keep-alive'}


def make_soup(input_url):
    try:
        req = urllib.request.Request(input_url, headers=html_hdrs)
        html = urllib.request.urlopen(req).read()
    except http.client.IncompleteRead as e:
        html = e.partial
    return BeautifulSoup(html, "html.parser")



class Items(object):
    """Exporting all Matches Fashion website items
    to a list of soups"""

    def __init__(self):
        self._ul_tag = 'ul'
        self._li_tag = 'li'
        self._href_attr = 'href'
        self.main_url = 'https://www.matchesfashion.com'
        self._gender_urls = ('https://www.matchesfashion.com/intl/mens/shop/shoes',
                             'https://www.matchesfashion.com/intl/womens/shop/shoes',
                             'https://www.matchesfashion.com/intl/mens/sale/shoes',
                             'https://www.matchesfashion.com/intl/womens/sale/shoes')

    def _get_categories(self):
        category_box_class = 'filter__box__list'
        sale_category_box_class = 'innerFilterMobile catLevel3'
        out_categories = set()
        for gender in self._gender_urls:
            if '/sale' in gender:
                raw_page = make_soup(gender)
                raw_categories = raw_page.findAll(name=self._ul_tag, class_=sale_category_box_class)
                raw_categories = raw_categories[0].findAll(name=self._li_tag)
                for raw_category in raw_categories:
                    out_categories.add('{}{}'.format(self.main_url, raw_category.a[self._href_attr]))
            else:
                raw_page = make_soup(gender)
                raw_categories = raw_page.findAll(name=self._ul_tag, class_=category_box_class)
                raw_categories = raw_categories[0].findAll(name=self._li_tag)
                for raw_category in raw_categories:
                    out_categories.add('{}{}'.format(self.main_url, raw_category.a[self._href_attr]))
        self._category_urls = out_categories

    def _get_pages(self):
        pagination_class = 'redefine__right__pager'
        self._pages = list()

        def _paginate(input_url, input_num):
            out_url = '{}?page={}&noOfRecordsPerPage=240&sort='.format(input_url, input_num)
            return out_url

        for url in self._category_urls:
            page_soup = make_soup(url)
            last_page_num = page_soup.findAll(name=self._ul_tag, class_=pagination_class)
            if len(last_page_num) != 0:
                last_page_num = int(last_page_num[0].findAll(name=self._li_tag)[-3].text)
                for page_num in range(1, last_page_num+1):
                    page_list_url = _paginate(url, page_num)
                    self._pages.append(page_list_url)
            else:
                self._pages.append(url)

    def get(self):
        def get_gender(input_url):
            if '/womens/' in input_url:
                return 'women'
            elif '/mens/' in input_url:
                return 'men'

        def is_sale(input_url):
            if '/sale' in input_url:
                return True
            else:
                return False

        def get_category(input_url):
            if '?page=' in input_url:
                out_category = re.search(r'/shoes/(.*)(\?)', input_url).group(1)
            else:
                out_category = re.search(r'/shoes/(.*)', input_url).group(1)
            return out_category
        out_items = list()
        item_class = 'lister__item'
        self._get_categories()
        self._get_pages()
        for url in self._pages:
            gender = get_gender(url)
            category = get_category(url)
            sale = is_sale(url)
            url_soup = make_soup(url)
            url_soup = url_soup.findAll(name=self._li_tag, class_=item_class)
            for item_soup in url_soup:
                out_items.append((sale, gender, category, item_soup))
        return out_items


class Info(object):
    """Exporting each"""Getting item data to a dictionary object
 info out of andsoup returningto listan ofInfo dictionaries"""object"""

    def __init__(self, input_item):
        self._div_tag = 'div'
        self._strike_tag = 'strike'
        self._span_tag = 'span'
        self._li_tag = 'li'
        self._alt_attr = 'alt'
        self._pic_attr = 'data-original'
        self._href_attr = 'href'
        self._shop_name = 'Matches Fashion'
        self.sale, self.gender, self.category, self.item_soup = input_item

    def _get_raw_info(self):
        vendor_class = 'lister__item__title'
        self.title = self.item_soup.img[self._alt_attr]
        self.description = self.title
        self.model = self.title
        self.time = datetime.now(timezone('Europe/Moscow')).strftime('%Y-%m-%d %H:%M:%S')
        self.picture = 'https:{}'.format(self.item_soup.img[self._pic_attr])
        self.url = 'https://www.matchesfashion.com{}'.format(self.item_soup.a[self._href_attr])
        self.vendor = self.item_soup.findAll(name=self._div_tag, class_=vendor_class)[0].text
        self.shop = self._shop_name
        self.freeship = False

    def _get_prices(self):
        full_price_class = 'lister__item__price-full'
        new_price_class = 'lister__item__price-down'
        discount_class = 'lister__item__price-save'
        try:
            self.price = self.item_soup.findAll(name=self._span_tag, class_=full_price_class)[0].text
            self.oldprice = self.price
            self.discount = 0
        except IndexError:
            self.price = self.item_soup.findAll(name=self._span_tag, class_=new_price_class)[0].text
            self.oldprice = self.item_soup.findAll(name=self._strike_tag)[0].text
            self.discount = self.item_soup.findAll(name=self._span_tag, class_=discount_class)[0].text

    def _get_sizes(self):
        no_stock_class = 'noStock'
        raw_sizes = self.item_soup.select("{}[class!={}]".format(self._li_tag, no_stock_class))
        self.sizes = []
        for raw_size in raw_sizes:
            size = raw_size.text
            self.sizes.append(size)
        if len(raw_sizes) != 0:
            size = raw_sizes[0].text
            if 'eu' in size.lower():
                self.sizetype = 'eu'
            if 'us' in size.lower():
                self.sizetype = 'us'
            if 'uk' in size.lower():
                self.sizetype = 'uk'

    def get(self):
        self._get_raw_info()
        self._get_prices()
        self._get_sizes()


raw_items = Items()
items = raw_items.get()

for item in items:
    info = Info(item)
    info.get()
#!/usr/bin/env python3
# coding=utf-8

import re
from datetime import datetime
from bs4 import BeautifulSoup
import http
import requests
import urllib.request


html_hdrs = {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 ('
         'KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11',
         'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
         'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
         'Accept-Encoding': 'none',
         'Accept-Language': 'en-US,en;q=0.8',
         'X-Requested-With': 'XMLHttpRequest',
         'Connection': 'keep-alive'}


def make_soup(input_url):
    try:
        req = urllib.request.Request(input_url, headers=html_hdrs)
        html = urllib.request.urlopen(req).read()
    except http.client.IncompleteRead as e:
        html = e.partial
    return BeautifulSoup(html, "html.parser")



class Items(object):
    """Exporting all Matches Fashion website items
    to a list of soups"""

    def __init__(self):
        self._ul_tag = 'ul'
        self._li_tag = 'li'
        self._href_attr = 'href'
        self.main_url = 'https://www.matchesfashion.com'
        self._gender_urls = ('https://www.matchesfashion.com/intl/mens/shop/shoes',
                             'https://www.matchesfashion.com/intl/womens/shop/shoes',
                             'https://www.matchesfashion.com/intl/mens/sale/shoes',
                             'https://www.matchesfashion.com/intl/womens/sale/shoes')

    def _get_categories(self):
        category_box_class = 'filter__box__list'
        sale_category_box_class = 'innerFilterMobile catLevel3'
        out_categories = set()
        for gender in self._gender_urls:
            if '/sale' in gender:
                raw_page = make_soup(gender)
                raw_categories = raw_page.findAll(name=self._ul_tag, class_=sale_category_box_class)
                raw_categories = raw_categories[0].findAll(name=self._li_tag)
                for raw_category in raw_categories:
                    out_categories.add('{}{}'.format(self.main_url, raw_category.a[self._href_attr]))
            else:
                raw_page = make_soup(gender)
                raw_categories = raw_page.findAll(name=self._ul_tag, class_=category_box_class)
                raw_categories = raw_categories[0].findAll(name=self._li_tag)
                for raw_category in raw_categories:
                    out_categories.add('{}{}'.format(self.main_url, raw_category.a[self._href_attr]))
        self._category_urls = out_categories

    def _get_pages(self):
        pagination_class = 'redefine__right__pager'
        self._pages = list()

        def _paginate(input_url, input_num):
            out_url = '{}?page={}&noOfRecordsPerPage=240&sort='.format(input_url, input_num)
            return out_url

        for url in self._category_urls:
            page_soup = make_soup(url)
            last_page_num = page_soup.findAll(name=self._ul_tag, class_=pagination_class)
            if len(last_page_num) != 0:
                last_page_num = int(last_page_num[0].findAll(name=self._li_tag)[-3].text)
                for page_num in range(1, last_page_num+1):
                    page_list_url = _paginate(url, page_num)
                    self._pages.append(page_list_url)
            else:
                self._pages.append(url)

    def get(self):
        def get_gender(input_url):
            if '/womens/' in input_url:
                return 'women'
            elif '/mens/' in input_url:
                return 'men'

        def is_sale(input_url):
            if '/sale' in input_url:
                return True
            else:
                return False

        def get_category(input_url):
            if '?page=' in input_url:
                out_category = re.search(r'/shoes/(.*)(\?)', input_url).group(1)
            else:
                out_category = re.search(r'/shoes/(.*)', input_url).group(1)
            return out_category
        out_items = list()
        item_class = 'lister__item'
        self._get_categories()
        self._get_pages()
        for url in self._pages:
            gender = get_gender(url)
            category = get_category(url)
            sale = is_sale(url)
            url_soup = make_soup(url)
            url_soup = url_soup.findAll(name=self._li_tag, class_=item_class)
            for item_soup in url_soup:
                out_items.append((sale, gender, category, item_soup))
        return out_items


class Info(object):
    """Exporting each item data to a dictionary object
    and returning list of dictionaries"""

    def __init__(self, input_item):
        self._div_tag = 'div'
        self._strike_tag = 'strike'
        self._span_tag = 'span'
        self._li_tag = 'li'
        self._alt_attr = 'alt'
        self._pic_attr = 'data-original'
        self._href_attr = 'href'
        self._shop_name = 'Matches Fashion'
        self.sale, self.gender, self.category, self.item_soup = input_item

    def _get_raw_info(self):
        vendor_class = 'lister__item__title'
        self.title = self.item_soup.img[self._alt_attr]
        self.description = self.title
        self.model = self.title
        self.time = datetime.now(timezone('Europe/Moscow')).strftime('%Y-%m-%d %H:%M:%S')
        self.picture = 'https:{}'.format(self.item_soup.img[self._pic_attr])
        self.url = 'https://www.matchesfashion.com{}'.format(self.item_soup.a[self._href_attr])
        self.vendor = self.item_soup.findAll(name=self._div_tag, class_=vendor_class)[0].text
        self.shop = self._shop_name
        self.freeship = False

    def _get_prices(self):
        full_price_class = 'lister__item__price-full'
        new_price_class = 'lister__item__price-down'
        discount_class = 'lister__item__price-save'
        try:
            self.price = self.item_soup.findAll(name=self._span_tag, class_=full_price_class)[0].text
            self.oldprice = self.price
            self.discount = 0
        except IndexError:
            self.price = self.item_soup.findAll(name=self._span_tag, class_=new_price_class)[0].text
            self.oldprice = self.item_soup.findAll(name=self._strike_tag)[0].text
            self.discount = self.item_soup.findAll(name=self._span_tag, class_=discount_class)[0].text

    def _get_sizes(self):
        no_stock_class = 'noStock'
        raw_sizes = self.item_soup.select("{}[class!={}]".format(self._li_tag, no_stock_class))
        self.sizes = []
        for raw_size in raw_sizes:
            size = raw_size.text
            self.sizes.append(size)
        if len(raw_sizes) != 0:
            size = raw_sizes[0].text
            if 'eu' in size.lower():
                self.sizetype = 'eu'
            if 'us' in size.lower():
                self.sizetype = 'us'
            if 'uk' in size.lower():
                self.sizetype = 'uk'

    def get(self):
        self._get_raw_info()
        self._get_prices()
        self._get_sizes()


raw_items = Items()
items = raw_items.get()

for item in items:
    info = Info(item)
    info.get()
#!/usr/bin/env python3
# coding=utf-8

import re
from datetime import datetime
from bs4 import BeautifulSoup
import http
import requests
import urllib.request


html_hdrs = {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 ('
         'KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11',
         'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
         'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
         'Accept-Encoding': 'none',
         'Accept-Language': 'en-US,en;q=0.8',
         'X-Requested-With': 'XMLHttpRequest',
         'Connection': 'keep-alive'}


def make_soup(input_url):
    try:
        req = urllib.request.Request(input_url, headers=html_hdrs)
        html = urllib.request.urlopen(req).read()
    except http.client.IncompleteRead as e:
        html = e.partial
    return BeautifulSoup(html, "html.parser")



class Items(object):
    """Exporting all Matches Fashion website items
    to a list of soups"""

    def __init__(self):
        self._ul_tag = 'ul'
        self._li_tag = 'li'
        self._href_attr = 'href'
        self.main_url = 'https://www.matchesfashion.com'
        self._gender_urls = ('https://www.matchesfashion.com/intl/mens/shop/shoes',
                             'https://www.matchesfashion.com/intl/womens/shop/shoes',
                             'https://www.matchesfashion.com/intl/mens/sale/shoes',
                             'https://www.matchesfashion.com/intl/womens/sale/shoes')

    def _get_categories(self):
        category_box_class = 'filter__box__list'
        sale_category_box_class = 'innerFilterMobile catLevel3'
        out_categories = set()
        for gender in self._gender_urls:
            if '/sale' in gender:
                raw_page = make_soup(gender)
                raw_categories = raw_page.findAll(name=self._ul_tag, class_=sale_category_box_class)
                raw_categories = raw_categories[0].findAll(name=self._li_tag)
                for raw_category in raw_categories:
                    out_categories.add('{}{}'.format(self.main_url, raw_category.a[self._href_attr]))
            else:
                raw_page = make_soup(gender)
                raw_categories = raw_page.findAll(name=self._ul_tag, class_=category_box_class)
                raw_categories = raw_categories[0].findAll(name=self._li_tag)
                for raw_category in raw_categories:
                    out_categories.add('{}{}'.format(self.main_url, raw_category.a[self._href_attr]))
        self._category_urls = out_categories

    def _get_pages(self):
        pagination_class = 'redefine__right__pager'
        self._pages = list()

        def _paginate(input_url, input_num):
            out_url = '{}?page={}&noOfRecordsPerPage=240&sort='.format(input_url, input_num)
            return out_url

        for url in self._category_urls:
            page_soup = make_soup(url)
            last_page_num = page_soup.findAll(name=self._ul_tag, class_=pagination_class)
            if len(last_page_num) != 0:
                last_page_num = int(last_page_num[0].findAll(name=self._li_tag)[-3].text)
                for page_num in range(1, last_page_num+1):
                    page_list_url = _paginate(url, page_num)
                    self._pages.append(page_list_url)
            else:
                self._pages.append(url)

    def get(self):
        def get_gender(input_url):
            if '/womens/' in input_url:
                return 'women'
            elif '/mens/' in input_url:
                return 'men'

        def is_sale(input_url):
            if '/sale' in input_url:
                return True
            else:
                return False

        def get_category(input_url):
            if '?page=' in input_url:
                out_category = re.search(r'/shoes/(.*)(\?)', input_url).group(1)
            else:
                out_category = re.search(r'/shoes/(.*)', input_url).group(1)
            return out_category
        out_items = list()
        item_class = 'lister__item'
        self._get_categories()
        self._get_pages()
        for url in self._pages:
            gender = get_gender(url)
            category = get_category(url)
            sale = is_sale(url)
            url_soup = make_soup(url)
            url_soup = url_soup.findAll(name=self._li_tag, class_=item_class)
            for item_soup in url_soup:
                out_items.append((sale, gender, category, item_soup))
        return out_items


class Info(object):
    """Getting item info out of soup to an Info object"""

    def __init__(self, input_item):
        self._div_tag = 'div'
        self._strike_tag = 'strike'
        self._span_tag = 'span'
        self._li_tag = 'li'
        self._alt_attr = 'alt'
        self._pic_attr = 'data-original'
        self._href_attr = 'href'
        self._shop_name = 'Matches Fashion'
        self.sale, self.gender, self.category, self.item_soup = input_item

    def _get_raw_info(self):
        vendor_class = 'lister__item__title'
        self.title = self.item_soup.img[self._alt_attr]
        self.description = self.title
        self.model = self.title
        self.time = datetime.now(timezone('Europe/Moscow')).strftime('%Y-%m-%d %H:%M:%S')
        self.picture = 'https:{}'.format(self.item_soup.img[self._pic_attr])
        self.url = 'https://www.matchesfashion.com{}'.format(self.item_soup.a[self._href_attr])
        self.vendor = self.item_soup.findAll(name=self._div_tag, class_=vendor_class)[0].text
        self.shop = self._shop_name
        self.freeship = False

    def _get_prices(self):
        full_price_class = 'lister__item__price-full'
        new_price_class = 'lister__item__price-down'
        discount_class = 'lister__item__price-save'
        try:
            self.price = self.item_soup.findAll(name=self._span_tag, class_=full_price_class)[0].text
            self.oldprice = self.price
            self.discount = 0
        except IndexError:
            self.price = self.item_soup.findAll(name=self._span_tag, class_=new_price_class)[0].text
            self.oldprice = self.item_soup.findAll(name=self._strike_tag)[0].text
            self.discount = self.item_soup.findAll(name=self._span_tag, class_=discount_class)[0].text

    def _get_sizes(self):
        no_stock_class = 'noStock'
        raw_sizes = self.item_soup.select("{}[class!={}]".format(self._li_tag, no_stock_class))
        self.sizes = []
        for raw_size in raw_sizes:
            size = raw_size.text
            self.sizes.append(size)
        if len(raw_sizes) != 0:
            size = raw_sizes[0].text
            if 'eu' in size.lower():
                self.sizetype = 'eu'
            if 'us' in size.lower():
                self.sizetype = 'us'
            if 'uk' in size.lower():
                self.sizetype = 'uk'

    def get(self):
        self._get_raw_info()
        self._get_prices()
        self._get_sizes()


raw_items = Items()
items = raw_items.get()

for item in items:
    info = Info(item)
    info.get()
3 Improved readability, fixed indentation error.
source | link

How to make this parser code more easier to be maintained in future? Footwear scraper

I've done this parser to scrape all footwear data, but I don't know if it is good to use OOP in this case. Can you please check this out and give me the strongest feedback? I'm wokingworking on improving my coding style and looking for an opportunity to do coding better. Thanks!How can I make this parser more maintainable for the future?

#!/usr/bin/env python3
# coding=utf-8

import re
from datetime import datetime
from bs4 import BeautifulSoup
import http
import requests
import urllib.request


html_hdrs = {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 ('
         'KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11',
         'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
         'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
         'Accept-Encoding': 'none',
         'Accept-Language': 'en-US,en;q=0.8',
         'X-Requested-With': 'XMLHttpRequest',
         'Connection': 'keep-alive'}


def make_soup(input_url):
    try:
        req = urllib.request.Request(input_url, headers=html_hdrs)
        html = urllib.request.urlopen(req).read()
    except http.client.IncompleteRead as e:
        html = e.partial
    return BeautifulSoup(html, "html.parser") 



class Items(object):
    """Exporting all Matches Fashion website items
    to a list of soups"""

    def __init__(self):
        self._ul_tag = 'ul'
        self._li_tag = 'li'
        self._href_attr = 'href'
        self.main_url = 'https://www.matchesfashion.com'
        self._gender_urls = ('https://www.matchesfashion.com/intl/mens/shop/shoes',
                             'https://www.matchesfashion.com/intl/womens/shop/shoes',
                             'https://www.matchesfashion.com/intl/mens/sale/shoes',
                             'https://www.matchesfashion.com/intl/womens/sale/shoes')

    def _get_categories(self):
        category_box_class = 'filter__box__list'
        sale_category_box_class = 'innerFilterMobile catLevel3'
        out_categories = set()
        for gender in self._gender_urls:
            if '/sale' in gender:
                raw_page = make_soup(gender)
                raw_categories = raw_page.findAll(name=self._ul_tag, class_=sale_category_box_class)
                raw_categories = raw_categories[0].findAll(name=self._li_tag)
                for raw_category in raw_categories:
                    out_categories.add('{}{}'.format(self.main_url, raw_category.a[self._href_attr]))
            else:
                raw_page = make_soup(gender)
                raw_categories = raw_page.findAll(name=self._ul_tag, class_=category_box_class)
                raw_categories = raw_categories[0].findAll(name=self._li_tag)
                for raw_category in raw_categories:
                    out_categories.add('{}{}'.format(self.main_url, raw_category.a[self._href_attr]))
        self._category_urls = out_categories

    def _get_pages(self):
        pagination_class = 'redefine__right__pager'
        self._pages = list()

        def _paginate(input_url, input_num):
            out_url = '{}?page={}&noOfRecordsPerPage=240&sort='.format(input_url, input_num)
            return out_url

        for url in self._category_urls:
            page_soup = make_soup(url)
            last_page_num = page_soup.findAll(name=self._ul_tag, class_=pagination_class)
            if len(last_page_num) != 0:
                last_page_num = int(last_page_num[0].findAll(name=self._li_tag)[-3].text)
                for page_num in range(1, last_page_num+1):
                    page_list_url = _paginate(url, page_num)
                    self._pages.append(page_list_url)
            else:
                self._pages.append(url)

    def get(self):
        def get_gender(input_url):
            if '/womens/' in input_url:
                return 'women'
            elif '/mens/' in input_url:
                return 'men'

        def is_sale(input_url):
            if '/sale' in input_url:
                return True
            else:
                return False

        def get_category(input_url):
            if '?page=' in input_url:
                out_category = re.search(r'/shoes/(.*)(\?)', input_url).group(1)
            else:
                out_category = re.search(r'/shoes/(.*)', input_url).group(1)
            return out_category
        out_items = list()
        item_class = 'lister__item'
        self._get_categories()
        self._get_pages()
        for url in self._pages:
            gender = get_gender(url)
            category = get_category(url)
            sale = is_sale(url)
            url_soup = make_soup(url)
            url_soup = url_soup.findAll(name=self._li_tag, class_=item_class)
            for item_soup in url_soup:
                out_items.append((sale, gender, category, item_soup))
        return out_items


class Info(object):
    """Exporting each item data to a dictionary object
    and returning list of dictionaries"""

    def __init__(self, input_item):
        self._div_tag = 'div'
        self._strike_tag = 'strike'
        self._span_tag = 'span'
        self._li_tag = 'li'
        self._alt_attr = 'alt'
        self._pic_attr = 'data-original'
        self._href_attr = 'href'
        self._shop_name = 'Matches Fashion'
        self.sale, self.gender, self.category, self.item_soup = input_item

    def _get_raw_info(self):
        vendor_class = 'lister__item__title'
        self.title = self.item_soup.img[self._alt_attr]
        self.description = self.title
        self.model = self.title
        self.time = datetime.now(timezone('Europe/Moscow')).strftime('%Y-%m-%d %H:%M:%S')
        self.picture = 'https:{}'.format(self.item_soup.img[self._pic_attr])
        self.url = 'https://www.matchesfashion.com{}'.format(self.item_soup.a[self._href_attr])
        self.vendor = self.item_soup.findAll(name=self._div_tag, class_=vendor_class)[0].text
        self.shop = self._shop_name
        self.freeship = False

    def _get_prices(self):
        full_price_class = 'lister__item__price-full'
        new_price_class = 'lister__item__price-down'
        discount_class = 'lister__item__price-save'
        try:
            self.price = self.item_soup.findAll(name=self._span_tag, class_=full_price_class)[0].text
            self.oldprice = self.price
            self.discount = 0
        except IndexError:
            self.price = self.item_soup.findAll(name=self._span_tag, class_=new_price_class)[0].text
            self.oldprice = self.item_soup.findAll(name=self._strike_tag)[0].text
            self.discount = self.item_soup.findAll(name=self._span_tag, class_=discount_class)[0].text

    def _get_sizes(self):
        no_stock_class = 'noStock'
        raw_sizes = self.item_soup.select("{}[class!={}]".format(self._li_tag, no_stock_class))
        self.sizes = []
        for raw_size in raw_sizes:
            size = raw_size.text
            self.sizes.append(size)
        if len(raw_sizes) != 0:
            size = raw_sizes[0].text
            if 'eu' in size.lower():
                self.sizetype = 'eu'
            if 'us' in size.lower():
                self.sizetype = 'us'
            if 'uk' in size.lower():
                self.sizetype = 'uk'

    def get(self):
        self._get_raw_info()
        self._get_prices()
        self._get_sizes()


raw_items = Items()
items = raw_items.get()

for item in items:
    info = Info(item)
    info.get()

This code is a bit raw, but the only thing that is left is the one more class Process(object)class Process(object) to process all data left.

How to make this parser code more easier to be maintained in future?

I've done this parser to scrape all footwear data, but I don't know if it is good to use OOP in this case. Can you please check this out and give me the strongest feedback? I'm woking on improving my coding style and looking for an opportunity to do coding better. Thanks!

#!/usr/bin/env python3
# coding=utf-8

import re
from datetime import datetime
from bs4 import BeautifulSoup
import http
import requests
import urllib.request


html_hdrs = {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 ('
         'KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11',
         'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
         'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
         'Accept-Encoding': 'none',
         'Accept-Language': 'en-US,en;q=0.8',
         'X-Requested-With': 'XMLHttpRequest',
         'Connection': 'keep-alive'}


def make_soup(input_url):
try:
    req = urllib.request.Request(input_url, headers=html_hdrs)
    html = urllib.request.urlopen(req).read()
except http.client.IncompleteRead as e:
    html = e.partial
return BeautifulSoup(html, "html.parser")


class Items(object):
    """Exporting all Matches Fashion website items
    to a list of soups"""

    def __init__(self):
        self._ul_tag = 'ul'
        self._li_tag = 'li'
        self._href_attr = 'href'
        self.main_url = 'https://www.matchesfashion.com'
        self._gender_urls = ('https://www.matchesfashion.com/intl/mens/shop/shoes',
                             'https://www.matchesfashion.com/intl/womens/shop/shoes',
                             'https://www.matchesfashion.com/intl/mens/sale/shoes',
                             'https://www.matchesfashion.com/intl/womens/sale/shoes')

    def _get_categories(self):
        category_box_class = 'filter__box__list'
        sale_category_box_class = 'innerFilterMobile catLevel3'
        out_categories = set()
        for gender in self._gender_urls:
            if '/sale' in gender:
                raw_page = make_soup(gender)
                raw_categories = raw_page.findAll(name=self._ul_tag, class_=sale_category_box_class)
                raw_categories = raw_categories[0].findAll(name=self._li_tag)
                for raw_category in raw_categories:
                    out_categories.add('{}{}'.format(self.main_url, raw_category.a[self._href_attr]))
            else:
                raw_page = make_soup(gender)
                raw_categories = raw_page.findAll(name=self._ul_tag, class_=category_box_class)
                raw_categories = raw_categories[0].findAll(name=self._li_tag)
                for raw_category in raw_categories:
                    out_categories.add('{}{}'.format(self.main_url, raw_category.a[self._href_attr]))
        self._category_urls = out_categories

    def _get_pages(self):
        pagination_class = 'redefine__right__pager'
        self._pages = list()

        def _paginate(input_url, input_num):
            out_url = '{}?page={}&noOfRecordsPerPage=240&sort='.format(input_url, input_num)
            return out_url

        for url in self._category_urls:
            page_soup = make_soup(url)
            last_page_num = page_soup.findAll(name=self._ul_tag, class_=pagination_class)
            if len(last_page_num) != 0:
                last_page_num = int(last_page_num[0].findAll(name=self._li_tag)[-3].text)
                for page_num in range(1, last_page_num+1):
                    page_list_url = _paginate(url, page_num)
                    self._pages.append(page_list_url)
            else:
                self._pages.append(url)

    def get(self):
        def get_gender(input_url):
            if '/womens/' in input_url:
                return 'women'
            elif '/mens/' in input_url:
                return 'men'

        def is_sale(input_url):
            if '/sale' in input_url:
                return True
            else:
                return False

        def get_category(input_url):
            if '?page=' in input_url:
                out_category = re.search(r'/shoes/(.*)(\?)', input_url).group(1)
            else:
                out_category = re.search(r'/shoes/(.*)', input_url).group(1)
            return out_category
        out_items = list()
        item_class = 'lister__item'
        self._get_categories()
        self._get_pages()
        for url in self._pages:
            gender = get_gender(url)
            category = get_category(url)
            sale = is_sale(url)
            url_soup = make_soup(url)
            url_soup = url_soup.findAll(name=self._li_tag, class_=item_class)
            for item_soup in url_soup:
                out_items.append((sale, gender, category, item_soup))
        return out_items


class Info(object):
    """Exporting each item data to a dictionary object
    and returning list of dictionaries"""

    def __init__(self, input_item):
        self._div_tag = 'div'
        self._strike_tag = 'strike'
        self._span_tag = 'span'
        self._li_tag = 'li'
        self._alt_attr = 'alt'
        self._pic_attr = 'data-original'
        self._href_attr = 'href'
        self._shop_name = 'Matches Fashion'
        self.sale, self.gender, self.category, self.item_soup = input_item

    def _get_raw_info(self):
        vendor_class = 'lister__item__title'
        self.title = self.item_soup.img[self._alt_attr]
        self.description = self.title
        self.model = self.title
        self.time = datetime.now(timezone('Europe/Moscow')).strftime('%Y-%m-%d %H:%M:%S')
        self.picture = 'https:{}'.format(self.item_soup.img[self._pic_attr])
        self.url = 'https://www.matchesfashion.com{}'.format(self.item_soup.a[self._href_attr])
        self.vendor = self.item_soup.findAll(name=self._div_tag, class_=vendor_class)[0].text
        self.shop = self._shop_name
        self.freeship = False

    def _get_prices(self):
        full_price_class = 'lister__item__price-full'
        new_price_class = 'lister__item__price-down'
        discount_class = 'lister__item__price-save'
        try:
            self.price = self.item_soup.findAll(name=self._span_tag, class_=full_price_class)[0].text
            self.oldprice = self.price
            self.discount = 0
        except IndexError:
            self.price = self.item_soup.findAll(name=self._span_tag, class_=new_price_class)[0].text
            self.oldprice = self.item_soup.findAll(name=self._strike_tag)[0].text
            self.discount = self.item_soup.findAll(name=self._span_tag, class_=discount_class)[0].text

    def _get_sizes(self):
        no_stock_class = 'noStock'
        raw_sizes = self.item_soup.select("{}[class!={}]".format(self._li_tag, no_stock_class))
        self.sizes = []
        for raw_size in raw_sizes:
            size = raw_size.text
            self.sizes.append(size)
        if len(raw_sizes) != 0:
            size = raw_sizes[0].text
            if 'eu' in size.lower():
                self.sizetype = 'eu'
            if 'us' in size.lower():
                self.sizetype = 'us'
            if 'uk' in size.lower():
                self.sizetype = 'uk'

    def get(self):
        self._get_raw_info()
        self._get_prices()
        self._get_sizes()


raw_items = Items()
items = raw_items.get()

for item in items:
    info = Info(item)
    info.get()

This code is a bit raw, but the only thing that is left is the one more class Process(object) to process all data left.

Footwear scraper

I've done this parser to scrape all footwear data, but I don't know if it is good to use OOP in this case. Can you please check this out and give me the strongest feedback? I'm working on improving my coding style and looking for an opportunity to do coding better. How can I make this parser more maintainable for the future?

#!/usr/bin/env python3
# coding=utf-8

import re
from datetime import datetime
from bs4 import BeautifulSoup
import http
import requests
import urllib.request


html_hdrs = {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 ('
         'KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11',
         'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
         'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
         'Accept-Encoding': 'none',
         'Accept-Language': 'en-US,en;q=0.8',
         'X-Requested-With': 'XMLHttpRequest',
         'Connection': 'keep-alive'}


def make_soup(input_url):
    try:
        req = urllib.request.Request(input_url, headers=html_hdrs)
        html = urllib.request.urlopen(req).read()
    except http.client.IncompleteRead as e:
        html = e.partial
    return BeautifulSoup(html, "html.parser") 



class Items(object):
    """Exporting all Matches Fashion website items
    to a list of soups"""

    def __init__(self):
        self._ul_tag = 'ul'
        self._li_tag = 'li'
        self._href_attr = 'href'
        self.main_url = 'https://www.matchesfashion.com'
        self._gender_urls = ('https://www.matchesfashion.com/intl/mens/shop/shoes',
                             'https://www.matchesfashion.com/intl/womens/shop/shoes',
                             'https://www.matchesfashion.com/intl/mens/sale/shoes',
                             'https://www.matchesfashion.com/intl/womens/sale/shoes')

    def _get_categories(self):
        category_box_class = 'filter__box__list'
        sale_category_box_class = 'innerFilterMobile catLevel3'
        out_categories = set()
        for gender in self._gender_urls:
            if '/sale' in gender:
                raw_page = make_soup(gender)
                raw_categories = raw_page.findAll(name=self._ul_tag, class_=sale_category_box_class)
                raw_categories = raw_categories[0].findAll(name=self._li_tag)
                for raw_category in raw_categories:
                    out_categories.add('{}{}'.format(self.main_url, raw_category.a[self._href_attr]))
            else:
                raw_page = make_soup(gender)
                raw_categories = raw_page.findAll(name=self._ul_tag, class_=category_box_class)
                raw_categories = raw_categories[0].findAll(name=self._li_tag)
                for raw_category in raw_categories:
                    out_categories.add('{}{}'.format(self.main_url, raw_category.a[self._href_attr]))
        self._category_urls = out_categories

    def _get_pages(self):
        pagination_class = 'redefine__right__pager'
        self._pages = list()

        def _paginate(input_url, input_num):
            out_url = '{}?page={}&noOfRecordsPerPage=240&sort='.format(input_url, input_num)
            return out_url

        for url in self._category_urls:
            page_soup = make_soup(url)
            last_page_num = page_soup.findAll(name=self._ul_tag, class_=pagination_class)
            if len(last_page_num) != 0:
                last_page_num = int(last_page_num[0].findAll(name=self._li_tag)[-3].text)
                for page_num in range(1, last_page_num+1):
                    page_list_url = _paginate(url, page_num)
                    self._pages.append(page_list_url)
            else:
                self._pages.append(url)

    def get(self):
        def get_gender(input_url):
            if '/womens/' in input_url:
                return 'women'
            elif '/mens/' in input_url:
                return 'men'

        def is_sale(input_url):
            if '/sale' in input_url:
                return True
            else:
                return False

        def get_category(input_url):
            if '?page=' in input_url:
                out_category = re.search(r'/shoes/(.*)(\?)', input_url).group(1)
            else:
                out_category = re.search(r'/shoes/(.*)', input_url).group(1)
            return out_category
        out_items = list()
        item_class = 'lister__item'
        self._get_categories()
        self._get_pages()
        for url in self._pages:
            gender = get_gender(url)
            category = get_category(url)
            sale = is_sale(url)
            url_soup = make_soup(url)
            url_soup = url_soup.findAll(name=self._li_tag, class_=item_class)
            for item_soup in url_soup:
                out_items.append((sale, gender, category, item_soup))
        return out_items


class Info(object):
    """Exporting each item data to a dictionary object
    and returning list of dictionaries"""

    def __init__(self, input_item):
        self._div_tag = 'div'
        self._strike_tag = 'strike'
        self._span_tag = 'span'
        self._li_tag = 'li'
        self._alt_attr = 'alt'
        self._pic_attr = 'data-original'
        self._href_attr = 'href'
        self._shop_name = 'Matches Fashion'
        self.sale, self.gender, self.category, self.item_soup = input_item

    def _get_raw_info(self):
        vendor_class = 'lister__item__title'
        self.title = self.item_soup.img[self._alt_attr]
        self.description = self.title
        self.model = self.title
        self.time = datetime.now(timezone('Europe/Moscow')).strftime('%Y-%m-%d %H:%M:%S')
        self.picture = 'https:{}'.format(self.item_soup.img[self._pic_attr])
        self.url = 'https://www.matchesfashion.com{}'.format(self.item_soup.a[self._href_attr])
        self.vendor = self.item_soup.findAll(name=self._div_tag, class_=vendor_class)[0].text
        self.shop = self._shop_name
        self.freeship = False

    def _get_prices(self):
        full_price_class = 'lister__item__price-full'
        new_price_class = 'lister__item__price-down'
        discount_class = 'lister__item__price-save'
        try:
            self.price = self.item_soup.findAll(name=self._span_tag, class_=full_price_class)[0].text
            self.oldprice = self.price
            self.discount = 0
        except IndexError:
            self.price = self.item_soup.findAll(name=self._span_tag, class_=new_price_class)[0].text
            self.oldprice = self.item_soup.findAll(name=self._strike_tag)[0].text
            self.discount = self.item_soup.findAll(name=self._span_tag, class_=discount_class)[0].text

    def _get_sizes(self):
        no_stock_class = 'noStock'
        raw_sizes = self.item_soup.select("{}[class!={}]".format(self._li_tag, no_stock_class))
        self.sizes = []
        for raw_size in raw_sizes:
            size = raw_size.text
            self.sizes.append(size)
        if len(raw_sizes) != 0:
            size = raw_sizes[0].text
            if 'eu' in size.lower():
                self.sizetype = 'eu'
            if 'us' in size.lower():
                self.sizetype = 'us'
            if 'uk' in size.lower():
                self.sizetype = 'uk'

    def get(self):
        self._get_raw_info()
        self._get_prices()
        self._get_sizes()


raw_items = Items()
items = raw_items.get()

for item in items:
    info = Info(item)
    info.get()

This code is a bit raw, but the only thing that is left is the one more class Process(object) to process all data left.

2 edited title
| link
1
source | link