I've written a Scrapy spider to scrape product details from a specific website. The spider is designed to navigate through various pages, locate specific products, and gather detailed information including price, size, color, and reviews. The spider also makes use of additional API endpoints to fetch color and review data.

Here is the code:

import scrapy
import json

class MarksAndSpencerSpider(scrapy.Spider):
    name = "marksandspencer"
    allowed_domains = ["marksandspencer.com"]
    start_urls = ["https://www.marksandspencer.com"]
    headers = {
        "Accept": "*/*",
        "Accept-Encoding": "gzip, deflate, br, zstd",
        "Accept-Language": "en-US,en;q=0.9",
        "Cookie": "GlobalE_Data=%7B%22countryISO%22%3A%22BG%22%2C%22cultureCode%22%3A%22en-GB%22%2C%22currencyCode%22%3A%22BGN%22%2C%22apiVersion%22%3A%222.1.4%22%7D; ... (long cookie string) ...",
        "Priority": "u=1, i",
        "Referer": "https://www.marksandspencer.com/bg/easy-iron-geometric-print-shirt/p/P60639302.html",
        "Sec-Ch-Ua": '"Not/A)Brand";v="8", "Chromium";v="126", "Google Chrome";v="126"',
        "Sec-Ch-Ua-Mobile": "?0",
        "Sec-Ch-Ua-Platform": "Windows",
        "Sec-Fetch-Dest": "empty",
        "Sec-Fetch-Mode": "cors",
        "Sec-Fetch-Site": "same-origin",
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/ Safari/537.36",
        "X-Requested-With": "XMLHttpRequest"

    def parse(self, response):
        option = response.css('#country-selector-country option[value="BG"]')
        if option:
            value = option.css('::attr(value)').get().lower()
            bg_version_url = f"https://www.marksandspencer.com/{value}"
            self.logger.info(f"BG URL found: {bg_version_url}")
            yield response.follow(bg_version_url, callback=self.parse_bg_version_url)
            self.logger.warning("BG option not found on the homepage.")

    def parse_bg_version_url(self, response):
        men_page_link = response.css('.nav-item.dropdown.order-lg-3 .subcategory a::attr(href)').get()
        if men_page_link:
            self.logger.info(f"Men's link found: {men_page_link}")
            yield response.follow(men_page_link, callback=self.parse_mens_page)
            self.logger.warning("Men's section link not found on the BG page.")

    def parse_mens_page(self, response):
        casual_shirts_link = response.xpath('//a[contains(text(), "Casual shirts")]/@href').get()
        if casual_shirts_link:
            self.logger.info(f"Casual shirts link found: {casual_shirts_link}")
            yield response.follow(casual_shirts_link, callback=self.parse_casual_shirts)
            self.logger.warning("Casual shirts link not found on the Men's page.")

    def parse_casual_shirts(self, response):
        product_link = response.css('div.pdp-link a:contains("Easy Iron Geometric Print Shirt")::attr(href)').get()
        if product_link:
            self.logger.info(f"Product link found: {product_link}")
            yield response.follow(product_link, callback=self.parse_product_page)
            self.logger.warning("Product link not found in the Casual Shirts section.")

    def parse_product_page(self, response):
        product_name = response.css('.product-name::text').get()
        price = response.css('.value::attr(content)').get()
        size = response.css('#plp-select').get()
        self.logger.info(f"In the parse_product_page method")

        if price:
                price = float(price)
            except ValueError:
                price = None
            price = None

        sizes = []
        if size:
            select_element = scrapy.Selector(text=size, type="html")
            option_tags = select_element.css('option')
            for option in option_tags:
                option_text = option.xpath('normalize-space(text())').get()
                if option_text != "Select Size":

        product_data = {
            "name": str(product_name) or '',
            "price": float(price) or '',
            "colour": '',
            "size": list(sizes),
            "reviews_count": 0,
            "reviews_score": 0.0,

        color_api_url = 'https://www.marksandspencer.com/on/demandware.store/Sites-mandslondon-Site/en_BG/Product-Variation?pid=P60639302&isCompleteTheLookABTest=false'
        reviews_api_url = 'https://www.marksandspencer.com/on/demandware.store/Sites-mandslondon-Site/en_BG/BazaarVoiceReviews-ReviewSummary?pid=P60639302&locale=en_BG&offset=0'
        self.logger.info(f"At the end: , {product_data}")
        self.logger.info(f"Requesting color API URL: {color_api_url}")
        yield scrapy.Request(color_api_url, callback=self.parse_api_color, cb_kwargs={'product_data': product_data, 'reviews_api_url': reviews_api_url}, dont_filter=True, headers=self.headers)

    def parse_api_color(self, response, **cb_kwargs):
        product_data = cb_kwargs['product_data']
        reviews_api_url = cb_kwargs['reviews_api_url']
        self.logger.info(f"Parsing color API URL: {response.url}")
        raw_data = response.body
        data = json.loads(raw_data)
        default_color = data.get('product', {}).get('defaultColor', 'N/A')

        product_data['colour'] = default_color

        yield scrapy.Request(url=reviews_api_url, callback=self.parse_api_reviews, cb_kwargs={'product_data': product_data}, dont_filter=True)

    def parse_api_reviews(self, response, **cb_kwargs):
        product_data = cb_kwargs['product_data']

        raw_data = response.body
        data = json.loads(raw_data)

        reviews_html = data.get('bvDetailReviewsHTML', '')

        reviews_selector = scrapy.Selector(text=reviews_html)

        average_rating = reviews_selector.css('span.review-summary__rating::text').get()
        if average_rating:
                average_rating = float(average_rating)
            except ValueError:
                average_rating = 0.0
            average_rating = 0.0

        review_count_text = reviews_selector.css('span.review-summary__total-review::text').get()
        review_count = review_count_text.split()[2] if review_count_text else '0'
        if review_count:
                review_count = int(review_count)
            except ValueError:
                review_count = 0
            review_count = 0

        product_data['reviews_score'] = average_rating
        product_data['reviews_count'] = review_count

        yield product_data


  • Optimization: Are there any areas in my spider that can be optimized for better performance or reduced complexity?

  • Best Practices: Am I following the best practices for Scrapy spiders, especially with respect to request handling and parsing?

  • Error Handling: How can I improve error handling, especially in scenarios where the expected elements or API responses might not be available?

  • Code Cleanliness: Are there any improvements that can be made for better readability and maintainability of the code?

  • Headers and Cookies: Is the way I handle headers and cookies appropriate, or are there improvements or security concerns I should be aware of?

Any suggestions or feedback would be greatly appreciated.

  • 1
    \$\begingroup\$ Please see Post a new question. What review advice is incorporated in this code? How has the code changed? Are the design goals similar? \$\endgroup\$
    – J_H
    Commented Jun 29 at 15:05


Your Answer

By clicking “Post Your Answer”, you agree to our terms of service and acknowledge you have read our privacy policy.

Browse other questions tagged or ask your own question.