I have been working with requests where I as easy as it is. Scraping the webpage and add it into a dict and print the payload if we find a new value or not. For secure reason, I have not added the correct link for scraping this exactly but the main code review im asking if I have done it correctly.

import json
import re

import requests
from selectolax.parser import HTMLParser

payload = {
    "store": "Not found",
    "name": "Not found",
    "price": "Not found",
    "image": "Not found",
    "sizes": []
}

response = requests.get("https://www.lipsum.com/")

if response.ok:

    bs4 = HTMLParser(response.text)

    product_name = bs4.css_first('h1[class="product-page-header"]')
    product_price = bs4.css_first('span[class="price"]')
    product_image = bs4.css_first('meta[property="og:image"]')

    if product_name:
        payload['name'] = product_name.text().strip()

    if product_price:
        payload['price'] = "{} Price".format(product_price.text().strip())

    if product_image:
        payload['image'] = product_image.attrs['content']

    try:
        attribute = json.loads(
            '{}'.format(
                re.search(
                    r'var\s*DataThrill\s*=\s*(.*?);',
                    response.text,
                    re.M | re.S
                ).group(1)
            )
        )

        payload['sizes'] = [
            f'{get_value["Variation"][0]}'
            for get_value in attribute['ProductInfo']['Attributes']['Variations']
            if get_value.get('variant_fruit')
        ]

    except Exception:  # noqa
        pass

    del bs4
    print("New payload!", payload)

else:
    print("No new payload!", payload)

I have been working with requests where I as easy as it is. Scraping the webpage and add it into a dict and print the payload if we find a new value or not.

import json
import re

import requests
from selectolax.parser import HTMLParser

payload = {
    "store": "Not found",
    "name": "Not found",
    "price": "Not found",
    "image": "Not found",
    "sizes": []
}

response = requests.get("https://shelta.se/sneakers/nike-air-zoom-type-whiteblack-cj2033-103")

if response.ok:

    bs4 = HTMLParser(response.text)

    product_name = bs4.css_first('h1[class="product-page-header"]')
    product_price = bs4.css_first('span[class="price"]')
    product_image = bs4.css_first('meta[property="og:image"]')

    if product_name:
        payload['name'] = product_name.text().strip()

    if product_price:
        payload['price'] = "{} Price".format(product_price.text().strip())

    if product_image:
        payload['image'] = product_image.attrs['content']

    try:
        attribute = json.loads(
            '{}'.format(
                re.search(
                    r'var\s*JetshopData\s*=\s*(.*?);',
                    response.text,
                    re.M | re.S
                ).group(1)
            )
        )

        payload['sizes'] = [
            f'{get_value["Variation"][0]}'
            for get_value in attribute['ProductInfo']['Attributes']['Variations']
            if get_value.get('IsBuyable')
        ]

    except Exception:  # noqa
        pass

    del bs4
    print("New payload!", payload)

else:
    print("No new payload!", payload)