5
\$\begingroup\$

I made a scraper for Steam that get different info about a Steam game, such as the price, the specs and the supported platforms. The reason I made this was because I have a super slow laptop, so looking at many games would take a long time :)

Some things I would like to improve is have better error handling, as the web is a messy place and not all the pages are going to be the same.

Another thing I was thinking about doing is having better data management, such as using classes and objects for each game instead of storing all the values in a dictionary, that would make for simpler and maybe even shorter code.

from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support.ui import Select
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver.firefox.options import Options
from sys import exit

games = {}
x = 0

# ask for games
while True:
    if x == 0:
        game = input('Type the game you want to find here: ')
    else:
        game = input('Type the game you want to find here (or enter nothing to continue): ')
    if not game:
        break
    games[game] = {}
    x += 1

# configure browser
print('Starting Browser')
firefox_options = Options()
firefox_options.headless = True
browser = webdriver.Firefox(options=firefox_options, service_log_path='/tmp/geckodriver.log')
print('Retrieving website')
browser.get('https://store.steampowered.com/')

for a_game in games:
    print('Finding info for "' + a_game + '"')

    # input & click
    print('Waiting for page to load')
    WebDriverWait(browser, 20).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "input#store_nav_search_term"))).send_keys(a_game)
    WebDriverWait(browser, 20).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "div#search_suggestion_contents>a"))).click()
    print('Navigating to game page')

    # if age-restricted:
    try:
        browser.find_element_by_css_selector('.agegate_birthday_selector')
        age_query = input('"' + a_game + '" is age-restricted, do you want to continue? y/n ')
        if age_query != 'y':
            print('Abort')
            exit()
        select = Select(browser.find_element_by_id('ageYear'))
        select.select_by_value('2000')
        browser.find_element_by_css_selector('a.btnv6_blue_hoverfade:nth-child(1)').click()
    except NoSuchElementException:
        pass

    print('Waiting for game page to load')
    # name of game
    games[a_game]['name'] = WebDriverWait(browser, 20).until(EC.element_to_be_clickable((By.CSS_SELECTOR, '.apphub_AppName'))).text

    # supported platforms
    print('Retrieving supported platforms')
    mac = False
    linux = False
    try:
        browser.find_element_by_css_selector('div.game_area_purchase_game_wrapper:nth-child(1) > div:nth-child(1) > div:nth-child(2) > '
                                             'span:nth-child(2)')
        mac = True
    except NoSuchElementException:
        pass

    try:
        browser.find_element_by_css_selector('div.game_area_purchase_game_wrapper:nth-child(1) > div:nth-child(1) > div:nth-child(2) > '
                                             'span:nth-child(3)')
        linux = True
    except NoSuchElementException:
        pass

    if mac and linux:
        games[a_game]['platform'] = 'all'
    elif mac:
        games[a_game]['platform'] = 'mac'
    elif linux:
        games[a_game]['platform'] = 'linux'
    else:
        games[a_game]['platform'] = 'windows'

    # price
    print('Retrieving price')
    discounted = False
    try:
        games[a_game]['price'] = browser.find_element_by_css_selector('div.game_purchase_action:nth-child(4) > div:nth-child(1) > div:nth-child(1)').text
    except NoSuchElementException:
        try:
            games[a_game]['before_price'] = browser.find_element_by_class_name('discount_original_price').text
            games[a_game]['after_price'] = browser.find_element_by_class_name('discount_final_price').text
        except NoSuchElementException:
            try:
                games[a_game]['price'] = 'FREE'
            except NoSuchElementException:
                games[a_game]['bundle_price'] = browser.find_element_by_css_selector('div.game_purchase_action_bg:nth-child(2) > div:nth-child(1)')
    except Exception:
        games[a_game]['price'] = 'Error: Unable to get price'

    # system requirements
    print('Retrieving system requirements')
    games[a_game]['specs'] = browser.find_element_by_css_selector('.game_area_sys_req').text

# close browser
print('Finished Retrieving data, closing browser \n')
print('********************************************')
browser.close()

for each_game in games.keys():
    print('GAME: ' + games[each_game]['name'].upper())

    # printing supported platforms
    if games[each_game]['platform'] == 'all':
        print('Supported Platforms: Windows, Mac and Linux')
    elif games[each_game]['platform'] == 'mac':
        print('Supported Platforms: Windows and Mac')
    elif games[each_game]['platform'] == 'linux':
        print('Supported Platforms: Windows and Linux')
    else:
        print('Supported Platforms: Windows Only')
    print('\n')

    # printing price
    try:
        print('Price: Discounted ' + games[each_game]['after_price'] + ' from ' + games[each_game]['before_price'])
    except KeyError:
        print('Price: ' + games[each_game]['price'])
    except Exception:
        print('Bundled Price: ' + games[each_game]['bundle_price'])
    print('\n')

    # printing system requirements
    print('System Requirements: \n')
    print('-------------------------------- \n')
    print(games[each_game]['specs'])
    print('--------------------------------')
    input('Press enter to continue ')

print('Finished Successfully')
\$\endgroup\$
  • \$\begingroup\$ That's a lot of effort, but you're making things more difficult for yourself by scraping the website instead of using API :( \$\endgroup\$ – yedpodtrzitko Aug 29 at 15:54
4
\$\begingroup\$

Break up logic into functions

Having separate functions for each of the following steps will make the code easier to read.

  • Get game names
  • Scrape game information
  • Display game information

Guard your script's entry point

I'd recommend moving the script execution flow under an if __name__ == "__main__": guard. Doing this allows you to import the functions from this file into other files without running the script.

Avoid using sys.exit() for control flow

Calling sys.exit() shuts down the Python interpreter, which makes any code that calls it difficult to test. You should instead refactor the script so it terminates normally for all recoverable cases.

For example, if the user doesn't want information for an age-restricted game, skip it and move on to the next game in the list. I think this would make for a better user experience anyway, because if we exit() we don't get to process the other remaining games in the list.

Supported platforms should be a list

In determining and printing supported platforms for a game, you have booleans mac and linux which are eventually translated to a string taking one of all, mac, linux, windows:

if mac and linux:
    games[a_game]['platform'] = 'all'      # windows, mac, linux
elif mac:
    games[a_game]['platform'] = 'mac'      # windows, mac
elif linux:
    games[a_game]['platform'] = 'linux'    # windows, linux
else:
    games[a_game]['platform'] = 'windows'  # windows

I think it makes more sense to model this as a list, e.g. ["windows", "mac"] so it's more explicit what platforms are supported. This will also save you from writing extra if/elif/else logic when printing these out.

Flat is better than nested

The nested try/except blocks in the price retrieval stage is very difficult to read.

If you delegate the price retrieval to a function, you can structure the logic so it is flat instead of nested, like in the following pseudocode:

def get_price() -> str:
    element = find_element_for_bundle_price()
    if element:
        return element.text

    element = find_element_for_non_discounted_price()
    if element:
        return element.text

    element = find_element_for_discounted_price()
    if element:
        return element.text

    # If we don't find a price on the page, it's free?
    # Actually this is not always true, but for this example
    # we'll assume this is the case.
    return 'FREE'

Misuse of exception handling

The script is catching KeyError and Exception to handle printing out three different types of prices: bundle, discounted, and standard. This is arguably a misuse of exception handling, especially since catching the general Exception is rarely a good idea because it can hide other errors you weren't expecting. Plus it's not needed here; we can just use an if/elif/else:

game_dict = games[each_game]
if 'bundle_price' in game_dict:
    # print bundle price
elif 'before_price' in game_dict and 'after_price' in game_dict:
    # print discounted price
else:
    # print standard price

Data management

You mentioned that you were thinking about having classes or objects for each game instead of using a dictionary. I think this is a good idea. It might not make the code shorter, but it would definitely improve the code's readability.

A good candidate for this would be a simple data container like typing.NamedTuple. As @MaartenFabré suggested in the comments, dataclasses.dataclass is another good choice for a data container like this.

Unhandled edge cases

The following search queries will result in the script timing out:

  • Any search query that returns no suggestions on Steam. This could be something like a game that doesn't exist (yet), e.g. "funkytown", or a non-empty string consisting only of whitespace, e.g. " ".

  • Any search query where the first suggestion is a Steam Package, e.g. "the witcher 3 wild hunt game of the year".

  • Any search query where the first suggestion is a Steam Bundle, e.g. "monkey island collection".

I mentioned this earlier, but the script incorrectly assumes that if a price is not displayed on the page, then the game is free. But there are unreleased games on Steam where the developer hasn't set a price, and at the same time they have not classified it as "free" or as a "free-to-play" game ("Spirit of Glace" is one concrete example I found). The price to display in this case should be something like "TBD" or "Unknown".

Fortunately, and as a way of introducing the next section, there's an API we can use to distinguish between free and not free. This API exposes an is_free field that is true when a game is free or free-to-play. If you jump to the end of this review you can see how it's getting retrieved in my example script.

Prefer APIs to scraping

With APIs, data retrieval is faster -- and often orders of magnitude faster than scraping with Selenium. With APIs, data extraction is easier since the format of the response is often JSON.

I always make a point of mentioning this whenever scraping comes up because the potential time and effort savings can be huge. Spend some time searching for an official API, or an unofficial API that's documented. If nothing turns up, poke around with an HTTP/S traffic inspector like Fiddler or Chrome DevTools and see if you can find any promising unofficial APIs. If at last you can't find anything, fall back to scraping as a last resort.

In this case, there is actually an unofficial Steam Store API that's available. To use it we need the Steam App ID or Steam Package ID of the item we're in interested in, but we can get that from the API that powers the search suggestion drop-down menu, https://store.steampowered.com/search/suggest.

Example script using API

The following is an example script using the unofficial Steam Store API.

#!/usr/bin/env python3

import re
import json
import requests
from enum import Enum
from bs4 import BeautifulSoup  # type: ignore
from typing import Any, Dict, List, Optional, NamedTuple, Union

SEARCH_SUGGEST_API = "https://store.steampowered.com/search/suggest"
APP_DETAILS_API = "https://store.steampowered.com/api/appdetails"
PACKAGE_DETAILS_API = "https://store.steampowered.com/api/packagedetails"


class Platform(Enum):
    WINDOWS = "windows"
    MAC = "mac"
    LINUX = "linux"

    def __str__(self) -> str:
        return str(self.value)


class Price(NamedTuple):
    initial: int  # price in cents
    final: int  # price in cents


class SteamGame(NamedTuple):
    app_id: int
    name: str
    platforms: List[Platform]
    is_released: bool
    is_free: bool
    price: Optional[Price]
    pc_requirements: str

    def __str__(self) -> str:
        if self.is_free:
            price = "Free"
        elif self.price:
            final = f"${self.price.final / 100}"
            if self.price.initial == self.price.final:
                price = final
            else:
                price = f"{final} (previously ${self.price.initial / 100})"
        else:
            price = "TBD"

        platforms = ", ".join(str(p) for p in self.platforms)
        is_released = "Yes" if self.is_released else "No"

        return "\n".join(
            (
                f"Name: {self.name}",
                f"Released: {is_released}",
                f"Supported Platforms: {platforms}",
                f"Price: {price}",
                "",
                "PC Requirements:",
                self.pc_requirements,
            )
        )


class SteamBundle(NamedTuple):
    bundle_id: int
    name: str
    price: Price
    application_names: List[str]

    def __str__(self) -> str:
        final = f"${self.price.final / 100}"
        if self.price.initial == self.price.final:
            price = final
        else:
            price = f"{final} (without bundle: ${self.price.initial / 100})"

        return "\n".join(
            (
                f"Name: {self.name}",
                f"Price: {price}",
                "",
                "Items included in this bundle:",
                *(f"  - {name}" for name in self.application_names),
            )
        )


class SteamPackage(NamedTuple):
    package_id: int
    name: str
    platforms: List[Platform]
    is_released: bool
    price: Optional[Price]
    application_names: List[str]

    def __str__(self) -> str:
        if self.price:
            final = f"${self.price.final / 100}"
            if self.price.initial == self.price.final:
                price = final
            else:
                initial = f"${self.price.initial / 100}"
                price = f"{final} (without package: {initial})"
        else:
            price = "TBD"

        platforms = ", ".join(str(p) for p in self.platforms)
        is_released = "Yes" if self.is_released else "No"

        return "\n".join(
            (
                f"Name: {self.name}",
                f"Released: {is_released}",
                f"Supported Platforms: {platforms}",
                f"Price: {price}",
                "",
                "Items included in this package:",
                *(f"  - {name}" for name in self.application_names),
            )
        )


SteamItem = Union[SteamGame, SteamBundle, SteamPackage]


def deserialize_bundle_data(encoded_bundle_json: str) -> Any:
    return json.loads(re.sub(r""", '"', encoded_bundle_json))


def extract_app_ids(bundle_data: Dict[str, Any]) -> List[int]:
    return [
        app_id
        for item in bundle_data["m_rgItems"]
        for app_id in item["m_rgIncludedAppIDs"]
    ]


def lookup_app_names(
    session: requests.Session, app_ids: List[int]
) -> List[str]:
    app_names = []
    for app_id in app_ids:
        params = {"appids": app_id, "filters": "basic"}
        response = session.get(APP_DETAILS_API, params=params)
        response.raise_for_status()

        app_names.append(response.json()[str(app_id)]["data"]["name"])

    return app_names


def extract_bundle_price(bundle_data: Dict[str, Any]) -> Price:
    total_price = sum(
        item["m_nFinalPriceInCents"] for item in bundle_data["m_rgItems"]
    )
    total_price_with_bundle_discount = sum(
        item["m_nFinalPriceWithBundleDiscount"]
        for item in bundle_data["m_rgItems"]
    )

    return Price(total_price, total_price_with_bundle_discount)


def extract_package_information(
    package_id: int, package_data: Dict[str, Any]
) -> SteamPackage:
    return SteamPackage(
        package_id=package_id,
        name=package_data["name"],
        platforms=[p for p in Platform if package_data["platforms"][str(p)]],
        is_released=not package_data["release_date"]["coming_soon"],
        price=Price(
            package_data["price"]["initial"], package_data["price"]["final"]
        ),
        application_names=[app["name"] for app in package_data["apps"]],
    )


def get_package(session: requests.Session, package_id: str) -> SteamPackage:
    params = {"packageids": package_id}
    response = session.get(PACKAGE_DETAILS_API, params=params)
    response.raise_for_status()

    return extract_package_information(
        int(package_id), response.json()[package_id]["data"]
    )


def extract_requirements_text(requirements_html: str) -> str:
    soup = BeautifulSoup(requirements_html, "html.parser")
    return "\n".join(tag.get_text() for tag in soup.find_all("li"))


def extract_game_information(game_data: Dict[str, Any]) -> SteamGame:
    price_overview = game_data.get("price_overview")
    price = (
        Price(price_overview["initial"], price_overview["final"])
        if price_overview
        else None
    )

    requirements = game_data["pc_requirements"]
    minimum = extract_requirements_text(requirements["minimum"])
    recommended_html = requirements.get("recommended")
    recommended = (
        extract_requirements_text(recommended_html)
        if recommended_html
        else None
    )

    minimum_requirements = f"[Minimum]\n{minimum}"
    if recommended:
        recommended_requirements = f"[Recommended]\n{recommended}"
        pc_requirements = (
            minimum_requirements + "\n\n" + recommended_requirements
        )
    else:
        pc_requirements = minimum_requirements

    return SteamGame(
        app_id=game_data["steam_appid"],
        name=game_data["name"],
        platforms=[p for p in Platform if game_data["platforms"][str(p)]],
        is_released=not game_data["release_date"]["coming_soon"],
        is_free=game_data["is_free"],
        price=price,
        pc_requirements=pc_requirements,
    )


def get_game(session: requests.Session, app_id: str) -> SteamGame:
    params = {"appids": app_id}
    response = session.get(APP_DETAILS_API, params=params)
    response.raise_for_status()

    return extract_game_information(response.json()[app_id]["data"])


def get_game_information(games: List[str]) -> Dict[str, Optional[SteamItem]]:
    game_to_info = {}

    with requests.Session() as session:
        for game in games:
            params = {"term": game, "f": "games", "cc": "US", "l": "english"}
            response = session.get(SEARCH_SUGGEST_API, params=params)
            response.raise_for_status()

            # get first search suggestion
            result = BeautifulSoup(response.text, "html.parser").find("a")

            if result:
                bundle_id = result.get("data-ds-bundleid")
                package_id = result.get("data-ds-packageid")
                app_id = result.get("data-ds-appid")

                if bundle_id:
                    name = result.find("div", class_="match_name").get_text()
                    bundle_data = deserialize_bundle_data(
                        result["data-ds-bundle-data"]
                    )
                    app_ids = extract_app_ids(bundle_data)
                    app_names = lookup_app_names(session, app_ids)
                    price = extract_bundle_price(bundle_data)

                    info: Optional[SteamItem] = SteamBundle(
                        bundle_id=int(bundle_id),
                        name=name,
                        price=price,
                        application_names=app_names,
                    )
                elif package_id:
                    info = get_package(session, package_id)
                elif app_id:
                    info = get_game(session, app_id)
                else:
                    info = None
            else:
                info = None

            game_to_info[game] = info

    return game_to_info


def display_game_information(
    game_information: Dict[str, Optional[SteamItem]]
) -> None:
    arrow = " =>"
    for game_query, game_info in game_information.items():
        result_header = f"{game_query}{arrow}"
        query_result = (
            game_info if game_info else f"No results found for {game_query!r}."
        )
        result = "\n".join(
            (
                result_header,
                "-" * (len(result_header) - len(arrow)),
                "",
                str(query_result),
                "\n",
            )
        )

        print(result)


if __name__ == "__main__":
    games = [
        "slay the spire",
        "civ 6",
        "funkytown",
        "path of exile",
        "bless unleashed",
        "the witcher 3 wild hunt game of the year",
        "divinity source",
        "monkey island collection",
        "star wars squadrons",
        "spirit of glace",
    ]
    game_info = get_game_information(games)
    display_game_information(game_info)
| improve this answer | |
\$\endgroup\$
  • \$\begingroup\$ Why use NamedTuple instead of dataclasses.datadclass? \$\endgroup\$ – Maarten Fabré Aug 29 at 18:37
  • \$\begingroup\$ @MaartenFabré Absolutely dataclass is another good option, which I can mention along with NamedTuple. The data containers (e.g. SteamGame) in my example script don't require mutability since we're just loading data into them once and printing them out later, so I chose to make them NamedTuples. I know that dataclasses can also fulfill the same purpose via dataclass(frozen=True), but whenever the situation calls for a simple, lightweight, immutable by default container, I generally prefer NamedTuple. \$\endgroup\$ – Setris Aug 29 at 19:26

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Not the answer you're looking for? Browse other questions tagged or ask your own question.