5
\$\begingroup\$

This is a scraper written to do most of what had been attempted by another user in this question:

How can I optimise this webscraping code

I did the rewrite because I felt bad that the new user didn't get an opportunity to talk about their code (even though the off-topic closure was correct). Also the site, as with so many others, is badly designed enough to be an interesting challenge. The design objective of this code is to scrape an arbitrary number of years and pages from OddsPortal without needing to involve a browser (i.e. Selenium).

On the balance, there are advantages and disadvantages to using Requests: when I lose the layer of abstraction provided by Selenium, I gain speed but also the code is probably more fragile given some of the hoops I have to jump through to decode their timestamp and odds formats. Any feedback welcome.

import json
import re
from dataclasses import dataclass, asdict
from datetime import datetime, timezone
from pprint import pprint
from typing import Dict, Any, Tuple, Optional, Iterable
from urllib.parse import urljoin

from requests import Session
from bs4 import BeautifulSoup, Tag


BASE = 'https://www.oddsportal.com'
TERRIBLE_TIMESTAMP_PAT = re.compile(r'^t(\d+)-')


def decode_terrible_timestamp_class(tag: Tag) -> datetime:
    for cls in tag['class']:
        match = TERRIBLE_TIMESTAMP_PAT.search(cls)
        if match:
            break
    else:
        raise ValueError(f'{tag} does not seem to contain a valid timestamp')

    stamp = int(match[1])
    return datetime.fromtimestamp(stamp, tz=timezone.utc)


# Refer to https://www.oddsportal.com/res/x/global-210609145530.js
# this.d = function (str)
ODDS_TABLE = str.maketrans(
    'axcteopzf',
    '1234567.|',
)


def decode_terrible_odds(decorated: Tag) -> Tuple[float, Optional[float]]:
    odds = decorated['xodd'].translate(ODDS_TABLE).split('|')

    if len(odds) == 1:
        return float(odds), None

    preferred, normal = odds
    return float(normal), float(preferred)


@dataclass
class GameData:
    sport: str
    sport_path: str
    country: str
    country_path: str
    season: str
    season_path: str

    when: datetime
    path: str

    home_team: str
    away_team: str
    home_team_won: bool
    away_team_won: bool
    home_score: int
    away_score: int

    home_odds: float
    home_odds_preferred: float
    draw_odds: float
    draw_odds_preferred: float
    away_odds: float
    away_odds_preferred: float

    bookmakers: int

    @classmethod
    def from_tags(
        cls,
        sport: Tag,
        country: Tag,
        season: Tag,
        date_span: Tag,
        time: Tag,
        teams: Tag,
        score: Tag,
        home_odds: Tag,
        draw_odds: Tag,
        away_odds: Tag,
        bookmakers: Tag,
    ):
        home_score, away_score = score.text.split(':')

        when = datetime.combine(
            decode_terrible_timestamp_class(date_span).date(),
            decode_terrible_timestamp_class(time).time(),
            tzinfo=timezone.utc,
        )

        team_anchor = teams.find('a')
        any_span = team_anchor.find('span')
        if any_span:
            home_team, away_team = (
                t.text if isinstance(t, Tag) else t.strip('- ')
                for t in team_anchor.children
            )
            home_team_won, away_team_won = (
                isinstance(t, Tag)
                for t in team_anchor.children
            )
        else:
            home_team, away_team = team_anchor.text.split('-')
            home_team_won, away_team_won = False, False

        home_odds_norm, home_odds_pref = decode_terrible_odds(home_odds)
        draw_odds_norm, draw_odds_pref = decode_terrible_odds(draw_odds)
        away_odds_norm, away_odds_pref = decode_terrible_odds(away_odds)

        game = cls(
            sport=sport.text.strip(),
            sport_path=sport['href'],
            country=country.text.strip(),
            country_path=country['href'],
            season=season.text.strip(),
            season_path=season['href'],
            when=when,
            path=team_anchor['href'],
            home_team=home_team,
            away_team=away_team,
            home_team_won=home_team_won,
            away_team_won=away_team_won,
            home_score=int(home_score),
            away_score=int(away_score),
            home_odds=home_odds_norm,
            home_odds_preferred=home_odds_pref,
            draw_odds=draw_odds_norm,
            draw_odds_preferred=draw_odds_pref,
            away_odds=away_odds_norm,
            away_odds_preferred=away_odds_pref,
            bookmakers=int(bookmakers.text),
        )
        return game


def start_search(session: Session, year: int) -> Dict[str, Any]:
    url = urljoin(BASE, f'/soccer/england/premier-league-{year}-{year+1}/results/')
    with session.get(url) as resp:
        resp.raise_for_status()
        doc = BeautifulSoup(resp.text, 'lxml')

    script_pat = re.compile(r'new PageTournament\(')
    for script in doc.select('body > script'):
        match = script_pat.search(script.string)
        if match:
            break
    else:
        raise ValueError('ID script not found')

    start = match.end()
    end = script.string.find('}', start) + 1
    json_str = script.string[start: end]
    ids = json.loads(json_str)
    return ids


def get_page(session: Session, sid: int, id: str, page: int) -> Iterable[GameData]:
    bookie_hash = 'X0'
    use_premium = 1
    timezone_offset = 0

    archive_url = urljoin(
        BASE,
        f'/ajax-sport-country-tournament-archive'
        f'/{sid}'
        f'/{id}'
        f'/{bookie_hash}'
        f'/{use_premium}'
        f'/{timezone_offset}'
        f'/{page}'
        f'/'
    )

    with session.get(archive_url) as resp:
        resp.raise_for_status()
        start = resp.text.find('{')
        end = resp.text.rfind('}') + 1
        json_str = resp.text[start: end]

    html = json.loads(json_str)['d']['html']
    doc = BeautifulSoup(html, 'lxml')

    head = doc.find('th')
    sport, country, season = head.find_all('a')

    for tr in doc.find_all('tr'):
        date_header = tr.select_one('th[colspan="3"]')
        if date_header:
            date_span = date_header.find('span')
            continue

        if tr.select_one('td.table-time'):
            yield GameData.from_tags(
                sport, country, season, date_span,
                *tr.find_all('td'),
            )


def main():
    with Session() as session:
        session.headers = {'User-Agent': 'Mozilla/5.0'}

        ids = start_search(session, year=2020)

        for page in range(1, 3):
            for game in get_page(session, ids['sid'], ids['id'], page):
                pprint(asdict(game))


if __name__ == '__main__':
    main()
\$\endgroup\$
6
  • \$\begingroup\$ What are your objectives? Your code looks quite WET, and the use of a single god class looks questionable. You could probably have more maintainable code by using dataclasses.field and a very 'meta programming' approach with untyped/partially typed input. However I think such an approach would conflict with your objectives. \$\endgroup\$
    – Peilonrayz
    Jun 25, 2021 at 18:38
  • \$\begingroup\$ This is great however the code looks to be customised for just one URL. f'/soccer/england/premier-league-{year}-{year+1}/results/' While there are more than one competitions covered by the webiste. Can we edit the code to accomodate more than one competitions? \$\endgroup\$
    – PyNoob
    Jul 7, 2021 at 3:19
  • \$\begingroup\$ @PyNoob Yes - just add the path as a parameter to start_search \$\endgroup\$
    – Reinderien
    Jul 7, 2021 at 3:21
  • \$\begingroup\$ @Reinderien could you elaborate on what's wrong with the design of OddsPortal? \$\endgroup\$ Sep 20, 2022 at 19:25
  • \$\begingroup\$ I know this is a year after but I'd like to ask HOW do you know what to decode and how? I've been scraping with Selenium for years but compared to requests it's so slow. Al though when I try to scrape Oddsportal.com I don't get the content when I just do requests/session.get('some_url'). I think I need to decode like you did but I can't figure out how to do that. I've been in developer tab and watched all JS queries and in network tab also but can't figure out. Like for example how did you how to build the "whole \$\endgroup\$
    – Jaz
    Dec 8, 2022 at 18:31

0

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service and acknowledge you have read our privacy policy.

Browse other questions tagged or ask your own question.