Design Pattern: Builder - BeautifulSoup directory navigation and scraping

Question

I wrote a class on top of BeautifulSoup using the builder design pattern that allows for the navigation of the necp data directory.

There are a couple navigation methods navto which just builds upon the base url to return a new instance, and inav useful when the urls are likely to change due to temporal updates.

import time
import re
import pandas as pd
import requests
from bs4 import BeautifulSoup

# for context TSRAGR is TAF code for a thunderstorm with hail

class TSragr:
    def __init__(self, base_url: str = None) -> None:

        self._baseurl = base_url

        r = requests.get(base_url)
        r.raise_for_status()

        soup = BeautifulSoup(r.content, "lxml").find_all("a")

        if soup[0].text == "Parent Directory":
            soup = soup[1:]

        self._soup = pd.Series([x.text for x in soup])

    def __repr__(self) -> str:
        return f"{self.url}\n"+ self._soup.__repr__()

    def __getitem__(self, args) -> "TSragr":
        self._soup = self._soup[args]
        return self

    @property
    def url(self) -> str:
        url = self._baseurl
        if not url.endswith("/"):
            url = url+ "/"
        return url

    def navto(self, *args: str) -> "TSragr":
        return TSragr(self.url + "/".join(args))

    def navup(self) -> "TSragr":
        return TSragr(re.match(r"^(.*[\/])", self.url).group())

    def inav(self, index: int) -> "TSragr":
        return TSragr(self.url + self._soup[index])

    def download(self, save_to="./", wait: float = 10) -> None:

        soup = self._soup.copy()
        soup.index = self.url + self._soup

        for url, filename in soup.items():
            print("DOWNLAODING FILE")
            local_filename = save_to + filename
            with requests.get(url, stream=True) as r:
                r.raise_for_status()
                with open(local_filename, "wb") as f:
                    for chunk in r.iter_content(chunk_size=8192):
                        f.write(chunk)

            print("FILE SAVED")
            time.sleep(60 * wait)

usage

>>> from wxlab.scrape import TSragr
>>> ragr = TSragr("https://nomads.ncep.noaa.gov/pub/data")
>>> ragr
https://nomads.ncep.noaa.gov/pub/data/
0     DSRC
1    nccf/
dtype: object
>>> ragr.navto("nccf")
https://nomads.ncep.noaa.gov/pub/data/nccf/
0            charts/
1               com/
2              dcom/
3    nonoperational/
4              pcom/
5             radar/
dtype: object
>>> ragr.navto("nccf","com")
https://nomads.ncep.noaa.gov/pub/data/nccf/com/
0               557ww/
1     amsu_estimation/
2                 aqm/
3                arch/
4               blend/
            ...       
61                uvi/
62               wave/
63        wave_nfcens/
64                wfs/
65          wsa_enlil/
Length: 66, dtype: object
>>> ragr.navto("nccf","com","blend")
https://nomads.ncep.noaa.gov/pub/data/nccf/com/blend/
0    prod/
1    v4.0/
dtype: object
>>> ragr.navto("nccf","com","blend","prod")
https://nomads.ncep.noaa.gov/pub/data/nccf/com/blend/prod/
0    blend.20220604/
1    blend.20220605/
dtype: object
>>> ragr.navto("nccf","com","blend","prod").inav(0)
https://nomads.ncep.noaa.gov/pub/data/nccf/com/blend/prod/blend.20220604/
0     00/
1     01/
2     02/
3     03/
4     04/
5     05/
6     06/
7     07/
8     08/
9     09/
10    10/
11    11/
12    12/
13    13/
14    14/
15    15/
16    16/
17    17/
18    18/
19    19/
20    20/
21    21/
22    22/
23    23/
dtype: object
>>> ragr.navto("nccf","com","blend","prod").inav(0).inav(0)
https://nomads.ncep.noaa.gov/pub/data/nccf/com/blend/prod/blend.20220604/00/
0    core/
1     qmd/
2    text/
dtype: object
>>> ragr.navto("nccf","com","blend","prod").inav(0).inav(0).navto("core")
https://nomads.ncep.noaa.gov/pub/data/nccf/com/blend/prod/blend.20220604/00/core/
0           blend.t00z.core.f001.ak.grib2
1       blend.t00z.core.f001.ak.grib2.idx
2           blend.t00z.core.f001.co.grib2
3       blend.t00z.core.f001.co.grib2.idx
4           blend.t00z.core.f001.gu.grib2
                      ...                
1148        blend.t00z.core.f264.oc.grib2
1149    blend.t00z.core.f264.oc.grib2.idx
1150        blend.t00z.core.f264.pr.grib2
1151    blend.t00z.core.f264.pr.grib2.idx
1152                                 ls-l
Length: 1153, dtype: object
>>> ragr.navto("nccf","com","blend","prod").inav(0).inav(0).navto("core")[0:6:2]
https://nomads.ncep.noaa.gov/pub/data/nccf/com/blend/prod/blend.20220604/00/core/
0    blend.t00z.core.f001.ak.grib2
2    blend.t00z.core.f001.co.grib2
4    blend.t00z.core.f001.gu.grib2
dtype: object
>>> ragr.navto("nccf","com","blend","prod").inav(0).inav(0).navto("core")[0:6:2].download(save_to="/media/external/data/", wait=1)
DOWNLAODING FILE
FILE SAVED
DOWNLAODING FILE
FILE SAVED

Answer 1

Fundamentally you're scraping the default directory listing format for Apache, which is the web server used by this NOAA site.

I see no use for Pandas here; I've removed it in my example code.

You should re-think your class. Rather than attempting to be an eager-fetched directory listing, it's better represented as a lazy-fetched directory listing that can be navigable either to child instances of subdirectories or of files, which amount to a different class. I don't think it's all that helpful to write your magic __getitem__ method; if you have an iterable, represent it directly as an iterable from a .children() method.

Use a soup strainer where applicable to narrow your DOM tree search.

Your Parent Directory check can be carried over to the BeautifulSoup DOM search by use of a regular expression.

I see no reason to wait; delete your sleep.

Suggested code

import re
from itertools import islice
from pathlib import Path
from shutil import copyfileobj
from typing import Iterator, Optional
from urllib.parse import urljoin

from requests import Session
from bs4 import BeautifulSoup
from bs4.element import SoupStrainer


class ApacheNode:
    """
    This represents a tree node in the Apache directory page for the NOAA NOMADS site;
    for more details see https://nomads.ncep.noaa.gov/

    It's used to look up TSRAGR data. TSRAGR is Terminal Aerodrome Forecast (TAF) code
    for a thunderstorm with hail.
    """
    def __init__(self, session: Session, url: str, name: Optional[str] = None) -> None:
        self.session = session
        self.url = url

        if name is None:
            name = url.removesuffix('/').rsplit('/', maxsplit=1)[-1]
        self.name = name


    def __repr__(self) -> str:
        return self.url


class ApacheFile(ApacheNode):
    def download(self, save_to: Path) -> None:
        print(f'Downloading {self.name}...', end=' ')
        local_filename = save_to / self.name

        with self.session.get(self.url, stream=True) as response:
            response.raise_for_status()
            with local_filename.open('wb') as f:
                copyfileobj(response.raw, f)

        print('saved')


class ApacheDir(ApacheNode):
    pre_strainer = SoupStrainer(name='pre')

    # Text has at least one character and cannot contain Parent Directory
    link_pattern = re.compile(
        '(?i)'                          # ignore case
        '^'                             # string start
            '(?:'                       # non-capturing group
                '(?!parent directory)'  # negative lookahead: don't match 'parent directory'
                '.'                     # match any one character
            ')+'                        # match one or more of the above chars
        '$'                             # string end
    )

    def __init__(self, session: Session, url: str, name: Optional[str] = None) -> None:
        if not url.endswith('/'):
            url += '/'
        super().__init__(session, url, name)

    def children(self) -> Iterator[ApacheNode]:
        with self.session.get(self.url) as response:
            response.raise_for_status()
            soup = BeautifulSoup(markup=response.text, features='lxml', parse_only=self.pre_strainer)
        pre = soup.pre
        anchors = pre.find_all(name='a', text=self.link_pattern, recursive=False)

        for anchor in anchors:
            child_name = anchor['href']
            child_url = urljoin(self.url, child_name)
            size_text = anchor.next_sibling.strip()
            if size_text.endswith('-'):
                child_type = ApacheDir
            else:
                child_type = ApacheFile
            yield child_type(self.session, child_url, child_name)

    def navto(self, *args: str) -> 'ApacheDir':
        url = urljoin(self.url, '/'.join(args))
        return ApacheDir(self.session, url=url, name=args[-1])

    def inav(self, index: int) -> 'ApacheNode':
        child, = islice(self.children(), index, index+1)
        return child


def test() -> None:
    with Session() as session:
        ragr = ApacheDir(session, url='https://nomads.ncep.noaa.gov/pub/data')
        print(ragr)
        print(ragr.navto('nccf'))
        print(ragr.navto('nccf', 'com'))
        print(ragr.navto('nccf', 'com', 'blend'))

        ragr = ragr.navto('nccf', 'com', 'blend', 'prod')
        print(ragr)
        ragr = ragr.inav(0)
        print(ragr)
        ragr = ragr.inav(0)
        print(ragr)
        ragr = ragr.navto('core')
        print(ragr)

        # anything that doesn't end in idx, first three
        first_three_files = islice(
            (
                child for child in ragr.children()
                if not child.name.endswith('idx')
            ), 3
        )

        save_to = Path('ragr-test')  # '/media/external/data/'
        save_to.mkdir(exist_ok=True)
        for file in first_three_files:
            file.download(save_to)


if __name__ == '__main__':
    test()

Output

https://nomads.ncep.noaa.gov/pub/data/
https://nomads.ncep.noaa.gov/pub/data/nccf/
https://nomads.ncep.noaa.gov/pub/data/nccf/com/
https://nomads.ncep.noaa.gov/pub/data/nccf/com/blend/
https://nomads.ncep.noaa.gov/pub/data/nccf/com/blend/prod/
https://nomads.ncep.noaa.gov/pub/data/nccf/com/blend/prod/blend.20220605/
https://nomads.ncep.noaa.gov/pub/data/nccf/com/blend/prod/blend.20220605/00/
https://nomads.ncep.noaa.gov/pub/data/nccf/com/blend/prod/blend.20220605/00/core/
Downloading blend.t00z.core.f001.ak.grib2... saved
Downloading blend.t00z.core.f001.co.grib2... saved
Downloading blend.t00z.core.f001.gu.grib2... saved