Football Web Scraper Part 2

Question

A revised version of the code in this question.

Things I have done so far:

• Adjusted some formatting things like constant & variable naming and indentation
• Wrapped most of the functions into a Scraper class
• Improved CSS selectors
• Improved performance by using a session instead of several requests
• Wrapped the execution part in a main() function

Doubts:

• I'm unsure if I use main() correctly here or if I should even use it at all. If I run the code as a script like this, is everything executed, even what's not inside the main function? I guess it has to, because otherwise the function definitions wouldn't be available.
• I think the part where I'm dropping and renaming columns could be done better, best would probably not to store them in the first place
• In the beginning of the Scraper class definition PyCharm shows me warning about instance attributes being assigned outside of the __init__ method. Why is that a problem?
• In the store_data function for all instances of seasons["Saison"] PyCharm shows me a warning "Expected type 'Union[int, slice]', got 'str' instead" and I don't know why.

New code:

import os
import requests
import pandas as pd
from bs4 import BeautifulSoup

# base url for own scrape
BASE_URL = "http://www.weltfussball.de/"

LIGEN = [["Liga1", "alle_spiele/bundesliga-2017-2018/"],
         ["Liga2", "alle_spiele/2-bundesliga-2017-2018/"],
         ["Liga3", "alle_spiele/3-liga-2017-2018/"]]


class Scraper(object):
    def __init__(self):
        self.session = requests.Session()

    def get_page(self, ext):
        # get the page and make soup
        if ext[0] == "/":
            self.ext = ext[1:]
        else:
            self.ext = ext
        url = BASE_URL + self.ext
        r = self.session.get(url)
        self.page = r.content.decode('utf-8', 'ignore')
        self.soup = BeautifulSoup(self.page, "lxml")

    def get_seasons(self, liga):
        # Getting all seasons
        seasons = [
            [season.get_text(), season.get("value")]
            for season in self.soup.select("select[name=saison] option")
        ]
        self.seasons_df = pd.DataFrame(seasons, columns=["Saison", "Link"])
        self.seasons_df["Spiele"] = "empty"
        self.seasons_df["Liga"] = str(liga)

    def get_games(self, season_n):
        data_table = self.soup.select(".data .standard_tabelle")
        # df = pd.read_html(str(data_table))
        record = []
        records = []
        trs = data_table.findAll("tr")
        for tr in range(len(trs)):
            if trs[tr].contents[1].name == "th":
                th = trs[n].find("th")
                record.append(th.text)
            else:
                # appending the previous header if none is given
                if not record:
                    record.append(records[-1][0])
                tds = trs[tr].findAll("td")
                for td in range(len(tds)):
                    record.append(tds[td].text)
                # filling empty date column with previous value
                if record[1] == "":
                    record[1] = records[-1][1]
                records.append(record)
                record = []

        # Pasting it into a DataFrame
        df = pd.DataFrame(records)

        # Dropping empty columns
        spdf = pd.DataFrame()
        z = 0
        for key in df.keys():
            if df[key].all() in (" - ", "\n", "\n\n"):
                print("Column %s is empty" % key)
            else:
                print("Adding %s to game DF" % key)
                spdf[z] = df[key]
                z += 1

        # Defining the new column names
        names = {
            0: "Spieltag",
            1: "Datum",
            2: "Zeit",
            3: "Heim",
            4: "Auswärts",
            5: "Score"
        }

        # Renaming the columns
        spdf.rename(columns=names, inplace=True)
        # Storing the season in the league's season DF
        self.seasons_df["Spiele"][season_n] = spdf


def create_dir(season):
    saison = "saison-" + season.replace("/", "-")
    os.mkdir("data/%s" % saison)


def get_data():
    scraper = Scraper()
    for liga in range(len(LIGEN)):
        scraper.get_page("%s" % LIGEN[liga][1])
        scraper.get_seasons(LIGEN[liga][0])
        # Getting the data
        for i in scraper.seasons_df.index:
            if LIGEN[liga][0] == "Liga1":
                try:
                    create_dir(scraper.seasons_df["Saison"][i])
                except FileExistsError:
                    print("Directory %s exists already" % scraper.seasons_df["Saison"][i])
                else:
                    create_dir(scraper.seasons_df["Saison"][i])
            scraper.get_page(scraper.seasons_df["Link"][i])
            scraper.get_games(i)
        LIGEN[liga].append(scraper.seasons_df)


def store_data():
    for liga in range(len(LIGEN)):
        seasons = LIGEN[liga][2]
        for s in range(len(seasons["Saison"])):
            season = seasons["Saison"][s].replace("/", "-")
            season = season[0:9]
            seasons["Spiele"][s].to_csv("data/saison-%s/%s_Spiele.csv" % (season, LIGEN[liga][0]), sep=";")


def main(store=True):
    get_data()
    if store:
        store_data()


if __name__ == '__main__':
    status = main()
    # sys.exit(status)