I wrote a web scraper to get football scores from here. I'm getting the data for all seasons for the three major German leagues. It all works at the moment, but I'm sure it's possible to make it a lot more concise.
import requests
import os
import pandas as pd
from bs4 import BeautifulSoup
# base url for scrape
base_url = "http://www.weltfussball.de/"
ligen = [["Liga1", "bundesliga-2017-2018/"],
["Liga2", "2-bundesliga-2017-2018/"],
["Liga3", "3-liga-2017-2018/"]]
def get_page(ext):
# get the page and make soup
global soup, page
if (ext[0] == "/"):
ext = ext[1:]
url = base_url + ext
r = requests.get(url)
page = r.content.decode('utf-8','ignore')
soup = BeautifulSoup(page, "lxml")
def create_dir(s):
saison = "saison-" + s.replace("/", "-")
os.mkdir("data/%s" % saison)
def get_seasons(liga):
global seasons_df
# Getting all seasons
form = soup.find("select", attrs={"name": "saison"})
seasons = []
season = []
for s in form.find_all("option"):
season.append(s.text)
season.append(s.get("value"))
seasons.append(season)
season = []
seasons_df = pd.DataFrame(seasons, columns=["Saison", "Link"])
seasons_df["Spiele"] = "empty"
seasons_df["Liga"] = str(liga)
def get_games(season_n):
tables = []
for t in soup.find_all("table", "standard_tabelle"):
print(t.contents)
tables.append(t)
data_table = tables[0]
record = []
records = []
trs = data_table.findAll("tr")
for n in range(len(trs)):
if (trs[n].contents[1].name == "th"):
th = trs[n].find("th")
record.append(th.text)
else:
if not record:
record.append(records[-1][0])
tds = trs[n].findAll("td")
for p in range(len(tds)):
record.append(tds[p].text)
# filling empty date column with previous value
if (record[1] == ""):
record[1] = records[-1][1]
records.append(record)
record = []
# Pasting it into a DataFrame
df = pd.DataFrame(records)
# Dropping empty columns
spdf = pd.DataFrame()
z = 0
for key in df.keys():
if (df[key].all() in (" - ", "\n", "\n\n")):
print("Column %s is empty" % key)
else:
print("Adding %s to game DF" % key)
spdf[z] = df[key]
z += 1
# Renaming the columns
# Defining the new column names
names = {
0 : "Spieltag",
1 : "Datum",
2 : "Zeit",
3 : "Heim",
4 : "Auswärts",
5 : "Score"
}
spdf.rename(columns=names, inplace=True)
seasons_df["Spiele"][season_n] = spdf
# the execution starts here
for l in range(len(ligen)):
get_page("alle_spiele/%s" % ligen[l][1])
get_seasons(ligen[l][0])
# Getting the data
for i in seasons_df.index:
if (ligen[l][0] == "Liga1"):
try:
create_dir(seasons_df["Saison"][i])
except FileExistsError:
print("Directory %s exists already" % seasons_df["Saison"][i])
else:
create_dir(seasons_df["Saison"][i])
get_page(seasons_df["Link"][i])
get_games(i)
ligen[l].append(seasons_df)
for l in range(len(ligen)):
l1 = ligen[l][2]
for s in range(len(l1["Saison"])):
sss = l1["Saison"][s].replace("/", "-")
sss = sss[0:9]
l1["Spiele"][s].to_csv("data/saison-%s/%s_Spiele.csv" % (sss, ligen[l][0]), sep=";")
For example, I have to get all tables with class standard_tabelle
, because there are two in the page that have no otherwise distinctive attribute.
Data Table:
<div class="data">
<table class="standard_tabelle" cellpadding="3" cellspacing="1">
<tbody>
...
</tbody>
</table>
</div>
Table 2:
<table class="standard_tabelle" cellpadding="3" cellspacing="1">
<tr>
<td align="right"><b><a href="/news/2-bundesliga/1/">Aktuelle Meldungen »</a></b>
</td>
</tr>
</table>
I would like to select the data table based on its parent <div class="data">
, but there are other divs with the data class, so I can't just find div with class "data".