I'm building a selenium web scraper for basketball-reference.com that takes a player name and returns data in either a JSON format or Pandas DataFrame object. The class in question is one of many that will scrape a particular table on a player's basketball reference page, in this case the per-season totals. I have been using Python 3.x for a while now, and while my code does work, I am looking for ways to improve the structure, make it more 'pythonic' and just generally improve the implementation itself.
Here is the abstract class blueprint of which the different scraping classes originate:
from abc import ABC, abstractmethod
"""
Structured base class for an NBA player's stat profils on their home page
(eg. https://www.basketball-reference.com/players/w/wembavi01.html)
Abstract class to scrape and parse player data, with selenium as basketball reference
now only loads dynamic data
Methods extract desired column header, row data, parses them and zips in one list of
dictionaries, cleans and packages data with pandas
"""
class BasePlayerStats(ABC):
@abstractmethod
def get_player_column_headers(self) -> list:
pass
@abstractmethod
def get_player_row_stats(self) -> list:
pass
@abstractmethod
def parse_player_stats(self, key_list, value_list) -> list:
pass
@abstractmethod
def clean_player_stats(self, player_data_dic) -> None:
pass
And here is the implementation of said base class for per-season totals:
from base_stats_class.base_player_stat_class import BasePlayerStats
from selenium import webdriver
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.options import Options
import pandas as pd
"""
Season total stats
Takes player name and uses it to reference basketball reference's player stats page,
Inits selenium webdriver with options for headless browser and not loading images for faster performance
__call__ method performs like runtime script, aggregating columns and rows of stats table
and concatenates them to dictionary, then to pandas dataframe
Other helper methods retrieve columns headers and rows, and perform utility functions
to package data into format that can be output to a json-like format and then to dataframe
"""
class PlayerSeasonTotalStats(BasePlayerStats):
def __init__(self, player_name):
self.player_name = player_name
self.options = Options()
self.options.add_argument("--headless=new")
self.options.add_experimental_option(
"prefs", {
"profile.managed_default_content_settings.images" : 2,
}
)
self.browser = webdriver.Chrome(options=self.options)
self.browser.get(f"https://www.basketball-reference.com/players/{self.player_name[0]}/{self.player_name}01.html")
def __call__(self):
print("Scraping per game column data...")
(columns := self.get_player_column_headers())
print("Scraping per game row data...")
(rows := self.get_player_row_stats())
print("Scrape ok...\n")
print("Parsing data...")
(player_dict := self.parse_player_stats(columns, rows))
print("Constructing dataframe...")
self.clean_player_stats(player_dict)
def get_player_column_headers(self) -> list:
"""
Scrapes page with selenium and xpath methods, returns list of column headers
"""
try:
table = self.browser.find_element(By.ID, 'totals')
headers = table.find_elements(By.XPATH, './thead/tr')
column_headers = [header.text for header in headers[0].find_elements(By.XPATH, './th[not(contains(@data-stat, "DUMMY"))]')]
#Test print
#print(column_headers)
return column_headers
except TimeoutException:
self.browser.quit()
def get_player_row_stats(self) -> list:
"""
Scrapes page with selenium and xpath methods, returns list of row stats for each row
"""
try:
table = self.browser.find_element(By.ID, 'totals')
rows = table.find_elements(By.XPATH, './tbody')
stat_rows = [row.text for row in rows[0].find_elements(By.XPATH, './tr')]
player_data = [y for x in stat_rows for y in x.split(' ')]
#Test print
#print(player_data)
return player_data
except TimeoutException:
self.browser.quit()
def parse_player_stats(self, key_list, value_list) -> list:
"""
Parses both column headers and row values, packages them into list of dictionaries for each row
Init empty list
Append empty list as dictionary, zip the key_list (column headers), and value_list(rows), slices row
list from zero to the length of the header list, for each value in range (0, start: length of row, step: length of
column headers)
"""
out = []
out += [dict(zip(key_list, value_list[i: i + len(key_list)])) for i in range(0, len(value_list), len(key_list))]
#Test print
#print(out)
return out
def clean_player_stats(self, player_data_dic) -> None:
player_df = pd.DataFrame(data=player_data_dic)
#Test print
#print(player_df.to_string())
return player_df
Looking at it, I can see it's not the most advanced implementation, and I would just like to know what can be changed and what to keep in mind as I extend this project further. Anything from cleaning up the object-oriented side of things, improving the selenium setup or code structure would be very helpful.
