This is my first program code ever, and it actually works. My goal is to scrape information from the website and store it in a database. It is a site that has historical data on sporting events and odds.
https://www.oddsportal.com/hockey/sweden/shl-2019-2020/results/
For example, there are 50 matches on this page and the program enters each of them and scrapes this data.
However, I don't think it really looks like the real code. I'm interested in what I can do to improve and optimize it. What do I pay the most attention to, what things do I do wrong?
from selenium import webdriver
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import NoSuchElementException, TimeoutException
from selenium.webdriver.common.keys import Keys
import sqlite3 as sql
import time
# I disable image display to get up to speed
options = webdriver.ChromeOptions()
prefs = {"profile.managed_default_content_settings.images": 2}
options.add_experimental_option("prefs", prefs)
driver = webdriver.Chrome(options=options)
# Database connection
conn = sql.connect('Hockey_data.db')
c = conn.cursor()
driver.get("https://www.oddsportal.com/hockey/sweden/shl-2019-2020/results/")
driver.maximize_window()
cur_win = driver.current_window_handle
list_of_links = driver.find_elements_by_xpath("//td[2]/a")[0:-2] # because 2 more elements have the same xpath and they are always at the end
for index, link in enumerate(list_of_links):
link.send_keys(Keys.CONTROL + Keys.RETURN)
driver.switch_to.window([win for win in driver.window_handles if win !=cur_win][0])
# Basic game info
league = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.XPATH, "/html/body/div[1]/div/div[2]/div[6]/div[1]/div/div[1]/div[1]/a[4]"))).text
game = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.XPATH, "/html/body/div[1]/div/div[2]/div[6]/div[1]/div/div[1]/div[2]/div[1]/h1"))).text
game_split = game.split('- ')
home_team, away_team = game_split[0], game_split[1]
time = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.XPATH, "/html/body/div[1]/div/div[2]/div[6]/div[1]/div/div[1]/div[2]/div[1]/p[1]"))).text
time_split = time.split(', ')
day_in_week = time_split[0]
date = time_split[1]
day = int(date[0:2])
month = date[3:7]
year = int(date[7:11])
hour = time_split[2]
# Result and goals by periods
try:
result = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.ID, "event-status"))).text
except NoSuchElementException:
result = "0000000000000000000000000000000000000000000000000000000000000000000000"
if len(result) == 32:
home_goals = int(result[13:14])
home_first = int(result[18:19])
home_second = int(result[23:24])
home_third = int(result[28:29])
home_overtime = 0
home_penalties = 0
away_goals = int(result[15:16])
away_first = int(result[20:21])
away_second = int(result[25:26])
away_third = int(result[30:31])
away_overtime = 0
away_penalties = 0
elif len(result) == 33:
home_goals = int(result[13:15])
home_first = int(result[19:20])
home_second = int(result[24:25])
home_third = int(result[29:30])
home_overtime = 0
home_penalties = 0
away_goals = int(result[16:17])
away_first = int(result[21:22])
away_second = int(result[26:27])
away_third = int(result[31:32])
away_overtime = 0
elif len(result) == 40:
home_goals = int(result[13:14])
home_first = int(result[21:22])
home_second = int(result[26:27])
home_third = int(result[31:32])
home_overtime = int(result[36:37])
home_penalties = 0
away_goals = int(result[15:16])
away_first = int(result[23:24])
away_second = int(result[28:29])
away_third = int(result[33:34])
away_overtime = int(result[38:39])
away_penalties = 0
elif len(result) == 52:
home_goals = int(result[13:14])
home_first = int(result[28:29])
home_second = int(result[33:34])
home_third = int(result[38:39])
home_overtime = int(result[43:44])
home_penalties = int(result[48:49])
away_goals = int(result[15:16])
away_first = int(result[30:31])
away_second = int(result[35:36])
away_third = int(result[40:41])
away_overtime = int(result[45:46])
away_penalties = int(result[50:51])
else:
home_goals = 0
home_first = 0
home_second = 0
home_third = 0
home_overtime = 0
home_penalties = 0
away_goals = 0
away_first = 0
away_second = 0
away_third = 0
away_overtime = 0
away_penalties = 0
# 1x2 odds
pinnacle = "Pinnacle"
try:
pinnacle_home = driver.find_element_by_xpath("//div[a[contains(text(), 'Pinnacle')]]/following::td[1]")
hov_pinnacle_home = ActionChains(driver).move_to_element(pinnacle_home)
hov_pinnacle_home.perform()
pinnacle_home_closing = driver.find_element_by_xpath("//*[@id='tooltiptext']/strong[1]").text
try:
pinnacle_home_opening = driver.find_element_by_xpath("//*[@id='tooltiptext']/strong[2]").text
except (NoSuchElementException, TimeoutException):
pinnacle_home_opening = pinnacle_home_closing
except (NoSuchElementException, TimeoutException):
pinnacle_home = "0000000000"
pinnacle_home_closing = 0.00
pinnacle_home_opening = 0.00
try:
pinnacle_draw = driver.find_element_by_xpath("//div[a[contains(text(), 'Pinnacle')]]/following::td[2]")
hov_pinnacle_draw = ActionChains(driver).move_to_element(pinnacle_draw)
hov_pinnacle_draw.perform()
pinnacle_draw_closing = driver.find_element_by_xpath("//*[@id='tooltiptext']/strong[1]").text
try:
pinnacle_draw_opening = driver.find_element_by_xpath("//*[@id='tooltiptext']/strong[2]").text
except (NoSuchElementException, TimeoutException):
pinnacle_draw_opening = pinnacle_draw_closing
except (NoSuchElementException, TimeoutException):
pinnacle_draw = "0000000000"
pinnacle_draw_closing = 0.00
pinnacle_draw_opening = 0.00
try:
pinnacle_away = driver.find_element_by_xpath("//div[a[contains(text(), 'Pinnacle')]]/following::td[3]")
hov_pinnacle_away = ActionChains(driver).move_to_element(pinnacle_away)
hov_pinnacle_away.perform()
pinnacle_away_closing = driver.find_element_by_xpath("//*[@id='tooltiptext']/strong[1]").text
try:
pinnacle_away_opening = driver.find_element_by_xpath("//*[@id='tooltiptext']/strong[2]").text
except (NoSuchElementException, TimeoutException):
pinnacle_away_opening = pinnacle_away_closing
except (NoSuchElementException, TimeoutException):
pinnacle_away = "0000000000"
pinnacle_away_closing = 0.00
pinnacle_away_opening = 0.00
average = "Average"
try:
average_home = driver.find_element_by_xpath("//td[strong[contains(text(), 'Average')]]/following::td[1]").text
except NoSuchElementException:
average_home = 0.00
try:
average_draw = driver.find_element_by_xpath("//td[strong[contains(text(), 'Average')]]/following::td[2]").text
except NoSuchElementException:
average_draw = 0.00
try:
average_away = driver.find_element_by_xpath("//td[strong[contains(text(), 'Average')]]/following::td[3]").text
except NoSuchElementException:
average_away = 0.00
highest = "Highest"
try:
highest_home = driver.find_element_by_xpath("//td[strong[contains(text(), 'Highest')]]/following::td[1]").text
except NoSuchElementException:
highest_home = 0.00
try:
highest_draw = driver.find_element_by_xpath("//td[strong[contains(text(), 'Highest')]]/following::td[2]").text
except NoSuchElementException:
highest_draw = 0.00
try:
highest_away = driver.find_element_by_xpath("//td[strong[contains(text(), 'Highest')]]/following::td[3]").text
except NoSuchElementException:
highest_away = 0.00
# Over-Under tab
OU_link = WebDriverWait(driver, 5).until(EC.presence_of_element_located((By.XPATH, "/html/body/div[1]/div/div[2]/div[6]/div[1]/div/div[1]/div[2]/div[1]/div[5]/div[1]/ul/li[5]/a/span"))).click()
pinnacle_over_under = "Pinnacle 5.5"
# 5.5 over-under tab
try:
OU_55 = WebDriverWait(driver, 5).until(EC.presence_of_element_located((By.XPATH, "//div/div/strong/a[contains(text(), 'Over/Under +5.5')]"))).click()
try:
pinnacle_over = driver.find_element_by_xpath("//div[a[contains(text(), 'Pinnacle')]]/following::td[2]")
hov_pinnacle_over = ActionChains(driver).move_to_element(pinnacle_over)
hov_pinnacle_over.perform()
pinnacle_over_closing = driver.find_element_by_xpath("//*[@id='tooltiptext']/strong[1]").text
try:
pinnacle_over_opening = driver.find_element_by_xpath("//*[@id='tooltiptext']/strong[2]").text
except (NoSuchElementException, TimeoutException):
pinnacle_over_opening = pinnacle_over_closing
except (NoSuchElementException, TimeoutException):
pinnacle_over = "0000000000"
pinnacle_over_opening = 0.00
pinnacle_over_closing = 0.00
try:
pinnacle_under = driver.find_element_by_xpath("//div[a[contains(text(), 'Pinnacle')]]/following::td[3]")
hov_pinnacle_under = ActionChains(driver).move_to_element(pinnacle_under)
hov_pinnacle_under.perform()
pinnacle_under_closing = driver.find_element_by_xpath("//*[@id='tooltiptext']/strong[1]").text
try:
pinnacle_under_opening = driver.find_element_by_xpath("//*[@id='tooltiptext']/strong[2]").text
except (NoSuchElementException, TimeoutException):
pinnacle_under_opening = pinnacle_over_closing
except (NoSuchElementException, TimeoutException):
pinnacle_under = "0000000000"
pinnacle_under_opening = 0.00
pinnacle_under_closing = 0.00
except (NoSuchElementException, TimeoutException):
OU_55 = "0000000000"
pinnacle_under_opening = 0.00
pinnacle_under_closing = 0.00
average_over_under = "Average 5.5"
try:
average_over = driver.find_element_by_xpath("//td[strong[contains(text(), 'Average')]]/following::td[2]").text
except NoSuchElementException:
average_over = 0.00
try:
average_under = driver.find_element_by_xpath("//td[strong[contains(text(), 'Average')]]/following::td[3]").text
except NoSuchElementException:
average_under = 0.00
highest_over_under = "Highest 5.5"
try:
highest_over = driver.find_element_by_xpath("//td[strong[contains(text(), 'Highest')]]/following::td[2]").text
except NoSuchElementException:
highest_over = 0.00
try:
highest_under = driver.find_element_by_xpath("//td[strong[contains(text(), 'Highest')]]/following::td[3]").text
except NoSuchElementException:
highest_under = 0.00
c.execute('insert into Model values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)',
(league, home_team, away_team, day_in_week, day, month, year, hour,
home_goals, away_goals, home_first, away_first, home_second,away_second, home_third, away_third,home_overtime, away_overtime, home_penalties, away_penalties,
pinnacle, pinnacle_home_closing, pinnacle_draw_closing, pinnacle_away_closing, pinnacle_home_opening, pinnacle_draw_opening, pinnacle_away_opening,
average, average_home, average_draw, average_away, highest, highest_home, highest_draw, highest_away,
pinnacle_over_under, pinnacle_over_closing, pinnacle_under_closing, pinnacle_over_opening, pinnacle_under_opening,
average_over_under, average_over, average_under, highest_over_under, highest_over, highest_under,))
conn.commit()
print("Number: ", index)
driver.close()
driver.switch_to.window(cur_win)