Scraping scores from flashscore.com

Question

I built a bot with Python to scrape scores on flashscore.com but the data scraped from the site loads into its listbox very slowly. I am curious about the speed of selenium so I made a button that prints all the text and it turns out the speed was fast so it must be the if elif block that is slowing down the program.

from tkinter import *
from selenium import webdriver
import threading



def LoadSite():

    lblStatus2.config(text="loading")
    m = 0
    Table = browser.find_elements_by_tag_name('table')
    for tables in Table:
        abc = tables.find_elements_by_class_name('country_part')
        aaa = tables.find_elements_by_class_name('tournament_part')
        C = len(tables.find_elements_by_class_name("padr"))

        for countrys in abc:
                    LbCountry.insert(END, countrys.text+aaa[m].text)
                    n = 1
                    while (n < C):
                        LbCountry.insert(END, "")
                        n += 1
    m +=1


    Time = browser.find_elements_by_tag_name('td')
    for g in Time:
        if (g.get_attribute('class').find('cell_ab team-home')) != -1 or (g.get_attribute('class').find('cell_ab team-home  bold')) != -1:
            LbHome.insert(END,g.text)
            if g.get_attribute('innerHTML').find('rhcard rhcard1') != -1:
                LbHRed.insert(END,"1")
            elif g.get_attribute('innerHTML').find('rhcard rhcard2') != -1:
                LbHRed.insert(END,"2")
            else:
                LbHRed.insert(END,"")

        elif (g.get_attribute('class').find('cell_ac team-away')) != -1 or (g.get_attribute('class').find('cell_ac team-away  bold')) != -1:
            LbAway.insert(END,g.text)
            if g.get_attribute('innerHTML').find('racard racard1') != -1:
                LbARed.insert(END,"1")
            elif g.get_attribute('innerHTML').find('racard racard2') != -1:
                LbARed.insert(END,"2")
            else:
                LbARed.insert(END,"")

        elif g.get_attribute('class').find('cell_sa score  bold') != -1:
            LbScore.insert(END,g.text)
        elif g.get_attribute('class').find('cell_sa score') != -1:
            LbScore.insert(END,g.text)
        elif g.get_attribute('class').find('cell_ad time  time-playing') != -1:
            LbScore.insert(END,g.text)
        elif g.get_attribute('class').find('cell_ad time') != -1:
            LbTime.insert(END,g.text)
        elif g.get_attribute('class').find('cell_ad time  time-playing') != -1:
            LbTime.insert(END,g.text)
        elif g.get_attribute('class').find('cell_aa timer') != -1:
            LbStatus.insert(END,g.text)
        elif g.get_attribute('class').find('cell_aa timer  playing') != -1:
            LbStatus.insert(END,g.text)
    lblStatus2.config(text="complete")




def tick():
    t = threading.Thread(name='LoadSite', target=LoadSite, args=())
    t.start()

def a():
    print(browser.page_source)

def b():
    Time = browser.find_elements_by_tag_name('td')
    for g in Time:
        print(g.text)

root = Tk()

LbCountry = Listbox(height = '30', width='10')
LbCountry.grid(row=1, column=0)
lblCountry = Label(text="Country")
lblCountry.grid(row=0, column=0)
LbTime = Listbox(height = '30', width='7')
LbTime.grid(row =1, column = 1)
lblTime = Label(text="Time")
lblTime.grid(row=0, column=1)
LbStatus = Listbox(height = '30')
LbStatus.grid(row =1, column = 2)
lblStatus = Label(text="Status")
lblStatus.grid(row=0, column=2)
LbHRed = Listbox(height = '30', width='3')
LbHRed.grid(row=1, column = 3)
lblHRed = Label(text="Red")
lblHRed.grid(row=0, column=3)
LbHome = Listbox(height = '30')
LbHome.grid(row =1, column = 4)
lblHome = Label(text="Home")
lblHome.grid(row=0, column=4)
LbScore = Listbox(height = '30', width='7')
LbScore.grid(row =1, column = 5)
lblScore = Label(text="Score")
lblScore.grid(row=0, column=5)
LbAway = Listbox(height = '30')
LbAway.grid(row =1, column = 6)
lblAway = Label(text="Away")
lblAway.grid(row=0, column=6)
LbARed = Listbox(height = '30', width='3')
LbARed.grid(row=1, column = 7)
lblARed = Label(text="Red")
lblARed.grid(row=0, column=7)

LbPossession = Listbox(height = '30', width='7')
LbPossession.grid(row=1, column = 8)
LbSOnT = Listbox(height = '30', width='7')
LbSOnT.grid(row=1, column = 9)
LbSOffT = Listbox(height = '30', width='7')
LbSOffT.grid(row=1, column = 10)
LbCorner = Listbox(height = '30', width='7')
LbCorner.grid(row=1, column = 11)
LbFoul = Listbox(height = '30', width='7')
LbFoul.grid(row=1, column = 12)
LbYellow = Listbox(height = '30', width='7')
LbYellow.grid(row=1, column = 13)
LbPIndex = Listbox(height = '30', width='7')
LbPIndex.grid(row=1, column = 14)

cmdLoad = Button(text="Load Games", command=tick)
cmdLoad.grid(row=2, column=0)
cmdLoad2 = Button(text="get source", command=a)
cmdLoad2.grid(row=2, column=1)
cmdLoad3 = Button(text="print TD", command=b)
cmdLoad3.grid(row=2, column=3)

lblStatus2 = Label(text="hi!")
lblStatus2.grid(row=2, column=2)


browser = webdriver.Firefox()
browser.get('http://www.flashscore.com/')



root.mainloop()

Answer 1

I do not think that the slowness is coming from the if/elif statement; I think it is coming from using Selenium.

Selenium is a nice web-scraper. However, it is fairly slow as you actually have to open up and load the webpage to actually do any scraping.

This is not very efficient, so the best solution would be to find another library for web-scraping that does not rely on opening up and loading the webpage.

---

My solution for this would be to use two libraries: urllib2 and BeautifulSoup.

urllib2 - This library will be used to read the HTML document into memory. I believe this library comes with Python.

BeautifulSoup - This library will be used to parse the HTML document. You will have to download this.

See this StackOverflow post for an example.