7
\$\begingroup\$

I built a bot with Python to scrap scores on flashscore.com but the data scrap from the site loads into its listbox very slowly. I am curious about the speed of selenium so I made a button that prints all the text and it turns out the speed was fast so it must be the if elif block that is slowing down the program.

from tkinter import *
from selenium import webdriver
import threading



def LoadSite():

    lblStatus2.config(text="loading")
    m = 0
    Table = browser.find_elements_by_tag_name('table')
    for tables in Table:
        abc = tables.find_elements_by_class_name('country_part')
        aaa = tables.find_elements_by_class_name('tournament_part')
        C = len(tables.find_elements_by_class_name("padr"))

        for countrys in abc:
                    LbCountry.insert(END, countrys.text+aaa[m].text)
                    n = 1
                    while (n < C):
                        LbCountry.insert(END, "")
                        n += 1
    m +=1


    Time = browser.find_elements_by_tag_name('td')
    for g in Time:
        if (g.get_attribute('class').find('cell_ab team-home')) != -1 or (g.get_attribute('class').find('cell_ab team-home  bold')) != -1:
            LbHome.insert(END,g.text)
            if g.get_attribute('innerHTML').find('rhcard rhcard1') != -1:
                LbHRed.insert(END,"1")
            elif g.get_attribute('innerHTML').find('rhcard rhcard2') != -1:
                LbHRed.insert(END,"2")
            else:
                LbHRed.insert(END,"")

        elif (g.get_attribute('class').find('cell_ac team-away')) != -1 or (g.get_attribute('class').find('cell_ac team-away  bold')) != -1:
            LbAway.insert(END,g.text)
            if g.get_attribute('innerHTML').find('racard racard1') != -1:
                LbARed.insert(END,"1")
            elif g.get_attribute('innerHTML').find('racard racard2') != -1:
                LbARed.insert(END,"2")
            else:
                LbARed.insert(END,"")

        elif g.get_attribute('class').find('cell_sa score  bold') != -1:
            LbScore.insert(END,g.text)
        elif g.get_attribute('class').find('cell_sa score') != -1:
            LbScore.insert(END,g.text)
        elif g.get_attribute('class').find('cell_ad time  time-playing') != -1:
            LbScore.insert(END,g.text)
        elif g.get_attribute('class').find('cell_ad time') != -1:
            LbTime.insert(END,g.text)
        elif g.get_attribute('class').find('cell_ad time  time-playing') != -1:
            LbTime.insert(END,g.text)
        elif g.get_attribute('class').find('cell_aa timer') != -1:
            LbStatus.insert(END,g.text)
        elif g.get_attribute('class').find('cell_aa timer  playing') != -1:
            LbStatus.insert(END,g.text)
    lblStatus2.config(text="complete")




def tick():
    t = threading.Thread(name='LoadSite', target=LoadSite, args=())
    t.start()

def a():
    print(browser.page_source)

def b():
    Time = browser.find_elements_by_tag_name('td')
    for g in Time:
        print(g.text)

root = Tk()

LbCountry = Listbox(height = '30', width='10')
LbCountry.grid(row=1, column=0)
lblCountry = Label(text="Country")
lblCountry.grid(row=0, column=0)
LbTime = Listbox(height = '30', width='7')
LbTime.grid(row =1, column = 1)
lblTime = Label(text="Time")
lblTime.grid(row=0, column=1)
LbStatus = Listbox(height = '30')
LbStatus.grid(row =1, column = 2)
lblStatus = Label(text="Status")
lblStatus.grid(row=0, column=2)
LbHRed = Listbox(height = '30', width='3')
LbHRed.grid(row=1, column = 3)
lblHRed = Label(text="Red")
lblHRed.grid(row=0, column=3)
LbHome = Listbox(height = '30')
LbHome.grid(row =1, column = 4)
lblHome = Label(text="Home")
lblHome.grid(row=0, column=4)
LbScore = Listbox(height = '30', width='7')
LbScore.grid(row =1, column = 5)
lblScore = Label(text="Score")
lblScore.grid(row=0, column=5)
LbAway = Listbox(height = '30')
LbAway.grid(row =1, column = 6)
lblAway = Label(text="Away")
lblAway.grid(row=0, column=6)
LbARed = Listbox(height = '30', width='3')
LbARed.grid(row=1, column = 7)
lblARed = Label(text="Red")
lblARed.grid(row=0, column=7)

LbPossession = Listbox(height = '30', width='7')
LbPossession.grid(row=1, column = 8)
LbSOnT = Listbox(height = '30', width='7')
LbSOnT.grid(row=1, column = 9)
LbSOffT = Listbox(height = '30', width='7')
LbSOffT.grid(row=1, column = 10)
LbCorner = Listbox(height = '30', width='7')
LbCorner.grid(row=1, column = 11)
LbFoul = Listbox(height = '30', width='7')
LbFoul.grid(row=1, column = 12)
LbYellow = Listbox(height = '30', width='7')
LbYellow.grid(row=1, column = 13)
LbPIndex = Listbox(height = '30', width='7')
LbPIndex.grid(row=1, column = 14)

cmdLoad = Button(text="Load Games", command=tick)
cmdLoad.grid(row=2, column=0)
cmdLoad2 = Button(text="get source", command=a)
cmdLoad2.grid(row=2, column=1)
cmdLoad3 = Button(text="print TD", command=b)
cmdLoad3.grid(row=2, column=3)

lblStatus2 = Label(text="hi!")
lblStatus2.grid(row=2, column=2)


browser = webdriver.Firefox()
browser.get('http://www.flashscore.com/')



root.mainloop()
\$\endgroup\$
  • \$\begingroup\$ Are you certain it's the if block, not the for loop? \$\endgroup\$ – wei2912 May 9 '15 at 12:06
  • \$\begingroup\$ At the first glance looks like you do a lot of selenium queries inside that loop, each one is very time consuming since there is a java app wrapped in a python app that comunicates remotely with a browser plugin. We used selenium a lot in our integration test, to speed them up we retreived all the the data with a javascript script injected into the page while making just one selenium call. \$\endgroup\$ – Tiberiu C. May 9 '15 at 12:13
  • \$\begingroup\$ @wei2912, it might be the for loop. haha didn't thought about that. \$\endgroup\$ – GnomeMage May 9 '15 at 15:19
  • \$\begingroup\$ @tiberiu corbu so what do you suggest me to do now? Take the source code and parse it? Like that I only make one selenium call. \$\endgroup\$ – GnomeMage May 9 '15 at 15:20
  • 3
    \$\begingroup\$ I'd suggest some profiling, to find out where the time is actually going, rather than making assumptions. \$\endgroup\$ – jonrsharpe May 9 '15 at 16:42
2
\$\begingroup\$

I do not think that the slowness is coming from the if/elif statement; I think it is coming from using Selenium.

Selenium is a nice web-scraper. However, it is fairly slow as you actually have to open up and load the webpage to actually do any scraping.

This is not very efficient, so the best solution would be to find another library for web-scraping that does not rely on opening up and loading the webpage.


My solution for this would be to use two libraries: urllib2 and BeautifulSoup.

urllib2 - This library will be used to read the HTML document into memory. I believe this library comes with Python.

BeautifulSoup - This library will be used to parse the HTML document. You will have to download this.

See this StackOverflow post for an example.

\$\endgroup\$

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Not the answer you're looking for? Browse other questions tagged or ask your own question.