6
\$\begingroup\$

I have created a small selenium script that checks for available times to write a test for a drivers license. The program runs every minute and takes approx 50 seconds to run. I have noticed that it's quite unstable, and do not preform optimally because the web-page loads elements dynamically. Even if I use seleniums wait I am unable to optimise it fully as connectivity vary on my wifi, and may lead to longer loading times than expected. Feedback on error-handling and code optimisation for greater stability would be highly appreciated and how to schedule the task to run without a back-log of jobs waiting to be executed as memory is small on my mac-book air 2013.

import datetime
import schedule
import threading

from datetime import date

import smtplib
import time

import multiprocessing as mp

from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText

from selenium import webdriver
from selenium.common.exceptions import NoSuchElementException
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait



def selenium_get_time(ort):
    """Checks if time is available for a county"""
    options = Options()
    options.headless = True
    driver = webdriver.Chrome(chrome_options=options, executable_path='/Users/andreas/.wdm/chromedriver/83.0.4103.39/mac64/chromedriver')

    
    driver.get("https://fp.trafikverket.se/boka/#/search/dIccADaISRCIi/5/0/0/0")
    element = WebDriverWait(driver, 40).until(EC.element_to_be_clickable((By.CLASS_NAME, "form-control")))
    driver.find_element_by_xpath("//select[@id='examination-type-select']/option[@value='3']").click()
    driver.find_element_by_xpath("//select[@id='language-select']/option[@value='13']").click()
    driver.find_element_by_id('id-control-searchText').clear()
    inputElement = driver.find_element_by_id("id-control-searchText")
    inputElement.send_keys(ort)
    inputElement.send_keys(Keys.ENTER)
    # time.sleep(10)
    
    try:
        element = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.XPATH, "//div[@class='col-sm-3 text-center']/button[@data-bind='click:$parent.select']")))
        first_time = driver.find_element_by_xpath("//div[@class='col-xs-6']/strong")
        return first_time.text
    except (NoSuchElementException, TimeoutException) as e:
        driver.close()
        if NoSuchElementException:
            print('Nothing found for: ', ort, ' NoElemFound')
            driver.close()
            driver.quit()
        else:
            print('Nothing found for: ', ort, ' TimedOut')
            driver.close()
            driver.quit()

def convert_time(time_stamp):
    """converts a timestamp"""
    date_time_obj = datetime.datetime.strptime(time_stamp, '%Y-%m-%d %H:%M')
    return date_time_obj

def check_schedule(date, start_date, end_date):
    start_date = datetime.datetime.strptime(start_date, '%Y-%m-%d')
    end_date = datetime.datetime.strptime(end_date, '%Y-%m-%d')
    if start_date <= date <= end_date:
        return True
    else:
        return False

def send_email(first_availible, ort):
    """sends an email"""
    gmailUser = 'nor###rov@###.com'
    gmailPassword = '######'
    recipient = '###########'
    message=msg = 'Första lediga tid i'+' '+ str(ort) +' '+ str(first_availible) +' '+ 'https://fp.trafikverket.se/boka/#/search/SPHHISiPAfhpP/5/0/0/0'

    msg = MIMEMultipart()
    msg['From'] = gmailUser
    msg['To'] = recipient
    msg['Subject'] = "Ledig tid körkortsprov"
    msg.attach(MIMEText(message))

    mailServer = smtplib.SMTP('smtp.gmail.com', 587)
    mailServer.ehlo()
    mailServer.starttls()
    mailServer.ehlo()
    mailServer.login(gmailUser, gmailPassword)
    mailServer.sendmail(gmailUser, recipient, msg.as_string())
    mailServer.close()

def main(ort):
   """main program"""
    first_availible = selenium_get_time(ort)
    if first_availible:
        date = convert_time(first_availible)
        if check_schedule(date, '2020-07-01', '2020-07-05'):
            print('FOUND: ', ort +' '+ first_availible)
            send_email(first_availible, ort)
        else:
            now = datetime.datetime.now()
            dt_string = now.strftime("%H:%M:%S")
            print('Found Nothing for: ', ort, ' ', dt_string)



def run():
    """runs program"""
    ORTER = ['Södertälje', 'Stockholm', 'Järfälla', 'Sollentuna']
    for ort in ORTER:
        main(ort)

def worker():
    """"spans processes for program"""
    p = mp.Process(target=run)
    # run `worker` in a subprocess
    p.start()
    # make the main process wait for `worker` to end
    p.join()
    # all memory used by the subprocess will be freed to the OS




if __name__ == '__main__':

    """schedule to run every minute"""
    schedule.every(55).seconds.do(worker)
    while True:
        try:
            schedule.run_pending()
            time.sleep(1)

        except Exception as e:
            schedule.run_pending()
            time.sleep(1)
\$\endgroup\$
5
  • \$\begingroup\$ Scrapy with splash can be used but I haven't used it personally because it's not easy to use. It's a well known fact that selenium is slow. \$\endgroup\$ Jul 1 '20 at 19:31
  • 1
    \$\begingroup\$ @VisheshMangla I haven't used it either, and barely know enough to execute a proper selenium script. Thank you for pointing out that selenium is sub-par on speed, will keep that for future reference! \$\endgroup\$ Jul 1 '20 at 20:27
  • \$\begingroup\$ Learning scrapy basics is super easy, youtube.com/… but I left watching the tutorial on 8th video. \$\endgroup\$ Jul 2 '20 at 6:17
  • \$\begingroup\$ @VisheshMangla Thank you for the resource, will check it out! \$\endgroup\$ Jul 2 '20 at 7:24
  • \$\begingroup\$ Selenium is made for testing your created websites, not for scrapping them. \$\endgroup\$ Jul 2 '20 at 15:22
0
\$\begingroup\$

convert_time

This function can just be

def convert_time(time_stamp):
    return datetime.datetime.strptime(time_stamp, '%Y-%m-%d %H:%M')

check_schedule

This function can just be

def check_schedule(date, start_date, end_date):
    start_date = datetime.datetime.strptime(start_date, '%Y-%m-%d')
    end_date = datetime.datetime.strptime(end_date, '%Y-%m-%d')
    return start_date <= date <= end_date

Since the last line is a boolean comparison in itself.

Proper scraping tools

Selenium doesn't have a reputation for being a good website scraper. Here are a few that do:

\$\endgroup\$
0
\$\begingroup\$

Optimized code: Runs fast, stable and does not consume more RAM over time.

###For written test

import datetime
import schedule
import threading

from datetime import date

import smtplib
import time

import multiprocessing as mp

from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from urllib3.exceptions import MaxRetryError
from urllib3.exceptions import NewConnectionError

from selenium import webdriver
from selenium.common.exceptions import NoSuchElementException
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait

driver = None 


def tear_down():
    driver.quit()


def selenium_get_time(ort):
    global driver
    options = Options()
    options.headless = True
    driver = webdriver.Chrome(chrome_options=options, executable_path='/Users/andreas/.wdm/chromedriver/83.0.4103.39/mac64/chromedriver')
    driver.get("https://fp.trafikverket.se/boka/#/search/dIccADaISRCIi/5/0/0/0")
    element = WebDriverWait(driver, 40).until(EC.element_to_be_clickable((By.CLASS_NAME, "form-control")))
    driver.find_element_by_xpath("//select[@id='examination-type-select']/option[@value='3']").click()
    driver.find_element_by_xpath("//select[@id='language-select']/option[@value='13']").click()
    driver.find_element_by_id('id-control-searchText').clear()
    inputElement = driver.find_element_by_id("id-control-searchText")
    inputElement.send_keys(ort)
    inputElement.send_keys(Keys.ENTER)
    # time.sleep(10)
    try:
        element = WebDriverWait(driver, 40).until(EC.element_to_be_clickable((By.XPATH, "//div[@class='col-sm-3 text-center']/button[@data-bind='click:$parent.select']")))
        first_time = driver.find_element_by_xpath("//div[@class='col-xs-6']/strong")
        return first_time.text
    except (NoSuchElementException, TimeoutException, MaxRetryError, ConnectionRefusedError, NewConnectionError) as e:
        if NoSuchElementException:
            print('Nothing found for: ', ort, ' NoElemFound')
        elif MaxRetryError or ConnectionRefusedError or NewConnectionError:
            print('Connection TimedOut: ', ort)
        else:
            print('Nothing found for: ', ort, ' TimedOut')
    finally:
        driver.close()
        driver.quit()



def convert_time(time_stamp):
    date_time_obj = datetime.datetime.strptime(time_stamp, '%Y-%m-%d %H:%M')
    return date_time_obj

def check_schedule(date, start_date, end_date):
    start_date = datetime.datetime.strptime(start_date, '%Y-%m-%d')
    end_date = datetime.datetime.strptime(end_date, '%Y-%m-%d')
    if start_date <= date <= end_date:
        return True
    else:
        return False

def send_email(first_availible, ort):
    gmailUser = '####@gmail.com'
    gmailPassword = '#######'
    recipient = '########@gmail.com'
    message=msg = 'Första lediga tid i'+' '+ str(ort) +' '+ str(first_availible) +' '+ 'https://fp.trafikverket.se/boka/#/search/SPHHISiPAfhpP/5/0/0/0'

    msg = MIMEMultipart()
    msg['From'] = gmailUser
    msg['To'] = recipient
    msg['Subject'] = "Ledig tid körkortsprov"
    msg.attach(MIMEText(message))

    mailServer = smtplib.SMTP('smtp.gmail.com', 587)
    mailServer.ehlo()
    mailServer.starttls()
    mailServer.ehlo()
    mailServer.login(gmailUser, gmailPassword)
    mailServer.sendmail(gmailUser, recipient, msg.as_string())
    mailServer.close()

def main(ort):
    first_availible = selenium_get_time(ort)
    if first_availible:
        date = convert_time(first_availible)
        if check_schedule(date, '2020-07-01', '2020-07-05'):
            print('FOUND: ', ort +' '+ first_availible)
            send_email(first_availible, ort)
        else:
            now = datetime.datetime.now()
            dt_string = now.strftime("%H:%M:%S")
            print('Found Nothing for: ', ort, ' ', dt_string)
    tear_down()



def run():
    ORTER = ['Södertälje', 'Stockholm', 'Järfälla', 'Sollentuna']
    for ort in ORTER:
        main(ort)


def worker():
    p = mp.Process(target=run)
    # run `worker` in a subprocess
    p.start()
    # make the main process wait for `worker` to end
    p.join()
    # all memory used by the subprocess will be freed to the OS




if __name__ == '__main__':


    schedule.every(10).seconds.do(worker)
    while True:
        try:
            schedule.run_pending()
            time.sleep(1)

        except Exception as e:
            schedule.run_pending()
            time.sleep(1)
    

\$\endgroup\$

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Not the answer you're looking for? Browse other questions tagged or ask your own question.