Selenium Kijiji web scraper

Question

I have this script working pretty well but I know that there must be many things that I could do better to make it more efficient.

from selenium import webdriver
import re
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import NoSuchElementException, TimeoutException
import csv
import xlsxwriter
import sys

reload(sys)
sys.setdefaultencoding('utf-8')

new_driver = webdriver.Firefox()
driver = webdriver.Firefox()
new_driver_handle = new_driver.current_window_handle
driver_handle = driver.current_window_handle
driver.get('http://www.kijiji.com')

try:
    element = WebDriverWait(driver, 30).until(EC.presence_of_element_located((By.ID, "SignInLink")))
finally:
    linkElem = driver.find_element_by_id('SignInLink')
    type(linkElem)
    linkElem.click()  # follows the "Read It Online" link

try:
    element = WebDriverWait(driver, 30).until(EC.presence_of_element_located((By.ID, "LoginEmailOrNickname")))
finally:
    emailElem = driver.find_element_by_id('LoginEmailOrNickname')
    emailElem.send_keys('[email protected]')
    passwordElem = driver.find_element_by_id('login-password')
    passwordElem.send_keys('123456789')
    passwordElem.submit()

try:
    element = WebDriverWait(driver, 30).until(EC.presence_of_element_located((By.PARTIAL_LINK_TEXT, "houses for sale"))
                                              )
finally:
    linkElem = driver.find_element_by_partial_link_text("houses for sale")
    type(linkElem)
    linkElem.click()  # follows the "Read It Online" link

try:
    element = WebDriverWait(driver, 30).until(EC.presence_of_element_located((By.PARTIAL_LINK_TEXT, "Propri")))
finally:
    linkElem = driver.find_element_by_partial_link_text("Propri")
    type(linkElem)
    linkElem.click()  # follows the "Read It Online" link

try:
    element = WebDriverWait(driver, 30).until(EC.presence_of_element_located((By.CLASS_NAME, "titlecount")))
finally:
    linkcount_label = driver.find_element_by_class_name("titlecount")
    titlecount_text = linkcount_label.get_attribute('textContent')

adnumber = 0
title_count = re.sub("[^0-9.]", "", titlecount_text)
title_count = int(title_count)
s = title_count / 20
# helloFile = open('houses.txt', 'a')
xbook = xlsxwriter.Workbook('Test.xlsx')
xsheet = xbook.add_worksheet('Test')
outputFile = open('output.csv', 'a')
outputWriter = csv.writer(outputFile)
outputWriter.writerow(
    ['label1', 'label2', 'label3', 'label4', 'label5', 'label6', 'label7', 'label8', 'label9', 'label10'])

for a in range(s):

    driver.get(driver.current_url)
    list_links = driver.find_elements_by_css_selector("*[class^='title enable-search-navigation-flag']")
    list_links1 = []
    for i1 in list_links:
        list_links1.append(i1.get_attribute('href'))

    b = 0

    for i in list_links:

        # Open a new tab and open individual kijiji ad

        print list_links1[b]

        if b > 0:
            new_driver.get(list_links1[b])

        if adnumber == 0:
            new_driver.switch_to.window(new_driver_handle)
            new_driver.get(list_links1[b])
            try:
                element = WebDriverWait(new_driver, 30).until(
                    EC.presence_of_element_located((By.ID, "SignInLink"))
                )
            finally:
                linkElem = new_driver.find_element_by_id('SignInLink')
                type(linkElem)
                linkElem.click()  # follows the "Read It Online" link

            try:
                element = WebDriverWait(new_driver, 30).until(
                    EC.presence_of_element_located((By.ID, "LoginEmailOrNickname")))
            finally:
                emailElem = new_driver.find_element_by_id('LoginEmailOrNickname')
                emailElem.send_keys('[email protected]')
                passwordElem = new_driver.find_element_by_id('login-password')
                passwordElem.send_keys('WJ1029vc1')
                passwordElem.submit()

        new_driver.save_screenshot('out1.png')

        texts = []

        try:
            element = WebDriverWait(new_driver, 30).until(
                EC.element_to_be_clickable((By.CSS_SELECTOR, "*[class*='phoneShowNumberButton']"))
            )
        except (NoSuchElementException, TimeoutException):
            pass
        else:
            linkElem = new_driver.find_element_by_css_selector("*[class*='phoneShowNumberButton']")
            type(linkElem)
            new_driver.implicitly_wait(2)
            linkElem.click()
            phone_num = linkElem.text
            print linkElem.text

        # if len(new_driver.find_elements_by_css_selector("*[class*='phoneShowNumberButton']")) > 0:
        #     linkElem = new_driver.find_element_by_css_selector("*[class*='phoneShowNumberButton']")
        #     type(linkElem)
        #     linkElem.click()
        #     new_driver.implicitly_wait(5)
        #     phone_num = linkElem.text
        #     print linkElem.text

        if 'phone_num' in locals():
            texts.append(phone_num)
        else:
            texts.append(',')

        try:
            element = WebDriverWait(new_driver, 30).until(EC.presence_of_element_located(
                (By.CSS_SELECTOR, "[class='ad-attributes']"))
            )
        except (NoSuchElementException, TimeoutException):
            continue
        else:
            info = new_driver.find_element_by_css_selector("[class='ad-attributes']")

        # info = driver.find_element_by_css_selector("[class='ad-attributes']") was replaced by 161 to 165
        description = new_driver.find_element_by_id("AdDescriptionTabs")
        getinfo = info.find_elements_by_tag_name("td")
        for a in getinfo:
            texts.append(a.get_attribute('textContent'))
        # print texts
        # print description.get_attribute('textContent')
        home_desc = description.get_attribute('textContent')

        texts.append(home_desc)
        texts1 = texts

        str1 = '|'.join(texts)

        str1 = re.sub(r'\s+', ' ', str1.replace(' |', ' |'))
        # str1 = str1.strip('\n')
        texts = str1.split('|')

        outputWriter.writerow([str1])

        if adnumber <= title_count:
            xsheet.write_row(adnumber, 0, texts)
            adnumber += 1

        if 'phone_num' in locals():
            del phone_num

        b += 1

    driver.switch_to.window(driver_handle)

    try:
        element = WebDriverWait(driver, 60).until(
            EC.presence_of_element_located((By.PARTIAL_LINK_TEXT, "Suivante"))
        )
    finally:
        linkElem = driver.find_element_by_partial_link_text("Suivante")
        type(linkElem)
        linkElem.click()

xbook.close()
driver.quit()

Answer 1

• If you haven't looked at PEP8, do that. In general following the style guidelines described in that document is a very good idea and the code has a number of variable names (linkElem) that don't match it. Also the imports aren't ordered in any way. Keeping just a single style of quotes would also be nice.
• Single character variable names are super non-descript. Unless there's a lot of precedent (like i for indexes), choose something more readable.
• (More) compatibility with Python 3 would be nice, so e.g. print should be called as a function (print(...)).
• The reload(sys) thing shouldn't be necessary.
• There lots of no-ops (type(linkElem)) and commented out code. Remove both and don't keep them in (production) code, but perhaps in a separate notebook or something.
• If you have comments they should be accurate; "follows the 'Read It Online' link" appears a couple of times, but it's probably only correct once.
• Assignments to unused variables (element) isn't useful, it just keeps the reader guessing at where it's finally being used instead.
• There lots of duplicate code. Put that into functions and reuse them. You should probably also have a main function instead of putting it "raw" in a script like this in order to reuse it later.
• The a in for a in range(s) and the i in for i in list_links aren't being used, so use _ as the name instead to clarify the intent.
• Use list comprehensions instead of append if possible.
• Use enumerate if possible instead of manually counting (b). You could even argue that the b > 0 would be better done as a single flag first or so.
• Use zip/itertools.izip instead of manually indexing (list_link1[b]). And reuse what you've indexed.
• Using locals is ... wrong. Use a different approach, e.g. set a flag or something.
• Ensure that things are closed by using with on files you open / things that can be closed.
• The label1 to label10 can be easily generated instead of hardcoded.

Well, there's still lots to do, but here's somewhat of a cleanup:

# -*- coding: utf-8 -*-

import csv
import re
import sys
import xlsxwriter

from selenium import webdriver
from selenium.common.exceptions import NoSuchElementException, TimeoutException
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

new_driver = webdriver.Firefox()
driver = webdriver.Firefox()
new_driver_handle = new_driver.current_window_handle
driver_handle = driver.current_window_handle
driver.get('http://www.kijiji.com')

try:
    WebDriverWait(driver, 30).until(EC.presence_of_element_located((By.ID, "SignInLink")))
finally:
    driver.find_element_by_id('SignInLink').click()  # follows the "Read It Online" link

try:
    WebDriverWait(driver, 30).until(EC.presence_of_element_located((By.ID, "LoginEmailOrNickname")))
finally:
    driver.find_element_by_id('LoginEmailOrNickname').send_keys('[email protected]')

    password_element = driver.find_element_by_id('login-password')
    password_element.send_keys('123456789')
    password_element.submit()

try:
    WebDriverWait(driver, 30).until(EC.presence_of_element_located((By.PARTIAL_LINK_TEXT, "houses for sale"))
finally:
    driver.find_element_by_partial_link_text("houses for sale").click()

try:
    WebDriverWait(driver, 30).until(EC.presence_of_element_located((By.PARTIAL_LINK_TEXT, "Propri")))
finally:
    driver.find_element_by_partial_link_text("Propri").click()

try:
    WebDriverWait(driver, 30).until(EC.presence_of_element_located((By.CLASS_NAME, "titlecount")))
finally:
    titlecount_text = driver.find_element_by_class_name("titlecount").get_attribute('textContent')

adnumber = 0
title_count = int(re.sub("[^0-9.]", "", titlecount_text))
xbook = xlsxwriter.Workbook('Test.xlsx')
xsheet = xbook.add_worksheet('Test')
outputFile = open('output.csv', 'a')
outputWriter = csv.writer(outputFile)
outputWriter.writerow(['label1', 'label2', 'label3', 'label4', 'label5', 'label6', 'label7', 'label8', 'label9', 'label10'])

for _ in range(title_count / 20):
    driver.get(driver.current_url)

    list_links = driver.find_elements_by_css_selector("*[class^='title enable-search-navigation-flag']")

    # Open a new tab and open individual kijiji ad
    for i, b_link in enumerate(i1.get_attribute('href') for i1 in list_links):
        print(b_link)

        if i > 0:
            new_driver.get(b_link)

        if adnumber == 0:
            new_driver.switch_to.window(new_driver_handle)
            new_driver.get(b_link)
            try:
                WebDriverWait(new_driver, 30).until(EC.presence_of_element_located((By.ID, "SignInLink")))
            finally:
                new_driver.find_element_by_id('SignInLink').click()

            try:
                WebDriverWait(new_driver, 30).until(EC.presence_of_element_located((By.ID, "LoginEmailOrNickname")))
            finally:
                new_driver.find_element_by_id('LoginEmailOrNickname').send_keys('[email protected]')

                password_element = new_driver.find_element_by_id('login-password')
                password_element.send_keys('WJ1029vc1')
                password_element.submit()

        new_driver.save_screenshot('out1.png')

        try:
            WebDriverWait(new_driver, 30).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "*[class*='phoneShowNumberButton']")))
        except (NoSuchElementException, TimeoutException):
            pass
        else:
            link_element = new_driver.find_element_by_css_selector("*[class*='phoneShowNumberButton']")
            new_driver.implicitly_wait(2)
            link_element.click()
            phone_num = link_element.text
            print(phone_num)

        texts = []

        if 'phone_num' in locals():
            texts.append(phone_num)
        else:
            texts.append(',')

        try:
            WebDriverWait(new_driver, 30).until(EC.presence_of_element_located((By.CSS_SELECTOR, "[class='ad-attributes']")))
        except (NoSuchElementException, TimeoutException):
            continue
        else:
            info = new_driver.find_element_by_css_selector("[class='ad-attributes']")

        description = new_driver.find_element_by_id("AdDescriptionTabs")
        texts.extend(a.get_attribute('textContent') for a in info.find_elements_by_tag_name("td"))
        texts.append(description.get_attribute('textContent'))

        str1 = '|'.join(texts)

        str1 = re.sub(r'\s+', ' ', str1.replace(' |', ' |'))
        texts = str1.split('|')

        outputWriter.writerow([str1])

        if adnumber <= title_count:
            xsheet.write_row(adnumber, 0, texts)
            adnumber += 1

        if 'phone_num' in locals():
            del phone_num

    driver.switch_to.window(driver_handle)

    try:
        element = WebDriverWait(driver, 60).until(EC.presence_of_element_located((By.PARTIAL_LINK_TEXT, "Suivante")))
    finally:
        driver.find_element_by_partial_link_text("Suivante").click()

xbook.close()
driver.quit()