Parser for Facebook friend list

Question

I have written some code in python in combination with selenium to parse all the names from facebook friend list. It was hard to manage the pop up notification and the process of scrolling to the end of that page. However, my scraper can do that successfully. I tried to do the whole thing very carefully. There are always rooms for improvement, though. here is the working code:

from selenium import webdriver
from selenium.webdriver.chrome.options import Options
import time

chrome_options = webdriver.ChromeOptions()
prefs = {"profile.default_content_setting_values.notifications" : 2}
chrome_options.add_experimental_option("prefs",prefs)

def Facebook_Crawler(driver):

    driver.get('https://www.facebook.com/login.php?login_attempt=1&lwv=111')
    driver.find_element_by_xpath('//*[@id="email"]').send_keys('SomeEmail')
    driver.find_element_by_xpath('//*[@id="pass"]').send_keys('Password')
    driver.find_element_by_xpath('//*[@id="loginbutton"]').click()
    time.sleep(5)

    driver.find_element_by_xpath("//a[@id='findFriendsNav']").click()
    time.sleep(5)

    num_Of_links = -1
    num = 0

    while num != num_Of_links:

        num_Of_links = num
        driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
        time.sleep(8)

        items = [post.text for post in driver.find_elements_by_xpath("//div[contains(@class,'friendBrowserNameTitle')]/a")]
        num = len(items)

    for item in items:
        print(item)


if __name__ == '__main__':
    driver = webdriver.Chrome(chrome_options = chrome_options)
    try:
        Facebook_Crawler(driver)
    finally:
        driver.quit()

Answer 1

I would improve the following things:

• implement a "crawler" class that will store the "driver" instance as a variable and will also have a better separation of concerns - a separate method to log in, a separate method to get friends etc
• stop using time.sleep() for waits with selenium - hardcoded time delays are seriously unreliable - use Explicit Waits instead
• stop using XPaths - they are the least readable and the least reliable - use the fastest "by id" or the more concise "by css selector" locators

Improved code:

from selenium import webdriver
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.common.by import By
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC


class FacebookCrawler:
    LOGIN_URL = 'https://www.facebook.com/login.php?login_attempt=1&lwv=111'

    def __init__(self, login, password):
        chrome_options = webdriver.ChromeOptions()
        prefs = {"profile.default_content_setting_values.notifications": 2}
        chrome_options.add_experimental_option("prefs", prefs)

        self.driver = webdriver.Chrome(chrome_options=chrome_options)
        self.wait = WebDriverWait(self.driver, 10)

        self.login(login, password)

    def login(self, login, password):
        self.driver.get(self.LOGIN_URL)

        # wait for the login page to load
        self.wait.until(EC.visibility_of_element_located((By.ID, "email")))

        self.driver.find_element_by_id('email').send_keys(login)
        self.driver.find_element_by_id('pass').send_keys(password)
        self.driver.find_element_by_id('loginbutton').click()

        # wait for the main page to load
        self.wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR, "a#findFriendsNav")))

    def _get_friends_list(self):
        return self.driver.find_elements_by_css_selector(".friendBrowserNameTitle > a")

    def get_friends(self):
        # navigate to "friends" page
        self.driver.find_element_by_css_selector("a#findFriendsNav").click()

        # continuous scroll until no more new friends loaded
        num_of_loaded_friends = len(self._get_friends_list())
        while True:
            self.driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
            try:
                self.wait.until(lambda driver: len(self._get_friends_list()) > num_of_loaded_friends)
                num_of_loaded_friends = len(self._get_friends_list())
            except TimeoutException:
                break  # no more friends loaded

        return [friend.text for friend in self._get_friends_list()]


if __name__ == '__main__':
    crawler = FacebookCrawler(login='login', password='password')

    for friend in crawler.get_friends():
        print(friend)

(not tested)

Answer 2

The main problem with your code is the design itself. I mean your program is sequential which thing impedes its scalability. At least delegate the instructions to functions or, why not, restructure your program following the object-oriented design.