Webscraping With Selenium - a Course Downloader and Sorter

Question

I've been working on a program that automates the painful task of downloading each file from a website for courses one by one and sorting them using python. This is my first python webscraping project on this scale, as such I would love some advice and lessons on how to make my code better! Here is the website I'm scraping.

from selenium import webdriver
import time
import os
import shutil
import re

path = r'https://coursevania.courses.workers.dev/[coursevania.com]%20Udemy%20-%20Master%20the%20Coding%20Interview%20Data%20Structures%20+%20Algorithms/'

# For changing the download location for this browser temporarily
options = webdriver.ChromeOptions()
preferences = {"download.default_directory": r"E:\Utilities_and_Apps\Python\MY PROJECTS\Test data\Downloads", "safebrowsing.enabled": "false"}
options.add_experimental_option("prefs", preferences)

# Acquire the Course Link and Get all the directories
browser = webdriver.Chrome(chrome_options=options)
browser.get(r"https://coursevania.courses.workers.dev/[coursevania.com]%20Udemy%20-%20Master%20the%20Coding%20Interview%20Data%20Structures%20+%20Algorithms/")
time.sleep(2)
elements = browser.find_elements_by_css_selector(".mdui-text-truncate")

# loop for as many directories there are
for i in range(15, len(elements)):

    # At each directory, it refreshes the page to update the webelements in the list, and returns the current directory that is being worked on
    browser.get(path)
    time.sleep(2)
    elements = browser.find_elements_by_css_selector(".mdui-text-truncate")
    element = elements[i]

    # checks if the folder for the directory already exists
    current_directory_name = element.text[11:].strip(" .")
    current_folder_path = "E:\\Utilities_and_Apps\\Python\\MY PROJECTS\\Test data\Downloads\\" + current_directory_name
    if os.path.exists(current_folder_path):
        pass
    else:
        os.mkdir(current_folder_path)

    # Formatting what has been downloaded and sorted, and 
    print(current_directory_name, "------------------------------", sep="\n")

    # moves on to the directory to get the page with the files
    element.click()

    # pausing for a few secs for the page to load, and running the same mechanism to get each file using the same method used in directory 
    time.sleep(3)
    files = browser.find_elements_by_css_selector(".mdui-text-truncate")
    for j in range(len(files)):
        files = browser.find_elements_by_css_selector(".mdui-text-truncate")
        _file = files[j]
    # constants for some if statements
        download = True
        move = True
        current_file_name = _file.text[17:].strip()

    # If file exists, then pass over it, and don't do anything, and moveon to next file
        if os.path.exists(current_folder_path + "\\" + current_file_name):
            pass

    # If it doesnt exist, then depending on its extension, do specific actions with it 
        else:
            # Downloads the mp4 files by clicking on it, and finding the input tag which contains the download link for vid in its value attribute
            if ".mp4" in current_file_name:
                _file.click()
                time.sleep(2)  
                download_path = browser.find_element_by_css_selector("input").get_attribute("value")
                current_file_name = re.search(r'https://coursevania.courses.workers.dev/\[coursevania.com\]%20Udemy%20-%20Master%20the%20Coding%20Interview%20Data%20Structures%20\+%20Algorithms/.+/(.+)', download_path, re.DOTALL).group(1)
                # Checks if file exists again, incase the filename is different then the predicted filename orderly generated.
                if os.path.exists(current_folder_path + "\\" + current_file_name):
                    move = False
                    download = False
                # returns to the previous page with the files
                browser.back()

            # self explanatory
            elif ".html" in current_file_name:
                download_path = path + current_directory_name + "/" + current_file_name
                if os.path.exists(current_folder_path + "\\" + current_file_name):
                    move = False
                    download = False

            else:
            # acquires the download location by going to the parent tag which is an a tag containing the link for html in its 'href' attribute
                download_path = _file.find_element_by_xpath('..').get_attribute('href').replace(r"%5E", "^")
                current_file_name = re.search(r'https://coursevania.courses.workers.dev/\[coursevania.com\]%20Udemy%20-%20Master%20the%20Coding%20Interview%20Data%20Structures%20\+%20Algorithms/.+/(.+)', download_path, re.DOTALL).group(1).replace("%20", " ")

            time.sleep(2)
            current_file_path = "E:\\Utilities_and_Apps\\Python\\MY PROJECTS\\Test data\Downloads\\" + current_file_name
            # responsible for downloading it using a path, get allows downloading, by source links
            if download:
                browser.get(download_path)

                # while the file doesn't exist/ it hasn't been downloaded yet, do nothing
                while True:
                    if os.path.exists(current_file_path):
                        break
                time.sleep(1)

            # moves the file from the download spot to its own folder
            if move:
                shutil.move(current_file_path, current_folder_path + "\\" + current_file_name)
        print(current_file_name)

    # formatter
    print("------------------------------", "", sep="\n")
    time.sleep(3)

Output:

1. Introduction
------------------------------
1. How To Succeed In This Course.mp4
1. How To Succeed In This Course.vtt
1.1 Interview Mind Map.html
1.2 Technical Interview Mind Map.html
2. Join Our Online Classroom!.html
3. Exercise Meet The Community!.html
------------------------------

10. Data Structures Trees
------------------------------
1. Trees Introduction.mp4
1. Trees Introduction.vtt
1.1 Technical Interview Mind Map.html
10. Solution lookup().mp4
10. Solution lookup().vtt
10.1 Solution Code.html
11. Bonus Exercise remove().mp4
11. Bonus Exercise remove().vtt
11.1 Exercise Repl.html
11.2 Binary Search Tree VisuAlgo.html
12. Solution remove().mp4
12. Solution remove().vtt
12.1 Solution Code.html
13. AVL Trees + Red Black Trees.mp4
13. AVL Trees + Red Black Trees.vtt
14. Resources AVL Trees + Red Black Trees.html
15. Binary Heaps.mp4
15. Binary Heaps.vtt
15.1 VisuAlgo Binary Heap.html
16. Quick Note on Heaps.mp4
16. Quick Note on Heaps.vtt
16.1 A great explanation of why.html
17. Priority Queue.mp4
17. Priority Queue.vtt
17.1 Priority Queue Javascript Code.html
18. Trie.mp4
18. Trie.vtt
19. Tree Review.mp4
19. Tree Review.vtt
19.1 Technical Interview Mind Map.html
2. Binary Trees.mp4
2. Binary Trees.vtt
3. O(log n).mp4
3. O(log n).vtt
4. Correction Binary Search Trees.html
5. Binary Search Trees.mp4
5. Binary Search Trees.vtt
5.1 Binary Search Tree VisuAlgo.html
6. Balanced VS Unbalanced BST.mp4
6. Balanced VS Unbalanced BST.vtt
6.1 Big O Cheat Sheet.html
7. BST Pros and Cons.mp4
7. BST Pros and Cons.vtt
8. Exercise Binary Search Tree.mp4
8. Exercise Binary Search Tree.vtt
8.1 Exercise Repl.html
9. Solution insert().mp4
9. Solution insert().vtt
9.1 Solution Code.html
------------------------------

... To Be Continued

Answer 1

Nice.

Alas I don't have enough time for a comprehensive review so I will not refactor your whole code. Instead I will just focus on a few points.

First of all, I would add a few constants (written as UPPERCASE) for more flexibility and to avoid repetition:

ROOT_URL = r'https://coursevania.courses.workers.dev/[coursevania.com]%20Udemy%20-%20Master%20the%20Coding%20Interview%20Data%20Structures%20+%20Algorithms/'

The path: "E:\\Utilities_and_Apps\\Python\\MY PROJECTS\\Test data\Downloads\\" appears multiple times in your code, this is unnecessary. Add another constant:

DOWNLOAD_PATH = "E:\\Utilities_and_Apps\\Python\\MY PROJECTS\\Test data\Downloads\"

---

The DOM selection method could be improved because you use a mix of Selenium and regex:

download_path = browser.find_element_by_css_selector("input").get_attribute("value")
current_file_name = re.search(r'https://coursevania.courses.workers.dev/\[coursevania.com\]%20Udemy%20-%20Master%20the%20Coding%20Interview%20Data%20Structures%20\+%20Algorithms/.+/(.+)', download_path, re.DOTALL).group(1)

Parsing HTML with regex can quickly become a nightmare and is generally discouraged. Instead you can use a parsing library like Beautiful Soup. This is often done in conjunction with the requests module but since you are using Selenium here you can use the built-in functions available to you (notably the find_elements_by_... functions).

I advise you to break up the code in small functions to separate functionality, and make the code easier to maintain. For instance you could have one function that retrieves all the links of interest in a given page, and another function to fetch those URLs and download the files.

---

Delays: Instead of setting arbitrary waits with time.sleep (that will either be too long or too short depending on the network conditions) you can use Selenium functions again (WebDriverWait), to determine when the page is 'loaded' or at least half-ready, for example by waiting for certain elements to appear. And if they do not appear or take too long (timeout), then there is no point proceeding with the rest of the code.

This can be tricky, and the criteria will vary from one site to another. Sometimes you spend more time looking for the right signals than on coding. My own approach in this particular case is to wait until the progress bar disappears but YMMV (I have tried to wait until the list of courses is loaded but that didn't seem to work well). This is not perfect and probably can be improved.

See chapter: 5. Waits

So in this case I am waiting until the control with class name .mdui-progress becomes invisible. I have determined this by using the Firefox inspector (under menu Tools/Web Developer) and setting up some breakpoints to freeze the page while it is reloading. This is not so straightforward but it's a question of practice.

---

Now for some code. I have used Selenium with Firefox on Linux so the code has been adapted to run on my platform. Please disregard the Firefox directives and use yours instead.
I note that your are on Windows but the code should be OK.
NB: I added a few imports and removed re.

---

import time
import os, sys
import shutil
from typing import (Dict, List)
from selenium import webdriver
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.webdriver.firefox.options import Options as FirefoxOptions

# constants
ROOT_URL = r'https://coursevania.courses.workers.dev/[coursevania.com]%20Udemy%20-%20Master%20the%20Coding%20Interview%20Data%20Structures%20+%20Algorithms/'
DOWNLOAD_PATH = "E:\\Utilities_and_Apps\\Python\\MY PROJECTS\\Test data\Downloads\\"

options = FirefoxOptions()
#options.add_argument("--headless")
options.add_argument("--private-window")
driver = webdriver.Firefox(options=options)


def get_links(driver: webdriver.firefox, xpath_selector: str) -> List[Dict]:
    links = []
    elems = driver.find_elements_by_xpath(xpath_selector)
    for elem in elems:
        url = elem.get_attribute("href")
        class_name = elem.get_attribute("class")
        links.append({
            "url": url,
            "class_name": class_name
            })
    return links


# could return bool   
def wait_for_page_load():
    # borrowed code: https://stackoverflow.com/questions/26566799/wait-until-page-is-loaded-with-selenium-webdriver-for-python
    try:
        # wait for page to load
        print("Waiting for page load...")
        WebDriverWait(driver, 10).until(EC.invisibility_of_element_located((By.CSS_SELECTOR, '.mdui-progress')))
        print("Page should now be ready, continue")
    except TimeoutException:
        print("Timed out/failed to load page")
        sys.exit()


# load the main page and wait
driver.get(ROOT_URL)
wait_for_page_load()
print(f'Links from {ROOT_URL}:')
links = get_links(driver=driver, xpath_selector="//li/a")
for link in links:
    url = link["url"]
    class_name = link["class_name"]
    print(f'Link: {url}: class name: {class_name}')
    if class_name.startswith('folder'):
        print('=> Folder: to be crawled')
    if class_name.startswith('file'):
        print('=> File: to be downloaded')

Details:

• The function get_links returns a list of dictionaries, for each link found I am returning the URL + the class name for the href tag: this is useful to differentiate between folders and files. Then all you have to do is enumerate the links and decide on whether to crawl further or download the file. The process should be made recursive et voilà.
• For more flexibility you can specify the xpath selector so as to reuse the function on other sites
• The xpath selector here is simply to look for A tags embedded in LI tags.
• The function wait_for_page_load waits for the progress bar to disappear, then I consider the page 'loaded' and ready to be inspected
• I have not implemented a download function
• I have attempted to use proper type hinting for the functions, but no docstrings

If you add the recursion I think the final code could be quite short.

Regarding the download, I am not sure how to determine it has finished. Probably by waiting for the file to appear in the Downloads folder. on Linux I might use inotifywait but this is an OS-dependent approach.

---

Future improvements:

Adding some parallel processing would be nice, to crawl multiple pages at the same time and download several files at once but be gentle with third-party websites. They could block you if they think you are bot and crawling too fast to be human.

Or you could collect all the file links and download them in bulk at the end of the process.

To get the file name from the URL you can simply do this:

from urllib.parse import urlparse

def get_file_name_from_url(url: str) -> str:
    u = urlparse(url)
    return os.path.basename(u.path))

But you should sanitize the file names as they may contain special characters that your OS will not accept (I think Windows does not accept the colon in file names for example). Unless your browser does that automatically.

---

Sample output: folders

Wating for page load...
Page should now be ready, continue
Links from https://coursevania.courses.workers.dev/[coursevania.com]%20Udemy%20-%20Master%20the%20Coding%20Interview%20Data%20Structures%20+%20Algorithms/:
Link: https://coursevania.courses.workers.dev/[coursevania.com]%20Udemy%20-%20Master%20the%20Coding%20Interview%20Data%20Structures%20+%20Algorithms/1.%20Introduction/: class name: folder
=> Folder: to be crawled
Link: https://coursevania.courses.workers.dev/[coursevania.com]%20Udemy%20-%20Master%20the%20Coding%20Interview%20Data%20Structures%20+%20Algorithms/10.%20Data%20Structures%20Trees/: class name: folder
=> Folder: to be crawled
Link: https://coursevania.courses.workers.dev/[coursevania.com]%20Udemy%20-%20Master%20the%20Coding%20Interview%20Data%20Structures%20+%20Algorithms/11.%20Data%20Structures%20Graphs/: class name: folder
=> Folder: to be crawled
Link: https://coursevania.courses.workers.dev/[coursevania.com]%20Udemy%20-%20Master%20the%20Coding%20Interview%20Data%20Structures%20+%20Algorithms/12.%20Algorithms%20Recursion/: class name: folder
=> Folder: to be crawled
Link: https://coursevania.courses.workers.dev/[coursevania.com]%20Udemy%20-%20Master%20the%20Coding%20Interview%20Data%20Structures%20+%20Algorithms/13.%20Algorithms%20Sorting/: class name: folder
=> Folder: to be crawled
Link: https://coursevania.courses.workers.dev/[coursevania.com]%20Udemy%20-%20Master%20the%20Coding%20Interview%20Data%20Structures%20+%20Algorithms/14.%20Algorithms%20Searching%20+%20BFS%20+%20DFS/: class name: folder
=> Folder: to be crawled
Link: https://coursevania.courses.workers.dev/[coursevania.com]%20Udemy%20-%20Master%20the%20Coding%20Interview%20Data%20Structures%20+%20Algorithms/15.%20Algorithms%20Dynamic%20Programming/: class name: folder
=> Folder: to be crawled
Link: https://coursevania.courses.workers.dev/[coursevania.com]%20Udemy%20-%20Master%20the%20Coding%20Interview%20Data%20Structures%20+%20Algorithms/16.%20Non%20Technical%20Interviews/: class name: folder
=> Folder: to be crawled
Link: https://coursevania.courses.workers.dev/[coursevania.com]%20Udemy%20-%20Master%20the%20Coding%20Interview%20Data%20Structures%20+%20Algorithms/17.%20Offer%20+%20Negotiation/: class name: folder
=> Folder: to be crawled
Link: https://coursevania.courses.workers.dev/[coursevania.com]%20Udemy%20-%20Master%20the%20Coding%20Interview%20Data%20Structures%20+%20Algorithms/18.%20Thank%20You/: class name: folder
=> Folder: to be crawled
Link: https://coursevania.courses.workers.dev/[coursevania.com]%20Udemy%20-%20Master%20the%20Coding%20Interview%20Data%20Structures%20+%20Algorithms/19.%20Extras%20Google,%20Amazon,%20Facebook%20Interview%20Questions/: class name: folder
=> Folder: to be crawled
Link: https://coursevania.courses.workers.dev/[coursevania.com]%20Udemy%20-%20Master%20the%20Coding%20Interview%20Data%20Structures%20+%20Algorithms/2.%20Getting%20More%20Interviews/: class name: folder
=> Folder: to be crawled
Link: https://coursevania.courses.workers.dev/[coursevania.com]%20Udemy%20-%20Master%20the%20Coding%20Interview%20Data%20Structures%20+%20Algorithms/20.%20Contributing%20To%20Open%20Source%20To%20Gain%20Experience/: class name: folder
=> Folder: to be crawled
Link: https://coursevania.courses.workers.dev/[coursevania.com]%20Udemy%20-%20Master%20the%20Coding%20Interview%20Data%20Structures%20+%20Algorithms/21.%20Bonus%20Extra%20Bits/: class name: folder
=> Folder: to be crawled
Link: https://coursevania.courses.workers.dev/[coursevania.com]%20Udemy%20-%20Master%20the%20Coding%20Interview%20Data%20Structures%20+%20Algorithms/22.%20Extras/: class name: folder
=> Folder: to be crawled
Link: https://coursevania.courses.workers.dev/[coursevania.com]%20Udemy%20-%20Master%20the%20Coding%20Interview%20Data%20Structures%20+%20Algorithms/3.%20Big%20O/: class name: folder
=> Folder: to be crawled
Link: https://coursevania.courses.workers.dev/[coursevania.com]%20Udemy%20-%20Master%20the%20Coding%20Interview%20Data%20Structures%20+%20Algorithms/4.%20How%20To%20Solve%20Coding%20Problems/: class name: folder
=> Folder: to be crawled
Link: https://coursevania.courses.workers.dev/[coursevania.com]%20Udemy%20-%20Master%20the%20Coding%20Interview%20Data%20Structures%20+%20Algorithms/5.%20Data%20Structures%20Introduction/: class name: folder
=> Folder: to be crawled
Link: https://coursevania.courses.workers.dev/[coursevania.com]%20Udemy%20-%20Master%20the%20Coding%20Interview%20Data%20Structures%20+%20Algorithms/6.%20Data%20Structures%20Arrays/: class name: folder
=> Folder: to be crawled
Link: https://coursevania.courses.workers.dev/[coursevania.com]%20Udemy%20-%20Master%20the%20Coding%20Interview%20Data%20Structures%20+%20Algorithms/7.%20Data%20Structures%20Hash%20Tables/: class name: folder
=> Folder: to be crawled
Link: https://coursevania.courses.workers.dev/[coursevania.com]%20Udemy%20-%20Master%20the%20Coding%20Interview%20Data%20Structures%20+%20Algorithms/8.%20Data%20Structures%20Linked%20Lists/: class name: folder
=> Folder: to be crawled
Link: https://coursevania.courses.workers.dev/[coursevania.com]%20Udemy%20-%20Master%20the%20Coding%20Interview%20Data%20Structures%20+%20Algorithms/9.%20Data%20Structures%20Stacks%20+%20Queues/: class name: folder
=> Folder: to be crawled

Sample output: files

Links from https://coursevania.courses.workers.dev/[coursevania.com]%20Udemy%20-%20Master%20the%20Coding%20Interview%20Data%20Structures%20+%20Algorithms/1.%20Introduction/:
Wating for page load...
Page should now be ready, continue
Link: https://coursevania.courses.workers.dev/[coursevania.com]%20Udemy%20-%20Master%20the%20Coding%20Interview%20Data%20Structures%20+%20Algorithms/1.%20Introduction/1.%20How%20To%20Succeed%20In%20This%20Course.mp4?a=view: class name: file view
=> File: to be downloaded
Link: https://coursevania.courses.workers.dev/[coursevania.com]%20Udemy%20-%20Master%20the%20Coding%20Interview%20Data%20Structures%20+%20Algorithms/1.%20Introduction/1.%20How%20To%20Succeed%20In%20This%20Course.vtt: class name: file
=> File: to be downloaded
Link: https://coursevania.courses.workers.dev/[coursevania.com]%20Udemy%20-%20Master%20the%20Coding%20Interview%20Data%20Structures%20+%20Algorithms/1.%20Introduction/1.1%20Interview%20Mind%20Map.html?a=view: class name: file view
=> File: to be downloaded
Link: https://coursevania.courses.workers.dev/[coursevania.com]%20Udemy%20-%20Master%20the%20Coding%20Interview%20Data%20Structures%20+%20Algorithms/1.%20Introduction/1.2%20Technical%20Interview%20Mind%20Map.html?a=view: class name: file view
=> File: to be downloaded
Link: https://coursevania.courses.workers.dev/[coursevania.com]%20Udemy%20-%20Master%20the%20Coding%20Interview%20Data%20Structures%20+%20Algorithms/1.%20Introduction/2.%20Join%20Our%20Online%20Classroom!.html?a=view: class name: file view
=> File: to be downloaded
Link: https://coursevania.courses.workers.dev/[coursevania.com]%20Udemy%20-%20Master%20the%20Coding%20Interview%20Data%20Structures%20+%20Algorithms/1.%20Introduction/3.%20Exercise%20Meet%20The%20Community!.html?a=view: class name: file view
=> File: to be downloaded