0
\$\begingroup\$

There's a well-defensed job-hunting website in China,
I refactored my code which could retrieve data unlimited and effectively

import os
import time
import selenium
import random
import json
from selenium import webdriver

browser = webdriver.Chrome()
start_url = "https://www.lagou.com/jobs/list_Python?px=default&gj=3%E5%B9%B4%E5%8F%8A%E4%BB%A5%E4%B8%8B&city=%E5%8C%97%E4%BA%AC#filterBox"
position_links = set()
errors_set = set()
#kick off
browser.get(start_url)
page_element = browser.find_element_by_class_name("pager_next")
page_element_attribute = page_element.get_attribute("class").strip()
while True:
    if len(position_links) >= 450:
        break
    if not page_element_attribute.endswith('disabled'):
        for i in range(60):    
            try:
                #scrap the current page.
                position_elements = browser.find_elements_by_class_name(
                    "position_link")  # Retrieve the postions link elements
                position_links.update({p.get_attribute("href") for p in position_elements})

                #Pager Forwards
                print("Page Forward...")
                page_element.click()
                page_element = browser.find_element_by_class_name("pager_next")
                page_element_attribute = page_element.get_attribute("class").strip()
                print(page_element_attribute)

            except Exception as e:
                errors_set.add(e)

    if page_element_attribute.endswith('disabled'):
        for i in range(60):
            try:  
                page_element = browser.find_element_by_class_name("pager_prev")
                page_element_attribute = page_element.get_attribute(
                    "class").strip()
                print(page_element_attribute)
                page_element.click()
                print("Page backwords...")
                position_elements = browser.find_elements_by_class_name(
                    "position_link")
                position_links.update({p.get_attribute("href")
                                        for p in position_elements})

            except Exception as e:
                errors_set.add(e)
                print(e)
        #reset the flag
        page_element = browser.find_element_by_class_name("pager_next")
        page_element_attribute = page_element.get_attribute("class").strip()

It perform as a metal ball oscillating between the paginator brutally.

Deeply appreciate any of your review feedback.

\$\endgroup\$

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Browse other questions tagged or ask your own question.