I am trying to use Scrapy for one of the sites I've scraped before using Selenium over here.
Because the search field for this site is dynamically generated and requires the user to hover the cursor over a button before it appears, I can't seem to find a way to POST the query using Requests or Scrapy's spider alone.
In scrapy shell, though I can:
fetch(FormRequest.from_response(response,
formdata={'.search-left input':"尹至"},
callback=self.search_result))
I have no way to tell whether the search query is successful or not.
Here is a simple working code which I will be using for my spider below.
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver import Firefox, FirefoxProfile
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
import time
def parse(url):
with Firefox() as driver:
driver.get(url)
wait = WebDriverWait(driver, 100)
xpath = "//form/button/input"
element_to_hover_over = driver.find_element_by_xpath(xpath)
hover = ActionChains(driver).move_to_element(element_to_hover_over)
hover.perform()
search = wait.until(
EC.presence_of_element_located((By.ID, 'showkeycode1015273'))
)
search.send_keys("尹至")
search.submit()
time.sleep(5)
rows = driver.find_elements_by_css_selector(".search_list > li")
for row in rows:
caption_elems = row.find_element_by_tag_name('a')
yield {
"caption" : caption_elems.text,
"date": row.find_element_by_class_name('time').text,
"url": caption_elems.get_attribute('href')
}
x = parse('https://www.ctwx.tsinghua.edu.cn')
for rslt in x:
print(rslt)
The scrapy spider below stops short at entering the search query when I run scrapy crawl qinghua.
import scrapy
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver import Firefox, FirefoxProfile
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
import time
class QinghuaSpider(scrapy.Spider):
name = 'qinghua'
allowed_domains = ['https://www.ctwx.tsinghua.edu.cn']
start_urls = ['https://www.ctwx.tsinghua.edu.cn']
def __init__(self):
self.driver = webdriver.Firefox()
def parse(self, response):
with Firefox() as driver:
driver.get(response.url)
wait = WebDriverWait(self.driver, 100)
xpath = "//form/button/input"
element_to_hover_over = driver.find_element_by_xpath(xpath)
hover = ActionChains(driver).move_to_element(element_to_hover_over)
hover.perform()
search = wait.until(
EC.presence_of_element_located((By.ID, 'showkeycode1015273'))
)
search.send_keys("尹至")
search.submit()
time.sleep(5)
rows = self.driver.find_elements_by_css_selector(".search_list > li")
for row in rows:
caption_elems = row.find_element_by_tag_name('a')
yield {
"caption" : caption_elems.text,
"date": row.find_element_by_class_name('time').text,
"url": caption_elems.get_attribute('href')
}
# return FormRequest.from_response(
# response,
# formdata={'.search-left input':"尹至"},
# callback=self.search_result)
def search_result(self, response):
pass
I would like to ask:
- Why the spider code doesn't work, and
- How to do this properly in Scrapy, with or (preferably) without the help of Selenium.
I suspect this website has a robust anti-bot infrastructure that can prevent spiders from operating properly.
