This is a follow-up of my question over here.
I am extending the functionality of @Reinderien's suggested code to enable the automatic looping through of pages in the search result.
I've mainly modified the final search() function (originally cnki_search) and added a number_of_articles_and_pages() function to the SearchResults class.
There seems to be a minor bug related to the ContentFilterPlugin in @Reinderien original code. MainPage.max_content failed to register, so 20 instead of 50 items are retrieved at a time.
It can be considered as trivial because given the higher efficiency when using the plugin, all content would still be parsed, perhaps even in a shorter time, with the implementation of a loop through the pages.
Issues:
- I've moved
next_pageout of theMainPageclass into theSearchResultsclass. - Tasks such as looping through the search result seem to be simpler when they are written in functions outside the classes.
- Data that is yield from the generators can either be printed or written to file. If we want to do both concurrently or one after the other, the generator would have to be re-initiated each time.
- Because I was limiting the maximum number of pages to be scraped, the
ContentFilterPluginbug had the effect of retrieving 180 articles instead of 424, which did not tally with what is printed on the console:
424 found. A maximum of 500 will be retrieved.
Scraping page 1/9
Navigating to Next Page
Scraping page 2/9
Navigating to Next Page
Scraping page 3/9
Navigating to Next Page
Scraping page 4/9
Navigating to Next Page
Scraping page 5/9
Navigating to Next Page
Scraping page 6/9
Navigating to Next Page
Scraping page 7/9
Navigating to Next Page
Scraping page 8/9
Navigating to Next Page
Scraping page 9/9
cnki.py
from contextlib import contextmanager
from dataclasses import dataclass
from datetime import date
from pathlib import Path
from typing import Generator, Iterable, Optional, List, ContextManager, Dict
from urllib.parse import unquote
from itertools import chain, count
import re
import json
from math import ceil
# pip install proxy.py
import proxy
from proxy.http.exception import HttpRequestRejected
from proxy.http.parser import HttpParser
from proxy.http.proxy import HttpProxyBasePlugin
from selenium.common.exceptions import (
NoSuchElementException,
StaleElementReferenceException,
TimeoutException,
WebDriverException,
)
from selenium.webdriver import Firefox, FirefoxProfile
from selenium.webdriver.common.by import By
from selenium.webdriver.common.proxy import ProxyType
from selenium.webdriver.remote.webdriver import WebDriver
from selenium.webdriver.remote.webelement import WebElement
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait
# from urllib3.packages.six import X
@dataclass
class Result:
title: str # Mozi's Theory of Human Nature and Politics
title_link: str # http://big5.oversea.cnki.net/kns55/detail/detail.aspx?recid=&FileName=ZDXB202006009&DbName=CJFDLAST2021&DbCode=CJFD
html_link: Optional[str] # http%3a%2f%2fkns.cnki.net%2fKXReader%2fDetail%3fdbcode%3dCJFD%26filename%3dZDXB202006009
author: str # Xie Qiyang
source: str # Vocational University News
source_link: str # http://big5.oversea.cnki.net/kns55/Navi/ScdbBridge.aspx?DBCode=CJFD&BaseID=ZDXB&UnitCode=&NaviLink=%e8%81%8c%e5%a4%a7%e5%ad%a6%e6%8a%a5
date: date # 2020-12-28
download: str #
database: str # Periodical
@classmethod
def from_row(cls, row: WebElement) -> 'Result':
number, title, author, source, published, database = row.find_elements_by_xpath('td')
title_links = title.find_elements_by_tag_name('a')
if len(title_links) > 1:
# 'http://big5.oversea.cnki.net/kns55/ReadRedirectPage.aspx?flag=html&domain=http%3a%2f%2fkns.cnki.net%2fKXReader%2fDetail%3fdbcode%3dCJFD%26filename%3dZDXB202006009'
html_link = unquote(
title_links[1]
.get_attribute('href')
.split('domain=', 1)[1])
else:
html_link = None
dl_links, sno = number.find_elements_by_tag_name('a')
published_date = date.fromisoformat(
published.text.split(maxsplit=1)[0]
)
return cls(
title=title_links[0].text,
title_link=title_links[0].get_attribute('href'),
html_link=html_link,
author=author.text,
source=source.text,
source_link=source.get_attribute('href'),
date=published_date,
download=dl_links.get_attribute('href'),
database=database.text,
)
def __str__(self):
return (
f'題名 {self.title}'
f'\n作者 {self.author}'
f'\n來源 {self.source}'
f'\n發表時間 {self.date}'
f'\n下載連結 {self.download}'
f'\n來源數據庫 {self.database}'
)
def as_dict(self) -> Dict[str, str]:
return {
'author': self.author,
'title': self.title,
'date': self.date.isoformat(),
'download': self.download,
'url': self.html_link,
'database': self.database,
}
class MainPage:
def __init__(self, driver: WebDriver):
self.driver = driver
def submit_search(self, keyword: str) -> None:
wait = WebDriverWait(self.driver, 50)
search = wait.until(
EC.presence_of_element_located((By.NAME, 'txt_1_value1'))
)
search.send_keys(keyword)
search.submit()
def switch_to_frame(self) -> None:
wait = WebDriverWait(self.driver, 100)
wait.until(
EC.presence_of_element_located((By.XPATH, '//iframe[@name="iframeResult"]'))
)
self.driver.switch_to.default_content()
self.driver.switch_to.frame('iframeResult')
wait.until(
EC.presence_of_element_located((By.XPATH, '//table[@class="GridTableContent"]'))
)
def max_content(self) -> None:
"""Maximize the number of items on display in the search results."""
max_content = self.driver.find_element(
By.CSS_SELECTOR, '#id_grid_display_num > a:nth-child(3)',
)
max_content.click()
# def get_element_and_stop_page(self, *locator) -> WebElement:
# ignored_exceptions = (NoSuchElementException, StaleElementReferenceException)
# wait = WebDriverWait(self.driver, 30, ignored_exceptions=ignored_exceptions)
# elm = wait.until(EC.presence_of_element_located(locator))
# self.driver.execute_script("window.stop();")
# return elm
class SearchResults:
def __init__(self, driver: WebDriver):
self.driver = driver
def number_of_articles_and_pages(self) -> int:
elem = self.driver.find_element_by_xpath(
'//table//tr[3]//table//table//td[1]/table//td[1]'
)
n_articles = re.search("共有記錄(.+)條", elem.text).group(1)
n_pages = ceil(int(n_articles)/50)
return n_articles, n_pages
def get_structured_elements(self) -> Iterable[Result]:
rows = self.driver.find_elements_by_xpath(
'//table[@class="GridTableContent"]//tr[position() > 1]'
)
for row in rows:
yield Result.from_row(row)
def get_element_and_stop_page(self, *locator) -> WebElement:
ignored_exceptions = (NoSuchElementException, StaleElementReferenceException)
wait = WebDriverWait(self.driver, 30, ignored_exceptions=ignored_exceptions)
elm = wait.until(EC.presence_of_element_located(locator))
self.driver.execute_script("window.stop();")
return elm
def next_page(self) -> None:
link = self.get_element_and_stop_page(By.LINK_TEXT, "下頁")
try:
link.click()
print("Navigating to Next Page")
except (TimeoutException, WebDriverException):
print("Last page reached")
class ContentFilterPlugin(HttpProxyBasePlugin):
HOST_WHITELIST = {
b'ocsp.digicert.com',
b'ocsp.sca1b.amazontrust.com',
b'big5.oversea.cnki.net',
}
def handle_client_request(self, request: HttpParser) -> Optional[HttpParser]:
host = request.host or request.header(b'Host')
if host not in self.HOST_WHITELIST:
raise HttpRequestRejected(403)
if any(
suffix in request.path
for suffix in (
b'png', b'ico', b'jpg', b'gif', b'css',
)
):
raise HttpRequestRejected(403)
return request
def before_upstream_connection(self, request):
return super().before_upstream_connection(request)
def handle_upstream_chunk(self, chunk):
return super().handle_upstream_chunk(chunk)
def on_upstream_connection_close(self):
pass
@contextmanager
def run_driver() -> ContextManager[WebDriver]:
prox_type = ProxyType.MANUAL['ff_value']
prox_host = '127.0.0.1'
prox_port = 8889
profile = FirefoxProfile()
profile.set_preference('network.proxy.type', prox_type)
profile.set_preference('network.proxy.http', prox_host)
profile.set_preference('network.proxy.ssl', prox_host)
profile.set_preference('network.proxy.http_port', prox_port)
profile.set_preference('network.proxy.ssl_port', prox_port)
profile.update_preferences()
plugin = f'{Path(__file__).stem}.{ContentFilterPlugin.__name__}'
with proxy.start((
'--hostname', prox_host,
'--port', str(prox_port),
'--plugins', plugin,
)), Firefox(profile) as driver:
yield driver
def loop_through_results(driver):
result_page = SearchResults(driver)
n_articles, n_pages = result_page.number_of_articles_and_pages()
print(f"{n_articles} found. A maximum of 500 will be retrieved.")
for page in count(1):
print(f"Scraping page {page}/{n_pages}")
print()
result = result_page.get_structured_elements()
yield from result
if page >= n_pages or page >= 10:
break
result_page.next_page()
result_page = SearchResults(driver)
def save_articles(articles: Iterable, file_prefix: str) -> None:
file_path = Path(file_prefix).with_suffix('.json')
with file_path.open('w') as file:
file.write('[\n')
first = True
for article in articles:
if first:
first = False
else:
file.write(',\n')
json.dump(article.as_dict(), file, ensure_ascii=False, indent=4)
file.write('\n]\n')
def query(keyword, driver) -> None:
page = MainPage(driver)
page.submit_search(keyword)
page.switch_to_frame()
page.max_content()
def search(keyword):
with Firefox() as driver:
driver.get('http://big5.oversea.cnki.net/kns55/')
query(keyword, driver)
result = loop_through_results(driver)
save_articles(result, 'cnki_search_result.json')
if __name__ == '__main__':
search('古文尚書')
Output (truncated):
[
{
"author": "王祥辰",
"title": "惠棟與吳派經學研究",
"date": "2020-06-10",
"download": "http://big5.oversea.cnki.net/kns55/download.aspx?filename=jZVFjZ5UDZKVEUnFTSxUGcNJlNJBDZDxWeiNlez52az8GMBV1QLFlNStmQolUQxUjTzoUbyFmYjJXS=0TQHZ0MWhmWhRlQsNUQWFkQD5WaMBzV5ZkM0MnUUtCVysSSGN3aCRnRy8kb0V1KKRVbiJkeSlWdrU&tablename=CDFDLAST2021&dflag=pdfdown",
"url": null,
"database": "博士"
},
{
"author": "余康",
"title": "章太炎《尚書》研究述論",
"date": "2017-05-01",
"download": "http://big5.oversea.cnki.net/kns55/download.aspx?filename=jZVFjZ5UDZKVEUnFTSxUGcNJlNJBDZDxWeiNlez52az8GMBV1QLFlNStmQolUQxUjTzoUbyFmYjJXS=0DNKtWZyolbytmRrJFbxgHbtBXY1glQIF2Kys2aKtCVysSSGN3aCRnRy8kb0V1KKRVbiJkeSlWdrU&tablename=CDFDLAST2020&dflag=pdfdown",
"url": null,
"database": "博士"
},
{
"author": "崔海鷹",
"title": "孔傳《古文尚書》淵源與成書問題探論",
"date": "2014-04-01",
"download": "http://big5.oversea.cnki.net/kns55/download.aspx?filename=rUjZVFjZ5UDZKVEUnFTSxUGcNJlNJBDZDxWeiNlez52az8GMBV1QLFlNStmQolUQxUjTzoUbyFmYjJXSStWd4JFVJhVQxRmNjZWWKxmZxZFUrg1asdzSV9CW5RTQT52TysSSGN3aCRnRy8kb0V1KKRVbiJkeSlWd&tablename=CDFD1214&dflag=pdfdown",
"url": null,
"database": "博士"
},
{
"author": "王祥辰",
"title": "清代吳派《尚書》學疑辨成就管窺——以《古文尚書考》為中心",
"date": "2018-03-20",
"download": "http://big5.oversea.cnki.net/kns55/download.aspx?filename=rUjZVFjZ5UDZKVEUnFTSxUGcNJlNJBDZDxWeiNlez52az8GMBV1QLFlNStmQolUQxUjTzoUbyFmYjJXSH1WQXdGTKNDdRR0VuNWTyVnT3BTel9meOlVO4lDavF1U5o0Q6djcmpWQodjRy8kb0V1KKRVbiJkeSlWd&tablename=CJFDLAST2018&dflag=pdfdown",
"url": null,
"database": "期刊"
},
{
"author": "蘆倩",
"title": "古文《尚書》復合詞研究",
"date": "2015-05-01",
"download": "http://big5.oversea.cnki.net/kns55/download.aspx?filename=jZVFjZ5UDZKVEUnFTSxUGcNJlNJBDZDxWeiNlez52az8GMBV1QLFlNStmQolUQxUjTzoUbyFmYjJXS=0DMFdWSDNjQYhXaMJEUKF3M5gzLxN1Y2MTOwUFZJR1UNZENChUUSZnRy8kb0V1KKRVbiJkeSlWdrU&tablename=CMFD201601&dflag=pdfdown",
"url": null,
"database": "碩士"
},
{
"author": "盧秀松",
"title": "古文《尚書》代詞研究",
"date": "2015-05-01",
"download": "http://big5.oversea.cnki.net/kns55/download.aspx?filename=jZVFjZ5UDZKVEUnFTSxUGcNJlNJBDZDxWeiNlez52az8GMBV1QLFlNStmQolUQxUjTzoUbyFmYjJXS=0TRQ9EaI1mQYhXaMJEUKF3M5gzLxN1Y2MTOwUFZJR1UNZENChUUSZnRy8kb0V1KKRVbiJkeSlWdrU&tablename=CMFD201601&dflag=pdfdown",
"url": null,
"database": "碩士"
},
{
"author": "嚴璐",
"title": "今古文《尚書》復音單純詞研究",
"date": "2013-05-01",
"download": "http://big5.oversea.cnki.net/kns55/download.aspx?filename=rUjZVFjZ5UDZKVEUnFTSxUGcNJlNJBDZDxWeiNlez52az8GMBV1QLFlNStmQolUQxUjTzoUbyFmYjJXSGNGdTd0ZwhGZpRTcGtCRw8yc0xke0NVcPhnSCNDdi90SoN1UNZENChUUSZnRy8kb0V1KKRVbiJkeSlWd&tablename=CMFD201401&dflag=pdfdown",
"url": null,
"database": "碩士"
},
{
"author": "史振卿",
"title": "清代《尚書》學若干問題研究",
"date": "2011-05-01",
"download": "http://big5.oversea.cnki.net/kns55/download.aspx?filename=jZVFjZ5UDZKVEUnFTSxUGcNJlNJBDZDxWeiNlez52az8GMBV1QLFlNStmQolUQxUjTzoUbyFmYjJXS=0zYqRla54EONFlUJdlYuRmQGlmd6JGa0c2K6tke4p2TysSSGN3aCRnRy8kb0V1KKRVbiJkeSlWdrU&tablename=CDFD0911&dflag=pdfdown",
"url": null,
"database": "博士"
},
{
"author": "丁鼎",
"title": "“偽《古文尚書》案”平議",
"date": "2010-03-25",
"download": "http://big5.oversea.cnki.net/kns55/download.aspx?filename=jZVFjZ5UDZKVEUnFTSxUGcNJlNJBDZDxWeiNlez52az8GMBV1QLFlNStmQolUQxUjTzoUbyFmYjJXS=0TSG92ZrpGck9mR4Jzaw9yRRFjVURFbw0WZxgmM2EWOFRVZBtWVBNnRy8kb0V1KKRVbiJkeSlWdrU&tablename=CJFD2010&dflag=pdfdown",
"url": "http://kns.cnki.net/KXReader/Detail?dbcode=CJFD&filename=GJZL201002003",
"database": "期刊"
},
......
{
"author": "藏生",
"title": "《今文尚書》校詁(三)——《禹貢》《甘誓》《湯誓》四十二則",
"date": "1998-08-15",
"download": "http://big5.oversea.cnki.net/kns55/download.aspx?filename=kN2d3UTbyI3d4MkRslmQzl1KvcEVK1UVTNTVYJzLkFVNGN0M6JURwgHbUJ2bEZFN4gHZ6BVdw9EawdUa3hmaPlWcOdWUapHeRBFUxpGWQZmZmhlSGNzTqRXWvsWMjN3SLRkQWRHb5FFZNFlazdUcDlGV3EzSvBlM&tablename=CJFD9899&dflag=pdfdown",
"url": null,
"database": "期刊"
},
{
"author": "劉起釪",
"title": "《尚書》的隸古定本、古寫本",
"date": "1980-06-29",
"download": "http://big5.oversea.cnki.net/kns55/download.aspx?filename=2d3UTbyI3d4MkRslmQzl1KvcEVK1UVTNTVYJzLkFVNGN0M6JURwgHbUJ2bEZFN4gHZ6BVdw9EawdUa=0TU4tSav5GZDlGcUZzc50UTHlXdhRXYJVUYOd0S6VVRHh0MhZDbrUFZNFlazdUcDlGV3EzSvBlMkN&tablename=CJFD7984&dflag=pdfdown",
"url": null,
"database": "期刊"
},
{
"author": "劉起釪",
"title": "關于隸古定與河圖洛書問題",
"date": "1997-04-15",
"download": "http://big5.oversea.cnki.net/kns55/download.aspx?filename=kN2d3UTbyI3d4MkRslmQzl1KvcEVK1UVTNTVYJzLkFVNGN0M6JURwgHbUJ2bEZFN4gHZ6BVdw9EawdUaUNmNxETO0AFR5A1bYZlamxkWw4Uc4FTSFlza1MXZzdWYWBHeN50ZRVnMwdFZNFlazdUcDlGV3EzSvBlM&tablename=CJFD9697&dflag=pdfdown",
"url": "http://kns.cnki.net/KXReader/Detail?dbcode=CJFD&filename=CTWH199702004",
"database": "期刊"
},
{
"author": "劉家和",
"title": "《史記》與漢代經學",
"date": "1991-05-01",
"download": "http://big5.oversea.cnki.net/kns55/download.aspx?filename=2d3UTbyI3d4MkRslmQzl1KvcEVK1UVTNTVYJzLkFVNGN0M6JURwgHbUJ2bEZFN4gHZ6BVdw9EawdUa=0TQEJ1NLdEdzwkZWFHZ2F1bORjRVl2c69EO3c0S6VVRHh0MhZDbrUFZNFlazdUcDlGV3EzSvBlMkN&tablename=CJFD9093&dflag=pdfdown",
"url": "http://kns.cnki.net/KXReader/Detail?dbcode=CJFD&filename=SYSJ199102003",
"database": "期刊"
},
{
"author": "陸建初",
"title": "《尚書》史詩略考",
"date": "2009-03-18",
"download": "http://big5.oversea.cnki.net/kns55/download.aspx?filename=kN2d3UTbyI3d4MkRslmQzl1KvcEVK1UVTNTVYJzLkFVNGN0M6JURwgHbUJ2bEZFN4gHZ6BVdw9EawdUaHZDWOBlbMhjaU10UBBFeGR1K19iSRZ3NRZ1QONEVsR1U0E2YLRkQWRHb5FFZNFlazdUcDlGV3EzSvBlM&tablename=CJFD2009&dflag=pdfdown",
"url": "http://kns.cnki.net/KXReader/Detail?dbcode=CJFD&filename=YNXS200902010",
"database": "期刊"
},
{
"author": "盂光宇",
"title": "《千字文》批注",
"date": "1974-06-15",
"download": "http://big5.oversea.cnki.net/kns55/download.aspx?filename=2d3UTbyI3d4MkRslmQzl1KvcEVK1UVTNTVYJzLkFVNGN0M6JURwgHbUJ2bEZFN4gHZ6BVdw9EawdUa=0DNmN2VBJlVsR3YXBTQxQ3RtJUZaZEU0E3aYlDWOpmZvZ2MhZDbrUFZNFlazdUcDlGV3EzSvBlMkN&tablename=CJFD7984&dflag=pdfdown",
"url": null,
"database": "期刊"
},
{
"author": "金敏",
"title": "“法”的故事的另一種講法",
"date": "2018-12-15",
"download": "http://big5.oversea.cnki.net/kns55/download.aspx?filename=2d3UTbyI3d4MkRslmQzl1KvcEVK1UVTNTVYJzLkFVNGN0M6JURwgHbUJ2bEZFN4gHZ6BVdw9EawdUa=0DOYF2dUp1bYtyMxMVcXZkWoZTbM9UOyZHbFJnQx0kZUtyY15UeaVFZNFlazdUcDlGV3EzSvBlMkN&tablename=CJFDLAST2019&dflag=pdfdown",
"url": "http://kns.cnki.net/KXReader/Detail?dbcode=CJFD&filename=FLPL201806021",
"database": "期刊"
},
{
"author": "趙穹天",
"title": "近出楚簡疑難字詞匯考",
"date": "2013-03-01",
"download": "http://big5.oversea.cnki.net/kns55/download.aspx?filename=3EzSvBlMkN2d3UTbyI3d4MkRslmQzl1KvcEVK1UVTNTVYJzLkFVNGN0M6JURwgHbUJ2bEZFN4gHZ6BVdw9EawdUa9cXUnJXTENWSGZXb0tmWroEd3p3MDZHehVlU5ZHczJzRvwkaXdUWRVzUTRXQBVWMIRnZnhzbYdFZNFlazdUcDlGV&tablename=CMFD201402&dflag=pdfdown",
"url": null,
"database": "碩士"
}
]
as_dictindentation is incorrect \$\endgroup\$