This is a follow-up to my question over here.
Having worked through most of the issues highlighted by @Reinderien in his detailed answer, I somehow got the code to work...
Some quick observations:
elements.py
and theSearchTextElement
class frompage.py
seem to be obsolete to me, after revising thesubmit_search()
function. But I still feel that given the fact that this design approach is suggested by the author(s) of the Selenium Manual, there ought to be some merit in it. One thing I had in mind when adopting their approach was to make use of theBasePageElement
class (this is the "class of descriptors" described in my title) to do all the "getting" and "setting" of data on a webpage instance. The "getting" can be done via a static variable that represents an instance ofSearchTextElement
, as demonstrated in the manual. The downside to this is: the locator for the search text field would not be part oflocators.py
, but secretly tucked away in an obscure class whose sole purpose is to declare that variable. I still haven't figured out howBasePageElement
can be invoked to do the "getting" work.- I tried putting
driver.get(CNKI.URL)
inMainPage
's__init__
function (it has since been commented out) but the code did not run well. I got caught in a "Catch-22"-like situation whereby initiation of bothMainPage
andSearchResult
requiredriver
as an argument. Which means I cannot do:
class MainPage(BasePage):
"""Home page action methods come here. I.e. Python.org"""
def __init__(self, driver, SRC):
super().__init__(driver, SRC)
self.driver = driver()
self.SRC = SRC
self.driver.get(SRC.URL)
Because even though the browser and webpage get launched smoothly over here, back in test.py
, driver
still has to be defined so that it can be plugged into MainPage
and SearchResults
respectively:
driver = webdriver.Chrome # I use inert driver over here to avoid launching another empty browser.
cnki_search = MainPage(driver, CNKI) # driver gets passed to `MainPage.__init__` and activated.
result = SearchResults(driver, CNKI) # inert driver gets passed to `SearchResults` and spoils the soup.
I tried making use of class inheritance to solve the problem by defining driver
and SRC
as class variables (as opposed to instance) in BasePage
so that SearchResults
can somehow inherit that activated driver
instance from its parent class, but so long as SearchResults
needs to be instantiated in test.py
, the active driver
would be overwritten by the inert.
Any advice pertaining the above issues and/or other suggestions to improve the code would be greatly appreciated.
The Code:
test.py
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from selenium import webdriver
from selenium.webdriver.chrome.webdriver import WebDriver
from locators import CNKI
from page import BasePage, MainPage, SearchResults
def cnki_search():
driver = webdriver.Chrome()
driver.get(CNKI.URL)
cnki_search = MainPage(driver, CNKI)
cnki_search.submit_search('人性論')
# cnki_search.search_keyword = '人性論'
# cnki_search.select_dropdown_item('篇名')
# cnki_search.click_search_button()
cnki_search.switch_to_frame()
cnki_search.max_content()
result = SearchResults(driver, CNKI)
titles_grp = result.get_structured_elements('titles')
# cnki_search.next_page()
cnki_search()
page.py
# -*- coding: utf-8 -*-
from element import BasePageElement
from selenium import webdriver
from selenium.common.exceptions import TimeoutException, WebDriverException
from selenium.common.exceptions import StaleElementReferenceException
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver.support.ui import WebDriverWait
class SearchTextElement(BasePageElement):
"""This class gets the search text from the specified locator"""
#The locator for search box where search string is entered
locator = None
class BasePage:
"""Base class to initialize the base page that will be called from all
pages"""
def __init__(self, driver, SRC):
BasePage.driver = driver
BasePage.SRC = SRC
class MainPage(BasePage):
"""Home page action methods come here. I.e. Python.org"""
# SRC = None
# search_keyword = SearchTextElement()
def __init__(self, driver, SRC):
super().__init__(driver, SRC)
self.driver = driver
self.SRC = SRC
# self.driver.get(SRC.URL)
def submit_search(self, keyword):
driver = self.driver
wait = WebDriverWait(driver, 50)
search = wait.until(EC.presence_of_element_located
(self.SRC.INPUT['SEARCH_FIELD']))
search.send_keys(keyword)
search.submit()
# THIS ONE MAKES USE OF SearchTextElement; the above doesn't.
#
# def submit_search(self, keyword):
# """Submits keyword and triggers the search"""
# SearchTextElement.locator = self.SRC.INPUT['SEARCH_FIELD']
# self.search_keyword = keyword
def select_dropdown_item(self, item):
driver = self.driver
by, val = self.SRC.INPUT['SEARCH_ATTR']
driver.find_element(by, val + "/option[text()='" + item + "']").click()
def click_search_button(self):
driver = self.driver
element = driver.find_element(*self.SRC.INPUT['SEARCH_BUTTON'])
element.click()
def switch_to_frame(self):
"""Use this function to get access to hidden elements. """
driver = self.driver
wait = WebDriverWait(driver, 100)
wait.until(
lambda driver:
driver.find_element_by_xpath('//iframe[@name="iframeResult"]'))
driver.switch_to.default_content()
driver.switch_to.frame('iframeResult')
def max_content(self):
"""Maximize the number of items on display in the search results."""
driver = self.driver
max_content = driver.find_element(*self.SRC.INPUT['MAX_CONTENT'])
max_content.click()
def stop_loading_page_when_element_is_present(self, locator):
driver = self.driver
ignored_exceptions = (NoSuchElementException, StaleElementReferenceException)
wait = WebDriverWait(driver, 30, ignored_exceptions=ignored_exceptions)
wait.until(
EC.presence_of_element_located(locator))
driver.execute_script("window.stop();")
def next_page(self):
driver = self.driver
self.stop_loading_page_when_element_is_present(self.SRC.INPUT['NEXT_PAGE'])
driver.execute_script("window.stop();")
try:
driver.find_element(*self.SRC.INPUT['NEXT_PAGE']).click()
print("Navigating to Next Page")
except (TimeoutException, WebDriverException):
print("Last page reached")
class SearchResults(BasePage):
"""Search results page action methods come here"""
def __init__(self, driver, SRC):
super().__init__(driver, SRC)
self.driver = driver
self.SRC = SRC
def wait_for_page_to_load(self):
driver = self.driver
wait = WebDriverWait(driver, 100)
wait.until(
EC.presence_of_element_located(*self.SRC.INPUT['MAIN_BODY']))
def get_single_element(self, name):
"""Returns a single value as target data."""
driver = self.driver
target_data = driver.find_element(*self.SRC.OUTPUT[str(name.upper())])
return target_data
def number_of_items_found(self):
"""Return the number of items found on a single page."""
driver = self.driver
target_data = driver.find_elements(*self.SRC.OUTPUT['INDEX'])
return len(target_data)
def get_elements(self, name):
"""Returns simple list of values in specific data field in a table."""
driver = self.driver
target_data = driver.find_elements(*self.SRC.OUTPUT[str(name.upper())])
for item in target_data:
yield item.text
def get_structured_elements(self, name):
"""For target data that is nested and structured,
such as a table with multiple values in a single cell."""
driver = self.driver
i = 2 # keep track of 'i' to retain the document structure.
number_of_items = self.number_of_items_found()
elements = [None] * number_of_items
# Manage split-xpaths for iteration.
front_half, back_half = self.SRC.OUTPUT['SPLIT_XPATHS'][str(name.upper())]
while i - 2 < number_of_items:
target_data = driver.find_elements_by_xpath(front_half + str(i) + back_half)
for item in target_data:
print(item.text, i - 1)
if elements[i - 2] == None:
elements[i - 2] = item.text
elif isinstance(elements[i - 2], list):
elements[i - 2].append(item.text)
else:
elements[i - 2] = [elements[i - 2]]
elements[i - 2].append(item.text)
i += 1
return elements
def simple_get_structured_elements(self):
"""Simple structured elements code with fixed xpath.
FOR DEBUGGING."""
driver = self.driver
i = 2 # keep track of 'i' to retain the document structure.
number_of_items = self.number_of_items_found()
elements = [None] * number_of_items
while i - 2 < number_of_items:
target_data = driver.find_elements_by_xpath
('//*[@id="Form1"]/table/tbody/tr[2]/td/table/tbody/tr['
+ str(i) + ']/td[2]/a')
for item in target_data:
print(item.text, i-1)
if elements[i - 2] == None:
elements[i - 2] = item.text
elif isinstance(elements[i - 2], list):
elements[i - 2].append(item.text)
else:
elements[i - 2] = [elements[i - 2]]
elements[i - 2].append(item.text)
i += 1
return elements
locators.py
# -*- coding: utf-8 -*-
from selenium.webdriver.common.by import By
# All locators specific to a certian web domain goes here, organized into classes.
class CNKI:
URL = 'http://big5.oversea.cnki.net/kns55/'
INPUT = {
"MAIN_BODY": (By.XPATH, '//GridTableContent/tbody'),
"SEARCH_FIELD": (By.NAME, 'txt_1_value1'),
"SEARCH_ATTR": (By.XPATH, '//select[@name = txt_1_sel]'), # dropdown list xpath.
"SEARCH_BUTTON": (By.ID, 'btnSearch'),
"NEXT_PAGE": (By.LINK_TEXT, "下頁"),
"MAX_CONTENT": (By.CSS_SELECTOR, '#id_grid_display_num > a:nth-child(3)'),
}
OUTPUT = {
# Xpaths split at row number to be iterated, "i".
"SPLIT_XPATHS": {
"TITLES": ('//*[@id="Form1"]/table/tbody/tr[2]/td/table/tbody/tr[', ']/td[2]/a'),
"AUTHORS": ('//*[@id="Form1"]/table/tbody/tr[2]/td/table/tbody/tr[', ']/td[3]/a')
},
"JOURNALS": (By.XPATH,'//*[@id="Form1"]/table/tbody/tr[2]/td/table/tbody/tr/td[4]/a'),
"YEAR_ISSUE": (By.XPATH,'//*[@id="Form1"]/table/tbody/tr[2]/td/table/tbody/tr/td[5]/a'),
"DOWNLOAD_PATHS": (By.XPATH,'//*[@id="Form1"]/table/tbody/tr[2]/td/table/tbody/tr/td[1]/table/tbody/tr/td/a[1]'),
"INDEX": (By.XPATH, '//*[@id="Form1"]/table/tbody/tr[2]/td/table/tbody/tr/td[1]/table/tbody/tr/td/a[2]')
}
# //*[@id="Form1"]/table/tbody/tr[2]/td/table/tbody/tr[2]/td[1]/table/tbody/tr/td/a[2]
# //*[@id="Form1"]/table/tbody/tr[2]/td/table/tbody/tr[3]/td[1]/table/tbody/tr/td/a[2]
element.py
# -*- coding: utf-8 -*-
from selenium.webdriver.support.ui import WebDriverWait
class BasePageElement:
"""Base page class that is initialized on every page object class."""
def __set__(self, obj, value):
"""Sets the text to the value supplied"""
driver = obj.driver
text_field = WebDriverWait(driver, 100).until(
lambda driver: driver.find_element(*self.locator))
text_field.clear()
text_field.send_keys(value)
text_field.submit()
def __get__(self, obj, owner):
"""Gets the text of the specified object"""
driver = obj.driver
WebDriverWait(driver, 100).until(
lambda driver: driver.find_element(*self.locator))
element = driver.find_element(*self.locator)
return element.get_attribute("value")