Hi I'm fairly new to coding and would appreciate some feedback on the code. This is for a site that has a dynamic login page and infinite scrolling.
I wanted to use scrapy not necessarily because it needs the framework, but because I hadn't done infinite scrolling and dynamic content with scrapy.
I'm aware that it's essentially one big long list of code. I'm not actually too sure how to split this up because I'm not requesting another page till the end of the code. I've tried an iteration of using a loop and response.follow for each link but the scrapy output was abit messed up.
Is there anyway to make this into discrete functions and abit more readable ? I know some people like splash for dynamic content with scrapy.
I also think the drop down menu code is abit repetitive.
Minor details but I have noticed that the sellersusername and sellerscompany_name come out as a list but the others don't. I cant really account for that! I've tried explicitly defining input processors without too much success! I can also edit this in pandas so not too concerned by it but if there's a more efficient way that would be awesome.
Also I feel there's probably a way to drop the column file_url and file from the output after it's been put to the FilePipeLine, but haven't quite worked that out!
Appreciate the feedback
import scrapy
import logging
from scrapy.utils.log import configure_logging
from selenium import webdriver
from selenium.webdriver.support import ui
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.action_chains import ActionChains
from scrape.items import ScrapeItem
from scrapy.loader import ItemLoader
from scrape.items import MyItemLoader
from scrapy.selector import Selector
import time
class DataSpider(scrapy.Spider):
name = 'data'
allowed_domains = ['scrapo.com']
start_urls = ['http://scrapo.com/']
custom_settings = {
'LOG_FILE': 'my_log.log',
'LOG_LEVEL': 'ERROR'}
logging.getLogger().addHandler(logging.StreamHandler())
def parse(self, response):
self.driver = webdriver.Chrome(executable_path = r'C:/Users/Aaron/Downloads/chromedriver_win32/chromedriver.exe')
self.driver.get('http://scrapo.com/')
self.driver.find_element_by_xpath("/html/body/header/div[2]/div/div[1]/a[2]").clck()
time.sleep(1)
#login procedure
self.driver.find_element_by_id('loginform-email').send_keys('xxxx')
self.driver.find_element_by_id('loginform-password').send_keys('xxxxx')
self.driver.find_element_by_id('btnSignin').click()
time.sleep(10)
#Dynamic drop down menu. Waiting for each element before click()
x = self.driver
wait = ui.WebDriverWait(self.driver,10)
action = ActionChains(self.driver)
wait.until(lambda x: self.driver.find_element_by_xpath('//*[@id="form_search_filter"]/div[2]/div[2]/div[1]/div'))
firstLevelMenu = self.driver.find_element_by_xpath('//*[@id="form_search_filter"]/div[2]/div[2]/div[1]/div')
action.move_to_element(firstLevelMenu).perform()
wait.until(lambda x: self.driver.find_element_by_xpath('//*[@id="mCSB_2_container"]/li[2]/label/span'))
secondLevelMenu = self.driver.find_element_by_xpath('//*[@id="mCSB_2_container"]/li[2]/label/span')
secondLevelMenu.click()
wait.until(lambda x: self.driver.find_element_by_xpath('//*[@id="form_search_filter"]/div[2]/div[3]/div[1]/div'))
menu = self.driver.find_element_by_xpath('//*[@id="form_search_filter"]/div[2]/div[3]/div[1]/div')
action.move_to_element(menu).perform()
wait.until(lambda x: self.driver.find_element_by_xpath('//*[@id="mCSB_3_container"]/li[8]/label/span'))
thirdLevelMenu = self.driver.find_element_by_xpath('//*[@id="mCSB_3_container"]/li[8]/label/span')
thirdLevelMenu.click()
wait.until(lambda x: self.driver.find_element_by_xpath('//*[@id="mCSB_3_container"]/li[11]/label/span'))
fourthLevelMenu = self.driver.find_element_by_xpath('//*[@id="mCSB_3_container"]/li[11]/label/span')
action.move_to_element(fourthLevelMenu).perform()
fourthLevelMenu.click()
wait.until(lambda x: self.driver.find_element_by_xpath('//*[@id="mCSB_3_container"]/li[12]/label/span'))
fifthLevelMenu = self.driver.find_element_by_xpath('//*[@id="mCSB_3_container"]/li[12]/label/span')
action.move_to_element(fifthLevelMenu).perform()
fifthLevelMenu.click()
wait.until(lambda x: self.driver.find_element_by_xpath('//*[@id="mCSB_3_container"]/li[25]/label/span'))
sixthLevelMenu = self.driver.find_element_by_xpath('//*[@id="mCSB_3_container"]/li[25]/label/span')
action.move_to_element(sixthLevelMenu).perform()
sixthLevelMenu.click()
wait.until(lambda x: self.driver.find_element_by_xpath('//*[@id="mCSB_3_container"]/li[26]/label/span'))
seventhLevelMenu = self.driver.find_element_by_xpath('//*[@id="mCSB_3_container"]/li[26]/label/span')
action.move_to_element(seventhLevelMenu).perform()
seventhLevelMenu.click()
wait.until(lambda x: self.driver.find_element_by_xpath('//*[@id="form_search_filter"]/div[2]/div[3]/div[2]/div/div/button'))
eigthLevelMenu = self.driver.find_element_by_xpath('//*[@id="form_search_filter"]/div[2]/div[3]/div[2]/div/div/button')
eigthLevelMenu.click()
#Infinite scroll logic
pre_scroll_height = self.driver.execute_script('return document.body.scrollHeight;')
run_time, max_run_time = 0, 10
while True:
iteration_start = time.time()
# Scroll webpage, the 100 allows for a more 'aggressive' scroll
self.driver.execute_script('window.scrollTo(0, 1*document.body.scrollHeight);')
time.sleep(1)
post_scroll_height = self.driver.execute_script('return document.body.scrollHeight;')
scrolled = post_scroll_height != pre_scroll_height
timed_out = run_time >= max_run_time
if scrolled:
run_time = 0
pre_scroll_height = post_scroll_height
elif not scrolled and not timed_out:
run_time += time.time() - iteration_start
elif not scrolled and timed_out:
self.logger.info('Reached end of page')
break
#Grab links and store in list after infinite scroll finished
scrapy_selector = Selector(text = self.driver.page_source)
rel_links_list = scrapy_selector.xpath('.//div[@class="grid_product_header"]/a/@href').extract()
self.logger.info('Theres a total of ' + str(len(rel_links_list)) + ' links.')
links = ['https://www.scrapo.com' + rel_link for rel_link in rel_links_list]
#looping over every link
#Item loader for fields given
q = 1
for link in links:
self.logger.info('Item #' + str(q))
self.driver.get(link)
self.driver.implicitly_wait(50)
q = q+1
plastic_scrapy_selector = Selector(text = self.driver.page_source)
l = MyItemLoader(item = ScrapeItem(),selector = plastic_scrapy_selector)
l.add_xpath('Title','.//h1[@class="product-small-info_title"]/@title')
l.add_xpath('Category','//div[@class="additional-info"]/div/span[@class="additional-info-item_description"]/@title')
l.add_xpath('Condition','.//div[@class="additional-info"]/div[2]/span[@class="additional-info-item_description"]/@title')
l.add_xpath('Quantity','.//div[@class="additional-info"]/div[3]//span[@class="additional-info-item_description"]/@title')
l.add_xpath('Supply','.//div[@class="additional-info"]/div[4]//span[@class="additional-info-item_description"]/@title')
l.add_xpath('Pricingterms','.//div[@class="additional-info"]/div[5]//span[@class="additional-info-item_description"]/@title')
l.add_xpath('Location','.//div[@class="additional-info"]/div[6]//span[@class="additional-info-item_description"]/@title')
l.add_xpath('Sellersusername','.//div[@class="user-info-name"]/text()')
l.add_xpath('Sellerscompany_name','.//p[@class="company-name"]/text()')
l.add_xpath('Price', '//span[@class="value price"]/b/text()')
l.add_xpath('Price', '//span[@class="value price"]/text()')
l.add_xpath('file_urls','//*[@id="app"]/div[1]/div[1]/div[1]/div[1]/div[1]/a/@href')
yield l.load_item()
self.driver.close()
Here is the settings.py
import scrapy
from scrapy.loader import ItemLoader
from scrapy.loader.processors import TakeFirst, MapCompose
class ScrapeItem(scrapy.Item):
name = scrapy.Field()
Title = scrapy.Field()
Category = scrapy.Field()
Condition = scrapy.Field()
Quantity = scrapy.Field()
Supply = scrapy.Field()
Pricingterms = scrapy.Field()
Location = scrapy.Field()
Sellersusername = scrapy.Field()
Sellerscompany_name = scrapy.Field()
Sellerslocation = scrapy.Field()
Price = scrapy.Field()
file_urls = scrapy.Field()
files = scrapy.Field()
class MyItemLoader(ItemLoader):
Title_out = TakeFirst()
Category_out = TakeFirst()
Condition_out = TakeFirst()
Quantity_out = TakeFirst()
Supply_out = TakeFirst()
Pricingterms_out = TakeFirst()
Location_out = TakeFirst()
Sellerslocation_out = TakeFirst()
Here's the settings.py
BOT_NAME = 'scrape'
SPIDER_MODULES = ['scrape.spiders']
NEWSPIDER_MODULE = 'scrape.spiders'
FILES_STORE = 'c:/Users/Aaron/python/output'
RANDOMIZE_DOWNLOAD_DELAY = True
ROBOTSTXT_OBEY = True
CONCURRENT_REQUESTS = 1
DOWNLOAD_DELAY = 5
CONCURRENT_REQUESTS_PER_IP = 3
DOWNLOADER_MIDDLEWARES = {
'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': None,
'scrapy_user_agents.middlewares.RandomUserAgentMiddleware': 400
}
ITEM_PIPELINES = {'scrapy.pipelines.files.FilesPipeline': 1}
AUTOTHROTTLE_ENABLED = True
AUTOTHROTTLE_START_DELAY = 2
HTTPCACHE_ENABLED = True
HTTPCACHE_DIR = 'httpcache'
HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
There are no errors from the scrapy log and it managed to grab all data that is there.
Output:
{"Title": "LDPE Film",
"Category": "LDPE", "Condition": "Other condition",
"Quantity": "40,000 lbs",
"Supply": "Ongoing", "Pricingterms": "Other",
"Location": "Wisconsin, United States",
"Sellersusername": ["Fulfilled by Scrapo"],
"Sellerscompany_name": ["Scrapo, Inc"],
"Price": ["\u00a30.20",
"\n ", " / lb "],
"file_urls":
["https://d1vpmfwd72pjy6.cloudfront.net/uploads/product/514e36e92b2f.jpeg"],
"files": [{"url":
"https://d1vpmfwd72pjy6.cloudfront.net/uploads/product/514e36e92b2f.jpeg",
"path": "full/c07f35499c3dc329d54c2fd3d85c461a1598a172.jpeg", "checksum":
"b20c16f6e99b37ee6e2d9c89d6503def"}]}
