Scrapy -- Very Procedural -- Better Alternatives?

Question

   class PropFinal(CrawlSpider):
   name = "propfinal"
   allowed_domains = [over 200 allowed domains here]
   start_urls = ["http://www.putnam-fl.com/coc/index.php?option=com_content&view=article&id=385&Itemid=114"       
    ]



def __init__(self, *args, **kwargs):
          ##Is there a better way to add all 200 or more requests?
            self.requests_list=[]
            link = 'http://www.putnam-fl.com/clerks_web_apps/foreclosure_trans/get_cases.php'
            request = Request(link, callback=self.parse_putnam)
            self.requests_list.append(request)
            link = 'https://mypalmbeachclerk.clerkauction.com/'
            request = Request(link, callback=self.parse_palmbeach)
            self.requests_list.append(request)
            link = 'http://www.clerk.co.okeechobee.fl.us/Foreclosurecalender.htm'
            request = Request(link, callback=self.parse_okeechobee)
            self.requests_list.append(request)

This code populates my requests_list and then the parse function iterates over it to add requests to the scheduler. This just doesn't seem like the do not repeat yourself programming mantra. There's got to be a better way?

def parse(self, response):
    for r in self.requests_list:
        yield r

def parse_realforeclose(self, response):
         ##i hate realforeclose and javascript + selenium
         global county
##code to set the county based on the URL
         county = str(response.url).replace('https://www.', '').replace('.realforeclose.com/index.cfm', '').title()
         print county         
         logdir=os.getcwd()
##this try catch is not needed if the code at the end of the script doing the same thing worked correctly
         try:
            logfiles = sorted([ f for f in os.listdir(logdir) if f.startswith('QuickSearch')])
            logfiles = str(logfiles).replace("['", "").replace("']", "")
            os.remove('QuickSearch.csv')
         except:
            print ">>>>>>>>>>>>QuickSearch deleted previously"
            pass
         item = PropfinalItem()
##set firefox preferences to auto download csv files
         fp = webdriver.FirefoxProfile()
         fp.set_preference("browser.download.folderList",2)
         fp.set_preference("browser.download.manager.showWhenStarting",False)
         fp.set_preference("browser.download.dir",  os.getcwd())
         fp.set_preference("browser.helperApps.neverAsk.saveToDisk","text/csv")
         self.selenium = webdriver.Firefox(firefox_profile=fp)
         sel = self.selenium
         sel.get(response.url)
         userName = sel.find_element_by_id("LogName")
         userName.send_keys(username)
         passWord = sel.find_element_by_id("LogPass")
         passWord.send_keys(password)
         submit = sel.find_element(By.CSS_SELECTOR, "#LogButton").click()
         time.sleep(3)
##if the county is volusia, babysit the clicking of buttons, weird responses
         if "volusia" in response.url:
            clear = sel.find_element(By.XPATH, '//*[@id="BNOTACC"]').click()
            print ">>>>>>>>>>>>>VOLUSIA"
            time.sleep(3)
            print "sleep"
            clear = sel.find_element(By.XPATH, '//*[@id="BNOTACC"]').click()
            time.sleep(3)
            print "sleep"
            clear
            time.sleep(3)
            print "sleep"
            clear
            time.sleep(3)
            print "sleep"
            clear
         try:
             clear = sel.find_element(By.XPATH, '//*[@id="BNOTACC"]').click()
             clear
             clear
             clear
             clear
         except:
             pass
         w = WebDriverWait(sel, 15, 1)
         w.until(lambda sel: sel.find_element(By.XPATH, "/html/body/table/tbody/tr/td/div[2]/div/div/div[8]/a/span"))
         inputs = sel.find_element(By.XPATH, "/html/body/table/tbody/tr/td/div[2]/div/div/div[8]/a/span").click()
         time.sleep(9)
         i=0
         while i < 10:
            i+=1
            try:
               searchType = sel.find_element(By.XPATH, '/html/body/table/tbody/tr/td/div[2]/div/div[2]/div[2]/div[3]/select/option[2]').click()
               filters = sel.find_element(By.XPATH, '/html/body/table/tbody/tr/td/div[2]/div/div[2]/div[2]/div[7]/button').click()
               removeSold = sel.find_element(By.XPATH, '/html/body/table/tbody/tr/td/div[2]/div/div[2]/div[2]/div[7]/div/ul/li[3]/label').click()
               removeCancel = sel.find_element(By.XPATH, '//*[@id="ui-multiselect-CaseStatus-option-3"]').click()
               submit = sel.find_element(By.XPATH, '//*[@id="testBut"]').click()
               pass
            except ElementNotVisibleException as e:
               print e
               time.sleep(2)
               continue
            break
         time.sleep(2)
##grid showing dataset, go ahead and download
         download = sel.find_element(By.XPATH, '/html/body/table/tbody/tr/td[2]/div[2]/div/div/div[3]/div[2]/a/img').click()
##broward has enormous CSV files
         if 'broward' in response.url: ##cheap fix for fucking broward
            time.sleep(15)
         else:
            time.sleep(10)
         logfiles = sorted([ f for f in os.listdir(logdir) if f.startswith('QuickSearch')])
         logfiles = str(logfiles).replace("['", "").replace("']", "")
         try:
            t0 = time.clock()
            t1 = time.time()
            with open(logfiles, 'rU') as f:
                print ">>>>>>>>>>>>>>CSV PARSING NOW"
                reader = csv.reader(f)
                ##need to skip first line eventually
                j = 0
                for row in reader:
                   j+=1
##scrapy only parses 50 of these before starting the next request
                   item['state'] = 'Florida'
                   item['county'] = county
                   item['saleDate'] = row[0]
                   item['caseNumber'] = row[2]
                   item['finalJudgement'] = row[4]
                   item['aV'] = row[6]
                   item['plaintiff'] = row[7]
                   item['propertyAddress'] = row[10]
                   item['propertyCity'] = row[11]
                   item['propertyZip'] = row[12]
                   item['parcelID'] = row[13]
                   print ">>>>>>>>>>>>>>>>", j
                   yield item
            f.close()
            print time.clock() - t0, "seconds process time"
            print time.time() - t1, "seconds wall time"
         except:
            pass
         print ">>>>>>>>>>>F CLOSED"
         os.remove(logfiles)
         sel.close()

The biggest issue here is the bottom portion of the parse_realforeclose function. You can review the issue with my stackoverflow post here: https://stackoverflow.com/questions/18538644/scrapy-hanging-on-csv-parse

Basically, I've got every available scrapy setting for concurrency set to 1. concurrent_requests, concurrent_requests_per_ip, etc. If the CSV is large, Scrapy will parse 50-100 of the items in the CSV file, and then it pauses and sends out the next requests. I'm not sure why this is happening as I didn't get any feedback on the StackOverflow post. I have a feeling scrapy thinks the request / response is dead since it get's taken by selenium to finish the javascript parsing.

Any help cleaning up this code and or debugging the CSV parsing issue would be phenomenal.

Answer 1

For your requests list, I suggest defining a dictionary like that:

requests_dict = {
    'putnam': 'http://www.putnam-fl.com/clerks_web_apps/foreclosure_trans/get_cases.php',
    'palmbeach': 'https://mypalmbeachclerk.clerkauction.com/',
    ...
}

and then doing something like:

requests_list = [Request(url, callback=getattr(self, 'parse_'+name)) for name, url in requests_dict.items()]

---

EDIT

Other possible solution, attach the URL as an attribute of the callback function:

class PropFinal(CrawlSpider):

    def __init__(self, *args, **kwargs):
        self.requests_list = []
        for name, obj in self.__class__.__dict__.items():
            if name.startswith('parse_'):
                self.requests_list.append(Request(obj.url, callback=obj))
        ...

    def parse_foo(...):
        ...
    parse_foo.url = 'http://foo.example.org/'

    def parse_bar(...):
        ...
    parse_bar.url = 'http://bar.example.net/'

Or, if you prefer to have the URL before the function, you can use a decorator:

def attach(**params):
    def f(g):
        g.__dict__.update(params)
        return g
    return f

class PropFinal(CrawlSpider):

    def __init__(self, *args, **kwargs):
        self.requests_list = []
        for name, obj in self.__class__.__dict__.items():
            if name.startswith('parse_'):
                self.requests_list.append(Request(obj.url, callback=obj))
        ...

    @attach(url='http://foo.example.org/')
    def parse_foo(...):
        ...