2
\$\begingroup\$

enter image description here

This code is super slow. I'm looking for advice on how to improve its performance.

#!/usr/bin/python
# -*- coding: utf-8 -*-

#
# rss feed indexing bot
# 


import requests
import urlparse
import time
import cookielib
import psutil
import feedparser
import socket
import datetime
import sys
import urllibfthe reason for which  
import re
import nltk
import unicodedata
import xmlrpclib
import thread
from contextlib import closing
from bs4 import BeautifulSoup as bs
from feedfinder import feeds as feedfinder # http://www.aaronsw.com/2002/feedfinder/
from dateutil import parser as dateparser
import threading
from multiprocessing import Process
from nltk.tokenize import RegexpTokenizer
import MySQLdb as mdb
from SimpleXMLRPCServer import SimpleXMLRPCServer
from SimpleXMLRPCServer import SimpleXMLRPCRequestHandler
import pickle


def _pickle(obj):
  return pickle.dumps(obj)

def _unpickle(this):
  return pickle.loads(this)

# Requests handler for bot server 
class RequestHandler(SimpleXMLRPCRequestHandler):
  rpc_paths = ('/RPC2',)

# Bot server functions
class functionFactory:
  def div(self, x, y):
    return x // y

# Restrict to a particular path.
class botServer(threading.Thread,RequestHandler,functionFactory):
  try:
    def run(self):
      try:
        server = SimpleXMLRPCServer( ( "localhost",8000 ), requestHandler=RequestHandler )
        server.register_introspection_functions()
        server.register_instance(functionFactory())
        now = datetime.datetime.now()
        print "Starting server thread %s" % (str(now))
        server.serve_forever()
      except Exception as error:
        if "Address" in str(error):
          print "Bot server is up"
  except Exception as error:
    print str(error)

serverThread = botServer()
serverThread.start()

client = xmlrpclib.ServerProxy('http://localhost:8000')
Q = client.div(100,2)
print Q
#   DATABASE VARS
"""***************"""
HOST     = 'localhost'
USER     = 'root'
PASSWORD = 'foobar'
DATABASE = 'rss-bot'
"""****************"""

def _install():
  con = mdb.connect(HOST, USER, PASSWORD, DATABASE);
  with con:    
      cur = con.cursor()
      #cur.execute("DROP TABLE IF EXISTS feeds")
      cur.execute("DROP TABLE IF EXISTS articles")
      cur.execute("CREATE TABLE articles (Id INT PRIMARY KEY AUTO_INCREMENT, title varchar(255), summary longtext, link varchar(255))")
      #cur.execute("CREATE TABLE feeds (Id INT PRIMARY KEY AUTO_INCREMENT, URL VARCHAR(255))")
      #cur.execute("ALTER TABLE `feeds` ADD UNIQUE( `URL`);")
      cur.execute("ALTER TABLE `articles` ADD UNIQUE( `title`);")
      cur.execute("ALTER TABLE `articles` ADD UNIQUE( `link`);")



con = mdb.connect(HOST, USER, PASSWORD, DATABASE,charset='utf8',use_unicode=True)

def insertFeed(url):
  url = urllib.unquote(url)
  if len(url) <= 255:
    url = con.escape_string(url)
    try:
      with con:
        cur = con.cursor()
        cur.execute("INSERT INTO feeds(URL) VALUES('%s')" % (url))
    except:pass

# strip of crap characters (based on the Unicode database
# categorization:
# http://www.sql-und-xml.de/unicode-database/#kategorien

PRINTABLE = set(('Lu', 'Ll', 'Nd', 'Zs'))

def filter_non_printable(s):
    result = []
    ws_last = False
    for c in s:
        c = unicodedata.category(c) in PRINTABLE and c or u'#'
        result.append(c)
    return u''.join(result).replace(u'#', u' ')

def insertArticle(title,link,summary):
  try:
    with con:
      cur = con.cursor()
      if title and link and summary:
        summary = con.escape_string(summary)
        title = con.escape_string(title)
        link = con.escape_string(link)
        cur.execute('INSERT INTO articles(title,link,summary) VALUES("%s","%s","%s")' % (title,link,summary))
  except Exception as error:
    print "Error in insertArticle"
    print error

def pullFeeds():  
  with con:
      feeds = []
      cur = con.cursor()
      cur.execute("SELECT * FROM feeds")
      rows = cur.fetchall()
      for row in rows:
          feeds.append(row[1])
  return feeds

newswords = nltk.corpus.brown.words(categories='news')
tokenizer = RegexpTokenizer(r'([A-z])+')

def is_news(text):
  if text:
    text = nltk.clean_html(text)
    words = sorted(set(tokenizer.tokenize(text)))
    try:
      worthParsing = len(words) >=1 
      if worthParsing == True:
        content = set([w for w in words if w.lower() in newswords])
      if content and len(content) >=1:
        return True
      else:
        return False
    except Exception as error:
      print "Exception %s\nin function is_news" % (str(error))

def parseQuery(term):
  if term.decode('utf-8', 'ignore'):
    try:
      # search?q=foo&num=100&safe=off&client=ubuntu&channel=fs&ei=O2A_U8niLra3sATtnoH4DA&start=100&sa=N&biw=1065&bih=569
      perams = urllib.urlencode({'client':'ubuntu',
                                 'q':term,
                                 'num':100,
                                 'safe':'off',
                                 'start':100,
                                 'bih':569,
                                 'ei':'O2A_U8niLra3sATtnoH4DA'
                                 })
      query = "https://www.google.com/search?%s" % (perams)
      return query
    except Exception as error:
      print "Exception %s\nIn function parseQuery() " % (str(error))
      return False
  else: return False

def submitQuery(query):
  if query:
    try:
      markup = requests.get(query).content
      if markup:
        return markup.decode('utf-8', 'ignore')
      else:
        return False
    except Exception as error:
      print "Exception %s\n In submitQuery" % (str(error))
  else:
    return False

def cleanLink(link):
  return re.sub(' ','',link)

def GoogleLinkResults(raw):
  if raw:
    raw = raw.decode('utf-8', 'ignore')
    try:
      sites = bs(raw).select('cite')
      urls = []
      for site in sites:
        link = cleanLink(site.text)
        if link.startswith("http") and "." in link:
          urls.append(link)
        elif link.startswith("http") == False and "." in link:
          urls.append("http://%s" % (cleanLink(link)))
      return urls
    except Exception as error:
      print "Exception %s\nin finction GoogleLinkResults"% str(error)
  else:
    return False

def Google(terms):
  try:
    print "Google Link Results"
    if terms:
      terms = " ".join(re.findall('([A-z ]+)',terms))
      print terms
      links = GoogleLinkResults(submitQuery(parseQuery(terms)))
    if links:
      return links
  except Exception as error:
    print "Exception %s\n in function Google" % (str(error))
    return False


delay = 0

def currentDelay(delay): return delay

def lastModified(reqHeader):
  if 'last-modified' in reqHeader:
    last_modified = reqHeader['last-modified']
    return dateparser.parse(last_modified)

def extractLinks(markup,TYPE='all'):
  try:
    soup = bs(markup.decode('utf-8', 'ignore'))
    _rel_urls = [] # Holds Relative url objects
    _abs_urls = [] # Holds Absolute url objects
    hrefs = sorted(set([ l for l in soup.find_all('a') if l and l.get('href') ]))
    if hrefs:
      for href in hrefs:
        url = urlparse.urlparse(href.get('href'))
        if url.netloc: # If url is absolute
          _abs_urls.append(url)
        else: # If the url is relative
          _rel_urls.append(url)
      if TYPE == "abs":
        return _abs_urls
      elif TYPE == 'rel':
        return _rel_urls
      else: # If no TYPE argument was passed return both absolute & relative urls
        return _rel_urls + _abs_urls
    else:
      return False
  except Exception as error:
    print "Exception %s in function extractLinks" % (str(error))
    return False

def isHTML(reqHeader):
  return 'content-type' in reqHeader and reqHeader['content-type'].startswith('text/html') 

def agent(url,delay=0):
  cookiejar = cookielib.CookieJar()
  UserAgent = """
  Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:26.0)Gecko/20100101 Firefox/26.0
  """
  headers = {
    'user-agent':UserAgent,
    'Accept': 'text/html',
    'Accept-Language': 'en-US',
    'Accept-encoding': 'gzip,deflate'
  }
  try:
    time.sleep(delay)
    with closing(requests.get(url, stream=True, headers=headers, cookies=cookiejar, timeout=1.50, allow_redirects=True)) \
    as req:
      if isHTML(req.headers)  : return req.content
      else: return None
  except Exception as error:
    print "Exception %s\nin function agent" % (str(error))
    return None   

def stack(seeds):
  try:
    stack = {}
    stack['tocrawl'] = set([])
    stack['crawled'] = set([])
    stack['domains'] = set([])
    for i in seeds:  stack['tocrawl'].add(i)
    return stack
  except Exception as error:
    print "Exception %s\nin function stack" % (str(error))

def getfeeds(url,ignore):
  try:
    temp = feedfinder(url)
    feeds = []
    if temp and len(temp) > 0:
      for feed in temp:
        if feed not in ignore:
          feeds.append(feed)
          print "PUSHING %s TO DATABASE" % (feed)
          insertFeed(feed)

    if len(feeds) > 0:
      feed = feedparser.parse(feeds[0])['entries'][0]['title']
      temp = Google("rss feed  %s" % (feed))
      if is_news(feed):
        if temp:
          for this in temp:
            if this.startswith("http"):
              this = "http://%s" % (urlparse.urlparse(this).netloc)
              newfeeds = feedfinder(this)
              if newfeeds:
                for feed in newfeeds:
                  feeds.append(feed)
                  print "PUSHING %s TO DATABASE" % (feed)
                  insertFeed(feed)

    return list(set(feeds))
  except Exception as error:
    print str(error)
    return False    

def acceptableTLD(link):
  acceptable = ['.com','.net','.edu','.gov','.org']
  link = link.lower()
  return len([d for d in acceptable if link.endswith(d)]) >=1

def updateStack(urls,tocrawl,crawled,feeds):
  total_feeds  = []
  for url in urls:
    time.sleep(0)
    #ip = socket.gethostbyname(url.netloc)
    link = "http://%s" % (url.netloc)
    foundfeeds = getfeeds(link,feeds)
    if link not in crawled and link not in tocrawl and link:
        tocrawl.add(link)
    if foundfeeds:
      for feed in foundfeeds:
        if feed not in feeds:
          print "Found Feed > %s " % (feed)
          crawled.add(feed)
          feeds.add(feed)

def crawler(seeds,delay):
  try:
    _stack = stack(seeds)
    _tocrawl = _stack['tocrawl']
    _crawled = _stack['crawled']
    _domains = _stack['domains']
    _feeds   = set([])

    while len(_tocrawl) > 0 :
      link = _tocrawl.pop()    
      _crawled.add(link)
      time.sleep(delay)
      newLinks = extractLinks(agent(link,delay),'abs')

      print "Crawling      : %s" % (link)
      print "Crawled       : %d" % (len(_crawled))
      print "Left to crawl : %d" % (len(_tocrawl))
      print "Feeds found   : %d" % (len(_feeds))
      print "Delay         : %d" % (delay) 

      if newLinks:
        updateStack(newLinks,_tocrawl,_crawled,_feeds)
  except KeyboardInterrupt:
    print "Quiting..."
    serverThread._Thread__stop()
    time.sleep(5)
    quit()
  except Exception as error:
    print "Exception %s\nin function crawler"

crawler(['http://msnbc.com'],delay)
\$\endgroup\$
  • 1
    \$\begingroup\$ As the diagram shows, your system consists of several components, any of which may be slow. You should carry out some experiments to find the bottleneck. \$\endgroup\$ – Janne Karila Apr 8 '14 at 13:14

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Browse other questions tagged or ask your own question.