3
\$\begingroup\$

I have written a web crawler. I hope anyone can help me make this code be more better.

This is the task to implement a web crawler in Python.

# Task:
# Write a crawler code by python
# for example:
# myspider.py  -u www.google.com -d 3 -t 10 -l logfile -v log_level
# -d  ---  the depth of crawl
# -t  --- the thread's number
# -v  --- the detail of the log file, the more higher, the more detail
# -l  --- the log file's name
# convert the web page to utf-8, and store to the sqlite3 database
# use the logger module to record the log information

This is the code I have written:

# -*- coding:utf-8 -*-
# myspider.py
# Tanky Woo

import urllib
import threading
from Queue import Queue
from sgmllib import SGMLParser
from optparse import OptionParser
import sqlite3
import logging
import re
import os
import stat

# Parser for command line options
def cmdParser():
    usage = "usage: %prog [options] arg"
    cmdLine = OptionParser(usage=usage, version="%prog 1.0")
    cmdLine.add_option("-u", "--url", action="store", dest="siteUrl", type="string",
                       help=u"the website's url")
    cmdLine.add_option("-d", "--deep", action="store", dest="depthNum", type="int",
                       default="3", help=u"the depth of crawler,default is 3")
    cmdLine.add_option("-t", "--thread", action="store", dest="threadNum", type="int",
                       default="10", help=u"the number of the thread,default is 10")
    cmdLine.add_option("-v", "--level", action="store", dest="logLevel", type="int",
                       default="0", help=u"the level of the logfile, default is 5")
    cmdLine.add_option("-l", "--logfile", action="store", dest="logFile", type="string",
                       default="spider.log", help=u"the name of the logfile, default is spider.log")

    (options, args) = cmdLine.parse_args()
    return options

# Initialize the log file
def logInit(logfile="spider.log", level=5, quiet=False):
    logger = logging.getLogger()
    # 0~5 level
    # level 0 -- CRITICAL -- 50
    # level 1 -- ERROR -- 40
    # level 2 -- WARNING -- 30
    # level 3 -- INFO -- 20
    # level 4 -- DEBUG -- 10
    # level 5 -- NOTSET -- 0
    logger.setLevel(50-level*10)
    hdlr = logging.FileHandler(logfile)
    formatter = logging.Formatter("%(asctime)s %(levelname)s %(message)s")
    hdlr.setFormatter(formatter)
    logger.addHandler(hdlr)
    # if the quiet equal False, then output the log information in the standard stream also
    if not quiet:
        hdlr = logging.StreamHandler()
        hdlr.setFormatter(formatter)
        logger.addHandler(hdlr)
    return logger


class Node:
    def __init__(self, deep, url):
        self.url = url
        self.deep = deep


# the urlParser class
# parser the link in the web page and store in the url list
class urlParser(SGMLParser):
    def reset(self):
        self.urls = []
        SGMLParser.reset(self)
    # get the link which begin with "href"
    def start_a(self, attrs):
        href = [v for k, v in attrs if k == 'href']
        if href:
            self.urls.extend(href)

# A mult-thread class
# read the url
class readUrlThread(threading.Thread):
    # deep is the depth of the crawler
    def __init__(self, urlque, htmlque, deep):
        threading.Thread.__init__(self)
        self.urlque = urlque
        self.htmlque = htmlque
        self.deep = deep

    def getUrl(self, node):            
        try:
            htmlSource = urllib.urlopen(node.url).read()
        except UnicodeError, e: # if there is a Unicode exception, switch and put back to the queue
            self.urlque.put(Node(node.deep, node.url.encode('raw_unicode_escape')))
            mylog.error("unicode error" + e)
            return None
        except Exception, e:
            mylog.error("crawl error" + str(e), )
            return None

        self.htmlque.put((node.url, htmlSource))

        print "get url: " + node.url
        mylog.info("get url: " + node.url)

        if node.deep <= self.deep:  
            parser = urlParser()
            parser.feed(htmlSource)
            for url in parser.urls:
                myurl = url
                self.urlque.put(Node(node.deep+1, myurl))
        return htmlSource

    def run(self):
        while True:
            node = self.urlque.get()
            htmlSource = self.getUrl(node)
            self.urlque.task_done()

class writeDatabaseThread(threading.Thread):
    def __init__(self, htmlque, sqlfile):
        threading.Thread.__init__(self)
        self.htmlque = htmlque
        self.sqlfile = sqlfile

    def run(self):
        conn = sqlite3.connect(self.sqlfile)
        conn.text_factory = str
        cursor = conn.cursor()

        cursor.execute('''
        create table data (
            id INTEGER PRIMARY KEY AUTOINCREMENT,
            url text,
            html text
        )
        ''')

        conn.commit()

        print 'Begin<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'
        while True:
            url, html = self.htmlque.get()
            self.htmlque.task_done()
            try:
                # insert the data into the database
                cursor.execute('insert into data (url, html) values(?, ?)', (url, html))
                conn.commit()
                mylog.info("Write to: " + url)
                print u'Write to: ', url, self.htmlque.qsize()
            except Exception, e:
                mylog.error("database error: " + e)
                print 'database error: ', e
                self.htmlque.put((url, html))

        #############################################
        # This statement was not be printed, why??? #
        #############################################
        print u'End<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'
        cursor.close()
        conn.close()


def work(url, deep, threads, dbfile):

    urlque = Queue(0)
    htmlque = Queue(0)

    # if the dbfile is exist, remove it
    if os.path.isfile(dbfile):
        os.chmod(dbfile, stat.S_IWRITE)
        os.remove(dbfile)

    # read url from the urlque and parser it
    for i in range(threads):
        r = readUrlThread(urlque, htmlque, deep)
        r.setDaemon(True)
        r.start()

    urlque.put(Node(1, url))

    # store the html source into the database
    w = writeDatabaseThread(htmlque, dbfile)
    w.setDaemon(True)
    w.start()

    urlque.join()
    htmlque.join()
    print u'Done...'

if __name__ == '__main__':
    options = cmdParser()
    mylog = logInit("spider.log", 5)
    work('http://www.google.com', 5, 100, "data.db")

Can everyone give me suggestions on improving this code?

\$\endgroup\$
2
  • 1
    \$\begingroup\$ You need a code review site. Is this homework? \$\endgroup\$ – duffymo Mar 26 '12 at 13:05
  • 1
    \$\begingroup\$ @duffymo: I have implemented all the task. I only hope everyone can give me better suggestion to improve the code. why treat it as homework? and close it? \$\endgroup\$ – Tanky Woo Mar 26 '12 at 13:14
1
\$\begingroup\$

It is easy to write an abusive crawler you need to keep a couple of things in mind.

  • You should always respect the robots.txt file
  • You should respect the meta "ROBOTS" tag.
  • You should not hit a site too quickly.

    • A good rule of thumb is that you should not hit a site more often than every 0.7 seconds
      Big sites (like google.com) can be hit quicker.
      Smaller sites (like my moms web pages) should be hit much slower.
    • Also check "Crawl-Delay" (It is a non standard extension to robots.txt that some sites use)
  • You should always provide a user agent string that identifies your bot.

If you don't follow the above guidelines then organized Web Masters will block your IP quickly or worse send your bot into a bot trap (were you get no useful information).

Worth checking here:
http://www.robotstxt.org/robotstxt.html

\$\endgroup\$

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy