This program will scrape Wikipedia to create a list of all English Wikipedia articles.

How can I improve this program as it currently performs very badly performance-wise? On my Internet connection it takes about 1-2 secs per processed link.

import wget             # I use wget for downloading the files
import os               # for deleting the cached html file + clearing the screen
import re               # for extracting the urls out of the html file
import timeit           # for timing the runtime of every processed page
import sys

base_url = 'http://en.wikipedia.org/wiki/'  # standard url if the queue is empty.
session_urls = 0
runtimes = []
average_runtime = 0

if os.path.isfile('cache/page'):

def serialize_urls(urls):
    convert a list like ['article1','article2','article3']
    into a string like 'article1|article2|article3' for saving it to file
    buffer = ''
    for url in urls:
        buffer += url + '|'
    return buffer

def get_urls(path):
    reads string (format: 'article1|article2|article3') from either 'queue' or
    'database/page_index' and converts it to a list (format: ['article1','article2','article3'])
    index_file = open(path, 'r', encoding='utf8')
    raw_data = index_file.read()
    urls = raw_data.split('|')
    if path == 'queue':
        urls = [x for x in urls if x != '']
    return urls

def create_url():
    concatenates the base_url (= 'http://en.wikipedia.org/wiki/') and
    the url (wikipedia article name) to get the full http url
    global queue
    if queue == [''] or not queue:  # fix this, refactor
        queue = ['Main_Page']

    url = base_url + queue[0]
    del queue[0]
    return url

def fetch_page(url):
    downloads the html file at 'url', saves it's contents to the var 'page'
    and deletes the temp html file from 'cache/page'
    wget.download(url, out='cache/page', bar=None)
    page_file = open('cache/page', 'r', encoding='utf8')
    page = page_file.read()
    return page

def parse_urls(page):
    searches for all internal wikipedia urls in the html file and returns them
    raw_urls = re.findall(r'href=[\'"]?/wiki/([^\'" >]+)', page)
    return raw_urls

def filter_urls(raw_urls):
    removes all internal wikipedia help urls from the url list


    all of these contain a ':' so I can just remove urls containing a ':'
    cnt = 0
    while cnt < len(raw_urls):
        if ':' in raw_urls[cnt]:
            del raw_urls[cnt]
            cnt += 1
    return raw_urls

def remove_existing_urls(url_base, urls):
    removes all urls from the new urls that were extracted from the html if they:
    1. are already in the queue
    2. have already been processed (and therefore are in 'database/page_index')
    cnt = 0
    while cnt < len(urls):
        if urls[cnt] in url_base:
            del urls[cnt]
            cnt += 1
    return list(set(urls))

def write_to_disk(url, new_urls):
    queue_obj = open('queue', 'a', encoding='utf8')
    page_index_obj = open('database/page_index', 'a', encoding='utf8')
    page_index_obj.write(url[29:] + '|')

def process_next_url():
    global average_runtime, session_urls, queue, page_index

    start = timeit.default_timer()

    url = create_url()
    page = fetch_page(url)
    raw_urls = parse_urls(page)
    urls = filter_urls(raw_urls)


    new_urls = remove_existing_urls(queue + page_index, urls)

    queue = queue + new_urls  # refactor queue += new_urls(?)

    stop = timeit.default_timer()
    time = stop - start
    if queue == []: #refactor? # own function for output
        print('Article: Main_Page')
        print('Article: ' + queue[0])
    print('Page Index: ' + str(len(page_index)))
    print('Queue: ' + str(len(queue)))
    print('Average Runtime: ' + str(average_runtime))
    print('Runtime: ' + str(time))
    print('URLs Processed: ' + str(session_urls))
    print('Full URL: ' + url)
    return time, url, new_urls

def check_cmd():
    file = open('cmd', 'r', encoding='utf8')
    cmd = file.read()
    if cmd == 'stop\n':
        file = open('cmd', 'w', encoding='utf8')

queue = get_urls('queue')
page_index = get_urls('database/page_index')

while True:
    for cnt in range(100):
        session_urls += 1
        time, url, new_urls = process_next_url()
    write_to_disk(url, new_urls)
    average_runtime = str(sum(runtimes) / len(runtimes))
    runtimes = []

1 Answer 1


I'll just focus on fetch_page(). Using wget to download a Wikipedia article to a file, just to delete it immediately afterwards, is complicated and wasteful. You could just do it all in memory, using the built-in urllib library.

def fetch_page(url):
    returns the contents of the html file at 'url'
    with urllib.request.urlopen(url) as page:
        return page.read().decode('UTF-8')

However, if your goal is just to list all English Wikipedia articles, web-scraping is an anti-social way to do it. Wikimedia content is all Creative Commons licensed, and they make data dumps available. Analyze that instead, and save bandwidth and time.

  • \$\begingroup\$ I am downloading and deleting the html files because I wanted to change it later so I can keep them as an offline wikipedia archive. Also I know about the Wikimedia data dumps but I only created this project to learn the basics of web-scraping in a simple environment like wikipedia with just basic html and css files that all follow along the same pattern, but I won't actually be using it. Thanks for the help anyway! \$\endgroup\$ Commented Nov 27, 2014 at 17:18

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service and acknowledge you have read our privacy policy.

Not the answer you're looking for? Browse other questions tagged or ask your own question.