I've written a short web scraper in Lua (run in LuaJIT for speed) to download every post from a specific forum (not general purpose... yet). It takes about 12 hours to run and download 500,000 posts and there are no errors. I'm wondering how I can make it faster. I know different places where the code can be better in readability and structure, but not how it could be better in speed.

local https = require("ssl.https")
local tcount, pcount = 0,0
local threads = {}
local posts = {}
local root = --the forum url

local output = io.open("output.txt", "a")

function scrape(address, p)
    local request_body = [[]]
    local response_body = {}

    local res, code, response_headers = https.request{
        url = address,
        method = "GET",
        source = ltn12.source.string(request_body),
        sink = ltn12.sink.table(response_body),

    if res ~= 1 or code ~= 200 then print(res, code) end

    if type(response_headers) == "table" and p then --if the function is called with a p param, print the response headers
        for k,v in pairs(response_headers) do
            print(k, v)
    if code == 301 then
        print("Location", response_headers["location"])

    if type(response_body) == "table" then
        response_body = table.concat(response_body) --every response in this instance is a table, so I would probably be able to take out this if statement.
        print("Not a table:", type(response_body))
    return response_body

function parse_posts(response_body)
    for match in string.gmatch(response_body, '<div class="post%-body">.-</div>') do
        local m = string.gsub(match, '<span class="quote%-attr">.-</span>', ""):gsub([[</?[^>q]+>]], ""):gsub("\n", "<br>")
        pcount = pcount+1

function get_threads(response_body)
    string.gsub(response_body, [[<a href="(/%d+/[^"]+)" class="title"]], function (m) local d = tonumber(m:match("%d+")) threads[d] = root..m end)

function get_pages(response_body, url)
    local p = 0
    string.gsub(response_body, [[class="btn mod%-page">(%d+)</a>]], function (m) if tonumber(m) > p then p = tonumber(m) end end)
    for i = p, 1, -1 do
        --print("Scraping", url.."/?page="..tostring(i))

    print("Getting threads")
    for i = 1528, 1, -1 do
        local url = root .. '/threads/?page='..tostring(i)

    print("Getting pages on each thread")
    for id,url in pairs(threads) do
        print("Writing for id", id)
        if id ~= 1121 and id ~= 1217 and id ~= 441 then     --exclude three specific large threads, i intend to remove this next iteration in favor of just removing them: threads[1121] = nil, but i do not believe that will improve speed at all.
            get_pages(scrape(url), url)
        tcount = tcount+1

print(string.format("Finished scraping %d posts across %d threads.", pcount, tcount))
  • \$\begingroup\$ Would the user that voted down on this question please indicate in a comment why they voted down. \$\endgroup\$
    – pacmaninbw
    Commented Apr 7, 2020 at 21:11
  • \$\begingroup\$ Out of interest how long does it take you to download an equivalent amount of data on your internet connection? I haven't looked at your code, but wonder if you may be optimising at the wrong side of a bottleneck \$\endgroup\$ Commented May 30, 2020 at 12:56
  • \$\begingroup\$ The entire txt post of everything I downloaded was like 120 MB, my internet DL speed is 73 Mbps, without doing the math, that definitely doesn't add to 12 hr run time. \$\endgroup\$
    – Allister
    Commented May 30, 2020 at 17:46
  • \$\begingroup\$ Have you tried benchmarking the different sections of the program? (downloading vs. parsing, mostly) \$\endgroup\$ Commented Oct 27, 2020 at 7:43
  • \$\begingroup\$ @DarkWiiPlayer I did not. I probably would try it but its been 6 months and I've lost interest / had no need of the project since I originally wrote it. \$\endgroup\$
    – Allister
    Commented Oct 28, 2020 at 5:15


Your Answer

By clicking “Post Your Answer”, you agree to our terms of service and acknowledge you have read our privacy policy.

Browse other questions tagged or ask your own question.