3
\$\begingroup\$

I have a table in Lua, which contains two 1-dimensional arrays in which each array contains approximately 800,000 elements. I want to serialize this Lua table to file efficiently. Hence, I planned to use Lua C bindings.

#include "lua.h"
#include "lauxlib.h"
#include <stdio.h>
#include <assert.h>

static int do_it(lua_State *L) {
  assert(L && lua_type(L, -1) == LUA_TTABLE);

  int len, idx;
  void *ptr;
  FILE *f;
  size_t r;

  lua_pushstring(L, "p");
  lua_gettable(L, -2);
  len = lua_objlen(L, -1);
  // instead of using lua_rawlen, i used lua_objlen. see below
  // len = lua_rawlen(L, -1); // it throws the following error
  // lua: error loading module 'savetable' from file './savetable.so':
  //    ./savetable.so: undefined symbol: lua_rawlen
  int p_values[len];
  for (idx = 1; idx <= len; idx++) {
    lua_rawgeti(L, -1, idx);
    p_values[idx - 1] = (int)lua_tonumber(L, -1);
    lua_pop(L, 1);
  }

  f = fopen("p.bin", "wb");
  assert(f);
  r = fwrite(p_values, sizeof(int), len, f);
  printf("[p] wrote %zu elements out of %d requested\n", r, len);
  fclose(f);
  lua_pop(L, 1);

  lua_pushstring(L, "q");
  lua_gettable(L, -2);
  len = lua_objlen(L, -1);
  double q_values[len];
  for (idx = 1; idx <= len; idx++) {
    lua_rawgeti(L, -1, idx);
    q_values[idx - 1] = (double)lua_tonumber(L, -1);
    lua_pop(L, 1);
  }

  f = fopen("q.bin", "wb");
  assert(f);
  r = fwrite(q_values, sizeof(double), len, f);
  printf("[q] wrote %zu elements out of %d requested\n", r, len);
  fclose(f);
  lua_pop(L, 1);

  return 1;
}

int luaopen_savetable(lua_State *L) {
  static const luaL_reg Map[] = {{"do_it", do_it}, {NULL, NULL}};

  luaL_register(L, "mytask", Map);
  return 1;
}

Please note that for debugging purpose, I have defined two very small 1-dimensional arrays:

my_table = {p = {11, 22, 33, 44}, q = {0.12, 0.23, 0.34, 0.45, 0.56}}

require "savetable"
mytask.do_it(my_table)

I used the following commands to compile and run it:

> gcc -I/usr/include/lua5.1 -o savetable.so -shared savetable.c -fPIC
> lua wrapper.lua

The code works, however, I am looking for suggestions to make table serialization to file much faster than current.

Please note that I am using Lua 5.1 on a 64-bit Ubuntu PC.

\$\endgroup\$
  • \$\begingroup\$ Tip: you may write your array implementation in C and push it into Lua as a light userdata with custom metatable (with insert/remove methods). If you make your array continuous you may use it directly from C w/o Lua stack functions. \$\endgroup\$ – sineemore Jul 3 '18 at 9:54
  • \$\begingroup\$ @sineemore: Thank you very much for the suggestion. Unfortunately, I didn't get it completely. Let me tell you that I have a table of two large arrays inside Lua. In order to serialize it quickly, I am using Lua C bindings. Well, can you please explain your suggestion a bit more? Or may I ask you to provide a reference to what exactly you are suggesting? \$\endgroup\$ – Ravi Joshi Jul 3 '18 at 10:33
1
\$\begingroup\$
// undefined symbol: lua_rawlen

This happens because there is no such function in Lua 5.1 C API. As stated by documentation lua_objlen is preffered way to get table length.

FYI lua_rawlen function first appeared in Lua 5.2.

It is very handy to check Lua Reference manual to see availiable C API for particular Lua version.


Serialization with zero latency

Wow, that title stands out. Let me explain what I mean: serialization with zero latency means there is no serialization at all.

Watch out: it heavely depends on the origin of your table.

The caveats:

  • you must control the table creation
  • only inserts, no removes

If this is the case, follow on:

As you stated your table is actually an array of Lua numbers. In C Lua number is lua_Number which is probably a double value.

We can create a custom table implementation to only store Lua numbers in it. The table will be continuos memory with lua_Number members in it.

typedef struct {
    lua_Number *items;
    size_t len; // number of entries in items
    size_t cap; // maximum `len` value before we will `realloc(items)`
} fasterarray;

To provide fasterarray we'll use lua_newuserdata. It will allocate memory that will be watched by Lua garbage collector. In particular when there is no more references the allocated userdata will be collected.

static int new(lua_State *L) {
    fasterarray *fa = lua_newuserdata(L, sizeof(fasterarray));
    luaL_getmetatable(L, NAME); // <-- setting metatable, see below
    lua_setmetatable(L, -2);    // <--
    fa->len = 0;
    fa->cap = 8;
    fa->items = calloc(fa->cap * sizeof(lua_Number), 1);
    return 1;
}

Insert function will realloc items when more space is needed.

static int insert(lua_State *L) {
    fasterarray *fa = lua_touserdata(L, 1);
    lua_Number num = lua_tonumber(L, 2);
    if (fa->len == fa->cap) {
        fa->cap *= 2;
        fa->items = realloc(fa->items, fa->cap * sizeof(lua_Number));
        assert(fa && fa->items);
    }
    fa->items[fa->len++] = num;
    return 0;
}

Important part of this implementation is garbage collection. userdata will be collected by Lua automagically, but the memory behind items must be freed directly in C. To handle this we will use __gc metatable method.

static int gc(lua_State *L) {
    fasterarray *fa = lua_touserdata(L, 1);
    if (fa && fa->items) free(fa->items);
    printf("gc\n");
    return 0;
}

Full example:

fasterarray.c

#include <lua.h>
#include <lauxlib.h>
#include <stdlib.h>
#include <stdio.h>
#include <assert.h>

#define NAME "fasterarray"

typedef struct {
    lua_Number *items;
    size_t len;
    size_t cap;
} fasterarray;

static int new(lua_State *L) {
    fasterarray *fa = lua_newuserdata(L, sizeof(fasterarray));
    luaL_getmetatable(L, NAME);
    lua_setmetatable(L, -2);
    fa->len = 0;
    fa->cap = 8;
    fa->items = calloc(fa->cap * sizeof(lua_Number), 1);
    return 1;
}

static int gc(lua_State *L) {
    fasterarray *fa = lua_touserdata(L, 1);
    if (fa && fa->items) free(fa->items);
    printf("gc\n");
    return 0;
}

static int insert(lua_State *L) {
    fasterarray *fa = lua_touserdata(L, 1);
    lua_Number num = lua_tonumber(L, 2);
    if (fa->len == fa->cap) {
        fa->cap *= 2;
        fa->items = realloc(fa->items, fa->cap * sizeof(lua_Number));
        assert(fa && fa->items);
    }
    fa->items[fa->len++] = num;
    return 0;
}

int luaopen_fasterarray(lua_State *L) {

    // fasterarray metatable
    luaL_newmetatable(L, NAME);
    luaL_register(L, NULL, (luaL_Reg []) {
        {"__gc", gc},
        {NULL, NULL}
    });

    // exported funcs
    luaL_register(L, NAME, (luaL_Reg []) {
        {"new", new},
        {"insert", insert},
        {NULL, NULL}
    });
    return 1;
}

example.lua

require"fasterarray"

local ar = fasterarray.new()
local i = 700000
while i > 0 do
        fasterarray.insert(ar, i)
        i = i - 1
end
\$\endgroup\$
  • \$\begingroup\$ Thanks for the detailed suggestion. I must tell you that the large Lua arrays are returned from an external API (inside Lua script). Hence, I have no control over the creation of the array. Please also note that I am looking for serialization to file. Therefore I want to pass the file name along with the array while serializing the file. Any suggestions, please? \$\endgroup\$ – Ravi Joshi Jul 4 '18 at 6:12
  • \$\begingroup\$ @RaviJoshi, This might help: you may create this specific array I provided and insert all elements from vendor array into it. Since it resides in continuos memory you may dump it directly to file. \$\endgroup\$ – sineemore Jul 16 '18 at 12:01

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Not the answer you're looking for? Browse other questions tagged or ask your own question.