Accessing multiple dynamic libraries with the same extern C methods

Question

I have multiple pre-compiled dynamic libraries that use the same extern "C" function names. The functions can behave differently for each dynamic library. Ultimately these methods will be accessed in a SystemVerilog simulation via DPI (Direct Program Interface).

When trying to link all the libraries to my simulator, I noticed that the visibility of the methods is dependent on link order. This blocks me from accessing the desired method of the same name if it is not the first library.

One solution would have been to require each extern "C" function to have a unique name. But this fails in my case for two reasons:

1. I do not own the source code of the dynamic libraries. I can make requests, but there is no guarantee whether or when they will be fulfilled.
2. It would add a lot of verbosity and static code to my SystemVerilog code as each component would need to figure out which DPI method it needs to access. It also doesn't scale well if a library is added or removed.

My solution is to create my own dynamic library that uses dlopen() and dlsym() (from dlfcn.h) to dynamically access methods from the pre-compiled dynamic libraries. Fortunately, all the pre-compiled library flavors use the same root class. They also contain a publicly accessible variable identifying the compiled flavor. I can use this identifier to decide which library to dynamically reference.

I cannot share the real code. Bellow is a runnable proof of concept. I am getting the desired output. It has been a few years since I have done this kind of coding with C++, so I'm hoping I not missing something.

my_model.h

#include <stdio.h>
#include <iostream>
#include <unordered_map>
#define CONCAT_(A, B) A ## B
#define CONCAT(A, B) CONCAT_(A, B)
#define EXPORTED __attribute__((visibility("default")))
#ifndef FLAVOR
#define FLAVOR PASSION
#endif

namespace my_ns{
  enum Flavor { PASSION=0, ORANGE=1, GUAVA=2, POG=-1 };
    class my_model {
        public:
            my_model(int cfg);
            virtual ~my_model() { close(); }
            virtual void close() { um.clear(); }
            virtual int get_info(std::string key, int* value) { return -1; }
            virtual int set_info(std::string key, int  value) { return -1; }
            virtual int del_info(std::string key)             { return -1; }
            const Flavor flavor = FLAVOR;
        protected:
            std::unordered_map<std::string,int> um;
    };
#if FLAVOR>=0
    class CONCAT(my_model_,FLAVOR) : public my_model {
        public:
            CONCAT(my_model_,FLAVOR)(int cfg) : my_model(cfg) {}
            virtual int get_info(std::string key, int* value);
            virtual int set_info(std::string key, int  value);
            virtual int del_info(std::string key);
    };
#endif
    extern "C" {
        int EXPORTED create_model(void** model_h, int flavor, int cfg);
        int EXPORTED delete_model(void** model_h);
        int EXPORTED get_info(const void* model_h, const char* key, int* value);
        int EXPORTED set_info(const void* model_h, const char* key, int  value);
        int EXPORTED del_info(const void* model_h, const char* key);
    }
}

my_model.cpp

#include "my_model.h"
namespace my_ns {

    my_model::my_model(int cfg) {
        um["cfg"] = cfg;
        um["flavor"] = static_cast<int>(FLAVOR);
        printf("Info: cfg:%0d FLAVOR:%0d (%s:%s:%d)\n", um["cfg"],FLAVOR,__func__,__FILE__,__LINE__);
    }

    int CONCAT(my_model_,FLAVOR)::get_info(std::string key, int* value) {
        printf("Info: %s(%s,%%d) : %s:%d\n", __func__,key.c_str(), __FILE__,__LINE__);
        if (um.count(key)==0) return -1;
        *value = um[key] + static_cast<int>(FLAVOR);
        return 0;
    }
    int CONCAT(my_model_,FLAVOR)::set_info(std::string key, int value) {
        printf("Info: %s(%s,%0d) : %s:%d\n", __func__,key.c_str(),value, __FILE__,__LINE__);
        um[key] = value * um["cfg"] * um["cfg"];
        return 0;
    }
    int CONCAT(my_model_,FLAVOR)::del_info(std::string key) {
        printf("Info: %s(%s):%s:%d)\n", __func__,key.c_str(), __FILE__,__LINE__);
        if (um.count(key)==0) return -1;
        um.erase(key);
        return 0;
    }

    int create_model(void** model_h, int flavor, int cfg) {
        printf("Info: cfg:%0d FLAVOR:%d (%s:%s:%d)\n", cfg, FLAVOR, __func__,__FILE__,__LINE__);
        *model_h = (void*) new CONCAT(my_model_,FLAVOR)(cfg);
        return (!*model_h ? -1 : 0);
    }
    int delete_model(void** model_h) {
        CONCAT(my_model_,FLAVOR)* _model = (CONCAT(my_model_,FLAVOR)*) *model_h;
        if (!_model) { fprintf(stderr, "Error: NULL model_h"); return -1; }
        delete _model;
    *model_h = (void*) NULL;
        return 0;
    }
    int get_info(const void* model_h, const char* key, int* value) {
        CONCAT(my_model_,FLAVOR)* _model = (CONCAT(my_model_,FLAVOR)*) model_h;
        if (!_model) { fprintf(stderr, "Error: NULL model_h"); return -1; }
        return  _model->get_info(std::string(key), value);
    }
    int set_info(const void* model_h, const char* key, int value) {
        CONCAT(my_model_,FLAVOR)* _model = (CONCAT(my_model_,FLAVOR)*) model_h;
        if (!_model) { fprintf(stderr, "Error: NULL model_h"); return -1; }
        return  _model->set_info(std::string(key), value);
    }
    int del_info(const void* model_h, const char* key) {
        CONCAT(my_model_,FLAVOR)* _model = (CONCAT(my_model_,FLAVOR)*) model_h;
        if (!_model) { fprintf(stderr, "Error: NULL model_h"); return -1; }
        return  _model->del_info(std::string(key));
    }
}

my_top.cpp

#include "my_model.h"
#include <dlfcn.h>
namespace my_ns {

    my_model::my_model(int cfg) {}

    const char lib[3][16] = {"./libPassion.so","./libOrange.so","./libGuava.so"};
    void *my_so[3];
    int link_so(Flavor flavor) {
        if (my_so[flavor] == NULL) {
            printf("linking %s (%s:%s:%d)\n", lib[flavor], __func__,__FILE__,__LINE__);
            my_so[flavor] = dlopen(lib[flavor], RTLD_NOW);
        }
        if (!my_so[flavor]) {
            /* fail to load the library */
            fprintf(stderr, "Error: %s\n", dlerror());
            return -1;
        }
        return 0;
    }

    int create_model(void** model_h, int flavor, int cfg) {
        link_so(static_cast<Flavor>(flavor));
        int (*dlsym_create_model)(void** model_h, int flavor, int cfg);
        *(void**)(&dlsym_create_model) = dlsym(my_so[flavor], "create_model");
        return dlsym_create_model(model_h, flavor, cfg);
    }
    int delete_model(void** model_h) {
        my_model* _model = (my_model*) model_h;
        link_so(_model->flavor);
        int (*dlsym_delete_model)(void** model_h);
        *(void**)(&dlsym_delete_model) = dlsym(my_so[_model->flavor], "delete_model");
        return dlsym_delete_model(model_h);
    }
    int get_info(const void* model_h, const char* key, int* value) {
        my_model* _model = (my_model*) model_h;
        link_so(_model->flavor);
        int (*dlsym_get_info)(const void* model_h, const char* key, int* value);
        *(void**)(&dlsym_get_info) = dlsym(my_so[_model->flavor], "get_info");
        return dlsym_get_info(model_h, key, value);
    }
    int set_info(const void* model_h, const char* key, int value) {
        my_model* _model = (my_model*) model_h;
        link_so(_model->flavor);
        int (*dlsym_set_info)(const void* model_h, const char* key, int value);
        *(void**)(&dlsym_set_info) = dlsym(my_so[_model->flavor], "set_info");
        return dlsym_set_info(model_h, key, value);
    }
    int del_info(const void* model_h, const char* key) {
        my_model* _model = (my_model*) model_h;
        link_so(_model->flavor);
        int (*dlsym_del_info)(const void* model_h, const char* key);
        *(void**)(&dlsym_del_info) = dlsym(my_so[_model->flavor], "del_info");
        return dlsym_del_info(model_h, key);
    }
}

my_dpi.sv

package my_dpi_pkg;
  import "DPI-C" function int create_model( output chandle handle, input int flavor, cfg );
  import "DPI-C" function int delete_model(  inout chandle handle );
  import "DPI-C" function int get_info(      input chandle handle, input string key, output int value );
  import "DPI-C" function int set_info(      input chandle handle, input string key,  input int value );
  import "DPI-C" function int del_info(      input chandle handle, input string key );
endpackage : my_dpi_pkg
module tb;
  import my_dpi_pkg::*;
  initial begin
    chandle passion_h,orange_h, guava_h;
    string str;
    int flavor,val;
    $display("Create");
    assert(create_model(passion_h, 0, 10)==0);
    assert(create_model( orange_h, 1, 16)==0);
    assert(create_model(  guava_h, 2,  8)==0);
    $display("\nInfo via set_info()");
    assert(set_info(passion_h, "alpha", 'd13)==0);
    assert(set_info( orange_h, "beta",  'h13)==0);
    assert(set_info(  guava_h, "gamma", 'o13)==0);
    $display("\nInfo via get_info()");
    assert(get_info(passion_h,"flavor", flavor)==0);
    assert(get_info(passion_h,"alpha", val)==0);
    $display("passion_h flavor:%0d, alpha:'d%0d",flavor,val);
    assert(get_info(orange_h,"flavor", flavor)==0);
    assert(get_info(orange_h,"beta",  val)==0);
    $display("orange_h flavor:%0d, beta:'h%0h",flavor,val);
    assert(get_info(guava_h,"flavor", flavor)==0);
    assert(get_info(guava_h,"gamma", val)==0);
    $display("guava_h flavor:%0d, gamma:'o%0o",flavor,val);
    assert(get_info(guava_h,"cfg", flavor)==0);
    val = 0;
    assert(get_info(guava_h,"alpha", val)==-1);
    $display("guava_h cfg:%0d, alpha:'d%0d",flavor,val);

    $display("Delete");
    assert(delete_model(passion_h)==0);
    assert(delete_model(orange_h)==0);
    assert(delete_model(guava_h)==0);

    $finish(0);
  end
endmodule

---

Commands to build and run:

g++ -fvisibility=hidden -fvisibility-inlines-hidden -s -shared -fPIC -std=gnu++11 -DFLAVOR=PASSION -o libPassion.so my_model.h my_model.cpp -Wall -g || exit 1
g++ -fvisibility=hidden -fvisibility-inlines-hidden -s -shared -fPIC -std=gnu++11 -DFLAVOR=ORANGE  -o libOrange.so  my_model.h my_model.cpp -Wall -g || exit 1
g++ -fvisibility=hidden -fvisibility-inlines-hidden -s -shared -fPIC -std=gnu++11 -DFLAVOR=GUAVA   -o libGuava.so   my_model.h my_model.cpp -Wall -g || exit 1
g++ -fvisibility=hidden -fvisibility-inlines-hidden -s -shared -fPIC -std=gnu++11 -DFLAVOR=POG     -o libPOG.so     my_model.h my_top.cpp   -Wall -g
<sv_simulator> -<dpi_keyword> libPOG.so my_dpi.sv

Output:

Create
linking ./libPassion.so (link_so:my_top.cpp:11)
Info: cfg:10 FLAVOR:0 (create_model:my_model.cpp:29)
Info: cfg:10 FLAVOR:0 (my_model:my_model.cpp:7)
linking ./libOrange.so (link_so:my_top.cpp:11)
Info: cfg:16 FLAVOR:1 (create_model:my_model.cpp:29)
Info: cfg:16 FLAVOR:1 (my_model:my_model.cpp:7)
linking ./libGuava.so (link_so:my_top.cpp:11)
Info: cfg:8 FLAVOR:2 (create_model:my_model.cpp:29)
Info: cfg:8 FLAVOR:2 (my_model:my_model.cpp:7)

Info via set_info()
Info: set_info(alpha,13) : my_model.cpp:17
Info: set_info(beta,19) : my_model.cpp:17
Info: set_info(gamma,11) : my_model.cpp:17

Info via get_info()
Info: get_info(flavor,%d) : my_model.cpp:11
Info: get_info(alpha,%d) : my_model.cpp:11
passion_h flavor:0, alpha:'d1300
Info: get_info(flavor,%d) : my_model.cpp:11
Info: get_info(beta,%d) : my_model.cpp:11
orange_h flavor:2, beta:'h1301
Info: get_info(flavor,%d) : my_model.cpp:11
Info: get_info(gamma,%d) : my_model.cpp:11
guava_h flavor:4, gamma:'o1302
Info: get_info(cfg,%d) : my_model.cpp:11
Info: get_info(alpha,%d) : my_model.cpp:11
guava_h cfg:10, alpha:'d0
Delete
./my_dpi.sv:42     $finish(0);

Answer 1

The code looks correct. It looks like a weird mix of C and C++ though.

Assumptions

There was a statement that they share the same base class, but I am not sure what it means if they all use C style interface (not only linkage, but dealing in void*). I will assume that the library only has the free standing functions and classes are free for modification.

The ABI looks quite safe with it using only C style built in types, so I will assume that ABI needs to be preserved.

Code Review

Dangers of dynamic loading. Dynamic loading has a lot of weird implicit behavior. If there are common dependency libraries and they have different versions, it is one way ticket to DLL hell. dlclose is also tricky because calling it can actually leave the library linked because it is a dependency to something else. There is also RPATH vs RUNPATH vs environment variables (LD_PRELOAD, LD_LIBRARY_PATH) vs system wide config ... Well, it is a pain to deal with dynamic loading.

Not using RTLD_LOCAL. If there is a common dependency symbol and one has it defined and the other does not, instead of failing it will silently link the wrong one.

Inefficiency. The code always dlsyms on a function call. It would be better to link class member functions once and reuse the retrieved function pointer.

Propagating C style interface. Is the code meant to be used on C++ side? If so, I believe it would be better to use C++ style error handling (there is expected library and std::optional with error code taken as reference, or straight up exceptions if they are supported). The reason I'm saying this is that the classes provide minimal abstraction even if it seems they could do more.

"Sticking out" symbols. I believe it would be better to not declare the functions to be linked from the loaded libraries. The last time I used dynamic loading I just declared a local function pointer variable and casted the result of dlsym. When the functions are declared, they might be accidentally linked and then cause some confusion. It is better to not expose symbols to be dynamically loaded to avoid accidental linkage.

No lifetime management. If libraries are loaded and unloaded, it is important to keep track of instances that use to-be-unloaded library. If the binary for the function to be called is unloaded then the program will terminate, probably with SIGSEGV as the page containing the binary does not belong to the process.

Better abstraction

I cannot believe I'm saying this, but GObject architecture looks good here. The idea is that the class needs to be loaded and linked exactly once, afterwards the function pointers will be reused. I did this outside of GObject where I had a factory function dlsymed and retrieve all of the information from there. After dlopening another one I would just ask for another factory function pointer. It just worked.

The idea is to deal in local pointers that the linker does not see, otherwise it will think we want to link it to something. The function definitions need to be in cpp file, I wrote them inline for brevity (overall it is untested sketch code to illustrate the proposed interface with some implementation guidelines).

class Model {
public:
    virtual int get_info(std::string_view key, int* value) = 0;
    virtual void set_info(std::string_view key, int  value) = 0;
    virtual bool del_info(std::string_view key) = 0;

    virtual ~Model() {}
};

class ModelClass {
    const Flavor flavour;
    void* so_handle;
    int (*create_model)(void**, int, int);
    /* and the others */
public:
    ModelClass(const ModelClass& other) = delete;
    ModelClass& operator=(const ModelClass& other) = delete;
    /* move operations are automatically deleted due to copy being deleted too*/

    std::unique_ptr<Model> create_instance(/*args?*/) {
        /*call the linked function and wrap it in something depending on the flavor,
        cpp file will hide the definition of the class so the class is better defined there*/
    }

    Flavour get_flavour() const noexcept;

    ~ModelClass() {
        /*unload the library, preferably track if no instances of this class are left out*/
    }
private:
    ModelClass(const char* lib_path) {
        so_handle = dlopen(lib_path, RTLD_NOW | RTLD_LOCAL);
        if (!so_handle) {
            throw std::runtime_error(dlerror());
        }

        create_model = reinterpret_cast<int (*)(void**, int, int)>(dlsym(so_handle), "create_model");
        if (!create_model) {
            throw runtime_error(dlerror());
        }

        /*link the rest*/
    }
};

There is also unncessary copying when passing std::string by value where it is clearly meant to be read only. As JDługosz mentioned, it should std::string_view.

After having all of the above, one could create a manager class and befirend them with ModelClass. The manager can be a singleton that will load everything needed at program start providing either static variables of ModelClass or a map if needed. The manager class could store the paths, constants for function names, etc.

---

If Boost.DLL is an option, I would just use that. I do not have experience with it, but I believe it should contain the boilerplate I wrote above.

Answer 2

virtual int get_info(std::string key, int* value) { return -1; }
virtual int set_info(std::string key, int  value) { return -1; }
virtual int del_info(std::string key)             { return -1; }

Why are you passing key by value in these functions? I'm guessing you are not used to C++ with its value semantics, so watch out for this when you code.

Here, use std::string_view. The previous best practice was to use const string&.

---

In the derived class:

virtual int get_info(std::string key, int* value);
virtual int set_info(std::string key, int  value);
virtual int del_info(std::string key);

⧺C.128 Virtual functions should specify exactly one of virtual, override, or final You should be writing: int del_info(std::string key) override; etc.

---