C++ program to format byte sequences into Python like string representations

Question

This is a C++ program I wrote, that includes functions to convert integers to byte sequences in little endian or big endian order, and functions to convert a byte sequence to a string representation.

There are two representation formats I use, one is very simple in which each byte is represented by a two digit hexadecimal numeral with leading 0, and the bytes are delimited by space characters.

The other is more complicated, it converts each byte into corresponding character if the byte is printable, else it converts the byte to a two digit hexadecimal prefixed by "\\x", except for bytes 0x09, 0x0a and 0x0d which are converted to "\\t", "\\n" and "\\r" respectively, following the convention of Python.

The logic itself is very simple, but because I am a beginner in C++ and I don't know the intricacies of C++ many mistakes were made. I wrote many different versions of these functions and I have fixed all bugs, and optimized them to the best of my ability.

I didn't use std::ostringstream because according to my benchmark somehow it is much slower than string concatenation.

Below are the most performant of the functions I wrote:

#include <chrono>
#include <iostream>
#include <string>
#include <vector>

using std::vector;
typedef vector<uint8_t> bytes;
using std::string;
using std::cout;
using namespace std::literals;
using std::chrono::steady_clock;
using std::chrono::duration;

constexpr char DIGITS[] = "0123456789abcdef";
const vector<string> ASCII = [] {
    vector<string> data(256);
    for (uint8_t i = 0; i < 128; i++) {
        if (std::isprint(i)) {
            data[i] = char(i);
        }
        else {
            data[i] = "\\x"s + DIGITS[i >> 4] + DIGITS[i & 15];
        }
    }
    for (int i = 128; i < 256; i++) {
        data[i] = "\\x"s + DIGITS[i >> 4] + DIGITS[i & 15];
    }
    data[9] = "\\t"s;
    data[10] = "\\n"s;
    data[13] = "\\r"s;
    data[92] = "\\\\"s;
    return data;
}();


template<class T>
inline bytes LittleEndian(const T& data) {
    size_t size = sizeof(T);
    bytes _bytes(size);
    uint64_t mask = 255;
    int shift = 0;
    for (size_t i = 0; i < size; i++) {
        _bytes[i] = (data & mask) >> shift;
        mask <<= 8;
        shift += 8;
    }
    return _bytes;
}

template<class T>
inline bytes BigEndian(const T& data) {
    bytes _bytes = LittleEndian<T>(data);
    std::reverse(_bytes.begin(), _bytes.end());
    return _bytes;
}

template<class T>
inline bytes Little_Endian(const T& data) {
    size_t size = sizeof(T);
    bytes _bytes(size);
    std::memcpy(_bytes.data(), &data, size);
    return _bytes;
}

template<class T>
inline bytes Big_Endian(const T& data) {
    bytes _bytes = Little_Endian(data);
    std::reverse(_bytes.begin(), _bytes.end());
    return _bytes;
}


static inline string hexify(const bytes& arr) {
    string repr(arr.size() * 3 - 1, ' ');
    uint64_t i = 0;
    for (auto& chr : arr) {
        repr[i++] = DIGITS[chr >> 4];
        repr[i] = DIGITS[chr & 15];
        i += 2;
    }
    return repr;
}

static inline string bin_ascii(const bytes& arr) {
    string repr = ""s;
    for (auto& chr : arr) {
        if (chr == 9) {
            repr += "\\t"s;
        }
        else if (chr == 10) {
            repr += "\\n"s;
        }
        else if (chr == 13) {
            repr += "\\r"s;
        }
        else if (chr == 92) {
            repr += "\\\\"s;
        }
        else if (std::isprint(chr)) {
            repr += char(chr);
        }
        else {
            repr += "\\x"s + DIGITS[chr >> 4] + DIGITS[chr & 15];
        }
    }
    return repr;
}


static inline string bin_ascii2(const bytes& arr) {
    string repr = ""s;
    for (auto& chr : arr) {
        repr += ASCII[chr];
    }
    return repr;
}


const bytes text = {
    84, 111, 32, 98, 101, 32,
    111, 114, 32, 110, 111, 116,
    32, 116, 111, 32, 98, 101
}; // To be or not to be

constexpr char lorem_ipsum[] = "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.";
const bytes placeholder = [] {
    bytes _bytes(124);
    std::memcpy(_bytes.data(), &lorem_ipsum[0], 124);
    return _bytes;
} ();
constexpr char answer[] = "@\x9b\x90\x00\x00\x00\x00\x00, the great answer to life, the universe and everything, squared";
const bytes _answer = [] {
    bytes _bytes(124, ' ');
    std::memcpy(_bytes.data(), &answer[0], 72);
    return _bytes;
} ();
string temp;
string accrete;
bytes dump;
bytes dumpster;
constexpr uint64_t magic = 0x5fe6eb50c7b537a9;

int main() {
    cout << hexify(text) << "\n";
    cout << bin_ascii(_answer) << "\n";
    cout << bin_ascii2(_answer) << "\n";
    cout << hexify(LittleEndian(magic)) << "\n";
    cout << hexify(BigEndian(magic)) << "\n";
    cout << hexify(Little_Endian(magic)) << "\n";
    cout << hexify(Big_Endian(magic)) << "\n";
    auto start = steady_clock::now();
    for (int i = 0; i < 65536; i++) {
        temp = hexify(placeholder);
    }
    auto end = steady_clock::now();
    duration<double, std::nano> time = end - start;
    accrete += temp;
    cout << "hexify(placeholder): " << time.count() / 65536 << " nanoseconds\n";
    start = steady_clock::now();
    for (int i = 0; i < 65536; i++) {
        temp = bin_ascii(_answer);
    }
    end = steady_clock::now();
    time = end - start;
    accrete += temp;
    cout << "bin_ascii(_answer): " << time.count() / 65536 << " nanoseconds\n";
    start = steady_clock::now();
    for (int i = 0; i < 65536; i++) {
        temp = bin_ascii2(_answer);
    }
    end = steady_clock::now();
    time = end - start;
    accrete += temp;
    cout << "bin_ascii2(_answer): " << time.count() / 65536 << " nanoseconds\n";
    start = steady_clock::now();
    for (int i = 0; i < 65536; i++) {
        temp = bin_ascii(placeholder);
    }
    end = steady_clock::now();
    time = end - start;
    accrete += temp;
    cout << "bin_ascii(placeholder): " << time.count() / 65536 << " nanoseconds\n";
    start = steady_clock::now();
    for (int i = 0; i < 65536; i++) {
        temp = bin_ascii2(placeholder);
    }
    end = steady_clock::now();
    time = end - start;
    accrete += temp;
    cout << "bin_ascii2(placeholder): " << time.count() / 65536 << " nanoseconds\n";
    start = steady_clock::now();
    for (int i = 0; i < 1048576; i++) {
        dump = LittleEndian(magic);
    }
    end = steady_clock::now();
    time = end - start;
    dumpster.insert(dumpster.end(), dump.begin(), dump.end());
    cout << "LittleEndian(magic): " << time.count() / 1048576 << " nanoseconds\n";
    start = steady_clock::now();
    for (int i = 0; i < 1048576; i++) {
        dump = BigEndian(magic);
    }
    end = steady_clock::now();
    time = end - start;
    dumpster.insert(dumpster.end(), dump.begin(), dump.end());
    cout << "BigEndian(magic): " << time.count() / 1048576 << " nanoseconds\n";
    start = steady_clock::now();
    for (int i = 0; i < 1048576; i++) {
        dump = Little_Endian(magic);
    }
    end = steady_clock::now();
    time = end - start;
    dumpster.insert(dumpster.end(), dump.begin(), dump.end());
    cout << "Little_Endian(magic): " << time.count() / 1048576 << " nanoseconds\n";
    start = steady_clock::now();
    for (int i = 0; i < 1048576; i++) {
        dump = Big_Endian(magic);
    }
    end = steady_clock::now();
    time = end - start;
    dumpster.insert(dumpster.end(), dump.begin(), dump.end());
    cout << "Big_Endian(magic): " << time.count() / 1048576 << " nanoseconds\n";
}

54 6f 20 62 65 20 6f 72 20 6e 6f 74 20 74 6f 20 62 65
@\x9b\x90\x00\x00\x00\x00\x00, the great answer to life, the universe and everything, squared
@\x9b\x90\x00\x00\x00\x00\x00, the great answer to life, the universe and everything, squared
a9 37 b5 c7 50 eb e6 5f
5f e6 eb 50 c7 b5 37 a9
a9 37 b5 c7 50 eb e6 5f
5f e6 eb 50 c7 b5 37 a9
hexify(placeholder): 320.486 nanoseconds
bin_ascii(_answer): 1282.06 nanoseconds
bin_ascii2(_answer): 1132.71 nanoseconds
bin_ascii(placeholder): 1193 nanoseconds
bin_ascii2(placeholder): 1191.95 nanoseconds
LittleEndian(magic): 76.7182 nanoseconds
BigEndian(magic): 103.861 nanoseconds
Little_Endian(magic): 85.1413 nanoseconds
Big_Endian(magic): 87.5931 nanoseconds

Compiled with Visual Studio 2022 17.9.7, compiler flags:

/permissive- /ifcOutput "hexlify_test\x64\Release\" /GS /GL /W3 /Gy /Zc:wchar_t /Zi /Gm- /O2 /sdl /Fd"hexlify_test\x64\Release\vc143.pdb" /Zc:inline /fp:precise /D "NDEBUG" /D "_CONSOLE" /D "_UNICODE" /D "UNICODE" /errorReport:prompt /WX- /Zc:forScope /std:c17 /Gd /Oi /MD /std:c++20 /FC /Fa"hexlify_test\x64\Release\" /EHsc /nologo /Fo"hexlify_test\x64\Release\" /Ot /Fp"hexlify_test\x64\Release\hexlify_test.pch" /diagnostics:column 

How do I make the code faster?

Answer 1

We're missing an include of <cstring> (required for std::memcpy).

Most C++ programmers prefer using to typedef.

Many of the usings can be reduced in scope. I think that std::cout and the chrono classes are used only in main().

Please don't use all-caps for C++ names - convention is that we use them for macros, to alert us that they don't obey the rules of scope (and for function-like macros, that they may evaluate arguments more than once).

ASCII is a poor name for strings in the platform's encoding (possibly, but not unequivocally, ASCII-compatible). Why choose std::vector rather than std::array? Why is it globally visible, rather than file scope (static)? Or even better: a function-scope constant, since it's used in only one place.

There seems to be a pervasive assumption that CHAR_BIT is 8. That's quite common, but not guaranteed.

sizeof (T) can be sizeof data to more obviously be the correct value (sizeof a reference yields the size of the corresponding object type).

Consider using Concepts to constrain the templates to suitable T (perhaps std::unsigned for the ones that otherwise have mixed signed/unsigned arithmetic?).

It's probably better to swap operations here:

for (size_t i = 0; i < size; i++) {
    _bytes[i] = (data & mask) >> shift;
    mask <<= 8;
    shift += 8;
}

If we shift first, we don't need to change mask:

for (size_t i = 0; i < size; i++) {
    _bytes[i] = (data >> shift) & mask;
    shift += 8;
}

We have a very misleading name: Little_Endian(), which seems to actually be host-endian. This obviously also causes Big_Endian() to have unexpected behaviour too when built for a big-endian target.

I would probably use a switch instead of if/else chain in bin_ascii().

The repeated loops in main() could be refactored by extracting a function to time a user-supplied std::function.

The timing loops may well get optimised to a single iteration, given that only the last time around executes a non-dead write.

Instead of extracting time.count() and writing "nanoseconds" explicitly, just divide the duration and let it produce the units when streamed. That can be correctly localised if you add a line to set user's locale at that start of main().

The main() function only times the functions, but does not check they produce correct results. I'd prefer a slow but correct function to a fast wrong one any day!