Efficient C Tokenizer/Lexer in C++

Question

I wanted to create my own compiler for C and it's the first prototype of the Lexer. Any thoughts on improvements?

Note: I forgot to add comment lines in header files while I was coding, but I assure you code in header files is clear and easy to understand.

Main.cpp

Main code which has Lexer inside

#include "Lists.h"
#include "Tokens.h"
#include <stream>

// checks for is it real end of string
// ex:
// "\" string is still open " string is now closed
// " \\" string is closed
inline static bool is_escape_char(char* c)
{
    int i = -1;
    while (c[i] == '\\') i--;
    return i & 1;
}

int main()
{
    char *buffer;

    // Read file
    {
        std::ifstream file("Main.cpp");
        assert(file.is_open());

        file.seekg(0, std::ios::end);
        size_t file_size = file.tellg();
        file.seekg(0, std::ios::beg);

        std::cout << "File size: " << file_size << '\n';

        buffer = (char*)malloc(file_size + 1);
        assert(buffer != nullptr);

        file.read(buffer, file_size);
        file.close();

        buffer[file_size] = '\0';
    }

    List<Token> tokens;

    // for each char in file
    for (char *c = buffer; *c != '\0';)
    {
        char C = c[0], *end;
        TokenKind kind;
        bool sign = (c[0] == '-') || (c[0] == '+');

        // skip if space (' ', '\n', '\t', '\r' ...)
        if (isspace(C))
        {
            c++;
            continue;
        }
        else if (isdigit(c[sign]))
        {
            // lambda for looping to find the first char which is not a digit or underscore
            static auto cond = [](int x) -> int {
                return static_cast<int>((isdigit(x) != 0) || (x == '_'));
            };

            // a function for finding base of number (4324 (decimal), 0x3 (hex)...)
            // Tokens.h
            uint8_t base = tokenKinds::secondary::number::get_base(c + sign);
            kind = TokenKind(tokenKinds::number, base);

            // First the first not digit or underscore char
            end = find_first_not(c + sign + 1 + tokenKinds::secondary::number::start_indices[base], cond);
        
            // if char is '.' it is a float or double
            if (*end == '.')
            {
                // if number already has a base abort app (ex: 0x32.3213)
                if (base != 0)
                    mexit("Float number with non-decimal system.");
                kind.set_sub_kind(tokenKinds::secondary::number::floatK);
                end = find_first_not(end + 1, cond);
            }
        }
        else
        {
            char *underscore = nullptr;

            // check for alpha values (int, a, return) starts with underscore (__int64)
            if (C == '_')
            {
                for (underscore = c + 1; isalpha(underscore[0]) == 0; underscore++)
                {
                    if (underscore[0] != '_') // if its finds a char not underscore and not alpha it is not a alpha value instead it is a underscore symbol
                    {
                        underscore = nullptr;
                        break;
                    }
                }
            }

            if (underscore != nullptr || isalpha(C))
            {
                static auto cond = [](int x) -> int {
                    return static_cast<int>((isalnum(x) != 0) || (x == '_'));
                };

                // find_first_not (Tokens.h) finds first char it is not according to function
                end = find_first_not((underscore ? underscore : c) + 1, cond);
                kind = TokenKind(tokenKinds::alpha, tokenKinds::secondary::alpha::get_sub_kind(c, end));
            }
            else
            {
                // check if comment
                if (C == '/')
                {
                    if (c[1] == '/')
                    {
                        c = find_first(c + 2, '\n') + 1;
                        continue;
                    } // check if multi-line comment
                    else if (c[1] == '*')
                    {
                        c += 2;
                        while (*c != '/')
                            c = find_first(c, '*') + 1;
                        c++;
                        continue;
                    }
                }

                // check doublable symbols (==, *=, !=, <<, >>, ||)
                static const char* doublable_and_equal_chars = "<>|&";
                static const char* doublable_or_equal_chars = "-+=";
                static const char* restof_equal_chars = "*/%!^";

                uint8_t length;
                if (find_first(doublable_and_equal_chars, C) != doublable_and_equal_chars + 4)
                {
                    length = (C == c[1]) + 1;
                    length = length + (c[length] == '=');
                }
                else if (find_first(doublable_or_equal_chars, C) != doublable_or_equal_chars + 3)
                    length = 1 + (c[1] == C || c[1] == '=');
                else if (find_first(restof_equal_chars, C) != restof_equal_chars + 5)
                    length = 1 + (c[1] == '=');
                else
                    length = 1;

                String str(c, length);
                kind = TokenKind(tokenKinds::symbol, tokenKinds::secondary::symbol::get_sub_kind(str));
                tokens.push_back(Token(str, kind));

                // check for strings & chars
                bool is_char;
                if ((is_char = C == '\'') || C == '"')
                {
                    end = c++;

                    do
                        end = find_first(end + 1, C);
                    while (is_escape_char(end) == false);

                    tokens.push_back(Token(String(c, end - c), TokenKind(tokenKinds::other, tokenKinds::secondary::other::string + is_char)));
                    tokens.push_back(Token(String(c = end, 1), kind));
                }

                c += length;
                continue;
            }
        }

        tokens.push_back(Token(String(c, end - c), kind));
        c = end;
    }

    std::cout << "Token amount: " << tokens.size << '\n';

    // print tokens
    for (ListNode<Token>* t = tokens.start; t != nullptr; t = t->next)
    {
        std::cout << t->value << '\n';
    }

    // free the memory
    tokens.clear();

    return 0;
}

Tokens.h

Header file for Token Struct and some Datas (Keywords, Token kinds, handing ostream (<<) ...)

#pragma once

#include "String.h"

struct TokenKind
{
    uint16_t kind;

    inline constexpr TokenKind() : kind(0) { }

    inline constexpr TokenKind(uint16_t kind) : kind(kind) { }

    inline constexpr TokenKind(uint8_t basic, uint16_t sub) : kind(basic | (sub << 2)) { }

    constexpr operator uint16_t() const
    {
        return kind;
    }

    constexpr inline uint8_t get_basic_kind() const
    {
        return kind & 0b11;
    }

    constexpr inline uint16_t get_sub_kind() const
    {
        return kind >> 2;
    }

    inline void set_basic_kind(uint8_t value)
    {
        kind = (kind & (~0b11)) | value;
    }

    inline void set_sub_kind(uint16_t value)
    {
        kind = (kind & 0b11) | (value << 2);
    }
};

struct Token
{
    String str;
    TokenKind kind;

    Token() : str(), kind() { }

    Token(String str, TokenKind kind) : str(str), kind(kind) { }
};

namespace tokenKinds
{
    constexpr uint8_t number = 0, alpha = 1, symbol = 2, other = 3;

    const char *names[] = {
        "number", "alpha", "symbol", "other",
        "decimal", "hex", "octal", "binary", "float",
        names[3], "keyword",
        "unknown", "string", "char"
    };

    namespace secondary
    {
        namespace number
        {
            const char **names = tokenKinds::names + 4;

            constexpr uint8_t
                decimal = 0,
                hex = 1,
                octal = 2,
                binary = 3,
                floatK = 4;

            uint8_t start_indices[] = {
                0,
                2,
                1,
                2
            };

            inline constexpr uint8_t get_base(char *str)
            {
                if (str[0] == '0')
                {
                    if (str[1] == 'x')
                        return hex;
                    if (str[1] == 'b')
                        return binary;
                    if (isdigit(str[1]) || str[1] == '_')
                        return octal;
                }
                return decimal;
            }
        }

        namespace alpha
        {
            const char **names = tokenKinds::names + 9;

            const char *keywords[] = {
                "void",
                "int",
                "unsigned",
                "float",
                "double",
                "bool",
                "char",
                "short",
                "long",
                "inline",
                "static",
                "const",
                "return",
                "namespace",
                "struct",
                "operator",
                "case",
                "if",
                "else",
                "switch",
                "default",
                "break",
                "continue",
                "include",
                "define",
                "undef"
            };

            constexpr uint8_t other = 0;

            inline uint8_t get_sub_kind(String str)
            {
                str.set_cstring();
                uint8_t v = find_in(str.start, keywords, 26);
                str.undo_cstring();
                if (v == 26)
                    return other;
                return v + 1;
            }

            inline uint8_t get_sub_kind(char* start, char* end)
            {
                char ei = *end;
                *end = '\0';
                uint8_t v = find_in(start, keywords, 26);
                *end = ei;
                if (v == 26)
                    return other;
                return v + 1;
            }
        }

        namespace symbol
        {
            inline uint16_t get_sub_kind(String str)
            {
                char c = str.start[0];
                uint16_t sk = c - '!';
                if (c >= 'A')
                {
                    sk -= 'Z' - 'A';
                    if (c >= 'a')
                        sk -= 'z' - 'a';
                }

                if (str.length == 1)
                    return sk;

                c = str.start[1];
                sk += (c - '!') << 6;
                if (c >= 'A')
                {
                    sk -= ('Z' - 'A') << 6;
                    if (c >= 'a')
                        sk -= ('z' - 'a') << 6;
                }

                return sk;
            }
        }

        namespace other
        {
            const char **names = tokenKinds::names + 11;

            constexpr uint8_t
                unknown = 0,
                string = 1,
                charK = 2;
        }
    }

} // namespace tokenKinds

inline std::ostream &operator<<(std::ostream &os, TokenKind kind)
{
    return os << +kind.get_basic_kind() << ", " << kind.get_sub_kind();
}

std::ostream &operator<<(std::ostream &os, Token token)
{
    uint8_t basic_kind = token.kind.get_basic_kind();
    uint16_t sub_kind = token.kind.get_sub_kind();

    os << "{'type': \033[32m" << tokenKinds::names[basic_kind];

    switch (basic_kind)
    {
    case tokenKinds::number:
        os << '-' << tokenKinds::secondary::number::names[sub_kind];
        break;
    case tokenKinds::alpha:
        os << '-' << tokenKinds::secondary::alpha::names[sub_kind != tokenKinds::secondary::alpha::other];
        break;
    case tokenKinds::other:
        os << '-' << tokenKinds::secondary::other::names[sub_kind];
        break;
    default:
        break;
    }

    return os
        << "\033[0m (\033[90m"
        << token.kind
        << "\033[0m), 'value': '\033[36m"
        << token.str
        << "\033[0m'}";
}

Lists.h

Custom Linked list for storing Tokens

#pragma once

#include <cstdlib>

#include "Basics.h"

template <typename T>
struct ListNode
{
    ListNode<T> *next;
    T value;
};

#define newv(T, name) T *name = reinterpret_cast<T *>(malloc(sizeof(T))); assert(name != nullptr)

template <typename T>
struct List
{
    ListNode<T> *start, *end;
    uint16_t size;

    List()
    {
        start = end = nullptr;
        size = 0;
    }

    void clear()
    {
        ListNode<T> *node = start, *next;
        while (true)
        {
            next = node->next;
            free(node);
            if (next == nullptr)
                break;
            node = next;
        }

        start = end = nullptr;
        size = 0;
    }

    void push_back(T value)
    {
        newv(ListNode<T>, nnode);

        nnode->value = value;
        nnode->next = nullptr;

        if (size)
            end->next = nnode;
        else
            start = nnode;

        end = nnode;
        size++;
    }
};

template <typename T>
std::ostream &operator<<(std::ostream &os, List<T> &list)
{
    os << '{';
    ListNode<T> *node = list.start;
    for (; node->next != nullptr; node = node->next)
        os << node->value << ", ";
    return os << node->value << '}';
}

template <typename T>
struct ListIterator
{
    ListNode<T> *current, *past;

    ListIterator(List<T>& list)
    {
        if (list.start == nullptr)
            mexit("List is empty can't created an iterator!");
        current = list.start;
        past = nullptr;
    }
};

template <typename T>
inline void operator++(ListIterator<T>& it)
{
    it.past = it.current;
    it.current = it.current->next;
}

Strings.h

#pragma once

#include "Basics.h"
#include <cstdint>

inline char *find_end(char *str)
{
    while (*str != '\0')
        str++;
    return str;
}

inline const char *find_end(const char *str)
{
    while (*str != '\0')
        str++;
    return str;
}

inline char *find_first(char *str, char c)
{
    while (*str != '\0' && *str != c)
        str++;
    return str;
}

inline const char *find_first(const char *str, char c)
{
    while (*str != '\0' && *str != c)
        str++;
    return str;
}

inline char *find_first(char *str, const char *chars)
{
    while (*str != '\0')
    {
        for (const char *c = chars; *c != '\0'; c++)
            if (*str == *c)
                return str;
        str++;
    }
    return str;
}

inline char *find_first(char *str, int cond(int))
{
    while (*str != '\0' && cond(*str) == 0)
        str++;
    return str;
}

inline char *find_first_not(char *str, int cond(int))
{
    while (*str != '\0' && cond(*str))
        str++;
    return str;
}

inline uint8_t find_in(char *str, const char **list, uint8_t amount)
{
    for (uint8_t i = 0; i < amount; i++)
        if (strcmp(str, list[i]) == 0)
            return i;
    return amount;
}

char _string_struct_ei_temp = '\0';

struct String
{
    char *start;
    uint8_t length;

    constexpr String() : start(nullptr), length(0) { }

    constexpr String(char *start, uint8_t length) : start(start), length(length) { }

    inline void set_cstring() const
    {
        if (_string_struct_ei_temp != '\0')
            mexit("Can't setup multiple cstring at a time!");

        _string_struct_ei_temp = start[length];
        start[length] = '\0';
    }

    inline void undo_cstring() const
    {
        if (_string_struct_ei_temp == '\0')
        {
            puts("The String is not set as cstring.");
            return;
        }

        start[length] = _string_struct_ei_temp;
        _string_struct_ei_temp = '\0';
    }

    long long to_int() const
    {
        uint8_t base, start_index;
        bool sign = start[0] == '-';

        if (start[sign] == '0' && length != sign + 1)
        {
            switch (start[sign + 1])
            {
            case 'b':
                base = 0b10;
                start_index = 2;
                break;
            case 'x':
                base = 0x10;
                start_index = 2;
                break;
            default:
                base = 010;
                start_index = 1;
                break;
            }
        }
        else
        {
            base = 10;
            start_index = 0;
        }

        long long value = 0;
        for (uint8_t i = start_index + sign; i < length; i++)
        {
            char c = start[i];
            if (c != '_')
                value = value * base + c - (isdigit(c) ? '0' : (islower(c) ? 'a' : 'A'));
        }

        return sign ? -value : value;
    }

    inline double to_float() const
    {
        set_cstring();
        double value = atof(start);
        undo_cstring();
        return value;
    }
};

inline std::ostream &operator<<(std::ostream &os, String &str)
{
    for (uint8_t i = 0; i < str.length; i++)
        os << str.start[i];
    return os;
}

Basics.h

#pragma once
#pragma pack(2)

#undef NDEBUG
#include <cassert>
#include <iostream>

inline void todo(const char *str)
{
    std::cout << "TODO: " << str << '\n';
    exit(0);
}

inline void mexit(const char *str)
{
    puts(str);
    exit(0);
}

Answer 1

You specified "Efficient". My thoughts on this are:

1. Ignore all the answers trying to tell you how to do a more C++-ish way.
2. Minimize function calls. If you can tokenize the entire file in one call, DO IT! Obviously, the output would be an array of tokens.
3. Minimize memory allocation. You probably should only allocate the data buffer being Tokenized, and an output array.
4. You might want to look at old solutions to things, and re-implement as needed. For instance, you've used isalpha(), which is classically a good macro, but the implementations you will find now are localized. (And you really don't want your compiler changing behavior based on the current language (as in English/French/Japanese/...).)
5. You might want to consider a finite state machine. (But I wouldn't consider compressing tables unless code size is a serious issue for you.)

Answer 2

Memory leak:

buffer = (char*)malloc(file_size + 1);

You allocated memory for buffer, but there is no corresponding call to free() in the entire code.

Moreover, (char *) is a C-style cast. The C++ way is:

buffer = static_cast<char*> (malloc(file_size + 1));

And it is quite unusual to see malloc() in C++ code. I would have expected new/delete instead, or no manual memory allocation at all.

If you run the program with valgrind:

$ valgrind ./a.out # replace a.out with program name

It'd show you all the places where your program has leaked memory. It is a tool I use any time I am writing C or C++ code. Apart from valgrind, Address Sanitizer and Static Analyzer are also good.

I see that you're terminating the bytes read with the null byte. Is there any reason why you're using C-strings? Had you used C++'s std::string from <string>, the code would have been simpler, and you wouldn't have leaked memory either.

In C++, we keep the type modifiers with the type. In other words:

• char *buffer: this is C-style.
• char* buffer: this is C++-style.

Missing and extra header files:

"Basics.h" does not use anything from <cassert>. It should be removed and included in "Main.cpp" where it is actually used. "Main.cpp" should not depend on "Basics.h" for its own dependencies.

"Basics.h" uses puts() without including <cstdio>.

Unqualified puts comes from the C compatibility header <stdio.h> - when using the C++ header <cstdio> one should use standard-namespace std::puts. Same goes for exit() that you've used in "Basics.h". See also: How to end C++ code?.

"Main.cpp" includes <stream>, but only uses ifstream from it. Replace it with <fstream>.

"Main.cpp" uses isspace(), isdigit() et cetera, but does not include <cctype>, and neither does "Basics.h", which "Main.cpp" includes.

"Main.cpp" also uses uint8_t without including <cstdint>.

"Lists.h" uses uint16_t without including <cstdint>.

The compiler would have told you as much, had you enabled warnings.

The ISO C++ language standard does not require the compilers to support any pragmas:

Whilst pragma is well-supported, I'd replace:

#pragma once

with:

#ifndef STRINGS_H
#define STRINGS_H 1

// File contents go here

#endif

The Standard library provides a way to find the size of the file:

From <filesystem>, use std::filesystem::file_size(p) instead of calling seekg() and tellg(), which aren't required to return the file size.

There's no need to manually close an ifstream:

Let the nested scope and destructor do its job, this is what RAII is for.

The functions from cctype expect an int cast to an unsigned char, not a char:

This:

if (isspace(C))

should be replaced with:

if (isspace(static_cast<unsigned char>(C)))

Note that the behavior is undefined if the value of the argument is not representable as unsigned char and is not equal to EOF.

Don't roll out your own String type and API:

The C++ Standard library already has a string type. Use that, it would provide all the functions you've written for C-strings.

But even if you don't, the C library string.h already provides functions like strchr(), strrchr() et cetera that you could have used. There was zero need to roll out your own find_first(), find_last() et cetera.

Furthermore, there's also std::list, which is a doubly-linked list, that could have been used instead of defining your own list. I do not see how a linked list is useful here, or what benefit it is providing. I'd suggest replacing it with std::vector.

Redundant code:

return 0 is the default behavior for main(). There's no need to add it yourself. But even if you're going to do so, use EXIT_SUCCESS and EXIT_FAILURE from cstdlib instead of magic numbers.

Reserved identifiers:

The ISO C++ Standard specifies that all identifiers preceded with an underscore are reserved at file-scope.

char _string_struct_ei_temp = '\0';

should just be:

char string_struct_ei_temp = '\0';

Don't mix C's and C++'s output functions;

Either use, preferably, std::cout, or puts(). Do not mix them.

Unit-tests:

I'd have expected to see unit tests for the lexer, and not a main() function that hardly exercises it. It also doesn't help that the main() function is 167 lines long and a mixture of C and C++ code.

ANSI escape sequences:

return os
        << "\033[0m (\033[90m"
        << token.kind
        << "\033[0m), 'value': '\033[36m"
        << token.str
        << "\033[0m'}";

If output is connected to a terminal (you can check if it is with isatty() on UNIX-like operating systems, or GetConsoleMode() on Windows), the chances that it will support VT100 (or "ANSI") sequences are infinitesimally close to 100%.

Yet a bigger problem is that I see no context to these sequences? What pray, are they doing? Or, what is their purpose? Most programs that use them have defined-constants for them. I'd suggest doing the same for this program.

C does not support the operator or namespace keywords:

const char *keywords[] = {
                "void",
                "int",
                "unsigned",
                "float",
                "double",
                "bool",
                "char",
                "short",
                "long",
                "inline",
                "static",
                "const",
                "return",
                "namespace",
                "struct",
                "operator",
                "case",
                "if",
                "else",
                "switch",
                "default",
                "break",
                "continue",
                "include",
                "define",
                "undef"
            };

namespace and operator are both C++ keywords. Moreover, bool is only a keyword in C23/C24. Before that, bool was only a macro that expanded to the actual keyword, which is _Bool. Now _Bool is just an alternate spelling for bool.

Furthermore, you're missing a lot of keywords. Here's a more comprehensive list:

alignas (C23)
alignof (C23)
auto
bool (C23)
break
case
char
const
constexpr (C23)
continue
default
do
double
else
enum
extern
false (C23)
float
for
goto
if
inline (C99)
int
long
nullptr (C23)
register
restrict (C99)
return
short
signed
sizeof
static
static_assert (C23)
struct
switch
thread_local (C23)
true (C23)
typedef
typeof (C23)
typeof_unqual (C23)
union
unsigned
void
volatile
while
_Alignas (C11)
_Alignof (C11)
_Atomic (C11)
_BitInt (C23)
_Bool (C99)
_Complex (C99)
_Decimal128 (C23)
_Decimal32 (C23)
_Decimal64 (C23)
_Generic (C11)
_Imaginary (C99)
_Noreturn (C11)
_Static_assert (C11)
_Thread_local (C11)

You may not be targeting C23, or even C11, but it is good to keep this in mind.

Error messages go to stderr, not stdout:

Change this:

inline void mexit(const char *str)
{
    puts(str);
    exit(0);

to:

inline void mexit(const char *str)
{
    fputs(str, stderr);
    exit(0);

Note that a return code of 0 signifies a successful termination. Consider using EXIT_SUCCESS and EXIT_FAILURE from <cstdlib> instead of magic numbers.

Answer 3

// free the memory
tokens.clear();

Typically in C++, the container would have a destructor that frees the whole thing, without requiring manual intervention.

You could also just use the old reliable std::vector (which you wouldn't need to manually clear); it doesn't look like the linked nature of the linked list is adding any value here (which is not to say that it never adds value, but it's rare niche thing).

You could also use the usual std::string and save the effort of implementing your own string (which typically comes with its own bugs etc), but it works a bit differently; up to you.

The widespread use of 16-bit integers is quite mysterious, they have some legitimate uses but they're not a good "default integer type". They also often don't even save any space, because the space they would have saved is added back in the form of padding. For example, in this struct:

struct List
{
    ListNode<T> *start, *end;
    uint16_t size;

The start and end pointers give the List an alignment of 8 (on a 64-bit platform, which most are) or 4 (on a 32-bit platform). The size of the struct will be 24 or 12 bytes respectively, and the std::uint16_t size effectively costs 8 or 4 bytes respectively, even though only 2 bytes are used usefully (the rest being padding).

For the std::uint8_t for the string length something similar applies, but also: strings longer than 255 characters are not unusual.

return os
    << "\033[0m (\033[90m"
    << token.kind
    << "\033[0m), 'value': '\033[36m"
    << token.str
    << "\033[0m'}";

This is fine (probably) but note that ANSI escape codes are not portable, and on some platforms will render as garbage. You don't have to write portable code of course, that's a choice that you can make.

Answer 4

Careful with <cctype> functions - these accept positive int values, and only one negative value: EOF. Passing char to std::isalpha() results in Undefined Behaviour because it may be sign-extended in the promotion. Cast it to unsigned char and promotion is safe:

   !std::isalpha((static_cast<unsigned char>(underscore[0]))

Alternatively, use the isalpha<>() template from <locales>, which accepts character types (including wide characters, when you need them). That will also allow you to select a specific locale (probably std::locale::classic()) so that parsing doesn't depend on global state.

---

int i = -1;
while (c[i] == '\\') i--;
return i & 1;

Logical operations on signed types are at the mercy of implementations' representations, just as in C. This code will give opposite results on ones-complement systems to twos-complement or sign-magnitude systems.

Avoid that issue completely by using a positive value:

    int i = 1;
    while (c[+i] == '\\') {
        ++i;
    }
    return i & 1;

Or just toggle a bool:

    bool odd = true;
    while (*--c == '\\') {
        odd = !odd;
    }
    return odd;

In any case, this function is flawed, since if pointed into a string such as "\\b", it won't terminate when it reaches the beginning, and you'll attempt to access memory you shouldn't (i.e. Undefined Behaviour).

---

This function looks wrong:

inline void mexit(const char *str)
{
    puts(str);
    exit(0);
}

It appears to be called when there's an error condition. So we should write the message to the standard error stream std::cerr (or, if you insist on using C-style I/O, with std::fputs(str, stderr)). I don't recommend calling std::exit() - instead it's better to return from main() - and we should be using a non-zero exit status if the program was not successful. <cstdlib> defines a macro EXIT_FAILURE for exactly that purpose.

Also, inline probably doesn't have the effect you seem to expect. It's rare that we need it in C++ (or, indeed in modern C).

I'd write this function as

int warn(std::string_view s)
{
    std::cerr << s << '\n';
    return EXIT_FAILURE;
}

And then in main() (for example):

                if (base != 0) {
                    return warn("Float number with non-decimal system.");
                }

Answer 5

Don’t Reinvent the Wheel

This is especially true of your String class. You additionally sometimes #include "String.h" and sometimes #include "Strings.h". Avoid either of these names: there is a <string.h> in the standard library and a <strings.h> on many operating systems.

The TokenKind class has a tag to identify the stored type, making it effectively a discriminated union. There’s a more-efficient implementation in the Standard Library already, std::variant.

Consider Making String Tokens, Views

Tokens are tiny snippets of the source file, so if you memory-map the file (which requires Boost or some other library to do portably), your tokens could each be a string_view of the file contents in memory, eliminating any need to copy their data at all. In that case, the token-list class should own the memory mapping, and its destructor should unmap/close the file at the same time it destroys the vector of tokens, to prevent dangling references.

Here, though, it looks like all tokens are either constant keywords/operators, identifiers, or numeric constants. You already have special handling of numeric constants. You might therefore store identifiers in a hash table (unordered_set). Every identifier token could hold a string_view or pointer to the string stored in this table. This way, you only need to store one copy of each identifier in memory. Comparing equality also becomes much simpler, since any two tokens are identical if and only if they point to the same address.

Keyword and operator tokens could be either a string_view or an index into a table of string constants. The fanciest representation would be a perfect hash, but you might also store them in lexicographic order and check each word with a binary, instead of linear, search. You probably also want to check first whether a token is the right length to be a keyword. You want this lookup to be fast, since you need to check every word in the input.

Use RAII to Manage Resources

Your classes should be cleaning up their resources when their instances’ lifetimes expire. There’s no reason to be using C-style memory management, or manually calling .clear() before program exit: all memory is freed on exit automatically.

For example, if you don’t memory-map the source file, your input buffer could be declared as std::vector<char> buffer(file_size); and you could read into its data() member. Rather than .clear() it, you could declare it within a scope other than main, and it will automatically be destroyed when the program returns from that scope.

You will often be able to implement a pipeline like this by moving data structures to functions that consume them and return a new data structure, e.g. auto ast = parser(lexer(std::move(source))); makes the data structures that hold the source text and token list into temporary objects that are destroyed when no longer needed.

Error Handling

Use assert to catch logic errors, not runtime errors that could happen to correct code. A user should never see an assertion fail in production.

Your todo function should probably be an uncaught exception, such as std::logic_error. The best way to handle an out-of-memory error is to throw std::bad_alloc as the Standard Library does, or to allocate a smart object such as std::unique_ptr or std::vector that manages its own memory.

Your mexit debugging function would be better off printing the source line where the error occurred. You could add a parameter, const std::source_location location = std::source_location::current(), which passes in the line that called it.

Both should flush cout, print their error message to cerr, and return EXIT_FAILURE.

This module should have a more descriptive name than Basics.

Answer 6

I hate this:

    buffer = (char*)malloc(file_size + 1);
    assert(buffer != nullptr);

The trouble is that if you have a build system set up to build this it will probably define NDEBUG for production builds. This will disable all the assert() and this no longer does anything, so your check in production (when you need it most) will not help you find issues.

Bugs:

You don't check if find_first() takes you to the end of buffer?

            if (C == '/')
            {
                if (c[1] == '/')
                {
                    // What if the last line in the file
                    // is a comment that does not have a '\n'?
                    //
                    // Then find_first() will return a pointer
                    // to the '\0' character. But you add + 1 to this
                    // without checking.
                    //
                    // So `c` is now out beyond the end of your
                    // buffer.
                    c = find_first(c + 2, '\n') + 1;
                    continue;
                } // check if multi-line comment
                else if (c[1] == '*')
                {
                    // Same problem here if your file
                    // contains an unterminated comment.
                    //
                    // Sure technically that's not valid input
                    // But it would be nice to spot that and tell
                    // the developer rather than go and build some
                    // random code from beyond the end of the buffer.
                    c += 2;
                    while (*c != '/')
                        c = find_first(c, '*') + 1;
                    c++;
                    continue;
                }
            }