We’re rewarding the question askers & reputations are being recalculated! Read more.
3 edited tags
| link
2 Fix stupid bug
source | link
#include <set>
#include "Lexer.h"

namespace
{
bool startsIdentifier(char character)
{
    return std::isalpha(character) ||
        character == '_' ||
        character == '\\';);
}

bool isInIdentifier(char character)
{
    return startsIdentifier(character) ||
        std::isdigit(character);
}

bool isHexChar(char character)
{
    return std::isdigit(character) ||
        (character >= 'A' && character <= 'F') ||
        (character >= 'a' && character <= 'f');
}

bool isOctal(char character)
{
    return std::isdigit(character) && '8' - character > 0;
}

bool isDigit(char character)
{
    return std::isdigit(character);
}

bool isWhitespace(char character)
{
    return character == ' ' || character == '\n' || character == '\t';
}

bool isExponentChar(char character)
{
    return (character == 'e' ||
            character == 'E' ||
            character == 'p' ||
            character == 'P');
}

static std::set<std::string> keywords = {
    "auto", "break", "case", "char", "const", "continue", "default",
    "do", "double", "else", "enum", "extern", "float", "for", "goto",
    "if", "inline", "int", "long", "register", "restrict", "return",
    "short", "signed", "sizeof", "static", "struct", "switch", "typedef",
    "union", "unsigned", "void", "volatile", "while", "_Bool",
    "_Complex", "_Imaginary"
};
}

Lexer::Lexer(std::istream& stream):
    mStream(stream)
{
}

char Lexer::nextChar()
{
    char c;
    mStream.read(&c, 1);
    return c;
}

char Lexer::skipWhitespace()
{
    char c;
    do
        c = nextChar();
    while (isWhitespace(c));
    return c;
}

char Lexer::peekChar()
{
    return mStream.peek();
}

bool Lexer::peekWord(const std::string& word)
{
    std::string lexedWord;

    // We've already read the first character, so set that in advance
    lexedWord.resize(word.size());
    lexedWord[0] = word[0];

    const size_t bytesToRead = lexedWord.size() - 1;
    mStream.read(&lexedWord[1], bytesToRead);
    if (lexedWord == word) {
        return true;
    }
    else {
        mStream.seekg((-bytesToRead), std::ios::cur);
        return false;
    }
}

std::string Lexer::lexHexQuad(bool& isValid)
{
    std::string hexQuad;
    for (size_t i = 0; i < 4; ++i) {
        char character = nextChar();
        // First check if the character is okay
        if (!isHexChar(character))
            isValid = false;

        // Get a better error message (I hope?) if we keep reading
        // until a "logical" break, so no matter what keep going unless
        // there's whitespace
        if (!isWhitespace(character) && mStream)
            hexQuad += character;

        // If we hit whitespace, there's no way it's valid
        // We're at a logical boundary, so just return here
        else return hexQuad;

    }
    return hexQuad;
}

std::string Lexer::lexUniversalCharacter(char character, bool& isValid)
{
    std::string universalCharacter(1, character);

    character = nextChar();
    universalCharacter += character;
    if (character == 'u')
        universalCharacter += lexHexQuad(isValid);
    else if (character == 'U') {
        universalCharacter += lexHexQuad(isValid);
        universalCharacter += lexHexQuad(isValid);
    }
    else
        isValid = false;
    return universalCharacter;
}

std::string Lexer::nextIdentifier(char character, bool& isValid)
{
    std::string identifier;
    while (isInIdentifier(character)) {
        if (character == '\\') {
            identifier += lexUniversalCharacter(character, isValid);
            if (!isValid)
                return identifier;
        }
        else
            identifier += character;
        character = nextChar();
    }
    mStream.seekg(-1, std::ios::cur);
    return identifier;
}



LexResult Lexer::checkInvalidToken(const LexResult& token)
{
    std::string lexeme = token.lexeme;
    if (!isWhitespace(peekChar()) && peekChar() != EOF) {
        while (!isWhitespace(peekChar()) && peekChar() != EOF)
            lexeme += nextChar();
        return LexResult{Token::ERROR, lexeme};
    }
    return token;
}

LexResult Lexer::lexIdentifier(char character)
{
    bool isValid = true;
    const std::string identifier = nextIdentifier(character, isValid);
    if (!isValid)
        return LexResult{Token::ERROR, identifier};
    if (keywords.count(identifier))
        return LexResult{Token::KEYWORD, identifier};
    return LexResult{Token::IDENTIFIER, identifier};
}

std::string Lexer::readIntConstant(bool predicate(char))
{
    std::string lexeme;
    while (predicate(peekChar()))
        lexeme += nextChar();
    return lexeme;
}

std::string Lexer::readLongSuffix()
{
    std::string suffix;
    suffix += nextChar();
    if (peekChar() == 'L' || peekChar() == 'l')
        suffix += nextChar();
    return suffix;
}

std::string Lexer::readIntSuffix()
{
    std::string suffix;
    if (peekChar() == 'L' || peekChar() == 'l') {
        suffix += readLongSuffix();
        if (peekChar() == 'u' || peekChar() == 'U')
            suffix += nextChar();
    }
    else if (peekChar() == 'u' || peekChar() == 'U') {
        suffix += nextChar();
        if (peekChar() == 'l' || peekChar() == 'L')
            suffix += readLongSuffix();
    }
    return suffix;
}

LexResult Lexer::readFloatConstant(const std::string& prefix,
                                   bool predicate(char))
{
    std::string result = prefix;
    if (peekChar() == '.') {
        result += nextChar();
        result += readIntConstant(predicate);
    }
    if (isExponentChar(peekChar())) {
        result += nextChar();
        if (peekChar() == '+' || peekChar() == '-')
            result += nextChar();

        const std::string exponentSequence = readIntConstant(predicate);
        if (exponentSequence.empty())
            return checkInvalidToken(LexResult{Token::ERROR, result});

        result += exponentSequence;
    }

    const char suffix = peekChar();
    if (suffix == 'f' || suffix == 'F' ||
        suffix == 'l' || suffix == 'L')
        result += nextChar();
    return checkInvalidToken(LexResult{Token::FLOAT_CONST, result});
}

LexResult Lexer::lexConstant(char character)
{
    std::string lexeme(1, character);
    Token token = Token::INT_CONST;
    auto predicate = isDigit;

    if (peekChar() == 'x' || peekChar() == 'X') {
        predicate = isHexChar;
        lexeme += nextChar();
    }
    else if (character == '0')
        predicate = isOctal;

    lexeme += readIntConstant(predicate);

    if (peekChar() == '.' || isExponentChar(peekChar()))
        return readFloatConstant(lexeme, predicate);

    lexeme += readIntSuffix();
    return checkInvalidToken(LexResult{token, lexeme});
}

std::string Lexer::lexEscapeSequence(bool& isValid)
{
    std::string lexeme(1, nextChar());
    const char c = peekChar();

    if (c == '\'' || c == '"' || c == '?' ||
        c == '\\' || c == 'a' || c == 'b' ||
        c == 'f' || c == 'n' || c == 'r' ||
        c == 't' || c == 'v' || c == 'x' || isHexChar(c))
        return lexeme + nextChar();

    isValid = false;
    return lexeme;
}

LexResult Lexer::lexCharConst(const std::string& initial, char delimiter)
{
    bool isValid = true;
    std::string lexeme = initial + delimiter;
    while (peekChar() != delimiter && peekChar() != EOF) {
        if (peekChar() == '\n')
            isValid = false;
        if (peekChar() == '\\') {
            lexeme += lexEscapeSequence(isValid);
            continue;
        }

        lexeme += nextChar();
    }
    lexeme += nextChar();
    return LexResult{isValid ? Token::CHAR_CONST : Token::ERROR, lexeme};
}

void Lexer::skipLineComment()
{
    nextChar(); nextChar();
    while (peekChar() != '\n')
        nextChar();
}

void Lexer::skipBlockComment()
{
    nextChar(); // Skip the first *
    char c;
    do
        c = nextChar();
    while (!(c == '*' && peekChar() == '/'));
    nextChar(); // Pass the last /
}

LexResult Lexer::next()
{
    char character = skipWhitespace();
    while (character == '/' && peekChar() == '/') {
        skipLineComment();
        character = skipWhitespace();
    }
    while (character == '/' && peekChar() == '*') {
        skipBlockComment();
        character = skipWhitespace();
    }
    if (isdigit(character))
        return lexConstant(character);
    if (character == '\'' || character == '"')
        return lexCharConst("", character);
    if (character == 'L')
        if (peekChar() == '\'' || peekChar() == '"') {
            return lexCharConst("L", nextChar());
        }
    if (startsIdentifier(character))
        return lexIdentifier(character);
    switch (character) {
        case '+':
            if (peekWord("++"))
                return LexResult{Token::PUNCTUATOR, "++"};
            if (peekWord("+="))
                return LexResult{Token::PUNCTUATOR, "+="};
            return LexResult{Token::PUNCTUATOR, "+"};
        case '-':
            if (peekWord("->"))
                return LexResult{Token::PUNCTUATOR, "->"};
            if (peekWord("--"))
                return LexResult{Token::PUNCTUATOR, "--"};
            if (peekWord("-="))
                return LexResult{Token::PUNCTUATOR, "-="};
            return LexResult{Token::PUNCTUATOR, "-"};
        case '*':
            if (peekWord("*="))
                return LexResult{Token::PUNCTUATOR, "*="};
            return LexResult{Token::PUNCTUATOR, "*"};
        case '/':
            if (peekWord("/="))
                return LexResult{Token::PUNCTUATOR, "/="};
            return LexResult{Token::PUNCTUATOR, "/"};
        case '=':
            if (peekWord("=="))
                return LexResult{Token::PUNCTUATOR, "=="};
            return LexResult{Token::PUNCTUATOR, "="};
        case '[':
            return LexResult{Token::PUNCTUATOR, "["};
        case ']':
            return LexResult{Token::PUNCTUATOR, "]"};
        case '(':
            return LexResult{Token::PUNCTUATOR, "("};
        case ')':
            return LexResult{Token::PUNCTUATOR, ")"};
        case '.':
            if (peekWord("..."))
                return LexResult{Token::PUNCTUATOR, "..."};
            return LexResult{Token::PUNCTUATOR, "."};
        case '&':
            if (peekWord("&&"))
                return LexResult{Token::PUNCTUATOR, "&&"};
            if (peekWord("&="))
                return LexResult{Token::PUNCTUATOR, "&="};
            return LexResult{Token::PUNCTUATOR, "&"};
        case '~':
            return LexResult{Token::PUNCTUATOR, "~"};
        case '!':
            if (peekWord("!="))
                return LexResult{Token::PUNCTUATOR, "!="};
            return LexResult{Token::PUNCTUATOR, "!"};
        case '%':
            if (peekWord("%:%:"))
                return LexResult{Token::PUNCTUATOR, "%:%:"};
            if (peekWord("%:"))
                return LexResult{Token::PUNCTUATOR, "%:"};
            if (peekWord("%="))
                return LexResult{Token::PUNCTUATOR, "%="};
            if (peekWord("%>"))
               return LexResult{Token::PUNCTUATOR, "%>"};
            return LexResult{Token::PUNCTUATOR, "%"};
        case '<':
            if (peekWord("<<="))
                return LexResult{Token::PUNCTUATOR, "<<="};
            if (peekWord("<:"))
                return LexResult{Token::PUNCTUATOR, "<:"};
            if (peekWord("<%"))
                return LexResult{Token::PUNCTUATOR, "<%"};
            if (peekWord("<<"))
                return LexResult{Token::PUNCTUATOR, "<<"};
            if (peekWord("<="))
                return LexResult{Token::PUNCTUATOR, "<="};
            return LexResult{Token::PUNCTUATOR, "<"};
        case '>':
            if (peekWord(">>="))
                return LexResult{Token::PUNCTUATOR, ">>="};
            if (peekWord(">>"))
                return LexResult{Token::PUNCTUATOR, ">>"};
            if (peekWord(">="))
                return LexResult{Token::PUNCTUATOR, ">="};
            return LexResult{Token::PUNCTUATOR, ">"};
        case '^':
            if (peekWord("^="))
                return LexResult{Token::PUNCTUATOR, "^="};
            return LexResult{Token::PUNCTUATOR, "^"};
        case '|':
            if (peekWord("||"))
                return LexResult{Token::PUNCTUATOR, "||"};
            if (peekWord("|="))
                return LexResult{Token::PUNCTUATOR, "|="};
            return LexResult{Token::PUNCTUATOR, "|"};
        case '?':
            return LexResult{Token::PUNCTUATOR, "?"};
        case ':':
            if (peekWord(":>"))
                return LexResult{Token::PUNCTUATOR, ":>"};
            return LexResult{Token::PUNCTUATOR, ":"};
        case ';':
            return LexResult{Token::PUNCTUATOR, ";"};
        case ',':
            return LexResult{Token::PUNCTUATOR, ","};
        case '#':
            if (peekWord("##"))
                return LexResult{Token::PUNCTUATOR, "##"};
            return LexResult{Token::PUNCTUATOR, "#"};
        default:
            return LexResult{Token::ERROR, std::string(character, 1)};
    }
}
#include <set>
#include "Lexer.h"

namespace
{
bool startsIdentifier(char character)
{
    return std::isalpha(character) ||
        character == '_' ||
        character == '\\';
}

bool isInIdentifier(char character)
{
    return startsIdentifier(character) ||
        std::isdigit(character);
}

bool isHexChar(char character)
{
    return std::isdigit(character) ||
        (character >= 'A' && character <= 'F') ||
        (character >= 'a' && character <= 'f');
}

bool isOctal(char character)
{
    return std::isdigit(character) && '8' - character > 0;
}

bool isDigit(char character)
{
    return std::isdigit(character);
}

bool isWhitespace(char character)
{
    return character == ' ' || character == '\n' || character == '\t';
}

bool isExponentChar(char character)
{
    return (character == 'e' ||
            character == 'E' ||
            character == 'p' ||
            character == 'P');
}

static std::set<std::string> keywords = {
    "auto", "break", "case", "char", "const", "continue", "default",
    "do", "double", "else", "enum", "extern", "float", "for", "goto",
    "if", "inline", "int", "long", "register", "restrict", "return",
    "short", "signed", "sizeof", "static", "struct", "switch", "typedef",
    "union", "unsigned", "void", "volatile", "while", "_Bool",
    "_Complex", "_Imaginary"
};
}

Lexer::Lexer(std::istream& stream):
    mStream(stream)
{
}

char Lexer::nextChar()
{
    char c;
    mStream.read(&c, 1);
    return c;
}

char Lexer::skipWhitespace()
{
    char c;
    do
        c = nextChar();
    while (isWhitespace(c));
    return c;
}

char Lexer::peekChar()
{
    return mStream.peek();
}

bool Lexer::peekWord(const std::string& word)
{
    std::string lexedWord;

    // We've already read the first character, so set that in advance
    lexedWord.resize(word.size());
    lexedWord[0] = word[0];

    const size_t bytesToRead = lexedWord.size() - 1;
    mStream.read(&lexedWord[1], bytesToRead);
    if (lexedWord == word) {
        return true;
    }
    else {
        mStream.seekg((-bytesToRead), std::ios::cur);
        return false;
    }
}

std::string Lexer::lexHexQuad(bool& isValid)
{
    std::string hexQuad;
    for (size_t i = 0; i < 4; ++i) {
        char character = nextChar();
        // First check if the character is okay
        if (!isHexChar(character))
            isValid = false;

        // Get a better error message (I hope?) if we keep reading
        // until a "logical" break, so no matter what keep going unless
        // there's whitespace
        if (!isWhitespace(character) && mStream)
            hexQuad += character;

        // If we hit whitespace, there's no way it's valid
        // We're at a logical boundary, so just return here
        else return hexQuad;

    }
    return hexQuad;
}

std::string Lexer::lexUniversalCharacter(char character, bool& isValid)
{
    std::string universalCharacter(1, character);

    character = nextChar();
    universalCharacter += character;
    if (character == 'u')
        universalCharacter += lexHexQuad(isValid);
    else if (character == 'U') {
        universalCharacter += lexHexQuad(isValid);
        universalCharacter += lexHexQuad(isValid);
    }
    else
        isValid = false;
    return universalCharacter;
}

std::string Lexer::nextIdentifier(char character, bool& isValid)
{
    std::string identifier;
    while (isInIdentifier(character)) {
        if (character == '\\') {
            identifier += lexUniversalCharacter(character, isValid);
            if (!isValid)
                return identifier;
        }
        else
            identifier += character;
        character = nextChar();
    }
    mStream.seekg(-1, std::ios::cur);
    return identifier;
}



LexResult Lexer::checkInvalidToken(const LexResult& token)
{
    std::string lexeme = token.lexeme;
    if (!isWhitespace(peekChar()) && peekChar() != EOF) {
        while (!isWhitespace(peekChar()) && peekChar() != EOF)
            lexeme += nextChar();
        return LexResult{Token::ERROR, lexeme};
    }
    return token;
}

LexResult Lexer::lexIdentifier(char character)
{
    bool isValid = true;
    const std::string identifier = nextIdentifier(character, isValid);
    if (!isValid)
        return LexResult{Token::ERROR, identifier};
    if (keywords.count(identifier))
        return LexResult{Token::KEYWORD, identifier};
    return LexResult{Token::IDENTIFIER, identifier};
}

std::string Lexer::readIntConstant(bool predicate(char))
{
    std::string lexeme;
    while (predicate(peekChar()))
        lexeme += nextChar();
    return lexeme;
}

std::string Lexer::readLongSuffix()
{
    std::string suffix;
    suffix += nextChar();
    if (peekChar() == 'L' || peekChar() == 'l')
        suffix += nextChar();
    return suffix;
}

std::string Lexer::readIntSuffix()
{
    std::string suffix;
    if (peekChar() == 'L' || peekChar() == 'l') {
        suffix += readLongSuffix();
        if (peekChar() == 'u' || peekChar() == 'U')
            suffix += nextChar();
    }
    else if (peekChar() == 'u' || peekChar() == 'U') {
        suffix += nextChar();
        if (peekChar() == 'l' || peekChar() == 'L')
            suffix += readLongSuffix();
    }
    return suffix;
}

LexResult Lexer::readFloatConstant(const std::string& prefix,
                                   bool predicate(char))
{
    std::string result = prefix;
    if (peekChar() == '.') {
        result += nextChar();
        result += readIntConstant(predicate);
    }
    if (isExponentChar(peekChar())) {
        result += nextChar();
        if (peekChar() == '+' || peekChar() == '-')
            result += nextChar();

        const std::string exponentSequence = readIntConstant(predicate);
        if (exponentSequence.empty())
            return checkInvalidToken(LexResult{Token::ERROR, result});

        result += exponentSequence;
    }

    const char suffix = peekChar();
    if (suffix == 'f' || suffix == 'F' ||
        suffix == 'l' || suffix == 'L')
        result += nextChar();
    return checkInvalidToken(LexResult{Token::FLOAT_CONST, result});
}

LexResult Lexer::lexConstant(char character)
{
    std::string lexeme(1, character);
    Token token = Token::INT_CONST;
    auto predicate = isDigit;

    if (peekChar() == 'x' || peekChar() == 'X') {
        predicate = isHexChar;
        lexeme += nextChar();
    }
    else if (character == '0')
        predicate = isOctal;

    lexeme += readIntConstant(predicate);

    if (peekChar() == '.' || isExponentChar(peekChar()))
        return readFloatConstant(lexeme, predicate);

    lexeme += readIntSuffix();
    return checkInvalidToken(LexResult{token, lexeme});
}

std::string Lexer::lexEscapeSequence(bool& isValid)
{
    std::string lexeme(1, nextChar());
    const char c = peekChar();

    if (c == '\'' || c == '"' || c == '?' ||
        c == '\\' || c == 'a' || c == 'b' ||
        c == 'f' || c == 'n' || c == 'r' ||
        c == 't' || c == 'v' || c == 'x' || isHexChar(c))
        return lexeme + nextChar();

    isValid = false;
    return lexeme;
}

LexResult Lexer::lexCharConst(const std::string& initial, char delimiter)
{
    bool isValid = true;
    std::string lexeme = initial + delimiter;
    while (peekChar() != delimiter && peekChar() != EOF) {
        if (peekChar() == '\n')
            isValid = false;
        if (peekChar() == '\\') {
            lexeme += lexEscapeSequence(isValid);
            continue;
        }

        lexeme += nextChar();
    }
    lexeme += nextChar();
    return LexResult{isValid ? Token::CHAR_CONST : Token::ERROR, lexeme};
}

void Lexer::skipLineComment()
{
    nextChar(); nextChar();
    while (peekChar() != '\n')
        nextChar();
}

void Lexer::skipBlockComment()
{
    nextChar(); // Skip the first *
    char c;
    do
        c = nextChar();
    while (!(c == '*' && peekChar() == '/'));
    nextChar(); // Pass the last /
}

LexResult Lexer::next()
{
    char character = skipWhitespace();
    while (character == '/' && peekChar() == '/') {
        skipLineComment();
        character = skipWhitespace();
    }
    while (character == '/' && peekChar() == '*') {
        skipBlockComment();
        character = skipWhitespace();
    }
    if (isdigit(character))
        return lexConstant(character);
    if (character == '\'' || character == '"')
        return lexCharConst("", character);
    if (character == 'L')
        if (peekChar() == '\'' || peekChar() == '"') {
            return lexCharConst("L", nextChar());
        }
    if (startsIdentifier(character))
        return lexIdentifier(character);
    switch (character) {
        case '+':
            if (peekWord("++"))
                return LexResult{Token::PUNCTUATOR, "++"};
            if (peekWord("+="))
                return LexResult{Token::PUNCTUATOR, "+="};
            return LexResult{Token::PUNCTUATOR, "+"};
        case '-':
            if (peekWord("->"))
                return LexResult{Token::PUNCTUATOR, "->"};
            if (peekWord("--"))
                return LexResult{Token::PUNCTUATOR, "--"};
            if (peekWord("-="))
                return LexResult{Token::PUNCTUATOR, "-="};
            return LexResult{Token::PUNCTUATOR, "-"};
        case '*':
            if (peekWord("*="))
                return LexResult{Token::PUNCTUATOR, "*="};
            return LexResult{Token::PUNCTUATOR, "*"};
        case '/':
            if (peekWord("/="))
                return LexResult{Token::PUNCTUATOR, "/="};
            return LexResult{Token::PUNCTUATOR, "/"};
        case '=':
            if (peekWord("=="))
                return LexResult{Token::PUNCTUATOR, "=="};
            return LexResult{Token::PUNCTUATOR, "="};
        case '[':
            return LexResult{Token::PUNCTUATOR, "["};
        case ']':
            return LexResult{Token::PUNCTUATOR, "]"};
        case '(':
            return LexResult{Token::PUNCTUATOR, "("};
        case ')':
            return LexResult{Token::PUNCTUATOR, ")"};
        case '.':
            if (peekWord("..."))
                return LexResult{Token::PUNCTUATOR, "..."};
            return LexResult{Token::PUNCTUATOR, "."};
        case '&':
            if (peekWord("&&"))
                return LexResult{Token::PUNCTUATOR, "&&"};
            if (peekWord("&="))
                return LexResult{Token::PUNCTUATOR, "&="};
            return LexResult{Token::PUNCTUATOR, "&"};
        case '~':
            return LexResult{Token::PUNCTUATOR, "~"};
        case '!':
            if (peekWord("!="))
                return LexResult{Token::PUNCTUATOR, "!="};
            return LexResult{Token::PUNCTUATOR, "!"};
        case '%':
            if (peekWord("%:%:"))
                return LexResult{Token::PUNCTUATOR, "%:%:"};
            if (peekWord("%:"))
                return LexResult{Token::PUNCTUATOR, "%:"};
            if (peekWord("%="))
                return LexResult{Token::PUNCTUATOR, "%="};
            if (peekWord("%>"))
               return LexResult{Token::PUNCTUATOR, "%>"};
            return LexResult{Token::PUNCTUATOR, "%"};
        case '<':
            if (peekWord("<<="))
                return LexResult{Token::PUNCTUATOR, "<<="};
            if (peekWord("<:"))
                return LexResult{Token::PUNCTUATOR, "<:"};
            if (peekWord("<%"))
                return LexResult{Token::PUNCTUATOR, "<%"};
            if (peekWord("<<"))
                return LexResult{Token::PUNCTUATOR, "<<"};
            if (peekWord("<="))
                return LexResult{Token::PUNCTUATOR, "<="};
            return LexResult{Token::PUNCTUATOR, "<"};
        case '>':
            if (peekWord(">>="))
                return LexResult{Token::PUNCTUATOR, ">>="};
            if (peekWord(">>"))
                return LexResult{Token::PUNCTUATOR, ">>"};
            if (peekWord(">="))
                return LexResult{Token::PUNCTUATOR, ">="};
            return LexResult{Token::PUNCTUATOR, ">"};
        case '^':
            if (peekWord("^="))
                return LexResult{Token::PUNCTUATOR, "^="};
            return LexResult{Token::PUNCTUATOR, "^"};
        case '|':
            if (peekWord("||"))
                return LexResult{Token::PUNCTUATOR, "||"};
            if (peekWord("|="))
                return LexResult{Token::PUNCTUATOR, "|="};
            return LexResult{Token::PUNCTUATOR, "|"};
        case '?':
            return LexResult{Token::PUNCTUATOR, "?"};
        case ':':
            if (peekWord(":>"))
                return LexResult{Token::PUNCTUATOR, ":>"};
            return LexResult{Token::PUNCTUATOR, ":"};
        case ';':
            return LexResult{Token::PUNCTUATOR, ";"};
        case ',':
            return LexResult{Token::PUNCTUATOR, ","};
        case '#':
            if (peekWord("##"))
                return LexResult{Token::PUNCTUATOR, "##"};
            return LexResult{Token::PUNCTUATOR, "#"};
        default:
            return LexResult{Token::ERROR, std::string(character, 1)};
    }
}
#include <set>
#include "Lexer.h"

namespace
{
bool startsIdentifier(char character)
{
    return std::isalpha(character) || character == '_');
}

bool isInIdentifier(char character)
{
    return startsIdentifier(character) ||
        std::isdigit(character);
}

bool isHexChar(char character)
{
    return std::isdigit(character) ||
        (character >= 'A' && character <= 'F') ||
        (character >= 'a' && character <= 'f');
}

bool isOctal(char character)
{
    return std::isdigit(character) && '8' - character > 0;
}

bool isDigit(char character)
{
    return std::isdigit(character);
}

bool isWhitespace(char character)
{
    return character == ' ' || character == '\n' || character == '\t';
}

bool isExponentChar(char character)
{
    return (character == 'e' ||
            character == 'E' ||
            character == 'p' ||
            character == 'P');
}

static std::set<std::string> keywords = {
    "auto", "break", "case", "char", "const", "continue", "default",
    "do", "double", "else", "enum", "extern", "float", "for", "goto",
    "if", "inline", "int", "long", "register", "restrict", "return",
    "short", "signed", "sizeof", "static", "struct", "switch", "typedef",
    "union", "unsigned", "void", "volatile", "while", "_Bool",
    "_Complex", "_Imaginary"
};
}

Lexer::Lexer(std::istream& stream):
    mStream(stream)
{
}

char Lexer::nextChar()
{
    char c;
    mStream.read(&c, 1);
    return c;
}

char Lexer::skipWhitespace()
{
    char c;
    do
        c = nextChar();
    while (isWhitespace(c));
    return c;
}

char Lexer::peekChar()
{
    return mStream.peek();
}

bool Lexer::peekWord(const std::string& word)
{
    std::string lexedWord;

    // We've already read the first character, so set that in advance
    lexedWord.resize(word.size());
    lexedWord[0] = word[0];

    const size_t bytesToRead = lexedWord.size() - 1;
    mStream.read(&lexedWord[1], bytesToRead);
    if (lexedWord == word) {
        return true;
    }
    else {
        mStream.seekg((-bytesToRead), std::ios::cur);
        return false;
    }
}

std::string Lexer::lexHexQuad(bool& isValid)
{
    std::string hexQuad;
    for (size_t i = 0; i < 4; ++i) {
        char character = nextChar();
        // First check if the character is okay
        if (!isHexChar(character))
            isValid = false;

        // Get a better error message (I hope?) if we keep reading
        // until a "logical" break, so no matter what keep going unless
        // there's whitespace
        if (!isWhitespace(character) && mStream)
            hexQuad += character;

        // If we hit whitespace, there's no way it's valid
        // We're at a logical boundary, so just return here
        else return hexQuad;

    }
    return hexQuad;
}

std::string Lexer::lexUniversalCharacter(char character, bool& isValid)
{
    std::string universalCharacter(1, character);

    character = nextChar();
    universalCharacter += character;
    if (character == 'u')
        universalCharacter += lexHexQuad(isValid);
    else if (character == 'U') {
        universalCharacter += lexHexQuad(isValid);
        universalCharacter += lexHexQuad(isValid);
    }
    else
        isValid = false;
    return universalCharacter;
}

std::string Lexer::nextIdentifier(char character, bool& isValid)
{
    std::string identifier;
    while (isInIdentifier(character)) {
        if (character == '\\') {
            identifier += lexUniversalCharacter(character, isValid);
            if (!isValid)
                return identifier;
        }
        else
            identifier += character;
        character = nextChar();
    }
    mStream.seekg(-1, std::ios::cur);
    return identifier;
}



LexResult Lexer::checkInvalidToken(const LexResult& token)
{
    std::string lexeme = token.lexeme;
    if (!isWhitespace(peekChar()) && peekChar() != EOF) {
        while (!isWhitespace(peekChar()) && peekChar() != EOF)
            lexeme += nextChar();
        return LexResult{Token::ERROR, lexeme};
    }
    return token;
}

LexResult Lexer::lexIdentifier(char character)
{
    bool isValid = true;
    const std::string identifier = nextIdentifier(character, isValid);
    if (!isValid)
        return LexResult{Token::ERROR, identifier};
    if (keywords.count(identifier))
        return LexResult{Token::KEYWORD, identifier};
    return LexResult{Token::IDENTIFIER, identifier};
}

std::string Lexer::readIntConstant(bool predicate(char))
{
    std::string lexeme;
    while (predicate(peekChar()))
        lexeme += nextChar();
    return lexeme;
}

std::string Lexer::readLongSuffix()
{
    std::string suffix;
    suffix += nextChar();
    if (peekChar() == 'L' || peekChar() == 'l')
        suffix += nextChar();
    return suffix;
}

std::string Lexer::readIntSuffix()
{
    std::string suffix;
    if (peekChar() == 'L' || peekChar() == 'l') {
        suffix += readLongSuffix();
        if (peekChar() == 'u' || peekChar() == 'U')
            suffix += nextChar();
    }
    else if (peekChar() == 'u' || peekChar() == 'U') {
        suffix += nextChar();
        if (peekChar() == 'l' || peekChar() == 'L')
            suffix += readLongSuffix();
    }
    return suffix;
}

LexResult Lexer::readFloatConstant(const std::string& prefix,
                                   bool predicate(char))
{
    std::string result = prefix;
    if (peekChar() == '.') {
        result += nextChar();
        result += readIntConstant(predicate);
    }
    if (isExponentChar(peekChar())) {
        result += nextChar();
        if (peekChar() == '+' || peekChar() == '-')
            result += nextChar();

        const std::string exponentSequence = readIntConstant(predicate);
        if (exponentSequence.empty())
            return checkInvalidToken(LexResult{Token::ERROR, result});

        result += exponentSequence;
    }

    const char suffix = peekChar();
    if (suffix == 'f' || suffix == 'F' ||
        suffix == 'l' || suffix == 'L')
        result += nextChar();
    return checkInvalidToken(LexResult{Token::FLOAT_CONST, result});
}

LexResult Lexer::lexConstant(char character)
{
    std::string lexeme(1, character);
    Token token = Token::INT_CONST;
    auto predicate = isDigit;

    if (peekChar() == 'x' || peekChar() == 'X') {
        predicate = isHexChar;
        lexeme += nextChar();
    }
    else if (character == '0')
        predicate = isOctal;

    lexeme += readIntConstant(predicate);

    if (peekChar() == '.' || isExponentChar(peekChar()))
        return readFloatConstant(lexeme, predicate);

    lexeme += readIntSuffix();
    return checkInvalidToken(LexResult{token, lexeme});
}

std::string Lexer::lexEscapeSequence(bool& isValid)
{
    std::string lexeme(1, nextChar());
    const char c = peekChar();

    if (c == '\'' || c == '"' || c == '?' ||
        c == '\\' || c == 'a' || c == 'b' ||
        c == 'f' || c == 'n' || c == 'r' ||
        c == 't' || c == 'v' || c == 'x' || isHexChar(c))
        return lexeme + nextChar();

    isValid = false;
    return lexeme;
}

LexResult Lexer::lexCharConst(const std::string& initial, char delimiter)
{
    bool isValid = true;
    std::string lexeme = initial + delimiter;
    while (peekChar() != delimiter && peekChar() != EOF) {
        if (peekChar() == '\n')
            isValid = false;
        if (peekChar() == '\\') {
            lexeme += lexEscapeSequence(isValid);
            continue;
        }

        lexeme += nextChar();
    }
    lexeme += nextChar();
    return LexResult{isValid ? Token::CHAR_CONST : Token::ERROR, lexeme};
}

void Lexer::skipLineComment()
{
    nextChar(); nextChar();
    while (peekChar() != '\n')
        nextChar();
}

void Lexer::skipBlockComment()
{
    nextChar(); // Skip the first *
    char c;
    do
        c = nextChar();
    while (!(c == '*' && peekChar() == '/'));
    nextChar(); // Pass the last /
}

LexResult Lexer::next()
{
    char character = skipWhitespace();
    while (character == '/' && peekChar() == '/') {
        skipLineComment();
        character = skipWhitespace();
    }
    while (character == '/' && peekChar() == '*') {
        skipBlockComment();
        character = skipWhitespace();
    }
    if (isdigit(character))
        return lexConstant(character);
    if (character == '\'' || character == '"')
        return lexCharConst("", character);
    if (character == 'L')
        if (peekChar() == '\'' || peekChar() == '"') {
            return lexCharConst("L", nextChar());
        }
    if (startsIdentifier(character))
        return lexIdentifier(character);
    switch (character) {
        case '+':
            if (peekWord("++"))
                return LexResult{Token::PUNCTUATOR, "++"};
            if (peekWord("+="))
                return LexResult{Token::PUNCTUATOR, "+="};
            return LexResult{Token::PUNCTUATOR, "+"};
        case '-':
            if (peekWord("->"))
                return LexResult{Token::PUNCTUATOR, "->"};
            if (peekWord("--"))
                return LexResult{Token::PUNCTUATOR, "--"};
            if (peekWord("-="))
                return LexResult{Token::PUNCTUATOR, "-="};
            return LexResult{Token::PUNCTUATOR, "-"};
        case '*':
            if (peekWord("*="))
                return LexResult{Token::PUNCTUATOR, "*="};
            return LexResult{Token::PUNCTUATOR, "*"};
        case '/':
            if (peekWord("/="))
                return LexResult{Token::PUNCTUATOR, "/="};
            return LexResult{Token::PUNCTUATOR, "/"};
        case '=':
            if (peekWord("=="))
                return LexResult{Token::PUNCTUATOR, "=="};
            return LexResult{Token::PUNCTUATOR, "="};
        case '[':
            return LexResult{Token::PUNCTUATOR, "["};
        case ']':
            return LexResult{Token::PUNCTUATOR, "]"};
        case '(':
            return LexResult{Token::PUNCTUATOR, "("};
        case ')':
            return LexResult{Token::PUNCTUATOR, ")"};
        case '.':
            if (peekWord("..."))
                return LexResult{Token::PUNCTUATOR, "..."};
            return LexResult{Token::PUNCTUATOR, "."};
        case '&':
            if (peekWord("&&"))
                return LexResult{Token::PUNCTUATOR, "&&"};
            if (peekWord("&="))
                return LexResult{Token::PUNCTUATOR, "&="};
            return LexResult{Token::PUNCTUATOR, "&"};
        case '~':
            return LexResult{Token::PUNCTUATOR, "~"};
        case '!':
            if (peekWord("!="))
                return LexResult{Token::PUNCTUATOR, "!="};
            return LexResult{Token::PUNCTUATOR, "!"};
        case '%':
            if (peekWord("%:%:"))
                return LexResult{Token::PUNCTUATOR, "%:%:"};
            if (peekWord("%:"))
                return LexResult{Token::PUNCTUATOR, "%:"};
            if (peekWord("%="))
                return LexResult{Token::PUNCTUATOR, "%="};
            if (peekWord("%>"))
               return LexResult{Token::PUNCTUATOR, "%>"};
            return LexResult{Token::PUNCTUATOR, "%"};
        case '<':
            if (peekWord("<<="))
                return LexResult{Token::PUNCTUATOR, "<<="};
            if (peekWord("<:"))
                return LexResult{Token::PUNCTUATOR, "<:"};
            if (peekWord("<%"))
                return LexResult{Token::PUNCTUATOR, "<%"};
            if (peekWord("<<"))
                return LexResult{Token::PUNCTUATOR, "<<"};
            if (peekWord("<="))
                return LexResult{Token::PUNCTUATOR, "<="};
            return LexResult{Token::PUNCTUATOR, "<"};
        case '>':
            if (peekWord(">>="))
                return LexResult{Token::PUNCTUATOR, ">>="};
            if (peekWord(">>"))
                return LexResult{Token::PUNCTUATOR, ">>"};
            if (peekWord(">="))
                return LexResult{Token::PUNCTUATOR, ">="};
            return LexResult{Token::PUNCTUATOR, ">"};
        case '^':
            if (peekWord("^="))
                return LexResult{Token::PUNCTUATOR, "^="};
            return LexResult{Token::PUNCTUATOR, "^"};
        case '|':
            if (peekWord("||"))
                return LexResult{Token::PUNCTUATOR, "||"};
            if (peekWord("|="))
                return LexResult{Token::PUNCTUATOR, "|="};
            return LexResult{Token::PUNCTUATOR, "|"};
        case '?':
            return LexResult{Token::PUNCTUATOR, "?"};
        case ':':
            if (peekWord(":>"))
                return LexResult{Token::PUNCTUATOR, ":>"};
            return LexResult{Token::PUNCTUATOR, ":"};
        case ';':
            return LexResult{Token::PUNCTUATOR, ";"};
        case ',':
            return LexResult{Token::PUNCTUATOR, ","};
        case '#':
            if (peekWord("##"))
                return LexResult{Token::PUNCTUATOR, "##"};
            return LexResult{Token::PUNCTUATOR, "#"};
        default:
            return LexResult{Token::ERROR, std::string(character, 1)};
    }
}
1
source | link

Simple C99 Lexer

I'm building a compiler for fun, and this is my first pass at the lexer for it. It should handle all tokens for C99, with the exception of the preprocessor tokens. It's very minimal, only grabbing the token and lexeme. Specific areas where I'd like feedback:

  • I rarely get to use anything beyond C++98. Are there any more recent language features I could be taking advantage of? Anything that could be more idiomatic?

  • My error-handling seems a little ad-hoc. Is there a smarter approach?

  • I'm not sure about the best way to choose tokens. The C99 standard just breaks things down into punctuators, keywords, identifiers, and constants. But the compilers I've looked at tend to be more granular. Is there a better approach to choosing tokens? Why?

  • I think there has to be a more maintainable way of structuring this.

Things I don't care about:

  • I know there are more efficient ways to handle input. My aim here is to just do the simplest thing and worry about fancy stuff/performance tuning later.

  • You probably don't like my brace style. Sorry. :/

Lexer.h

#ifndef __JMCOMP_LEXER_H__
#define __JMCOMP_LEXER_H__

#include <iostream>
#include <string>

enum class Token
{
    PUNCTUATOR,
    KEYWORD,
    IDENTIFIER,
    INT_CONST,
    FLOAT_CONST,
    CHAR_CONST,
    ERROR
};

struct LexResult
{
    Token token = Token::ERROR;
    std::string lexeme = "";
};

class Lexer
{
public:
    explicit Lexer(std::istream& stream);
    LexResult next();
private:
    std::istream& mStream;
    char nextChar();
    char skipWhitespace();
    char peekChar();
    bool peekWord(const std::string& word);
    std::string nextIdentifier(char character, bool& isValid);
    std::string lexUniversalCharacter(char character, bool& isValid);
    std::string lexHexQuad(bool& isValid);
    LexResult lexIdentifier(char character);
    LexResult lexConstant(char character);
    std::string readIntConstant(bool predicate(char));
    std::string readIntSuffix();
    std::string readLongSuffix();
    LexResult checkInvalidToken(const LexResult& token);
    LexResult readFloatConstant(const std::string& prefix,
                                bool predicate(char));
    LexResult lexCharConst(const std::string& initial, char delimiter);
    std::string lexEscapeSequence(bool& isValid);
    void skipLineComment();
    void skipBlockComment();
};

#endif

Lexer.cpp

#include <set>
#include "Lexer.h"

namespace
{
bool startsIdentifier(char character)
{
    return std::isalpha(character) ||
        character == '_' ||
        character == '\\';
}

bool isInIdentifier(char character)
{
    return startsIdentifier(character) ||
        std::isdigit(character);
}

bool isHexChar(char character)
{
    return std::isdigit(character) ||
        (character >= 'A' && character <= 'F') ||
        (character >= 'a' && character <= 'f');
}

bool isOctal(char character)
{
    return std::isdigit(character) && '8' - character > 0;
}

bool isDigit(char character)
{
    return std::isdigit(character);
}

bool isWhitespace(char character)
{
    return character == ' ' || character == '\n' || character == '\t';
}

bool isExponentChar(char character)
{
    return (character == 'e' ||
            character == 'E' ||
            character == 'p' ||
            character == 'P');
}

static std::set<std::string> keywords = {
    "auto", "break", "case", "char", "const", "continue", "default",
    "do", "double", "else", "enum", "extern", "float", "for", "goto",
    "if", "inline", "int", "long", "register", "restrict", "return",
    "short", "signed", "sizeof", "static", "struct", "switch", "typedef",
    "union", "unsigned", "void", "volatile", "while", "_Bool",
    "_Complex", "_Imaginary"
};
}

Lexer::Lexer(std::istream& stream):
    mStream(stream)
{
}

char Lexer::nextChar()
{
    char c;
    mStream.read(&c, 1);
    return c;
}

char Lexer::skipWhitespace()
{
    char c;
    do
        c = nextChar();
    while (isWhitespace(c));
    return c;
}

char Lexer::peekChar()
{
    return mStream.peek();
}

bool Lexer::peekWord(const std::string& word)
{
    std::string lexedWord;

    // We've already read the first character, so set that in advance
    lexedWord.resize(word.size());
    lexedWord[0] = word[0];

    const size_t bytesToRead = lexedWord.size() - 1;
    mStream.read(&lexedWord[1], bytesToRead);
    if (lexedWord == word) {
        return true;
    }
    else {
        mStream.seekg((-bytesToRead), std::ios::cur);
        return false;
    }
}

std::string Lexer::lexHexQuad(bool& isValid)
{
    std::string hexQuad;
    for (size_t i = 0; i < 4; ++i) {
        char character = nextChar();
        // First check if the character is okay
        if (!isHexChar(character))
            isValid = false;

        // Get a better error message (I hope?) if we keep reading
        // until a "logical" break, so no matter what keep going unless
        // there's whitespace
        if (!isWhitespace(character) && mStream)
            hexQuad += character;

        // If we hit whitespace, there's no way it's valid
        // We're at a logical boundary, so just return here
        else return hexQuad;

    }
    return hexQuad;
}

std::string Lexer::lexUniversalCharacter(char character, bool& isValid)
{
    std::string universalCharacter(1, character);

    character = nextChar();
    universalCharacter += character;
    if (character == 'u')
        universalCharacter += lexHexQuad(isValid);
    else if (character == 'U') {
        universalCharacter += lexHexQuad(isValid);
        universalCharacter += lexHexQuad(isValid);
    }
    else
        isValid = false;
    return universalCharacter;
}

std::string Lexer::nextIdentifier(char character, bool& isValid)
{
    std::string identifier;
    while (isInIdentifier(character)) {
        if (character == '\\') {
            identifier += lexUniversalCharacter(character, isValid);
            if (!isValid)
                return identifier;
        }
        else
            identifier += character;
        character = nextChar();
    }
    mStream.seekg(-1, std::ios::cur);
    return identifier;
}



LexResult Lexer::checkInvalidToken(const LexResult& token)
{
    std::string lexeme = token.lexeme;
    if (!isWhitespace(peekChar()) && peekChar() != EOF) {
        while (!isWhitespace(peekChar()) && peekChar() != EOF)
            lexeme += nextChar();
        return LexResult{Token::ERROR, lexeme};
    }
    return token;
}

LexResult Lexer::lexIdentifier(char character)
{
    bool isValid = true;
    const std::string identifier = nextIdentifier(character, isValid);
    if (!isValid)
        return LexResult{Token::ERROR, identifier};
    if (keywords.count(identifier))
        return LexResult{Token::KEYWORD, identifier};
    return LexResult{Token::IDENTIFIER, identifier};
}

std::string Lexer::readIntConstant(bool predicate(char))
{
    std::string lexeme;
    while (predicate(peekChar()))
        lexeme += nextChar();
    return lexeme;
}

std::string Lexer::readLongSuffix()
{
    std::string suffix;
    suffix += nextChar();
    if (peekChar() == 'L' || peekChar() == 'l')
        suffix += nextChar();
    return suffix;
}

std::string Lexer::readIntSuffix()
{
    std::string suffix;
    if (peekChar() == 'L' || peekChar() == 'l') {
        suffix += readLongSuffix();
        if (peekChar() == 'u' || peekChar() == 'U')
            suffix += nextChar();
    }
    else if (peekChar() == 'u' || peekChar() == 'U') {
        suffix += nextChar();
        if (peekChar() == 'l' || peekChar() == 'L')
            suffix += readLongSuffix();
    }
    return suffix;
}

LexResult Lexer::readFloatConstant(const std::string& prefix,
                                   bool predicate(char))
{
    std::string result = prefix;
    if (peekChar() == '.') {
        result += nextChar();
        result += readIntConstant(predicate);
    }
    if (isExponentChar(peekChar())) {
        result += nextChar();
        if (peekChar() == '+' || peekChar() == '-')
            result += nextChar();

        const std::string exponentSequence = readIntConstant(predicate);
        if (exponentSequence.empty())
            return checkInvalidToken(LexResult{Token::ERROR, result});

        result += exponentSequence;
    }

    const char suffix = peekChar();
    if (suffix == 'f' || suffix == 'F' ||
        suffix == 'l' || suffix == 'L')
        result += nextChar();
    return checkInvalidToken(LexResult{Token::FLOAT_CONST, result});
}

LexResult Lexer::lexConstant(char character)
{
    std::string lexeme(1, character);
    Token token = Token::INT_CONST;
    auto predicate = isDigit;

    if (peekChar() == 'x' || peekChar() == 'X') {
        predicate = isHexChar;
        lexeme += nextChar();
    }
    else if (character == '0')
        predicate = isOctal;

    lexeme += readIntConstant(predicate);

    if (peekChar() == '.' || isExponentChar(peekChar()))
        return readFloatConstant(lexeme, predicate);

    lexeme += readIntSuffix();
    return checkInvalidToken(LexResult{token, lexeme});
}

std::string Lexer::lexEscapeSequence(bool& isValid)
{
    std::string lexeme(1, nextChar());
    const char c = peekChar();

    if (c == '\'' || c == '"' || c == '?' ||
        c == '\\' || c == 'a' || c == 'b' ||
        c == 'f' || c == 'n' || c == 'r' ||
        c == 't' || c == 'v' || c == 'x' || isHexChar(c))
        return lexeme + nextChar();

    isValid = false;
    return lexeme;
}

LexResult Lexer::lexCharConst(const std::string& initial, char delimiter)
{
    bool isValid = true;
    std::string lexeme = initial + delimiter;
    while (peekChar() != delimiter && peekChar() != EOF) {
        if (peekChar() == '\n')
            isValid = false;
        if (peekChar() == '\\') {
            lexeme += lexEscapeSequence(isValid);
            continue;
        }

        lexeme += nextChar();
    }
    lexeme += nextChar();
    return LexResult{isValid ? Token::CHAR_CONST : Token::ERROR, lexeme};
}

void Lexer::skipLineComment()
{
    nextChar(); nextChar();
    while (peekChar() != '\n')
        nextChar();
}

void Lexer::skipBlockComment()
{
    nextChar(); // Skip the first *
    char c;
    do
        c = nextChar();
    while (!(c == '*' && peekChar() == '/'));
    nextChar(); // Pass the last /
}

LexResult Lexer::next()
{
    char character = skipWhitespace();
    while (character == '/' && peekChar() == '/') {
        skipLineComment();
        character = skipWhitespace();
    }
    while (character == '/' && peekChar() == '*') {
        skipBlockComment();
        character = skipWhitespace();
    }
    if (isdigit(character))
        return lexConstant(character);
    if (character == '\'' || character == '"')
        return lexCharConst("", character);
    if (character == 'L')
        if (peekChar() == '\'' || peekChar() == '"') {
            return lexCharConst("L", nextChar());
        }
    if (startsIdentifier(character))
        return lexIdentifier(character);
    switch (character) {
        case '+':
            if (peekWord("++"))
                return LexResult{Token::PUNCTUATOR, "++"};
            if (peekWord("+="))
                return LexResult{Token::PUNCTUATOR, "+="};
            return LexResult{Token::PUNCTUATOR, "+"};
        case '-':
            if (peekWord("->"))
                return LexResult{Token::PUNCTUATOR, "->"};
            if (peekWord("--"))
                return LexResult{Token::PUNCTUATOR, "--"};
            if (peekWord("-="))
                return LexResult{Token::PUNCTUATOR, "-="};
            return LexResult{Token::PUNCTUATOR, "-"};
        case '*':
            if (peekWord("*="))
                return LexResult{Token::PUNCTUATOR, "*="};
            return LexResult{Token::PUNCTUATOR, "*"};
        case '/':
            if (peekWord("/="))
                return LexResult{Token::PUNCTUATOR, "/="};
            return LexResult{Token::PUNCTUATOR, "/"};
        case '=':
            if (peekWord("=="))
                return LexResult{Token::PUNCTUATOR, "=="};
            return LexResult{Token::PUNCTUATOR, "="};
        case '[':
            return LexResult{Token::PUNCTUATOR, "["};
        case ']':
            return LexResult{Token::PUNCTUATOR, "]"};
        case '(':
            return LexResult{Token::PUNCTUATOR, "("};
        case ')':
            return LexResult{Token::PUNCTUATOR, ")"};
        case '.':
            if (peekWord("..."))
                return LexResult{Token::PUNCTUATOR, "..."};
            return LexResult{Token::PUNCTUATOR, "."};
        case '&':
            if (peekWord("&&"))
                return LexResult{Token::PUNCTUATOR, "&&"};
            if (peekWord("&="))
                return LexResult{Token::PUNCTUATOR, "&="};
            return LexResult{Token::PUNCTUATOR, "&"};
        case '~':
            return LexResult{Token::PUNCTUATOR, "~"};
        case '!':
            if (peekWord("!="))
                return LexResult{Token::PUNCTUATOR, "!="};
            return LexResult{Token::PUNCTUATOR, "!"};
        case '%':
            if (peekWord("%:%:"))
                return LexResult{Token::PUNCTUATOR, "%:%:"};
            if (peekWord("%:"))
                return LexResult{Token::PUNCTUATOR, "%:"};
            if (peekWord("%="))
                return LexResult{Token::PUNCTUATOR, "%="};
            if (peekWord("%>"))
               return LexResult{Token::PUNCTUATOR, "%>"};
            return LexResult{Token::PUNCTUATOR, "%"};
        case '<':
            if (peekWord("<<="))
                return LexResult{Token::PUNCTUATOR, "<<="};
            if (peekWord("<:"))
                return LexResult{Token::PUNCTUATOR, "<:"};
            if (peekWord("<%"))
                return LexResult{Token::PUNCTUATOR, "<%"};
            if (peekWord("<<"))
                return LexResult{Token::PUNCTUATOR, "<<"};
            if (peekWord("<="))
                return LexResult{Token::PUNCTUATOR, "<="};
            return LexResult{Token::PUNCTUATOR, "<"};
        case '>':
            if (peekWord(">>="))
                return LexResult{Token::PUNCTUATOR, ">>="};
            if (peekWord(">>"))
                return LexResult{Token::PUNCTUATOR, ">>"};
            if (peekWord(">="))
                return LexResult{Token::PUNCTUATOR, ">="};
            return LexResult{Token::PUNCTUATOR, ">"};
        case '^':
            if (peekWord("^="))
                return LexResult{Token::PUNCTUATOR, "^="};
            return LexResult{Token::PUNCTUATOR, "^"};
        case '|':
            if (peekWord("||"))
                return LexResult{Token::PUNCTUATOR, "||"};
            if (peekWord("|="))
                return LexResult{Token::PUNCTUATOR, "|="};
            return LexResult{Token::PUNCTUATOR, "|"};
        case '?':
            return LexResult{Token::PUNCTUATOR, "?"};
        case ':':
            if (peekWord(":>"))
                return LexResult{Token::PUNCTUATOR, ":>"};
            return LexResult{Token::PUNCTUATOR, ":"};
        case ';':
            return LexResult{Token::PUNCTUATOR, ";"};
        case ',':
            return LexResult{Token::PUNCTUATOR, ","};
        case '#':
            if (peekWord("##"))
                return LexResult{Token::PUNCTUATOR, "##"};
            return LexResult{Token::PUNCTUATOR, "#"};
        default:
            return LexResult{Token::ERROR, std::string(character, 1)};
    }
}