I had some spare time so I have decided to write a simple tokenizer in C# for my own programming language. The tokenizer is not finished, but it is working and I would like to hear your opinion on things I can change a bit, improve or completely modify. It is lacking FloatLiteral
token and it assumes the input is not empty, but I will add proper exception handling later. I have decided not to use Regex, as it kills all of the fun in creating this.
Language keywords
PRINT
VAR
LET
GOTO
IF
WHILE
Source Code
The source code is split in three files.
TokenType.cs
using System;
namespace Lang.Lexer
{
public enum TokenType : ushort
{
// keyword
PrintKeyword = 1, // PRINT
VarKeyword = 2, // VAR
LetKeyword = 3, // LET
GotoKeyword = 4, // GOTO
IfKeyword = 5, // IF
WhileKeyword = 6, // WHILE
// literal
IntegerLiteral = 10,
// identifier
Identifier = 20,
// operator
Assignment = 30, // =
Plus = 31, // +
Minus = 32, // -
Multiplication = 33, // *
Division = 34, // /
Modulo = 35, // %
EqualTo = 36, // ==
NotEqualTo = 37, // !=
GeaterThan = 38, // >
LessThan = 39, // <
GreaterThanOrEqualTo = 40, // >=
LessThanOrEqualTo = 41, // <=
// logical operators
LogicalNOT = 42, // !
LogicalAND = 43, // &&
LogicalOR = 44, // ||
// trivia
Space = 60,
}
}
Token.cs
namespace Lang.Lexer
{
public sealed class Token
{
public TokenType Type { get; private set; }
public string Lexeme { get; private set; }
public int CurrentLine { get; private set; }
public Token(TokenType type, string lexeme, int currentLine)
{
Type = type;
Lexeme = lexeme;
CurrentLine = currentLine;
}
}
}
Tokenizer.cs
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
namespace Lang.Lexer
{
public sealed class Tokenizer
{
readonly List<string> Lines;
int Line = 0;
int Position = 0;
bool EOF = false;
int lexemeLength = 0;
public Tokenizer(string source)
{
Lines = new List<string>(Regex.Split(source, Environment.NewLine));
}
char GetChar()
{
if (EOF) return (char)0;
char c = Lines[Line][Position];
if (Position + 1 < Lines[Line].Length)
{
Position++;
}
else
{
if (Line + 1 < Lines.Count)
{
Line++;
Position = 0;
}
else
{
EOF = true;
Position++;
}
}
return c;
}
void UngetString(int count)
{
for (int i = 0; i < count; i++)
{
UngetChar();
}
}
void UngetChar()
{
if (Position != 0)
{
if (!EOF)
{
Position--;
}
else
{
Position--;
EOF = false;
}
}
else
{
Line--;
Position = Lines[Line].Length - 1;
}
}
char PeekChar()
{
char c = GetChar();
if (c != (char)0) UngetChar();
return c;
}
public void Unget()
{
UngetString(lexemeLength);
}
public Token Peek()
{
Token token = Get();
Unget();
return token;
}
public Token Get()
{
if (EOF) return null;
TokenType type;
string lexeme = string.Empty;
if ((type = IsSpace()) != 0)
{
return new Token(type, lexeme, Line);
}
if ((type = IsOperator()) != 0)
{
return new Token(type, lexeme, Line);
}
if ((type = IsKeyword()) != 0)
{
return new Token(type, lexeme, Line);
}
Tuple<TokenType, String> identifier = IsIdentifier();
if (identifier.Item1 != 0)
{
return new Token(TokenType.Identifier, identifier.Item2, Line);
}
Tuple<TokenType, String> integerLiteral = IsIntegerLiteral();
if (integerLiteral.Item1 != 0)
{
return new Token(TokenType.IntegerLiteral, integerLiteral.Item2, Line);
}
//bad token
return null;
}
Tuple<TokenType, String> IsIntegerLiteral()
{
if (!char.IsDigit(PeekChar()))
return new Tuple<TokenType, string>(0, string.Empty);
string lexeme = GetChar().ToString();
int count = 1;
int line = Line;
while (char.IsDigit(PeekChar()))
{
lexeme = lexeme + GetChar();
count++;
if (line != Line)
{
UngetString(count);
return new Tuple<TokenType, string>(0, string.Empty);
}
}
lexemeLength = count;
return new Tuple<TokenType, string>(TokenType.Identifier, lexeme);
}
TokenType IsKeyword()
{
if (!char.IsLetter(PeekChar())) return 0;
string lexeme = GetChar().ToString();
int count = 1;
int line = Line;
while (char.IsLetter(PeekChar()))
{
lexeme = lexeme + GetChar();
count++;
if (line != Line) break;
}
switch (lexeme.ToUpper())
{
case "PRINT":
{
lexemeLength = count;
return TokenType.PrintKeyword;
}
case "VAR":
{
lexemeLength = count;
return TokenType.VarKeyword;
}
case "LET":
{
lexemeLength = count;
return TokenType.LetKeyword;
}
case "GOTO":
{
lexemeLength = count;
return TokenType.GotoKeyword;
}
case "IF":
{
lexemeLength = count;
return TokenType.IfKeyword;
}
case "WHILE":
{
lexemeLength = count;
return TokenType.WhileKeyword;
}
}
UngetString(count);
return 0;
}
Tuple<TokenType, String> IsIdentifier()
{
if (!(char.IsLetter(PeekChar()) || PeekChar() == '_'))
return new Tuple<TokenType, string>(0, string.Empty);
string lexeme = GetChar().ToString();
int count = 1;
int line = Line;
while ((char.IsLetter(PeekChar()) || char.IsDigit(PeekChar()) || PeekChar() == '_'))
{
lexeme = lexeme + GetChar();
count++;
if (line != Line)
{
UngetString(count);
return new Tuple<TokenType, string>(0, string.Empty);
}
}
lexemeLength = count;
return new Tuple<TokenType, string>(TokenType.Identifier, lexeme);
}
TokenType IsSpace()
{
if (char.IsWhiteSpace(PeekChar()))
{
GetChar();
lexemeLength = 1;
return TokenType.Space;
}
return 0;
}
TokenType IsOperator()
{
char c = PeekChar();
switch (c)
{
case '=':
{
GetChar();
if (PeekChar() == '=')
{
GetChar();
lexemeLength = 2;
return TokenType.EqualTo;
}
lexemeLength = 1;
return TokenType.Assignment;
}
case '+':
{
GetChar();
lexemeLength = 1;
return TokenType.Plus;
}
case '-':
{
GetChar();
lexemeLength = 1;
return TokenType.Minus;
}
case '*':
{
GetChar();
lexemeLength = 1;
return TokenType.Multiplication;
}
case '/':
{
GetChar();
lexemeLength = 1;
return TokenType.Division;
}
case '%':
{
GetChar();
lexemeLength = 1;
return TokenType.Modulo;
}
case '!':
{
GetChar();
if (PeekChar() == '=')
{
GetChar();
lexemeLength = 2;
return TokenType.NotEqualTo;
}
lexemeLength = 1;
return TokenType.LogicalNOT;
}
case '>':
{
GetChar();
if (PeekChar() == '=')
{
GetChar();
lexemeLength = 2;
return TokenType.GreaterThanOrEqualTo;
}
lexemeLength = 1;
return TokenType.GeaterThan;
}
case '<':
{
GetChar();
if (PeekChar() == '=')
{
GetChar();
lexemeLength = 2;
return TokenType.LessThanOrEqualTo;
}
lexemeLength = 1;
return TokenType.LessThan;
}
case '&':
{
GetChar();
if (PeekChar() == '&')
{
GetChar();
lexemeLength = 2;
return TokenType.LogicalAND;
}
lexemeLength = 1;
return 0;
}
case '|':
{
GetChar();
if (PeekChar() == '|')
{
GetChar();
lexemeLength = 2;
return TokenType.LogicalOR;
}
lexemeLength = 1;
return 0;
}
default:
{
return 0;
}
}
}
}
}