I've written a simple text tokenizer in JavaScript which works in my specific case, but I do not think it is maintainable (for example, if the requirement ever changed to allow newlines when surrounded by double quotes). How can I simplify the algorithm or abstract it better?
It should:
- strip out single-line comments (lines which begin with
#
) - discard empty and whitespace-only tokens, except in the case where they are inside double quotes
throw
an error when there is an odd number of double quotes in the input
Example input:
# a "comment" with special characters
example 1
"with multiple spaces" " kept only inside double quotes" and not outside
"side-by-side""double quotes without space in between""count as separate tokens"
Example output:
[
"example",
"1",
"with multiple spaces",
" kept only inside double quotes",
"and",
"not",
"outside",
"side-by-side",
"double quotes without space in between",
"count as separate tokens"
]
A different approach I thought of instead of going through character-by-character is to use regular expressions (comments: ^#.*
, quoted tokens: ".*"
, unquoted tokens: [^ ]+
, token separators: \s
).
const logTokens = (id) => console.log(parseTokens(id));
const parseTokens = (id) => {
const input = document.getElementById(id).value;
const tokens = [];
var tokenBuffer = "";
var insideDoubleQuote = false;
var insideComment = false;
for (const c of input) {
// single-line comments end at EOF and upon new line
if (insideComment) {
if (c === "\n") {
insideComment = false;
}
continue;
}
switch (c) {
// single-line comments start with a "#"
case "#":
// single-line comments can only occur at the start of the line (token buffer empty)
if (tokenBuffer === "") {
insideComment = true;
}
break;
case " ":
case "\n":
if (insideDoubleQuote) {
if (c === "\n") {
throw new Error("unterminated double quote");
}
// spaces inside double quotes should be kept (example: "this is a single token")
tokenBuffer += c;
break;
}
// ignore empty and whitespace-only tokens
if (tokenBuffer.trim() !== "") {
tokens.push(tokenBuffer);
}
// start next token
tokenBuffer = "";
break;
// double quotes are not part of tokens
case '"':
// upon a matching double quote, it signifies the end of a token
// this is the only case where empty and whitespace-only tokens are possible (i.e., "" and " ")
if (insideDoubleQuote) {
tokens.push(tokenBuffer);
// start next token
tokenBuffer = "";
}
insideDoubleQuote = !insideDoubleQuote;
break;
// all other characters have no special meaning and are part of tokens
default:
tokenBuffer += c;
break;
}
}
if (insideDoubleQuote) {
throw new Error("unterminated double quote");
}
// handle the case when the last line does not have a trailing newline
if (tokenBuffer.trim() !== "") {
tokens.push(tokenBuffer);
}
return tokens;
};
button,
textarea {
box-sizing: border-box;
width: 100%;
}
<textarea id="input" rows=10>
# a "comment" with special characters
example 1
"with multiple spaces" " kept only inside double quotes" and not outside
"side-by-side""double quotes without space in between""count as separate tokens"
</textarea>
<button onclick="logTokens('input');">parse</button>