package main
import (
"regexp"
"strconv"
"strings"
"fmt"
)
// Is it best practice to enumerate string constants like this?
const (
Plus = "Plus"
Minus = "Minus"
Number = "Number"
Skip = "Skip"
Newline = "Newline"
)
// Token struct for keeping track of tokens
type Token struct {
Type string
// How to best signify that a value could be a string or an int?
Value interface{}
Line int
Column int
}
// Tokenize converts a string into a slice of tokens
func Tokenize(text string) []Token {
// Is this mapping a good way of keeping track of token patterns?
groupNamesPatterns := map[string]string{
Plus: `\+`,
Minus: `-`,
Number: `[\d]+`,
Skip: `[ \t]`,
Newline: `\n`,
}
// Create the Regex pattern with all the named groups
var patternStrings []string
for groupName, pattern := range groupNamesPatterns {
groupPattern := fmt.Sprintf(`(?P<%s>%s)`, groupName, pattern)
patternStrings = append(patternStrings, groupPattern)
}
pattern := regexp.MustCompile(strings.Join(patternStrings, "|"))
groupNames := pattern.SubexpNames()
// Create the tokens list
var tokens []Token
line := 1
column := 0
matches := pattern.FindAllStringSubmatch(text, -1)
for _, m := range matches {
// Iterate through match group names to find the text namedname of the
// matched group
for i, matchedText := range m[1:] {
if matchedText != "" {
groupName := groupNames[i+1]
if groupName == Newline {
line += 1
column = 0
break
}
if groupName != Skip {
t := Token{groupName, "", line, column}
// Set the value of the token to either the parsed string
// or raw string
if parsed, err := strconv.Atoi(matchedText); err != nil {
t.Value = matchedText
} else {
t.Value = parsed
}
tokens = append(tokens, t)
}
column += len(matchedText)
break
}
}
}
return tokens
}
func main() {
// Example of tokenizing string spanning multiple lines
text := `
1
+ 2
- 3`
tokens := Tokenize(text)
for _, token := range tokens {
fmt.Printf("%#v\n", token)
}
}
package main
import (
"regexp"
"strconv"
"strings"
"fmt"
)
// Is it best practice to enumerate string constants like this?
const (
Plus = "Plus"
Minus = "Minus"
Number = "Number"
Skip = "Skip"
Newline = "Newline"
)
// Token struct for keeping track of tokens
type Token struct {
Type string
// How to best signify that a value could be a string or an int?
Value interface{}
Line int
Column int
}
// Tokenize converts a string into a slice of tokens
func Tokenize(text string) []Token {
// Is this mapping a good way of keeping track of token patterns?
groupNamesPatterns := map[string]string{
Plus: `\+`,
Minus: `-`,
Number: `[\d]+`,
Skip: `[ \t]`,
Newline: `\n`,
}
// Create the Regex pattern with all the named groups
var patternStrings []string
for groupName, pattern := range groupNamesPatterns {
groupPattern := fmt.Sprintf(`(?P<%s>%s)`, groupName, pattern)
patternStrings = append(patternStrings, groupPattern)
}
pattern := regexp.MustCompile(strings.Join(patternStrings, "|"))
groupNames := pattern.SubexpNames()
// Create the tokens list
var tokens []Token
line := 1
column := 0
matches := pattern.FindAllStringSubmatch(text, -1)
for _, m := range matches {
// Iterate through match group names to find the text named of the
// matched group
for i, matchedText := range m[1:] {
if matchedText != "" {
groupName := groupNames[i+1]
if groupName == Newline {
line += 1
column = 0
break
}
if groupName != Skip {
t := Token{groupName, "", line, column}
// Set the value of the token to either the parsed string
// or raw string
if parsed, err := strconv.Atoi(matchedText); err != nil {
t.Value = matchedText
} else {
t.Value = parsed
}
tokens = append(tokens, t)
}
column += len(matchedText)
break
}
}
}
return tokens
}
func main() {
// Example of tokenizing string spanning multiple lines
text := `
1
+ 2
- 3`
tokens := Tokenize(text)
for _, token := range tokens {
fmt.Printf("%#v\n", token)
}
}
package main
import (
"regexp"
"strconv"
"strings"
"fmt"
)
// Is it best practice to enumerate string constants like this?
const (
Plus = "Plus"
Minus = "Minus"
Number = "Number"
Skip = "Skip"
Newline = "Newline"
)
// Token struct for keeping track of tokens
type Token struct {
Type string
// How to best signify that a value could be a string or an int?
Value interface{}
Line int
Column int
}
// Tokenize converts a string into a slice of tokens
func Tokenize(text string) []Token {
// Is this mapping a good way of keeping track of token patterns?
groupNamesPatterns := map[string]string{
Plus: `\+`,
Minus: `-`,
Number: `[\d]+`,
Skip: `[ \t]`,
Newline: `\n`,
}
// Create the Regex pattern with all the named groups
var patternStrings []string
for groupName, pattern := range groupNamesPatterns {
groupPattern := fmt.Sprintf(`(?P<%s>%s)`, groupName, pattern)
patternStrings = append(patternStrings, groupPattern)
}
pattern := regexp.MustCompile(strings.Join(patternStrings, "|"))
groupNames := pattern.SubexpNames()
// Create the tokens list
var tokens []Token
line := 1
column := 0
matches := pattern.FindAllStringSubmatch(text, -1)
for _, m := range matches {
// Iterate through match group names to find the name of the
// matched group
for i, matchedText := range m[1:] {
if matchedText != "" {
groupName := groupNames[i+1]
if groupName == Newline {
line += 1
column = 0
break
}
if groupName != Skip {
t := Token{groupName, "", line, column}
// Set the value of the token to either the parsed string
// or raw string
if parsed, err := strconv.Atoi(matchedText); err != nil {
t.Value = matchedText
} else {
t.Value = parsed
}
tokens = append(tokens, t)
}
column += len(matchedText)
break
}
}
}
return tokens
}
func main() {
// Example of tokenizing string spanning multiple lines
text := `
1
+ 2
- 3`
tokens := Tokenize(text)
for _, token := range tokens {
fmt.Printf("%#v\n", token)
}
}
Idiomatic Go Lexer
I have written some Go code that produces the output I would like, but I am unsure if the code itself "smells" good. I would like to know if it contains any anti-patterns, exemplifies bad practices, or lacks the proper idioms one would expect to see in a program written in Golang. Details of the program follow.
I have recently started programming in Go, and have been looking for ways to practice my skills. I come from a Python background and lately have been reading up on compiler design. So in an attempt to practice both of these nascent interests, this morning I wrote a bit of Go code to lex a string into a series of tokens. The lexer recognizes numbers, common whitespace characters, and the binary operators for addition and subtraction.
I based this lexer on an example of creating a tokenizer I found in the Python documentation for the re (regex) library. Thus my code uses the regexp package of the Go standard library. I plan on segmenting the code into separate files / packages, but for now I have it all in one file for readability.
The code itself is below. Specific questions are contained in the comments, but please share any issues you find with the code. I am liking the imperative and "simple" style of Go, and would like to make sure I do not start any poor coding habits when writing Go code in the future.
Here is a Go Playground link to the code as well: https://play.golang.org/p/jfuWRyOipMM
package main
import (
"regexp"
"strconv"
"strings"
"fmt"
)
// Is it best practice to enumerate string constants like this?
const (
Plus = "Plus"
Minus = "Minus"
Number = "Number"
Skip = "Skip"
Newline = "Newline"
)
// Token struct for keeping track of tokens
type Token struct {
Type string
// How to best signify that a value could be a string or an int?
Value interface{}
Line int
Column int
}
// Tokenize converts a string into a slice of tokens
func Tokenize(text string) []Token {
// Is this mapping a good way of keeping track of token patterns?
groupNamesPatterns := map[string]string{
Plus: `\+`,
Minus: `-`,
Number: `[\d]+`,
Skip: `[ \t]`,
Newline: `\n`,
}
// Create the Regex pattern with all the named groups
var patternStrings []string
for groupName, pattern := range groupNamesPatterns {
groupPattern := fmt.Sprintf(`(?P<%s>%s)`, groupName, pattern)
patternStrings = append(patternStrings, groupPattern)
}
pattern := regexp.MustCompile(strings.Join(patternStrings, "|"))
groupNames := pattern.SubexpNames()
// Create the tokens list
var tokens []Token
line := 1
column := 0
matches := pattern.FindAllStringSubmatch(text, -1)
for _, m := range matches {
// Iterate through match group names to find the text named of the
// matched group
for i, matchedText := range m[1:] {
if matchedText != "" {
groupName := groupNames[i+1]
if groupName == Newline {
line += 1
column = 0
break
}
if groupName != Skip {
t := Token{groupName, "", line, column}
// Set the value of the token to either the parsed string
// or raw string
if parsed, err := strconv.Atoi(matchedText); err != nil {
t.Value = matchedText
} else {
t.Value = parsed
}
tokens = append(tokens, t)
}
column += len(matchedText)
break
}
}
}
return tokens
}
func main() {
// Example of tokenizing string spanning multiple lines
text := `
1
+ 2
- 3`
tokens := Tokenize(text)
for _, token := range tokens {
fmt.Printf("%#v\n", token)
}
}
lang-golang