fix: minor bugs && example

lexer/lexer.go

@@ -1,7 +1,7 @@
 package lexer
 
 import (
-	//"fmt"
+	"fmt"
 	"regexp"
 )
 
@@ -23,7 +23,9 @@ func createLexer(source string) *lexer {
 		source:   source,
 		Tokens:   make([]Token, 0),
 		patterns: []regexPattern{
-			{regexp.MustCompile(`\s+`), defaultHandler(SPACE, " ")},
+			{regexp.MustCompile(`\s+`), skipHandler},
+			{regexp.MustCompile(`\+`), defaultHandler(PLUS, "+")},
+			{regexp.MustCompile(`\-`), defaultHandler(MINUS, "-")},
 			{regexp.MustCompile(`\:`), defaultHandler(COLON, ":")},
 			{regexp.MustCompile(`>=`), defaultHandler(MORE_EQUAL, ">=")},
 			{regexp.MustCompile(`>`), defaultHandler(MORE, ">")},
@@ -111,3 +113,27 @@ func commentHandler(lex *lexer, regex *regexp.Regexp) {
 		lex.incrementPosition(match[1])
 	}
 }
+
+func Tokenize(source string) []Token {
+	lex := createLexer(source)
+
+	for !lex.atEof() {
+		matched := false
+
+		for _, pattern := range lex.patterns {
+			location := pattern.regex.FindStringIndex(lex.currentString())
+			if location != nil && location[0] == 0 {
+				pattern.handler(lex, pattern.regex)
+				matched = true
+				break
+			}
+		}
+
+		if !matched {
+			panic(fmt.Sprintf("lexer error: unrecognized token near '%v'", lex.currentString()))
+		}
+	}
+
+	lex.push(Token{EOF, "EOF"})
+	return lex.Tokens
+}