fix: minor bugs && example

lexer/tokens.go

@@ -1,5 +1,7 @@
 package lexer
 
+import "fmt"
+
 // что есть в запросе?
 // строковые литералы
 // двоеточия
@@ -17,7 +19,8 @@ package lexer
 type TokenType int
 
 const (
-	SYMBOL TokenType = iota
+	EOF TokenType = iota
+	SYMBOL
 	COLON
 	EXCLAMATION
 	EQUAL
@@ -31,6 +34,8 @@ const (
 	LESS_EQUAL
 	OPEN_BRACE
 	CLOSED_BRACE
+	MINUS
+	PLUS
 	COMMENT
 	PIPE
 	NUMBER
@@ -44,6 +49,73 @@ type Token struct {
 	value     string
 }
 
-func Parse(str string) {
-
+func TokenKindString(kind TokenType) string {
+	switch kind {
+	case EOF:
+		return "eof"
+	case SYMBOL:
+		return "symbol"
+	case NUMBER:
+		return "number"
+	case STRING_LITERAL:
+		return "string"
+	case FLOAT_NUMBER:
+		return "float"
+	case OPEN_BRACE:
+		return "open_paren"
+	case CLOSED_BRACE:
+		return "close_paren"
+	case EQUAL:
+		return "equals"
+	case NOT_EQUAL:
+		return "not_equals"
+	case NOT:
+		return "not"
+	case OR:
+		return "or"
+	case AND:
+		return "and"
+	case COLON:
+		return "colon"
+	case EXCLAMATION:
+		return "exclamation"
+	case MORE:
+		return "more"
+	case LESS:
+		return "less"
+	case MORE_EQUAL:
+		return "more_equal"
+	case LESS_EQUAL:
+		return "less_equal"
+	case COMMENT:
+		return "comment"
+	case PIPE:
+		return "pipe"
+	case MINUS:
+		return "minus"
+	case PLUS:
+		return "plus"
+	case SPACE:
+		return "space"
+	default:
+		return fmt.Sprintf("unknown(%d)", kind)
+	}
+}
+
+func (tk Token) IsOneOfMany(expectedTokens ...TokenType) bool {
+	for _, expected := range expectedTokens {
+		if expected == tk.tokenType {
+			return true
+		}
+	}
+
+	return false
+}
+
+func Debug(token Token) {
+	if token.tokenType == SYMBOL || token.tokenType == NUMBER || token.tokenType == STRING_LITERAL {
+		fmt.Printf("%s(%s)\n", TokenKindString(token.tokenType), token.value)
+	} else {
+		fmt.Printf("%s()\n", TokenKindString(token.tokenType))
+	}
 }