parser (still doesn't work)

tokenizer/token.go

@@ -14,13 +14,16 @@ func (t Rule) String() string {
 	return fmt.Sprintf("TokenRule(%s)", t.Name)
 }
 
+var RULES = []Rule{}
+
 func NewRule(name string, pattern regexp.Regexp) Rule {
-	return Rule{Name: name, Pattern: pattern}
+	var rule = Rule{Name: name, Pattern: pattern}
+	RULES = append(RULES, rule)
+	return rule
 }
 
 var (
 	NUMBER     = NewRule("NUMBER", *regexp.MustCompile(`^\d+`))
 	IDENTIFIER = NewRule("IDENTIFIER", *regexp.MustCompile("^[A-Za-z_][A-Za-z_0-9]*"))
+	SEMICOLON  = NewRule("SEMICOLON", *regexp.MustCompile("^;"))
 )
-
-var RULES = []Rule{IDENTIFIER, NUMBER}

tokenizer/tokenizer.go

@@ -23,6 +23,7 @@ func (t Token) String() string {
 func Tokenize(code string) ([]Token, State, error) {
 	var tokens = []Token{}
 	var state = State{1, 0}
+	var err error = nil
 	for code != "" {
 		matched := false
 		for i := 0; i < len(RULES); i++ {
@@ -45,8 +46,9 @@ func Tokenize(code string) ([]Token, State, error) {
 			}
 		}
 		if !matched {
-			return tokens, state, fmt.Errorf("invalid character or token")
+			code = code[1:]
+			err = fmt.Errorf("invalid character or token")
 		}
 	}
-	return tokens, state, nil
+	return tokens, state, err
 }