parser (still doesn't work)

tokenizer/token.go

@@ -14,13 +14,16 @@ func (t Rule) String() string {
 	return fmt.Sprintf("TokenRule(%s)", t.Name)
 }
 
+var RULES = []Rule{}
+
 func NewRule(name string, pattern regexp.Regexp) Rule {
-	return Rule{Name: name, Pattern: pattern}
+	var rule = Rule{Name: name, Pattern: pattern}
+	RULES = append(RULES, rule)
+	return rule
 }
 
 var (
 	NUMBER     = NewRule("NUMBER", *regexp.MustCompile(`^\d+`))
 	IDENTIFIER = NewRule("IDENTIFIER", *regexp.MustCompile("^[A-Za-z_][A-Za-z_0-9]*"))
+	SEMICOLON  = NewRule("SEMICOLON", *regexp.MustCompile("^;"))
 )
-
-var RULES = []Rule{IDENTIFIER, NUMBER}