parser (still doesn't work)

main.go

@@ -7,11 +7,10 @@ import (
 )
 
 func main() {
-	code := "int x"
-	ast, state, err := parser.Parse(code, parser.RULE_SET)
+	code := "int x;"
+	ast, err := parser.ParseString(code)
 	if err != nil {
-		fmt.Printf("SyntaxErr at %d:%d: %s\n", state.Line, state.Column, err)
-		return
+		fmt.Printf("SyntaxErr: %s\n", err)
 	}
 	for i := 0; i < len(ast); i++ {
 		fmt.Printf("%s\n", ast[i])

parser/ast.go

@@ -4,32 +4,62 @@ import (
 	"fmt"
 )
 
+type RuleKind int
+
+const (
+	RK_Required RuleKind = iota
+	RK_Optional
+	RK_Repeat
+	RK_Port
+)
+
 type Rule struct {
 	Name  string
+	Kind  RuleKind
 	Inner []Rule
 }
 
 type AstNode struct {
 	Rule  Rule
+	Value string
 	Inner []AstNode
 }
 
 func (t AstNode) String() string {
-	return fmt.Sprintf("AstNode(\n  rule=%v,\n  inner=%v,\n)", t.Rule, t.Inner)
+	return fmt.Sprintf("AstNode(\n  value=%s\n  rule=%v,\n  inner=%v,\n)", t.Value, t.Rule, t.Inner)
 }
 func (t Rule) String() string {
 	return t.Name
 }
 
 func NewRule(name string, inner []Rule) Rule {
-	return Rule{name, inner}
+	return Rule{name, RK_Required, inner}
+}
+
+func NewPort(name string, inner []Rule) Rule {
+	return Rule{name, RK_Port, inner}
+}
+
+func Optional(rule Rule) Rule {
+	rule.Kind = RK_Optional
+	return rule
+}
+
+func Repeat(rule Rule) Rule {
+	rule.Kind = RK_Repeat
+	return rule
 }
 
 var (
-	IDENTIFIER = NewRule("IDENTIFIER", []Rule{})
+	IDENTIFIER = NewPort("IDENTIFIER", []Rule{})
+	NUMBER     = NewPort("NUMBER", []Rule{})
+	SEMICOLON  = NewPort("SEMICOLON", []Rule{})
+	EXPR       = NewRule("EXPR", []Rule{IDENTIFIER})
+	CODE_BLOCK = NewRule("CODE_BLOCK", []Rule{EXPR, SEMICOLON})
 	VAR_DEF    = NewRule("VAR_DEF", []Rule{IDENTIFIER, IDENTIFIER})
+	FUNC_DEF   = NewRule("FUNC_DEF", []Rule{IDENTIFIER, IDENTIFIER, CODE_BLOCK})
 )
 
 var (
-	RULE_SET = []Rule{IDENTIFIER, VAR_DEF}
+	TOP_RULE = []Rule{FUNC_DEF}
 )

parser/parser.go

@@ -1,34 +1,71 @@
 package parser
 
 import (
+	"fmt"
+
 	"github.com/wyst-lang/wyst/tokenizer"
 )
 
-func Parse(code string, rule_set []Rule) ([]AstNode, tokenizer.State, error) {
-	var ast = []AstNode{}
+func ParseString(code string) ([]AstNode, error) {
 	tokens, state, err := tokenizer.Tokenize(code)
+	ast, err1 := Parse(tokens, TOP_RULE)
+	if err != nil {
+		return ast, fmt.Errorf("LexingError at %d:%d: %s", state.Line, state.Column, err)
+	} else if err1 != nil {
+		return ast, fmt.Errorf("ParserError")
+	}
+	return ast, nil
+}
+
+func Parse(tokens []tokenizer.Token, rule_set []Rule) ([]AstNode, error) {
+	var ast = []AstNode{}
 	for i := 0; i < len(tokens); i++ {
 		for r := 0; r < len(rule_set); r++ {
 			matching := false
 			var match_nodes = []AstNode{}
+			if rule_set[r].Kind == RK_Port {
+				if tokens[i].Rule.Name == rule_set[r].Name {
+					matching = true
+					match_nodes = append(match_nodes, AstNode{Rule: rule_set[r], Value: tokens[i].Value})
+				}
+			} else {
 				for v := 0; v < len(rule_set[r].Inner); v++ {
+					if rule_set[r].Inner[v].Kind == RK_Required {
 						if len(tokens)-i > v && tokens[i].Rule.Name == rule_set[r].Inner[v].Name {
 							matching = true
-					match_nodes = append(match_nodes, AstNode{Rule: rule_set[r].Inner[v]})
+							match_nodes = append(match_nodes, AstNode{Rule: rule_set[r].Inner[v], Value: tokens[i].Value})
 						} else {
 							matching = false
 							break
 						}
+					} else if rule_set[r].Inner[v].Kind == RK_Repeat {
+						for l := 0; len(tokens)-i > l && tokens[i+l].Rule.Name == rule_set[r].Inner[v].Name; l++ {
+							match_nodes = append(match_nodes, AstNode{Rule: rule_set[r].Inner[v], Value: tokens[i+l].Value})
+							matching = true
+						}
+					} else if rule_set[r].Inner[v].Kind == RK_Optional {
+						match_nodes = append(match_nodes, AstNode{Rule: rule_set[r].Inner[v], Value: tokens[i].Value})
+						matching = true
+					}
+				}
 			}
 			if matching {
-				ast = append(ast, AstNode{Rule: rule_set[r], Inner: match_nodes})
+				ast = append(ast, AstNode{Rule: rule_set[r], Inner: match_nodes, Value: merge_nodes(match_nodes)})
 				i += len(match_nodes) - 1
 				break
 			}
 		}
 	}
-	if err != nil {
-		return ast, state, err
+	return ast, nil
 }
-	return ast, state, nil
+
+func merge_nodes(match_nodes []AstNode) string {
+	str := ""
+	for i := 0; i < len(match_nodes); i++ {
+		str += match_nodes[i].Value
+		if len(match_nodes) > i {
+			str += " "
+		}
+	}
+	return str
 }

tokenizer/token.go

@@ -14,13 +14,16 @@ func (t Rule) String() string {
 	return fmt.Sprintf("TokenRule(%s)", t.Name)
 }
 
+var RULES = []Rule{}
+
 func NewRule(name string, pattern regexp.Regexp) Rule {
-	return Rule{Name: name, Pattern: pattern}
+	var rule = Rule{Name: name, Pattern: pattern}
+	RULES = append(RULES, rule)
+	return rule
 }
 
 var (
 	NUMBER     = NewRule("NUMBER", *regexp.MustCompile(`^\d+`))
 	IDENTIFIER = NewRule("IDENTIFIER", *regexp.MustCompile("^[A-Za-z_][A-Za-z_0-9]*"))
+	SEMICOLON  = NewRule("SEMICOLON", *regexp.MustCompile("^;"))
 )
-
-var RULES = []Rule{IDENTIFIER, NUMBER}

tokenizer/tokenizer.go

@@ -23,6 +23,7 @@ func (t Token) String() string {
 func Tokenize(code string) ([]Token, State, error) {
 	var tokens = []Token{}
 	var state = State{1, 0}
+	var err error = nil
 	for code != "" {
 		matched := false
 		for i := 0; i < len(RULES); i++ {
@@ -45,8 +46,9 @@ func Tokenize(code string) ([]Token, State, error) {
 			}
 		}
 		if !matched {
-			return tokens, state, fmt.Errorf("invalid character or token")
+			code = code[1:]
+			err = fmt.Errorf("invalid character or token")
 		}
 	}
-	return tokens, state, nil
+	return tokens, state, err
 }