diff --git a/main.go b/main.go index c685d41..2d3a6db 100644 --- a/main.go +++ b/main.go @@ -7,11 +7,10 @@ import ( ) func main() { - code := "int x" - ast, state, err := parser.Parse(code, parser.RULE_SET) + code := "int x;" + ast, err := parser.ParseString(code) if err != nil { - fmt.Printf("SyntaxErr at %d:%d: %s\n", state.Line, state.Column, err) - return + fmt.Printf("SyntaxErr: %s\n", err) } for i := 0; i < len(ast); i++ { fmt.Printf("%s\n", ast[i]) diff --git a/parser/ast.go b/parser/ast.go index a580a02..659f456 100644 --- a/parser/ast.go +++ b/parser/ast.go @@ -4,32 +4,62 @@ import ( "fmt" ) +type RuleKind int + +const ( + RK_Required RuleKind = iota + RK_Optional + RK_Repeat + RK_Port +) + type Rule struct { Name string + Kind RuleKind Inner []Rule } type AstNode struct { Rule Rule + Value string Inner []AstNode } func (t AstNode) String() string { - return fmt.Sprintf("AstNode(\n rule=%v,\n inner=%v,\n)", t.Rule, t.Inner) + return fmt.Sprintf("AstNode(\n value=%s\n rule=%v,\n inner=%v,\n)", t.Value, t.Rule, t.Inner) } func (t Rule) String() string { return t.Name } func NewRule(name string, inner []Rule) Rule { - return Rule{name, inner} + return Rule{name, RK_Required, inner} +} + +func NewPort(name string, inner []Rule) Rule { + return Rule{name, RK_Port, inner} +} + +func Optional(rule Rule) Rule { + rule.Kind = RK_Optional + return rule +} + +func Repeat(rule Rule) Rule { + rule.Kind = RK_Repeat + return rule } var ( - IDENTIFIER = NewRule("IDENTIFIER", []Rule{}) + IDENTIFIER = NewPort("IDENTIFIER", []Rule{}) + NUMBER = NewPort("NUMBER", []Rule{}) + SEMICOLON = NewPort("SEMICOLON", []Rule{}) + EXPR = NewRule("EXPR", []Rule{IDENTIFIER}) + CODE_BLOCK = NewRule("CODE_BLOCK", []Rule{EXPR, SEMICOLON}) VAR_DEF = NewRule("VAR_DEF", []Rule{IDENTIFIER, IDENTIFIER}) + FUNC_DEF = NewRule("FUNC_DEF", []Rule{IDENTIFIER, IDENTIFIER, CODE_BLOCK}) ) var ( - RULE_SET = []Rule{IDENTIFIER, VAR_DEF} + TOP_RULE = []Rule{FUNC_DEF} ) diff --git a/parser/parser.go b/parser/parser.go index 57d42ab..25407b0 100644 --- a/parser/parser.go +++ b/parser/parser.go @@ -1,34 +1,71 @@ package parser import ( + "fmt" + "github.com/wyst-lang/wyst/tokenizer" ) -func Parse(code string, rule_set []Rule) ([]AstNode, tokenizer.State, error) { - var ast = []AstNode{} +func ParseString(code string) ([]AstNode, error) { tokens, state, err := tokenizer.Tokenize(code) + ast, err1 := Parse(tokens, TOP_RULE) + if err != nil { + return ast, fmt.Errorf("LexingError at %d:%d: %s", state.Line, state.Column, err) + } else if err1 != nil { + return ast, fmt.Errorf("ParserError") + } + return ast, nil +} + +func Parse(tokens []tokenizer.Token, rule_set []Rule) ([]AstNode, error) { + var ast = []AstNode{} for i := 0; i < len(tokens); i++ { for r := 0; r < len(rule_set); r++ { matching := false var match_nodes = []AstNode{} - for v := 0; v < len(rule_set[r].Inner); v++ { - if len(tokens)-i > v && tokens[i].Rule.Name == rule_set[r].Inner[v].Name { + if rule_set[r].Kind == RK_Port { + if tokens[i].Rule.Name == rule_set[r].Name { matching = true - match_nodes = append(match_nodes, AstNode{Rule: rule_set[r].Inner[v]}) - } else { - matching = false - break + match_nodes = append(match_nodes, AstNode{Rule: rule_set[r], Value: tokens[i].Value}) + } + } else { + for v := 0; v < len(rule_set[r].Inner); v++ { + if rule_set[r].Inner[v].Kind == RK_Required { + if len(tokens)-i > v && tokens[i].Rule.Name == rule_set[r].Inner[v].Name { + matching = true + match_nodes = append(match_nodes, AstNode{Rule: rule_set[r].Inner[v], Value: tokens[i].Value}) + } else { + matching = false + break + } + } else if rule_set[r].Inner[v].Kind == RK_Repeat { + for l := 0; len(tokens)-i > l && tokens[i+l].Rule.Name == rule_set[r].Inner[v].Name; l++ { + match_nodes = append(match_nodes, AstNode{Rule: rule_set[r].Inner[v], Value: tokens[i+l].Value}) + matching = true + } + } else if rule_set[r].Inner[v].Kind == RK_Optional { + match_nodes = append(match_nodes, AstNode{Rule: rule_set[r].Inner[v], Value: tokens[i].Value}) + matching = true + } } } if matching { - ast = append(ast, AstNode{Rule: rule_set[r], Inner: match_nodes}) + ast = append(ast, AstNode{Rule: rule_set[r], Inner: match_nodes, Value: merge_nodes(match_nodes)}) i += len(match_nodes) - 1 break } } } - if err != nil { - return ast, state, err + return ast, nil +} + +func merge_nodes(match_nodes []AstNode) string { + str := "" + for i := 0; i < len(match_nodes); i++ { + str += match_nodes[i].Value + if len(match_nodes) > i { + str += " " + } } - return ast, state, nil + return str } diff --git a/tokenizer/token.go b/tokenizer/token.go index 1380dcd..3caf7c3 100644 --- a/tokenizer/token.go +++ b/tokenizer/token.go @@ -14,13 +14,16 @@ func (t Rule) String() string { return fmt.Sprintf("TokenRule(%s)", t.Name) } +var RULES = []Rule{} + func NewRule(name string, pattern regexp.Regexp) Rule { - return Rule{Name: name, Pattern: pattern} + var rule = Rule{Name: name, Pattern: pattern} + RULES = append(RULES, rule) + return rule } var ( NUMBER = NewRule("NUMBER", *regexp.MustCompile(`^\d+`)) IDENTIFIER = NewRule("IDENTIFIER", *regexp.MustCompile("^[A-Za-z_][A-Za-z_0-9]*")) + SEMICOLON = NewRule("SEMICOLON", *regexp.MustCompile("^;")) ) - -var RULES = []Rule{IDENTIFIER, NUMBER} diff --git a/tokenizer/tokenizer.go b/tokenizer/tokenizer.go index 0dbae53..3075aff 100644 --- a/tokenizer/tokenizer.go +++ b/tokenizer/tokenizer.go @@ -23,6 +23,7 @@ func (t Token) String() string { func Tokenize(code string) ([]Token, State, error) { var tokens = []Token{} var state = State{1, 0} + var err error = nil for code != "" { matched := false for i := 0; i < len(RULES); i++ { @@ -45,8 +46,9 @@ func Tokenize(code string) ([]Token, State, error) { } } if !matched { - return tokens, state, fmt.Errorf("invalid character or token") + code = code[1:] + err = fmt.Errorf("invalid character or token") } } - return tokens, state, nil + return tokens, state, err }