From 459c7f47cda96376793bcd216564c56316d301b1 Mon Sep 17 00:00:00 2001 From: Leo dev Date: Sat, 10 Aug 2024 16:49:47 +0200 Subject: [PATCH] please help --- main.go | 11 +++--- parser/ast.go | 60 ++++++++++++++++++++++++++++++ parser/parser.go | 17 +++++++++ token/token.go | 26 ------------- tokenizer/token.go | 26 +++++++++++++ lexer.go => tokenizer/tokenizer.go | 20 +++++----- 6 files changed, 118 insertions(+), 42 deletions(-) create mode 100644 parser/ast.go create mode 100644 parser/parser.go delete mode 100644 token/token.go create mode 100644 tokenizer/token.go rename lexer.go => tokenizer/tokenizer.go (64%) diff --git a/main.go b/main.go index c94e1af..33d2dda 100644 --- a/main.go +++ b/main.go @@ -3,16 +3,17 @@ package main import ( "fmt" - "github.com/wyst-lang/wyst/token" + "github.com/wyst-lang/wyst/parser" ) func main() { - tokens, err := Tokenize("test 123", token.RULES_TOP) + code := "hello" + ast, state, err := parser.Parse(code, parser.TOP) if err != nil { - fmt.Printf("Syntax error at %s \n", err) + fmt.Printf("SyntaxErr at %d:%d: %s\n", state.Line, state.Column, err) return } - for i := 0; i < len(tokens); i++ { - fmt.Printf("%s\n", tokens[i]) + for i := 0; i < len(ast); i++ { + fmt.Printf("%s\n", ast[i]) } } diff --git a/parser/ast.go b/parser/ast.go new file mode 100644 index 0000000..4323a2c --- /dev/null +++ b/parser/ast.go @@ -0,0 +1,60 @@ +package parser + +import ( + "fmt" + + "github.com/wyst-lang/wyst/tokenizer" +) + +type inner interface { + isInner() +} + +type Rule struct { + Name string + Inner inner +} + +func (t Rule) String() string { + return fmt.Sprintf("AstRule(%s)", t.Name) +} + +type Rules struct { + Value []Rule +} + +func (v Rules) isInner() {} + +type Token struct { + Value tokenizer.Rule +} + +func (v Token) isInner() {} + +type String struct { + Value string +} + +func (v String) isInner() {} + +type AstNode struct { + Rule Rule + Value string + Inner []AstNode +} + +func (t AstNode) String() string { + return fmt.Sprintf("AstNode(\n rule=%s,\n value=%s,\n inner=%v\n)", t.Rule, t.Value, t.Inner) +} + +// Ported rules from the tokenizer +var ( + IDENTIFIER = Rule{"IDENTIFIER", Token{tokenizer.IDENTIFIER}} +) + +var ( + TOP = Rule{"TOP", Rules{ + []Rule{ + IDENTIFIER, + }}} +) diff --git a/parser/parser.go b/parser/parser.go new file mode 100644 index 0000000..fce8f93 --- /dev/null +++ b/parser/parser.go @@ -0,0 +1,17 @@ +package parser + +import ( + "github.com/wyst-lang/wyst/tokenizer" +) + +func Parse(code string, rule Rule) ([]AstNode, tokenizer.State, error) { + tokens, state, _ := tokenizer.Tokenize(code) + var ast = []AstNode{} + for i := 0; i < len(tokens); i++ { + // token := tokens[i] + if rule.Inner == (String{""}) { + + } + } + return ast, state, nil +} diff --git a/token/token.go b/token/token.go deleted file mode 100644 index 1d7aba7..0000000 --- a/token/token.go +++ /dev/null @@ -1,26 +0,0 @@ -package token - -import ( - "fmt" - "regexp" -) - -type Rule struct { - Pattern regexp.Regexp - Name string -} - -func (t Rule) String() string { - return fmt.Sprintf("TokenRule(%s)", t.Name) -} - -func NewToken(name string, pattern regexp.Regexp) Rule { - return Rule{Name: name, Pattern: pattern} -} - -var ( - NUMBER = NewToken("NUMBER", *regexp.MustCompile("^[0-9]*")) - IDENTIFIER = NewToken("IDENTIFIER", *regexp.MustCompile("^[A-Za-z_][A-Za-z_0-9]*")) -) - -var RULES_TOP = []Rule{IDENTIFIER, NUMBER} diff --git a/tokenizer/token.go b/tokenizer/token.go new file mode 100644 index 0000000..1380dcd --- /dev/null +++ b/tokenizer/token.go @@ -0,0 +1,26 @@ +package tokenizer + +import ( + "fmt" + "regexp" +) + +type Rule struct { + Pattern regexp.Regexp + Name string +} + +func (t Rule) String() string { + return fmt.Sprintf("TokenRule(%s)", t.Name) +} + +func NewRule(name string, pattern regexp.Regexp) Rule { + return Rule{Name: name, Pattern: pattern} +} + +var ( + NUMBER = NewRule("NUMBER", *regexp.MustCompile(`^\d+`)) + IDENTIFIER = NewRule("IDENTIFIER", *regexp.MustCompile("^[A-Za-z_][A-Za-z_0-9]*")) +) + +var RULES = []Rule{IDENTIFIER, NUMBER} diff --git a/lexer.go b/tokenizer/tokenizer.go similarity index 64% rename from lexer.go rename to tokenizer/tokenizer.go index dba8230..0dbae53 100644 --- a/lexer.go +++ b/tokenizer/tokenizer.go @@ -1,10 +1,8 @@ -package main +package tokenizer import ( "fmt" "strings" - - "github.com/wyst-lang/wyst/token" ) type State struct { @@ -13,7 +11,7 @@ type State struct { } type Token struct { - Rule token.Rule + Rule Rule Value string Position State } @@ -22,17 +20,17 @@ func (t Token) String() string { return fmt.Sprintf("Token(\n rule=%s,\n value=%s\n)", t.Rule, t.Value) } -func Tokenize(code string, rules []token.Rule) ([]Token, error) { +func Tokenize(code string) ([]Token, State, error) { var tokens = []Token{} var state = State{1, 0} for code != "" { matched := false - for i := 0; i < len(rules); i++ { - if rules[i].Pattern.MatchString(code) { - match := rules[i].Pattern.FindString(code) + for i := 0; i < len(RULES); i++ { + if RULES[i].Pattern.MatchString(code) { + match := RULES[i].Pattern.FindString(code) matched = true code = code[len(match):] - tokens = append(tokens, Token{Rule: rules[i], Value: match, Position: state}) + tokens = append(tokens, Token{Rule: RULES[i], Value: match, Position: state}) state.Column += len(match) break } else if strings.HasPrefix(code, "\n") { @@ -47,8 +45,8 @@ func Tokenize(code string, rules []token.Rule) ([]Token, error) { } } if !matched { - return tokens, fmt.Errorf("%d:%d", state.Line, state.Column) + return tokens, state, fmt.Errorf("invalid character or token") } } - return tokens, nil + return tokens, state, nil }