please help

main.go

@@ -3,16 +3,17 @@ package main
 import (
 	"fmt"
 
-	"github.com/wyst-lang/wyst/token"
+	"github.com/wyst-lang/wyst/parser"
 )
 
 func main() {
-	tokens, err := Tokenize("test 123", token.RULES_TOP)
+	code := "hello"
+	ast, state, err := parser.Parse(code, parser.TOP)
 	if err != nil {
-		fmt.Printf("Syntax error at %s \n", err)
+		fmt.Printf("SyntaxErr at %d:%d: %s\n", state.Line, state.Column, err)
 		return
 	}
-	for i := 0; i < len(tokens); i++ {
+	for i := 0; i < len(ast); i++ {
-		fmt.Printf("%s\n", tokens[i])
+		fmt.Printf("%s\n", ast[i])
 	}
 }

parser/ast.go

@@ -0,0 +1,60 @@
+package parser
+
+import (
+	"fmt"
+
+	"github.com/wyst-lang/wyst/tokenizer"
+)
+
+type inner interface {
+	isInner()
+}
+
+type Rule struct {
+	Name  string
+	Inner inner
+}
+
+func (t Rule) String() string {
+	return fmt.Sprintf("AstRule(%s)", t.Name)
+}
+
+type Rules struct {
+	Value []Rule
+}
+
+func (v Rules) isInner() {}
+
+type Token struct {
+	Value tokenizer.Rule
+}
+
+func (v Token) isInner() {}
+
+type String struct {
+	Value string
+}
+
+func (v String) isInner() {}
+
+type AstNode struct {
+	Rule  Rule
+	Value string
+	Inner []AstNode
+}
+
+func (t AstNode) String() string {
+	return fmt.Sprintf("AstNode(\n  rule=%s,\n  value=%s,\n  inner=%v\n)", t.Rule, t.Value, t.Inner)
+}
+
+// Ported rules from the tokenizer
+var (
+	IDENTIFIER = Rule{"IDENTIFIER", Token{tokenizer.IDENTIFIER}}
+)
+
+var (
+	TOP = Rule{"TOP", Rules{
+		[]Rule{
+			IDENTIFIER,
+		}}}
+)

parser/parser.go

@@ -0,0 +1,17 @@
+package parser
+
+import (
+	"github.com/wyst-lang/wyst/tokenizer"
+)
+
+func Parse(code string, rule Rule) ([]AstNode, tokenizer.State, error) {
+	tokens, state, _ := tokenizer.Tokenize(code)
+	var ast = []AstNode{}
+	for i := 0; i < len(tokens); i++ {
+		// token := tokens[i]
+		if rule.Inner == (String{""}) {
+			
+		}
+	}
+	return ast, state, nil
+}

token/token.go

@@ -1,26 +0,0 @@
-package token
-
-import (
-	"fmt"
-	"regexp"
-)
-
-type Rule struct {
-	Pattern regexp.Regexp
-	Name    string
-}
-
-func (t Rule) String() string {
-	return fmt.Sprintf("TokenRule(%s)", t.Name)
-}
-
-func NewToken(name string, pattern regexp.Regexp) Rule {
-	return Rule{Name: name, Pattern: pattern}
-}
-
-var (
-	NUMBER     = NewToken("NUMBER", *regexp.MustCompile("^[0-9]*"))
-	IDENTIFIER = NewToken("IDENTIFIER", *regexp.MustCompile("^[A-Za-z_][A-Za-z_0-9]*"))
-)
-
-var RULES_TOP = []Rule{IDENTIFIER, NUMBER}

tokenizer/token.go

@@ -0,0 +1,26 @@
+package tokenizer
+
+import (
+	"fmt"
+	"regexp"
+)
+
+type Rule struct {
+	Pattern regexp.Regexp
+	Name    string
+}
+
+func (t Rule) String() string {
+	return fmt.Sprintf("TokenRule(%s)", t.Name)
+}
+
+func NewRule(name string, pattern regexp.Regexp) Rule {
+	return Rule{Name: name, Pattern: pattern}
+}
+
+var (
+	NUMBER     = NewRule("NUMBER", *regexp.MustCompile(`^\d+`))
+	IDENTIFIER = NewRule("IDENTIFIER", *regexp.MustCompile("^[A-Za-z_][A-Za-z_0-9]*"))
+)
+
+var RULES = []Rule{IDENTIFIER, NUMBER}

tokenizer/tokenizer.go

@@ -1,10 +1,8 @@
-package main
+package tokenizer
 
 import (
 	"fmt"
 	"strings"
-
-	"github.com/wyst-lang/wyst/token"
 )
 
 type State struct {
@@ -13,7 +11,7 @@ type State struct {
 }
 
 type Token struct {
-	Rule     token.Rule
+	Rule     Rule
 	Value    string
 	Position State
 }
@@ -22,17 +20,17 @@ func (t Token) String() string {
 	return fmt.Sprintf("Token(\n  rule=%s,\n  value=%s\n)", t.Rule, t.Value)
 }
 
-func Tokenize(code string, rules []token.Rule) ([]Token, error) {
+func Tokenize(code string) ([]Token, State, error) {
 	var tokens = []Token{}
 	var state = State{1, 0}
 	for code != "" {
 		matched := false
-		for i := 0; i < len(rules); i++ {
+		for i := 0; i < len(RULES); i++ {
-			if rules[i].Pattern.MatchString(code) {
+			if RULES[i].Pattern.MatchString(code) {
-				match := rules[i].Pattern.FindString(code)
+				match := RULES[i].Pattern.FindString(code)
 				matched = true
 				code = code[len(match):]
-				tokens = append(tokens, Token{Rule: rules[i], Value: match, Position: state})
+				tokens = append(tokens, Token{Rule: RULES[i], Value: match, Position: state})
 				state.Column += len(match)
 				break
 			} else if strings.HasPrefix(code, "\n") {
@@ -47,8 +45,8 @@ func Tokenize(code string, rules []token.Rule) ([]Token, error) {
 			}
 		}
 		if !matched {
-			return tokens, fmt.Errorf("%d:%d", state.Line, state.Column)
+			return tokens, state, fmt.Errorf("invalid character or token")
 		}
 	}
-	return tokens, nil
+	return tokens, state, nil
 }