parser change

Cargo.toml

@@ -1,5 +1,5 @@
 [package]
-name = "wisp"
+name = "wyst"
 version = "0.1.0"
 edition = "2021"
 

src/lexer.rs

@@ -17,6 +17,7 @@ pub enum TokenType {
     Identifier,
     Ptr,
     Operator,
+    SecondOperator,
     Round,
     Curly,
     Square,
@@ -27,14 +28,20 @@ pub enum TokenType {
 #[derive(Clone)]
 pub struct Token {
     pub token_type: TokenType,
-    pub token_values: Vec<String>,
+    pub value: String,
     pub line: usize,
     pub column: usize
 }
 
-impl<'a> fmt::Debug for Token {
+impl fmt::Debug for Token {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "(TokenType: {:?}, TokenValue: {}, Line: {}, Column {})", self.token_type, self.token_values.join(", "), self.line, self.column)
+        write!(f, "(TokenType: {:?}, TokenValue: {}, Line: {}, Column {})", self.token_type, self.value, self.line, self.column)
+    }
+}
+
+impl fmt::Display for Token {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "(TokenType: {:?}, TokenValue: {}, Line: {}, Column {})", self.token_type, self.value, self.line, self.column)
     }
 }
 
@@ -43,7 +50,7 @@ pub struct Node {
     token_regex: Lazy<Regex>
 }
 
-const SYNTAX: [Node; 11] = [
+const SYNTAX: [Node; 12] = [
     Node {
         token_type: TokenType::Newline,
         token_regex: Lazy::new(|| Regex::new(r"^\n+").unwrap()),
@@ -88,6 +95,10 @@ const SYNTAX: [Node; 11] = [
         token_type: TokenType::Angle,
         token_regex: Lazy::new(|| Regex::new(r"<(?:[^<>]|(?R))*>").unwrap())
     },
+    Node {
+        token_type: TokenType::SecondOperator,
+        token_regex: Lazy::new(|| Regex::new(r"^,|;").unwrap())
+    },
 ];
 
 pub fn lex(mut code: &str, use_whitespace: bool) -> Vec<Token> {
@@ -100,18 +111,10 @@ pub fn lex(mut code: &str, use_whitespace: bool) -> Vec<Token> {
             if let Some(caps) = s.token_regex.captures(code) {
                 is_match = true;
                 code = code.strip_prefix(&caps[0]).unwrap_or(code);
-                let mut vcaps: Vec<String> = Vec::new();
-                let capsl = caps.len();
-                let mut x = 0;
-                while x < capsl {
-                    vcaps.push(caps[x].to_string());
-                    x+=1;
-                }
-
                 if (!use_whitespace && s.token_type!=TokenType::Whitespace) || use_whitespace {
                     tokens.push(Token {
                         token_type: s.token_type,
-                        token_values: vcaps,
+                        value: caps[0].to_string(),
                         line: state.line,
                         column: state.column
                     });

src/main.rs

@@ -1,24 +1,14 @@
 mod lexer;
 use lexer::lex;
 mod parser;
-use parser::parse;
+use parser::Parser;
 
 fn main() {
-    let input = "void main() {} <>";
+    let input = "int main() {}";
     println!("\n\n\n\n");
-    // let input = "test->xy";
+    let tokens = lex(input, false);
-    let mut tokens = lex(input, false);
+    let mut ast = Parser::new(tokens.clone());
-    let ast = *parse(&mut tokens);
+    while ast.tokens.len() > ast.index as usize {
-    // for t in tokens {
+        println!("{}\n\n", ast.next());
-    //     println!("{:?}", t);
-    // }
-    let mut _x = 1;
-    for a in ast {
-        println!("{:?}: [", a.type_);
-        for v in a.values {
-            println!("    {:?},", v);
-        }
-        println!("]");
-        _x+=1;
     }
 }

src/parser.rs

@@ -1,127 +1,85 @@
-// use std::collections::HashMap;
 use  crate::lexer::{Token, TokenType};
 use std::fmt;
-use crate::parser::AstDef::{Else, Normal};
-use crate::parser::AstTypes::{FunctionDecleration, Other};
 
-#[derive(Debug, Clone)]
+#[derive(Clone, Debug)]
-pub enum AstTypes {
+pub enum AstType {
     FunctionDecleration,
     VariableDecleration,
     Other
 }
 
 pub struct Ast {
-    pub type_: AstTypes,
+    pub tokens: Vec<Token>,
-    pub values: Vec<Token>
+    pub ast_type: AstType
 }
 
-impl<'a> fmt::Debug for Ast {
+impl fmt::Display for Ast {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "type={:?} values={:?}", self.type_, self.values)
+        write!(f, "{:?}: [\n", self.ast_type)?;
+        for (i, token) in self.tokens.iter().enumerate() {
+            if i < self.tokens.len() - 1 {
+                write!(f, "    {},\n", token)?;
+            } else {
+                write!(f, "    {}\n", token)?;
+            }
+        }
+        write!(f, "]")
     }
 }
 
-#[derive(Debug, PartialEq, Clone)]
+
-pub enum AstDef {
+
-    Normal(TokenType),
+pub struct Parser {
-    NormalValue(TokenType, Vec<String>),
+    pub tokens: Vec<Token>,
-    Repeated(Vec<TokenType>),
+    pub index: u32
-    Optional(TokenType),
-    Else
 }
 
-pub struct PNode {
-    ast_type: AstTypes,
-    ast_match: Vec<AstDef>
-}
 
-pub fn parse(tokens: &mut Vec<Token>) -> Box<Vec<Ast>> {
+impl Parser {
-    let ast_def: Vec<PNode> = vec![
+    pub fn new(tokens: Vec<Token>) -> Parser {
-        PNode {
+        Parser {tokens: tokens, index: 0}
-            ast_type: FunctionDecleration,
+    }
-            ast_match: vec![Normal(TokenType::Identifier), Normal(TokenType::Identifier), Normal(TokenType::Round), Normal(TokenType::Curly)]
+    pub fn next(&mut self) -> Ast {
-        },
+        let mut ast_res: Ast = Ast {tokens: vec![], ast_type: AstType::Other};
-        PNode {
+        let mut index = self.index as usize;
-            ast_type: AstTypes::VariableDecleration,
+        if index == self.tokens.len() {panic!("Reached the end of tokens")}
-            ast_match: vec![Normal(TokenType::Identifier), Normal(TokenType::Identifier)]
+        let token = &self.tokens[index];
-        },
+        match token.token_type {
-        PNode {
+            TokenType::Identifier => {
-            ast_type: Other,
+                if self.tokens[index+1].token_type==TokenType::Identifier && self.tokens[index+2].token_type==TokenType::Round && self.tokens[index+3].token_type==TokenType::Curly {
-            ast_match: vec![Else]
+                    ast_res.tokens.push(self.tokens[index].clone());
-        }
+                    ast_res.tokens.push(self.tokens[index+1].clone());
-    ];
+                    ast_res.tokens.push(self.tokens[index+2].clone());
-    let tokens_len = tokens.len();
+                    ast_res.tokens.push(self.tokens[index+3].clone());
-    let mut ast: Vec<Ast> = Vec::new(); 
+                    ast_res.ast_type = AstType::FunctionDecleration;
-    // let mut i = 0;
+                    self.index += 3;
-    while tokens.len()>0 {
+                } else {
-        for ast_node in &ast_def {
+                    loop {
-            if tokens_len < ast_node.ast_match.len() {
+                        if index+1 >= self.tokens.len() {break;}
-                continue;
+                        index = self.index as usize; // Update the index
-            }
+                        let ntk = &self.tokens[index]; // Stands for next token
-            let mut is_match = false;
+                        if ntk.value=="," {
-            let mut matched: Vec<Token> = Vec::new();
+                            ast_res.tokens.push(ntk.clone());
-            for ast_match in &ast_node.ast_match {
+                            self.index += 1;
-                match ast_match {
+                        } else if ntk.token_type==TokenType::Identifier {
-                    AstDef::Normal(token_type) => {
+                            ast_res.tokens.push(ntk.clone());
-                        if &tokens[0].token_type == token_type {
+                            ast_res.ast_type = AstType::VariableDecleration;
-                            matched.push(tokens[0].clone());
+                            self.index += 1;
-                            tokens.drain(0..1);
+                        } else if ntk.token_type==TokenType::Angle {
-                            is_match = true;
+                            ast_res.tokens.push(ntk.clone());
+                            self.index += 1;
                         } else {
-                            is_match = false;
+                            self.index -= 1;
                             break;
-                        }
+                        } 
                     }
-                    AstDef::NormalValue(token_type, token_value) => {
-                        if &tokens[0].token_type == token_type && &tokens[0].token_values == token_value {
-                            matched.push(tokens[0].clone());
-                            tokens.drain(0..1);
-                            is_match = true;
-                        } else {
-                            is_match = false;
-                            break;
-                        }
-                    }
-                    AstDef::Repeated(nodes) => {
-                        loop {
-                            let mut node_match = false;
-                            for &node_ in nodes {
-                                if node_ == tokens[0].token_type {
-                                    matched.push(tokens[0].clone());
-                                    node_match = true;
-                                    tokens.drain(0..1);
-                                }
-                            }
-                            if node_match {
-                                is_match = true;
-                            } else {
-                                break;
-                            }
-                        }
-                    }
-                    AstDef::Optional(token_type) => {
-                        if &tokens[0].token_type == token_type {
-                            matched.push(tokens[0].clone());
-                            tokens.drain(0..1);
-                            is_match = true;
-                            break;
-                        }
-                    }
-                    AstDef::Else => {
-                        matched.push(tokens[0].clone());
-                        tokens.drain(0..1);
-                        is_match = true;
-                    }
-                    _ => {}
                 }
             }
-            if is_match {
+            _ => {
-                ast.push(Ast {values: matched, type_: ast_node.ast_type.clone()});
+                ast_res.tokens.push(token.clone());
-                matched = Vec::new();
-                break;
             }
         }
+        self.index += 1;
+        ast_res
     }
-    Box::new(ast)
+}
-}