From 55fe45717c62fa3bfde97bfba12c19ba929b5488 Mon Sep 17 00:00:00 2001 From: Leo Dev Date: Tue, 2 Apr 2024 21:49:30 +0200 Subject: [PATCH] parser beta --- src/lexer.rs | 25 ++++++++++++---- src/main.rs | 16 ++++++++--- src/parser.rs | 79 +++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 110 insertions(+), 10 deletions(-) create mode 100644 src/parser.rs diff --git a/src/lexer.rs b/src/lexer.rs index ed07b6c..2a8c54d 100644 --- a/src/lexer.rs +++ b/src/lexer.rs @@ -5,6 +5,7 @@ use std::fmt; // Define token types #[derive(Debug, PartialEq, Clone, Copy)] pub enum TokenType { + Keyword, Newline, Whitespace, Number, @@ -16,14 +17,15 @@ pub enum TokenType { // EOF, } +#[derive(Clone)] pub struct Token { pub token_type: TokenType, - pub token_value: String + pub token_value: Vec } -impl<'a> fmt::Display for Token { +impl<'a> fmt::Debug for Token { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "TokenType: {:?}, TokenValue: {}", self.token_type, self.token_value) + write!(f, "(TokenType: {:?}, TokenValue: {})", self.token_type, self.token_value.join(", ")) } } @@ -32,7 +34,7 @@ pub struct Node { token_regex: Lazy } -const SYNTAX: [Node; 8] = [ +const SYNTAX: [Node; 9] = [ Node { token_type: TokenType::Newline, token_regex: Lazy::new(|| Regex::new(r"^\n+").unwrap()) @@ -45,6 +47,10 @@ const SYNTAX: [Node; 8] = [ token_type: TokenType::Number, token_regex: Lazy::new(|| Regex::new(r"^\b(:?.)?(:?0[x|X])?\d+(:?.\d+)?\b").unwrap()) }, + Node { + token_type: TokenType::Keyword, + token_regex: Lazy::new(|| Regex::new(r"^mut|try|catch|return|fn").unwrap()) + }, Node { token_type: TokenType::Identifier, token_regex: Lazy::new(|| Regex::new(r"^[._a-zA-Z][a-zA-Z0-9_]*").unwrap()) @@ -67,7 +73,7 @@ const SYNTAX: [Node; 8] = [ } ]; -pub fn lex(mut code: &str, use_whitespace: bool ) -> Vec { +pub fn lex(mut code: &str, use_whitespace: bool) -> Vec { let mut tokens: Vec = Vec::new(); while !code.is_empty() { let mut is_match = false; @@ -75,10 +81,17 @@ pub fn lex(mut code: &str, use_whitespace: bool ) -> Vec { if let Some(caps) = s.token_regex.captures(code) { is_match = true; code = code.strip_prefix(&caps[0]).unwrap_or(code); + let mut vcaps: Vec = Vec::new(); + let capsl = caps.len(); + let mut x = 0; + while x < capsl { + vcaps.push(caps[x].to_string()); + x+=1; + } if (!use_whitespace && s.token_type!=TokenType::Whitespace) || use_whitespace { tokens.push(Token { token_type: s.token_type, - token_value: caps[0].to_string(), + token_value: vcaps, }); } } else { diff --git a/src/main.rs b/src/main.rs index 32439bb..f69b2ba 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,11 +1,19 @@ mod lexer; use lexer::lex; +mod parser; +use parser::parse; fn main() { - let input = "int test() {}"; + let input = "int main() {"; // let input = "test->xy"; - let tokens = lex(input, true); - for token in tokens { - println!("{:?} {}", token.token_type, token.token_value); + let mut tokens = lex(input, false); + let ast = *parse(&mut tokens); + for t in tokens { + println!("{:?}", t); + } + let mut _x = 1; + for a in ast { + println!("{:?}: {:?}", a.type_, a.values); + _x+=1; } } diff --git a/src/parser.rs b/src/parser.rs new file mode 100644 index 0000000..dca0d5b --- /dev/null +++ b/src/parser.rs @@ -0,0 +1,79 @@ +// use std::collections::HashMap; +use crate::lexer::{Token, TokenType}; +use std::fmt; + +#[derive(Debug, Clone)] +pub enum AstTypes { + FunctionDecleration +} + +pub struct Ast { + pub type_: AstTypes, + pub values: Vec +} + +impl<'a> fmt::Debug for Ast { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "type={:?} values={:?}", self.type_, self.values) + } +} + +#[derive(Debug, PartialEq, Clone, Copy)] +pub enum AstDef { + Normal(TokenType), + Repeated(TokenType), + Optional(TokenType) +} + +pub struct PNode { + ast_type: AstTypes, + ast_match: Vec +} + +pub fn parse(tokens: &mut Vec) -> Box> { + let ast_def: Vec = vec![ + PNode { + ast_type: AstTypes::FunctionDecleration, + ast_match: vec![AstDef::Normal(TokenType::Identifier), AstDef::Normal(TokenType::Identifier), AstDef::Normal(TokenType::Round), AstDef::Normal(TokenType::Curly)] + }, + PNode { + ast_type: AstTypes::FunctionDecleration, + ast_match: vec![AstDef::Normal(TokenType::Keyword)] + }, + ]; + let tokens_len = tokens.len(); + let mut ast: Vec = Vec::new(); + let mut current = 0; + let mut i = 0; + while tokens.len()>0 { + for ast_node in &ast_def { + if tokens_len < ast_node.ast_match.len() { + continue; + } + let mut ast_types: Vec = vec![]; + let mut is_match = false; + let mut matched: Vec = Vec::new(); + for ast_match in &ast_node.ast_match { + match ast_match { + AstDef::Normal(token_type) => { + if &tokens[i].token_type == token_type { + matched.push(tokens[i].clone()); + tokens.drain(0..1); + is_match = true; + } else { + is_match = false; + } + } + _ => {} + } + } + if is_match { + i = 0; + ast.push(Ast {values: matched, type_: ast_node.ast_type.clone()}); + matched = Vec::new(); + break; + } + } + } + Box::new(ast) +}