From e909e001449bf3d487a17bed496219638e668f53 Mon Sep 17 00:00:00 2001 From: Leo Dev Date: Mon, 1 Apr 2024 09:01:46 +0200 Subject: [PATCH] lexer change --- .vscode/settings.json | 6 ++ Cargo.toml | 2 + src/lexer.rs | 139 ++++++++++++++++++++++++++++++++++++++++++ src/lexer/lexer.rs | 97 ----------------------------- src/lexer/mod.rs | 1 - src/main.rs | 19 ++---- 6 files changed, 152 insertions(+), 112 deletions(-) create mode 100644 .vscode/settings.json create mode 100644 src/lexer.rs delete mode 100644 src/lexer/lexer.rs delete mode 100644 src/lexer/mod.rs diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..4c9aee3 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,6 @@ +{ + "rust-analyzer.linkedProjects": [ + "./Cargo.toml" + ], + "rust-analyzer.showUnlinkedFileNotification": false +} \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml index 0cf9d5d..2edb360 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,3 +6,5 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +regex = "1.10.4" +once_cell = "1.19.0" \ No newline at end of file diff --git a/src/lexer.rs b/src/lexer.rs new file mode 100644 index 0000000..e7b55b2 --- /dev/null +++ b/src/lexer.rs @@ -0,0 +1,139 @@ +use regex::Regex; +use once_cell::sync::Lazy; + +// Define token types +#[derive(Debug, PartialEq)] +pub enum Token { + Number, + Dot, + Plus, + Minus, + Multiply, + Divide, + LParen, + RParen, + EOF, +} + + +pub struct Node<'a> { + token_type: Token, + token_regex: Lazy, + token_value: &'a str +} + +const SYNTAX: [Node; 1] = [ + Node { + token_type: Token::Dot, + token_regex: Lazy::new(|| Regex::new(r"^test").unwrap()), + token_value: "" + } +]; + +pub fn lex(code: &str) -> Vec { + let mut tokens: Vec = Vec::new(); + while (code!="") { + for s in SYNTAX { + if (s.token_regex.is_match(code)) { + println!("It works!"); + tokens.push(Node { + token_type: s.token_type, + token_regex: s.token_regex, + token_value: s.token_regex.captures(wisp) + }); + } + } + } { + + } + tokens +} + +// Define Lexer struct +// pub struct Lexer<'a> { +// input: &'a str, +// position: usize, +// } + +// impl<'a> Lexer<'a> { +// // Constructor +// pub fn new(input: &'a str) -> Self { +// // Lexer { input, position: 0 } +// } + +// // Advance position in input +// fn advance(&mut self) { +// self.position += 1; +// } + +// // Get current character without advancing position +// fn current_char(&self) -> Option { +// self.input.chars().nth(self.position) +// } + +// // Lexical analysis +// pub fn next_token(&mut self) -> Token { +// // Skip whitespace +// while let Some(c) = self.current_char() { +// if c.is_whitespace() { +// self.advance(); +// } else { +// break; +// } +// } + +// // Check for end of input +// if let None = self.current_char() { +// return Token::EOF; +// } + +// // Match characters to tokens +// match self.current_char().unwrap() { +// '+' => { +// self.advance(); +// Token::Plus +// } +// '-' => { +// self.advance(); +// Token::Minus +// } +// '*' => { +// self.advance(); +// Token::Multiply +// } +// '/' => { +// self.advance(); +// Token::Divide +// } +// '(' => { +// self.advance(); +// Token::LParen +// } +// ')' => { +// self.advance(); +// Token::RParen +// } +// '.' => { +// self.advance(); +// Token::Dot +// } +// // Match numbers +// digit if digit.is_digit(10) => { +// let mut num_str = String::new(); +// while let Some(digit) = self.current_char() { +// if digit.is_digit(10) { +// num_str.push(digit); +// self.advance(); +// } else { +// break; +// } +// } +// Token::Number(num_str.parse().unwrap()) +// } +// // Unknown character +// _ => { +// panic!("Invalid character: {}", self.current_char().unwrap()); +// } +// } +// } +// } diff --git a/src/lexer/lexer.rs b/src/lexer/lexer.rs deleted file mode 100644 index 5b7e60d..0000000 --- a/src/lexer/lexer.rs +++ /dev/null @@ -1,97 +0,0 @@ -// Define token types -#[derive(Debug, PartialEq)] -pub enum Token { - Number(i32), - Plus, - Minus, - Multiply, - Divide, - LParen, - RParen, - EOF, -} - -// Define Lexer struct -pub struct Lexer<'a> { - input: &'a str, - position: usize, -} - -impl<'a> Lexer<'a> { - // Constructor - pub fn new(input: &'a str) -> Self { - Lexer { input, position: 0 } - } - - // Advance position in input - fn advance(&mut self) { - self.position += 1; - } - - // Get current character without advancing position - fn current_char(&self) -> Option { - self.input.chars().nth(self.position) - } - - // Lexical analysis - pub fn next_token(&mut self) -> Token { - // Skip whitespace - while let Some(c) = self.current_char() { - if c.is_whitespace() { - self.advance(); - } else { - break; - } - } - - // Check for end of input - if let None = self.current_char() { - return Token::EOF; - } - - // Match characters to tokens - match self.current_char().unwrap() { - '+' => { - self.advance(); - Token::Plus - } - '-' => { - self.advance(); - Token::Minus - } - '*' => { - self.advance(); - Token::Multiply - } - '/' => { - self.advance(); - Token::Divide - } - '(' => { - self.advance(); - Token::LParen - } - ')' => { - self.advance(); - Token::RParen - } - // Match numbers - digit if digit.is_digit(10) => { - let mut num_str = String::new(); - while let Some(digit) = self.current_char() { - if digit.is_digit(10) { - num_str.push(digit); - self.advance(); - } else { - break; - } - } - Token::Number(num_str.parse().unwrap()) - } - // Unknown character - _ => { - panic!("Invalid character: {}", self.current_char().unwrap()); - } - } - } -} diff --git a/src/lexer/mod.rs b/src/lexer/mod.rs deleted file mode 100644 index 8c99512..0000000 --- a/src/lexer/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod lexer; diff --git a/src/main.rs b/src/main.rs index 372c52d..40ba416 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,17 +1,8 @@ -mod lexer; // Import the lexer module - -use lexer::lexer::Lexer; // Import the Lexer struct -use lexer::lexer::Token; // Import the Token enum +mod lexer; +use lexer::lex; fn main() { - let input = "3 + 4 * (10 - 2)"; - let mut lexer = Lexer::new(input); - - loop { - let token = lexer.next_token(); - println!("{:?}", token); - if token == Token::EOF { - break; - } - } + // let input = "-3.232313 + 4 * (10 - 2)"; + let input = "test"; + let mut tokens = lex(input); }