use regex::Regex; use once_cell::sync::Lazy; use std::fmt; // Define token types #[derive(Debug, PartialEq, Clone, Copy)] pub enum TokenType { Newline, Whitespace, Number, Identifier, Ptr, Operator, Round, Curly, // EOF, } pub struct Token { pub token_type: TokenType, pub token_value: String } impl<'a> fmt::Display for Token { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "TokenType: {:?}, TokenValue: {}", self.token_type, self.token_value) } } pub struct Node { token_type: TokenType, token_regex: Lazy } const SYNTAX: [Node; 8] = [ Node { token_type: TokenType::Newline, token_regex: Lazy::new(|| Regex::new(r"^\n+").unwrap()) }, Node { token_type: TokenType::Whitespace, token_regex: Lazy::new(|| Regex::new(r"^\s+").unwrap()) }, Node { token_type: TokenType::Number, token_regex: Lazy::new(|| Regex::new(r"^\b(:?.)?(:?0[x|X])?\d+(:?.\d+)?\b").unwrap()) }, Node { token_type: TokenType::Identifier, token_regex: Lazy::new(|| Regex::new(r"^[._a-zA-Z][a-zA-Z0-9_]*").unwrap()) }, Node { token_type: TokenType::Ptr, token_regex: Lazy::new(|| Regex::new(r"^->").unwrap()) }, Node { token_type: TokenType::Operator, token_regex: Lazy::new(|| Regex::new(r"^[\-|\+|\*]").unwrap()) }, Node { token_type: TokenType::Round, token_regex: Lazy::new(|| Regex::new(r"\((?:[^()]|(?R))*\)").unwrap()) }, Node { token_type: TokenType::Curly, token_regex: Lazy::new(|| Regex::new(r"^[\{|\}]").unwrap()) } ]; pub fn lex(mut code: &str, use_whitespace: bool ) -> Vec { let mut tokens: Vec = Vec::new(); while !code.is_empty() { let mut is_match = false; for s in &SYNTAX { if let Some(caps) = s.token_regex.captures(code) { is_match = true; code = code.strip_prefix(&caps[0]).unwrap_or(code); if (!use_whitespace && s.token_type!=TokenType::Whitespace) || use_whitespace { tokens.push(Token { token_type: s.token_type, token_value: caps[0].to_string(), }); } } else { continue; }; break; } if !is_match { println!("Error: syntax -> {code}"); break; } } tokens }