Merge pull request #2 from xquantxz/wip/lexer-info

Add line and column counting

src/lexer.rs

@@ -1,108 +1,128 @@
-use regex::Regex;
+use regex::Regex;
-use once_cell::sync::Lazy;
+use once_cell::sync::Lazy;
-use std::fmt;
+use std::fmt;
-
+
-// Define token types
+pub struct LexerState {
-#[derive(Debug, PartialEq, Clone, Copy)]
+    line: usize,
-pub enum TokenType {
+    column: usize
-    Keyword,
+}
-    Newline,
+
-    Whitespace,
+// Define token types
-    Number,
+#[derive(Debug, PartialEq, Clone, Copy)]
-    Identifier,
+pub enum TokenType {
-    Ptr,
+    Keyword,
-    Operator,
+    Newline,
-    Round,
+    Whitespace,
-    Curly,
+    Number,
-    // EOF,
+    Identifier,
-}
+    Ptr,
-
+    Operator,
-#[derive(Clone)]
+    Round,
-pub struct Token {
+    Curly,
-    pub token_type: TokenType,
+    // EOF,
-    pub token_values: Vec<String>
+}
-}
+
-
+#[derive(Clone)]
-impl<'a> fmt::Debug for Token {
+pub struct Token {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    pub token_type: TokenType,
-        write!(f, "(TokenType: {:?}, TokenValue: {})", self.token_type, self.token_values.join(", "))
+    pub token_values: Vec<String>,
-    }
+    pub line: usize,
-}
+    pub column: usize
-
+}
-pub struct Node {
+
-    token_type: TokenType,
+impl<'a> fmt::Debug for Token {
-    token_regex: Lazy<Regex>
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-}
+        write!(f, "(TokenType: {:?}, TokenValue: {}, Line: {}, Column {})", self.token_type, self.token_values.join(", "), self.line, self.column)
-
+    }
-const SYNTAX: [Node; 9] = [
+}
-    Node {
+
-        token_type: TokenType::Newline,
+pub struct Node {
-        token_regex: Lazy::new(|| Regex::new(r"^\n+").unwrap())
+    token_type: TokenType,
-    },
+    token_regex: Lazy<Regex>
-    Node {
+}
-        token_type: TokenType::Whitespace,
+
-        token_regex: Lazy::new(|| Regex::new(r"^\s+").unwrap())
+const SYNTAX: [Node; 9] = [
-    },
+    Node {
-    Node {
+        token_type: TokenType::Newline,
-        token_type: TokenType::Number,
+        token_regex: Lazy::new(|| Regex::new(r"^\n+").unwrap()),
-        token_regex: Lazy::new(|| Regex::new(r"^\b(:?.)?(:?0[x|X])?\d+(:?.\d+)?\b").unwrap())
+    },
-    },
+    Node {
-    Node {
+        token_type: TokenType::Whitespace,
-        token_type: TokenType::Keyword,
+        token_regex: Lazy::new(|| Regex::new(r"^\s+").unwrap())
-        token_regex: Lazy::new(|| Regex::new(r"^if|mut|try|catch|return|fn").unwrap())
+    },
-    },
+    Node {
-    Node {
+        token_type: TokenType::Number,
-        token_type: TokenType::Identifier,
+        token_regex: Lazy::new(|| Regex::new(r"^\b(:?.)?(:?0[x|X])?\d+(:?.\d+)?\b").unwrap())
-        token_regex: Lazy::new(|| Regex::new(r"^[._a-zA-Z][a-zA-Z0-9_]*").unwrap())
+    },
-    },
+    Node {
-    Node {
+        token_type: TokenType::Keyword,
-        token_type: TokenType::Ptr,
+        token_regex: Lazy::new(|| Regex::new(r"^mut|try|catch|return|fn").unwrap())
-        token_regex: Lazy::new(|| Regex::new(r"^->").unwrap())
+    },
-    },
+    Node {
-    Node {
+        token_type: TokenType::Identifier,
-        token_type: TokenType::Operator,
+        token_regex: Lazy::new(|| Regex::new(r"^[._a-zA-Z][a-zA-Z0-9_]*").unwrap())
-        token_regex: Lazy::new(|| Regex::new(r"^[\-|\+|\*]").unwrap())
+    },
-    },
+    Node {
-    Node {
+        token_type: TokenType::Ptr,
-        token_type: TokenType::Round,
+        token_regex: Lazy::new(|| Regex::new(r"^->").unwrap())
-        token_regex: Lazy::new(|| Regex::new(r"\((?:[^()]|(?R))*\)").unwrap())
+    },
-    },
+    Node {
-    Node {
+        token_type: TokenType::Operator,
-        token_type: TokenType::Curly,
+        token_regex: Lazy::new(|| Regex::new(r"^[\-|\+|\*]").unwrap())
-        token_regex: Lazy::new(|| Regex::new(r"^[\{|\}]").unwrap())
+    },
-    }
+    Node {
-];
+        token_type: TokenType::Round,
-
+        token_regex: Lazy::new(|| Regex::new(r"\((?:[^()]|(?R))*\)").unwrap())
-pub fn lex(mut code: &str, use_whitespace: bool) -> Vec<Token> {
+    },
-    let mut tokens: Vec<Token> = Vec::new();
+    Node {
-    while !code.is_empty() {
+        token_type: TokenType::Curly,
-        let mut is_match = false;
+        token_regex: Lazy::new(|| Regex::new(r"^[\{|\}]").unwrap())
-        for s in &SYNTAX {
+    }
-            if let Some(caps) = s.token_regex.captures(code) {
+];
-                is_match = true;
+
-                code = code.strip_prefix(&caps[0]).unwrap_or(code);
+pub fn lex(mut code: &str, use_whitespace: bool) -> Vec<Token> {
-                let mut vcaps: Vec<String> = Vec::new();
+    let mut state = LexerState { line: 1, column: 1 };
-                let capsl = caps.len();
+    let mut tokens: Vec<Token> = Vec::new();
-                let mut x = 0;
+
-                while x < capsl {
+    while !code.is_empty() {
-                    vcaps.push(caps[x].to_string());
+        let mut is_match = false;
-                    x+=1;
+        for s in &SYNTAX {
-                }
+            if let Some(caps) = s.token_regex.captures(code) {
-                if (!use_whitespace && s.token_type!=TokenType::Whitespace) || use_whitespace {
+                is_match = true;
-                    tokens.push(Token {
+                code = code.strip_prefix(&caps[0]).unwrap_or(code);
-                        token_type: s.token_type,
+                let mut vcaps: Vec<String> = Vec::new();
-                        token_values: vcaps,
+                let capsl = caps.len();
-                    });
+                let mut x = 0;
-                }
+                while x < capsl {
-            } else {
+                    vcaps.push(caps[x].to_string());
-                continue;
+                    x+=1;
-            };
+                }
-            break;
+
-        }
+                if (!use_whitespace && s.token_type!=TokenType::Whitespace) || use_whitespace {
-        if !is_match {
+                    tokens.push(Token {
-            println!("Error: syntax -> {code}");
+                        token_type: s.token_type,
-            break;
+                        token_values: vcaps,
-        }
+                        line: state.line,
-    }
+                        column: state.column
-    tokens
+                    });
-}
+                }
+
+                match s.token_type {
+                    TokenType::Newline => {
+                        state.line += caps[0].len();
+                        state.column = 1;
+                    },
+                    _ => { state.column += caps[0].len(); }
+                }
+            } else {
+                continue;
+            };
+            break;
+        }
+        if !is_match {
+            println!("Error: syntax -> {code}");
+            break;
+        }
+    }
+    tokens
+}