Merge pull request #2 from xquantxz/wip/lexer-info

Add line and column counting

src/lexer.rs

@@ -2,6 +2,11 @@ use regex::Regex;
 use once_cell::sync::Lazy;
 use std::fmt;
 
+pub struct LexerState {
+    line: usize,
+    column: usize
+}
+
 // Define token types
 #[derive(Debug, PartialEq, Clone, Copy)]
 pub enum TokenType {
@@ -20,12 +25,14 @@ pub enum TokenType {
 #[derive(Clone)]
 pub struct Token {
     pub token_type: TokenType,
-    pub token_values: Vec<String>
+    pub token_values: Vec<String>,
+    pub line: usize,
+    pub column: usize
 }
 
 impl<'a> fmt::Debug for Token {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "(TokenType: {:?}, TokenValue: {})", self.token_type, self.token_values.join(", "))
+        write!(f, "(TokenType: {:?}, TokenValue: {}, Line: {}, Column {})", self.token_type, self.token_values.join(", "), self.line, self.column)
     }
 }
 
@@ -37,7 +44,7 @@ pub struct Node {
 const SYNTAX: [Node; 9] = [
     Node {
         token_type: TokenType::Newline,
-        token_regex: Lazy::new(|| Regex::new(r"^\n+").unwrap())
+        token_regex: Lazy::new(|| Regex::new(r"^\n+").unwrap()),
     },
     Node {
         token_type: TokenType::Whitespace,
@@ -49,7 +56,7 @@ const SYNTAX: [Node; 9] = [
     },
     Node {
         token_type: TokenType::Keyword,
-        token_regex: Lazy::new(|| Regex::new(r"^if|mut|try|catch|return|fn").unwrap())
+        token_regex: Lazy::new(|| Regex::new(r"^mut|try|catch|return|fn").unwrap())
     },
     Node {
         token_type: TokenType::Identifier,
@@ -74,7 +81,9 @@ const SYNTAX: [Node; 9] = [
 ];
 
 pub fn lex(mut code: &str, use_whitespace: bool) -> Vec<Token> {
+    let mut state = LexerState { line: 1, column: 1 };
     let mut tokens: Vec<Token> = Vec::new();
+
     while !code.is_empty() {
         let mut is_match = false;
         for s in &SYNTAX {
@@ -88,12 +97,23 @@ pub fn lex(mut code: &str, use_whitespace: bool) -> Vec<Token> {
                     vcaps.push(caps[x].to_string());
                     x+=1;
                 }
+
                 if (!use_whitespace && s.token_type!=TokenType::Whitespace) || use_whitespace {
                     tokens.push(Token {
                         token_type: s.token_type,
                         token_values: vcaps,
+                        line: state.line,
+                        column: state.column
                     });
                 }
+
+                match s.token_type {
+                    TokenType::Newline => {
+                        state.line += caps[0].len();
+                        state.column = 1;
+                    },
+                    _ => { state.column += caps[0].len(); }
+                }
             } else {
                 continue;
             };