using pest

Cargo.toml

@@ -6,9 +6,6 @@ edition = "2021"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-regex = "1.10.4"
-once_cell = "1.19.0"
-clap = "3.0"
 rand = "0.8.5"
 serde_json = "1.0"
 serde = "1.0.203"
@@ -16,3 +13,6 @@ serde_derive = "1.0.203"
 lsp-types = "0.97.0"
 zip = "2.1.3"
 dirs = "5.0.1"
+pest = "2"
+pest_derive = "2"
+clap = "3"

src/parser.rs

@@ -1,367 +1,22 @@
-use std::usize;
+use pest::Parser;
+use pest_derive::Parser;
 
-use once_cell::sync::Lazy;
+#[derive(Parser)]
-use regex::Regex;
+#[grammar = "wyst.pest"]
+pub struct WystParser;
 
-use crate::{
+pub fn parse(input: String, rule: Rule) {
-    transpiler::{State, Transpiler},
+    match WystParser::parse(rule, &input) {
-    utils::{Problem, ProblemCap, ProblemType, Variables},
+        Ok(parsed) => {
-};
+            for pair in parsed {
-
+                for inner in pair.into_inner() {
-pub struct Node {
+                    println!("{:?}", inner.into_inner())
-    pub regex_pattern: Lazy<Regex>,
-    pub token: u32,
-}
-
-const NODES: [Node; 3] = [
-    Node {
-        token: 1,
-        regex_pattern: Lazy::new(|| Regex::new(r"^(0(x|X)\d+|\d+)").unwrap()),
-    },
-    Node {
-        token: 2,
-        regex_pattern: Lazy::new(|| Regex::new(r"^[.:_a-zA-Z][a-zA-Z0-9_.:]*").unwrap()),
-    },
-    Node {
-        token: 5,
-        regex_pattern: Lazy::new(|| Regex::new(r"^(\;|\!|\=)").unwrap()),
-    },
-];
-
-#[derive(Clone, Debug)]
-#[allow(dead_code)]
-pub enum Token {
-    Number(String, State),
-    Keyword(String, State),
-    SKeyword(String, State),
-    DKeyword(String, State),
-    Identifier(String, State),
-    Curly(String, State),
-    Round(String, State),
-    Operator(String, State),
-    String(String, State),
-    // Square(String),
-}
-
-pub fn extract_values(token: Token) -> (u8, String, State) {
-    match token {
-        Token::Number(x, state) => (0, x, state),
-        Token::Identifier(x, state) => (1, x, state),
-        Token::Curly(x, state) => (2, x, state),
-        Token::Round(x, state) => (3, x, state),
-        Token::DKeyword(x, state) => (4, x, state),
-        Token::SKeyword(x, state) => (5, x, state),
-        Token::Keyword(x, state) => (6, x, state),
-        Token::Operator(x, state) => (7, x, state),
-        Token::String(x, state) => (8, x, state),
-    }
-}
-
-pub fn extract_ast(ast: Ast) -> Vec<String> {
-    let mut vstr: Vec<String> = Vec::new();
-    match ast {
-        Ast::Struct(a, _) => {
-            vstr.push(a);
-        }
-        Ast::Single(a) => match a {
-            Token::Identifier(b, _) => vstr.push(b),
-            _ => {}
-        },
-        Ast::Variable(a, b) => {
-            vstr.push(a);
-            vstr.push(b);
-        }
-        Ast::Function(a, b, _, _) => {
-            vstr.push(a);
-            vstr.push(b);
-        }
-        _ => {}
-    }
-    vstr
-}
-
-#[derive(Clone, Debug)]
-pub enum Ast {
-    Variable(String, String),
-    Function(String, String, String, String),
-    Struct(String, String),
-    Rust(String),
-    Single(Token),
-}
-
-pub fn get_token(v: String, t: u32, state: State) -> Token {
-    match t {
-        1 => Token::Number(v, state),
-        2 => match v.as_str() {
-            "enum" | "struct" | "namespace" => Token::DKeyword(v, state),
-            "rust" => Token::SKeyword(v, state),
-            _ => Token::Identifier(v, state),
-        },
-        3 => Token::Round(v, state),
-        4 => Token::Curly(v, state),
-        5 => Token::Operator(v, state),
-        6 => Token::String(v, state),
-        _ => Token::Number(String::from(""), state),
-    }
-}
-
-pub fn tokenzie_regex(code: String, state: &mut State) -> (u32, String) {
-    let mut token = (0, String::new());
-    if !code.is_empty() {
-        for node in &NODES {
-            if let Some(caps) = node.regex_pattern.captures(&code) {
-                token.0 = node.token;
-                token.1 = caps[0].to_string();
-                state.column += token.1.len() as u32;
-                if token.1.contains("\n") {
-                    state.column = 0;
-                    state.line += token.1.chars().filter(|c| *c == '\n').count() as u32;
                 }
             }
+            // println!("{:#?}", parsed);
+        }
+        Err(e) => {
+            println!("Ok: {}", e);
         }
     }
-    token
-}
-
-pub fn tokenize(code: String, state: &mut State, root: &mut Transpiler) -> Vec<Token> {
-    let mut code = code;
-    code += " ";
-    let mut tokens: Vec<Token> = Vec::new();
-    let mut token_value: String = String::new();
-    let mut token_type: u32 = 0;
-    let mut stoken: (u8, u8) = (0, 0);
-    let mut token_state = State {
-        line: 0,
-        column: 0,
-        file: None,
-    };
-    while !code.is_empty() {
-        if let Some(c) = code.chars().next() {
-            code.remove(0);
-            let tc = c.to_string();
-            let mut tval = tc.as_str();
-            let mut t: u32 = 0;
-            let mut ignore = false;
-            state.column += 1;
-            if root.inject.inject
-                && root.inject.state.line == state.line
-                && root.inject.state.column == state.column
-            {
-                token_value += "list_vx";
-            }
-            if stoken.1 > 0 {
-                match c {
-                    '(' => {
-                        if stoken.0 == 1 {
-                            stoken.1 += 1;
-                            token_value += c.to_string().as_str();
-                        }
-                    }
-                    ')' => {
-                        if stoken.0 == 1 {
-                            stoken.1 -= 1;
-                            if stoken.1 == 0 {
-                                stoken.0 = 0;
-                                tokens.push(get_token(
-                                    token_value.clone(),
-                                    token_type,
-                                    token_state.clone(),
-                                ));
-                                token_type = 0;
-                                token_value = String::new();
-                            } else {
-                                token_value += c.to_string().as_str();
-                            }
-                        }
-                    }
-                    '{' => {
-                        if stoken.0 == 2 {
-                            stoken.1 += 1;
-                            token_value += c.to_string().as_str();
-                        }
-                    }
-                    '}' => {
-                        if stoken.0 == 2 {
-                            stoken.1 -= 1;
-                            if stoken.1 == 0 {
-                                stoken.0 = 0;
-                                tokens.push(get_token(
-                                    token_value.clone(),
-                                    token_type,
-                                    token_state.clone(),
-                                ));
-                                token_type = 0;
-                                token_value = String::new();
-                            } else {
-                                token_value += c.to_string().as_str();
-                            }
-                        }
-                    }
-                    '"' => {
-                        token_value += c.to_string().as_str();
-                        if stoken.0 == 3 {
-                            stoken.1 -= 1;
-                            if stoken.1 == 0 {
-                                stoken.0 = 0;
-                                tokens.push(get_token(
-                                    token_value.clone(),
-                                    token_type,
-                                    token_state.clone(),
-                                ));
-                                token_type = 0;
-                                token_value = String::new();
-                            }
-                        }
-                    }
-                    '\\' => {
-                        token_value += c.to_string().as_str();
-                        token_value += code
-                            .chars()
-                            .next()
-                            .expect("Error expected a char for \\")
-                            .to_string()
-                            .as_str();
-                    }
-                    _ => token_value += c.to_string().as_str(),
-                }
-            } else {
-                match c {
-                    '\t' | ' ' => {
-                        ignore = true;
-                    }
-                    '\n' | '\r' => {
-                        ignore = true;
-                        state.line += 1;
-                        state.column = 0;
-                    }
-                    '(' => {
-                        stoken.0 = 1;
-                        stoken.1 = 1;
-                        tval = " ";
-                        t = 3;
-                    }
-                    '{' => {
-                        stoken.0 = 2;
-                        stoken.1 = 1;
-                        tval = " ";
-                        t = 4;
-                    }
-                    '"' => {
-                        stoken.0 = 3;
-                        stoken.1 = 1;
-                        t = 6;
-                        ignore = true;
-                    }
-                    x => {
-                        ignore = true;
-                        let vals = tokenzie_regex(format!("{}{}", c, code.clone()), state);
-                        if vals.0 != 0 {
-                            tokens.push(get_token(vals.1.clone(), vals.0, state.clone()));
-                            code.drain(0..vals.1.len() - 1);
-                        } else {
-                            root.problems.push(ProblemCap::Error(Problem {
-                                problem_msg: format!("Invalid syntax '{}'", x,),
-                                problem_type: ProblemType::SyntaxError,
-                                state: state.clone(),
-                            }));
-                        }
-                    }
-                }
-                if token_type != t && token_type != 0 {
-                    tokens.push(get_token(
-                        token_value.clone(),
-                        token_type,
-                        token_state.clone(),
-                    ));
-                    token_type = 0;
-                    token_value = String::new();
-                }
-                if token_type != t {
-                    token_state = state.clone();
-                    token_type = t;
-                    token_value += tval.to_string().as_str();
-                } else if !ignore {
-                    token_value += tval.to_string().as_str();
-                }
-            }
-        }
-    }
-    if token_type != 0 {
-        tokens.push(get_token(token_value.clone(), token_type, state.clone()));
-    }
-    tokens
-}
-
-pub fn parse(
-    code: String,
-    vars: &mut Variables,
-    state: &mut State,
-    root: &mut Transpiler,
-) -> Vec<Ast> {
-    let mut ast: Vec<Ast> = Vec::new();
-    let mut tokens = tokenize(code, state, root);
-    while tokens.len() > 0 {
-        let mut drain: usize = 1;
-        let vals = extract_values(tokens[0].clone());
-        if vals.1.contains("list_vx") {
-            ast.push(Ast::Single(Token::Identifier(vals.1, vals.2)));
-            tokens.drain(0..1);
-            continue;
-        }
-        match tokens.as_slice() {
-            [Token::Identifier(var_type, state0), Token::Identifier(var_name, _state1), Token::Round(round, _state2), Token::Curly(curly, _state3), ..] =>
-            {
-                ast.push(Ast::Function(
-                    if var_type == "void" {
-                        "()".to_string()
-                    } else {
-                        var_type.clone()
-                    },
-                    vars.new_func(var_name.clone(), state0.clone(), "".to_string()),
-                    round.clone(),
-                    curly.clone(),
-                ));
-                drain += 3;
-            }
-            [Token::Identifier(var_type, _state0), Token::Identifier(var_name, state1), ..] => {
-                ast.push(Ast::Variable(
-                    var_type.clone(),
-                    vars.new_var(var_name.clone(), state1.clone(), "".to_string()),
-                ));
-                drain += 1;
-            }
-            [Token::DKeyword(keyword, _), Token::Identifier(name, state), Token::Curly(curly, _), ..] =>
-            {
-                match keyword.as_str() {
-                    "struct" => ast.push(Ast::Struct(
-                        vars.new_struct(name.clone(), state.clone(), String::from("")),
-                        curly.clone(),
-                    )),
-                    _ => {}
-                }
-                drain += 2;
-            }
-            [Token::SKeyword(keyword, _), Token::Curly(curly, _), ..] => {
-                match keyword.as_str() {
-                    "rust" => ast.push(Ast::Rust(curly.clone())),
-                    _ => {}
-                }
-                drain += 1;
-            }
-            x => {
-                ast.push(Ast::Single(x[0].clone()));
-                //let estate = extract_values(x[0].clone()).1;
-                //if !pcon.contains(&(estate.line, estate.column)) {
-                //    root.problems.push(ProblemCap::Error(Problem {
-                //        problem_msg: format!("Invalid placement"),
-                //        problem_type: ProblemType::SyntaxError,
-                //        state: estate,
-                //    }));
-                //    pcon.push((root.state.line, root.state.column));
-                //}
-            }
-        }
-        tokens.drain(0..drain);
-    }
-    ast
 }

src/transpiler.rs

@@ -1,24 +1,24 @@
 use crate::{
-    parser::{self, extract_ast, extract_values, Ast},
+    parser::{self, Rule},
-    utils::{ProblemCap, VariableType, Variables},
+    utils::{ProblemCap, Variables},
 };
 use serde::{Deserialize, Serialize};
 
-#[derive(Clone, Debug, Serialize, Deserialize)]
+#[derive(Deserialize, Serialize, Debug, Clone)]
 pub struct State {
     pub line: u32,
     pub column: u32,
     pub file: Option<String>,
 }
 
-#[derive(Clone, Debug, Serialize, Deserialize)]
+#[derive(Deserialize, Serialize, Debug, Clone)]
 pub struct Transpiler {
     pub matched_vars: Variables,
     pub problems: Vec<ProblemCap>,
     pub state: State,
     pub inject: Inject,
 }
-#[derive(Clone, Debug, Serialize, Deserialize)]
+#[derive(Deserialize, Serialize, Debug, Clone)]
 pub struct Inject {
     pub state: State,
     pub inject: bool,
@@ -54,71 +54,72 @@ impl Default for Transpiler {
 
 impl Transpiler {
     pub fn transpile(&mut self, code: String, indent: usize, vars: &mut Variables) -> String {
-        let ast = parser::parse(code, vars, &mut self.state.clone(), self);
+        // args: code, vars, &mut self.state.clone(), self
+        let ast = parser::parse(code, Rule::top_expr);
         let mut res = String::new();
         if indent > 0 {
             res += " ".repeat(indent * 2).as_str();
         }
-        for a in ast {
+        // for a in ast {
-            let astval = extract_ast(a.clone());
+        //     let astval = extract_ast(a.clone());
-            if self.inject.inject {
+        //     if self.inject.inject {
-                for val in astval {
+        //         for val in astval {
-                    if val.contains("list_vx") {
+        //             if val.contains("list_vx") {
-                        for (name, var) in vars.clone().iter_mut() {
+        //                 for (name, var) in vars.clone().iter_mut() {
-                            if self.inject.state.line > var.state.line
+        //                     if self.inject.state.line > var.state.line
-                                || var.vtype != VariableType::Var
+        //                         || var.vtype != VariableType::Var
-                            {
+        //                     {
-                                self.matched_vars.vars.insert(name.clone(), var.clone());
+        //                         self.matched_vars.vars.insert(name.clone(), var.clone());
-                            }
+        //                     }
-                        }
+        //                 }
-                    }
+        //             }
-                }
+        //         }
-            }
+        //     }
-            match a {
+        //     match a {
-                Ast::Variable(var_type, var_name) => {
+        //         Ast::Variable(var_type, var_name) => {
-                    res += format!("let mut {}: {}", var_name, var_type).as_str();
+        //             res += format!("let mut {}: {}", var_name, var_type).as_str();
-                }
+        //         }
-                Ast::Function(var_type, var_name, round, curly) => {
+        //         Ast::Function(var_type, var_name, round, curly) => {
-                    res += format!(
+        //             res += format!(
-                        "fn {}({}) -> {} {}\n{}{}\n",
+        //                 "fn {}({}) -> {} {}\n{}{}\n",
-                        var_name,
+        //                 var_name,
-                        round,
+        //                 round,
-                        var_type,
+        //                 var_type,
-                        "{",
+        //                 "{",
-                        self.transpile(curly, indent + 1, &mut vars.clone()),
+        //                 self.transpile(curly, indent + 1, &mut vars.clone()),
-                        "}"
+        //                 "}"
-                    )
+        //             )
-                    .as_str();
+        //             .as_str();
-                }
+        //         }
-                Ast::Struct(name, curly) => {
+        //         Ast::Struct(name, curly) => {
-                    res += format!(
+        //             res += format!(
-                        "struct {} {}\n{}{}\n",
+        //                 "struct {} {}\n{}{}\n",
-                        name,
+        //                 name,
-                        "{",
+        //                 "{",
-                        self.transpile(curly, indent + 1, &mut vars.clone()),
+        //                 self.transpile(curly, indent + 1, &mut vars.clone()),
-                        "}"
+        //                 "}"
-                    )
+        //             )
-                    .as_str()
+        //             .as_str()
-                }
+        //         }
-                Ast::Rust(rs) => {
+        //         Ast::Rust(rs) => {
-                    res += "{";
+        //             res += "{";
-                    res += rs.as_str();
+        //             res += rs.as_str();
-                    res += "}";
+        //             res += "}";
-                }
+        //         }
-                Ast::Single(x) => {
+        //         Ast::Single(x) => {
-                    println!("{:?}", x);
+        //             println!("{:?}", x);
-                    let vals = extract_values(x);
+        //             let vals = extract_values(x);
-                    match vals.0 {
+        //             match vals.0 {
-                        2 => res += vars.get_var(vals.1, self, vals.2).as_str(),
+        //                 2 => res += vars.get_var(vals.1, self, vals.2).as_str(),
-                        3 => res += format!("({})", vals.1).as_str(),
+        //                 3 => res += format!("({})", vals.1).as_str(),
-                        8 => res += vals.1.as_str(),
+        //                 8 => res += vals.1.as_str(),
-                        _ => {
+        //                 _ => {
-                            res += vals.1.as_str();
+        //                     res += vals.1.as_str();
-                        }
+        //                 }
-                    }
+        //             }
-                }
+        //         }
-            }
+        //     }
-        }
+        // }
         if indent > 0 {
             res += "\n";
             res += " ".repeat((indent - 1) * 2).as_str();

src/wyst.pest

@@ -0,0 +1,58 @@
+WHITESPACE = _{ " " | "\t" | "\n" }
+
+// [literals]
+round = { "(" ~ (expr)* ~ ")" }
+curly = { "{" ~ (code_expr)* ~ "}" }
+escape = { "\\" ~ ANY }
+raw_string = { (!("\\" | "\"") ~ ANY)+ }
+string = { "\"" ~ (raw_string | escape )* ~ "\"" }
+char = { "'" ~ (escape | ANY) ~ "'" }
+bool = @{"true" | "false"}
+semicolon = {";"}
+
+// [numbers]
+hex = @{ "0" ~ ("x"|"X") ~ ('0'..'9' | 'a'..'f' | 'A'..'F')*}
+float = @{ ('0'..'9')* ~ "." ~ ('0'..'9')+ }
+int = @{ '0'..'9'+ }
+
+// [identifiers]
+identifier = @{ !(hex|float|int) ~ ("_" | 'a'..'z' | 'a'..'z') ~ ( "_" | 'a'..'z' | 'a'..'z' | int )* }
+
+// [definitions]
+var_def = { identifier ~ identifier }
+var_def_set = { var_def ~ "=" ~ expr ~ semicolon }
+func_def = { identifier ~ identifier ~ round ~ curly }
+
+expr = {(
+    round |
+    curly |
+    char |
+    hex |
+    float |
+    int |
+    bool |
+    identifier |
+    string |
+    semicolon
+)+}
+
+code_expr = {(
+    round |
+    curly |
+    char |
+    var_def_set |
+    (var_def ~ semicolon) |
+    hex |
+    float |
+    int |
+    bool |
+    // identifier |
+    string |
+    semicolon
+)+}
+
+top_expr = {
+    (
+        func_def
+    )+
+}