wyst-lang/wyst
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

rewrite 1

Browse Source
This commit is contained in:
selimaj-dev
2024-06-28 14:57:47 +02:00
parent 910b638458
commit 3d694d4931
10 changed files with 109 additions and 2526 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/parser.rs
+65 -372
View File
@@ -1,387 +1,80 @@
use crate::{
lexer::{LexerState, Token, TokenType},
variable::Variables,
};
use once_cell::sync::Lazy;
use regex::Regex;
use std::fmt;
use crate::{transpiler::State, variable::Variables};
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum AstType {
Ref,
FunctionDeceleration,
StructDeceleration,
StructCall,
StructVar,
VoidFunctionDeceleration,
Namespace,
VariableDeceleration,
PointerDeceleration,
MutVariableDeceleration,
State3,
State2,
Include,
IncludeLocal,
CodeBlock,
Json,
Impl,
StaticExecution,
EnumDeceleration,
Other,
#[derive(Clone, Debug)]
pub enum Token {
Number(String),
Identifier(String),
// Curly(String),
// Round(String),
// Square(String),
}
pub enum Ast {
Variable(String, String),
}
pub fn is_decl(ast: &Ast) -> bool {
ast.ast_type == AstType::FunctionDeceleration
|| ast.ast_type == AstType::VoidFunctionDeceleration
|| ast.ast_type == AstType::Namespace
|| ast.ast_type == AstType::VariableDeceleration
|| ast.ast_type == AstType::PointerDeceleration
|| ast.ast_type == AstType::MutVariableDeceleration
|| ast.ast_type == AstType::StructDeceleration
}
pub struct Ast {
pub tokens: Vec<Token>,
pub ast_type: AstType,
}
impl fmt::Display for Ast {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(f, "\x1b[36m{:?}:\x1b[0m [\n", self.ast_type)?;
for (i, token) in self.tokens.iter().enumerate() {
if i < self.tokens.len() - 1 {
write!(f, " {},\n", token)?;
} else {
write!(f, " {}\n", token)?;
}
}
write!(f, "]")
pub fn get_token(v: String, t: u32) -> Token {
match t {
1 => Token::Number(v),
2 => Token::Identifier(v),
_ => Token::Number(String::from("")),
}
}
impl fmt::Debug for Ast {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(f, "{:?}: [\n", self.ast_type)?;
for (i, token) in self.tokens.iter().enumerate() {
if i < self.tokens.len() - 1 {
write!(f, " {:?},\n", token)?;
} else {
write!(f, " {:?}\n", token)?;
pub fn tokenize(code: String) -> Vec<Token> {
let mut code = code;
let mut tokens: Vec<Token> = Vec::new();
let mut token_value: String = String::new();
let mut token_type: u32 = 0;
while !code.is_empty() {
if let Some(c) = code.chars().next() {
code.remove(0);
let mut tval: char = c;
let mut t: u32 = 0;
match c {
'0'..='9' => {
if token_type == 2 {
t = 2;
} else {
t = 1;
}
}
'a'..='z' | 'A'..='Z' => {
t = 2;
}
'\t' | ' ' => tval = ' ',
_ => {}
}
if t != token_type && token_type != 0 || tval == ' ' {
tokens.push(get_token(token_value.clone(), token_type));
token_type = 0;
token_value = String::new();
}
if tval != ' ' {
token_type = t;
token_value += tval.to_string().as_str();
}
}
write!(f, "]")
}
if token_type != 0 {
tokens.push(get_token(token_value.clone(), token_type));
}
tokens
}
pub struct Parser {
pub tokens: Vec<Token>,
pub index: u32,
pub include_regex: Lazy<Regex>,
pub include_regex_local: Lazy<Regex>,
pub variables: Variables,
pub json: bool,
}
impl Parser {
pub fn new(tokens: Vec<Token>, variables: Variables) -> Parser {
Parser {
tokens: tokens,
index: 0,
include_regex: Lazy::new(|| Regex::new(r"^(#include *)<(.*?)>").unwrap()),
include_regex_local: Lazy::new(|| Regex::new(r#"^(#include *)"(.*?)""#).unwrap()),
variables: variables,
json: false,
}
}
pub fn parse(&mut self) -> Vec<Ast> {
let mut full_ast: Vec<Ast> = Vec::new();
while self.tokens.len() > self.index as usize {
let mut ast_res: Ast = Ast {
tokens: vec![],
ast_type: AstType::Other,
};
let index = self.index as usize;
if index == self.tokens.len() {
panic!("Reached the end of tokens")
pub fn parse(code: String, vars: &mut Variables) -> Vec<Ast> {
let mut ast: Vec<Ast> = Vec::new();
let mut tokens = tokenize(code);
while tokens.len() > 0 {
let mut drain: usize = 0;
match tokens.as_slice() {
[Token::Identifier(var_type), Token::Identifier(var_name), ..] => {
ast.push(Ast::Variable(var_type.clone(), var_name.clone()));
drain += 2;
// vars.new_var(var_name, State {}, desc)
}
let token = &self.tokens[index];
if self.json
&& self.tokens.len() - (self.index as usize) > 2
&& self.tokens[index + 1].value == ":"
{
ast_res.ast_type = AstType::Json;
ast_res.tokens.push(self.tokens[index].clone());
ast_res.tokens.push(self.tokens[index + 2].clone());
self.index += 2;
} else if self.tokens.len() - index > 1
&& self.tokens[index].value == "&"
&& self.tokens[index + 1].token_type == TokenType::Identifier
{
ast_res.tokens.push(self.tokens[index + 1].clone());
ast_res.ast_type = AstType::Ref;
self.index += 1;
} else if self.tokens.len() - index > 2
&& self.tokens[index].value == "enum"
&& self.tokens[index + 1].token_type == TokenType::Identifier
&& self.tokens[index + 2].token_type == TokenType::Curly
{
ast_res.tokens.push(self.tokens[index + 1].clone());
ast_res.tokens.push(self.tokens[index + 2].clone());
ast_res.ast_type = AstType::EnumDeceleration;
self.index += 2;
let mut desc = String::new();
if index > 0 && self.tokens[index - 1].token_type == TokenType::Comment {
desc = self.tokens[index - 1].value.clone()
}
self.variables.new_enum(
self.tokens[index + 1].clone().value,
LexerState {
line: self.tokens[index + 1].clone().line,
column: self.tokens[index + 1].clone().column,
},
desc,
);
} else if self.tokens.len() - index > 2
&& self.tokens[index].value == "struct"
&& self.tokens[index + 1].token_type == TokenType::Identifier
&& self.tokens[index + 2].token_type == TokenType::Curly
{
ast_res.tokens.push(self.tokens[index + 1].clone());
ast_res.tokens.push(self.tokens[index + 2].clone());
ast_res.ast_type = AstType::StructDeceleration;
self.index += 2;
let mut desc = String::new();
if index > 0 && self.tokens[index - 1].token_type == TokenType::Comment {
desc = self.tokens[index - 1].value.clone()
}
self.variables.new_struct(
self.tokens[index + 1].clone().value,
LexerState {
line: self.tokens[index + 1].clone().line,
column: self.tokens[index + 1].clone().column,
},
desc,
);
} else if self.tokens.len() - index > 2
&& self.tokens[index].value == "namespace"
&& self.tokens[index + 1].token_type == TokenType::Identifier
&& self.tokens[index + 2].token_type == TokenType::Curly
{
ast_res.tokens.push(self.tokens[index + 1].clone());
ast_res.tokens.push(self.tokens[index + 2].clone());
ast_res.ast_type = AstType::Namespace;
self.index += 2;
let mut desc = String::new();
if index > 0 && self.tokens[index - 1].token_type == TokenType::Comment {
desc = self.tokens[index - 1].value.clone()
}
self.variables.new_namespace(
self.tokens[index + 1].clone().value,
LexerState {
line: self.tokens[index + 1].clone().line,
column: self.tokens[index + 1].clone().column,
},
desc,
);
} else if self.tokens.len() - index > 2
&& self.tokens[index].value == "impl"
&& self.tokens[index + 1].token_type == TokenType::Identifier
&& self.tokens[index + 2].token_type == TokenType::Curly
{
ast_res.tokens.push(self.tokens[index + 1].clone());
ast_res.tokens.push(self.tokens[index + 2].clone());
ast_res.ast_type = AstType::Impl;
self.index += 2;
} else if self.tokens.len() - index > 2
&& self.tokens[index].token_type == TokenType::Keyword1
&& self.tokens[index + 1].token_type == TokenType::Round
&& self.tokens[index + 2].token_type == TokenType::Curly
{
ast_res.tokens.push(self.tokens[index].clone());
ast_res.tokens.push(self.tokens[index + 1].clone());
ast_res.tokens.push(self.tokens[index + 2].clone());
ast_res.ast_type = AstType::State3;
self.index += 2;
} else if self.tokens.len() - index > 1
&& self.tokens[index].token_type == TokenType::Keyword2
&& self.tokens[index + 1].token_type == TokenType::Curly
{
ast_res.tokens.push(self.tokens[index].clone());
ast_res.tokens.push(self.tokens[index + 1].clone());
ast_res.ast_type = AstType::State2;
self.index += 1;
} else {
match token.token_type {
TokenType::Identifier => {
ast_res.tokens.push(self.tokens[index].clone());
if self.tokens.len() - index > 3
&& self.tokens[index + 1].token_type == TokenType::Identifier
&& self.tokens[index + 2].token_type == TokenType::Round
&& self.tokens[index + 3].token_type == TokenType::Curly
{
ast_res.tokens.push(self.tokens[index + 1].clone());
ast_res.tokens.push(self.tokens[index + 2].clone());
ast_res.tokens.push(self.tokens[index + 3].clone());
if token.value == "void" {
ast_res.ast_type = AstType::VoidFunctionDeceleration;
} else {
ast_res.ast_type = AstType::FunctionDeceleration;
}
self.index += 3;
let mut desc = String::new();
if index > 0 && self.tokens[index - 1].token_type == TokenType::Comment
{
desc = self.tokens[index - 1].value.clone()
}
self.variables.new_func(
self.tokens[index + 1].clone().value,
LexerState {
line: self.tokens[index + 1].clone().line,
column: self.tokens[index + 1].clone().column,
},
desc,
);
} else if self.tokens.len() - index > 1
&& self.tokens[index + 1].token_type == TokenType::Curly
{
ast_res.tokens.push(self.tokens[index + 1].clone());
ast_res.ast_type = AstType::StructCall;
self.index += 1;
} else if self.tokens.len() - index > 2
&& self.tokens[index + 1].token_type == TokenType::Identifier
&& self.tokens[index + 2].token_type == TokenType::Curly
{
ast_res.tokens.push(self.tokens[index + 1].clone());
ast_res.tokens.push(self.tokens[index + 2].clone());
ast_res.ast_type = AstType::StructVar;
self.index += 2;
let mut desc = String::new();
if index > 0 && self.tokens[index - 1].token_type == TokenType::Comment
{
desc = self.tokens[index - 1].value.clone()
}
self.variables.new_var(
self.tokens[index + 1].clone().value,
LexerState {
line: self.tokens[index + 1].clone().line,
column: self.tokens[index + 1].clone().column,
},
desc,
);
} else if self.tokens.len() - index > 1 {
if self.tokens[index + 1].token_type == TokenType::Identifier {
ast_res.tokens.push(self.tokens[index + 1].clone());
ast_res.ast_type = AstType::VariableDeceleration;
self.index += 1;
let mut desc = String::new();
if index > 0
&& self.tokens[index - 1].token_type == TokenType::Comment
{
desc = self.tokens[index - 1].value.clone()
}
self.variables.new_var(
self.tokens[index + 1].clone().value,
LexerState {
line: self.tokens[index + 1].clone().line,
column: self.tokens[index + 1].clone().column,
},
desc,
);
} else if self.tokens.len() - index > 2
&& self.tokens[index + 2].token_type == TokenType::Identifier
&& self.tokens[index + 1].token_type == TokenType::Angle
{
ast_res.tokens.push(self.tokens[index + 2].clone());
ast_res.tokens[0].value += "<";
ast_res.tokens[0].value += self.tokens[index + 1].value.as_str();
ast_res.tokens[0].value += ">";
ast_res.ast_type = AstType::VariableDeceleration;
self.index += 2;
let mut desc = String::new();
if index > 0
&& self.tokens[index - 1].token_type == TokenType::Comment
{
desc = self.tokens[index - 1].value.clone()
}
self.variables.new_var(
self.tokens[index + 1].clone().value,
LexerState {
line: self.tokens[index + 1].clone().line,
column: self.tokens[index + 1].clone().column,
},
desc,
);
} else if self.tokens.len() - index > 2
&& self.tokens[index + 1].value == "*"
&& self.tokens[index + 2].token_type == TokenType::Identifier
{
ast_res.tokens.push(self.tokens[index + 2].clone());
ast_res.ast_type = AstType::PointerDeceleration;
self.index += 2;
let mut desc = String::new();
if index > 0
&& self.tokens[index - 1].token_type == TokenType::Comment
{
desc = self.tokens[index - 1].value.clone()
}
self.variables.new_var(
self.tokens[index + 1].clone().value,
LexerState {
line: self.tokens[index + 1].clone().line,
column: self.tokens[index + 1].clone().column,
},
desc,
);
}
}
}
TokenType::Include => {
if let Some(caps) = self.include_regex.captures(&token.value) {
ast_res.tokens.push(Token {
token_type: TokenType::String,
value: caps[2].to_owned().to_string(),
line: 0,
column: 0,
});
ast_res.ast_type = AstType::Include;
} else if let Some(caps) = self.include_regex_local.captures(&token.value) {
ast_res.tokens.push(Token {
token_type: TokenType::String,
value: caps[2].to_owned().to_string(),
line: 0,
column: 0,
});
ast_res.ast_type = AstType::IncludeLocal;
} else {
ast_res.tokens.push(token.clone());
ast_res.ast_type = AstType::Include;
}
}
TokenType::Keyword => {
if token.value == "cb"
&& self.tokens[index + 1].token_type == TokenType::Curly
{
ast_res.tokens.push(self.tokens[index + 1].clone());
ast_res.ast_type = AstType::CodeBlock;
self.index += 1;
} else {
ast_res.tokens.push(token.clone());
}
}
TokenType::StaticExecution => {
if self.tokens[index + 1].token_type == TokenType::Square {
ast_res.tokens.push(self.tokens[index + 1].clone());
ast_res.ast_type = AstType::StaticExecution;
}
}
_ => {
ast_res.tokens.push(token.clone());
}
}
}
self.index += 1;
full_ast.push(ast_res);
_ => {}
}
full_ast
tokens.drain(0..drain);
}
ast
}