wyst-lang/wyst
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

A LOT of bug fixes

Browse Source
This commit is contained in:
selimaj-dev
2024-04-25 02:37:07 +02:00
parent 1626137ad6
commit cf588ab7e7
4 changed files with 140 additions and 206 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lexer.rs
+91 -120
View File
@@ -115,155 +115,132 @@ fn get_first_char(value: &str) -> String {
value.chars().next().unwrap().to_string() value.chars().next().unwrap().to_string()
} }
fn get_second_char(value: &str) -> String {
let mut sv2 = value.chars();
sv2.next();
sv2.next().unwrap().to_string()
}
pub fn lex(mut code: &str, use_whitespace: bool) -> Result<Vec<Token>, (LexerState, Vec<Token>)> { pub fn lex(mut code: &str, use_whitespace: bool) -> Result<Vec<Token>, (LexerState, Vec<Token>)> {
let mut state = LexerState { line: 1, column: 1 }; let mut state = LexerState { line: 1, column: 1 };
let mut tokens: Vec<Token> = Vec::new(); let mut tokens: Vec<Token> = Vec::new();
let mut bracket_vec: Vec<char> = Vec::new(); let mut brstr: String = String::new();
let mut inner_string = String::new(); let mut brvct: Vec<u8> = Vec::new();
// let mut bracket_state: LexerState = LexerState {line: 0, column: 0}; let mut inside_str: u8 = 0;
while !code.is_empty() { while !code.is_empty() {
let mut is_match = false; let mut is_match = false;
let svl = bracket_vec.len(); let fch = get_first_char(code);
match code.chars().next().unwrap() { match fch.as_str() {
'\"' => { "\"" => {
is_match = true; brstr += "\"";
code = code.strip_prefix("\"").unwrap_or(code); code = code.strip_prefix(fch.as_str()).expect("");
inner_string += "\""; if inside_str == 1 {
if svl == 0 { inside_str = 0;
bracket_vec.push('\"'); if brvct.len() == 0 {
} else if bracket_vec[svl-1] == '\"' {
bracket_vec.pop();
if svl == 1 {
tokens.push(Token { tokens.push(Token {
token_type: TokenType::String, token_type: TokenType::String,
value: inner_string.clone(), value: brstr.clone(),
line: state.line, column: state.line,
column: state.column line: state.column
}); });
inner_string = String::new(); brstr = String::new();
}
} else if inside_str == 0 {
inside_str = 1;
} }
} }
} "'" => {
'\'' => { brstr += "'";
is_match = true; code = code.strip_prefix(fch.as_str()).expect("");
code = code.strip_prefix("\'").unwrap_or(code); if inside_str == 2 {
inner_string += "\'"; inside_str = 0;
if svl == 0 { if brvct.len() == 0 {
bracket_vec.push('\'');
} else if bracket_vec[svl-1] == '\'' {
bracket_vec.pop();
if svl == 1 {
tokens.push(Token { tokens.push(Token {
token_type: TokenType::String, token_type: TokenType::String,
value: inner_string.clone(), value: brstr.clone(),
line: state.line, column: state.line,
column: state.column line: state.column
}); });
inner_string = String::new(); brstr = String::new();
}
} else if inside_str == 0 {
inside_str = 2;
} }
} }
} "\\" => {
'{' => { brstr += "\\";
is_match = true; code = code.strip_prefix(fch.as_str()).expect("");
code = code.strip_prefix("{").unwrap_or(code); if code.len() > 0 {
if svl == 0 { let sch = get_first_char(code);
bracket_vec.push('{'); code = code.strip_prefix(sch.as_str()).expect("");
brstr += &sch;
} }
} }
'}' => { "{" => {
is_match = true; code = code.strip_prefix(fch.as_str()).expect("");
code = code.strip_prefix("}").unwrap_or(code); if brvct.len() > 0 {
if bracket_vec[svl-1] == '{' { brstr += "{";
bracket_vec.pop(); }
brvct.push(0);
}
"}" => {
code = code.strip_prefix(fch.as_str()).expect("");
brvct.pop();
if brvct.len() == 0 {
tokens.push(Token { tokens.push(Token {
token_type: TokenType::Curly, token_type: TokenType::Curly,
value: inner_string.clone(), value: brstr.clone(),
line: state.line, column: state.line,
column: state.column line: state.column
}); });
inner_string = String::new(); } else {
state.column += inner_string.len(); brstr += "}";
} }
} }
'(' => { "(" => {
is_match = true; code = code.strip_prefix(fch.as_str()).expect("");
code = code.strip_prefix("(").unwrap_or(code); if brvct.len() > 0 {
if svl == 0 { brstr += "(";
bracket_vec.push('(');
} }
brvct.push(1);
} }
')' => { ")" => {
is_match = true; code = code.strip_prefix(fch.as_str()).expect("");
code = code.strip_prefix(")").unwrap_or(code); brvct.pop();
if bracket_vec[svl-1] == '(' { if brvct.len() == 0 {
bracket_vec.pop();
tokens.push(Token { tokens.push(Token {
token_type: TokenType::Round, token_type: TokenType::Round,
value: inner_string.clone(), value: brstr.clone(),
line: state.line, column: state.line,
column: state.column line: state.column
}); });
inner_string = String::new(); } else {
state.column += inner_string.len(); brstr += ")";
} }
} }
'[' => { "[" => {
is_match = true; code = code.strip_prefix(fch.as_str()).expect("");
code = code.strip_prefix("[").unwrap_or(code); if brvct.len() > 0 {
brstr += "[";
if svl == 0 {
bracket_vec.push('[');
} }
brvct.push(2);
} }
']' => { "]" => {
is_match = true; code = code.strip_prefix(fch.as_str()).expect("");
code = code.strip_prefix("]").unwrap_or(code); brvct.pop();
if bracket_vec[svl-1] == '[' { if brvct.len() == 0 {
bracket_vec.pop();
tokens.push(Token { tokens.push(Token {
token_type: TokenType::Square, token_type: TokenType::Square,
value: inner_string.clone(), value: brstr.clone(),
line: state.line, column: state.line,
column: state.column line: state.column
}); });
inner_string = String::new();
state.column += inner_string.len();
}
}
'\n' => {
is_match = true;
code = code.strip_prefix("\n").unwrap_or(code);
state.line += 1;
state.column = 1;
if svl > 0 {
inner_string+="\n";
} else { } else {
tokens.push(Token { brstr += "]";
token_type: TokenType::Newline,
value: "\n".to_string(),
line: state.line,
column: state.column
});
} }
} }
"\n" => {
code = code.strip_prefix(fch.as_str()).expect("");
state.line += 1;
}
_ => { _ => {
if svl > 0 { if inside_str > 0 || brvct.len() > 0 {
let cfc = get_first_char(&code); code = code.strip_prefix(fch.as_str()).expect("");
let cfc1 = get_second_char(&code); brstr += fch.as_str();
inner_string += cfc.as_str();
code = code.strip_prefix(&cfc).unwrap_or(code);
is_match = true;
if cfc == "\\" && (cfc1 == "\"" || cfc1 == "'") {
inner_string += cfc1.as_str();
code = code.strip_prefix(&cfc1).unwrap_or(code);
is_match = true;
}
} else { } else {
for s in &SYNTAX { for s in &SYNTAX {
if let Some(caps) = s.token_regex.captures(code) { if let Some(caps) = s.token_regex.captures(code) {
@@ -277,23 +254,17 @@ pub fn lex(mut code: &str, use_whitespace: bool) -> Result<Vec<Token>, (LexerSta
column: state.column column: state.column
}); });
} }
match s.token_type {
_ => {
state.column += caps[0].len(); state.column += caps[0].len();
}
}
} else { } else {
continue; continue;
}; };
break; break;
} }
}
}
}
if !is_match { if !is_match {
return Err((state, tokens)); return Err((state, tokens));
// break; }
}
}
} }
} }
Ok(tokens) Ok(tokens)
src/main.rs
+1 -1
View File
@@ -5,7 +5,7 @@ use std::fs;
fn main() { fn main() {
let code_file = "code.ws"; let code_file = "code.wst";
let contents = fs::read_to_string(code_file).expect("cannot read for some reason"); let contents = fs::read_to_string(code_file).expect("cannot read for some reason");
let result = transpiler::transpile(contents, 0); let result = transpiler::transpile(contents, 0);
println!("{result}") println!("{result}")
src/parser.rs
+11 -21
View File
@@ -11,6 +11,7 @@ pub enum AstType {
MutVariableDeceleration, MutVariableDeceleration,
Include, Include,
IncludeLocal, IncludeLocal,
CodeBlock,
Other Other
} }
@@ -65,13 +66,12 @@ impl Parser {
} }
pub fn next(&mut self) -> Ast { pub fn next(&mut self) -> Ast {
let mut ast_res: Ast = Ast {tokens: vec![], ast_type: AstType::Other}; let mut ast_res: Ast = Ast {tokens: vec![], ast_type: AstType::Other};
let mut index = self.index as usize; let index = self.index as usize;
if index == self.tokens.len() {panic!("Reached the end of tokens")} if index == self.tokens.len() {panic!("Reached the end of tokens")}
let token = &self.tokens[index]; let token = &self.tokens[index];
match token.token_type { match token.token_type {
TokenType::Identifier => { TokenType::Identifier => {
ast_res.tokens.push(self.tokens[index].clone()); ast_res.tokens.push(self.tokens[index].clone());
self.index += 1;
if self.tokens.len()-index > 3 && self.tokens[index+1].token_type==TokenType::Identifier && self.tokens[index+2].token_type==TokenType::Round && self.tokens[index+3].token_type==TokenType::Curly { if self.tokens.len()-index > 3 && self.tokens[index+1].token_type==TokenType::Identifier && self.tokens[index+2].token_type==TokenType::Round && self.tokens[index+3].token_type==TokenType::Curly {
ast_res.tokens.push(self.tokens[index+1].clone()); ast_res.tokens.push(self.tokens[index+1].clone());
ast_res.tokens.push(self.tokens[index+2].clone()); ast_res.tokens.push(self.tokens[index+2].clone());
@@ -81,29 +81,19 @@ impl Parser {
} else { } else {
ast_res.ast_type = AstType::FunctionDeceleration; ast_res.ast_type = AstType::FunctionDeceleration;
} }
self.index += 2; self.index += 3;
} else { } else if self.tokens.len()-index > 1 && token.value == "cb" && self.tokens[index+1].token_type == TokenType::Curly {
loop { ast_res.tokens.push(self.tokens[index+1].clone());
if index+1 >= self.tokens.len() {break;} ast_res.ast_type = AstType::CodeBlock;
index = self.index as usize; // Update the index
let ntk = &self.tokens[index]; // Stands for next token
if ntk.value=="," {
ast_res.tokens.push(ntk.clone());
self.index += 1; self.index += 1;
} else if ntk.token_type==TokenType::Identifier { } else if self.tokens[index+1].token_type==TokenType::Identifier {
ast_res.tokens.push(ntk.clone()); ast_res.tokens.push(self.tokens[index+1].clone());
if ast_res.ast_type != AstType::VariableDeceleration && ast_res.ast_type != AstType::MutVariableDeceleration {
ast_res.ast_type = AstType::VariableDeceleration; ast_res.ast_type = AstType::VariableDeceleration;
}
self.index += 1; self.index += 1;
} else if ntk.value=="mut" { } else if self.tokens[index+1].value=="mut" && self.tokens[index+2].token_type==TokenType::Identifier {
ast_res.tokens.push(self.tokens[index+2].clone());
ast_res.ast_type = AstType::MutVariableDeceleration; ast_res.ast_type = AstType::MutVariableDeceleration;
self.index += 1; self.index += 2;
} else {
self.index -= 1;
break;
}
}
} }
} }
TokenType::Include => { TokenType::Include => {
src/transpiler.rs
+17 -44
View File
@@ -3,69 +3,42 @@ use crate::parser::{Parser, AstType};
pub fn transpile(input: String, indent: u32) -> String { pub fn transpile(input: String, indent: u32) -> String {
let mut result = String::new(); let mut result = String::new();
if indent > 0 {
result += " ".repeat((indent as usize)*2).as_str();
}
let lexer_out = lex(input.as_str(), false); let lexer_out = lex(input.as_str(), false);
match lexer_out { match lexer_out {
Ok(tokens) => { Ok(tokens) => {
println!("\n\n\n\n");
let mut full_ast = Parser::new(tokens.clone()); let mut full_ast = Parser::new(tokens.clone());
while full_ast.tokens.len() > full_ast.index as usize { while full_ast.tokens.len() > full_ast.index as usize {
let ast = full_ast.next(); let ast = full_ast.next();
println!("{ast}"); println!("{ast}");
if ast.ast_type == AstType::FunctionDeceleration { if ast.ast_type == AstType::FunctionDeceleration {
result += format!("fn {}{} -> {} {}", ast.tokens[1].value, ast.tokens[2].value, ast.tokens[0].value, transpile(ast.tokens[3].value.clone(), indent+1)).as_str(); result += format!("fn {}({}) -> {} {}", ast.tokens[1].value, ast.tokens[2].value, ast.tokens[0].value, transpile(ast.tokens[3].value.clone(), indent+1)).as_str();
} else if ast.ast_type == AstType::VoidFunctionDeceleration { } else if ast.ast_type == AstType::VoidFunctionDeceleration {
result += format!("fn {}{} {}", ast.tokens[1].value, ast.tokens[2].value, transpile(ast.tokens[3].value.clone(), indent+1)).as_str(); result += format!("fn {}({}) {}", ast.tokens[1].value, ast.tokens[2].value, transpile(ast.tokens[3].value.clone(), indent+1)).as_str();
} else if ast.ast_type == AstType::VariableDeceleration { } else if ast.ast_type == AstType::VariableDeceleration {
let _o = String::new();
let mut oy = ast.tokens.clone();
oy.remove(0);
if oy.len() > 1 {
result += "(";
for t in &oy {
result+=t.value.as_str()
}
result += "): (";
for t in oy {
if t.value == "," {
result+=", "
} else {
result+=ast.tokens[0].value.as_str()
}
}
result += ")";
} else {
result += format!("let {}: {}", ast.tokens[1].value, ast.tokens[0].value).as_str(); result += format!("let {}: {}", ast.tokens[1].value, ast.tokens[0].value).as_str();
}
} else if ast.ast_type == AstType::MutVariableDeceleration { } else if ast.ast_type == AstType::MutVariableDeceleration {
let _o = String::new();
let mut oy = ast.tokens.clone();
oy.remove(0);
if oy.len() > 1 {
result += "(";
for t in &oy {
result+=t.value.as_str()
}
result += "): (";
for t in oy {
if t.value == "," {
result+=", "
} else {
result+=ast.tokens[0].value.as_str()
}
}
result += ")";
} else {
result += format!("let mut {}: {}", ast.tokens[1].value, ast.tokens[0].value).as_str(); result += format!("let mut {}: {}", ast.tokens[1].value, ast.tokens[0].value).as_str();
}
} else if ast.tokens.len() == 1 && ast.tokens[0].token_type == TokenType::Round { } else if ast.tokens.len() == 1 && ast.tokens[0].token_type == TokenType::Round {
result += format!("({})", ast.tokens[0].value).as_str(); result += format!("({})", ast.tokens[0].value).as_str();
} else { } else if ast.tokens.len() == 1 && ast.tokens[0].token_type == TokenType::Curly {
if ast.tokens[0].token_type==TokenType::Newline { if ast.tokens[0].token_type==TokenType::Newline {
result += (ast.tokens[0].value.as_str().to_owned() + (" ".repeat((indent as usize)*2).as_str())).as_str() result += (ast.tokens[0].value.as_str().to_owned() + (" ".repeat((indent as usize)*2).as_str())).as_str()
} }
else { else {
result += ast.tokens[0].value.as_str() result += ast.tokens[0].value.as_str()
} }
} else {
if ast.tokens[0].token_type==TokenType::Newline {
result += (ast.tokens[0].value.as_str().to_owned() + (" ".repeat((indent as usize)*2).as_str())).as_str()
} else if ast.tokens[0].token_type==TokenType::Semicolon {
result += ";\n";
result += " ".repeat((indent as usize)*2).as_str();
} else {
result += ast.tokens[0].value.as_str()
}
} }
} }
@@ -73,13 +46,13 @@ pub fn transpile(input: String, indent: u32) -> String {
if indent > 0 { if indent > 0 {
result += "\n"; result += "\n";
result += " ".repeat((indent as usize-1)*2).as_str(); result += " ".repeat((indent as usize-1)*2).as_str();
return "{".to_owned()+result.as_str()+"}" return "{\n".to_owned()+result.as_str()+"}"
} else { } else {
return result return result
} }
}, },
Err((state, _tokens)) => { Err((state, _tokens)) => {
println!("Invalid character at code.ws:{}:{}", state.line, state.column); println!("Invalid syntax at code.ws:{}:{}", state.line, state.column);
return "".to_string(); return "".to_string();
} }
} }