Merge pull request #75 from mist-go/module-system

Module system

Cargo.lock

@@ -122,7 +122,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856"
 dependencies = [
  "cfg-if",
- "hashbrown",
+ "hashbrown 0.14.5",
  "lock_api",
  "once_cell",
  "parking_lot_core",
@@ -136,7 +136,7 @@ checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf"
 dependencies = [
  "cfg-if",
  "crossbeam-utils",
- "hashbrown",
+ "hashbrown 0.14.5",
  "lock_api",
  "once_cell",
  "parking_lot_core",
@@ -163,6 +163,12 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "equivalent"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f"
+
 [[package]]
 name = "errno"
 version = "0.3.14"
@@ -274,6 +280,12 @@ version = "0.14.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1"
 
+[[package]]
+name = "hashbrown"
+version = "0.17.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a"
+
 [[package]]
 name = "httparse"
 version = "1.10.1"
@@ -383,6 +395,16 @@ dependencies = [
  "icu_properties",
 ]
 
+[[package]]
+name = "indexmap"
+version = "2.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9"
+dependencies = [
+ "equivalent",
+ "hashbrown 0.17.1",
+]
+
 [[package]]
 name = "itoa"
 version = "1.0.18"
@@ -464,6 +486,7 @@ dependencies = [
  "mist-parser",
  "serde",
  "serde_json",
+ "toml",
 ]
 
 [[package]]
@@ -711,6 +734,15 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "serde_spanned"
+version = "1.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6662b5879511e06e8999a8a235d848113e942c9124f211511b16466ee2995f26"
+dependencies = [
+ "serde_core",
+]
+
 [[package]]
 name = "sha2"
 version = "0.10.9"
@@ -859,6 +891,45 @@ dependencies = [
  "tokio",
 ]
 
+[[package]]
+name = "toml"
+version = "1.1.2+spec-1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "81f3d15e84cbcd896376e6730314d59fb5a87f31e4b038454184435cd57defee"
+dependencies = [
+ "indexmap",
+ "serde_core",
+ "serde_spanned",
+ "toml_datetime",
+ "toml_parser",
+ "toml_writer",
+ "winnow",
+]
+
+[[package]]
+name = "toml_datetime"
+version = "1.1.1+spec-1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7"
+dependencies = [
+ "serde_core",
+]
+
+[[package]]
+name = "toml_parser"
+version = "1.1.2+spec-1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526"
+dependencies = [
+ "winnow",
+]
+
+[[package]]
+name = "toml_writer"
+version = "1.1.1+spec-1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "756daf9b1013ebe47a8776667b466417e2d4c5679d441c26230efd9ef78692db"
+
 [[package]]
 name = "tower"
 version = "0.4.13"
@@ -1014,6 +1085,12 @@ dependencies = [
  "windows-link",
 ]
 
+[[package]]
+name = "winnow"
+version = "1.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0592e1c9d151f854e6fd382574c3a0855250e1d9b2f99d9281c6e6391af352f1"
+
 [[package]]
 name = "writeable"
 version = "0.6.3"

Mist.toml

@@ -1 +1,2 @@
 package = "main.mist"
+packages = ["analyzer.mist"]

crates/analyzer/src/transpiler.rs

@@ -3,7 +3,10 @@ use std::{
     path::{Path, PathBuf},
 };
 
-use mist_parser::{ast::TopLevel, error::ParseError};
+use mist_parser::{
+    ast::{Identifier, TopLevel, Visibility},
+    error::ParseError,
+};
 
 pub fn build(root: &PathBuf) {
     let src_dir = root.join("src");
@@ -133,7 +136,9 @@ pub fn transpile_file(path: &Path, output_path: &Path) -> Result<(), String> {
     });
 
     // If parsing fails, return the error string back gracefully so the LSP can show it
-    let ast = parser_result.map_err(|e| format!("parse failed in {}:\n{}", path.display(), e))?;
+    let ast = parser_result
+        .map_err(|e| format!("parse failed in {}:\n{}", path.display(), e))?
+        .1;
 
     let mut gc = mist_codegen::RustCodegen::new();
     let output = gc.generate(ast);
@@ -144,10 +149,12 @@ pub fn transpile_file(path: &Path, output_path: &Path) -> Result<(), String> {
     Ok(())
 }
 
-pub fn transpile_text<'a>(source: &'a str) -> Result<String, ParseError<'a, Vec<TopLevel>>> {
+pub fn transpile_text<'a>(
+    source: &'a str,
+) -> Result<String, ParseError<'a, (Option<(Visibility, Identifier)>, Vec<TopLevel>)>> {
     let mut gc = mist_codegen::RustCodegen::new();
 
-    Ok(gc.generate(mist_parser::parse(&source)?))
+    Ok(gc.generate(mist_parser::parse(&source)?.1))
 }
 
 fn should_skip(source: &Path, output: &Path) -> bool {

crates/codegen/src/top_level.rs

@@ -177,7 +177,7 @@ impl GenRust for TopLevelKind {
             Self::Import(vis, path) => {
                 cg.addln(&format!("{}use {};", vis.get_rust(), path.get_rust()))
             }
-            Self::Mod(vis, id) => cg.addln(&format!("{}mod {};", vis.get_rust(), id.get_rust())),
+            Self::DeclareModule(_, _) => {}
             Self::FunctionDecl(decl) => decl.gen_rust(ctx, cg),
             Self::ImplDecl(impl_) => impl_.gen_rust(ctx, cg),
             Self::StructDecl {

crates/mist-api/Cargo.toml

@@ -20,6 +20,7 @@ serde_json = "1.0.149"
 cargo_metadata = "0.23.1"
 mist-parser = { workspace = true }
 mist-codegen = { workspace = true }
+toml = "1.1.2"
 
 [lib]
 name = "mist_api"

crates/mist-api/src/modules.mist

@@ -3,13 +3,15 @@ use std::path::PathBuf;
 use std::fs;
 use std::process;
 
+use crate::transpiler::MistConfig;
+
 #[derive(Debug)]
 pub class Module {
     pub name String;
     pub path PathBuf;
-    pub children Vec<Self>;
+    pub children Vec<Box<Module>>;
 
-    pub constructor(mist_path PathBuf) {
+    pub constructor(mist_path PathBuf, config *MistConfig) {
         self.children = Vec::new();
         self.name = mist_path.file_stem()
             .and_then(fn(s) -> s.to_str())
@@ -25,11 +27,15 @@ pub class Module {
                 self.path = pkg_gate;
             }
 
-            self.scan_directory(&dir);
+            self.scan_directory(&dir, config);
+        } else if (config.should_ignore(&self.path)) {
+            let dir = self.path.parent().expect("Failed to get parent directory to scan").to_path_buf();
+
+            self.scan_directory(&dir, config);
         }
     }
 
-    fn scan_directory(*mut self, search_dir *Path) {
+    fn scan_directory(*mut self, search_dir *Path, config *MistConfig) {
         let entries = match (fs::read_dir(search_dir)) {
             Ok[entries] => entries;
             Err[e] => {
@@ -49,16 +55,20 @@ pub class Module {
 
             let child_path = entry.path();
 
-            if (child_path == self.path) {
+            if (child_path == self.path || child_path == config.package) {
                 continue;
             }
 
-            self.children.push(Module::new(child_path));
+            self.children.push(Module::new(child_path, config));
         }
     }
 
+    pub fn is_package(*self, parent_dir *PathBuf) bool {
+        self.path.file_name().map(fn(v) -> v.to_str()).unwrap_or_default() == Some("package.mist")
+    }
+
     pub fn output_dir(*self, parent_dir *PathBuf) PathBuf {
-        if (self.path.is_dir() || Some("package.mist") == self.path.file_name().unwrap().to_str()) {
+        if (self.path.is_dir() || self.is_package(parent_dir)) {
             parent_dir.join(&self.name)
         } else {
             parent_dir.clone()
@@ -66,21 +76,10 @@ pub class Module {
     }
 
     pub fn output_path(*self, parent_dir *PathBuf) PathBuf {
-        if (self.path.is_dir() || Some("package.mist") == self.path.file_name().unwrap().to_str()) {
+        if (self.path.is_dir() || self.is_package(parent_dir)) {
             parent_dir.join(&self.name).join("mod.rs")
         } else {
             parent_dir.join(&self.name).with_extension("rs")
         }
     }
 }
-
-pub fn master_package(mist_path PathBuf) Module {
-    let mut root_mod = Module::new(mist_path.clone());
-
-    let parent_dir = mist_path.parent()
-        .expect("Failed to get parent directory of root file");
-
-    root_mod.scan_directory(parent_dir);
-
-    root_mod
-}

crates/mist-api/src/transpiler.mist

@@ -1,4 +1,5 @@
 use std::fs;
+use std::path::Path;
 use std::path::PathBuf;
 use std::process;
 use std::time;
@@ -6,21 +7,72 @@ use std::env;
 
 use mist_parser::error;
 use crate::modules;
+use mist_codegen::GetRust;
+
+use serde::Deserialize;
+
+#[derive(Debug, Deserialize)]
+#[serde(default)]
+pub struct MistConfig {
+    pub package PathBuf,
+    pub packages Vec<PathBuf>,
+    pub main_mods String
+}
+impl MistConfig {
+    pub fn should_ignore(*self, path *Path) bool {
+        if (path == self.package) {
+            return true;
+        }
+
+        for (pkg : &self.packages) {
+            if (path.ends_with(pkg)) {
+                return true;
+            }
+        }
+
+        false
+    }
+}
+impl Default for MistConfig {
+    fn default() Self {
+        Self {
+            package: PathBuf::from("main.mist"),
+            packages: Vec::new(),
+            main_mods: String::new(),
+        }
+    }
+}
 
 pub fn transpile() PathBuf {
     let start = time::Instant::now();
 
     let root = env::current_dir().expect("Unable to find project root");
 
+    let toml_path = root.join("Mist.toml");
     let src = root.join("src");
     let out = root.join(".mist/src");
 
-    let master = modules::master_package(src.join("main.mist"));
+    let toml_content = fs::read_to_string(&toml_path)
+        .unwrap_or_else(fn(_) -> panic!("Could not read file at {:?}", toml_path));
 
-    master.transpile(&out);
+    let mut config MistConfig = toml::from_str(&toml_content)
+        .unwrap_or_else(fn(_) -> panic!("Failed to parse Mist.toml from {:?}", toml_path));
+    
+    for (pkg : &mut config.packages) {
+        *pkg = src.join(&*pkg);
+    }
+
+    config.package = src.join(&*config.package);
+
+    let master = modules::Module::new(
+        config.package.clone(),
+        &config
+    );
+
+    master.transpile(&out, &mut config);
 
     let elapsed = start.elapsed();
-    
+
     println!(
         "\x1b[32m\nTranspile successful\x1b[0m in \x1b[34m{:.2?}\x1b[0m",
         elapsed
@@ -30,7 +82,7 @@ pub fn transpile() PathBuf {
 }
 
 impl modules::Module {
-    fn transpile(*self, parent_dir *PathBuf) String {
+    fn transpile(*self, parent_dir *PathBuf, config *mut MistConfig) String {
         if (!self.path.is_dir() && self.path.extension().and_then(fn(v) -> v.to_str()) != Some("mist")) {
             let file_name = self.path.file_name()
                 .and_then(fn(s) -> s.to_str())
@@ -49,9 +101,21 @@ impl modules::Module {
         let mut output = String::new();
 
         for (child : &self.children) {
-            output.push_str(&child.transpile(&dir));
+            if (!config.should_ignore(&child.path)) {
+                output.push_str(&child.transpile(&dir, config));
+            }
         }
 
+        if (self.path == config.package) {
+            config.main_mods = output.clone();
+        } else if (config.should_ignore(&self.path)) {
+            output.push_str(&config.main_mods)
+        }
+
+        self.write_transpile(output, output_file)
+    }
+
+    fn write_transpile(*self, output String, output_file PathBuf) String {
         if (self.path.is_dir()) {
             if (output.len() == 0) {
                 return String::new();
@@ -68,16 +132,16 @@ impl modules::Module {
 
                 process::exit(1);
             }
-        } else {
-            transpile_file(&self.path, &output_file, &output);
-        }
 
-        format!("pub mod {};\n", self.name)
+            format!("pub mod {};\n", self.name)
+        } else {
+            transpile_file(&self.path, output_file, &output)
+        }
     }
 }
 
-fn transpile_file(path *PathBuf, output_file *PathBuf, mod_decl *str) {
+fn transpile_file(path *PathBuf, mut output_file PathBuf, mod_decl *str) String {
-    let source = match (fs::read_to_string(path)) {
+    let source = match (fs::read_to_string(&path)) {
         Ok[s] => s;
         Err[e] => {
             eprintln!("error: failed to read file {}\n  {}", path.display(), e);
@@ -88,7 +152,7 @@ fn transpile_file(path *PathBuf, output_file *PathBuf, mod_decl *str) {
 
     let mut gc = mist_codegen::RustCodegen::new();
 
-    let output = gc.generate(match (mist_parser::parse(&source)) {
+    let parsed = match (mist_parser::parse(&source)) {
         Ok[ast] => ast;
         Err[e] => {
             match (e) {
@@ -118,18 +182,51 @@ fn transpile_file(path *PathBuf, output_file *PathBuf, mod_decl *str) {
                 }
             }
         }
-    });
+    };
 
-    let res = fs::write(&output_file, format!("{mod_decl}{output}"));
+    let output = gc.generate(parsed.1);
 
-    if (res.is_err()) {
+    if (let [Some[[vis, name]], _] = parsed) {
-        eprintln!(
+        if (output_file.file_name().map(fn(v) -> v.to_str()).unwrap_or_default() != Some("mod.rs")) {
-            "error: failed to write output {}\n  {}",
+            if (let Some[ext] = output_file.extension().map(fn(e) -> e.to_owned())) {
-            output_file.display(),
+                let mut new_name = std::ffi::OsString::from(&name.0);
-            res.unwrap_err(),
-        );
 
-        process::exit(1);
+                new_name.push(".");
+                new_name.push(ext);
+
+                output_file.set_file_name(new_name);
+            } else {
+                output_file.set_file_name(&name.0);
+            }
+        }
+
+        let res = fs::write(&output_file, format!("{mod_decl}{output}"));
+
+        if (res.is_err()) {
+            eprintln!(
+                "error: failed to write output {}\n  {}",
+                output_file.display(),
+                res.unwrap_err(),
+            );
+
+            process::exit(1);
+        }
+
+        format!("{}mod {};\n", vis.get_rust(), name.get_rust())
+    } else {
+        let res = fs::write(&output_file, format!("{mod_decl}{output}"));
+
+        if (res.is_err()) {
+            eprintln!(
+                "error: failed to write output {}\n  {}",
+                output_file.display(),
+                res.unwrap_err(),
+            );
+
+            process::exit(1);
+        }
+
+        String::new()
     }
 }
 

crates/parser/src/ast/top_level.rs

@@ -28,7 +28,7 @@ pub struct TopLevel(pub Spanned<TopLevelKind>, pub Vec<Attribute>);
 pub enum TopLevelKind {
     ModAttribute,
     Import(Visibility, Path),
-    Mod(Visibility, Identifier),
+    DeclareModule(Visibility, Identifier),
     ImplDecl(ImplDecl),
     TraitDecl {
         visibility: Visibility,

crates/parser/src/error.rs

@@ -24,6 +24,7 @@ pub struct AstError<'a, T> {
 pub enum ErrorCode {
     InvalidStatement,
     AstGenBug,
+    Module,
 }
 
 impl<T> From<pest::error::Error<Rule>> for ParseError<'_, T> {

crates/parser/src/grammar.pest

@@ -220,8 +220,8 @@ import = {
     visibility? ~ "use" ~ static_path ~ semicolon
 }
 
-mod_package = {
+declare_module = {
-    visibility? ~ "mod" ~ identifier ~ semicolon
+    visibility? ~ "module" ~ identifier ~ semicolon
 }
 
 // ======================================================
@@ -334,7 +334,7 @@ function_decl = {
 // ======================================================
 
 top_level = {
-    (attributes ~ (import | impl_for_decl | impl_decl | trait_decl | struct_decl | enum_decl | class_decl | mod_package | function_decl))
+    (attributes ~ (import | impl_for_decl | impl_decl | trait_decl | struct_decl | enum_decl | class_decl | declare_module | function_decl))
   | mod_attribute
 }
 

crates/parser/src/lib.rs

@@ -14,7 +14,12 @@ use crate::error::{IntoErr, ParseError};
 #[grammar = "./src/grammar.pest"]
 pub struct MistParser;
 
-pub fn parse<'a>(source: &'a str) -> Result<Vec<TopLevel>, ParseError<'a, Vec<TopLevel>>> {
+pub fn parse<'a>(
+    source: &'a str,
+) -> Result<
+    (Option<(Visibility, Identifier)>, Vec<TopLevel>),
+    ParseError<'a, (Option<(Visibility, Identifier)>, Vec<TopLevel>)>,
+> {
     let mut pairs = MistParser::parse(Rule::program, source)?;
 
     let mut statements = vec![];
@@ -27,9 +32,27 @@ pub fn parse<'a>(source: &'a str) -> Result<Vec<TopLevel>, ParseError<'a, Vec<To
         }
     }
 
+    let m = if let Some(v) = statements.get(0) {
+        if let TopLevelKind::DeclareModule(vis, name) = &v.0.item {
+            Some((vis.clone(), name.clone()))
+        } else {
+            None
+        }
+    } else {
+        None
+    };
+
     match analyzer.build(statements) {
-        Ok(v) => Ok(v),
+        Ok(v) => Ok((m, v)),
-        Err(e) => Err(ParseError::Ast(e)),
+        Err(e) => {
+            let rec = e.recovered.clone().map(move |r| (m, r));
+
+            let mut e2 = e.get();
+
+            e2.recovered = rec;
+
+            Err(ParseError::Ast(e2))
+        }
     }
 }
 

crates/parser/src/parser/items/mod.rs

@@ -93,7 +93,7 @@ impl<'a> TryFrom<pest::iterators::Pair<'a, Rule>> for TopLevelKind {
                 fields: collect_recovered(inner),
             }),
 
-            Rule::mod_package => ast_expr!(TopLevelKind::Mod(
+            Rule::declare_module => ast_expr!(TopLevelKind::DeclareModule(
                 Visibility::try_from(&mut inner),
                 inner.next().unwrap().try_into(),
             )),

mist-test/Mist.toml

@@ -1 +1,2 @@
 package = "main.mist"
+packages = []

mist-test/src/common/attribute_test.mist

@@ -1,21 +0,0 @@
-#[derive(Debug)]
-pub struct Debuggable {
-    value i32,
-}
-
-pub fn test_derive_attribute() {
-    let d = Debuggable { value: 42 };
-    assert_eq!(d.value, 42);
-}
-
-#[allow(unused_variables)]
-pub fn test_allow_attribute() {
-    let x = 42;
-    let y = "unused";
-    assert_eq!(x, 42);
-}
-
-pub fn run_tests() {
-    test_derive_attribute();
-    test_allow_attribute();
-}

mist-test/src/common/decl_test.mist

@@ -1,3 +1,5 @@
+pub module decl_test;
+
 pub fn test_let_basic() {
     let x = 42;
     assert_eq!(x, 42);

mist-test/src/common/expr_test.mist

@@ -1,3 +1,5 @@
+pub module expr_test;
+
 pub fn test_literals() {
     assert_eq!(42, 42);
     assert_eq!(3.14, 3.14);

mist-test/src/common/package.mist

@@ -0,0 +1,3 @@
+pub module common;
+
+pub fn test_package() {}

mist-test/src/common/statement_test.mist

@@ -1,3 +1,5 @@
+pub module statement_test;
+
 pub fn test_var_decl() {
     let x = 42;
     assert_eq!(x, 42);

mist-test/src/common/types_test.mist

@@ -1,3 +1,5 @@
+pub module types_test;
+
 pub fn test_integer_type() {
     let x i32 = 42;
     assert_eq!(x, 42);

mist-test/src/items/attribute_test.mist

@@ -1,3 +1,5 @@
+pub module attribute_test;
+
 #[derive(Debug)]
 pub struct Debuggable {
     value i32,

mist-test/src/items/class_test.mist

@@ -1,3 +1,5 @@
+pub module class_test;
+
 pub class Animal<T> {
     pub name String;
     pub val T;

mist-test/src/items/enum_test.mist

@@ -1,3 +1,5 @@
+pub module enum_test;
+
 enum OptionInt {
     None,
     Some[i32],

mist-test/src/items/function_test.mist

@@ -1,3 +1,5 @@
+pub module function_test;
+
 fn greet() {
     let mut x = 1;
     x++;

mist-test/src/items/impl_test.mist

@@ -1,3 +1,5 @@
+pub module impl_test;
+
 struct Vector2 {
     x i32,
     y i32,

mist-test/src/items/struct_test.mist

@@ -1,3 +1,5 @@
+pub module struct_test;
+
 struct Point {
     x i32,
     y i32,

mist-test/src/items/trait_test.mist

@@ -1,3 +1,5 @@
+pub module trait_test;
+
 trait Speak {
     fn speak(*self) String;
 }

mist-test/src/main.mist

@@ -1,5 +1,6 @@
 fn main() {
     println!("running common tests...");
+    common::test_package();
     common::expr_test::run_tests();
     common::statement_test::run_tests();
     common::decl_test::run_tests();