Merge pull request #99 from mist-go/subcrates-and-diagnostics

Subcrates and diagnostics

Cargo.lock

@@ -481,6 +481,7 @@ dependencies = [
  "cargo_metadata",
  "dashmap 6.1.0",
  "heck",
+ "mist-api",
  "mist-codegen",
  "mist-parser",
  "pest",

crates/analyzer/Cargo.toml

@@ -9,6 +9,7 @@ keywords = ["language-server", "language", "mist"]
 categories = ["compilers"]
 
 [dependencies]
+mist-api = { workspace = true }
 mist-parser = { workspace = true }
 mist-codegen = { workspace = true }
 pest = "2.8.6"

crates/analyzer/src/lib.rs

@@ -7,18 +7,21 @@ use std::sync::Arc;
 use std::sync::atomic::{AtomicU64, Ordering};
 
 use heck::ToSnakeCase;
+use mist_api::transpiler::MistConfig;
 use mist_parser::MistFmtConfig;
 use mist_parser::error::ParseError;
 use mist_parser::rev_mapper::{Mapping, MistMap, RustMap};
 use ropey::Rope;
-use serde::{Deserialize, Serialize};
+use serde::Deserialize;
 use serde_json::Value;
 use tokio::sync::Mutex;
 use tower_lsp::lsp_types::{self, *};
 use tower_lsp::{Client, LanguageServer, LspService, Server};
 
 use crate::rust_analyzer::RustAnalyzer;
-use crate::transpiler::{TranspileError, format_mist, transpile_mist, transpile_mist_no_sem};
+use crate::transpiler::{
+    TranspileError, TranspiledFile, format_mist, transpile_mist, transpile_mist_no_sem,
+};
 
 static MARKER_COUNTER: AtomicU64 = AtomicU64::new(0);
 
@@ -519,8 +522,9 @@ impl Backend {
             Some(p) => p.clone(),
             None => return String::new(),
         };
-        let src_root = ws.join("src");
+        let crate_root = find_crate_root(mist_path).unwrap_or(ws);
-        let package_mist = read_mist_package(&ws);
+        let src_root = crate_root.join("src");
+        let package_mist = read_mist_package(&crate_root);
         let docs = self.documents.lock().await;
         let mod_decls = compute_mod_decls(&docs, &src_root, &package_mist);
         mod_decls.get(mist_path).cloned().unwrap_or_default()
@@ -547,7 +551,7 @@ impl Backend {
         mist_path: &Path,
         source: &str,
         extra_mod_decl: &str,
-    ) {
+    ) -> Option<TranspiledFile> {
         let transpiled = match transpile_mist(mist_path, source, extra_mod_decl) {
             Ok(t) => t,
             Err(e) => {
@@ -556,7 +560,7 @@ impl Backend {
                 if let Some(uri) = clean_lsp_url(mist_path) {
                     self.publish_diagnostics(uri, diag).await;
                 }
-                return;
+                return None;
             }
         };
 
@@ -577,28 +581,28 @@ impl Backend {
                 let _ = ra.did_open(rust_uri, &transpiled.rust_content).await;
             }
 
-            map.insert(transpiled.rust_path, transpiled.mapping);
+            map.insert(transpiled.rust_path.clone(), transpiled.mapping.clone());
             self.last_rust_contents
                 .lock()
                 .await
-                .insert(mist_path.to_path_buf(), transpiled.rust_content);
+                .insert(mist_path.to_path_buf(), transpiled.rust_content.clone());
         }
 
         // Clear any previous diagnostics — transpile succeeded.
         if let Some(mist_uri) = clean_lsp_url(mist_path) {
             self.publish_diagnostics(mist_uri, vec![]).await;
         }
+
+        Some(transpiled)
     }
 
-    async fn rebuild_module_tree(&self) {
+    async fn rebuild_module_tree(&self, persist: bool) {
         let ws = match &*self.workspace_folder.lock().await {
             Some(p) => p.clone(),
             None => return,
         };
-        let src_root = ws.join("src");
-        let package_mist = read_mist_package(&ws);
 
-        self.ensure_implicit_packages(&src_root).await;
+        self.ensure_implicit_packages().await;
 
         // Collect sources first to avoid holding locks during transpile
         let sources: Vec<(PathBuf, String, String)> = {
@@ -606,25 +610,72 @@ impl Backend {
             if docs.is_empty() {
                 return;
             }
+            let crate_roots = collect_crate_roots(&docs, &ws);
+            let mut result = Vec::new();
+            for crate_root in &crate_roots {
+                let src_root = crate_root.join("src");
+                let package_mist = read_mist_package(crate_root);
                 let mod_decls = compute_mod_decls(&docs, &src_root, &package_mist);
-            mod_decls
+                for (path, decl) in mod_decls {
-                .into_iter()
+                    if !decl.is_empty() {
-                .filter(|(_, decl)| !decl.is_empty())
+                        if let Some(source) = docs.get(&path) {
-                .filter_map(|(path, decl)| docs.get(&path).map(|r| (path, r.to_string(), decl)))
+                            result.push((path.clone(), source.to_string(), decl));
-                .collect()
+                        }
+                    }
+                }
+            }
+            result
         };
 
         for (mist_path, source, decl) in sources {
-            self.handle_transpile_and_notify(&mist_path, &source, &decl)
+            if let Some(transpiled) = self
-                .await;
+                .handle_transpile_and_notify(&mist_path, &source, &decl)
+                .await
+            {
+                if persist {
+                    Self::write_transpiled_to_disk(&transpiled);
+                }
+            }
         }
     }
 
     /// Create synthetic `package.mist` documents for subdirectories that contain
     /// .mist files but no real package.mist (implicit packages).
-    async fn ensure_implicit_packages(&self, src_root: &Path) {
+    async fn ensure_implicit_packages(&self) {
+        let ws = match &*self.workspace_folder.lock().await {
+            Some(p) => p.clone(),
+            None => return,
+        };
         let mut docs = self.documents.lock().await;
-        ensure_implicit_packages_impl(&mut *docs, src_root);
+        let crate_roots = collect_crate_roots(&docs, &ws);
+        for crate_root in &crate_roots {
+            let src_root = crate_root.join("src");
+            ensure_implicit_packages_impl(&mut *docs, &src_root);
+        }
+    }
+
+    /// Write a transpiled file's `.rs` content and `.map.json` mapping to disk.
+    /// The output directory is created if it doesn't exist.
+    fn write_transpiled_to_disk(transpiled: &TranspiledFile) {
+        if let Some(parent) = transpiled.rust_path.parent() {
+            if let Err(e) = std::fs::create_dir_all(parent) {
+                eprintln!("Failed to create output directory {:?}: {e}", parent);
+                return;
+            }
+        }
+        if let Err(e) = std::fs::write(&transpiled.rust_path, &transpiled.rust_content) {
+            eprintln!(
+                "Failed to write transpiled file {:?}: {e}",
+                transpiled.rust_path
+            );
+            return;
+        }
+        let map_path = transpiled.rust_path.with_extension("map.json");
+        if let Ok(map_json) = serde_json::to_string(&transpiled.mapping) {
+            if let Err(e) = std::fs::write(&map_path, &map_json) {
+                eprintln!("Failed to write mapping file {:?}: {e}", map_path);
+            }
+        }
     }
 
     async fn publish_diagnostics(&self, uri: Url, diagnostics: Vec<Diagnostic>) {
@@ -730,8 +781,6 @@ impl LanguageServer for Backend {
 
         tokio::spawn(async move {
             if let Some(root) = &*ws.lock().await {
-                let src_root = root.join("src");
-
                 // Hold the lock across both initialize + initialized so the
                 // editor's didOpen cannot race in-between and send notifications
                 // to rust-analyzer before it has received Initialized.
@@ -748,7 +797,21 @@ impl LanguageServer for Backend {
                 }
 
                 let mut files = Vec::new();
-                collect_mist_files(&src_root, &mut files);
+                collect_mist_files(&root.join("src"), &mut files);
+
+                // Discover subcrates by scanning for Mist.toml in immediate
+                // subdirectories of the workspace root.
+                let mut subcrate_roots = Vec::new();
+                if let Ok(entries) = std::fs::read_dir(root) {
+                    for entry in entries.flatten() {
+                        let entry_path = entry.path();
+                        if entry_path.is_dir() && entry_path.join("Mist.toml").is_file() {
+                            let sub_src = entry_path.join("src");
+                            collect_mist_files(&sub_src, &mut files);
+                            subcrate_roots.push(entry_path);
+                        }
+                    }
+                }
 
                 for file in &files {
                     if let Ok(text) = std::fs::read_to_string(file) {
@@ -761,15 +824,32 @@ impl LanguageServer for Backend {
 
                 eprintln!("Loaded {} mist files", files.len());
 
-                ensure_implicit_packages_impl(&mut *documents.lock().await, &src_root);
+                // Create implicit packages per crate root
+                {
+                    let mut docs = documents.lock().await;
+                    let all_crate_roots: Vec<PathBuf> = std::iter::once(root.clone())
+                        .chain(subcrate_roots.iter().cloned())
+                        .collect();
+                    for crate_root in &all_crate_roots {
+                        ensure_implicit_packages_impl(&mut *docs, &crate_root.join("src"));
+                    }
+                }
 
-                // Compute mod_decls before any transpile so the root file gets its
+                // Compute mod_decls per crate root before any transpile so each
-                // child module declarations from the very first didOpen.
+                // crate's root file gets its child module declarations.
-                let package_mist = read_mist_package(root);
+                let all_crate_roots: Vec<PathBuf> = std::iter::once(root.clone())
-                let src_root = root.join("src");
+                    .chain(subcrate_roots.iter().cloned())
+                    .collect();
                 let mod_decls = {
                     let docs = documents.lock().await;
-                    compute_mod_decls(&docs, &src_root, &package_mist)
+                    let mut merged = HashMap::new();
+                    for crate_root in &all_crate_roots {
+                        let cr_src = crate_root.join("src");
+                        let pkg = read_mist_package(crate_root);
+                        let decls = compute_mod_decls(&docs, &cr_src, &pkg);
+                        merged.extend(decls);
+                    }
+                    merged
                 };
 
                 for file in &files {
@@ -819,14 +899,22 @@ impl LanguageServer for Backend {
                     .cloned()
                     .collect();
                 for syn_path in &synthetic_paths {
-                    let decl = mod_decls.get(syn_path).map(String::as_str).unwrap_or("");
+                    // Find the crate root for this synthetic file
+                    let syn_crate_root = find_crate_root(syn_path).unwrap_or_else(|| root.clone());
+                    let pkg = read_mist_package(&syn_crate_root);
+                    let cr_src = syn_crate_root.join("src");
+                    let decl = {
+                        let docs = documents.lock().await;
+                        let crate_decls = compute_mod_decls(&docs, &cr_src, &pkg);
+                        crate_decls.get(syn_path).cloned().unwrap_or_default()
+                    };
                     let source = documents
                         .lock()
                         .await
                         .get(syn_path)
                         .map(|r| r.to_string())
                         .unwrap_or_default();
-                    if let Ok(transpiled) = transpile_mist(syn_path, &source, decl) {
+                    if let Ok(transpiled) = transpile_mist(syn_path, &source, &decl) {
                         if let Some(rust_uri) = clean_lsp_url(&transpiled.rust_path) {
                             let _ = ra
                                 .lock()
@@ -949,9 +1037,10 @@ impl LanguageServer for Backend {
             .await
             .insert(mist_path.clone(), Rope::from_str(&source));
 
-        self.handle_transpile_and_notify(&mist_path, &source, "")
+        let extra_decl = self.compute_mod_decl_for_file(&mist_path).await;
+        self.handle_transpile_and_notify(&mist_path, &source, &extra_decl)
             .await;
-        self.rebuild_module_tree().await;
+        self.rebuild_module_tree(false).await;
     }
 
     async fn did_change(&self, params: DidChangeTextDocumentParams) {
@@ -971,9 +1060,10 @@ impl LanguageServer for Backend {
                 .await
                 .insert(mist_path.clone(), Rope::from_str(&source));
 
-            self.handle_transpile_and_notify(&mist_path, &source, "")
+            let extra_decl = self.compute_mod_decl_for_file(&mist_path).await;
+            self.handle_transpile_and_notify(&mist_path, &source, &extra_decl)
                 .await;
-            self.rebuild_module_tree().await;
+            self.rebuild_module_tree(false).await;
         }
     }
 
@@ -986,17 +1076,36 @@ impl LanguageServer for Backend {
             }
         };
 
-        if let Some(text) = params.text {
+        let text = if let Some(text) = &params.text {
+            text.clone()
+        } else {
+            tokio::fs::read_to_string(&mist_path)
+                .await
+                .expect("Failed to read file")
+        };
+
         self.documents
             .lock()
             .await
             .insert(mist_path.clone(), Rope::from_str(&text));
 
-            self.handle_transpile_and_notify(&mist_path, &text, "")
+        let extra_decl = self.compute_mod_decl_for_file(&mist_path).await;
+        if let Some(transpiled) = self
+            .handle_transpile_and_notify(&mist_path, &text, &extra_decl)
+            .await
+        {
+            Self::write_transpiled_to_disk(&transpiled);
+            if let Some(rust_uri) = clean_lsp_url(&transpiled.rust_path) {
+                let _ = self
+                    .rust_analyzer
+                    .lock()
+                    .await
+                    .did_save(rust_uri, &transpiled.rust_content)
                     .await;
-            self.rebuild_module_tree().await;
             }
         }
+        self.rebuild_module_tree(true).await;
+    }
 
     async fn did_close(&self, params: DidCloseTextDocumentParams) {
         let uri = params.text_document.uri;
@@ -1775,9 +1884,6 @@ async fn handle_ra_notifications(
                     };
 
                     if let Some(mist_uri) = mist_uri.or(Some(rust_uri.clone())) {
-                        let prev = previous_diagnostics.lock().await.get(&mist_uri).cloned();
-
-                        if prev.as_ref() != Some(&mapped_diagnostics) {
                         previous_diagnostics
                             .lock()
                             .await
@@ -1788,7 +1894,6 @@ async fn handle_ra_notifications(
                     }
                 }
             }
-            }
             _ => {}
         }
     }
@@ -1842,13 +1947,6 @@ fn read_mist_package(workspace_root: &Path) -> String {
         .unwrap_or_else(|| "main.mist".to_string())
 }
 
-#[derive(Debug, Serialize, Deserialize)]
-pub struct MistConfig {
-    pub package: PathBuf,
-    pub packages: Vec<PathBuf>,
-    pub fmt: Option<MistFmtConfig>,
-}
-
 fn read_mist_fmt(workspace_root: &Path) -> Option<MistFmtConfig> {
     let toml_path = workspace_root.join("Mist.toml");
     let content = match std::fs::read_to_string(&toml_path) {
@@ -1860,9 +1958,31 @@ fn read_mist_fmt(workspace_root: &Path) -> Option<MistFmtConfig> {
         }
     };
 
-    toml::from_str::<MistConfig>(&content)
+    toml::from_str::<MistConfig>(&content).ok().map(|v| v.fmt)
-        .ok()
+}
-        .and_then(|v| v.fmt)
+
+fn find_crate_root(file_path: &Path) -> Option<PathBuf> {
+    let mut dir = file_path.parent()?;
+    loop {
+        if dir.join("Mist.toml").is_file() {
+            return Some(dir.to_path_buf());
+        }
+        dir = match dir.parent() {
+            Some(p) => p,
+            None => return None,
+        };
+    }
+}
+
+fn collect_crate_roots(docs: &HashMap<PathBuf, Rope>, workspace_root: &Path) -> HashSet<PathBuf> {
+    let mut roots = HashSet::new();
+    roots.insert(workspace_root.to_path_buf());
+    for path in docs.keys() {
+        if let Some(crate_root) = find_crate_root(path) {
+            roots.insert(crate_root);
+        }
+    }
+    roots
 }
 
 fn compute_mod_decls(

crates/analyzer/src/rust_analyzer.rs

@@ -374,6 +374,23 @@ impl RustAnalyzer {
         .await
     }
 
+    pub async fn did_save(
+        &mut self,
+        uri: Url,
+        text: &str,
+    ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+        self.notify(
+            "textDocument/didSave",
+            json!({
+                "textDocument": {
+                    "uri": uri,
+                },
+                "text": text,
+            }),
+        )
+        .await
+    }
+
     pub async fn did_close(
         &mut self,
         uri: Url,

crates/mist-api/src/transpiler.mist

@@ -26,7 +26,8 @@ pub struct MistConfig
 {
     pub PathBuf package,
     pub Vec<PathBuf> packages,
-    pub String main_mods,
+    pub Vec<PathBuf> include,
+    pub mist_parser::MistFmtConfig fmt,
 }
 
 impl MistConfig
@@ -52,15 +53,18 @@ impl Default for MistConfig
         Self {
             package: PathBuf::from("main.mist"),
             packages: Vec::new(),
-            main_mods: String::new(),
+            include: Vec::new(),
+            fmt: mist_parser::MistFmtConfig {
+                allman_bracket_style: true,
+            },
         }
     }
 }
 
-pub PathBuf transpile()
+pub void transpile(Path& root)
 {
+    println!("\x1b[32m\nTranspiling\x1b[0m \x1b[94m{:?}\x1b[0m", root.file_name().unwrap_or_default());
     let start = time::Instant::now();
-    let root = env::current_dir().expect("Unable to find project root");
     let toml_path = root.join("Mist.toml");
     let src = root.join("src");
     let out = root.join(".mist/src");
@@ -71,16 +75,41 @@ pub PathBuf transpile()
     }
     config.package = src.join(&*config.package);
     let master = modules::Module::new(config.package.clone(), &config);
-    master.transpile(&out, &mut config);
+    master.transpile(&out, &mut config, &mut String::new());
     let elapsed = start.elapsed();
-    println!("\x1b[32m\nTranspile successful\x1b[0m in \x1b[34m{:.2?}\x1b[0m",
+    println!("\x1b[32m\nTranspile successful\x1b[0m in \x1b[34m{:.2?}\x1b[0m", elapsed);
-        elapsed);
+    for incl in &config.include {
-    root
+        if ends_with_star(&incl) {
+            let sub_crates = incl.parent().expect("Failed to get parent");
+            let entries = match fs::read_dir(sub_crates) {
+                Ok(entries) => entries,
+                Err(e) => {
+                    eprintln!("error: failed to read directory {}\n  {}", sub_crates.display(), e);
+                    process::exit(1);
+                }
+            };
+            for sub_crate in entries {
+                if let Ok(sub_crate) = sub_crate {
+                    let sub_crate = sub_crate.path();
+                    if fs::exists(sub_crate.join("Mist.toml")).unwrap_or_default() {
+                        transpile(&sub_crate);
+                    }
+                }
+            }
+        } else {
+            transpile(incl);
+        }
+    }
+}
+
+bool ends_with_star(Path& path)
+{
+    path.file_name().and_then((name) => name.to_str()).is_some_and((name) => name.ends_with("*"))
 }
 
 impl modules::Module
 {
-    (bool, String) transpile(&self, PathBuf& parent_dir, MistConfig mut& config)
+    (bool, String) transpile(&self, PathBuf& parent_dir, MistConfig mut& config, String mut& main_mods)
     {
         if !self.path.is_dir() && self.path.extension().and_then((v) => v.to_str()) != Some("mist") {
             let file_name = self.path.file_name().and_then((s) => s.to_str()).expect("Failed to get file name");
@@ -93,7 +122,7 @@ impl modules::Module
         let mut child_changed = false;
         for child in &self.children {
             if !config.should_ignore(&child.path) {
-                let out = child.transpile(&dir, config);
+                let out = child.transpile(&dir, config, main_mods);
                 if out.0 {
                     child_changed = true;
                 }
@@ -101,9 +130,9 @@ impl modules::Module
             }
         }
         if self.path == config.package {
-            config.main_mods = output.clone();
+            *main_mods = output.clone();
         } else if config.should_ignore(&self.path) {
-            output.push_str(&config.main_mods)
+            output.push_str(main_mods)
         }
         self.write_transpile(output, parent_dir, child_changed)
     }

src/main.mist

@@ -17,7 +17,8 @@ void main()
     }
     match args[1].as_str() {
         "run" | "build" | "check" | "r" | "b" | "c" => {
-            let root = transpiler::transpile();
+            let root = env::current_dir().expect("Unable to find project root");
+            transpiler::transpile(&root);
             println!("");
             if builder::build(args, root) {
                 println!("\x1b[32m\nBuild successful\x1b[0m");
@@ -26,7 +27,8 @@ void main()
             }
         }
         "transpile" | "t" => {
-            transpiler::transpile();
+            let root = env::current_dir().expect("Unable to find project root");
+            transpiler::transpile(&root);
         }
         "init" => {
             init_project(args);