From 04e9ef86056891363fb3bb17db8ef76fa1ac1f55 Mon Sep 17 00:00:00 2001 From: itowlson Date: Thu, 22 Jan 2026 10:06:41 +1300 Subject: [PATCH 1/6] Extract dependency WITs during build Signed-off-by: itowlson --- Cargo.lock | 17 ++++ Cargo.toml | 1 + crates/build/Cargo.toml | 1 + crates/build/src/lib.rs | 49 +++++++++- crates/dependency-wit/Cargo.toml | 16 ++++ crates/dependency-wit/src/lib.rs | 150 +++++++++++++++++++++++++++++++ 6 files changed, 230 insertions(+), 4 deletions(-) create mode 100644 crates/dependency-wit/Cargo.toml create mode 100644 crates/dependency-wit/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 0ccc6f3d74..1fecffdddf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8336,6 +8336,7 @@ dependencies = [ "anyhow", "serde", "spin-common", + "spin-dependency-wit", "spin-environments", "spin-manifest", "spin-serde", @@ -8392,6 +8393,7 @@ dependencies = [ "spin-app", "spin-build", "spin-common", + "spin-dependency-wit", "spin-doctor", "spin-environments", "spin-factor-outbound-networking", @@ -8499,6 +8501,21 @@ dependencies = [ "wasmtime-wasi", ] +[[package]] +name = "spin-dependency-wit" +version = "3.6.0-pre0" +dependencies = [ + "anyhow", + "indexmap 2.12.0", + "spin-loader", + "spin-manifest", + "spin-serde", + "tokio", + "wasmparser 0.243.0", + "wit-component 0.243.0", + "wit-parser 0.243.0", +] + [[package]] name = "spin-doctor" version = "3.7.0-pre0" diff --git a/Cargo.toml b/Cargo.toml index 04c48088b3..a8e116e1fa 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -56,6 +56,7 @@ watchexec-filterer-globset = "8.0" spin-app = { path = "crates/app" } spin-build = { path = "crates/build" } spin-common = { path = "crates/common" } +spin-dependency-wit = { path = "crates/dependency-wit" } spin-factors-executor = { path = "crates/factors-executor" } spin-doctor = { path = "crates/doctor" } spin-environments = { path = "crates/environments" } diff --git a/crates/build/Cargo.toml b/crates/build/Cargo.toml index ecedecb9ab..09ebe10860 100644 --- a/crates/build/Cargo.toml +++ b/crates/build/Cargo.toml @@ -8,6 +8,7 @@ edition = { workspace = true } anyhow = { workspace = true } serde = { workspace = true } spin-common = { path = "../common" } +spin-dependency-wit = { path = "../dependency-wit" } spin-environments = { path = "../environments" } spin-manifest = { path = "../manifest" } spin-serde = { path = "../serde" } diff --git a/crates/build/src/lib.rs b/crates/build/src/lib.rs index 6b517896e2..cbc7e2dd95 100644 --- a/crates/build/src/lib.rs +++ b/crates/build/src/lib.rs @@ -33,7 +33,11 @@ pub async fn build( })?; let app_dir = parent_dir(manifest_file)?; - let build_result = build_components(component_ids, build_info.components(), &app_dir); + let components_to_build = components_to_build(component_ids, build_info.components())?; + + regenerate_wits(&components_to_build, &app_dir).await?; + + let build_result = build_components(components_to_build, &app_dir); // Emit any required warnings now, so that they don't bury any errors. if let Some(e) = build_info.load_error() { @@ -94,11 +98,20 @@ pub async fn build_default(manifest_file: &Path, cache_root: Option) -> build(manifest_file, &[], TargetChecking::Check, cache_root).await } -fn build_components( +// fn build_components( +// component_ids: &[String], +// components: Vec, +// app_dir: &Path, +// ) -> Result<(), anyhow::Error> { +// let components_to_build = components_to_build(component_ids, components)?; + +// build_components_2(app_dir, components_to_build) +// } + +fn components_to_build( component_ids: &[String], components: Vec, - app_dir: &Path, -) -> Result<(), anyhow::Error> { +) -> anyhow::Result> { let components_to_build = if component_ids.is_empty() { components } else { @@ -119,6 +132,34 @@ fn build_components( .collect() }; + Ok(components_to_build) +} + +async fn regenerate_wits( + components_to_build: &[ComponentBuildInfo], + app_root: &Path, +) -> anyhow::Result<()> { + for component in components_to_build { + let component_dir = match component.build.as_ref().and_then(|b| b.workdir.as_ref()) { + None => app_root.to_owned(), + Some(d) => app_root.join(d), + }; + let dest_file = component_dir.join("spin-dependencies.wit"); + spin_dependency_wit::extract_wits_into( + component.dependencies.inner.iter(), + app_root, + dest_file, + ) + .await?; + } + + Ok(()) +} + +fn build_components( + components_to_build: Vec, + app_dir: &Path, +) -> anyhow::Result<()> { if components_to_build.iter().all(|c| c.build.is_none()) { println!("None of the components have a build command."); println!("For information on specifying a build command, see https://spinframework.dev/build#setting-up-for-spin-build."); diff --git a/crates/dependency-wit/Cargo.toml b/crates/dependency-wit/Cargo.toml new file mode 100644 index 0000000000..8d46b141a8 --- /dev/null +++ b/crates/dependency-wit/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "spin-dependency-wit" +version = { workspace = true } +authors = { workspace = true } +edition = { workspace = true } + +[dependencies] +anyhow = { workspace = true } +indexmap = { workspace = true } +spin-loader = { path = "../loader" } +spin-manifest = { path = "../manifest" } +spin-serde = { path = "../serde" } +tokio = { workspace = true, features = ["rt", "time"] } +wasmparser = { workspace = true } +wit-component = { workspace = true } +wit-parser = { workspace = true } diff --git a/crates/dependency-wit/src/lib.rs b/crates/dependency-wit/src/lib.rs new file mode 100644 index 0000000000..78521ba3ce --- /dev/null +++ b/crates/dependency-wit/src/lib.rs @@ -0,0 +1,150 @@ +use std::path::Path; + +use anyhow::Context; +use spin_loader::WasmLoader; +use spin_manifest::schema::v2::ComponentDependency; +use spin_serde::DependencyName; +use wit_component::DecodedWasm; + +pub async fn extract_wits_into( + source: impl Iterator, + app_root: impl AsRef, + dest_file: impl AsRef, +) -> anyhow::Result<()> { + let loader = WasmLoader::new(app_root.as_ref().to_owned(), None, None).await?; + + let mut package_wits = indexmap::IndexMap::new(); + let mut world_wits = vec![]; + + // TODO: figure out what to do if we import two itfs from same dep + for (dependency_name, dependency) in source { + // TODO: map `export` + let (wasm_path, _export) = loader + .load_component_dependency(dependency_name, dependency) + .await?; + let wasm_bytes = tokio::fs::read(&wasm_path).await?; + + let decoded = read_wasm(&wasm_bytes)?; // this erases the package name, hurrah + let importised = importize(decoded, None, None)?; + + let root_pkg = importised.package(); // the useless root:component one + let useful_pkgs = importised + .resolve() + .packages + .iter() + .map(|p| p.0) + .filter(|pid| *pid != root_pkg) + .collect::>(); + + for p in &useful_pkgs { + let pkg_name = importised.resolve().packages.get(*p).unwrap().name.clone(); + let output = wit_component::OutputToString::default(); + let mut printer = wit_component::WitPrinter::new(output); + printer.print_package(importised.resolve(), *p, false)?; + package_wits.insert(pkg_name, printer.output.to_string()); + } + + let output = wit_component::OutputToString::default(); + let mut printer = wit_component::WitPrinter::new(output); + // TODO: limit to the imported interfaces + printer.print(importised.resolve(), root_pkg, &[])?; + world_wits.push(format!("{}", printer.output)); + } + + tokio::fs::create_dir_all(dest_file.as_ref().parent().unwrap()).await?; + + use tokio::io::AsyncWriteExt; + let mut dest_file = tokio::fs::File::create(dest_file.as_ref()).await?; + + // TODO: ugh! + dest_file + .write_all("package root:component;\n\nworld root {\n".as_bytes()) + .await?; + for world_wit in world_wits { + let text = world_wit.replace("package root:component;", ""); + let text = text.replace("world root-importized", ""); + let text = text.trim(); + let text = text.trim_matches('{').trim_matches('}'); + let text = text.trim(); + dest_file.write_all(text.trim().as_bytes()).await?; + dest_file.write_all("\n".as_bytes()).await?; + } + dest_file.write_all("}\n\n".as_bytes()).await?; + + for package_wit in package_wits.values() { + dest_file.write_all(package_wit.as_bytes()).await?; + } + + dest_file.flush().await?; + + Ok(()) +} + +fn read_wasm(wasm_bytes: &[u8]) -> anyhow::Result { + if wasmparser::Parser::is_component(wasm_bytes) { + wit_component::decode(wasm_bytes) + } else { + let (wasm, bindgen) = wit_component::metadata::decode(wasm_bytes)?; + if wasm.is_none() { + anyhow::bail!( + "input is a core wasm module with no `component-type*` \ + custom sections meaning that there is not WIT information; \ + is the information not embedded or is this supposed \ + to be a component?" + ) + } + Ok(DecodedWasm::Component(bindgen.resolve, bindgen.world)) + } +} + +fn importize( + decoded: DecodedWasm, + world: Option<&str>, + out_world_name: Option<&String>, +) -> anyhow::Result { + let (mut resolve, world_id) = match (decoded, world) { + (DecodedWasm::Component(resolve, world), None) => (resolve, world), + (DecodedWasm::Component(..), Some(_)) => { + anyhow::bail!( + "the `--importize-world` flag is not compatible with a \ + component input, use `--importize` instead" + ); + } + (DecodedWasm::WitPackage(resolve, id), world) => { + let world = resolve.select_world(&[id], world)?; + (resolve, world) + } + }; + + resolve + .importize(world_id, out_world_name.cloned()) + .context("failed to move world exports to imports")?; + + Ok(DecodedWasm::Component(resolve, world_id)) +} + +// fn all_imports(wasm: &DecodedWasm) -> Vec<(wit_parser::PackageName, String)> { +// let mut itfs = vec![]; + +// for (_pid, pp) in &wasm.resolve().packages { +// for (_w, wid) in &pp.worlds { +// if let Some(world) = wasm.resolve().worlds.get(*wid) { +// for (_wk, witem) in &world.imports { +// if let wit_parser::WorldItem::Interface { id, .. } = witem { +// if let Some(itf) = wasm.resolve().interfaces.get(*id) { +// if let Some(itfp) = itf.package.as_ref() { +// if let Some(ppp) = wasm.resolve().packages.get(*itfp) { +// if let Some(itfname) = itf.name.as_ref() { +// itfs.push((ppp.name.clone(), itfname.clone())); +// } +// } +// } +// } +// } +// } +// } +// } +// } + +// itfs +// } From b4540de9eacb3facb35ab4db3ab50ef5096c7cb6 Mon Sep 17 00:00:00 2001 From: itowlson Date: Fri, 23 Jan 2026 11:39:33 +1300 Subject: [PATCH 2/6] Emitting the world is now less gross Signed-off-by: itowlson --- crates/dependency-wit/src/lib.rs | 100 ++++++++++++++++++++++++------- 1 file changed, 77 insertions(+), 23 deletions(-) diff --git a/crates/dependency-wit/src/lib.rs b/crates/dependency-wit/src/lib.rs index 78521ba3ce..62a35ea010 100644 --- a/crates/dependency-wit/src/lib.rs +++ b/crates/dependency-wit/src/lib.rs @@ -4,6 +4,7 @@ use anyhow::Context; use spin_loader::WasmLoader; use spin_manifest::schema::v2::ComponentDependency; use spin_serde::DependencyName; +use tokio::io::AsyncWriteExt; use wit_component::DecodedWasm; pub async fn extract_wits_into( @@ -14,20 +15,30 @@ pub async fn extract_wits_into( let loader = WasmLoader::new(app_root.as_ref().to_owned(), None, None).await?; let mut package_wits = indexmap::IndexMap::new(); - let mut world_wits = vec![]; + + let mut aggregating_resolve = wit_parser::Resolve::default(); + let aggregating_pkg_id = + aggregating_resolve.push_str("dummy.wit", "package root:component;\n\nworld root {}")?; + let aggregating_world_id = + aggregating_resolve.select_world(&[aggregating_pkg_id], Some("root"))?; // TODO: figure out what to do if we import two itfs from same dep - for (dependency_name, dependency) in source { + for (index, (dependency_name, dependency)) in source.enumerate() { // TODO: map `export` let (wasm_path, _export) = loader .load_component_dependency(dependency_name, dependency) .await?; let wasm_bytes = tokio::fs::read(&wasm_path).await?; - let decoded = read_wasm(&wasm_bytes)?; // this erases the package name, hurrah - let importised = importize(decoded, None, None)?; + let decoded = read_wasm(&wasm_bytes)?; + let impo_world = format!("impo-world{index}"); + let importised = importize(decoded, None, Some(&impo_world))?; + + // Capture WITs for all packages used in the importised thing. + // Things like WASI packages may be depended on my multiple packages + // so we index on the package name to avoid emitting them twice. - let root_pkg = importised.package(); // the useless root:component one + let root_pkg = importised.package(); let useful_pkgs = importised .resolve() .packages @@ -44,33 +55,50 @@ pub async fn extract_wits_into( package_wits.insert(pkg_name, printer.output.to_string()); } - let output = wit_component::OutputToString::default(); - let mut printer = wit_component::WitPrinter::new(output); - // TODO: limit to the imported interfaces - printer.print(importised.resolve(), root_pkg, &[])?; - world_wits.push(format!("{}", printer.output)); + // Now add the imports to the aggregating component import world + + let imports = match dependency_name { + DependencyName::Plain(_) => all_imports(&importised), + DependencyName::Package(dependency_package_name) => { + match dependency_package_name.interface.as_ref() { + Some(itf) => one_import(&importised, itf), + None => all_imports(&importised), + } + } + }; + + let remap = aggregating_resolve.merge(importised.resolve().clone())?; + for iid in imports { + let mapped_iid = remap.map_interface(iid, None)?; + let wk = wit_parser::WorldKey::Interface(mapped_iid); + let world_item = wit_parser::WorldItem::Interface { + id: mapped_iid, + stability: wit_parser::Stability::Unknown, + }; + aggregating_resolve + .worlds + .get_mut(aggregating_world_id) + .unwrap() + .imports + .insert(wk, world_item); + } } + // Text for the root package and world(s) + let world_output = wit_component::OutputToString::default(); + let mut world_printer = wit_component::WitPrinter::new(world_output); + world_printer.print(&aggregating_resolve, aggregating_pkg_id, &[])?; + tokio::fs::create_dir_all(dest_file.as_ref().parent().unwrap()).await?; - use tokio::io::AsyncWriteExt; let mut dest_file = tokio::fs::File::create(dest_file.as_ref()).await?; - // TODO: ugh! + // Print the root package and the world(s) with the imports dest_file - .write_all("package root:component;\n\nworld root {\n".as_bytes()) + .write_all(world_printer.output.to_string().as_bytes()) .await?; - for world_wit in world_wits { - let text = world_wit.replace("package root:component;", ""); - let text = text.replace("world root-importized", ""); - let text = text.trim(); - let text = text.trim_matches('{').trim_matches('}'); - let text = text.trim(); - dest_file.write_all(text.trim().as_bytes()).await?; - dest_file.write_all("\n".as_bytes()).await?; - } - dest_file.write_all("}\n\n".as_bytes()).await?; + // Print each package for package_wit in package_wits.values() { dest_file.write_all(package_wit.as_bytes()).await?; } @@ -80,6 +108,32 @@ pub async fn extract_wits_into( Ok(()) } +fn all_imports(wasm: &DecodedWasm) -> Vec { + wasm.resolve() + .worlds + .iter() + .flat_map(|(_wid, w)| w.imports.values()) + .flat_map(as_interface) + .collect() +} + +fn as_interface(wi: &wit_parser::WorldItem) -> Option { + match wi { + wit_parser::WorldItem::Interface { id, .. } => Some(*id), + _ => None, + } +} + +fn one_import(wasm: &DecodedWasm, name: &spin_serde::KebabId) -> Vec { + let id = wasm + .resolve() + .interfaces + .iter() + .find(|i| i.1.name == Some(name.to_string())) + .map(|t| t.0); + id.into_iter().collect() +} + fn read_wasm(wasm_bytes: &[u8]) -> anyhow::Result { if wasmparser::Parser::is_component(wasm_bytes) { wit_component::decode(wasm_bytes) From 66a6706f2a2f74f3f9faabf4ef268a7b05b4f1f6 Mon Sep 17 00:00:00 2001 From: itowlson Date: Mon, 26 Jan 2026 09:57:36 +1300 Subject: [PATCH 3/6] `spin build` option to skip WIT generation Signed-off-by: itowlson --- crates/build/src/lib.rs | 77 +++++++++++++++++++++++++++++------------ src/commands/build.rs | 18 ++++++++++ 2 files changed, 73 insertions(+), 22 deletions(-) diff --git a/crates/build/src/lib.rs b/crates/build/src/lib.rs index cbc7e2dd95..58771f251e 100644 --- a/crates/build/src/lib.rs +++ b/crates/build/src/lib.rs @@ -21,6 +21,7 @@ pub async fn build( manifest_file: &Path, component_ids: &[String], target_checks: TargetChecking, + wit_generation: GenerateDependencyWits, cache_root: Option, ) -> Result<()> { let build_info = component_build_configs(manifest_file) @@ -35,7 +36,9 @@ pub async fn build( let components_to_build = components_to_build(component_ids, build_info.components())?; - regenerate_wits(&components_to_build, &app_dir).await?; + if wit_generation.generate() { + regenerate_wits(&components_to_build, &app_dir).await?; + } let build_result = build_components(components_to_build, &app_dir); @@ -95,19 +98,16 @@ pub async fn build( /// components, perform target checking). We run a "default build" in several /// places and this centralises the logic of what such a "default build" means. pub async fn build_default(manifest_file: &Path, cache_root: Option) -> Result<()> { - build(manifest_file, &[], TargetChecking::Check, cache_root).await + build( + manifest_file, + &[], + TargetChecking::Check, + GenerateDependencyWits::Generate, + cache_root, + ) + .await } -// fn build_components( -// component_ids: &[String], -// components: Vec, -// app_dir: &Path, -// ) -> Result<(), anyhow::Error> { -// let components_to_build = components_to_build(component_ids, components)?; - -// build_components_2(app_dir, components_to_build) -// } - fn components_to_build( component_ids: &[String], components: Vec, @@ -372,6 +372,21 @@ impl TargetChecking { } } +/// Specifies dependency WIT generation behaviour +pub enum GenerateDependencyWits { + /// The build should generate WITs for component dependencies. + Generate, + /// The build should not generate WITs. + Skip, +} + +impl GenerateDependencyWits { + /// Should the build generate dependency WITs? + fn generate(&self) -> bool { + matches!(self, Self::Generate) + } +} + #[cfg(test)] mod tests { use super::*; @@ -384,26 +399,44 @@ mod tests { #[tokio::test] async fn can_load_even_if_trigger_invalid() { let bad_trigger_file = test_data_root().join("bad_trigger.toml"); - build(&bad_trigger_file, &[], TargetChecking::Skip, None) - .await - .unwrap(); + build( + &bad_trigger_file, + &[], + TargetChecking::Skip, + GenerateDependencyWits::Skip, + None, + ) + .await + .unwrap(); } #[tokio::test] async fn succeeds_if_target_env_matches() { let manifest_path = test_data_root().join("good_target_env.toml"); - build(&manifest_path, &[], TargetChecking::Check, None) - .await - .unwrap(); + build( + &manifest_path, + &[], + TargetChecking::Check, + GenerateDependencyWits::Skip, + None, + ) + .await + .unwrap(); } #[tokio::test] async fn fails_if_target_env_does_not_match() { let manifest_path = test_data_root().join("bad_target_env.toml"); - let err = build(&manifest_path, &[], TargetChecking::Check, None) - .await - .expect_err("should have failed") - .to_string(); + let err = build( + &manifest_path, + &[], + TargetChecking::Check, + GenerateDependencyWits::Skip, + None, + ) + .await + .expect_err("should have failed") + .to_string(); // build prints validation errors rather than returning them to top level // (because there could be multiple errors) - see has_meaningful_error_if_target_env_does_not_match diff --git a/src/commands/build.rs b/src/commands/build.rs index 35dd3e286a..676ce1c851 100644 --- a/src/commands/build.rs +++ b/src/commands/build.rs @@ -39,6 +39,15 @@ pub struct BuildCommand { )] skip_target_checks: bool, + /// By default, the build command generates WIT files for components' dependencies. Specify + /// this option to bypass generating WITs. + #[clap( + long = "skip-generate-wits", + alias = "skip-generate-wit", + takes_value = false + )] + skip_generate_wits: bool, + /// Run the application after building. #[clap(name = BUILD_UP_OPT, short = 'u', long = "up")] pub up: bool, @@ -57,6 +66,7 @@ impl BuildCommand { &manifest_file, &self.component_id, self.target_checking(), + self.wit_generation(), None, ) .await?; @@ -83,4 +93,12 @@ impl BuildCommand { spin_build::TargetChecking::Check } } + + fn wit_generation(&self) -> spin_build::GenerateDependencyWits { + if self.skip_generate_wits { + spin_build::GenerateDependencyWits::Skip + } else { + spin_build::GenerateDependencyWits::Generate + } + } } From bd8b18004526e938e37113789f27cccd06ecd0b6 Mon Sep 17 00:00:00 2001 From: itowlson Date: Mon, 26 Jan 2026 13:26:22 +1300 Subject: [PATCH 4/6] Extract via string for testability Signed-off-by: itowlson --- crates/dependency-wit/src/lib.rs | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/crates/dependency-wit/src/lib.rs b/crates/dependency-wit/src/lib.rs index 62a35ea010..99622ac513 100644 --- a/crates/dependency-wit/src/lib.rs +++ b/crates/dependency-wit/src/lib.rs @@ -4,7 +4,6 @@ use anyhow::Context; use spin_loader::WasmLoader; use spin_manifest::schema::v2::ComponentDependency; use spin_serde::DependencyName; -use tokio::io::AsyncWriteExt; use wit_component::DecodedWasm; pub async fn extract_wits_into( @@ -12,6 +11,18 @@ pub async fn extract_wits_into( app_root: impl AsRef, dest_file: impl AsRef, ) -> anyhow::Result<()> { + let wit_text = extract_wits(source, app_root).await?; + + tokio::fs::create_dir_all(dest_file.as_ref().parent().unwrap()).await?; + tokio::fs::write(dest_file, wit_text.as_bytes()).await?; + + Ok(()) +} + +pub async fn extract_wits( + source: impl Iterator, + app_root: impl AsRef, +) -> anyhow::Result { let loader = WasmLoader::new(app_root.as_ref().to_owned(), None, None).await?; let mut package_wits = indexmap::IndexMap::new(); @@ -89,23 +100,17 @@ pub async fn extract_wits_into( let mut world_printer = wit_component::WitPrinter::new(world_output); world_printer.print(&aggregating_resolve, aggregating_pkg_id, &[])?; - tokio::fs::create_dir_all(dest_file.as_ref().parent().unwrap()).await?; - - let mut dest_file = tokio::fs::File::create(dest_file.as_ref()).await?; + let mut buf = String::new(); // Print the root package and the world(s) with the imports - dest_file - .write_all(world_printer.output.to_string().as_bytes()) - .await?; + buf.push_str(&world_printer.output.to_string()); // Print each package for package_wit in package_wits.values() { - dest_file.write_all(package_wit.as_bytes()).await?; + buf.push_str(package_wit); } - dest_file.flush().await?; - - Ok(()) + Ok(buf) } fn all_imports(wasm: &DecodedWasm) -> Vec { From fee97e5bf3b4073a5cfaf58f42a6d9cdd463bc49 Mon Sep 17 00:00:00 2001 From: itowlson Date: Thu, 5 Feb 2026 16:26:36 +1300 Subject: [PATCH 5/6] Handle `export` modifier, hopefully Signed-off-by: itowlson --- Cargo.lock | 8 +- crates/dependency-wit/Cargo.toml | 5 + crates/dependency-wit/src/lib.rs | 304 +++++++++++++++++++++++++++---- 3 files changed, 276 insertions(+), 41 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1fecffdddf..c49c4910b6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8510,10 +8510,12 @@ dependencies = [ "spin-loader", "spin-manifest", "spin-serde", + "tempfile", "tokio", - "wasmparser 0.243.0", - "wit-component 0.243.0", - "wit-parser 0.243.0", + "wasm-pkg-common", + "wasmparser 0.244.0", + "wit-component 0.244.0", + "wit-parser 0.244.0", ] [[package]] diff --git a/crates/dependency-wit/Cargo.toml b/crates/dependency-wit/Cargo.toml index 8d46b141a8..364756ace3 100644 --- a/crates/dependency-wit/Cargo.toml +++ b/crates/dependency-wit/Cargo.toml @@ -14,3 +14,8 @@ tokio = { workspace = true, features = ["rt", "time"] } wasmparser = { workspace = true } wit-component = { workspace = true } wit-parser = { workspace = true } + +[dev-dependencies] +tempfile = { workspace = true } +wasm-pkg-common = { workspace = true } +wit-component = { workspace = true, features = ["dummy-module"] } diff --git a/crates/dependency-wit/src/lib.rs b/crates/dependency-wit/src/lib.rs index 99622ac513..fd1e2f3d05 100644 --- a/crates/dependency-wit/src/lib.rs +++ b/crates/dependency-wit/src/lib.rs @@ -35,16 +35,35 @@ pub async fn extract_wits( // TODO: figure out what to do if we import two itfs from same dep for (index, (dependency_name, dependency)) in source.enumerate() { - // TODO: map `export` - let (wasm_path, _export) = loader + let import_name = match dependency_name { + DependencyName::Plain(_) => None, + DependencyName::Package(dependency_package_name) => { + dependency_package_name.interface.as_ref() + // match dependency_package_name.interface.as_ref() { + // Some(itf) => Some(itf), + // None => None, + // } + } + }; + + let (wasm_path, export) = loader .load_component_dependency(dependency_name, dependency) .await?; let wasm_bytes = tokio::fs::read(&wasm_path).await?; let decoded = read_wasm(&wasm_bytes)?; + let decoded = match export { + None => decoded, + Some(export) => munge_aliased_export(decoded, &export, dependency_name)?, + }; let impo_world = format!("impo-world{index}"); let importised = importize(decoded, None, Some(&impo_world))?; + let imports = match import_name { + None => all_imports(&importised), + Some(itf) => one_import(&importised, itf.as_ref()), + }; + // Capture WITs for all packages used in the importised thing. // Things like WASI packages may be depended on my multiple packages // so we index on the package name to avoid emitting them twice. @@ -68,16 +87,6 @@ pub async fn extract_wits( // Now add the imports to the aggregating component import world - let imports = match dependency_name { - DependencyName::Plain(_) => all_imports(&importised), - DependencyName::Package(dependency_package_name) => { - match dependency_package_name.interface.as_ref() { - Some(itf) => one_import(&importised, itf), - None => all_imports(&importised), - } - } - }; - let remap = aggregating_resolve.merge(importised.resolve().clone())?; for iid in imports { let mapped_iid = remap.map_interface(iid, None)?; @@ -113,6 +122,122 @@ pub async fn extract_wits( Ok(buf) } +fn munge_aliased_export( + decoded: DecodedWasm, + export: &str, + new_name: &DependencyName, +) -> anyhow::Result { + // TODO: I am not sure how `export` is meant to work if you are + // depping on a package rather than an itf + + let export_qname = spin_serde::DependencyPackageName::try_from(export.to_string())?; + let Some(export_itf_name) = export_qname.interface.as_ref() else { + anyhow::bail!("the export name should be a qualified interface name - missing interface"); + }; + let export_pkg_name = wit_parser::PackageName { + namespace: export_qname.package.namespace().to_string(), + name: export_qname.package.name().to_string(), + version: export_qname.version, + }; + + let DependencyName::Package(new_name) = new_name else { + anyhow::bail!("the dependency name should be a qualified interface name - not qualified"); + }; + let Some(new_itf_name) = new_name.interface.as_ref() else { + anyhow::bail!( + "the dependency name should be a qualified interface name - missing interface" + ); + }; + let new_pkg_name = wit_parser::PackageName { + namespace: new_name.package.namespace().to_string(), + name: new_name.package.name().to_string(), + version: new_name.version.clone(), + }; + + let (mut resolve, decode_id) = match decoded { + DecodedWasm::WitPackage(resolve, id) => (resolve, WorldOrPackageId::Package(id)), + DecodedWasm::Component(resolve, id) => (resolve, WorldOrPackageId::World(id)), + }; + + // Two scenarios: + // 1. The new name is in a package that is already in the Resolve + // 1a. The package already contains an interface with the right name + // 1b. The package does not already contain an interface with the right name + // 2. The new name is in a package that is NOT already in the Resolve + + let existing_pkg = resolve + .packages + .iter() + .find(|(_pkg_id, pkg)| pkg.name == new_pkg_name); + + // We address the first level by creating the new-name package if it doesn't exist + let (inserting_into_pkg_id, inserting_into_pkg) = match existing_pkg { + Some(tup) => tup, + None => { + // insert the needed package + let package_wit = format!("package {new_pkg_name};"); + let pkg_id = resolve + .push_str(std::env::current_dir().unwrap(), &package_wit) + .context("failed at setting up fake pkg")?; + let pkg = resolve.packages.get(pkg_id).unwrap(); + (pkg_id, pkg) + } + }; + + // Second level asks if the package already contains the interface + let existing_itf = inserting_into_pkg.interfaces.get(new_itf_name.as_ref()); + if existing_itf.is_some() { + // no rename is needed, but we might need to do some extra work to make sure + // that the export, rather than the import, gets included in the aggregated world + return Ok(decode_id.make_decoded_wasm(resolve)); + } + + // It does not: we need to slurp the EXPORTED itf into the `inserting_into` + // package under the NEW (importing) interface name + let Some(export_pkg_id) = resolve.package_names.get(&export_pkg_name) else { + anyhow::bail!("export is from a package that doesn't exist"); + }; + let Some(export_pkg) = resolve.packages.get(*export_pkg_id) else { + anyhow::bail!("export pkg id doesn't point to a package wtf"); + }; + let Some(export_itf_id) = export_pkg.interfaces.get(export_itf_name.as_ref()) else { + anyhow::bail!("export pkg doesn't contain export itf"); + }; + let Some(export_itf) = resolve.interfaces.get(*export_itf_id) else { + anyhow::bail!("export pkg doesn't contain export itf part 2"); + }; + + let mut export_itf = export_itf.clone(); + export_itf.package = Some(inserting_into_pkg_id); + export_itf.name = Some(new_itf_name.to_string()); + let export_itf_id_2 = resolve.interfaces.alloc(export_itf); + + // OKAY TIME TO ADD THIS UNDER THE WRONG NAME TO THE THINGY + // oh man there is some nonsense about worlds as well + let inserting_into_pkg_mut = resolve.packages.get_mut(inserting_into_pkg_id).unwrap(); // SHENANIGANS to get around a "mutable borrow at the same time as immutable borrow" woe + inserting_into_pkg_mut + .interfaces + .insert(new_itf_name.to_string(), export_itf_id_2); + + let thingy = decode_id.make_decoded_wasm(resolve); + + Ok(thingy) +} + +enum WorldOrPackageId { + Package(wit_parser::PackageId), + World(wit_parser::WorldId), +} + +impl WorldOrPackageId { + pub fn make_decoded_wasm(&self, resolve: wit_parser::Resolve) -> DecodedWasm { + match self { + Self::Package(id) => DecodedWasm::WitPackage(resolve, *id), + Self::World(id) => DecodedWasm::Component(resolve, *id), + } + } +} + fn all_imports(wasm: &DecodedWasm) -> Vec { wasm.resolve() .worlds @@ -129,7 +254,7 @@ fn as_interface(wi: &wit_parser::WorldItem) -> Option { } } -fn one_import(wasm: &DecodedWasm, name: &spin_serde::KebabId) -> Vec { +fn one_import(wasm: &DecodedWasm, name: &str) -> Vec { let id = wasm .resolve() .interfaces @@ -182,28 +307,131 @@ fn importize( Ok(DecodedWasm::Component(resolve, world_id)) } -// fn all_imports(wasm: &DecodedWasm) -> Vec<(wit_parser::PackageName, String)> { -// let mut itfs = vec![]; - -// for (_pid, pp) in &wasm.resolve().packages { -// for (_w, wid) in &pp.worlds { -// if let Some(world) = wasm.resolve().worlds.get(*wid) { -// for (_wk, witem) in &world.imports { -// if let wit_parser::WorldItem::Interface { id, .. } = witem { -// if let Some(itf) = wasm.resolve().interfaces.get(*id) { -// if let Some(itfp) = itf.package.as_ref() { -// if let Some(ppp) = wasm.resolve().packages.get(*itfp) { -// if let Some(itfname) = itf.name.as_ref() { -// itfs.push((ppp.name.clone(), itfname.clone())); -// } -// } -// } -// } -// } -// } -// } -// } -// } - -// itfs -// } +#[cfg(test)] +mod test { + use super::*; + + fn parse_wit(wit: &str) -> anyhow::Result { + let mut resolve = wit_parser::Resolve::new(); + resolve.push_str("dummy.wit", wit)?; + Ok(resolve) + } + + fn generate_dummy_component(wit: &str, world: &str) -> Vec { + let mut resolve = wit_parser::Resolve::default(); + let package_id = resolve.push_str("test", wit).expect("should parse WIT"); + let world_id = resolve + .select_world(&[package_id], Some(world)) + .expect("should select world"); + + let mut wasm = wit_component::dummy_module( + &resolve, + world_id, + wit_parser::ManglingAndAbi::Legacy(wit_parser::LiftLowerAbi::Sync), + ); + wit_component::embed_component_metadata( + &mut wasm, + &resolve, + world_id, + wit_component::StringEncoding::UTF8, + ) + .expect("should embed component metadata"); + + let mut encoder = wit_component::ComponentEncoder::default() + .validate(true) + .module(&wasm) + .expect("should set module"); + encoder.encode().expect("should encode component") + } + + #[tokio::test] + async fn if_no_dependencies_then_empty_valid_wit() -> anyhow::Result<()> { + let wit = extract_wits(std::iter::empty(), ".").await?; + + let resolve = parse_wit(&wit).expect("should have emitted valid WIT"); + + assert_eq!(1, resolve.packages.len()); + assert_eq!( + "root:component", + resolve.packages.iter().next().unwrap().1.name.to_string() + ); + + assert_eq!(0, resolve.interfaces.len()); + + assert_eq!(1, resolve.worlds.len()); + + let world = resolve.worlds.iter().next().unwrap().1; + assert_eq!("root", world.name); + assert_eq!(0, world.imports.len()); + + Ok(()) + } + + #[tokio::test] + async fn single_dep_wit_extracted() -> anyhow::Result<()> { + let tempdir = tempfile::TempDir::new()?; + let dep_file = tempdir.path().join("regex.wasm"); + + let dep_wit = "package my:regex@1.0.0;\n\ninterface regex {\n matches: func(s: string) -> bool;\n}\nworld matcher {\n export regex;\n}"; + let dep_wasm = generate_dummy_component(dep_wit, "matcher"); + tokio::fs::write(&dep_file, &dep_wasm).await?; + + let dep_name = + DependencyName::Package("my:regex/regex@1.0.0".to_string().try_into().unwrap()); + let dep_src = ComponentDependency::Local { + path: dep_file, + export: None, + }; + let deps = std::iter::once((&dep_name, &dep_src)); + + let wit = extract_wits(deps, ".").await?; + + let resolve = parse_wit(&wit).expect("should have emitted valid WIT"); + + assert_eq!(2, resolve.packages.len()); // root:component and my:regex + let (_rc_pkg_id, rc_pkg) = resolve + .packages + .iter() + .find(|(_, p)| p.name.to_string() == "root:component") + .expect("should have had `root:component`"); + let (_mr_pkg_id, _mr_pkg) = resolve + .packages + .iter() + .find(|(_, p)| p.name.to_string() == "my:regex@1.0.0") + .expect("should have had `my:regex`"); + + assert_eq!(1, resolve.interfaces.len()); + assert_eq!( + "regex", + resolve + .interfaces + .iter() + .next() + .unwrap() + .1 + .name + .as_ref() + .unwrap() + ); + let regex_itf_id = resolve.interfaces.iter().next().unwrap().0; + + assert_eq!(2, rc_pkg.worlds.len()); // root and synthetic "impo*" wart + let root_world_id = rc_pkg + .worlds + .iter() + .find(|w| w.0 == "root") + .expect("should have had `root` world") + .1; + + let world = resolve.worlds.get(*root_world_id).unwrap(); + assert_eq!(1, world.imports.len()); + let expected_import = wit_parser::WorldItem::Interface { + id: regex_itf_id, + stability: wit_parser::Stability::Unknown, + }; + let import = world.imports.values().next().unwrap(); + assert_eq!(&expected_import, import); + + Ok(()) + } +} From ed40fa4c31d7561c1aa5041f762ea13df55cc2af Mon Sep 17 00:00:00 2001 From: itowlson Date: Wed, 11 Feb 2026 15:56:03 +1300 Subject: [PATCH 6/6] Tidying, error noodling, etc. Signed-off-by: itowlson --- Cargo.lock | 2 +- crates/build/src/lib.rs | 27 ++++++-- crates/dependency-wit/src/lib.rs | 108 ++++++++++++++++++++++--------- 3 files changed, 99 insertions(+), 38 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c49c4910b6..e44357c6ad 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8503,7 +8503,7 @@ dependencies = [ [[package]] name = "spin-dependency-wit" -version = "3.6.0-pre0" +version = "3.7.0-pre0" dependencies = [ "anyhow", "indexmap 2.12.0", diff --git a/crates/build/src/lib.rs b/crates/build/src/lib.rs index 58771f251e..2d4d639b48 100644 --- a/crates/build/src/lib.rs +++ b/crates/build/src/lib.rs @@ -37,7 +37,18 @@ pub async fn build( let components_to_build = components_to_build(component_ids, build_info.components())?; if wit_generation.generate() { - regenerate_wits(&components_to_build, &app_dir).await?; + let wit_gen_errs = regenerate_wits(&components_to_build, &app_dir).await; + if !wit_gen_errs.is_empty() { + terminal::warn!("One or more components specified dependencies for which Spin couldn't generate import bindings."); + eprintln!( + "If these components rely on Spin-generated bindings they may fail to build." + ); + eprintln!("Otherwise, to skip binding generation, use the --skip-generate-wits flag."); + eprintln!("Error details:"); + for (component, err) in wit_gen_errs { + terminal::einfo!("{component}:", "{err:#}"); + } + } } let build_result = build_components(components_to_build, &app_dir); @@ -135,25 +146,31 @@ fn components_to_build( Ok(components_to_build) } +#[must_use] async fn regenerate_wits( components_to_build: &[ComponentBuildInfo], app_root: &Path, -) -> anyhow::Result<()> { +) -> Vec<(String, anyhow::Error)> { + let mut errors = vec![]; + for component in components_to_build { let component_dir = match component.build.as_ref().and_then(|b| b.workdir.as_ref()) { None => app_root.to_owned(), Some(d) => app_root.join(d), }; let dest_file = component_dir.join("spin-dependencies.wit"); - spin_dependency_wit::extract_wits_into( + let extract_result = spin_dependency_wit::extract_wits_into( component.dependencies.inner.iter(), app_root, dest_file, ) - .await?; + .await; + if let Err(e) = extract_result { + errors.push((component.id.clone(), e)); + } } - Ok(()) + errors } fn build_components( diff --git a/crates/dependency-wit/src/lib.rs b/crates/dependency-wit/src/lib.rs index fd1e2f3d05..b437242027 100644 --- a/crates/dependency-wit/src/lib.rs +++ b/crates/dependency-wit/src/lib.rs @@ -13,7 +13,13 @@ pub async fn extract_wits_into( ) -> anyhow::Result<()> { let wit_text = extract_wits(source, app_root).await?; - tokio::fs::create_dir_all(dest_file.as_ref().parent().unwrap()).await?; + tokio::fs::create_dir_all( + dest_file + .as_ref() + .parent() + .context("file is root directory")?, + ) + .await?; tokio::fs::write(dest_file, wit_text.as_bytes()).await?; Ok(()) @@ -39,25 +45,27 @@ pub async fn extract_wits( DependencyName::Plain(_) => None, DependencyName::Package(dependency_package_name) => { dependency_package_name.interface.as_ref() - // match dependency_package_name.interface.as_ref() { - // Some(itf) => Some(itf), - // None => None, - // } } }; let (wasm_path, export) = loader .load_component_dependency(dependency_name, dependency) - .await?; + .await + .with_context(|| format!("failed to load dependency {dependency_name}"))?; let wasm_bytes = tokio::fs::read(&wasm_path).await?; let decoded = read_wasm(&wasm_bytes)?; let decoded = match export { None => decoded, - Some(export) => munge_aliased_export(decoded, &export, dependency_name)?, + Some(export) => { + munge_aliased_export(decoded, &export, dependency_name).with_context(|| { + format!("failed to map named export {export} to dependency {dependency_name}") + })? + } }; let impo_world = format!("impo-world{index}"); - let importised = importize(decoded, None, Some(&impo_world))?; + let importised = importize(decoded, None, Some(&impo_world)) + .with_context(|| format!("failed to map importize dependency {dependency_name}"))?; let imports = match import_name { None => all_imports(&importised), @@ -78,7 +86,13 @@ pub async fn extract_wits( .collect::>(); for p in &useful_pkgs { - let pkg_name = importised.resolve().packages.get(*p).unwrap().name.clone(); + let pkg_name = importised + .resolve() + .packages + .get(*p) + .context("package not found in importised (id lookup failed)")? // shouldn't happen + .name + .clone(); let output = wit_component::OutputToString::default(); let mut printer = wit_component::WitPrinter::new(output); printer.print_package(importised.resolve(), *p, false)?; @@ -98,7 +112,7 @@ pub async fn extract_wits( aggregating_resolve .worlds .get_mut(aggregating_world_id) - .unwrap() + .context("aggregated dependency world doesn't exist")? // shouldn't happen .imports .insert(wk, world_item); } @@ -132,7 +146,7 @@ fn munge_aliased_export( let export_qname = spin_serde::DependencyPackageName::try_from(export.to_string())?; let Some(export_itf_name) = export_qname.interface.as_ref() else { - anyhow::bail!("the export name should be a qualified interface name - missing interface"); + anyhow::bail!("the export name should be a qualified interface name - {export_qname} doesn't specify interface"); }; let export_pkg_name = wit_parser::PackageName { namespace: export_qname.package.namespace().to_string(), @@ -141,11 +155,13 @@ fn munge_aliased_export( }; let DependencyName::Package(new_name) = new_name else { - anyhow::bail!("the dependency name should be a qualified interface name - not qualified"); + anyhow::bail!( + "the dependency name should be a qualified interface name - {new_name} not qualified" + ); }; let Some(new_itf_name) = new_name.interface.as_ref() else { anyhow::bail!( - "the dependency name should be a qualified interface name - missing interface" + "the dependency name should be a qualified interface name - {new_name} doesn't specify an interface" ); }; let new_pkg_name = wit_parser::PackageName { @@ -172,56 +188,84 @@ fn munge_aliased_export( // We address the first level by creating the new-name package if it doesn't exist let (inserting_into_pkg_id, inserting_into_pkg) = match existing_pkg { - Some(tup) => tup, + Some(tuple) => tuple, None => { // insert the needed package let package_wit = format!("package {new_pkg_name};"); let pkg_id = resolve - .push_str(std::env::current_dir().unwrap(), &package_wit) - .context("failed at setting up fake pkg")?; - let pkg = resolve.packages.get(pkg_id).unwrap(); + .push_str( + std::env::current_dir().context("no current dir")?, // unused + &package_wit, + ) + .with_context(|| format!("failed to create import alias package {new_pkg_name}"))?; + let pkg = resolve + .packages + .get(pkg_id) + .context("export alias package created but doesn't exist")?; // shouldn't happen (pkg_id, pkg) } }; - // Second level asks if the package already contains the interface + // Second level asks if the new-name package already contains the interface let existing_itf = inserting_into_pkg.interfaces.get(new_itf_name.as_ref()); if existing_itf.is_some() { - // no rename is needed, but we might need to do some extra work to make sure - // that the export, rather than the import, gets included in the aggregated world + // This makes the questionable assumption that the matchingly interface already + // in the package is the same as the export. E.g. given "a:b/i" = { export = "c:d/i" } + // where the dep contains an `a:b` package with an interface named `i`, we will generate + // from that rather than emitting the `c:d/i` WIT for `a:b/i`. We could be smarter about + // this but we don't know if other things depend on `a:b/i` so replacing it could result + // in a bad WIT. If this becomes a problem we could maybe try emitting it as something + // like `a:b/i-from-c-d` but I'd prefer to cross that bridge when we come to it. return Ok(decode_id.make_decoded_wasm(resolve)); } - // It does not: we need to slurp the EXPORTED itf into the `inserting_into` + // The new-name package does not contain the interface: we need to slurp the EXPORTED itf into the `inserting_into` // package under the NEW (importing) interface name let Some(export_pkg_id) = resolve.package_names.get(&export_pkg_name) else { - anyhow::bail!("export is from a package that doesn't exist"); + anyhow::bail!( + "export is from a package ({}) that doesn't exist (name lookup failed)", + export_pkg_name + ); }; let Some(export_pkg) = resolve.packages.get(*export_pkg_id) else { - anyhow::bail!("export pkg id doesn't point to a package wtf"); + anyhow::bail!( + "export is from a package ({}) that doesn't exist (id lookup failed)", + export_pkg_name + ); }; let Some(export_itf_id) = export_pkg.interfaces.get(export_itf_name.as_ref()) else { - anyhow::bail!("export pkg doesn't contain export itf"); + anyhow::bail!( + "export package ({}) doesn't contain interface {} (name lookup failed)", + export_pkg_name, + export_itf_name + ); }; let Some(export_itf) = resolve.interfaces.get(*export_itf_id) else { - anyhow::bail!("export pkg doesn't contain export itf part 2"); + anyhow::bail!( + "export package ({}) doesn't contain interface {} (id lookup failed)", + export_pkg_name, + export_itf_name + ); }; + // Create the new-name interface by cloning the export interface let mut export_itf = export_itf.clone(); export_itf.package = Some(inserting_into_pkg_id); export_itf.name = Some(new_itf_name.to_string()); - let export_itf_id_2 = resolve.interfaces.alloc(export_itf); - // OKAY TIME TO ADD THIS UNDER THE WRONG NAME TO THE THINGY - // oh man there is some nonsense about worlds as well - let inserting_into_pkg_mut = resolve.packages.get_mut(inserting_into_pkg_id).unwrap(); // SHENANIGANS to get around a "mutable borrow at the same time as immutable borrow" woe + // Add the new-name interface to the resolve and to the new-name package + let export_itf_id_new = resolve.interfaces.alloc(export_itf); + let inserting_into_pkg_mut = resolve + .packages + .get_mut(inserting_into_pkg_id) + .context("package id lookup that succeeded before failed now")?; // we re-lookup to get around a "mutable borrow at the same time as immutable borrow" woe inserting_into_pkg_mut .interfaces - .insert(new_itf_name.to_string(), export_itf_id_2); + .insert(new_itf_name.to_string(), export_itf_id_new); - let thingy = decode_id.make_decoded_wasm(resolve); + let decoded = decode_id.make_decoded_wasm(resolve); - Ok(thingy) + Ok(decoded) } enum WorldOrPackageId {