From e7c67107d1cf11db9a7bd7e90db7f74ebc446321 Mon Sep 17 00:00:00 2001 From: Zach Vorhies Date: Mon, 14 Sep 2026 16:55:01 -0700 Subject: [PATCH 1/4] feat(cache): payload cache slices and hit/miss lines in build output (#1433) - `fbuild cache` gains opt-in `core`, `framework-libs` and `library-selection` slices. A default save stays packages-only; CI saves the per-board build payload separately with `--include`. - Builds now print one line per framework core cache hydrate and store, and per ESP32 framework-libs hydrate and store. Hits, misses and stores were only in the daemon log, so CI could not tell whether a restored cache was used. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01StasPkdQ3D1gYGj6WnaQ3q --- .../src/framework_core_cache.rs | 85 +++++++++++++++ .../src/pipeline/sequential.rs | 16 ++- .../src/esp32/orchestrator/build.rs | 16 ++- .../orchestrator/framework_library_cache.rs | 35 ++++++ .../src/esp32/orchestrator/framework_libs.rs | 20 +++- crates/fbuild-cli/src/cli/args.rs | 4 +- crates/fbuild-cli/src/cli/cache.rs | 4 +- .../src/cache_archive.rs | 100 +++++++++++++++++- 8 files changed, 264 insertions(+), 16 deletions(-) diff --git a/crates/fbuild-build-engine/src/framework_core_cache.rs b/crates/fbuild-build-engine/src/framework_core_cache.rs index 88119ade5..5c03c530f 100644 --- a/crates/fbuild-build-engine/src/framework_core_cache.rs +++ b/crates/fbuild-build-engine/src/framework_core_cache.rs @@ -127,6 +127,36 @@ impl FrameworkCoreCache { Ok(copy_artifacts(core_build_dir, &self.path, true, true)?.stats) } + /// Build-output line for a [`Self::hydrate`] outcome. CI logs only carry + /// build output, so this is how a restored cache shows it was used + /// (FastLED/fbuild#1433). + pub fn hydrate_summary(&self, outcome: &std::io::Result) -> String { + let key = short_key(&self.key); + match outcome { + Ok(stats) if stats.copied > 0 || stats.skipped > 0 => format!( + "framework core cache: hit key={key} copied={} skipped={}", + stats.copied, stats.skipped + ), + Ok(_) => format!("framework core cache: miss key={key}"), + Err(error) => format!("framework core cache: hydrate failed key={key}: {error}"), + } + } + + /// Build-output line for a [`Self::store`] outcome. + pub fn store_summary(&self, outcome: &std::io::Result) -> String { + let key = short_key(&self.key); + match outcome { + Ok(stats) if stats.copied > 0 => { + format!( + "framework core cache: stored key={key} copied={}", + stats.copied + ) + } + Ok(_) => format!("framework core cache: up to date key={key}"), + Err(error) => format!("framework core cache: store failed key={key}: {error}"), + } + } + /// Remove only this content-addressed cache entry. /// /// The parent cache root may contain entries for other projects, @@ -140,6 +170,11 @@ impl FrameworkCoreCache { } } +/// Enough of a sha256 key to tell entries apart in a log line. +fn short_key(key: &str) -> &str { + &key[..key.len().min(12)] +} + fn core_cache_key( project_dir: &Path, platform_label: &str, @@ -364,6 +399,56 @@ mod tests { } } + fn cache_with_key(key: &str) -> FrameworkCoreCache { + FrameworkCoreCache { + key: key.to_string(), + path: PathBuf::from("/cache/core").join(key), + } + } + + #[test] + fn hydrate_summary_names_hit_miss_and_failure() { + let cache = cache_with_key("0123456789abcdef0123"); + let hit = cache.hydrate_summary(&Ok(ArtifactCopyStats { + copied: 3, + skipped: 1, + })); + assert_eq!( + hit, + "framework core cache: hit key=0123456789ab copied=3 skipped=1" + ); + let miss = cache.hydrate_summary(&Ok(ArtifactCopyStats::default())); + assert_eq!(miss, "framework core cache: miss key=0123456789ab"); + let failed = cache.hydrate_summary(&Err(std::io::Error::other("disk gone"))); + assert_eq!( + failed, + "framework core cache: hydrate failed key=0123456789ab: disk gone" + ); + } + + #[test] + fn store_summary_names_stored_up_to_date_and_failure() { + let cache = cache_with_key("fedcba9876543210"); + let stored = cache.store_summary(&Ok(ArtifactCopyStats { + copied: 5, + skipped: 0, + })); + assert_eq!( + stored, + "framework core cache: stored key=fedcba987654 copied=5" + ); + let unchanged = cache.store_summary(&Ok(ArtifactCopyStats::default())); + assert_eq!( + unchanged, + "framework core cache: up to date key=fedcba987654" + ); + let failed = cache.store_summary(&Err(std::io::Error::other("read-only"))); + assert_eq!( + failed, + "framework core cache: store failed key=fedcba987654: read-only" + ); + } + #[test] fn key_changes_when_source_flags_change() { let compiler = FakeCompiler::new(); diff --git a/crates/fbuild-build-engine/src/pipeline/sequential.rs b/crates/fbuild-build-engine/src/pipeline/sequential.rs index 1da628b6a..a01ad9457 100644 --- a/crates/fbuild-build-engine/src/pipeline/sequential.rs +++ b/crates/fbuild-build-engine/src/pipeline/sequential.rs @@ -145,12 +145,17 @@ pub async fn run_sequential_build_with_libs( { let cache = &core_cache; let _g = perf.phase("core-cache-hydrate"); - match cache.hydrate( + let outcome = cache.hydrate( &ctx.core_build_dir, compiler, &core_and_variant, &user_overlay, - ) { + ); + build_log_mutex + .lock() + .unwrap_or_else(|e| e.into_inner()) + .push(cache.hydrate_summary(&outcome)); + match &outcome { Ok(stats) if stats.copied > 0 || stats.skipped > 0 => tracing::info!( "framework core cache hydrate key={} copied={} skipped={} from {}", cache.key(), @@ -201,7 +206,12 @@ pub async fn run_sequential_build_with_libs( { let cache = &core_cache; let _g = perf.phase("core-cache-store"); - match cache.store(&ctx.core_build_dir) { + let outcome = cache.store(&ctx.core_build_dir); + build_log_mutex + .lock() + .unwrap_or_else(|e| e.into_inner()) + .push(cache.store_summary(&outcome)); + match &outcome { Ok(stats) if stats.copied > 0 => tracing::info!( "framework core cache store key={} copied={} to {}", cache.key(), diff --git a/crates/fbuild-build-esp/src/esp32/orchestrator/build.rs b/crates/fbuild-build-esp/src/esp32/orchestrator/build.rs index dc6d2994a..d7e25f0d3 100644 --- a/crates/fbuild-build-esp/src/esp32/orchestrator/build.rs +++ b/crates/fbuild-build-esp/src/esp32/orchestrator/build.rs @@ -489,6 +489,7 @@ impl BuildOrchestrator for Esp32Orchestrator { build_dir, compiler_cache.as_deref(), &mut library_archives, + &mut ctx.build_log, ) .await?; } @@ -648,7 +649,13 @@ impl BuildOrchestrator for Esp32Orchestrator { let build_log_mutex = std::sync::Mutex::new(ctx.build_log); { let _g = perf.phase("core-cache-hydrate"); - match core_cache.hydrate(core_build_dir, &compiler, &all_core_sources, &user_overlay) { + let outcome = + core_cache.hydrate(core_build_dir, &compiler, &all_core_sources, &user_overlay); + build_log_mutex + .lock() + .unwrap_or_else(|e| e.into_inner()) + .push(core_cache.hydrate_summary(&outcome)); + match &outcome { Ok(stats) if stats.copied > 0 || stats.skipped > 0 => tracing::info!( "framework core cache hydrate key={} copied={} skipped={} from {}", core_cache.key(), @@ -683,7 +690,12 @@ impl BuildOrchestrator for Esp32Orchestrator { }; { let _g = perf.phase("core-cache-store"); - match core_cache.store(core_build_dir) { + let outcome = core_cache.store(core_build_dir); + build_log_mutex + .lock() + .unwrap_or_else(|e| e.into_inner()) + .push(core_cache.store_summary(&outcome)); + match &outcome { Ok(stats) if stats.copied > 0 => tracing::info!( "framework core cache store key={} copied={} to {}", core_cache.key(), diff --git a/crates/fbuild-build-esp/src/esp32/orchestrator/framework_library_cache.rs b/crates/fbuild-build-esp/src/esp32/orchestrator/framework_library_cache.rs index 0376a7f9e..026b897f6 100644 --- a/crates/fbuild-build-esp/src/esp32/orchestrator/framework_library_cache.rs +++ b/crates/fbuild-build-esp/src/esp32/orchestrator/framework_library_cache.rs @@ -107,6 +107,26 @@ impl FrameworkLibraryCache { } } +/// Build-output line for a [`FrameworkLibraryCache::hydrate`] outcome. CI logs +/// only carry build output, so this is how a restored cache shows it was used +/// (FastLED/fbuild#1433). +pub(super) fn hydrate_summary(outcome: &std::io::Result) -> String { + match outcome { + Ok(copied) if *copied > 0 => format!("framework-libs cache: hit restored={copied}"), + Ok(_) => "framework-libs cache: miss".to_string(), + Err(error) => format!("framework-libs cache: hydrate failed: {error}"), + } +} + +/// Build-output line for the archives compiled and stored by this build. +pub(super) fn store_summary(stored: usize) -> String { + if stored > 0 { + format!("framework-libs cache: stored {stored}") + } else { + "framework-libs cache: up to date".to_string() + } +} + fn cache_key( project_dir: &Path, profile: BuildProfile, @@ -186,6 +206,21 @@ fn is_header(path: &Path) -> bool { mod tests { use super::*; + #[test] + fn summaries_name_hit_miss_failure_and_stores() { + assert_eq!( + hydrate_summary(&Ok(4)), + "framework-libs cache: hit restored=4" + ); + assert_eq!(hydrate_summary(&Ok(0)), "framework-libs cache: miss"); + assert_eq!( + hydrate_summary(&Err(std::io::Error::other("gone"))), + "framework-libs cache: hydrate failed: gone" + ); + assert_eq!(store_summary(2), "framework-libs cache: stored 2"); + assert_eq!(store_summary(0), "framework-libs cache: up to date"); + } + #[test] fn cache_key_is_independent_of_project_path_when_headers_match() { let tmp = tempfile::tempdir().unwrap(); diff --git a/crates/fbuild-build-esp/src/esp32/orchestrator/framework_libs.rs b/crates/fbuild-build-esp/src/esp32/orchestrator/framework_libs.rs index a290bf600..5e4ff4fd4 100644 --- a/crates/fbuild-build-esp/src/esp32/orchestrator/framework_libs.rs +++ b/crates/fbuild-build-esp/src/esp32/orchestrator/framework_libs.rs @@ -10,7 +10,7 @@ use fbuild_packages::Framework; use super::super::esp32_compiler::Esp32Compiler; use super::super::mcu_config::Esp32McuConfig; -use super::framework_library_cache::FrameworkLibraryCache; +use super::framework_library_cache::{FrameworkLibraryCache, hydrate_summary, store_summary}; use super::helpers::{ framework_failure_marker, framework_signature, record_failed_framework_lib, should_skip_failed_framework_lib, @@ -36,6 +36,7 @@ pub(super) async fn compile_framework_builtin_libs( build_dir: &Path, compiler_cache: Option<&Path>, library_archives: &mut Vec, + build_log: &mut fbuild_core::BuildLog, ) -> Result<()> { use fbuild_packages::Toolchain; @@ -113,8 +114,10 @@ pub(super) async fn compile_framework_builtin_libs( if params.clean_only { return Ok(()); } - match framework_cache.hydrate(&fw_libs_build_dir) { - Ok(copied) if copied > 0 => tracing::info!( + let hydrate_outcome = framework_cache.hydrate(&fw_libs_build_dir); + build_log.push(hydrate_summary(&hydrate_outcome)); + match &hydrate_outcome { + Ok(copied) if *copied > 0 => tracing::info!( "hydrated {} cached ESP32 framework library archives", copied ), @@ -122,6 +125,7 @@ pub(super) async fn compile_framework_builtin_libs( Err(error) => tracing::warn!("failed to hydrate ESP32 framework library cache: {}", error), } + let mut fw_lib_stored = 0; let mut fw_lib_count = 0; let mut fw_lib_seen = 0; if let Ok(entries) = std::fs::read_dir(&builtin_libs_dir) { @@ -232,8 +236,13 @@ pub(super) async fn compile_framework_builtin_libs( { Ok(Some(archive)) => { let _ = std::fs::remove_file(&failure_marker); - if let Err(error) = framework_cache.store_archive(&archive) { - tracing::warn!("failed to cache framework library {}: {}", lib_name, error); + match framework_cache.store_archive(&archive) { + Ok(()) => fw_lib_stored += 1, + Err(error) => tracing::warn!( + "failed to cache framework library {}: {}", + lib_name, + error + ), } library_archives.push(archive); fw_lib_count += 1; @@ -279,6 +288,7 @@ pub(super) async fn compile_framework_builtin_libs( if fw_lib_count > 0 { tracing::info!("compiled {} framework built-in libraries", fw_lib_count); } + build_log.push(store_summary(fw_lib_stored)); perf.record("fw-libs", fw_libs_started.elapsed()); perf.checkpoint("fw-libs-finish"); Ok(()) diff --git a/crates/fbuild-cli/src/cli/args.rs b/crates/fbuild-cli/src/cli/args.rs index 581fe6262..25c35c4f6 100644 --- a/crates/fbuild-cli/src/cli/args.rs +++ b/crates/fbuild-cli/src/cli/args.rs @@ -766,7 +766,9 @@ pub enum Commands { /// Save / restore / list / verify the fbuild cache as a single /// portable `.tar.zst` archive (toolchains, platforms, framework, - /// downloaded archives, sqlite index). FastLED/fbuild#527. + /// downloaded archives, sqlite index). FastLED/fbuild#527. Per-board + /// build payloads (`core`, `framework-libs`, `library-selection`) and + /// `zccache` are opt-in via `--include` (FastLED/fbuild#1433). Cache { #[command(subcommand)] action: super::cache::CacheAction, diff --git a/crates/fbuild-cli/src/cli/cache.rs b/crates/fbuild-cli/src/cli/cache.rs index 98aa681dc..f1e845cde 100644 --- a/crates/fbuild-cli/src/cli/cache.rs +++ b/crates/fbuild-cli/src/cli/cache.rs @@ -142,7 +142,7 @@ fn render_manifest( for s in &m.slices { let _ = writeln!( out, - " {:<12} {:>6} file(s) {:>10} {}", + " {:<18} {:>6} file(s) {:>10} {}", s.name, s.file_count, human_bytes(s.byte_count), @@ -151,7 +151,7 @@ fn render_manifest( } let _ = write!( out, - " {:<12} {:>6} file(s) {:>10}", + " {:<18} {:>6} file(s) {:>10}", "TOTAL", m.total_files(), human_bytes(m.total_bytes()) diff --git a/crates/fbuild-packages-fetch/src/cache_archive.rs b/crates/fbuild-packages-fetch/src/cache_archive.rs index 85d284f7a..ad60371e9 100644 --- a/crates/fbuild-packages-fetch/src/cache_archive.rs +++ b/crates/fbuild-packages-fetch/src/cache_archive.rs @@ -66,9 +66,12 @@ struct SliceDef { default: bool, } -/// The full slice registry. Everything fbuild owns under the cache root is on -/// by default; the `zccache` engine store (a different root, content-addressed, -/// often better left to zccache itself) is opt-in. +/// The full slice registry. Downloaded packages are on by default. Build +/// payloads (`core`, `framework-libs`, `library-selection`) are opt-in: they +/// are per board, so CI keeps them in their own cache entry instead of mixing +/// them into the packages every board shares (FastLED/fbuild#1433). The +/// `zccache` engine store (a different root, content-addressed, often better +/// left to zccache itself) is opt-in too. const SLICES: &[SliceDef] = &[ SliceDef { name: "toolchains", @@ -119,6 +122,27 @@ const SLICES: &[SliceDef] = &[ is_file: true, default: true, }, + SliceDef { + name: "core", + root: Root::Cache, + rel: "core", + is_file: false, + default: false, + }, + SliceDef { + name: "framework-libs", + root: Root::Cache, + rel: "framework-libs", + is_file: false, + default: false, + }, + SliceDef { + name: "library-selection", + root: Root::Cache, + rel: "library-selection", + is_file: false, + default: false, + }, SliceDef { name: "zccache", root: Root::Fbuild, @@ -541,6 +565,76 @@ mod tests { write(cache, "index.sqlite", "SQLITE-DB"); } + fn seed_build_payload(cache: &Path) { + write(cache, "core/0123abcd/main.cpp.o", "OBJ"); + write(cache, "framework-libs/4567ef/libWiFi.a", "LIB"); + write(cache, "library-selection/esp32/89ab.pb", "SEL"); + } + + const BUILD_PAYLOAD: [&str; 3] = ["core", "framework-libs", "library-selection"]; + + #[test] + fn default_save_leaves_out_build_payload_slices() { + let src = tempfile::tempdir().unwrap(); + seed_cache(src.path()); + seed_build_payload(src.path()); + let archive = src.path().join("out.tar.zst"); + + let saved = save( + src.path(), + &archive, + &SliceSelection::Default, + &[], + DEFAULT_ZSTD_LEVEL, + ) + .unwrap(); + + let names: Vec<_> = saved.slices.iter().map(|s| s.name.as_str()).collect(); + for payload in BUILD_PAYLOAD { + assert!( + !names.contains(&payload), + "per-board payload slice {payload} must be opt-in: {names:?}" + ); + } + } + + #[test] + fn build_payload_slices_round_trip_when_included() { + let src = tempfile::tempdir().unwrap(); + seed_cache(src.path()); + seed_build_payload(src.path()); + let archive = src.path().join("payload.tar.zst"); + + let saved = save( + src.path(), + &archive, + &SliceSelection::Explicit(BUILD_PAYLOAD.iter().map(|s| s.to_string()).collect()), + &[], + DEFAULT_ZSTD_LEVEL, + ) + .unwrap(); + let names: Vec<_> = saved.slices.iter().map(|s| s.name.as_str()).collect(); + assert_eq!(names, BUILD_PAYLOAD.to_vec()); + + let dst = tempfile::tempdir().unwrap(); + restore(&archive, dst.path()).unwrap(); + for rel in [ + "core/0123abcd/main.cpp.o", + "framework-libs/4567ef/libWiFi.a", + "library-selection/esp32/89ab.pb", + ] { + assert_eq!( + std::fs::read(src.path().join(rel)).unwrap(), + std::fs::read(dst.path().join(rel)).unwrap(), + "mismatch restoring {rel}" + ); + } + assert!( + !dst.path().join("toolchains").exists(), + "a payload archive must not carry packages" + ); + } + #[test] fn round_trip_default_slices_reproduces_tree() { let src = tempfile::tempdir().unwrap(); From d03cc2fc9270cdabfe22c2ff714b1e3e3def3170 Mon Sep 17 00:00:00 2001 From: Zach Vorhies Date: Mon, 14 Sep 2026 17:17:20 -0700 Subject: [PATCH 2/4] feat(cli): `fbuild install` provisions an env's packages without compiling (#1433) Adds `fbuild install [project_dir] [-e ]... [--all-envs] [--check] [--dry-run] [--json] [-j N]`, which downloads everything an environment's build needs and stops: platform package, toolchains, framework, SDK libs, tools (esptool, picotool, CMSIS) and lib_deps. One line per package with status (present / fetched / would-fetch / failed), size from the cache index, duration, URL and sha256; `--json` adds a `packages_hash` over the sorted (kind, name, version, url, sha256) tuples for CI cache keys. `--check` and `--dry-run` never call `ensure_installed`, so they never touch the network; `--check` exits 2 when anything is missing. - `PlatformSupport::install_deps` (toolchain only, and the wrong one on ESP32 RISC-V and Teensy) is replaced by `provision`, implemented for all 13 platforms through one package helper per orchestrator that `build()` also uses, so install fetches exactly what a build would. - `fbuild_build::provision_env` resolves board and platform without a BuildContext (no extra_scripts, no build dirs) and also provisions lib_deps into the release build's `libs/` dir. `ensure_libraries` is split so its download half runs without compiling. - ESP32: check/dry-run resolve the toolchain from metadata already on disk (`resolve_toolchain_url_cached`); SDK libs and esptool get their own rows. - The daemon's `POST /api/install-deps` (used by `fbuild ide`) now calls `provision_env`, so it installs the full set too. - `PackageInfo` gains `checksum` and `installed_bytes` (read-only index lookup). The CLI runs in-process; `install.rs` is allowlisted in the cli_no_build_deploy_direct_use dylint with its reason. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01StasPkdQ3D1gYGj6WnaQ3q --- README.md | 1 + agents/docs/commands-reference.md | 1 + crates/CLAUDE.md | 2 +- crates/fbuild-build-arm/src/apollo3/mod.rs | 18 +- .../src/apollo3/orchestrator.rs | 48 +- crates/fbuild-build-arm/src/nrf52/mod.rs | 19 +- .../src/nrf52/orchestrator.rs | 46 +- crates/fbuild-build-arm/src/nxplpc/mod.rs | 28 +- .../src/nxplpc/orchestrator.rs | 62 ++- crates/fbuild-build-arm/src/renesas/mod.rs | 18 +- .../src/renesas/orchestrator.rs | 43 +- crates/fbuild-build-arm/src/rp2040/mod.rs | 35 +- .../src/rp2040/orchestrator.rs | 43 +- crates/fbuild-build-arm/src/sam/mod.rs | 35 +- .../fbuild-build-arm/src/sam/orchestrator.rs | 180 ++++--- crates/fbuild-build-arm/src/silabs/mod.rs | 18 +- .../src/silabs/orchestrator.rs | 38 +- crates/fbuild-build-arm/src/stm32/mod.rs | 27 +- .../src/stm32/orchestrator/arduino_mbed.rs | 14 +- .../src/stm32/orchestrator/mod.rs | 80 ++- crates/fbuild-build-arm/src/teensy/mod.rs | 18 +- .../src/teensy/orchestrator.rs | 38 +- crates/fbuild-build-engine/src/lib.rs | 32 +- crates/fbuild-build-engine/src/provision.rs | 480 ++++++++++++++++++ crates/fbuild-build-esp/src/esp32/mod.rs | 16 +- .../src/esp32/orchestrator/mod.rs | 2 + .../src/esp32/orchestrator/packages.rs | 401 ++++++++++++--- crates/fbuild-build-esp/src/esp8266/mod.rs | 18 +- .../src/esp8266/orchestrator.rs | 44 +- crates/fbuild-build-mcu/src/avr/mod.rs | 20 +- .../fbuild-build-mcu/src/avr/orchestrator.rs | 129 +++-- crates/fbuild-build-mcu/src/ch32v/mod.rs | 18 +- .../src/ch32v/orchestrator.rs | 41 +- crates/fbuild-build/src/lib.rs | 96 +++- crates/fbuild-cli/src/cli/README.md | 1 + crates/fbuild-cli/src/cli/args.rs | 29 ++ crates/fbuild-cli/src/cli/cache.rs | 2 +- crates/fbuild-cli/src/cli/dispatch.rs | 20 + crates/fbuild-cli/src/cli/install.rs | 295 +++++++++++ crates/fbuild-cli/src/cli/mod.rs | 1 + crates/fbuild-cli/src/cli/tests.rs | 37 ++ .../src/handlers/operations/install_deps.rs | 68 +-- .../src/library/esp32_framework/libs.rs | 11 + crates/fbuild-library/src/library/esptool.rs | 21 + .../src/library/library_downloader.rs | 12 +- .../src/library/library_manager.rs | 155 +++--- crates/fbuild-packages-fetch/src/lib.rs | 20 + .../src/toolchain/esp32_metadata.rs | 13 + docs/reference/cli.md | 33 ++ .../src/allowlist.txt | 6 + 50 files changed, 2253 insertions(+), 580 deletions(-) create mode 100644 crates/fbuild-build-engine/src/provision.rs create mode 100644 crates/fbuild-cli/src/cli/install.rs diff --git a/README.md b/README.md index 1d035cb4e..9b124ac59 100644 --- a/README.md +++ b/README.md @@ -183,6 +183,7 @@ These commands extend beyond the PlatformIO workflow surface: | `fbuild reset` | Reset a device without flashing it. | | `fbuild purge` | Purge downloaded packages or run cache garbage collection. | | `fbuild sync` | Resolve `platformio.ini` dependencies into a deterministic lock file. | +| `fbuild install` | Download an environment's platform, toolchains, framework, tools and `lib_deps` without compiling (`--check` exits 2 when something is missing). | | `fbuild daemon` | Manage the background build daemon, locks, and cache. | | `fbuild show` | Show daemon logs and other runtime information. | | `fbuild device` | List devices and manage device leases. | diff --git a/agents/docs/commands-reference.md b/agents/docs/commands-reference.md index 8a2253da2..34b5b9031 100644 --- a/agents/docs/commands-reference.md +++ b/agents/docs/commands-reference.md @@ -51,6 +51,7 @@ help text). | `fbuild show` | Show daemon logs or other introspection. | `fbuild help show` | | `fbuild device` | List / inspect connected devices the daemon knows about. | `fbuild help device` | | `fbuild purge` | Purge cached packages — full purge or LRU-only via `--gc`. | `fbuild help purge` | +| `fbuild install [project_dir] [-e ]... [--all-envs] [--check] [--dry-run] [--json]` | You want an env's platform, toolchains, framework, tools and `lib_deps` downloaded without compiling — e.g. a separate, observable CI step before the build. One line per package (`present` / `fetched` / `would-fetch` / `failed`); `--json` adds a `packages_hash` to key a packages cache on. `--check` and `--dry-run` never touch the network; `--check` exits 2 when anything is missing. Runs in-process, no daemon. | `fbuild help install`, FastLED/fbuild#1433, `docs/reference/cli.md#fbuild-install` | | `fbuild lnk` | Manage `.fetch` blob pointers (fetch / verify / add). `.lnk` is still read for pointers written before FastLED/fbuild#1369; FastLED's runtime `.lnk` asset links are a different format and are skipped. | `fbuild help lnk` | ## Serial-port introspection (FastLED/fbuild#686) diff --git a/crates/CLAUDE.md b/crates/CLAUDE.md index 98493cd6e..19132cd8b 100644 --- a/crates/CLAUDE.md +++ b/crates/CLAUDE.md @@ -77,7 +77,7 @@ fbuild-test-support (test utilities) ────────────── **HTTP API boundary:** CLI sends JSON requests to daemon over HTTP. Build output streams via WebSocket. Serial monitor data streams via `/ws/serial-monitor`. All endpoints match the Python FastAPI daemon's contract. -**Diagnostic subcommand exception:** A small, growing set of `fbuild-cli` subcommands (`clang-tidy`, `clang-query`, `iwyu`, `mcp`, `lnk`, `lib-select`) run in-process and intentionally bypass the daemon. They are read-only diagnostics that don't need build orchestration, so a round-trip through the HTTP API would only add latency. The "thin HTTP client" rule still applies to every command that touches the build pipeline (`build`, `deploy`, `monitor`, `test-emu`, etc.). +**Diagnostic subcommand exception:** A small, growing set of `fbuild-cli` subcommands (`clang-tidy`, `clang-query`, `iwyu`, `mcp`, `lnk`, `lib-select`) run in-process and intentionally bypass the daemon. They are read-only diagnostics that don't need build orchestration, so a round-trip through the HTTP API would only add latency. `install` also runs in-process: it only resolves and downloads packages into the cache (no compile, no build lock), and CI runs it as its own step before any build (FastLED/fbuild#1433). The "thin HTTP client" rule still applies to every command that touches the build pipeline (`build`, `deploy`, `monitor`, `test-emu`, etc.). **PyO3 consumer contract:** FastLED imports `SerialMonitor` as a Python context manager with `read_lines()`, `write()`, `write_json_rpc()`. The `fbuild-python` crate must preserve this API exactly. diff --git a/crates/fbuild-build-arm/src/apollo3/mod.rs b/crates/fbuild-build-arm/src/apollo3/mod.rs index 809dcd5d2..05af8e0c3 100644 --- a/crates/fbuild-build-arm/src/apollo3/mod.rs +++ b/crates/fbuild-build-arm/src/apollo3/mod.rs @@ -14,12 +14,18 @@ impl crate::PlatformSupport for Apollo3PlatformSupport { orchestrator::create() } - async fn install_deps(&self, project_dir: &std::path::Path) -> fbuild_core::Result<()> { - use fbuild_packages::Package; - let tc = fbuild_packages::toolchain::ArmGcc8Toolchain::new(project_dir); - Package::ensure_installed(&tc).await?; - tracing::info!("ARM GCC 8 toolchain installed"); - Ok(()) + async fn provision( + &self, + inputs: &crate::provision::ProvisionInputs<'_>, + mode: crate::provision::ProvisionMode, + ) -> fbuild_core::Result> { + use crate::provision::{PackageKind, provision_package}; + let (toolchain, cores) = + orchestrator::apollo3_packages(inputs.project_dir, Some(inputs.env_config)); + Ok(vec![ + provision_package(PackageKind::Toolchain, &toolchain, mode).await, + provision_package(PackageKind::Framework, &cores, mode).await, + ]) } fn default_board_id(&self) -> &str { diff --git a/crates/fbuild-build-arm/src/apollo3/orchestrator.rs b/crates/fbuild-build-arm/src/apollo3/orchestrator.rs index 1036cf829..c490e1861 100644 --- a/crates/fbuild-build-arm/src/apollo3/orchestrator.rs +++ b/crates/fbuild-build-arm/src/apollo3/orchestrator.rs @@ -36,6 +36,29 @@ fn profile_label(profile: fbuild_core::BuildProfile) -> &'static str { } } +/// Apollo3's ARM GCC 8 toolchain (mbed-os requires GCC 8) and SparkFun Apollo3 +/// cores for an env, honoring the `framework-arduinoambiqapollo3` +/// `platform_packages` override (FastLED/fbuild#664, #681). Shared by the +/// build and `fbuild install`, so both provision the same packages +/// (FastLED/fbuild#1433). +pub(crate) fn apollo3_packages( + project_dir: &Path, + env_config: Option<&std::collections::HashMap>, +) -> ( + fbuild_packages::toolchain::ArmGcc8Toolchain, + fbuild_packages::library::Apollo3Cores, +) { + let toolchain = fbuild_packages::toolchain::ArmGcc8Toolchain::new(project_dir); + let override_pin = env_config.and_then(|env| { + crate::package_override::resolve_override(env, "framework-arduinoambiqapollo3") + }); + let cores = match override_pin { + Some(o) => fbuild_packages::library::Apollo3Cores::with_override(project_dir, o), + None => fbuild_packages::library::Apollo3Cores::new(project_dir), + }; + (toolchain, cores) +} + #[async_trait::async_trait] impl BuildOrchestrator for Apollo3Orchestrator { fn platform(&self) -> Platform { @@ -52,8 +75,11 @@ impl BuildOrchestrator for Apollo3Orchestrator { let eh_frame_policy = crate::eh_frame_policy_compute::compute_eh_frame_policy(&ctx, params.profile, None); - // 3. Ensure ARM GCC 8 toolchain (Apollo3/mbed-os requires GCC 8) - let toolchain = fbuild_packages::toolchain::ArmGcc8Toolchain::new(¶ms.project_dir); + // 3-4. ARM GCC 8 toolchain and Apollo3 cores + let (toolchain, framework) = apollo3_packages( + ¶ms.project_dir, + ctx.config.get_env_config(¶ms.env_name).ok(), + ); let toolchain_dir = fbuild_packages::Package::ensure_installed(&toolchain).await?; tracing::info!("arm-gcc8 toolchain at {}", toolchain_dir.display()); @@ -65,24 +91,6 @@ impl BuildOrchestrator for Apollo3Orchestrator { ) .await; - // 4. Ensure Apollo3 cores (SparkFun Arduino Apollo3 core) - // Honor `platform_packages` override from the env section - // (FastLED/fbuild#664, #681): if set, the override URL replaces the - // const-pinned default and gets its own cache subdir via - // `PackageBase::with_override`. - let __ovr = ctx - .config - .get_env_config(¶ms.env_name) - .ok() - .and_then(|env| { - crate::package_override::resolve_override(env, "framework-arduinoambiqapollo3") - }); - let framework = match __ovr { - Some(o) => { - fbuild_packages::library::Apollo3Cores::with_override(¶ms.project_dir, o) - } - None => fbuild_packages::library::Apollo3Cores::new(¶ms.project_dir), - }; let framework_dir = fbuild_packages::Package::ensure_installed(&framework).await?; tracing::info!("Apollo3 cores at {}", framework_dir.display()); diff --git a/crates/fbuild-build-arm/src/nrf52/mod.rs b/crates/fbuild-build-arm/src/nrf52/mod.rs index a7dbf0a1c..5bf64f461 100644 --- a/crates/fbuild-build-arm/src/nrf52/mod.rs +++ b/crates/fbuild-build-arm/src/nrf52/mod.rs @@ -18,12 +18,19 @@ impl crate::PlatformSupport for Nrf52PlatformSupport { orchestrator::create() } - async fn install_deps(&self, project_dir: &std::path::Path) -> fbuild_core::Result<()> { - use fbuild_packages::Package; - let tc = fbuild_packages::toolchain::ArmToolchain::new(project_dir); - Package::ensure_installed(&tc).await?; - tracing::info!("ARM toolchain installed"); - Ok(()) + async fn provision( + &self, + inputs: &crate::provision::ProvisionInputs<'_>, + mode: crate::provision::ProvisionMode, + ) -> fbuild_core::Result> { + use crate::provision::{PackageKind, provision_package}; + let (toolchain, cores, cmsis) = + orchestrator::nrf52_packages(inputs.project_dir, Some(inputs.env_config)); + Ok(vec![ + provision_package(PackageKind::Toolchain, &toolchain, mode).await, + provision_package(PackageKind::Framework, &cores, mode).await, + provision_package(PackageKind::Framework, &cmsis, mode).await, + ]) } fn default_board_id(&self) -> &str { diff --git a/crates/fbuild-build-arm/src/nrf52/orchestrator.rs b/crates/fbuild-build-arm/src/nrf52/orchestrator.rs index 68060b21e..5608c4f92 100644 --- a/crates/fbuild-build-arm/src/nrf52/orchestrator.rs +++ b/crates/fbuild-build-arm/src/nrf52/orchestrator.rs @@ -57,6 +57,30 @@ fn profile_label(profile: fbuild_core::BuildProfile) -> &'static str { } } +/// NRF52's ARM GCC toolchain, Adafruit nRF52 cores and CMSIS for an env, +/// honoring the `framework-arduinoadafruitnrf52` `platform_packages` override +/// (FastLED/fbuild#664, #681). Shared by the build and `fbuild install`, so +/// both provision the same packages (FastLED/fbuild#1433). +pub(crate) fn nrf52_packages( + project_dir: &Path, + env_config: Option<&std::collections::HashMap>, +) -> ( + fbuild_packages::toolchain::ArmToolchain, + fbuild_packages::library::Nrf52Cores, + fbuild_packages::library::CmsisFramework, +) { + let toolchain = fbuild_packages::toolchain::ArmToolchain::new(project_dir); + let override_pin = env_config.and_then(|env| { + crate::package_override::resolve_override(env, "framework-arduinoadafruitnrf52") + }); + let cores = match override_pin { + Some(o) => fbuild_packages::library::Nrf52Cores::with_override(project_dir, o), + None => fbuild_packages::library::Nrf52Cores::new(project_dir), + }; + let cmsis = fbuild_packages::library::CmsisFramework::new(project_dir); + (toolchain, cores, cmsis) +} + #[async_trait::async_trait] impl BuildOrchestrator for Nrf52Orchestrator { fn platform(&self) -> Platform { @@ -70,8 +94,12 @@ impl BuildOrchestrator for Nrf52Orchestrator { // 1-2. Parse config, load board, setup build dirs, resolve src dir, collect flags let mut ctx = pipeline::BuildContext::new(params).await?; - // 3. Ensure ARM GCC toolchain - let toolchain = fbuild_packages::toolchain::ArmToolchain::new(¶ms.project_dir); + // 3-4. ARM GCC toolchain and NRF52 cores (Adafruit nRF52 Arduino + // core). CMSIS is installed later, once include discovery needs it. + let (toolchain, framework, cmsis) = nrf52_packages( + ¶ms.project_dir, + ctx.config.get_env_config(¶ms.env_name).ok(), + ); let toolchain_dir = fbuild_packages::Package::ensure_installed(&toolchain).await?; tracing::info!("arm-none-eabi toolchain at {}", toolchain_dir.display()); @@ -83,19 +111,6 @@ impl BuildOrchestrator for Nrf52Orchestrator { ) .await; - // 4. Ensure NRF52 cores (Adafruit nRF52 Arduino core) - // Honor `platform_packages` override (FastLED/fbuild#664, #681). - let __ovr = ctx - .config - .get_env_config(¶ms.env_name) - .ok() - .and_then(|env| { - crate::package_override::resolve_override(env, "framework-arduinoadafruitnrf52") - }); - let framework = match __ovr { - Some(o) => fbuild_packages::library::Nrf52Cores::with_override(¶ms.project_dir, o), - None => fbuild_packages::library::Nrf52Cores::new(¶ms.project_dir), - }; let framework_dir = fbuild_packages::Package::ensure_installed(&framework).await?; tracing::info!("NRF52 cores at {}", framework_dir.display()); @@ -244,7 +259,6 @@ impl BuildOrchestrator for Nrf52Orchestrator { // Toolchain sysroot includes include_dirs.extend(toolchain.get_include_dirs()); // CMSIS Core includes (core_cm4.h, etc.) - let cmsis = fbuild_packages::library::CmsisFramework::new(¶ms.project_dir); let _cmsis_dir = fbuild_packages::Package::ensure_installed(&cmsis).await?; tracing::info!("CMSIS framework installed"); include_dirs.push(cmsis.get_core_include_dir()); diff --git a/crates/fbuild-build-arm/src/nxplpc/mod.rs b/crates/fbuild-build-arm/src/nxplpc/mod.rs index aef41131b..846ee6d0e 100644 --- a/crates/fbuild-build-arm/src/nxplpc/mod.rs +++ b/crates/fbuild-build-arm/src/nxplpc/mod.rs @@ -20,8 +20,6 @@ pub mod orchestrator; // `fbuild_config::platform_packages` so every orchestrator gets the same // parser without duplication. -use std::path::Path; - use fbuild_core::Result; /// NXP LPC8xx platform support. @@ -33,19 +31,19 @@ impl crate::PlatformSupport for NxpLpcPlatformSupport { orchestrator::create() } - async fn install_deps(&self, project_dir: &Path) -> Result<()> { - // ARM GCC is the right toolchain for Cortex-M0+ bare metal. - // Pre-install it (+ CMSIS + the Arduino core framework) so the - // orchestrator can `ensure_installed` cheaply. - use fbuild_packages::Package; - let tc = fbuild_packages::toolchain::ArmToolchain::new(project_dir); - Package::ensure_installed(&tc).await?; - let cmsis = fbuild_packages::library::CmsisFramework::new(project_dir); - Package::ensure_installed(&cmsis).await?; - let core = fbuild_packages::library::ArduinoCoreLpc8xx::new(project_dir); - Package::ensure_installed(&core).await?; - tracing::info!("ARM GCC toolchain + ArduinoCore-LPC8xx installed for NXP LPC8xx"); - Ok(()) + async fn provision( + &self, + inputs: &crate::provision::ProvisionInputs<'_>, + mode: crate::provision::ProvisionMode, + ) -> Result> { + use crate::provision::{PackageKind, provision_package}; + let (toolchain, cmsis, core) = + orchestrator::nxplpc_packages(inputs.project_dir, Some(inputs.env_config)); + Ok(vec![ + provision_package(PackageKind::Toolchain, &toolchain, mode).await, + provision_package(PackageKind::Framework, &cmsis, mode).await, + provision_package(PackageKind::Framework, &core, mode).await, + ]) } fn default_board_id(&self) -> &str { diff --git a/crates/fbuild-build-arm/src/nxplpc/orchestrator.rs b/crates/fbuild-build-arm/src/nxplpc/orchestrator.rs index 7004881d8..93467d398 100644 --- a/crates/fbuild-build-arm/src/nxplpc/orchestrator.rs +++ b/crates/fbuild-build-arm/src/nxplpc/orchestrator.rs @@ -91,6 +91,29 @@ fn profile_label(profile: fbuild_core::BuildProfile) -> &'static str { } } +/// NXP LPC8xx's ARM GCC toolchain, CMSIS and ArduinoCore-LPC8xx for an env, +/// honoring the `framework-arduino-lpc8xx` `platform_packages` override +/// (FastLED/fbuild#663, #681). Shared by the build and `fbuild install`, so +/// both provision the same packages (FastLED/fbuild#1433). +pub(crate) fn nxplpc_packages( + project_dir: &std::path::Path, + env_config: Option<&std::collections::HashMap>, +) -> ( + fbuild_packages::toolchain::ArmToolchain, + fbuild_packages::library::CmsisFramework, + fbuild_packages::library::ArduinoCoreLpc8xx, +) { + let toolchain = fbuild_packages::toolchain::ArmToolchain::new(project_dir); + let cmsis = fbuild_packages::library::CmsisFramework::new(project_dir); + let override_pin = env_config + .and_then(|env| crate::package_override::resolve_override(env, "framework-arduino-lpc8xx")); + let core = match override_pin { + Some(o) => fbuild_packages::library::ArduinoCoreLpc8xx::with_override(project_dir, o), + None => fbuild_packages::library::ArduinoCoreLpc8xx::new(project_dir), + }; + (toolchain, cmsis, core) +} + #[async_trait::async_trait] impl BuildOrchestrator for NxpLpcOrchestrator { fn platform(&self) -> Platform { @@ -108,14 +131,17 @@ impl BuildOrchestrator for NxpLpcOrchestrator { let eh_frame_policy = crate::eh_frame_policy_compute::compute_eh_frame_policy(&ctx, params.profile, None); - // 3. Ensure ARM GCC. `install_deps` already pre-installs this when - // the platform is dispatched, but ensure_installed is idempotent - // and cheap when the toolchain is already on disk. - let toolchain = fbuild_packages::toolchain::ArmToolchain::new(¶ms.project_dir); + let env_config = ctx.config.get_env_config(¶ms.env_name).ok(); + let core_override = env_config.and_then(|env| { + crate::package_override::resolve_override(env, "framework-arduino-lpc8xx") + }); + let (toolchain, cmsis, core) = nxplpc_packages(¶ms.project_dir, env_config); + + // 3. Ensure ARM GCC. ensure_installed is idempotent and cheap when + // the toolchain is already on disk. let toolchain_dir = fbuild_packages::Package::ensure_installed(&toolchain).await?; tracing::info!("arm-none-eabi-gcc toolchain at {}", toolchain_dir.display()); - let cmsis = fbuild_packages::library::CmsisFramework::new(¶ms.project_dir); let cmsis_dir = fbuild_packages::Package::ensure_installed(&cmsis).await?; tracing::info!("CMSIS framework at {}", cmsis_dir.display()); @@ -138,25 +164,13 @@ impl BuildOrchestrator for NxpLpcOrchestrator { // cache subdir via `PackageBase::with_override`. The parser // + resolver are shared across every framework orchestrator so // nxplpc carries no platform-specific platform_packages logic. - let core_override = ctx - .config - .get_env_config(¶ms.env_name) - .ok() - .and_then(|env| { - crate::package_override::resolve_override(env, "framework-arduino-lpc8xx") - }); - let core = match core_override { - Some(ovr) => { - let banner = format!( - "ArduinoCore-LPC8xx OVERRIDE: {} (default pinned: {})", - ovr.url, - fbuild_packages::library::ArduinoCoreLpc8xx::commit() - ); - ctx.build_log.push(banner); - fbuild_packages::library::ArduinoCoreLpc8xx::with_override(¶ms.project_dir, ovr) - } - None => fbuild_packages::library::ArduinoCoreLpc8xx::new(¶ms.project_dir), - }; + if let Some(ovr) = &core_override { + ctx.build_log.push(format!( + "ArduinoCore-LPC8xx OVERRIDE: {} (default pinned: {})", + ovr.url, + fbuild_packages::library::ArduinoCoreLpc8xx::commit() + )); + } let core_root = fbuild_packages::Package::ensure_installed(&core).await?; tracing::info!("ArduinoCore-LPC8xx at {}", core_root.display()); diff --git a/crates/fbuild-build-arm/src/renesas/mod.rs b/crates/fbuild-build-arm/src/renesas/mod.rs index c3b9a457c..3649a425e 100644 --- a/crates/fbuild-build-arm/src/renesas/mod.rs +++ b/crates/fbuild-build-arm/src/renesas/mod.rs @@ -18,12 +18,18 @@ impl crate::PlatformSupport for RenesasPlatformSupport { orchestrator::create() } - async fn install_deps(&self, project_dir: &std::path::Path) -> fbuild_core::Result<()> { - use fbuild_packages::Package; - let tc = fbuild_packages::toolchain::ArmToolchain::new(project_dir); - Package::ensure_installed(&tc).await?; - tracing::info!("ARM toolchain installed"); - Ok(()) + async fn provision( + &self, + inputs: &crate::provision::ProvisionInputs<'_>, + mode: crate::provision::ProvisionMode, + ) -> fbuild_core::Result> { + use crate::provision::{PackageKind, provision_package}; + let (toolchain, cores) = + orchestrator::renesas_packages(inputs.project_dir, Some(inputs.env_config)); + Ok(vec![ + provision_package(PackageKind::Toolchain, &toolchain, mode).await, + provision_package(PackageKind::Framework, &cores, mode).await, + ]) } fn default_board_id(&self) -> &str { diff --git a/crates/fbuild-build-arm/src/renesas/orchestrator.rs b/crates/fbuild-build-arm/src/renesas/orchestrator.rs index e4992b1f7..1301b1f08 100644 --- a/crates/fbuild-build-arm/src/renesas/orchestrator.rs +++ b/crates/fbuild-build-arm/src/renesas/orchestrator.rs @@ -58,6 +58,27 @@ fn profile_label(profile: fbuild_core::BuildProfile) -> &'static str { } } +/// Renesas RA's ARM GCC toolchain and ArduinoCore-renesas for an env, honoring +/// the `framework-arduinorenesas` `platform_packages` override +/// (FastLED/fbuild#664, #681). Shared by the build and `fbuild install`, so +/// both provision the same packages (FastLED/fbuild#1433). +pub(crate) fn renesas_packages( + project_dir: &Path, + env_config: Option<&std::collections::HashMap>, +) -> ( + fbuild_packages::toolchain::ArmToolchain, + fbuild_packages::library::RenesasCores, +) { + let toolchain = fbuild_packages::toolchain::ArmToolchain::new(project_dir); + let override_pin = env_config + .and_then(|env| crate::package_override::resolve_override(env, "framework-arduinorenesas")); + let cores = match override_pin { + Some(o) => fbuild_packages::library::RenesasCores::with_override(project_dir, o), + None => fbuild_packages::library::RenesasCores::new(project_dir), + }; + (toolchain, cores) +} + #[async_trait::async_trait] impl BuildOrchestrator for RenesasOrchestrator { fn platform(&self) -> Platform { @@ -71,8 +92,11 @@ impl BuildOrchestrator for RenesasOrchestrator { // 1-2. Parse config, load board, setup build dirs, resolve src dir, collect flags let mut ctx = pipeline::BuildContext::new(params).await?; - // 3. Ensure ARM GCC toolchain - let toolchain = fbuild_packages::toolchain::ArmToolchain::new(¶ms.project_dir); + // 3-4. ARM GCC toolchain and Renesas cores (ArduinoCore-renesas) + let (toolchain, framework) = renesas_packages( + ¶ms.project_dir, + ctx.config.get_env_config(¶ms.env_name).ok(), + ); let toolchain_dir = fbuild_packages::Package::ensure_installed(&toolchain).await?; tracing::info!("arm-gcc toolchain at {}", toolchain_dir.display()); @@ -84,21 +108,6 @@ impl BuildOrchestrator for RenesasOrchestrator { ) .await; - // 4. Ensure Renesas cores (ArduinoCore-renesas) - // Honor `platform_packages` override (FastLED/fbuild#664, #681). - let __ovr = ctx - .config - .get_env_config(¶ms.env_name) - .ok() - .and_then(|env| { - crate::package_override::resolve_override(env, "framework-arduinorenesas") - }); - let framework = match __ovr { - Some(o) => { - fbuild_packages::library::RenesasCores::with_override(¶ms.project_dir, o) - } - None => fbuild_packages::library::RenesasCores::new(¶ms.project_dir), - }; let framework_dir = fbuild_packages::Package::ensure_installed(&framework).await?; tracing::info!("Renesas cores at {}", framework_dir.display()); diff --git a/crates/fbuild-build-arm/src/rp2040/mod.rs b/crates/fbuild-build-arm/src/rp2040/mod.rs index bf547015b..540b7a713 100644 --- a/crates/fbuild-build-arm/src/rp2040/mod.rs +++ b/crates/fbuild-build-arm/src/rp2040/mod.rs @@ -15,12 +15,37 @@ impl crate::PlatformSupport for Rp2040PlatformSupport { orchestrator::create() } - async fn install_deps(&self, project_dir: &std::path::Path) -> fbuild_core::Result<()> { + async fn provision( + &self, + inputs: &crate::provision::ProvisionInputs<'_>, + mode: crate::provision::ProvisionMode, + ) -> fbuild_core::Result> { + use crate::provision::{PackageKind, provision_package}; + let (toolchain, picotool, cores) = + orchestrator::rp2040_packages(inputs.project_dir, Some(inputs.env_config)); + Ok(vec![ + provision_package(PackageKind::Toolchain, &toolchain, mode).await, + provision_package(PackageKind::Tool, &picotool, mode).await, + provision_package(PackageKind::Framework, &cores, mode).await, + ]) + } + + /// arduino-pico bundles libraries (Wire, SPI, ...) that `lib_deps` may + /// name; the build filters them out before downloading, so provisioning + /// must too. That needs the installed cores — without them, every entry + /// is reported. + fn downloadable_lib_deps( + &self, + inputs: &crate::provision::ProvisionInputs<'_>, + lib_deps: Vec, + ) -> Vec { use fbuild_packages::Package; - let tc = fbuild_packages::toolchain::ArmToolchain::new(project_dir); - Package::ensure_installed(&tc).await?; - tracing::info!("ARM toolchain installed"); - Ok(()) + let (_, _, cores) = + orchestrator::rp2040_packages(inputs.project_dir, Some(inputs.env_config)); + if !cores.is_installed() { + return lib_deps; + } + fbuild_library_select::external_declared_deps(&lib_deps, &cores.get_framework_libraries()) } fn default_board_id(&self) -> &str { diff --git a/crates/fbuild-build-arm/src/rp2040/orchestrator.rs b/crates/fbuild-build-arm/src/rp2040/orchestrator.rs index 0aa579fc0..ca203b663 100644 --- a/crates/fbuild-build-arm/src/rp2040/orchestrator.rs +++ b/crates/fbuild-build-arm/src/rp2040/orchestrator.rs @@ -65,6 +65,30 @@ fn profile_label(profile: fbuild_core::BuildProfile) -> &'static str { } } +/// RP2040's arduino-pico-matched pqt-gcc toolchain, managed picotool and +/// arduino-pico cores for an env, honoring the `framework-arduinopico` +/// `platform_packages` override (FastLED/fbuild#664, #681). Shared by the +/// build and `fbuild install`, so both provision the same packages +/// (FastLED/fbuild#1433). +pub(crate) fn rp2040_packages( + project_dir: &Path, + env_config: Option<&HashMap>, +) -> ( + fbuild_packages::toolchain::Rp2040PqtToolchain, + fbuild_packages::toolchain::Rp2040Picotool, + fbuild_packages::library::Rp2040Cores, +) { + let toolchain = fbuild_packages::toolchain::Rp2040PqtToolchain::new(project_dir); + let picotool = fbuild_packages::toolchain::Rp2040Picotool::new(project_dir); + let override_pin = env_config + .and_then(|env| crate::package_override::resolve_override(env, "framework-arduinopico")); + let cores = match override_pin { + Some(o) => fbuild_packages::library::Rp2040Cores::with_override(project_dir, o), + None => fbuild_packages::library::Rp2040Cores::new(project_dir), + }; + (toolchain, picotool, cores) +} + #[async_trait::async_trait] impl BuildOrchestrator for Rp2040Orchestrator { fn platform(&self) -> Platform { @@ -82,15 +106,18 @@ impl BuildOrchestrator for Rp2040Orchestrator { let eh_frame_policy = crate::eh_frame_policy_compute::compute_eh_frame_policy(&ctx, params.profile, None); + let (toolchain, picotool, framework) = rp2040_packages( + ¶ms.project_dir, + ctx.config.get_env_config(¶ms.env_name).ok(), + ); + // 3. Ensure the arduino-pico-matched pqt-gcc toolchain - let toolchain = fbuild_packages::toolchain::Rp2040PqtToolchain::new(¶ms.project_dir); let toolchain_dir = fbuild_packages::Package::ensure_installed(&toolchain).await?; tracing::info!("rp2040 pqt-gcc toolchain at {}", toolchain_dir.display()); // Arduino-Pico generates its canonical UF2 from the linked ELF with // the managed pqt-picotool package. Do the same here rather than // flattening ELF segments in an fbuild-specific encoder. - let picotool = fbuild_packages::toolchain::Rp2040Picotool::new(¶ms.project_dir); let picotool_dir = fbuild_packages::Package::ensure_installed(&picotool).await?; tracing::info!("managed picotool at {}", picotool_dir.display()); @@ -103,18 +130,6 @@ impl BuildOrchestrator for Rp2040Orchestrator { .await; // 4. Ensure RP2040 cores (arduino-pico by earlephilhower) - // Honor `platform_packages` override (FastLED/fbuild#664, #681). - let __ovr = ctx - .config - .get_env_config(¶ms.env_name) - .ok() - .and_then(|env| { - crate::package_override::resolve_override(env, "framework-arduinopico") - }); - let framework = match __ovr { - Some(o) => fbuild_packages::library::Rp2040Cores::with_override(¶ms.project_dir, o), - None => fbuild_packages::library::Rp2040Cores::new(¶ms.project_dir), - }; let framework_dir = fbuild_packages::Package::ensure_installed(&framework).await?; tracing::info!("RP2040 cores at {}", framework_dir.display()); let board_id = ctx diff --git a/crates/fbuild-build-arm/src/sam/mod.rs b/crates/fbuild-build-arm/src/sam/mod.rs index 1021069fc..ff3e699b3 100644 --- a/crates/fbuild-build-arm/src/sam/mod.rs +++ b/crates/fbuild-build-arm/src/sam/mod.rs @@ -18,12 +18,35 @@ impl crate::PlatformSupport for SamPlatformSupport { orchestrator::create() } - async fn install_deps(&self, project_dir: &std::path::Path) -> fbuild_core::Result<()> { - use fbuild_packages::Package; - let tc = fbuild_packages::toolchain::ArmToolchain::new(project_dir); - Package::ensure_installed(&tc).await?; - tracing::info!("ARM toolchain installed"); - Ok(()) + async fn provision( + &self, + inputs: &crate::provision::ProvisionInputs<'_>, + mode: crate::provision::ProvisionMode, + ) -> fbuild_core::Result> { + use crate::provision::{PackageKind, provision_package}; + use orchestrator::SamCore; + let (toolchain, core) = + orchestrator::sam_packages(inputs.project_dir, Some(inputs.env_config), inputs.board); + let mut rows = vec![provision_package(PackageKind::Toolchain, &*toolchain, mode).await]; + match core { + SamCore::Sam(cores) => { + rows.push(provision_package(PackageKind::Framework, &cores, mode).await); + } + SamCore::Samd { + cores, + cmsis, + cmsis_atmel, + } => { + rows.push(provision_package(PackageKind::Framework, &cores, mode).await); + rows.push(provision_package(PackageKind::Framework, &cmsis, mode).await); + rows.push(provision_package(PackageKind::Framework, &cmsis_atmel, mode).await); + } + SamCore::ClearCore { cores, cmsis } => { + rows.push(provision_package(PackageKind::Framework, &cores, mode).await); + rows.push(provision_package(PackageKind::Framework, &cmsis, mode).await); + } + } + Ok(rows) } fn default_board_id(&self) -> &str { diff --git a/crates/fbuild-build-arm/src/sam/orchestrator.rs b/crates/fbuild-build-arm/src/sam/orchestrator.rs index cd14cac1c..da7e3d576 100644 --- a/crates/fbuild-build-arm/src/sam/orchestrator.rs +++ b/crates/fbuild-build-arm/src/sam/orchestrator.rs @@ -102,6 +102,83 @@ fn profile_label(profile: fbuild_core::BuildProfile) -> &'static str { } } +/// The Arduino core (and its external CMSIS packages) a SAM-family board +/// builds against. +// Built once per build or provision and consumed immediately, so the size gap +// between variants never multiplies; boxing would only add indirection. +#[allow(clippy::large_enum_variant)] +pub(crate) enum SamCore { + /// ArduinoCore-sam for classic SAM3X boards (Due). + Sam(fbuild_packages::library::SamCores), + /// Adafruit ArduinoCore-samd for SAMD21/51 and SAME5x, plus the CMSIS and + /// CMSIS-Atmel device headers it does not bundle. + Samd { + cores: fbuild_packages::library::SamdCores, + cmsis: fbuild_packages::library::CmsisFramework, + cmsis_atmel: fbuild_packages::library::CmsisAtmel, + }, + /// Teknic's ClearCore Arduino package for ATSAME53, plus generic CMSIS for + /// the Cortex-M4 core headers and DSP library. + ClearCore { + cores: fbuild_packages::library::ClearCoreCores, + cmsis: fbuild_packages::library::CmsisFramework, + }, +} + +/// SAM's toolchain and Arduino core packages for an env and board. +/// +/// Teknic's precompiled ClearCore/LwIP libraries are built with GCC 7, so +/// ClearCore gets fbuild's GCC 9 package — the closest supported toolchain, +/// avoiding the much larger GCC 15 ABI/version gap; every other board gets +/// the current ARM GCC. SAM (Arduino ArduinoCore-sam), SAMD (Adafruit +/// ArduinoCore-samd, PIO package `framework-arduino-samd-adafruit` per +/// FastLED/fbuild#677) and ClearCore are distinct PIO packages, each honoring +/// its own `platform_packages` override (FastLED/fbuild#664, #681). Shared by +/// the build and `fbuild install`, so both provision the same packages +/// (FastLED/fbuild#1433). +pub(crate) fn sam_packages( + project_dir: &Path, + env_config: Option<&std::collections::HashMap>, + board: &fbuild_config::BoardConfig, +) -> (Box, SamCore) { + let override_for = |package: &str| { + env_config.and_then(|env| crate::package_override::resolve_override(env, package)) + }; + if is_clearcore_board(board) { + let cores = match override_for("framework-arduino-sam-clearcore") { + Some(o) => fbuild_packages::library::ClearCoreCores::with_override(project_dir, o), + None => fbuild_packages::library::ClearCoreCores::new(project_dir), + }; + return ( + Box::new(fbuild_packages::toolchain::ArmGcc8Toolchain::new( + project_dir, + )), + SamCore::ClearCore { + cores, + cmsis: fbuild_packages::library::CmsisFramework::new(project_dir), + }, + ); + } + let toolchain: Box = + Box::new(fbuild_packages::toolchain::ArmToolchain::new(project_dir)); + let core = if is_samd_mcu(&board.mcu) { + SamCore::Samd { + cores: match override_for("framework-arduino-samd-adafruit") { + Some(o) => fbuild_packages::library::SamdCores::with_override(project_dir, o), + None => fbuild_packages::library::SamdCores::new(project_dir), + }, + cmsis: fbuild_packages::library::CmsisFramework::new(project_dir), + cmsis_atmel: fbuild_packages::library::CmsisAtmel::new(project_dir), + } + } else { + SamCore::Sam(match override_for("framework-arduino-sam") { + Some(o) => fbuild_packages::library::SamCores::with_override(project_dir, o), + None => fbuild_packages::library::SamCores::new(project_dir), + }) + }; + (toolchain, core) +} + #[async_trait::async_trait] impl BuildOrchestrator for SamOrchestrator { fn platform(&self) -> Platform { @@ -116,20 +193,12 @@ impl BuildOrchestrator for SamOrchestrator { let mut ctx = pipeline::BuildContext::new(params).await?; let clearcore = is_clearcore_board(&ctx.board); - // 3. Ensure ARM GCC toolchain - // Teknic's precompiled ClearCore/LwIP libraries are built with GCC 7. - // fbuild's GCC 9 package is the closest supported toolchain and avoids - // needlessly crossing the much larger GCC 15 ABI/version gap. - use fbuild_packages::Toolchain; - let toolchain: Box = if clearcore { - Box::new(fbuild_packages::toolchain::ArmGcc8Toolchain::new( - ¶ms.project_dir, - )) - } else { - Box::new(fbuild_packages::toolchain::ArmToolchain::new( - ¶ms.project_dir, - )) - }; + // 3. Ensure ARM GCC toolchain (GCC 9 for ClearCore; see `sam_packages`) + let (toolchain, core) = sam_packages( + ¶ms.project_dir, + ctx.config.get_env_config(¶ms.env_name).ok(), + &ctx.board, + ); let toolchain_dir = toolchain.ensure_installed().await?; tracing::info!("arm-none-eabi toolchain at {}", toolchain_dir.display()); @@ -141,36 +210,6 @@ impl BuildOrchestrator for SamOrchestrator { .await; // 4. Ensure correct Arduino core based on MCU family - // Honor `platform_packages` override (FastLED/fbuild#664, #681). Only - // SAM (Due) is wired through this PR — SAMD's framework_name needs - // separate verification (see issue thread). - // FastLED/fbuild#664, #681: honor `platform_packages` override per - // framework. SAMD (Adafruit ArduinoCore-samd) and SAM (Arduino - // ArduinoCore-sam) are distinct PIO packages, so resolve both - // overrides and route to the branch that runs. - let __sam_ovr = ctx - .config - .get_env_config(¶ms.env_name) - .ok() - .and_then(|env| { - crate::package_override::resolve_override(env, "framework-arduino-sam") - }); - // FastLED/fbuild#677: fbuild ships Adafruit's `ArduinoCore-samd`, - // so the matching PIO package name is `framework-arduino-samd-adafruit`. - let __samd_ovr = ctx - .config - .get_env_config(¶ms.env_name) - .ok() - .and_then(|env| { - crate::package_override::resolve_override(env, "framework-arduino-samd-adafruit") - }); - let __clearcore_ovr = ctx - .config - .get_env_config(¶ms.env_name) - .ok() - .and_then(|env| { - crate::package_override::resolve_override(env, "framework-arduino-sam-clearcore") - }); let SamCoreInstall { framework_dir, core_dir, @@ -179,13 +218,27 @@ impl BuildOrchestrator for SamOrchestrator { system_includes, library_dirs, libraries, - } = if clearcore { - install_clearcore_core(params, &ctx.board.core, &ctx.board.variant, __clearcore_ovr) + } = match core { + SamCore::ClearCore { cores, cmsis } => { + install_clearcore_core(cores, cmsis, &ctx.board.core, &ctx.board.variant).await? + } + SamCore::Samd { + cores, + cmsis, + cmsis_atmel, + } => { + install_samd_core( + cores, + cmsis, + cmsis_atmel, + &ctx.board.core, + &ctx.board.variant, + ) .await? - } else if is_samd_mcu(&ctx.board.mcu) { - install_samd_core(params, &ctx.board.core, &ctx.board.variant, __samd_ovr).await? - } else { - install_sam_core(params, &ctx.board.core, &ctx.board.variant, __sam_ovr).await? + } + SamCore::Sam(cores) => { + install_sam_core(cores, &ctx.board.core, &ctx.board.variant).await? + } }; let build_dir = &ctx.build_dir; @@ -440,15 +493,10 @@ impl BuildOrchestrator for SamOrchestrator { /// Install ArduinoCore-sam for classic SAM3X boards (Due). async fn install_sam_core( - params: &BuildParams, + framework: fbuild_packages::library::SamCores, core_name: &str, variant_name: &str, - ovr: Option, ) -> Result { - let framework = match ovr { - Some(o) => fbuild_packages::library::SamCores::with_override(¶ms.project_dir, o), - None => fbuild_packages::library::SamCores::new(¶ms.project_dir), - }; let framework_dir = fbuild_packages::Package::ensure_installed(&framework).await?; tracing::info!("SAM cores at {}", framework_dir.display()); @@ -479,15 +527,12 @@ async fn install_sam_core( /// Install Adafruit ArduinoCore-samd for SAMD21/SAMD51 boards. async fn install_samd_core( - params: &BuildParams, + framework: fbuild_packages::library::SamdCores, + cmsis: fbuild_packages::library::CmsisFramework, + cmsis_atmel: fbuild_packages::library::CmsisAtmel, core_name: &str, variant_name: &str, - override_: Option, ) -> Result { - let framework = match override_ { - Some(o) => fbuild_packages::library::SamdCores::with_override(¶ms.project_dir, o), - None => fbuild_packages::library::SamdCores::new(¶ms.project_dir), - }; let framework_dir = fbuild_packages::Package::ensure_installed(&framework).await?; tracing::info!("SAMD cores at {}", framework_dir.display()); @@ -496,11 +541,9 @@ async fn install_samd_core( let linker_script = framework.get_linker_script(variant_name); // SAMD core needs external CMSIS and CMSIS-Atmel packages for device headers - let cmsis = fbuild_packages::library::CmsisFramework::new(¶ms.project_dir); let cmsis_dir = fbuild_packages::Package::ensure_installed(&cmsis).await?; tracing::info!("CMSIS at {}", cmsis_dir.display()); - let cmsis_atmel = fbuild_packages::library::CmsisAtmel::new(¶ms.project_dir); let _cmsis_atmel_dir = fbuild_packages::Package::ensure_installed(&cmsis_atmel).await?; tracing::info!("CMSIS-Atmel installed"); @@ -543,15 +586,11 @@ async fn install_samd_core( /// device headers and precompiled ClearCore/LwIP libraries. Generic CMSIS /// supplies the Cortex-M4 core headers and DSP library. async fn install_clearcore_core( - params: &BuildParams, + framework: fbuild_packages::library::ClearCoreCores, + cmsis: fbuild_packages::library::CmsisFramework, core_name: &str, variant_name: &str, - override_: Option, ) -> Result { - let framework = match override_ { - Some(o) => fbuild_packages::library::ClearCoreCores::with_override(¶ms.project_dir, o), - None => fbuild_packages::library::ClearCoreCores::new(¶ms.project_dir), - }; let framework_dir = fbuild_packages::Package::ensure_installed(&framework).await?; tracing::info!("ClearCore Arduino core at {}", framework_dir.display()); @@ -559,7 +598,6 @@ async fn install_clearcore_core( let variant_dir = framework.get_variant_dir(variant_name); let linker_script = framework.get_linker_script(variant_name); - let cmsis = fbuild_packages::library::CmsisFramework::new(¶ms.project_dir); let cmsis_dir = fbuild_packages::Package::ensure_installed(&cmsis).await?; tracing::info!("CMSIS at {}", cmsis_dir.display()); diff --git a/crates/fbuild-build-arm/src/silabs/mod.rs b/crates/fbuild-build-arm/src/silabs/mod.rs index bc39a5609..1d2e7fc5e 100644 --- a/crates/fbuild-build-arm/src/silabs/mod.rs +++ b/crates/fbuild-build-arm/src/silabs/mod.rs @@ -18,12 +18,18 @@ impl crate::PlatformSupport for SilabsPlatformSupport { orchestrator::create() } - async fn install_deps(&self, project_dir: &std::path::Path) -> fbuild_core::Result<()> { - use fbuild_packages::Package; - let tc = fbuild_packages::toolchain::ArmToolchain::new(project_dir); - Package::ensure_installed(&tc).await?; - tracing::info!("ARM toolchain installed"); - Ok(()) + async fn provision( + &self, + inputs: &crate::provision::ProvisionInputs<'_>, + mode: crate::provision::ProvisionMode, + ) -> fbuild_core::Result> { + use crate::provision::{PackageKind, provision_package}; + let (toolchain, cores) = + orchestrator::silabs_packages(inputs.project_dir, Some(inputs.env_config)); + Ok(vec![ + provision_package(PackageKind::Toolchain, &toolchain, mode).await, + provision_package(PackageKind::Framework, &cores, mode).await, + ]) } fn default_board_id(&self) -> &str { diff --git a/crates/fbuild-build-arm/src/silabs/orchestrator.rs b/crates/fbuild-build-arm/src/silabs/orchestrator.rs index 40ce73c97..a69602b4a 100644 --- a/crates/fbuild-build-arm/src/silabs/orchestrator.rs +++ b/crates/fbuild-build-arm/src/silabs/orchestrator.rs @@ -25,6 +25,27 @@ fn profile_label(profile: fbuild_core::BuildProfile) -> &'static str { } } +/// Silicon Labs' ARM GCC toolchain and Arduino cores for an env, honoring the +/// `framework-arduinosilabs` `platform_packages` override (FastLED/fbuild#664, +/// #681). Shared by the build and `fbuild install`, so both provision the same +/// packages (FastLED/fbuild#1433). +pub(crate) fn silabs_packages( + project_dir: &Path, + env_config: Option<&HashMap>, +) -> ( + fbuild_packages::toolchain::ArmToolchain, + fbuild_packages::library::SilabsCores, +) { + let toolchain = fbuild_packages::toolchain::ArmToolchain::new(project_dir); + let override_pin = env_config + .and_then(|env| crate::package_override::resolve_override(env, "framework-arduinosilabs")); + let cores = match override_pin { + Some(o) => fbuild_packages::library::SilabsCores::with_override(project_dir, o), + None => fbuild_packages::library::SilabsCores::new(project_dir), + }; + (toolchain, cores) +} + #[async_trait::async_trait] impl BuildOrchestrator for SilabsOrchestrator { fn platform(&self) -> Platform { @@ -36,7 +57,10 @@ impl BuildOrchestrator for SilabsOrchestrator { let mut ctx = pipeline::BuildContext::new(params).await?; - let toolchain = fbuild_packages::toolchain::ArmToolchain::new(¶ms.project_dir); + let (toolchain, framework) = silabs_packages( + ¶ms.project_dir, + ctx.config.get_env_config(¶ms.env_name).ok(), + ); let toolchain_dir = fbuild_packages::Package::ensure_installed(&toolchain).await?; tracing::info!("arm-gcc toolchain at {}", toolchain_dir.display()); @@ -48,18 +72,6 @@ impl BuildOrchestrator for SilabsOrchestrator { ) .await; - // Honor `platform_packages` override (FastLED/fbuild#664, #681). - let __ovr = ctx - .config - .get_env_config(¶ms.env_name) - .ok() - .and_then(|env| { - crate::package_override::resolve_override(env, "framework-arduinosilabs") - }); - let framework = match __ovr { - Some(o) => fbuild_packages::library::SilabsCores::with_override(¶ms.project_dir, o), - None => fbuild_packages::library::SilabsCores::new(¶ms.project_dir), - }; let framework_dir = fbuild_packages::Package::ensure_installed(&framework).await?; tracing::info!("Silicon Labs cores at {}", framework_dir.display()); diff --git a/crates/fbuild-build-arm/src/stm32/mod.rs b/crates/fbuild-build-arm/src/stm32/mod.rs index 040ce4426..e9e3d9901 100644 --- a/crates/fbuild-build-arm/src/stm32/mod.rs +++ b/crates/fbuild-build-arm/src/stm32/mod.rs @@ -1,4 +1,4 @@ -//! STM32 platform build support (STM32F1, STM32F4, STM32H7, etc.) +//! STM32 platform build support (STM32F1, STM32F4, STM32H7, etc.) pub mod mcu_config; pub mod orchestrator; @@ -14,12 +14,25 @@ impl crate::PlatformSupport for Stm32PlatformSupport { orchestrator::create() } - async fn install_deps(&self, project_dir: &std::path::Path) -> fbuild_core::Result<()> { - use fbuild_packages::Package; - let tc = fbuild_packages::toolchain::ArmToolchain::new(project_dir); - Package::ensure_installed(&tc).await?; - tracing::info!("ARM toolchain installed"); - Ok(()) + async fn provision( + &self, + inputs: &crate::provision::ProvisionInputs<'_>, + mode: crate::provision::ProvisionMode, + ) -> fbuild_core::Result> { + use crate::provision::{PackageKind, provision_package}; + let (toolchain, core) = + orchestrator::stm32_packages(inputs.project_dir, Some(inputs.env_config), inputs.board); + let mut rows = vec![provision_package(PackageKind::Toolchain, &toolchain, mode).await]; + match core { + orchestrator::Stm32Core::Stm32duino { cores, cmsis } => { + rows.push(provision_package(PackageKind::Framework, &cores, mode).await); + rows.push(provision_package(PackageKind::Framework, &cmsis, mode).await); + } + orchestrator::Stm32Core::ArduinoMbed(core) => { + rows.push(provision_package(PackageKind::Framework, &core, mode).await); + } + } + Ok(rows) } fn default_board_id(&self) -> &str { diff --git a/crates/fbuild-build-arm/src/stm32/orchestrator/arduino_mbed.rs b/crates/fbuild-build-arm/src/stm32/orchestrator/arduino_mbed.rs index 39ff4e37d..a3232ed92 100644 --- a/crates/fbuild-build-arm/src/stm32/orchestrator/arduino_mbed.rs +++ b/crates/fbuild-build-arm/src/stm32/orchestrator/arduino_mbed.rs @@ -32,23 +32,15 @@ pub(super) async fn build_arduino_mbed_stm32( params: &BuildParams, ctx: pipeline::BuildContext, toolchain: &fbuild_packages::toolchain::ArmToolchain, + framework: fbuild_packages::library::ArduinoMbedCore, start: Instant, ) -> Result { // Compute eh_frame strip policy once per build (FastLED/fbuild#244). let eh_frame_policy = crate::eh_frame_policy_compute::compute_eh_frame_policy(&ctx, params.profile, None); - // Honor `platform_packages` override from the env section - // (FastLED/fbuild#664, #681). - let __ovr = ctx - .config - .get_env_config(¶ms.env_name) - .ok() - .and_then(|env| crate::package_override::resolve_override(env, "framework-arduino-mbed")); - let framework = match __ovr { - Some(o) => fbuild_packages::library::ArduinoMbedCore::with_override(¶ms.project_dir, o), - None => fbuild_packages::library::ArduinoMbedCore::new(¶ms.project_dir), - }; + // `framework` honors the `framework-arduino-mbed` `platform_packages` + // override (FastLED/fbuild#664, #681); see `super::stm32_packages`. let framework_dir = fbuild_packages::Package::ensure_installed(&framework).await?; tracing::info!("Arduino mbed core at {}", framework_dir.display()); diff --git a/crates/fbuild-build-arm/src/stm32/orchestrator/mod.rs b/crates/fbuild-build-arm/src/stm32/orchestrator/mod.rs index 0e472dc82..e98ee8dbb 100644 --- a/crates/fbuild-build-arm/src/stm32/orchestrator/mod.rs +++ b/crates/fbuild-build-arm/src/stm32/orchestrator/mod.rs @@ -59,6 +59,54 @@ fn profile_label(profile: fbuild_core::BuildProfile) -> &'static str { } } +/// The Arduino core an STM32 board builds against. +// Built once per build or provision and consumed immediately, so the size gap +// between variants never multiplies; boxing would only add indirection. +#[allow(clippy::large_enum_variant)] +pub(crate) enum Stm32Core { + /// STM32duino cores plus the CMSIS Core headers they do not bundle. + Stm32duino { + cores: fbuild_packages::library::Stm32Cores, + cmsis: fbuild_packages::library::CmsisFramework, + }, + /// Arduino's prebuilt mbed core for GIGA, PORTENTA, ... variants. + ArduinoMbed(fbuild_packages::library::ArduinoMbedCore), +} + +/// STM32's ARM GCC toolchain and Arduino core for an env: the Arduino mbed +/// core for mbed variants, otherwise STM32duino + CMSIS. Honors the +/// `framework-arduino-mbed` / `framework-arduinoststm32` `platform_packages` +/// overrides (FastLED/fbuild#664, #681). Shared by the build and `fbuild +/// install`, so both provision the same packages (FastLED/fbuild#1433). +pub(crate) fn stm32_packages( + project_dir: &Path, + env_config: Option<&std::collections::HashMap>, + board: &fbuild_config::BoardConfig, +) -> (fbuild_packages::toolchain::ArmToolchain, Stm32Core) { + let toolchain = fbuild_packages::toolchain::ArmToolchain::new(project_dir); + let core = if is_arduino_mbed_stm32_variant(&board.variant) { + let override_pin = env_config.and_then(|env| { + crate::package_override::resolve_override(env, "framework-arduino-mbed") + }); + Stm32Core::ArduinoMbed(match override_pin { + Some(o) => fbuild_packages::library::ArduinoMbedCore::with_override(project_dir, o), + None => fbuild_packages::library::ArduinoMbedCore::new(project_dir), + }) + } else { + let override_pin = env_config.and_then(|env| { + crate::package_override::resolve_override(env, "framework-arduinoststm32") + }); + Stm32Core::Stm32duino { + cores: match override_pin { + Some(o) => fbuild_packages::library::Stm32Cores::with_override(project_dir, o), + None => fbuild_packages::library::Stm32Cores::new(project_dir), + }, + cmsis: fbuild_packages::library::CmsisFramework::new(project_dir), + } + }; + (toolchain, core) +} + #[async_trait::async_trait] impl BuildOrchestrator for Stm32Orchestrator { fn platform(&self) -> Platform { @@ -75,8 +123,12 @@ impl BuildOrchestrator for Stm32Orchestrator { let eh_frame_policy = crate::eh_frame_policy_compute::compute_eh_frame_policy(&ctx, params.profile, None); - // 3. Ensure ARM GCC toolchain - let toolchain = fbuild_packages::toolchain::ArmToolchain::new(¶ms.project_dir); + // 3. ARM GCC toolchain and the board's Arduino core + let (toolchain, core) = stm32_packages( + ¶ms.project_dir, + ctx.config.get_env_config(¶ms.env_name).ok(), + &ctx.board, + ); let toolchain_dir = fbuild_packages::Package::ensure_installed(&toolchain).await?; tracing::info!("arm-gcc toolchain at {}", toolchain_dir.display()); @@ -87,22 +139,13 @@ impl BuildOrchestrator for Stm32Orchestrator { ) .await; - if is_arduino_mbed_stm32_variant(&ctx.board.variant) { - return build_arduino_mbed_stm32(params, ctx, &toolchain, start).await; - } - - // 4. Ensure STM32duino cores - // Honor `platform_packages` override (FastLED/fbuild#664, #681). - let __ovr = ctx - .config - .get_env_config(¶ms.env_name) - .ok() - .and_then(|env| { - crate::package_override::resolve_override(env, "framework-arduinoststm32") - }); - let framework = match __ovr { - Some(o) => fbuild_packages::library::Stm32Cores::with_override(¶ms.project_dir, o), - None => fbuild_packages::library::Stm32Cores::new(¶ms.project_dir), + // 4. Ensure STM32duino cores (CMSIS is installed later, once include + // discovery needs it). + let (framework, cmsis) = match core { + Stm32Core::ArduinoMbed(framework) => { + return build_arduino_mbed_stm32(params, ctx, &toolchain, framework, start).await; + } + Stm32Core::Stm32duino { cores, cmsis } => (cores, cmsis), }; let framework_dir = fbuild_packages::Package::ensure_installed(&framework).await?; tracing::info!("STM32 cores at {}", framework_dir.display()); @@ -344,7 +387,6 @@ impl BuildOrchestrator for Stm32Orchestrator { add_stm32_system_includes(&system_dir, family, &mut include_dirs); // CMSIS Core includes (core_cm3.h, core_cm4.h, etc.) — not bundled in STM32duino - let cmsis = fbuild_packages::library::CmsisFramework::new(¶ms.project_dir); let _cmsis_dir = fbuild_packages::Package::ensure_installed(&cmsis).await?; tracing::info!("CMSIS framework installed"); include_dirs.push(cmsis.get_core_include_dir()); diff --git a/crates/fbuild-build-arm/src/teensy/mod.rs b/crates/fbuild-build-arm/src/teensy/mod.rs index 0fe139d5b..d8358983e 100644 --- a/crates/fbuild-build-arm/src/teensy/mod.rs +++ b/crates/fbuild-build-arm/src/teensy/mod.rs @@ -18,12 +18,18 @@ impl crate::PlatformSupport for TeensyPlatformSupport { orchestrator::create() } - async fn install_deps(&self, project_dir: &std::path::Path) -> fbuild_core::Result<()> { - use fbuild_packages::Package; - let tc = fbuild_packages::toolchain::ArmToolchain::new(project_dir); - Package::ensure_installed(&tc).await?; - tracing::info!("ARM toolchain installed"); - Ok(()) + async fn provision( + &self, + inputs: &crate::provision::ProvisionInputs<'_>, + mode: crate::provision::ProvisionMode, + ) -> fbuild_core::Result> { + use crate::provision::{PackageKind, provision_package}; + let (toolchain, cores) = + orchestrator::teensy_packages(inputs.project_dir, Some(inputs.env_config)); + Ok(vec![ + provision_package(PackageKind::Toolchain, &toolchain, mode).await, + provision_package(PackageKind::Framework, &cores, mode).await, + ]) } fn default_board_id(&self) -> &str { diff --git a/crates/fbuild-build-arm/src/teensy/orchestrator.rs b/crates/fbuild-build-arm/src/teensy/orchestrator.rs index 9eacfba76..998d6a55f 100644 --- a/crates/fbuild-build-arm/src/teensy/orchestrator.rs +++ b/crates/fbuild-build-arm/src/teensy/orchestrator.rs @@ -62,6 +62,27 @@ fn profile_label(profile: fbuild_core::BuildProfile) -> &'static str { } } +/// Teensy's ARM GCC toolchain and cores for an env, honoring the +/// `framework-arduinoteensy` `platform_packages` override (FastLED/fbuild#664, +/// #681). Shared by the build and `fbuild install`, so both provision the same +/// packages (FastLED/fbuild#1433). +pub(crate) fn teensy_packages( + project_dir: &Path, + env_config: Option<&std::collections::HashMap>, +) -> ( + fbuild_packages::toolchain::TeensyArmToolchain, + fbuild_packages::library::TeensyCores, +) { + let toolchain = fbuild_packages::toolchain::TeensyArmToolchain::new(project_dir); + let override_pin = env_config + .and_then(|env| crate::package_override::resolve_override(env, "framework-arduinoteensy")); + let cores = match override_pin { + Some(o) => fbuild_packages::library::TeensyCores::with_override(project_dir, o), + None => fbuild_packages::library::TeensyCores::new(project_dir), + }; + (toolchain, cores) +} + #[async_trait::async_trait] impl BuildOrchestrator for TeensyOrchestrator { fn platform(&self) -> Platform { @@ -86,8 +107,8 @@ impl BuildOrchestrator for TeensyOrchestrator { fbuild_core::FbuildError::ConfigError("missing 'board' in environment config".into()) })?; - // 3. Ensure Teensy-compatible ARM GCC toolchain - let toolchain = fbuild_packages::toolchain::TeensyArmToolchain::new(¶ms.project_dir); + // 3-4. Teensy-compatible ARM GCC toolchain and Teensy cores + let (toolchain, framework) = teensy_packages(¶ms.project_dir, Some(env_config)); let toolchain_dir = fbuild_packages::Package::ensure_installed(&toolchain).await?; tracing::info!("Teensy ARM GCC toolchain at {}", toolchain_dir.display()); @@ -99,19 +120,6 @@ impl BuildOrchestrator for TeensyOrchestrator { ) .await; - // 4. Ensure Teensy cores - // Honor `platform_packages` override (FastLED/fbuild#664, #681). - let __ovr = ctx - .config - .get_env_config(¶ms.env_name) - .ok() - .and_then(|env| { - crate::package_override::resolve_override(env, "framework-arduinoteensy") - }); - let framework = match __ovr { - Some(o) => fbuild_packages::library::TeensyCores::with_override(¶ms.project_dir, o), - None => fbuild_packages::library::TeensyCores::new(¶ms.project_dir), - }; let framework_dir = fbuild_packages::Package::ensure_installed(&framework).await?; tracing::info!("Teensy cores at {}", framework_dir.display()); diff --git a/crates/fbuild-build-engine/src/lib.rs b/crates/fbuild-build-engine/src/lib.rs index 65ace7068..073bbdeb8 100644 --- a/crates/fbuild-build-engine/src/lib.rs +++ b/crates/fbuild-build-engine/src/lib.rs @@ -28,6 +28,7 @@ pub mod package_override; pub mod parallel; pub mod perf_log; pub mod pipeline; +pub mod provision; pub mod rebuild_signature; pub mod resolution; pub mod script_runtime; @@ -39,25 +40,42 @@ pub mod zccache_embedded; pub use source_scanner::SourceScanner; -use std::path::{Path, PathBuf}; +use std::path::PathBuf; use fbuild_core::{BuildProfile, Platform, Result, SizeInfo}; /// Trait for platform-specific build support. /// /// Each platform crate implements this to provide orchestrator creation, -/// dependency installation, and configuration. The `fbuild-build` facade's +/// package provisioning, and configuration. The `fbuild-build` facade's /// `get_platform_support()` factory maps a [`Platform`] to the right impl. -/// -/// FastLED/fbuild#820 (Phase B of #813): `install_deps` is `async` so -/// per-platform impls can `.await` `fbuild_packages::Package::ensure_installed`. #[async_trait::async_trait] pub trait PlatformSupport: Send + Sync { /// Create the build orchestrator for this platform. fn create_orchestrator(&self) -> Box; - /// Install platform-specific dependencies (toolchain, framework). - async fn install_deps(&self, project_dir: &Path) -> Result<()>; + /// Provision every package this env's build downloads — platform, + /// toolchain, framework, SDK libs and tools — without compiling, one + /// report row per package. Implementations resolve packages through the + /// same helpers their orchestrator uses, so `fbuild install` fetches + /// exactly what a build would. `lib_deps` are provisioned by the caller + /// (FastLED/fbuild#1433). + async fn provision( + &self, + inputs: &provision::ProvisionInputs<'_>, + mode: provision::ProvisionMode, + ) -> Result>; + + /// The `lib_deps` entries this platform's build downloads. Defaults to + /// all of them; a platform whose framework bundles libraries filters + /// those out. + fn downloadable_lib_deps( + &self, + _inputs: &provision::ProvisionInputs<'_>, + lib_deps: Vec, + ) -> Vec { + lib_deps + } /// Default board ID used as fallback when none is specified. fn default_board_id(&self) -> &str; diff --git a/crates/fbuild-build-engine/src/provision.rs b/crates/fbuild-build-engine/src/provision.rs new file mode 100644 index 000000000..3d8c9ab83 --- /dev/null +++ b/crates/fbuild-build-engine/src/provision.rs @@ -0,0 +1,480 @@ +//! Provision an environment's packages without compiling — the engine behind +//! `fbuild install` (FastLED/fbuild#1433). +//! +//! Each platform lists what its build downloads through +//! [`crate::PlatformSupport::provision`]; [`provision_package`] turns one +//! [`Package`] into a report row, so every platform reports presence, fetches, +//! durations and sizes the same way. [`ProvisionMode::Check`] and +//! [`ProvisionMode::DryRun`] never call `ensure_installed`, so they never touch +//! the network. + +use std::collections::HashMap; +use std::path::Path; +use std::time::Instant; + +use fbuild_config::BoardConfig; +use fbuild_packages::Package; +use serde::Serialize; +use sha2::{Digest, Sha256}; + +/// What provisioning is allowed to do. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ProvisionMode { + /// Fetch anything missing. + Install, + /// Report what is missing without fetching; callers exit non-zero when + /// something would need fetching. + Check, + /// Report the resolved set without fetching. + DryRun, +} + +impl ProvisionMode { + /// Whether this mode may download and install. + pub fn fetches(self) -> bool { + matches!(self, Self::Install) + } +} + +/// Outcome for one package. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "kebab-case")] +pub enum ProvisionStatus { + /// Already installed before this run. + Present, + /// Installed by this run. + Fetched, + /// Missing; a check or dry run did not fetch it. + WouldFetch, + /// Fetching failed. + Failed, +} + +impl ProvisionStatus { + pub fn as_str(self) -> &'static str { + match self { + Self::Present => "present", + Self::Fetched => "fetched", + Self::WouldFetch => "would-fetch", + Self::Failed => "failed", + } + } +} + +/// What role a package plays in the build. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize)] +#[serde(rename_all = "kebab-case")] +pub enum PackageKind { + Platform, + Toolchain, + Framework, + SdkLibs, + Tool, + Library, +} + +impl PackageKind { + pub fn as_str(self) -> &'static str { + match self { + Self::Platform => "platform", + Self::Toolchain => "toolchain", + Self::Framework => "framework", + Self::SdkLibs => "sdk-libs", + Self::Tool => "tool", + Self::Library => "library", + } + } +} + +/// One row of a provisioning report. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct ProvisionedPackage { + pub kind: PackageKind, + pub name: String, + pub version: String, + pub url: String, + pub sha256: Option, + pub status: ProvisionStatus, + /// Installed size from the package cache index, when it has a row. + pub bytes: Option, + pub duration_ms: u64, + pub install_path: Option, + pub error: Option, +} + +impl ProvisionedPackage { + /// A row for a package whose presence was decided outside [`Package`] + /// (SDK libs, standalone tools, libraries). + pub fn new(kind: PackageKind, name: impl Into, status: ProvisionStatus) -> Self { + Self { + kind, + name: name.into(), + version: String::new(), + url: String::new(), + sha256: None, + status, + bytes: None, + duration_ms: 0, + install_path: None, + error: None, + } + } +} + +/// Everything a platform needs to resolve its packages for one env. +pub struct ProvisionInputs<'a> { + pub project_dir: &'a Path, + pub env_name: &'a str, + pub env_config: &'a HashMap, + pub board: &'a BoardConfig, +} + +/// Provision one [`Package`] and describe the outcome. +pub async fn provision_package( + kind: PackageKind, + package: &dyn Package, + mode: ProvisionMode, +) -> ProvisionedPackage { + let started = Instant::now(); + let (status, error) = if package.is_installed() { + (ProvisionStatus::Present, None) + } else if !mode.fetches() { + (ProvisionStatus::WouldFetch, None) + } else { + match package.ensure_installed().await { + Ok(_) => (ProvisionStatus::Fetched, None), + Err(error) => (ProvisionStatus::Failed, Some(error.to_string())), + } + }; + let info = package.get_info(); + let installed = matches!(status, ProvisionStatus::Present | ProvisionStatus::Fetched); + ProvisionedPackage { + kind, + name: info.name, + version: info.version, + url: info.url, + sha256: info.checksum, + status, + bytes: if installed { + info.installed_bytes + } else { + None + }, + duration_ms: started.elapsed().as_millis() as u64, + install_path: Some(info.install_path.display().to_string()), + error, + } +} + +/// Provision `lib_deps` into `libs_dir` — the directory the build downloads +/// them into — without compiling. A check or dry run reports direct +/// dependencies only: transitive ones are only known once the direct ones are +/// downloaded. +pub async fn provision_lib_deps( + project_dir: &Path, + lib_deps: &[String], + lib_ignore: &[String], + libs_dir: &Path, + mode: ProvisionMode, +) -> Vec { + use fbuild_packages::library::{library_downloader, library_manager}; + + let specs = library_manager::parse_lib_specs(lib_deps, lib_ignore); + let started = Instant::now(); + let present_before: Vec = specs + .iter() + .map(|spec| spec.local_path.is_some() || library_downloader::is_downloaded(spec, libs_dir)) + .collect(); + let mut rows: Vec = specs + .iter() + .zip(&present_before) + .map(|(spec, present)| { + let status = if *present { + ProvisionStatus::Present + } else { + ProvisionStatus::WouldFetch + }; + library_row(spec, libs_dir, status) + }) + .collect(); + if !mode.fetches() || present_before.iter().all(|present| *present) { + return rows; + } + + match library_manager::download_libraries(lib_deps, lib_ignore, project_dir, libs_dir).await { + Ok(installed) => { + let elapsed_ms = started.elapsed().as_millis() as u64; + for row in rows + .iter_mut() + .filter(|row| row.status == ProvisionStatus::WouldFetch) + { + row.status = ProvisionStatus::Fetched; + row.duration_ms = elapsed_ms; + } + for library in installed { + let path = library.lib_dir.display().to_string(); + if rows + .iter() + .all(|row| row.install_path.as_deref() != Some(path.as_str())) + { + rows.push(ProvisionedPackage { + install_path: Some(path), + duration_ms: elapsed_ms, + ..ProvisionedPackage::new( + PackageKind::Library, + library.name, + ProvisionStatus::Fetched, + ) + }); + } + } + } + Err(error) => { + for row in rows + .iter_mut() + .filter(|row| row.status == ProvisionStatus::WouldFetch) + { + row.status = ProvisionStatus::Failed; + row.error = Some(error.to_string()); + } + } + } + rows +} + +fn library_row( + spec: &fbuild_packages::library::library_spec::LibrarySpec, + libs_dir: &Path, + status: ProvisionStatus, +) -> ProvisionedPackage { + let name = if spec.owner.is_empty() { + spec.name.clone() + } else { + format!("{}/{}", spec.owner, spec.name) + }; + let (url, install_path) = match (&spec.local_path, &spec.github_url) { + (Some(local), _) => ( + format!("file://{}", local.display()), + local.display().to_string(), + ), + (None, Some(github)) => ( + github.clone(), + libs_dir.join(spec.sanitized_name()).display().to_string(), + ), + (None, None) => ( + format!("registry:{name}"), + libs_dir.join(spec.sanitized_name()).display().to_string(), + ), + }; + ProvisionedPackage { + version: spec.version.clone().unwrap_or_default(), + url, + install_path: Some(install_path), + ..ProvisionedPackage::new(PackageKind::Library, name, status) + } +} + +/// An env's provisioning report. +#[derive(Debug, Clone, Serialize)] +pub struct ProvisionReport { + pub env: String, + pub platform: String, + pub packages: Vec, +} + +impl ProvisionReport { + /// True when a check or dry run found something missing. + pub fn needs_fetch(&self) -> bool { + self.packages + .iter() + .any(|p| p.status == ProvisionStatus::WouldFetch) + } + + /// True when any package failed to install. + pub fn failed(&self) -> bool { + self.packages + .iter() + .any(|p| p.status == ProvisionStatus::Failed) + } +} + +/// Content hash of a package set: sha256 over the sorted +/// `(kind, name, version, url, sha256)` tuples. Status, sizes and paths are +/// left out, so a cold and a warm run of the same set hash the same — it is a +/// cache key for "which packages", not "what state are they in". +pub fn packages_hash<'a>(packages: impl IntoIterator) -> String { + let mut tuples: Vec<_> = packages + .into_iter() + .map(|p| { + ( + p.kind.as_str(), + p.name.as_str(), + p.version.as_str(), + p.url.as_str(), + p.sha256.as_deref().unwrap_or(""), + ) + }) + .collect(); + tuples.sort(); + tuples.dedup(); + let mut hasher = Sha256::new(); + for (kind, name, version, url, sha256) in tuples { + for field in [kind, name, version, url, sha256] { + hasher.update(field.as_bytes()); + hasher.update([0]); + } + hasher.update([0xff]); + } + format!("{:x}", hasher.finalize()) +} + +#[cfg(test)] +mod tests { + use super::*; + use fbuild_packages::PackageInfo; + use std::path::PathBuf; + use std::sync::atomic::{AtomicBool, Ordering}; + + struct FakePackage { + installed: AtomicBool, + install_fails: bool, + } + + impl FakePackage { + fn new(installed: bool, install_fails: bool) -> Self { + Self { + installed: AtomicBool::new(installed), + install_fails, + } + } + } + + #[async_trait::async_trait] + impl Package for FakePackage { + async fn ensure_installed(&self) -> fbuild_core::Result { + if self.install_fails { + return Err(fbuild_core::FbuildError::PackageError("404".into())); + } + self.installed.store(true, Ordering::SeqCst); + Ok(PathBuf::from("/cache/fake")) + } + + fn is_installed(&self) -> bool { + self.installed.load(Ordering::SeqCst) + } + + fn get_info(&self) -> PackageInfo { + PackageInfo { + name: "toolchain-fake".into(), + version: "1.2.3".into(), + url: "https://example.com/fake.tar.gz".into(), + install_path: PathBuf::from("/cache/fake"), + checksum: Some("abc123".into()), + installed_bytes: Some(4096), + } + } + } + + #[tokio::test] + async fn present_package_is_reported_present_in_every_mode() { + for mode in [ + ProvisionMode::Install, + ProvisionMode::Check, + ProvisionMode::DryRun, + ] { + let row = + provision_package(PackageKind::Toolchain, &FakePackage::new(true, true), mode) + .await; + assert_eq!(row.status, ProvisionStatus::Present, "{mode:?}"); + assert_eq!(row.bytes, Some(4096)); + assert_eq!(row.sha256.as_deref(), Some("abc123")); + } + } + + #[tokio::test] + async fn missing_package_is_fetched_only_by_install() { + let fake = FakePackage::new(false, false); + let row = provision_package(PackageKind::Toolchain, &fake, ProvisionMode::Check).await; + assert_eq!(row.status, ProvisionStatus::WouldFetch); + assert_eq!(row.bytes, None); + assert!(!fake.is_installed(), "a check must not install"); + + let row = provision_package(PackageKind::Toolchain, &fake, ProvisionMode::DryRun).await; + assert_eq!(row.status, ProvisionStatus::WouldFetch); + assert!(!fake.is_installed(), "a dry run must not install"); + + let row = provision_package(PackageKind::Toolchain, &fake, ProvisionMode::Install).await; + assert_eq!(row.status, ProvisionStatus::Fetched); + assert!(fake.is_installed()); + } + + #[tokio::test] + async fn failed_install_carries_the_error() { + let row = provision_package( + PackageKind::Framework, + &FakePackage::new(false, true), + ProvisionMode::Install, + ) + .await; + assert_eq!(row.status, ProvisionStatus::Failed); + assert!(row.error.as_deref().unwrap_or("").contains("404")); + } + + fn row(name: &str, status: ProvisionStatus, bytes: Option) -> ProvisionedPackage { + ProvisionedPackage { + version: "1".into(), + url: format!("https://example.com/{name}"), + bytes, + ..ProvisionedPackage::new(PackageKind::Toolchain, name, status) + } + } + + #[test] + fn packages_hash_ignores_order_status_and_size() { + let cold = [ + row("a", ProvisionStatus::WouldFetch, None), + row("b", ProvisionStatus::WouldFetch, None), + ]; + let warm = [ + row("b", ProvisionStatus::Present, Some(10)), + row("a", ProvisionStatus::Fetched, Some(20)), + ]; + assert_eq!(packages_hash(&cold), packages_hash(&warm)); + } + + #[test] + fn packages_hash_changes_with_the_package_set() { + let one = [row("a", ProvisionStatus::Present, None)]; + let two = [ + row("a", ProvisionStatus::Present, None), + row("b", ProvisionStatus::Present, None), + ]; + assert_ne!(packages_hash(&one), packages_hash(&two)); + } + + #[test] + fn report_flags_needs_fetch_and_failure() { + let report = ProvisionReport { + env: "uno".into(), + platform: "AtmelAvr".into(), + packages: vec![ + row("a", ProvisionStatus::Present, None), + row("b", ProvisionStatus::WouldFetch, None), + ], + }; + assert!(report.needs_fetch()); + assert!(!report.failed()); + } + + #[test] + fn statuses_serialize_kebab_case() { + assert_eq!( + serde_json::to_string(&ProvisionStatus::WouldFetch).unwrap(), + "\"would-fetch\"" + ); + assert_eq!( + serde_json::to_string(&PackageKind::SdkLibs).unwrap(), + "\"sdk-libs\"" + ); + } +} diff --git a/crates/fbuild-build-esp/src/esp32/mod.rs b/crates/fbuild-build-esp/src/esp32/mod.rs index 54bc06daa..f7095f661 100644 --- a/crates/fbuild-build-esp/src/esp32/mod.rs +++ b/crates/fbuild-build-esp/src/esp32/mod.rs @@ -20,16 +20,12 @@ impl crate::PlatformSupport for Esp32PlatformSupport { orchestrator::create() } - async fn install_deps(&self, project_dir: &std::path::Path) -> fbuild_core::Result<()> { - use fbuild_packages::Package; - let tc = fbuild_packages::toolchain::esp32::Esp32Toolchain::new( - project_dir, - false, - "xtensa-esp-elf", - ); - Package::ensure_installed(&tc).await?; - tracing::info!("ESP32 toolchain installed"); - Ok(()) + async fn provision( + &self, + inputs: &crate::provision::ProvisionInputs<'_>, + mode: crate::provision::ProvisionMode, + ) -> fbuild_core::Result> { + orchestrator::provision_esp32(inputs, mode).await } fn default_board_id(&self) -> &str { diff --git a/crates/fbuild-build-esp/src/esp32/orchestrator/mod.rs b/crates/fbuild-build-esp/src/esp32/orchestrator/mod.rs index f9d2a78ad..ea478d2f0 100644 --- a/crates/fbuild-build-esp/src/esp32/orchestrator/mod.rs +++ b/crates/fbuild-build-esp/src/esp32/orchestrator/mod.rs @@ -34,6 +34,8 @@ mod helpers; mod local_libs; mod packages; +pub(crate) use packages::provision_esp32; + #[cfg(test)] mod tests; diff --git a/crates/fbuild-build-esp/src/esp32/orchestrator/packages.rs b/crates/fbuild-build-esp/src/esp32/orchestrator/packages.rs index 6188c13d0..72a4a4517 100644 --- a/crates/fbuild-build-esp/src/esp32/orchestrator/packages.rs +++ b/crates/fbuild-build-esp/src/esp32/orchestrator/packages.rs @@ -1,11 +1,22 @@ //! Package resolution for pioarduino (platform.json, framework, toolchain). +//! +//! The build ([`resolve_pioarduino_packages`]) and `fbuild install` +//! ([`provision_esp32`]) construct packages through the same helpers, so they +//! always agree on what an env needs (FastLED/fbuild#1433). use std::collections::HashMap; use std::path::Path; +use std::time::Instant; use fbuild_core::Result; use fbuild_core::path::NormalizedPath; +use super::super::mcu_config::{Esp32McuConfig, get_mcu_config}; +use crate::provision::{ + PackageKind, ProvisionInputs, ProvisionMode, ProvisionStatus, ProvisionedPackage, + provision_package, +}; + /// Resolve framework + toolchain for pioarduino mode (GCC 14 + ESP-IDF 5.x). /// /// Downloads pioarduino platform.json, resolves toolchain via metadata, @@ -19,7 +30,7 @@ use fbuild_core::path::NormalizedPath; pub(super) async fn resolve_pioarduino_packages( project_dir: &Path, mcu: &str, - mcu_config: &super::super::mcu_config::Esp32McuConfig, + mcu_config: &Esp32McuConfig, env_config: Option<&HashMap>, ) -> Result<( fbuild_packages::toolchain::Esp32Toolchain, @@ -27,20 +38,7 @@ pub(super) async fn resolve_pioarduino_packages( Option, )> { // Ensure pioarduino platform (contains platform.json with metadata URLs). - // Honor `platform_packages = platform-espressif32@#` - // (FastLED/fbuild#672), then `platform = ` - // (FastLED/fbuild#1432): the pin replaces the const-pinned default and gets - // its own cache subdir via `PackageBase::with_override`. - let platform_ovr = env_config.and_then(|env| { - crate::package_override::resolve_platform_override(env, "platform-espressif32") - }); - let platform = match platform_ovr { - Some(o) => fbuild_packages::library::Esp32Platform::with_override(project_dir, o), - None => { - warn_unhonored_platform_pin(env_config); - fbuild_packages::library::Esp32Platform::new(project_dir) - } - }; + let platform = pioarduino_platform(project_dir, env_config); fbuild_packages::Package::ensure_installed(&platform).await?; // Resolve toolchain via metadata @@ -55,29 +53,7 @@ pub(super) async fn resolve_pioarduino_packages( // See fbuild#401. provision_helper_toolchains(&platform, project_dir, mcu_config); - // Resolve framework. Override precedence (FastLED/fbuild#672): - // 1. `platform_packages = framework-arduinoespressif32@#` wins - // outright — consumer-supplied URL replaces the platform.json-derived - // URL and gets its own cache subdir. - // 2. Otherwise, derive the URL from platform.json. - // 3. Otherwise (very old / missing platform.json), fall back to the - // legacy hardcoded URL via `Esp32Framework::new`. - let framework_ovr = env_config.and_then(|env| { - crate::package_override::resolve_override(env, "framework-arduinoespressif32") - }); - let framework = match framework_ovr { - Some(o) => fbuild_packages::library::Esp32Framework::with_override(project_dir, o), - None => match platform.get_package_url("framework-arduinoespressif32") { - Ok(url) => { - tracing::info!("resolved framework URL from platform.json"); - fbuild_packages::library::Esp32Framework::from_url(project_dir, &url) - } - Err(e) => { - tracing::warn!("could not resolve framework URL, using legacy: {}", e); - fbuild_packages::library::Esp32Framework::new(project_dir, mcu) - } - }, - }; + let framework = pioarduino_framework(&platform, project_dir, mcu, env_config); // Download the GCC toolchain (~100+ MB) CONCURRENTLY with the framework + // SDK libs (~hundreds of MB). Once the platform's metadata URLs are @@ -88,30 +64,18 @@ pub(super) async fn resolve_pioarduino_packages( // (FastLED/fbuild#953). The framework chain stays internally ordered: // the framework must be installed before its SDK libs extract into // `tools/`. - let mcu_suffix = mcu.strip_prefix("esp32").unwrap_or(""); - let libs_url = platform - .get_package_url("framework-arduinoespressif32-libs") - .ok(); - let skeleton_url = if mcu_suffix.is_empty() { - None - } else { - platform - .get_package_url(&format!("framework-arduino-{}-skeleton-lib", mcu_suffix)) - .ok() - }; + let (libs_url, skeleton_url) = sdk_libs_urls(&platform, mcu); let toolchain_fut = fbuild_packages::Package::ensure_installed(&toolchain); let framework_fut = async { fbuild_packages::Package::ensure_installed(&framework).await?; - // Ensure SDK libs (split package in pioarduino 3.3.7+). - if let Some(url) = &libs_url { - framework.ensure_libs(url, mcu).await?; - } - // Ensure MCU-specific skeleton libs (e.g. ESP32-C2, ESP32-C61). - if let Some(url) = &skeleton_url { - framework.ensure_mcu_libs(url, mcu).await?; - } - Ok::<(), fbuild_core::FbuildError>(()) + ensure_sdk_libs( + &framework, + mcu, + libs_url.as_deref(), + skeleton_url.as_deref(), + ) + .await }; // Provision the managed `tool-esptoolpy` package CONCURRENTLY with the // toolchain + framework. esptool converts firmware.elf → firmware.bin at @@ -144,6 +108,153 @@ pub(super) async fn resolve_pioarduino_packages( Ok((toolchain, framework, esptool_py)) } +/// Provision what [`resolve_pioarduino_packages`] installs — platform, +/// MCU-primary toolchain, framework, SDK libs and esptool — one report row +/// each, for `fbuild install` (FastLED/fbuild#1433). Check and dry-run modes +/// resolve the toolchain from metadata already on disk and never download. +/// Helper toolchains are left out: the build only resolves their metadata and +/// never installs them. +pub(crate) async fn provision_esp32( + inputs: &ProvisionInputs<'_>, + mode: ProvisionMode, +) -> Result> { + let project_dir = inputs.project_dir; + let env_config = Some(inputs.env_config); + let mcu = inputs.board.mcu.as_str(); + let mcu_config = get_mcu_config(mcu)?; + let mut rows = Vec::new(); + + let platform = pioarduino_platform(project_dir, env_config); + let platform_row = provision_package(PackageKind::Platform, &platform, mode).await; + let platform_ready = is_installed(&platform_row); + rows.push(platform_row); + if !platform_ready { + // Every other package is named by the platform's platform.json. + return Ok(rows); + } + + rows.push(provision_toolchain(&platform, project_dir, &mcu_config, mode).await); + + let framework = pioarduino_framework(&platform, project_dir, mcu, env_config); + let framework_row = provision_package(PackageKind::Framework, &framework, mode).await; + let framework_ready = is_installed(&framework_row); + rows.push(framework_row); + + let (libs_url, skeleton_url) = sdk_libs_urls(&platform, mcu); + if libs_url.is_some() || skeleton_url.is_some() { + rows.push( + provision_sdk_libs( + &framework, + framework_ready, + mcu, + libs_url.as_deref(), + skeleton_url.as_deref(), + mode, + ) + .await, + ); + } + + if let Some(row) = provision_esptool(&platform, project_dir, mode).await { + rows.push(row); + } + Ok(rows) +} + +fn is_installed(row: &ProvisionedPackage) -> bool { + matches!( + row.status, + ProvisionStatus::Present | ProvisionStatus::Fetched + ) +} + +/// The pioarduino platform package. Honors +/// `platform_packages = platform-espressif32@#` (FastLED/fbuild#672), +/// then `platform = ` (FastLED/fbuild#1432): the pin +/// replaces the const-pinned default and gets its own cache subdir via +/// `PackageBase::with_override`. +fn pioarduino_platform( + project_dir: &Path, + env_config: Option<&HashMap>, +) -> fbuild_packages::library::Esp32Platform { + let platform_ovr = env_config.and_then(|env| { + crate::package_override::resolve_platform_override(env, "platform-espressif32") + }); + match platform_ovr { + Some(o) => fbuild_packages::library::Esp32Platform::with_override(project_dir, o), + None => { + warn_unhonored_platform_pin(env_config); + fbuild_packages::library::Esp32Platform::new(project_dir) + } + } +} + +/// The Arduino framework package. Override precedence (FastLED/fbuild#672): +/// 1. `platform_packages = framework-arduinoespressif32@#` wins +/// outright — consumer-supplied URL replaces the platform.json-derived +/// URL and gets its own cache subdir. +/// 2. Otherwise, derive the URL from platform.json. +/// 3. Otherwise (very old / missing platform.json), fall back to the +/// legacy hardcoded URL via `Esp32Framework::new`. +fn pioarduino_framework( + platform: &fbuild_packages::library::Esp32Platform, + project_dir: &Path, + mcu: &str, + env_config: Option<&HashMap>, +) -> fbuild_packages::library::Esp32Framework { + let framework_ovr = env_config.and_then(|env| { + crate::package_override::resolve_override(env, "framework-arduinoespressif32") + }); + match framework_ovr { + Some(o) => fbuild_packages::library::Esp32Framework::with_override(project_dir, o), + None => match platform.get_package_url("framework-arduinoespressif32") { + Ok(url) => { + tracing::info!("resolved framework URL from platform.json"); + fbuild_packages::library::Esp32Framework::from_url(project_dir, &url) + } + Err(e) => { + tracing::warn!("could not resolve framework URL, using legacy: {}", e); + fbuild_packages::library::Esp32Framework::new(project_dir, mcu) + } + }, + } +} + +/// URLs of the split SDK libs package (pioarduino 3.3.7+) and, for MCUs that +/// ship one, the MCU skeleton libs (e.g. ESP32-C2, ESP32-C61). +fn sdk_libs_urls( + platform: &fbuild_packages::library::Esp32Platform, + mcu: &str, +) -> (Option, Option) { + let mcu_suffix = mcu.strip_prefix("esp32").unwrap_or(""); + let libs_url = platform + .get_package_url("framework-arduinoespressif32-libs") + .ok(); + let skeleton_url = if mcu_suffix.is_empty() { + None + } else { + platform + .get_package_url(&format!("framework-arduino-{}-skeleton-lib", mcu_suffix)) + .ok() + }; + (libs_url, skeleton_url) +} + +async fn ensure_sdk_libs( + framework: &fbuild_packages::library::Esp32Framework, + mcu: &str, + libs_url: Option<&str>, + skeleton_url: Option<&str>, +) -> Result<()> { + if let Some(url) = libs_url { + framework.ensure_libs(url, mcu).await?; + } + if let Some(url) = skeleton_url { + framework.ensure_mcu_libs(url, mcu).await?; + } + Ok(()) +} + /// Name a `platform` pin fbuild cannot honor instead of dropping it silently /// (FastLED/fbuild#1407). Registry pins and git URLs fall back to the /// pioarduino stable platform, which carries a different framework release. @@ -234,13 +345,9 @@ async fn resolve_esptool( fn provision_helper_toolchains( platform: &fbuild_packages::library::Esp32Platform, project_dir: &Path, - mcu_config: &super::super::mcu_config::Esp32McuConfig, + mcu_config: &Esp32McuConfig, ) { - let primary = if mcu_config.is_riscv() { - "toolchain-riscv32-esp" - } else { - "toolchain-xtensa-esp-elf" - }; + let primary = primary_toolchain_name(mcu_config.is_riscv()); let entries = match platform.enumerate_packages() { Ok(e) => e, @@ -286,10 +393,18 @@ fn provision_helper_toolchains( } } +fn primary_toolchain_name(is_riscv: bool) -> &'static str { + if is_riscv { + "toolchain-riscv32-esp" + } else { + "toolchain-xtensa-esp-elf" + } +} + fn resolve_and_create_toolchain( platform: &fbuild_packages::library::Esp32Platform, project_dir: &Path, - mcu_config: &super::super::mcu_config::Esp32McuConfig, + mcu_config: &Esp32McuConfig, ) -> Result { let is_riscv = mcu_config.is_riscv(); let prefix = mcu_config.toolchain_prefix(); @@ -297,11 +412,7 @@ fn resolve_and_create_toolchain( // Try metadata-based resolution match platform.get_toolchain_metadata_url(is_riscv) { Ok(metadata_url) => { - let toolchain_name = if is_riscv { - "toolchain-riscv32-esp" - } else { - "toolchain-xtensa-esp-elf" - }; + let toolchain_name = primary_toolchain_name(is_riscv); let cache = fbuild_packages::Cache::new(project_dir); let cache_dir = cache.toolchains_dir().join(toolchain_name); @@ -344,3 +455,153 @@ fn resolve_and_create_toolchain( } } } + +/// The MCU-primary toolchain row. Install resolves metadata exactly as the +/// build does; a check or dry run reads only metadata already on disk and +/// reports a would-fetch row when it has never been downloaded. +async fn provision_toolchain( + platform: &fbuild_packages::library::Esp32Platform, + project_dir: &Path, + mcu_config: &Esp32McuConfig, + mode: ProvisionMode, +) -> ProvisionedPackage { + let is_riscv = mcu_config.is_riscv(); + let name = primary_toolchain_name(is_riscv); + let toolchain = if mode.fetches() { + resolve_and_create_toolchain(platform, project_dir, mcu_config).map(Some) + } else { + cached_toolchain(platform, project_dir, mcu_config) + }; + match toolchain { + Ok(Some(toolchain)) => provision_package(PackageKind::Toolchain, &toolchain, mode).await, + Ok(None) => ProvisionedPackage { + url: platform + .get_toolchain_metadata_url(is_riscv) + .unwrap_or_default(), + ..ProvisionedPackage::new(PackageKind::Toolchain, name, ProvisionStatus::WouldFetch) + }, + Err(error) => ProvisionedPackage { + error: Some(error.to_string()), + ..ProvisionedPackage::new(PackageKind::Toolchain, name, ProvisionStatus::Failed) + }, + } +} + +/// [`resolve_and_create_toolchain`] without the network: `Ok(None)` when the +/// toolchain metadata has not been downloaded yet. +fn cached_toolchain( + platform: &fbuild_packages::library::Esp32Platform, + project_dir: &Path, + mcu_config: &Esp32McuConfig, +) -> Result> { + let is_riscv = mcu_config.is_riscv(); + let prefix = mcu_config.toolchain_prefix(); + if platform.get_toolchain_metadata_url(is_riscv).is_err() { + return Ok(Some(fbuild_packages::toolchain::Esp32Toolchain::new( + project_dir, + is_riscv, + &prefix, + ))); + } + let name = primary_toolchain_name(is_riscv); + let cache_dir = fbuild_packages::Cache::new(project_dir) + .toolchains_dir() + .join(name); + let resolved = + fbuild_packages::toolchain::esp32_metadata::resolve_toolchain_url_cached(name, &cache_dir)?; + Ok(resolved.map(|resolved| { + fbuild_packages::toolchain::Esp32Toolchain::from_resolved( + project_dir, + &resolved.url, + resolved.sha256.as_deref(), + is_riscv, + &prefix, + ) + })) +} + +/// The SDK libs row. They extract into the framework's `tools/` dir rather +/// than being a `Package`, so presence is the framework's own completeness +/// check. +async fn provision_sdk_libs( + framework: &fbuild_packages::library::Esp32Framework, + framework_ready: bool, + mcu: &str, + libs_url: Option<&str>, + skeleton_url: Option<&str>, + mode: ProvisionMode, +) -> ProvisionedPackage { + let started = Instant::now(); + let url = libs_url.or(skeleton_url).unwrap_or_default(); + let mut row = ProvisionedPackage { + version: url.rsplit('/').next().unwrap_or_default().to_string(), + url: url.to_string(), + ..ProvisionedPackage::new( + PackageKind::SdkLibs, + format!("framework-arduinoespressif32-libs ({mcu})"), + ProvisionStatus::WouldFetch, + ) + }; + if framework_ready && framework.sdk_libs_installed(mcu) { + row.status = ProvisionStatus::Present; + } else if mode.fetches() { + if framework_ready { + match ensure_sdk_libs(framework, mcu, libs_url, skeleton_url).await { + Ok(()) => row.status = ProvisionStatus::Fetched, + Err(error) => { + row.status = ProvisionStatus::Failed; + row.error = Some(error.to_string()); + } + } + } else { + row.status = ProvisionStatus::Failed; + row.error = Some("the framework is not installed".to_string()); + } + } + row.duration_ms = started.elapsed().as_millis() as u64; + row +} + +/// The esptool row, or `None` when `platform.json` names no esptool (the build +/// then relies on an `esptool` on PATH). +async fn provision_esptool( + platform: &fbuild_packages::library::Esp32Platform, + project_dir: &Path, + mode: ProvisionMode, +) -> Option { + let metadata_url = platform.get_package_url("tool-esptoolpy").ok()?; + let esptool = fbuild_packages::library::Esptool::from_metadata_url(project_dir, &metadata_url); + let started = Instant::now(); + let mut row = ProvisionedPackage { + version: esptool.version().to_string(), + url: esptool.download_url().unwrap_or(metadata_url), + ..ProvisionedPackage::new( + PackageKind::Tool, + "tool-esptoolpy", + ProvisionStatus::WouldFetch, + ) + }; + match esptool.installed_binary() { + Ok(Some(binary)) => { + row.status = ProvisionStatus::Present; + row.install_path = Some(binary.display().to_string()); + } + Ok(None) if mode.fetches() => match esptool.ensure_installed().await { + Ok(binary) => { + row.status = ProvisionStatus::Fetched; + row.install_path = Some(binary.display().to_string()); + } + Err(error) => { + row.status = ProvisionStatus::Failed; + row.error = Some(error.to_string()); + } + }, + Ok(None) => {} + Err(error) => { + row.status = ProvisionStatus::Failed; + row.error = Some(error.to_string()); + } + } + row.duration_ms = started.elapsed().as_millis() as u64; + Some(row) +} diff --git a/crates/fbuild-build-esp/src/esp8266/mod.rs b/crates/fbuild-build-esp/src/esp8266/mod.rs index 7e29e5d05..4659c64af 100644 --- a/crates/fbuild-build-esp/src/esp8266/mod.rs +++ b/crates/fbuild-build-esp/src/esp8266/mod.rs @@ -18,12 +18,18 @@ impl crate::PlatformSupport for Esp8266PlatformSupport { orchestrator::create() } - async fn install_deps(&self, project_dir: &std::path::Path) -> fbuild_core::Result<()> { - use fbuild_packages::Package; - let tc = fbuild_packages::toolchain::Esp8266Toolchain::new(project_dir); - Package::ensure_installed(&tc).await?; - tracing::info!("ESP8266 toolchain installed"); - Ok(()) + async fn provision( + &self, + inputs: &crate::provision::ProvisionInputs<'_>, + mode: crate::provision::ProvisionMode, + ) -> fbuild_core::Result> { + use crate::provision::{PackageKind, provision_package}; + let (toolchain, framework) = + orchestrator::esp8266_packages(inputs.project_dir, Some(inputs.env_config)); + Ok(vec![ + provision_package(PackageKind::Toolchain, &toolchain, mode).await, + provision_package(PackageKind::Framework, &framework, mode).await, + ]) } fn default_board_id(&self) -> &str { diff --git a/crates/fbuild-build-esp/src/esp8266/orchestrator.rs b/crates/fbuild-build-esp/src/esp8266/orchestrator.rs index 0f4407da7..b10383fa1 100644 --- a/crates/fbuild-build-esp/src/esp8266/orchestrator.rs +++ b/crates/fbuild-build-esp/src/esp8266/orchestrator.rs @@ -39,6 +39,28 @@ fn profile_label(profile: fbuild_core::BuildProfile) -> &'static str { } } +/// The ESP8266 toolchain and Arduino framework for an env, honoring the +/// `framework-arduinoespressif8266` `platform_packages` override +/// (FastLED/fbuild#664, #681). Shared by the build and `fbuild install`, so +/// both provision the same packages (FastLED/fbuild#1433). +pub(crate) fn esp8266_packages( + project_dir: &Path, + env_config: Option<&HashMap>, +) -> ( + fbuild_packages::toolchain::Esp8266Toolchain, + fbuild_packages::library::Esp8266Framework, +) { + let toolchain = fbuild_packages::toolchain::Esp8266Toolchain::new(project_dir); + let override_pin = env_config.and_then(|env| { + crate::package_override::resolve_override(env, "framework-arduinoespressif8266") + }); + let framework = match override_pin { + Some(o) => fbuild_packages::library::Esp8266Framework::with_override(project_dir, o), + None => fbuild_packages::library::Esp8266Framework::new(project_dir), + }; + (toolchain, framework) +} + #[async_trait::async_trait] impl BuildOrchestrator for Esp8266Orchestrator { fn platform(&self) -> Platform { @@ -56,8 +78,11 @@ impl BuildOrchestrator for Esp8266Orchestrator { let eh_frame_policy = crate::eh_frame_policy_compute::compute_eh_frame_policy(&ctx, params.profile, None); - // 3. Ensure toolchain - let toolchain = fbuild_packages::toolchain::Esp8266Toolchain::new(¶ms.project_dir); + // 3-4. Toolchain and framework + let (toolchain, framework) = esp8266_packages( + ¶ms.project_dir, + ctx.config.get_env_config(¶ms.env_name).ok(), + ); let _toolchain_dir = fbuild_packages::Package::ensure_installed(&toolchain).await?; tracing::info!("ESP8266 toolchain ready"); @@ -69,21 +94,6 @@ impl BuildOrchestrator for Esp8266Orchestrator { ) .await; - // 4. Ensure framework - // Honor `platform_packages` override (FastLED/fbuild#664, #681). - let __ovr = ctx - .config - .get_env_config(¶ms.env_name) - .ok() - .and_then(|env| { - crate::package_override::resolve_override(env, "framework-arduinoespressif8266") - }); - let framework = match __ovr { - Some(o) => { - fbuild_packages::library::Esp8266Framework::with_override(¶ms.project_dir, o) - } - None => fbuild_packages::library::Esp8266Framework::new(¶ms.project_dir), - }; let _framework_dir = fbuild_packages::Package::ensure_installed(&framework).await?; tracing::info!("ESP8266 framework ready"); let board_id = ctx diff --git a/crates/fbuild-build-mcu/src/avr/mod.rs b/crates/fbuild-build-mcu/src/avr/mod.rs index 1eda621a8..07bca26c0 100644 --- a/crates/fbuild-build-mcu/src/avr/mod.rs +++ b/crates/fbuild-build-mcu/src/avr/mod.rs @@ -1,4 +1,4 @@ -//! AVR platform build support (Arduino Uno, Mega, Nano, etc.) +//! AVR platform build support (Arduino Uno, Mega, Nano, etc.) pub mod avr_compiler; pub mod avr_linker; @@ -18,12 +18,18 @@ impl crate::PlatformSupport for AvrPlatformSupport { orchestrator::create() } - async fn install_deps(&self, project_dir: &std::path::Path) -> fbuild_core::Result<()> { - use fbuild_packages::Package; - let tc = fbuild_packages::toolchain::AvrToolchain::new(project_dir); - Package::ensure_installed(&tc).await?; - tracing::info!("AVR toolchain installed"); - Ok(()) + async fn provision( + &self, + inputs: &crate::provision::ProvisionInputs<'_>, + mode: crate::provision::ProvisionMode, + ) -> fbuild_core::Result> { + use crate::provision::{PackageKind, provision_package}; + let (toolchain, framework) = + orchestrator::avr_packages(inputs.project_dir, Some(inputs.env_config), inputs.board)?; + Ok(vec![ + provision_package(PackageKind::Toolchain, &toolchain, mode).await, + provision_package(PackageKind::Framework, &framework, mode).await, + ]) } fn default_board_id(&self) -> &str { diff --git a/crates/fbuild-build-mcu/src/avr/orchestrator.rs b/crates/fbuild-build-mcu/src/avr/orchestrator.rs index 4b8b685ce..e2b558cb8 100644 --- a/crates/fbuild-build-mcu/src/avr/orchestrator.rs +++ b/crates/fbuild-build-mcu/src/avr/orchestrator.rs @@ -96,20 +96,7 @@ impl BuildOrchestrator for AvrOrchestrator { let eh_frame_policy = crate::eh_frame_policy_compute::compute_eh_frame_policy(&ctx, params.profile, None); - // 3. Ensure toolchain - let (toolchain, toolchain_dir) = { - let _g = perf.phase("toolchain-ensure"); - let toolchain = fbuild_packages::toolchain::AvrToolchain::new(¶ms.project_dir); - let toolchain_dir = fbuild_packages::Package::ensure_installed(&toolchain).await?; - (toolchain, toolchain_dir) - }; - tracing::info!("avr-gcc toolchain at {}", toolchain_dir.display()); - - use fbuild_packages::Toolchain as _; - pipeline::log_toolchain_version(&toolchain.get_gcc_path(), "avr-gcc", &mut ctx.build_log) - .await; - - // 4. Ensure Arduino core + // 3-4. Resolve the avr-gcc toolchain and Arduino core packages. // // Honor `platform_packages = framework-arduino-avr@` (FastLED/fbuild#667) // and `platform_packages = framework-arduino-avr-attiny@` @@ -123,24 +110,27 @@ impl BuildOrchestrator for AvrOrchestrator { // they all fall under the atmelavr platform — but only the two PIO // canonical names are resolved here. If those alt-core PIO package // names land in the registry later, add additional resolution branches. - let env_cfg = ctx.config.get_env_config(¶ms.env_name).ok(); - let __avr_ovr = env_cfg.as_ref().and_then(|env| { - crate::package_override::resolve_override(env, "framework-arduino-avr") - }); - let __attiny_ovr = env_cfg.as_ref().and_then(|env| { - crate::package_override::resolve_override(env, "framework-arduino-avr-attiny") - }); + let (toolchain, framework) = avr_packages( + ¶ms.project_dir, + ctx.config.get_env_config(¶ms.env_name).ok(), + &ctx.board, + )?; + + // 3. Ensure toolchain + let toolchain_dir = { + let _g = perf.phase("toolchain-ensure"); + fbuild_packages::Package::ensure_installed(&toolchain).await? + }; + tracing::info!("avr-gcc toolchain at {}", toolchain_dir.display()); + + use fbuild_packages::Toolchain as _; + pipeline::log_toolchain_version(&toolchain.get_gcc_path(), "avr-gcc", &mut ctx.build_log) + .await; + + // 4. Ensure Arduino core let (_framework_dir, core_dir, variant_dir) = { let _g = perf.phase("framework-ensure"); - ensure_avr_framework( - ¶ms.project_dir, - &ctx.board.core, - &ctx.board.variant, - ctx.board.platform(), - __avr_ovr, - __attiny_ovr, - ) - .await? + ensure_avr_framework(&framework, &ctx.board.core, &ctx.board.variant).await? }; // 4.5. Warm-build fast path. @@ -387,7 +377,57 @@ pub fn create() -> Box { Box::new(AvrOrchestrator) } -/// Select and install the correct AVR Arduino framework based on the board's core name. +/// The avr-gcc toolchain and the Arduino framework package for an env's board, +/// honoring the `framework-arduino-avr` / `framework-arduino-avr-attiny` +/// `platform_packages` overrides. Shared by the build and `fbuild install`, so +/// both provision the same packages (FastLED/fbuild#1433). +pub(crate) fn avr_packages( + project_dir: &Path, + env_config: Option<&std::collections::HashMap>, + board: &fbuild_config::BoardConfig, +) -> Result<( + fbuild_packages::toolchain::AvrToolchain, + fbuild_packages::library::AvrFramework, +)> { + let toolchain = fbuild_packages::toolchain::AvrToolchain::new(project_dir); + let avr_override = env_config + .and_then(|env| crate::package_override::resolve_override(env, "framework-arduino-avr")); + let attiny_override = env_config.and_then(|env| { + crate::package_override::resolve_override(env, "framework-arduino-avr-attiny") + }); + let framework = avr_framework_package( + project_dir, + &board.core, + board.platform(), + avr_override, + attiny_override, + )?; + Ok((toolchain, framework)) +} + +/// Install an AVR framework package and resolve the board's core and variant +/// directories inside it. +/// +/// Returns (framework_root, core_dir, variant_dir). +async fn ensure_avr_framework( + framework: &fbuild_packages::library::AvrFramework, + core_name: &str, + variant_name: &str, +) -> fbuild_core::Result<(PathBuf, PathBuf, PathBuf)> { + use fbuild_packages::Package; + + let framework_dir = framework.ensure_installed().await?; + tracing::info!( + "AVR framework for core '{}' at {}", + core_name, + framework_dir.display() + ); + let core_dir = framework.get_core_dir(core_name); + let variant_dir = framework.get_variant_dir(variant_name); + Ok((framework_dir, core_dir, variant_dir)) +} + +/// Select the correct AVR Arduino framework package based on the board's core name. /// /// Uses the data-driven `avr_frameworks.json` registry to resolve the correct /// framework package (GitHub URL, version) for any board core. @@ -401,18 +441,13 @@ pub fn create() -> Box { /// supersedes the registry-pinned default; the cache subdir is derived from /// the override URL via `PackageBase::with_override` so an override doesn't /// collide with the default cache entry. -/// -/// Returns (framework_root, core_dir, variant_dir). -async fn ensure_avr_framework( +fn avr_framework_package( project_dir: &Path, core_name: &str, - variant_name: &str, platform: Option, avr_override: Option, attiny_override: Option, -) -> fbuild_core::Result<(PathBuf, PathBuf, PathBuf)> { - use fbuild_packages::Package; - +) -> fbuild_core::Result { // megaAVR boards (e.g. nano_every) share core name "arduino" with standard AVR // but need ArduinoCore-megaavr instead of ArduinoCore-avr. let lookup_key = @@ -435,24 +470,14 @@ async fn ensure_avr_framework( _ => None, }; - let framework = match routed_override { + match routed_override { Some(ovr) => fbuild_packages::library::AvrFramework::for_core_with_override( lookup_key, project_dir, ovr, - )?, - None => fbuild_packages::library::AvrFramework::for_core(lookup_key, project_dir)?, - }; - let framework_dir = framework.ensure_installed().await?; - tracing::info!( - "AVR framework for core '{}' (lookup '{}') at {}", - core_name, - lookup_key, - framework_dir.display() - ); - let core_dir = framework.get_core_dir(core_name); - let variant_dir = framework.get_variant_dir(variant_name); - Ok((framework_dir, core_dir, variant_dir)) + ), + None => fbuild_packages::library::AvrFramework::for_core(lookup_key, project_dir), + } } /// Check if a project is configured for AVR by reading its platformio.ini. diff --git a/crates/fbuild-build-mcu/src/ch32v/mod.rs b/crates/fbuild-build-mcu/src/ch32v/mod.rs index 1954e3353..69ccf308a 100644 --- a/crates/fbuild-build-mcu/src/ch32v/mod.rs +++ b/crates/fbuild-build-mcu/src/ch32v/mod.rs @@ -18,12 +18,18 @@ impl crate::PlatformSupport for Ch32vPlatformSupport { orchestrator::create() } - async fn install_deps(&self, project_dir: &std::path::Path) -> fbuild_core::Result<()> { - use fbuild_packages::Package; - let tc = fbuild_packages::toolchain::RiscvToolchain::new(project_dir); - Package::ensure_installed(&tc).await?; - tracing::info!("RISC-V toolchain installed"); - Ok(()) + async fn provision( + &self, + inputs: &crate::provision::ProvisionInputs<'_>, + mode: crate::provision::ProvisionMode, + ) -> fbuild_core::Result> { + use crate::provision::{PackageKind, provision_package}; + let (toolchain, cores) = + orchestrator::ch32v_packages(inputs.project_dir, Some(inputs.env_config)); + Ok(vec![ + provision_package(PackageKind::Toolchain, &toolchain, mode).await, + provision_package(PackageKind::Framework, &cores, mode).await, + ]) } fn default_board_id(&self) -> &str { diff --git a/crates/fbuild-build-mcu/src/ch32v/orchestrator.rs b/crates/fbuild-build-mcu/src/ch32v/orchestrator.rs index 7b7a64bd9..6d971f6bd 100644 --- a/crates/fbuild-build-mcu/src/ch32v/orchestrator.rs +++ b/crates/fbuild-build-mcu/src/ch32v/orchestrator.rs @@ -57,8 +57,11 @@ impl BuildOrchestrator for Ch32vOrchestrator { .map(String::as_str); validate_ch32v_framework(framework_name)?; - // 3. Ensure RISC-V GCC toolchain - let toolchain = fbuild_packages::toolchain::RiscvToolchain::new(¶ms.project_dir); + // 3-4. RISC-V GCC toolchain and OpenWCH CH32V cores + let (toolchain, framework) = ch32v_packages( + ¶ms.project_dir, + ctx.config.get_env_config(¶ms.env_name).ok(), + ); let toolchain_dir = fbuild_packages::Package::ensure_installed(&toolchain).await?; tracing::info!("riscv-gcc toolchain at {}", toolchain_dir.display()); @@ -70,19 +73,6 @@ impl BuildOrchestrator for Ch32vOrchestrator { ) .await; - // 4. Ensure OpenWCH CH32V cores - // Honor `platform_packages` override (FastLED/fbuild#664, #681). - let __ovr = ctx - .config - .get_env_config(¶ms.env_name) - .ok() - .and_then(|env| { - crate::package_override::resolve_override(env, "framework-arduino-ch32v") - }); - let framework = match __ovr { - Some(o) => fbuild_packages::library::Ch32vCores::with_override(¶ms.project_dir, o), - None => fbuild_packages::library::Ch32vCores::new(¶ms.project_dir), - }; let framework_dir = fbuild_packages::Package::ensure_installed(&framework).await?; tracing::info!("CH32V cores at {}", framework_dir.display()); @@ -382,6 +372,27 @@ impl BuildOrchestrator for Ch32vOrchestrator { } } +/// The RISC-V GCC toolchain and OpenWCH CH32V cores for an env, honoring the +/// `framework-arduino-ch32v` `platform_packages` override (FastLED/fbuild#664, +/// #681). Shared by the build and `fbuild install`, so both provision the same +/// packages (FastLED/fbuild#1433). +pub(crate) fn ch32v_packages( + project_dir: &Path, + env_config: Option<&std::collections::HashMap>, +) -> ( + fbuild_packages::toolchain::RiscvToolchain, + fbuild_packages::library::Ch32vCores, +) { + let toolchain = fbuild_packages::toolchain::RiscvToolchain::new(project_dir); + let override_pin = env_config + .and_then(|env| crate::package_override::resolve_override(env, "framework-arduino-ch32v")); + let cores = match override_pin { + Some(o) => fbuild_packages::library::Ch32vCores::with_override(project_dir, o), + None => fbuild_packages::library::Ch32vCores::new(project_dir), + }; + (toolchain, cores) +} + /// Create a CH32V orchestrator (convenience for get_orchestrator dispatch). pub fn create() -> Box { Box::new(Ch32vOrchestrator) diff --git a/crates/fbuild-build/src/lib.rs b/crates/fbuild-build/src/lib.rs index 13b59ca8a..ffda94ba5 100644 --- a/crates/fbuild-build/src/lib.rs +++ b/crates/fbuild-build/src/lib.rs @@ -66,17 +66,103 @@ pub fn get_orchestrator(platform: Platform) -> Result get_platform_support(platform).map(|s| s.create_orchestrator()) } -/// Install platform-specific dependencies (toolchain, framework). -pub async fn install_platform_deps(platform: Platform, project_dir: &Path) -> Result<()> { - get_platform_support(platform)? - .install_deps(project_dir) - .await +/// Resolve an env's board and platform, then provision everything its build +/// downloads — platform packages, toolchains, framework, tools and `lib_deps` +/// — without compiling. The engine behind `fbuild install` and the daemon's +/// `POST /api/install-deps` (FastLED/fbuild#1433). +pub async fn provision_env( + project_dir: &Path, + env_name: &str, + mode: provision::ProvisionMode, +) -> Result { + let config = fbuild_config::PlatformIOConfig::from_path(&project_dir.join("platformio.ini"))?; + let env_config = config.get_env_config(env_name)?; + let board = + resolution::ResolutionContext::new(project_dir, env_name, &config).resolve_board()?; + let platform = env_config + .get("platform") + .and_then(|value| Platform::from_platform_str(value)) + .or_else(|| board.platform()) + .ok_or_else(|| { + fbuild_core::FbuildError::ConfigError(format!( + "could not determine the platform for environment '{env_name}'" + )) + })?; + let support = get_platform_support(platform)?; + let inputs = provision::ProvisionInputs { + project_dir, + env_name, + env_config, + board: &board, + }; + + let mut packages = support.provision(&inputs, mode).await?; + let lib_deps = support.downloadable_lib_deps(&inputs, config.get_lib_deps(env_name)?); + let lib_ignore = config.get_lib_ignore(env_name)?; + // `fbuild build` downloads lib_deps into the release build dir's `libs/`. + let libs_dir = fbuild_paths::BuildLayout::new( + project_dir.to_path_buf(), + env_name.to_string(), + fbuild_core::BuildProfile::Release, + ) + .resolve() + .join("libs"); + packages.extend( + provision::provision_lib_deps(project_dir, &lib_deps, &lib_ignore, &libs_dir, mode).await, + ); + + Ok(provision::ProvisionReport { + env: env_name.to_string(), + platform: format!("{platform:?}"), + packages, + }) } #[cfg(test)] mod tests { use super::*; + fn project(ini: &str) -> tempfile::TempDir { + let dir = tempfile::tempdir().unwrap(); + std::fs::write(dir.path().join("platformio.ini"), ini).unwrap(); + dir + } + + #[tokio::test] + async fn provision_env_rejects_an_unknown_environment() { + let dir = project("[env:teensy41]\nplatform = teensy\nboard = teensy41\n"); + let error = provision_env(dir.path(), "nope", provision::ProvisionMode::DryRun) + .await + .unwrap_err(); + assert!(error.to_string().contains("nope"), "{error}"); + } + + #[tokio::test] + async fn provision_env_dry_run_lists_packages_without_fetching() { + let dir = project("[env:teensy41]\nplatform = teensy\nboard = teensy41\n"); + let report = provision_env(dir.path(), "teensy41", provision::ProvisionMode::DryRun) + .await + .unwrap(); + assert_eq!(report.env, "teensy41"); + let kinds: Vec<_> = report.packages.iter().map(|p| p.kind).collect(); + assert_eq!( + kinds, + vec![ + provision::PackageKind::Toolchain, + provision::PackageKind::Framework + ] + ); + for package in &report.packages { + assert!( + matches!( + package.status, + provision::ProvisionStatus::Present | provision::ProvisionStatus::WouldFetch + ), + "a dry run must not fetch: {package:?}" + ); + } + } + #[test] fn test_get_orchestrator_atmelmegaavr() { let orch = get_orchestrator(Platform::AtmelMegaAvr).unwrap(); diff --git a/crates/fbuild-cli/src/cli/README.md b/crates/fbuild-cli/src/cli/README.md index 9e5239f2b..8b51caae5 100644 --- a/crates/fbuild-cli/src/cli/README.md +++ b/crates/fbuild-cli/src/cli/README.md @@ -25,4 +25,5 @@ here and is dispatched from `cli::async_main`. - **`show.rs`** -- `run_show`, `show_daemon_logs` - **`reset.rs`** -- `run_reset` - **`lnk.rs`** -- `run_lnk` (pull / check / add) +- **`install.rs`** -- `run_install`: in-process `fbuild install` over `fbuild_build::provision_env`, text / JSON rendering, `--check` exit codes (FastLED/fbuild#1433) - **`tests.rs`** -- unit tests for argument normalization and `fbuild ci` parsing diff --git a/crates/fbuild-cli/src/cli/args.rs b/crates/fbuild-cli/src/cli/args.rs index 25c35c4f6..cf975e8f7 100644 --- a/crates/fbuild-cli/src/cli/args.rs +++ b/crates/fbuild-cli/src/cli/args.rs @@ -452,6 +452,34 @@ pub enum Commands { #[arg(long = "upgrade-package")] upgrade_package: Option, }, + /// Download everything an environment's build needs — platform, + /// toolchains, framework, tools and lib_deps — without compiling, one + /// line per package. `--check` and `--dry-run` never touch the network; + /// `--check` exits 2 when anything would need fetching. + /// FastLED/fbuild#1433. + Install { + /// Project directory. + project_dir: Option, + /// Environment to provision (repeatable). Defaults to the project's + /// default environment. + #[arg(short = 'e', long = "environment", conflicts_with = "all_envs")] + environments: Vec, + /// Provision every environment in platformio.ini. + #[arg(long)] + all_envs: bool, + /// Report what is missing without fetching; exit 2 if anything is. + #[arg(long, conflicts_with = "dry_run")] + check: bool, + /// List the resolved packages without fetching. + #[arg(long = "dry-run")] + dry_run: bool, + /// Print a JSON manifest (rows plus `packages_hash`) instead of text. + #[arg(long)] + json: bool, + /// Environments provisioned in parallel. + #[arg(short = 'j', long, value_parser = parse_jobs)] + jobs: Option, + }, /// Manage the fbuild daemon Daemon { #[command(subcommand)] @@ -1072,6 +1100,7 @@ pub const KNOWN_SUBCOMMANDS: &[&str] = &[ "symbols", "bloat", "sync", + "install", ]; /// Rewrite `fbuild ...` → `fbuild ...` diff --git a/crates/fbuild-cli/src/cli/cache.rs b/crates/fbuild-cli/src/cli/cache.rs index f1e845cde..a2957a9d5 100644 --- a/crates/fbuild-cli/src/cli/cache.rs +++ b/crates/fbuild-cli/src/cli/cache.rs @@ -159,7 +159,7 @@ fn render_manifest( out } -fn human_bytes(n: u64) -> String { +pub(crate) fn human_bytes(n: u64) -> String { const UNITS: &[&str] = &["B", "KB", "MB", "GB", "TB"]; let mut v = n as f64; let mut i = 0; diff --git a/crates/fbuild-cli/src/cli/dispatch.rs b/crates/fbuild-cli/src/cli/dispatch.rs index c94cc95a4..1584d41ce 100644 --- a/crates/fbuild-cli/src/cli/dispatch.rs +++ b/crates/fbuild-cli/src/cli/dispatch.rs @@ -418,6 +418,26 @@ pub async fn async_main() { std::process::exit(code); } } + Some(Commands::Install { + project_dir, + environments, + all_envs, + check, + dry_run, + json, + jobs, + }) => { + super::install::run_install(super::install::InstallArgs { + project_dir: resolve_project_dir(project_dir, &top_level_project_dir), + environments, + all_envs, + check, + dry_run, + json, + jobs, + }) + .await + } Some(Commands::Daemon { action }) => run_daemon(action).await, Some(Commands::Show { target, diff --git a/crates/fbuild-cli/src/cli/install.rs b/crates/fbuild-cli/src/cli/install.rs new file mode 100644 index 000000000..af8babb44 --- /dev/null +++ b/crates/fbuild-cli/src/cli/install.rs @@ -0,0 +1,295 @@ +//! `fbuild install`: provision an environment's packages without compiling +//! (FastLED/fbuild#1433). +//! +//! Runs in-process rather than through the daemon: it only touches the package +//! cache, and CI runs it as its own step before any build, so a daemon +//! round-trip (and the compile backend's startup) would only add time. + +use std::path::Path; + +use fbuild_build::provision::{ + ProvisionMode, ProvisionReport, ProvisionStatus, ProvisionedPackage, packages_hash, +}; +use fbuild_core::{FbuildError, Result}; + +use super::cache::human_bytes; +use crate::output; + +/// Exit code when `--check` finds a package that would need fetching. +pub const CHECK_NEEDS_FETCH_EXIT: i32 = 2; + +/// Parsed `fbuild install` arguments. +pub struct InstallArgs { + pub project_dir: String, + pub environments: Vec, + pub all_envs: bool, + pub check: bool, + pub dry_run: bool, + pub json: bool, + pub jobs: Option, +} + +pub async fn run_install(args: InstallArgs) -> Result<()> { + let project_dir = Path::new(&args.project_dir); + let envs = select_envs(project_dir, &args)?; + let mode = if args.check { + ProvisionMode::Check + } else if args.dry_run { + ProvisionMode::DryRun + } else { + ProvisionMode::Install + }; + let reports = provision_envs(project_dir, &envs, mode, args.jobs.unwrap_or(1)).await?; + if args.json { + output::result(render_json(&reports)); + } else { + output::result(render_text(&reports)); + } + exit_status(&reports, mode) +} + +/// `--all-envs`, the named envs, or the project's default env. +fn select_envs(project_dir: &Path, args: &InstallArgs) -> Result> { + let config = fbuild_config::PlatformIOConfig::from_path(&project_dir.join("platformio.ini"))?; + if args.all_envs { + return Ok(config + .get_environments() + .iter() + .map(|env| env.to_string()) + .collect()); + } + if !args.environments.is_empty() { + return Ok(args.environments.clone()); + } + config + .get_default_environment() + .map(|env| vec![env.to_string()]) + .ok_or_else(|| FbuildError::ConfigError("platformio.ini defines no environments".into())) +} + +/// Provision each env, up to `jobs` at a time, keeping the envs' order. +async fn provision_envs( + project_dir: &Path, + envs: &[String], + mode: ProvisionMode, + jobs: usize, +) -> Result> { + use futures::stream::{self, StreamExt}; + stream::iter( + envs.iter() + .map(|env| fbuild_build::provision_env(project_dir, env, mode)), + ) + .buffered(jobs.max(1)) + .collect::>() + .await + .into_iter() + .collect() +} + +fn count(reports: &[ProvisionReport], status: ProvisionStatus) -> usize { + reports + .iter() + .flat_map(|report| &report.packages) + .filter(|package| package.status == status) + .count() +} + +fn render_text(reports: &[ProvisionReport]) -> String { + use std::fmt::Write as _; + let mut out = String::new(); + for report in reports { + let _ = writeln!(out, "[{}] {}", report.env, report.platform); + for package in &report.packages { + let _ = writeln!(out, " {}", package_line(package)); + if let Some(error) = &package.error { + let _ = writeln!(out, " error: {error}"); + } + } + } + let total: usize = reports.iter().map(|report| report.packages.len()).sum(); + let bytes: u64 = reports + .iter() + .flat_map(|report| &report.packages) + .filter_map(|package| package.bytes) + .sum(); + let _ = write!( + out, + "{total} package(s): {} present, {} fetched, {} would-fetch, {} failed; {} on disk; packages_hash {}", + count(reports, ProvisionStatus::Present), + count(reports, ProvisionStatus::Fetched), + count(reports, ProvisionStatus::WouldFetch), + count(reports, ProvisionStatus::Failed), + human_bytes(bytes), + packages_hash(reports.iter().flat_map(|report| &report.packages)), + ); + out +} + +/// `status kind name version bytes duration url sha256`. +fn package_line(package: &ProvisionedPackage) -> String { + fn or_dash(value: &str) -> &str { + if value.is_empty() { "-" } else { value } + } + format!( + "{:<11} {:<9} {} {} {} {}ms {} {}", + package.status.as_str(), + package.kind.as_str(), + package.name, + or_dash(&package.version), + package.bytes.map_or_else(|| "-".to_string(), human_bytes), + package.duration_ms, + or_dash(&package.url), + package.sha256.as_deref().unwrap_or("-"), + ) +} + +/// The manifest CI keys caches on: every row plus a `packages_hash` per env +/// and across all of them. +fn render_json(reports: &[ProvisionReport]) -> String { + let environments: Vec<_> = reports + .iter() + .map(|report| { + serde_json::json!({ + "env": report.env, + "platform": report.platform, + "packages_hash": packages_hash(&report.packages), + "packages": report.packages, + }) + }) + .collect(); + let manifest = serde_json::json!({ + "packages_hash": packages_hash(reports.iter().flat_map(|report| &report.packages)), + "environments": environments, + }); + serde_json::to_string_pretty(&manifest) + .expect("fbuild-cli: install manifest is built from serializable rows") +} + +/// 1 when a package failed; [`CHECK_NEEDS_FETCH_EXIT`] when `--check` found +/// something missing; otherwise success. +fn exit_status(reports: &[ProvisionReport], mode: ProvisionMode) -> Result<()> { + let failed = count(reports, ProvisionStatus::Failed); + if failed > 0 { + return Err(FbuildError::CommandFailed { + message: format!("{failed} package(s) failed to install"), + exit_code: 1, + }); + } + let missing = count(reports, ProvisionStatus::WouldFetch); + if mode == ProvisionMode::Check && missing > 0 { + return Err(FbuildError::CommandFailed { + message: format!("{missing} package(s) would need fetching"), + exit_code: CHECK_NEEDS_FETCH_EXIT, + }); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use fbuild_build::provision::PackageKind; + + fn args(environments: &[&str], all_envs: bool) -> InstallArgs { + InstallArgs { + project_dir: ".".into(), + environments: environments.iter().map(|env| env.to_string()).collect(), + all_envs, + check: false, + dry_run: false, + json: false, + jobs: None, + } + } + + fn project() -> tempfile::TempDir { + let dir = tempfile::tempdir().unwrap(); + std::fs::write( + dir.path().join("platformio.ini"), + "[platformio]\ndefault_envs = uno\n\n[env:uno]\nplatform = atmelavr\nboard = uno\n\n[env:teensy41]\nplatform = teensy\nboard = teensy41\n", + ) + .unwrap(); + dir + } + + #[test] + fn select_envs_prefers_all_then_named_then_default() { + let dir = project(); + let mut all = select_envs(dir.path(), &args(&[], true)).unwrap(); + all.sort(); + assert_eq!(all, vec!["teensy41", "uno"]); + assert_eq!( + select_envs(dir.path(), &args(&["teensy41"], false)).unwrap(), + vec!["teensy41"] + ); + assert_eq!( + select_envs(dir.path(), &args(&[], false)).unwrap(), + vec!["uno"] + ); + } + + fn report(statuses: &[ProvisionStatus]) -> ProvisionReport { + ProvisionReport { + env: "uno".into(), + platform: "AtmelAvr".into(), + packages: statuses + .iter() + .enumerate() + .map(|(i, status)| ProvisionedPackage { + version: "1.0".into(), + url: format!("https://example.com/{i}"), + bytes: Some(2048), + error: (*status == ProvisionStatus::Failed).then(|| "404".to_string()), + ..ProvisionedPackage::new(PackageKind::Toolchain, format!("pkg-{i}"), *status) + }) + .collect(), + } + } + + #[test] + fn text_lists_each_package_and_a_summary() { + let text = render_text(&[report(&[ProvisionStatus::Present, ProvisionStatus::Failed])]); + assert!(text.contains("[uno] AtmelAvr")); + assert!(text.contains("present toolchain pkg-0 1.0 2.0 KB")); + assert!(text.contains("error: 404")); + assert!(text.contains("2 package(s): 1 present, 0 fetched, 0 would-fetch, 1 failed")); + } + + #[test] + fn json_manifest_carries_rows_and_hashes() { + let reports = [report(&[ProvisionStatus::WouldFetch])]; + let manifest: serde_json::Value = serde_json::from_str(&render_json(&reports)).unwrap(); + let env = &manifest["environments"][0]; + assert_eq!(env["env"], "uno"); + assert_eq!(env["packages"][0]["status"], "would-fetch"); + assert_eq!(env["packages"][0]["kind"], "toolchain"); + assert_eq!( + manifest["packages_hash"], + packages_hash(&reports[0].packages) + ); + } + + #[test] + fn check_exits_two_only_when_something_would_be_fetched() { + let missing = [report(&[ProvisionStatus::WouldFetch])]; + match exit_status(&missing, ProvisionMode::Check) { + Err(FbuildError::CommandFailed { exit_code, .. }) => { + assert_eq!(exit_code, CHECK_NEEDS_FETCH_EXIT) + } + other => panic!("expected exit 2, got {other:?}"), + } + assert!(exit_status(&missing, ProvisionMode::DryRun).is_ok()); + assert!(exit_status(&[report(&[ProvisionStatus::Present])], ProvisionMode::Check).is_ok()); + } + + #[test] + fn a_failed_package_exits_one() { + match exit_status( + &[report(&[ProvisionStatus::Failed])], + ProvisionMode::Install, + ) { + Err(FbuildError::CommandFailed { exit_code, .. }) => assert_eq!(exit_code, 1), + other => panic!("expected exit 1, got {other:?}"), + } + } +} diff --git a/crates/fbuild-cli/src/cli/mod.rs b/crates/fbuild-cli/src/cli/mod.rs index d3cfa8224..5b8f50b6c 100644 --- a/crates/fbuild-cli/src/cli/mod.rs +++ b/crates/fbuild-cli/src/cli/mod.rs @@ -28,6 +28,7 @@ pub mod dispatch; pub mod graph_cmd; pub mod ide; pub mod ide_debug; +pub mod install; pub mod libraries; pub mod lnk; pub mod monitor_parse; diff --git a/crates/fbuild-cli/src/cli/tests.rs b/crates/fbuild-cli/src/cli/tests.rs index b178abf82..e8c86756a 100644 --- a/crates/fbuild-cli/src/cli/tests.rs +++ b/crates/fbuild-cli/src/cli/tests.rs @@ -9,6 +9,43 @@ fn deploy_admin_and_no_admin_conflict() { assert!(Cli::try_parse_from(["fbuild", "deploy", "--admin", "--no-admin"]).is_err()); } +// ---------- `fbuild install` CLI shape (FastLED/fbuild#1433) ---------- + +#[test] +fn install_parses_repeated_envs_and_flags() { + let cli = Cli::try_parse_from([ + "fbuild", "install", "proj", "-e", "uno", "-e", "esp32s3", "--check", "--json", "-j", "4", + ]) + .expect("parse"); + match cli.command { + Some(Commands::Install { + project_dir, + environments, + all_envs, + check, + dry_run, + json, + jobs, + }) => { + assert_eq!(project_dir.as_deref(), Some("proj")); + assert_eq!(environments, vec!["uno", "esp32s3"]); + assert!(check && json && !dry_run && !all_envs); + assert_eq!(jobs, Some(4)); + } + _ => panic!("expected Commands::Install"), + } +} + +#[test] +fn install_env_conflicts_with_all_envs() { + assert!(Cli::try_parse_from(["fbuild", "install", "-e", "uno", "--all-envs"]).is_err()); +} + +#[test] +fn install_check_conflicts_with_dry_run() { + assert!(Cli::try_parse_from(["fbuild", "install", "--check", "--dry-run"]).is_err()); +} + // ---------- `fbuild ide` / `fbuild ide select` CLI shape ---------- #[test] diff --git a/crates/fbuild-daemon/src/handlers/operations/install_deps.rs b/crates/fbuild-daemon/src/handlers/operations/install_deps.rs index 0be8c7457..319839c29 100644 --- a/crates/fbuild-daemon/src/handlers/operations/install_deps.rs +++ b/crates/fbuild-daemon/src/handlers/operations/install_deps.rs @@ -83,43 +83,53 @@ pub async fn install_deps( .or_else(|| config.get_default_environment().map(|s| s.to_string())) .unwrap_or_else(|| "default".to_string()); - let env_config = match config.get_env_config(&env_name) { - Ok(c) => c, - Err(e) => { - return ( - StatusCode::BAD_REQUEST, - Json(OperationResponse::fail( - request_id, - format!("invalid environment '{}': {}", env_name, e), - )), - ); - } - }; + if let Err(e) = config.get_env_config(&env_name) { + return ( + StatusCode::BAD_REQUEST, + Json(OperationResponse::fail( + request_id, + format!("invalid environment '{}': {}", env_name, e), + )), + ); + } - let platform_str = env_config.get("platform").cloned().unwrap_or_default(); - let platform = match fbuild_core::Platform::from_platform_str(&platform_str) { - Some(p) => p, - None => { - return ( - StatusCode::BAD_REQUEST, + // Provision everything the env's build downloads — the same set + // `fbuild install` reports (FastLED/fbuild#1433). + let result = fbuild_build::provision_env( + &project_dir, + &env_name, + fbuild_build::provision::ProvisionMode::Install, + ) + .await; + + match result { + Ok(report) if report.failed() => { + let failures = report + .packages + .iter() + .filter(|p| p.status == fbuild_build::provision::ProvisionStatus::Failed) + .map(|p| { + format!( + "{}: {}", + p.name, + p.error.as_deref().unwrap_or("unknown error") + ) + }) + .collect::>() + .join("; "); + ( + StatusCode::INTERNAL_SERVER_ERROR, Json(OperationResponse::fail( request_id, - format!("unsupported platform: {}", platform_str), + format!("install-deps error: {failures}"), )), - ); + ) } - }; - - // Install dependencies via the package manager - let env_label = env_name.clone(); - let result = fbuild_build::install_platform_deps(platform, &project_dir).await; - - match result { - Ok(()) => ( + Ok(_) => ( StatusCode::OK, Json(OperationResponse::ok( request_id, - format!("Dependencies installed for environment '{}'", env_label), + format!("Dependencies installed for environment '{}'", env_name), )), ), Err(e) => ( diff --git a/crates/fbuild-library/src/library/esp32_framework/libs.rs b/crates/fbuild-library/src/library/esp32_framework/libs.rs index 2a2900242..3cc0058e8 100644 --- a/crates/fbuild-library/src/library/esp32_framework/libs.rs +++ b/crates/fbuild-library/src/library/esp32_framework/libs.rs @@ -144,6 +144,17 @@ fn patch_mcu_compatibility(mcu_dir: &Path, mcu: &str) -> fbuild_core::Result<()> } impl Esp32Framework { + /// Whether the per-MCU SDK is already complete under the framework's + /// `tools/` dir — the test [`Self::ensure_libs`] and + /// [`Self::ensure_mcu_libs`] use to skip work. Offline, so `fbuild install + /// --check` can ask it (FastLED/fbuild#1433). + pub fn sdk_libs_installed(&self, mcu: &str) -> bool { + let tools_dir = self.resolved_dir().join("tools"); + mcu_sdk_dir_candidates(&tools_dir, mcu) + .iter() + .any(|mcu_dir| mcu_sdk_complete(mcu_dir)) + } + /// Ensure the SDK libs are downloaded and extracted into the framework's `tools/` dir. pub async fn ensure_libs(&self, libs_url: &str, mcu: &str) -> fbuild_core::Result<()> { let root = self.resolved_dir(); diff --git a/crates/fbuild-library/src/library/esptool.rs b/crates/fbuild-library/src/library/esptool.rs index dfa07ff99..ae7a377f6 100644 --- a/crates/fbuild-library/src/library/esptool.rs +++ b/crates/fbuild-library/src/library/esptool.rs @@ -114,6 +114,27 @@ impl Esptool { ) } + /// The esptool binary [`Self::ensure_installed`] would return, if it is + /// already usable: the [`ESPTOOL_PATH_ENV_VAR`] override, or a cached + /// install that contains the executable. Never installs anything, so + /// `fbuild install --check` can ask it (FastLED/fbuild#1433). + pub fn installed_binary(&self) -> Result> { + if let Some(override_path) = esptool_path_override()? { + return Ok(Some(override_path)); + } + let url = Self::release_url(&self.version, host_platform_tag()?); + let base = PackageBase::new( + "tool-esptoolpy", + &self.version, + &url, + &url, + None, + CacheSubdir::Toolchains, + self.project_dir.as_path(), + ); + Ok(find_esptool_binary(&base.install_path())) + } + /// Ensure the standalone esptool binary is installed and return its path. /// The caller runs it directly as ` --chip elf2image …`. /// diff --git a/crates/fbuild-library/src/library/library_downloader.rs b/crates/fbuild-library/src/library/library_downloader.rs index 574b1c45e..3af045013 100644 --- a/crates/fbuild-library/src/library/library_downloader.rs +++ b/crates/fbuild-library/src/library/library_downloader.rs @@ -10,6 +10,14 @@ use fbuild_core::{FbuildError, Result}; use super::library_spec::LibrarySpec; use super::registry; +/// Whether `spec` is already downloaded into `libs_dir` — the test +/// [`download_library`] uses to skip the network. Offline, so `fbuild install +/// --check` can ask it (FastLED/fbuild#1433). +pub fn is_downloaded(spec: &LibrarySpec, libs_dir: &Path) -> bool { + let lib_dir = libs_dir.join(spec.sanitized_name()); + lib_dir.join("library.json").exists() && lib_dir.join("src").exists() +} + /// Download a library from its spec, returning the library directory. /// /// - GitHub URL deps: download archive from `{url}/archive/refs/heads/main.zip` @@ -20,9 +28,7 @@ pub async fn download_library(spec: &LibrarySpec, libs_dir: &Path) -> Result, } +/// Parse `lib_deps` and drop `lib_ignore` entries. +pub fn parse_lib_specs(lib_specs: &[String], lib_ignore: &[String]) -> Vec { + lib_specs + .iter() + .filter_map(|s| LibrarySpec::parse(s)) + .filter(|spec| { + !lib_ignore + .iter() + .any(|ig| ig.eq_ignore_ascii_case(&spec.name)) + }) + .collect() +} + /// Ensure all library dependencies are downloaded and compiled. /// /// Flow: -/// 1. Parse specs from `lib_deps` -/// 2. Filter out `lib_ignore` entries -/// 3. Download all libraries -/// 4. Collect all include dirs (needed before compilation for cross-includes) -/// 5. Compile each library -/// 6. Return include dirs + archives +/// 1. Download every library with [`download_libraries`] +/// 2. Collect all include dirs (needed before compilation for cross-includes) +/// 3. Compile each library +/// 4. Return include dirs + archives #[allow(clippy::too_many_arguments)] pub async fn ensure_libraries( lib_specs: &[String], @@ -94,74 +105,14 @@ pub async fn ensure_libraries( jobs: usize, compiler_cache: Option<&Path>, ) -> Result { - // 1. Parse specs, filter ignored - let specs: Vec = lib_specs - .iter() - .filter_map(|s| LibrarySpec::parse(s)) - .filter(|spec| { - !lib_ignore - .iter() - .any(|ig| ig.eq_ignore_ascii_case(&spec.name)) - }) - .collect(); - - if specs.is_empty() { + let installed = download_libraries(lib_specs, lib_ignore, project_dir, libs_dir).await?; + if installed.is_empty() { return Ok(LibraryResult { include_dirs: Vec::new(), archives: Vec::new(), }); } - tracing::info!("resolving {} library dependencies", specs.len()); - - // 2. Resolve named local libraries and download remote libraries in parallel. - // Local libraries compile into `libs_dir`, never their checked-out source - // directory, so a build cannot leave generated artifacts in a dependency. - std::fs::create_dir_all(libs_dir)?; - let mut installed: Vec = Vec::new(); - let mut downloaded_names: std::collections::HashSet = std::collections::HashSet::new(); - - let libs_dir_owned = libs_dir.to_path_buf(); - let mut tasks: tokio::task::JoinSet< - std::result::Result<(std::path::PathBuf, String, String), fbuild_core::FbuildError>, - > = tokio::task::JoinSet::new(); - for spec in &specs { - if let Some(local_path) = &spec.local_path { - let lib_dir = resolve_local_library_dir(project_dir, local_path, &spec.name)?; - let sanitized = spec.sanitized_name(); - installed.push(InstalledLibrary::with_build_dir( - &lib_dir, - &sanitized, - &libs_dir.join(&sanitized), - )); - downloaded_names.insert(spec.name.to_lowercase()); - continue; - } - let spec_clone = spec.clone(); - let dir = libs_dir_owned.clone(); - tasks.spawn(async move { - let lib_dir = library_downloader::download_library(&spec_clone, &dir).await?; - Ok(( - lib_dir, - spec_clone.sanitized_name(), - spec_clone.name.to_lowercase(), - )) - }); - } - - while let Some(joined) = tasks.join_next().await { - let (lib_dir, sanitized, name_lower) = joined.map_err(|e| { - fbuild_core::FbuildError::PackageError(format!("library download task failed: {}", e)) - })??; - installed.push(InstalledLibrary::new(&lib_dir, &sanitized)); - downloaded_names.insert(name_lower); - } - - // 2b. Resolve transitive dependencies from library.json files - let ignore_set: std::collections::HashSet = - lib_ignore.iter().map(|s| s.to_lowercase()).collect(); - resolve_transitive_deps(&mut installed, &mut downloaded_names, &ignore_set, libs_dir).await?; - // 3. Collect all include dirs (needed for cross-library includes) let mut all_include_dirs: Vec = base_includes.to_vec(); for lib in &installed { @@ -219,6 +170,74 @@ pub async fn ensure_libraries( }) } +/// Download (or resolve locally) every `lib_deps` library and its transitive +/// dependencies, without compiling. `fbuild install` stops here; builds go on +/// to compile in [`ensure_libraries`] (FastLED/fbuild#1433). +pub async fn download_libraries( + lib_specs: &[String], + lib_ignore: &[String], + project_dir: &Path, + libs_dir: &Path, +) -> Result> { + // 1. Parse specs, filter ignored + let specs = parse_lib_specs(lib_specs, lib_ignore); + if specs.is_empty() { + return Ok(Vec::new()); + } + + tracing::info!("resolving {} library dependencies", specs.len()); + + // 2. Resolve named local libraries and download remote libraries in parallel. + // Local libraries compile into `libs_dir`, never their checked-out source + // directory, so a build cannot leave generated artifacts in a dependency. + std::fs::create_dir_all(libs_dir)?; + let mut installed: Vec = Vec::new(); + let mut downloaded_names: std::collections::HashSet = std::collections::HashSet::new(); + + let libs_dir_owned = libs_dir.to_path_buf(); + let mut tasks: tokio::task::JoinSet< + std::result::Result<(std::path::PathBuf, String, String), fbuild_core::FbuildError>, + > = tokio::task::JoinSet::new(); + for spec in &specs { + if let Some(local_path) = &spec.local_path { + let lib_dir = resolve_local_library_dir(project_dir, local_path, &spec.name)?; + let sanitized = spec.sanitized_name(); + installed.push(InstalledLibrary::with_build_dir( + &lib_dir, + &sanitized, + &libs_dir.join(&sanitized), + )); + downloaded_names.insert(spec.name.to_lowercase()); + continue; + } + let spec_clone = spec.clone(); + let dir = libs_dir_owned.clone(); + tasks.spawn(async move { + let lib_dir = library_downloader::download_library(&spec_clone, &dir).await?; + Ok(( + lib_dir, + spec_clone.sanitized_name(), + spec_clone.name.to_lowercase(), + )) + }); + } + + while let Some(joined) = tasks.join_next().await { + let (lib_dir, sanitized, name_lower) = joined.map_err(|e| { + fbuild_core::FbuildError::PackageError(format!("library download task failed: {}", e)) + })??; + installed.push(InstalledLibrary::new(&lib_dir, &sanitized)); + downloaded_names.insert(name_lower); + } + + // 2b. Resolve transitive dependencies from library.json files + let ignore_set: std::collections::HashSet = + lib_ignore.iter().map(|s| s.to_lowercase()).collect(); + resolve_transitive_deps(&mut installed, &mut downloaded_names, &ignore_set, libs_dir).await?; + + Ok(installed) +} + /// Resolve transitive dependencies by scanning library.json files. /// /// For each installed library, reads its `library.json` (checking both diff --git a/crates/fbuild-packages-fetch/src/lib.rs b/crates/fbuild-packages-fetch/src/lib.rs index bfde7cdcc..2082aea84 100644 --- a/crates/fbuild-packages-fetch/src/lib.rs +++ b/crates/fbuild-packages-fetch/src/lib.rs @@ -150,6 +150,11 @@ pub struct PackageInfo { pub version: String, pub url: String, pub install_path: PathBuf, + /// Pinned sha256 of the download archive, when the package has one. + pub checksum: Option, + /// Installed size recorded in the package cache index, when it has a row + /// for this package (FastLED/fbuild#1433). + pub installed_bytes: Option, } /// Shared base for package implementations. @@ -483,8 +488,21 @@ impl PackageBase { version: self.version.clone(), url: self.url.clone(), install_path: self.install_path(), + checksum: self.checksum.clone(), + installed_bytes: self.recorded_installed_bytes(), } } + + /// Installed size from the cache index row written by + /// [`Self::record_install_in_disk_cache`]. Read-only: unlike + /// [`Self::is_cached`] it does not bump the LRU timestamp. + fn recorded_installed_bytes(&self) -> Option { + let dc = self.disk_cache.as_ref()?; + let entry = dc + .lookup(self.cache_subdir.into(), &self.cache_key, &self.version) + .ok()??; + u64::try_from(entry.installed_bytes?).ok() + } } fn package_touch_key( @@ -571,6 +589,8 @@ mod toolchain_gcc_ar_tests { version: "0.0".to_string(), url: String::new(), install_path: PathBuf::new(), + checksum: None, + installed_bytes: None, } } } diff --git a/crates/fbuild-toolchain/src/toolchain/esp32_metadata.rs b/crates/fbuild-toolchain/src/toolchain/esp32_metadata.rs index 56f234f82..9de575bb3 100644 --- a/crates/fbuild-toolchain/src/toolchain/esp32_metadata.rs +++ b/crates/fbuild-toolchain/src/toolchain/esp32_metadata.rs @@ -109,6 +109,19 @@ pub fn resolve_toolchain_url_sync( } } +/// Resolve from a metadata package that is already on disk, without touching +/// the network. `Ok(None)` when the metadata has never been fetched, which +/// `fbuild install --check` reports as something to fetch (FastLED/fbuild#1433). +pub fn resolve_toolchain_url_cached( + toolchain_name: &str, + cache_dir: &Path, +) -> Result> { + match find_tools_json(&cache_dir.join("metadata")) { + Some(tools_json) => parse_tools_json(&tools_json, toolchain_name).map(Some), + None => Ok(None), + } +} + /// Find tools.json in a directory (may be at root or one level deep). fn find_tools_json(dir: &Path) -> Option { let direct = dir.join("tools.json"); diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 18c4d468f..58521e0b5 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -467,6 +467,39 @@ generated for. ## Batch And CI Commands +### `fbuild install` + +Download everything an environment's build needs — platform package, +toolchain, framework, SDK libs, tools such as esptool, and `lib_deps` — without +compiling. It runs in-process (no daemon) and resolves packages through the +same helpers the build uses, so a following `fbuild build` fetches nothing. + +```bash +fbuild install -e esp32s3 # fetch what is missing +fbuild install -e uno -e teensy41 --json # manifest with a packages_hash +fbuild install --all-envs --check # exit 2 if anything is missing +fbuild install -e esp32s3 --dry-run # list the resolved set +``` + +Each package prints one line: status (`present`, `fetched`, `would-fetch`, +`failed`), kind, name, version, installed size, duration, URL and sha256. +`--json` emits the same rows per environment plus a `packages_hash` — sha256 +over the sorted `(kind, name, version, url, sha256)` tuples, independent of +status and size — which CI can use as a packages-cache key. + +| Flag | Behavior | +|---|---| +| `-e ` | Environment to provision; repeatable. Defaults to the project's default environment. | +| `--all-envs` | Provision every environment in `platformio.ini`. | +| `--check` | Report missing packages without fetching; exit 2 if any. Never touches the network. | +| `--dry-run` | List the resolved packages without fetching; exit 0. Never touches the network. | +| `--json` | Print the JSON manifest instead of text. | +| `-j ` | Provision up to N environments in parallel. | + +Exit codes: `0` success, `1` a package failed to install, `2` `--check` found +something to fetch. `--check` and `--dry-run` list direct `lib_deps` only; +transitive library dependencies are known once the direct ones are downloaded. + ### `fbuild compile-many` Build many sketches against one board using a two-stage pipeline: framework and diff --git a/dylints/cli_no_build_deploy_direct_use/src/allowlist.txt b/dylints/cli_no_build_deploy_direct_use/src/allowlist.txt index 4e3705163..fe6d11f6b 100644 --- a/dylints/cli_no_build_deploy_direct_use/src/allowlist.txt +++ b/dylints/cli_no_build_deploy_direct_use/src/allowlist.txt @@ -24,6 +24,12 @@ crates/fbuild-cli/src/cli/graph_cmd.rs crates/fbuild-cli/src/cli/symbols_cmd.rs crates/fbuild-cli/src/cli/compile_many.rs +# `fbuild install` calls `fbuild_build::provision_env` in-process: it only +# resolves and downloads packages (no compile, no build lock), and CI runs it +# as its own step before any build, so a daemon round-trip and its compile +# backend startup would only add time (FastLED/fbuild#1433). +crates/fbuild-cli/src/cli/install.rs + # `fbuild reset` calls into `fbuild_deploy::reset` directly because # resetting a device is a single, fast, no-build-state operation — # wrapping it in a daemon HTTP round-trip adds latency for no From 26c547b464353833977f1d56318d43a3343e2965 Mon Sep 17 00:00:00 2001 From: Zach Vorhies Date: Mon, 14 Sep 2026 17:23:14 -0700 Subject: [PATCH 3/4] feat(setup-action): split packages and per-board build-payload caches (#1433) `cache-mode: split` restores a packages cache shared per platform family, runs `fbuild install` as its own step and saves that cache under the family prefix plus `packages_hash`, then restores a per-board payload cache (core, framework-libs, library-selection, zccache) with no cross-board fallback. A new `save` input restores without saving in either mode, so pull requests stop adding entries to the repository's cache budget. The default combined mode is unchanged. The CI caching docs drop the stale claims that cache export/import is not implemented and that `fbuild cache stats` exists, and document the payload slices, `fbuild install` and split mode. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01StasPkdQ3D1gYGj6WnaQ3q --- .github/actions/setup/README.md | 35 ++++++- .github/actions/setup/action.yml | 167 ++++++++++++++++++++++++++++++- docs/CI_CACHE.md | 17 ++++ docs/CI_CACHING.md | 31 +++++- 4 files changed, 241 insertions(+), 9 deletions(-) diff --git a/.github/actions/setup/README.md b/.github/actions/setup/README.md index 80cf1e8bc..befa52092 100644 --- a/.github/actions/setup/README.md +++ b/.github/actions/setup/README.md @@ -51,6 +51,28 @@ jobs: } ``` +## Split caches (`cache-mode: split`) + +For large board matrices, split mode keeps packages shared per platform family and build payloads per board (FastLED/fbuild#1433). It needs an fbuild release with `fbuild install`. + +```yaml +- uses: FastLED/fbuild/.github/actions/setup@main + id: fbuild + with: + cache-mode: split + environments: ${{ matrix.board }} + cache-key-extra: ${{ hashFiles('platformio.ini') }} + save: ${{ github.event_name != 'pull_request' }} +- run: fbuild build examples/Blink -e ${{ matrix.board }} +``` + +In split mode the action: + +1. Runs `fbuild install --dry-run --json` to learn each environment's platform, without touching the network. +2. Restores the packages cache (`toolchains`, `platforms`, `packages`, `libraries`, `archives`, `installed`, `index.sqlite`) by the prefix `fbuild-pkgs-----`. +3. Runs `fbuild install` as its own step, then saves the packages cache under that prefix plus the `packages_hash`. It skips the save when the restored key already matches. +4. Restores the build-payload cache (`core`, `framework-libs`, `library-selection`, the zccache store) keyed by fbuild hash, board and `cache-key-extra`, without falling back to other boards. It is saved at the end of the job unless `save` is `false`. + ## Caching the zccache store The built-in `cache: true` wiring covers the fbuild package/tool cache rooted at `FBUILD_CACHE_DIR`. If you also want cross-run reuse of zccache's object store, add your own `actions/cache@v5` step for the resolved zccache directory and your project build outputs: @@ -94,6 +116,11 @@ Use `steps..outputs.zccache-store-path` inside workflow expressions. The sam | `fbuild-version` | `latest` | PyPI version spec. Pin to an exact version (`2.1.16`) for reproducible CI. | | `python-version` | `3.12` | Python used to install fbuild. Must be >= 3.9. | | `cache` | `true` | Set to `false` to install fbuild without wiring `actions/cache`. | +| `cache-mode` | `combined` | `combined` keeps one `FBUILD_CACHE_DIR` entry per key. `split` restores a packages cache shared per platform family and a per-board build-payload cache, and runs `fbuild install` as its own step. | +| `save` | `true` | `false` restores caches without saving; use it on pull requests. | +| `project-dir` | `.` | Split mode: project passed to `fbuild install`. | +| `environments` | `""` | Split mode: space-separated environments to provision. Required in split mode. | +| `board` | `""` | Split mode: name in the build-payload cache key. Defaults to `environments` joined with `_`. | | `cache-key-extra` | `""` | String baked into the cache key. Use `hashFiles(...)` over your graph inputs so edits invalidate stale artifacts. | | `cache-version` | `v1` | Manual cache bump. Increment when you want to force-invalidate across your matrix. | | `cache-dir` | `$RUNNER_TEMP/fbuild-cache` | Override if you need a different cache root. | @@ -103,7 +130,11 @@ Use `steps..outputs.zccache-store-path` inside workflow expressions. The sam | Output | Description | |---|---| -| `cache-hit` | `true` if the cache was restored from a previous run, `false` on miss. | +| `cache-hit` | Combined mode: `true` if the cache was restored from an exact key match, `false` otherwise. | +| `platform-family` | Split mode: the platform family the packages cache is shared across. | +| `packages-hash` | Split mode: `packages_hash` from `fbuild install --json`. | +| `packages-cache-hit` | Split mode: the restored packages cache key, empty on a miss. | +| `build-cache-hit` | Split mode: `true` if the build-payload cache was restored from an exact key match. | | `cache-dir` | Resolved cache directory path. Useful for diagnostic steps. | | `fbuild-hash` | sha256 prefix (16 hex chars) of the installed fbuild wheel's `RECORD` file. Baked into the cache key so any fbuild change, including a re-released wheel at the same version, invalidates stale cache artifacts. | | `zccache-store-path` | Resolved zccache object-store directory. The same path is exported to later steps as `ZCCACHE_DIR`, so consumer-managed `actions/cache@v5` blocks can reuse it without guessing platform-specific defaults. | @@ -117,7 +148,7 @@ Use `steps..outputs.zccache-store-path` inside workflow expressions. The sam 5. Installs fbuild from PyPI at the requested version (skipped on install-cache hit). Install uses `pip install --target=$RUNNER_TEMP/fbuild-install` so the cached directory is the entire install surface. 6. Activates the install dir by appending `bin/` (POSIX) and `Scripts/` (Windows) to `$GITHUB_PATH` and prepending `PYTHONPATH`. 7. **Computes the installed fbuild's content hash** (sha256 of its dist-info `RECORD`) and bakes it into the **build artifact** cache key. This guarantees the artifact cache is tied to the exact fbuild you're running, not just the PyPI version string, so `latest` is safe and a re-released wheel won't poison the cache. -8. Restores (and on job-end, saves) the fbuild build artifact cache via `actions/cache@v5`. +8. Restores (and on job-end, saves) the fbuild build artifact cache via `actions/cache@v5`. With `save: false` it only restores. In `cache-mode: split` this step is replaced by the packages and build-payload caches described in [Split caches](#split-caches-cache-mode-split). ### Why hash-pinning matters diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 41775e15a..3e3c0ecbc 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -18,6 +18,26 @@ inputs: description: "Whether to save/restore the fbuild cache via actions/cache. Set to 'false' to skip caching (install only)." required: false default: "true" + cache-mode: + description: "'combined' (default) keeps one FBUILD_CACHE_DIR entry per key. 'split' restores a packages cache shared per platform family plus a per-board build-payload cache, and runs `fbuild install` as its own step (FastLED/fbuild#1433). 'split' needs `environments` and an fbuild release with `fbuild install`." + required: false + default: "combined" + save: + description: "Whether to save caches. 'false' only restores; use it on pull requests so PR runs don't add entries of their own to the repository's cache budget." + required: false + default: "true" + project-dir: + description: "Project directory passed to `fbuild install` in split mode." + required: false + default: "." + environments: + description: "Space-separated environments to provision with `fbuild install` in split mode." + required: false + default: "" + board: + description: "Name for the per-board build-payload cache key in split mode. Defaults to `environments` joined with '_'." + required: false + default: "" cache-key-extra: description: "Extra string baked into the cache key. Typically hashFiles('platformio.ini', ...) so graph-input changes invalidate stale artifacts." required: false @@ -37,8 +57,8 @@ inputs: outputs: cache-hit: - description: "true if the fbuild cache was restored from a previous run, false on miss." - value: ${{ steps.fbuild-cache.outputs.cache-hit }} + description: "Combined mode: true if the fbuild cache was restored from an exact key match, false otherwise." + value: ${{ steps.fbuild-cache.outputs.cache-hit || steps.fbuild-cache-restore-only.outputs.cache-hit }} cache-dir: description: "Resolved cache directory path (useful for diagnostic steps)." value: ${{ steps.resolve-paths.outputs.cache-dir }} @@ -48,6 +68,18 @@ outputs: fbuild-hash: description: "sha256 prefix of the installed fbuild wheel's RECORD file. Baked into the cache key so an fbuild upgrade invalidates stale artifacts." value: ${{ steps.fbuild-hash.outputs.fbuild-hash }} + platform-family: + description: "Split mode: the platform family (e.g. 'espressif32') the packages cache is shared across." + value: ${{ steps.split-plan.outputs.family }} + packages-hash: + description: "Split mode: `packages_hash` reported by `fbuild install --json` for the provisioned environments." + value: ${{ steps.split-install.outputs.packages-hash }} + packages-cache-hit: + description: "Split mode: the packages cache key that was restored, or empty on a miss." + value: ${{ steps.packages-restore.outputs.cache-matched-key }} + build-cache-hit: + description: "Split mode: true if the per-board build-payload cache was restored from an exact key match." + value: ${{ steps.build-cache.outputs.cache-hit || steps.build-cache-restore-only.outputs.cache-hit }} runs: using: "composite" @@ -177,8 +209,10 @@ runs: echo "fbuild-hash=${FBUILD_HASH}" >> "$GITHUB_OUTPUT" echo "Resolved fbuild content hash: ${FBUILD_HASH}" + # ── Combined mode (default): one FBUILD_CACHE_DIR entry per key ───────── + - name: Restore fbuild cache - if: ${{ inputs.cache == 'true' }} + if: ${{ inputs.cache == 'true' && inputs.cache-mode != 'split' && inputs.save == 'true' }} id: fbuild-cache uses: actions/cache@v5 with: @@ -187,3 +221,130 @@ runs: restore-keys: | fbuild-${{ inputs.cache-version }}-${{ runner.os }}-${{ runner.arch }}-${{ steps.fbuild-hash.outputs.fbuild-hash }}- fbuild-${{ inputs.cache-version }}-${{ runner.os }}-${{ runner.arch }}- + + - name: Restore fbuild cache (no save) + if: ${{ inputs.cache == 'true' && inputs.cache-mode != 'split' && inputs.save != 'true' }} + id: fbuild-cache-restore-only + uses: actions/cache/restore@v5 + with: + path: ${{ steps.resolve-paths.outputs.cache-dir }} + key: fbuild-${{ inputs.cache-version }}-${{ runner.os }}-${{ runner.arch }}-${{ steps.fbuild-hash.outputs.fbuild-hash }}-${{ inputs.cache-key-extra }} + restore-keys: | + fbuild-${{ inputs.cache-version }}-${{ runner.os }}-${{ runner.arch }}-${{ steps.fbuild-hash.outputs.fbuild-hash }}- + fbuild-${{ inputs.cache-version }}-${{ runner.os }}-${{ runner.arch }}- + + # ── Split mode (FastLED/fbuild#1433) ─────────────────────────────────── + # Packages (toolchains, platforms, frameworks, libraries) are shared per + # platform family and saved as soon as `fbuild install` finishes: they do + # not change during the build. Build payloads (core objects, framework + # library archives, library selection, the zccache store) are per board + # and saved at the end of the job. Neither falls back across boards, so a + # board's entry never grows into an all-toolchain blob. + + - name: Plan split fbuild caches + if: ${{ inputs.cache == 'true' && inputs.cache-mode == 'split' }} + id: split-plan + shell: bash + env: + FBUILD_PROJECT_DIR: ${{ inputs.project-dir }} + FBUILD_ENVIRONMENTS: ${{ inputs.environments }} + FBUILD_BOARD: ${{ inputs.board }} + run: | + set -euo pipefail + read -r -a envs <<< "$FBUILD_ENVIRONMENTS" + if [ "${#envs[@]}" -eq 0 ]; then + echo "::error::cache-mode: split needs the 'environments' input" >&2 + exit 1 + fi + env_args=() + for env in "${envs[@]}"; do env_args+=(-e "$env"); done + # A dry run resolves the package set without touching the network; + # it names each env's platform before anything is restored. + plan="${RUNNER_TEMP}/fbuild-install-plan.json" + fbuild install "$FBUILD_PROJECT_DIR" "${env_args[@]}" --dry-run --json > "$plan" + family=$(python -c 'import json,sys; print("-".join(sorted({e["platform"].lower() for e in json.load(open(sys.argv[1]))["environments"]})))' "$plan") + board="${FBUILD_BOARD:-$(IFS=_; echo "${envs[*]}")}" + echo "family=${family}" >> "$GITHUB_OUTPUT" + echo "board=${board}" >> "$GITHUB_OUTPUT" + echo "Platform family: ${family}; board payload key: ${board}" + + - name: Restore fbuild packages cache + if: ${{ inputs.cache == 'true' && inputs.cache-mode == 'split' }} + id: packages-restore + uses: actions/cache/restore@v5 + with: + path: | + ${{ steps.resolve-paths.outputs.cache-dir }}/toolchains + ${{ steps.resolve-paths.outputs.cache-dir }}/platforms + ${{ steps.resolve-paths.outputs.cache-dir }}/packages + ${{ steps.resolve-paths.outputs.cache-dir }}/libraries + ${{ steps.resolve-paths.outputs.cache-dir }}/archives + ${{ steps.resolve-paths.outputs.cache-dir }}/installed + ${{ steps.resolve-paths.outputs.cache-dir }}/index.sqlite + # Saved keys end in the packages_hash, which is unknown until install + # runs; restore the newest entry for this platform family by prefix. + key: fbuild-pkgs-${{ inputs.cache-version }}-${{ runner.os }}-${{ runner.arch }}-${{ steps.split-plan.outputs.family }}-restore + restore-keys: | + fbuild-pkgs-${{ inputs.cache-version }}-${{ runner.os }}-${{ runner.arch }}-${{ steps.split-plan.outputs.family }}- + + - name: Install fbuild packages + if: ${{ inputs.cache == 'true' && inputs.cache-mode == 'split' }} + id: split-install + shell: bash + env: + FBUILD_PROJECT_DIR: ${{ inputs.project-dir }} + FBUILD_ENVIRONMENTS: ${{ inputs.environments }} + FBUILD_PACKAGES_KEY_PREFIX: fbuild-pkgs-${{ inputs.cache-version }}-${{ runner.os }}-${{ runner.arch }}-${{ steps.split-plan.outputs.family }}- + run: | + set -euo pipefail + read -r -a envs <<< "$FBUILD_ENVIRONMENTS" + env_args=() + for env in "${envs[@]}"; do env_args+=(-e "$env"); done + manifest="${RUNNER_TEMP}/fbuild-install.json" + fbuild install "$FBUILD_PROJECT_DIR" "${env_args[@]}" --json > "$manifest" + cat "$manifest" + hash=$(python -c 'import json,sys; print(json.load(open(sys.argv[1]))["packages_hash"])' "$manifest") + echo "packages-hash=${hash}" >> "$GITHUB_OUTPUT" + echo "packages-key=${FBUILD_PACKAGES_KEY_PREFIX}${hash}" >> "$GITHUB_OUTPUT" + + - name: Save fbuild packages cache + if: ${{ inputs.cache == 'true' && inputs.cache-mode == 'split' && inputs.save == 'true' && steps.packages-restore.outputs.cache-matched-key != steps.split-install.outputs.packages-key }} + uses: actions/cache/save@v5 + with: + path: | + ${{ steps.resolve-paths.outputs.cache-dir }}/toolchains + ${{ steps.resolve-paths.outputs.cache-dir }}/platforms + ${{ steps.resolve-paths.outputs.cache-dir }}/packages + ${{ steps.resolve-paths.outputs.cache-dir }}/libraries + ${{ steps.resolve-paths.outputs.cache-dir }}/archives + ${{ steps.resolve-paths.outputs.cache-dir }}/installed + ${{ steps.resolve-paths.outputs.cache-dir }}/index.sqlite + key: ${{ steps.split-install.outputs.packages-key }} + + - name: Restore fbuild build-payload cache + if: ${{ inputs.cache == 'true' && inputs.cache-mode == 'split' && inputs.save == 'true' }} + id: build-cache + uses: actions/cache@v5 + with: + path: | + ${{ steps.resolve-paths.outputs.cache-dir }}/core + ${{ steps.resolve-paths.outputs.cache-dir }}/framework-libs + ${{ steps.resolve-paths.outputs.cache-dir }}/library-selection + ${{ steps.resolve-paths.outputs.zccache-store-path }} + key: fbuild-build-${{ inputs.cache-version }}-${{ runner.os }}-${{ runner.arch }}-${{ steps.fbuild-hash.outputs.fbuild-hash }}-${{ steps.split-plan.outputs.board }}-${{ inputs.cache-key-extra }} + restore-keys: | + fbuild-build-${{ inputs.cache-version }}-${{ runner.os }}-${{ runner.arch }}-${{ steps.fbuild-hash.outputs.fbuild-hash }}-${{ steps.split-plan.outputs.board }}- + + - name: Restore fbuild build-payload cache (no save) + if: ${{ inputs.cache == 'true' && inputs.cache-mode == 'split' && inputs.save != 'true' }} + id: build-cache-restore-only + uses: actions/cache/restore@v5 + with: + path: | + ${{ steps.resolve-paths.outputs.cache-dir }}/core + ${{ steps.resolve-paths.outputs.cache-dir }}/framework-libs + ${{ steps.resolve-paths.outputs.cache-dir }}/library-selection + ${{ steps.resolve-paths.outputs.zccache-store-path }} + key: fbuild-build-${{ inputs.cache-version }}-${{ runner.os }}-${{ runner.arch }}-${{ steps.fbuild-hash.outputs.fbuild-hash }}-${{ steps.split-plan.outputs.board }}-${{ inputs.cache-key-extra }} + restore-keys: | + fbuild-build-${{ inputs.cache-version }}-${{ runner.os }}-${{ runner.arch }}-${{ steps.fbuild-hash.outputs.fbuild-hash }}-${{ steps.split-plan.outputs.board }}- diff --git a/docs/CI_CACHE.md b/docs/CI_CACHE.md index bd6a70bc6..43a9dbd56 100644 --- a/docs/CI_CACHE.md +++ b/docs/CI_CACHE.md @@ -34,6 +34,23 @@ against local installs, dev builds, or a wheel re-uploaded with the same version The setup action computes the hash for you and uses it in its built-in `FBUILD_CACHE_DIR` cache key. +## Split packages and build payloads + +For a large board matrix, set the setup action's `cache-mode: split` and pass +the board's `environments` (FastLED/fbuild#1433). The action then: + +1. Restores a **packages cache** shared per platform family (toolchains, + platforms, frameworks, libraries, `index.sqlite`). +2. Runs `fbuild install` as its own step and saves the packages cache under the + family prefix plus the `packages_hash` from `fbuild install --json`. +3. Restores a **per-board build-payload cache** (`core/`, `framework-libs/`, + `library-selection/`, zccache) keyed by fbuild hash, board and + `cache-key-extra`, with no fallback across boards, and saves it at job end. + +Pass `save: ${{ github.event_name != 'pull_request' }}` so pull requests restore +the default branch's entries without adding their own to the repository's +cache budget. See [CI_CACHING.md](CI_CACHING.md#split-caches-for-large-board-matrices-fastledfbuild1433). + ## Invalidation pattern Keep normal invalidation automatic by hashing graph inputs. Keep forced diff --git a/docs/CI_CACHING.md b/docs/CI_CACHING.md index 16fd7e028..3259d25b0 100644 --- a/docs/CI_CACHING.md +++ b/docs/CI_CACHING.md @@ -47,6 +47,26 @@ Keep the action's built-in cache for `FBUILD_CACHE_DIR`, then add a second cache Use `steps..outputs.zccache-store-path` inside workflow expressions such as `path:`. Use `ZCCACHE_DIR` in later shell steps when you need the resolved directory at runtime. +### Split caches for large board matrices (FastLED/fbuild#1433) + +A single `FBUILD_CACHE_DIR` entry per board mixes packages every board shares with build payloads only that board uses. With cross-board `restore-keys`, each board's entry also inherits whatever toolchains the previous board had, so entries grow to 1–2 GB and a large matrix evicts itself from the repository's 10 GB cache budget. `cache-mode: split` keeps the two apart: + +```yaml +- uses: FastLED/fbuild/.github/actions/setup@main + with: + cache-mode: split + environments: ${{ matrix.board }} + cache-key-extra: ${{ hashFiles('platformio.ini') }} + # Pull requests restore main's entries but add none of their own. + save: ${{ github.event_name != 'pull_request' }} +- run: fbuild build examples/Blink -e ${{ matrix.board }} +``` + +- **Packages cache**, shared per platform family: the `toolchains`, `platforms`, `packages`, `libraries`, `archives`, `installed` and `index.sqlite` slices. It is restored by the family prefix `fbuild-pkgs-----`, then `fbuild install` runs as its own step and the cache is saved under that prefix plus the `packages_hash` from `fbuild install --json`. An unchanged package set maps to an existing key, so nothing new is written. +- **Build-payload cache**, per board: `core/`, `framework-libs/`, `library-selection/` and the zccache store, keyed by fbuild hash, board and `cache-key-extra`, with no fallback across boards. It is saved at the end of the job. + +Builds print one line per framework core cache and framework-libs cache hydrate and store (`framework core cache: hit …`, `framework-libs cache: miss`), so the CI log shows whether a restored payload was used. + ### Raw snippet (if you don't want the action dependency) If you skip the composite action, you MUST still bake the fbuild content hash into the cache key - see [Cache-key strategy](#cache-key-strategy) below for why. Minimal version: @@ -92,6 +112,8 @@ Adjust `hashFiles(...)` inputs to whatever files actually change the build graph | `~/.fbuild/prod/cache/archives/` | Downloaded toolchain + framework + library tarballs (pre-extract) | **Yes** | | `~/.fbuild/prod/cache/installed/` | Extracted, usable toolchains, frameworks, libraries | **Yes** | | `~/.fbuild/prod/cache/index.sqlite` | LRU index that pairs entries to URLs/versions | **Yes** (must match archives + installed) | +| `~/.fbuild/prod/cache/{toolchains,platforms,packages,libraries}/` | Installed toolchains, platform packages, frameworks and libraries | **Yes** (the packages cache in split mode) | +| `~/.fbuild/prod/cache/{core,framework-libs,library-selection}/` | Reusable framework core objects, ESP32 framework library archives, library-selection results | **Yes, per board** (the build-payload cache in split mode) | | `$ZCCACHE_DIR` | zccache object store for compiled translation units | **Yes, if you want cross-run zccache hits** | | `/.fbuild/build/` | Per-project build outputs (object files, archives, compile DB, firmware) | **Yes** (the warm-build fast path depends on this) | | `~/.fbuild/prod/daemon/` | Daemon PID, port, log, status - **ephemeral runtime state** | **No** | @@ -192,9 +214,10 @@ Matrix jobs sharing one `actions/cache` key will each read the same restore atom As of this doc: -- `fbuild cache export ` / `fbuild cache import `: **not implemented**. `actions/cache@v5` handles archive+extract. A native helper would only be needed for non-GHA CI systems. +- `fbuild cache save|restore|list|verify` (FastLED/fbuild#527): **implemented**. Packs cache slices into one zstd `.tar.zst` with a manifest of per-slice file counts, sizes and content hashes. The default slices are the packages; `core`, `framework-libs`, `library-selection` and `zccache` are opt-in with `--include`, which is how a per-board build payload is archived separately. Useful on CI systems without `actions/cache`; the setup action still uses `actions/cache@v5`. +- `fbuild install -e [--check] [--json]` (FastLED/fbuild#1433): **implemented**. Provisions an env's packages without compiling and reports each one; `--check` exits 2 when anything is missing and never touches the network, and `--json` reports the `packages_hash` the split setup mode keys its packages cache on. - `fbuild cache pin `: **not implemented**. LRU eviction is based on recency; if you need to guarantee a toolchain never evicts on a shared cache, file a follow-up. -- `fbuild cache stats`: yes - `DiskCache::stats()` exposes size and entry counts. Useful for CI debug output. +- `fbuild cache stats`: **not implemented** as a subcommand. `fbuild daemon cache-stats` reports the daemon's view, and `DiskCache::stats()` exposes size and entry counts to code. ## Worked example: FastLED's matrix @@ -230,10 +253,10 @@ steps: - name: Build run: fbuild build examples/Blink -e ${{ matrix.board }} - - name: Print cache stats + - name: Confirm packages are still installed run: | echo "zccache store: $ZCCACHE_DIR" - fbuild cache stats + fbuild install examples/Blink -e ${{ matrix.board }} --check shell: bash if: always() ``` From a9baadc7e0f5ade29b1a850584d2d200d4921b95 Mon Sep 17 00:00:00 2001 From: Zach Vorhies Date: Mon, 14 Sep 2026 17:44:29 -0700 Subject: [PATCH 4/4] fix(install): address review on #1438 and drop PathBuf from provision tests - Setup action: split mode runs the plan and `fbuild install` steps even with `cache: false`; only the cache restore/save steps stay gated on `cache`. - Local `file://`/`symlink://` lib_deps are reported present only when the project-relative directory exists, so `--check` no longer passes a missing one. - ESP32 esptool row checks `FBUILD_ESPTOOL_PATH` before platform metadata: a valid override is present, an invalid one fails, as the build resolves it. - CH32V provisioning rejects a non-Arduino framework before downloading, as the build does. - `Esptool::installed_binary` requires the install-complete sentinel, so an interrupted install is not reported present. - `provision_package` delegates to `provision_with`, which the tests drive directly; the test double no longer implements `Package`, whose signature names `std::path::PathBuf` and failed the `ban_std_pathbuf` Dylint. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01StasPkdQ3D1gYGj6WnaQ3q --- .github/actions/setup/action.yml | 4 +- crates/fbuild-build-engine/src/provision.rs | 102 ++++++++++++------ .../src/esp32/orchestrator/packages.rs | 21 ++++ crates/fbuild-build-mcu/src/ch32v/mod.rs | 4 + .../src/ch32v/orchestrator.rs | 2 +- crates/fbuild-library/src/library/esptool.rs | 5 + 6 files changed, 103 insertions(+), 35 deletions(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 3e3c0ecbc..b592668ef 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -242,7 +242,7 @@ runs: # board's entry never grows into an all-toolchain blob. - name: Plan split fbuild caches - if: ${{ inputs.cache == 'true' && inputs.cache-mode == 'split' }} + if: ${{ inputs.cache-mode == 'split' }} id: split-plan shell: bash env: @@ -288,7 +288,7 @@ runs: fbuild-pkgs-${{ inputs.cache-version }}-${{ runner.os }}-${{ runner.arch }}-${{ steps.split-plan.outputs.family }}- - name: Install fbuild packages - if: ${{ inputs.cache == 'true' && inputs.cache-mode == 'split' }} + if: ${{ inputs.cache-mode == 'split' }} id: split-install shell: bash env: diff --git a/crates/fbuild-build-engine/src/provision.rs b/crates/fbuild-build-engine/src/provision.rs index 3d8c9ab83..8333891f3 100644 --- a/crates/fbuild-build-engine/src/provision.rs +++ b/crates/fbuild-build-engine/src/provision.rs @@ -135,18 +135,41 @@ pub async fn provision_package( package: &dyn Package, mode: ProvisionMode, ) -> ProvisionedPackage { + provision_with( + kind, + package.is_installed(), + mode, + || async { package.ensure_installed().await.map(|_| ()) }, + || package.get_info(), + ) + .await +} + +/// [`provision_package`] over a package's parts: whether it is installed, how +/// to install it, and its metadata once that is settled. +async fn provision_with( + kind: PackageKind, + installed: bool, + mode: ProvisionMode, + install: Install, + info: impl FnOnce() -> fbuild_packages::PackageInfo, +) -> ProvisionedPackage +where + Install: FnOnce() -> Installing, + Installing: std::future::Future>, +{ let started = Instant::now(); - let (status, error) = if package.is_installed() { + let (status, error) = if installed { (ProvisionStatus::Present, None) } else if !mode.fetches() { (ProvisionStatus::WouldFetch, None) } else { - match package.ensure_installed().await { - Ok(_) => (ProvisionStatus::Fetched, None), + match install().await { + Ok(()) => (ProvisionStatus::Fetched, None), Err(error) => (ProvisionStatus::Failed, Some(error.to_string())), } }; - let info = package.get_info(); + let info = info(); let installed = matches!(status, ProvisionStatus::Present | ProvisionStatus::Fetched); ProvisionedPackage { kind, @@ -183,7 +206,11 @@ pub async fn provision_lib_deps( let started = Instant::now(); let present_before: Vec = specs .iter() - .map(|spec| spec.local_path.is_some() || library_downloader::is_downloaded(spec, libs_dir)) + .map(|spec| match &spec.local_path { + // Resolved the way the build resolves it: relative to the project. + Some(path) => project_dir.join(path.as_path()).is_dir(), + None => library_downloader::is_downloaded(spec, libs_dir), + }) .collect(); let mut rows: Vec = specs .iter() @@ -332,47 +359,55 @@ pub fn packages_hash<'a>(packages: impl IntoIterator Self { Self { installed: AtomicBool::new(installed), install_fails, } } - } - #[async_trait::async_trait] - impl Package for FakePackage { - async fn ensure_installed(&self) -> fbuild_core::Result { + fn is_installed(&self) -> bool { + self.installed.load(Ordering::SeqCst) + } + + async fn install(&self) -> fbuild_core::Result<()> { if self.install_fails { return Err(fbuild_core::FbuildError::PackageError("404".into())); } self.installed.store(true, Ordering::SeqCst); - Ok(PathBuf::from("/cache/fake")) - } - - fn is_installed(&self) -> bool { - self.installed.load(Ordering::SeqCst) + Ok(()) } - fn get_info(&self) -> PackageInfo { + fn info(&self) -> PackageInfo { PackageInfo { name: "toolchain-fake".into(), version: "1.2.3".into(), url: "https://example.com/fake.tar.gz".into(), - install_path: PathBuf::from("/cache/fake"), + install_path: "/cache/fake".into(), checksum: Some("abc123".into()), installed_bytes: Some(4096), } } + + async fn provision(&self, kind: PackageKind, mode: ProvisionMode) -> ProvisionedPackage { + provision_with( + kind, + self.is_installed(), + mode, + || self.install(), + || self.info(), + ) + .await + } } #[tokio::test] @@ -382,9 +417,9 @@ mod tests { ProvisionMode::Check, ProvisionMode::DryRun, ] { - let row = - provision_package(PackageKind::Toolchain, &FakePackage::new(true, true), mode) - .await; + let row = Fake::new(true, true) + .provision(PackageKind::Toolchain, mode) + .await; assert_eq!(row.status, ProvisionStatus::Present, "{mode:?}"); assert_eq!(row.bytes, Some(4096)); assert_eq!(row.sha256.as_deref(), Some("abc123")); @@ -393,29 +428,32 @@ mod tests { #[tokio::test] async fn missing_package_is_fetched_only_by_install() { - let fake = FakePackage::new(false, false); - let row = provision_package(PackageKind::Toolchain, &fake, ProvisionMode::Check).await; + let fake = Fake::new(false, false); + let row = fake + .provision(PackageKind::Toolchain, ProvisionMode::Check) + .await; assert_eq!(row.status, ProvisionStatus::WouldFetch); assert_eq!(row.bytes, None); assert!(!fake.is_installed(), "a check must not install"); - let row = provision_package(PackageKind::Toolchain, &fake, ProvisionMode::DryRun).await; + let row = fake + .provision(PackageKind::Toolchain, ProvisionMode::DryRun) + .await; assert_eq!(row.status, ProvisionStatus::WouldFetch); assert!(!fake.is_installed(), "a dry run must not install"); - let row = provision_package(PackageKind::Toolchain, &fake, ProvisionMode::Install).await; + let row = fake + .provision(PackageKind::Toolchain, ProvisionMode::Install) + .await; assert_eq!(row.status, ProvisionStatus::Fetched); assert!(fake.is_installed()); } #[tokio::test] async fn failed_install_carries_the_error() { - let row = provision_package( - PackageKind::Framework, - &FakePackage::new(false, true), - ProvisionMode::Install, - ) - .await; + let row = Fake::new(false, true) + .provision(PackageKind::Framework, ProvisionMode::Install) + .await; assert_eq!(row.status, ProvisionStatus::Failed); assert!(row.error.as_deref().unwrap_or("").contains("404")); } diff --git a/crates/fbuild-build-esp/src/esp32/orchestrator/packages.rs b/crates/fbuild-build-esp/src/esp32/orchestrator/packages.rs index 72a4a4517..b28b27cdc 100644 --- a/crates/fbuild-build-esp/src/esp32/orchestrator/packages.rs +++ b/crates/fbuild-build-esp/src/esp32/orchestrator/packages.rs @@ -564,11 +564,32 @@ async fn provision_sdk_libs( /// The esptool row, or `None` when `platform.json` names no esptool (the build /// then relies on an `esptool` on PATH). +/// +/// `FBUILD_ESPTOOL_PATH` is checked first, as `resolve_esptool` does: a valid +/// override is the tool, and an invalid one fails the build before any +/// metadata is read. async fn provision_esptool( platform: &fbuild_packages::library::Esp32Platform, project_dir: &Path, mode: ProvisionMode, ) -> Option { + let override_row = + |status| ProvisionedPackage::new(PackageKind::Tool, "tool-esptoolpy", status); + match fbuild_packages::library::esptool_path_override() { + Ok(Some(path)) => { + return Some(ProvisionedPackage { + install_path: Some(path.display().to_string()), + ..override_row(ProvisionStatus::Present) + }); + } + Err(error) => { + return Some(ProvisionedPackage { + error: Some(error.to_string()), + ..override_row(ProvisionStatus::Failed) + }); + } + Ok(None) => {} + } let metadata_url = platform.get_package_url("tool-esptoolpy").ok()?; let esptool = fbuild_packages::library::Esptool::from_metadata_url(project_dir, &metadata_url); let started = Instant::now(); diff --git a/crates/fbuild-build-mcu/src/ch32v/mod.rs b/crates/fbuild-build-mcu/src/ch32v/mod.rs index 69ccf308a..41e1819a1 100644 --- a/crates/fbuild-build-mcu/src/ch32v/mod.rs +++ b/crates/fbuild-build-mcu/src/ch32v/mod.rs @@ -24,6 +24,10 @@ impl crate::PlatformSupport for Ch32vPlatformSupport { mode: crate::provision::ProvisionMode, ) -> fbuild_core::Result> { use crate::provision::{PackageKind, provision_package}; + // Reject what the build would reject before downloading for it. + orchestrator::validate_ch32v_framework( + inputs.env_config.get("framework").map(String::as_str), + )?; let (toolchain, cores) = orchestrator::ch32v_packages(inputs.project_dir, Some(inputs.env_config)); Ok(vec![ diff --git a/crates/fbuild-build-mcu/src/ch32v/orchestrator.rs b/crates/fbuild-build-mcu/src/ch32v/orchestrator.rs index 6d971f6bd..869a9cafe 100644 --- a/crates/fbuild-build-mcu/src/ch32v/orchestrator.rs +++ b/crates/fbuild-build-mcu/src/ch32v/orchestrator.rs @@ -399,7 +399,7 @@ pub fn create() -> Box { } /// Only the Arduino framework is implemented for CH32V. -fn validate_ch32v_framework(framework: Option<&str>) -> fbuild_core::Result<()> { +pub(crate) fn validate_ch32v_framework(framework: Option<&str>) -> fbuild_core::Result<()> { match framework.map(str::trim) { None | Some("") | Some("arduino") => Ok(()), Some(other) => Err(fbuild_core::FbuildError::ConfigError(format!( diff --git a/crates/fbuild-library/src/library/esptool.rs b/crates/fbuild-library/src/library/esptool.rs index ae7a377f6..c1573fc52 100644 --- a/crates/fbuild-library/src/library/esptool.rs +++ b/crates/fbuild-library/src/library/esptool.rs @@ -132,6 +132,11 @@ impl Esptool { CacheSubdir::Toolchains, self.project_dir.as_path(), ); + // An interrupted install can hold the executable without the + // completion sentinel; that is not an install. + if !base.is_cached() { + return Ok(None); + } Ok(find_esptool_binary(&base.install_path())) }