diff --git a/Cargo.lock b/Cargo.lock index fcb49da3..52c5e1cc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1030,7 +1030,7 @@ checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" [[package]] name = "fbuild-bench-fastled-examples" -version = "2.5.24" +version = "2.5.25" dependencies = [ "fbuild-core", "fbuild-library-select", @@ -1044,7 +1044,7 @@ dependencies = [ [[package]] name = "fbuild-build" -version = "2.5.24" +version = "2.5.25" dependencies = [ "async-trait", "blake3", @@ -1081,7 +1081,7 @@ dependencies = [ [[package]] name = "fbuild-build-arm" -version = "2.5.24" +version = "2.5.25" dependencies = [ "async-trait", "blake3", @@ -1115,7 +1115,7 @@ dependencies = [ [[package]] name = "fbuild-build-engine" -version = "2.5.24" +version = "2.5.25" dependencies = [ "async-trait", "blake3", @@ -1148,7 +1148,7 @@ dependencies = [ [[package]] name = "fbuild-build-esp" -version = "2.5.24" +version = "2.5.25" dependencies = [ "async-trait", "blake3", @@ -1182,7 +1182,7 @@ dependencies = [ [[package]] name = "fbuild-build-mcu" -version = "2.5.24" +version = "2.5.25" dependencies = [ "async-trait", "blake3", @@ -1216,7 +1216,7 @@ dependencies = [ [[package]] name = "fbuild-cli" -version = "2.5.24" +version = "2.5.25" dependencies = [ "blake3", "clap", @@ -1248,7 +1248,7 @@ dependencies = [ [[package]] name = "fbuild-config" -version = "2.5.24" +version = "2.5.25" dependencies = [ "fbuild-core", "fbuild-paths", @@ -1263,7 +1263,7 @@ dependencies = [ [[package]] name = "fbuild-core" -version = "2.5.24" +version = "2.5.25" dependencies = [ "async-trait", "fs2", @@ -1291,7 +1291,7 @@ dependencies = [ [[package]] name = "fbuild-daemon" -version = "2.5.24" +version = "2.5.25" dependencies = [ "async-trait", "axum 0.7.9", @@ -1328,7 +1328,7 @@ dependencies = [ [[package]] name = "fbuild-deploy" -version = "2.5.24" +version = "2.5.25" dependencies = [ "async-trait", "espflash", @@ -1356,7 +1356,7 @@ dependencies = [ [[package]] name = "fbuild-header-scan" -version = "2.5.24" +version = "2.5.25" dependencies = [ "criterion", "fbuild-paths", @@ -1367,7 +1367,7 @@ dependencies = [ [[package]] name = "fbuild-library" -version = "2.5.24" +version = "2.5.25" dependencies = [ "async-trait", "axum 0.7.9", @@ -1397,7 +1397,7 @@ dependencies = [ [[package]] name = "fbuild-library-select" -version = "2.5.24" +version = "2.5.25" dependencies = [ "bincode", "blake3", @@ -1418,7 +1418,7 @@ dependencies = [ [[package]] name = "fbuild-packages" -version = "2.5.24" +version = "2.5.25" dependencies = [ "async-trait", "axum 0.7.9", @@ -1450,7 +1450,7 @@ dependencies = [ [[package]] name = "fbuild-packages-fetch" -version = "2.5.24" +version = "2.5.25" dependencies = [ "async-trait", "axum 0.7.9", @@ -1479,7 +1479,7 @@ dependencies = [ [[package]] name = "fbuild-paths" -version = "2.5.24" +version = "2.5.25" dependencies = [ "blake3", "fbuild-core", @@ -1491,7 +1491,7 @@ dependencies = [ [[package]] name = "fbuild-python" -version = "2.5.24" +version = "2.5.25" dependencies = [ "base64", "fbuild-core", @@ -1513,7 +1513,7 @@ dependencies = [ [[package]] name = "fbuild-serial" -version = "2.5.24" +version = "2.5.25" dependencies = [ "async-trait", "base64", @@ -1536,7 +1536,7 @@ dependencies = [ [[package]] name = "fbuild-test-support" -version = "2.5.24" +version = "2.5.25" dependencies = [ "fbuild-config", "fbuild-core", @@ -1556,7 +1556,7 @@ dependencies = [ [[package]] name = "fbuild-toolchain" -version = "2.5.24" +version = "2.5.25" dependencies = [ "async-trait", "axum 0.7.9", diff --git a/Cargo.toml b/Cargo.toml index 45124746..11400f14 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -93,7 +93,7 @@ targets = [ ] [workspace.package] -version = "2.5.24" +version = "2.5.25" edition = "2021" rust-version = "1.95.0" license = "AGPL-3.0-only" diff --git a/crates/fbuild-build-engine/src/framework_libs.rs b/crates/fbuild-build-engine/src/framework_libs.rs index 9643abcd..c707501d 100644 --- a/crates/fbuild-build-engine/src/framework_libs.rs +++ b/crates/fbuild-build-engine/src/framework_libs.rs @@ -61,10 +61,63 @@ pub fn resolve_framework_library_sources_active_declared( defines: &HashMap, declared: &[String], ) -> Vec { + resolve_framework_library_selection_active_declared( + libraries, + project_dir, + src_dir, + defines, + declared, + ) + .source_files +} + +/// Resolve the selected framework-library records using active branches and +/// explicit declarations. +/// +/// Most orchestrators only need the flattened source list. ESP32 also needs +/// the selected include roots and library names so it can retain its one-archive +/// per library layout without compiling every bundled Arduino library. +pub fn resolve_framework_library_selection_active_declared( + libraries: &[FrameworkLibrary], + project_dir: &Path, + src_dir: &Path, + defines: &HashMap, + declared: &[String], +) -> fbuild_library_select::Selection { + resolve_framework_library_selection_active_declared_with_extra( + libraries, + project_dir, + src_dir, + defines, + declared, + &[], + &[], + ) +} + +/// Active framework selection with additional translation-unit seeds and +/// include roots supplied by externally declared libraries. +/// +/// An external library can include a framework header from one of its own +/// `.cpp` files. The compiler sees that dependency, so the LDF must see it as +/// well or the selected framework archive is omitted from the final link. +pub fn resolve_framework_library_selection_active_declared_with_extra( + libraries: &[FrameworkLibrary], + project_dir: &Path, + src_dir: &Path, + defines: &HashMap, + declared: &[String], + extra_source_files: &[PathBuf], + extra_include_dirs: &[PathBuf], +) -> fbuild_library_select::Selection { let roots = framework_include_scan_roots(project_dir, src_dir); let filtered = filter_framework_libs_shadowed_by_project(libraries, &roots); - let seeds = collect_project_seeds(&roots); - let search_paths = project_search_paths(&roots); + let mut seeds = collect_project_seeds(&roots); + seeds.extend_from_slice(extra_source_files); + let mut search_paths = project_search_paths(&roots); + for include_dir in extra_include_dirs { + push_existing_unique(&mut search_paths, include_dir.clone()); + } fbuild_library_select::resolve_with_stats_active_declared( &seeds, &search_paths, @@ -73,7 +126,6 @@ pub fn resolve_framework_library_sources_active_declared( declared, ) .0 - .source_files } /// Warn when a project sets `lib_ldf_mode`, which fbuild does not implement. diff --git a/crates/fbuild-build-engine/src/framework_libs_tests.rs b/crates/fbuild-build-engine/src/framework_libs_tests.rs index f04c7c7c..2b90b815 100644 --- a/crates/fbuild-build-engine/src/framework_libs_tests.rs +++ b/crates/fbuild-build-engine/src/framework_libs_tests.rs @@ -686,3 +686,94 @@ fn cached_resolution_round_trips_through_file_store() { assert!(hit_second, "second call must hit the cache"); assert_eq!(first, second, "cache hit must yield identical sources"); } + +#[test] +fn active_selection_excludes_unreached_matter_library() { + // Regression guard for #1449: ESP32's framework library compiler must be + // able to receive the selected library records, not just a flattened source + // list, so it never compiles Arduino Matter for a Blink-like sketch. + let tmp = tempfile::TempDir::new().unwrap(); + let project_dir = tmp.path().join("project"); + let src_dir = project_dir.join("src"); + std::fs::create_dir_all(&src_dir).unwrap(); + std::fs::write(src_dir.join("main.cpp"), "#include \n").unwrap(); + + let wifi_dir = tmp.path().join("framework").join("libraries").join("WiFi"); + std::fs::create_dir_all(&wifi_dir).unwrap(); + std::fs::write(wifi_dir.join("WiFi.h"), "").unwrap(); + std::fs::write(wifi_dir.join("WiFi.cpp"), "int wifi;\n").unwrap(); + + let matter_dir = tmp + .path() + .join("framework") + .join("libraries") + .join("Matter"); + std::fs::create_dir_all(&matter_dir).unwrap(); + std::fs::write(matter_dir.join("Matter.h"), "").unwrap(); + std::fs::write(matter_dir.join("Matter.cpp"), "int matter;\n").unwrap(); + + let libraries = vec![ + FrameworkLibrary { + name: "Matter".to_string(), + dir: matter_dir.clone(), + include_dirs: vec![matter_dir.clone()], + source_files: vec![matter_dir.join("Matter.cpp")], + }, + FrameworkLibrary { + name: "WiFi".to_string(), + dir: wifi_dir.clone(), + include_dirs: vec![wifi_dir.clone()], + source_files: vec![wifi_dir.join("WiFi.cpp")], + }, + ]; + + let selection = resolve_framework_library_selection_active_declared( + &libraries, + &project_dir, + &src_dir, + &HashMap::new(), + &[], + ); + + assert_eq!(selection.required_libraries, vec!["WiFi"]); + assert_eq!(selection.source_files, vec![wifi_dir.join("WiFi.cpp")]); + assert_eq!(selection.include_dirs, vec![wifi_dir]); +} + +#[test] +fn external_library_source_selects_framework_dependency() { + // An external lib is compiled from its own .cpp files, so those files must + // seed ESP32's LDF pass or a dependency is omitted at link time. + let tmp = tempfile::TempDir::new().unwrap(); + let project_dir = tmp.path().join("project"); + let src_dir = project_dir.join("src"); + std::fs::create_dir_all(&src_dir).unwrap(); + std::fs::write(src_dir.join("main.cpp"), "void setup() {}\n").unwrap(); + + let external_dir = tmp.path().join("external").join("src"); + std::fs::create_dir_all(&external_dir).unwrap(); + let external_source = external_dir.join("transport.cpp"); + std::fs::write(&external_source, "#include \n").unwrap(); + + let wifi_dir = tmp.path().join("framework").join("libraries").join("WiFi"); + std::fs::create_dir_all(&wifi_dir).unwrap(); + std::fs::write(wifi_dir.join("WiFi.h"), "").unwrap(); + std::fs::write(wifi_dir.join("WiFi.cpp"), "int wifi;\n").unwrap(); + + let selection = resolve_framework_library_selection_active_declared_with_extra( + &[FrameworkLibrary { + name: "WiFi".to_string(), + dir: wifi_dir.clone(), + include_dirs: vec![wifi_dir.clone()], + source_files: vec![wifi_dir.join("WiFi.cpp")], + }], + &project_dir, + &src_dir, + &HashMap::new(), + &[], + &[external_source], + &[external_dir], + ); + + assert_eq!(selection.required_libraries, vec!["WiFi"]); +} diff --git a/crates/fbuild-build-esp/src/esp32/orchestrator/build.rs b/crates/fbuild-build-esp/src/esp32/orchestrator/build.rs index 5ae32286..11e2b56b 100644 --- a/crates/fbuild-build-esp/src/esp32/orchestrator/build.rs +++ b/crates/fbuild-build-esp/src/esp32/orchestrator/build.rs @@ -5,6 +5,7 @@ use std::time::Instant; +use fbuild_build_engine::framework_libs::resolve_framework_library_selection_active_declared_with_extra; use fbuild_core::{Platform, Result}; use fbuild_packages::Framework; @@ -17,7 +18,9 @@ use super::cdc::warn_if_cdc_on_boot; use super::embed_stage::stage_embed_files; use super::fingerprint::Esp32FingerprintMetadata; use super::framework_libs::compile_framework_builtin_libs; -use super::helpers::{compile_db_is_current, framework_macro_prefix_map, profile_label}; +use super::helpers::{ + apply_effective_define_flags, compile_db_is_current, framework_macro_prefix_map, profile_label, +}; use super::local_libs::compile_local_libraries; use super::packages::resolve_pioarduino_packages; @@ -260,21 +263,7 @@ impl BuildOrchestrator for Esp32Orchestrator { include_dirs .extend(framework.get_sdk_include_dirs(&sdk_variant, sdk_memory_type.as_deref())); - // Add built-in Arduino library includes (Wire, SPI, WiFi, etc.) let builtin_libs_dir = framework.get_libraries_dir(); - if builtin_libs_dir.is_dir() { - if let Ok(entries) = std::fs::read_dir(&builtin_libs_dir) { - for entry in entries.flatten() { - let path = entry.path(); - if path.is_dir() { - let lib_src = path.join("src"); - if lib_src.is_dir() { - include_dirs.push(lib_src); - } - } - } - } - } include_dirs.push(ctx.src_dir.clone()); crate::pipeline::discover_project_includes(¶ms.project_dir, &mut include_dirs); @@ -301,12 +290,13 @@ impl BuildOrchestrator for Esp32Orchestrator { // already on the include path above and compiled with the framework, // so drop them rather than sending them to the registry, which does // not carry them (FastLED/fbuild#1442). RP2040 does the same. - let lib_deps = fbuild_library_select::external_declared_deps( - &ctx.config.get_lib_deps(¶ms.env_name)?, - &fbuild_packages::library::framework_library::discover_framework_libraries( + let declared_lib_deps = ctx.config.get_lib_deps(¶ms.env_name)?; + let framework_libraries = + fbuild_packages::library::framework_library::discover_framework_libraries( &builtin_libs_dir, - ), - ); + ); + let lib_deps = + fbuild_library_select::external_declared_deps(&declared_lib_deps, &framework_libraries); let lib_ignore = ctx.config.get_lib_ignore(¶ms.env_name)?; use fbuild_packages::Toolchain; @@ -314,7 +304,7 @@ impl BuildOrchestrator for Esp32Orchestrator { // Read user build_flags early — needed for both library and sketch compilation. // SDK defines (from flags/defines) are prepended so user flags can override them. - let mut user_flags = sdk_defines; + let mut user_flags = sdk_defines.clone(); // Before the user's build_flags, so their own prefix maps still win. user_flags.extend(framework_macro_prefix_map(&core_dir)); let mut user_build_flags = ctx.config.get_build_flags(¶ms.env_name)?; @@ -333,6 +323,19 @@ impl BuildOrchestrator for Esp32Orchestrator { ); crate::warn_debug_build_flags(&user_build_flags); + // External libraries compile before their framework dependencies are + // selected. Give that compilation all bundled include roots, then seed + // the later LDF pass with its source files to select only the archives + // that external code actually reaches. + let mut external_base_includes = include_dirs.clone(); + external_base_includes.extend( + framework_libraries + .iter() + .flat_map(|library| library.include_dirs.iter().cloned()), + ); + let mut external_library_sources = Vec::new(); + let mut external_library_include_dirs = Vec::new(); + if !lib_deps.is_empty() { let libs_dir = build_dir.join("libs"); @@ -346,7 +349,7 @@ impl BuildOrchestrator for Esp32Orchestrator { mcu_config.clone(), &ctx.board.f_cpu, defines.clone(), - include_dirs.clone(), + external_base_includes.clone(), params.profile, params.verbose, build_dir.join("tmp"), @@ -377,7 +380,7 @@ impl BuildOrchestrator for Esp32Orchestrator { dep_lib_ar_path, &c_flags, &cpp_flags, - &include_dirs, + &external_base_includes, ¶ms.project_dir, &libs_dir, params.verbose, @@ -393,7 +396,9 @@ impl BuildOrchestrator for Esp32Orchestrator { // `-I` flags and changes each TU's zccache context key, defeating // cross-project cache hits. Library includes are same-tier, so a // stable sort is safe for include resolution. - let mut lib_include_dirs = lib_result.include_dirs; + external_library_sources = lib_result.source_files; + external_library_include_dirs = lib_result.include_dirs; + let mut lib_include_dirs = external_library_include_dirs.clone(); lib_include_dirs.sort(); include_dirs.extend(lib_include_dirs); library_archives = lib_result.archives; @@ -405,6 +410,44 @@ impl BuildOrchestrator for Esp32Orchestrator { ); } + // Unlike section GC, Arduino's Matter archive can retain global roots + // which pull its entire Wi-Fi/BLE stack into an otherwise empty sketch. + // Select only active, reachable framework libraries (or explicit + // `lib_deps`) after external library translation units are available. + let mut library_selection_defines = ctx.board.get_defines(); + library_selection_defines.extend(mcu_config.defines_map()); + apply_effective_define_flags( + &mut library_selection_defines, + &sdk_defines, + &user_build_flags, + &ctx.build_unflags, + ); + let framework_selection = resolve_framework_library_selection_active_declared_with_extra( + &framework_libraries, + ¶ms.project_dir, + &ctx.src_dir, + &library_selection_defines, + &declared_lib_deps, + &external_library_sources, + &external_library_include_dirs, + ); + let selected_framework_libraries: Vec<_> = framework_libraries + .iter() + .filter(|library| { + framework_selection + .required_libraries + .iter() + .any(|name| name == &library.name) + }) + .cloned() + .collect(); + tracing::info!( + libraries = ?framework_selection.required_libraries, + "ESP32 framework libraries selected: {}", + selected_framework_libraries.len() + ); + include_dirs.extend(framework_selection.include_dirs); + // 8.5b. Project-as-library compilation — shared with sequential pipeline. // When the project root contains library.json or library.properties (e.g., FastLED), // the project's own src/ directory is compiled as a library archive so that example @@ -482,8 +525,7 @@ impl BuildOrchestrator for Esp32Orchestrator { tracing::info!("include paths: {} total", include_dirs.len()); - // 8.6. Compile framework built-in libraries (WiFi, FS, SPIFFS, Network, etc.) - // The linker's --gc-sections will strip any unused code. + // 8.6. Compile the active LDF-selected framework libraries. // Skip when only generating compile_commands.json. if !params.compiledb_only { compile_framework_builtin_libs( @@ -499,6 +541,7 @@ impl BuildOrchestrator for Esp32Orchestrator { &user_overlay, build_dir, compiler_cache.as_deref(), + &selected_framework_libraries, &mut library_archives, &mut ctx.build_log, ) diff --git a/crates/fbuild-build-esp/src/esp32/orchestrator/framework_libs.rs b/crates/fbuild-build-esp/src/esp32/orchestrator/framework_libs.rs index 5e4ff4fd..dc56e4f4 100644 --- a/crates/fbuild-build-esp/src/esp32/orchestrator/framework_libs.rs +++ b/crates/fbuild-build-esp/src/esp32/orchestrator/framework_libs.rs @@ -1,6 +1,10 @@ -//! Compile framework built-in libraries (WiFi, FS, SPIFFS, Network, etc.) -//! shipped under `framework/libraries//src/`. Linker `--gc-sections` -//! strips unused code, so we err on the side of compiling everything. +//! Compile selected framework built-in libraries (WiFi, FS, SPIFFS, Network, +//! etc.) shipped under `framework/libraries//src/`. +//! +//! The ESP32 linker cannot reliably garbage-collect every bundled Arduino +//! library: Matter carries global roots that pull its full networking stack into +//! otherwise empty sketches. Callers must therefore pass only the libraries +//! selected by the active-branch LDF resolver (FastLED/fbuild#1449). use std::path::{Path, PathBuf}; use std::time::Instant; @@ -19,7 +23,7 @@ use crate::BuildParams; use crate::compiler::Compiler as _; use crate::flag_overlay::{LanguageExtraFlags, apply_overlay_flags}; -/// Compile every Arduino built-in library shipped with the ESP32 framework. +/// Compile LDF-selected Arduino framework libraries for ESP32. /// Library archives are appended to `library_archives`. #[allow(clippy::too_many_arguments)] pub(super) async fn compile_framework_builtin_libs( @@ -35,6 +39,7 @@ pub(super) async fn compile_framework_builtin_libs( user_overlay: &LanguageExtraFlags, build_dir: &Path, compiler_cache: Option<&Path>, + selected_libraries: &[fbuild_packages::library::FrameworkLibrary], library_archives: &mut Vec, build_log: &mut fbuild_core::BuildLog, ) -> Result<()> { @@ -128,158 +133,138 @@ pub(super) async fn compile_framework_builtin_libs( let mut fw_lib_stored = 0; let mut fw_lib_count = 0; let mut fw_lib_seen = 0; - if let Ok(entries) = std::fs::read_dir(&builtin_libs_dir) { - for entry in entries.flatten() { - let path = entry.path(); - if !path.is_dir() { - continue; - } - let lib_name = path - .file_name() - .unwrap_or_default() - .to_string_lossy() - .to_lowercase(); - if lib_name.starts_with('.') || already_compiled.contains(&lib_name) { - continue; - } + for library in selected_libraries { + let lib_name = library.name.to_lowercase(); + if already_compiled.contains(&lib_name) { + continue; + } + + fw_lib_seen += 1; - let lib_src = path.join("src"); - if !lib_src.is_dir() { - continue; + // Check if archive already exists + let archive_path = fw_libs_build_dir.join(format!("lib{}.a", lib_name)); + if archive_path.exists() { + if perf.is_active() { + perf.checkpoint(format!( + "fw-lib-cache-hit name={} index={}", + lib_name, fw_lib_seen + )); } + library_archives.push(archive_path); + fw_lib_count += 1; + continue; + } - fw_lib_seen += 1; + let sources = &library.source_files; + if sources.is_empty() { + continue; + } + let failure_marker = framework_failure_marker(&fw_libs_build_dir, &lib_name); + if should_skip_failed_framework_lib(&failure_marker, &fw_signature, sources)? { + if perf.is_active() { + perf.checkpoint(format!( + "fw-lib-skip-failed name={} index={} sources={}", + lib_name, + fw_lib_seen, + sources.len() + )); + } + tracing::debug!( + "skipping previously failed framework library '{}'", + lib_name + ); + continue; + } + if framework_cache.has_failed(&lib_name) { + if perf.is_active() { + perf.checkpoint(format!( + "fw-lib-cache-skip-failed name={} index={}", + lib_name, fw_lib_seen + )); + } + continue; + } - // Check if archive already exists - let archive_path = fw_libs_build_dir.join(format!("lib{}.a", lib_name)); - if archive_path.exists() { + let fw_jobs = crate::parallel::effective_jobs(params.jobs); + if perf.is_active() { + perf.checkpoint(format!( + "fw-lib-compile-start name={} index={} sources={} jobs={}", + lib_name, + fw_lib_seen, + sources.len(), + fw_jobs + )); + } + // Use gcc-ar for LTO archives so the linker-plugin index is written. + let fw_ar_path = toolchain.get_ar_path(); + let fw_gcc_ar_path = toolchain.get_gcc_ar_path(); + let fw_lib_ar_path = crate::pipeline::pick_archiver( + &fw_ar_path, + &fw_gcc_ar_path, + &fw_c_flags, + &fw_cpp_flags, + ); + match fbuild_packages::library::library_compiler::compile_library_with_jobs( + &lib_name, + sources, + include_dirs, + &toolchain.get_gcc_path(), + &toolchain.get_gxx_path(), + fw_lib_ar_path, + &fw_c_flags, + &fw_cpp_flags, + &fw_libs_build_dir, + params.verbose, + fw_jobs, + compiler_cache, + fw_compile_cwd.clone(), + Some(lib_backend.clone()), + ) + .await + { + Ok(Some(archive)) => { + let _ = std::fs::remove_file(&failure_marker); + match framework_cache.store_archive(&archive) { + Ok(()) => fw_lib_stored += 1, + Err(error) => { + tracing::warn!("failed to cache framework library {}: {}", lib_name, error) + } + } + library_archives.push(archive); + fw_lib_count += 1; if perf.is_active() { perf.checkpoint(format!( - "fw-lib-cache-hit name={} index={}", - lib_name, fw_lib_seen + "fw-lib-compile-finish name={} index={} count={}", + lib_name, fw_lib_seen, fw_lib_count )); } - library_archives.push(archive_path); - fw_lib_count += 1; - continue; } - - // Collect source files - let lib_info = - fbuild_packages::library::library_info::InstalledLibrary::new(&path, &lib_name); - let sources = lib_info.get_source_files(); - if sources.is_empty() { - continue; - } - let failure_marker = framework_failure_marker(&fw_libs_build_dir, &lib_name); - if should_skip_failed_framework_lib(&failure_marker, &fw_signature, &sources)? { + Ok(None) => { if perf.is_active() { perf.checkpoint(format!( - "fw-lib-skip-failed name={} index={} sources={}", - lib_name, - fw_lib_seen, - sources.len() + "fw-lib-header-only name={} index={}", + lib_name, fw_lib_seen )); } - tracing::debug!( - "skipping previously failed framework library '{}'", - lib_name - ); - continue; } - if framework_cache.has_failed(&lib_name) { + Err(e) => { + // Non-fatal: some framework libs may fail to compile + // (e.g., platform-specific ones). The linker will report + // if any actually-needed symbols are missing. if perf.is_active() { perf.checkpoint(format!( - "fw-lib-cache-skip-failed name={} index={}", + "fw-lib-compile-error name={} index={}", lib_name, fw_lib_seen )); } - continue; - } - - let fw_jobs = crate::parallel::effective_jobs(params.jobs); - if perf.is_active() { - perf.checkpoint(format!( - "fw-lib-compile-start name={} index={} sources={} jobs={}", - lib_name, - fw_lib_seen, - sources.len(), - fw_jobs - )); - } - // Use gcc-ar for LTO archives so the linker-plugin index is written. - let fw_ar_path = toolchain.get_ar_path(); - let fw_gcc_ar_path = toolchain.get_gcc_ar_path(); - let fw_lib_ar_path = crate::pipeline::pick_archiver( - &fw_ar_path, - &fw_gcc_ar_path, - &fw_c_flags, - &fw_cpp_flags, - ); - match fbuild_packages::library::library_compiler::compile_library_with_jobs( - &lib_name, - &sources, - include_dirs, - &toolchain.get_gcc_path(), - &toolchain.get_gxx_path(), - fw_lib_ar_path, - &fw_c_flags, - &fw_cpp_flags, - &fw_libs_build_dir, - params.verbose, - fw_jobs, - compiler_cache, - fw_compile_cwd.clone(), - Some(lib_backend.clone()), - ) - .await - { - Ok(Some(archive)) => { - let _ = std::fs::remove_file(&failure_marker); - match framework_cache.store_archive(&archive) { - Ok(()) => fw_lib_stored += 1, - Err(error) => tracing::warn!( - "failed to cache framework library {}: {}", - lib_name, - error - ), - } - library_archives.push(archive); - fw_lib_count += 1; - if perf.is_active() { - perf.checkpoint(format!( - "fw-lib-compile-finish name={} index={} count={}", - lib_name, fw_lib_seen, fw_lib_count - )); - } - } - Ok(None) => { - if perf.is_active() { - perf.checkpoint(format!( - "fw-lib-header-only name={} index={}", - lib_name, fw_lib_seen - )); - } - } - Err(e) => { - // Non-fatal: some framework libs may fail to compile - // (e.g., platform-specific ones). The linker will report - // if any actually-needed symbols are missing. - if perf.is_active() { - perf.checkpoint(format!( - "fw-lib-compile-error name={} index={}", - lib_name, fw_lib_seen - )); - } - tracing::debug!("framework library {} failed to compile: {}", lib_name, e); - record_failed_framework_lib(&failure_marker, &fw_signature, &e.to_string()); - if let Err(error) = framework_cache.record_failure(&lib_name) { - tracing::warn!( - "failed to cache framework library failure {}: {}", - lib_name, - error - ); - } + tracing::debug!("framework library {} failed to compile: {}", lib_name, e); + record_failed_framework_lib(&failure_marker, &fw_signature, &e.to_string()); + if let Err(error) = framework_cache.record_failure(&lib_name) { + tracing::warn!( + "failed to cache framework library failure {}: {}", + lib_name, + error + ); } } } diff --git a/crates/fbuild-build-esp/src/esp32/orchestrator/helpers.rs b/crates/fbuild-build-esp/src/esp32/orchestrator/helpers.rs index ecda4830..c26dce46 100644 --- a/crates/fbuild-build-esp/src/esp32/orchestrator/helpers.rs +++ b/crates/fbuild-build-esp/src/esp32/orchestrator/helpers.rs @@ -8,10 +8,59 @@ //! overlay) can reach them without depending on `esp32::orchestrator::helpers`. //! See FastLED/fbuild#587. +use std::collections::HashMap; use std::path::{Path, PathBuf}; use fbuild_core::Result; +/// Apply the effective `-D` / `-U` compiler flags used by library selection. +/// SDK flags are inherited before `build_unflags`; user flags apply afterward +/// unless an exact user token is also unflagged by the compiler. +pub(super) fn apply_effective_define_flags( + defines: &mut HashMap, + sdk_flags: &[String], + user_flags: &[String], + build_unflags: &[String], +) { + apply_define_flags(defines, sdk_flags, build_unflags); + for flag in build_unflags { + if let Some(name) = define_name(flag) { + defines.remove(name); + } + } + apply_define_flags(defines, user_flags, build_unflags); +} + +fn apply_define_flags( + defines: &mut HashMap, + flags: &[String], + build_unflags: &[String], +) { + for flag in flags { + if build_unflags.contains(flag) { + continue; + } + if let Some((name, value)) = define_value(flag) { + defines.insert(name.to_string(), value.to_string()); + } else if let Some(name) = flag.strip_prefix("-U") { + defines.remove(name.trim()); + } + } +} + +fn define_name(flag: &str) -> Option<&str> { + let raw = flag.strip_prefix("-D")?; + let name = raw.split_once('=').map_or(raw, |(name, _)| name).trim(); + (!name.is_empty()).then_some(name) +} + +fn define_value(flag: &str) -> Option<(&str, &str)> { + let raw = flag.strip_prefix("-D")?.trim(); + let (name, value) = raw.split_once('=').unwrap_or((raw, "1")); + let name = name.trim(); + (!name.is_empty()).then_some((name, value)) +} + pub(super) fn framework_failure_marker(build_dir: &Path, lib_name: &str) -> PathBuf { build_dir.join(format!(".{lib_name}.failed")) } diff --git a/crates/fbuild-build-esp/src/esp32/orchestrator/tests.rs b/crates/fbuild-build-esp/src/esp32/orchestrator/tests.rs index 9d81fa36..0e05182e 100644 --- a/crates/fbuild-build-esp/src/esp32/orchestrator/tests.rs +++ b/crates/fbuild-build-esp/src/esp32/orchestrator/tests.rs @@ -2,6 +2,7 @@ use super::Esp32Orchestrator; use super::cdc::{cdc_on_boot_enabled, is_esp32_project, warn_if_cdc_on_boot}; +use super::helpers::apply_effective_define_flags; use super::helpers::{ framework_failure_marker, framework_signature, record_failed_framework_lib, should_skip_failed_framework_lib, @@ -190,3 +191,40 @@ fn test_retry_failed_framework_lib_after_signature_change() { assert!(!should_skip_failed_framework_lib(&marker, &sig_b, &[source]).unwrap()); } + +#[test] +fn effective_define_flags_match_compiler_overlay_order() { + let mut defines = std::collections::HashMap::from([ + ("BOARD_ONLY".to_string(), "1".to_string()), + ("DISABLED_BY_UNFLAG".to_string(), "1".to_string()), + ]); + let sdk_flags = vec![ + "-DDISABLED_BY_UNFLAG".to_string(), + "-DENABLE_WIFI".to_string(), + ]; + let user_flags = vec![ + "-DDISABLED_BY_UNFLAG=0".to_string(), + "-DREMOVED_USER_DEFINE".to_string(), + "-DVALUE=42".to_string(), + "-UVALUE".to_string(), + ]; + let build_unflags = vec![ + "-DDISABLED_BY_UNFLAG".to_string(), + "-DREMOVED_USER_DEFINE".to_string(), + ]; + + apply_effective_define_flags(&mut defines, &sdk_flags, &user_flags, &build_unflags); + + assert_eq!(defines.get("BOARD_ONLY"), Some(&"1".to_string())); + assert_eq!(defines.get("ENABLE_WIFI"), Some(&"1".to_string())); + assert_eq!( + defines.get("DISABLED_BY_UNFLAG"), + Some(&"0".to_string()), + "user flags must override an unflagged SDK definition" + ); + assert!( + !defines.contains_key("REMOVED_USER_DEFINE"), + "an exact user flag listed in build_unflags must remain removed" + ); + assert!(!defines.contains_key("VALUE")); +} diff --git a/crates/fbuild-library/src/library/library_manager.rs b/crates/fbuild-library/src/library/library_manager.rs index 558af8ca..9975a26b 100644 --- a/crates/fbuild-library/src/library/library_manager.rs +++ b/crates/fbuild-library/src/library/library_manager.rs @@ -65,6 +65,11 @@ fn resolve_local_library_dir(project_dir: &Path, local_path: &Path, name: &str) pub struct LibraryResult { /// All include directories from all libraries (for compiler `-I` flags). pub include_dirs: Vec, + /// Translation units compiled into the returned archives. + /// + /// Platform orchestrators use these as LDF seeds when an external library + /// includes a framework-bundled header. + pub source_files: Vec, /// All compiled library archives (`.a` files) for the linker. pub archives: Vec, } @@ -109,6 +114,7 @@ pub async fn ensure_libraries( if installed.is_empty() { return Ok(LibraryResult { include_dirs: Vec::new(), + source_files: Vec::new(), archives: Vec::new(), }); } @@ -163,9 +169,14 @@ pub async fn ensure_libraries( .iter() .flat_map(|lib| lib.get_include_dirs()) .collect(); + let source_files: Vec = installed + .iter() + .flat_map(|lib| lib.get_source_files()) + .collect(); Ok(LibraryResult { include_dirs: lib_include_dirs, + source_files, archives, }) } diff --git a/pyproject.toml b/pyproject.toml index 213090a6..1704a002 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "fbuild" -version = "2.5.24" +version = "2.5.25" description = "PlatformIO-compatible embedded build tool (Rust implementation)" readme = "README.md" license = "AGPL-3.0-only"