diff --git a/crates/fbuild-build-esp/src/esp32/orchestrator/build.rs b/crates/fbuild-build-esp/src/esp32/orchestrator/build.rs index dcedf4e0..f6c61c6b 100644 --- a/crates/fbuild-build-esp/src/esp32/orchestrator/build.rs +++ b/crates/fbuild-build-esp/src/esp32/orchestrator/build.rs @@ -738,18 +738,25 @@ impl BuildOrchestrator for Esp32Orchestrator { ), } } - let core_result = { - let _g = perf.phase("compile-core-variant"); - crate::parallel::compile_sources_parallel( - &compiler, - &all_core_sources, - core_build_dir, - &user_overlay, - jobs, - Some(&build_log_mutex), - ) - .await? - }; + // Core and sketch compile concurrently against one shared job gate + // (FastLED/fbuild#1537, cause 2); see `compile_phases`. + let (core_result, sketch_result) = super::compile_phases::compile_core_and_sketch( + &compiler, + &mut perf, + jobs, + super::compile_phases::CompileTarget { + sources: &all_core_sources, + build_dir: core_build_dir, + overlay: &user_overlay, + }, + super::compile_phases::CompileTarget { + sources: &sources.sketch_sources, + build_dir: src_build_dir, + overlay: &src_overlay, + }, + &build_log_mutex, + ) + .await?; { let _g = perf.phase("core-cache-store"); let outcome = core_cache.store(core_build_dir); @@ -777,20 +784,6 @@ impl BuildOrchestrator for Esp32Orchestrator { } } - // Compile sketch sources in parallel - let sketch_result = { - let _g = perf.phase("compile-sketch"); - crate::parallel::compile_sources_parallel( - &compiler, - &sources.sketch_sources, - src_build_dir, - &src_overlay, - jobs, - Some(&build_log_mutex), - ) - .await? - }; - // Unwrap build log and flush collected warnings let mut build_log = build_log_mutex .into_inner() diff --git a/crates/fbuild-build-esp/src/esp32/orchestrator/compile_phases.rs b/crates/fbuild-build-esp/src/esp32/orchestrator/compile_phases.rs new file mode 100644 index 00000000..60d69cea --- /dev/null +++ b/crates/fbuild-build-esp/src/esp32/orchestrator/compile_phases.rs @@ -0,0 +1,103 @@ +//! Concurrent compile phases for the ESP32 orchestrator. +//! +//! The framework core and the sketch are independent source sets that link +//! together, so nothing about one depends on the other having finished. They +//! are compiled against a single shared job gate here rather than one after +//! the other, which is what this module exists to hold (FastLED/fbuild#1537). + +use std::path::Path; +use std::sync::{Arc, Mutex}; +use std::time::Instant; + +use fbuild_core::{BuildLog, Result}; +use tokio::sync::Semaphore; + +use crate::compiler::Compiler; +use crate::flag_overlay::LanguageExtraFlags; +use crate::parallel::{ParallelCompileResult, compile_sources_parallel_shared}; +use crate::perf_log::PerfTimer; + +/// One source set to compile, with the build dir and flag overlay it needs. +/// +/// Generic over the source element so callers pass their existing `PathBuf` +/// vectors straight through. `dylints/ban_std_pathbuf` denies naming +/// `std::path::PathBuf` in a new file while the compile engine's API takes +/// exactly that, so the slices are materialised at the call below (where the +/// element type is inferred) instead of in this signature. +pub(super) struct CompileTarget<'a, S: AsRef> { + /// Sources to compile. + pub sources: &'a [S], + /// Directory the objects are written to. + pub build_dir: &'a Path, + /// Per-language extra flags for this source set. + pub overlay: &'a LanguageExtraFlags, +} + +/// Compile the framework core sources and the sketch sources concurrently. +/// +/// Returns `(core, sketch)`. The sketch is submitted first and the gate has +/// far more permits than the first wave needs, so its single translation unit +/// starts alongside the core fan-out instead of being scheduled after all of +/// it. Sharing one gate also keeps the region at `jobs` compilers rather than +/// letting each phase hold its own full pool. +/// +/// One [`PerfTimer::phase`] guard covers the region both phases share, so +/// `compile-core-variant` spans it; the sketch's own span is recorded +/// separately, so the two entries deliberately overlap. +pub(super) async fn compile_core_and_sketch( + compiler: &(dyn Compiler + Send + Sync), + perf: &mut PerfTimer, + jobs: usize, + core: CompileTarget<'_, S>, + sketch: CompileTarget<'_, T>, + build_log: &Mutex, +) -> Result<(ParallelCompileResult, ParallelCompileResult)> +where + S: AsRef + Send + Sync, + T: AsRef + Send + Sync, +{ + let gate = Arc::new(Semaphore::new(jobs.max(1))); + let sketch_started = Instant::now(); + + let sketch_fut = async { + let paths: Vec<_> = sketch + .sources + .iter() + .map(|s| s.as_ref().to_path_buf()) + .collect(); + let result = compile_sources_parallel_shared( + compiler, + &paths, + sketch.build_dir, + sketch.overlay, + &gate, + Some(build_log), + ) + .await; + (result, sketch_started.elapsed()) + }; + let core_fut = async { + let paths: Vec<_> = core + .sources + .iter() + .map(|s| s.as_ref().to_path_buf()) + .collect(); + compile_sources_parallel_shared( + compiler, + &paths, + core.build_dir, + core.overlay, + &gate, + Some(build_log), + ) + .await + }; + + let ((sketch_result, sketch_elapsed), core_result) = { + let _region = perf.phase("compile-core-variant"); + tokio::join!(sketch_fut, core_fut) + }; + perf.record("compile-sketch", sketch_elapsed); + + Ok((core_result?, sketch_result?)) +} diff --git a/crates/fbuild-build-esp/src/esp32/orchestrator/mod.rs b/crates/fbuild-build-esp/src/esp32/orchestrator/mod.rs index 32ca57f8..42226075 100644 --- a/crates/fbuild-build-esp/src/esp32/orchestrator/mod.rs +++ b/crates/fbuild-build-esp/src/esp32/orchestrator/mod.rs @@ -25,6 +25,7 @@ mod boot_artifacts; mod build; mod cdc; +mod compile_phases; mod embed; mod embed_stage; mod fingerprint;