perf(esp32): compile the sketch concurrently with the framework core

crates/fbuild-build-esp/src/esp32/orchestrator/build.rs

@@ -738,18 +738,25 @@ impl BuildOrchestrator for Esp32Orchestrator {
                 ),
             }
         }
-        let core_result = {
-            let _g = perf.phase("compile-core-variant");
-            crate::parallel::compile_sources_parallel(
-                &compiler,
-                &all_core_sources,
-                core_build_dir,
-                &user_overlay,
-                jobs,
-                Some(&build_log_mutex),
-            )
-            .await?
-        };
+        // Core and sketch compile concurrently against one shared job gate
+        // (FastLED/fbuild#1537, cause 2); see `compile_phases`.
+        let (core_result, sketch_result) = super::compile_phases::compile_core_and_sketch(
+            &compiler,
+            &mut perf,
+            jobs,
+            super::compile_phases::CompileTarget {
+                sources: &all_core_sources,
+                build_dir: core_build_dir,
+                overlay: &user_overlay,
+            },
+            super::compile_phases::CompileTarget {
+                sources: &sources.sketch_sources,
+                build_dir: src_build_dir,
+                overlay: &src_overlay,
+            },
+            &build_log_mutex,
+        )
+        .await?;
         {
             let _g = perf.phase("core-cache-store");
             let outcome = core_cache.store(core_build_dir);
@@ -777,20 +784,6 @@ impl BuildOrchestrator for Esp32Orchestrator {
             }
         }
 
-        // Compile sketch sources in parallel
-        let sketch_result = {
-            let _g = perf.phase("compile-sketch");
-            crate::parallel::compile_sources_parallel(
-                &compiler,
-                &sources.sketch_sources,
-                src_build_dir,
-                &src_overlay,
-                jobs,
-                Some(&build_log_mutex),
-            )
-            .await?
-        };
-
         // Unwrap build log and flush collected warnings
         let mut build_log = build_log_mutex
             .into_inner()

crates/fbuild-build-esp/src/esp32/orchestrator/compile_phases.rs

@@ -0,0 +1,103 @@
+//! Concurrent compile phases for the ESP32 orchestrator.
+//!
+//! The framework core and the sketch are independent source sets that link
+//! together, so nothing about one depends on the other having finished. They
+//! are compiled against a single shared job gate here rather than one after
+//! the other, which is what this module exists to hold (FastLED/fbuild#1537).
+
+use std::path::Path;
+use std::sync::{Arc, Mutex};
+use std::time::Instant;
+
+use fbuild_core::{BuildLog, Result};
+use tokio::sync::Semaphore;
+
+use crate::compiler::Compiler;
+use crate::flag_overlay::LanguageExtraFlags;
+use crate::parallel::{ParallelCompileResult, compile_sources_parallel_shared};
+use crate::perf_log::PerfTimer;
+
+/// One source set to compile, with the build dir and flag overlay it needs.
+///
+/// Generic over the source element so callers pass their existing `PathBuf`
+/// vectors straight through. `dylints/ban_std_pathbuf` denies naming
+/// `std::path::PathBuf` in a new file while the compile engine's API takes
+/// exactly that, so the slices are materialised at the call below (where the
+/// element type is inferred) instead of in this signature.
+pub(super) struct CompileTarget<'a, S: AsRef<Path>> {
+    /// Sources to compile.
+    pub sources: &'a [S],
+    /// Directory the objects are written to.
+    pub build_dir: &'a Path,
+    /// Per-language extra flags for this source set.
+    pub overlay: &'a LanguageExtraFlags,
+}
+
+/// Compile the framework core sources and the sketch sources concurrently.
+///
+/// Returns `(core, sketch)`. The sketch is submitted first and the gate has
+/// far more permits than the first wave needs, so its single translation unit
+/// starts alongside the core fan-out instead of being scheduled after all of
+/// it. Sharing one gate also keeps the region at `jobs` compilers rather than
+/// letting each phase hold its own full pool.
+///
+/// One [`PerfTimer::phase`] guard covers the region both phases share, so
+/// `compile-core-variant` spans it; the sketch's own span is recorded
+/// separately, so the two entries deliberately overlap.
+pub(super) async fn compile_core_and_sketch<S, T>(
+    compiler: &(dyn Compiler + Send + Sync),
+    perf: &mut PerfTimer,
+    jobs: usize,
+    core: CompileTarget<'_, S>,
+    sketch: CompileTarget<'_, T>,
+    build_log: &Mutex<BuildLog>,
+) -> Result<(ParallelCompileResult, ParallelCompileResult)>
+where
+    S: AsRef<Path> + Send + Sync,
+    T: AsRef<Path> + Send + Sync,
+{
+    let gate = Arc::new(Semaphore::new(jobs.max(1)));
+    let sketch_started = Instant::now();
+
+    let sketch_fut = async {
+        let paths: Vec<_> = sketch
+            .sources
+            .iter()
+            .map(|s| s.as_ref().to_path_buf())
+            .collect();
+        let result = compile_sources_parallel_shared(
+            compiler,
+            &paths,
+            sketch.build_dir,
+            sketch.overlay,
+            &gate,
+            Some(build_log),
+        )
+        .await;
+        (result, sketch_started.elapsed())
+    };
+    let core_fut = async {
+        let paths: Vec<_> = core
+            .sources
+            .iter()
+            .map(|s| s.as_ref().to_path_buf())
+            .collect();
+        compile_sources_parallel_shared(
+            compiler,
+            &paths,
+            core.build_dir,
+            core.overlay,
+            &gate,
+            Some(build_log),
+        )
+        .await
+    };
+
+    let ((sketch_result, sketch_elapsed), core_result) = {
+        let _region = perf.phase("compile-core-variant");
+        tokio::join!(sketch_fut, core_fut)
+    };
+    perf.record("compile-sketch", sketch_elapsed);
+
+    Ok((core_result?, sketch_result?))
+}

crates/fbuild-build-esp/src/esp32/orchestrator/mod.rs

@@ -25,6 +25,7 @@
 mod boot_artifacts;
 mod build;
 mod cdc;
+mod compile_phases;
 mod embed;
 mod embed_stage;
 mod fingerprint;