fix(packages): fill or excuse declared submodules so CH32V and SiLabs build again (#1420, #1421, #1422)

crates/fbuild-library/src/library/ch32v_core.rs

@@ -10,6 +10,15 @@ use crate::{CacheSubdir, Framework, PackageBase, PackageInfo};
 const CH32V_CORE_VERSION: &str = "1.0.4+d767162.ch32l103";
 const CH32V_CORE_URL: &str = "https://github.com/openwch/arduino_core_ch32/archive/d76716239cdf8a084a5045c3dfd3151b3f69eeec.tar.gz";
 
+// The core declares `libraries/Adafruit_TinyUSB_Arduino` as a submodule, and
+// GitHub's source archive ships it empty, which the unpack-time check (#1401)
+// rejects. openwch publishes no archive that bundles it, and the pin above
+// postdates every release, so the submodule is fetched on its own at the
+// commit the core's gitlink records (FastLED/fbuild#1420).
+const TINYUSB_SUBMODULE_PATH: &str = "libraries/Adafruit_TinyUSB_Arduino";
+const TINYUSB_URL: &str = "https://github.com/adafruit/Adafruit_TinyUSB_Arduino/archive/1f9da4918f2c05441a9bfc3866a5b9cc03f2da62.tar.gz";
+const TINYUSB_SHA256: &str = "e9ddbe3ac62adc402b33c1015bac0825c0b0bd1bad9a9e13d89183623b164080";
+
 /// OpenWCH CH32V Arduino core framework manager.
 pub struct Ch32vCores {
     base: PackageBase,
@@ -27,7 +36,8 @@ impl Ch32vCores {
                 None,
                 CacheSubdir::Platforms,
                 project_dir,
-            ),
+            )
+            .with_submodule_source(TINYUSB_SUBMODULE_PATH, TINYUSB_URL, TINYUSB_SHA256),
             install_dir: None,
         }
     }
@@ -64,7 +74,8 @@ impl Ch32vCores {
                 CacheSubdir::Platforms,
                 project_dir,
                 cache_root,
-            ),
+            )
+            .with_submodule_source(TINYUSB_SUBMODULE_PATH, TINYUSB_URL, TINYUSB_SHA256),
             install_dir: None,
         }
     }
@@ -214,6 +225,17 @@ mod tests {
         assert_eq!(find_core_root(tmp.path()), nested);
     }
 
+    /// The TinyUSB pin must stay on the path the core's `.gitmodules` declares
+    /// and the commit its gitlink records at `d767162`. Moving the core pin
+    /// means re-reading both from the new commit, not keeping these.
+    #[test]
+    fn test_tinyusb_pin_matches_the_core_gitlink() {
+        assert!(CH32V_CORE_URL.contains("d76716239cdf8a084a5045c3dfd3151b3f69eeec"));
+        assert_eq!(TINYUSB_SUBMODULE_PATH, "libraries/Adafruit_TinyUSB_Arduino");
+        assert!(TINYUSB_URL.contains("/archive/1f9da4918f2c05441a9bfc3866a5b9cc03f2da62."));
+        assert_eq!(TINYUSB_SHA256.len(), 64);
+    }
+
     #[test]
     fn test_validate_missing_cores() {
         let tmp = tempfile::TempDir::new().unwrap();

crates/fbuild-library/src/library/silabs_core.rs

@@ -11,6 +11,15 @@ const SILABS_CORE_VERSION: &str = "2.2.0";
 const SILABS_CORE_URL: &str =
     "https://github.com/SiliconLabs/arduino/archive/refs/tags/2.2.0.tar.gz";
 
+// The core declares `extra/core-api` as a submodule, and the source archive
+// ships it empty. Nothing reads it: `ensure_arduino_api()` installs
+// ArduinoCore-API into `cores/silabs/api` after every install. The release
+// asset that bundles it is 448 MB, so the directory is declared expected-empty
+// instead of fetched (FastLED/fbuild#1421).
+const CORE_API_SUBMODULE_PATH: &str = "extra/core-api";
+const CORE_API_SUBMODULE_REASON: &str =
+    "ArduinoCore-API is installed into cores/silabs/api by ensure_arduino_api()";
+
 /// Silicon Labs Arduino core framework manager.
 pub struct SilabsCores {
     base: PackageBase,
@@ -28,7 +37,8 @@ impl SilabsCores {
                 None,
                 CacheSubdir::Platforms,
                 project_dir,
-            ),
+            )
+            .expect_empty_submodule(CORE_API_SUBMODULE_PATH, CORE_API_SUBMODULE_REASON),
             install_dir: None,
         }
     }
@@ -65,7 +75,8 @@ impl SilabsCores {
                 CacheSubdir::Platforms,
                 project_dir,
                 cache_root,
-            ),
+            )
+            .expect_empty_submodule(CORE_API_SUBMODULE_PATH, CORE_API_SUBMODULE_REASON),
             install_dir: None,
         }
     }

crates/fbuild-packages-fetch/src/lib.rs

@@ -169,6 +169,9 @@ pub struct PackageBase {
     pub cache_subdir: CacheSubdir,
     /// Optional DiskCache for LRU tracking. Best-effort: `None` if SQLite open fails.
     disk_cache: Option<DiskCache>,
+    /// How declared submodules are handled at unpack (FastLED/fbuild#1420,
+    /// #1422). Empty for almost every package.
+    submodules: submodules::SubmodulePlan,
 }
 
 /// Which cache subdirectory to use.
@@ -208,6 +211,7 @@ impl PackageBase {
             cache,
             cache_subdir,
             disk_cache,
+            submodules: submodules::SubmodulePlan::default(),
         }
     }
 
@@ -233,6 +237,7 @@ impl PackageBase {
             cache: Cache::with_cache_root(project_dir, cache_root),
             cache_subdir,
             disk_cache,
+            submodules: submodules::SubmodulePlan::default(),
         }
     }
 
@@ -248,6 +253,9 @@ impl PackageBase {
     /// `checksum: None` skips sha256 verification — consumer-trusted, which is
     /// the right policy for `platform_packages` overrides (#681, sibling of #663).
     ///
+    /// The submodule plan is dropped too: it describes the *default* archive,
+    /// and an override's commit may declare different submodules.
+    ///
     /// Emits a single INFO log so the override is visible in build scrollback
     /// across every framework package, without each orchestrator having to
     /// remember to log it themselves.
@@ -257,6 +265,7 @@ impl PackageBase {
         self.cache_key = ovr.url;
         self.version = ovr.version;
         self.checksum = ovr.checksum;
+        self.submodules = submodules::SubmodulePlan::default();
         self
     }
 
@@ -433,16 +442,9 @@ impl PackageBase {
         // A core whose archive dropped its submodules extracts to something
         // that looks complete. Catch it here rather than letting the compiler
         // report a missing header from inside the core (FastLED/fbuild#1380,
-        // #1400). Checked against the extracted root and one level down,
-        // since most archives nest under a single version directory.
-        for root in submodule_scan_roots(&staging_path) {
-            let empty = submodules::find_empty_submodules(&root);
-            if !empty.is_empty() {
-                return Err(fbuild_core::FbuildError::PackageError(
-                    submodules::empty_submodule_error(&self.name, &self.url, &empty),
-                ));
-            }
-        }
+        // #1400), after filling or excusing what the package's plan covers.
+        submodules::prepare_submodules(&self.name, &self.url, &staging_path, &self.submodules)
+            .await?;
 
         // Validate
         validate(&staging_path)?;
@@ -940,21 +942,3 @@ mod package_override_tests {
         );
     }
 }
-
-/// Where to look for a `.gitmodules` after extraction.
-///
-/// Archives usually nest everything under one directory named for the
-/// version (`esp8266-3.1.2/`), so the repo root is one level down from the
-/// staging dir — but not always. Checking both costs one `read_dir`.
-fn submodule_scan_roots(staging: &Path) -> Vec<PathBuf> {
-    let mut roots = vec![staging.to_path_buf()];
-    if let Ok(entries) = std::fs::read_dir(staging) {
-        for entry in entries.flatten() {
-            let path = entry.path();
-            if path.is_dir() {
-                roots.push(path);
-            }
-        }
-    }
-    roots
-}