From 9bb17540318ccb8360f4d1f2aa3943be4e201a0c Mon Sep 17 00:00:00 2001 From: Zach Vorhies Date: Sun, 27 Sep 2026 15:26:05 -0700 Subject: [PATCH 1/5] fix(ch32v): resolve pinned PlatformIO source packages (#1495) --- .../fbuild-build-mcu/src/ch32v/mcu_config.rs | 41 +- crates/fbuild-build-mcu/src/ch32v/mod.rs | 11 +- .../src/ch32v/orchestrator.rs | 356 +++++++++- .../src/ch32v/platform_source.rs | 671 ++++++++++++++++++ crates/fbuild-build/src/lib.rs | 40 ++ .../tests/platformio_ch32v_resolution.rs | 68 ++ .../fbuild-library/src/library/ch32v_core.rs | 13 + crates/fbuild-packages-fetch/src/lib.rs | 1 + .../src/platformio_repository.rs | 301 ++++++++ .../fbuild-packages-fetch/src/submodules.rs | 284 +++++++- .../fbuild-toolchain/src/toolchain/riscv.rs | 194 ++++- 11 files changed, 1916 insertions(+), 64 deletions(-) create mode 100644 crates/fbuild-build-mcu/src/ch32v/platform_source.rs create mode 100644 crates/fbuild-core/tests/platformio_ch32v_resolution.rs create mode 100644 crates/fbuild-packages-fetch/src/platformio_repository.rs diff --git a/crates/fbuild-build-mcu/src/ch32v/mcu_config.rs b/crates/fbuild-build-mcu/src/ch32v/mcu_config.rs index c8bbecec9..b2e6f3008 100644 --- a/crates/fbuild-build-mcu/src/ch32v/mcu_config.rs +++ b/crates/fbuild-build-mcu/src/ch32v/mcu_config.rs @@ -98,6 +98,17 @@ pub fn normalize_march(march: &str) -> String { /// Apply board ISA and ABI values to compiler and linker flags. pub fn apply_board_isa(config: &mut Ch32vMcuConfig, march: Option<&str>, mabi: Option<&str>) { + apply_board_isa_for_toolchain(config, march, mabi, "riscv-none-elf"); +} + +/// Preserve WCH's vendor ISA extensions for its own compiler; only the xPack +/// compiler requires the stripped/modernized `zicsr` spelling. +pub fn apply_board_isa_for_toolchain( + config: &mut Ch32vMcuConfig, + march: Option<&str>, + mabi: Option<&str>, + executable_prefix: &str, +) { fn replace(flags: &mut [String], prefix: &str, value: &str) { for flag in flags { if flag.starts_with(prefix) { @@ -107,7 +118,11 @@ pub fn apply_board_isa(config: &mut Ch32vMcuConfig, march: Option<&str>, mabi: O } if let Some(march) = march { - let normalized = normalize_march(march); + let normalized = if executable_prefix == "riscv-none-elf" { + normalize_march(march) + } else { + march.to_ascii_lowercase() + }; replace(&mut config.compiler_flags.common, "-march=", &normalized); replace(&mut config.linker_flags, "-march=", &normalized); } @@ -129,6 +144,30 @@ mod tests { assert_eq!(normalize_march("rv32ecxw"), "rv32ec_zicsr"); } + #[test] + fn platformio_gcc8_preserves_wch_vendor_isa() { + let mut config = get_ch32v_config_for_mcu("ch32v003").unwrap(); + apply_board_isa_for_toolchain( + &mut config, + Some("rv32ecxw"), + Some("ilp32e"), + "riscv-none-embed", + ); + assert!( + config + .compiler_flags + .common + .contains(&"-march=rv32ecxw".into()) + ); + assert!(config.linker_flags.contains(&"-march=rv32ecxw".into())); + assert!( + !config + .compiler_flags + .common + .contains(&"-march=rv32ec_zicsr".into()) + ); + } + #[test] fn test_apply_board_isa_updates_compiler_and_linker() { let mut config = get_ch32v_config_for_mcu("ch32v003").unwrap(); diff --git a/crates/fbuild-build-mcu/src/ch32v/mod.rs b/crates/fbuild-build-mcu/src/ch32v/mod.rs index 41e1819a1..6df5c857d 100644 --- a/crates/fbuild-build-mcu/src/ch32v/mod.rs +++ b/crates/fbuild-build-mcu/src/ch32v/mod.rs @@ -1,9 +1,10 @@ -//! CH32V RISC-V platform build support (WCH CH32V003, CH32V203, etc.) +//! CH32V RISC-V platform build support (WCH CH32V003, CH32V203, etc.) pub mod ch32v_compiler; pub mod ch32v_linker; pub mod mcu_config; pub mod orchestrator; +mod platform_source; pub use ch32v_compiler::Ch32vCompiler; pub use ch32v_linker::Ch32vLinker; @@ -28,8 +29,12 @@ impl crate::PlatformSupport for Ch32vPlatformSupport { orchestrator::validate_ch32v_framework( inputs.env_config.get("framework").map(String::as_str), )?; - let (toolchain, cores) = - orchestrator::ch32v_packages(inputs.project_dir, Some(inputs.env_config)); + let (toolchain, cores, _, _) = orchestrator::ch32v_packages( + inputs.project_dir, + Some(inputs.env_config), + &inputs.board.core, + ) + .await?; Ok(vec![ provision_package(PackageKind::Toolchain, &toolchain, mode).await, provision_package(PackageKind::Framework, &cores, mode).await, diff --git a/crates/fbuild-build-mcu/src/ch32v/orchestrator.rs b/crates/fbuild-build-mcu/src/ch32v/orchestrator.rs index bea96038f..57069a664 100644 --- a/crates/fbuild-build-mcu/src/ch32v/orchestrator.rs +++ b/crates/fbuild-build-mcu/src/ch32v/orchestrator.rs @@ -38,6 +38,32 @@ fn profile_label(profile: fbuild_core::BuildProfile) -> &'static str { } } +fn preferred_board_value<'a>( + env: Option<&'a std::collections::HashMap>, + key: &str, + selected: Option<&'a str>, + fallback: Option<&'a str>, +) -> Option<&'a str> { + env.and_then(|values| values.get(key).map(String::as_str)) + .or(selected) + .or(fallback) +} + +fn board_fingerprint_fields( + march: Option<&str>, + mabi: Option<&str>, + variant_h: Option<&str>, +) -> std::collections::BTreeMap { + std::collections::BTreeMap::from([ + ("board_march".into(), march.unwrap_or_default().into()), + ("board_mabi".into(), mabi.unwrap_or_default().into()), + ( + "board_variant_h".into(), + variant_h.unwrap_or_default().into(), + ), + ]) +} + #[async_trait::async_trait] impl BuildOrchestrator for Ch32vOrchestrator { fn platform(&self) -> Platform { @@ -49,26 +75,58 @@ impl BuildOrchestrator for Ch32vOrchestrator { // 1-2. Parse config, load board, setup build dirs, resolve src dir, collect flags let mut ctx = pipeline::BuildContext::new(params).await?; - let framework_name = ctx - .config - .get_env_config(¶ms.env_name) - .ok() + let env_config = ctx.config.get_env_config(¶ms.env_name).ok(); + let framework_name = env_config .and_then(|env| env.get("framework")) .map(String::as_str); validate_ch32v_framework(framework_name)?; // 3-4. RISC-V GCC toolchain and OpenWCH CH32V cores - let (toolchain, framework) = ch32v_packages( - ¶ms.project_dir, - ctx.config.get_env_config(¶ms.env_name).ok(), - ); + let (toolchain, framework, selected_board, platform_resolution) = + ch32v_packages(¶ms.project_dir, env_config, &ctx.board.core).await?; + use fbuild_packages::Package as _; + let toolchain_info = toolchain.get_info(); + let framework_info = framework.get_info(); + if let Some(env) = env_config { + let platform_request = env.get("platform").map(String::as_str).unwrap_or("ch32v"); + let package_request = env + .get("platform_packages") + .map(|value| value.split_whitespace().collect::>().join(", ")) + .unwrap_or_else(|| "(default)".to_string()); + ctx.build_log.push(format!( + "CH32V requested: platform={platform_request}; platform_packages={package_request}" + )); + } + if let Some(platform) = platform_resolution.as_deref() { + ctx.build_log + .push(format!("CH32V resolved platform: {platform}")); + } + ctx.build_log.push(format!( + "CH32V resolved: {}@{} ({}{}); {}@{} ({}{})", + toolchain_info.name, + toolchain_info.version, + toolchain_info.url, + toolchain_info + .checksum + .as_ref() + .map(|sha| format!("; sha256={sha}")) + .unwrap_or_default(), + framework_info.name, + framework_info.version, + framework_info.url, + framework_info + .checksum + .as_ref() + .map(|sha| format!("; sha256={sha}")) + .unwrap_or_default(), + )); let toolchain_dir = fbuild_packages::Package::ensure_installed(&toolchain).await?; tracing::info!("riscv-gcc toolchain at {}", toolchain_dir.display()); use fbuild_packages::Toolchain; pipeline::log_toolchain_version( &toolchain.get_gcc_path(), - "riscv-none-elf-gcc", + "RISC-V GCC", &mut ctx.build_log, ) .await; @@ -99,10 +157,65 @@ impl BuildOrchestrator for Ch32vOrchestrator { let system_series = series_to_system_dir(&series); // 6. Scan sources - let core_dir = framework.get_core_dir(&ctx.board.core); - let variant_dir = resolve_variant_dir(&framework_dir, &ctx.board.variant, &system_series); + let effective_core = selected_board + .as_ref() + .map(|board| board.core.as_str()) + .unwrap_or(&ctx.board.core); + let effective_variant = selected_board + .as_ref() + .map(|board| board.variant.as_str()) + .unwrap_or(&ctx.board.variant); + let effective_variant_h = selected_board + .as_ref() + .and_then(|board| board.variant_h.as_deref()) + .or(ctx.board.variant_h.as_deref()); + let core_dir = framework.get_core_dir(effective_core); + let variant_dir = resolve_variant_dir(&framework_dir, effective_variant, &system_series); + let effective_variant_h = resolve_variant_h(&variant_dir, effective_variant_h); + let selected_march = preferred_board_value( + env_config, + "board_build.march", + selected_board.as_ref().map(|board| board.march.as_str()), + ctx.board.march.as_deref(), + ); + let selected_mabi = preferred_board_value( + env_config, + "board_build.mabi", + selected_board.as_ref().map(|board| board.mabi.as_str()), + ctx.board.mabi.as_deref(), + ); let build_dir = &ctx.build_dir; + let mut fingerprint_extra = std::collections::BTreeMap::from([ + ("series".to_string(), series.clone()), + ("toolchain_url".to_string(), toolchain_info.url.clone()), + ( + "toolchain_version".to_string(), + toolchain_info.version.clone(), + ), + ( + "toolchain_sha256".to_string(), + toolchain_info.checksum.clone().unwrap_or_default(), + ), + ("framework_url".to_string(), framework_info.url.clone()), + ( + "framework_version".to_string(), + framework_info.version.clone(), + ), + ( + "framework_sha256".to_string(), + framework_info.checksum.clone().unwrap_or_default(), + ), + ( + "platform_resolution".to_string(), + platform_resolution.clone().unwrap_or_default(), + ), + ]); + fingerprint_extra.extend(board_fingerprint_fields( + selected_march, + selected_mabi, + effective_variant_h.as_deref(), + )); let metadata_hash = stable_hash_with_build_config( &CoreFingerprintMetadata { version: crate::build_fingerprint::BUILD_FINGERPRINT_VERSION, @@ -111,19 +224,16 @@ impl BuildOrchestrator for Ch32vOrchestrator { board_name: ctx.board.name.clone(), board_mcu: ctx.board.mcu.clone(), board_define: ctx.board.board.clone(), - board_core: ctx.board.core.clone(), + board_core: effective_core.to_string(), board_f_cpu: ctx.board.f_cpu.clone(), board_extra_flags: ctx.board.extra_flags.clone(), board_ldscript: ctx.board.ldscript.clone(), - board_variant: Some(ctx.board.variant.clone()), + board_variant: Some(effective_variant.to_string()), platform: "ch32v".to_string(), max_flash: ctx.board.max_flash, max_ram: ctx.board.max_ram, eh_frame_policy: None, - extra: Some(std::collections::BTreeMap::from([( - "series".to_string(), - series.clone(), - )])), + extra: Some(fingerprint_extra), }, &ctx, )?; @@ -180,10 +290,12 @@ impl BuildOrchestrator for Ch32vOrchestrator { // 7. Build include dirs + compiler let mut mcu_config = super::mcu_config::get_ch32v_config_for_mcu(&series)?; - super::mcu_config::apply_board_isa( + let toolchain_prefix = toolchain.executable_prefix(); + super::mcu_config::apply_board_isa_for_toolchain( &mut mcu_config, - ctx.board.march.as_deref(), - ctx.board.mabi.as_deref(), + selected_march, + selected_mabi, + &toolchain_prefix, ); let mut defines = ctx.board.get_defines(); defines.extend(mcu_config.defines_map()); @@ -195,7 +307,7 @@ impl BuildOrchestrator for Ch32vOrchestrator { )?; defines.insert(sysclk_name, sysclk_value); // CH32V cores use `#include VARIANT_H` — define it from the variant dir - if let Some(vh) = resolve_variant_h(&variant_dir, ctx.board.variant_h.as_deref()) { + if let Some(vh) = effective_variant_h { defines.insert("VARIANT_H".to_string(), format!("\\\"{}\\\"", vh)); } // Use resolved core_dir/variant_dir directly — board.get_include_paths() @@ -379,21 +491,61 @@ impl BuildOrchestrator for Ch32vOrchestrator { /// `framework-arduino-ch32v` `platform_packages` override (FastLED/fbuild#664, /// #681). Shared by the build and `fbuild install`, so both provision the same /// packages (FastLED/fbuild#1433). -pub(crate) fn ch32v_packages( +pub(crate) async fn ch32v_packages( project_dir: &Path, env_config: Option<&std::collections::HashMap>, -) -> ( + board_core: &str, +) -> fbuild_core::Result<( fbuild_packages::toolchain::RiscvToolchain, fbuild_packages::library::Ch32vCores, -) { - let toolchain = fbuild_packages::toolchain::RiscvToolchain::new(project_dir); + Option, + Option, +)> { + if let Some(env) = env_config { + let selected_core = env + .get("board_build.core") + .map(String::as_str) + .unwrap_or(board_core); + if let Some(packages) = + super::platform_source::resolve_source_packages(project_dir, env, selected_core).await? + { + return Ok((packages.0, packages.1, Some(packages.2), Some(packages.3))); + } + } + let registry_overrides = if let Some(env) = env_config { + crate::package_override::resolve_registry_overrides( + project_dir, + env, + "ch32v", + &["toolchain-riscv", "framework-arduino-openwch-ch32"], + &[], + ) + .await? + } else { + std::collections::HashMap::new() + }; + let toolchain = registry_overrides.get("toolchain-riscv").map_or_else( + || fbuild_packages::toolchain::RiscvToolchain::new(project_dir), + |pin| { + fbuild_packages::toolchain::RiscvToolchain::with_platform_override( + project_dir, + pin.clone(), + "auto", + ) + }, + ); let override_pin = env_config - .and_then(|env| crate::package_override::resolve_override(env, "framework-arduino-ch32v")); + .and_then(|env| crate::package_override::resolve_override(env, "framework-arduino-ch32v")) + .or_else(|| { + registry_overrides + .get("framework-arduino-openwch-ch32") + .cloned() + }); let cores = match override_pin { Some(o) => fbuild_packages::library::Ch32vCores::with_override(project_dir, o), None => fbuild_packages::library::Ch32vCores::new(project_dir), }; - (toolchain, cores) + Ok((toolchain, cores, None, None)) } /// Create a CH32V orchestrator (convenience for get_orchestrator dispatch). @@ -620,6 +772,156 @@ mod tests { assert_eq!(orch.platform(), Platform::Ch32v); } + #[test] + fn explicit_board_isa_and_abi_override_selected_platform_defaults() { + let env = std::collections::HashMap::from([ + ("board_build.march".into(), "rv32imac".into()), + ("board_build.mabi".into(), "ilp32".into()), + ]); + assert_eq!( + preferred_board_value( + Some(&env), + "board_build.march", + Some("rv32ecxw"), + Some("rv32ec") + ), + Some("rv32imac") + ); + assert_eq!( + preferred_board_value( + Some(&env), + "board_build.mabi", + Some("ilp32e"), + Some("ilp32e") + ), + Some("ilp32") + ); + assert_eq!( + preferred_board_value(None, "board_build.march", Some("rv32ecxw"), Some("rv32ec")), + Some("rv32ecxw") + ); + let mut config = crate::ch32v::mcu_config::get_ch32v_config_for_mcu("ch32v003").unwrap(); + crate::ch32v::mcu_config::apply_board_isa_for_toolchain( + &mut config, + preferred_board_value( + Some(&env), + "board_build.march", + Some("rv32ecxw"), + Some("rv32ec"), + ), + preferred_board_value( + Some(&env), + "board_build.mabi", + Some("ilp32e"), + Some("ilp32e"), + ), + "riscv-none-embed", + ); + assert!( + config + .compiler_flags + .common + .contains(&"-march=rv32imac".into()) + ); + assert!(config.compiler_flags.common.contains(&"-mabi=ilp32".into())); + } + + #[test] + fn effective_board_settings_invalidate_fingerprint() { + let baseline = board_fingerprint_fields( + Some("rv32ecxw"), + Some("ilp32e"), + Some("variant_CH32V003F4.h"), + ); + let hash = crate::build_fingerprint::stable_hash_json(&baseline).unwrap(); + for changed in [ + board_fingerprint_fields( + Some("rv32imac"), + Some("ilp32e"), + Some("variant_CH32V003F4.h"), + ), + board_fingerprint_fields( + Some("rv32ecxw"), + Some("ilp32"), + Some("variant_CH32V003F4.h"), + ), + board_fingerprint_fields( + Some("rv32ecxw"), + Some("ilp32e"), + Some("variant_CH32V003J4.h"), + ), + ] { + assert_ne!( + hash, + crate::build_fingerprint::stable_hash_json(&changed).unwrap() + ); + } + } + + #[tokio::test(flavor = "multi_thread")] + #[ignore = "downloads CH32V platform, compiler, and framework packages"] + async fn pinned_github_platform_builds_ch32v003_with_gcc8() { + let backend = crate::compile_backend::CompileBackend::start() + .await + .expect("compile backend starts"); + crate::compile_backend::install_global(backend); + let project = tempdir(); + std::fs::create_dir_all(project.path().join("src")).unwrap(); + std::fs::write( + project.path().join("platformio.ini"), + "[env:ch32v003]\nplatform = https://github.com/Community-PIO-CH32V/platform-ch32v.git#b7397c29a71101175bfc94f6ab06f9daac336458\nboard = genericCH32V003F4P6\nframework = arduino\n", + ) + .unwrap(); + std::fs::copy( + concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../tests/platform/ch32v003/ch32v003.ino" + ), + project.path().join("src/ch32v003.ino"), + ) + .unwrap(); + let build_dir = fbuild_paths::BuildLayout::new( + project.path().to_path_buf(), + "ch32v003".into(), + fbuild_core::BuildProfile::Release, + ) + .resolve(); + let params = BuildParams { + project_dir: project.path().to_path_buf(), + env_name: "ch32v003".into(), + clean_all: false, + clean_only: false, + clean: false, + profile: fbuild_core::BuildProfile::Release, + build_dir, + verbose: false, + jobs: Some(2), + generate_compiledb: false, + compiledb_only: false, + log_sender: None, + symbol_analysis: false, + symbol_analysis_path: None, + no_timestamp: true, + src_dir: None, + pio_env: std::collections::BTreeMap::new(), + extra_build_flags: Vec::new(), + watch_set_cache: None, + bloat_analysis: false, + caller_path: None, + }; + let built = Ch32vOrchestrator.build(¶ms).await.unwrap(); + assert!(built.success); + assert!(built.elf_path.as_ref().is_some_and(|path| path.is_file())); + let log = built.build_log.into_lines().join("\n"); + assert!(log.contains("CH32V requested: platform=https://github.com/Community-PIO-CH32V/platform-ch32v.git#b7397c29a71101175bfc94f6ab06f9daac336458")); + assert!( + log.contains("CH32V resolved platform: ch32v@1.1.0 (source_ref=b7397c29a71101175bfc94f6ab06f9daac336458") + ); + assert!(log.contains("toolchain-riscv-linux/archive/")); + assert!(log.contains("arduino_core_ch32/archive/")); + assert!(log.contains("8.2.0")); + } + #[test] fn test_is_ch32v_project() { let tmp = tempdir(); diff --git a/crates/fbuild-build-mcu/src/ch32v/platform_source.rs b/crates/fbuild-build-mcu/src/ch32v/platform_source.rs new file mode 100644 index 000000000..db373a08e --- /dev/null +++ b/crates/fbuild-build-mcu/src/ch32v/platform_source.rs @@ -0,0 +1,671 @@ +//! Select the packages described by a CH32V PlatformIO source checkout. + +use std::collections::HashMap; +use std::path::{Path, PathBuf}; + +use fbuild_config::PackageOverride; +use fbuild_core::platformio_package::{ + PackageKind, PackageRequirement, PackageSource, PackageSpec, parse_package_spec, + require_platform_package, resolve_platform_requirements, +}; +use fbuild_packages::{CacheSubdir, PackageBase}; + +#[derive(Debug)] +pub(crate) struct SelectedRequirements { + pub framework: PackageRequirement, + pub toolchain: PackageRequirement, +} + +#[derive(Debug, Clone)] +pub(crate) struct SelectedBoardBuild { + pub core: String, + pub variant: String, + pub variant_h: Option, + pub march: String, + pub mabi: String, +} + +/// Resolve an explicitly selected CH32V platform source before constructing +/// any native packages. Unpinned `platform = ch32v` keeps the legacy native +/// defaults; a nonexistent registry pin never reaches that fallback. +pub(crate) async fn resolve_source_packages( + project_dir: &Path, + env: &HashMap, + board_core: &str, +) -> fbuild_core::Result< + Option<( + fbuild_packages::toolchain::RiscvToolchain, + fbuild_packages::library::Ch32vCores, + SelectedBoardBuild, + String, + )>, +> { + let raw_platform = env.get("platform").map(String::as_str).unwrap_or("ch32v"); + let platform = parse_package_spec(raw_platform).map_err(package_error)?; + let host = fbuild_core::platformio_package::host_system(fbuild_core::platform::host::current()) + .ok_or_else(|| package_error("unsupported PlatformIO host"))?; + let (archive, identity) = match &platform.source { + PackageSource::Registry(registry) if is_legacy_default(registry) => { + return Ok(None); + } + PackageSource::Registry(registry) => { + let cache_root = fbuild_packages::Cache::new(project_dir).platforms_dir(); + let payload = fbuild_packages::platformio_registry::RegistryClient::default() + .resolve_cached(registry, PackageKind::Platform, host, &cache_root, true) + .await + .map_err(package_error)? + .ok_or_else(|| package_error("CH32V platform registry pin is unavailable"))?; + ( + PackageOverride { + url: payload.url.clone(), + version: payload.version.clone(), + checksum: Some(payload.sha256.clone()), + }, + payload.cache_identity(), + ) + } + PackageSource::Repository { .. } => { + let resolved = + fbuild_packages::platformio_repository::resolve_github_repository(&platform.source) + .await + .map_err(package_error)?; + let identity = resolved.lock.cache_identity(); + (resolved.archive, identity) + } + PackageSource::Archive { url, revision } => { + let resolved = fbuild_packages::platformio_repository::resolve_archive_source( + url, + revision.as_deref(), + ) + .await + .map_err(package_error)?; + let identity = resolved.lock.cache_identity(); + (resolved.archive, identity) + } + PackageSource::LocalPath { .. } => { + return Err(package_error( + "local CH32V platform directories are not supported by the native adapter", + )); + } + }; + let platform_base = PackageBase::new( + "ch32v-platform", + &archive.version, + &archive.url, + &identity, + archive.checksum.as_deref(), + CacheSubdir::Platforms, + project_dir, + ); + let installed = platform_base + .staged_install(|root| { + find_platform_manifest(root) + .ok_or_else(|| package_error("selected CH32V platform has no platform.json"))?; + Ok(()) + }) + .await?; + let manifest_path = find_platform_manifest(&installed) + .ok_or_else(|| package_error("selected CH32V platform has no platform.json"))?; + let manifest = std::fs::read_to_string(&manifest_path).map_err(package_error)?; + let declared_version = serde_json::from_str::(&manifest) + .map_err(package_error)? + .get("version") + .and_then(serde_json::Value::as_str) + .ok_or_else(|| package_error("selected CH32V platform has no version"))? + .to_string(); + let builder = std::fs::read_to_string(manifest_path.with_file_name("platform.py")) + .map_err(package_error)?; + let board_name = env + .get("board") + .ok_or_else(|| package_error("CH32V platform source requires a board name"))?; + if !board_name + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_')) + { + return Err(package_error(format!( + "invalid CH32V board name `{board_name}`" + ))); + } + let board_path = manifest_path + .parent() + .ok_or_else(|| package_error("CH32V platform manifest has no parent"))? + .join("boards") + .join(format!("{board_name}.json")); + let board: serde_json::Value = + serde_json::from_slice(&std::fs::read(&board_path).map_err(package_error)?) + .map_err(package_error)?; + let board_build = board + .get("build") + .ok_or_else(|| package_error("selected CH32V board has no build metadata"))?; + let selected_board = select_board_build(board_build, env, board_core)?; + let selected_core = selected_board.core.as_str(); + let explicit = parse_explicit_packages(env)?; + let selected = select_requirements(&manifest, &builder, selected_core, host, &explicit)?; + if selected.framework.name != "framework-arduino-openwch-ch32" { + return Err(package_error(format!( + "CH32V core `{selected_core}` selects `{}`; the native adapter only supports OpenWCH", + selected.framework.name + ))); + } + for entry in &explicit { + let name = entry.package_name().unwrap_or_default(); + if name != selected.framework.name && name != selected.toolchain.name { + return Err(package_error(format!( + "CH32V platform package `{name}` is not consumed by the native adapter" + ))); + } + } + let toolchain_override = package_override_for_spec( + project_dir, + &selected.toolchain.spec, + PackageKind::Tool, + host, + ) + .await?; + let framework_override = package_override_for_spec( + project_dir, + &selected.framework.spec, + PackageKind::Framework, + host, + ) + .await?; + tracing::info!( + "resolved CH32V platform {} to {} (identity {}), toolchain {} at {}, framework {} at {}", + raw_platform, + archive.url, + identity, + selected.toolchain.name, + toolchain_override.url, + selected.framework.name, + framework_override.url, + ); + let platform_resolution = format!( + "{}@{} (source_ref={}; url={}{})", + platform.package_name().unwrap_or("ch32v"), + declared_version, + archive.version, + archive.url, + archive + .checksum + .as_ref() + .map(|sha| format!("; sha256={sha}")) + .unwrap_or_default() + ); + Ok(Some(( + fbuild_packages::toolchain::RiscvToolchain::with_platform_override( + project_dir, + toolchain_override, + "auto", + ), + fbuild_packages::library::Ch32vCores::with_platform_override( + project_dir, + framework_override, + ), + selected_board, + platform_resolution, + ))) +} + +fn select_board_build( + board_build: &serde_json::Value, + env: &HashMap, + fallback_core: &str, +) -> fbuild_core::Result { + let core = env + .get("board_build.core") + .map(String::as_str) + .or_else(|| board_build.get("core").and_then(serde_json::Value::as_str)) + .unwrap_or(fallback_core); + let arduino_core = board_build.get("arduino").and_then(|value| value.get(core)); + let variant_override_key = format!("board_build.arduino.{core}.variant"); + let variant_h_override_key = format!("board_build.arduino.{core}.variant_h"); + let variant = env + .get(&variant_override_key) + .or_else(|| env.get("board_build.variant")) + .map(String::as_str) + .or_else(|| { + arduino_core + .and_then(|value| value.get("variant")) + .and_then(serde_json::Value::as_str) + }) + .or_else(|| { + board_build + .get("variant") + .and_then(serde_json::Value::as_str) + }) + .ok_or_else(|| package_error("selected CH32V board has no variant"))?; + let variant_h = env + .get(&variant_h_override_key) + .or_else(|| env.get("board_build.variant_h")) + .map(String::as_str) + .or_else(|| { + arduino_core + .and_then(|value| value.get("variant_h")) + .and_then(serde_json::Value::as_str) + }); + Ok(SelectedBoardBuild { + core: core.into(), + variant: variant.into(), + variant_h: variant_h.map(str::to_string), + march: board_build + .get("march") + .and_then(serde_json::Value::as_str) + .ok_or_else(|| package_error("selected CH32V board has no march"))? + .into(), + mabi: board_build + .get("mabi") + .and_then(serde_json::Value::as_str) + .ok_or_else(|| package_error("selected CH32V board has no mabi"))? + .into(), + }) +} + +fn is_legacy_default(registry: &fbuild_core::platformio_package::RegistrySpec) -> bool { + registry.requirement.is_none() + && registry.owner.is_none() + && registry.registry_type.is_none() + && registry.name == "ch32v" +} + +fn find_platform_manifest(root: &Path) -> Option { + walkdir::WalkDir::new(root) + .max_depth(3) + .into_iter() + .filter_map(Result::ok) + .find(|entry| entry.file_type().is_file() && entry.file_name() == "platform.json") + .map(|entry| entry.path().to_path_buf()) +} + +fn parse_explicit_packages(env: &HashMap) -> fbuild_core::Result> { + env.get("platform_packages") + .into_iter() + .flat_map(|value| value.lines()) + .map(str::trim) + .filter(|line| !line.is_empty()) + .map(|line| parse_package_spec(line).map_err(package_error)) + .collect() +} + +async fn package_override_for_spec( + project_dir: &Path, + spec: &PackageSpec, + kind: PackageKind, + host: &str, +) -> fbuild_core::Result { + match &spec.source { + PackageSource::Registry(registry) => { + let cache_root = fbuild_packages::Cache::new(project_dir).platforms_dir(); + let payload = fbuild_packages::platformio_registry::RegistryClient::default() + .resolve_cached(registry, kind, host, &cache_root, true) + .await + .map_err(package_error)? + .ok_or_else(|| package_error("CH32V platform package is unavailable"))?; + Ok(PackageOverride { + url: payload.url, + version: payload.version, + checksum: Some(payload.sha256), + }) + } + PackageSource::Repository { .. } => Ok( + fbuild_packages::platformio_repository::resolve_github_repository(&spec.source) + .await + .map_err(package_error)? + .archive, + ), + PackageSource::Archive { url, revision } => Ok( + fbuild_packages::platformio_repository::resolve_archive_source( + url, + revision.as_deref(), + ) + .await + .map_err(package_error)? + .archive, + ), + PackageSource::LocalPath { .. } => Err(package_error( + "local CH32V package directories are not supported by the native adapter", + )), + } +} + +/// Apply the selected platform's own `platform.py` board/host choices to its +/// manifest, then give consumer `platform_packages` overrides final precedence. +/// This recognizes literal CH32V builder assignments; unexpected dynamic code +/// fails rather than silently retaining the manifest's Windows toolchain. +pub(crate) fn select_requirements( + manifest: &str, + builder: &str, + board_core: &str, + host: &str, + explicit: &[PackageSpec], +) -> fbuild_core::Result { + let framework_condition = format!("build_core == \"{board_core}\""); + let framework_name = branch_assignment( + builder, + &framework_condition, + "self.frameworks[\"arduino\"][\"package\"]", + ) + .ok_or_else(|| { + package_error(format!( + "CH32V platform has no Arduino package for core `{board_core}`" + )) + })?; + + let mut selections = explicit.to_vec(); + if !explicit + .iter() + .any(|spec| spec.package_name() == Some("toolchain-riscv")) + { + let template = toolchain_template( + builder, + host, + "self.packages[\"toolchain-riscv\"][\"version\"]", + ) + .ok_or_else(|| { + package_error(format!("CH32V platform has no toolchain for host `{host}`")) + })?; + let host_condition = format!("sys_type == \"{host}\""); + let gcc_branch = + branch_assignment(builder, "\"arduino\" in selected_frameworks", "gcc_branch") + .or_else(|| branch_assignment(builder, &host_condition, "gcc_branch")) + .or_else(|| root_assignment(builder, "gcc_branch")) + .ok_or_else(|| package_error("CH32V platform has no GCC branch selection"))?; + let source = template + .strip_suffix("%s") + .ok_or_else(|| package_error("CH32V toolchain builder expression is unsupported"))?; + let version = format!("{source}{gcc_branch}"); + selections.push( + parse_package_spec(&format!("toolchain-riscv@{version}")).map_err(package_error)?, + ); + } + let requirements = + resolve_platform_requirements(manifest, &selections).map_err(package_error)?; + Ok(SelectedRequirements { + framework: require_platform_package(&requirements, &framework_name) + .map_err(package_error)? + .clone(), + toolchain: require_platform_package(&requirements, "toolchain-riscv") + .map_err(package_error)? + .clone(), + }) +} + +fn toolchain_template(builder: &str, host: &str, assignment: &str) -> Option { + // CH32V 1.1.0 uses IS_LINUX/IS_MAC/else. The manifest's Windows URL + // cannot stand in for an unsupported host. + match host { + "linux_x86_64" => branch_assignment(builder, "IS_LINUX", assignment), + "darwin_x86_64" | "darwin_arm64" => branch_assignment(builder, "IS_MAC", assignment), + "windows_amd64" | "windows_x86" => { + branch_assignment_after(builder, "elif IS_MAC:", "else:", assignment) + } + _ => None, + } +} + +fn branch_assignment_after( + source: &str, + preceding: &str, + branch: &str, + assignment: &str, +) -> Option { + let lines: Vec<_> = source.lines().collect(); + let prior = lines.iter().position(|line| line.trim() == preceding)?; + let indent = indentation(lines[prior]); + let branch_index = lines + .iter() + .enumerate() + .skip(prior + 1) + .find_map(|(index, line)| { + (line.trim() == branch && indentation(line) == indent).then_some(index) + })?; + for line in lines.iter().skip(branch_index + 1) { + if !line.trim().is_empty() && indentation(line) <= indent { + break; + } + if let Some(value) = quoted_assignment(line.trim(), assignment) { + return Some(value); + } + } + None +} + +fn branch_assignment(source: &str, condition: &str, assignment: &str) -> Option { + let lines: Vec<_> = source.lines().collect(); + for (branch_index, branch) in lines.iter().enumerate() { + let trimmed = branch.trim(); + if !(trimmed.starts_with("if ") || trimmed.starts_with("elif ")) + || !trimmed.ends_with(':') + || !trimmed.contains(condition) + { + continue; + } + let branch_indent = indentation(branch); + for line in lines.iter().skip(branch_index + 1) { + let trimmed = line.trim(); + if trimmed.is_empty() || trimmed.starts_with('#') { + continue; + } + if indentation(line) <= branch_indent { + break; + } + if let Some(value) = quoted_assignment(trimmed, assignment) { + return Some(value); + } + } + } + None +} + +fn root_assignment(source: &str, assignment: &str) -> Option { + source + .lines() + .find_map(|line| quoted_assignment(line.trim(), assignment)) +} + +fn quoted_assignment(line: &str, assignment: &str) -> Option { + let rhs = line + .strip_prefix(assignment)? + .trim_start() + .strip_prefix('=')? + .trim_start(); + let quote = rhs.chars().next()?; + if quote != '\'' && quote != '"' { + return None; + } + let rest = &rhs[1..]; + let end = rest.find(quote)?; + Some(rest[..end].to_string()) +} + +fn indentation(line: &str) -> usize { + line.len() - line.trim_start().len() +} + +fn package_error(error: impl std::fmt::Display) -> fbuild_core::FbuildError { + fbuild_core::FbuildError::PackageError(error.to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + use fbuild_core::platformio_package::{PackageSource, parse_package_spec}; + use fbuild_packages::{Package, Toolchain}; + + const MANIFEST: &str = r#"{ + "frameworks": {"arduino": {"package": "framework-arduinoch32v003"}}, + "packages": { + "toolchain-riscv": {"type": "toolchain", "owner": "platformio", "version": "https://github.com/Community-PIO-CH32V/toolchain-riscv-windows.git"}, + "framework-arduinoch32v003": {"type": "framework", "optional": true, "version": "https://github.com/Community-PIO-CH32V/arduino-wch32v003"}, + "framework-arduino-openwch-ch32": {"type": "framework", "optional": true, "version": "https://github.com/Community-PIO-CH32V/arduino_core_ch32.git"} + } + }"#; + + const BUILDER: &str = r##" + gcc_branch = "#gcc12" + FORCE_DOWNGRADE_TO_GCC8 = True + if "arduino" in selected_frameworks or FORCE_DOWNGRADE_TO_GCC8: + gcc_branch = "" + if IS_LINUX: + self.packages["toolchain-riscv"]["version"] = "https://github.com/Community-PIO-CH32V/toolchain-riscv-linux.git%s" % gcc_branch + elif IS_MAC: + self.packages["toolchain-riscv"]["version"] = "https://github.com/Community-PIO-CH32V/toolchain-riscv-mac.git%s" % gcc_branch + else: + self.packages["toolchain-riscv"]["version"] = "https://github.com/Community-PIO-CH32V/toolchain-riscv-windows.git%s" % gcc_branch + if build_core == "ch32v003": + self.frameworks["arduino"]["package"] = "framework-arduinoch32v003" + elif build_core == "openwch": + self.frameworks["arduino"]["package"] = "framework-arduino-openwch-ch32" + "##; + + #[test] + fn pinned_source_selects_openwch_and_linux_gcc8() { + let selected = + select_requirements(MANIFEST, BUILDER, "openwch", "linux_x86_64", &[]).unwrap(); + assert_eq!(selected.framework.name, "framework-arduino-openwch-ch32"); + assert_eq!(selected.toolchain.name, "toolchain-riscv"); + assert!(matches!( + selected.toolchain.spec.source, + PackageSource::Repository { ref url, reference: None } + if url.ends_with("/toolchain-riscv-linux.git") + )); + } + + #[test] + fn pinned_source_selects_host_specific_mac_and_windows_toolchains() { + for (host, repository) in [ + ("darwin_x86_64", "toolchain-riscv-mac.git"), + ("darwin_arm64", "toolchain-riscv-mac.git"), + ("windows_amd64", "toolchain-riscv-windows.git"), + ("windows_x86", "toolchain-riscv-windows.git"), + ] { + let selected = select_requirements(MANIFEST, BUILDER, "openwch", host, &[]).unwrap(); + assert!( + matches!( + selected.toolchain.spec.source, + PackageSource::Repository { ref url, reference: None } + if url.ends_with(repository) + ), + "{host}" + ); + } + } + + #[test] + fn explicit_toolchain_pin_wins_over_host_default() { + let explicit = parse_package_spec("community-ch32v/toolchain-riscv@1.2.3").unwrap(); + let selected = + select_requirements(MANIFEST, BUILDER, "openwch", "linux_x86_64", &[explicit]).unwrap(); + assert_eq!( + selected + .toolchain + .spec + .registry() + .unwrap() + .requirement + .as_deref(), + Some("1.2.3") + ); + } + + #[test] + fn explicit_framework_archive_wins_over_manifest_repository() { + let url = "https://example.invalid/openwch-core.tar.gz"; + let explicit = + parse_package_spec(&format!("framework-arduino-openwch-ch32@{url}")).unwrap(); + let selected = + select_requirements(MANIFEST, BUILDER, "openwch", "linux_x86_64", &[explicit]).unwrap(); + assert!(matches!( + selected.framework.spec.source, + PackageSource::Archive { url: ref selected_url, revision: None } + if selected_url == url + )); + } + + #[test] + fn unsupported_host_does_not_fall_back_to_windows_toolchain() { + let error = + select_requirements(MANIFEST, BUILDER, "openwch", "linux_aarch64", &[]).unwrap_err(); + assert!(error.to_string().contains("linux_aarch64")); + } + + #[test] + fn only_bare_unpinned_ch32v_uses_native_defaults() { + for (raw, expected) in [ + ("ch32v", true), + ("ch32v@1.1.0", false), + ("community-ch32v/ch32v", false), + ("another-platform", false), + ] { + let spec = parse_package_spec(raw).unwrap(); + assert_eq!( + is_legacy_default(spec.registry().unwrap()), + expected, + "{raw}" + ); + } + } + + #[test] + fn selected_platform_board_core_and_variant_are_not_taken_from_builtin_board() { + let selected = serde_json::json!({ + "core": "openwch", + "variant": "GENERIC", + "march": "rv32ecxw", + "mabi": "ilp32e", + "arduino": {"openwch": { + "variant": "CH32V00x/CH32V003F4", + "variant_h": "variant_CH32V003F4.h" + }} + }); + let board = select_board_build(&selected, &HashMap::new(), "stale_builtin_core").unwrap(); + assert_eq!(board.core, "openwch"); + assert_eq!(board.variant, "CH32V00x/CH32V003F4"); + assert_eq!(board.variant_h.as_deref(), Some("variant_CH32V003F4.h")); + + let env = HashMap::from([ + ("board_build.variant".into(), "custom_variant".into()), + ("board_build.variant_h".into(), "custom.h".into()), + ]); + let overridden = select_board_build(&selected, &env, "stale_builtin_core").unwrap(); + assert_eq!(overridden.variant, "custom_variant"); + assert_eq!(overridden.variant_h.as_deref(), Some("custom.h")); + } + + #[tokio::test] + #[ignore = "downloads a pinned CH32V platform archive and resolves GitHub package refs"] + async fn pinned_github_platform_resolves_real_ch32v_package_sources() { + let project = tempfile::TempDir::new().unwrap(); + let env = HashMap::from([( + "platform".to_string(), + "https://github.com/Community-PIO-CH32V/platform-ch32v.git#b7397c29a71101175bfc94f6ab06f9daac336458".to_string(), + ), ("board".to_string(), "genericCH32V003F4P6".to_string())]); + let (toolchain, framework, board_isa, platform_resolution) = + resolve_source_packages(project.path(), &env, "openwch") + .await + .unwrap() + .unwrap(); + assert!(platform_resolution.contains("b7397c29a71101175bfc94f6ab06f9daac336458")); + let toolchain_info = toolchain.get_info(); + let framework_info = framework.get_info(); + assert!( + toolchain_info + .url + .contains("toolchain-riscv-linux/archive/") + ); + assert!(!toolchain_info.url.contains("xpack")); + assert!(framework_info.url.contains("arduino_core_ch32/archive/")); + assert_eq!(board_isa.march, "rv32ecxw"); + let toolchain_root = toolchain.ensure_installed().await.unwrap(); + let framework_root = framework.ensure_installed().await.unwrap(); + assert!(toolchain_root.is_dir()); + assert!(framework_root.is_dir()); + assert!(toolchain.get_gcc_path().is_file()); + // allow-direct-spawn: integration test probes the selected compiler binary. + let version = std::process::Command::new(toolchain.get_gcc_path()) + .arg("--version") + .output() + .unwrap(); + assert!(version.status.success()); + assert!(String::from_utf8_lossy(&version.stdout).contains("8.2.0")); + } +} diff --git a/crates/fbuild-build/src/lib.rs b/crates/fbuild-build/src/lib.rs index 1566d7f08..e3cf5efd8 100644 --- a/crates/fbuild-build/src/lib.rs +++ b/crates/fbuild-build/src/lib.rs @@ -94,6 +94,12 @@ pub async fn provision_env( "offline install check/dry-run cannot resolve a registry PlatformIO platform or package without fetching its metadata; run `fbuild install`".into(), )); } + if !mode.fetches() && platform == Platform::Ch32v && has_repository_platform_source(env_config)? + { + return Err(fbuild_core::FbuildError::PackageError( + "offline install check/dry-run cannot resolve a CH32V repository/archive platform without fetching its manifest; run `fbuild install`".into(), + )); + } let support = get_platform_support(platform)?; let inputs = provision::ProvisionInputs { project_dir, @@ -265,6 +271,19 @@ fn has_registry_resolution(env: &std::collections::HashMap) -> R Ok(false) } +fn has_repository_platform_source(env: &std::collections::HashMap) -> Result { + let Some(raw) = env.get("platform") else { + return Ok(false); + }; + let spec = fbuild_core::platformio_package::parse_package_spec(raw) + .map_err(|error| fbuild_core::FbuildError::PackageError(error.to_string()))?; + Ok(matches!( + spec.source, + fbuild_core::platformio_package::PackageSource::Repository { .. } + | fbuild_core::platformio_package::PackageSource::Archive { .. } + )) +} + #[cfg(test)] mod tests { use super::*; @@ -335,6 +354,27 @@ mod tests { } } + #[tokio::test] + async fn ch32v_repository_dry_run_never_fetches_platform_source() { + let dir = project( + "[env:ch32v003]\nplatform = https://github.com/Community-PIO-CH32V/platform-ch32v.git#b7397c29a71101175bfc94f6ab06f9daac336458\nboard = genericCH32V003F4P6\nframework = arduino\n", + ); + for mode in [ + provision::ProvisionMode::DryRun, + provision::ProvisionMode::Check, + ] { + let error = provision_env(dir.path(), "ch32v003", mode) + .await + .unwrap_err(); + assert!( + error + .to_string() + .contains("cannot resolve a CH32V repository") + ); + assert!(!dir.path().join(fbuild_paths::FBUILD_DIR_NAME).exists()); + } + } + #[test] fn test_get_orchestrator_atmelmegaavr() { let orch = get_orchestrator(Platform::AtmelMegaAvr).unwrap(); diff --git a/crates/fbuild-core/tests/platformio_ch32v_resolution.rs b/crates/fbuild-core/tests/platformio_ch32v_resolution.rs new file mode 100644 index 000000000..b87888961 --- /dev/null +++ b/crates/fbuild-core/tests/platformio_ch32v_resolution.rs @@ -0,0 +1,68 @@ +//! Offline CH32V VCS platform/package identity fixture. + +use fbuild_core::platformio_package::{ + PackageKind, PackageLock, PackageSource, parse_package_spec, require_platform_package, + resolve_platform_requirements, +}; + +const MANIFEST: &str = r#"{ + "version": "1.1.0", + "frameworks": {"arduino": {"package": "framework-arduinoch32v003"}}, + "packages": { + "toolchain-riscv": {"type": "toolchain", "owner": "platformio", "version": "https://github.com/Community-PIO-CH32V/toolchain-riscv-windows.git"}, + "framework-arduino-openwch-ch32": {"type": "framework", "optional": true, "version": "https://github.com/Community-PIO-CH32V/arduino_core_ch32.git"} + } +}"#; + +#[test] +fn pinned_ch32v_platform_source_has_immutable_identity_and_selected_packages() { + let platform = parse_package_spec("https://github.com/Community-PIO-CH32V/platform-ch32v.git#b7397c29a71101175bfc94f6ab06f9daac336458").unwrap(); + let PackageSource::Repository { url, reference } = platform.source else { + panic!("CH32V platform must remain a VCS source"); + }; + let lock = PackageLock::Repository { + url: url.clone(), + commit: reference.unwrap(), + }; + assert_eq!(lock.cache_identity().len(), 64); + + let linux_toolchain = parse_package_spec( + "toolchain-riscv@https://github.com/Community-PIO-CH32V/toolchain-riscv-linux.git", + ) + .unwrap(); + let requirements = resolve_platform_requirements(MANIFEST, &[linux_toolchain]).unwrap(); + let toolchain = require_platform_package(&requirements, "toolchain-riscv").unwrap(); + assert_eq!(toolchain.kind, PackageKind::Tool); + assert!(matches!( + &toolchain.spec.source, + PackageSource::Repository { url, reference: None } + if url.ends_with("toolchain-riscv-linux.git") + )); + let framework = + require_platform_package(&requirements, "framework-arduino-openwch-ch32").unwrap(); + assert_eq!(framework.kind, PackageKind::Framework); + assert!(matches!( + &framework.spec.source, + PackageSource::Repository { url, reference: None } + if url.ends_with("arduino_core_ch32.git") + )); + assert_ne!( + lock.cache_identity(), + PackageLock::Repository { + url, + commit: "0000000000000000000000000000000000000000".into(), + } + .cache_identity() + ); +} + +#[test] +fn explicit_ch32v_package_pin_precedes_selected_manifest_source() { + let explicit = parse_package_spec("platformio/toolchain-riscv@1.2.3").unwrap(); + let requirements = resolve_platform_requirements(MANIFEST, &[explicit]).unwrap(); + let toolchain = require_platform_package(&requirements, "toolchain-riscv").unwrap(); + assert_eq!( + toolchain.spec.registry().unwrap().requirement.as_deref(), + Some("1.2.3") + ); +} diff --git a/crates/fbuild-library/src/library/ch32v_core.rs b/crates/fbuild-library/src/library/ch32v_core.rs index 4c5e4daee..ce1274124 100644 --- a/crates/fbuild-library/src/library/ch32v_core.rs +++ b/crates/fbuild-library/src/library/ch32v_core.rs @@ -62,6 +62,19 @@ impl Ch32vCores { } } + /// Use a CH32V platform-selected GitHub core source. Resolve its declared + /// gitlinks from that exact parent commit, rather than reusing the default + /// core's fixed TinyUSB submodule pin. + pub fn with_platform_override(project_dir: &Path, ovr: fbuild_config::PackageOverride) -> Self { + let github_commit_archive = + fbuild_packages_fetch::submodules::is_github_commit_archive_url(&ovr.url); + let mut core = Self::with_override(project_dir, ovr); + if github_commit_archive { + core.base = core.base.with_github_gitlinks(); + } + core + } + #[cfg(test)] fn with_cache_root(project_dir: &Path, cache_root: &Path) -> Self { Self { diff --git a/crates/fbuild-packages-fetch/src/lib.rs b/crates/fbuild-packages-fetch/src/lib.rs index 549642c17..48bb96806 100644 --- a/crates/fbuild-packages-fetch/src/lib.rs +++ b/crates/fbuild-packages-fetch/src/lib.rs @@ -12,6 +12,7 @@ pub mod downloader; pub mod extractor; pub mod http; pub mod platformio_registry; +pub mod platformio_repository; pub mod submodules; mod install_lock; diff --git a/crates/fbuild-packages-fetch/src/platformio_repository.rs b/crates/fbuild-packages-fetch/src/platformio_repository.rs new file mode 100644 index 000000000..e8d0a8f36 --- /dev/null +++ b/crates/fbuild-packages-fetch/src/platformio_repository.rs @@ -0,0 +1,301 @@ +//! Lock GitHub-hosted PlatformIO repository packages to immutable commits. + +use fbuild_config::PackageOverride; +use fbuild_core::platformio_package::{PackageLock, PackageSource}; +use sha2::{Digest, Sha256}; +use std::time::Duration; +use tokio::io::AsyncReadExt; + +#[derive(Debug, thiserror::Error)] +pub enum RepositoryError { + #[error("unsupported PlatformIO repository URL: {0}")] + UnsupportedUrl(String), + #[error("invalid PlatformIO repository ref: {0}")] + InvalidRef(String), + #[error("PlatformIO repository Git command failed: {0}")] + Git(String), + #[error("PlatformIO repository returned an invalid commit: {0}")] + InvalidCommit(String), + #[error("PlatformIO archive resolution failed: {0}")] + Archive(String), +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ResolvedRepository { + pub lock: PackageLock, + pub archive: PackageOverride, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ResolvedArchive { + pub lock: PackageLock, + pub archive: PackageOverride, +} + +/// Fetch a mutable archive URL to establish its content identity before it +/// becomes a package-cache key. The later staged install verifies this digest, +/// so a server changing between resolution and installation fails closed. +pub async fn resolve_archive_source( + url: &str, + revision: Option<&str>, +) -> Result { + let temp = + tempfile::TempDir::new().map_err(|error| RepositoryError::Archive(error.to_string()))?; + let path = crate::downloader::download_file(url, temp.path()) + .await + .map_err(|error| RepositoryError::Archive(error.to_string()))?; + let mut file = tokio::fs::File::open(path) + .await + .map_err(|error| RepositoryError::Archive(error.to_string()))?; + let mut hasher = Sha256::new(); + let mut buffer = [0_u8; 64 * 1024]; + loop { + let read = file + .read(&mut buffer) + .await + .map_err(|error| RepositoryError::Archive(error.to_string()))?; + if read == 0 { + break; + } + hasher.update(&buffer[..read]); + } + let sha256 = format!("{:x}", hasher.finalize()); + Ok(ResolvedArchive { + lock: PackageLock::Archive { + url: url.into(), + sha256: sha256.clone(), + }, + archive: PackageOverride { + url: url.into(), + version: revision.unwrap_or("archive").into(), + checksum: Some(sha256), + }, + }) +} + +/// Resolve a GitHub repository source to an immutable commit. Other VCS hosts +/// fail explicitly until a transport for them exists; they never become a +/// default fbuild package. The archive URL is keyed by the resolved SHA. +pub async fn resolve_github_repository( + source: &PackageSource, +) -> Result { + let PackageSource::Repository { url, reference } = source else { + return Err(RepositoryError::UnsupportedUrl(format!("{source:?}"))); + }; + let (owner, repo) = github_owner_repo(url)?; + let reference = reference.as_deref().unwrap_or("HEAD"); + if reference.is_empty() || reference.len() > 1024 || reference.contains(['\n', '\r', '\0']) { + return Err(RepositoryError::InvalidRef(reference.into())); + } + let commit = if is_full_sha(reference) { + reference.to_ascii_lowercase() + } else { + let candidates = candidate_refs(reference); + // allow-direct-spawn: PlatformIO VCS package ref lookup, no shell. + let mut command = tokio::process::Command::new("git"); + command + .args(["ls-remote", url]) + .args(&candidates) + .kill_on_drop(true); + let output = tokio::time::timeout(Duration::from_secs(30), command.output()) + .await + .map_err(|_| RepositoryError::Git(format!("timed out resolving {reference}")))? + .map_err(|error| RepositoryError::Git(error.to_string()))?; + if !output.status.success() { + return Err(RepositoryError::Git( + String::from_utf8_lossy(&output.stderr).trim().to_string(), + )); + } + let stdout = String::from_utf8_lossy(&output.stdout); + let commit = select_remote_commit(&stdout, &candidates) + .ok_or_else(|| RepositoryError::InvalidRef(reference.into()))?; + if !is_full_sha(commit) { + return Err(RepositoryError::InvalidCommit(commit.into())); + } + commit.to_ascii_lowercase() + }; + Ok(github_archive(url, &owner, &repo, &commit)) +} + +fn candidate_refs(reference: &str) -> Vec { + if reference == "HEAD" { + return vec!["HEAD".into()]; + } + if reference.starts_with("refs/tags/") { + return vec![reference.into(), format!("{reference}^{{}}")]; + } + if reference.starts_with("refs/") { + return vec![reference.into()]; + } + vec![ + format!("refs/heads/{reference}"), + format!("refs/tags/{reference}"), + format!("refs/tags/{reference}^{{}}"), + ] +} + +fn select_remote_commit<'a>(stdout: &'a str, candidates: &[String]) -> Option<&'a str> { + let refs = stdout + .lines() + .filter_map(|line| line.split_once('\t')) + .map(|(sha, name)| (name, sha)) + .collect::>(); + // Branch names win if a remote has both a branch and a tag; an annotated + // tag must use its peeled commit, not the tag object's SHA. + candidates + .iter() + .filter(|name| !name.starts_with("refs/tags/") || name.ends_with("^{}")) + .chain( + candidates + .iter() + .filter(|name| name.starts_with("refs/tags/") && !name.ends_with("^{}")), + ) + .find_map(|name| refs.get(name.as_str()).copied()) +} + +fn github_owner_repo(url: &str) -> Result<(String, String), RepositoryError> { + let parsed = + reqwest::Url::parse(url).map_err(|_| RepositoryError::UnsupportedUrl(url.into()))?; + if parsed.scheme() != "https" + || parsed.host_str() != Some("github.com") + || !parsed.username().is_empty() + || parsed.password().is_some() + || parsed.query().is_some() + || parsed.fragment().is_some() + { + return Err(RepositoryError::UnsupportedUrl(url.into())); + } + let segments: Vec<_> = parsed + .path_segments() + .ok_or_else(|| RepositoryError::UnsupportedUrl(url.into()))? + .filter(|segment| !segment.is_empty()) + .collect(); + let [owner, repo] = segments.as_slice() else { + return Err(RepositoryError::UnsupportedUrl(url.into())); + }; + let repo = repo.strip_suffix(".git").unwrap_or(repo); + if !valid_segment(owner) || !valid_segment(repo) { + return Err(RepositoryError::UnsupportedUrl(url.into())); + } + Ok(((*owner).into(), repo.into())) +} + +fn valid_segment(value: &str) -> bool { + !value.is_empty() + && value != "." + && value != ".." + && value + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.')) +} + +fn is_full_sha(value: &str) -> bool { + value.len() == 40 && value.bytes().all(|byte| byte.is_ascii_hexdigit()) +} + +fn github_archive(url: &str, owner: &str, repo: &str, commit: &str) -> ResolvedRepository { + ResolvedRepository { + lock: PackageLock::Repository { + url: url.into(), + commit: commit.into(), + }, + archive: PackageOverride::new( + format!("https://github.com/{owner}/{repo}/archive/{commit}.tar.gz"), + commit, + ), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use fbuild_core::platformio_package::parse_package_spec; + + #[tokio::test] + async fn archive_lock_changes_when_bytes_at_the_same_url_change() { + let bytes = std::sync::Arc::new(tokio::sync::RwLock::new(b"first archive".to_vec())); + let served = bytes.clone(); + let app = axum::Router::new().route( + "/platform.tar.gz", + axum::routing::get(move || { + let served = served.clone(); + async move { axum::body::Bytes::from(served.read().await.clone()) } + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = format!("http://{}/platform.tar.gz", listener.local_addr().unwrap()); + let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + + let first = resolve_archive_source(&url, Some("test-revision")) + .await + .unwrap(); + *bytes.write().await = b"second archive".to_vec(); + let second = resolve_archive_source(&url, Some("test-revision")) + .await + .unwrap(); + server.abort(); + + assert_eq!(first.archive.url, second.archive.url); + assert_eq!(first.archive.version, second.archive.version); + assert_ne!(first.archive.checksum, second.archive.checksum); + assert_ne!(first.lock.cache_identity(), second.lock.cache_identity()); + } + + #[tokio::test] + async fn immutable_ch32v_platform_source_becomes_archive_lock() { + let spec = parse_package_spec("https://github.com/Community-PIO-CH32V/platform-ch32v.git#b7397c29a71101175bfc94f6ab06f9daac336458").unwrap(); + let resolved = resolve_github_repository(&spec.source).await.unwrap(); + assert_eq!( + resolved.archive.url, + "https://github.com/Community-PIO-CH32V/platform-ch32v/archive/b7397c29a71101175bfc94f6ab06f9daac336458.tar.gz" + ); + assert_eq!( + resolved.archive.version, + "b7397c29a71101175bfc94f6ab06f9daac336458" + ); + assert_eq!(resolved.lock.cache_identity().len(), 64); + } + + #[tokio::test] + async fn unsupported_repository_host_fails_without_substitution() { + let spec = parse_package_spec( + "https://gitlab.com/example/ch32v.git#0123456789abcdef0123456789abcdef01234567", + ) + .unwrap(); + assert!(matches!( + resolve_github_repository(&spec.source).await, + Err(RepositoryError::UnsupportedUrl(_)) + )); + } + + #[test] + fn annotated_tags_resolve_to_the_peeled_commit() { + let candidates = candidate_refs("v1.1.0"); + let tag = "a".repeat(40); + let commit = "b".repeat(40); + let listing = format!("{tag}\trefs/tags/v1.1.0\n{commit}\trefs/tags/v1.1.0^{{}}\n"); + assert_eq!( + select_remote_commit(&listing, &candidates), + Some(commit.as_str()) + ); + } + + #[test] + fn branch_takes_precedence_over_same_named_tag() { + let candidates = candidate_refs("release"); + let branch = "c".repeat(40); + let tag = "d".repeat(40); + let listing = format!("{tag}\trefs/tags/release\n{branch}\trefs/heads/release\n"); + assert_eq!( + select_remote_commit(&listing, &candidates), + Some(branch.as_str()) + ); + } + + #[test] + fn unavailable_ref_cannot_resolve_to_head_or_a_default_package() { + let candidates = candidate_refs("missing-branch"); + let listing = format!("{}\tHEAD\n", "e".repeat(40)); + assert_eq!(select_remote_commit(&listing, &candidates), None); + } +} diff --git a/crates/fbuild-packages-fetch/src/submodules.rs b/crates/fbuild-packages-fetch/src/submodules.rs index 1d1c09ec0..ffc9cea3c 100644 --- a/crates/fbuild-packages-fetch/src/submodules.rs +++ b/crates/fbuild-packages-fetch/src/submodules.rs @@ -137,7 +137,7 @@ pub struct SubmoduleSource { /// Archive of the submodule at the commit the parent's gitlink records. pub url: String, /// SHA-256 of that archive. - pub sha256: String, + pub sha256: Option, } /// A declared submodule left empty on purpose. @@ -155,11 +155,12 @@ pub struct ExpectedEmpty { pub struct SubmodulePlan { pub sources: Vec, pub expected_empty: Vec, + pub github_gitlinks: bool, } impl SubmodulePlan { fn is_empty(&self) -> bool { - self.sources.is_empty() && self.expected_empty.is_empty() + self.sources.is_empty() && self.expected_empty.is_empty() && !self.github_gitlinks } } @@ -171,11 +172,18 @@ impl PackageBase { self.submodules.sources.push(SubmoduleSource { path: path.to_string(), url: url.to_string(), - sha256: sha256.to_string(), + sha256: Some(sha256.to_string()), }); self } + /// Resolve empty submodules from the selected GitHub archive's immutable + /// parent gitlinks, instead of reusing pins from another core revision. + pub fn with_github_gitlinks(mut self) -> Self { + self.submodules.github_gitlinks = true; + self + } + /// Accept a declared submodule that extracts empty because fbuild supplies /// its contents another way. `reason` says how (FastLED/fbuild#1422). pub fn expect_empty_submodule(mut self, path: &str, reason: &str) -> Self { @@ -214,11 +222,23 @@ pub async fn prepare_submodules( continue; }; let declared = declared_submodule_paths(&text); - check_plan(package, root, &declared, plan)?; - populate(package, root, &plan.sources).await?; + let mut resolved_plan = plan.clone(); + if plan.github_gitlinks { + resolved_plan.sources.extend( + github_submodule_sources(url, root, &text, &declared) + .await + .map_err(|error| { + fbuild_core::FbuildError::PackageError(format!( + "{package}: resolving GitHub submodules failed: {error}" + )) + })?, + ); + } + check_plan(package, root, &declared, &resolved_plan)?; + populate(package, root, &resolved_plan.sources).await?; applied = true; - let empty = unexpected_empty_submodules(root, plan); + let empty = unexpected_empty_submodules(root, &resolved_plan); if !empty.is_empty() { return Err(fbuild_core::FbuildError::PackageError( empty_submodule_error(package, url, &empty), @@ -242,6 +262,212 @@ pub async fn prepare_submodules( Ok(()) } +async fn github_submodule_sources( + parent_archive: &str, + root: &Path, + gitmodules: &str, + declared: &[String], +) -> fbuild_core::Result> { + let (owner, repo, commit) = github_archive_identity(parent_archive)?; + let checkout = tempfile::TempDir::new().map_err(package_error)?; + let git_dir = checkout + .path() + .to_str() + .ok_or_else(|| package_error("invalid temporary Git path"))?; + git_command(["init", "--bare", "--quiet", git_dir]).await?; + let repository = format!("https://github.com/{owner}/{repo}.git"); + git_command([ + "-C", + git_dir, + "fetch", + "--quiet", + "--depth", + "1", + "--filter=blob:none", + "--no-tags", + &repository, + &commit, + ]) + .await?; + let declarations = declared_submodule_urls(gitmodules); + let mut sources = Vec::new(); + for path in declared { + if !is_empty_dir(&root.join(path)) { + continue; + } + checked_submodule_dest(root, path)?; + let source_url = declarations + .iter() + .find(|(declared_path, _)| declared_path == path) + .map(|(_, url)| url) + .ok_or_else(|| { + package_error(format!("submodule `{path}` has no URL in .gitmodules")) + })?; + let listing = git_command(["-C", git_dir, "ls-tree", &commit, "--", path]).await?; + let mut words = listing.split_whitespace(); + if words.next() != Some("160000") || words.next() != Some("commit") { + return Err(package_error(format!("submodule `{path}` has no gitlink"))); + } + let gitlink = words + .next() + .ok_or_else(|| package_error("missing gitlink SHA"))?; + if !full_sha(gitlink) { + return Err(package_error(format!( + "submodule `{path}` has invalid gitlink" + ))); + } + let (sub_owner, sub_repo) = github_repository_identity(source_url)?; + sources.push(SubmoduleSource { + path: path.clone(), + url: format!("https://github.com/{sub_owner}/{sub_repo}/archive/{gitlink}.tar.gz"), + // The immutable gitlink is the source identity. GitHub does not + // publish a SHA-256 for its generated archive. + sha256: None, + }); + } + Ok(sources) +} + +fn checked_submodule_dest(root: &Path, path: &str) -> fbuild_core::Result { + if path.starts_with('/') + || path + .split('/') + .any(|component| matches!(component, "" | "." | "..")) + || path.contains(['\\', '\n', '\r']) + { + return Err(package_error(format!("unsafe submodule path `{path}`"))); + } + let canonical_root = root.canonicalize().map_err(package_error)?; + let canonical_dest = root.join(path).canonicalize().map_err(package_error)?; + if !canonical_dest.starts_with(&canonical_root) { + return Err(package_error(format!( + "submodule `{path}` escapes the extracted package" + ))); + } + Ok(canonical_dest) +} + +async fn git_command(args: [&str; N]) -> fbuild_core::Result { + // allow-direct-spawn: Git resolves immutable VCS gitlinks without a shell + // or GitHub REST API quota, matching PlatformIO's VCS dependency. + let mut command = tokio::process::Command::new("git"); + command.args(args).kill_on_drop(true); + let output = tokio::time::timeout(std::time::Duration::from_secs(60), command.output()) + .await + .map_err(|_| package_error("timed out resolving GitHub submodule gitlink"))? + .map_err(package_error)?; + if !output.status.success() { + return Err(package_error(format!( + "git exited {}: {}", + output.status, + String::from_utf8_lossy(&output.stderr).trim() + ))); + } + String::from_utf8(output.stdout).map_err(package_error) +} + +fn github_archive_identity(url: &str) -> fbuild_core::Result<(String, String, String)> { + let parsed = reqwest::Url::parse(url).map_err(package_error)?; + if parsed.scheme() != "https" + || parsed.host_str() != Some("github.com") + || !parsed.username().is_empty() + || parsed.password().is_some() + || parsed.query().is_some() + || parsed.fragment().is_some() + { + return Err(package_error(format!("not a GitHub archive: {url}"))); + } + let parts: Vec<_> = parsed + .path_segments() + .ok_or_else(|| package_error("GitHub archive has no path"))? + .collect(); + let [owner, repo, "archive", file] = parts.as_slice() else { + return Err(package_error(format!( + "unsupported GitHub archive URL: {url}" + ))); + }; + let commit = file + .strip_suffix(".tar.gz") + .ok_or_else(|| package_error("GitHub archive is not a tar.gz"))?; + if !valid_github_segment(owner) || !valid_github_segment(repo) || !full_sha(commit) { + return Err(package_error(format!( + "invalid GitHub archive identity: {url}" + ))); + } + Ok(((*owner).into(), (*repo).into(), commit.into())) +} + +/// Whether a package URL names an immutable GitHub commit archive whose +/// submodule gitlinks can be resolved from the same parent commit. +pub fn is_github_commit_archive_url(url: &str) -> bool { + github_archive_identity(url).is_ok() +} + +fn github_repository_identity(url: &str) -> fbuild_core::Result<(String, String)> { + let parsed = reqwest::Url::parse(url).map_err(package_error)?; + if parsed.scheme() != "https" || parsed.host_str() != Some("github.com") { + return Err(package_error(format!( + "unsupported submodule repository: {url}" + ))); + } + let parts: Vec<_> = parsed + .path_segments() + .ok_or_else(|| package_error("submodule repository has no path"))? + .collect(); + let [owner, repo] = parts.as_slice() else { + return Err(package_error(format!( + "invalid submodule repository: {url}" + ))); + }; + let repo = repo.strip_suffix(".git").unwrap_or(repo); + if !valid_github_segment(owner) || !valid_github_segment(repo) { + return Err(package_error(format!( + "invalid submodule repository: {url}" + ))); + } + Ok(((*owner).into(), repo.into())) +} + +fn valid_github_segment(value: &str) -> bool { + !value.is_empty() + && value != "." + && value != ".." + && value + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.')) +} + +fn full_sha(value: &str) -> bool { + value.len() == 40 && value.bytes().all(|byte| byte.is_ascii_hexdigit()) +} + +fn declared_submodule_urls(gitmodules: &str) -> Vec<(String, String)> { + let mut result = Vec::new(); + let mut path = None; + let mut url = None; + for line in gitmodules.lines().chain(std::iter::once("[end]")) { + let line = line.trim(); + if line.starts_with('[') { + if let (Some(path), Some(url)) = (path.take(), url.take()) { + result.push((path, url)); + } + continue; + } + if let Some((key, value)) = line.split_once('=') { + match key.trim() { + "path" => path = Some(value.trim().to_string()), + "url" => url = Some(value.trim().to_string()), + _ => {} + } + } + } + result +} + +fn package_error(error: impl std::fmt::Display) -> fbuild_core::FbuildError { + fbuild_core::FbuildError::PackageError(error.to_string()) +} + /// Every plan entry must name a path `.gitmodules` declares and that /// extracted empty; anything else means upstream moved on. fn check_plan( @@ -323,7 +549,9 @@ async fn populate( async fn fetch_into(source: &SubmoduleSource, work: &Path, dest: &Path) -> fbuild_core::Result<()> { let archive = crate::downloader::download_file(&source.url, work).await?; - crate::downloader::verify_checksum(&archive, &source.sha256)?; + if let Some(sha256) = &source.sha256 { + crate::downloader::verify_checksum(&archive, sha256)?; + } let extracted = work.join("extracted"); std::fs::create_dir_all(&extracted)?; crate::extractor::extract(&archive, &extracted)?; @@ -374,14 +602,53 @@ mod tests { const TINYUSB: &str = "libraries/Adafruit_TinyUSB_Arduino"; + #[test] + fn gitlink_resolution_only_accepts_immutable_github_commit_archives() { + let sha = "0123456789abcdef0123456789abcdef01234567"; + assert!(is_github_commit_archive_url(&format!( + "https://github.com/openwch/arduino_core_ch32/archive/{sha}.tar.gz" + ))); + for url in [ + "https://registry.platformio.org/download/framework.tar.gz", + "https://github.com/openwch/arduino_core_ch32/archive/main.tar.gz", + "https://github.com/openwch/arduino_core_ch32/archive/0123456789abcdef0123456789abcdef01234567.tar.gz?unexpected=1", + "https://attacker@github.com/openwch/arduino_core_ch32/archive/0123456789abcdef0123456789abcdef01234567.tar.gz", + ] { + assert!(!is_github_commit_archive_url(url), "{url}"); + } + } + + #[test] + fn gitlink_paths_must_stay_inside_extracted_package() { + let root = tempfile::TempDir::new().unwrap(); + std::fs::create_dir(root.path().join("safe")).unwrap(); + assert_eq!( + checked_submodule_dest(root.path(), "safe").unwrap(), + root.path().join("safe").canonicalize().unwrap() + ); + for path in ["../escape", "/escape", "safe/../escape", "safe\\escape"] { + assert!(checked_submodule_dest(root.path(), path).is_err(), "{path}"); + } + } + + #[cfg(unix)] + #[test] + fn gitlink_paths_reject_symlink_escape() { + let root = tempfile::TempDir::new().unwrap(); + let outside = tempfile::TempDir::new().unwrap(); + std::os::unix::fs::symlink(outside.path(), root.path().join("outside")).unwrap(); + assert!(checked_submodule_dest(root.path(), "outside").is_err()); + } + fn tinyusb_plan() -> SubmodulePlan { SubmodulePlan { sources: vec![SubmoduleSource { path: TINYUSB.to_string(), url: "https://example.invalid/tinyusb.tar.gz".to_string(), - sha256: "0".repeat(64), + sha256: Some("0".repeat(64)), }], expected_empty: Vec::new(), + github_gitlinks: false, } } @@ -392,6 +659,7 @@ mod tests { path: path.to_string(), reason: "supplied another way".to_string(), }], + github_gitlinks: false, } } diff --git a/crates/fbuild-toolchain/src/toolchain/riscv.rs b/crates/fbuild-toolchain/src/toolchain/riscv.rs index 95f767c97..ab7727571 100644 --- a/crates/fbuild-toolchain/src/toolchain/riscv.rs +++ b/crates/fbuild-toolchain/src/toolchain/riscv.rs @@ -18,6 +18,7 @@ const RISCV_GCC_BASE_URL: &str = /// RISC-V GCC toolchain manager. pub struct RiscvToolchain { base: PackageBase, + executable_prefix: String, /// Resolved install path (set after ensure_installed) install_dir: Option, } @@ -35,10 +36,42 @@ impl RiscvToolchain { CacheSubdir::Toolchains, project_dir, ), + executable_prefix: "riscv-none-elf".into(), install_dir: None, } } + /// Use a selected PlatformIO toolchain package rather than the built-in + /// xPack release. Pass `auto` to discover the executable family from the + /// installed package instead of assuming that a registry pin uses xPack's + /// or one particular CH32V branch's layout. + pub fn with_platform_override( + project_dir: &Path, + override_package: fbuild_config::PackageOverride, + executable_prefix: &str, + ) -> Self { + let mut toolchain = Self::new(project_dir); + toolchain.base = toolchain.base.with_override(override_package); + toolchain.executable_prefix = executable_prefix.into(); + toolchain + } + + /// Executable family supplied by this package (`riscv-none-elf`, + /// `riscv-none-embed`, or `riscv-wch-elf`). CH32V's ISA spelling depends on + /// the selected compiler family. + pub fn executable_prefix(&self) -> String { + self.effective_prefix() + } + + fn effective_prefix(&self) -> String { + if self.executable_prefix == "auto" { + detect_tool_prefix(&self.resolved_dir().join("bin")) + .unwrap_or_else(|| "auto".to_string()) + } else { + self.executable_prefix.clone() + } + } + #[cfg(test)] fn with_cache_root(project_dir: &Path, cache_root: &Path) -> Self { let (url, checksum) = platform_package(); @@ -53,6 +86,7 @@ impl RiscvToolchain { project_dir, cache_root, ), + executable_prefix: "riscv-none-elf".into(), install_dir: None, } } @@ -77,6 +111,7 @@ impl RiscvToolchain { /// 6. `/riscv-none-elf/include/` — target system headers pub fn get_cxx_system_includes(&self, march: &str, mabi: &str) -> Vec { let root = self.resolved_dir(); + let prefix = self.effective_prefix(); let mut dirs = Vec::new(); // Ask GCC for the selected multilib instead of deriving its directory from @@ -86,7 +121,7 @@ impl RiscvToolchain { .unwrap_or_else(|| PathBuf::from(march.split('_').next().unwrap_or(march)).join(mabi)); // C++ headers: find the version directory dynamically - let cxx_base = root.join("riscv-none-elf").join("include").join("c++"); + let cxx_base = root.join(&prefix).join("include").join("c++"); if let Ok(entries) = std::fs::read_dir(&cxx_base) { for entry in entries.flatten() { let version_dir = entry.path(); @@ -94,7 +129,7 @@ impl RiscvToolchain { // 1. Base C++ headers dirs.push(version_dir.clone()); // 2. Multilib-specific - let multilib = multilib_include_path(&version_dir, &multilib_dir); + let multilib = multilib_include_path(&version_dir, &prefix, &multilib_dir); if multilib.is_dir() { dirs.push(multilib); } @@ -109,7 +144,7 @@ impl RiscvToolchain { } // GCC internal headers: find the version directory dynamically - let gcc_lib = root.join("lib").join("gcc").join("riscv-none-elf"); + let gcc_lib = root.join("lib").join("gcc").join(&prefix); if let Ok(entries) = std::fs::read_dir(&gcc_lib) { for entry in entries.flatten() { let ver_dir = entry.path(); @@ -130,7 +165,7 @@ impl RiscvToolchain { } // 6. Target system headers - let sys_inc = root.join("riscv-none-elf").join("include"); + let sys_inc = root.join(&prefix).join("include"); if sys_inc.is_dir() { dirs.push(sys_inc); } @@ -160,7 +195,7 @@ impl RiscvToolchain { } /// Validate that the toolchain installation has all required files. - fn validate(install_dir: &Path) -> fbuild_core::Result<()> { + fn validate(install_dir: &Path, executable_prefix: &str) -> fbuild_core::Result<()> { let root = find_bin_root(install_dir); let bin_dir = root.join("bin"); @@ -171,15 +206,21 @@ impl RiscvToolchain { ))); } - let required_tools = [ - "riscv-none-elf-gcc", - "riscv-none-elf-g++", - "riscv-none-elf-ar", - "riscv-none-elf-objcopy", - "riscv-none-elf-size", - ]; - for tool in &required_tools { - let tool_path = tool_binary(&bin_dir, tool); + let detected; + let executable_prefix = if executable_prefix == "auto" { + detected = detect_tool_prefix(&bin_dir).ok_or_else(|| { + fbuild_core::FbuildError::PackageError(format!( + "no complete RISC-V GCC tool suite found in {}", + bin_dir.display() + )) + })?; + detected.as_str() + } else { + executable_prefix + }; + for suffix in ["gcc", "g++", "ar", "objcopy", "size"] { + let tool = format!("{executable_prefix}-{suffix}"); + let tool_path = tool_binary(&bin_dir, &tool); if !tool_path.exists() { return Err(fbuild_core::FbuildError::PackageError(format!( "required tool {} not found at {}", @@ -193,8 +234,8 @@ impl RiscvToolchain { } } -fn multilib_include_path(version_dir: &Path, multilib_dir: &Path) -> PathBuf { - version_dir.join("riscv-none-elf").join(multilib_dir) +fn multilib_include_path(version_dir: &Path, prefix: &str, multilib_dir: &Path) -> PathBuf { + version_dir.join(prefix).join(multilib_dir) } #[async_trait::async_trait] @@ -204,7 +245,11 @@ impl crate::Package for RiscvToolchain { return Ok(self.resolved_dir()); } - let install_path = self.base.staged_install(Self::validate).await?; + let prefix = self.executable_prefix.clone(); + let install_path = self + .base + .staged_install(move |dir| Self::validate(dir, &prefix)) + .await?; Ok(find_bin_root(&install_path)) } @@ -213,8 +258,11 @@ impl crate::Package for RiscvToolchain { return false; } let root = find_bin_root(&self.base.install_path()); + if self.executable_prefix == "auto" { + return detect_tool_prefix(&root.join("bin")).is_some(); + } root.join("bin") - .join(tool_name("riscv-none-elf-gcc")) + .join(tool_name(&format!("{}-gcc", self.executable_prefix))) .exists() } @@ -225,23 +273,38 @@ impl crate::Package for RiscvToolchain { impl Toolchain for RiscvToolchain { fn get_gcc_path(&self) -> PathBuf { - tool_binary(&self.resolved_dir().join("bin"), "riscv-none-elf-gcc") + tool_binary( + &self.resolved_dir().join("bin"), + &format!("{}-gcc", self.effective_prefix()), + ) } fn get_gxx_path(&self) -> PathBuf { - tool_binary(&self.resolved_dir().join("bin"), "riscv-none-elf-g++") + tool_binary( + &self.resolved_dir().join("bin"), + &format!("{}-g++", self.effective_prefix()), + ) } fn get_ar_path(&self) -> PathBuf { - tool_binary(&self.resolved_dir().join("bin"), "riscv-none-elf-ar") + tool_binary( + &self.resolved_dir().join("bin"), + &format!("{}-ar", self.effective_prefix()), + ) } fn get_objcopy_path(&self) -> PathBuf { - tool_binary(&self.resolved_dir().join("bin"), "riscv-none-elf-objcopy") + tool_binary( + &self.resolved_dir().join("bin"), + &format!("{}-objcopy", self.effective_prefix()), + ) } fn get_size_path(&self) -> PathBuf { - tool_binary(&self.resolved_dir().join("bin"), "riscv-none-elf-size") + tool_binary( + &self.resolved_dir().join("bin"), + &format!("{}-size", self.effective_prefix()), + ) } fn get_bin_dir(&self) -> PathBuf { @@ -350,6 +413,33 @@ fn tool_binary(bin_dir: &Path, name: &str) -> PathBuf { bin_dir.join(tool_name(name)) } +/// Find one complete RISC-V compiler suite in an extracted PlatformIO +/// package. A package with multiple suites is ambiguous and must not be +/// selected by directory iteration order. +fn detect_tool_prefix(bin_dir: &Path) -> Option { + let gcc_suffix = tool_name("-gcc"); + let mut candidates = std::fs::read_dir(bin_dir) + .ok()? + .filter_map(Result::ok) + .filter_map(|entry| { + let name = entry.file_name().to_string_lossy().into_owned(); + let prefix = name.strip_suffix(&gcc_suffix)?; + if !prefix.starts_with("riscv") || !entry.path().is_file() { + return None; + } + ["g++", "ar", "objcopy", "size"] + .iter() + .all(|suffix| tool_binary(bin_dir, &format!("{prefix}-{suffix}")).is_file()) + .then(|| prefix.to_string()) + }) + .collect::>(); + if candidates.len() == 1 { + candidates.pop() + } else { + None + } +} + #[cfg(test)] mod tests { use super::*; @@ -372,6 +462,56 @@ mod tests { ); } + #[test] + fn ch32v_platform_toolchain_uses_wch_executable_prefix() { + let tmp = tempfile::TempDir::new().unwrap(); + let override_package = fbuild_config::PackageOverride::new( + "https://github.com/Community-PIO-CH32V/toolchain-riscv-linux/archive/fde267fb356efc11dee4c48ea58a1fd6dc787603.tar.gz", + "fde267fb356efc11dee4c48ea58a1fd6dc787603", + ); + let toolchain = + RiscvToolchain::with_platform_override(tmp.path(), override_package, "riscv-wch-elf"); + assert_eq!( + toolchain.get_gcc_path().file_name().unwrap(), + std::ffi::OsStr::new(&tool_name("riscv-wch-elf-gcc")) + ); + assert_eq!( + toolchain.get_ar_path().file_name().unwrap(), + std::ffi::OsStr::new(&tool_name("riscv-wch-elf-ar")) + ); + assert!(toolchain.get_info().url.contains("toolchain-riscv-linux")); + } + + #[test] + fn platformio_toolchain_prefix_comes_from_complete_installed_suite() { + let tmp = tempfile::TempDir::new().unwrap(); + let mut toolchain = RiscvToolchain::with_cache_root(tmp.path(), tmp.path()); + toolchain.executable_prefix = "auto".into(); + let bin = toolchain.base.install_path().join("bin"); + std::fs::create_dir_all(&bin).unwrap(); + for suffix in ["gcc", "g++", "ar", "objcopy", "size"] { + std::fs::write(tool_binary(&bin, &format!("riscv-none-embed-{suffix}")), "").unwrap(); + } + std::fs::write( + crate::disk_cache::paths::install_complete_sentinel(&toolchain.base.install_path()), + "", + ) + .unwrap(); + assert_eq!( + detect_tool_prefix(&bin).as_deref(), + Some("riscv-none-embed") + ); + assert!(toolchain.is_installed()); + assert_eq!(toolchain.executable_prefix(), "riscv-none-embed"); + assert_eq!( + toolchain.get_gcc_path(), + tool_binary(&bin, "riscv-none-embed-gcc") + ); + std::fs::remove_file(tool_binary(&bin, "riscv-none-embed-ar")).unwrap(); + assert_eq!(detect_tool_prefix(&bin), None); + assert!(!toolchain.is_installed()); + } + #[test] fn test_find_bin_root_direct() { let tmp = tempfile::TempDir::new().unwrap(); @@ -410,7 +550,7 @@ mod tests { fn test_multilib_include_path_maps_default_directory_to_sysroot() { let version_dir = Path::new("toolchain/include/c++/14.2.0"); assert_eq!( - multilib_include_path(version_dir, Path::new("")), + multilib_include_path(version_dir, "riscv-none-elf", Path::new("")), PathBuf::from("toolchain/include/c++/14.2.0/riscv-none-elf") ); } @@ -419,7 +559,11 @@ mod tests { fn test_multilib_include_path_preserves_extension_directory() { let version_dir = Path::new("toolchain/include/c++/14.2.0"); assert_eq!( - multilib_include_path(version_dir, Path::new("rv32imafc_zicsr/ilp32f")), + multilib_include_path( + version_dir, + "riscv-none-elf", + Path::new("rv32imafc_zicsr/ilp32f") + ), PathBuf::from("toolchain/include/c++/14.2.0/riscv-none-elf/rv32imafc_zicsr/ilp32f") ); } From 3ac992865cd068eed7b6dc43517772dab09a0fbb Mon Sep 17 00:00:00 2001 From: Zach Vorhies Date: Sun, 27 Sep 2026 15:42:15 -0700 Subject: [PATCH 2/5] test(ch32v): split orchestrator tests and annotate Git spawn --- .../src/ch32v/orchestrator.rs | 273 +----------------- .../src/ch32v/orchestrator_tests.rs | 269 +++++++++++++++++ .../fbuild-packages-fetch/src/submodules.rs | 4 +- 3 files changed, 273 insertions(+), 273 deletions(-) create mode 100644 crates/fbuild-build-mcu/src/ch32v/orchestrator_tests.rs diff --git a/crates/fbuild-build-mcu/src/ch32v/orchestrator.rs b/crates/fbuild-build-mcu/src/ch32v/orchestrator.rs index 57069a664..efc930d25 100644 --- a/crates/fbuild-build-mcu/src/ch32v/orchestrator.rs +++ b/crates/fbuild-build-mcu/src/ch32v/orchestrator.rs @@ -759,274 +759,5 @@ pub fn is_ch32v_project(project_dir: &Path, env_name: &str) -> bool { } #[cfg(test)] -mod tests { - use super::*; - - fn tempdir() -> tempfile::TempDir { - tempfile::TempDir::new_in(fbuild_paths::temp_subdir("fbuild-ch32v-tests")).unwrap() - } - - #[test] - fn test_ch32v_orchestrator_platform() { - let orch = Ch32vOrchestrator; - assert_eq!(orch.platform(), Platform::Ch32v); - } - - #[test] - fn explicit_board_isa_and_abi_override_selected_platform_defaults() { - let env = std::collections::HashMap::from([ - ("board_build.march".into(), "rv32imac".into()), - ("board_build.mabi".into(), "ilp32".into()), - ]); - assert_eq!( - preferred_board_value( - Some(&env), - "board_build.march", - Some("rv32ecxw"), - Some("rv32ec") - ), - Some("rv32imac") - ); - assert_eq!( - preferred_board_value( - Some(&env), - "board_build.mabi", - Some("ilp32e"), - Some("ilp32e") - ), - Some("ilp32") - ); - assert_eq!( - preferred_board_value(None, "board_build.march", Some("rv32ecxw"), Some("rv32ec")), - Some("rv32ecxw") - ); - let mut config = crate::ch32v::mcu_config::get_ch32v_config_for_mcu("ch32v003").unwrap(); - crate::ch32v::mcu_config::apply_board_isa_for_toolchain( - &mut config, - preferred_board_value( - Some(&env), - "board_build.march", - Some("rv32ecxw"), - Some("rv32ec"), - ), - preferred_board_value( - Some(&env), - "board_build.mabi", - Some("ilp32e"), - Some("ilp32e"), - ), - "riscv-none-embed", - ); - assert!( - config - .compiler_flags - .common - .contains(&"-march=rv32imac".into()) - ); - assert!(config.compiler_flags.common.contains(&"-mabi=ilp32".into())); - } - - #[test] - fn effective_board_settings_invalidate_fingerprint() { - let baseline = board_fingerprint_fields( - Some("rv32ecxw"), - Some("ilp32e"), - Some("variant_CH32V003F4.h"), - ); - let hash = crate::build_fingerprint::stable_hash_json(&baseline).unwrap(); - for changed in [ - board_fingerprint_fields( - Some("rv32imac"), - Some("ilp32e"), - Some("variant_CH32V003F4.h"), - ), - board_fingerprint_fields( - Some("rv32ecxw"), - Some("ilp32"), - Some("variant_CH32V003F4.h"), - ), - board_fingerprint_fields( - Some("rv32ecxw"), - Some("ilp32e"), - Some("variant_CH32V003J4.h"), - ), - ] { - assert_ne!( - hash, - crate::build_fingerprint::stable_hash_json(&changed).unwrap() - ); - } - } - - #[tokio::test(flavor = "multi_thread")] - #[ignore = "downloads CH32V platform, compiler, and framework packages"] - async fn pinned_github_platform_builds_ch32v003_with_gcc8() { - let backend = crate::compile_backend::CompileBackend::start() - .await - .expect("compile backend starts"); - crate::compile_backend::install_global(backend); - let project = tempdir(); - std::fs::create_dir_all(project.path().join("src")).unwrap(); - std::fs::write( - project.path().join("platformio.ini"), - "[env:ch32v003]\nplatform = https://github.com/Community-PIO-CH32V/platform-ch32v.git#b7397c29a71101175bfc94f6ab06f9daac336458\nboard = genericCH32V003F4P6\nframework = arduino\n", - ) - .unwrap(); - std::fs::copy( - concat!( - env!("CARGO_MANIFEST_DIR"), - "/../../tests/platform/ch32v003/ch32v003.ino" - ), - project.path().join("src/ch32v003.ino"), - ) - .unwrap(); - let build_dir = fbuild_paths::BuildLayout::new( - project.path().to_path_buf(), - "ch32v003".into(), - fbuild_core::BuildProfile::Release, - ) - .resolve(); - let params = BuildParams { - project_dir: project.path().to_path_buf(), - env_name: "ch32v003".into(), - clean_all: false, - clean_only: false, - clean: false, - profile: fbuild_core::BuildProfile::Release, - build_dir, - verbose: false, - jobs: Some(2), - generate_compiledb: false, - compiledb_only: false, - log_sender: None, - symbol_analysis: false, - symbol_analysis_path: None, - no_timestamp: true, - src_dir: None, - pio_env: std::collections::BTreeMap::new(), - extra_build_flags: Vec::new(), - watch_set_cache: None, - bloat_analysis: false, - caller_path: None, - }; - let built = Ch32vOrchestrator.build(¶ms).await.unwrap(); - assert!(built.success); - assert!(built.elf_path.as_ref().is_some_and(|path| path.is_file())); - let log = built.build_log.into_lines().join("\n"); - assert!(log.contains("CH32V requested: platform=https://github.com/Community-PIO-CH32V/platform-ch32v.git#b7397c29a71101175bfc94f6ab06f9daac336458")); - assert!( - log.contains("CH32V resolved platform: ch32v@1.1.0 (source_ref=b7397c29a71101175bfc94f6ab06f9daac336458") - ); - assert!(log.contains("toolchain-riscv-linux/archive/")); - assert!(log.contains("arduino_core_ch32/archive/")); - assert!(log.contains("8.2.0")); - } - - #[test] - fn test_is_ch32v_project() { - let tmp = tempdir(); - std::fs::write( - tmp.path().join("platformio.ini"), - "[env:ch32v003]\nplatform = ch32v\nboard = genericCH32V003F4P6\nframework = arduino\n", - ) - .unwrap(); - assert!(is_ch32v_project(tmp.path(), "ch32v003")); - assert!(!is_ch32v_project(tmp.path(), "uno")); - } - - #[test] - fn test_is_not_ch32v_project() { - let tmp = tempdir(); - std::fs::write( - tmp.path().join("platformio.ini"), - "[env:uno]\nplatform = atmelavr\nboard = uno\nframework = arduino\n", - ) - .unwrap(); - assert!(!is_ch32v_project(tmp.path(), "uno")); - } - - #[test] - fn test_validate_ch32v_framework() { - assert!(validate_ch32v_framework(None).is_ok()); - assert!(validate_ch32v_framework(Some("arduino")).is_ok()); - assert!(validate_ch32v_framework(Some(" arduino ")).is_ok()); - let error = validate_ch32v_framework(Some("noneos-sdk")).unwrap_err(); - assert!(error.to_string().contains("1108")); - } - - #[test] - fn test_sysclk_define_uses_series_spelling_and_clock_source() { - assert_eq!( - sysclk_define("ch32v203", "144000000L", "hsi+pll").unwrap(), - ( - "SYSCLK_FREQ_144MHz_HSI".to_string(), - "144000000".to_string() - ) - ); - assert_eq!( - sysclk_define("ch32v003", "48000000L", "hsi+pll").unwrap(), - ("SYSCLK_FREQ_48MHZ_HSI".to_string(), "48000000".to_string()) - ); - } - - /// ch32v006 must NOT get ch32v003's uppercase spelling. Its vendor file - /// declares the setter under `SYSCLK_FREQ_48MHz_HSI` but dispatches on - /// `SYSCLK_FREQ_48MHZ_HSI`, so the uppercase form selects a call to an - /// undeclared `SetSysClockTo_48MHZ_HSI` and the core fails to compile. - #[test] - fn test_sysclk_define_ch32v006_uses_lowercase_mhz() { - assert_eq!( - sysclk_define("ch32v006", "48000000L", "hsi+pll").unwrap(), - ("SYSCLK_FREQ_48MHz_HSI".to_string(), "48000000".to_string()) - ); - // HSE was never part of the uppercase carve-out; keep it lowercase too. - assert_eq!( - sysclk_define("ch32v006", "24000000L", "hse").unwrap(), - ("SYSCLK_FREQ_24MHz_HSE".to_string(), "24000000".to_string()) - ); - } - - #[test] - fn test_sysclk_define_rejects_unsupported_frequency() { - let error = sysclk_define("ch32v203", "8000000L", "hsi").unwrap_err(); - assert!(error.to_string().contains("supported values")); - } - - #[test] - fn test_series_to_system_dir() { - // CH32V series: last digit replaced with 'x' - assert_eq!(series_to_system_dir("ch32v003"), "CH32V00x"); - assert_eq!(series_to_system_dir("ch32v006"), "CH32VM00X"); - assert_eq!(series_to_system_dir("ch32v103"), "CH32V10x"); - assert_eq!(series_to_system_dir("ch32v203"), "CH32V20x"); - assert_eq!(series_to_system_dir("ch32v303"), "CH32V30x"); - assert_eq!(series_to_system_dir("ch32v307"), "CH32V30x"); - // CH32L follows the same family-directory pattern as CH32V. - assert_eq!(series_to_system_dir("ch32l103"), "CH32L10x"); - // CH32X: exact uppercase name - assert_eq!(series_to_system_dir("ch32x035"), "CH32X035"); - } - - #[test] - fn test_resolve_variant_dir_falls_back_to_family_variant() { - let tmp = tempdir(); - let fallback = tmp - .path() - .join("variants") - .join("CH32V00x") - .join("CH32V003F4"); - std::fs::create_dir_all(&fallback).unwrap(); - - let resolved = resolve_variant_dir(tmp.path(), "CH32V00x/CH32V006K8", "CH32V00x"); - assert_eq!(resolved, fallback); - } - - #[test] - fn test_resolve_variant_h_ignores_missing_preferred_header() { - let tmp = tempdir(); - std::fs::write(tmp.path().join("variant_CH32V003F4.h"), "").unwrap(); - - let resolved = resolve_variant_h(tmp.path(), Some("variant_CH32V006K8.h")); - assert_eq!(resolved.as_deref(), Some("variant_CH32V003F4.h")); - } -} +#[path = "orchestrator_tests.rs"] +mod tests; diff --git a/crates/fbuild-build-mcu/src/ch32v/orchestrator_tests.rs b/crates/fbuild-build-mcu/src/ch32v/orchestrator_tests.rs new file mode 100644 index 000000000..c8186cca6 --- /dev/null +++ b/crates/fbuild-build-mcu/src/ch32v/orchestrator_tests.rs @@ -0,0 +1,269 @@ +use super::*; + +fn tempdir() -> tempfile::TempDir { + tempfile::TempDir::new_in(fbuild_paths::temp_subdir("fbuild-ch32v-tests")).unwrap() +} + +#[test] +fn test_ch32v_orchestrator_platform() { + let orch = Ch32vOrchestrator; + assert_eq!(orch.platform(), Platform::Ch32v); +} + +#[test] +fn explicit_board_isa_and_abi_override_selected_platform_defaults() { + let env = std::collections::HashMap::from([ + ("board_build.march".into(), "rv32imac".into()), + ("board_build.mabi".into(), "ilp32".into()), + ]); + assert_eq!( + preferred_board_value( + Some(&env), + "board_build.march", + Some("rv32ecxw"), + Some("rv32ec") + ), + Some("rv32imac") + ); + assert_eq!( + preferred_board_value( + Some(&env), + "board_build.mabi", + Some("ilp32e"), + Some("ilp32e") + ), + Some("ilp32") + ); + assert_eq!( + preferred_board_value(None, "board_build.march", Some("rv32ecxw"), Some("rv32ec")), + Some("rv32ecxw") + ); + let mut config = crate::ch32v::mcu_config::get_ch32v_config_for_mcu("ch32v003").unwrap(); + crate::ch32v::mcu_config::apply_board_isa_for_toolchain( + &mut config, + preferred_board_value( + Some(&env), + "board_build.march", + Some("rv32ecxw"), + Some("rv32ec"), + ), + preferred_board_value( + Some(&env), + "board_build.mabi", + Some("ilp32e"), + Some("ilp32e"), + ), + "riscv-none-embed", + ); + assert!( + config + .compiler_flags + .common + .contains(&"-march=rv32imac".into()) + ); + assert!(config.compiler_flags.common.contains(&"-mabi=ilp32".into())); +} + +#[test] +fn effective_board_settings_invalidate_fingerprint() { + let baseline = board_fingerprint_fields( + Some("rv32ecxw"), + Some("ilp32e"), + Some("variant_CH32V003F4.h"), + ); + let hash = crate::build_fingerprint::stable_hash_json(&baseline).unwrap(); + for changed in [ + board_fingerprint_fields( + Some("rv32imac"), + Some("ilp32e"), + Some("variant_CH32V003F4.h"), + ), + board_fingerprint_fields( + Some("rv32ecxw"), + Some("ilp32"), + Some("variant_CH32V003F4.h"), + ), + board_fingerprint_fields( + Some("rv32ecxw"), + Some("ilp32e"), + Some("variant_CH32V003J4.h"), + ), + ] { + assert_ne!( + hash, + crate::build_fingerprint::stable_hash_json(&changed).unwrap() + ); + } +} + +#[tokio::test(flavor = "multi_thread")] +#[ignore = "downloads CH32V platform, compiler, and framework packages"] +async fn pinned_github_platform_builds_ch32v003_with_gcc8() { + let backend = crate::compile_backend::CompileBackend::start() + .await + .expect("compile backend starts"); + crate::compile_backend::install_global(backend); + let project = tempdir(); + std::fs::create_dir_all(project.path().join("src")).unwrap(); + std::fs::write( + project.path().join("platformio.ini"), + "[env:ch32v003]\nplatform = https://github.com/Community-PIO-CH32V/platform-ch32v.git#b7397c29a71101175bfc94f6ab06f9daac336458\nboard = genericCH32V003F4P6\nframework = arduino\n", + ) + .unwrap(); + std::fs::copy( + concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../tests/platform/ch32v003/ch32v003.ino" + ), + project.path().join("src/ch32v003.ino"), + ) + .unwrap(); + let build_dir = fbuild_paths::BuildLayout::new( + project.path().to_path_buf(), + "ch32v003".into(), + fbuild_core::BuildProfile::Release, + ) + .resolve(); + let params = BuildParams { + project_dir: project.path().to_path_buf(), + env_name: "ch32v003".into(), + clean_all: false, + clean_only: false, + clean: false, + profile: fbuild_core::BuildProfile::Release, + build_dir, + verbose: false, + jobs: Some(2), + generate_compiledb: false, + compiledb_only: false, + log_sender: None, + symbol_analysis: false, + symbol_analysis_path: None, + no_timestamp: true, + src_dir: None, + pio_env: std::collections::BTreeMap::new(), + extra_build_flags: Vec::new(), + watch_set_cache: None, + bloat_analysis: false, + caller_path: None, + }; + let built = Ch32vOrchestrator.build(¶ms).await.unwrap(); + assert!(built.success); + assert!(built.elf_path.as_ref().is_some_and(|path| path.is_file())); + let log = built.build_log.into_lines().join("\n"); + assert!(log.contains("CH32V requested: platform=https://github.com/Community-PIO-CH32V/platform-ch32v.git#b7397c29a71101175bfc94f6ab06f9daac336458")); + assert!(log.contains( + "CH32V resolved platform: ch32v@1.1.0 (source_ref=b7397c29a71101175bfc94f6ab06f9daac336458" + )); + assert!(log.contains("toolchain-riscv-linux/archive/")); + assert!(log.contains("arduino_core_ch32/archive/")); + assert!(log.contains("8.2.0")); +} + +#[test] +fn test_is_ch32v_project() { + let tmp = tempdir(); + std::fs::write( + tmp.path().join("platformio.ini"), + "[env:ch32v003]\nplatform = ch32v\nboard = genericCH32V003F4P6\nframework = arduino\n", + ) + .unwrap(); + assert!(is_ch32v_project(tmp.path(), "ch32v003")); + assert!(!is_ch32v_project(tmp.path(), "uno")); +} + +#[test] +fn test_is_not_ch32v_project() { + let tmp = tempdir(); + std::fs::write( + tmp.path().join("platformio.ini"), + "[env:uno]\nplatform = atmelavr\nboard = uno\nframework = arduino\n", + ) + .unwrap(); + assert!(!is_ch32v_project(tmp.path(), "uno")); +} + +#[test] +fn test_validate_ch32v_framework() { + assert!(validate_ch32v_framework(None).is_ok()); + assert!(validate_ch32v_framework(Some("arduino")).is_ok()); + assert!(validate_ch32v_framework(Some(" arduino ")).is_ok()); + let error = validate_ch32v_framework(Some("noneos-sdk")).unwrap_err(); + assert!(error.to_string().contains("1108")); +} + +#[test] +fn test_sysclk_define_uses_series_spelling_and_clock_source() { + assert_eq!( + sysclk_define("ch32v203", "144000000L", "hsi+pll").unwrap(), + ( + "SYSCLK_FREQ_144MHz_HSI".to_string(), + "144000000".to_string() + ) + ); + assert_eq!( + sysclk_define("ch32v003", "48000000L", "hsi+pll").unwrap(), + ("SYSCLK_FREQ_48MHZ_HSI".to_string(), "48000000".to_string()) + ); +} + +/// ch32v006 must NOT get ch32v003's uppercase spelling. Its vendor file +/// declares the setter under `SYSCLK_FREQ_48MHz_HSI` but dispatches on +/// `SYSCLK_FREQ_48MHZ_HSI`, so the uppercase form selects a call to an +/// undeclared `SetSysClockTo_48MHZ_HSI` and the core fails to compile. +#[test] +fn test_sysclk_define_ch32v006_uses_lowercase_mhz() { + assert_eq!( + sysclk_define("ch32v006", "48000000L", "hsi+pll").unwrap(), + ("SYSCLK_FREQ_48MHz_HSI".to_string(), "48000000".to_string()) + ); + // HSE was never part of the uppercase carve-out; keep it lowercase too. + assert_eq!( + sysclk_define("ch32v006", "24000000L", "hse").unwrap(), + ("SYSCLK_FREQ_24MHz_HSE".to_string(), "24000000".to_string()) + ); +} + +#[test] +fn test_sysclk_define_rejects_unsupported_frequency() { + let error = sysclk_define("ch32v203", "8000000L", "hsi").unwrap_err(); + assert!(error.to_string().contains("supported values")); +} + +#[test] +fn test_series_to_system_dir() { + // CH32V series: last digit replaced with 'x' + assert_eq!(series_to_system_dir("ch32v003"), "CH32V00x"); + assert_eq!(series_to_system_dir("ch32v006"), "CH32VM00X"); + assert_eq!(series_to_system_dir("ch32v103"), "CH32V10x"); + assert_eq!(series_to_system_dir("ch32v203"), "CH32V20x"); + assert_eq!(series_to_system_dir("ch32v303"), "CH32V30x"); + assert_eq!(series_to_system_dir("ch32v307"), "CH32V30x"); + // CH32L follows the same family-directory pattern as CH32V. + assert_eq!(series_to_system_dir("ch32l103"), "CH32L10x"); + // CH32X: exact uppercase name + assert_eq!(series_to_system_dir("ch32x035"), "CH32X035"); +} + +#[test] +fn test_resolve_variant_dir_falls_back_to_family_variant() { + let tmp = tempdir(); + let fallback = tmp + .path() + .join("variants") + .join("CH32V00x") + .join("CH32V003F4"); + std::fs::create_dir_all(&fallback).unwrap(); + + let resolved = resolve_variant_dir(tmp.path(), "CH32V00x/CH32V006K8", "CH32V00x"); + assert_eq!(resolved, fallback); +} + +#[test] +fn test_resolve_variant_h_ignores_missing_preferred_header() { + let tmp = tempdir(); + std::fs::write(tmp.path().join("variant_CH32V003F4.h"), "").unwrap(); + + let resolved = resolve_variant_h(tmp.path(), Some("variant_CH32V006K8.h")); + assert_eq!(resolved.as_deref(), Some("variant_CH32V003F4.h")); +} diff --git a/crates/fbuild-packages-fetch/src/submodules.rs b/crates/fbuild-packages-fetch/src/submodules.rs index ffc9cea3c..a7a1dddfa 100644 --- a/crates/fbuild-packages-fetch/src/submodules.rs +++ b/crates/fbuild-packages-fetch/src/submodules.rs @@ -348,8 +348,8 @@ fn checked_submodule_dest(root: &Path, path: &str) -> fbuild_core::Result(args: [&str; N]) -> fbuild_core::Result { - // allow-direct-spawn: Git resolves immutable VCS gitlinks without a shell - // or GitHub REST API quota, matching PlatformIO's VCS dependency. + // Git avoids GitHub REST API quota for PlatformIO's VCS dependencies. + // allow-direct-spawn: resolve immutable gitlinks with Git, without a shell. let mut command = tokio::process::Command::new("git"); command.args(args).kill_on_drop(true); let output = tokio::time::timeout(std::time::Duration::from_secs(60), command.output()) From b5cfed5870b335bce8bc9d8909143a1156511110 Mon Sep 17 00:00:00 2001 From: Zach Vorhies Date: Sun, 27 Sep 2026 15:44:07 -0700 Subject: [PATCH 3/5] test(packages): use platform symlink facade in gitlink test --- crates/fbuild-packages-fetch/src/submodules.rs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/crates/fbuild-packages-fetch/src/submodules.rs b/crates/fbuild-packages-fetch/src/submodules.rs index a7a1dddfa..31bdd18fe 100644 --- a/crates/fbuild-packages-fetch/src/submodules.rs +++ b/crates/fbuild-packages-fetch/src/submodules.rs @@ -631,12 +631,19 @@ mod tests { } } - #[cfg(unix)] #[test] fn gitlink_paths_reject_symlink_escape() { let root = tempfile::TempDir::new().unwrap(); let outside = tempfile::TempDir::new().unwrap(); - std::os::unix::fs::symlink(outside.path(), root.path().join("outside")).unwrap(); + if let Err(error) = + fbuild_core::platform::fs::symlink_dir(outside.path(), &root.path().join("outside")) + { + // Windows symlink creation can require a privilege absent on CI runners. + if error.kind() == std::io::ErrorKind::PermissionDenied { + return; + } + panic!("failed to create test symlink: {error}"); + } assert!(checked_submodule_dest(root.path(), "outside").is_err()); } From cde5ab0a653ea9aabf2b033860bb5a15e397fb29 Mon Sep 17 00:00:00 2001 From: Zach Vorhies Date: Sun, 27 Sep 2026 15:47:14 -0700 Subject: [PATCH 4/5] fix(ci): distinguish YAML toolchain pins from Rust fields --- ci/check_rust_toolchain_pins.py | 4 ++++ ci/test_rust_toolchain_pins.py | 8 ++++++++ 2 files changed, 12 insertions(+) diff --git a/ci/check_rust_toolchain_pins.py b/ci/check_rust_toolchain_pins.py index 9a93d67ed..305bffef1 100644 --- a/ci/check_rust_toolchain_pins.py +++ b/ci/check_rust_toolchain_pins.py @@ -100,6 +100,10 @@ def validate_discovered_pins(relative: str, text: str) -> list[str]: for line_number, line in enumerate(text.splitlines(), 1): versions: set[str] = set() for pattern in patterns: + if pattern is PIN_FIELD_PATTERNS[1] and Path(relative).suffix not in {".yml", ".yaml"}: + # `toolchain:` is also an ordinary Rust struct field; only + # workflow YAML declares a Rust toolchain with that spelling. + continue match = pattern.search(line) if match is not None: versions.add(match.group("selector")) diff --git a/ci/test_rust_toolchain_pins.py b/ci/test_rust_toolchain_pins.py index fba3361f9..342250bdc 100644 --- a/ci/test_rust_toolchain_pins.py +++ b/ci/test_rust_toolchain_pins.py @@ -61,6 +61,14 @@ def test_unrelated_toolchain_package_version_is_not_a_rust_pin(self) -> None: self.assertEqual(failures, []) + def test_rust_toolchain_struct_field_is_not_a_workflow_pin(self) -> None: + failures = check_rust_toolchain_pins.validate_discovered_pins( + "crates/example/src/platform_source.rs", + " toolchain: require_platform_package(&requirements, name),\n", + ) + + self.assertEqual(failures, []) + if __name__ == "__main__": unittest.main() From bd031d687bc4963b259b341fc360d8fc71ab3727 Mon Sep 17 00:00:00 2001 From: Zach Vorhies Date: Sun, 27 Sep 2026 16:08:21 -0700 Subject: [PATCH 5/5] fix(ch32v): cache source locks and satisfy Dylint policies --- crates/fbuild-build-mcu/src/ch32v/mod.rs | 1 + .../src/ch32v/orchestrator.rs | 12 +- .../src/ch32v/platform_source.rs | 70 +++---- .../src/platformio_repository.rs | 181 ++++++++++++++++-- .../fbuild-packages-fetch/src/submodules.rs | 62 ++++-- 5 files changed, 252 insertions(+), 74 deletions(-) diff --git a/crates/fbuild-build-mcu/src/ch32v/mod.rs b/crates/fbuild-build-mcu/src/ch32v/mod.rs index 6df5c857d..03449eaac 100644 --- a/crates/fbuild-build-mcu/src/ch32v/mod.rs +++ b/crates/fbuild-build-mcu/src/ch32v/mod.rs @@ -33,6 +33,7 @@ impl crate::PlatformSupport for Ch32vPlatformSupport { inputs.project_dir, Some(inputs.env_config), &inputs.board.core, + mode.fetches(), ) .await?; Ok(vec![ diff --git a/crates/fbuild-build-mcu/src/ch32v/orchestrator.rs b/crates/fbuild-build-mcu/src/ch32v/orchestrator.rs index efc930d25..ead08e775 100644 --- a/crates/fbuild-build-mcu/src/ch32v/orchestrator.rs +++ b/crates/fbuild-build-mcu/src/ch32v/orchestrator.rs @@ -83,7 +83,7 @@ impl BuildOrchestrator for Ch32vOrchestrator { // 3-4. RISC-V GCC toolchain and OpenWCH CH32V cores let (toolchain, framework, selected_board, platform_resolution) = - ch32v_packages(¶ms.project_dir, env_config, &ctx.board.core).await?; + ch32v_packages(¶ms.project_dir, env_config, &ctx.board.core, false).await?; use fbuild_packages::Package as _; let toolchain_info = toolchain.get_info(); let framework_info = framework.get_info(); @@ -495,6 +495,7 @@ pub(crate) async fn ch32v_packages( project_dir: &Path, env_config: Option<&std::collections::HashMap>, board_core: &str, + refresh: bool, ) -> fbuild_core::Result<( fbuild_packages::toolchain::RiscvToolchain, fbuild_packages::library::Ch32vCores, @@ -506,8 +507,13 @@ pub(crate) async fn ch32v_packages( .get("board_build.core") .map(String::as_str) .unwrap_or(board_core); - if let Some(packages) = - super::platform_source::resolve_source_packages(project_dir, env, selected_core).await? + if let Some(packages) = super::platform_source::resolve_source_packages( + project_dir, + env, + selected_core, + refresh, + ) + .await? { return Ok((packages.0, packages.1, Some(packages.2), Some(packages.3))); } diff --git a/crates/fbuild-build-mcu/src/ch32v/platform_source.rs b/crates/fbuild-build-mcu/src/ch32v/platform_source.rs index db373a08e..aada13b9a 100644 --- a/crates/fbuild-build-mcu/src/ch32v/platform_source.rs +++ b/crates/fbuild-build-mcu/src/ch32v/platform_source.rs @@ -1,9 +1,10 @@ //! Select the packages described by a CH32V PlatformIO source checkout. use std::collections::HashMap; -use std::path::{Path, PathBuf}; +use std::path::Path; use fbuild_config::PackageOverride; +use fbuild_core::path::NormalizedPath; use fbuild_core::platformio_package::{ PackageKind, PackageRequirement, PackageSource, PackageSpec, parse_package_spec, require_platform_package, resolve_platform_requirements, @@ -32,6 +33,7 @@ pub(crate) async fn resolve_source_packages( project_dir: &Path, env: &HashMap, board_core: &str, + refresh: bool, ) -> fbuild_core::Result< Option<( fbuild_packages::toolchain::RiscvToolchain, @@ -64,18 +66,12 @@ pub(crate) async fn resolve_source_packages( payload.cache_identity(), ) } - PackageSource::Repository { .. } => { - let resolved = - fbuild_packages::platformio_repository::resolve_github_repository(&platform.source) - .await - .map_err(package_error)?; - let identity = resolved.lock.cache_identity(); - (resolved.archive, identity) - } - PackageSource::Archive { url, revision } => { - let resolved = fbuild_packages::platformio_repository::resolve_archive_source( - url, - revision.as_deref(), + PackageSource::Repository { .. } | PackageSource::Archive { .. } => { + let cache_root = fbuild_packages::Cache::new(project_dir).platforms_dir(); + let resolved = fbuild_packages::platformio_repository::resolve_cached_source( + &platform.source, + &cache_root, + refresh, ) .await .map_err(package_error)?; @@ -160,6 +156,7 @@ pub(crate) async fn resolve_source_packages( &selected.toolchain.spec, PackageKind::Tool, host, + refresh, ) .await?; let framework_override = package_override_for_spec( @@ -167,6 +164,7 @@ pub(crate) async fn resolve_source_packages( &selected.framework.spec, PackageKind::Framework, host, + refresh, ) .await?; tracing::info!( @@ -267,13 +265,13 @@ fn is_legacy_default(registry: &fbuild_core::platformio_package::RegistrySpec) - && registry.name == "ch32v" } -fn find_platform_manifest(root: &Path) -> Option { +fn find_platform_manifest(root: &Path) -> Option { walkdir::WalkDir::new(root) .max_depth(3) .into_iter() .filter_map(Result::ok) .find(|entry| entry.file_type().is_file() && entry.file_name() == "platform.json") - .map(|entry| entry.path().to_path_buf()) + .map(|entry| NormalizedPath::from(entry.path())) } fn parse_explicit_packages(env: &HashMap) -> fbuild_core::Result> { @@ -291,6 +289,7 @@ async fn package_override_for_spec( spec: &PackageSpec, kind: PackageKind, host: &str, + refresh: bool, ) -> fbuild_core::Result { match &spec.source { PackageSource::Registry(registry) => { @@ -306,21 +305,19 @@ async fn package_override_for_spec( checksum: Some(payload.sha256), }) } - PackageSource::Repository { .. } => Ok( - fbuild_packages::platformio_repository::resolve_github_repository(&spec.source) + PackageSource::Repository { .. } | PackageSource::Archive { .. } => { + let cache_root = fbuild_packages::Cache::new(project_dir).platforms_dir(); + Ok( + fbuild_packages::platformio_repository::resolve_cached_source( + &spec.source, + &cache_root, + refresh, + ) .await .map_err(package_error)? .archive, - ), - PackageSource::Archive { url, revision } => Ok( - fbuild_packages::platformio_repository::resolve_archive_source( - url, - revision.as_deref(), ) - .await - .map_err(package_error)? - .archive, - ), + } PackageSource::LocalPath { .. } => Err(package_error( "local CH32V package directories are not supported by the native adapter", )), @@ -640,7 +637,7 @@ mod tests { "https://github.com/Community-PIO-CH32V/platform-ch32v.git#b7397c29a71101175bfc94f6ab06f9daac336458".to_string(), ), ("board".to_string(), "genericCH32V003F4P6".to_string())]); let (toolchain, framework, board_isa, platform_resolution) = - resolve_source_packages(project.path(), &env, "openwch") + resolve_source_packages(project.path(), &env, "openwch", false) .await .unwrap() .unwrap(); @@ -660,12 +657,17 @@ mod tests { assert!(toolchain_root.is_dir()); assert!(framework_root.is_dir()); assert!(toolchain.get_gcc_path().is_file()); - // allow-direct-spawn: integration test probes the selected compiler binary. - let version = std::process::Command::new(toolchain.get_gcc_path()) - .arg("--version") - .output() - .unwrap(); - assert!(version.status.success()); - assert!(String::from_utf8_lossy(&version.stdout).contains("8.2.0")); + let gcc = toolchain.get_gcc_path(); + let gcc = gcc.to_str().expect("compiler path is UTF-8"); + let version = fbuild_core::subprocess::run_command( + &[gcc, "--version"], + None, + None, + Some(std::time::Duration::from_secs(10)), + ) + .await + .unwrap(); + assert!(version.success()); + assert!(version.stdout.contains("8.2.0")); } } diff --git a/crates/fbuild-packages-fetch/src/platformio_repository.rs b/crates/fbuild-packages-fetch/src/platformio_repository.rs index e8d0a8f36..3182d05e3 100644 --- a/crates/fbuild-packages-fetch/src/platformio_repository.rs +++ b/crates/fbuild-packages-fetch/src/platformio_repository.rs @@ -3,6 +3,8 @@ use fbuild_config::PackageOverride; use fbuild_core::platformio_package::{PackageLock, PackageSource}; use sha2::{Digest, Sha256}; +use std::io::Write; +use std::path::Path; use std::time::Duration; use tokio::io::AsyncReadExt; @@ -18,6 +20,8 @@ pub enum RepositoryError { InvalidCommit(String), #[error("PlatformIO archive resolution failed: {0}")] Archive(String), + #[error("PlatformIO source lock storage failed: {0}")] + Storage(String), } #[derive(Debug, Clone, PartialEq, Eq)] @@ -32,6 +36,116 @@ pub struct ResolvedArchive { pub archive: PackageOverride, } +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ResolvedSource { + pub lock: PackageLock, + pub archive: PackageOverride, +} + +/// Reuse a previously resolved source identity for ordinary builds. Explicit +/// install refreshes it; a changed request receives a different cache key. +/// This keeps warm/offline builds from fetching mutable archives or running +/// `git ls-remote` before their firmware fast-path check. +pub async fn resolve_cached_source( + source: &PackageSource, + cache_root: &Path, + refresh: bool, +) -> Result { + if !matches!( + source, + PackageSource::Repository { .. } | PackageSource::Archive { .. } + ) { + return Err(RepositoryError::UnsupportedUrl(format!("{source:?}"))); + } + let request = + serde_json::to_vec(source).map_err(|error| RepositoryError::Storage(error.to_string()))?; + let key = format!("{:x}", Sha256::digest(request)); + let directory = cache_root.join("platformio-source-locks"); + let path = directory.join(format!("{key}.json")); + if !refresh { + match std::fs::read(&path) { + Ok(bytes) => { + let lock: PackageLock = serde_json::from_slice(&bytes) + .map_err(|error| RepositoryError::Storage(error.to_string()))?; + return source_from_lock(source, lock); + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(RepositoryError::Storage(error.to_string())), + } + } + let resolved = match source { + PackageSource::Repository { .. } => { + let result = resolve_github_repository(source).await?; + ResolvedSource { + lock: result.lock, + archive: result.archive, + } + } + PackageSource::Archive { url, revision } => { + let result = resolve_archive_source(url, revision.as_deref()).await?; + ResolvedSource { + lock: result.lock, + archive: result.archive, + } + } + _ => unreachable!("source kind checked above"), + }; + std::fs::create_dir_all(&directory) + .map_err(|error| RepositoryError::Storage(error.to_string()))?; + let mut staged = tempfile::NamedTempFile::new_in(&directory) + .map_err(|error| RepositoryError::Storage(error.to_string()))?; + staged + .write_all( + &serde_json::to_vec(&resolved.lock) + .map_err(|error| RepositoryError::Storage(error.to_string()))?, + ) + .map_err(|error| RepositoryError::Storage(error.to_string()))?; + staged + .persist(&path) + .map_err(|error| RepositoryError::Storage(error.error.to_string()))?; + Ok(resolved) +} + +fn source_from_lock( + source: &PackageSource, + lock: PackageLock, +) -> Result { + let archive = match (source, &lock) { + ( + PackageSource::Repository { url, .. }, + PackageLock::Repository { + url: locked_url, + commit, + }, + ) if url == locked_url && is_full_sha(commit) => { + let (owner, repo) = github_owner_repo(url)?; + github_archive(url, &owner, &repo, commit).archive + } + ( + PackageSource::Archive { url, revision }, + PackageLock::Archive { + url: locked_url, + sha256, + }, + ) if url == locked_url + && sha256.len() == 64 + && sha256.bytes().all(|byte| byte.is_ascii_hexdigit()) => + { + PackageOverride { + url: url.clone(), + version: revision.clone().unwrap_or_else(|| "archive".into()), + checksum: Some(sha256.clone()), + } + } + _ => { + return Err(RepositoryError::Storage( + "cached source lock does not match request".into(), + )); + } + }; + Ok(ResolvedSource { lock, archive }) +} + /// Fetch a mutable archive URL to establish its content identity before it /// becomes a package-cache key. The later staged install verifies this digest, /// so a server changing between resolution and installation fails closed. @@ -39,8 +153,8 @@ pub async fn resolve_archive_source( url: &str, revision: Option<&str>, ) -> Result { - let temp = - tempfile::TempDir::new().map_err(|error| RepositoryError::Archive(error.to_string()))?; + let temp = tempfile::TempDir::new_in(fbuild_paths::temp_subdir("platformio-archive-resolve")) + .map_err(|error| RepositoryError::Archive(error.to_string()))?; let path = crate::downloader::download_file(url, temp.path()) .await .map_err(|error| RepositoryError::Archive(error.to_string()))?; @@ -91,23 +205,16 @@ pub async fn resolve_github_repository( reference.to_ascii_lowercase() } else { let candidates = candidate_refs(reference); - // allow-direct-spawn: PlatformIO VCS package ref lookup, no shell. - let mut command = tokio::process::Command::new("git"); - command - .args(["ls-remote", url]) - .args(&candidates) - .kill_on_drop(true); - let output = tokio::time::timeout(Duration::from_secs(30), command.output()) - .await - .map_err(|_| RepositoryError::Git(format!("timed out resolving {reference}")))? - .map_err(|error| RepositoryError::Git(error.to_string()))?; - if !output.status.success() { - return Err(RepositoryError::Git( - String::from_utf8_lossy(&output.stderr).trim().to_string(), - )); + let mut args = vec!["git", "ls-remote", url]; + args.extend(candidates.iter().map(String::as_str)); + let output = + fbuild_core::subprocess::run_command(&args, None, None, Some(Duration::from_secs(30))) + .await + .map_err(|error| RepositoryError::Git(error.to_string()))?; + if !output.success() { + return Err(RepositoryError::Git(output.stderr.trim().to_string())); } - let stdout = String::from_utf8_lossy(&output.stdout); - let commit = select_remote_commit(&stdout, &candidates) + let commit = select_remote_commit(&output.stdout, &candidates) .ok_or_else(|| RepositoryError::InvalidRef(reference.into()))?; if !is_full_sha(commit) { return Err(RepositoryError::InvalidCommit(commit.into())); @@ -241,6 +348,44 @@ mod tests { assert_ne!(first.lock.cache_identity(), second.lock.cache_identity()); } + #[tokio::test] + async fn cached_archive_lock_supports_offline_rebuild_and_explicit_refresh() { + let bytes = std::sync::Arc::new(tokio::sync::RwLock::new(b"first archive".to_vec())); + let served = bytes.clone(); + let app = axum::Router::new().route( + "/platform.tar.gz", + axum::routing::get(move || { + let served = served.clone(); + async move { axum::body::Bytes::from(served.read().await.clone()) } + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = format!("http://{}/platform.tar.gz", listener.local_addr().unwrap()); + let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + let source = PackageSource::Archive { + url, + revision: Some("test-revision".into()), + }; + let cache = tempfile::TempDir::new().unwrap(); + let first = resolve_cached_source(&source, cache.path(), false) + .await + .unwrap(); + *bytes.write().await = b"second archive".to_vec(); + let warm = resolve_cached_source(&source, cache.path(), false) + .await + .unwrap(); + assert_eq!(first, warm, "warm builds must keep the locked payload"); + let refreshed = resolve_cached_source(&source, cache.path(), true) + .await + .unwrap(); + assert_ne!(first.lock, refreshed.lock); + server.abort(); + let offline = resolve_cached_source(&source, cache.path(), false) + .await + .unwrap(); + assert_eq!(offline, refreshed, "cached rebuild must work offline"); + } + #[tokio::test] async fn immutable_ch32v_platform_source_becomes_archive_lock() { let spec = parse_package_spec("https://github.com/Community-PIO-CH32V/platform-ch32v.git#b7397c29a71101175bfc94f6ab06f9daac336458").unwrap(); diff --git a/crates/fbuild-packages-fetch/src/submodules.rs b/crates/fbuild-packages-fetch/src/submodules.rs index 31bdd18fe..c519c3fb5 100644 --- a/crates/fbuild-packages-fetch/src/submodules.rs +++ b/crates/fbuild-packages-fetch/src/submodules.rs @@ -160,7 +160,9 @@ pub struct SubmodulePlan { impl SubmodulePlan { fn is_empty(&self) -> bool { - self.sources.is_empty() && self.expected_empty.is_empty() && !self.github_gitlinks + // Dynamic gitlink resolution is optional when the archive declares + // no submodules; only explicit entries imply a stale plan. + self.sources.is_empty() && self.expected_empty.is_empty() } } @@ -269,7 +271,9 @@ async fn github_submodule_sources( declared: &[String], ) -> fbuild_core::Result> { let (owner, repo, commit) = github_archive_identity(parent_archive)?; - let checkout = tempfile::TempDir::new().map_err(package_error)?; + let checkout = + tempfile::TempDir::new_in(fbuild_paths::temp_subdir("platformio-gitlink-resolve")) + .map_err(package_error)?; let git_dir = checkout .path() .to_str() @@ -328,7 +332,7 @@ async fn github_submodule_sources( Ok(sources) } -fn checked_submodule_dest(root: &Path, path: &str) -> fbuild_core::Result { +fn checked_submodule_dest(root: &Path, path: &str) -> fbuild_core::Result { if path.starts_with('/') || path .split('/') @@ -337,9 +341,10 @@ fn checked_submodule_dest(root: &Path, path: &str) -> fbuild_core::Result fbuild_core::Result(args: [&str; N]) -> fbuild_core::Result { - // Git avoids GitHub REST API quota for PlatformIO's VCS dependencies. - // allow-direct-spawn: resolve immutable gitlinks with Git, without a shell. - let mut command = tokio::process::Command::new("git"); - command.args(args).kill_on_drop(true); - let output = tokio::time::timeout(std::time::Duration::from_secs(60), command.output()) - .await - .map_err(|_| package_error("timed out resolving GitHub submodule gitlink"))? - .map_err(package_error)?; - if !output.status.success() { + let mut command = vec!["git"]; + command.extend(args); + let output = fbuild_core::subprocess::run_command( + &command, + None, + None, + Some(std::time::Duration::from_secs(60)), + ) + .await?; + if !output.success() { return Err(package_error(format!( "git exited {}: {}", - output.status, - String::from_utf8_lossy(&output.stderr).trim() + output.exit_code, + output.stderr.trim() ))); } - String::from_utf8(output.stdout).map_err(package_error) + Ok(output.stdout) } fn github_archive_identity(url: &str) -> fbuild_core::Result<(String, String, String)> { @@ -624,7 +630,7 @@ mod tests { std::fs::create_dir(root.path().join("safe")).unwrap(); assert_eq!( checked_submodule_dest(root.path(), "safe").unwrap(), - root.path().join("safe").canonicalize().unwrap() + NormalizedPath::from(root.path().join("safe").canonicalize().unwrap()) ); for path in ["../escape", "/escape", "safe/../escape", "safe\\escape"] { assert!(checked_submodule_dest(root.path(), path).is_err(), "{path}"); @@ -732,6 +738,24 @@ mod tests { assert!(find_empty_submodules(tmp.path()).is_empty()); } + #[tokio::test] + async fn dynamic_gitlinks_allow_an_archive_without_gitmodules() { + let tmp = tempfile::TempDir::new().unwrap(); + write(tmp.path(), "cores/arduino/main.cpp", "int main(){}"); + let plan = SubmodulePlan { + github_gitlinks: true, + ..SubmodulePlan::default() + }; + prepare_submodules( + "core", + "https://example.invalid/core.tar.gz", + tmp.path(), + &plan, + ) + .await + .unwrap(); + } + /// A declared submodule whose directory is missing entirely is a /// different failure — an incomplete extract, not a submodule-less /// archive. Reporting it here would send the reader after the wrong