perf(test): consolidate all integration tests into 8 category binaries (#1577)

.github/workflows/acceptance-205.yml

@@ -51,30 +51,39 @@ jobs:
             test_fn: teensy41_cold_library_selection_meets_205_ac6
           # FastLED/fbuild#1588: GCC resolves the same headers with and without the include farm.
           - gate: esp32dev include-farm parity
+            test_target: env_isolated
             test_bin: esp32_include_farm_parity
             test_fn: farm_parity_esp32dev
           - gate: esp32s2 include-farm parity
+            test_target: env_isolated
             test_bin: esp32_include_farm_parity
             test_fn: farm_parity_esp32s2
           - gate: esp32s3 include-farm parity
+            test_target: env_isolated
             test_bin: esp32_include_farm_parity
             test_fn: farm_parity_esp32s3
           - gate: esp32c2 include-farm parity
+            test_target: env_isolated
             test_bin: esp32_include_farm_parity
             test_fn: farm_parity_esp32c2
           - gate: esp32c3 include-farm parity
+            test_target: env_isolated
             test_bin: esp32_include_farm_parity
             test_fn: farm_parity_esp32c3
           - gate: esp32c5 include-farm parity
+            test_target: env_isolated
             test_bin: esp32_include_farm_parity
             test_fn: farm_parity_esp32c5
           - gate: esp32c6 include-farm parity
+            test_target: env_isolated
             test_bin: esp32_include_farm_parity
             test_fn: farm_parity_esp32c6
           - gate: esp32h2 include-farm parity
+            test_target: env_isolated
             test_bin: esp32_include_farm_parity
             test_fn: farm_parity_esp32h2
           - gate: esp32p4 include-farm parity
+            test_target: env_isolated
             test_bin: esp32_include_farm_parity
             test_fn: farm_parity_esp32p4
     steps:
@@ -101,4 +110,4 @@ jobs:
 
       - name: Run ${{ matrix.gate }} acceptance gate
         run: |
-          soldr cargo test -p fbuild-build --test it -- --ignored --exact ${{ matrix.test_bin }}::${{ matrix.test_fn }} --nocapture --test-threads=1
+          soldr cargo test -p fbuild-build --test ${{ matrix.test_target || 'it' }} -- --ignored --exact ${{ matrix.test_bin }}::${{ matrix.test_fn }} --nocapture --test-threads=1

.github/workflows/check-ubuntu.yml

@@ -44,6 +44,11 @@ jobs:
         id: setup-soldr
         uses: zackees/setup-soldr@67ed4018aca013f8388050ac9bc264244f9b742c
         with:
+          # Own build-cache store (setup-soldr#237): with no suffix this job
+          # shared one Cargo.lock-keyed entry with ~10 other workflows, and
+          # the first small job to save it (165 MB) froze it -- an exact hit
+          # never re-saves -- so every run recompiled ~240 crates.
+          cache-key-suffix: check-ubuntu
           cache: true
           build-cache: true
           target-cache: true
@@ -103,6 +108,7 @@ jobs:
       - name: Setup soldr
         uses: zackees/setup-soldr@67ed4018aca013f8388050ac9bc264244f9b742c
         with:
+          cache-key-suffix: python-facades
           cache: true
           build-cache: true
           target-cache: true

.github/workflows/crate-gate.yml

@@ -39,3 +39,8 @@ jobs:
         # which requires `soldr` (absent in this job) and fails the gate for an
         # unrelated reason. The script declares its own inline metadata.
         run: uv run --script ci/check_workspace_crates.py
+
+      - name: Check integration tests stay in their 8 category binaries
+        # FastLED/fbuild#1577: a new top-level crates/*/tests/*.rs is a new
+        # test binary that re-links the whole dependency graph.
+        run: uv run --script ci/check_test_layout.py

.github/workflows/qemu-linux-runtime.yml

@@ -72,5 +72,5 @@ jobs:
 
       - name: QEMU runtime provisioning test
         run: |
-          soldr cargo test -p fbuild-toolchain --test qemu_linux_runtime -- \
+          soldr cargo test -p fbuild-packages --test it -- qemu_linux_runtime:: \
             --ignored --nocapture

bench/fastled-examples/Cargo.toml

@@ -10,6 +10,8 @@ publish = false
 [[bin]]
 name = "bench-fastled-examples"
 path = "src/main.rs"
+# No #[test] in this bin; skip its empty harness (FastLED/fbuild#1577).
+test = false
 
 [[bin]]
 name = "bench-build-comparison"

ci/check_test_layout.py

@@ -0,0 +1,99 @@
+#!/usr/bin/env -S uv run --script
+# /// script
+# requires-python = ">=3.10"
+# ///
+"""Linter: keep the workspace at no more than 8 integration-test binaries.
+
+Every top-level `crates/<crate>/tests/*.rs` file, and every
+`tests/<dir>/main.rs`, is a separate test binary. Each one links its crate's
+whole dependency graph again, and before FastLED/fbuild#1577 that linking took
+most of CI's `Test` step. The workspace now has a fixed set of category
+binaries (soldr's layout, zackees/ci.yml RUST-005). A new integration test goes
+into one of them as a module, never into a new top-level file or directory.
+
+Adding a category is a deliberate act: add it to `CATEGORIES` below with a
+one-line reason, and keep the total at no more than `MAX_CATEGORIES`.
+
+Usage:
+    uv run --script ci/check_test_layout.py
+"""
+
+from __future__ import annotations
+
+import re
+import sys
+from pathlib import Path
+
+MAX_CATEGORIES = 8
+
+# (crate dir, tests/ subdir) -> what belongs there.
+CATEGORIES: dict[tuple[str, str], str] = {
+    ("fbuild-build", "it"): "build: real and synthetic build pipelines",
+    ("fbuild-build", "env_isolated"): "tests that mutate FBUILD_*/ZCCACHE_* env vars, behind ENV_LOCK",
+    ("fbuild-daemon", "it"): "daemon: HTTP/WS server and managers",
+    ("fbuild-cli", "it"): "cli: command-line front end",
+    ("fbuild-core", "it"): "config_resolution: PlatformIO resolution and core types",
+    ("fbuild-packages", "it"): "packages: caches, lnk, toolchain provisioning",
+    ("fbuild-library-select", "it"): "library_select: LDF and library selection",
+    ("fbuild-python", "python_facades"): "python: needs libpython at link time",
+}
+
+REPO = Path(__file__).resolve().parent.parent
+
+
+def violations(repo: Path = REPO) -> list[str]:
+    problems: list[str] = []
+    if len(CATEGORIES) > MAX_CATEGORIES:
+        problems.append(f"{len(CATEGORIES)} categories declared; the limit is {MAX_CATEGORIES}")
+    for tests in sorted((repo / "crates").glob("*/tests")):
+        crate = tests.parent.name
+        for entry in sorted(tests.iterdir()):
+            rel = entry.relative_to(repo).as_posix()
+            if entry.is_file() and entry.suffix == ".rs":
+                problems.append(f"{rel}: top-level test file is its own binary; add it as a module of a category in ci/check_test_layout.py")
+            elif entry.is_dir() and (entry / "main.rs").exists() and (crate, entry.name) not in CATEGORIES:
+                problems.append(f"{rel}/main.rs: unknown test category; use an existing one")
+    manifests = sorted((repo / "crates").glob("*/Cargo.toml")) + sorted((repo / "bench").glob("*/Cargo.toml"))
+    for manifest in manifests:
+        text = manifest.read_text(encoding="utf-8")
+        rel = manifest.relative_to(repo).as_posix()
+        if re.search(r"^\[\[test\]\]", text, re.MULTILINE):
+            problems.append(f"{rel}: [[test]] targets bypass the category layout")
+        for source in test_disabled_sources(manifest, text):
+            if TEST_ATTR.search(source.read_text(encoding="utf-8")):
+                problems.append(
+                    f"{source.relative_to(repo).as_posix()}: has #[test] but its target sets `test = false` in {rel}, so it never runs"
+                )
+    return problems
+
+
+TEST_ATTR = re.compile(r"#\[(tokio::)?test\b")
+
+
+def test_disabled_sources(manifest: Path, text: str) -> list[Path]:
+    """Source files of `[lib]`/`[[bin]]` targets that set `test = false`."""
+    crate = manifest.parent
+    sources: list[Path] = []
+    for header, body in re.findall(r"^(\[lib\]|\[\[bin\]\])\n((?:(?!\[).*\n?)*)", text, re.MULTILINE):
+        if not re.search(r"^test\s*=\s*false", body, re.MULTILINE):
+            continue
+        path = re.search(r'^path\s*=\s*"([^"]+)"', body, re.MULTILINE)
+        entry = crate / (path.group(1) if path else ("src/lib.rs" if header == "[lib]" else "src/main.rs"))
+        # A lib's unit tests can live anywhere under src/; a bin's in its file.
+        sources.extend(sorted(entry.parent.rglob("*.rs")) if header == "[lib]" else [entry])
+    return [s for s in sources if s.is_file()]
+
+
+def main() -> int:
+    problems = violations()
+    for problem in problems:
+        print(f"error: {problem}", file=sys.stderr)
+    if problems:
+        print(f"\n{len(problems)} test-layout violation(s); see FastLED/fbuild#1577", file=sys.stderr)
+        return 1
+    print(f"ok: {len(CATEGORIES)} integration-test categories, no stray test binaries")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

ci/platform_boundary_ledger.tsv

@@ -1,6 +1,6 @@
 path	kind	normalized	ordinal	capability	classification
-crates/fbuild-core/tests/platformio_package_resolution.rs	attr_cfg	#[cfg(windows)]	0	host	host_mechanic
-crates/fbuild-core/tests/platformio_package_resolution.rs	attr_cfg	#[cfg(windows)]	1	host	host_mechanic
+crates/fbuild-core/tests/it/platformio_package_resolution.rs	attr_cfg	#[cfg(windows)]	0	host	host_mechanic
+crates/fbuild-core/tests/it/platformio_package_resolution.rs	attr_cfg	#[cfg(windows)]	1	host	host_mechanic
 crates/fbuild-daemon/Cargo.toml	native_dependency	libc	0	process	host_mechanic
 crates/fbuild-daemon/src/handlers/emulator/tests_npm_cache.rs	attr_cfg	#[cfg(windows)]	0	host_executable	host_artifact_policy
 crates/fbuild-daemon/src/handlers/emulator/tests_process.rs	attr_cfg	#[cfg(not(windows))]	0	host_executable	host_artifact_policy
@@ -24,6 +24,7 @@ crates/fbuild-daemon/src/handlers/websockets_tests.rs	native_path	libc::	6	proce
 crates/fbuild-daemon/src/handlers/websockets_tests.rs	native_path	libc::	7	process	host_mechanic
 crates/fbuild-daemon/src/handlers/websockets_tests.rs	native_path	libc::	8	process	host_mechanic
 crates/fbuild-daemon/src/handlers/websockets_tests.rs	native_path	libc::	9	process	host_mechanic
+crates/fbuild-packages/tests/it/qemu_linux_runtime.rs	attr_cfg	#![cfg(target_os=)]	0	host	host_mechanic
 crates/fbuild-paths/src/dev_daemon_namespace.rs	native_path	std::env::current_exe	0	host_executable	host_mechanic
 crates/fbuild-paths/src/executable_hash.rs	attr_cfg	#[cfg(all(test,unix))]	0	host	host_mechanic
 crates/fbuild-paths/src/executable_hash.rs	attr_cfg	#[cfg(not(unix))]	0	host	host_mechanic
@@ -32,7 +33,7 @@ crates/fbuild-paths/src/executable_hash.rs	attr_cfg	#[cfg(unix)]	1	host	host_mec
 crates/fbuild-paths/src/executable_hash.rs	attr_cfg	#[cfg(unix)]	2	host	host_mechanic
 crates/fbuild-paths/src/executable_hash.rs	attr_cfg	#[cfg(unix)]	3	host	host_mechanic
 crates/fbuild-paths/src/executable_hash.rs	attr_cfg	#[cfg(unix)]	4	host	host_mechanic
-crates/fbuild-python/tests/python_facades.rs	native_path	std::env::current_exe	0	host_executable	host_mechanic
+crates/fbuild-python/tests/python_facades/main.rs	native_path	std::env::current_exe	0	host_executable	host_mechanic
 crates/fbuild-toolchain/src/toolchain/esp_qemu.rs	attr_cfg	#[cfg(not(windows))]	0	host_executable	host_mechanic
 crates/fbuild-toolchain/src/toolchain/esp_qemu.rs	attr_cfg	#[cfg(not(windows))]	1	host_executable	host_mechanic
 crates/fbuild-toolchain/src/toolchain/esp_qemu.rs	attr_cfg	#[cfg(windows)]	0	host_executable	host_mechanic
@@ -42,4 +43,3 @@ crates/fbuild-toolchain/src/toolchain/esp_qemu.rs	attr_cfg	#[cfg(windows)]	3	hos
 crates/fbuild-toolchain/src/toolchain/esp_qemu_runtime.rs	attr_cfg	#[cfg(not(target_os=))]	0	host_executable	host_artifact_policy
 crates/fbuild-toolchain/src/toolchain/esp_qemu_runtime.rs	attr_cfg	#[cfg(target_os=)]	0	host_executable	host_artifact_policy
 crates/fbuild-toolchain/src/toolchain/esp_qemu_runtime.rs	attr_cfg	#[cfg_attr(not(target_os=),allow(dead_code))]	0	host_executable	host_artifact_policy
-crates/fbuild-toolchain/tests/qemu_linux_runtime.rs	attr_cfg	#![cfg(target_os=)]	0	host_executable	host_artifact_policy

ci/platform_boundary_research.tsv

@@ -71,9 +71,9 @@ crates/fbuild-core/src/platform/windows/usb_pnp.rs	32	native_path	windows_sys::
 crates/fbuild-core/src/platform/windows/usb_pnp.rs	35	native_path	windows_sys::	process	host_mechanic
 crates/fbuild-core/src/platform/windows/usb_pnp.rs	39	native_path	windows_sys::	process	host_mechanic
 crates/fbuild-core/src/platform/windows/usb_pnp.rs	40	native_path	windows_sys::	process	host_mechanic
-crates/fbuild-core/tests/platformio_package_resolution.rs	5	attr_cfg	#[cfg(windows)]	host	host_mechanic
-crates/fbuild-core/tests/platformio_package_resolution.rs	107	attr_cfg	#[cfg(windows)]	host	host_mechanic
-crates/fbuild-daemon/Cargo.toml	68	native_dependency	libc	process	host_mechanic
+crates/fbuild-core/tests/it/platformio_package_resolution.rs	5	attr_cfg	#[cfg(windows)]	host	host_mechanic
+crates/fbuild-core/tests/it/platformio_package_resolution.rs	107	attr_cfg	#[cfg(windows)]	host	host_mechanic
+crates/fbuild-daemon/Cargo.toml	74	native_dependency	libc	process	host_mechanic
 crates/fbuild-daemon/src/handlers/emulator/tests_npm_cache.rs	146	attr_cfg	#[cfg(windows)]	host_executable	host_artifact_policy
 crates/fbuild-daemon/src/handlers/emulator/tests_process.rs	9	attr_cfg	#[cfg(windows)]	host_executable	host_artifact_policy
 crates/fbuild-daemon/src/handlers/emulator/tests_process.rs	21	attr_cfg	#[cfg(not(windows))]	host_executable	host_artifact_policy
@@ -96,6 +96,7 @@ crates/fbuild-daemon/src/handlers/websockets_tests.rs	340	native_path	libc::	pro
 crates/fbuild-daemon/src/handlers/websockets_tests.rs	342	native_path	libc::	process	host_mechanic
 crates/fbuild-daemon/src/handlers/websockets_tests.rs	345	native_path	libc::	process	host_mechanic
 crates/fbuild-daemon/src/handlers/websockets_tests.rs	348	native_path	libc::	process	host_mechanic
+crates/fbuild-packages/tests/it/qemu_linux_runtime.rs	18	attr_cfg	#![cfg(target_os=)]	host	host_mechanic
 crates/fbuild-paths/src/dev_daemon_namespace.rs	91	native_path	std::env::current_exe	host_executable	host_mechanic
 crates/fbuild-paths/src/executable_hash.rs	6	attr_cfg	#[cfg(unix)]	host	host_mechanic
 crates/fbuild-paths/src/executable_hash.rs	9	attr_cfg	#[cfg(unix)]	host	host_mechanic
@@ -104,7 +105,7 @@ crates/fbuild-paths/src/executable_hash.rs	66	attr_cfg	#[cfg(not(unix))]	host	ho
 crates/fbuild-paths/src/executable_hash.rs	71	attr_cfg	#[cfg(unix)]	host	host_mechanic
 crates/fbuild-paths/src/executable_hash.rs	76	attr_cfg	#[cfg(unix)]	host	host_mechanic
 crates/fbuild-paths/src/executable_hash.rs	85	attr_cfg	#[cfg(all(test,unix))]	host	host_mechanic
-crates/fbuild-python/tests/python_facades.rs	317	native_path	std::env::current_exe	host_executable	host_mechanic
+crates/fbuild-python/tests/python_facades/main.rs	317	native_path	std::env::current_exe	host_executable	host_mechanic
 crates/fbuild-toolchain/src/toolchain/esp_qemu.rs	528	attr_cfg	#[cfg(windows)]	host_executable	host_mechanic
 crates/fbuild-toolchain/src/toolchain/esp_qemu.rs	534	attr_cfg	#[cfg(windows)]	host_executable	host_mechanic
 crates/fbuild-toolchain/src/toolchain/esp_qemu.rs	565	attr_cfg	#[cfg(not(windows))]	host_executable	host_mechanic
@@ -114,4 +115,3 @@ crates/fbuild-toolchain/src/toolchain/esp_qemu.rs	755	attr_cfg	#[cfg(windows)]	h
 crates/fbuild-toolchain/src/toolchain/esp_qemu_runtime.rs	253	attr_cfg	#[cfg_attr(not(target_os=),allow(dead_code))]	host_executable	host_artifact_policy
 crates/fbuild-toolchain/src/toolchain/esp_qemu_runtime.rs	295	attr_cfg	#[cfg(not(target_os=))]	host_executable	host_artifact_policy
 crates/fbuild-toolchain/src/toolchain/esp_qemu_runtime.rs	301	attr_cfg	#[cfg(target_os=)]	host_executable	host_artifact_policy
-crates/fbuild-toolchain/tests/qemu_linux_runtime.rs	18	attr_cfg	#![cfg(target_os=)]	host_executable	host_artifact_policy

ci/select_boards.py

@@ -53,10 +53,24 @@ def path_matches(pattern: str, path: str) -> bool:
     return _compile(pattern).match(path) is not None
 
 
+# Test-only paths never select boards (FastLED/fbuild#1577): the Linux test
+# job runs them, and a test refactor must not fan out to every platform.
+TEST_ONLY_PATTERNS = (
+    "crates/*/tests/**",
+    "crates/**/*_tests.rs",
+    "ci/test_*.py",
+)
+
+
+def is_test_only(path: str) -> bool:
+    return any(path_matches(p, path) for p in TEST_ONLY_PATTERNS)
+
+
 def select_workflows(boards, families, common_paths, changed):
     """Workflows to run for `changed` files; `None` means the diff is unknown."""
     if changed is None:
         return sorted(b["workflow"] for b in boards)
+    changed = [f for f in changed if not is_test_only(f)]
     selected = []
     for board in boards:
         patterns = render_paths_for_board(board, families, common_paths)

ci/test_check_test_layout.py

@@ -0,0 +1,75 @@
+"""Tests for ci/check_test_layout.py (FastLED/fbuild#1577)."""
+from __future__ import annotations
+
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+
+from check_test_layout import CATEGORIES, MAX_CATEGORIES, violations  # noqa: E402
+
+
+def make(root: Path, rel: str, body: str = "") -> None:
+    path = root / rel
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(body, encoding="utf-8")
+
+
+class LayoutTests(unittest.TestCase):
+    def test_repository_is_clean(self):
+        self.assertEqual([], violations())
+
+    def test_at_most_eight_categories(self):
+        self.assertLessEqual(len(CATEGORIES), MAX_CATEGORIES)
+        self.assertEqual(8, MAX_CATEGORIES)
+
+    def test_top_level_test_file_is_rejected(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            make(root, "crates/fbuild-core/tests/it/main.rs")
+            make(root, "crates/fbuild-core/tests/new_thing.rs")
+            problems = violations(root)
+        self.assertEqual(1, len(problems))
+        self.assertIn("new_thing.rs", problems[0])
+
+    def test_unknown_category_dir_is_rejected(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            make(root, "crates/fbuild-core/tests/extra/main.rs")
+            self.assertIn("unknown test category", violations(root)[0])
+
+    def test_explicit_test_target_is_rejected(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            make(root, "crates/fbuild-core/Cargo.toml", '[package]\nname = "x"\n\n[[test]]\nname = "y"\n')
+            self.assertIn("[[test]]", violations(root)[0])
+
+    def test_helper_dirs_without_main_are_allowed(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            make(root, "crates/fbuild-core/tests/it/main.rs")
+            make(root, "crates/fbuild-core/tests/fixtures/data.json")
+            self.assertEqual([], violations(root))
+
+    def test_test_false_target_with_tests_is_rejected(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            make(root, "crates/fbuild-x/Cargo.toml", '[package]\nname = "x"\n\n[lib]\ntest = false\n\n[dependencies]\n')
+            make(root, "crates/fbuild-x/src/lib.rs", "mod a;\n")
+            make(root, "crates/fbuild-x/src/a.rs", "#[test]\nfn t() {}\n")
+            problems = violations(root)
+        self.assertEqual(1, len(problems))
+        self.assertIn("never runs", problems[0])
+
+    def test_test_false_target_without_tests_is_fine(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            make(root, "crates/fbuild-x/Cargo.toml", '[package]\nname = "x"\n\n[[bin]]\nname = "b"\npath = "src/bin/b.rs"\ntest = false\n')
+            make(root, "crates/fbuild-x/src/bin/b.rs", "fn main() {}\n")
+            self.assertEqual([], violations(root))
+
+
+if __name__ == "__main__":
+    unittest.main()