From 531ef48fd55d43aa98b7adc43004cb03e9a8a08d Mon Sep 17 00:00:00 2001 From: Michael Kinney Date: Wed, 11 Nov 2026 23:43:08 -0600 Subject: [PATCH] feat(ci): a Cargo cache directory per job The pipeline's eight jobs each compiled the workspace from scratch: Cargo's target output went to a disposable container /tmp directory, so no job ever started warm. A run's chain took about seven hours before its last job began, which is the binding constraint on every merge receipt. The runner can mount a host directory at /cargo-target and export CARGO_TARGET_DIR, so the compiled artifacts survive the job. It also takes an EXCLUSIVE advisory lock on every writable host directory a job mounts, for the job's whole lifetime (GitForge GF-11): two jobs writing one target tree clobber each other's binaries. One directory per image would therefore serialize every Cargo job in the fleet into a single lane, which is the opposite of what the cache is for. Each job now declares its own cache directory, named for the job. The offline validator enforces that shape - a per-job source, no source shared between jobs, no disposable container /tmp target - and it now runs in the GitHub backup CI's fmt job, because a malformed .gitforge.yml fails every run at parse time and the GitForge API has no dry-run endpoint. Decision 0098. Co-Authored-By: Claude Code --- .gitforge.yml | 78 ++++++++++++++------- .github/workflows/ci.yml | 14 ++++ .gitignore | 3 + docs/TASKS.md | 1 + docs/decisions/0098-per-job-cargo-cache.md | 81 ++++++++++++++++++++++ scripts/validate_gitforge_pipeline.py | 42 +++++++++++ 6 files changed, 193 insertions(+), 26 deletions(-) create mode 100644 docs/decisions/0098-per-job-cargo-cache.md diff --git a/.gitforge.yml b/.gitforge.yml index ca1c2777..72cea73f 100644 --- a/.gitforge.yml +++ b/.gitforge.yml @@ -20,11 +20,18 @@ # multi-command block here sets it: without it a mid-block failure is # invisible whenever a later command happens to succeed. That masking # once hid a failing gate for dozens of runs (decision 0086). -# - CARGO_TARGET_DIR=/tmp/ci-target: no rustc write pattern survives the -# sshfs workspace bridge under load (clippy metadata bursts and codegen -# both EPERM'd on fedora), so all compilation lands on the -# container-local overlay2 disk. Steps within a job share one -# container, so heavy steps are grouped to reuse the warm target dir. +# - `cache:` mounts a host-local Cargo target directory at `/cargo-target` +# and exports `CARGO_TARGET_DIR`. This keeps rustc writes off the sshfs +# workspace bridge while reusing compiled artifacts across runs. +# The runner holds an EXCLUSIVE advisory lock on every writable host +# directory a job mounts, for the job's whole lifetime (GitForge GF-11): +# two jobs writing one target tree would clobber each other's binaries. +# So the cache directory is PER JOB — `.../cargo/{image}-` — +# and not one directory per image. A single shared directory would +# serialize every Cargo job in the fleet into one lane, which is the +# throughput problem this cache is meant to fix. Each job keeps its own +# warm tree; no two jobs contend for it. Two runs of the SAME job still +# contend, by design: that is the case the lock exists for. # - Job sandboxes run with CARGO_NET_OFFLINE=false: the image's # registry cache is a FAST FLOOR, not a wall. A crate already in the # cache touches the network not at all; a crate that is not costs one @@ -66,12 +73,14 @@ environment: {} jobs: - name: fmt-layers image: firecraft-ci-rust:1 + cache: + source: "{cache_root}/cargo/{image}-fmt-layers" steps: - name: Rust formatting check run: cargo fmt --all -- --check - name: Layer-table gate # Compiles xtask alone; the gate itself is cargo metadata. - run: CARGO_TARGET_DIR=/tmp/ci-target cargo run --locked -p xtask -- layers + run: cargo run --locked -p xtask -- layers # 45m: the gate compiles xtask, so it carries the workspace's # build wave. Measured 20.1 min on 2026-11-11 with 9 runs in # flight (0.3 min idle); the previous 20m budget killed a run that @@ -80,16 +89,21 @@ jobs: - name: clippy-test image: firecraft-ci-rust:1 + cache: + source: "{cache_root}/cargo/{image}-clippy-test" needs: [fmt-layers] steps: - name: Cargo clippy (deny warnings, all targets) - run: CARGO_TARGET_DIR=/tmp/ci-target cargo clippy --workspace --all-targets --locked -- -D warnings + run: cargo clippy --workspace --all-targets --locked -- -D warnings - name: Web crate clippy (workspace-excluded, trunk) - run: cd web && CARGO_TARGET_DIR=/tmp/ci-web-target cargo clippy --all-targets --locked -- -D warnings + # The web crate is a separate workspace with its own lockfile, so it + # keeps its own target tree — inside the job's cache mount, so the + # artifacts survive the job. + run: cd web && CARGO_TARGET_DIR=/cargo-target/web cargo clippy --all-targets --locked -- -D warnings - name: Workspace tests # Same container as the clippy step: the warm target dir carries # over, so the test build only completes what clippy checked. - run: CARGO_TARGET_DIR=/tmp/ci-target cargo test --workspace --locked --no-fail-fast -- --test-threads=2 + run: cargo test --workspace --locked --no-fail-fast -- --test-threads=2 # The egui/eframe graph is the bulk of the workspace; the wall # absorbs co-tenant build waves on these shared hosts. # 150m: measured 76.9 min on 2026-11-11 with 9 runs in flight. @@ -97,38 +111,44 @@ jobs: - name: wasm-web image: firecraft-ci-rust:1 + cache: + source: "{cache_root}/cargo/{image}-wasm-web" needs: [clippy-test] steps: - name: Wasm compile gate + smoke suite + trunk bundle - # CARGO_TARGET_DIR is inherited by xtask's child cargo calls and - # by trunk's cargo build, keeping every rustc write off the + # The cache exports CARGO_TARGET_DIR to xtask's child cargo calls + # and trunk's cargo build, keeping every rustc write off the # sshfs bridge. With wasm-bindgen-test-runner + node baked into # the image, the fwpng smoke suite executes for real (no # compile-only downgrade), and trunk builds the web bundle. - run: CARGO_TARGET_DIR=/tmp/ci-target cargo run --locked -p xtask -- wasm + run: cargo run --locked -p xtask -- wasm # 90m: measured 45.6 min on 2026-11-11 with 9 runs in flight. timeout: 90m - name: corpus-golden image: firecraft-ci-rust:1 + cache: + source: "{cache_root}/cargo/{image}-corpus-golden" needs: [wasm-web] steps: - name: Corpus digests + doc-model round-trips - run: CARGO_TARGET_DIR=/tmp/ci-target cargo run --locked -p xtask -- corpus + run: cargo run --locked -p xtask -- corpus - name: Golden render pins + determinism - run: CARGO_TARGET_DIR=/tmp/ci-target cargo run --locked -p xtask -- golden + run: cargo run --locked -p xtask -- golden # 45m: measured 19.6 min on 2026-11-11 with 9 runs in flight. timeout: 45m - name: perf-budgets image: firecraft-ci-rust:1 + cache: + source: "{cache_root}/cargo/{image}-perf-budgets" needs: [corpus-golden] steps: - name: Performance budgets (open / frame / save / peak memory) # M12: the parity table's ceilings, enforced on the runner # fleet. The report line names the host, so a trip is # diagnosable as regression vs environment shift. - run: CARGO_TARGET_DIR=/tmp/ci-target cargo run --locked -p xtask -- perf + run: cargo run --locked -p xtask -- perf # 45m: measured 14.3 min on 2026-11-11 with 9 runs in flight, but # two runs were killed at the previous 20m budget while still # working (one on 2026-11-11 at 20.2 min, one at 20.3 min). The @@ -137,7 +157,9 @@ jobs: - name: fuzz image: firecraft-ci-rust:1 - needs: [corpus-golden] + cache: + source: "{cache_root}/cargo/{image}-fuzz" + needs: [perf-budgets] steps: - name: Stable fuzz drivers (anchors + panic_hunt + scripting arms) # The stable-toolchain driver set from `xtask ci`; the @@ -150,7 +172,7 @@ jobs: # needed). A crate missing from the cache costs one small # fetch; the full error still reaches only the job log ledger, # not the bounded receipt. - run: CARGO_TARGET_DIR=/tmp/ci-target cargo run --locked -p xtask -- fuzz-quick + run: cargo run --locked -p xtask -- fuzz-quick # 30m proved too tight: a cold sandbox target dir spends ~20m # compiling before fuzz-quick starts (dfd7f46 run was killed by # the watchdog 10m into the campaign itself). @@ -160,6 +182,8 @@ jobs: - name: generated-docs image: firecraft-ci-rust:1 + cache: + source: "{cache_root}/cargo/{image}-generated-docs" needs: [fuzz] steps: - name: Parity floors + scorecard + stamps + reachability + panel snapshot, then drift check @@ -191,11 +215,11 @@ jobs: set -e cp docs/planning/generated/parity.md /tmp/parity.committed cp docs/planning/generated/scorecard.md /tmp/scorecard.committed - CARGO_TARGET_DIR=/tmp/ci-target cargo run --locked -p xtask -- parity - CARGO_TARGET_DIR=/tmp/ci-target cargo run --locked -p xtask -- scorecard - CARGO_TARGET_DIR=/tmp/ci-target cargo run --locked -p xtask -- stamps - CARGO_TARGET_DIR=/tmp/ci-target cargo run --locked -p xtask -- reachability - CARGO_TARGET_DIR=/tmp/ci-target cargo run --locked -p xtask -- panel-snapshot + cargo run --locked -p xtask -- parity + cargo run --locked -p xtask -- scorecard + cargo run --locked -p xtask -- stamps + cargo run --locked -p xtask -- reachability + cargo run --locked -p xtask -- panel-snapshot cmp /tmp/parity.committed docs/planning/generated/parity.md cmp /tmp/scorecard.committed docs/planning/generated/scorecard.md # 45m: measured 16.1 min on 2026-11-11 with 9 runs in flight. @@ -203,6 +227,8 @@ jobs: - name: release image: firecraft-ci-rust:1 + cache: + source: "{cache_root}/cargo/{image}-release" needs: [generated-docs] steps: - name: Ref report @@ -238,8 +264,8 @@ jobs: set -e echo "release-path build — the CI image ships no git, so this step cannot detect the ref; the ref is on the pipeline run (pipeline_runs.ref_name) and in the release workflow's arguments" - name: "Web bundle (bridge smoke)" - # Same container as the tag report; the fresh job target dir - # builds the release profile from scratch, which is the point: + # Same container as the tag report; the release profile remains + # distinct under the shared target cache's release subdirectory. # the bundle must come from tag bytes, not a warm debug cache. # # The bridge smoke RUNS here: `dist --web` serves the staged @@ -250,9 +276,9 @@ jobs: # `chromium` package), so no skip switch is needed — and none is # accepted: a missing browser or a failed round trip must fail # the job, not pass it quietly. - run: CARGO_TARGET_DIR=/tmp/ci-dist-target cargo run --locked -p xtask -- dist --web + run: cargo run --locked -p xtask -- dist --web - name: Native set + deb read-back - run: CARGO_TARGET_DIR=/tmp/ci-dist-target cargo run --locked -p xtask -- dist --native + run: cargo run --locked -p xtask -- dist --native - name: DEB install verification (clean Debian 12) # The package must be installable, launchable and removable on # the platform it declares. `packaging/deb/verify-install.sh` is diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e9c72951..e4a40101 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,6 +47,20 @@ jobs: - name: Check web formatting run: cargo fmt --manifest-path web/Cargo.toml -- --check + # The GitForge pipeline definition is parsed server-side by + # `PipelineDefinition::parse`, which has no dry-run endpoint: a + # malformed `.gitforge.yml` fails every run at parse time, after + # the commit has already landed. This is the offline check of the + # same contract, and it also enforces the per-job Cargo cache + # directories that keep the runner's exclusive mount lock from + # serializing the whole fleet. + - name: Validate the GitForge pipeline definition + run: | + set -euo pipefail + python3 -m venv /tmp/pipeline-venv + /tmp/pipeline-venv/bin/pip install --quiet pyyaml + /tmp/pipeline-venv/bin/python scripts/validate_gitforge_pipeline.py .gitforge.yml + gate: name: gate runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index 6ecd21f7..0408dc75 100644 --- a/.gitignore +++ b/.gitignore @@ -36,3 +36,6 @@ lcov.info # Session scratch: throwaway helper scripts and probe artifacts /.claude/scratch/ + +# Python bytecode from the offline pipeline validator +__pycache__/ diff --git a/docs/TASKS.md b/docs/TASKS.md index 81f4f2ab..6e15e23e 100644 --- a/docs/TASKS.md +++ b/docs/TASKS.md @@ -65,6 +65,7 @@ | 20 | 20.1 In-canvas text editing (egui TextEdit overlay; kittest focus tests) | [tasks/T20-text-editing.md](tasks/T20-text-editing.md) | `IMPLEMENTED` — landed 2026-11-06 (`shell::begin_text_edit` / `text_edit_overlay`) | editing lane | the remaining Phase 20 items (Optimize, animation timeline, mask stack, tabs, history rewind, slice behavior) stay queued below | | 8 | 20.3 Animation timeline (frames strip, per-state timing, onion skinning, `symbol.tween`) | [tasks/T8-animation-timeline.md](tasks/T8-animation-timeline.md) | `IMPLEMENTED` — the shell half landed 2026-11-11 (decision [0094](decisions/0094-animation-timeline-shell.md)): the States panel is the strip, the delay field writes `frame.setDuration`, the onion toggle writes `page.setOnionSkinning` and the canvas calls `compose::render_onion` | editing lane | playback (CS6's `Play Animation`), state thumbnails, and the screenshot acceptance are still open — see the ticket | | 7 | 20.2 Optimize panel (palette/bitrate over `ExportOptions` + live preview) | [tasks/T7-optimize-panel.md](tasks/T7-optimize-panel.md) | `IMPLEMENTED` — landed 2026-11-11 (decision [0095](decisions/0095-optimize-panel-honors-only-what-export-honors.md)): the controls are the fields the export honors, the preview encodes the page through the export's own `io::expv` resolution, and PNG's Adam7 interlace is disclosed and clearable | editing lane | Adam7 interlacing, CS6's selective-JPEG quality table, and the screenshot acceptance are still open — see the ticket | +| — | CI throughput: a warm Cargo cache per job, so a pipeline run starts warm instead of rebuilding the world | [decisions/0098-per-job-cargo-cache.md](decisions/0098-per-job-cargo-cache.md) | `IN PROGRESS` — the per-job `cache:` blocks, the offline validator and the validator's CI step are in the tree on this branch; no run has yet measured the warm cost | CI/tooling lane | land the change, then read the first warm run's per-job durations from the authoritative store before claiming a throughput win; the cache's own growth needs a prune policy (recorded as debt in decision 0098) | ## Queued (Phase 20 — daily-driver features) diff --git a/docs/decisions/0098-per-job-cargo-cache.md b/docs/decisions/0098-per-job-cargo-cache.md new file mode 100644 index 00000000..71edfe04 --- /dev/null +++ b/docs/decisions/0098-per-job-cargo-cache.md @@ -0,0 +1,81 @@ +# 0098 — The Cargo cache is one directory per job, not one per image + +Date: 2026-11-11 · Tickets: [T27-15](../release/EXECUTION-BOARD.md) · Status: accepted + +## Context + +The GitForge pipeline is the authoritative receipt for a merge, and it is +slow enough to be the binding constraint on the whole release campaign. + +The `firecraft-ci` pipeline is a chain of eight jobs — `fmt-layers`, +`clippy-test`, `wasm-web`, `corpus-golden`, `perf-budgets`, `fuzz`, +`generated-docs`, `release` — and every one of them compiles the workspace +from scratch. Cargo's target output went to a disposable directory inside +the job container, so no job ever started warm. Measured on the +`swarmone-docker` runner under fleet load on 2026-11-11: `fmt-layers` +10 min, `clippy-test` 31 min, `wasm-web` 27 min, `corpus-golden` 11 min, +`perf-budgets` 10.5 min, `fuzz` 29 min, `generated-docs` 11 min, and +`release` still running after ~90 min. One run's chain took about seven +hours before its last job started; a merge could not be claimed on a +receipt that had not been produced yet. + +The runner already has the mechanism this needs. A job's `cache:` block +bind-mounts a host directory at `/cargo-target` and exports +`CARGO_TARGET_DIR` into the container, so the compiled artifacts live on +the host and survive the job. + +That mechanism comes with a rule that decides the whole shape of the +change: **the runner takes an exclusive advisory lock on every writable +host directory a job mounts, and holds it for the job's whole lifetime** +(GitForge GF-11). The lock exists because two jobs writing one Cargo +target tree clobber each other's binaries — a measured failure on the DIL +repository on 2026-11-05, where two coverage jobs lost their test +binaries to a concurrent rebuild. + +So the cache directory is a serialization point, not just a directory. + +## Decision + +1. **One cache directory per job, named for the job.** + Every Cargo job declares its own source, for example + `{cache_root}/cargo/{image}-clippy-test`, so no two jobs in the fleet + ever contend for the same tree. A single directory per image — the + bare default — would serialize every Cargo job in the fleet into one + lane, which is the opposite of what the cache is for. Each job keeps + its own warm tree across runs; two runs of the *same* job still + contend, which is exactly the case the lock exists for. +2. **The offline validator enforces the shape.** It requires a `cache:` + mapping on any job whose steps run Cargo, requires the source to name + the job, rejects a source another job already uses, and rejects + `CARGO_TARGET_DIR=/tmp/` in a step — the disposable-container shape + this change replaces. +3. **The validator runs in CI.** The GitForge API has no dry-run + endpoint and creating a pipeline needs authentication, so a malformed + `.gitforge.yml` fails every run at parse time, after the commit has + already landed. The GitHub backup workflow's `fmt` job now runs + `scripts/validate_gitforge_pipeline.py`, which checks the same + contract offline in seconds. + +## Consequences + +- The first run after this change is still cold: the cache directories + do not exist yet, so that run pays the full compile cost. Every later + run of every branch reuses the artifacts and starts warm. The change is + therefore a net loss for exactly one run and a large win for all of + them after it. +- The cache is keyed by image and job, not by branch: the branches share + one tree, which is what makes a warm run possible for work that has + not been merged yet. +- A cache that grows without bound is a real possibility. It lives under + the runner's own home on the NAS mount, which had 2.2 TB free on + 2026-11-11; nothing prunes it yet. A prune policy is owed before the + tree becomes a disk problem, and it is recorded as debt rather than + silently assumed. +- The pipeline definition that governs a push is the one in the pushed + commit, so a branch without this change keeps running cold. The change + reaches other branches by merging `main` into them. +- The cache does not shorten the `release` job's work: `xtask -- dist` + still builds the web bundle and the native set. It shortens the + compile inside it, from the measured 88 min toward the warm cost. + Whether the release path should run on every branch push at all is a + separate question, still open in [T27-15](../release/EXECUTION-BOARD.md). diff --git a/scripts/validate_gitforge_pipeline.py b/scripts/validate_gitforge_pipeline.py index 0f620abe..34a4ba32 100644 --- a/scripts/validate_gitforge_pipeline.py +++ b/scripts/validate_gitforge_pipeline.py @@ -57,6 +57,7 @@ def validate(path: Path) -> list[str]: names: list[str] = [] indegree: dict[str, int] = {} + cache_sources: set[str] = set() for job in jobs: name = job.get("name") if not name: @@ -78,11 +79,52 @@ def validate(path: Path) -> list[str]: if timeout is not None and not TIMEOUT_RE.match(str(timeout)): errors.append(f"job {name}: timeout {timeout!r} must match ") + cache = job.get("cache") + if cache is not None and not isinstance(cache, dict): + errors.append(f"job {name}: cache must be a mapping") + elif isinstance(cache, dict): + source = cache.get("source") + if source is None: + errors.append( + f"job {name}: cache must name its own directory (source: " + "\"{cache_root}/cargo/{image}-\"); the bare default is " + "shared by every job and serializes the fleet" + ) + elif not isinstance(source, str): + errors.append(f"job {name}: cache source must be a string") + else: + # The runner takes an EXCLUSIVE lock on every writable host + # directory a job mounts (GitForge GF-11), so two jobs on one + # cache directory cannot run at the same time. A source shared + # by every job would serialize the whole fleet into one lane. + if not source.endswith(name): + errors.append( + f"job {name}: cache source must end with the job name so each job " + f"owns its own cache directory (got {source!r})" + ) + if source in cache_sources: + errors.append( + f"job {name}: cache source {source!r} is already used by another job; " + "two jobs on one cache directory serialize each other" + ) + cache_sources.add(source) + steps = job.get("steps") or [] if isinstance(steps, list): + runs_cargo = False for step in steps: if not isinstance(step, dict) or not step.get("name") or not step.get("run"): errors.append(f"job {name}: every step needs name and run") + continue + run = str(step["run"]) + runs_cargo |= re.search(r"\bcargo(?:\s|$)", run) is not None + if re.search(r"CARGO_TARGET_DIR=/tmp/", run): + errors.append( + f"job {name}: Cargo target output must use the declared host cache, " + "not a disposable container /tmp directory" + ) + if runs_cargo and not isinstance(cache, dict): + errors.append(f"job {name}: Cargo work must declare a cache: block") for ref in (n for job in jobs for n in job.get("needs", []) or []): if ref not in names: