diff --git a/.github/workflows/benchmark_pages.yml b/.github/workflows/benchmark_pages.yml
index eb12156398..e9dbaaa412 100644
--- a/.github/workflows/benchmark_pages.yml
+++ b/.github/workflows/benchmark_pages.yml
@@ -73,6 +73,8 @@ jobs:
python3 source/tools/benchmark_landscape.py --site-root=site/benchmarks \
--renderer-js=source/tools/benchmark_web/dist/landscape.js --pulls=pulls.json \
--repository="${GITHUB_REPOSITORY}"
+ python3 source/tools/benchmark_extended_site.py publish \
+ --root=site/benchmarks --repository="${GITHUB_REPOSITORY}"
python3 source/tools/site_artwork.py source/docs/assets site
python3 source/tools/compact_site.py site
git -C site add --all
diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
index fd343f91bd..1334833fc7 100644
--- a/.github/workflows/benchmarks.yml
+++ b/.github/workflows/benchmarks.yml
@@ -17,6 +17,112 @@ concurrency:
cancel-in-progress: false
jobs:
+ extended:
+ name: Extended production (${{ matrix.platform }})
+ strategy:
+ fail-fast: false
+ matrix:
+ include:
+ - os: ubuntu-latest
+ platform: linux
+ timeout: 120
+ - os: macos-latest
+ platform: macos
+ timeout: 180
+ runs-on: ${{ matrix.os }}
+ timeout-minutes: ${{ matrix.timeout }}
+ steps:
+ - uses: actions/checkout@v7
+ with:
+ ref: ${{ github.sha }}
+ fetch-depth: 0
+ persist-credentials: false
+ - uses: bazel-contrib/setup-bazel@0.19.1
+ - uses: actions/setup-python@v7
+ with:
+ python-version: "3.13"
+ - name: Check extended runner capacity before building
+ run: |
+ python3 - <<'PY'
+ import sys
+ sys.path.insert(0, 'tools')
+ from benchmark_extended_ci import cpu_allocation
+ print('Extended allocation for three workers:', cpu_allocation(3, ci=True))
+ PY
+ - uses: ./.github/actions/bazel-cache-restore
+ with:
+ namespace: default
+ - name: Install extended reference tools
+ run: |
+ if [[ "${RUNNER_OS}" == Linux ]]; then
+ sudo apt-get update && sudo apt-get install -y findutils ripgrep tar
+ else
+ brew install findutils ripgrep gnu-tar
+ fi
+ - name: Freeze native accounting helper before production builds
+ run: |
+ set -euo pipefail
+ bazel build //tools:process_resources_native --config=clang_opt --disk_cache="${HOME}/.cache/bazel-disk"
+ launcher="$(bazel cquery 'config(//tools:process_resources_native, target)' --config=clang_opt \
+ --output=starlark --starlark:expr='target.files_to_run.executable.path')"
+ cp "${launcher}" "${RUNNER_TEMP}/extended-resource-launcher"
+ - name: Freeze full production head and first parent
+ run: |
+ set -euo pipefail
+ git worktree add --detach "${RUNNER_TEMP}/extended-base" HEAD^1
+ python3 tools/benchmark_extended_ci.py build --repo="${GITHUB_WORKSPACE}" --role=head \
+ --root="${RUNNER_TEMP}/extended-build" --disk-cache="${HOME}/.cache/bazel-disk"
+ python3 tools/benchmark_extended_ci.py build --repo="${RUNNER_TEMP}/extended-base" --role=base \
+ --root="${RUNNER_TEMP}/extended-build" --disk-cache="${HOME}/.cache/bazel-disk"
+ tar -cf "${RUNNER_TEMP}/benchmark-extended-build.tar" -C "${RUNNER_TEMP}/extended-build" .
+ bazel --nosystem_rc --nohome_rc shutdown
+ (cd "${RUNNER_TEMP}/extended-base" && bazel --nosystem_rc --nohome_rc shutdown)
+ - uses: actions/upload-artifact@v7
+ with:
+ name: benchmark-extended-build-${{ matrix.platform }}
+ path: ${{ runner.temp }}/benchmark-extended-build.tar
+ retention-days: 30
+ if-no-files-found: error
+ - uses: actions/download-artifact@v8
+ with:
+ name: benchmark-extended-build-${{ matrix.platform }}
+ path: ${{ runner.temp }}/extended-retention
+ - name: Restore the original retained production bytes
+ run: |
+ mkdir -p "${RUNNER_TEMP}/extended-retained-build"
+ tar -xf "${RUNNER_TEMP}/extended-retention/benchmark-extended-build.tar" -C "${RUNNER_TEMP}/extended-retained-build"
+ - name: Measure the complete extended contract in memory
+ run: |
+ set -euo pipefail
+ cat >"${RUNNER_TEMP}/extended-collection.json" <<'JSON'
+ {"run_id":${{ github.run_id }},"run_attempt":${{ github.run_attempt }},"head_sha":"${{ github.sha }}","repository":"${{ github.repository }}"}
+ JSON
+ python3 tools/benchmark_extended_ci.py collect \
+ --build-root="${RUNNER_TEMP}/extended-retained-build" --root="${RUNNER_TEMP}/extended-measurements" \
+ --resource-launcher="${RUNNER_TEMP}/extended-resource-launcher" --collection="${RUNNER_TEMP}/extended-collection.json"
+ - uses: actions/upload-artifact@v7
+ if: always()
+ with:
+ name: benchmark-extended-report-${{ matrix.platform }}
+ path: |
+ ${{ runner.temp }}/extended-measurements/observation.json
+ ${{ runner.temp }}/extended-measurements/measurements
+ ${{ runner.temp }}/extended-measurements/storage.json
+ ${{ runner.temp }}/extended-measurements/storage-release.json
+ retention-days: 30
+ if-no-files-found: warn
+ - name: Retain extended build diagnostics after success or failure
+ uses: actions/upload-artifact@v7
+ if: always()
+ with:
+ name: benchmark-extended-build-logs-${{ matrix.platform }}
+ path: |
+ ${{ runner.temp }}/extended-build/binaries/*/build.log
+ ${{ runner.temp }}/extended-build/binaries/*/build.json
+ ${{ runner.temp }}/extended-build/binaries/*/link-actions.json
+ retention-days: 30
+ if-no-files-found: warn
+
build:
strategy:
fail-fast: false
diff --git a/docs/benchmark-extended-history-campaign.md b/docs/benchmark-extended-history-campaign.md
index 01c750d52a..370d9f21c1 100644
--- a/docs/benchmark-extended-history-campaign.md
+++ b/docs/benchmark-extended-history-campaign.md
@@ -250,3 +250,98 @@ shard setup against merged PR #1149. It includes Linux/macOS production builds,
the 1/3/10-worker grid, identical controls, rejected experiments and small-task
regression repeats. Its experiment populations remain separate from the original
130-revision history and the core CI benchmark contract.
+
+## Continuous publication
+
+The `Benchmarks` workflow collects `extended-production-v1` independently on Linux and macOS.
+It uses the complete 60-cell grid above, one warm-up and nine measured rounds, retaining seven.
+Each platform builds the full production head and its exact first parent with ThinLTO. Each
+configured executable is copied immediately after its own build/query, before any later Bazel
+invocation. Both Bazel servers stop before measurement. The original executables and build/link
+evidence are uploaded, downloaded again, and verified before their bytes are measured.
+
+Linux creates a new directory on verified `/dev/shm` tmpfs. macOS creates a disposable 2 GiB
+case-sensitive HFS+ RAM image, records its device and volume UUID, and checks live ownership before
+formatting, use and release. Cleanup refuses changed ownership. Mac worker counts are requests;
+Linux records an allowed CPU mask. The fixture, executable, reference tools and native accounting
+launcher are checked again after collection. No timing result is accepted after an integrity
+failure. Local Linux validation can reuse an existing verified fixture with `--fixture`; this
+does not remove that fixture and is forbidden for CI collections.
+
+CI Linux budgets three allowed logical CPUs, including hosts where those logical CPUs share
+guest-topology cores. The original local-history collector retains its distinct-core policy.
+The workflow checks capacity before production builds and records the allocation policy and
+logical CPU count. Actual post-merge CI collection still needs to be verified on its runner.
+
+Head cells rotate head XFF, first-parent XFF and the traditional tool through execution order.
+A second complete grid measures the parent against itself as a diagnostic control. All successful
+outputs must match the independent oracle. Raw warm-ups, measured samples, command lines,
+environment, pressure context and failed attempts remain in the report artifact. A control
+failure leaves a complete, checked head observation recoverable, with an explicit control error;
+a failed or incomplete head never becomes a publishable observation.
+
+The existing 3D browser offers two separate views per series: traditional tools and parent XFF.
+The actual parent remains visible for each version; changing parents does not create a new
+dataset. Original local history keeps its fixed PR #1130 reference, local origin and separate
+series. Core benchmarks, local fixed-reference history and CI parent comparisons retain their
+own contracts. Extended observations do not satisfy the core merged-PR coverage inventory.
+No unavailable sizes or worker allocations are invented.
+
+The publisher recovers valid completed platform artifacts even when another job failed. A
+retry of failed jobs preserves successful platforms from their original attempts. CI publication
+checks the trusted workflow, main push, repository, run/attempt, exact head and GitHub's actual
+first parent. An artifact's own claims cannot establish that provenance. Complete observations
+are immutable under `benchmarks/extended/observations/`; generated views live under
+`benchmarks/extended/views/`. The dataset registry includes those views without putting them into
+the core benchmark normalizer or overview. Build/report artifacts retain the existing 30-day
+retention policy; publication retains the validated observations.
+
+### Import the original measured history
+
+Use the original measurement directory and retained build tree. The importer validates every
+declared cell, exact commands, output oracle, sample population, source identity and production
+link evidence. It reads and hashes the retained executables; rebuilding a revision cannot
+retroactively verify its original measurements. Incomplete or invalid campaigns are rejected.
+
+```sh
+python3.13 tools/benchmark_extended_site.py import \
+ --campaign=MEASUREMENT_ROOT --build-root=BUILD_ROOT \
+ --root=SITE/benchmarks --series=linux-zen5-extended-20261107 \
+ --platform=linux --machine=x86_64
+
+python3.13 tools/benchmark_extended_site.py publish --root=SITE/benchmarks
+```
+
+Publication runs after the ordinary core renderer and before site compaction. Existing core
+catalog entries remain intact. Tests cover known chart heights, warm-up exclusion, missing cells,
+stable parent datasets, local/CI separation, packed observations, idempotent publication,
+untrusted provenance, partial workflow recovery and RAM-resource ownership. Synthetic test
+observations are explicitly labeled and are never performance evidence.
+
+### Native pipeline and imported-view qualification (2026-11-09)
+
+The [qualification summary](benchmarks/extended-history-continuous-qualification-20261109.json)
+records the actual import, compacted-browser and native collector checks. All 130 original
+revisions and 7,800 cells passed validation against their original retained executables and link
+evidence. Both 130-version datasets joined a copy of the published catalog without changing any
+of its 588 existing entries. Every chart order reproduces the recorded ratios: 60 measured
+coordinates and 20 unavailable coordinates per view. Chromium selected the oldest, middle and
+latest versions in both datasets with correct binary evidence and no page errors.
+
+Each native collector completed 60 head cells and 60 identical-parent control cells on Linux
+and macOS: 3,600 independently checked invocations per platform. Both source revisions produce
+byte-identical production executables on their respective host. This validates the pipeline;
+it is not evidence of a performance improvement. The frozen driver source is
+`96765e1859c3d48ac0f04092c0613b4e8ad8ae21`, against first parent
+`52950f937ad371e1450538faf7983024b324688d`. Subsequent changes tighten validation and improve
+build logging/publication without changing native commands, sampling or C++ sources.
+
+Linux preserved its explicitly borrowed tmpfs fixture. The new macOS RAM volume recorded UUID
+`A03B359F-E90E-38C0-96B4-439E7BF90389`; its image, device and volume identity were checked before
+release, and device/mount absence was checked afterward. Its former device number is not an
+authorization to detach any future device. Raw observations, builds and failed-attempt support
+remain outside the source tree, with hashes and evidence locations in the summary.
+
+The original imported observations remain local history with a fixed reference. Publishing that
+data and verifying the first actual post-merge extended CI run are separate final steps.
+Changing workloads, sampling or resource accounting requires a separately identified contract.
diff --git a/docs/benchmark-merged-pr-coverage.md b/docs/benchmark-merged-pr-coverage.md
index 73e53e9c00..a0cc51797b 100644
--- a/docs/benchmark-merged-pr-coverage.md
+++ b/docs/benchmark-merged-pr-coverage.md
@@ -1,156 +1,196 @@
# Missing merged PR benchmark inventory
-Snapshot: November 9, 2026, 01:02 UTC. Scope: PRs merged directly into `main` on or after
-September 19, 2026. This is a dated inventory; publishing and active measurements may change it.
-
-A platform is covered only when the exact merge commit is selectable in the published merged-CI
-catalog. Local and extended-contract datasets are excluded. A retained artifact awaiting publication
-is recorded separately from a catalog entry.
-
-## Current coverage
-
-- 237 directly merged PRs: 162 published on both Linux and macOS; 75 absent on both.
-- Of the 75, PR #1150 has validated reports for both platforms awaiting publication.
-- The remaining 74 require recovery according to the read-only planner. Two already have an active
- targeted campaign: #868 and #876 have successful Linux jobs and macOS jobs still running in
- [run 42538554174](https://github.com/mboworks/xff/actions/runs/42538554174). Do not repeat those jobs.
-- PRs #914 and #986 are now published on both platforms. Their Linux recovery completed successfully
- in [run 42534282665](https://github.com/mboworks/xff/actions/runs/42534282665). PR #1143 is also present
- on both platforms. Live browser selection verification is recorded below.
-
-The November 8 total of 252 included PRs merged into other development branches. This refresh
-counts only direct merges into `main`; merges into development branches are excluded from this audit.
-The present metadata contains 256 merges across all base branches, of which 19 target other branches.
-
-## Missing PRs
-
-Every row currently lacks both Linux and macOS catalog entries. Workflow conclusions are observed
-states, not established root causes. A canceled run may have been canceled manually or superseded;
-a failed run requires its logs to establish the cause. No validated report was discovered for a
-row unless the table explicitly identifies one.
-
-| PR | Merged (UTC) | Merge commit | Observed state |
-| -------------------------------------------------- | ------------ | -------------- | ----------------------------------------------------------------------------------------------------------------------------- |
-| [#1150](https://github.com/mboworks/xff/pull/1150) | 2026-11-08 | `52950f937ad3` | Validated Linux/macOS reports awaiting publication |
-| [#1136](https://github.com/mboworks/xff/pull/1136) | 2026-11-07 | `1f25ba6d4ae0` | Main run cancelled; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/42398990564)) |
-| [#1132](https://github.com/mboworks/xff/pull/1132) | 2026-11-07 | `5d413a35cbec` | Main run cancelled; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/42378502027)) |
-| [#1130](https://github.com/mboworks/xff/pull/1130) | 2026-11-07 | `ae8c5af3eb11` | Main run cancelled; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/42367122125)) |
-| [#1128](https://github.com/mboworks/xff/pull/1128) | 2026-11-07 | `1278c8943a6a` | Main run cancelled; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/42354657340)) |
-| [#1121](https://github.com/mboworks/xff/pull/1121) | 2026-11-06 | `d9bf84b04784` | Main run cancelled; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/42265512655)) |
-| [#1118](https://github.com/mboworks/xff/pull/1118) | 2026-11-06 | `c7bb70cc6f3e` | Main run cancelled; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/42234848791)) |
-| [#1115](https://github.com/mboworks/xff/pull/1115) | 2026-11-06 | `ab909839da7d` | Main run cancelled; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/42218900004)) |
-| [#1113](https://github.com/mboworks/xff/pull/1113) | 2026-11-06 | `5729026a8482` | Main run cancelled; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/42200480427)) |
-| [#1111](https://github.com/mboworks/xff/pull/1111) | 2026-11-06 | `5cade08b6701` | Main run cancelled; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/42184472882)) |
-| [#1106](https://github.com/mboworks/xff/pull/1106) | 2026-11-06 | `e7e42d01e067` | Main run cancelled; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/42146475737)) |
-| [#1099](https://github.com/mboworks/xff/pull/1099) | 2026-11-05 | `76185cb37fa2` | Main run failure; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/41999670147)) |
-| [#1098](https://github.com/mboworks/xff/pull/1098) | 2026-11-05 | `7c8cd64e02a8` | Main run failure; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/41986760505)) |
-| [#1097](https://github.com/mboworks/xff/pull/1097) | 2026-11-05 | `910e2a7bbae5` | Main run cancelled; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/41977457900)) |
-| [#1095](https://github.com/mboworks/xff/pull/1095) | 2026-11-05 | `6792b5030de7` | Main run cancelled; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/41962227281)) |
-| [#1092](https://github.com/mboworks/xff/pull/1092) | 2026-11-04 | `a89459ed819a` | Main run cancelled; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/41920310179)) |
-| [#1088](https://github.com/mboworks/xff/pull/1088) | 2026-11-04 | `9bd70fc9b460` | Main run failure; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/41851695947)) |
-| [#1084](https://github.com/mboworks/xff/pull/1084) | 2026-11-04 | `f06cea15f5b1` | Main run cancelled; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/41798644066)) |
-| [#1082](https://github.com/mboworks/xff/pull/1082) | 2026-11-04 | `6e6b25ffc869` | Main run failure; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/41783344161)) |
-| [#1080](https://github.com/mboworks/xff/pull/1080) | 2026-11-04 | `669f68a3e263` | Main run failure; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/41771864734)) |
-| [#1078](https://github.com/mboworks/xff/pull/1078) | 2026-11-03 | `76054b3a9c60` | Main run failure; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/41711975639)) |
-| [#1073](https://github.com/mboworks/xff/pull/1073) | 2026-11-03 | `6db2de9e49f1` | Main run cancelled; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/41628320126)) |
-| [#1067](https://github.com/mboworks/xff/pull/1067) | 2026-11-03 | `fb726c946063` | Main run cancelled; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/41573418793)) |
-| [#1059](https://github.com/mboworks/xff/pull/1059) | 2026-11-03 | `22702c34a0bc` | Main run cancelled; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/41559497818)) |
-| [#1057](https://github.com/mboworks/xff/pull/1057) | 2026-11-02 | `6bebe940ba36` | Main run cancelled; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/41524744291)) |
-| [#1053](https://github.com/mboworks/xff/pull/1053) | 2026-11-02 | `dc4e70237eff` | Main run cancelled; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/41451327523)) |
-| [#1050](https://github.com/mboworks/xff/pull/1050) | 2026-11-02 | `2d615ea1e447` | Main run failure; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/41425575717)) |
-| [#1044](https://github.com/mboworks/xff/pull/1044) | 2026-11-02 | `d546fcb3d864` | Main run cancelled; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/41378115229)) |
-| [#1042](https://github.com/mboworks/xff/pull/1042) | 2026-11-02 | `2b21790502fd` | Main run cancelled; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/41367091220)) |
-| [#1027](https://github.com/mboworks/xff/pull/1027) | 2026-10-29 | `d406f4132d06` | Main run cancelled; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/40816218296)) |
-| [#1011](https://github.com/mboworks/xff/pull/1011) | 2026-10-23 | `801087f3e9a8` | Main run failure; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/39886813900)) |
-| [#1007](https://github.com/mboworks/xff/pull/1007) | 2026-10-14 | `33a01e13d7cd` | Main run cancelled; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/38653097649)) |
-| [#1006](https://github.com/mboworks/xff/pull/1006) | 2026-10-14 | `20095dd53c31` | Main run failure; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/38644322359)) |
-| [#994](https://github.com/mboworks/xff/pull/994) | 2026-10-10 | `88d4facd15eb` | Main run failure; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/38049489783)) |
-| [#993](https://github.com/mboworks/xff/pull/993) | 2026-10-10 | `dac3747d63ca` | Main run cancelled; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/38045263735)) |
-| [#976](https://github.com/mboworks/xff/pull/976) | 2026-10-06 | `73b2cfcd7101` | Main run cancelled; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/37393233041)) |
-| [#979](https://github.com/mboworks/xff/pull/979) | 2026-10-05 | `378d3540691e` | Main run cancelled; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/37388497650)) |
-| [#973](https://github.com/mboworks/xff/pull/973) | 2026-10-05 | `dfd5447644f0` | Main run cancelled; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/37388437775)) |
-| [#954](https://github.com/mboworks/xff/pull/954) | 2026-10-03 | `2817247f59cb` | Main run failure; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/37138060568)) |
-| [#950](https://github.com/mboworks/xff/pull/950) | 2026-10-03 | `c262c0a71360` | Main run failure; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/37114542638)) |
-| [#939](https://github.com/mboworks/xff/pull/939) | 2026-10-01 | `cc01d2edbab6` | No main-push benchmark run found; no retained report discovered |
-| [#935](https://github.com/mboworks/xff/pull/935) | 2026-10-01 | `481c32aaf2bd` | Main run failure; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/36831896372)) |
-| [#936](https://github.com/mboworks/xff/pull/936) | 2026-10-01 | `be954012e048` | No main-push benchmark run found; no retained report discovered |
-| [#928](https://github.com/mboworks/xff/pull/928) | 2026-09-27 | `f0dfc3edefc7` | No main-push benchmark run found; no retained report discovered |
-| [#916](https://github.com/mboworks/xff/pull/916) | 2026-09-27 | `ce5ee3c40579` | No main-push benchmark run found; no retained report discovered |
-| [#902](https://github.com/mboworks/xff/pull/902) | 2026-09-22 | `be2f400392dc` | Main run success; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/35677724823)) |
-| [#904](https://github.com/mboworks/xff/pull/904) | 2026-09-21 | `80d75576c32b` | Main run success; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/35667805574)) |
-| [#905](https://github.com/mboworks/xff/pull/905) | 2026-09-21 | `77c00ec8cbea` | Main run success; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/35666152368)) |
-| [#903](https://github.com/mboworks/xff/pull/903) | 2026-09-21 | `9b04999acb79` | Main run success; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/35664420733)) |
-| [#906](https://github.com/mboworks/xff/pull/906) | 2026-09-21 | `79278f51e63c` | Main run success; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/35662475460)) |
-| [#901](https://github.com/mboworks/xff/pull/901) | 2026-09-21 | `f76015ff96f8` | Main run success; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/35653591054)) |
-| [#899](https://github.com/mboworks/xff/pull/899) | 2026-09-21 | `98d31e1616a7` | Main run success; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/35564630217)) |
-| [#898](https://github.com/mboworks/xff/pull/898) | 2026-09-20 | `c0c0b6d39e26` | Main run success; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/35538423416)) |
-| [#897](https://github.com/mboworks/xff/pull/897) | 2026-09-20 | `6e882aea6d2c` | Main run success; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/35536193393)) |
-| [#896](https://github.com/mboworks/xff/pull/896) | 2026-09-20 | `ced00ac473e6` | Main run success; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/35529346711)) |
-| [#895](https://github.com/mboworks/xff/pull/895) | 2026-09-20 | `a185ae12d70d` | Main run success; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/35521619198)) |
-| [#894](https://github.com/mboworks/xff/pull/894) | 2026-09-20 | `42febc0e8d0a` | Main run success; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/35510379808)) |
-| [#893](https://github.com/mboworks/xff/pull/893) | 2026-09-20 | `29c75be40529` | Main run success; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/35503333615)) |
-| [#892](https://github.com/mboworks/xff/pull/892) | 2026-09-20 | `c9b56c2f009e` | Main run success; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/35502743136)) |
-| [#891](https://github.com/mboworks/xff/pull/891) | 2026-09-20 | `a39758eff8ac` | Main run success; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/35499899635)) |
-| [#890](https://github.com/mboworks/xff/pull/890) | 2026-09-19 | `1f568ceecd28` | Main run success; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/35474269925)) |
-| [#888](https://github.com/mboworks/xff/pull/888) | 2026-09-19 | `b7d28411a9cc` | Main run success; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/35473674929)) |
-| [#887](https://github.com/mboworks/xff/pull/887) | 2026-09-19 | `0bcb461dbb08` | Main run success; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/35470455554)) |
-| [#886](https://github.com/mboworks/xff/pull/886) | 2026-09-19 | `2bd76d6e682f` | No main-push benchmark run found; no retained report discovered |
-| [#885](https://github.com/mboworks/xff/pull/885) | 2026-09-19 | `2448257a8d4d` | No main-push benchmark run found; no retained report discovered |
-| [#884](https://github.com/mboworks/xff/pull/884) | 2026-09-19 | `4fc594246067` | No main-push benchmark run found; no retained report discovered |
-| [#883](https://github.com/mboworks/xff/pull/883) | 2026-09-19 | `b1df434503a3` | No main-push benchmark run found; no retained report discovered |
-| [#882](https://github.com/mboworks/xff/pull/882) | 2026-09-19 | `9282e3d1da7c` | No main-push benchmark run found; no retained report discovered |
-| [#881](https://github.com/mboworks/xff/pull/881) | 2026-09-19 | `990045bf2d3b` | No main-push benchmark run found; no retained report discovered |
-| [#880](https://github.com/mboworks/xff/pull/880) | 2026-09-19 | `c8b35dfa8249` | No main-push benchmark run found; no retained report discovered |
-| [#879](https://github.com/mboworks/xff/pull/879) | 2026-09-19 | `7f48d0d888cb` | No main-push benchmark run found; no retained report discovered |
-| [#878](https://github.com/mboworks/xff/pull/878) | 2026-09-19 | `889c652ec8cf` | No main-push benchmark run found; no retained report discovered |
-| [#877](https://github.com/mboworks/xff/pull/877) | 2026-09-19 | `99b78c8f1bad` | No main-push benchmark run found; no retained report discovered |
-| [#876](https://github.com/mboworks/xff/pull/876) | 2026-09-19 | `e1cc1f72605e` | Linux backfill succeeded; macOS measuring ([campaign](https://github.com/mboworks/xff/actions/runs/42538554174)) |
-| [#868](https://github.com/mboworks/xff/pull/868) | 2026-09-19 | `c092e8245ff8` | Linux backfill succeeded; macOS measuring ([campaign](https://github.com/mboworks/xff/actions/runs/42538554174)) |
+Snapshot: November 9, 2026, 09:37 UTC. Scope: PRs merged directly into main on or after
+September 19, 2026, by that snapshot. Later merges, including #1155, are outside this dated
+inventory. Refresh before selecting another backfill campaign.
+
+A platform is covered only when the exact merge commit appears in the published merged-CI
+catalog. Local and extended-contract datasets do not fill gaps in this contract. A validated
+report awaiting publication is recorded separately from a selectable catalog entry.
+
+## Current coverage at the snapshot
+
+- 240 directly merged PRs: 171 published on both platforms; 69 missing from both viewer datasets.
+- PR #1154 has validated Linux and macOS reports awaiting publication. It needs publication,
+ not another measurement run.
+- The other 68 missing PRs lack validated retained reports at the snapshot. Four of them,
+ #881-884, already have an active bounded backfill campaign; do not dispatch them again.
+- PRs #877-880 are now published and browser-verified on both platforms. All eight selections
+ loaded their exact merge commit and chart data at 09:35 UTC, without page errors.
+- PR #1151 is also published on both platforms. PRs #868/#876 were browser-verified at 03:58 UTC;
+ #914/#986/#1143 were verified at 01:14 UTC.
+
+The earlier 05:10 snapshot counted 239 direct merges, 166 covered and 73 missing. Since then,
+#1154 merged while #1151 and #877-880 were published, producing 240/171/69. The November 8
+total of 252 included PRs targeting other development branches; those are outside this scope.
+
+## Missing PRs at the snapshot
+
+Every row lacks both published catalog entries. Historical workflow conclusions are observations,
+not established root causes: a successful workflow does not prove that usable reports remain,
+and a cancellation needs its own logs to establish why. Available reports and active backfills
+are identified separately.
+
+| PR | Merged (UTC) | Merge commit | Observed state |
+| -------------------------------------------------- | ------------ | -------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| [#1154](https://github.com/mboworks/xff/pull/1154) | 2026-11-09 | `ad3be8a12e05` | Both validated reports retained; awaiting publication ([main run](https://github.com/mboworks/xff/actions/runs/42583205104)) |
+| [#1136](https://github.com/mboworks/xff/pull/1136) | 2026-11-07 | `1f25ba6d4ae0` | Main run cancelled; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/42398990564)) |
+| [#1132](https://github.com/mboworks/xff/pull/1132) | 2026-11-07 | `5d413a35cbec` | Main run cancelled; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/42378502027)) |
+| [#1130](https://github.com/mboworks/xff/pull/1130) | 2026-11-07 | `ae8c5af3eb11` | Main run cancelled; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/42367122125)) |
+| [#1128](https://github.com/mboworks/xff/pull/1128) | 2026-11-07 | `1278c8943a6a` | Main run cancelled; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/42354657340)) |
+| [#1121](https://github.com/mboworks/xff/pull/1121) | 2026-11-06 | `d9bf84b04784` | Main run cancelled; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/42265512655)) |
+| [#1118](https://github.com/mboworks/xff/pull/1118) | 2026-11-06 | `c7bb70cc6f3e` | Main run cancelled; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/42234848791)) |
+| [#1115](https://github.com/mboworks/xff/pull/1115) | 2026-11-06 | `ab909839da7d` | Main run cancelled; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/42218900004)) |
+| [#1113](https://github.com/mboworks/xff/pull/1113) | 2026-11-06 | `5729026a8482` | Main run cancelled; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/42200480427)) |
+| [#1111](https://github.com/mboworks/xff/pull/1111) | 2026-11-06 | `5cade08b6701` | Main run cancelled; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/42184472882)) |
+| [#1106](https://github.com/mboworks/xff/pull/1106) | 2026-11-06 | `e7e42d01e067` | Main run cancelled; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/42146475737)) |
+| [#1099](https://github.com/mboworks/xff/pull/1099) | 2026-11-05 | `76185cb37fa2` | Main run failure; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/41999670147)) |
+| [#1098](https://github.com/mboworks/xff/pull/1098) | 2026-11-05 | `7c8cd64e02a8` | Main run failure; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/41986760505)) |
+| [#1097](https://github.com/mboworks/xff/pull/1097) | 2026-11-05 | `910e2a7bbae5` | Main run cancelled; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/41977457900)) |
+| [#1095](https://github.com/mboworks/xff/pull/1095) | 2026-11-05 | `6792b5030de7` | Main run cancelled; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/41962227281)) |
+| [#1092](https://github.com/mboworks/xff/pull/1092) | 2026-11-04 | `a89459ed819a` | Main run cancelled; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/41920310179)) |
+| [#1088](https://github.com/mboworks/xff/pull/1088) | 2026-11-04 | `9bd70fc9b460` | Main run failure; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/41851695947)) |
+| [#1084](https://github.com/mboworks/xff/pull/1084) | 2026-11-04 | `f06cea15f5b1` | Main run cancelled; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/41798644066)) |
+| [#1082](https://github.com/mboworks/xff/pull/1082) | 2026-11-04 | `6e6b25ffc869` | Main run failure; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/41783344161)) |
+| [#1080](https://github.com/mboworks/xff/pull/1080) | 2026-11-04 | `669f68a3e263` | Main run failure; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/41771864734)) |
+| [#1078](https://github.com/mboworks/xff/pull/1078) | 2026-11-03 | `76054b3a9c60` | Main run failure; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/41711975639)) |
+| [#1073](https://github.com/mboworks/xff/pull/1073) | 2026-11-03 | `6db2de9e49f1` | Main run cancelled; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/41628320126)) |
+| [#1067](https://github.com/mboworks/xff/pull/1067) | 2026-11-03 | `fb726c946063` | Main run cancelled; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/41573418793)) |
+| [#1059](https://github.com/mboworks/xff/pull/1059) | 2026-11-03 | `22702c34a0bc` | Main run cancelled; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/41559497818)) |
+| [#1057](https://github.com/mboworks/xff/pull/1057) | 2026-11-02 | `6bebe940ba36` | Main run cancelled; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/41524744291)) |
+| [#1053](https://github.com/mboworks/xff/pull/1053) | 2026-11-02 | `dc4e70237eff` | Main run cancelled; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/41451327523)) |
+| [#1050](https://github.com/mboworks/xff/pull/1050) | 2026-11-02 | `2d615ea1e447` | Main run failure; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/41425575717)) |
+| [#1044](https://github.com/mboworks/xff/pull/1044) | 2026-11-02 | `d546fcb3d864` | Main run cancelled; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/41378115229)) |
+| [#1042](https://github.com/mboworks/xff/pull/1042) | 2026-11-02 | `2b21790502fd` | Main run cancelled; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/41367091220)) |
+| [#1027](https://github.com/mboworks/xff/pull/1027) | 2026-10-29 | `d406f4132d06` | Main run cancelled; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/40816218296)) |
+| [#1011](https://github.com/mboworks/xff/pull/1011) | 2026-10-23 | `801087f3e9a8` | Main run failure; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/39886813900)) |
+| [#1007](https://github.com/mboworks/xff/pull/1007) | 2026-10-14 | `33a01e13d7cd` | Main run cancelled; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/38653097649)) |
+| [#1006](https://github.com/mboworks/xff/pull/1006) | 2026-10-14 | `20095dd53c31` | Main run failure; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/38644322359)) |
+| [#994](https://github.com/mboworks/xff/pull/994) | 2026-10-10 | `88d4facd15eb` | Main run failure; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/38049489783)) |
+| [#993](https://github.com/mboworks/xff/pull/993) | 2026-10-10 | `dac3747d63ca` | Main run cancelled; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/38045263735)) |
+| [#979](https://github.com/mboworks/xff/pull/979) | 2026-10-05 | `378d3540691e` | Main run cancelled; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/37388497650)) |
+| [#976](https://github.com/mboworks/xff/pull/976) | 2026-10-06 | `73b2cfcd7101` | Main run cancelled; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/37393233041)) |
+| [#973](https://github.com/mboworks/xff/pull/973) | 2026-10-05 | `dfd5447644f0` | Main run cancelled; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/37388437775)) |
+| [#954](https://github.com/mboworks/xff/pull/954) | 2026-10-03 | `2817247f59cb` | Main run failure; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/37138060568)) |
+| [#950](https://github.com/mboworks/xff/pull/950) | 2026-10-03 | `c262c0a71360` | Main run failure; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/37114542638)) |
+| [#939](https://github.com/mboworks/xff/pull/939) | 2026-10-01 | `cc01d2edbab6` | No main-push benchmark run found; no validated retained report discovered |
+| [#936](https://github.com/mboworks/xff/pull/936) | 2026-10-01 | `be954012e048` | No main-push benchmark run found; no validated retained report discovered |
+| [#935](https://github.com/mboworks/xff/pull/935) | 2026-10-01 | `481c32aaf2bd` | Main run failure; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/36831896372)) |
+| [#928](https://github.com/mboworks/xff/pull/928) | 2026-09-27 | `f0dfc3edefc7` | No main-push benchmark run found; no validated retained report discovered |
+| [#916](https://github.com/mboworks/xff/pull/916) | 2026-09-27 | `ce5ee3c40579` | No main-push benchmark run found; no validated retained report discovered |
+| [#906](https://github.com/mboworks/xff/pull/906) | 2026-09-21 | `79278f51e63c` | Main run success; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/35662475460)) |
+| [#905](https://github.com/mboworks/xff/pull/905) | 2026-09-21 | `77c00ec8cbea` | Main run success; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/35666152368)) |
+| [#904](https://github.com/mboworks/xff/pull/904) | 2026-09-21 | `80d75576c32b` | Main run success; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/35667805574)) |
+| [#903](https://github.com/mboworks/xff/pull/903) | 2026-09-21 | `9b04999acb79` | Main run success; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/35664420733)) |
+| [#902](https://github.com/mboworks/xff/pull/902) | 2026-09-22 | `be2f400392dc` | Main run success; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/35677724823)) |
+| [#901](https://github.com/mboworks/xff/pull/901) | 2026-09-21 | `f76015ff96f8` | Main run success; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/35653591054)) |
+| [#899](https://github.com/mboworks/xff/pull/899) | 2026-09-21 | `98d31e1616a7` | Main run success; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/35564630217)) |
+| [#898](https://github.com/mboworks/xff/pull/898) | 2026-09-20 | `c0c0b6d39e26` | Main run success; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/35538423416)) |
+| [#897](https://github.com/mboworks/xff/pull/897) | 2026-09-20 | `6e882aea6d2c` | Main run success; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/35536193393)) |
+| [#896](https://github.com/mboworks/xff/pull/896) | 2026-09-20 | `ced00ac473e6` | Main run success; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/35529346711)) |
+| [#895](https://github.com/mboworks/xff/pull/895) | 2026-09-20 | `a185ae12d70d` | Main run success; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/35521619198)) |
+| [#894](https://github.com/mboworks/xff/pull/894) | 2026-09-20 | `42febc0e8d0a` | Main run success; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/35510379808)) |
+| [#893](https://github.com/mboworks/xff/pull/893) | 2026-09-20 | `29c75be40529` | Main run success; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/35503333615)) |
+| [#892](https://github.com/mboworks/xff/pull/892) | 2026-09-20 | `c9b56c2f009e` | Main run success; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/35502743136)) |
+| [#891](https://github.com/mboworks/xff/pull/891) | 2026-09-20 | `a39758eff8ac` | Main run success; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/35499899635)) |
+| [#890](https://github.com/mboworks/xff/pull/890) | 2026-09-19 | `1f568ceecd28` | Main run success; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/35474269925)) |
+| [#888](https://github.com/mboworks/xff/pull/888) | 2026-09-19 | `b7d28411a9cc` | Main run success; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/35473674929)) |
+| [#887](https://github.com/mboworks/xff/pull/887) | 2026-09-19 | `0bcb461dbb08` | Main run success; no validated retained report discovered ([run](https://github.com/mboworks/xff/actions/runs/35470455554)) |
+| [#886](https://github.com/mboworks/xff/pull/886) | 2026-09-19 | `2bd76d6e682f` | No main-push benchmark run found; no validated retained report discovered |
+| [#885](https://github.com/mboworks/xff/pull/885) | 2026-09-19 | `2448257a8d4d` | No main-push benchmark run found; no validated retained report discovered |
+| [#884](https://github.com/mboworks/xff/pull/884) | 2026-09-19 | `4fc594246067` | No main-push benchmark run found; no validated retained report discovered; already in [active backfill](https://github.com/mboworks/xff/actions/runs/42589939294) |
+| [#883](https://github.com/mboworks/xff/pull/883) | 2026-09-19 | `b1df434503a3` | No main-push benchmark run found; no validated retained report discovered; already in [active backfill](https://github.com/mboworks/xff/actions/runs/42589939294) |
+| [#882](https://github.com/mboworks/xff/pull/882) | 2026-09-19 | `9282e3d1da7c` | No main-push benchmark run found; no validated retained report discovered; already in [active backfill](https://github.com/mboworks/xff/actions/runs/42589939294) |
+| [#881](https://github.com/mboworks/xff/pull/881) | 2026-09-19 | `990045bf2d3b` | No main-push benchmark run found; no validated retained report discovered; already in [active backfill](https://github.com/mboworks/xff/actions/runs/42589939294) |
+
+## Completed backfill and browser publication
+
+[Campaign 42566817872](https://github.com/mboworks/xff/actions/runs/42566817872) measured
+PRs #877-880 on Linux and macOS. All eight attempt-1 jobs failed on HTTP 504 dependency
+downloads before compiling or measuring. The exact failure logs and download preflights remain.
+After downloads recovered, one failed-only rerun was requested. Attempt 2 completed successfully
+on every configuration. No successful configuration needs another measurement run.
+
+The first browser check at 08:47 UTC found none of these eight selections. The subsequent
+09:35 UTC check loaded all eight exact commits and landscape payloads, without page errors.
+Both observations remain. Successful measurement was verified separately from publication;
+the publication delay did not require repeating the measurements.
+
+## Active bounded campaign
+
+After a fresh check confirmed that no backfill was active, exactly PRs #881-884 were dispatched
+for both platforms at 08:59 UTC in
+[campaign 42589939294](https://github.com/mboworks/xff/actions/runs/42589939294).
+The 09:37 inventory still observes this original campaign as active. PRs #877-880 were excluded
+because their successful artifacts were already available. The dispatch receipt retains the
+selected commits, inventory hash, live campaign preflight, exact command, timestamp and result.
+Observe the original campaign to completion; preserve successes and retry failed configurations
+only after investigating terminal failures. Its dispatch is not proof of published viewer entries.
## Reproduction and retained evidence
-Refresh the inventory without starting measurements:
+Refresh the read-only planner with:
-```sh
-bazel run //tools:benchmark_recovery -- --repository mboworks/xff
-```
+ bazel run //tools:benchmark_recovery -- --repository mboworks/xff
-The read-only planner reports missing platforms, live main measurements and validated retained
-reports awaiting publication. Active backfill campaigns require an additional workflow-status
-check; their incomplete plans are not yet included in the planner's completed-campaign inventory.
+The planner distinguishes missing platforms, active measurements and validated reports awaiting
+publication. Check active backfill workflows separately. PR #1153 contains the planner fix that
+subtracts available platforms before grouping requests, and propagates API observation errors;
+it remains unmerged at this snapshot. Every dispatch must preserve available configurations.
-This snapshot retains the complete responses locally as:
+The complete 09:37 inventory, request index and raw API responses are under
+`/home/marcus/Documents/xff-coverage-refresh-post-backfill-20261109/post-browser-877-880/`
+on drogon. The audit rejects non-GET API requests and identifies source revision
+`d7dee8aca41fea4c9d287800946a546377e5a85e`. The 94-response index records content hashes.
+Document generation independently checked all three main-run response pages against that index.
+The complete missing rows and artifact classifications come from the retained inventory.
-- `/private/tmp/xff-missing-merged-prs-20261109-refresh.json` and its diagnostic log;
-- `/private/tmp/xff-coverage-pulls-20261109.json`: all 1,139 closed PRs across 12 API pages;
-- `/private/tmp/xff-coverage-runs-20261109.json`: all 225 main benchmark runs across three pages;
-- `/private/tmp/xff-published-catalog-live-20261109.json`: 590 published entries, catalog Git blob
- `854b5bf4cf46515b5d0ca70dd0fa6f83de384233`.
-
-These are observation snapshots, not pending instructions to dispatch a new recovery campaign.
+Earlier snapshots remain separate under the same root's `snapshot/` and
+`post-campaign-42566817872/` directories. Failed attempt-1 logs and the rerun receipt remain
+under `campaign-42566817872/`. The next dispatch is recorded in
+`dispatch-881-884-receipt.json`. Browser evidence for #877-880 is retained as
+`xff-recovered-877-880-browser-20261109.{json,png,log}` for 08:47 and
+`xff-recovered-877-880-after-publisher-20261109.{json,png,log}` for 09:35.
## Preservation and remaining work
-- Preserve successful backfills and let the active #868/#876 jobs finish. Reconciliation publishes
- valid configurations independently; a failed sibling must not force successful work to rerun.
-- Retry only a genuinely missing or failed configuration. PR #1147 added targeted revision inputs;
- #1148 added the reusable inventory/recovery command. #1149/#1150 fixed historical Linux runtime
- preparation. The additional planner fixes in #1153 avoid repeating available platforms across
- different missing-platform sets and propagate API observation failures.
-- The publisher coalesces pending requests and reconciles available artifacts under the shared site
- lock, with a six-hour scheduled reconciliation. This republishes existing evidence; it does not
- automatically measure every missing merge.
-- A durable per-PR/platform retry queue, pending/failed viewer entries and automatic completeness
- enforcement remain proposed work. A finite GitHub concurrency queue alone cannot prove coverage.
-- Historical backfill entries currently report `Not verified` with a workflow-provenance mismatch.
- Their workflow driver commit differs from the measured historical commit. Validating that relation
- requires the retained campaign/build provenance; do not bypass the check or label them verified.
+- Preserve all successful reports; retry only missing or failed configurations after observing
+ the original run's terminal state and investigating its failure. Do not cancel active work
+ merely because observation timed out.
+- The coalescing publisher and six-hour scheduled reconciliation can publish retained reports.
+ They do not automatically measure every missing merge. A durable retry queue, pending/failed
+ viewer entries and automatic completeness enforcement remain proposed work.
+- Historical entries conservatively show Not verified when workflow-driver provenance differs
+ from the measured source. Do not retrofit unsupported binary verification.
+- The 130-revision extended-production-v1 history is a separate contract and does not fill these
+ original merged-CI gaps. Its continuous integration and live publication remain separate work.
+
+### Earlier historical Linux compatibility and timeouts
+
+The original #914/#986 Linux campaign failed on missing historical linker libraries. Selecting
+Ubuntu 22.04 then allowed compilation but produced binaries that required a newer glibc
+(`GLIBC_ABI_DT_RELR`). The deployed follow-up uses Ubuntu 24.04 and obtains `libicu70` from signed
+Jammy package indexes without changing the runner's configured sources. Runtime receipts record
+package and loaded-library hashes, OS, glibc and preparer identity. Historical compiler/source/LTO
+configurations remain fixed. See
+[the retained runtime smoke proof](benchmarks/historical-runtime-smoke-20261108.json).
+The successful [Linux retry](https://github.com/mboworks/xff/actions/runs/42534282665) has now been
+published; the obsolete Ubuntu 22.04 campaign requires no further retry.
+
+The original macOS #914 warmup exceeded a 60-second command limit. New backfill plans record a
+300-second command limit; [its macOS retry](https://github.com/mboworks/xff/actions/runs/42519073200)
+succeeded. Preserve that report and the original successful macOS #986 report. Timeouts terminate
+the worker process group, and no timed-out sample is retained. These historical failures are
+resolved for those two PRs; the #877-880 attempt-1 HTTP 504 failures occurred before that runtime was exercised.
## Browser verification
-Chromium selected PRs #914, #986 and #1143 in both CI datasets on the
-[published viewer](https://mboworks.github.io/xff/benchmarks/) at 01:14 UTC on November 9.
-All six selections loaded their actual chart payloads and displayed the expected merge commit,
-with no page errors. Their binary state remains `Not verified`, matching the conservative
-provenance limitation above. The retained check is
-`/private/tmp/xff-recovered-browser-20261109.json` and its companion screenshot.
+The published viewer loaded #914, #986 and #1143 in both CI datasets at 01:14 UTC, and #868 and
+#876 on both at 03:58 UTC. Those checks retained exact merge commits, actual loaded landscape
+payloads, conservative binary states, screenshots and absence of page errors.
+
+The 09:35 check for #877-880 retained all eight successful selections. Each loaded its own
+merge commit and landscape in both `linux / x86_64` and `macos / arm64`. Binary state remained
+Not verified; publication does not manufacture provenance. The catalog had 570 entries, and
+the browser reported neither missing selections nor page errors. Future recovered entries
+require the same actual selection and payload checks.
diff --git a/docs/benchmarks/extended-history-continuous-qualification-20261109.json b/docs/benchmarks/extended-history-continuous-qualification-20261109.json
new file mode 100644
index 0000000000..01f2529bdd
--- /dev/null
+++ b/docs/benchmarks/extended-history-continuous-qualification-20261109.json
@@ -0,0 +1,418 @@
+{
+ "schema": 1,
+ "purpose": "Extended-history import and collector correctness qualification; not an optimization claim",
+ "import": {
+ "schema": 1,
+ "original_observations": 130,
+ "validated_original_cells": 7800,
+ "validated_original_invocations": 234000,
+ "retained_core_catalog_entries": 588,
+ "extended_catalog_entries": 260,
+ "series": {
+ "extended-production-v1 / Fixed XFF ae8c5af3eb11": 130,
+ "extended-production-v1 / Traditional tools": 130
+ },
+ "measured_points_per_view": 60,
+ "unavailable_points_per_view": 20,
+ "all_three_order_modes_match_actual_samples": true,
+ "normalization_cells": {
+ "available": 15600
+ },
+ "archive_sha256": "985d4c471bb91b917e54202406e662e5c19c0b5825e06e095b4b754a42b62ecc",
+ "binary_evidence": "Verified against Retention (original frozen executable bytes, both views)",
+ "published_snapshot_commit": "08a0e7ad53e559f672035e0adcbfd5ee39b6bcec"
+ },
+ "browser": {
+ "url": "http://127.0.0.1:63790/benchmarks/",
+ "checked": [
+ {
+ "dataset": "Local / linux-zen5-extended-20261107 / extended-production-v1 / Fixed XFF ae8c5af3eb11",
+ "index": 0,
+ "commit": "21c7f3656b005b23113d936c1cd2ca0fd2221ab2",
+ "binary": "f8577e4753dc90640bd926ac80ce7277be9791491bf4e15bfdcd1a3777ef8e47"
+ },
+ {
+ "dataset": "Local / linux-zen5-extended-20261107 / extended-production-v1 / Fixed XFF ae8c5af3eb11",
+ "index": 65,
+ "commit": "09e6bbfff3275ed36eba7b98052cf5d0877d94f7",
+ "binary": "c21c424a876f82db3ac4de2458c8f47b5161dd49271c1b739f2d1ac6ee0f689b"
+ },
+ {
+ "dataset": "Local / linux-zen5-extended-20261107 / extended-production-v1 / Fixed XFF ae8c5af3eb11",
+ "index": 129,
+ "commit": "ae8c5af3eb11dc8571f11db9b307b0db948f78c6",
+ "binary": "c47c904b693c8c114bb87284497b98a10e4925f25afdc3f5ec4f26896ad1a2e1"
+ },
+ {
+ "dataset": "Local / linux-zen5-extended-20261107 / extended-production-v1 / Traditional tools",
+ "index": 0,
+ "commit": "21c7f3656b005b23113d936c1cd2ca0fd2221ab2",
+ "binary": "f8577e4753dc90640bd926ac80ce7277be9791491bf4e15bfdcd1a3777ef8e47"
+ },
+ {
+ "dataset": "Local / linux-zen5-extended-20261107 / extended-production-v1 / Traditional tools",
+ "index": 65,
+ "commit": "09e6bbfff3275ed36eba7b98052cf5d0877d94f7",
+ "binary": "c21c424a876f82db3ac4de2458c8f47b5161dd49271c1b739f2d1ac6ee0f689b"
+ },
+ {
+ "dataset": "Local / linux-zen5-extended-20261107 / extended-production-v1 / Traditional tools",
+ "index": 129,
+ "commit": "ae8c5af3eb11dc8571f11db9b307b0db948f78c6",
+ "binary": "c47c904b693c8c114bb87284497b98a10e4925f25afdc3f5ec4f26896ad1a2e1"
+ }
+ ],
+ "page_errors": []
+ },
+ "native": {
+ "linux": {
+ "source": "96765e1859c3d48ac0f04092c0613b4e8ad8ae21",
+ "parent": "52950f937ad371e1450538faf7983024b324688d",
+ "origin": "local",
+ "platform": "Linux-7.0.0-38-generic-x86_64-with-glibc2.43",
+ "target": "//xff/cli:xff_full",
+ "configuration": ["clang_release", "xff_full"],
+ "builds": {
+ "96765e1859c3d48ac0f04092c0613b4e8ad8ae21": {
+ "source": {
+ "revision": "96765e1859c3d48ac0f04092c0613b4e8ad8ae21",
+ "tree": "28719926a988f24150f67f9eb7947066d2488071",
+ "clean": true
+ },
+ "sha256": "4fd6b2f73f912a5f6f50b958e1a75842bcce57fcb35bbcdce13833b2ffc16877",
+ "bytes": 18473632,
+ "command": [
+ "bazel",
+ "--nosystem_rc",
+ "--nohome_rc",
+ "build",
+ "//xff/cli:xff_full",
+ "--config=clang_release",
+ "--config=xff_full",
+ "--disk_cache=/home/marcus/.cache/bazel-disk",
+ "--jobs=3"
+ ],
+ "query": [
+ "bazel",
+ "--nosystem_rc",
+ "--nohome_rc",
+ "cquery",
+ "config(//xff/cli:xff_full, target)",
+ "--config=clang_release",
+ "--config=xff_full",
+ "--disk_cache=/home/marcus/.cache/bazel-disk",
+ "--jobs=3",
+ "--output=starlark",
+ "--starlark:expr=target.files_to_run.executable.path"
+ ],
+ "artifact": "bazel-out/k8-opt/bin/xff/cli/xff_full",
+ "configuration": {
+ ".bazelrc": "d194a75cb8f8e5d39dd3dc8a853aae4d1215a4539aafa92fb044713ef1ddbea1",
+ ".bazelversion": "4d9dfd02f463b487bd0bff68cf9292b3626b31c81febb54e8911e493f64793d0",
+ "MODULE.bazel": "14ac2e29445ee90680e1ec7d6debaeb0b2fb22ef29326eb74acf9d5ac52ce75d",
+ "MODULE.bazel.lock": "c5448fb90798589c71e18295711c61ca471d8088e05e194245905bc5dcc68e71",
+ "bazelmod/llvm.MODULE.bazel": "91fe1ae8bc722dc8af130906212774bfc28ade948d265cd3e6c72f67c50e08aa"
+ }
+ },
+ "52950f937ad371e1450538faf7983024b324688d": {
+ "source": {
+ "revision": "52950f937ad371e1450538faf7983024b324688d",
+ "tree": "f99e563e2f1d6762190aee03c9ed0fd2c3e846f1",
+ "clean": true
+ },
+ "sha256": "4fd6b2f73f912a5f6f50b958e1a75842bcce57fcb35bbcdce13833b2ffc16877",
+ "bytes": 18473632,
+ "command": [
+ "bazel",
+ "--nosystem_rc",
+ "--nohome_rc",
+ "build",
+ "//xff/cli:xff_full",
+ "--config=clang_release",
+ "--config=xff_full",
+ "--disk_cache=/home/marcus/.cache/bazel-disk",
+ "--jobs=3"
+ ],
+ "query": [
+ "bazel",
+ "--nosystem_rc",
+ "--nohome_rc",
+ "cquery",
+ "config(//xff/cli:xff_full, target)",
+ "--config=clang_release",
+ "--config=xff_full",
+ "--disk_cache=/home/marcus/.cache/bazel-disk",
+ "--jobs=3",
+ "--output=starlark",
+ "--starlark:expr=target.files_to_run.executable.path"
+ ],
+ "artifact": "bazel-out/k8-opt/bin/xff/cli/xff_full",
+ "configuration": {
+ ".bazelrc": "d194a75cb8f8e5d39dd3dc8a853aae4d1215a4539aafa92fb044713ef1ddbea1",
+ ".bazelversion": "4d9dfd02f463b487bd0bff68cf9292b3626b31c81febb54e8911e493f64793d0",
+ "MODULE.bazel": "14ac2e29445ee90680e1ec7d6debaeb0b2fb22ef29326eb74acf9d5ac52ce75d",
+ "MODULE.bazel.lock": "c5448fb90798589c71e18295711c61ca471d8088e05e194245905bc5dcc68e71",
+ "bazelmod/llvm.MODULE.bazel": "91fe1ae8bc722dc8af130906212774bfc28ade948d265cd3e6c72f67c50e08aa"
+ }
+ }
+ },
+ "head_cells": 60,
+ "control_cells": 60,
+ "validated_invocations": 3600,
+ "sampling": {
+ "warmup": 1,
+ "measured": 9,
+ "retained": 7
+ },
+ "control_error": null,
+ "fixture": "/dev/shm/xff-per-entry-followup-20261108",
+ "storage_release": {
+ "filesystem": "tmpfs",
+ "storage": "memory",
+ "fixture": "/dev/shm/xff-per-entry-followup-20261108",
+ "borrowed": true,
+ "preserved": true,
+ "released_at": "2026-11-08T23:59:03.795281+00:00"
+ },
+ "binary_evidence": {
+ "traditional": {
+ "schema": 1,
+ "method": "sha256-retained-executable",
+ "sha256": "4fd6b2f73f912a5f6f50b958e1a75842bcce57fcb35bbcdce13833b2ffc16877",
+ "measurement_identity": "f8ff95dfcaf473df6ba5fdd2b7110b731e08c062bc824cbb541670c65d9cda70",
+ "retention_reference": "original frozen full-production executable used for local paired validation",
+ "verified_at": "2026-11-08T23:59:03.960422+00:00"
+ },
+ "anchor": {
+ "schema": 1,
+ "method": "sha256-retained-executable",
+ "sha256": "4fd6b2f73f912a5f6f50b958e1a75842bcce57fcb35bbcdce13833b2ffc16877",
+ "measurement_identity": "74d24587d2d8b9ce656166758b7db15a084a56628d97a1fe0018290129ed3688",
+ "retention_reference": "original frozen full-production executable used for local paired validation",
+ "verified_at": "2026-11-08T23:59:03.975087+00:00"
+ }
+ },
+ "raw_observation_sha256": "71fa57acbc15958232e2c245343bb470b99c55d8eafa965b1d5fa5820d635f3d",
+ "raw_evidence": "/home/marcus/Documents/xff-per-entry-followup-20261108/extended-native-validation/collection/observation.json"
+ },
+ "macos": {
+ "source": "96765e1859c3d48ac0f04092c0613b4e8ad8ae21",
+ "parent": "52950f937ad371e1450538faf7983024b324688d",
+ "origin": "local",
+ "platform": "macOS-26.6-arm64-arm-64bit-Mach-O",
+ "target": "//xff/cli:xff_full",
+ "configuration": ["clang_release", "xff_full"],
+ "builds": {
+ "96765e1859c3d48ac0f04092c0613b4e8ad8ae21": {
+ "source": {
+ "revision": "96765e1859c3d48ac0f04092c0613b4e8ad8ae21",
+ "tree": "28719926a988f24150f67f9eb7947066d2488071",
+ "clean": true
+ },
+ "sha256": "b014fc984a68514826c2e32c3be7816baa32ac441753a18e12c75ed7e721b8b3",
+ "bytes": 8278888,
+ "command": [
+ "bazel",
+ "--nosystem_rc",
+ "--nohome_rc",
+ "build",
+ "//xff/cli:xff_full",
+ "--config=clang_release",
+ "--config=xff_full",
+ "--disk_cache=/Users/marcus/.cache/bazel-disk",
+ "--jobs=3"
+ ],
+ "query": [
+ "bazel",
+ "--nosystem_rc",
+ "--nohome_rc",
+ "cquery",
+ "config(//xff/cli:xff_full, target)",
+ "--config=clang_release",
+ "--config=xff_full",
+ "--disk_cache=/Users/marcus/.cache/bazel-disk",
+ "--jobs=3",
+ "--output=starlark",
+ "--starlark:expr=target.files_to_run.executable.path"
+ ],
+ "artifact": "bazel-out/darwin_arm64-opt/bin/xff/cli/xff_full",
+ "configuration": {
+ ".bazelrc": "d194a75cb8f8e5d39dd3dc8a853aae4d1215a4539aafa92fb044713ef1ddbea1",
+ ".bazelversion": "4d9dfd02f463b487bd0bff68cf9292b3626b31c81febb54e8911e493f64793d0",
+ "MODULE.bazel": "14ac2e29445ee90680e1ec7d6debaeb0b2fb22ef29326eb74acf9d5ac52ce75d",
+ "MODULE.bazel.lock": "585f641b4aa3c00c77634b2ef3d8f73da48cf99e03ea6c3727b5d7b89fcda26f",
+ "bazelmod/llvm.MODULE.bazel": "91fe1ae8bc722dc8af130906212774bfc28ade948d265cd3e6c72f67c50e08aa"
+ }
+ },
+ "52950f937ad371e1450538faf7983024b324688d": {
+ "source": {
+ "revision": "52950f937ad371e1450538faf7983024b324688d",
+ "tree": "f99e563e2f1d6762190aee03c9ed0fd2c3e846f1",
+ "clean": true
+ },
+ "sha256": "b014fc984a68514826c2e32c3be7816baa32ac441753a18e12c75ed7e721b8b3",
+ "bytes": 8278888,
+ "command": [
+ "bazel",
+ "--nosystem_rc",
+ "--nohome_rc",
+ "build",
+ "//xff/cli:xff_full",
+ "--config=clang_release",
+ "--config=xff_full",
+ "--disk_cache=/Users/marcus/.cache/bazel-disk",
+ "--jobs=3"
+ ],
+ "query": [
+ "bazel",
+ "--nosystem_rc",
+ "--nohome_rc",
+ "cquery",
+ "config(//xff/cli:xff_full, target)",
+ "--config=clang_release",
+ "--config=xff_full",
+ "--disk_cache=/Users/marcus/.cache/bazel-disk",
+ "--jobs=3",
+ "--output=starlark",
+ "--starlark:expr=target.files_to_run.executable.path"
+ ],
+ "artifact": "bazel-out/darwin_arm64-opt/bin/xff/cli/xff_full",
+ "configuration": {
+ ".bazelrc": "d194a75cb8f8e5d39dd3dc8a853aae4d1215a4539aafa92fb044713ef1ddbea1",
+ ".bazelversion": "4d9dfd02f463b487bd0bff68cf9292b3626b31c81febb54e8911e493f64793d0",
+ "MODULE.bazel": "14ac2e29445ee90680e1ec7d6debaeb0b2fb22ef29326eb74acf9d5ac52ce75d",
+ "MODULE.bazel.lock": "585f641b4aa3c00c77634b2ef3d8f73da48cf99e03ea6c3727b5d7b89fcda26f",
+ "bazelmod/llvm.MODULE.bazel": "91fe1ae8bc722dc8af130906212774bfc28ade948d265cd3e6c72f67c50e08aa"
+ }
+ }
+ },
+ "head_cells": 60,
+ "control_cells": 60,
+ "validated_invocations": 3600,
+ "sampling": {
+ "warmup": 1,
+ "measured": 9,
+ "retained": 7
+ },
+ "control_error": null,
+ "fixture": "/private/tmp/xff-extended-native-validation-macos-20261108/collection/ram-volume/fixture",
+ "storage_release": {
+ "device": "/dev/disk4",
+ "image": "ram://4194304",
+ "storage": "memory",
+ "filesystem": "case-sensitive HFS+",
+ "created_at": "2026-11-08T23:57:15.208935+00:00",
+ "mount": "/private/tmp/xff-extended-native-validation-macos-20261108/collection/ram-volume",
+ "uuid": "A03B359F-E90E-38C0-96B4-439E7BF90389",
+ "released_at": "2026-11-09T00:20:53.956904+00:00"
+ },
+ "binary_evidence": {
+ "traditional": {
+ "schema": 1,
+ "method": "sha256-retained-executable",
+ "sha256": "b014fc984a68514826c2e32c3be7816baa32ac441753a18e12c75ed7e721b8b3",
+ "measurement_identity": "d24dba4764229a0b5e7e2f60c1e57aabc635b33efd89639736acf226fec126dd",
+ "retention_reference": "original frozen full-production executable used for local paired validation",
+ "verified_at": "2026-11-09T00:20:54.189423+00:00"
+ },
+ "anchor": {
+ "schema": 1,
+ "method": "sha256-retained-executable",
+ "sha256": "b014fc984a68514826c2e32c3be7816baa32ac441753a18e12c75ed7e721b8b3",
+ "measurement_identity": "8ea324153b17ab9845c7cc56278ac9ecfe80faaee85707943fb0464732097c8b",
+ "retention_reference": "original frozen full-production executable used for local paired validation",
+ "verified_at": "2026-11-09T00:20:54.203892+00:00"
+ }
+ },
+ "raw_observation_sha256": "4ee36606de59184ca94d958a3dd0b4ba16fb57a45271e05d4e83835f4aaa6786",
+ "raw_evidence": "/private/tmp/xff-extended-native-validation-macos-20261108/collection/observation.json"
+ }
+ },
+ "limits": [
+ "Native pipeline qualification uses byte-identical head/parent production executables and makes no performance-improvement claim.",
+ "Frozen native driver source is 96765e1859c3d48ac0f04092c0613b4e8ad8ae21; follow-ups tighten validation and improve build logging/publication without changing measured commands, sampling, or C++ sources.",
+ "Original imported observations remain local fixed-reference history; GitHub collection must establish CI provenance separately.",
+ "Historical data publication and the first real post-merge extended CI run remain required."
+ ],
+ "tests": {
+ "macos_bazel": {
+ "benchmark_extended_ci_test": {
+ "tests": 1,
+ "failures": 0,
+ "errors": 0
+ },
+ "benchmark_extended_records_test": {
+ "tests": 1,
+ "failures": 0,
+ "errors": 0
+ },
+ "benchmark_extended_site_test": {
+ "tests": 1,
+ "failures": 0,
+ "errors": 0
+ },
+ "benchmark_extended_storage_test": {
+ "tests": 1,
+ "failures": 0,
+ "errors": 0
+ },
+ "benchmark_extended_build_test": {
+ "tests": 1,
+ "failures": 0,
+ "errors": 0
+ },
+ "benchmark_extended_history_test": {
+ "tests": 1,
+ "failures": 0,
+ "errors": 0
+ },
+ "benchmark_extended_report_test": {
+ "tests": 1,
+ "failures": 0,
+ "errors": 0
+ },
+ "benchmark_extended_test": {
+ "tests": 1,
+ "failures": 0,
+ "errors": 0
+ },
+ "benchmark_reconcile_ci_test": {
+ "tests": 1,
+ "failures": 0,
+ "errors": 0
+ },
+ "benchmark_datasets_test": {
+ "tests": 1,
+ "failures": 0,
+ "errors": 0
+ },
+ "benchmark_history_test": {
+ "tests": 1,
+ "failures": 0,
+ "errors": 0
+ },
+ "benchmark_recovery_test": {
+ "tests": 1,
+ "failures": 0,
+ "errors": 0
+ }
+ },
+ "linux_direct": {
+ "extended": 54,
+ "reconcile": 8,
+ "recovery": 8,
+ "failures": 0
+ },
+ "real_build_logging_smoke": {
+ "source": "52950f937ad371e1450538faf7983024b324688d",
+ "sha256": "b014fc984a68514826c2e32c3be7816baa32ac441753a18e12c75ed7e721b8b3",
+ "result": "passed"
+ }
+ },
+ "cpu_allocation_policy": {
+ "ci_linux": "Three allowed logical CPUs, including sibling threads; capacity checked before builds.",
+ "local_linux": "Distinct allowed guest-topology cores, preserving original history policy.",
+ "macos": "Worker requests without CPU affinity.",
+ "verification": "Allocation policy is unit-tested on Linux and macOS; the first actual extended GitHub run remains required."
+ }
+}
diff --git a/release-site.json b/release-site.json
index ef3df53f3e..d75ef07a37 100644
--- a/release-site.json
+++ b/release-site.json
@@ -182,6 +182,7 @@
}
],
"files": {
+ "docs/benchmarks/extended-history-continuous-qualification-20261109.json": "docs/benchmarks/extended-history-continuous-qualification-20261109.json",
"docs/benchmarks/transactional-rg-records-20261107.json.gz": "docs/benchmarks/transactional-rg-records-20261107.json.gz",
"docs/benchmarks/language-json-storage-20261107.json.gz": "docs/benchmarks/language-json-storage-20261107.json.gz",
"docs/benchmarks/archive-ancestor-hint-20261107.json.gz": "docs/benchmarks/archive-ancestor-hint-20261107.json.gz",
diff --git a/tools/BUILD.bazel b/tools/BUILD.bazel
index 9f7740d850..d2bd454735 100644
--- a/tools/BUILD.bazel
+++ b/tools/BUILD.bazel
@@ -926,8 +926,12 @@ test_suite(
":benchmark_compare_test",
":benchmark_datasets_test",
":benchmark_extended_build_test",
+ ":benchmark_extended_ci_test",
":benchmark_extended_history_test",
+ ":benchmark_extended_records_test",
":benchmark_extended_report_test",
+ ":benchmark_extended_site_test",
+ ":benchmark_extended_storage_test",
":benchmark_extended_test",
":benchmark_fixture_test",
":benchmark_history_test",
@@ -940,6 +944,7 @@ test_suite(
":benchmark_preview_ci_test",
":benchmark_preview_test",
":benchmark_provenance_test",
+ ":benchmark_reconcile_ci_test",
":benchmark_regex_test",
":benchmark_shards_test",
":benchmark_test",
@@ -1263,6 +1268,131 @@ py_test(
deps = [":benchmark_extended_report_lib"],
)
+py_library(
+ name = "benchmark_extended_records_lib",
+ srcs = ["benchmark_extended_records.py"],
+ imports = ["."],
+ deps = [
+ ":benchmark_extended_lib",
+ ":benchmark_extended_report_lib",
+ ":benchmark_provenance_lib",
+ ],
+)
+
+py_library(
+ name = "benchmark_extended_storage_lib",
+ srcs = ["benchmark_extended_storage.py"],
+ imports = ["."],
+)
+
+py_library(
+ name = "benchmark_extended_ci_lib",
+ srcs = ["benchmark_extended_ci.py"],
+ imports = ["."],
+ deps = [
+ ":benchmark_extended_build_lib",
+ ":benchmark_extended_history_lib",
+ ":benchmark_extended_lib",
+ ":benchmark_extended_records_lib",
+ ":benchmark_extended_storage_lib",
+ ":process_resources_lib",
+ ],
+)
+
+py_binary(
+ name = "benchmark_extended_ci",
+ srcs = ["benchmark_extended_ci.py"],
+ deps = [
+ ":benchmark_extended_build_lib",
+ ":benchmark_extended_history_lib",
+ ":benchmark_extended_lib",
+ ":benchmark_extended_records_lib",
+ ":benchmark_extended_storage_lib",
+ ":process_resources_lib",
+ ],
+)
+
+py_library(
+ name = "benchmark_extended_site_lib",
+ srcs = ["benchmark_extended_site.py"],
+ imports = ["."],
+ deps = [
+ ":benchmark_datasets_lib",
+ ":benchmark_extended_lib",
+ ":benchmark_extended_records_lib",
+ ":benchmark_landscape_lib",
+ ":benchmark_matrix_lib",
+ ":benchmark_normalization_lib",
+ ],
+)
+
+py_binary(
+ name = "benchmark_extended_site",
+ srcs = ["benchmark_extended_site.py"],
+ deps = [
+ ":benchmark_datasets_lib",
+ ":benchmark_extended_lib",
+ ":benchmark_extended_records_lib",
+ ":benchmark_landscape_lib",
+ ":benchmark_matrix_lib",
+ ":benchmark_normalization_lib",
+ ],
+)
+
+py_library(
+ name = "benchmark_extended_test_support_lib",
+ testonly = True,
+ srcs = ["benchmark_extended_test_support.py"],
+ imports = ["."],
+ deps = [
+ ":benchmark_extended_lib",
+ ":benchmark_extended_records_lib",
+ ],
+)
+
+py_test(
+ name = "benchmark_extended_records_test",
+ size = "small",
+ srcs = ["benchmark_extended_records_test.py"],
+ deps = [
+ ":benchmark_extended_records_lib",
+ ":benchmark_extended_test_support_lib",
+ ],
+)
+
+py_test(
+ name = "benchmark_extended_storage_test",
+ size = "small",
+ srcs = ["benchmark_extended_storage_test.py"],
+ deps = [
+ ":benchmark_extended_storage_lib",
+ ],
+)
+
+py_test(
+ name = "benchmark_extended_ci_test",
+ size = "small",
+ srcs = ["benchmark_extended_ci_test.py"],
+ data = ["//:python_workflow_test_data"],
+ deps = [
+ ":benchmark_extended_ci_lib",
+ ":benchmark_extended_test_support_lib",
+ ":test_paths_lib",
+ ],
+)
+
+py_test(
+ name = "benchmark_extended_site_test",
+ size = "small",
+ srcs = ["benchmark_extended_site_test.py"],
+ deps = [
+ ":benchmark_extended_lib",
+ ":benchmark_extended_records_lib",
+ ":benchmark_extended_site_lib",
+ ":benchmark_extended_test_support_lib",
+ ],
+)
+
py_binary(
name = "benchmark_compare",
srcs = ["benchmark_compare.py"],
@@ -1371,28 +1501,59 @@ py_library(
],
)
+py_library(
+ name = "benchmark_reconcile_ci_lib",
+ srcs = ["benchmark_reconcile_ci.py"],
+ imports = ["."],
+ deps = [
+ ":benchmark_campaign_lib",
+ ":benchmark_extended_records_lib",
+ ":benchmark_history_lib",
+ ":benchmark_preview_ci_lib",
+ ":benchmark_records_lib",
+ ],
+)
+
py_binary(
name = "benchmark_reconcile_ci",
srcs = ["benchmark_reconcile_ci.py"],
deps = [
":benchmark_campaign_lib",
+ ":benchmark_extended_records_lib",
":benchmark_history_lib",
":benchmark_preview_ci_lib",
":benchmark_records_lib",
],
)
+py_test(
+ name = "benchmark_reconcile_ci_test",
+ size = "small",
+ srcs = ["benchmark_reconcile_ci_test.py"],
+ deps = [
+ ":benchmark_extended_records_lib",
+ ":benchmark_extended_test_support_lib",
+ ":benchmark_reconcile_ci_lib",
+ ],
+)
+
py_library(
name = "benchmark_recovery_lib",
srcs = ["benchmark_recovery.py"],
imports = ["."],
- deps = [":benchmark_preview_ci_lib"],
+ deps = [
+ ":benchmark_history_lib",
+ ":benchmark_preview_ci_lib",
+ ],
)
py_binary(
name = "benchmark_recovery",
srcs = ["benchmark_recovery.py"],
- deps = [":benchmark_preview_ci_lib"],
+ deps = [
+ ":benchmark_history_lib",
+ ":benchmark_preview_ci_lib",
+ ],
)
py_test(
diff --git a/tools/benchmark_datasets.py b/tools/benchmark_datasets.py
index 98f5fc335b..597ee6473f 100644
--- a/tools/benchmark_datasets.py
+++ b/tools/benchmark_datasets.py
@@ -36,13 +36,17 @@ def recipe(report, local_contract=None):
cache=contract.get('cache'), output=contract.get('output'))
if contract.get('layouts'):
result['layouts'] = [dict(layout) for layout in contract['layouts']]
+ if contract.get('suite'):
+ result.update(suite=contract['suite'], comparison=contract.get('comparison'),
+ production_target=contract.get('production_target'))
return result
def build_catalog(root):
paths = {*records.run_paths(root), *root.glob('backfills/*/*/*/*/report.json'),
- *root.glob('local/*/*/*/report.json')}
- paths.update(path.with_suffix('') for pattern in ('backfills/*/*/*/*/report.json.gz', 'local/*/*/*/report.json.gz')
+ *root.glob('local/*/*/*/report.json'), *root.glob('extended/views/*/*/report.json')}
+ paths.update(path.with_suffix('') for pattern in ('backfills/*/*/*/*/report.json.gz', 'local/*/*/*/report.json.gz',
+ 'extended/views/*/*/report.json.gz')
for path in root.glob(pattern))
datasets = {}
for path in sorted(paths):
diff --git a/tools/benchmark_extended_build.py b/tools/benchmark_extended_build.py
index d9e5e535e7..13686a7124 100644
--- a/tools/benchmark_extended_build.py
+++ b/tools/benchmark_extended_build.py
@@ -10,6 +10,7 @@
from pathlib import Path
import shutil
import subprocess
+import sys
import time
@@ -30,11 +31,72 @@ def git(checkout, *args):
def clean_source(checkout, sha):
if git(checkout, 'rev-parse', 'HEAD') != sha or git(checkout, 'status', '--porcelain'):
raise ValueError('historical checkout must exactly match its clean revision')
- if (checkout / '.bazelrc.user').exists():
+ if any((checkout / name).exists() for name in ('.bazelrc.user', '.bazelrc.user.end')):
raise ValueError('workspace Bazel override is forbidden')
return {'revision': sha, 'tree': git(checkout, 'rev-parse', 'HEAD^{tree}'), 'clean': True}
+def build_logged(command, checkout, log):
+ """Expose native build progress and keep the exact diagnostic output."""
+ with subprocess.Popen(command, cwd=checkout, stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT, text=True, bufsize=1) as process:
+ for line in process.stdout:
+ log.write(line)
+ log.flush()
+ print(line, end='', file=sys.stderr, flush=True)
+ if process.wait():
+ raise subprocess.CalledProcessError(process.returncode, command)
+
+
+def freeze_build(checkout, directory, sha, disk_cache, jobs=6, build_library_path=None):
+ """Build and immediately freeze one exact checkout without selecting another revision."""
+ directory.mkdir(parents=True, exist_ok=True)
+ executable = directory / 'xff_full'
+ if (directory / 'build.json').exists():
+ saved = json.loads((directory / 'build.json').read_text())
+ if saved['source'] != clean_source(checkout, sha) or saved['sha256'] != digest(executable):
+ raise ValueError('retained executable mismatch: ' + sha)
+ return saved
+ started = time.monotonic()
+ source = clean_source(checkout, sha)
+ bazel = ['bazel', '--nosystem_rc', '--nohome_rc']
+ options = ['--config=clang_release', '--config=xff_full',
+ '--disk_cache=' + str(disk_cache.resolve()), '--jobs=' + str(jobs)]
+ if build_library_path:
+ options.append('--action_env=LD_LIBRARY_PATH=' + str(build_library_path.resolve()))
+ command = [*bazel, 'build', '//xff/cli:xff_full', *options]
+ print(f'BUILD {sha} //xff/cli:xff_full --config=clang_release --config=xff_full ({checkout})', flush=True)
+ with (directory / 'build.log').open('w') as log:
+ build_logged(command, checkout, log)
+ query = [*bazel, 'cquery', 'config(//xff/cli:xff_full, target)', *options,
+ '--output=starlark', '--starlark:expr=target.files_to_run.executable.path']
+ artifact = subprocess.check_output(query, cwd=checkout, stderr=log, text=True).strip()
+ if not artifact or len(artifact.splitlines()) != 1:
+ raise ValueError('configured executable query is ambiguous')
+ if clean_source(checkout, sha) != source:
+ raise ValueError('source identity changed during build')
+ built = checkout / artifact
+ binary_hash = digest(built)
+ shutil.copy2(built, executable)
+ if digest(executable) != binary_hash:
+ raise ValueError('frozen executable copy mismatch')
+ print(f'FROZEN {sha} SHA256={binary_hash} bytes={executable.stat().st_size} {executable}', flush=True)
+ # Link actions record the actual production LTO command for this revision.
+ with (directory / 'link-actions.json').open('w') as actions:
+ subprocess.run([*bazel, 'aquery', 'mnemonic("CppLink", //xff/cli:xff_full)',
+ *options, '--output=jsonproto'], cwd=checkout, stdout=actions,
+ stderr=log, check=True)
+ saved = {'source': source, 'sha256': binary_hash, 'bytes': executable.stat().st_size,
+ 'command': command, 'query': query, 'artifact': artifact,
+ 'elapsed_seconds': time.monotonic() - started,
+ 'completed_at': datetime.now(timezone.utc).isoformat(),
+ 'configuration': {name: digest(checkout / name) for name in
+ ('.bazelrc', '.bazelversion', 'MODULE.bazel', 'MODULE.bazel.lock',
+ 'bazelmod/llvm.MODULE.bazel') if (checkout / name).is_file()}}
+ write(directory / 'build.json', saved)
+ return saved
+
+
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--root', type=Path, required=True)
@@ -79,40 +141,8 @@ def main():
write(status_path, status)
try:
subprocess.run(['git', '-C', str(checkout), 'checkout', '--detach', sha], check=True)
- source = clean_source(checkout, sha)
- bazel = ['bazel', *policy['startup_flags']]
- options = ['--config=clang_release', '--config=xff_full',
- '--disk_cache=' + str(args.disk_cache.resolve()), '--jobs=' + str(args.jobs)]
- if args.build_library_path:
- options.append('--action_env=LD_LIBRARY_PATH=' + str(args.build_library_path.resolve()))
- command = [*bazel, 'build', '//xff/cli:xff_full', *options]
- with (directory / 'build.log').open('w') as log:
- subprocess.run(command, cwd=checkout, stdout=log, stderr=subprocess.STDOUT, check=True)
- query = [*bazel, 'cquery', 'config(//xff/cli:xff_full, target)', *options,
- '--output=starlark', '--starlark:expr=target.files_to_run.executable.path']
- artifact = subprocess.check_output(query, cwd=checkout, stderr=log, text=True).strip()
- if not artifact or len(artifact.splitlines()) != 1:
- raise ValueError('configured executable query is ambiguous')
- if clean_source(checkout, sha) != source:
- raise ValueError('source identity changed during build')
- built = checkout / artifact
- binary_hash = digest(built)
- shutil.copy2(built, executable)
- if digest(executable) != binary_hash:
- raise ValueError('frozen executable copy mismatch')
- # Link actions record the actual production LTO command for this revision.
- with (directory / 'link-actions.json').open('w') as actions:
- subprocess.run([*bazel, 'aquery', 'mnemonic("CppLink", //xff/cli:xff_full)',
- *options, '--output=jsonproto'], cwd=checkout, stdout=actions,
- stderr=log, check=True)
- saved = {'source': source, 'sha256': binary_hash, 'bytes': executable.stat().st_size,
- 'command': command, 'query': query, 'artifact': artifact,
- 'elapsed_seconds': time.monotonic() - started,
- 'completed_at': datetime.now(timezone.utc).isoformat(),
- 'configuration': {name: digest(checkout / name) for name in
- ('.bazelrc', '.bazelversion', 'MODULE.bazel', 'MODULE.bazel.lock',
- 'bazelmod/llvm.MODULE.bazel') if (checkout / name).is_file()}}
- write(identity, saved)
+ saved = freeze_build(checkout, directory, sha, args.disk_cache, args.jobs, args.build_library_path)
+ binary_hash = saved['sha256']
status[sha] = {'status': 'built', 'sha256': binary_hash, 'seconds': saved['elapsed_seconds']}
print(f'BUILT {index}/{len(order)} {sha[:10]} {binary_hash} {saved["elapsed_seconds"]:.1f}s', flush=True)
except (subprocess.CalledProcessError, ValueError, OSError) as error:
diff --git a/tools/benchmark_extended_build_test.py b/tools/benchmark_extended_build_test.py
index 491b0dea47..af15cee97b 100644
--- a/tools/benchmark_extended_build_test.py
+++ b/tools/benchmark_extended_build_test.py
@@ -3,6 +3,9 @@
"""Keep historical source and binary provenance fail closed without invoking Bazel."""
from pathlib import Path
+import json
+import io
+import shutil
import tempfile
import unittest
from unittest import mock
@@ -39,6 +42,70 @@ def test_identity_hash_changes_with_artifact_contents(self):
build.write(Path(temporary) / 'identity.json', {'hash': old})
self.assertFalse((Path(temporary) / 'identity.json.tmp').exists())
+ def test_each_configured_build_is_copied_before_any_later_bazel_invocation(self):
+ with tempfile.TemporaryDirectory() as temporary:
+ root = Path(temporary)
+ checkout, directory = root / 'checkout', root / 'frozen'
+ artifact = 'bazel-out/production/bin/xff/cli/xff_full'
+ built = checkout / artifact
+ built.parent.mkdir(parents=True)
+ frozen_bytes = b'synthetic production binary'
+ events = []
+ copy = shutil.copy2
+
+ def run(command, **kwargs):
+ if 'build' in command:
+ events.append('build')
+ built.write_bytes(frozen_bytes)
+ else:
+ events.append('aquery')
+ self.assertEqual((directory / 'xff_full').read_bytes(), frozen_bytes)
+ kwargs['stdout'].write(json.dumps({'actions': [
+ {'arguments': ['clang++', '-flto=thin', '-o', artifact]}]}))
+ built.write_bytes(b'alias changed after the frozen copy')
+
+ def logged(command, checkout, log):
+ run(command)
+
+ def query(command, **kwargs):
+ events.append('configured query')
+ self.assertIn('--config=clang_release', command)
+ self.assertIn('--config=xff_full', command)
+ self.assertIn('config(//xff/cli:xff_full, target)', command)
+ return artifact + '\n'
+
+ def freeze(source, destination):
+ events.append('copy')
+ return copy(source, destination)
+
+ with mock.patch.object(build, 'clean_source', return_value={'revision': 'revision', 'tree': 'tree', 'clean': True}), \
+ mock.patch.object(build.subprocess, 'run', side_effect=run), \
+ mock.patch.object(build, 'build_logged', side_effect=logged), \
+ mock.patch.object(build.subprocess, 'check_output', side_effect=query), \
+ mock.patch.object(build.shutil, 'copy2', side_effect=freeze):
+ saved = build.freeze_build(checkout, directory, 'revision', root / 'cache')
+ self.assertEqual(events, ['build', 'configured query', 'copy', 'aquery'])
+ self.assertEqual(saved['sha256'], build.digest(directory / 'xff_full'))
+ self.assertNotEqual(saved['sha256'], build.digest(built))
+
+ def test_build_progress_is_visible_retained_and_failure_propagates(self):
+ for status in (0, 1):
+ with self.subTest(status=status):
+ process = mock.Mock(stdout=io.StringIO('building source.cc\nlinking xff_full\n'), returncode=status)
+ process.wait.return_value = status
+ process.__enter__ = mock.Mock(return_value=process)
+ process.__exit__ = mock.Mock(return_value=False)
+ output, log = io.StringIO(), io.StringIO()
+ with mock.patch.object(build.subprocess, 'Popen', return_value=process), \
+ mock.patch.object(build.sys, 'stderr', output):
+ if status:
+ with self.assertRaises(build.subprocess.CalledProcessError):
+ build.build_logged(['bazel', 'build'], Path('.'), log)
+ else:
+ build.build_logged(['bazel', 'build'], Path('.'), log)
+ self.assertEqual(log.getvalue(), 'building source.cc\nlinking xff_full\n')
+ self.assertEqual(output.getvalue(), log.getvalue())
+
if __name__ == '__main__':
unittest.main()
diff --git a/tools/benchmark_extended_ci.py b/tools/benchmark_extended_ci.py
new file mode 100644
index 0000000000..66de7b4c93
--- /dev/null
+++ b/tools/benchmark_extended_ci.py
@@ -0,0 +1,177 @@
+# SPDX-FileCopyrightText: Copyright (c) M. Boerger, the MBO Works authors
+# SPDX-License-Identifier: Apache-2.0
+"""Freeze paired production builds and collect the separate extended CI contract."""
+
+import argparse
+from dataclasses import asdict
+import json
+import os
+from pathlib import Path
+import platform
+import re
+import shutil
+import subprocess
+import sys
+
+import benchmark_extended as suite
+import benchmark_extended_build as build
+import benchmark_extended_history as history
+import benchmark_extended_records as records
+import benchmark_extended_storage as storage
+import process_resources
+
+
+def cpu_allocation(count, ci):
+ """CI budgets logical CPUs; local history keeps distinct guest-topology cores."""
+ if ci and sys.platform == 'linux':
+ allowed = sorted(os.sched_getaffinity(0))
+ records.require(len(allowed) >= count, 'not enough allowed logical CPUs')
+ return allowed[:count]
+ return history.cpu_allocation(count)
+
+
+def revision(repo):
+ sha = build.git(repo, 'rev-parse', 'HEAD')
+ subject = build.git(repo, 'show', '-s', '--format=%s', sha)
+ numbers = re.findall(r'\(#([0-9]+)\)', subject)
+ return {'sha': sha, 'date': build.git(repo, 'show', '-s', '--format=%cI', sha),
+ 'subject': subject, 'pr': int(numbers[-1]) if numbers else 0}
+
+
+def build_endpoint(repo, root, role, disk_cache, jobs):
+ selected = revision(repo)
+ build.freeze_build(repo, root / 'binaries' / selected['sha'], selected['sha'], disk_cache, jobs)
+ history.freeze(root / (role + '.json'), dict(revision=selected, parent=build.git(repo, 'rev-parse', 'HEAD^1')))
+ if all((root / (side + '.json')).exists() for side in ('head', 'base')):
+ head, base = (records.read(root / (side + '.json')) for side in ('head', 'base'))
+ records.require(head['parent'] == base['revision']['sha'], 'baseline must be the first parent')
+ history.freeze(root / 'revisions.json', [head['revision'], base['revision']])
+ history.freeze(root / 'build-contract.json', {'target': '//xff/cli:xff_full',
+ 'configs': ['clang_release', 'xff_full'], 'revisions': [head['revision'], base['revision']],
+ 'drivers': {Path(module).name: build.digest(Path(module)) for module in (__file__, build.__file__)}})
+
+
+def collect(build_root, root, launcher_path, collection=None, series=None, existing_fixture=None):
+ platform_name = {'linux': 'linux', 'darwin': 'macos'}.get(sys.platform)
+ records.require(platform_name is not None, 'requires Linux or macOS')
+ revisions = records.read(build_root / 'revisions.json')
+ records.require(len(revisions) == 2, 'requires exactly head and first-parent builds')
+ builds = {}
+ for selected in revisions:
+ directory = build_root / 'binaries' / selected['sha']
+ manifest = records.read(directory / 'build.json')
+ selected.update(binary=str(directory / 'xff_full'), binary_sha256=manifest['sha256'],
+ build_sha256=build.digest(directory / 'build.json'),
+ link_actions_sha256=build.digest(directory / 'link-actions.json'))
+ builds[selected['sha']] = records.build_proof(directory, selected)
+ head, anchor = revisions
+ origin = 'ci' if collection is not None else 'local'
+ records.require(collection is None or existing_fixture is None, 'CI must create its own isolated fixture')
+ if collection is not None:
+ records.require(collection['head_sha'] == head['sha'], 'workflow head/build mismatch')
+ collection = dict(collection, base_sha=anchor['sha'])
+ series = series or ('github-ci-' + platform_name)
+ cases = suite.cases()
+ affinity = cpu_allocation(max(case.cpus for case in cases), origin == 'ci')
+ tools = {}
+ for name in ('find', 'rg', 'tar'):
+ executable = shutil.which({'find': 'gfind', 'tar': 'gtar'}.get(name, name) if sys.platform == 'darwin' else name)
+ records.require(executable is not None, 'missing reference tool: ' + name)
+ path = Path(executable).resolve(strict=True)
+ tools[name] = {'path': str(path), 'sha256': build.digest(path),
+ 'version': subprocess.check_output([str(path), '--version'], text=True).strip()}
+ launcher = process_resources.launcher_identity(launcher_path)
+ output = root / 'measurements'
+ output.mkdir(parents=True, exist_ok=True)
+ controls, control_error = [], None
+ with storage.fixture(root, existing_fixture) as (fixture, filesystem):
+ contract = {'recipe': suite.RECIPE, 'cases': [asdict(case) for case in cases], 'revisions': revisions,
+ 'repetitions': 9, 'keep': 7, 'estimator': 'mean-fastest', 'warmup_rounds': 1,
+ 'anchor': anchor, 'fixture': str(fixture),
+ 'filesystem': 'tmpfs' if sys.platform == 'linux' else filesystem,
+ 'tools': tools, 'launcher': launcher, 'affinity': affinity, 'platform': platform.platform(),
+ 'logical_cpu_count': os.cpu_count(),
+ 'cpu_policy': ('allowed-logical-cpus' if origin == 'ci' else 'guest-topology-cores')
+ if sys.platform == 'linux' else 'unpinned-workers',
+ 'comparison': 'parent', 'python': sys.version,
+ 'drivers': {Path(module).name: build.digest(Path(module)) for module in
+ (__file__, suite.__file__, history.__file__, storage.__file__, process_resources.__file__)},
+ 'build_contract_sha256': build.digest(build_root / 'build-contract.json'),
+ 'order': 'head cells then identical-parent controls; participant start rotates every round',
+ 'output_retention': 'all outputs checked; successful hash/count retained; failed output gzip retained'}
+ history.freeze(output / 'contract.json', contract)
+ if existing_fixture is None:
+ suite.prepare(fixture, cases)
+ manifest = suite.verify(fixture, cases)
+ history.freeze(output / 'fixtures.json', manifest)
+ rows = []
+ for index, case in enumerate(cases):
+ print(f'HEAD {index + 1}/{len(cases)} {case.key}', flush=True)
+ rows.append(history.collect_cell(case, head, anchor, output, fixture, {k: v['path'] for k, v in tools.items()},
+ launcher['path'], affinity, index, 9))
+ # Head results remain usable if an independent diagnostic control fails.
+ try:
+ for index, case in enumerate(cases):
+ print(f'CONTROL {index + 1}/{len(cases)} {case.key}', flush=True)
+ controls.append(history.collect_cell(case, anchor, anchor, output, fixture, {k: v['path'] for k, v in tools.items()},
+ launcher['path'], affinity, index + len(cases), 9))
+ except (ValueError, OSError) as error:
+ control_error = repr(error)
+ controls = []
+ try:
+ records.require(suite.verify(fixture, cases) == manifest, 'fixture identity changed')
+ for selected in revisions:
+ records.build_proof(build_root / 'binaries' / selected['sha'], selected)
+ for tool in tools.values():
+ records.require(build.digest(Path(tool['path'])) == tool['sha256'], 'reference executable changed')
+ records.require(process_resources.launcher_identity(launcher_path) == launcher, 'resource launcher changed')
+ except BaseException as error:
+ history.write(output / 'invalid.json', {'error': repr(error)})
+ raise
+ record = records.make_record(contract, head, rows, builds, platform_name, platform.machine().lower(), series,
+ 'parent', origin, collection, controls,
+ ('https://github.com/' + collection['repository'] + '/actions/runs/'
+ + str(collection['run_id']) + '/attempts/' + str(collection['run_attempt'])
+ + ' (benchmark-extended-build-' + platform_name + ')') if collection is not None
+ else 'original frozen full-production executable used for local paired validation',
+ control_error=control_error)
+ record['storage_release'] = records.read(root / 'storage-release.json')
+ record['control_error'] = control_error
+ record['identity'] = records.observation_identity(record)
+ records.validate(record)
+ history.write(root / 'observation.json', record)
+ history.write(output / 'progress.json', {'state': 'complete' if control_error is None else 'head-complete-controls-failed',
+ 'head_cells': len(rows), 'control_cells': len(controls)})
+ if control_error is not None:
+ raise ValueError('head measurements retained; control diagnostic failed: ' + control_error)
+ return record
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__)
+ modes = parser.add_subparsers(dest='mode', required=True)
+ freeze = modes.add_parser('build')
+ freeze.add_argument('--repo', type=Path, required=True)
+ freeze.add_argument('--root', type=Path, required=True)
+ freeze.add_argument('--disk-cache', type=Path, required=True)
+ freeze.add_argument('--role', choices=('head', 'base'), required=True)
+ freeze.add_argument('--jobs', type=int, default=3)
+ measure = modes.add_parser('collect')
+ measure.add_argument('--build-root', type=Path, required=True)
+ measure.add_argument('--root', type=Path, required=True)
+ measure.add_argument('--resource-launcher', type=Path, required=True)
+ measure.add_argument('--collection', type=Path, help='Trusted workflow identity JSON; absence means local validation')
+ measure.add_argument('--series', help='Explicit series for local paired validation')
+ measure.add_argument('--fixture', type=Path, help='Reuse a verified tmpfs fixture for local validation; never removed')
+ args = parser.parse_args()
+ if args.mode == 'build':
+ build_endpoint(args.repo.resolve(), args.root.resolve(), args.role, args.disk_cache.resolve(), args.jobs)
+ else:
+ if args.collection is None and not args.series:
+ parser.error('local collection requires an explicit --series')
+ collect(args.build_root.resolve(), args.root.resolve(), args.resource_launcher.resolve(),
+ records.read(args.collection) if args.collection else None, args.series, args.fixture)
+
+
+if __name__ == '__main__':
+ main()
diff --git a/tools/benchmark_extended_ci_test.py b/tools/benchmark_extended_ci_test.py
new file mode 100644
index 0000000000..4238010e52
--- /dev/null
+++ b/tools/benchmark_extended_ci_test.py
@@ -0,0 +1,160 @@
+# SPDX-FileCopyrightText: Copyright (c) M. Boerger, the MBO Works authors
+# SPDX-License-Identifier: Apache-2.0
+"""Exercise the complete collector contract with explicitly synthetic samples."""
+
+from contextlib import contextmanager
+import hashlib
+from pathlib import Path
+import unittest
+from unittest import mock
+
+import benchmark_extended_ci as ci
+from benchmark_extended_test_support import ExtendedFixture
+from test_paths import repository_file
+
+
+class BenchmarkExtendedCiTest(unittest.TestCase):
+ def setUp(self):
+ self.fixture = ExtendedFixture()
+ self.addCleanup(self.fixture.close)
+ self.root = self.fixture.root / 'collection'
+ ci.history.write(self.fixture.root / 'revisions.json', self.fixture.revisions)
+ ci.history.write(self.fixture.root / 'build-contract.json', {'synthetic-only': True})
+ self.tools = {}
+ for name in ('find', 'rg', 'tar'):
+ path = self.fixture.root / name
+ path.write_bytes(('synthetic ' + name).encode())
+ self.tools[name] = str(path)
+ self.launcher = self.fixture.root / 'launcher'
+ self.launcher.write_bytes(b'synthetic native launcher')
+ self.calls = 0
+ self.fail_at = None
+ self.mutate_at = None
+
+ @contextmanager
+ def memory(self, root, existing=None):
+ root.mkdir(parents=True, exist_ok=True)
+ try:
+ yield root / 'fixture', {'storage': 'memory', 'filesystem': 'tmpfs'}
+ finally:
+ ci.history.write(root / 'storage-release.json', {'synthetic-only': True, 'released_at': '2026-11-08T12:00:00Z'})
+
+ def invoke(self, spec, wanted, separator, failure):
+ self.calls += 1
+ if self.fail_at == self.calls:
+ raise ValueError('synthetic failed output')
+ if self.mutate_at == self.calls:
+ Path(self.tools['find']).write_bytes(b'changed reference executable')
+ output = separator.join(wanted) + separator
+ return {'elapsed_seconds': 1, 'user_cpu_seconds': .5, 'system_cpu_seconds': .2,
+ 'first_stdout_seconds': .1, 'peak_child_rss_bytes': 1024, 'exit_codes': [0],
+ 'stderr_bytes': 0, 'stderr': '', 'stdout_bytes': len(output),
+ 'stdout_sha256': hashlib.sha256(output).hexdigest(), 'validated_records': len(wanted)}
+
+ def collect(self, collection=None, existing=None):
+ with mock.patch.object(ci.sys, 'platform', 'linux'), \
+ mock.patch.object(ci.platform, 'platform', return_value='Linux-synthetic-x86_64'), \
+ mock.patch.object(ci.platform, 'machine', return_value='x86_64'), \
+ mock.patch.object(ci.os, 'sched_getaffinity', return_value={0, 1, 2}, create=True), \
+ mock.patch.object(ci.shutil, 'which', side_effect=self.tools.get), \
+ mock.patch.object(ci.subprocess, 'check_output', return_value='synthetic tool'), \
+ mock.patch.object(ci.history, 'cpu_allocation', return_value=[0, 1, 2]), \
+ mock.patch.object(ci.history, 'host_context', return_value={'utc': '2026-11-08T12:00:00Z'}), \
+ mock.patch.object(ci.history, 'invoke', side_effect=self.invoke), \
+ mock.patch.object(ci.storage, 'fixture', side_effect=self.memory), \
+ mock.patch.object(ci.process_resources, 'launcher_identity', return_value={
+ 'path': str(self.launcher), 'sha256': ci.build.digest(self.launcher),
+ 'wall_method': 'synthetic wall', 'memory_method': 'synthetic RSS'}):
+ return ci.collect(self.fixture.root, self.root, self.launcher, collection, 'synthetic-only', existing)
+
+ def test_complete_head_and_identical_parent_controls_are_retained_after_final_checks(self):
+ value = self.collect()
+ self.assertEqual(value['origin'], 'local')
+ self.assertEqual(value['comparison'], 'parent')
+ self.assertEqual(len(value['rows']), len(self.fixture.cases))
+ self.assertEqual(len(value['controls']), len(self.fixture.cases))
+ self.assertEqual(self.calls, len(self.fixture.cases) * 30 * 2)
+ self.assertEqual(value['controls'][0]['revision'], value['contract']['anchor'])
+ self.assertTrue((self.root / 'observation.json').exists())
+ self.assertEqual(ci.records.read(self.root / 'measurements/progress.json')['state'], 'complete')
+
+ def test_control_failure_retains_complete_head_and_failed_attempt_diagnostics(self):
+ self.fail_at = len(self.fixture.cases) * 30 + 1
+ with self.assertRaisesRegex(ValueError, 'head measurements retained'):
+ self.collect()
+ value = ci.records.read(self.root / 'observation.json')
+ ci.records.validate(value)
+ self.assertEqual(value['controls'], [])
+ self.assertIn('synthetic failed output', value['control_error'])
+ self.assertTrue(list((self.root / 'measurements/cells').glob('**/failure.json')))
+ self.assertEqual(ci.records.read(self.root / 'measurements/progress.json')['state'], 'head-complete-controls-failed')
+
+ def test_head_failure_never_creates_publishable_observation(self):
+ self.fail_at = 1
+ with self.assertRaisesRegex(ValueError, 'synthetic failed output'):
+ self.collect()
+ self.assertFalse((self.root / 'observation.json').exists())
+ self.assertTrue((self.root / 'storage-release.json').exists())
+
+ def test_ci_control_failure_retains_head_with_explicit_error_and_valid_workflow_binding(self):
+ self.fail_at = len(self.fixture.cases) * 30 + 1
+ collection = self.fixture.record(origin='ci', comparison='parent')['collection']
+ with self.assertRaisesRegex(ValueError, 'head measurements retained'):
+ self.collect(collection)
+ value = ci.records.read(self.root / 'observation.json')
+ self.assertEqual(value['origin'], 'ci')
+ self.assertEqual(value['controls'], [])
+ self.assertIn('synthetic failed output', value['control_error'])
+ ci.records.validate(value, self.fixture.source())
+
+ def test_changed_reference_invalidates_whole_collection_after_successful_samples(self):
+ self.mutate_at = len(self.fixture.cases) * 30 * 2
+ with self.assertRaisesRegex(ValueError, 'reference executable changed'):
+ self.collect()
+ self.assertFalse((self.root / 'observation.json').exists())
+ self.assertTrue((self.root / 'measurements/invalid.json').exists())
+
+ def test_ci_cannot_reuse_a_local_fixture_or_measure_a_different_head(self):
+ collection = self.fixture.record(origin='ci', comparison='parent')['collection']
+ with self.assertRaisesRegex(ValueError, 'own isolated fixture'):
+ self.collect(collection, Path('/shared-fixture'))
+ with self.assertRaisesRegex(ValueError, 'workflow head/build mismatch'):
+ self.collect(dict(collection, head_sha='f' * 40))
+ self.assertEqual(self.calls, 0)
+
+ def test_ci_accepts_allowed_logical_cpus_without_requiring_three_distinct_cores(self):
+ with mock.patch.object(ci.sys, 'platform', 'linux'), \
+ mock.patch.object(ci.os, 'sched_getaffinity', return_value={0, 1, 2, 3}, create=True), \
+ mock.patch.object(ci.history, 'cpu_allocation', side_effect=ValueError('not enough distinct cores')) as physical:
+ self.assertEqual(ci.cpu_allocation(3, ci=True), [0, 1, 2])
+ physical.assert_not_called()
+ with self.assertRaisesRegex(ValueError, 'not enough distinct cores'):
+ ci.cpu_allocation(3, ci=False)
+ with mock.patch.object(ci.sys, 'platform', 'linux'), \
+ mock.patch.object(ci.os, 'sched_getaffinity', return_value={0, 1}, create=True), \
+ self.assertRaisesRegex(ValueError, 'not enough allowed logical CPUs'):
+ ci.cpu_allocation(3, ci=True)
+
+ def test_workflow_keeps_collection_independent_and_measures_downloaded_production_bytes(self):
+ workflow = repository_file('.github/workflows/benchmarks.yml').read_text()
+ extended = workflow.split(' extended:\n', 1)[1].split('\n build:', 1)[0]
+ self.assertNotIn('needs:', extended)
+ self.assertIn('fail-fast: false', extended)
+ helper = extended.index('cp "${launcher}" "${RUNNER_TEMP}/extended-resource-launcher"')
+ self.assertLess(extended.index('Check extended runner capacity before building'), helper)
+ head = extended.index('--role=head')
+ parent = extended.index('--role=base')
+ upload = extended.index('name: benchmark-extended-build-${{ matrix.platform }}')
+ download = extended.index('actions/download-artifact@')
+ restore = extended.index('tar -xf "${RUNNER_TEMP}/extended-retention/benchmark-extended-build.tar"')
+ measure = extended.index('benchmark_extended_ci.py collect')
+ self.assertEqual(sorted((helper, head, parent, upload, download, restore, measure)),
+ [helper, head, parent, upload, download, restore, measure])
+ self.assertIn('--build-root="${RUNNER_TEMP}/extended-retained-build"', extended)
+ diagnostics = extended.split('name: Retain extended build diagnostics', 1)[1]
+ self.assertIn('if: always()', diagnostics)
+ self.assertIn('/extended-build/binaries/*/build.log', diagnostics)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/tools/benchmark_extended_history.py b/tools/benchmark_extended_history.py
index 27a8f35b3a..475af6be24 100644
--- a/tools/benchmark_extended_history.py
+++ b/tools/benchmark_extended_history.py
@@ -49,6 +49,10 @@ def freeze(path, value):
def cpu_allocation(count):
"""Prefer separate physical cores over sibling hardware threads."""
+ if sys.platform == 'darwin':
+ if count > (os.cpu_count() or 0):
+ raise ValueError('worker count exceeds available logical CPUs')
+ return []
cores = {}
for cpu in sorted(os.sched_getaffinity(0)):
topology = Path(f'/sys/devices/system/cpu/cpu{cpu}/topology')
@@ -65,7 +69,8 @@ def worker(connection, spec, wanted, separator, failure_path):
sample = None
try:
os.setsid()
- os.sched_setaffinity(0, spec['affinity'])
+ if spec['affinity']:
+ os.sched_setaffinity(0, spec['affinity'])
with open(spec['stdin'], 'rb') as source:
sample = process_resources.measure_pipeline(
[spec['command']], source=source, cwd=spec['cwd'], environment=spec['environment'],
@@ -110,6 +115,10 @@ def invoke(spec, wanted, separator, failure_path, timeout=120):
def host_context():
+ if sys.platform == 'darwin':
+ return {'utc': datetime.now(timezone.utc).isoformat(), 'loadavg': os.getloadavg(),
+ 'vm_stat': subprocess.check_output(['vm_stat'], text=True),
+ 'memory_pressure': subprocess.check_output(['sysctl', 'vm.swapusage'], text=True)}
return {'utc': datetime.now(timezone.utc).isoformat(), 'loadavg': os.getloadavg(),
'meminfo': Path('/proc/meminfo').read_text(),
'cpu_stat': Path('/proc/stat').read_text(),
diff --git a/tools/benchmark_extended_records.py b/tools/benchmark_extended_records.py
new file mode 100644
index 0000000000..809c35c4cc
--- /dev/null
+++ b/tools/benchmark_extended_records.py
@@ -0,0 +1,257 @@
+# SPDX-FileCopyrightText: Copyright (c) M. Boerger, the MBO Works authors
+# SPDX-License-Identifier: Apache-2.0
+"""Validate and retain extended observations separately from core benchmark history."""
+
+from dataclasses import asdict
+from datetime import datetime
+from functools import lru_cache
+import gzip
+import hashlib
+import json
+import math
+from pathlib import Path
+import re
+import tempfile
+
+import benchmark_extended as suite
+import benchmark_extended_report as report
+import benchmark_provenance
+
+
+def require(condition, message):
+ if not condition:
+ raise ValueError(message)
+
+
+def identity(value):
+ return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(',', ':'), allow_nan=False).encode()).hexdigest()
+
+
+def read(path):
+ return json.loads(path.read_text() if path.exists() else gzip.decompress(path.with_suffix('.json.gz').read_bytes()))
+
+
+def digest(path):
+ with path.open('rb') as source:
+ return hashlib.file_digest(source, 'sha256').hexdigest()
+
+
+def observation_identity(record):
+ # Retention attestations and trusted publication metadata can be added later;
+ # the observation identity binds the original contract, builds and raw rows.
+ return identity({key: value for key, value in record.items()
+ if key not in ('identity', 'view_verification', 'publication_source')})
+
+
+def build_proof(directory, revision):
+ manifest = read(directory / 'build.json')
+ require(manifest['source']['revision'] == revision['sha'] and manifest['source']['clean'] is True,
+ 'build source identity changed')
+ require(digest(directory / 'build.json') == revision['build_sha256']
+ and digest(directory / 'link-actions.json') == revision['link_actions_sha256'], 'build evidence changed')
+ require(digest(directory / 'xff_full') == manifest['sha256'] == revision['binary_sha256'], 'frozen binary changed')
+ links = [action['arguments'] for action in read(directory / 'link-actions.json')['actions']
+ if manifest['artifact'] in action.get('arguments', [])]
+ require(len(links) == 1 and '-flto=thin' in links[0], 'production ThinLTO link evidence missing')
+ require(all(flag in manifest['command'] for flag in ('//xff/cli:xff_full', '--config=clang_release', '--config=xff_full')),
+ 'full production configuration missing')
+ return {'manifest': manifest, 'link_arguments': links[0],
+ 'manifest_sha256': revision['build_sha256'], 'link_actions_sha256': revision['link_actions_sha256']}
+
+
+@lru_cache(maxsize=256)
+def oracle(name, shape, files, fixture):
+ wanted = suite.expected(suite.Case(name, shape, files, 1), Path(fixture))
+ return len(wanted), hashlib.sha256(b'\0'.join(sorted(wanted))).hexdigest(), sum(map(len, wanted)) + len(wanted)
+
+
+@lru_cache(maxsize=64)
+def fixture_identity(shape, files, archive):
+ return suite.fixture_identity(shape, files, archive)
+
+
+def validate_rows(rows, contract, revision):
+ expected = {suite.Case(**value).key: value for value in contract['cases']}
+ require(len(rows) == len(expected), 'incomplete extended revision')
+ seen = set()
+ for row in rows:
+ case = suite.Case(**row['case'])
+ require(case.key in expected and case.key not in seen and row['case'] == expected[case.key],
+ 'unexpected or repeated extended cell')
+ seen.add(case.key)
+ require(row['revision'] == revision and row['anchor'] == contract['anchor'], 'cell provenance changed')
+ count, expected_hash, expected_bytes = oracle(case.name, case.shape, case.files, contract['fixture'])
+ require(row['expected_records'] == count and row['expected_sha256'] == expected_hash, 'cell oracle changed')
+ commands, cwd, _ = suite.commands(case, Path(contract['fixture']), revision['binary'],
+ {key: value['path'] for key, value in contract['tools'].items()})
+ anchor, _, _ = suite.commands(case, Path(contract['fixture']), contract['anchor']['binary'],
+ {key: value['path'] for key, value in contract['tools'].items()})
+ commands['anchor'] = anchor['xff']
+ require(row['commands'] == commands and row['cwd'] == str(cwd)
+ and row['affinity'] == contract['affinity'][:case.cpus], 'cell commands or allocation changed')
+ require(row['environment'] == {'PATH': '/usr/bin:/bin', 'HOME': str(Path(contract['fixture']) / 'home'),
+ 'LC_ALL': 'C', 'TZ': 'UTC'}, 'cell environment changed')
+ samples = row['samples']
+ keys = {(label, phase, round_number) for label in commands
+ for phase, rounds in [('warmup', [0]), ('measured', range(1, contract['repetitions'] + 1))]
+ for round_number in rounds}
+ require(len(samples) == len(keys) and {(s['label'], s['phase'], s['round']) for s in samples} == keys,
+ 'incomplete or repeated extended samples')
+ for sample in samples:
+ require(sample['exit_codes'] == [0] and sample['stderr_bytes'] == 0 and not sample.get('stderr')
+ and sample['validated_records'] == count and sample['stdout_bytes'] == expected_bytes,
+ 'unsuccessful or unvalidated extended sample')
+ require(re.fullmatch('[0-9a-f]{64}', sample['stdout_sha256']) is not None, 'missing output hash')
+ for field in ('elapsed_seconds', 'user_cpu_seconds', 'system_cpu_seconds', 'peak_child_rss_bytes'):
+ value = sample[field]
+ require(type(value) in (int, float) and math.isfinite(value) and value >= 0, 'invalid resource sample')
+ require(sample['elapsed_seconds'] > 0, 'invalid elapsed sample')
+ # Also validate the existing chart's estimator and complete participant populations.
+ adapted = report.as_report([row], contract, 'traditional')
+ for task in adapted['tasks']:
+ for participant in task['participants'].values():
+ report.matrix.selected_samples(adapted, participant)
+
+
+def validate(record, source=None):
+ require(record.get('schema') == 1 and record.get('kind') == 'extended-production', 'unsupported extended record')
+ require(record['identity'] == observation_identity(record),
+ 'extended record identity changed')
+ contract = record['contract']
+ require(contract['recipe'] == suite.RECIPE and contract['cases'] == [asdict(case) for case in suite.cases()],
+ 'extended workload contract changed')
+ require(contract['warmup_rounds'] == 1 and contract['estimator'] == 'mean-fastest'
+ and (contract['repetitions'], contract['keep']) == (9, 7), 'invalid extended sampling')
+ require(record['comparison'] in ('parent', 'fixed-anchor'), 'unknown extended comparison')
+ require(record['origin'] in ('local', 'ci') and re.fullmatch('[A-Za-z0-9_.-]+', record['series']), 'invalid extended series')
+ require(record['platform'] in ('linux', 'macos'), 'unknown extended platform')
+ require(record['machine'] in contract['platform'].lower(), 'extended architecture does not match platform')
+ require(contract['filesystem'] == 'tmpfs' if record['platform'] == 'linux' else
+ isinstance(contract['filesystem'], dict) and contract['filesystem'].get('storage') == 'memory',
+ 'unverified memory fixture')
+ affinity = contract['affinity']
+ require((len(affinity) >= 3 and len(affinity) == len(set(affinity))
+ and all(type(cpu) is int and cpu >= 0 for cpu in affinity)) if record['platform'] == 'linux'
+ else affinity == [], 'invalid extended CPU allocation')
+ revision = record['revision']
+ require(revision in contract['revisions'], 'revision not in extended contract')
+ for entry in (revision, contract['anchor']):
+ require(re.fullmatch('[0-9a-f]{40}', entry['sha']) is not None, 'invalid revision')
+ require(datetime.fromisoformat(entry['date']).tzinfo is not None, 'revision date needs a timezone')
+ proof = record['builds'][entry['sha']]
+ manifest = proof['manifest']
+ require(manifest['source']['revision'] == entry['sha'] and manifest['source']['clean'] is True
+ and manifest['sha256'] == entry['binary_sha256']
+ and proof['manifest_sha256'] == entry['build_sha256']
+ and proof['link_actions_sha256'] == entry['link_actions_sha256'], 'binary/build identity mismatch')
+ require('-flto=thin' in proof['link_arguments'] and manifest['artifact'] in proof['link_arguments']
+ and all(flag in manifest['command'] for flag in ('//xff/cli:xff_full', '--config=clang_release', '--config=xff_full')),
+ 'production link/configuration evidence missing')
+ if record['origin'] == 'ci':
+ require(bool(record.get('controls')) or
+ isinstance(record.get('control_error'), str) and bool(record['control_error']),
+ 'CI controls require a complete grid or an explicit failure')
+ collection = record['collection']
+ require(all(type(collection[key]) is int and collection[key] > 0 for key in ('run_id', 'run_attempt')),
+ 'invalid extended workflow identity')
+ require(record['comparison'] == 'parent' and collection['head_sha'] == revision['sha']
+ and collection['base_sha'] == contract['anchor']['sha'], 'CI revision/parent changed')
+ if source is not None:
+ require(source['name'] == 'Benchmarks' and source['path'] == '.github/workflows/benchmarks.yml'
+ and source['event'] == 'push' and source['head_branch'] == 'main' and source['status'] == 'completed',
+ 'untrusted extended workflow')
+ require(all(str(collection[key]) == str(source[field]) for key, field in
+ [('run_id', 'id'), ('run_attempt', 'run_attempt'), ('head_sha', 'head_sha')]),
+ 'extended workflow provenance mismatch')
+ require(collection['repository'] == source['repository']['full_name'], 'extended repository mismatch')
+ require(source['base_sha'] == contract['anchor']['sha'], 'extended anchor is not the trusted first parent')
+ validate_rows(record['rows'], contract, revision)
+ if record.get('controls'):
+ validate_rows(record['controls'], contract, contract['anchor'])
+ for view in ('traditional', 'anchor'):
+ checked = benchmark_provenance.describe(view_report(record, view), record['view_verification'][view])
+ require(checked['state'] == 'Verified against Retention', checked['reason'])
+
+
+def view_report(record, view):
+ adapted = report.as_report(record['rows'], record['contract'], view)
+ anchor = record['contract']['anchor']
+ tools = {key: dict(value, status='available') for key, value in record['contract']['tools'].items()}
+ tools['xff'] = {'path': record['revision']['binary'], 'sha256': record['revision']['binary_sha256'], 'status': 'available'}
+ if view == 'anchor':
+ old = next(key for key in adapted['tasks'][0]['participants'] if key != 'xff')
+ name = 'xff-parent' if record['comparison'] == 'parent' else old
+ tools[name] = {'path': anchor['binary'], 'sha256': anchor['binary_sha256'], 'status': 'available'}
+ if name != old:
+ for task in adapted['tasks']:
+ task['participants'][name] = task['participants'].pop(old)
+ adapted['tools'] = tools
+ if 'cpu_policy' in record['contract']:
+ adapted['contract']['cpu_policy'] = record['contract']['cpu_policy']
+ for task in adapted['tasks']:
+ task['fixture_identity'] = fixture_identity(task['dataset'], task['files'], task['name'] == 'archive-tar')
+ adapted['contract'].update(comparison=record['comparison'] if view == 'anchor' else 'traditional',
+ production_target='//xff/cli:xff_full',
+ machine=record['machine'], warmup_rounds=1,
+ storage={'filesystem': 'tmpfs' if record['platform'] == 'linux' else 'case-sensitive HFS+',
+ 'memory_required': True},
+ resource_accounting={key: record['contract']['launcher'][key]
+ for key in ('wall_method', 'memory_method')})
+ # Empty means unpinned workers, never an enforceable zero-length mask.
+ if not record['contract']['affinity']:
+ adapted['contract']['affinity_by_cpu_count'] = {str(n): None for n in adapted['contract']['cpu_counts']}
+ source = record.get('publication_source', {'head_sha': record['revision']['sha']})
+ source = dict(source, reference_time=record['revision']['date'])
+ if record['origin'] == 'local':
+ source = dict(source, created_at=record['completed_at'], reference_time=record['revision']['date'],
+ event='local', head_branch='local')
+ result = {'schema': 1, 'kind': 'extended-production-view', 'head': record['revision']['sha'],
+ 'base': anchor['sha'], 'revision': record['revision'], 'platform': record['platform'],
+ 'series': record['series'], 'completed_at': record['completed_at'],
+ 'source': source, 'build': record['builds'][record['revision']['sha']]['manifest'],
+ 'tool_comparisons': adapted}
+ if record['origin'] == 'local':
+ result.update(kind='backfill', purpose='local-addition')
+ if view in record.get('view_verification', {}):
+ result['binary_verification'] = record['view_verification'][view]
+ return result
+
+
+def make_record(contract, revision, rows, builds, platform, machine, series, comparison,
+ origin='local', collection=None, controls=None, retention_reference='original frozen executable',
+ control_error=None):
+ result = {'schema': 1, 'kind': 'extended-production', 'contract': contract, 'revision': revision,
+ 'rows': rows, 'builds': builds, 'platform': platform, 'machine': machine, 'series': series,
+ 'comparison': comparison, 'origin': origin, 'collection': collection,
+ 'controls': controls or [], 'completed_at': max(row['after']['utc'] for row in rows)}
+ if control_error is not None:
+ result['control_error'] = control_error
+ result['view_verification'] = {}
+ for view in ('traditional', 'anchor'):
+ wrapper = view_report(result, view)
+ with Path(revision['binary']).open('rb') as executable:
+ result['view_verification'][view] = benchmark_provenance.verify_retention(wrapper, executable, retention_reference)
+ result['identity'] = observation_identity(result)
+ validate(result)
+ return result
+
+
+def retain(root, record, source=None):
+ require(record['origin'] != 'ci' or source is not None, 'CI retention requires trusted workflow metadata')
+ validate(record, source)
+ if source is not None:
+ record = dict(record, publication_source={key: source[key] for key in
+ ('id', 'run_attempt', 'created_at', 'head_sha', 'head_branch', 'event',
+ 'name', 'path', 'status', 'repository', 'base_sha')})
+ destination = root / 'extended' / 'observations' / record['identity'] / 'report.json'
+ if destination.exists() or destination.with_suffix('.json.gz').exists():
+ require(observation_identity(read(destination)) == observation_identity(record),
+ 'retained extended observation cannot change')
+ return destination
+ destination.parent.parent.mkdir(parents=True, exist_ok=True)
+ with tempfile.TemporaryDirectory(prefix='.extended-', dir=destination.parent.parent) as temporary:
+ stage = Path(temporary) / 'complete'
+ stage.mkdir()
+ (stage / 'report.json').write_text(json.dumps(record, allow_nan=False) + '\n')
+ stage.rename(destination.parent)
+ return destination
diff --git a/tools/benchmark_extended_records_test.py b/tools/benchmark_extended_records_test.py
new file mode 100644
index 0000000000..de377c989f
--- /dev/null
+++ b/tools/benchmark_extended_records_test.py
@@ -0,0 +1,97 @@
+# SPDX-FileCopyrightText: Copyright (c) M. Boerger, the MBO Works authors
+# SPDX-License-Identifier: Apache-2.0
+"""Reject incomplete, changed, incompatible or incorrectly attributed observations."""
+
+import copy
+import gzip
+import json
+import unittest
+
+import benchmark_extended_records as records
+from benchmark_extended_test_support import ExtendedFixture
+
+
+class BenchmarkExtendedRecordsTest(unittest.TestCase):
+ def setUp(self):
+ self.fixture = ExtendedFixture()
+ self.addCleanup(self.fixture.close)
+
+ def test_real_retained_bytes_bind_both_views_without_claiming_ci(self):
+ value = self.fixture.record()
+ records.validate(value)
+ for view in ('traditional', 'anchor'):
+ wrapper = records.view_report(value, view)
+ self.assertEqual(records.benchmark_provenance.describe(wrapper)['state'], 'Verified against Retention')
+ self.assertEqual(wrapper['source']['event'], 'local')
+ directory = self.fixture.root / 'binaries' / value['revision']['sha']
+ (directory / 'xff_full').write_bytes(b'changed')
+ with self.assertRaisesRegex(ValueError, 'frozen binary changed'):
+ records.build_proof(directory, value['revision'])
+
+ def test_changed_rows_oracle_commands_environment_and_sampling_are_rejected(self):
+ value = self.fixture.record()
+ mutations = [lambda r: r['rows'].pop(),
+ lambda r: r['rows'][0]['samples'].pop(),
+ lambda r: r['rows'][0].update(expected_records=999),
+ lambda r: r['rows'][0]['environment'].update(TZ='wrong'),
+ lambda r: r['rows'][0]['commands']['xff'].append('--wrong'),
+ lambda r: r['rows'][0]['samples'][0].update(stdout_bytes=1),
+ lambda r: r['rows'][0]['samples'][0].update(exit_codes=[1]),
+ lambda r: r['contract'].update(keep=8),
+ lambda r: r['builds'][r['revision']['sha']]['link_arguments'].remove('-flto=thin')]
+ for mutate in mutations:
+ with self.subTest(mutation=mutate):
+ changed = copy.deepcopy(value)
+ mutate(changed)
+ changed['identity'] = records.observation_identity(changed)
+ with self.assertRaises(ValueError):
+ records.validate(changed)
+
+ def test_parent_label_is_stable_and_concrete_parent_is_retained(self):
+ value = self.fixture.record(origin='ci', comparison='parent')
+ records.validate(value, self.fixture.source())
+ view = records.view_report(value, 'anchor')
+ self.assertEqual(set(view['tool_comparisons']['tasks'][0]['participants']), {'xff', 'xff-parent'})
+ self.assertEqual(view['base'], self.fixture.revisions[-1]['sha'])
+
+ def test_ci_requires_actual_run_attempt_repository_and_first_parent(self):
+ value = self.fixture.record(origin='ci', comparison='parent')
+ for change in ({'head_sha': 'f' * 40}, {'base_sha': 'f' * 40}, {'run_attempt': 2},
+ {'event': 'pull_request'}, {'repository': {'full_name': 'other/repo'}}):
+ with self.subTest(change=change), self.assertRaises(ValueError):
+ records.validate(value, dict(self.fixture.source(), **change))
+
+ def test_idempotent_retention_preserves_original_proof_and_supports_packed_records(self):
+ value = self.fixture.record()
+ root = self.fixture.root / 'site'
+ path = records.retain(root, value)
+ before = path.read_bytes()
+ repeated = self.fixture.record()
+ self.assertEqual(repeated['identity'], value['identity'])
+ records.retain(root, repeated)
+ self.assertEqual(path.read_bytes(), before)
+ path.with_suffix('.json.gz').write_bytes(gzip.compress(before, mtime=0))
+ path.unlink()
+ records.retain(root, repeated)
+ self.assertEqual(records.read(path), json.loads(before))
+
+ def test_complete_head_can_be_retained_from_failed_workflow(self):
+ value = self.fixture.record(origin='ci', comparison='parent')
+ path = records.retain(self.fixture.root / 'site', value, self.fixture.source())
+ saved = records.read(path)
+ self.assertEqual(saved['publication_source']['id'], 12)
+ self.assertEqual(saved['origin'], 'ci')
+
+ def test_ci_control_omission_requires_an_explicit_diagnostic_failure(self):
+ value = self.fixture.record(origin='ci', comparison='parent')
+ value['controls'] = []
+ value['identity'] = records.observation_identity(value)
+ with self.assertRaisesRegex(ValueError, 'explicit failure'):
+ records.validate(value, self.fixture.source())
+ value['control_error'] = 'synthetic control failure; complete head retained'
+ value['identity'] = records.observation_identity(value)
+ records.validate(value, self.fixture.source())
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/tools/benchmark_extended_site.py b/tools/benchmark_extended_site.py
new file mode 100644
index 0000000000..c4d2fc9df1
--- /dev/null
+++ b/tools/benchmark_extended_site.py
@@ -0,0 +1,166 @@
+# SPDX-FileCopyrightText: Copyright (c) M. Boerger, the MBO Works authors
+# SPDX-License-Identifier: Apache-2.0
+"""Import measured extended history and expose it in the existing benchmark viewer."""
+
+import argparse
+from datetime import datetime, timezone
+import html
+import json
+import os
+from pathlib import Path
+import re
+
+import benchmark_datasets
+import benchmark_extended as suite
+import benchmark_extended_records as records
+import benchmark_landscape as landscape
+import benchmark_matrix as matrix
+import benchmark_normalization
+
+
+def import_campaign(campaign, builds_root, root, series, platform, machine):
+ records.require(not (campaign / 'invalid.json').exists(), 'campaign integrity check failed')
+ progress = records.read(campaign / 'progress.json')
+ records.require(progress['state'] == 'complete' and progress['completed'] == progress['total'],
+ 'campaign is incomplete')
+ contract = records.read(campaign / 'contract.json')
+ records.require(progress['total'] == len(contract['cases']) * len(contract['revisions']), 'campaign grid changed')
+ anchor = contract['anchor']
+ anchor_build = records.build_proof(builds_root / 'binaries' / anchor['sha'], anchor)
+ result = []
+ for revision in contract['revisions']:
+ rows = [records.read(campaign / 'cells' / revision['sha'] / suite.Case(**case).key / 'complete.json')
+ for case in contract['cases']]
+ builds = {anchor['sha']: anchor_build,
+ revision['sha']: records.build_proof(builds_root / 'binaries' / revision['sha'], revision)}
+ record = records.make_record(contract, revision, rows, builds, platform, machine, series, 'fixed-anchor',
+ retention_reference=str(builds_root / 'binaries' / revision['sha'] / 'xff_full')
+ + ' (original retained production executable)')
+ path = records.retain(root, record)
+ result.append(path)
+ print(f'IMPORTED {len(result)}/{len(contract["revisions"])} {revision["sha"]} {path}', flush=True)
+ return result
+
+
+def descriptor(record, view):
+ return {'series': record['series'], 'origin': record['origin'], 'platform': record['platform'],
+ 'machine': record['machine'], 'recipe': record['contract']['recipe'], 'view': view,
+ 'comparison': record['comparison'],
+ 'fixed_anchor': record['contract']['anchor']['sha'] if record['comparison'] == 'fixed-anchor' else None,
+ 'target': '//xff/cli:xff_full', 'repetitions': 9, 'keep': 7,
+ 'storage': 'tmpfs' if record['platform'] == 'linux' else 'case-sensitive HFS+ RAM',
+ 'accounting': {key: record['contract']['launcher'][key] for key in ('wall_method', 'memory_method')}}
+
+
+def retained(root):
+ paths = {*root.glob('extended/observations/*/report.json'),
+ *(path.with_suffix('') for path in root.glob('extended/observations/*/report.json.gz'))}
+ selected = {}
+ for path in sorted(paths):
+ record = records.read(path)
+ records.require(record['origin'] != 'ci' or record.get('publication_source'),
+ 'CI publication requires trusted workflow metadata')
+ records.validate(record, record.get('publication_source'))
+ for view in ('traditional', 'anchor'):
+ description = descriptor(record, view)
+ series_id = records.identity(description)
+ key = series_id, record['revision']['sha']
+ collection = record.get('collection') or {}
+ rank = (record['completed_at'], int(collection.get('run_id', 0)), int(collection.get('run_attempt', 0)),
+ bool(record.get('controls')), record['identity'])
+ if key not in selected or rank > selected[key][0]:
+ selected[key] = (rank, path, record, view, description)
+ return sorted(selected.values(), key=lambda item: (item[2]['revision']['date'], item[2]['revision']['sha'], item[3]))
+
+
+def publish(root, repository='mboworks/xff'):
+ selected = retained(root)
+ catalog_path = root / 'catalog.json'
+ catalog = [item for item in records.read(catalog_path) if not item.get('extended')] if catalog_path.exists() else []
+ links_path = root / 'version-links.json'
+ version_links = records.read(links_path) if links_path.exists() else {}
+ groups = {}
+ for _, path, record, view, description in selected:
+ series_id = records.identity(description)
+ wrapper = records.view_report(record, view)
+ groups.setdefault(series_id, []).append((path, record, view, description, wrapper))
+ for series_id, group in groups.items():
+ order = landscape.common_task_order([matrix.relative_results(wrapper['tool_comparisons']) for *_, wrapper in group])
+ windows = benchmark_normalization.reference_windows([(record['identity'], wrapper) for _, record, _, _, wrapper in group])
+ for path, record, view, description, wrapper in group:
+ revision = record['revision']
+ anchor = record['contract']['anchor']
+ directory = root / 'extended' / 'views' / series_id / revision['sha']
+ directory.mkdir(parents=True, exist_ok=True)
+ (directory / 'report.json').write_text(json.dumps(wrapper, allow_nan=False) + '\n')
+ report = wrapper['tool_comparisons']
+ normalization = windows.get(record['identity'], {})
+ data = landscape.figures(report, normalization, order)
+ (directory / 'landscape.json').write_text(json.dumps(data, allow_nan=False) + '\n')
+ (directory / 'normalization.json').write_text(json.dumps(normalization, allow_nan=False) + '\n')
+ document = landscape.render(report, '', normalization, order)
+ asset = html.escape(os.path.relpath(root / 'assets/three-landscape.js', directory), quote=True)
+ document = document.replace('', f'')
+ raw = html.escape(os.path.relpath(path, directory), quote=True)
+ title = ('Traditional tools' if view == 'traditional' else 'Parent XFF' if record['comparison'] == 'parent'
+ else 'Fixed XFF ' + anchor['sha'][:12])
+ introduction = ('Extended production suite: ' + suite.RECIPE + '; full production CLI. '
+ + html.escape(title) + '. Original observations and provenance.
')
+ document = document.replace('', '' + introduction, 1)
+ (directory / 'index.html').write_text(document)
+ local = record['origin'] == 'local'
+ experiment = local and revision['pr'] <= 0
+ label = ('Experiment ' + revision['sha'][:12] if experiment else
+ ('PR #' + str(revision['pr']) if revision['pr'] > 0 else 'main') + ' / ' + revision['sha'][:10])
+ links = version_links.get(revision['sha'], [])
+ if not links and revision['pr'] > 0:
+ links = [{'label': 'PR #' + str(revision['pr']), 'href': f'https://github.com/{repository}/pull/{revision["pr"]}'}]
+ pr_labels = [link['label'] for link in links if re.fullmatch(r'PR #[0-9]+', link.get('label', ''))]
+ if not experiment and pr_labels:
+ label = ', '.join(pr_labels) + ' / ' + revision['sha'][:10]
+ links = [*links, {'label': ('Parent ' if record['comparison'] == 'parent' else 'Fixed reference ') + anchor['sha'][:12],
+ 'href': f'https://github.com/{repository}/commit/{anchor["sha"]}'}]
+ origin_label = 'Local / ' + record['series'] if local else 'CI / ' + record['platform']
+ collection = record.get('collection') or {}
+ catalog.append({'platform': origin_label + ' / ' + suite.RECIPE + ' / ' + title,
+ 'platform_name': record['platform'] + ' / ' + record['machine'],
+ 'dataset_type': suite.RECIPE + ' / ' + title, 'commit': revision['sha'],
+ 'label': label, 'date': datetime.fromisoformat(revision['date']).astimezone(timezone.utc).isoformat(),
+ 'measurement_date': record['completed_at'],
+ 'source': 'experiment' if experiment else 'merged',
+ 'source_label': 'Unmerged experiments' if experiment else 'Merged history',
+ 'comparison_role': ('Parent ' if record['comparison'] == 'parent' else 'Fixed reference ') + anchor['sha'][:12],
+ 'local': local, 'extended': True, 'series_id': series_id,
+ 'measured': record['completed_at'], 'run': collection.get('run_id', 0),
+ 'attempt': collection.get('run_attempt', 0), 'links': links,
+ 'binary': landscape.binary_evidence(directory / 'report.json', wrapper),
+ 'report': directory.relative_to(root).as_posix() + '/',
+ 'figures': (directory / 'landscape.json').relative_to(root).as_posix(),
+ 'identity': record['contract']['platform'] + ' / ' + suite.RECIPE,
+ 'platform_details': record['contract']['platform']})
+ catalog_path.write_text(json.dumps(catalog, indent=2, allow_nan=False) + '\n')
+ landscape.publish_history(root, catalog)
+ benchmark_datasets.publish(root)
+ return len(selected)
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__)
+ modes = parser.add_subparsers(dest='mode', required=True)
+ importer = modes.add_parser('import')
+ for name in ('campaign', 'build-root', 'root'):
+ importer.add_argument('--' + name, type=Path, required=True)
+ for name in ('series', 'platform', 'machine'):
+ importer.add_argument('--' + name, required=True)
+ renderer = modes.add_parser('publish')
+ renderer.add_argument('--root', type=Path, required=True)
+ renderer.add_argument('--repository', default='mboworks/xff')
+ args = parser.parse_args()
+ if args.mode == 'import':
+ import_campaign(args.campaign, args.build_root, args.root, args.series, args.platform, args.machine)
+ else:
+ print(f'Published {publish(args.root, args.repository)} extended views')
+
+
+if __name__ == '__main__':
+ main()
diff --git a/tools/benchmark_extended_site_test.py b/tools/benchmark_extended_site_test.py
new file mode 100644
index 0000000000..33549582ef
--- /dev/null
+++ b/tools/benchmark_extended_site_test.py
@@ -0,0 +1,134 @@
+# SPDX-FileCopyrightText: Copyright (c) M. Boerger, the MBO Works authors
+# SPDX-License-Identifier: Apache-2.0
+"""Keep extended observations separate, measured, discoverable and reproducible."""
+
+import gzip
+import copy
+import json
+import unittest
+
+import benchmark_extended as suite
+import benchmark_extended_records as records
+import benchmark_extended_site as site
+from benchmark_extended_test_support import ExtendedFixture
+
+
+class BenchmarkExtendedSiteTest(unittest.TestCase):
+ def setUp(self):
+ self.fixture = ExtendedFixture()
+ self.addCleanup(self.fixture.close)
+ self.root = self.fixture.root / 'site'
+ self.root.mkdir()
+ (self.root / 'index.html').write_text('Benchmarks
')
+ self.core = {'platform': 'CI core linux', 'commit': 'c' * 40, 'date': '2026-11-01T00:00:00Z',
+ 'report': 'runs/core/linux/', 'figures': 'runs/core/linux/landscape.json',
+ 'label': 'Core fixture', 'source': 'merged'}
+ (self.root / 'catalog.json').write_text(json.dumps([self.core]))
+
+ def test_both_views_join_existing_browser_without_replacing_core_or_inventing_points(self):
+ value = self.fixture.record()
+ records.retain(self.root, value)
+ self.assertEqual(site.publish(self.root), 2)
+ catalog = records.read(self.root / 'catalog.json')
+ self.assertEqual(catalog[0], self.core)
+ self.assertEqual(len(catalog), 3)
+ self.assertIn('XffBenchmarkHistory', (self.root / 'index.html').read_text())
+ for entry in catalog[1:]:
+ self.assertTrue(entry['local'])
+ self.assertTrue(entry['extended'])
+ self.assertEqual(entry['binary']['state'], 'Verified against Retention')
+ wrapper = records.read(self.root / entry['report'] / 'report.json')
+ self.assertEqual(len(wrapper['tool_comparisons']['tasks']), len(self.fixture.cases))
+ data = records.read(self.root / entry['figures'])
+ surfaces = [trace for trace in data['alphabetical']['percent']['data'] if trace['type'] == 'surface']
+ heights = [height for trace in surfaces for row in trace['y'] for height in row if height is not None]
+ expected = 50 if 'Traditional tools' in entry['platform'] else -100
+ self.assertEqual(heights, [expected] * len(self.fixture.cases))
+ datasets = records.read(self.root / 'datasets.json')['datasets']
+ self.assertEqual(len(datasets), 2)
+ self.assertEqual({item['kind'] for item in datasets}, {'local'})
+ self.assertTrue(all(item['recipe']['suite'] == suite.RECIPE for item in datasets))
+
+ def test_repeated_publication_and_packed_observations_are_idempotent(self):
+ path = records.retain(self.root, self.fixture.record())
+ site.publish(self.root)
+ before = records.read(self.root / 'catalog.json')
+ path.with_suffix('.json.gz').write_bytes(gzip.compress(path.read_bytes(), mtime=0))
+ path.unlink()
+ site.publish(self.root)
+ self.assertEqual(records.read(self.root / 'catalog.json'), before)
+ page = (self.root / 'index.html').read_text()
+ self.assertEqual(page.count(''), 1)
+ self.assertEqual(page.count(''), 1)
+
+ def test_changing_ci_parent_keeps_same_dataset_and_exposes_actual_parent_per_version(self):
+ first = self.fixture.record(origin='ci', comparison='parent')
+ records.retain(self.root, first, self.fixture.source())
+ self.fixture.revisions.reverse()
+ self.fixture.contract['anchor'] = self.fixture.revisions[-1]
+ second = self.fixture.record(origin='ci', comparison='parent')
+ records.retain(self.root, second, self.fixture.source())
+ self.assertEqual(site.publish(self.root), 4)
+ catalog = [item for item in records.read(self.root / 'catalog.json') if item.get('extended')]
+ self.assertEqual(len({item['series_id'] for item in catalog}), 2)
+ self.assertEqual({item['comparison_role'] for item in catalog}, {'Parent ' + 'a' * 12, 'Parent ' + 'b' * 12})
+ for entry in catalog:
+ self.assertFalse(entry['local'])
+ self.assertEqual(entry['source'], 'merged')
+ self.assertIn('github.com/mboworks/xff/commit/', entry['links'][-1]['href'])
+ self.assertEqual(len(records.read(self.root / 'datasets.json')['datasets']), 2)
+
+ def test_publication_rejects_untrusted_ci_and_modified_raw_rows(self):
+ value = self.fixture.record(origin='ci', comparison='parent')
+ with self.assertRaisesRegex(ValueError, 'trusted workflow metadata'):
+ records.retain(self.root, value)
+ path = records.retain(self.root, value, self.fixture.source())
+ changed = records.read(path)
+ changed['publication_source']['event'] = 'pull_request'
+ path.write_text(json.dumps(changed))
+ with self.assertRaisesRegex(ValueError, 'untrusted extended workflow'):
+ site.publish(self.root)
+
+ def test_same_head_observations_prefer_completed_controls_when_measurement_times_match(self):
+ value = self.fixture.record()
+ records.retain(self.root, value)
+ complete = copy.deepcopy(value)
+ complete['controls'] = self.fixture.rows(self.fixture.revisions[-1])
+ complete['identity'] = records.observation_identity(complete)
+ records.retain(self.root, complete)
+ selected = site.retained(self.root)
+ self.assertEqual(len(selected), 2)
+ self.assertTrue(all(item[2]['controls'] for item in selected))
+
+ def test_ci_pr_labels_use_trusted_version_links_when_subject_has_no_pr_suffix(self):
+ self.fixture.revisions[0]['pr'] = 0
+ value = self.fixture.record(origin='ci', comparison='parent')
+ records.retain(self.root, value, self.fixture.source())
+ (self.root / 'version-links.json').write_text(json.dumps({value['revision']['sha']: [
+ {'label': 'PR #999', 'href': 'https://github.com/mboworks/xff/pull/999'}]}))
+ site.publish(self.root)
+ entries = [item for item in records.read(self.root / 'catalog.json') if item.get('extended')]
+ self.assertTrue(all(item['label'] == 'PR #999 / ' + value['revision']['sha'][:10] for item in entries))
+
+ def test_import_uses_complete_original_cells_and_actual_retained_builds(self):
+ campaign = self.fixture.root / 'campaign'
+ campaign.mkdir()
+ (campaign / 'contract.json').write_text(json.dumps(self.fixture.contract))
+ total = len(self.fixture.cases) * len(self.fixture.revisions)
+ progress = {'state': 'complete', 'completed': total, 'total': total}
+ (campaign / 'progress.json').write_text(json.dumps(progress))
+ for revision in self.fixture.revisions:
+ for row in self.fixture.rows(revision):
+ path = campaign / 'cells' / revision['sha'] / suite.Case(**row['case']).key / 'complete.json'
+ path.parent.mkdir(parents=True)
+ path.write_text(json.dumps(row))
+ imported = site.import_campaign(campaign, self.fixture.root, self.root, 'synthetic-original', 'linux', 'x86_64')
+ self.assertEqual(len(imported), len(self.fixture.revisions))
+ self.assertEqual(site.publish(self.root), 4)
+ (campaign / 'progress.json').write_text(json.dumps(dict(progress, completed=total - 1)))
+ with self.assertRaisesRegex(ValueError, 'campaign is incomplete'):
+ site.import_campaign(campaign, self.fixture.root, self.root, 'synthetic-original', 'linux', 'x86_64')
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/tools/benchmark_extended_storage.py b/tools/benchmark_extended_storage.py
new file mode 100644
index 0000000000..bbf746043f
--- /dev/null
+++ b/tools/benchmark_extended_storage.py
@@ -0,0 +1,116 @@
+# SPDX-FileCopyrightText: Copyright (c) M. Boerger, the MBO Works authors
+# SPDX-License-Identifier: Apache-2.0
+"""Own disposable memory fixtures and record their creation and release."""
+
+from contextlib import contextmanager
+from datetime import datetime, timezone
+import json
+import hashlib
+from pathlib import Path
+import plistlib
+import re
+import shutil
+import subprocess
+import sys
+
+
+def write(path, value):
+ path.write_text(json.dumps(value, indent=2) + '\n')
+
+
+def disk_info(mount):
+ return plistlib.loads(subprocess.check_output(['diskutil', 'info', '-plist', str(mount)]))
+
+
+def verify_image(device, image):
+ images = plistlib.loads(subprocess.check_output(['hdiutil', 'info', '-plist']))['images']
+ found = [entry for entry in images if entry.get('image-path') == image
+ and any(item.get('dev-entry') == device for item in entry.get('system-entities', []))]
+ if len(found) != 1:
+ raise ValueError('device does not match the newly owned RAM image')
+
+
+def verify_volume(receipt):
+ verify_image(receipt['device'], receipt['image'])
+ info = disk_info(receipt['mount'])
+ if (info.get('DeviceNode') != receipt['device'] or info.get('VolumeUUID') != receipt['uuid']
+ or info.get('MountPoint') != receipt['mount']):
+ raise ValueError('RAM volume ownership changed; refusing to detach')
+
+
+@contextmanager
+def fixture(root, existing=None):
+ """Create a new fixture; never reformat an existing volume or remove another fixture."""
+ root = root.resolve()
+ root.mkdir(parents=True, exist_ok=True)
+ if existing is not None:
+ if sys.platform != 'linux':
+ raise ValueError('existing fixture reuse is Linux-only local validation')
+ existing = existing.resolve(strict=True)
+ filesystem = subprocess.check_output(['findmnt', '-n', '-o', 'FSTYPE', '-T', str(existing)], text=True).strip()
+ if filesystem != 'tmpfs' or not existing.is_dir():
+ raise ValueError('existing extended fixture must be a verified tmpfs directory')
+ receipt = {'filesystem': filesystem, 'storage': 'memory', 'fixture': str(existing), 'borrowed': True}
+ write(root / 'storage.json', receipt)
+ try:
+ yield existing, receipt
+ finally:
+ write(root / 'storage-release.json', dict(receipt, preserved=True,
+ released_at=datetime.now(timezone.utc).isoformat()))
+ return
+ if sys.platform == 'linux':
+ filesystem = subprocess.check_output(['findmnt', '-n', '-o', 'FSTYPE', '-T', '/dev/shm'], text=True).strip()
+ if filesystem != 'tmpfs':
+ raise ValueError('extended Linux fixtures require verified tmpfs')
+ # Stable paths keep repeated CI invocations comparable. mkdir refuses a
+ # collision; only this context's newly created directory may be removed.
+ parent = Path('/dev/shm') / ('xff-extended-' + hashlib.sha256(str(root).encode()).hexdigest()[:12])
+ parent.mkdir()
+ owned = parent.stat()
+ receipt = {'filesystem': filesystem, 'storage': 'memory', 'parent': str(parent)}
+ write(root / 'storage.json', receipt)
+ try:
+ yield parent / 'fixture', receipt
+ finally:
+ current = parent.lstat()
+ if parent.is_symlink() or (current.st_dev, current.st_ino) != (owned.st_dev, owned.st_ino):
+ raise ValueError('fixture ownership changed; refusing removal')
+ shutil.rmtree(parent)
+ write(root / 'storage-release.json', dict(receipt, released_at=datetime.now(timezone.utc).isoformat()))
+ return
+ if sys.platform != 'darwin':
+ raise ValueError('extended fixtures support Linux and macOS')
+ image = 'ram://4194304' # 2 GiB; the full recipe uses approximately 1 GiB.
+ result = subprocess.check_output(['hdiutil', 'attach', '-nomount', image], text=True).strip()
+ if not re.fullmatch(r'/dev/disk[0-9]+', result):
+ raise ValueError('ambiguous newly attached RAM device: ' + result)
+ device = result
+ verify_image(device, image)
+ # Record ownership before formatting; failure cleanup still checks the RAM image.
+ receipt = {'device': device, 'image': image, 'storage': 'memory', 'filesystem': 'case-sensitive HFS+',
+ 'created_at': datetime.now(timezone.utc).isoformat()}
+ write(root / 'storage.json', receipt)
+ try:
+ subprocess.run(['newfs_hfs', '-s', '-v', 'XffExtended', device], check=True)
+ mount = root / 'ram-volume'
+ mount.mkdir()
+ subprocess.run(['diskutil', 'mount', '-mountPoint', str(mount), device], check=True)
+ info = disk_info(mount)
+ if info.get('DeviceNode') != device or info.get('MountPoint') != str(mount) or not info.get('VolumeUUID'):
+ raise ValueError('new RAM volume identity is missing')
+ receipt.update(mount=str(mount), uuid=info['VolumeUUID'])
+ write(root / 'storage.json', receipt)
+ verify_volume(receipt)
+ yield mount / 'fixture', receipt
+ finally:
+ if 'uuid' in receipt:
+ verify_volume(receipt)
+ else:
+ verify_image(device, image)
+ subprocess.run(['hdiutil', 'detach', device], check=True)
+ images = plistlib.loads(subprocess.check_output(['hdiutil', 'info', '-plist']))['images']
+ if any(any(item.get('dev-entry') == device for item in entry.get('system-entities', [])) for entry in images):
+ raise ValueError('owned RAM device remains after detach')
+ if 'mount' in receipt and Path(receipt['mount']).is_mount():
+ raise ValueError('owned RAM mount remains after detach')
+ write(root / 'storage-release.json', dict(receipt, released_at=datetime.now(timezone.utc).isoformat()))
diff --git a/tools/benchmark_extended_storage_test.py b/tools/benchmark_extended_storage_test.py
new file mode 100644
index 0000000000..3fe805590c
--- /dev/null
+++ b/tools/benchmark_extended_storage_test.py
@@ -0,0 +1,134 @@
+# SPDX-FileCopyrightText: Copyright (c) M. Boerger, the MBO Works authors
+# SPDX-License-Identifier: Apache-2.0
+"""Memory fixtures release only resources whose ownership still matches."""
+
+import hashlib
+import json
+from pathlib import Path
+import plistlib
+import tempfile
+import unittest
+from unittest import mock
+
+import benchmark_extended_storage as storage
+
+
+class BenchmarkExtendedStorageTest(unittest.TestCase):
+ def setUp(self):
+ self.temporary = tempfile.TemporaryDirectory()
+ self.addCleanup(self.temporary.cleanup)
+ self.root = Path(self.temporary.name).resolve() / 'collection'
+ self.device = '/dev/disk987'
+ self.attached = False
+ self.image = 'ram://4194304'
+ self.uuid = 'synthetic-owned-uuid'
+ self.commands = []
+
+ def output(self, command, **kwargs):
+ self.commands.append(command)
+ if command[:2] == ['hdiutil', 'attach']:
+ self.attached = True
+ return self.device + '\n'
+ if command == ['hdiutil', 'info', '-plist']:
+ images = [{'image-path': self.image, 'system-entities': [{'dev-entry': self.device}]}] if self.attached else []
+ return plistlib.dumps({'images': images})
+ if command[:3] == ['diskutil', 'info', '-plist']:
+ return plistlib.dumps({'DeviceNode': self.device, 'MountPoint': str(self.root / 'ram-volume'),
+ 'VolumeUUID': self.uuid})
+ raise AssertionError(command)
+
+ def run_command(self, command, **kwargs):
+ self.commands.append(command)
+ if command == ['hdiutil', 'detach', self.device]:
+ self.attached = False
+ return mock.Mock(returncode=0)
+
+ def mac_mocks(self):
+ return (mock.patch.object(storage.sys, 'platform', 'darwin'),
+ mock.patch.object(storage.subprocess, 'check_output', side_effect=self.output),
+ mock.patch.object(storage.subprocess, 'run', side_effect=self.run_command))
+
+ def test_mac_formats_only_the_new_owned_ram_image_and_records_verified_release(self):
+ platform, output, run = self.mac_mocks()
+ with platform, output, run:
+ with storage.fixture(self.root) as (fixture, receipt):
+ self.assertEqual(fixture, self.root / 'ram-volume/fixture')
+ self.assertEqual(receipt['uuid'], self.uuid)
+ self.assertTrue(self.attached)
+ self.assertFalse(self.attached)
+ commands = self.commands
+ self.assertLess(commands.index(['hdiutil', 'info', '-plist']),
+ commands.index(['newfs_hfs', '-s', '-v', 'XffExtended', self.device]))
+ self.assertEqual(json.loads((self.root / 'storage-release.json').read_text())['uuid'], self.uuid)
+
+ def test_mac_body_failure_still_releases_owned_device(self):
+ platform, output, run = self.mac_mocks()
+ with platform, output, run, self.assertRaisesRegex(RuntimeError, 'measurement failed'):
+ with storage.fixture(self.root):
+ raise RuntimeError('measurement failed')
+ self.assertFalse(self.attached)
+ self.assertTrue((self.root / 'storage-release.json').is_file())
+
+ def test_mac_changed_volume_identity_refuses_detach(self):
+ platform, output, run = self.mac_mocks()
+ with platform, output, run, self.assertRaisesRegex(ValueError, 'ownership changed'):
+ with storage.fixture(self.root):
+ self.uuid = 'someone-else'
+ self.assertNotIn(['hdiutil', 'detach', self.device], self.commands)
+ self.assertFalse((self.root / 'storage-release.json').exists())
+
+ def test_mac_non_ram_device_is_never_formatted_or_detached(self):
+ platform, output, run = self.mac_mocks()
+ self.image = '/someone/elses/disk.dmg'
+ with platform, output, run, self.assertRaisesRegex(ValueError, 'newly owned RAM image'):
+ with storage.fixture(self.root):
+ self.fail('must not yield an unowned device')
+ self.assertFalse(any(command[0] == 'newfs_hfs' for command in self.commands))
+ self.assertNotIn(['hdiutil', 'detach', self.device], self.commands)
+
+ def test_linux_collision_preserves_existing_directory(self):
+ shm = Path(self.temporary.name) / 'shm'
+ shm.mkdir()
+ parent = shm / ('xff-extended-' + hashlib.sha256(str(self.root).encode()).hexdigest()[:12])
+ parent.mkdir()
+ (parent / 'foreign').write_text('preserve')
+ path = lambda value: shm if value == '/dev/shm' else Path(value)
+ with mock.patch.object(storage.sys, 'platform', 'linux'), mock.patch.object(storage, 'Path', side_effect=path), \
+ mock.patch.object(storage.subprocess, 'check_output', return_value='tmpfs\n'), \
+ self.assertRaises(FileExistsError):
+ with storage.fixture(self.root):
+ self.fail('must not adopt an existing directory')
+ self.assertEqual((parent / 'foreign').read_text(), 'preserve')
+
+ def test_linux_new_fixture_cleanup_and_borrowed_fixture_preservation_on_error(self):
+ shm = Path(self.temporary.name) / 'shm'
+ shm.mkdir()
+ path = lambda value: shm if value == '/dev/shm' else Path(value)
+ with mock.patch.object(storage.sys, 'platform', 'linux'), mock.patch.object(storage, 'Path', side_effect=path), \
+ mock.patch.object(storage.subprocess, 'check_output', return_value='tmpfs\n'):
+ with self.assertRaisesRegex(RuntimeError, 'failed'):
+ with storage.fixture(self.root) as (fixture, _):
+ fixture.mkdir()
+ (fixture / 'owned').write_text('owned')
+ raise RuntimeError('failed')
+ self.assertFalse(fixture.parent.exists())
+ borrowed = shm / 'borrowed'
+ borrowed.mkdir()
+ (borrowed / 'preserve').write_text('shared fixture')
+ with self.assertRaisesRegex(RuntimeError, 'failed'):
+ with storage.fixture(self.root, borrowed):
+ raise RuntimeError('failed')
+ self.assertEqual((borrowed / 'preserve').read_text(), 'shared fixture')
+ self.assertTrue(json.loads((self.root / 'storage-release.json').read_text())['preserved'])
+
+ def test_disk_filesystems_cannot_be_used_as_memory_fixtures(self):
+ self.root.mkdir()
+ with mock.patch.object(storage.sys, 'platform', 'linux'), \
+ mock.patch.object(storage.subprocess, 'check_output', return_value='ext4\n'), \
+ self.assertRaisesRegex(ValueError, 'verified tmpfs'):
+ with storage.fixture(self.root, self.root):
+ self.fail('must not measure disk fixtures')
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/tools/benchmark_extended_test_support.py b/tools/benchmark_extended_test_support.py
new file mode 100644
index 0000000000..d0f2d00b90
--- /dev/null
+++ b/tools/benchmark_extended_test_support.py
@@ -0,0 +1,99 @@
+# SPDX-FileCopyrightText: Copyright (c) M. Boerger, the MBO Works authors
+# SPDX-License-Identifier: Apache-2.0
+"""Small synthetic observations for extended-pipeline tests; never performance data."""
+
+import copy
+from dataclasses import asdict
+import hashlib
+import json
+from pathlib import Path
+import tempfile
+from unittest import mock
+
+import benchmark_extended as suite
+import benchmark_extended_records as records
+
+
+class ExtendedFixture:
+ def __init__(self):
+ self.temporary = tempfile.TemporaryDirectory()
+ self.root = Path(self.temporary.name)
+ self.cases = [suite.Case('files-exact', shape, files, cpus)
+ for shape in ('broad', 'deep') for files in (3, 6) for cpus in (1, 3)]
+ self.patch = mock.patch.object(suite, 'cases', return_value=self.cases)
+ self.patch.start()
+ self.revisions, self.builds = [], {}
+ for index, sha in enumerate(('a' * 40, 'b' * 40), 1):
+ directory = self.root / 'binaries' / sha
+ directory.mkdir(parents=True)
+ binary = directory / 'xff_full'
+ binary.write_bytes(('synthetic executable ' + sha).encode())
+ artifact = 'bazel-out/test/bin/xff/cli/xff_full'
+ manifest = {'source': {'revision': sha, 'tree': sha, 'clean': True}, 'sha256': records.digest(binary),
+ 'bytes': binary.stat().st_size, 'artifact': artifact,
+ 'command': ['bazel', 'build', '//xff/cli:xff_full', '--config=clang_release', '--config=xff_full'],
+ 'configuration': {'.bazelrc': 'c' * 64}}
+ (directory / 'build.json').write_text(json.dumps(manifest, indent=2) + '\n')
+ (directory / 'link-actions.json').write_text(json.dumps({'actions': [
+ {'arguments': ['clang++', '-flto=thin', '-o', artifact]}]}) + '\n')
+ revision = {'sha': sha, 'pr': index, 'date': f'2026-11-0{index}T00:00:00Z',
+ 'subject': 'Synthetic fixture', 'binary': str(binary), 'binary_sha256': manifest['sha256'],
+ 'build_sha256': records.digest(directory / 'build.json'),
+ 'link_actions_sha256': records.digest(directory / 'link-actions.json')}
+ self.revisions.append(revision)
+ self.builds[sha] = records.build_proof(directory, revision)
+ self.contract = {'recipe': suite.RECIPE, 'cases': [asdict(case) for case in self.cases],
+ 'revisions': self.revisions, 'anchor': self.revisions[-1], 'repetitions': 9, 'keep': 7,
+ 'estimator': 'mean-fastest', 'warmup_rounds': 1, 'fixture': '/synthetic-fixture',
+ 'filesystem': 'tmpfs', 'platform': 'Linux-test-x86_64', 'affinity': [0, 1, 2],
+ 'tools': {name: {'path': '/bin/' + name, 'sha256': 'd' * 64, 'version': 'synthetic ' + name}
+ for name in ('find', 'rg', 'tar')},
+ 'launcher': {'path': '/synthetic-launcher', 'sha256': 'e' * 64,
+ 'wall_method': 'synthetic wall', 'memory_method': 'synthetic RSS'}}
+
+ def close(self):
+ self.patch.stop()
+ self.temporary.cleanup()
+
+ def rows(self, revision=None):
+ revision = revision or self.revisions[0]
+ rows = []
+ for case in self.cases:
+ commands, cwd, separator = suite.commands(case, Path(self.contract['fixture']), revision['binary'],
+ {k: v['path'] for k, v in self.contract['tools'].items()})
+ anchor, _, _ = suite.commands(case, Path(self.contract['fixture']), self.revisions[-1]['binary'],
+ {k: v['path'] for k, v in self.contract['tools'].items()})
+ commands['anchor'] = anchor['xff']
+ wanted = suite.expected(case, Path(self.contract['fixture']))
+ output = separator.join(wanted) + separator
+ samples = []
+ for round_number in range(10):
+ for label in commands:
+ seconds = 100 if round_number == 0 else {'xff': 2, 'anchor': 1, 'find': 4}[label]
+ samples.append({'label': label, 'round': round_number,
+ 'phase': 'warmup' if round_number == 0 else 'measured',
+ 'elapsed_seconds': seconds, 'user_cpu_seconds': seconds / 2,
+ 'system_cpu_seconds': seconds / 4, 'peak_child_rss_bytes': 1024,
+ 'first_stdout_seconds': .01, 'exit_codes': [0], 'stderr_bytes': 0, 'stderr': '',
+ 'stdout_bytes': len(output), 'stdout_sha256': hashlib.sha256(output).hexdigest(),
+ 'validated_records': len(wanted)})
+ rows.append({'case': asdict(case), 'revision': revision, 'anchor': self.revisions[-1],
+ 'commands': commands, 'cwd': str(cwd), 'affinity': self.contract['affinity'][:case.cpus],
+ 'environment': {'PATH': '/usr/bin:/bin', 'HOME': '/synthetic-fixture/home', 'LC_ALL': 'C', 'TZ': 'UTC'},
+ 'expected_records': len(wanted), 'expected_sha256': hashlib.sha256(b'\0'.join(sorted(wanted))).hexdigest(),
+ 'samples': samples, 'after': {'utc': '2026-11-08T12:00:00Z'}})
+ return rows
+
+ def record(self, origin='local', comparison='fixed-anchor'):
+ collection = {'run_id': 12, 'run_attempt': 1, 'repository': 'owner/repo',
+ 'head_sha': self.revisions[0]['sha'], 'base_sha': self.revisions[-1]['sha']} if origin == 'ci' else None
+ return records.make_record(copy.deepcopy(self.contract), self.revisions[0], self.rows(), self.builds,
+ 'linux', 'x86_64', 'synthetic-only', comparison, origin, collection,
+ controls=self.rows(self.revisions[-1]) if origin == 'ci' else None)
+
+ def source(self):
+ return {'id': 12, 'run_attempt': 1, 'head_sha': self.revisions[0]['sha'],
+ 'base_sha': self.revisions[-1]['sha'], 'name': 'Benchmarks',
+ 'path': '.github/workflows/benchmarks.yml', 'event': 'push', 'head_branch': 'main',
+ 'created_at': '2026-11-01T00:00:00Z', 'status': 'completed', 'conclusion': 'failure',
+ 'repository': {'full_name': 'owner/repo'}}
diff --git a/tools/benchmark_history_test.py b/tools/benchmark_history_test.py
index e2c4c5e426..bcf90174ce 100644
--- a/tools/benchmark_history_test.py
+++ b/tools/benchmark_history_test.py
@@ -248,7 +248,7 @@ def test_publication_coalesces_requests_and_reconciles_sources(self):
self.assertIn('group: benchmark-publication-latest', workflow)
reconcile = repository_file('tools/benchmark_reconcile_ci.py').read_text()
self.assertIn('benchmark_preview_ci.collect', reconcile)
- self.assertIn("('benchmarks.yml', 'push', 'success', retain_measurement)", reconcile)
+ self.assertIn("('benchmarks.yml', 'push', 'completed', retain_benchmarks)", reconcile)
self.assertIn("('benchmark_backfill.yml', 'workflow_dispatch', 'completed', retain_backfill)", reconcile)
def test_fifth_candidate_requires_unresolved_fourth_agreement(self):
diff --git a/tools/benchmark_reconcile_ci.py b/tools/benchmark_reconcile_ci.py
index 80fdad40c8..34a4379d2d 100644
--- a/tools/benchmark_reconcile_ci.py
+++ b/tools/benchmark_reconcile_ci.py
@@ -13,6 +13,7 @@
import zipfile
import benchmark_campaign
+import benchmark_extended_records
import benchmark_history
import benchmark_preview_ci
import benchmark_records
@@ -55,13 +56,35 @@ def latest_artifacts(items):
return list(selected.values())
-def retain_measurement(api, root, run):
+def run_artifacts(api, root, run):
+ """Avoid re-enumerating settled runs on every coalesced publication request."""
+ now = datetime.now(timezone.utc)
+ source = {key: run.get(key) for key in ('id', 'run_attempt', 'head_sha', 'updated_at')}
+ marker = root / 'reconciliation' / str(run['id']) / str(run['run_attempt']) / 'artifacts.json'
+ if marker.exists():
+ saved = json.loads(marker.read_text())
+ if saved['source'] == source and now - datetime.fromisoformat(saved['checked_at']) < timedelta(hours=24):
+ return saved['artifacts']
+ inventory = api.pages(f"actions/runs/{run['id']}/artifacts", 'artifacts')
+ # Retry attempts get a different key. Recent completions are always refreshed
+ # to allow for artifact-index propagation; external deletion/expiry is refreshed daily.
+ if (run.get('status') == 'completed' and run.get('updated_at')
+ and now - datetime.fromisoformat(run['updated_at']) >= timedelta(minutes=15)):
+ marker.parent.mkdir(parents=True, exist_ok=True)
+ temporary = marker.with_suffix('.json.tmp')
+ temporary.write_text(json.dumps({'source': source, 'checked_at': now.isoformat(), 'artifacts': inventory}) + '\n')
+ temporary.replace(marker)
+ return inventory
+
+
+def retain_measurement(api, root, run, inventory=None):
destination = root / 'runs' / str(run['id']) / str(run['run_attempt'])
missing = {platform for platform in ('linux', 'macos')
if not benchmark_records.exists(destination / platform / 'report.json')}
if not missing:
return False
- selected = [item for item in latest_artifacts(artifacts(api, run, 'benchmark-report-'))
+ selected = [item for item in latest_artifacts(artifacts(api, run, 'benchmark-report-') if inventory is None else
+ [item for item in inventory if item['name'].startswith('benchmark-report-')])
if item['created_at'] >= run['run_started_at']]
reports = {}
for item in selected:
@@ -102,6 +125,73 @@ def retain_measurement(api, root, run):
return retained
+def retain_extended(api, root, run, inventory=None):
+ """Recover complete platforms independently, including successful earlier attempts."""
+ selected = latest_artifacts(artifacts(api, run, 'benchmark-extended-report-') if inventory is None else
+ [item for item in inventory if item['name'].startswith('benchmark-extended-report-')])
+ retained = False
+ for item in selected:
+ match = re.fullmatch(r'benchmark-extended-report-(linux|macos)', item['name'])
+ if not match:
+ continue
+ try:
+ retained = retain_extended_artifact(api, root, run, item, match.group(1)) or retained
+ except (ValueError, KeyError, OSError, zipfile.BadZipFile,
+ subprocess.CalledProcessError, subprocess.TimeoutExpired) as error:
+ print(f"::warning::extended run {run['id']} {item['name']}: {error}", flush=True)
+ return retained
+
+
+def retain_extended_artifact(api, root, run, item, platform):
+ collections = root / 'extended' / 'collections' / str(run['id'])
+ markers = {*collections.glob('*/' + platform + '.json'),
+ *(path.with_suffix('') for path in collections.glob('*/' + platform + '.json.gz'))}
+ for marker in markers:
+ saved = benchmark_extended_records.read(marker)
+ observation = root / 'extended' / 'observations' / saved['identity'] / 'report.json'
+ if saved['artifact_id'] == item['id'] and benchmark_records.exists(observation):
+ return False
+ # A rerun of failed jobs does not recreate a successful platform's artifact.
+ # Its observation and source must keep the original attempt identity.
+ with tempfile.TemporaryDirectory(prefix='xff-extended-report-') as temporary:
+ directory = Path(temporary)
+ extract(api.request(f"actions/artifacts/{item['id']}/zip", raw=True), directory)
+ record = json.loads((directory / 'observation.json').read_text())
+ collection = record['collection']
+ if (record['origin'] != 'ci' or record['platform'] != platform
+ or collection['run_id'] != run['id']
+ or type(collection['run_attempt']) is not int
+ or not 1 <= collection['run_attempt'] <= run['run_attempt']):
+ raise ValueError('extended artifact platform/run/attempt mismatch')
+ attempt = collection['run_attempt']
+ source = dict(run if attempt == run['run_attempt'] else
+ api.request(f"actions/runs/{run['id']}/attempts/{attempt}"))
+ commit = api.request(f"commits/{source['head_sha']}")
+ if not commit.get('parents'):
+ raise ValueError('extended source has no first parent')
+ source['base_sha'] = commit['parents'][0]['sha']
+ benchmark_extended_records.retain(root, record, source)
+ marker = collections / str(attempt) / (platform + '.json')
+ marker.parent.mkdir(parents=True, exist_ok=True)
+ marker.write_text(json.dumps({'identity': record['identity'], 'artifact_id': item['id']}) + '\n')
+ return True
+
+
+def retain_benchmarks(api, root, run):
+ """A failed independent job must not discard other valid platform reports."""
+ inventory = [item for item in run_artifacts(api, root, run)
+ if not item.get('expired') and (not item.get('expires_at')
+ or datetime.fromisoformat(item['expires_at']) > datetime.now(timezone.utc))]
+ retained = False
+ for name, retain in (('core', retain_measurement), ('extended', retain_extended)):
+ try:
+ retained = retain(api, root, run, inventory) or retained
+ except (ValueError, KeyError, OSError, zipfile.BadZipFile,
+ subprocess.CalledProcessError, subprocess.TimeoutExpired) as error:
+ print(f"::warning::benchmark run {run['id']} {name}: {error}", flush=True)
+ return retained
+
+
def retain_backfill(api, root, run):
destination = root / 'backfills' / str(run['id']) / str(run['run_attempt'])
if benchmark_records.exists(destination / 'campaign.json'):
@@ -132,7 +222,7 @@ def reconcile(api, root):
since = (datetime.now(timezone.utc) - timedelta(days=30)).date().isoformat()
failures = []
for workflow, event, status, retain in (
- ('benchmarks.yml', 'push', 'success', retain_measurement),
+ ('benchmarks.yml', 'push', 'completed', retain_benchmarks),
('benchmark_backfill.yml', 'workflow_dispatch', 'completed', retain_backfill)):
runs = api.pages(f'actions/workflows/{workflow}/runs?status={status}&branch=main&created=%3E%3D{since}',
'workflow_runs')
diff --git a/tools/benchmark_reconcile_ci_test.py b/tools/benchmark_reconcile_ci_test.py
new file mode 100644
index 0000000000..eb94aad094
--- /dev/null
+++ b/tools/benchmark_reconcile_ci_test.py
@@ -0,0 +1,156 @@
+# SPDX-FileCopyrightText: Copyright (c) M. Boerger, the MBO Works authors
+# SPDX-License-Identifier: Apache-2.0
+"""Recover independently valid reports without trusting workflow or artifact claims."""
+
+import copy
+from datetime import datetime, timezone
+import gzip
+import io
+import json
+import unittest
+from unittest import mock
+import zipfile
+
+import benchmark_extended_records as records
+import benchmark_reconcile_ci as reconcile
+from benchmark_extended_test_support import ExtendedFixture
+
+
+def archive(value):
+ output = io.BytesIO()
+ with zipfile.ZipFile(output, 'w', zipfile.ZIP_DEFLATED) as zipped:
+ zipped.writestr('observation.json', json.dumps(value))
+ return output.getvalue()
+
+
+class FakeAPI:
+ def __init__(self, fixture, value):
+ self.repository = 'owner/repo'
+ self.source = fixture.source()
+ self.commit = {'parents': [{'sha': self.source['base_sha']}],
+ 'commit': {'committer': {'date': '2026-11-01T00:00:00Z'}}}
+ self.items = [{'name': 'benchmark-extended-report-linux', 'id': 101,
+ 'created_at': '2026-11-08T00:00:00Z', 'expired': False}]
+ self.raw = {101: archive(value)}
+ self.requests = []
+
+ def request(self, path, raw=False):
+ self.requests.append(path)
+ if path.startswith('actions/artifacts/'):
+ return self.raw[int(path.split('/')[2])]
+ if path == 'actions/runs/12/attempts/1':
+ return copy.deepcopy(self.source)
+ if path.startswith('commits/'):
+ return copy.deepcopy(self.commit)
+ raise AssertionError(path)
+
+ def pages(self, path, key=None):
+ self.requests.append(path)
+ if path == 'actions/runs/12/artifacts' and key == 'artifacts':
+ return copy.deepcopy(self.items)
+ raise AssertionError(path)
+
+
+class BenchmarkReconcileCiTest(unittest.TestCase):
+ def setUp(self):
+ self.fixture = ExtendedFixture()
+ self.addCleanup(self.fixture.close)
+ self.record = self.fixture.record(origin='ci', comparison='parent')
+ self.api = FakeAPI(self.fixture, self.record)
+ self.root = self.fixture.root / 'site'
+
+ def test_failed_workflow_keeps_valid_extended_head_even_when_core_retention_fails(self):
+ with mock.patch.object(reconcile, 'retain_measurement', side_effect=ValueError('no core agreement')):
+ self.assertTrue(reconcile.retain_benchmarks(self.api, self.root, self.api.source))
+ path = self.root / 'extended/observations' / self.record['identity'] / 'report.json'
+ saved = records.read(path)
+ self.assertEqual(saved['publication_source']['status'], 'completed')
+ self.assertEqual(saved['publication_source']['base_sha'], self.fixture.revisions[-1]['sha'])
+ self.assertEqual(len(list(self.root.glob('extended/observations/*/report.json'))), 1)
+
+ def test_successful_earlier_attempt_is_preserved_and_repeated_import_needs_no_download(self):
+ latest = dict(self.api.source, run_attempt=2)
+ self.assertTrue(reconcile.retain_extended(self.api, self.root, latest))
+ self.assertIn('actions/runs/12/attempts/1', self.api.requests)
+ marker = self.root / 'extended/collections/12/1/linux.json'
+ self.assertTrue(marker.is_file())
+ path = self.root / 'extended/observations' / self.record['identity'] / 'report.json'
+ for file in (marker, path):
+ file.with_suffix('.json.gz').write_bytes(gzip.compress(file.read_bytes(), mtime=0))
+ file.unlink()
+ self.api.requests.clear()
+ self.assertFalse(reconcile.retain_extended(self.api, self.root, latest))
+ self.assertEqual(self.api.requests, ['actions/runs/12/artifacts'])
+ self.assertEqual(records.read(path)['publication_source']['run_attempt'], 1)
+
+ def test_invalid_platform_does_not_discard_other_valid_platform(self):
+ self.api.items.insert(0, dict(self.api.items[0], id=102, name='benchmark-extended-report-macos'))
+ self.api.raw[102] = archive(self.record)
+ self.assertTrue(reconcile.retain_extended(self.api, self.root, self.api.source))
+ self.assertTrue((self.root / 'extended/collections/12/1/linux.json').exists())
+ self.assertFalse((self.root / 'extended/collections/12/1/macos.json').exists())
+
+ def test_wrong_parent_head_repository_attempt_and_untrusted_workflow_are_rejected(self):
+ for key, replacement in (('base_sha', 'f' * 40), ('head_sha', 'f' * 40),
+ ('repository', {'full_name': 'other/repo'}),
+ ('run_attempt', 0), ('event', 'pull_request'), ('status', 'in_progress')):
+ with self.subTest(key=key):
+ source = dict(self.api.source, **{key: replacement})
+ api = FakeAPI(self.fixture, self.record)
+ if key == 'base_sha':
+ api.commit['parents'][0]['sha'] = replacement
+ self.assertFalse(reconcile.retain_extended(api, self.root, source))
+ self.assertFalse(list(self.root.glob('extended/observations/*/report.json')))
+
+ def test_invalid_extended_artifact_does_not_lose_valid_core_result(self):
+ self.api.raw[101] = b'not a zip'
+ with mock.patch.object(reconcile, 'retain_measurement', return_value=True) as core:
+ self.assertTrue(reconcile.retain_benchmarks(self.api, self.root, self.api.source))
+ core.assert_called_once()
+
+ def test_extract_rejects_escape_and_symlinks(self):
+ for name, mode in (('../outside', 0), ('/outside', 0), ('dir\\outside', 0), ('link', 0o120777)):
+ with self.subTest(name=name):
+ output = io.BytesIO()
+ with zipfile.ZipFile(output, 'w') as zipped:
+ info = zipfile.ZipInfo(name)
+ info.external_attr = mode << 16
+ zipped.writestr(info, 'data')
+ with self.assertRaises(ValueError):
+ reconcile.extract(output.getvalue(), self.fixture.root / 'extracted')
+
+ def test_settled_inventory_is_cached_but_retry_and_recent_completion_are_rechecked(self):
+ source = dict(self.api.source, updated_at='2026-11-01T00:00:00Z')
+ self.assertEqual(reconcile.run_artifacts(self.api, self.root, source), self.api.items)
+ self.api.requests.clear()
+ self.assertEqual(reconcile.run_artifacts(self.api, self.root, source), self.api.items)
+ self.assertEqual(self.api.requests, [])
+ reconcile.run_artifacts(self.api, self.root, dict(source, run_attempt=2))
+ self.assertEqual(self.api.requests, ['actions/runs/12/artifacts'])
+ self.api.requests.clear()
+ recent = dict(source, run_attempt=3, updated_at=datetime.now(timezone.utc).isoformat())
+ reconcile.run_artifacts(self.api, self.root, recent)
+ reconcile.run_artifacts(self.api, self.root, recent)
+ self.assertEqual(self.api.requests, ['actions/runs/12/artifacts', 'actions/runs/12/artifacts'])
+
+ def test_expired_cached_artifacts_are_not_downloaded_and_old_cache_is_refreshed(self):
+ source = dict(self.api.source, updated_at='2026-11-01T00:00:00Z')
+ self.api.items[0]['expires_at'] = '2000-01-01T00:00:00Z'
+ reconcile.run_artifacts(self.api, self.root, source)
+ self.api.requests.clear()
+ with mock.patch.object(reconcile, 'retain_measurement', return_value=False) as core, \
+ mock.patch.object(reconcile, 'retain_extended', return_value=False) as extended:
+ self.assertFalse(reconcile.retain_benchmarks(self.api, self.root, source))
+ self.assertEqual(core.call_args.args[-1], [])
+ self.assertEqual(extended.call_args.args[-1], [])
+ self.assertEqual(self.api.requests, [])
+ marker = self.root / 'reconciliation/12/1/artifacts.json'
+ cached = json.loads(marker.read_text())
+ cached['checked_at'] = '2000-01-01T00:00:00Z'
+ marker.write_text(json.dumps(cached))
+ reconcile.run_artifacts(self.api, self.root, source)
+ self.assertEqual(self.api.requests, ['actions/runs/12/artifacts'])
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/tools/benchmark_recovery.py b/tools/benchmark_recovery.py
index ef8e2ab0d0..a88935182f 100644
--- a/tools/benchmark_recovery.py
+++ b/tools/benchmark_recovery.py
@@ -8,15 +8,43 @@
from datetime import date
import io
import json
+from pathlib import Path
import re
import subprocess
+import sys
+import tempfile
import zipfile
+import benchmark_history
import benchmark_preview_ci
PLATFORMS = ('linux', 'macos')
+def report_available(api, run, artifact, platform):
+ # Retrieval failures propagate: an API timeout is not evidence that costly
+ # measurements must be repeated. Only received, invalid data is rejected.
+ raw = api.request(f"actions/artifacts/{artifact['id']}/zip", raw=True)
+ try:
+ if len(raw) > 128 * 1024 * 1024:
+ raise ValueError('benchmark report archive exceeds 128 MiB')
+ with zipfile.ZipFile(io.BytesIO(raw)) as archive:
+ entry = archive.getinfo(artifact['name'] + '.json')
+ if entry.file_size > 128 * 1024 * 1024:
+ raise ValueError('benchmark report exceeds 128 MiB')
+ report = json.loads(archive.read(entry))
+ if report.get('platform') != platform or report.get('head') != run['head_sha'] or not report.get('tool_comparisons'):
+ raise ValueError('report lacks matching head/platform/viewer measurements')
+ # Use the publisher's actual completeness, command and binary-identity
+ # checks, writing only into a disposable local directory.
+ with tempfile.TemporaryDirectory(prefix='xff-recovery-report-') as temporary:
+ benchmark_history.retain(Path(temporary), report, run)
+ return True
+ except (ValueError, KeyError, zipfile.BadZipFile) as error:
+ print(f"::warning::unpublishable run {run['id']} {platform}: {error}", file=sys.stderr, flush=True)
+ return False
+
+
def catalog(api):
metadata = api.request('contents/benchmarks/catalog.json?ref=coverage-pages')
if not metadata.get('content') and metadata.get('sha'):
@@ -33,7 +61,7 @@ def inventory(api, since, number=None):
entries = catalog(api)
covered = set()
for entry in entries:
- if entry.get('source') != 'merged' or entry.get('local'):
+ if entry.get('source') != 'merged' or entry.get('local') or entry.get('extended'):
continue
platform = entry.get('platform_name', '').split(' / ')[0].lower()
sha = entry.get('commit', '')
@@ -104,12 +132,13 @@ def inventory(api, since, number=None):
active = any(run['status'] != 'completed' for run in source_runs)
available = {platform for platform in PLATFORMS if (sha, platform) in pending_backfills}
for run in source_runs:
- if run.get('conclusion') != 'success':
+ if run.get('status') != 'completed':
continue
artifacts = api.pages(f"actions/runs/{run['id']}/artifacts", 'artifacts')
for artifact in artifacts:
match = re.fullmatch(r'benchmark-report-(linux|macos)-[A-Za-z0-9_.-]+', artifact['name'])
- if match and not artifact.get('expired') and artifact['created_at'] >= run['run_started_at']:
+ if (match and not artifact.get('expired') and artifact['created_at'] >= run['run_started_at']
+ and report_available(api, run, artifact, match.group(1))):
available.add(match.group(1))
state = ('measuring' if active else 'awaiting publication' if set(missing) <= available
else 'needs backfill')
@@ -142,8 +171,16 @@ def main():
campaigns = api.pages('actions/workflows/benchmark_backfill.yml/runs?branch=main', 'workflow_runs')
if any(run['status'] != 'completed' for run in campaigns):
raise ValueError('a backfill campaign is already active or queued; retry after it finishes')
- selected = candidates[:args.batch_size]
- platforms = sorted({platform for row in selected for platform in row['missing']})
+ # A cross-product of different missing-platform sets would repeat successful
+ # configurations. Recover the oldest gap and batch only identical requests.
+ def requested(row):
+ return tuple(platform for platform in PLATFORMS if platform in row['missing']
+ and platform not in row.get('available_artifacts', []))
+
+ platforms = requested(candidates[0])
+ if not platforms:
+ raise ValueError('backfill candidate has no unavailable platform')
+ selected = [row for row in candidates if requested(row) == platforms][:args.batch_size]
# The deployed workflow must support the revisions input added by PR #1147.
subprocess.run(['gh', 'workflow', 'run', 'benchmark_backfill.yml', '--repo', api.repository,
'--ref', 'main', '-f', 'revisions=' + ','.join(row['commit'] for row in selected),
diff --git a/tools/benchmark_recovery_test.py b/tools/benchmark_recovery_test.py
index 2434643b7c..58d17530c8 100644
--- a/tools/benchmark_recovery_test.py
+++ b/tools/benchmark_recovery_test.py
@@ -4,6 +4,7 @@
import base64
import io
import json
+import subprocess
import unittest
from unittest import mock
import zipfile
@@ -47,15 +48,38 @@ def test_pr_missing_one_platform_is_reported(self):
api.entries = [{'source': 'merged', 'commit': SHA, 'platform_name': 'linux / x86_64'}]
self.assertEqual(recovery.inventory(api, '2026-09-19')[0]['missing'], ['macos'])
+ def test_extended_views_do_not_hide_missing_core_history(self):
+ api = FakeGitHub()
+ api.entries = [{'source': 'merged', 'commit': SHA, 'platform_name': 'linux / x86_64', 'extended': True},
+ {'source': 'merged', 'commit': SHA, 'platform_name': 'macos / arm64', 'local': True}]
+ self.assertEqual(recovery.inventory(api, '2026-09-19')[0]['missing'], ['linux', 'macos'])
+
def test_active_run_and_available_reports_are_not_backfilled(self):
api = FakeGitHub()
api.runs = [{'id': 1, 'head_sha': SHA, 'event': 'push', 'status': 'in_progress', 'conclusion': None}]
self.assertEqual(recovery.inventory(api, '2026-09-19')[0]['state'], 'measuring')
- api.runs[0].update(status='completed', conclusion='success', run_started_at='2026-10-01T12:00:00Z')
+ api.runs[0].update(status='completed', conclusion='failure', run_started_at='2026-10-01T12:00:00Z')
api.artifacts['actions/runs/1/artifacts'] = [
- {'name': f'benchmark-report-{platform}-arch', 'created_at': '2026-10-01T13:00:00Z', 'expired': False}
- for platform in recovery.PLATFORMS]
- self.assertEqual(recovery.inventory(api, '2026-09-19')[0]['state'], 'awaiting publication')
+ {'id': index, 'name': f'benchmark-report-{platform}-arch',
+ 'created_at': '2026-10-01T13:00:00Z', 'expired': False}
+ for index, platform in enumerate(recovery.PLATFORMS, 10)]
+ for artifact, platform in zip(api.artifacts['actions/runs/1/artifacts'], recovery.PLATFORMS):
+ archive = io.BytesIO()
+ with zipfile.ZipFile(archive, 'w') as output:
+ output.writestr(artifact['name'] + '.json', json.dumps(
+ {'platform': platform, 'head': SHA, 'tool_comparisons': {'synthetic-only': True}}))
+ api.archives[f"actions/artifacts/{artifact['id']}/zip"] = archive.getvalue()
+ with mock.patch.object(recovery.benchmark_history, 'retain') as retain:
+ self.assertEqual(recovery.inventory(api, '2026-09-19')[0]['state'], 'awaiting publication')
+ self.assertEqual(retain.call_count, 2)
+ with mock.patch.object(recovery.benchmark_history, 'retain', side_effect=ValueError('incomplete report')):
+ self.assertEqual(recovery.inventory(api, '2026-09-19')[0]['state'], 'needs backfill')
+
+ def test_api_observation_timeout_never_becomes_a_request_to_repeat_measurements(self):
+ api = FakeGitHub()
+ with mock.patch.object(api, 'request', side_effect=subprocess.TimeoutExpired('gh', 120)), \
+ self.assertRaises(subprocess.TimeoutExpired):
+ recovery.report_available(api, {'id': 1, 'head_sha': SHA}, {'id': 2}, 'linux')
def test_retained_completed_campaign_is_not_dispatched_again(self):
api = FakeGitHub()
@@ -100,6 +124,22 @@ def test_dispatch_uses_the_targeted_main_workflow(self):
'owner/repo', '--ref', 'main', '-f', 'revisions=' + SHA,
'-f', 'platforms=linux'], check=True)
+ def test_dispatch_batches_equal_gaps_without_repeating_successful_or_available_platforms(self):
+ api = FakeGitHub()
+ rows = [{'state': 'needs backfill', 'pr': 7, 'commit': SHA, 'missing': ['linux', 'macos'],
+ 'available_artifacts': ['macos']},
+ {'state': 'needs backfill', 'pr': 8, 'commit': 'c' * 40, 'missing': ['linux', 'macos']},
+ {'state': 'needs backfill', 'pr': 9, 'commit': OTHER, 'missing': ['linux']}]
+ with mock.patch.object(recovery.benchmark_preview_ci, 'GitHub', return_value=api), \
+ mock.patch.object(recovery, 'inventory', return_value=rows), \
+ mock.patch.object(api, 'pages', return_value=[]), \
+ mock.patch.object(recovery.subprocess, 'run') as run, \
+ mock.patch('sys.argv', ['benchmark_recovery', '--repository', 'owner/repo', '--dispatch-next']):
+ recovery.main()
+ run.assert_called_once_with(['gh', 'workflow', 'run', 'benchmark_backfill.yml', '--repo',
+ 'owner/repo', '--ref', 'main', '-f', 'revisions=' + SHA + ',' + OTHER,
+ '-f', 'platforms=linux'], check=True)
+
if __name__ == '__main__':
unittest.main()