diff --git a/docs/benchmark-merged-pr-coverage.md b/docs/benchmark-merged-pr-coverage.md new file mode 100644 index 0000000000..dc360240c0 --- /dev/null +++ b/docs/benchmark-merged-pr-coverage.md @@ -0,0 +1,220 @@ +# Missing merged PR benchmark inventory + +Snapshot: November 8, 2026, using the published benchmark catalog, retained run tree, +GitHub merged PR metadata, and main benchmark workflow metadata captured during the audit. +Entries can become stale as publishers and workflows complete. This inventories PRs merged +on or after September 19, 2026; it is not an audit of earlier history. + +A PR is present when its merge commit appears in the merged CI viewer catalog. A catalog +entry may still be hard to find by PR number: existing post-merge entries use `main / ` +as the version label, with the PR only in the links. Local data is excluded from this count. + +The snapshot has 252 merged PRs: 157 represented and 95 absent. +The table records observed workflow conclusions, not inferred causes. A canceled run may +have been superseded or canceled for another reason. A failed run requires log investigation. +Older successful reports can exist without the viewer data required by the current catalog. + +| PR | Merged (UTC) | Merge commit | Observed reason or status | +| -------------------------------------------------- | ------------ | -------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------- | +| [#1145](https://github.com/mboworks/xff/pull/1145) | 2026-11-08 | `57a3cfb50b02` | Workflow pending ([run](https://github.com/mboworks/xff/actions/runs/42490418192)) | +| [#1146](https://github.com/mboworks/xff/pull/1146) | 2026-11-08 | `31dc55b400f3` | Workflow in_progress ([run](https://github.com/mboworks/xff/actions/runs/42486013719)) | +| [#1136](https://github.com/mboworks/xff/pull/1136) | 2026-11-07 | `1f25ba6d4ae0` | Measurement workflow canceled; cause not established ([run](https://github.com/mboworks/xff/actions/runs/42398990564)) | +| [#1132](https://github.com/mboworks/xff/pull/1132) | 2026-11-07 | `5d413a35cbec` | Measurement workflow canceled; cause not established ([run](https://github.com/mboworks/xff/actions/runs/42378502027)) | +| [#1130](https://github.com/mboworks/xff/pull/1130) | 2026-11-07 | `ae8c5af3eb11` | Measurement workflow canceled; cause not established ([run](https://github.com/mboworks/xff/actions/runs/42367122125)) | +| [#1128](https://github.com/mboworks/xff/pull/1128) | 2026-11-07 | `1278c8943a6a` | Measurement workflow canceled; cause not established ([run](https://github.com/mboworks/xff/actions/runs/42354657340)) | +| [#1121](https://github.com/mboworks/xff/pull/1121) | 2026-11-06 | `d9bf84b04784` | Measurement workflow canceled; cause not established ([run](https://github.com/mboworks/xff/actions/runs/42265512655)) | +| [#1118](https://github.com/mboworks/xff/pull/1118) | 2026-11-06 | `c7bb70cc6f3e` | Measurement workflow canceled; cause not established ([run](https://github.com/mboworks/xff/actions/runs/42234848791)) | +| [#1115](https://github.com/mboworks/xff/pull/1115) | 2026-11-06 | `ab909839da7d` | Measurement workflow canceled; cause not established ([run](https://github.com/mboworks/xff/actions/runs/42218900004)) | +| [#1113](https://github.com/mboworks/xff/pull/1113) | 2026-11-06 | `5729026a8482` | Measurement workflow canceled; cause not established ([run](https://github.com/mboworks/xff/actions/runs/42200480427)) | +| [#1111](https://github.com/mboworks/xff/pull/1111) | 2026-11-06 | `5cade08b6701` | Measurement workflow canceled; cause not established ([run](https://github.com/mboworks/xff/actions/runs/42184472882)) | +| [#1106](https://github.com/mboworks/xff/pull/1106) | 2026-11-06 | `e7e42d01e067` | Measurement workflow canceled; cause not established ([run](https://github.com/mboworks/xff/actions/runs/42146475737)) | +| [#1099](https://github.com/mboworks/xff/pull/1099) | 2026-11-05 | `76185cb37fa2` | Measurement workflow failed; exact cause not audited ([run](https://github.com/mboworks/xff/actions/runs/41999670147)) | +| [#1098](https://github.com/mboworks/xff/pull/1098) | 2026-11-05 | `7c8cd64e02a8` | Measurement workflow failed; exact cause not audited ([run](https://github.com/mboworks/xff/actions/runs/41986760505)) | +| [#1097](https://github.com/mboworks/xff/pull/1097) | 2026-11-05 | `910e2a7bbae5` | Measurement workflow canceled; cause not established ([run](https://github.com/mboworks/xff/actions/runs/41977457900)) | +| [#1095](https://github.com/mboworks/xff/pull/1095) | 2026-11-05 | `6792b5030de7` | Measurement workflow canceled; cause not established ([run](https://github.com/mboworks/xff/actions/runs/41962227281)) | +| [#1092](https://github.com/mboworks/xff/pull/1092) | 2026-11-04 | `a89459ed819a` | Measurement workflow canceled; cause not established ([run](https://github.com/mboworks/xff/actions/runs/41920310179)) | +| [#1088](https://github.com/mboworks/xff/pull/1088) | 2026-11-04 | `9bd70fc9b460` | Measurement workflow failed; exact cause not audited ([run](https://github.com/mboworks/xff/actions/runs/41851695947)) | +| [#1084](https://github.com/mboworks/xff/pull/1084) | 2026-11-04 | `f06cea15f5b1` | Measurement workflow canceled; cause not established ([run](https://github.com/mboworks/xff/actions/runs/41798644066)) | +| [#1082](https://github.com/mboworks/xff/pull/1082) | 2026-11-04 | `6e6b25ffc869` | Measurement workflow failed; exact cause not audited ([run](https://github.com/mboworks/xff/actions/runs/41783344161)) | +| [#1080](https://github.com/mboworks/xff/pull/1080) | 2026-11-04 | `669f68a3e263` | Measurement workflow failed; exact cause not audited ([run](https://github.com/mboworks/xff/actions/runs/41771864734)) | +| [#1078](https://github.com/mboworks/xff/pull/1078) | 2026-11-03 | `76054b3a9c60` | Measurement workflow failed; exact cause not audited ([run](https://github.com/mboworks/xff/actions/runs/41711975639)) | +| [#1073](https://github.com/mboworks/xff/pull/1073) | 2026-11-03 | `6db2de9e49f1` | Measurement workflow canceled; cause not established ([run](https://github.com/mboworks/xff/actions/runs/41628320126)) | +| [#1067](https://github.com/mboworks/xff/pull/1067) | 2026-11-03 | `fb726c946063` | Measurement workflow canceled; cause not established ([run](https://github.com/mboworks/xff/actions/runs/41573418793)) | +| [#1059](https://github.com/mboworks/xff/pull/1059) | 2026-11-03 | `22702c34a0bc` | Measurement workflow canceled; cause not established ([run](https://github.com/mboworks/xff/actions/runs/41559497818)) | +| [#1057](https://github.com/mboworks/xff/pull/1057) | 2026-11-02 | `6bebe940ba36` | Measurement workflow canceled; cause not established ([run](https://github.com/mboworks/xff/actions/runs/41524744291)) | +| [#1053](https://github.com/mboworks/xff/pull/1053) | 2026-11-02 | `dc4e70237eff` | Measurement workflow canceled; cause not established ([run](https://github.com/mboworks/xff/actions/runs/41451327523)) | +| [#1050](https://github.com/mboworks/xff/pull/1050) | 2026-11-02 | `2d615ea1e447` | Measurement workflow failed; exact cause not audited ([run](https://github.com/mboworks/xff/actions/runs/41425575717)) | +| [#1044](https://github.com/mboworks/xff/pull/1044) | 2026-11-02 | `d546fcb3d864` | Measurement workflow canceled; cause not established ([run](https://github.com/mboworks/xff/actions/runs/41378115229)) | +| [#1042](https://github.com/mboworks/xff/pull/1042) | 2026-11-02 | `2b21790502fd` | Measurement workflow canceled; cause not established ([run](https://github.com/mboworks/xff/actions/runs/41367091220)) | +| [#1027](https://github.com/mboworks/xff/pull/1027) | 2026-10-29 | `d406f4132d06` | Measurement workflow canceled; cause not established ([run](https://github.com/mboworks/xff/actions/runs/40816218296)) | +| [#1011](https://github.com/mboworks/xff/pull/1011) | 2026-10-23 | `801087f3e9a8` | Measurement workflow failed; exact cause not audited ([run](https://github.com/mboworks/xff/actions/runs/39886813900)) | +| [#1007](https://github.com/mboworks/xff/pull/1007) | 2026-10-14 | `33a01e13d7cd` | Measurement workflow canceled; cause not established ([run](https://github.com/mboworks/xff/actions/runs/38653097649)) | +| [#1006](https://github.com/mboworks/xff/pull/1006) | 2026-10-14 | `20095dd53c31` | Measurement workflow failed; exact cause not audited ([run](https://github.com/mboworks/xff/actions/runs/38644322359)) | +| [#994](https://github.com/mboworks/xff/pull/994) | 2026-10-10 | `88d4facd15eb` | Measurement workflow failed; exact cause not audited ([run](https://github.com/mboworks/xff/actions/runs/38049489783)) | +| [#993](https://github.com/mboworks/xff/pull/993) | 2026-10-10 | `dac3747d63ca` | Measurement workflow canceled; cause not established ([run](https://github.com/mboworks/xff/actions/runs/38045263735)) | +| [#986](https://github.com/mboworks/xff/pull/986) | 2026-10-07 | `61ab1ce41300` | Successful run absent from site; artifacts unavailable ([run](https://github.com/mboworks/xff/actions/runs/37689818933)) | +| [#976](https://github.com/mboworks/xff/pull/976) | 2026-10-06 | `73b2cfcd7101` | Measurement workflow canceled; cause not established ([run](https://github.com/mboworks/xff/actions/runs/37393233041)) | +| [#979](https://github.com/mboworks/xff/pull/979) | 2026-10-05 | `378d3540691e` | Measurement workflow canceled; cause not established ([run](https://github.com/mboworks/xff/actions/runs/37388497650)) | +| [#973](https://github.com/mboworks/xff/pull/973) | 2026-10-05 | `dfd5447644f0` | Measurement workflow canceled; cause not established ([run](https://github.com/mboworks/xff/actions/runs/37388437775)) | +| [#954](https://github.com/mboworks/xff/pull/954) | 2026-10-03 | `2817247f59cb` | Measurement workflow failed; exact cause not audited ([run](https://github.com/mboworks/xff/actions/runs/37138060568)) | +| [#950](https://github.com/mboworks/xff/pull/950) | 2026-10-03 | `c262c0a71360` | Measurement workflow failed; exact cause not audited ([run](https://github.com/mboworks/xff/actions/runs/37114542638)) | +| [#940](https://github.com/mboworks/xff/pull/940) | 2026-10-01 | `d2cd1e882428` | Measurement workflow failed; exact cause not audited ([run](https://github.com/mboworks/xff/actions/runs/36926663737)) | +| [#939](https://github.com/mboworks/xff/pull/939) | 2026-10-01 | `cc01d2edbab6` | No main benchmark workflow found | +| [#935](https://github.com/mboworks/xff/pull/935) | 2026-10-01 | `481c32aaf2bd` | Measurement workflow failed; exact cause not audited ([run](https://github.com/mboworks/xff/actions/runs/36831896372)) | +| [#937](https://github.com/mboworks/xff/pull/937) | 2026-10-01 | `c8a5fe9e22c3` | Measurement workflow failed; exact cause not audited ([run](https://github.com/mboworks/xff/actions/runs/36826183983)) | +| [#936](https://github.com/mboworks/xff/pull/936) | 2026-10-01 | `be954012e048` | No main benchmark workflow found | +| [#929](https://github.com/mboworks/xff/pull/929) | 2026-09-27 | `471284c4d8a5` | No main benchmark workflow found | +| [#928](https://github.com/mboworks/xff/pull/928) | 2026-09-27 | `f0dfc3edefc7` | No main benchmark workflow found | +| [#922](https://github.com/mboworks/xff/pull/922) | 2026-09-27 | `5db1240146be` | No main benchmark workflow found | +| [#921](https://github.com/mboworks/xff/pull/921) | 2026-09-27 | `df460ae32bc2` | No main benchmark workflow found | +| [#920](https://github.com/mboworks/xff/pull/920) | 2026-09-27 | `5a327394fbbb` | No main benchmark workflow found | +| [#919](https://github.com/mboworks/xff/pull/919) | 2026-09-27 | `7c8164de5b06` | No main benchmark workflow found | +| [#918](https://github.com/mboworks/xff/pull/918) | 2026-09-27 | `97c5e68ac92d` | No main benchmark workflow found | +| [#917](https://github.com/mboworks/xff/pull/917) | 2026-09-27 | `4ef35c0305de` | No main benchmark workflow found | +| [#916](https://github.com/mboworks/xff/pull/916) | 2026-09-27 | `ce5ee3c40579` | No main benchmark workflow found | +| [#914](https://github.com/mboworks/xff/pull/914) | 2026-09-24 | `9bbee34da11e` | Successful run absent from site; artifacts unavailable ([run](https://github.com/mboworks/xff/actions/runs/36068976975)) | +| [#902](https://github.com/mboworks/xff/pull/902) | 2026-09-22 | `be2f400392dc` | Retained historical report lacks a viewer catalog entry; format/eligibility not audited ([run](https://github.com/mboworks/xff/actions/runs/35677724823)) | +| [#904](https://github.com/mboworks/xff/pull/904) | 2026-09-21 | `80d75576c32b` | Retained historical report lacks a viewer catalog entry; format/eligibility not audited ([run](https://github.com/mboworks/xff/actions/runs/35667805574)) | +| [#905](https://github.com/mboworks/xff/pull/905) | 2026-09-21 | `77c00ec8cbea` | Retained historical report lacks a viewer catalog entry; format/eligibility not audited ([run](https://github.com/mboworks/xff/actions/runs/35666152368)) | +| [#903](https://github.com/mboworks/xff/pull/903) | 2026-09-21 | `9b04999acb79` | Retained historical report lacks a viewer catalog entry; format/eligibility not audited ([run](https://github.com/mboworks/xff/actions/runs/35664420733)) | +| [#906](https://github.com/mboworks/xff/pull/906) | 2026-09-21 | `79278f51e63c` | Retained historical report lacks a viewer catalog entry; format/eligibility not audited ([run](https://github.com/mboworks/xff/actions/runs/35662475460)) | +| [#901](https://github.com/mboworks/xff/pull/901) | 2026-09-21 | `f76015ff96f8` | Retained historical report lacks a viewer catalog entry; format/eligibility not audited ([run](https://github.com/mboworks/xff/actions/runs/35653591054)) | +| [#899](https://github.com/mboworks/xff/pull/899) | 2026-09-21 | `98d31e1616a7` | Retained historical report lacks a viewer catalog entry; format/eligibility not audited ([run](https://github.com/mboworks/xff/actions/runs/35564630217)) | +| [#898](https://github.com/mboworks/xff/pull/898) | 2026-09-20 | `c0c0b6d39e26` | Retained historical report lacks a viewer catalog entry; format/eligibility not audited ([run](https://github.com/mboworks/xff/actions/runs/35538423416)) | +| [#897](https://github.com/mboworks/xff/pull/897) | 2026-09-20 | `6e882aea6d2c` | Retained historical report lacks a viewer catalog entry; format/eligibility not audited ([run](https://github.com/mboworks/xff/actions/runs/35536193393)) | +| [#896](https://github.com/mboworks/xff/pull/896) | 2026-09-20 | `ced00ac473e6` | Retained historical report lacks a viewer catalog entry; format/eligibility not audited ([run](https://github.com/mboworks/xff/actions/runs/35529346711)) | +| [#895](https://github.com/mboworks/xff/pull/895) | 2026-09-20 | `a185ae12d70d` | Retained historical report lacks a viewer catalog entry; format/eligibility not audited ([run](https://github.com/mboworks/xff/actions/runs/35521619198)) | +| [#894](https://github.com/mboworks/xff/pull/894) | 2026-09-20 | `42febc0e8d0a` | Retained historical report lacks a viewer catalog entry; format/eligibility not audited ([run](https://github.com/mboworks/xff/actions/runs/35510379808)) | +| [#893](https://github.com/mboworks/xff/pull/893) | 2026-09-20 | `29c75be40529` | Retained historical report lacks a viewer catalog entry; format/eligibility not audited ([run](https://github.com/mboworks/xff/actions/runs/35503333615)) | +| [#892](https://github.com/mboworks/xff/pull/892) | 2026-09-20 | `c9b56c2f009e` | Retained historical report lacks a viewer catalog entry; format/eligibility not audited ([run](https://github.com/mboworks/xff/actions/runs/35502743136)) | +| [#891](https://github.com/mboworks/xff/pull/891) | 2026-09-20 | `a39758eff8ac` | Retained historical report lacks a viewer catalog entry; format/eligibility not audited ([run](https://github.com/mboworks/xff/actions/runs/35499899635)) | +| [#890](https://github.com/mboworks/xff/pull/890) | 2026-09-19 | `1f568ceecd28` | Retained historical report lacks a viewer catalog entry; format/eligibility not audited ([run](https://github.com/mboworks/xff/actions/runs/35474269925)) | +| [#888](https://github.com/mboworks/xff/pull/888) | 2026-09-19 | `b7d28411a9cc` | Retained historical report lacks a viewer catalog entry; format/eligibility not audited ([run](https://github.com/mboworks/xff/actions/runs/35473674929)) | +| [#889](https://github.com/mboworks/xff/pull/889) | 2026-09-19 | `1945c9ec032f` | No main benchmark workflow found | +| [#887](https://github.com/mboworks/xff/pull/887) | 2026-09-19 | `0bcb461dbb08` | Retained historical report lacks a viewer catalog entry; format/eligibility not audited ([run](https://github.com/mboworks/xff/actions/runs/35470455554)) | +| [#886](https://github.com/mboworks/xff/pull/886) | 2026-09-19 | `2bd76d6e682f` | No main benchmark workflow found | +| [#885](https://github.com/mboworks/xff/pull/885) | 2026-09-19 | `2448257a8d4d` | No main benchmark workflow found | +| [#884](https://github.com/mboworks/xff/pull/884) | 2026-09-19 | `4fc594246067` | No main benchmark workflow found | +| [#883](https://github.com/mboworks/xff/pull/883) | 2026-09-19 | `b1df434503a3` | No main benchmark workflow found | +| [#882](https://github.com/mboworks/xff/pull/882) | 2026-09-19 | `9282e3d1da7c` | No main benchmark workflow found | +| [#881](https://github.com/mboworks/xff/pull/881) | 2026-09-19 | `990045bf2d3b` | No main benchmark workflow found | +| [#880](https://github.com/mboworks/xff/pull/880) | 2026-09-19 | `c8b35dfa8249` | No main benchmark workflow found | +| [#879](https://github.com/mboworks/xff/pull/879) | 2026-09-19 | `7f48d0d888cb` | No main benchmark workflow found | +| [#878](https://github.com/mboworks/xff/pull/878) | 2026-09-19 | `889c652ec8cf` | No main benchmark workflow found | +| [#877](https://github.com/mboworks/xff/pull/877) | 2026-09-19 | `99b78c8f1bad` | No main benchmark workflow found | +| [#876](https://github.com/mboworks/xff/pull/876) | 2026-09-19 | `e1cc1f72605e` | No main benchmark workflow found | +| [#868](https://github.com/mboworks/xff/pull/868) | 2026-09-19 | `c092e8245ff8` | No main benchmark workflow found | +| [#875](https://github.com/mboworks/xff/pull/875) | 2026-09-19 | `9197abc259ba` | No main benchmark workflow found | +| [#874](https://github.com/mboworks/xff/pull/874) | 2026-09-19 | `a61f320479c9` | No main benchmark workflow found | +| [#873](https://github.com/mboworks/xff/pull/873) | 2026-09-19 | `13ba8e9474be` | No main benchmark workflow found | +| [#872](https://github.com/mboworks/xff/pull/872) | 2026-09-19 | `ecc9b1f7868a` | No main benchmark workflow found | +| [#871](https://github.com/mboworks/xff/pull/871) | 2026-09-19 | `05140bb49bf5` | No main benchmark workflow found | +| [#870](https://github.com/mboworks/xff/pull/870) | 2026-09-19 | `c7aa7ebea9b4` | No main benchmark workflow found | +| [#869](https://github.com/mboworks/xff/pull/869) | 2026-09-19 | `41e18f2748ff` | No main benchmark workflow found | + +## What the audit shows + +- PR #1143 is represented on Linux and macOS; it is labeled by main and commit, not by PR number. +- The publisher reconciles available successful artifacts, but cannot recreate a run that + never produced a final report or whose artifact expired. +- Main benchmark runs previously used a single pending concurrency slot; #1138 changed + the workflow to a 100-entry queue. That does not by itself retry failed or canceled runs, + and an exhausted queue can still lose requests. Individual cancellations were not audited. +- At the time of the audit the backfill planner selected commits already present in retained + history, and every job had to succeed before any result became visible. Neither behavior + provides a durable missing-PR queue. +- Merged PR #1147 provides explicit revisions for backfill, but that remains manual and does not + establish automatic completeness. + +## Proposed prevention + +1. Maintain a durable inventory keyed by merged PR number, merge commit, and platform. + Discover expected entries from trusted merged PR metadata, independently of workflow + events and retained reports. Store merge and tree identity, discovery time, state, + attempt identities, errors and retry time. Reconcile this inventory periodically. +2. For each platform, select a compatible retained measurement whose measured commit or + verified tree matches the merged PR. Accept PR measurements only after proving the + measured tree matches the merge result; otherwise schedule a new measurement. Keep + actual measurement and source identities and show backfill provenance in the viewer. +3. Expose a PR-labeled entry in the viewer for every merge. Show pending, running, failed + and retrying states instead of silently omitting unmeasured PRs. Do not plot a point + without a completed, validated measurement. A PR may be visible while coverage is + incomplete on one or both platforms. +4. Reconcile completed artifacts independently per PR and platform. Retry missing, + failed, canceled, expired and queue-rejected work with bounded concurrency and + backoff. Make retries idempotent and retain successful platforms even if another fails. + No workflow trigger or finite GitHub queue may be the only record of required work. +5. Limit remeasurement cost by reusing a trusted, compatible existing measurement when + it proves the same merge tree and benchmark contract; otherwise run one targeted + job for each missing platform. Do not equate different trees or measurement contracts. +6. Publish catalog and status from the durable inventory under the shared site lock, + coalesce redundant publishing requests, and report backlog age, failures, artifact + expiry and stalled progress. After a successful publisher, confirm every eligible + retained result is indexed; retry publication independently of measurement. + +## Recovery command + +After PR #1147 has deployed the targeted revision input, run: + +```sh +bazel run //tools:benchmark_recovery -- --repository mboworks/xff +``` + +The command compares merged PRs with the current CI viewer catalog and reports missing +platforms, active main measurements and retained artifacts awaiting publication. The +default starting date is September 19, 2026; use `--since=YYYY-MM-DD` to select another +inclusive date, or `--pr=986` to inspect one PR. It reads GitHub through the authenticated +`gh` CLI. An available artifact still needs validation and successful publication. + +Add `--dispatch-next` to dispatch up to four of the oldest PRs requiring new measurements. +Use `--batch-size=1` through `--batch-size=8` to change the bound. It starts one targeted +backfill campaign for both platforms and refuses if another backfill is active or queued. +Run the inventory again after that campaign and its publisher finish; +repeat until the desired range is covered. This is an operator-driven recovery step, +not the persistent automatic scheduler described above. A concurrent dispatch after +the active-campaign check can still race; avoid starting multiple operators at once. + +The publisher can retain each complete validated platform from a failed campaign, so a +failed platform does not discard another platform's results. A targeted retry currently +remeasures both platforms. The command does not cancel a pending publisher or +automatically retry a failed run. + +## Decisions and limits + +- Decide the supported historical starting point and platforms before scheduling a + complete backfill. The table includes older PRs whose original comparison contract or + reference tools may differ from the current contract. +- A successful backfill is a new measurement with its own host and method, not a + reconstruction of the original run. Preserve that distinction and do not mix incompatible + series silently. +- The implementation still needs a persistent scheduler; + merely increasing the concurrency queue, adding retries to one workflow, or adding + PR labels cannot guarantee eventual coverage. + +## Continuation after the current campaigns + +The main performance objective is to close the gap to equivalent plain `find` and `rg` +commands. Coverage recovery supports that work by restoring a trustworthy history. + +First let the [#914/#986 campaign](https://github.com/mboworks/xff/actions/runs/42498912253) +and the queued [#1143 campaign](https://github.com/mboworks/xff/actions/runs/42499244750) +finish, and merge PR #1148. Then: + +1. Inspect campaign results and publication, including failures and partial platforms. + Confirm each recovered PR is selectable by PR number in the viewer; workflow success + or a retained report directory alone does not establish that. +2. Recompute this inventory from the live catalog. Recover the remaining missing PRs in + bounded campaigns, preserving existing valid measurements and documenting failures + that need build or measurement fixes. Do not dispatch another campaign while one is + active or queued. +3. Resume concrete optimization of basic discovery and content search, comparing equivalent + output and semantics with `find` and `rg` on Linux and macOS. Use production binaries, + verify their identities, and keep paired measurements with reference tools. The local + investigation requests the full 1/3/10 worker grid; the existing CI backfill contract + remains 1/3. Keep those contracts distinguishable. +4. Develop a few representative combined workloads alongside the simple comparisons. + Select them for real use and matching semantics, rather than for a favorable result. + Establish correctness, repeatability, and sensitivity before adding them to the viewer. +5. Freeze and identify the additional workload contract before scheduling its historical + backfill. Preserve the existing simple comparisons and make compatibility explicit; + a new task must not silently appear as if it had been measured in earlier reports. diff --git a/tools/BUILD.bazel b/tools/BUILD.bazel index 29c83b1187..9f7740d850 100644 --- a/tools/BUILD.bazel +++ b/tools/BUILD.bazel @@ -1382,6 +1382,26 @@ py_binary( ], ) +py_library( + name = "benchmark_recovery_lib", + srcs = ["benchmark_recovery.py"], + imports = ["."], + deps = [":benchmark_preview_ci_lib"], +) + +py_binary( + name = "benchmark_recovery", + srcs = ["benchmark_recovery.py"], + deps = [":benchmark_preview_ci_lib"], +) + +py_test( + name = "benchmark_recovery_test", + size = "small", + srcs = ["benchmark_recovery_test.py"], + deps = [":benchmark_recovery_lib"], +) + py_test( name = "benchmark_preview_ci_test", size = "small", diff --git a/tools/benchmark_campaign.py b/tools/benchmark_campaign.py index a52324f3dc..a186264b3d 100644 --- a/tools/benchmark_campaign.py +++ b/tools/benchmark_campaign.py @@ -170,11 +170,12 @@ def validate_measurements(record, batch, revision, tasks): compare.render(report) # Validate all samples, successful commands and finite measurements. -def retain(root, artifacts, source, repository): - """Validate the entire campaign before making any replacement visible.""" +def retain(root, artifacts, source, repository, allow_partial=False): + """Retain validated campaign records; partial retention is explicitly opt-in.""" require(source['name'] == 'Benchmark backfill' and source['path'] == '.github/workflows/benchmark_backfill.yml' and source['event'] == 'workflow_dispatch' and source['head_branch'] == 'main' - and source['status'] == 'completed' and source['conclusion'] == 'success' + and source['status'] == 'completed' + and (source['conclusion'] == 'success' or allow_partial) and source['repository']['full_name'] == repository, 'expected a successful trusted main backfill workflow') run, attempt = int(source['id']), int(source['run_attempt']) require(run > 0 and attempt > 0, 'invalid source workflow identity') @@ -182,7 +183,9 @@ def retain(root, artifacts, source, repository): validate_plan(plan) require(plan['source_head'] == source['head_sha'], 'campaign source commit mismatch') expected_artifacts = {'benchmark-backfill-plan', *('benchmark-backfill-' + job['id'] for job in plan['jobs'])} - require({path.name for path in artifacts.iterdir()} == expected_artifacts, 'incomplete or unexpected campaign artifacts') + actual_artifacts = {path.name for path in artifacts.iterdir()} + require(actual_artifacts <= expected_artifacts and (allow_partial or actual_artifacts == expected_artifacts), + 'incomplete or unexpected campaign artifacts') trusted = {key: source[key] for key in ('id', 'run_attempt', 'created_at', 'head_sha', 'head_branch', 'event')} trusted['pull_requests'] = [] destination = root / 'backfills' / str(run) / str(attempt) @@ -192,24 +195,30 @@ def retain(root, artifacts, source, repository): originals.setdefault((benchmark_records.platform_key(old), old['head']), []).append(path.relative_to(root).as_posix()) records = [] for job in plan['jobs']: - folder = artifacts / ('benchmark-backfill-' + job['id']) - batch = json.loads((folder / 'batch.json').read_text()) - status = json.loads((folder / 'status.json').read_text()) - sha = job['revision']['sha'] - require(set(status) == {sha} and status[sha]['status'] == 'complete', 'incomplete campaign revision') - reports = list(folder.glob('reports/*/*.json')) - expected = folder / 'reports' / sha / f"benchmark-report-{job['platform']}-{job['arch']}.json" - require(reports == [expected], 'unexpected campaign report set') - record = json.loads(expected.read_text()) - validate_report(record, batch, job, plan) - collection = batch['contract']['collection'] - require(collection['id'] == run and 1 <= collection['run_attempt'] <= attempt - and collection['head_sha'] == source['head_sha'], 'batch source workflow mismatch') - record.update(source=trusted, collection=collection, campaign=plan['identity']) - saved = destination / job['platform'] / sha / 'report.json' - record['original_reports'] = (benchmark_records.read(saved)['original_reports'] if benchmark_records.exists(saved) - else sorted(originals.get((job['platform'], sha), []))) - records.append((job, record, batch, status)) + try: + folder = artifacts / ('benchmark-backfill-' + job['id']) + batch = json.loads((folder / 'batch.json').read_text()) + status = json.loads((folder / 'status.json').read_text()) + sha = job['revision']['sha'] + require(set(status) == {sha} and status[sha]['status'] == 'complete', 'incomplete campaign revision') + reports = list(folder.glob('reports/*/*.json')) + expected = folder / 'reports' / sha / f"benchmark-report-{job['platform']}-{job['arch']}.json" + require(reports == [expected], 'unexpected campaign report set') + record = json.loads(expected.read_text()) + validate_report(record, batch, job, plan) + collection = batch['contract']['collection'] + require(collection['id'] == run and 1 <= collection['run_attempt'] <= attempt + and collection['head_sha'] == source['head_sha'], 'batch source workflow mismatch') + record.update(source=trusted, collection=collection, campaign=plan['identity']) + saved = destination / job['platform'] / sha / 'report.json' + record['original_reports'] = (benchmark_records.read(saved)['original_reports'] if benchmark_records.exists(saved) + else sorted(originals.get((job['platform'], sha), []))) + records.append((job, record, batch, status)) + except (ValueError, KeyError, OSError) as error: + if not allow_partial: + raise + print(f"::warning::backfill {run}/{attempt} {job['id']}: {error}", flush=True) + require(records, 'campaign has no complete validated measurements') payloads = {'campaign.json': plan, 'source.json': trusted} for job, record, batch, status in records: prefix = job['platform'] + '/' + job['revision']['sha'] + '/' diff --git a/tools/benchmark_campaign_test.py b/tools/benchmark_campaign_test.py index 34446ce286..d52bbe0bea 100644 --- a/tools/benchmark_campaign_test.py +++ b/tools/benchmark_campaign_test.py @@ -137,6 +137,15 @@ def test_planner_freezes_each_complete_revision_once_per_platform(self): self.assertIn('content-needle', selected['contract']['tasks']) campaign.validate_plan(selected) + def test_planner_uses_explicit_revisions_instead_of_published_history(self): + revision = plan()['jobs'][0]['revision'] + with mock.patch.object(campaign.backfill, 'revisions', return_value=[revision]) as revisions, \ + mock.patch.object(campaign.backfill, 'git', side_effect=[DRIVER, 'c' * 40]): + selected = campaign.make_plan(Path('/repo'), Path('/history'), [HEAD]) + self.assertEqual(len(selected['jobs']), 2) + self.assertEqual([call.args[1:] for call in revisions.call_args_list], + [([HEAD], None, 'linux'), ([HEAD], None, 'macos')]) + def test_plan_rejects_mutation_duplicates_partial_grid_and_missing_platform(self): mutations = [lambda value: value['jobs'].append(value['jobs'][0]), lambda value: value['jobs'].pop(), @@ -233,6 +242,19 @@ def test_missing_artifact_or_failed_revision_never_promotes_a_partial_campaign(s campaign.retain(site, artifacts, source(), 'owner/repo') self.assertFalse(site.exists()) + def test_partial_recovery_retains_only_valid_completed_platforms(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + artifacts = root / 'artifacts' + write_artifacts(artifacts) + folder = next(artifacts.glob('benchmark-backfill-macos-*')) + (folder / 'status.json').write_text(json.dumps({HEAD: {'status': 'measurement-failed'}})) + failed = dict(source(), conclusion='failure') + destination = campaign.retain(root / 'site', artifacts, failed, 'owner/repo', allow_partial=True) + self.assertTrue((destination / 'linux' / HEAD / 'report.json').exists()) + self.assertFalse((destination / 'macos' / HEAD / 'report.json').exists()) + campaign.retain(root / 'site', artifacts, failed, 'owner/repo', allow_partial=True) + def test_report_rejects_missing_tasks_references_samples_and_changed_identity(self): selected = plan() job = selected['jobs'][0] diff --git a/tools/benchmark_history_test.py b/tools/benchmark_history_test.py index 8c7413809c..e2c4c5e426 100644 --- a/tools/benchmark_history_test.py +++ b/tools/benchmark_history_test.py @@ -248,8 +248,8 @@ def test_publication_coalesces_requests_and_reconciles_sources(self): self.assertIn('group: benchmark-publication-latest', workflow) reconcile = repository_file('tools/benchmark_reconcile_ci.py').read_text() self.assertIn('benchmark_preview_ci.collect', reconcile) - self.assertIn("('benchmarks.yml', 'push', retain_measurement)", reconcile) - self.assertIn("('benchmark_backfill.yml', 'workflow_dispatch', retain_backfill)", reconcile) + self.assertIn("('benchmarks.yml', 'push', 'success', retain_measurement)", reconcile) + self.assertIn("('benchmark_backfill.yml', 'workflow_dispatch', 'completed', retain_backfill)", reconcile) def test_fifth_candidate_requires_unresolved_fourth_agreement(self): workflow = repository_file('.github/workflows/benchmarks.yml').read_text() diff --git a/tools/benchmark_landscape.py b/tools/benchmark_landscape.py index 514e80488c..d69941b0da 100644 --- a/tools/benchmark_landscape.py +++ b/tools/benchmark_landscape.py @@ -643,6 +643,10 @@ def publish(root, javascript, previews=(), incremental=False, repository='mbowor if benchmark_records.is_backfill(record): label = 'CI backfill / ' + commit[:10] links = version_links.get(commit, []) + if not local and not preview: + pr_links = [link['label'] for link in links if re.fullmatch(r'PR #[0-9]+', link.get('label', ''))] + if pr_links: + label = ', '.join(pr_links) + ' / ' + commit[:10] if preview: number = record.get('pull_number') or source['pull_requests'][0]['number'] links = [{'label': f'PR #{number} preview', 'href': f'https://github.com/{repository}/pull/{number}'}] diff --git a/tools/benchmark_landscape_test.py b/tools/benchmark_landscape_test.py index d824d0e70a..f3f25db793 100644 --- a/tools/benchmark_landscape_test.py +++ b/tools/benchmark_landscape_test.py @@ -91,6 +91,23 @@ def test_layout_details_distinguish_legacy_and_versioned_datasets(self): {'name': 'broad', 'revision': 2}, {'name': 'deep', 'revision': 2}, ]}), 'broad/v2 + deep/v2') + def test_merged_version_uses_pr_label_from_version_links(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / 'index.html').write_text('

History

') + folder = root / 'runs/1/1/linux' + folder.mkdir(parents=True) + sha = 'a' * 40 + (folder / 'report.json').write_text(json.dumps(dict(tool_comparisons=report(), platform='linux', + head=sha, source=dict(id=1, run_attempt=1, head_branch='main', event='push', + created_at='2026-09-01T00:00:00Z')))) + (folder / 'index.html').write_text('

Main

') + (root / 'version-links.json').write_text(json.dumps({sha: [ + {'label': 'PR #1143', 'href': 'https://github.com/owner/repo/pull/1143'}]})) + landscape.publish(root, '/* renderer */') + catalog = json.loads((root / 'catalog.json').read_text()) + self.assertEqual(catalog[0]['label'], 'PR #1143 / ' + sha[:10]) + def test_incremental_preview_publication_preserves_merged_pages_and_removes_stale_selection(self): with tempfile.TemporaryDirectory() as directory: root = Path(directory) diff --git a/tools/benchmark_reconcile_ci.py b/tools/benchmark_reconcile_ci.py index d8850e3a08..80fdad40c8 100644 --- a/tools/benchmark_reconcile_ci.py +++ b/tools/benchmark_reconcile_ci.py @@ -116,19 +116,25 @@ def retain_backfill(api, root, run): if not re.fullmatch(r'benchmark-backfill-[A-Za-z0-9_.-]+', name): raise ValueError('invalid backfill artifact name') target = directory / name - target.mkdir() - extract(api.request(f"actions/artifacts/{item['id']}/zip", raw=True), target) - benchmark_campaign.retain(root, directory, run, api.repository) + try: + target.mkdir() + extract(api.request(f"actions/artifacts/{item['id']}/zip", raw=True), target) + except (ValueError, OSError, zipfile.BadZipFile, subprocess.CalledProcessError, + subprocess.TimeoutExpired) as error: + if name == 'benchmark-backfill-plan': + raise + print(f"::warning::backfill artifact {name}: {error}", flush=True) + benchmark_campaign.retain(root, directory, run, api.repository, allow_partial=True) return True def reconcile(api, root): since = (datetime.now(timezone.utc) - timedelta(days=30)).date().isoformat() failures = [] - for workflow, event, retain in ( - ('benchmarks.yml', 'push', retain_measurement), - ('benchmark_backfill.yml', 'workflow_dispatch', retain_backfill)): - runs = api.pages(f'actions/workflows/{workflow}/runs?status=success&branch=main&created=%3E%3D{since}', + for workflow, event, status, retain in ( + ('benchmarks.yml', 'push', 'success', retain_measurement), + ('benchmark_backfill.yml', 'workflow_dispatch', 'completed', retain_backfill)): + runs = api.pages(f'actions/workflows/{workflow}/runs?status={status}&branch=main&created=%3E%3D{since}', 'workflow_runs') for run in reversed(runs): if run.get('event') != event or run.get('head_branch') != 'main': diff --git a/tools/benchmark_recovery.py b/tools/benchmark_recovery.py new file mode 100644 index 0000000000..7331383755 --- /dev/null +++ b/tools/benchmark_recovery.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) M. Boerger, the MBO Works authors +# SPDX-License-Identifier: Apache-2.0 +"""Find missing merged PR benchmarks and start bounded targeted CI backfills.""" + +import argparse +import base64 +from datetime import date +import io +import json +import re +import subprocess +import zipfile + +import benchmark_preview_ci + +PLATFORMS = ('linux', 'macos') + + +def catalog(api): + metadata = api.request('contents/benchmarks/catalog.json?ref=coverage-pages') + if not metadata.get('content') and metadata.get('sha'): + metadata = api.request('git/blobs/' + metadata['sha']) + if metadata.get('encoding') != 'base64' or not metadata.get('content'): + raise ValueError('published benchmark catalog is unavailable') + value = json.loads(base64.b64decode(metadata['content'], validate=False)) + if not isinstance(value, list): + raise ValueError('invalid published benchmark catalog') + return value + + +def inventory(api, since, number=None): + entries = catalog(api) + covered = set() + for entry in entries: + if entry.get('source') != 'merged' or entry.get('local'): + continue + platform = entry.get('platform_name', '').split(' / ')[0].lower() + sha = entry.get('commit', '') + if platform in PLATFORMS and re.fullmatch(r'[0-9a-f]{40}', sha): + covered.add((sha, platform)) + pulls = api.pages('pulls?state=closed&sort=updated&direction=desc') + target_shas = {pull.get('merge_commit_sha') for pull in pulls + if pull.get('merged_at') and pull['merged_at'][:10] >= since + and pull.get('base', {}).get('ref') == 'main' + and (number is None or pull['number'] == number)} + runs = api.pages('actions/workflows/benchmarks.yml/runs?branch=main', 'workflow_runs') + by_commit = {} + for run in runs: + if run.get('event') == 'push': + by_commit.setdefault(run['head_sha'], []).append(run) + pending_backfills = set() + campaigns = api.pages('actions/workflows/benchmark_backfill.yml/runs?branch=main&status=completed', 'workflow_runs') + for run in campaigns: + artifacts = api.pages(f"actions/runs/{run['id']}/artifacts", 'artifacts') + available = {item['name']: item for item in artifacts if not item.get('expired')} + plan = available.get('benchmark-backfill-plan') + if not plan: + continue + raw = api.request(f"actions/artifacts/{plan['id']}/zip", raw=True) + if len(raw) > 16 * 1024 * 1024: + raise ValueError('backfill plan artifact exceeds 16 MiB') + with zipfile.ZipFile(io.BytesIO(raw)) as archive: + entry = archive.getinfo('campaign.json') + if entry.file_size > 16 * 1024 * 1024: + raise ValueError('backfill plan exceeds 16 MiB') + manifest = json.loads(archive.read(entry)) + jobs = manifest['jobs'] + for job in jobs: + sha = job['revision']['sha'] + if sha not in target_shas or (sha, job['platform']) in covered: + continue + item = available.get('benchmark-backfill-' + job['id']) + if not item: + continue + raw = api.request(f"actions/artifacts/{item['id']}/zip", raw=True) + if len(raw) > 128 * 1024 * 1024: + raise ValueError('backfill artifact exceeds 128 MiB') + with zipfile.ZipFile(io.BytesIO(raw)) as archive: + try: + info = archive.getinfo('status.json') + if info.file_size > 16 * 1024 * 1024: + raise ValueError('backfill status exceeds 16 MiB') + status = json.loads(archive.read(info)) + report = f"reports/{sha}/benchmark-report-{job['platform']}-{job['arch']}.json" + if status.get(sha, {}).get('status') == 'complete' and report in archive.namelist(): + pending_backfills.add((sha, job['platform'])) + except (KeyError, ValueError): + continue + rows = [] + for pull in pulls: + if number is not None and pull['number'] != number: + continue + merged = pull.get('merged_at') + sha = pull.get('merge_commit_sha') + if not merged or merged[:10] < since or pull.get('base', {}).get('ref') != 'main': + continue + if not isinstance(sha, str) or not re.fullmatch(r'[0-9a-f]{40}', sha): + raise ValueError(f"PR {pull['number']} has no valid merge commit") + missing = [platform for platform in PLATFORMS if (sha, platform) not in covered] + if not missing: + continue + source_runs = by_commit.get(sha, []) + active = any(run['status'] != 'completed' for run in source_runs) + available = {platform for platform in PLATFORMS if (sha, platform) in pending_backfills} + for run in source_runs: + if run.get('conclusion') != 'success': + continue + artifacts = api.pages(f"actions/runs/{run['id']}/artifacts", 'artifacts') + for artifact in artifacts: + match = re.fullmatch(r'benchmark-report-(linux|macos)-[A-Za-z0-9_.-]+', artifact['name']) + if match and not artifact.get('expired') and artifact['created_at'] >= run['run_started_at']: + available.add(match.group(1)) + state = ('measuring' if active else 'awaiting publication' if set(missing) <= available + else 'needs backfill') + rows.append({'pr': pull['number'], 'commit': sha, 'merged_at': merged, + 'missing': missing, 'available_artifacts': sorted(available), 'state': state}) + return sorted(rows, key=lambda row: (row['merged_at'], row['pr'])) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--repository', required=True, help='GitHub owner/repository') + parser.add_argument('--since', default='2026-09-19', help='Inclusive merge date (YYYY-MM-DD)') + parser.add_argument('--pr', type=int, help='Limit the inventory to one PR number') + parser.add_argument('--dispatch-next', action='store_true', + help='Start a bounded backfill after checking for an active campaign') + parser.add_argument('--batch-size', type=int, default=4, help='Maximum PRs per campaign (1 through 8)') + args = parser.parse_args() + date.fromisoformat(args.since) + if not 1 <= args.batch_size <= 8: + parser.error('--batch-size must be between 1 and 8') + api = benchmark_preview_ci.GitHub(args.repository) + rows = inventory(api, args.since, args.pr) + print(json.dumps({'since': args.since, 'missing': rows}, indent=2), flush=True) + if not args.dispatch_next: + return + candidates = [row for row in rows if row['state'] == 'needs backfill'] + if not candidates: + print('No PR requires backfill.', flush=True) + return + campaigns = api.pages('actions/workflows/benchmark_backfill.yml/runs?branch=main', 'workflow_runs') + if any(run['status'] != 'completed' for run in campaigns): + raise ValueError('a backfill campaign is already active or queued; retry after it finishes') + selected = candidates[:args.batch_size] + # The deployed workflow must support the revisions input added by PR #1147. + subprocess.run(['gh', 'workflow', 'run', 'benchmark_backfill.yml', '--repo', api.repository, + '--ref', 'main', '-f', 'revisions=' + ','.join(row['commit'] for row in selected)], check=True) + print('Dispatched both platforms for ' + ', '.join(f"PR #{row['pr']}" for row in selected) + '.', flush=True) + + +if __name__ == '__main__': + main() diff --git a/tools/benchmark_recovery_test.py b/tools/benchmark_recovery_test.py new file mode 100644 index 0000000000..69d3939d21 --- /dev/null +++ b/tools/benchmark_recovery_test.py @@ -0,0 +1,104 @@ +# SPDX-FileCopyrightText: Copyright (c) M. Boerger, the MBO Works authors +# SPDX-License-Identifier: Apache-2.0 + +import base64 +import io +import json +import unittest +from unittest import mock +import zipfile + +import benchmark_recovery as recovery + +SHA = 'a' * 40 +OTHER = 'b' * 40 + + +class FakeGitHub: + repository = 'owner/repo' + + def __init__(self): + self.pulls = [{'number': 7, 'merged_at': '2026-10-01T12:00:00Z', 'merge_commit_sha': SHA, + 'base': {'ref': 'main'}}] + self.runs = [] + self.campaigns = [] + self.artifacts = {} + self.archives = {} + self.entries = [] + + def request(self, path, raw=False): + if path.startswith('contents/'): + return {'encoding': 'base64', 'content': base64.b64encode(json.dumps(self.entries).encode()).decode()} + return self.archives[path] + + def pages(self, path, key=None): + if path.startswith('pulls?'): + return self.pulls + if 'benchmark_backfill.yml' in path: + return self.campaigns + if 'benchmarks.yml' in path: + return self.runs + return self.artifacts[path] + + +class BenchmarkRecoveryTest(unittest.TestCase): + def test_pr_missing_one_platform_is_reported(self): + api = FakeGitHub() + api.entries = [{'source': 'merged', 'commit': SHA, 'platform_name': 'linux / x86_64'}] + self.assertEqual(recovery.inventory(api, '2026-09-19')[0]['missing'], ['macos']) + + def test_active_run_and_available_reports_are_not_backfilled(self): + api = FakeGitHub() + api.runs = [{'id': 1, 'head_sha': SHA, 'event': 'push', 'status': 'in_progress', 'conclusion': None}] + self.assertEqual(recovery.inventory(api, '2026-09-19')[0]['state'], 'measuring') + api.runs[0].update(status='completed', conclusion='success', run_started_at='2026-10-01T12:00:00Z') + api.artifacts['actions/runs/1/artifacts'] = [ + {'name': f'benchmark-report-{platform}-arch', 'created_at': '2026-10-01T13:00:00Z', 'expired': False} + for platform in recovery.PLATFORMS] + self.assertEqual(recovery.inventory(api, '2026-09-19')[0]['state'], 'awaiting publication') + + def test_retained_completed_campaign_is_not_dispatched_again(self): + api = FakeGitHub() + api.campaigns = [{'id': 2}] + jobs = [{'id': platform + '-' + SHA, 'revision': {'sha': SHA}, 'platform': platform, 'arch': 'arch'} + for platform in recovery.PLATFORMS] + archive = io.BytesIO() + with zipfile.ZipFile(archive, 'w') as output: + output.writestr('campaign.json', json.dumps({'jobs': jobs})) + api.archives['actions/artifacts/3/zip'] = archive.getvalue() + api.artifacts['actions/runs/2/artifacts'] = [{'id': 3, 'name': 'benchmark-backfill-plan'}] + for index, job in enumerate(jobs, 4): + api.artifacts['actions/runs/2/artifacts'].append({'id': index, 'name': 'benchmark-backfill-' + job['id']}) + artifact = io.BytesIO() + with zipfile.ZipFile(artifact, 'w') as output: + output.writestr('status.json', json.dumps({SHA: {'status': 'complete'}})) + output.writestr(f"reports/{SHA}/benchmark-report-{job['platform']}-arch.json", '{}') + api.archives[f'actions/artifacts/{index}/zip'] = artifact.getvalue() + self.assertEqual(recovery.inventory(api, '2026-09-19')[0]['state'], 'awaiting publication') + + def test_dispatch_refuses_active_campaign(self): + api = FakeGitHub() + with mock.patch.object(recovery.benchmark_preview_ci, 'GitHub', return_value=api), \ + mock.patch.object(recovery, 'inventory', return_value=[{'state': 'needs backfill'}]), \ + mock.patch.object(api, 'pages', return_value=[{'status': 'queued'}]), \ + mock.patch.object(recovery.subprocess, 'run') as run, \ + mock.patch('sys.argv', ['benchmark_recovery', '--repository', 'owner/repo', '--dispatch-next']): + with self.assertRaisesRegex(ValueError, 'already active'): + recovery.main() + run.assert_not_called() + + def test_dispatch_uses_the_targeted_main_workflow(self): + api = FakeGitHub() + selected = {'state': 'needs backfill', 'pr': 7, 'commit': SHA} + with mock.patch.object(recovery.benchmark_preview_ci, 'GitHub', return_value=api), \ + mock.patch.object(recovery, 'inventory', return_value=[selected]), \ + mock.patch.object(api, 'pages', return_value=[]), \ + mock.patch.object(recovery.subprocess, 'run') as run, \ + mock.patch('sys.argv', ['benchmark_recovery', '--repository', 'owner/repo', '--dispatch-next']): + recovery.main() + run.assert_called_once_with(['gh', 'workflow', 'run', 'benchmark_backfill.yml', '--repo', + 'owner/repo', '--ref', 'main', '-f', 'revisions=' + SHA], check=True) + + +if __name__ == '__main__': + unittest.main()