From 7accf2caf5e5dde48b84cd0a672c3ce73ff84715 Mon Sep 17 00:00:00 2001 From: helly25 <6420169+helly25@users.noreply.github.com> Date: Sun, 8 Nov 2026 14:36:53 +0000 Subject: [PATCH 01/13] Run benchmark measurements after build in parallel --- .github/workflows/benchmarks.yml | 98 ++++++++++++++++++++++++-------- tools/benchmark_history.py | 3 + tools/benchmark_history_test.py | 16 ++++++ 3 files changed, 94 insertions(+), 23 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 6eddf3cc78d..fd343f91bd2 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -79,6 +79,11 @@ jobs: driver="$(pwd)/$(bazel cquery //tools:benchmark_resources --config=clang_opt \ --output=starlark --starlark:expr='target.files_to_run.executable.path')" echo "BENCHMARK_DRIVER=${driver}" >>"${GITHUB_ENV}" + for target in expression_benchmark expression_run_benchmark; do + executable="$(bazel cquery "config(//xff/engine:${target}, target)" --config=clang_opt \ + --output=starlark --starlark:expr='target.files_to_run.executable.path')" + cp "${executable}" "${RUNNER_TEMP}/${target}" + done - name: Build production benchmark CLI (ThinLTO) run: | bazel build //xff/cli:xff --config=clang_release --disk_cache="${HOME}/.cache/bazel-disk" @@ -102,9 +107,59 @@ jobs: --output=starlark --starlark:expr='target.files_to_run.executable.path')" cp "${binary}" "${RUNNER_TEMP}/benchmark-base-bin" shasum -a 256 "${RUNNER_TEMP}/benchmark-base-bin" + - name: Preserve build identities + run: | + cat "${RUNNER_TEMP}/benchmark-base/.bazelrc" "${RUNNER_TEMP}/benchmark-base/bazelmod/llvm.MODULE.bazel" | shasum -a 256 | cut -d ' ' -f 1 > "${RUNNER_TEMP}/base-build-identity" + cat .bazelrc bazelmod/llvm.MODULE.bazel | shasum -a 256 | cut -d ' ' -f 1 > "${RUNNER_TEMP}/head-build-identity" + echo "$(bazel --version); module-lock=$(shasum -a 256 MODULE.bazel.lock); base-module-lock=$(shasum -a 256 "${RUNNER_TEMP}/benchmark-base/MODULE.bazel.lock")" > "${RUNNER_TEMP}/benchmark-tools" + - name: Plan equal sample shards and stage shared binaries + run: | + scales=() + for count in 10 20 50 100 200 500 1000 2000 5000 10000 20000 50000 100000; do + scales+=(--files="${count}") + done + python3 tools/benchmark_shards.py --plan --adaptive --count=5 "${scales[@]}" \ + --partition=samples --repetitions=15 --keep=15 \ + --cpus=1 --cpus=${{ matrix.cpus }} --output=benchmark-plan.json + tar -cf benchmark-build.tar -C "${RUNNER_TEMP}" benchmark-head benchmark-resource-launcher expression_benchmark expression_run_benchmark benchmark-head-reads benchmark-base-reads benchmark-base-bin base-build-identity head-build-identity benchmark-tools + - uses: actions/upload-artifact@v7 + with: + name: benchmark-build-${{ matrix.platform }} + path: | + benchmark-build.tar + benchmark-plan.json + retention-days: 3 + if-no-files-found: error + + expression: + needs: build + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-latest + platform: linux + - os: macos-latest + platform: macos + runs-on: ${{ matrix.os }} + timeout-minutes: 60 + steps: + - uses: actions/checkout@v7 + with: + ref: ${{ github.sha }} + fetch-depth: 0 + persist-credentials: false + - uses: actions/setup-python@v7 + with: + python-version: "3.13" + - uses: actions/download-artifact@v8 + with: + name: benchmark-build-${{ matrix.platform }} + - name: Restore benchmark executables + run: tar -xf benchmark-build.tar -C "${RUNNER_TEMP}" - name: Measure expression preparation and execution baseline run: | - bazel run //xff/engine:expression_benchmark --config=clang_opt --disk_cache="${HOME}/.cache/bazel-disk" -- \ + "${RUNNER_TEMP}/expression_benchmark" \ --benchmark_out="${GITHUB_WORKSPACE}/expression-baseline-${{ matrix.platform }}.json" \ --benchmark_out_format=json --benchmark_display_aggregates_only=true \ --benchmark_enable_random_interleaving=true \ @@ -120,7 +175,7 @@ jobs: PY_CHECK - name: Measure whole-engine expression integration run: | - bazel run //xff/engine:expression_run_benchmark --config=clang_opt --disk_cache="${HOME}/.cache/bazel-disk" -- \ + "${RUNNER_TEMP}/expression_run_benchmark" \ --benchmark_out="${GITHUB_WORKSPACE}/expression-engine-${{ matrix.platform }}.json" \ --benchmark_out_format=json --benchmark_display_aggregates_only=true \ --benchmark_enable_random_interleaving=true \ @@ -145,36 +200,28 @@ jobs: - name: Measure alternating base and head workloads run: | set -euo pipefail + chmod +x tools/benchmark_resources.py + BENCHMARK_HEAD="$(git rev-parse HEAD)" + BENCHMARK_BASE="$(git rev-parse HEAD^1)" + BENCHMARK_PLATFORM="${{ matrix.platform }}-$(uname -m)" python3 tools/benchmark_history.py measure \ - --driver="${BENCHMARK_DRIVER}" \ + --driver="${GITHUB_WORKSPACE}/tools/benchmark_resources.py" \ + --resource-launcher="${RUNNER_TEMP}/benchmark-resource-launcher" \ --head-read-counter="${RUNNER_TEMP}/benchmark-head-reads" \ --base-read-counter="${RUNNER_TEMP}/benchmark-base-reads" \ --head-binary="${RUNNER_TEMP}/benchmark-head" \ --base-binary="${RUNNER_TEMP}/benchmark-base-bin" \ --head="${BENCHMARK_HEAD}" --base="${BENCHMARK_BASE}" \ - --base-build-identity="$(cat "${RUNNER_TEMP}/benchmark-base/.bazelrc" "${RUNNER_TEMP}/benchmark-base/bazelmod/llvm.MODULE.bazel" | shasum -a 256 | cut -d ' ' -f 1)" \ - --head-build-identity="$(cat .bazelrc bazelmod/llvm.MODULE.bazel | shasum -a 256 | cut -d ' ' -f 1)" \ + --base-build-identity="$(cat "${RUNNER_TEMP}/base-build-identity")" \ + --head-build-identity="$(cat "${RUNNER_TEMP}/head-build-identity")" \ --build='bazel --config=clang_release //xff/cli:xff' \ --runner-class='github-hosted ${{ matrix.os }}' \ - --tools="$(bazel --version); module-lock=$(shasum -a 256 MODULE.bazel.lock); base-module-lock=$(shasum -a 256 "${RUNNER_TEMP}/benchmark-base/MODULE.bazel.lock")" \ + --tools="$(cat "${RUNNER_TEMP}/benchmark-tools")" \ --platform=${{ matrix.platform }} --output=benchmark-report-${BENCHMARK_PLATFORM}.json - - name: Plan equal sample shards and stage shared binaries - run: | - scales=() - for count in 10 20 50 100 200 500 1000 2000 5000 10000 20000 50000 100000; do - scales+=(--files="${count}") - done - python3 tools/benchmark_shards.py --plan --adaptive --count=5 "${scales[@]}" \ - --partition=samples --repetitions=15 --keep=15 \ - --cpus=1 --cpus=${{ matrix.cpus }} --output=benchmark-plan.json - tar -cf benchmark-build.tar -C "${RUNNER_TEMP}" benchmark-head benchmark-resource-launcher - uses: actions/upload-artifact@v7 with: - name: benchmark-build-${{ matrix.platform }} - path: | - benchmark-build.tar - benchmark-plan.json - benchmark-report-${{ env.BENCHMARK_PLATFORM }}.json + name: benchmark-seed-${{ matrix.platform }} + path: benchmark-report-${{ matrix.platform }}-*.json retention-days: 3 if-no-files-found: error @@ -365,6 +412,7 @@ jobs: needs: [ build, + expression, measure, select-initial, measure-fourth-linux, @@ -373,7 +421,7 @@ jobs: measure-fifth-linux, measure-fifth-macos, ] - if: always() && needs.build.result == 'success' + if: always() && needs.build.result == 'success' && needs.expression.result == 'success' runs-on: ubuntu-latest permissions: actions: read @@ -401,13 +449,17 @@ jobs: with: pattern: benchmark-build-* path: build-metadata + - uses: actions/download-artifact@v8 + with: + pattern: benchmark-seed-* + path: seed-metadata - name: Select agreed candidates and render both platforms id: reports run: | set -euo pipefail git rev-list --first-parent HEAD^1 > benchmark-baseline-revisions.txt python3 tools/benchmark_shards.py --merge-adaptive-directory=candidate-shards \ - --output-directory=merged --seed-directory=build-metadata \ + --output-directory=merged --seed-directory=seed-metadata \ --baseline-root=benchmark-baseline/benchmarks --baseline-revisions=benchmark-baseline-revisions.txt for platform in linux macos; do arch="$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["tool_comparisons"]["contract"]["machine"])' "merged/benchmark-merged-${platform}.json")" diff --git a/tools/benchmark_history.py b/tools/benchmark_history.py index 3e8a967187e..6844427fd9b 100644 --- a/tools/benchmark_history.py +++ b/tools/benchmark_history.py @@ -105,6 +105,8 @@ def measure_pair(args): command = [str(args.driver), str(getattr(args, f"{side}_binary")), f"--output={output}", f"--files={args.files}", f"--depth={args.depth}", "--repetitions=1", f"--jobs={args.jobs}", f"--build-label={record[side]};{args.build}"] + if getattr(args, "resource_launcher", None) is not None: + command.append(f"--resource-launcher={args.resource_launcher}") counter = getattr(args, f"{side}_read_counter", None) if counter is not None: command.append(f"--read-counter={counter}") @@ -413,6 +415,7 @@ def main(): measure = commands.add_parser("measure") for name in ("driver", "base-binary", "head-binary", "output"): measure.add_argument("--" + name, type=Path, required=True) + measure.add_argument("--resource-launcher", type=Path) for side in ("base", "head"): measure.add_argument(f"--{side}-read-counter", type=Path) for name in ("base", "head", "build", "runner-class", "tools", "base-build-identity", "head-build-identity"): diff --git a/tools/benchmark_history_test.py b/tools/benchmark_history_test.py index dd7ffe0c7b4..1ebc1531f52 100644 --- a/tools/benchmark_history_test.py +++ b/tools/benchmark_history_test.py @@ -226,6 +226,22 @@ def test_workflow_keeps_measurement_unprivileged_and_publication_serialized(self self.assertNotIn('shards/*/*.json', measure) self.assertIn("git -C site add --all", publish) + def test_full_suite_runs_independently_of_paired_measurements(self): + workflow = repository_file('.github/workflows/benchmarks.yml').read_text() + build = workflow.split(' build:\n', 1)[1].split(' expression:\n', 1)[0] + expression = workflow.split(' expression:\n', 1)[1].split(' measure:\n', 1)[0] + aggregate = workflow.split(' aggregate:\n', 1)[1] + self.assertNotIn('Measure alternating base and head workloads', build) + self.assertNotIn('Measure expression preparation', build) + self.assertIn('name: benchmark-build-${{ matrix.platform }}', build) + self.assertIn('needs: build', expression) + self.assertIn('name: benchmark-seed-${{ matrix.platform }}', expression) + self.assertIn('--resource-launcher="${RUNNER_TEMP}/benchmark-resource-launcher"', expression) + self.assertIn(' measure:\n needs: build', workflow) + self.assertIn('pattern: benchmark-seed-*', aggregate) + self.assertIn('--seed-directory=seed-metadata', aggregate) + self.assertIn("needs.expression.result == 'success'", aggregate) + def test_publication_coalesces_requests_and_reconciles_sources(self): workflow = repository_file('.github/workflows/benchmark_pages.yml').read_text() self.assertIn('types: [completed]', workflow) From 93d65010001cb99cc797188ede65c3910b71f651 Mon Sep 17 00:00:00 2001 From: helly25 <6420169+helly25@users.noreply.github.com> Date: Sun, 8 Nov 2026 14:36:04 +0000 Subject: [PATCH 02/13] Allow selecting revisions for CI backfill --- .github/workflows/benchmark_backfill.yml | 19 +++++++++++++++++-- tools/benchmark_campaign.py | 7 ++++--- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/.github/workflows/benchmark_backfill.yml b/.github/workflows/benchmark_backfill.yml index c573b2df0d0..ce4a0424e97 100644 --- a/.github/workflows/benchmark_backfill.yml +++ b/.github/workflows/benchmark_backfill.yml @@ -3,7 +3,12 @@ name: Benchmark backfill on: - workflow_dispatch: {} + workflow_dispatch: + inputs: + revisions: + description: Comma-separated commit IDs (empty selects published history) + required: false + type: string permissions: contents: read @@ -35,10 +40,20 @@ jobs: python-version: "3.13" - name: Freeze the historical revision selection id: plan + env: + REVISIONS: ${{ inputs.revisions }} run: | set -euo pipefail + revision_args=() + IFS=',' read -ra revisions <<< "${REVISIONS}" + for revision in "${revisions[@]}"; do + revision="${revision//[[:space:]]/}" + if [[ -n "${revision}" ]]; then + revision_args+=(--revision="${revision}") + fi + done matrix="$(python3 tools/benchmark_campaign.py plan --repo=. \ - --history-root=benchmark-history/benchmarks --output=campaign.json)" + --history-root=benchmark-history/benchmarks --output=campaign.json "${revision_args[@]}")" echo "matrix=${matrix}" >>"${GITHUB_OUTPUT}" - uses: actions/upload-artifact@v7 with: diff --git a/tools/benchmark_campaign.py b/tools/benchmark_campaign.py index ef11a5f2cf5..a52324f3dc0 100644 --- a/tools/benchmark_campaign.py +++ b/tools/benchmark_campaign.py @@ -29,11 +29,11 @@ def require(condition, message): raise ValueError(message) -def make_plan(repo, history_root): +def make_plan(repo, history_root, references=None): tools = {'xff': {'path': 'xff'}, 'find': {}, 'rg': {}, 'fzf': {}} jobs = [] for platform, (runner, arch) in PLATFORMS.items(): - for revision in backfill.revisions(repo, [], history_root, platform): + for revision in backfill.revisions(repo, references or [], None if references else history_root, platform): jobs.append({'id': platform + '-' + revision['sha'], 'platform': platform, 'os': runner, 'arch': arch, 'revision': revision}) plan = {'schema': 1, 'source_head': backfill.git(repo, 'rev-parse', 'HEAD'), @@ -236,6 +236,7 @@ def main(): plan = modes.add_parser('plan') for name in ('repo', 'history-root', 'output'): plan.add_argument('--' + name, type=Path, required=True) + plan.add_argument('--revision', action='append', default=[], help='Measure this revision instead of site history') run = modes.add_parser('run') for name in ('plan', 'repo', 'output', 'fixture-parent'): run.add_argument('--' + name, type=Path, required=True) @@ -249,7 +250,7 @@ def main(): publish.add_argument('--repository', required=True) args = parser.parse_args() if args.mode == 'plan': - result = make_plan(args.repo, args.history_root) + result = make_plan(args.repo, args.history_root, args.revision) backfill.write_json(args.output, result) print(json.dumps({'include': [{key: job[key] for key in ('id', 'os', 'platform', 'arch')} for job in result['jobs']]})) From d4a7a00bc7053b22367f984c50d99a5dd082c0d5 Mon Sep 17 00:00:00 2001 From: helly25 <6420169+helly25@users.noreply.github.com> Date: Sun, 8 Nov 2026 14:41:01 +0000 Subject: [PATCH 03/13] Document targeted backfill workflow input --- .github/workflows/benchmark_backfill.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/benchmark_backfill.yml b/.github/workflows/benchmark_backfill.yml index ce4a0424e97..97ff888a532 100644 --- a/.github/workflows/benchmark_backfill.yml +++ b/.github/workflows/benchmark_backfill.yml @@ -3,6 +3,7 @@ name: Benchmark backfill on: + # trunk-ignore(checkov/CKV_GHA_7): The input selects measured revisions, not the workflow or driver. workflow_dispatch: inputs: revisions: From 96858394cc099d71e1d63055232cf4b987bee4f1 Mon Sep 17 00:00:00 2001 From: helly25 <6420169+helly25@users.noreply.github.com> Date: Sun, 8 Nov 2026 14:43:19 +0000 Subject: [PATCH 04/13] Attach Checkov suppression to revision input --- .github/workflows/benchmark_backfill.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmark_backfill.yml b/.github/workflows/benchmark_backfill.yml index 97ff888a532..44348bffa10 100644 --- a/.github/workflows/benchmark_backfill.yml +++ b/.github/workflows/benchmark_backfill.yml @@ -3,9 +3,9 @@ name: Benchmark backfill on: - # trunk-ignore(checkov/CKV_GHA_7): The input selects measured revisions, not the workflow or driver. workflow_dispatch: inputs: + # trunk-ignore(checkov/CKV_GHA_7): The input selects measured revisions, not the workflow or driver. revisions: description: Comma-separated commit IDs (empty selects published history) required: false From d33f4de32042fb1e8e21f1bb98d57088970ef0f4 Mon Sep 17 00:00:00 2001 From: helly25 <6420169+helly25@users.noreply.github.com> Date: Sun, 8 Nov 2026 15:12:58 +0000 Subject: [PATCH 05/13] Run PR expression benchmarks alongside shard measurements --- .github/workflows/main.yml | 75 ++++++++++++++++++++++++--------- tools/benchmark_history_test.py | 12 ++++++ 2 files changed, 68 insertions(+), 19 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0a515c31a36..eae7e0a51a4 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -789,11 +789,63 @@ jobs: --output=starlark --starlark:expr='target.files_to_run.executable.path')" cp "${binary}" "${RUNNER_TEMP}/benchmark-head" shasum -a 256 "${RUNNER_TEMP}/benchmark-head" + - name: Stage expression benchmark executables + run: | + for target in expression_benchmark expression_run_benchmark; do + executable="$(bazel cquery "config(//xff/engine:${target}, target)" --config=clang_opt \ + --output=starlark --starlark:expr='target.files_to_run.executable.path')" + cp "${executable}" "${RUNNER_TEMP}/${target}" + done - name: Package frozen binaries for all measurement candidates - run: tar -cf benchmark-build.tar -C "${RUNNER_TEMP}" benchmark-head benchmark-resource-launcher + run: tar -cf benchmark-build.tar -C "${RUNNER_TEMP}" benchmark-head benchmark-resource-launcher expression_benchmark expression_run_benchmark + - name: Plan three measured rounds per shard + run: | + scales=() + for count in 10 20 50 100 200 500 1000 2000 5000 10000; do + scales+=(--files="${count}") + done + python3 tools/benchmark_shards.py --plan --partition=samples --adaptive --count=5 \ + --repetitions=15 --keep=15 --cpus=1 --cpus=3 "${scales[@]}" --output=benchmark-plan.json + - uses: actions/upload-artifact@v7 + with: + name: benchmark-build-${{ matrix.platform }} + path: | + benchmark-build.tar + benchmark-plan.json + retention-days: 3 + if-no-files-found: error + + benchmark-expression: + name: Benchmark expression measurements (${{ matrix.platform }}) + needs: benchmark-build + if: github.event_name == 'pull_request' + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-latest + platform: linux + - os: macos-latest + platform: macos + runs-on: ${{ matrix.os }} + timeout-minutes: 60 + permissions: + contents: read + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - uses: actions/setup-python@v7 + with: + python-version: "3.13" + - uses: actions/download-artifact@v8 + with: + name: benchmark-build-${{ matrix.platform }} + - name: Restore benchmark executables + run: tar -xf benchmark-build.tar -C "${RUNNER_TEMP}" - name: Measure expression preparation and execution baseline run: | - bazel run //xff/engine:expression_benchmark --config=clang_opt --disk_cache="${HOME}/.cache/bazel-disk" -- \ + "${RUNNER_TEMP}/expression_benchmark" \ --benchmark_out="${GITHUB_WORKSPACE}/expression-baseline-${{ matrix.platform }}.json" \ --benchmark_out_format=json --benchmark_display_aggregates_only=true \ --benchmark_enable_random_interleaving=true \ @@ -809,7 +861,7 @@ jobs: PY_CHECK - name: Measure whole-engine expression integration run: | - bazel run //xff/engine:expression_run_benchmark --config=clang_opt --disk_cache="${HOME}/.cache/bazel-disk" -- \ + "${RUNNER_TEMP}/expression_run_benchmark" \ --benchmark_out="${GITHUB_WORKSPACE}/expression-engine-${{ matrix.platform }}.json" \ --benchmark_out_format=json --benchmark_display_aggregates_only=true \ --benchmark_enable_random_interleaving=true \ @@ -831,22 +883,6 @@ jobs: expression-engine-${{ matrix.platform }}.json retention-days: 30 if-no-files-found: error - - name: Plan three measured rounds per shard - run: | - scales=() - for count in 10 20 50 100 200 500 1000 2000 5000 10000; do - scales+=(--files="${count}") - done - python3 tools/benchmark_shards.py --plan --partition=samples --adaptive --count=5 \ - --repetitions=15 --keep=15 --cpus=1 --cpus=3 "${scales[@]}" --output=benchmark-plan.json - - uses: actions/upload-artifact@v7 - with: - name: benchmark-build-${{ matrix.platform }} - path: | - benchmark-build.tar - benchmark-plan.json - retention-days: 3 - if-no-files-found: error benchmark-measure: name: Benchmark measurement (${{ matrix.platform }}, shard ${{ matrix.shard }}) @@ -1124,6 +1160,7 @@ jobs: [ release-site-tests, benchmark-build, + benchmark-expression, benchmark-measure, benchmark-select-initial, benchmark-measure-fourth-linux, diff --git a/tools/benchmark_history_test.py b/tools/benchmark_history_test.py index 1ebc1531f52..8c7413809c9 100644 --- a/tools/benchmark_history_test.py +++ b/tools/benchmark_history_test.py @@ -262,6 +262,18 @@ def test_fifth_candidate_requires_unresolved_fourth_agreement(self): self.assertIn(f"needs.select-fourth.outputs.{platform_name}_fifth == 'true'", job) self.assertNotIn('needs.select-initial.outputs', job) + def test_pr_expression_measurements_do_not_block_full_suite(self): + workflow = repository_file('.github/workflows/main.yml').read_text() + build = workflow.split(' benchmark-build:\n', 1)[1].split(' benchmark-expression:\n', 1)[0] + expression = workflow.split(' benchmark-expression:\n', 1)[1].split(' benchmark-measure:\n', 1)[0] + self.assertNotIn('Measure expression preparation', build) + self.assertIn('expression_benchmark expression_run_benchmark', build) + self.assertIn('needs: benchmark-build', expression) + self.assertIn('"${RUNNER_TEMP}/expression_benchmark"', expression) + self.assertIn('"${RUNNER_TEMP}/expression_run_benchmark"', expression) + self.assertIn(' benchmark-measure:\n name:', workflow) + self.assertIn(' benchmark-expression,', workflow.split(' done:\n', 1)[1]) + def test_pr_and_main_measurements_use_the_same_runner_identity(self): main = repository_file(".github/workflows/benchmarks.yml").read_text() pr = repository_file(".github/workflows/main.yml").read_text() From 98d7bafd8fb06f1c375322e5b9062ad4ca2760fa Mon Sep 17 00:00:00 2001 From: helly25 <6420169+helly25@users.noreply.github.com> Date: Sun, 8 Nov 2026 15:28:00 +0000 Subject: [PATCH 06/13] Check cached action outputs after disk cache eviction --- .bazelrc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.bazelrc b/.bazelrc index 4a34cddd083..69f54e2875a 100644 --- a/.bazelrc +++ b/.bazelrc @@ -25,6 +25,9 @@ common --enable_bzlmod --noenable_workspace common --http_connector_attempts=20 common --http_connector_retry_max_timeout=5s common --experimental_repository_downloader_retries=20 +# The CI disk cache evicts individual CAS blobs. Check cached action results for +# missing outputs before reuse; Bazel 9 disables this check with action rewinding. +build --norewind_lost_inputs # Disable dependency ruleset telemetry for repository evaluation. common --repo_env=DO_NOT_TRACK=1 common --incompatible_disallow_empty_glob From 5da2a937e85414711341063a141bceac9501b858 Mon Sep 17 00:00:00 2001 From: helly25 <6420169+helly25@users.noreply.github.com> Date: Sun, 8 Nov 2026 15:30:55 +0000 Subject: [PATCH 07/13] Apply cache integrity setting to common Bazel commands --- .bazelrc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.bazelrc b/.bazelrc index 69f54e2875a..001de1468f0 100644 --- a/.bazelrc +++ b/.bazelrc @@ -27,7 +27,7 @@ common --http_connector_retry_max_timeout=5s common --experimental_repository_downloader_retries=20 # The CI disk cache evicts individual CAS blobs. Check cached action results for # missing outputs before reuse; Bazel 9 disables this check with action rewinding. -build --norewind_lost_inputs +common --norewind_lost_inputs # Disable dependency ruleset telemetry for repository evaluation. common --repo_env=DO_NOT_TRACK=1 common --incompatible_disallow_empty_glob From 139f394337fccc42a09549c41c387e47ec9abd58 Mon Sep 17 00:00:00 2001 From: helly25 <6420169+helly25@users.noreply.github.com> Date: Sun, 8 Nov 2026 15:39:11 +0000 Subject: [PATCH 08/13] Inventory missing merged PR benchmark coverage --- docs/benchmark-merged-pr-coverage.md | 165 +++++++++++++++++++++++++++ 1 file changed, 165 insertions(+) create mode 100644 docs/benchmark-merged-pr-coverage.md diff --git a/docs/benchmark-merged-pr-coverage.md b/docs/benchmark-merged-pr-coverage.md new file mode 100644 index 00000000000..39337479dd7 --- /dev/null +++ b/docs/benchmark-merged-pr-coverage.md @@ -0,0 +1,165 @@ +# Missing merged PR benchmark inventory + +Snapshot: November 8, 2026, using the published benchmark catalog, retained run tree, +GitHub merged PR metadata, and main benchmark workflow metadata captured during the audit. +Entries can become stale as publishers and workflows complete. This inventories PRs merged +on or after September 19, 2026; it is not an audit of earlier history. + +A PR is present when its merge commit appears in the merged CI viewer catalog. A catalog +entry may still be hard to find by PR number: existing post-merge entries use `main / ` +as the version label, with the PR only in the links. Local data is excluded from this count. + +The snapshot has 252 merged PRs: 157 represented and 95 absent. +The table records observed workflow conclusions, not inferred causes. A canceled run may +have been superseded or canceled for another reason. A failed run requires log investigation. +Older successful reports can exist without the viewer data required by the current catalog. + +| PR | Merged (UTC) | Merge commit | Observed reason or status | +| -------------------------------------------------- | ------------ | -------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------- | +| [#1145](https://github.com/mboworks/xff/pull/1145) | 2026-11-08 | `57a3cfb50b02` | Workflow pending ([run](https://github.com/mboworks/xff/actions/runs/42490418192)) | +| [#1146](https://github.com/mboworks/xff/pull/1146) | 2026-11-08 | `31dc55b400f3` | Workflow in_progress ([run](https://github.com/mboworks/xff/actions/runs/42486013719)) | +| [#1136](https://github.com/mboworks/xff/pull/1136) | 2026-11-07 | `1f25ba6d4ae0` | Measurement workflow canceled; cause not established ([run](https://github.com/mboworks/xff/actions/runs/42398990564)) | +| [#1132](https://github.com/mboworks/xff/pull/1132) | 2026-11-07 | `5d413a35cbec` | Measurement workflow canceled; cause not established ([run](https://github.com/mboworks/xff/actions/runs/42378502027)) | +| [#1130](https://github.com/mboworks/xff/pull/1130) | 2026-11-07 | `ae8c5af3eb11` | Measurement workflow canceled; cause not established ([run](https://github.com/mboworks/xff/actions/runs/42367122125)) | +| [#1128](https://github.com/mboworks/xff/pull/1128) | 2026-11-07 | `1278c8943a6a` | Measurement workflow canceled; cause not established ([run](https://github.com/mboworks/xff/actions/runs/42354657340)) | +| [#1121](https://github.com/mboworks/xff/pull/1121) | 2026-11-06 | `d9bf84b04784` | Measurement workflow canceled; cause not established ([run](https://github.com/mboworks/xff/actions/runs/42265512655)) | +| [#1118](https://github.com/mboworks/xff/pull/1118) | 2026-11-06 | `c7bb70cc6f3e` | Measurement workflow canceled; cause not established ([run](https://github.com/mboworks/xff/actions/runs/42234848791)) | +| [#1115](https://github.com/mboworks/xff/pull/1115) | 2026-11-06 | `ab909839da7d` | Measurement workflow canceled; cause not established ([run](https://github.com/mboworks/xff/actions/runs/42218900004)) | +| [#1113](https://github.com/mboworks/xff/pull/1113) | 2026-11-06 | `5729026a8482` | Measurement workflow canceled; cause not established ([run](https://github.com/mboworks/xff/actions/runs/42200480427)) | +| [#1111](https://github.com/mboworks/xff/pull/1111) | 2026-11-06 | `5cade08b6701` | Measurement workflow canceled; cause not established ([run](https://github.com/mboworks/xff/actions/runs/42184472882)) | +| [#1106](https://github.com/mboworks/xff/pull/1106) | 2026-11-06 | `e7e42d01e067` | Measurement workflow canceled; cause not established ([run](https://github.com/mboworks/xff/actions/runs/42146475737)) | +| [#1099](https://github.com/mboworks/xff/pull/1099) | 2026-11-05 | `76185cb37fa2` | Measurement workflow failed; exact cause not audited ([run](https://github.com/mboworks/xff/actions/runs/41999670147)) | +| [#1098](https://github.com/mboworks/xff/pull/1098) | 2026-11-05 | `7c8cd64e02a8` | Measurement workflow failed; exact cause not audited ([run](https://github.com/mboworks/xff/actions/runs/41986760505)) | +| [#1097](https://github.com/mboworks/xff/pull/1097) | 2026-11-05 | `910e2a7bbae5` | Measurement workflow canceled; cause not established ([run](https://github.com/mboworks/xff/actions/runs/41977457900)) | +| [#1095](https://github.com/mboworks/xff/pull/1095) | 2026-11-05 | `6792b5030de7` | Measurement workflow canceled; cause not established ([run](https://github.com/mboworks/xff/actions/runs/41962227281)) | +| [#1092](https://github.com/mboworks/xff/pull/1092) | 2026-11-04 | `a89459ed819a` | Measurement workflow canceled; cause not established ([run](https://github.com/mboworks/xff/actions/runs/41920310179)) | +| [#1088](https://github.com/mboworks/xff/pull/1088) | 2026-11-04 | `9bd70fc9b460` | Measurement workflow failed; exact cause not audited ([run](https://github.com/mboworks/xff/actions/runs/41851695947)) | +| [#1084](https://github.com/mboworks/xff/pull/1084) | 2026-11-04 | `f06cea15f5b1` | Measurement workflow canceled; cause not established ([run](https://github.com/mboworks/xff/actions/runs/41798644066)) | +| [#1082](https://github.com/mboworks/xff/pull/1082) | 2026-11-04 | `6e6b25ffc869` | Measurement workflow failed; exact cause not audited ([run](https://github.com/mboworks/xff/actions/runs/41783344161)) | +| [#1080](https://github.com/mboworks/xff/pull/1080) | 2026-11-04 | `669f68a3e263` | Measurement workflow failed; exact cause not audited ([run](https://github.com/mboworks/xff/actions/runs/41771864734)) | +| [#1078](https://github.com/mboworks/xff/pull/1078) | 2026-11-03 | `76054b3a9c60` | Measurement workflow failed; exact cause not audited ([run](https://github.com/mboworks/xff/actions/runs/41711975639)) | +| [#1073](https://github.com/mboworks/xff/pull/1073) | 2026-11-03 | `6db2de9e49f1` | Measurement workflow canceled; cause not established ([run](https://github.com/mboworks/xff/actions/runs/41628320126)) | +| [#1067](https://github.com/mboworks/xff/pull/1067) | 2026-11-03 | `fb726c946063` | Measurement workflow canceled; cause not established ([run](https://github.com/mboworks/xff/actions/runs/41573418793)) | +| [#1059](https://github.com/mboworks/xff/pull/1059) | 2026-11-03 | `22702c34a0bc` | Measurement workflow canceled; cause not established ([run](https://github.com/mboworks/xff/actions/runs/41559497818)) | +| [#1057](https://github.com/mboworks/xff/pull/1057) | 2026-11-02 | `6bebe940ba36` | Measurement workflow canceled; cause not established ([run](https://github.com/mboworks/xff/actions/runs/41524744291)) | +| [#1053](https://github.com/mboworks/xff/pull/1053) | 2026-11-02 | `dc4e70237eff` | Measurement workflow canceled; cause not established ([run](https://github.com/mboworks/xff/actions/runs/41451327523)) | +| [#1050](https://github.com/mboworks/xff/pull/1050) | 2026-11-02 | `2d615ea1e447` | Measurement workflow failed; exact cause not audited ([run](https://github.com/mboworks/xff/actions/runs/41425575717)) | +| [#1044](https://github.com/mboworks/xff/pull/1044) | 2026-11-02 | `d546fcb3d864` | Measurement workflow canceled; cause not established ([run](https://github.com/mboworks/xff/actions/runs/41378115229)) | +| [#1042](https://github.com/mboworks/xff/pull/1042) | 2026-11-02 | `2b21790502fd` | Measurement workflow canceled; cause not established ([run](https://github.com/mboworks/xff/actions/runs/41367091220)) | +| [#1027](https://github.com/mboworks/xff/pull/1027) | 2026-10-29 | `d406f4132d06` | Measurement workflow canceled; cause not established ([run](https://github.com/mboworks/xff/actions/runs/40816218296)) | +| [#1011](https://github.com/mboworks/xff/pull/1011) | 2026-10-23 | `801087f3e9a8` | Measurement workflow failed; exact cause not audited ([run](https://github.com/mboworks/xff/actions/runs/39886813900)) | +| [#1007](https://github.com/mboworks/xff/pull/1007) | 2026-10-14 | `33a01e13d7cd` | Measurement workflow canceled; cause not established ([run](https://github.com/mboworks/xff/actions/runs/38653097649)) | +| [#1006](https://github.com/mboworks/xff/pull/1006) | 2026-10-14 | `20095dd53c31` | Measurement workflow failed; exact cause not audited ([run](https://github.com/mboworks/xff/actions/runs/38644322359)) | +| [#994](https://github.com/mboworks/xff/pull/994) | 2026-10-10 | `88d4facd15eb` | Measurement workflow failed; exact cause not audited ([run](https://github.com/mboworks/xff/actions/runs/38049489783)) | +| [#993](https://github.com/mboworks/xff/pull/993) | 2026-10-10 | `dac3747d63ca` | Measurement workflow canceled; cause not established ([run](https://github.com/mboworks/xff/actions/runs/38045263735)) | +| [#986](https://github.com/mboworks/xff/pull/986) | 2026-10-07 | `61ab1ce41300` | Successful run absent from site; artifacts unavailable ([run](https://github.com/mboworks/xff/actions/runs/37689818933)) | +| [#976](https://github.com/mboworks/xff/pull/976) | 2026-10-06 | `73b2cfcd7101` | Measurement workflow canceled; cause not established ([run](https://github.com/mboworks/xff/actions/runs/37393233041)) | +| [#979](https://github.com/mboworks/xff/pull/979) | 2026-10-05 | `378d3540691e` | Measurement workflow canceled; cause not established ([run](https://github.com/mboworks/xff/actions/runs/37388497650)) | +| [#973](https://github.com/mboworks/xff/pull/973) | 2026-10-05 | `dfd5447644f0` | Measurement workflow canceled; cause not established ([run](https://github.com/mboworks/xff/actions/runs/37388437775)) | +| [#954](https://github.com/mboworks/xff/pull/954) | 2026-10-03 | `2817247f59cb` | Measurement workflow failed; exact cause not audited ([run](https://github.com/mboworks/xff/actions/runs/37138060568)) | +| [#950](https://github.com/mboworks/xff/pull/950) | 2026-10-03 | `c262c0a71360` | Measurement workflow failed; exact cause not audited ([run](https://github.com/mboworks/xff/actions/runs/37114542638)) | +| [#940](https://github.com/mboworks/xff/pull/940) | 2026-10-01 | `d2cd1e882428` | Measurement workflow failed; exact cause not audited ([run](https://github.com/mboworks/xff/actions/runs/36926663737)) | +| [#939](https://github.com/mboworks/xff/pull/939) | 2026-10-01 | `cc01d2edbab6` | No main benchmark workflow found | +| [#935](https://github.com/mboworks/xff/pull/935) | 2026-10-01 | `481c32aaf2bd` | Measurement workflow failed; exact cause not audited ([run](https://github.com/mboworks/xff/actions/runs/36831896372)) | +| [#937](https://github.com/mboworks/xff/pull/937) | 2026-10-01 | `c8a5fe9e22c3` | Measurement workflow failed; exact cause not audited ([run](https://github.com/mboworks/xff/actions/runs/36826183983)) | +| [#936](https://github.com/mboworks/xff/pull/936) | 2026-10-01 | `be954012e048` | No main benchmark workflow found | +| [#929](https://github.com/mboworks/xff/pull/929) | 2026-09-27 | `471284c4d8a5` | No main benchmark workflow found | +| [#928](https://github.com/mboworks/xff/pull/928) | 2026-09-27 | `f0dfc3edefc7` | No main benchmark workflow found | +| [#922](https://github.com/mboworks/xff/pull/922) | 2026-09-27 | `5db1240146be` | No main benchmark workflow found | +| [#921](https://github.com/mboworks/xff/pull/921) | 2026-09-27 | `df460ae32bc2` | No main benchmark workflow found | +| [#920](https://github.com/mboworks/xff/pull/920) | 2026-09-27 | `5a327394fbbb` | No main benchmark workflow found | +| [#919](https://github.com/mboworks/xff/pull/919) | 2026-09-27 | `7c8164de5b06` | No main benchmark workflow found | +| [#918](https://github.com/mboworks/xff/pull/918) | 2026-09-27 | `97c5e68ac92d` | No main benchmark workflow found | +| [#917](https://github.com/mboworks/xff/pull/917) | 2026-09-27 | `4ef35c0305de` | No main benchmark workflow found | +| [#916](https://github.com/mboworks/xff/pull/916) | 2026-09-27 | `ce5ee3c40579` | No main benchmark workflow found | +| [#914](https://github.com/mboworks/xff/pull/914) | 2026-09-24 | `9bbee34da11e` | Successful run absent from site; artifacts unavailable ([run](https://github.com/mboworks/xff/actions/runs/36068976975)) | +| [#902](https://github.com/mboworks/xff/pull/902) | 2026-09-22 | `be2f400392dc` | Retained historical report lacks a viewer catalog entry; format/eligibility not audited ([run](https://github.com/mboworks/xff/actions/runs/35677724823)) | +| [#904](https://github.com/mboworks/xff/pull/904) | 2026-09-21 | `80d75576c32b` | Retained historical report lacks a viewer catalog entry; format/eligibility not audited ([run](https://github.com/mboworks/xff/actions/runs/35667805574)) | +| [#905](https://github.com/mboworks/xff/pull/905) | 2026-09-21 | `77c00ec8cbea` | Retained historical report lacks a viewer catalog entry; format/eligibility not audited ([run](https://github.com/mboworks/xff/actions/runs/35666152368)) | +| [#903](https://github.com/mboworks/xff/pull/903) | 2026-09-21 | `9b04999acb79` | Retained historical report lacks a viewer catalog entry; format/eligibility not audited ([run](https://github.com/mboworks/xff/actions/runs/35664420733)) | +| [#906](https://github.com/mboworks/xff/pull/906) | 2026-09-21 | `79278f51e63c` | Retained historical report lacks a viewer catalog entry; format/eligibility not audited ([run](https://github.com/mboworks/xff/actions/runs/35662475460)) | +| [#901](https://github.com/mboworks/xff/pull/901) | 2026-09-21 | `f76015ff96f8` | Retained historical report lacks a viewer catalog entry; format/eligibility not audited ([run](https://github.com/mboworks/xff/actions/runs/35653591054)) | +| [#899](https://github.com/mboworks/xff/pull/899) | 2026-09-21 | `98d31e1616a7` | Retained historical report lacks a viewer catalog entry; format/eligibility not audited ([run](https://github.com/mboworks/xff/actions/runs/35564630217)) | +| [#898](https://github.com/mboworks/xff/pull/898) | 2026-09-20 | `c0c0b6d39e26` | Retained historical report lacks a viewer catalog entry; format/eligibility not audited ([run](https://github.com/mboworks/xff/actions/runs/35538423416)) | +| [#897](https://github.com/mboworks/xff/pull/897) | 2026-09-20 | `6e882aea6d2c` | Retained historical report lacks a viewer catalog entry; format/eligibility not audited ([run](https://github.com/mboworks/xff/actions/runs/35536193393)) | +| [#896](https://github.com/mboworks/xff/pull/896) | 2026-09-20 | `ced00ac473e6` | Retained historical report lacks a viewer catalog entry; format/eligibility not audited ([run](https://github.com/mboworks/xff/actions/runs/35529346711)) | +| [#895](https://github.com/mboworks/xff/pull/895) | 2026-09-20 | `a185ae12d70d` | Retained historical report lacks a viewer catalog entry; format/eligibility not audited ([run](https://github.com/mboworks/xff/actions/runs/35521619198)) | +| [#894](https://github.com/mboworks/xff/pull/894) | 2026-09-20 | `42febc0e8d0a` | Retained historical report lacks a viewer catalog entry; format/eligibility not audited ([run](https://github.com/mboworks/xff/actions/runs/35510379808)) | +| [#893](https://github.com/mboworks/xff/pull/893) | 2026-09-20 | `29c75be40529` | Retained historical report lacks a viewer catalog entry; format/eligibility not audited ([run](https://github.com/mboworks/xff/actions/runs/35503333615)) | +| [#892](https://github.com/mboworks/xff/pull/892) | 2026-09-20 | `c9b56c2f009e` | Retained historical report lacks a viewer catalog entry; format/eligibility not audited ([run](https://github.com/mboworks/xff/actions/runs/35502743136)) | +| [#891](https://github.com/mboworks/xff/pull/891) | 2026-09-20 | `a39758eff8ac` | Retained historical report lacks a viewer catalog entry; format/eligibility not audited ([run](https://github.com/mboworks/xff/actions/runs/35499899635)) | +| [#890](https://github.com/mboworks/xff/pull/890) | 2026-09-19 | `1f568ceecd28` | Retained historical report lacks a viewer catalog entry; format/eligibility not audited ([run](https://github.com/mboworks/xff/actions/runs/35474269925)) | +| [#888](https://github.com/mboworks/xff/pull/888) | 2026-09-19 | `b7d28411a9cc` | Retained historical report lacks a viewer catalog entry; format/eligibility not audited ([run](https://github.com/mboworks/xff/actions/runs/35473674929)) | +| [#889](https://github.com/mboworks/xff/pull/889) | 2026-09-19 | `1945c9ec032f` | No main benchmark workflow found | +| [#887](https://github.com/mboworks/xff/pull/887) | 2026-09-19 | `0bcb461dbb08` | Retained historical report lacks a viewer catalog entry; format/eligibility not audited ([run](https://github.com/mboworks/xff/actions/runs/35470455554)) | +| [#886](https://github.com/mboworks/xff/pull/886) | 2026-09-19 | `2bd76d6e682f` | No main benchmark workflow found | +| [#885](https://github.com/mboworks/xff/pull/885) | 2026-09-19 | `2448257a8d4d` | No main benchmark workflow found | +| [#884](https://github.com/mboworks/xff/pull/884) | 2026-09-19 | `4fc594246067` | No main benchmark workflow found | +| [#883](https://github.com/mboworks/xff/pull/883) | 2026-09-19 | `b1df434503a3` | No main benchmark workflow found | +| [#882](https://github.com/mboworks/xff/pull/882) | 2026-09-19 | `9282e3d1da7c` | No main benchmark workflow found | +| [#881](https://github.com/mboworks/xff/pull/881) | 2026-09-19 | `990045bf2d3b` | No main benchmark workflow found | +| [#880](https://github.com/mboworks/xff/pull/880) | 2026-09-19 | `c8b35dfa8249` | No main benchmark workflow found | +| [#879](https://github.com/mboworks/xff/pull/879) | 2026-09-19 | `7f48d0d888cb` | No main benchmark workflow found | +| [#878](https://github.com/mboworks/xff/pull/878) | 2026-09-19 | `889c652ec8cf` | No main benchmark workflow found | +| [#877](https://github.com/mboworks/xff/pull/877) | 2026-09-19 | `99b78c8f1bad` | No main benchmark workflow found | +| [#876](https://github.com/mboworks/xff/pull/876) | 2026-09-19 | `e1cc1f72605e` | No main benchmark workflow found | +| [#868](https://github.com/mboworks/xff/pull/868) | 2026-09-19 | `c092e8245ff8` | No main benchmark workflow found | +| [#875](https://github.com/mboworks/xff/pull/875) | 2026-09-19 | `9197abc259ba` | No main benchmark workflow found | +| [#874](https://github.com/mboworks/xff/pull/874) | 2026-09-19 | `a61f320479c9` | No main benchmark workflow found | +| [#873](https://github.com/mboworks/xff/pull/873) | 2026-09-19 | `13ba8e9474be` | No main benchmark workflow found | +| [#872](https://github.com/mboworks/xff/pull/872) | 2026-09-19 | `ecc9b1f7868a` | No main benchmark workflow found | +| [#871](https://github.com/mboworks/xff/pull/871) | 2026-09-19 | `05140bb49bf5` | No main benchmark workflow found | +| [#870](https://github.com/mboworks/xff/pull/870) | 2026-09-19 | `c7aa7ebea9b4` | No main benchmark workflow found | +| [#869](https://github.com/mboworks/xff/pull/869) | 2026-09-19 | `41e18f2748ff` | No main benchmark workflow found | + +## What the audit shows + +- PR #1143 is represented on Linux and macOS; it is labeled by main and commit, not by PR number. +- The publisher reconciles available successful artifacts, but cannot recreate a run that + never produced a final report or whose artifact expired. +- Main benchmark runs previously used a single pending concurrency slot; #1138 changed + the workflow to a 100-entry queue. That does not by itself retry failed or canceled runs, + and an exhausted queue can still lose requests. Individual cancellations were not audited. +- The existing backfill planner selects commits already present in retained history. Its + campaign requires every job to succeed before any result becomes visible; it is not a + durable missing-PR queue. +- PR #1147 proposes explicit revisions for backfill, but that remains manual and does not + establish automatic completeness. + +## Proposed prevention + +1. Maintain a durable inventory keyed by merged PR number, merge commit, and platform. + Discover expected entries from trusted merged PR metadata, independently of workflow + events and retained reports. Store merge and tree identity, discovery time, state, + attempt identities, errors and retry time. Reconcile this inventory periodically. +2. For each platform, select a compatible retained measurement whose measured commit or + verified tree matches the merged PR. Accept PR measurements only after proving the + measured tree matches the merge result; otherwise schedule a new measurement. Keep + actual measurement and source identities and show backfill provenance in the viewer. +3. Expose a PR-labeled entry in the viewer for every merge. Show pending, running, failed + and retrying states instead of silently omitting unmeasured PRs. Do not plot a point + without a completed, validated measurement. A PR may be visible while coverage is + incomplete on one or both platforms. +4. Reconcile completed artifacts independently per PR and platform. Retry missing, + failed, canceled, expired and queue-rejected work with bounded concurrency and + backoff. Make retries idempotent and retain successful platforms even if another fails. + No workflow trigger or finite GitHub queue may be the only record of required work. +5. Limit remeasurement cost by reusing a trusted, compatible existing measurement when + it proves the same merge tree and benchmark contract; otherwise run one targeted + job for each missing platform. Do not equate different trees or measurement contracts. +6. Publish catalog and status from the durable inventory under the shared site lock, + coalesce redundant publishing requests, and report backlog age, failures, artifact + expiry and stalled progress. After a successful publisher, confirm every eligible + retained result is indexed; retry publication independently of measurement. + +## Decisions and limits + +- Decide the supported historical starting point and platforms before scheduling a + complete backfill. The table includes older PRs whose original comparison contract or + reference tools may differ from the current contract. +- A successful backfill is a new measurement with its own host and method, not a + reconstruction of the original run. Preserve that distinction and do not mix incompatible + series silently. +- The implementation needs a persistent scheduler and partial campaign retention; + merely increasing the concurrency queue, adding retries to one workflow, or adding + PR labels cannot guarantee eventual coverage. From 83af157994b16dbe556ffbd3bdc04bc18cfa8c11 Mon Sep 17 00:00:00 2001 From: helly25 <6420169+helly25@users.noreply.github.com> Date: Sun, 8 Nov 2026 15:47:47 +0000 Subject: [PATCH 09/13] Add merged PR benchmark recovery inventory --- docs/benchmark-merged-pr-coverage.md | 25 ++++++ tools/BUILD.bazel | 20 +++++ tools/benchmark_landscape.py | 4 + tools/benchmark_landscape_test.py | 17 ++++ tools/benchmark_recovery.py | 127 +++++++++++++++++++++++++++ tools/benchmark_recovery_test.py | 98 +++++++++++++++++++++ 6 files changed, 291 insertions(+) create mode 100644 tools/benchmark_recovery.py create mode 100644 tools/benchmark_recovery_test.py diff --git a/docs/benchmark-merged-pr-coverage.md b/docs/benchmark-merged-pr-coverage.md index 39337479dd7..c164951f8dc 100644 --- a/docs/benchmark-merged-pr-coverage.md +++ b/docs/benchmark-merged-pr-coverage.md @@ -152,6 +152,31 @@ Older successful reports can exist without the viewer data required by the curre expiry and stalled progress. After a successful publisher, confirm every eligible retained result is indexed; retry publication independently of measurement. +## Recovery command + +After PR #1147 has deployed the targeted revision input, run: + +```sh +bazel run //tools:benchmark_recovery -- --repository mboworks/xff +``` + +The command compares merged PRs with the current CI viewer catalog and reports missing +platforms, active main measurements and retained artifacts awaiting publication. The +default starting date is September 19, 2026; use `--since=YYYY-MM-DD` to select another +inclusive date, or `--pr=986` to inspect one PR. It reads GitHub through the authenticated +`gh` CLI. An available artifact still needs validation and successful publication. + +Add `--dispatch-next` to dispatch the oldest PR requiring new measurements. It starts +one targeted backfill campaign for both platforms and refuses if another backfill is +active or queued. Run the inventory again after that campaign and its publisher finish; +repeat until the desired range is covered. This is an operator-driven recovery step, +not the persistent automatic scheduler described above. A concurrent dispatch after +the active-campaign check can still race; avoid starting multiple operators at once. + +The existing backfill campaign remains all-or-nothing. A failed platform prevents the +campaign from being published, and retrying the PR currently reruns both platforms. +The command does not cancel a pending publisher or automatically retry a failed run. + ## Decisions and limits - Decide the supported historical starting point and platforms before scheduling a diff --git a/tools/BUILD.bazel b/tools/BUILD.bazel index 29c83b11875..9f7740d8501 100644 --- a/tools/BUILD.bazel +++ b/tools/BUILD.bazel @@ -1382,6 +1382,26 @@ py_binary( ], ) +py_library( + name = "benchmark_recovery_lib", + srcs = ["benchmark_recovery.py"], + imports = ["."], + deps = [":benchmark_preview_ci_lib"], +) + +py_binary( + name = "benchmark_recovery", + srcs = ["benchmark_recovery.py"], + deps = [":benchmark_preview_ci_lib"], +) + +py_test( + name = "benchmark_recovery_test", + size = "small", + srcs = ["benchmark_recovery_test.py"], + deps = [":benchmark_recovery_lib"], +) + py_test( name = "benchmark_preview_ci_test", size = "small", diff --git a/tools/benchmark_landscape.py b/tools/benchmark_landscape.py index 514e80488ca..d69941b0da6 100644 --- a/tools/benchmark_landscape.py +++ b/tools/benchmark_landscape.py @@ -643,6 +643,10 @@ def publish(root, javascript, previews=(), incremental=False, repository='mbowor if benchmark_records.is_backfill(record): label = 'CI backfill / ' + commit[:10] links = version_links.get(commit, []) + if not local and not preview: + pr_links = [link['label'] for link in links if re.fullmatch(r'PR #[0-9]+', link.get('label', ''))] + if pr_links: + label = ', '.join(pr_links) + ' / ' + commit[:10] if preview: number = record.get('pull_number') or source['pull_requests'][0]['number'] links = [{'label': f'PR #{number} preview', 'href': f'https://github.com/{repository}/pull/{number}'}] diff --git a/tools/benchmark_landscape_test.py b/tools/benchmark_landscape_test.py index d824d0e70ac..f3f25db7932 100644 --- a/tools/benchmark_landscape_test.py +++ b/tools/benchmark_landscape_test.py @@ -91,6 +91,23 @@ def test_layout_details_distinguish_legacy_and_versioned_datasets(self): {'name': 'broad', 'revision': 2}, {'name': 'deep', 'revision': 2}, ]}), 'broad/v2 + deep/v2') + def test_merged_version_uses_pr_label_from_version_links(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / 'index.html').write_text('

History

') + folder = root / 'runs/1/1/linux' + folder.mkdir(parents=True) + sha = 'a' * 40 + (folder / 'report.json').write_text(json.dumps(dict(tool_comparisons=report(), platform='linux', + head=sha, source=dict(id=1, run_attempt=1, head_branch='main', event='push', + created_at='2026-09-01T00:00:00Z')))) + (folder / 'index.html').write_text('

Main

') + (root / 'version-links.json').write_text(json.dumps({sha: [ + {'label': 'PR #1143', 'href': 'https://github.com/owner/repo/pull/1143'}]})) + landscape.publish(root, '/* renderer */') + catalog = json.loads((root / 'catalog.json').read_text()) + self.assertEqual(catalog[0]['label'], 'PR #1143 / ' + sha[:10]) + def test_incremental_preview_publication_preserves_merged_pages_and_removes_stale_selection(self): with tempfile.TemporaryDirectory() as directory: root = Path(directory) diff --git a/tools/benchmark_recovery.py b/tools/benchmark_recovery.py new file mode 100644 index 00000000000..ed1e2bc2ba9 --- /dev/null +++ b/tools/benchmark_recovery.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) M. Boerger, the MBO Works authors +# SPDX-License-Identifier: Apache-2.0 +"""Find missing merged PR benchmarks and start one targeted CI backfill at a time.""" + +import argparse +import base64 +from datetime import date +import io +import json +import re +import subprocess +import zipfile + +import benchmark_preview_ci + +PLATFORMS = ('linux', 'macos') + + +def catalog(api): + metadata = api.request('contents/benchmarks/catalog.json?ref=coverage-pages') + if not metadata.get('content') and metadata.get('sha'): + metadata = api.request('git/blobs/' + metadata['sha']) + if metadata.get('encoding') != 'base64' or not metadata.get('content'): + raise ValueError('published benchmark catalog is unavailable') + value = json.loads(base64.b64decode(metadata['content'], validate=False)) + if not isinstance(value, list): + raise ValueError('invalid published benchmark catalog') + return value + + +def inventory(api, since, number=None): + entries = catalog(api) + covered = set() + for entry in entries: + if entry.get('source') != 'merged' or entry.get('local'): + continue + platform = entry.get('platform_name', '').split(' / ')[0].lower() + sha = entry.get('commit', '') + if platform in PLATFORMS and re.fullmatch(r'[0-9a-f]{40}', sha): + covered.add((sha, platform)) + pulls = api.pages('pulls?state=closed&sort=updated&direction=desc') + runs = api.pages('actions/workflows/benchmarks.yml/runs?branch=main', 'workflow_runs') + by_commit = {} + for run in runs: + if run.get('event') == 'push': + by_commit.setdefault(run['head_sha'], []).append(run) + pending_backfills = set() + campaigns = api.pages('actions/workflows/benchmark_backfill.yml/runs?branch=main&status=success', 'workflow_runs') + for run in campaigns: + artifacts = api.pages(f"actions/runs/{run['id']}/artifacts", 'artifacts') + available = {item['name']: item for item in artifacts if not item.get('expired')} + plan = available.get('benchmark-backfill-plan') + if not plan: + continue + raw = api.request(f"actions/artifacts/{plan['id']}/zip", raw=True) + if len(raw) > 16 * 1024 * 1024: + raise ValueError('backfill plan artifact exceeds 16 MiB') + with zipfile.ZipFile(io.BytesIO(raw)) as archive: + entry = archive.getinfo('campaign.json') + if entry.file_size > 16 * 1024 * 1024: + raise ValueError('backfill plan exceeds 16 MiB') + manifest = json.loads(archive.read(entry)) + jobs = manifest['jobs'] + if all('benchmark-backfill-' + job['id'] in available for job in jobs): + pending_backfills.update((job['revision']['sha'], job['platform']) for job in jobs) + rows = [] + for pull in pulls: + if number is not None and pull['number'] != number: + continue + merged = pull.get('merged_at') + sha = pull.get('merge_commit_sha') + if not merged or merged[:10] < since or pull.get('base', {}).get('ref') != 'main': + continue + if not isinstance(sha, str) or not re.fullmatch(r'[0-9a-f]{40}', sha): + raise ValueError(f"PR {pull['number']} has no valid merge commit") + missing = [platform for platform in PLATFORMS if (sha, platform) not in covered] + if not missing: + continue + source_runs = by_commit.get(sha, []) + active = any(run['status'] != 'completed' for run in source_runs) + available = {platform for platform in PLATFORMS if (sha, platform) in pending_backfills} + for run in source_runs: + if run.get('conclusion') != 'success': + continue + artifacts = api.pages(f"actions/runs/{run['id']}/artifacts", 'artifacts') + for artifact in artifacts: + match = re.fullmatch(r'benchmark-report-(linux|macos)-[A-Za-z0-9_.-]+', artifact['name']) + if match and not artifact.get('expired') and artifact['created_at'] >= run['run_started_at']: + available.add(match.group(1)) + state = ('measuring' if active else 'awaiting publication' if set(missing) <= available + else 'needs backfill') + rows.append({'pr': pull['number'], 'commit': sha, 'merged_at': merged, + 'missing': missing, 'available_artifacts': sorted(available), 'state': state}) + return sorted(rows, key=lambda row: (row['merged_at'], row['pr'])) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--repository', required=True, help='GitHub owner/repository') + parser.add_argument('--since', default='2026-09-19', help='Inclusive merge date (YYYY-MM-DD)') + parser.add_argument('--pr', type=int, help='Limit the inventory to one PR number') + parser.add_argument('--dispatch-next', action='store_true', + help='Start one missing PR backfill after checking for an active campaign') + args = parser.parse_args() + date.fromisoformat(args.since) + api = benchmark_preview_ci.GitHub(args.repository) + rows = inventory(api, args.since, args.pr) + print(json.dumps({'since': args.since, 'missing': rows}, indent=2), flush=True) + if not args.dispatch_next: + return + candidates = [row for row in rows if row['state'] == 'needs backfill'] + if not candidates: + print('No PR requires backfill.', flush=True) + return + campaigns = api.pages('actions/workflows/benchmark_backfill.yml/runs?branch=main', 'workflow_runs') + if any(run['status'] != 'completed' for run in campaigns): + raise ValueError('a backfill campaign is already active or queued; retry after it finishes') + selected = candidates[0] + # The deployed workflow must support the revisions input added by PR #1147. + subprocess.run(['gh', 'workflow', 'run', 'benchmark_backfill.yml', '--repo', api.repository, + '--ref', 'main', '-f', 'revisions=' + selected['commit']], check=True) + print(f"Dispatched PR #{selected['pr']} ({selected['commit']}) for both platforms.", flush=True) + + +if __name__ == '__main__': + main() diff --git a/tools/benchmark_recovery_test.py b/tools/benchmark_recovery_test.py new file mode 100644 index 00000000000..d994c208f0f --- /dev/null +++ b/tools/benchmark_recovery_test.py @@ -0,0 +1,98 @@ +# SPDX-FileCopyrightText: Copyright (c) M. Boerger, the MBO Works authors +# SPDX-License-Identifier: Apache-2.0 + +import base64 +import io +import json +import unittest +from unittest import mock +import zipfile + +import benchmark_recovery as recovery + +SHA = 'a' * 40 +OTHER = 'b' * 40 + + +class FakeGitHub: + repository = 'owner/repo' + + def __init__(self): + self.pulls = [{'number': 7, 'merged_at': '2026-10-01T12:00:00Z', 'merge_commit_sha': SHA, + 'base': {'ref': 'main'}}] + self.runs = [] + self.campaigns = [] + self.artifacts = {} + self.archives = {} + self.entries = [] + + def request(self, path, raw=False): + if path.startswith('contents/'): + return {'encoding': 'base64', 'content': base64.b64encode(json.dumps(self.entries).encode()).decode()} + return self.archives[path] + + def pages(self, path, key=None): + if path.startswith('pulls?'): + return self.pulls + if 'benchmark_backfill.yml' in path: + return self.campaigns + if 'benchmarks.yml' in path: + return self.runs + return self.artifacts[path] + + +class BenchmarkRecoveryTest(unittest.TestCase): + def test_pr_missing_one_platform_is_reported(self): + api = FakeGitHub() + api.entries = [{'source': 'merged', 'commit': SHA, 'platform_name': 'linux / x86_64'}] + self.assertEqual(recovery.inventory(api, '2026-09-19')[0]['missing'], ['macos']) + + def test_active_run_and_available_reports_are_not_backfilled(self): + api = FakeGitHub() + api.runs = [{'id': 1, 'head_sha': SHA, 'event': 'push', 'status': 'in_progress', 'conclusion': None}] + self.assertEqual(recovery.inventory(api, '2026-09-19')[0]['state'], 'measuring') + api.runs[0].update(status='completed', conclusion='success', run_started_at='2026-10-01T12:00:00Z') + api.artifacts['actions/runs/1/artifacts'] = [ + {'name': f'benchmark-report-{platform}-arch', 'created_at': '2026-10-01T13:00:00Z', 'expired': False} + for platform in recovery.PLATFORMS] + self.assertEqual(recovery.inventory(api, '2026-09-19')[0]['state'], 'awaiting publication') + + def test_retained_completed_campaign_is_not_dispatched_again(self): + api = FakeGitHub() + api.campaigns = [{'id': 2}] + jobs = [{'id': platform + '-' + SHA, 'revision': {'sha': SHA}, 'platform': platform} + for platform in recovery.PLATFORMS] + archive = io.BytesIO() + with zipfile.ZipFile(archive, 'w') as output: + output.writestr('campaign.json', json.dumps({'jobs': jobs})) + api.archives['actions/artifacts/3/zip'] = archive.getvalue() + api.artifacts['actions/runs/2/artifacts'] = [{'id': 3, 'name': 'benchmark-backfill-plan'}] + [ + {'name': 'benchmark-backfill-' + job['id']} for job in jobs] + self.assertEqual(recovery.inventory(api, '2026-09-19')[0]['state'], 'awaiting publication') + + def test_dispatch_refuses_active_campaign(self): + api = FakeGitHub() + with mock.patch.object(recovery.benchmark_preview_ci, 'GitHub', return_value=api), \ + mock.patch.object(recovery, 'inventory', return_value=[{'state': 'needs backfill'}]), \ + mock.patch.object(api, 'pages', return_value=[{'status': 'queued'}]), \ + mock.patch.object(recovery.subprocess, 'run') as run, \ + mock.patch('sys.argv', ['benchmark_recovery', '--repository', 'owner/repo', '--dispatch-next']): + with self.assertRaisesRegex(ValueError, 'already active'): + recovery.main() + run.assert_not_called() + + def test_dispatch_uses_the_targeted_main_workflow(self): + api = FakeGitHub() + selected = {'state': 'needs backfill', 'pr': 7, 'commit': SHA} + with mock.patch.object(recovery.benchmark_preview_ci, 'GitHub', return_value=api), \ + mock.patch.object(recovery, 'inventory', return_value=[selected]), \ + mock.patch.object(api, 'pages', return_value=[]), \ + mock.patch.object(recovery.subprocess, 'run') as run, \ + mock.patch('sys.argv', ['benchmark_recovery', '--repository', 'owner/repo', '--dispatch-next']): + recovery.main() + run.assert_called_once_with(['gh', 'workflow', 'run', 'benchmark_backfill.yml', '--repo', + 'owner/repo', '--ref', 'main', '-f', 'revisions=' + SHA], check=True) + + +if __name__ == '__main__': + unittest.main() From 7424d3b5f73a8cee2d8604adbbc971313fa806b0 Mon Sep 17 00:00:00 2001 From: helly25 <6420169+helly25@users.noreply.github.com> Date: Sun, 8 Nov 2026 15:52:20 +0000 Subject: [PATCH 10/13] Retain completed backfill results independently --- docs/benchmark-merged-pr-coverage.md | 20 ++++++----- tools/benchmark_campaign.py | 53 ++++++++++++++++------------ tools/benchmark_campaign_test.py | 22 ++++++++++++ tools/benchmark_history_test.py | 4 +-- tools/benchmark_reconcile_ci.py | 20 +++++++---- tools/benchmark_recovery.py | 42 +++++++++++++++++----- tools/benchmark_recovery_test.py | 12 +++++-- 7 files changed, 122 insertions(+), 51 deletions(-) diff --git a/docs/benchmark-merged-pr-coverage.md b/docs/benchmark-merged-pr-coverage.md index c164951f8dc..687a165df85 100644 --- a/docs/benchmark-merged-pr-coverage.md +++ b/docs/benchmark-merged-pr-coverage.md @@ -120,9 +120,9 @@ Older successful reports can exist without the viewer data required by the curre - Main benchmark runs previously used a single pending concurrency slot; #1138 changed the workflow to a 100-entry queue. That does not by itself retry failed or canceled runs, and an exhausted queue can still lose requests. Individual cancellations were not audited. -- The existing backfill planner selects commits already present in retained history. Its - campaign requires every job to succeed before any result becomes visible; it is not a - durable missing-PR queue. +- At the time of the audit the backfill planner selected commits already present in retained + history, and every job had to succeed before any result became visible. Neither behavior + provides a durable missing-PR queue. - PR #1147 proposes explicit revisions for backfill, but that remains manual and does not establish automatic completeness. @@ -166,16 +166,18 @@ default starting date is September 19, 2026; use `--since=YYYY-MM-DD` to select inclusive date, or `--pr=986` to inspect one PR. It reads GitHub through the authenticated `gh` CLI. An available artifact still needs validation and successful publication. -Add `--dispatch-next` to dispatch the oldest PR requiring new measurements. It starts -one targeted backfill campaign for both platforms and refuses if another backfill is -active or queued. Run the inventory again after that campaign and its publisher finish; +Add `--dispatch-next` to dispatch up to four of the oldest PRs requiring new measurements. +Use `--batch-size=1` through `--batch-size=8` to change the bound. It starts one targeted +backfill campaign for both platforms and refuses if another backfill is active or queued. +Run the inventory again after that campaign and its publisher finish; repeat until the desired range is covered. This is an operator-driven recovery step, not the persistent automatic scheduler described above. A concurrent dispatch after the active-campaign check can still race; avoid starting multiple operators at once. -The existing backfill campaign remains all-or-nothing. A failed platform prevents the -campaign from being published, and retrying the PR currently reruns both platforms. -The command does not cancel a pending publisher or automatically retry a failed run. +The publisher can retain each complete validated platform from a failed campaign, so a +failed platform does not discard another platform's results. A targeted retry currently +remeasures both platforms. The command does not cancel a pending publisher or +automatically retry a failed run. ## Decisions and limits diff --git a/tools/benchmark_campaign.py b/tools/benchmark_campaign.py index a52324f3dc0..a186264b3dd 100644 --- a/tools/benchmark_campaign.py +++ b/tools/benchmark_campaign.py @@ -170,11 +170,12 @@ def validate_measurements(record, batch, revision, tasks): compare.render(report) # Validate all samples, successful commands and finite measurements. -def retain(root, artifacts, source, repository): - """Validate the entire campaign before making any replacement visible.""" +def retain(root, artifacts, source, repository, allow_partial=False): + """Retain validated campaign records; partial retention is explicitly opt-in.""" require(source['name'] == 'Benchmark backfill' and source['path'] == '.github/workflows/benchmark_backfill.yml' and source['event'] == 'workflow_dispatch' and source['head_branch'] == 'main' - and source['status'] == 'completed' and source['conclusion'] == 'success' + and source['status'] == 'completed' + and (source['conclusion'] == 'success' or allow_partial) and source['repository']['full_name'] == repository, 'expected a successful trusted main backfill workflow') run, attempt = int(source['id']), int(source['run_attempt']) require(run > 0 and attempt > 0, 'invalid source workflow identity') @@ -182,7 +183,9 @@ def retain(root, artifacts, source, repository): validate_plan(plan) require(plan['source_head'] == source['head_sha'], 'campaign source commit mismatch') expected_artifacts = {'benchmark-backfill-plan', *('benchmark-backfill-' + job['id'] for job in plan['jobs'])} - require({path.name for path in artifacts.iterdir()} == expected_artifacts, 'incomplete or unexpected campaign artifacts') + actual_artifacts = {path.name for path in artifacts.iterdir()} + require(actual_artifacts <= expected_artifacts and (allow_partial or actual_artifacts == expected_artifacts), + 'incomplete or unexpected campaign artifacts') trusted = {key: source[key] for key in ('id', 'run_attempt', 'created_at', 'head_sha', 'head_branch', 'event')} trusted['pull_requests'] = [] destination = root / 'backfills' / str(run) / str(attempt) @@ -192,24 +195,30 @@ def retain(root, artifacts, source, repository): originals.setdefault((benchmark_records.platform_key(old), old['head']), []).append(path.relative_to(root).as_posix()) records = [] for job in plan['jobs']: - folder = artifacts / ('benchmark-backfill-' + job['id']) - batch = json.loads((folder / 'batch.json').read_text()) - status = json.loads((folder / 'status.json').read_text()) - sha = job['revision']['sha'] - require(set(status) == {sha} and status[sha]['status'] == 'complete', 'incomplete campaign revision') - reports = list(folder.glob('reports/*/*.json')) - expected = folder / 'reports' / sha / f"benchmark-report-{job['platform']}-{job['arch']}.json" - require(reports == [expected], 'unexpected campaign report set') - record = json.loads(expected.read_text()) - validate_report(record, batch, job, plan) - collection = batch['contract']['collection'] - require(collection['id'] == run and 1 <= collection['run_attempt'] <= attempt - and collection['head_sha'] == source['head_sha'], 'batch source workflow mismatch') - record.update(source=trusted, collection=collection, campaign=plan['identity']) - saved = destination / job['platform'] / sha / 'report.json' - record['original_reports'] = (benchmark_records.read(saved)['original_reports'] if benchmark_records.exists(saved) - else sorted(originals.get((job['platform'], sha), []))) - records.append((job, record, batch, status)) + try: + folder = artifacts / ('benchmark-backfill-' + job['id']) + batch = json.loads((folder / 'batch.json').read_text()) + status = json.loads((folder / 'status.json').read_text()) + sha = job['revision']['sha'] + require(set(status) == {sha} and status[sha]['status'] == 'complete', 'incomplete campaign revision') + reports = list(folder.glob('reports/*/*.json')) + expected = folder / 'reports' / sha / f"benchmark-report-{job['platform']}-{job['arch']}.json" + require(reports == [expected], 'unexpected campaign report set') + record = json.loads(expected.read_text()) + validate_report(record, batch, job, plan) + collection = batch['contract']['collection'] + require(collection['id'] == run and 1 <= collection['run_attempt'] <= attempt + and collection['head_sha'] == source['head_sha'], 'batch source workflow mismatch') + record.update(source=trusted, collection=collection, campaign=plan['identity']) + saved = destination / job['platform'] / sha / 'report.json' + record['original_reports'] = (benchmark_records.read(saved)['original_reports'] if benchmark_records.exists(saved) + else sorted(originals.get((job['platform'], sha), []))) + records.append((job, record, batch, status)) + except (ValueError, KeyError, OSError) as error: + if not allow_partial: + raise + print(f"::warning::backfill {run}/{attempt} {job['id']}: {error}", flush=True) + require(records, 'campaign has no complete validated measurements') payloads = {'campaign.json': plan, 'source.json': trusted} for job, record, batch, status in records: prefix = job['platform'] + '/' + job['revision']['sha'] + '/' diff --git a/tools/benchmark_campaign_test.py b/tools/benchmark_campaign_test.py index 34446ce286c..d52bbe0bea8 100644 --- a/tools/benchmark_campaign_test.py +++ b/tools/benchmark_campaign_test.py @@ -137,6 +137,15 @@ def test_planner_freezes_each_complete_revision_once_per_platform(self): self.assertIn('content-needle', selected['contract']['tasks']) campaign.validate_plan(selected) + def test_planner_uses_explicit_revisions_instead_of_published_history(self): + revision = plan()['jobs'][0]['revision'] + with mock.patch.object(campaign.backfill, 'revisions', return_value=[revision]) as revisions, \ + mock.patch.object(campaign.backfill, 'git', side_effect=[DRIVER, 'c' * 40]): + selected = campaign.make_plan(Path('/repo'), Path('/history'), [HEAD]) + self.assertEqual(len(selected['jobs']), 2) + self.assertEqual([call.args[1:] for call in revisions.call_args_list], + [([HEAD], None, 'linux'), ([HEAD], None, 'macos')]) + def test_plan_rejects_mutation_duplicates_partial_grid_and_missing_platform(self): mutations = [lambda value: value['jobs'].append(value['jobs'][0]), lambda value: value['jobs'].pop(), @@ -233,6 +242,19 @@ def test_missing_artifact_or_failed_revision_never_promotes_a_partial_campaign(s campaign.retain(site, artifacts, source(), 'owner/repo') self.assertFalse(site.exists()) + def test_partial_recovery_retains_only_valid_completed_platforms(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + artifacts = root / 'artifacts' + write_artifacts(artifacts) + folder = next(artifacts.glob('benchmark-backfill-macos-*')) + (folder / 'status.json').write_text(json.dumps({HEAD: {'status': 'measurement-failed'}})) + failed = dict(source(), conclusion='failure') + destination = campaign.retain(root / 'site', artifacts, failed, 'owner/repo', allow_partial=True) + self.assertTrue((destination / 'linux' / HEAD / 'report.json').exists()) + self.assertFalse((destination / 'macos' / HEAD / 'report.json').exists()) + campaign.retain(root / 'site', artifacts, failed, 'owner/repo', allow_partial=True) + def test_report_rejects_missing_tasks_references_samples_and_changed_identity(self): selected = plan() job = selected['jobs'][0] diff --git a/tools/benchmark_history_test.py b/tools/benchmark_history_test.py index 8c7413809c9..e2c4c5e426d 100644 --- a/tools/benchmark_history_test.py +++ b/tools/benchmark_history_test.py @@ -248,8 +248,8 @@ def test_publication_coalesces_requests_and_reconciles_sources(self): self.assertIn('group: benchmark-publication-latest', workflow) reconcile = repository_file('tools/benchmark_reconcile_ci.py').read_text() self.assertIn('benchmark_preview_ci.collect', reconcile) - self.assertIn("('benchmarks.yml', 'push', retain_measurement)", reconcile) - self.assertIn("('benchmark_backfill.yml', 'workflow_dispatch', retain_backfill)", reconcile) + self.assertIn("('benchmarks.yml', 'push', 'success', retain_measurement)", reconcile) + self.assertIn("('benchmark_backfill.yml', 'workflow_dispatch', 'completed', retain_backfill)", reconcile) def test_fifth_candidate_requires_unresolved_fourth_agreement(self): workflow = repository_file('.github/workflows/benchmarks.yml').read_text() diff --git a/tools/benchmark_reconcile_ci.py b/tools/benchmark_reconcile_ci.py index d8850e3a08c..80fdad40c86 100644 --- a/tools/benchmark_reconcile_ci.py +++ b/tools/benchmark_reconcile_ci.py @@ -116,19 +116,25 @@ def retain_backfill(api, root, run): if not re.fullmatch(r'benchmark-backfill-[A-Za-z0-9_.-]+', name): raise ValueError('invalid backfill artifact name') target = directory / name - target.mkdir() - extract(api.request(f"actions/artifacts/{item['id']}/zip", raw=True), target) - benchmark_campaign.retain(root, directory, run, api.repository) + try: + target.mkdir() + extract(api.request(f"actions/artifacts/{item['id']}/zip", raw=True), target) + except (ValueError, OSError, zipfile.BadZipFile, subprocess.CalledProcessError, + subprocess.TimeoutExpired) as error: + if name == 'benchmark-backfill-plan': + raise + print(f"::warning::backfill artifact {name}: {error}", flush=True) + benchmark_campaign.retain(root, directory, run, api.repository, allow_partial=True) return True def reconcile(api, root): since = (datetime.now(timezone.utc) - timedelta(days=30)).date().isoformat() failures = [] - for workflow, event, retain in ( - ('benchmarks.yml', 'push', retain_measurement), - ('benchmark_backfill.yml', 'workflow_dispatch', retain_backfill)): - runs = api.pages(f'actions/workflows/{workflow}/runs?status=success&branch=main&created=%3E%3D{since}', + for workflow, event, status, retain in ( + ('benchmarks.yml', 'push', 'success', retain_measurement), + ('benchmark_backfill.yml', 'workflow_dispatch', 'completed', retain_backfill)): + runs = api.pages(f'actions/workflows/{workflow}/runs?status={status}&branch=main&created=%3E%3D{since}', 'workflow_runs') for run in reversed(runs): if run.get('event') != event or run.get('head_branch') != 'main': diff --git a/tools/benchmark_recovery.py b/tools/benchmark_recovery.py index ed1e2bc2ba9..7331383755f 100644 --- a/tools/benchmark_recovery.py +++ b/tools/benchmark_recovery.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # SPDX-FileCopyrightText: Copyright (c) M. Boerger, the MBO Works authors # SPDX-License-Identifier: Apache-2.0 -"""Find missing merged PR benchmarks and start one targeted CI backfill at a time.""" +"""Find missing merged PR benchmarks and start bounded targeted CI backfills.""" import argparse import base64 @@ -40,13 +40,17 @@ def inventory(api, since, number=None): if platform in PLATFORMS and re.fullmatch(r'[0-9a-f]{40}', sha): covered.add((sha, platform)) pulls = api.pages('pulls?state=closed&sort=updated&direction=desc') + target_shas = {pull.get('merge_commit_sha') for pull in pulls + if pull.get('merged_at') and pull['merged_at'][:10] >= since + and pull.get('base', {}).get('ref') == 'main' + and (number is None or pull['number'] == number)} runs = api.pages('actions/workflows/benchmarks.yml/runs?branch=main', 'workflow_runs') by_commit = {} for run in runs: if run.get('event') == 'push': by_commit.setdefault(run['head_sha'], []).append(run) pending_backfills = set() - campaigns = api.pages('actions/workflows/benchmark_backfill.yml/runs?branch=main&status=success', 'workflow_runs') + campaigns = api.pages('actions/workflows/benchmark_backfill.yml/runs?branch=main&status=completed', 'workflow_runs') for run in campaigns: artifacts = api.pages(f"actions/runs/{run['id']}/artifacts", 'artifacts') available = {item['name']: item for item in artifacts if not item.get('expired')} @@ -62,8 +66,27 @@ def inventory(api, since, number=None): raise ValueError('backfill plan exceeds 16 MiB') manifest = json.loads(archive.read(entry)) jobs = manifest['jobs'] - if all('benchmark-backfill-' + job['id'] in available for job in jobs): - pending_backfills.update((job['revision']['sha'], job['platform']) for job in jobs) + for job in jobs: + sha = job['revision']['sha'] + if sha not in target_shas or (sha, job['platform']) in covered: + continue + item = available.get('benchmark-backfill-' + job['id']) + if not item: + continue + raw = api.request(f"actions/artifacts/{item['id']}/zip", raw=True) + if len(raw) > 128 * 1024 * 1024: + raise ValueError('backfill artifact exceeds 128 MiB') + with zipfile.ZipFile(io.BytesIO(raw)) as archive: + try: + info = archive.getinfo('status.json') + if info.file_size > 16 * 1024 * 1024: + raise ValueError('backfill status exceeds 16 MiB') + status = json.loads(archive.read(info)) + report = f"reports/{sha}/benchmark-report-{job['platform']}-{job['arch']}.json" + if status.get(sha, {}).get('status') == 'complete' and report in archive.namelist(): + pending_backfills.add((sha, job['platform'])) + except (KeyError, ValueError): + continue rows = [] for pull in pulls: if number is not None and pull['number'] != number: @@ -101,9 +124,12 @@ def main(): parser.add_argument('--since', default='2026-09-19', help='Inclusive merge date (YYYY-MM-DD)') parser.add_argument('--pr', type=int, help='Limit the inventory to one PR number') parser.add_argument('--dispatch-next', action='store_true', - help='Start one missing PR backfill after checking for an active campaign') + help='Start a bounded backfill after checking for an active campaign') + parser.add_argument('--batch-size', type=int, default=4, help='Maximum PRs per campaign (1 through 8)') args = parser.parse_args() date.fromisoformat(args.since) + if not 1 <= args.batch_size <= 8: + parser.error('--batch-size must be between 1 and 8') api = benchmark_preview_ci.GitHub(args.repository) rows = inventory(api, args.since, args.pr) print(json.dumps({'since': args.since, 'missing': rows}, indent=2), flush=True) @@ -116,11 +142,11 @@ def main(): campaigns = api.pages('actions/workflows/benchmark_backfill.yml/runs?branch=main', 'workflow_runs') if any(run['status'] != 'completed' for run in campaigns): raise ValueError('a backfill campaign is already active or queued; retry after it finishes') - selected = candidates[0] + selected = candidates[:args.batch_size] # The deployed workflow must support the revisions input added by PR #1147. subprocess.run(['gh', 'workflow', 'run', 'benchmark_backfill.yml', '--repo', api.repository, - '--ref', 'main', '-f', 'revisions=' + selected['commit']], check=True) - print(f"Dispatched PR #{selected['pr']} ({selected['commit']}) for both platforms.", flush=True) + '--ref', 'main', '-f', 'revisions=' + ','.join(row['commit'] for row in selected)], check=True) + print('Dispatched both platforms for ' + ', '.join(f"PR #{row['pr']}" for row in selected) + '.', flush=True) if __name__ == '__main__': diff --git a/tools/benchmark_recovery_test.py b/tools/benchmark_recovery_test.py index d994c208f0f..69d3939d219 100644 --- a/tools/benchmark_recovery_test.py +++ b/tools/benchmark_recovery_test.py @@ -60,14 +60,20 @@ def test_active_run_and_available_reports_are_not_backfilled(self): def test_retained_completed_campaign_is_not_dispatched_again(self): api = FakeGitHub() api.campaigns = [{'id': 2}] - jobs = [{'id': platform + '-' + SHA, 'revision': {'sha': SHA}, 'platform': platform} + jobs = [{'id': platform + '-' + SHA, 'revision': {'sha': SHA}, 'platform': platform, 'arch': 'arch'} for platform in recovery.PLATFORMS] archive = io.BytesIO() with zipfile.ZipFile(archive, 'w') as output: output.writestr('campaign.json', json.dumps({'jobs': jobs})) api.archives['actions/artifacts/3/zip'] = archive.getvalue() - api.artifacts['actions/runs/2/artifacts'] = [{'id': 3, 'name': 'benchmark-backfill-plan'}] + [ - {'name': 'benchmark-backfill-' + job['id']} for job in jobs] + api.artifacts['actions/runs/2/artifacts'] = [{'id': 3, 'name': 'benchmark-backfill-plan'}] + for index, job in enumerate(jobs, 4): + api.artifacts['actions/runs/2/artifacts'].append({'id': index, 'name': 'benchmark-backfill-' + job['id']}) + artifact = io.BytesIO() + with zipfile.ZipFile(artifact, 'w') as output: + output.writestr('status.json', json.dumps({SHA: {'status': 'complete'}})) + output.writestr(f"reports/{SHA}/benchmark-report-{job['platform']}-arch.json", '{}') + api.archives[f'actions/artifacts/{index}/zip'] = artifact.getvalue() self.assertEqual(recovery.inventory(api, '2026-09-19')[0]['state'], 'awaiting publication') def test_dispatch_refuses_active_campaign(self): From b4a1cac4f72e08da3479f642b4ed4eac4650e36b Mon Sep 17 00:00:00 2001 From: helly25 <6420169+helly25@users.noreply.github.com> Date: Sun, 8 Nov 2026 16:26:40 +0000 Subject: [PATCH 11/13] Record performance work after benchmark recovery --- docs/benchmark-merged-pr-coverage.md | 32 ++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/docs/benchmark-merged-pr-coverage.md b/docs/benchmark-merged-pr-coverage.md index 687a165df85..dc360240c04 100644 --- a/docs/benchmark-merged-pr-coverage.md +++ b/docs/benchmark-merged-pr-coverage.md @@ -123,7 +123,7 @@ Older successful reports can exist without the viewer data required by the curre - At the time of the audit the backfill planner selected commits already present in retained history, and every job had to succeed before any result became visible. Neither behavior provides a durable missing-PR queue. -- PR #1147 proposes explicit revisions for backfill, but that remains manual and does not +- Merged PR #1147 provides explicit revisions for backfill, but that remains manual and does not establish automatic completeness. ## Proposed prevention @@ -187,6 +187,34 @@ automatically retry a failed run. - A successful backfill is a new measurement with its own host and method, not a reconstruction of the original run. Preserve that distinction and do not mix incompatible series silently. -- The implementation needs a persistent scheduler and partial campaign retention; +- The implementation still needs a persistent scheduler; merely increasing the concurrency queue, adding retries to one workflow, or adding PR labels cannot guarantee eventual coverage. + +## Continuation after the current campaigns + +The main performance objective is to close the gap to equivalent plain `find` and `rg` +commands. Coverage recovery supports that work by restoring a trustworthy history. + +First let the [#914/#986 campaign](https://github.com/mboworks/xff/actions/runs/42498912253) +and the queued [#1143 campaign](https://github.com/mboworks/xff/actions/runs/42499244750) +finish, and merge PR #1148. Then: + +1. Inspect campaign results and publication, including failures and partial platforms. + Confirm each recovered PR is selectable by PR number in the viewer; workflow success + or a retained report directory alone does not establish that. +2. Recompute this inventory from the live catalog. Recover the remaining missing PRs in + bounded campaigns, preserving existing valid measurements and documenting failures + that need build or measurement fixes. Do not dispatch another campaign while one is + active or queued. +3. Resume concrete optimization of basic discovery and content search, comparing equivalent + output and semantics with `find` and `rg` on Linux and macOS. Use production binaries, + verify their identities, and keep paired measurements with reference tools. The local + investigation requests the full 1/3/10 worker grid; the existing CI backfill contract + remains 1/3. Keep those contracts distinguishable. +4. Develop a few representative combined workloads alongside the simple comparisons. + Select them for real use and matching semantics, rather than for a favorable result. + Establish correctness, repeatability, and sensitivity before adding them to the viewer. +5. Freeze and identify the additional workload contract before scheduling its historical + backfill. Preserve the existing simple comparisons and make compatibility explicit; + a new task must not silently appear as if it had been measured in earlier reports. From bd2b053fdb7ac8f6d1eeeb8fbf2347a9c4fdd302 Mon Sep 17 00:00:00 2001 From: helly25 <6420169+helly25@users.noreply.github.com> Date: Sun, 8 Nov 2026 16:44:49 +0000 Subject: [PATCH 12/13] Use compatible historical Linux backfill runners --- .github/workflows/benchmark_backfill.yml | 16 +++++++++++- docs/benchmark-merged-pr-coverage.md | 31 ++++++++++++++++++++++-- tools/benchmark_campaign.py | 22 +++++++++++++---- tools/benchmark_campaign_test.py | 19 +++++++++++++++ tools/benchmark_recovery.py | 7 ++++-- tools/benchmark_recovery_test.py | 5 ++-- 6 files changed, 88 insertions(+), 12 deletions(-) diff --git a/.github/workflows/benchmark_backfill.yml b/.github/workflows/benchmark_backfill.yml index 44348bffa10..9b09dd4c5c4 100644 --- a/.github/workflows/benchmark_backfill.yml +++ b/.github/workflows/benchmark_backfill.yml @@ -10,6 +10,11 @@ on: description: Comma-separated commit IDs (empty selects published history) required: false type: string + # Select the measurement platforms without changing the driver. + platforms: + description: Comma-separated platforms, linux or macos (empty selects both) + required: false + type: string permissions: contents: read @@ -43,6 +48,7 @@ jobs: id: plan env: REVISIONS: ${{ inputs.revisions }} + PLATFORMS: ${{ inputs.platforms }} run: | set -euo pipefail revision_args=() @@ -53,6 +59,13 @@ jobs: revision_args+=(--revision="${revision}") fi done + IFS=',' read -ra platforms <<< "${PLATFORMS}" + for platform in "${platforms[@]}"; do + platform="${platform//[[:space:]]/}" + if [[ -n "${platform}" ]]; then + revision_args+=(--platform="${platform}") + fi + done matrix="$(python3 tools/benchmark_campaign.py plan --repo=. \ --history-root=benchmark-history/benchmarks --output=campaign.json "${revision_args[@]}")" echo "matrix=${matrix}" >>"${GITHUB_OUTPUT}" @@ -100,7 +113,8 @@ jobs: - name: Install reference tools for this revision run: | if [[ "${RUNNER_OS}" == Linux ]]; then - sudo apt-get update && sudo apt-get install -y findutils ripgrep fzf + # Older LLVM linkers need these Ubuntu 22.04 runtime ABIs. + sudo apt-get update && sudo apt-get install -y findutils ripgrep fzf libicu70 libxml2 else brew install findutils ripgrep fzf fi diff --git a/docs/benchmark-merged-pr-coverage.md b/docs/benchmark-merged-pr-coverage.md index dc360240c04..dd5a9d82742 100644 --- a/docs/benchmark-merged-pr-coverage.md +++ b/docs/benchmark-merged-pr-coverage.md @@ -175,8 +175,9 @@ not the persistent automatic scheduler described above. A concurrent dispatch af the active-campaign check can still race; avoid starting multiple operators at once. The publisher can retain each complete validated platform from a failed campaign, so a -failed platform does not discard another platform's results. A targeted retry currently -remeasures both platforms. The command does not cancel a pending publisher or +failed platform does not discard another platform's results. Recovery selects the union +of missing platforms in a batch; `--pr=NUMBER` or `--batch-size=1` allows a platform-specific +retry without repeating a retained platform. The command does not cancel a pending publisher or automatically retry a failed run. ## Decisions and limits @@ -218,3 +219,29 @@ finish, and merge PR #1148. Then: 5. Freeze and identify the additional workload contract before scheduling its historical backfill. Preserve the existing simple comparisons and make compatibility explicit; a new task must not silently appear as if it had been measured in earlier reports. + +## Historical Linux linker compatibility + +The November 8 campaign reached compilation but failed before any Linux measurement: + +- [PR #986 Linux job](https://github.com/mboworks/xff/actions/runs/42498912253/job/128787591927): + historical `ld.lld` could not load `libicui18n.so.70`. +- [PR #914 Linux job](https://github.com/mboworks/xff/actions/runs/42498912253/job/128787591953): + historical `ld.lld` could not load `libxml2.so.2`. + +Both jobs used `ubuntu-latest`, which resolved to Ubuntu 26.04. New historical Linux +campaigns select `ubuntu-22.04` and explicitly install `libicu70` and `libxml2`. This +preserves each revision's compiler and LTO configuration. The selected runner remains +in the campaign plan, and validation still accepts older plans using `ubuntu-latest`. +The ordinary main benchmark workflow keeps its existing runner selection. + +The backfill dispatch also accepts `platforms=linux` or `platforms=macos`, so failed Linux +jobs can be remeasured in a new campaign without repeating completed macOS work. Earlier +plans without a platform selection still require both platforms, and the new selection +is part of the immutable campaign identity. For this campaign, retry the two original +revisions with `platforms=linux` after the fixed workflow is deployed on main. + +This pin needs a replacement before [GitHub retires Ubuntu 22.04 on April 17, 2027](https://github.com/actions/runner-images/issues/14254). +A replacement must provide the historical build dependencies, preserve build provenance, +and record the measurement environment. Replacing historical compilers would change +what the backfill measures. diff --git a/tools/benchmark_campaign.py b/tools/benchmark_campaign.py index a186264b3dd..0548189fdd3 100644 --- a/tools/benchmark_campaign.py +++ b/tools/benchmark_campaign.py @@ -16,7 +16,9 @@ import benchmark_compare as compare import benchmark_records -PLATFORMS = {'linux': ('ubuntu-latest', 'x86_64'), 'macos': ('macos-latest', 'arm64')} +# Historical LLVM 22/23 linkers require the Ubuntu 22.04 libxml2/ICU ABI. +# Keep the measurement runner pinned instead of changing historical toolchains. +PLATFORMS = {'linux': ('ubuntu-22.04', 'x86_64'), 'macos': ('macos-latest', 'arm64')} FILE_COUNTS = [*compare.DEFAULT_FILE_COUNTS, 20000, 50000, 100000] @@ -29,10 +31,12 @@ def require(condition, message): raise ValueError(message) -def make_plan(repo, history_root, references=None): +def make_plan(repo, history_root, references=None, platforms=None): tools = {'xff': {'path': 'xff'}, 'find': {}, 'rg': {}, 'fzf': {}} jobs = [] for platform, (runner, arch) in PLATFORMS.items(): + if platforms and platform not in platforms: + continue for revision in backfill.revisions(repo, references or [], None if references else history_root, platform): jobs.append({'id': platform + '-' + revision['sha'], 'platform': platform, 'os': runner, 'arch': arch, 'revision': revision}) @@ -43,6 +47,8 @@ def make_plan(repo, history_root, references=None): 'repetitions': 9, 'retained': 7, 'driver': backfill.driver_identity(), 'tasks': {name: sorted(participants) for name, _, participants in compare.scenarios(Path('.'), [], tools)}}} + if platforms: + plan['platforms'] = sorted(set(platforms)) plan['identity'] = identity(plan) validate_plan(plan) return plan @@ -66,11 +72,15 @@ def validate_plan(plan): require(re.fullmatch('[0-9a-f]{40}', sha), 'invalid historical revision') require(datetime.fromisoformat(job['revision']['date']).tzinfo is not None, 'historical commit date requires a timezone') - require(platform in PLATFORMS and (job['os'], job['arch']) == PLATFORMS[platform], + runners = {'ubuntu-latest', 'ubuntu-22.04'} if platform == 'linux' else {'macos-latest'} + require(platform in PLATFORMS and job['os'] in runners and job['arch'] == PLATFORMS[platform][1], 'invalid campaign platform') require(job['id'] == platform + '-' + sha and job['id'] not in ids, 'duplicate or invalid campaign job') ids.add(job['id']) - require({job['platform'] for job in plan['jobs']} == set(PLATFORMS), 'both CI platforms are required') + requested = plan.get('platforms', list(PLATFORMS)) + require(isinstance(requested, list) and requested and set(requested) <= set(PLATFORMS), + 'invalid requested campaign platforms') + require({job['platform'] for job in plan['jobs']} == set(requested), 'all requested CI platforms are required') def run_job(plan, job_id, repo, output, disk_cache, fixture_parent, resource_launcher=None): @@ -246,6 +256,8 @@ def main(): for name in ('repo', 'history-root', 'output'): plan.add_argument('--' + name, type=Path, required=True) plan.add_argument('--revision', action='append', default=[], help='Measure this revision instead of site history') + plan.add_argument('--platform', choices=tuple(PLATFORMS), action='append', default=[], + help='Measure only selected platforms (default: both)') run = modes.add_parser('run') for name in ('plan', 'repo', 'output', 'fixture-parent'): run.add_argument('--' + name, type=Path, required=True) @@ -259,7 +271,7 @@ def main(): publish.add_argument('--repository', required=True) args = parser.parse_args() if args.mode == 'plan': - result = make_plan(args.repo, args.history_root, args.revision) + result = make_plan(args.repo, args.history_root, args.revision, args.platform) backfill.write_json(args.output, result) print(json.dumps({'include': [{key: job[key] for key in ('id', 'os', 'platform', 'arch')} for job in result['jobs']]})) diff --git a/tools/benchmark_campaign_test.py b/tools/benchmark_campaign_test.py index d52bbe0bea8..9a59a4e563a 100644 --- a/tools/benchmark_campaign_test.py +++ b/tools/benchmark_campaign_test.py @@ -146,6 +146,25 @@ def test_planner_uses_explicit_revisions_instead_of_published_history(self): self.assertEqual([call.args[1:] for call in revisions.call_args_list], [([HEAD], None, 'linux'), ([HEAD], None, 'macos')]) + def test_historical_linux_runner_is_pinned_and_earlier_plans_remain_valid(self): + self.assertEqual(campaign.PLATFORMS['linux'], ('ubuntu-22.04', 'x86_64')) + selected = plan() + for job in selected['jobs']: + if job['platform'] == 'linux': + job['os'] = 'ubuntu-latest' + selected['identity'] = campaign.identity({key: value for key, value in selected.items() if key != 'identity'}) + campaign.validate_plan(selected) + + def test_explicit_linux_only_retry_does_not_schedule_macos(self): + revision = plan()['jobs'][0]['revision'] + with mock.patch.object(campaign.backfill, 'revisions', return_value=[revision]), \ + mock.patch.object(campaign.backfill, 'git', side_effect=[DRIVER, 'c' * 40]): + selected = campaign.make_plan(Path('/repo'), Path('/history'), [HEAD], ['linux']) + self.assertEqual(selected['platforms'], ['linux']) + self.assertEqual([(job['platform'], job['os']) for job in selected['jobs']], + [('linux', 'ubuntu-22.04')]) + campaign.validate_plan(selected) + def test_plan_rejects_mutation_duplicates_partial_grid_and_missing_platform(self): mutations = [lambda value: value['jobs'].append(value['jobs'][0]), lambda value: value['jobs'].pop(), diff --git a/tools/benchmark_recovery.py b/tools/benchmark_recovery.py index 7331383755f..ef8e2ab0d05 100644 --- a/tools/benchmark_recovery.py +++ b/tools/benchmark_recovery.py @@ -143,10 +143,13 @@ def main(): if any(run['status'] != 'completed' for run in campaigns): raise ValueError('a backfill campaign is already active or queued; retry after it finishes') selected = candidates[:args.batch_size] + platforms = sorted({platform for row in selected for platform in row['missing']}) # The deployed workflow must support the revisions input added by PR #1147. subprocess.run(['gh', 'workflow', 'run', 'benchmark_backfill.yml', '--repo', api.repository, - '--ref', 'main', '-f', 'revisions=' + ','.join(row['commit'] for row in selected)], check=True) - print('Dispatched both platforms for ' + ', '.join(f"PR #{row['pr']}" for row in selected) + '.', flush=True) + '--ref', 'main', '-f', 'revisions=' + ','.join(row['commit'] for row in selected), + '-f', 'platforms=' + ','.join(platforms)], check=True) + print('Dispatched ' + ', '.join(platforms) + ' for ' + + ', '.join(f"PR #{row['pr']}" for row in selected) + '.', flush=True) if __name__ == '__main__': diff --git a/tools/benchmark_recovery_test.py b/tools/benchmark_recovery_test.py index 69d3939d219..2434643b7c9 100644 --- a/tools/benchmark_recovery_test.py +++ b/tools/benchmark_recovery_test.py @@ -89,7 +89,7 @@ def test_dispatch_refuses_active_campaign(self): def test_dispatch_uses_the_targeted_main_workflow(self): api = FakeGitHub() - selected = {'state': 'needs backfill', 'pr': 7, 'commit': SHA} + selected = {'state': 'needs backfill', 'pr': 7, 'commit': SHA, 'missing': ['linux']} with mock.patch.object(recovery.benchmark_preview_ci, 'GitHub', return_value=api), \ mock.patch.object(recovery, 'inventory', return_value=[selected]), \ mock.patch.object(api, 'pages', return_value=[]), \ @@ -97,7 +97,8 @@ def test_dispatch_uses_the_targeted_main_workflow(self): mock.patch('sys.argv', ['benchmark_recovery', '--repository', 'owner/repo', '--dispatch-next']): recovery.main() run.assert_called_once_with(['gh', 'workflow', 'run', 'benchmark_backfill.yml', '--repo', - 'owner/repo', '--ref', 'main', '-f', 'revisions=' + SHA], check=True) + 'owner/repo', '--ref', 'main', '-f', 'revisions=' + SHA, + '-f', 'platforms=linux'], check=True) if __name__ == '__main__': From 70804911ce173e6ce69a06c14d0b102b581041c3 Mon Sep 17 00:00:00 2001 From: helly25 <6420169+helly25@users.noreply.github.com> Date: Sun, 8 Nov 2026 16:45:39 +0000 Subject: [PATCH 13/13] Revert "Use compatible historical Linux backfill runners" This reverts commit bd2b053fdb7ac8f6d1eeeb8fbf2347a9c4fdd302. --- .github/workflows/benchmark_backfill.yml | 16 +----------- docs/benchmark-merged-pr-coverage.md | 31 ++---------------------- tools/benchmark_campaign.py | 22 ++++------------- tools/benchmark_campaign_test.py | 19 --------------- tools/benchmark_recovery.py | 7 ++---- tools/benchmark_recovery_test.py | 5 ++-- 6 files changed, 12 insertions(+), 88 deletions(-) diff --git a/.github/workflows/benchmark_backfill.yml b/.github/workflows/benchmark_backfill.yml index 9b09dd4c5c4..44348bffa10 100644 --- a/.github/workflows/benchmark_backfill.yml +++ b/.github/workflows/benchmark_backfill.yml @@ -10,11 +10,6 @@ on: description: Comma-separated commit IDs (empty selects published history) required: false type: string - # Select the measurement platforms without changing the driver. - platforms: - description: Comma-separated platforms, linux or macos (empty selects both) - required: false - type: string permissions: contents: read @@ -48,7 +43,6 @@ jobs: id: plan env: REVISIONS: ${{ inputs.revisions }} - PLATFORMS: ${{ inputs.platforms }} run: | set -euo pipefail revision_args=() @@ -59,13 +53,6 @@ jobs: revision_args+=(--revision="${revision}") fi done - IFS=',' read -ra platforms <<< "${PLATFORMS}" - for platform in "${platforms[@]}"; do - platform="${platform//[[:space:]]/}" - if [[ -n "${platform}" ]]; then - revision_args+=(--platform="${platform}") - fi - done matrix="$(python3 tools/benchmark_campaign.py plan --repo=. \ --history-root=benchmark-history/benchmarks --output=campaign.json "${revision_args[@]}")" echo "matrix=${matrix}" >>"${GITHUB_OUTPUT}" @@ -113,8 +100,7 @@ jobs: - name: Install reference tools for this revision run: | if [[ "${RUNNER_OS}" == Linux ]]; then - # Older LLVM linkers need these Ubuntu 22.04 runtime ABIs. - sudo apt-get update && sudo apt-get install -y findutils ripgrep fzf libicu70 libxml2 + sudo apt-get update && sudo apt-get install -y findutils ripgrep fzf else brew install findutils ripgrep fzf fi diff --git a/docs/benchmark-merged-pr-coverage.md b/docs/benchmark-merged-pr-coverage.md index dd5a9d82742..dc360240c04 100644 --- a/docs/benchmark-merged-pr-coverage.md +++ b/docs/benchmark-merged-pr-coverage.md @@ -175,9 +175,8 @@ not the persistent automatic scheduler described above. A concurrent dispatch af the active-campaign check can still race; avoid starting multiple operators at once. The publisher can retain each complete validated platform from a failed campaign, so a -failed platform does not discard another platform's results. Recovery selects the union -of missing platforms in a batch; `--pr=NUMBER` or `--batch-size=1` allows a platform-specific -retry without repeating a retained platform. The command does not cancel a pending publisher or +failed platform does not discard another platform's results. A targeted retry currently +remeasures both platforms. The command does not cancel a pending publisher or automatically retry a failed run. ## Decisions and limits @@ -219,29 +218,3 @@ finish, and merge PR #1148. Then: 5. Freeze and identify the additional workload contract before scheduling its historical backfill. Preserve the existing simple comparisons and make compatibility explicit; a new task must not silently appear as if it had been measured in earlier reports. - -## Historical Linux linker compatibility - -The November 8 campaign reached compilation but failed before any Linux measurement: - -- [PR #986 Linux job](https://github.com/mboworks/xff/actions/runs/42498912253/job/128787591927): - historical `ld.lld` could not load `libicui18n.so.70`. -- [PR #914 Linux job](https://github.com/mboworks/xff/actions/runs/42498912253/job/128787591953): - historical `ld.lld` could not load `libxml2.so.2`. - -Both jobs used `ubuntu-latest`, which resolved to Ubuntu 26.04. New historical Linux -campaigns select `ubuntu-22.04` and explicitly install `libicu70` and `libxml2`. This -preserves each revision's compiler and LTO configuration. The selected runner remains -in the campaign plan, and validation still accepts older plans using `ubuntu-latest`. -The ordinary main benchmark workflow keeps its existing runner selection. - -The backfill dispatch also accepts `platforms=linux` or `platforms=macos`, so failed Linux -jobs can be remeasured in a new campaign without repeating completed macOS work. Earlier -plans without a platform selection still require both platforms, and the new selection -is part of the immutable campaign identity. For this campaign, retry the two original -revisions with `platforms=linux` after the fixed workflow is deployed on main. - -This pin needs a replacement before [GitHub retires Ubuntu 22.04 on April 17, 2027](https://github.com/actions/runner-images/issues/14254). -A replacement must provide the historical build dependencies, preserve build provenance, -and record the measurement environment. Replacing historical compilers would change -what the backfill measures. diff --git a/tools/benchmark_campaign.py b/tools/benchmark_campaign.py index 0548189fdd3..a186264b3dd 100644 --- a/tools/benchmark_campaign.py +++ b/tools/benchmark_campaign.py @@ -16,9 +16,7 @@ import benchmark_compare as compare import benchmark_records -# Historical LLVM 22/23 linkers require the Ubuntu 22.04 libxml2/ICU ABI. -# Keep the measurement runner pinned instead of changing historical toolchains. -PLATFORMS = {'linux': ('ubuntu-22.04', 'x86_64'), 'macos': ('macos-latest', 'arm64')} +PLATFORMS = {'linux': ('ubuntu-latest', 'x86_64'), 'macos': ('macos-latest', 'arm64')} FILE_COUNTS = [*compare.DEFAULT_FILE_COUNTS, 20000, 50000, 100000] @@ -31,12 +29,10 @@ def require(condition, message): raise ValueError(message) -def make_plan(repo, history_root, references=None, platforms=None): +def make_plan(repo, history_root, references=None): tools = {'xff': {'path': 'xff'}, 'find': {}, 'rg': {}, 'fzf': {}} jobs = [] for platform, (runner, arch) in PLATFORMS.items(): - if platforms and platform not in platforms: - continue for revision in backfill.revisions(repo, references or [], None if references else history_root, platform): jobs.append({'id': platform + '-' + revision['sha'], 'platform': platform, 'os': runner, 'arch': arch, 'revision': revision}) @@ -47,8 +43,6 @@ def make_plan(repo, history_root, references=None, platforms=None): 'repetitions': 9, 'retained': 7, 'driver': backfill.driver_identity(), 'tasks': {name: sorted(participants) for name, _, participants in compare.scenarios(Path('.'), [], tools)}}} - if platforms: - plan['platforms'] = sorted(set(platforms)) plan['identity'] = identity(plan) validate_plan(plan) return plan @@ -72,15 +66,11 @@ def validate_plan(plan): require(re.fullmatch('[0-9a-f]{40}', sha), 'invalid historical revision') require(datetime.fromisoformat(job['revision']['date']).tzinfo is not None, 'historical commit date requires a timezone') - runners = {'ubuntu-latest', 'ubuntu-22.04'} if platform == 'linux' else {'macos-latest'} - require(platform in PLATFORMS and job['os'] in runners and job['arch'] == PLATFORMS[platform][1], + require(platform in PLATFORMS and (job['os'], job['arch']) == PLATFORMS[platform], 'invalid campaign platform') require(job['id'] == platform + '-' + sha and job['id'] not in ids, 'duplicate or invalid campaign job') ids.add(job['id']) - requested = plan.get('platforms', list(PLATFORMS)) - require(isinstance(requested, list) and requested and set(requested) <= set(PLATFORMS), - 'invalid requested campaign platforms') - require({job['platform'] for job in plan['jobs']} == set(requested), 'all requested CI platforms are required') + require({job['platform'] for job in plan['jobs']} == set(PLATFORMS), 'both CI platforms are required') def run_job(plan, job_id, repo, output, disk_cache, fixture_parent, resource_launcher=None): @@ -256,8 +246,6 @@ def main(): for name in ('repo', 'history-root', 'output'): plan.add_argument('--' + name, type=Path, required=True) plan.add_argument('--revision', action='append', default=[], help='Measure this revision instead of site history') - plan.add_argument('--platform', choices=tuple(PLATFORMS), action='append', default=[], - help='Measure only selected platforms (default: both)') run = modes.add_parser('run') for name in ('plan', 'repo', 'output', 'fixture-parent'): run.add_argument('--' + name, type=Path, required=True) @@ -271,7 +259,7 @@ def main(): publish.add_argument('--repository', required=True) args = parser.parse_args() if args.mode == 'plan': - result = make_plan(args.repo, args.history_root, args.revision, args.platform) + result = make_plan(args.repo, args.history_root, args.revision) backfill.write_json(args.output, result) print(json.dumps({'include': [{key: job[key] for key in ('id', 'os', 'platform', 'arch')} for job in result['jobs']]})) diff --git a/tools/benchmark_campaign_test.py b/tools/benchmark_campaign_test.py index 9a59a4e563a..d52bbe0bea8 100644 --- a/tools/benchmark_campaign_test.py +++ b/tools/benchmark_campaign_test.py @@ -146,25 +146,6 @@ def test_planner_uses_explicit_revisions_instead_of_published_history(self): self.assertEqual([call.args[1:] for call in revisions.call_args_list], [([HEAD], None, 'linux'), ([HEAD], None, 'macos')]) - def test_historical_linux_runner_is_pinned_and_earlier_plans_remain_valid(self): - self.assertEqual(campaign.PLATFORMS['linux'], ('ubuntu-22.04', 'x86_64')) - selected = plan() - for job in selected['jobs']: - if job['platform'] == 'linux': - job['os'] = 'ubuntu-latest' - selected['identity'] = campaign.identity({key: value for key, value in selected.items() if key != 'identity'}) - campaign.validate_plan(selected) - - def test_explicit_linux_only_retry_does_not_schedule_macos(self): - revision = plan()['jobs'][0]['revision'] - with mock.patch.object(campaign.backfill, 'revisions', return_value=[revision]), \ - mock.patch.object(campaign.backfill, 'git', side_effect=[DRIVER, 'c' * 40]): - selected = campaign.make_plan(Path('/repo'), Path('/history'), [HEAD], ['linux']) - self.assertEqual(selected['platforms'], ['linux']) - self.assertEqual([(job['platform'], job['os']) for job in selected['jobs']], - [('linux', 'ubuntu-22.04')]) - campaign.validate_plan(selected) - def test_plan_rejects_mutation_duplicates_partial_grid_and_missing_platform(self): mutations = [lambda value: value['jobs'].append(value['jobs'][0]), lambda value: value['jobs'].pop(), diff --git a/tools/benchmark_recovery.py b/tools/benchmark_recovery.py index ef8e2ab0d05..7331383755f 100644 --- a/tools/benchmark_recovery.py +++ b/tools/benchmark_recovery.py @@ -143,13 +143,10 @@ def main(): if any(run['status'] != 'completed' for run in campaigns): raise ValueError('a backfill campaign is already active or queued; retry after it finishes') selected = candidates[:args.batch_size] - platforms = sorted({platform for row in selected for platform in row['missing']}) # The deployed workflow must support the revisions input added by PR #1147. subprocess.run(['gh', 'workflow', 'run', 'benchmark_backfill.yml', '--repo', api.repository, - '--ref', 'main', '-f', 'revisions=' + ','.join(row['commit'] for row in selected), - '-f', 'platforms=' + ','.join(platforms)], check=True) - print('Dispatched ' + ', '.join(platforms) + ' for ' - + ', '.join(f"PR #{row['pr']}" for row in selected) + '.', flush=True) + '--ref', 'main', '-f', 'revisions=' + ','.join(row['commit'] for row in selected)], check=True) + print('Dispatched both platforms for ' + ', '.join(f"PR #{row['pr']}" for row in selected) + '.', flush=True) if __name__ == '__main__': diff --git a/tools/benchmark_recovery_test.py b/tools/benchmark_recovery_test.py index 2434643b7c9..69d3939d219 100644 --- a/tools/benchmark_recovery_test.py +++ b/tools/benchmark_recovery_test.py @@ -89,7 +89,7 @@ def test_dispatch_refuses_active_campaign(self): def test_dispatch_uses_the_targeted_main_workflow(self): api = FakeGitHub() - selected = {'state': 'needs backfill', 'pr': 7, 'commit': SHA, 'missing': ['linux']} + selected = {'state': 'needs backfill', 'pr': 7, 'commit': SHA} with mock.patch.object(recovery.benchmark_preview_ci, 'GitHub', return_value=api), \ mock.patch.object(recovery, 'inventory', return_value=[selected]), \ mock.patch.object(api, 'pages', return_value=[]), \ @@ -97,8 +97,7 @@ def test_dispatch_uses_the_targeted_main_workflow(self): mock.patch('sys.argv', ['benchmark_recovery', '--repository', 'owner/repo', '--dispatch-next']): recovery.main() run.assert_called_once_with(['gh', 'workflow', 'run', 'benchmark_backfill.yml', '--repo', - 'owner/repo', '--ref', 'main', '-f', 'revisions=' + SHA, - '-f', 'platforms=linux'], check=True) + 'owner/repo', '--ref', 'main', '-f', 'revisions=' + SHA], check=True) if __name__ == '__main__':