diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 9921d232f9..05a830fe2f 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -240,7 +240,9 @@ jobs: measure-fourth-linux: name: Benchmark measurement (linux, shard 3) needs: select-initial - if: needs.select-initial.outputs.linux_fourth == 'true' + if: >- + !cancelled() && needs.select-initial.result == 'success' && + needs.select-initial.outputs.linux_fourth == 'true' runs-on: ubuntu-latest timeout-minutes: 60 permissions: @@ -260,7 +262,9 @@ jobs: measure-fourth-macos: name: Benchmark measurement (macos, shard 3) needs: select-initial - if: needs.select-initial.outputs.macos_fourth == 'true' + if: >- + !cancelled() && needs.select-initial.result == 'success' && + needs.select-initial.outputs.macos_fourth == 'true' runs-on: macos-latest timeout-minutes: 90 permissions: @@ -277,11 +281,48 @@ jobs: shard: 3 head: ${{ github.sha }} + select-fourth: + name: Check benchmark agreement after fourth candidates + needs: [select-initial, measure-fourth-linux, measure-fourth-macos] + if: always() && needs.select-initial.result == 'success' && + (needs.select-initial.outputs.linux_fourth == 'true' || + needs.select-initial.outputs.macos_fourth == 'true') && + needs.measure-fourth-linux.result != 'failure' && + needs.measure-fourth-macos.result != 'failure' + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + outputs: + linux_fifth: ${{ steps.decision.outputs.linux_fifth }} + macos_fifth: ${{ steps.decision.outputs.macos_fifth }} + steps: + - uses: actions/checkout@v7 + with: + ref: ${{ github.sha }} + persist-credentials: false + - uses: actions/setup-python@v7 + with: + python-version: "3.13" + - uses: actions/download-artifact@v8 + with: + pattern: benchmark-shard-* + path: candidate-shards + if-no-artifact-found: ignore + - name: Schedule a fifth candidate only without a three-shard majority + id: decision + run: | + set -euo pipefail + mkdir -p candidate-shards + python3 tools/benchmark_shards.py --decide-adaptive=candidate-shards \ + --decision-stage=fourth | tee -a "${GITHUB_OUTPUT}" + measure-fifth-linux: name: Benchmark measurement (linux, shard 4) - needs: [select-initial, measure-fourth-linux] - if: always() && needs.select-initial.outputs.linux_fourth == 'true' && - needs.measure-fourth-linux.result == 'success' + needs: select-fourth + if: >- + !cancelled() && needs.select-fourth.result == 'success' && + needs.select-fourth.outputs.linux_fifth == 'true' runs-on: ubuntu-latest timeout-minutes: 60 permissions: @@ -300,9 +341,10 @@ jobs: measure-fifth-macos: name: Benchmark measurement (macos, shard 4) - needs: [select-initial, measure-fourth-macos] - if: always() && needs.select-initial.outputs.macos_fourth == 'true' && - needs.measure-fourth-macos.result == 'success' + needs: select-fourth + if: >- + !cancelled() && needs.select-fourth.result == 'success' && + needs.select-fourth.outputs.macos_fifth == 'true' runs-on: macos-latest timeout-minutes: 90 permissions: @@ -327,6 +369,7 @@ jobs: select-initial, measure-fourth-linux, measure-fourth-macos, + select-fourth, measure-fifth-linux, measure-fifth-macos, ] diff --git a/CHANGELOG.md b/CHANGELOG.md index 743414a8b9..356b5f7f04 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,8 @@ # 0.11.0 +- Skip a fifth main-branch benchmark measurement once four candidates provide a compatible quorum. + - Reduce temporary allocations and copies when lazily decoding the embedded language database. - Transfer validated rg output directly into matcher results, reusing retained buffers while preserving diff --git a/TODO.md b/TODO.md index bb7a6e66d4..84635216bb 100644 --- a/TODO.md +++ b/TODO.md @@ -190,6 +190,10 @@ scope or declaring new flags implemented. See - [x] Use three complete-matrix sample shards on both hosted platforms: three measured rounds per shard for PR and main; pool raw samples before selecting the fastest seven of nine. Share compiled binaries, rotate participant order, and preserve per-sample shard provenance. +- [x] Apply the after-fourth agreement decision to main benchmarks as well as PR benchmarks. + Main run 42330247443 selected compatible macOS candidates 0/2/3 but still ran candidate 4; + gate the fifth measurement on the platform decision and preserve cancellation handling, + strict compatibility, all raw samples and the fastest-seven-of-nine estimator. - [x] Verify nine-round PR artifacts retain every raw observation and the fastest-seven estimate: PR #952 has 440 tasks / 800 tool-case entries per platform, each with 9 raw / 7 selected samples. Linux jobs took 4m31s-5m17s, macOS 4m33s-7m00s, versus #951's 2m19s-3m32s one-round jobs. diff --git a/tools/benchmark_history_test.py b/tools/benchmark_history_test.py index 5ea81fa7b4..e1e3c2ba36 100644 --- a/tools/benchmark_history_test.py +++ b/tools/benchmark_history_test.py @@ -224,16 +224,16 @@ def test_workflow_keeps_measurement_unprivileged_and_publication_serialized(self self.assertNotIn('shards/*/*.json', measure) self.assertIn("git -C site add --all", publish) - def test_fourth_candidate_always_schedules_fifth_after_unresolved_initial_set(self): + def test_fifth_candidate_requires_unresolved_fourth_agreement(self): workflow = repository_file('.github/workflows/benchmarks.yml').read_text() for platform_name in ('linux', 'macos'): with self.subTest(platform=platform_name): job = workflow.split(f' measure-fifth-{platform_name}:', 1)[1] job = re.split(r'\n [a-z][a-z-]*:', job, maxsplit=1)[0] - self.assertIn(f'needs: [select-initial, measure-fourth-{platform_name}]', job) - self.assertIn(f"needs.select-initial.outputs.{platform_name}_fourth == 'true'", job) - self.assertIn(f"needs.measure-fourth-{platform_name}.result == 'success'", job) - self.assertNotIn('select-fourth.outputs', job) + self.assertIn('needs: select-fourth', job) + self.assertIn("needs.select-fourth.result == 'success'", job) + self.assertIn(f"needs.select-fourth.outputs.{platform_name}_fifth == 'true'", job) + self.assertNotIn('needs.select-initial.outputs', job) def test_pr_and_main_measurements_use_the_same_runner_identity(self): main = repository_file(".github/workflows/benchmarks.yml").read_text() diff --git a/tools/benchmark_shards_test.py b/tools/benchmark_shards_test.py index 7335daf338..1963977041 100644 --- a/tools/benchmark_shards_test.py +++ b/tools/benchmark_shards_test.py @@ -5,6 +5,7 @@ import gzip import json from pathlib import Path +import re import tempfile import unittest from unittest import mock @@ -45,17 +46,36 @@ def records(cpus=(1, 4), count=3, partition='balanced', repetitions=9, keep=7, * class BenchmarkShardsTest(unittest.TestCase): def test_optional_candidates_override_skipped_ancestor_status(self): - workflow = repository_file('.github/workflows/main.yml').read_text() - for stage, selector in (('fourth', 'initial'), ('fifth', 'fourth')): - for platform in ('linux', 'macos'): - with self.subTest(stage=stage, platform=platform): - job = workflow.split(f' benchmark-measure-{stage}-{platform}:', 1)[1] - condition = job.split(' if:', 1)[1].split(' runs-on:', 1)[0] - # An explicit status function prevents GitHub's implicit success() from - # suppressing a requested candidate when the other platform was skipped. - self.assertIn('!cancelled()', condition) - self.assertIn(f"needs.benchmark-select-{selector}.result == 'success'", condition) - self.assertIn(f"outputs.{platform}_{stage} == 'true'", condition) + for path, prefix in (('main.yml', 'benchmark-'), ('benchmarks.yml', '')): + workflow = repository_file('.github/workflows/' + path).read_text() + for stage, selector in (('fourth', 'initial'), ('fifth', 'fourth')): + for platform in ('linux', 'macos'): + with self.subTest(workflow=path, stage=stage, platform=platform): + job = workflow.split(f' {prefix}measure-{stage}-{platform}:', 1)[1] + condition = job.split(' if:', 1)[1].split(' runs-on:', 1)[0] + # Explicit status handling permits requested candidates after a skipped + # optional sibling, while cancellation must still stop new measurements. + self.assertIn('!cancelled()', condition) + self.assertIn(f"needs.{prefix}select-{selector}.result == 'success'", condition) + self.assertIn(f"outputs.{platform}_{stage} == 'true'", condition) + + def test_both_workflows_decide_again_before_scheduling_fifth_candidates(self): + for path, prefix, aggregate in (('main.yml', 'benchmark-', 'benchmark-aggregate'), + ('benchmarks.yml', '', 'aggregate')): + with self.subTest(workflow=path): + workflow = repository_file('.github/workflows/' + path).read_text() + self.assertIn(f' {prefix}select-fourth:', workflow) + decision = re.split(r'\n (?=\S)', + workflow.split(f' {prefix}select-fourth:', 1)[1], maxsplit=1)[0] + self.assertIn(f'{prefix}select-initial', decision) + for platform in ('linux', 'macos'): + self.assertIn(f'{prefix}measure-fourth-{platform}', decision) + self.assertIn(f'{platform}_fifth: ${{{{ steps.decision.outputs.{platform}_fifth }}}}', + decision) + self.assertIn('--decision-stage=fourth', decision) + self.assertIn('tee -a "${GITHUB_OUTPUT}"', decision) + needs = workflow.split(f' {aggregate}:', 1)[1].split(' if:', 1)[0] + self.assertIn(f'{prefix}select-fourth', needs) def test_two_pairs_of_runner_environments_require_the_fifth_candidate(self): for platform, cpu_count in (('linux', 4), ('macos', 3)):