Stop main benchmarks once three of four candidates agree

.github/workflows/benchmarks.yml

@@ -240,7 +240,9 @@ jobs:
   measure-fourth-linux:
     name: Benchmark measurement (linux, shard 3)
     needs: select-initial
-    if: needs.select-initial.outputs.linux_fourth == 'true'
+    if: >-
+      !cancelled() && needs.select-initial.result == 'success' &&
+      needs.select-initial.outputs.linux_fourth == 'true'
     runs-on: ubuntu-latest
     timeout-minutes: 60
     permissions:
@@ -260,7 +262,9 @@ jobs:
   measure-fourth-macos:
     name: Benchmark measurement (macos, shard 3)
     needs: select-initial
-    if: needs.select-initial.outputs.macos_fourth == 'true'
+    if: >-
+      !cancelled() && needs.select-initial.result == 'success' &&
+      needs.select-initial.outputs.macos_fourth == 'true'
     runs-on: macos-latest
     timeout-minutes: 90
     permissions:
@@ -277,11 +281,48 @@ jobs:
           shard: 3
           head: ${{ github.sha }}
 
+  select-fourth:
+    name: Check benchmark agreement after fourth candidates
+    needs: [select-initial, measure-fourth-linux, measure-fourth-macos]
+    if: always() && needs.select-initial.result == 'success' &&
+      (needs.select-initial.outputs.linux_fourth == 'true' ||
+      needs.select-initial.outputs.macos_fourth == 'true') &&
+      needs.measure-fourth-linux.result != 'failure' &&
+      needs.measure-fourth-macos.result != 'failure'
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+    outputs:
+      linux_fifth: ${{ steps.decision.outputs.linux_fifth }}
+      macos_fifth: ${{ steps.decision.outputs.macos_fifth }}
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          ref: ${{ github.sha }}
+          persist-credentials: false
+      - uses: actions/setup-python@v7
+        with:
+          python-version: "3.13"
+      - uses: actions/download-artifact@v8
+        with:
+          pattern: benchmark-shard-*
+          path: candidate-shards
+          if-no-artifact-found: ignore
+      - name: Schedule a fifth candidate only without a three-shard majority
+        id: decision
+        run: |
+          set -euo pipefail
+          mkdir -p candidate-shards
+          python3 tools/benchmark_shards.py --decide-adaptive=candidate-shards \
+            --decision-stage=fourth | tee -a "${GITHUB_OUTPUT}"
+
   measure-fifth-linux:
     name: Benchmark measurement (linux, shard 4)
-    needs: [select-initial, measure-fourth-linux]
-    if: always() && needs.select-initial.outputs.linux_fourth == 'true' &&
-      needs.measure-fourth-linux.result == 'success'
+    needs: select-fourth
+    if: >-
+      !cancelled() && needs.select-fourth.result == 'success' &&
+      needs.select-fourth.outputs.linux_fifth == 'true'
     runs-on: ubuntu-latest
     timeout-minutes: 60
     permissions:
@@ -300,9 +341,10 @@ jobs:
 
   measure-fifth-macos:
     name: Benchmark measurement (macos, shard 4)
-    needs: [select-initial, measure-fourth-macos]
-    if: always() && needs.select-initial.outputs.macos_fourth == 'true' &&
-      needs.measure-fourth-macos.result == 'success'
+    needs: select-fourth
+    if: >-
+      !cancelled() && needs.select-fourth.result == 'success' &&
+      needs.select-fourth.outputs.macos_fifth == 'true'
     runs-on: macos-latest
     timeout-minutes: 90
     permissions:
@@ -327,6 +369,7 @@ jobs:
         select-initial,
         measure-fourth-linux,
         measure-fourth-macos,
+        select-fourth,
         measure-fifth-linux,
         measure-fifth-macos,
       ]

CHANGELOG.md

@@ -3,6 +3,8 @@
 
 # 0.11.0
 
+- Skip a fifth main-branch benchmark measurement once four candidates provide a compatible quorum.
+
 - Reduce temporary allocations and copies when lazily decoding the embedded language database.
 
 - Transfer validated rg output directly into matcher results, reusing retained buffers while preserving

TODO.md

@@ -190,6 +190,10 @@ scope or declaring new flags implemented. See
 - [x] Use three complete-matrix sample shards on both hosted platforms: three measured rounds
       per shard for PR and main; pool raw samples before selecting the fastest seven of nine.
       Share compiled binaries, rotate participant order, and preserve per-sample shard provenance.
+- [x] Apply the after-fourth agreement decision to main benchmarks as well as PR benchmarks.
+      Main run 42330247443 selected compatible macOS candidates 0/2/3 but still ran candidate 4;
+      gate the fifth measurement on the platform decision and preserve cancellation handling,
+      strict compatibility, all raw samples and the fastest-seven-of-nine estimator.
 - [x] Verify nine-round PR artifacts retain every raw observation and the fastest-seven estimate:
       PR #952 has 440 tasks / 800 tool-case entries per platform, each with 9 raw / 7 selected samples.
       Linux jobs took 4m31s-5m17s, macOS 4m33s-7m00s, versus #951's 2m19s-3m32s one-round jobs.

tools/benchmark_history_test.py

@@ -224,16 +224,16 @@ def test_workflow_keeps_measurement_unprivileged_and_publication_serialized(self
         self.assertNotIn('shards/*/*.json', measure)
         self.assertIn("git -C site add --all", publish)
 
-    def test_fourth_candidate_always_schedules_fifth_after_unresolved_initial_set(self):
+    def test_fifth_candidate_requires_unresolved_fourth_agreement(self):
         workflow = repository_file('.github/workflows/benchmarks.yml').read_text()
         for platform_name in ('linux', 'macos'):
             with self.subTest(platform=platform_name):
                 job = workflow.split(f'  measure-fifth-{platform_name}:', 1)[1]
                 job = re.split(r'\n  [a-z][a-z-]*:', job, maxsplit=1)[0]
-                self.assertIn(f'needs: [select-initial, measure-fourth-{platform_name}]', job)
-                self.assertIn(f"needs.select-initial.outputs.{platform_name}_fourth == 'true'", job)
-                self.assertIn(f"needs.measure-fourth-{platform_name}.result == 'success'", job)
-                self.assertNotIn('select-fourth.outputs', job)
+                self.assertIn('needs: select-fourth', job)
+                self.assertIn("needs.select-fourth.result == 'success'", job)
+                self.assertIn(f"needs.select-fourth.outputs.{platform_name}_fifth == 'true'", job)
+                self.assertNotIn('needs.select-initial.outputs', job)
 
     def test_pr_and_main_measurements_use_the_same_runner_identity(self):
         main = repository_file(".github/workflows/benchmarks.yml").read_text()

tools/benchmark_shards_test.py

@@ -5,6 +5,7 @@
 import gzip
 import json
 from pathlib import Path
+import re
 import tempfile
 import unittest
 from unittest import mock
@@ -45,17 +46,36 @@ def records(cpus=(1, 4), count=3, partition='balanced', repetitions=9, keep=7, *
 
 class BenchmarkShardsTest(unittest.TestCase):
     def test_optional_candidates_override_skipped_ancestor_status(self):
-        workflow = repository_file('.github/workflows/main.yml').read_text()
-        for stage, selector in (('fourth', 'initial'), ('fifth', 'fourth')):
-            for platform in ('linux', 'macos'):
-                with self.subTest(stage=stage, platform=platform):
-                    job = workflow.split(f'  benchmark-measure-{stage}-{platform}:', 1)[1]
-                    condition = job.split('    if:', 1)[1].split('    runs-on:', 1)[0]
-                    # An explicit status function prevents GitHub's implicit success() from
-                    # suppressing a requested candidate when the other platform was skipped.
-                    self.assertIn('!cancelled()', condition)
-                    self.assertIn(f"needs.benchmark-select-{selector}.result == 'success'", condition)
-                    self.assertIn(f"outputs.{platform}_{stage} == 'true'", condition)
+        for path, prefix in (('main.yml', 'benchmark-'), ('benchmarks.yml', '')):
+            workflow = repository_file('.github/workflows/' + path).read_text()
+            for stage, selector in (('fourth', 'initial'), ('fifth', 'fourth')):
+                for platform in ('linux', 'macos'):
+                    with self.subTest(workflow=path, stage=stage, platform=platform):
+                        job = workflow.split(f'  {prefix}measure-{stage}-{platform}:', 1)[1]
+                        condition = job.split('    if:', 1)[1].split('    runs-on:', 1)[0]
+                        # Explicit status handling permits requested candidates after a skipped
+                        # optional sibling, while cancellation must still stop new measurements.
+                        self.assertIn('!cancelled()', condition)
+                        self.assertIn(f"needs.{prefix}select-{selector}.result == 'success'", condition)
+                        self.assertIn(f"outputs.{platform}_{stage} == 'true'", condition)
+
+    def test_both_workflows_decide_again_before_scheduling_fifth_candidates(self):
+        for path, prefix, aggregate in (('main.yml', 'benchmark-', 'benchmark-aggregate'),
+                                        ('benchmarks.yml', '', 'aggregate')):
+            with self.subTest(workflow=path):
+                workflow = repository_file('.github/workflows/' + path).read_text()
+                self.assertIn(f'  {prefix}select-fourth:', workflow)
+                decision = re.split(r'\n  (?=\S)',
+                                    workflow.split(f'  {prefix}select-fourth:', 1)[1], maxsplit=1)[0]
+                self.assertIn(f'{prefix}select-initial', decision)
+                for platform in ('linux', 'macos'):
+                    self.assertIn(f'{prefix}measure-fourth-{platform}', decision)
+                    self.assertIn(f'{platform}_fifth: ${{{{ steps.decision.outputs.{platform}_fifth }}}}',
+                                  decision)
+                self.assertIn('--decision-stage=fourth', decision)
+                self.assertIn('tee -a "${GITHUB_OUTPUT}"', decision)
+                needs = workflow.split(f'  {aggregate}:', 1)[1].split('    if:', 1)[0]
+                self.assertIn(f'{prefix}select-fourth', needs)
 
     def test_two_pairs_of_runner_environments_require_the_fifth_candidate(self):
         for platform, cpu_count in (('linux', 4), ('macos', 3)):