Inventory missing merged PR benchmarks

tools/BUILD.bazel

@@ -1382,6 +1382,26 @@ py_binary(
     ],
 )
 
+py_library(
+    name = "benchmark_recovery_lib",
+    srcs = ["benchmark_recovery.py"],
+    imports = ["."],
+    deps = [":benchmark_preview_ci_lib"],
+)
+
+py_binary(
+    name = "benchmark_recovery",
+    srcs = ["benchmark_recovery.py"],
+    deps = [":benchmark_preview_ci_lib"],
+)
+
+py_test(
+    name = "benchmark_recovery_test",
+    size = "small",
+    srcs = ["benchmark_recovery_test.py"],
+    deps = [":benchmark_recovery_lib"],
+)
+
 py_test(
     name = "benchmark_preview_ci_test",
     size = "small",

tools/benchmark_campaign.py

@@ -170,19 +170,22 @@ def validate_measurements(record, batch, revision, tasks):
     compare.render(report)  # Validate all samples, successful commands and finite measurements.
 
 
-def retain(root, artifacts, source, repository):
-    """Validate the entire campaign before making any replacement visible."""
+def retain(root, artifacts, source, repository, allow_partial=False):
+    """Retain validated campaign records; partial retention is explicitly opt-in."""
     require(source['name'] == 'Benchmark backfill' and source['path'] == '.github/workflows/benchmark_backfill.yml'
             and source['event'] == 'workflow_dispatch' and source['head_branch'] == 'main'
-            and source['status'] == 'completed' and source['conclusion'] == 'success'
+            and source['status'] == 'completed'
+            and (source['conclusion'] == 'success' or allow_partial)
             and source['repository']['full_name'] == repository, 'expected a successful trusted main backfill workflow')
     run, attempt = int(source['id']), int(source['run_attempt'])
     require(run > 0 and attempt > 0, 'invalid source workflow identity')
     plan = json.loads((artifacts / 'benchmark-backfill-plan/campaign.json').read_text())
     validate_plan(plan)
     require(plan['source_head'] == source['head_sha'], 'campaign source commit mismatch')
     expected_artifacts = {'benchmark-backfill-plan', *('benchmark-backfill-' + job['id'] for job in plan['jobs'])}
-    require({path.name for path in artifacts.iterdir()} == expected_artifacts, 'incomplete or unexpected campaign artifacts')
+    actual_artifacts = {path.name for path in artifacts.iterdir()}
+    require(actual_artifacts <= expected_artifacts and (allow_partial or actual_artifacts == expected_artifacts),
+            'incomplete or unexpected campaign artifacts')
     trusted = {key: source[key] for key in ('id', 'run_attempt', 'created_at', 'head_sha', 'head_branch', 'event')}
     trusted['pull_requests'] = []
     destination = root / 'backfills' / str(run) / str(attempt)
@@ -192,24 +195,30 @@ def retain(root, artifacts, source, repository):
         originals.setdefault((benchmark_records.platform_key(old), old['head']), []).append(path.relative_to(root).as_posix())
     records = []
     for job in plan['jobs']:
-        folder = artifacts / ('benchmark-backfill-' + job['id'])
-        batch = json.loads((folder / 'batch.json').read_text())
-        status = json.loads((folder / 'status.json').read_text())
-        sha = job['revision']['sha']
-        require(set(status) == {sha} and status[sha]['status'] == 'complete', 'incomplete campaign revision')
-        reports = list(folder.glob('reports/*/*.json'))
-        expected = folder / 'reports' / sha / f"benchmark-report-{job['platform']}-{job['arch']}.json"
-        require(reports == [expected], 'unexpected campaign report set')
-        record = json.loads(expected.read_text())
-        validate_report(record, batch, job, plan)
-        collection = batch['contract']['collection']
-        require(collection['id'] == run and 1 <= collection['run_attempt'] <= attempt
-                and collection['head_sha'] == source['head_sha'], 'batch source workflow mismatch')
-        record.update(source=trusted, collection=collection, campaign=plan['identity'])
-        saved = destination / job['platform'] / sha / 'report.json'
-        record['original_reports'] = (benchmark_records.read(saved)['original_reports'] if benchmark_records.exists(saved)
-                                      else sorted(originals.get((job['platform'], sha), [])))
-        records.append((job, record, batch, status))
+        try:
+            folder = artifacts / ('benchmark-backfill-' + job['id'])
+            batch = json.loads((folder / 'batch.json').read_text())
+            status = json.loads((folder / 'status.json').read_text())
+            sha = job['revision']['sha']
+            require(set(status) == {sha} and status[sha]['status'] == 'complete', 'incomplete campaign revision')
+            reports = list(folder.glob('reports/*/*.json'))
+            expected = folder / 'reports' / sha / f"benchmark-report-{job['platform']}-{job['arch']}.json"
+            require(reports == [expected], 'unexpected campaign report set')
+            record = json.loads(expected.read_text())
+            validate_report(record, batch, job, plan)
+            collection = batch['contract']['collection']
+            require(collection['id'] == run and 1 <= collection['run_attempt'] <= attempt
+                    and collection['head_sha'] == source['head_sha'], 'batch source workflow mismatch')
+            record.update(source=trusted, collection=collection, campaign=plan['identity'])
+            saved = destination / job['platform'] / sha / 'report.json'
+            record['original_reports'] = (benchmark_records.read(saved)['original_reports'] if benchmark_records.exists(saved)
+                                          else sorted(originals.get((job['platform'], sha), [])))
+            records.append((job, record, batch, status))
+        except (ValueError, KeyError, OSError) as error:
+            if not allow_partial:
+                raise
+            print(f"::warning::backfill {run}/{attempt} {job['id']}: {error}", flush=True)
+    require(records, 'campaign has no complete validated measurements')
     payloads = {'campaign.json': plan, 'source.json': trusted}
     for job, record, batch, status in records:
         prefix = job['platform'] + '/' + job['revision']['sha'] + '/'

tools/benchmark_campaign_test.py

@@ -137,6 +137,15 @@ def test_planner_freezes_each_complete_revision_once_per_platform(self):
         self.assertIn('content-needle', selected['contract']['tasks'])
         campaign.validate_plan(selected)
 
+    def test_planner_uses_explicit_revisions_instead_of_published_history(self):
+        revision = plan()['jobs'][0]['revision']
+        with mock.patch.object(campaign.backfill, 'revisions', return_value=[revision]) as revisions, \
+                mock.patch.object(campaign.backfill, 'git', side_effect=[DRIVER, 'c' * 40]):
+            selected = campaign.make_plan(Path('/repo'), Path('/history'), [HEAD])
+        self.assertEqual(len(selected['jobs']), 2)
+        self.assertEqual([call.args[1:] for call in revisions.call_args_list],
+                         [([HEAD], None, 'linux'), ([HEAD], None, 'macos')])
+
     def test_plan_rejects_mutation_duplicates_partial_grid_and_missing_platform(self):
         mutations = [lambda value: value['jobs'].append(value['jobs'][0]),
                      lambda value: value['jobs'].pop(),
@@ -233,6 +242,19 @@ def test_missing_artifact_or_failed_revision_never_promotes_a_partial_campaign(s
                     campaign.retain(site, artifacts, source(), 'owner/repo')
                 self.assertFalse(site.exists())
 
+    def test_partial_recovery_retains_only_valid_completed_platforms(self):
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            artifacts = root / 'artifacts'
+            write_artifacts(artifacts)
+            folder = next(artifacts.glob('benchmark-backfill-macos-*'))
+            (folder / 'status.json').write_text(json.dumps({HEAD: {'status': 'measurement-failed'}}))
+            failed = dict(source(), conclusion='failure')
+            destination = campaign.retain(root / 'site', artifacts, failed, 'owner/repo', allow_partial=True)
+            self.assertTrue((destination / 'linux' / HEAD / 'report.json').exists())
+            self.assertFalse((destination / 'macos' / HEAD / 'report.json').exists())
+            campaign.retain(root / 'site', artifacts, failed, 'owner/repo', allow_partial=True)
+
     def test_report_rejects_missing_tasks_references_samples_and_changed_identity(self):
         selected = plan()
         job = selected['jobs'][0]

tools/benchmark_history_test.py

@@ -248,8 +248,8 @@ def test_publication_coalesces_requests_and_reconciles_sources(self):
         self.assertIn('group: benchmark-publication-latest', workflow)
         reconcile = repository_file('tools/benchmark_reconcile_ci.py').read_text()
         self.assertIn('benchmark_preview_ci.collect', reconcile)
-        self.assertIn("('benchmarks.yml', 'push', retain_measurement)", reconcile)
-        self.assertIn("('benchmark_backfill.yml', 'workflow_dispatch', retain_backfill)", reconcile)
+        self.assertIn("('benchmarks.yml', 'push', 'success', retain_measurement)", reconcile)
+        self.assertIn("('benchmark_backfill.yml', 'workflow_dispatch', 'completed', retain_backfill)", reconcile)
 
     def test_fifth_candidate_requires_unresolved_fourth_agreement(self):
         workflow = repository_file('.github/workflows/benchmarks.yml').read_text()

tools/benchmark_landscape.py

@@ -643,6 +643,10 @@ def publish(root, javascript, previews=(), incremental=False, repository='mbowor
             if benchmark_records.is_backfill(record):
                 label = 'CI backfill / ' + commit[:10]
             links = version_links.get(commit, [])
+            if not local and not preview:
+                pr_links = [link['label'] for link in links if re.fullmatch(r'PR #[0-9]+', link.get('label', ''))]
+                if pr_links:
+                    label = ', '.join(pr_links) + ' / ' + commit[:10]
             if preview:
                 number = record.get('pull_number') or source['pull_requests'][0]['number']
                 links = [{'label': f'PR #{number} preview', 'href': f'https://github.com/{repository}/pull/{number}'}]

tools/benchmark_landscape_test.py

@@ -91,6 +91,23 @@ def test_layout_details_distinguish_legacy_and_versioned_datasets(self):
             {'name': 'broad', 'revision': 2}, {'name': 'deep', 'revision': 2},
         ]}), 'broad/v2 + deep/v2')
 
+    def test_merged_version_uses_pr_label_from_version_links(self):
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            (root / 'index.html').write_text('<h1>History</h1>')
+            folder = root / 'runs/1/1/linux'
+            folder.mkdir(parents=True)
+            sha = 'a' * 40
+            (folder / 'report.json').write_text(json.dumps(dict(tool_comparisons=report(), platform='linux',
+                head=sha, source=dict(id=1, run_attempt=1, head_branch='main', event='push',
+                                     created_at='2026-09-01T00:00:00Z'))))
+            (folder / 'index.html').write_text('<h1>Main</h1>')
+            (root / 'version-links.json').write_text(json.dumps({sha: [
+                {'label': 'PR #1143', 'href': 'https://github.com/owner/repo/pull/1143'}]}))
+            landscape.publish(root, '/* renderer */')
+            catalog = json.loads((root / 'catalog.json').read_text())
+            self.assertEqual(catalog[0]['label'], 'PR #1143 / ' + sha[:10])
+
     def test_incremental_preview_publication_preserves_merged_pages_and_removes_stale_selection(self):
         with tempfile.TemporaryDirectory() as directory:
             root = Path(directory)

tools/benchmark_reconcile_ci.py

@@ -116,19 +116,25 @@ def retain_backfill(api, root, run):
             if not re.fullmatch(r'benchmark-backfill-[A-Za-z0-9_.-]+', name):
                 raise ValueError('invalid backfill artifact name')
             target = directory / name
-            target.mkdir()
-            extract(api.request(f"actions/artifacts/{item['id']}/zip", raw=True), target)
-        benchmark_campaign.retain(root, directory, run, api.repository)
+            try:
+                target.mkdir()
+                extract(api.request(f"actions/artifacts/{item['id']}/zip", raw=True), target)
+            except (ValueError, OSError, zipfile.BadZipFile, subprocess.CalledProcessError,
+                    subprocess.TimeoutExpired) as error:
+                if name == 'benchmark-backfill-plan':
+                    raise
+                print(f"::warning::backfill artifact {name}: {error}", flush=True)
+        benchmark_campaign.retain(root, directory, run, api.repository, allow_partial=True)
     return True
 
 
 def reconcile(api, root):
     since = (datetime.now(timezone.utc) - timedelta(days=30)).date().isoformat()
     failures = []
-    for workflow, event, retain in (
-            ('benchmarks.yml', 'push', retain_measurement),
-            ('benchmark_backfill.yml', 'workflow_dispatch', retain_backfill)):
-        runs = api.pages(f'actions/workflows/{workflow}/runs?status=success&branch=main&created=%3E%3D{since}',
+    for workflow, event, status, retain in (
+            ('benchmarks.yml', 'push', 'success', retain_measurement),
+            ('benchmark_backfill.yml', 'workflow_dispatch', 'completed', retain_backfill)):
+        runs = api.pages(f'actions/workflows/{workflow}/runs?status={status}&branch=main&created=%3E%3D{since}',
                          'workflow_runs')
         for run in reversed(runs):
             if run.get('event') != event or run.get('head_branch') != 'main':