diff --git a/.github/workflows/daily-client-performance.yml b/.github/workflows/daily-client-performance.yml index 8603eb2c9..f62362f65 100644 --- a/.github/workflows/daily-client-performance.yml +++ b/.github/workflows/daily-client-performance.yml @@ -176,6 +176,8 @@ jobs: if: needs.benchmark.result == 'success' && github.ref == 'refs/heads/main' runs-on: ubuntu-24.04 timeout-minutes: 15 + outputs: + publish: ${{ steps.evidence.outputs.publish }} permissions: actions: read contents: read @@ -203,21 +205,53 @@ jobs: with: name: daily-client-performance-${{ github.run_id }}-${{ github.run_attempt }} path: build/ci-evidence + - id: evidence + name: Require complete measurements or the exact GPU qualification skip + run: >- + python3 tools/ci/daily_performance_site.py classify-evidence + --evidence build/ci-evidence/movement-frame-time.json >>"${GITHUB_OUTPUT}" + - name: Preserve the published site while GPU measurements are unavailable + if: steps.evidence.outputs.publish == 'false' + run: | + echo 'GPU hardware qualification is required; no performance observation was recorded. The published site and alert state are unchanged.' >>"${GITHUB_STEP_SUMMARY}" - name: Load the exact predecessor checkpoint + if: steps.evidence.outputs.publish == 'true' env: GH_TOKEN: ${{ github.token }} CHECKPOINT_SOURCE: ${{ inputs.checkpoint_source || 'pages' }} run: | if test "${CHECKPOINT_SOURCE}" = pages; then - gh api \ - "repos/${GITHUB_REPOSITORY}/actions/workflows/daily-client-performance.yml/runs?branch=main&status=success&per_page=20" \ - >build/successful-performance-runs.json - # shellcheck disable=SC2016 - expected_run=$(jq -c ' - [.workflow_runs[] | - select(.event == "schedule" or .event == "workflow_dispatch")][0] // empty - ' build/successful-performance-runs.json) - test -n "${expected_run}" + expected_run= + # Keep the most recent deployed producer, even after successful no-op runs. + for page in {1..10}; do + gh api \ + "repos/${GITHUB_REPOSITORY}/actions/workflows/daily-client-performance.yml/runs?branch=main&status=success&per_page=20&page=${page}" \ + >build/successful-performance-runs.json + while IFS= read -r candidate; do + candidate_id=$(jq -r .id <<<"${candidate}") + gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${candidate_id}/jobs?filter=latest&per_page=100" \ + >build/predecessor-jobs.json + jq -e '.total_count == (.jobs | length) and .total_count <= 100' \ + build/predecessor-jobs.json >/dev/null + if jq -e ' + ([.jobs[] | select(.name == "Deploy the validated performance site" and + .status == "completed" and .conclusion == "success")] | length == 1) + ' build/predecessor-jobs.json >/dev/null; then + expected_run=${candidate} + break 2 + fi + done < <(jq -c ' + .workflow_runs[] | + select(.event == "schedule" or .event == "workflow_dispatch") + ' build/successful-performance-runs.json) + if test "$(jq '.workflow_runs | length' build/successful-performance-runs.json)" -lt 20; then + break + fi + done + if test -z "${expected_run}"; then + echo 'No successful Pages deployment found within 200 successful runs; preserve the existing site and investigate its producer.' >&2 + exit 2 + fi python3 tools/ci/daily_performance_site.py fetch \ --base-url "${PAGES_URL}" \ --output build/performance-checkpoint @@ -239,6 +273,7 @@ jobs: exit 2 fi - name: Build and validate the complete static deployment + if: steps.evidence.outputs.publish == 'true' env: CHECKPOINT_SOURCE: ${{ inputs.checkpoint_source || 'pages' }} run: | @@ -271,6 +306,7 @@ jobs: --site build/performance-site cat build/ci-evidence/site-summary.md >>"${GITHUB_STEP_SUMMARY}" - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + if: steps.evidence.outputs.publish == 'true' with: name: daily-client-performance-checkpoint-${{ github.run_id }}-${{ github.run_attempt }} path: | @@ -281,12 +317,14 @@ jobs: if-no-files-found: error retention-days: 90 - uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5.0.0 + if: steps.evidence.outputs.publish == 'true' with: path: build/performance-site deploy: name: Deploy the validated performance site needs: project + if: needs.project.outputs.publish == 'true' runs-on: ubuntu-24.04 timeout-minutes: 10 permissions: @@ -303,6 +341,7 @@ jobs: alerts: name: Reconcile deployed regression alert state needs: [project, deploy] + if: needs.project.outputs.publish == 'true' runs-on: ubuntu-24.04 timeout-minutes: 10 permissions: diff --git a/docs/DAILY_CLIENT_PERFORMANCE.md b/docs/DAILY_CLIENT_PERFORMANCE.md index 736892ac8..d9c165892 100644 --- a/docs/DAILY_CLIENT_PERFORMANCE.md +++ b/docs/DAILY_CLIENT_PERFORMANCE.md @@ -14,6 +14,14 @@ SHA, repository, workflow, run, attempt, and any explicit early-failure diagnostic. A successful benchmark that produces no movement JSON is an infrastructure failure, not a successful empty observation. +The retired player-view benchmark currently emits the exact +`gpu-hardware-qualification-required` skip record when its implementation is +unavailable. That record is retained in the raw artifact but is not a performance +observation. The workflow reports the unavailable GPU measurements and skips +checkpoint projection, Pages deployment, and alert reconciliation, preserving +the published site and alert state. Only this exact current-schema skip is +accepted; malformed, incomplete, or other skipped evidence still fails closed. + ## Public report and durable checkpoint Successful trusted `main` observations are published at @@ -30,7 +38,10 @@ surfaces are: - `points/run-RUN.json`: one immutable detailed point per retained logical run; - `reports/run-RUN/index.html`: one readable detailed report per point. -The normal predecessor is the last successfully deployed Pages tree. The +The normal predecessor is the last successfully deployed Pages tree. Successful +no-op runs are excluded when selecting its producer. The lookup checks at most +200 successful workflow runs and fails closed if none has a successful Pages +deployment; it never resets the checkpoint when that history bound is reached. The publisher downloads only the paths named by its manifest, rejects redirects, validates every digest and byte count, verifies the repository/workflow identity, and then constructs a new full tree. A benchmark, predecessor, diff --git a/tools/ci/daily_performance_site.py b/tools/ci/daily_performance_site.py index acf699629..62b417807 100644 --- a/tools/ci/daily_performance_site.py +++ b/tools/ci/daily_performance_site.py @@ -64,6 +64,22 @@ def _read_json(path: Path, context: str) -> Any: raise SiteError(f"cannot read {context}: {error}") from error +def publication_required(evidence: Any) -> bool: + """Accept only complete measurements or the producer's exact GPU skip.""" + if ( + isinstance(evidence, dict) + and type(evidence.get("schema_version")) is int + and evidence == { + "schema_version": report.benchmark_contract.EVIDENCE_SCHEMA_VERSION, + "status": "skipped", + "reason": "gpu-hardware-qualification-required", + } + ): + return False + report.validate_evidence(evidence) + return True + + def _safe_relative_path(value: str) -> PurePosixPath: path = PurePosixPath(value) if ( @@ -816,6 +832,8 @@ def fetch(relative: str, maximum: int) -> bytes: def main() -> int: parser = argparse.ArgumentParser() subparsers = parser.add_subparsers(dest="command", required=True) + classify = subparsers.add_parser("classify-evidence") + classify.add_argument("--evidence", type=Path, required=True) build = subparsers.add_parser("build") build.add_argument("--evidence", type=Path, required=True) build.add_argument("--output", type=Path, required=True) @@ -838,7 +856,10 @@ def main() -> int: fetch.add_argument("--output", type=Path, required=True) args = parser.parse_args() try: - if args.command == "build": + if args.command == "classify-evidence": + required = publication_required(_read_json(args.evidence, "movement evidence")) + print(f"publish={'true' if required else 'false'}") + elif args.command == "build": if (args.legacy_checkpoint is None) != (args.legacy_final_ref is None): raise SiteError("legacy checkpoint and final ref must be supplied together") build_site( diff --git a/tools/tests/test_daily_performance_site.py b/tools/tests/test_daily_performance_site.py index 23ed36ab1..674b3217d 100644 --- a/tools/tests/test_daily_performance_site.py +++ b/tools/tests/test_daily_performance_site.py @@ -3,6 +3,7 @@ import importlib.util import json from pathlib import Path +import subprocess import sys import tempfile import unittest @@ -70,6 +71,66 @@ def setUp(self) -> None: def tearDown(self) -> None: self.temporary.cleanup() + def test_exact_gpu_qualification_skip_preserves_publication(self) -> None: + # Observed producer record from daily run 41189141733, attempt 1. + skipped = { + "reason": "gpu-hardware-qualification-required", + "schema_version": 7, + "status": "skipped", + } + self.assertFalse(site.publication_required(skipped)) + self.evidence.write_text(json.dumps(skipped)) + result = self.classify() + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout, "publish=false\n") + self.assertFalse((self.root / "site").exists()) + # A skip never becomes a valid complete report. + with self.assertRaises(report.ReportError): + report.validate_evidence(skipped) + + def classify(self) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, str(ROOT / "ci" / "daily_performance_site.py"), + "classify-evidence", "--evidence", str(self.evidence)], + capture_output=True, text=True, check=False, + ) + + def test_complete_measurements_continue_to_publication(self) -> None: + self.assertTrue(site.publication_required(evidence())) + result = self.classify() + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout, "publish=true\n") + output = self.root / "site" + manifest = self.build(output) + self.assertEqual(site.validate_site(output), manifest) + + def test_unrecognized_or_incomplete_evidence_fails_closed(self) -> None: + skipped = { + "schema_version": 7, "status": "skipped", + "reason": "gpu-hardware-qualification-required", + } + complete = evidence() + invalid = [ + {}, None, [], + dict(skipped, reason="no-movement-sensitive-changes"), + dict(skipped, schema_version=6), + dict(skipped, schema_version=7.0), + dict(skipped, extra="untrusted"), + dict(skipped, status="error"), + {key: value for key, value in complete.items() if key != "records"}, + ] + for value in invalid: + with self.subTest(value=value): + self.evidence.write_text(json.dumps(value)) + result = self.classify() + self.assertEqual(result.returncode, 2, result.stderr) + self.assertEqual(result.stdout, "") + self.assertIn("error:", result.stderr) + self.evidence.write_text("{broken") + result = self.classify() + self.assertEqual(result.returncode, 2) + self.assertEqual(result.stdout, "") + def build(self, output: Path, *, run_id: str = "2", attempt: str = "1", checkpoint: Path | None = None) -> dict[str, object]: arguments = { diff --git a/tools/tests/test_workflow_contracts.py b/tools/tests/test_workflow_contracts.py index 5378446bb..381b2a35d 100644 --- a/tools/tests/test_workflow_contracts.py +++ b/tools/tests/test_workflow_contracts.py @@ -2276,6 +2276,11 @@ def test_daily_report_stages_verified_inputs_and_keeps_timing_informational( self.assertIn("--legacy-final-ref", project) self.assertIn("daily_performance_site.py build", project) self.assertIn("daily_performance_site.py validate", project) + self.assertIn("publish: ${{ steps.evidence.outputs.publish }}", project) + self.assertIn("daily_performance_site.py classify-evidence", project) + self.assertLess(project.index("daily_performance_site.py classify-evidence"), + project.index(" - name: Load the exact predecessor checkpoint")) + self.assertEqual(project.count("if: steps.evidence.outputs.publish == 'true'"), 4) self.assertRegex(project, r"actions/upload-pages-artifact@[0-9a-f]{40}\b") self.assertNotIn("benchmark-data:refs", workflow) self.assertNotIn("HEAD:benchmark-data", workflow) @@ -2286,18 +2291,83 @@ def test_daily_report_stages_verified_inputs_and_keeps_timing_informational( ] self.assertIn("pages: write", deploy) self.assertIn("id-token: write", deploy) + self.assertIn("if: needs.project.outputs.publish == 'true'", deploy) self.assertNotIn("issues: write", deploy) self.assertIn("environment:\n name: github-pages", deploy) self.assertRegex(deploy, r"actions/deploy-pages@[0-9a-f]{40}\b") alerts = workflow[workflow.index(" alerts:") :] self.assertIn("needs: [project, deploy]", alerts) + self.assertIn("if: needs.project.outputs.publish == 'true'", alerts) self.assertIn("issues: write", alerts) self.assertNotIn("pages: write", alerts) self.assertIn("reconcile_performance_alerts.py", alerts) self.assertIn("daily_performance_site.py validate-artifact", alerts) self.assertNotIn("pull_request:", workflow[: workflow.index("jobs:")]) + @unittest.skipUnless(shutil.which("jq"), "jq is required for workflow shell checks") + def test_daily_predecessor_skips_successful_runs_without_a_deployment(self) -> None: + workflow = self.text("daily-client-performance.yml") + step = workflow.split(" - name: Load the exact predecessor checkpoint\n", 1)[1] + script = textwrap.dedent(step.split(" run: |\n", 1)[1].split( + " - name: Build and validate the complete static deployment\n", 1)[0]) + with tempfile.TemporaryDirectory() as temporary: + directory = Path(temporary) + (directory / "build").mkdir() + commands = directory / "bin" + commands.mkdir() + gh = commands / "gh" + gh.write_text(f"#!{shutil.which('python3')}\n" + textwrap.dedent('''\ + import json, os, sys + endpoint = sys.argv[-1] + with open(os.environ["READ_LOG"], "a") as log: + log.write(endpoint + "\\n") + if "/workflows/" in endpoint: + if "page=1" in endpoint: + runs = [{"id": number, "event": "schedule", "run_attempt": 1, + "head_sha": "b" * 40} for number in range(22, 2, -1)] + else: + runs = [{"id": 2, "event": "schedule", "run_attempt": 1, + "head_sha": "a" * 40}] + value = {"workflow_runs": runs} + else: + success = "/runs/2/" in endpoint and os.environ["DEPLOY_RESULT"] == "success" + value = {"total_count": 1, "jobs": [{ + "name": "Deploy the validated performance site", "status": "completed", + "conclusion": "success" if success else "skipped"}]} + print(json.dumps(value)) + ''')) + gh.chmod(0o755) + fetch = commands / "python3" + fetch.write_text(f"#!{shutil.which('python3')}\n" + textwrap.dedent('''\ + import json + from pathlib import Path + path = Path("build/performance-checkpoint/v1") + path.mkdir(parents=True, exist_ok=True) + (path / "manifest.json").write_text(json.dumps({ + "run_id": "2", "run_attempt": "1", "source_sha": "a" * 40})) + ''')) + fetch.chmod(0o755) + for outcome in ("success", "skipped"): + with self.subTest(deployed=outcome): + log = directory / "reads.log" + log.unlink(missing_ok=True) + result = subprocess.run( + ["bash", "-euo", "pipefail", "-c", script], cwd=directory, + env=dict(os.environ, PATH=f"{commands}:{os.environ['PATH']}", + CHECKPOINT_SOURCE="pages", GITHUB_REPOSITORY="atrinik/classic", + PAGES_URL="https://atrinik.github.io/classic/", + READ_LOG=str(log), DEPLOY_RESULT=outcome), + capture_output=True, text=True, check=False, + ) + if outcome == "success": + self.assertEqual(result.returncode, 0, result.stderr) + else: + self.assertNotEqual(result.returncode, 0) + self.assertIn("per_page=20&page=2", log.read_text()) + self.assertIn("/runs/3/jobs", log.read_text()) + self.assertIn("/runs/2/jobs", log.read_text()) + def test_linux_checks_pin_image_and_isolate_compiler_caches(self) -> None: workflow = self.text("check.yml") match = re.search(