Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
57 changes: 48 additions & 9 deletions .github/workflows/daily-client-performance.yml
Original file line number Diff line number Diff line change
Expand Up @@ -176,6 +176,8 @@ jobs:
if: needs.benchmark.result == 'success' && github.ref == 'refs/heads/main'
runs-on: ubuntu-24.04
timeout-minutes: 15
outputs:
publish: ${{ steps.evidence.outputs.publish }}
permissions:
actions: read
contents: read
Expand Down Expand Up @@ -203,21 +205,53 @@ jobs:
with:
name: daily-client-performance-${{ github.run_id }}-${{ github.run_attempt }}
path: build/ci-evidence
- id: evidence
name: Require complete measurements or the exact GPU qualification skip
run: >-
python3 tools/ci/daily_performance_site.py classify-evidence
--evidence build/ci-evidence/movement-frame-time.json >>"${GITHUB_OUTPUT}"
- name: Preserve the published site while GPU measurements are unavailable
if: steps.evidence.outputs.publish == 'false'
run: |
echo 'GPU hardware qualification is required; no performance observation was recorded. The published site and alert state are unchanged.' >>"${GITHUB_STEP_SUMMARY}"
- name: Load the exact predecessor checkpoint
if: steps.evidence.outputs.publish == 'true'
env:
GH_TOKEN: ${{ github.token }}
CHECKPOINT_SOURCE: ${{ inputs.checkpoint_source || 'pages' }}
run: |
if test "${CHECKPOINT_SOURCE}" = pages; then
gh api \
"repos/${GITHUB_REPOSITORY}/actions/workflows/daily-client-performance.yml/runs?branch=main&status=success&per_page=20" \
>build/successful-performance-runs.json
# shellcheck disable=SC2016
expected_run=$(jq -c '
[.workflow_runs[] |
select(.event == "schedule" or .event == "workflow_dispatch")][0] // empty
' build/successful-performance-runs.json)
test -n "${expected_run}"
expected_run=
# Keep the most recent deployed producer, even after successful no-op runs.
for page in {1..10}; do
gh api \
"repos/${GITHUB_REPOSITORY}/actions/workflows/daily-client-performance.yml/runs?branch=main&status=success&per_page=20&page=${page}" \
>build/successful-performance-runs.json
while IFS= read -r candidate; do
candidate_id=$(jq -r .id <<<"${candidate}")
gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${candidate_id}/jobs?filter=latest&per_page=100" \
>build/predecessor-jobs.json
jq -e '.total_count == (.jobs | length) and .total_count <= 100' \
build/predecessor-jobs.json >/dev/null
if jq -e '
([.jobs[] | select(.name == "Deploy the validated performance site" and
.status == "completed" and .conclusion == "success")] | length == 1)
' build/predecessor-jobs.json >/dev/null; then
expected_run=${candidate}
break 2
fi
done < <(jq -c '
.workflow_runs[] |
select(.event == "schedule" or .event == "workflow_dispatch")
' build/successful-performance-runs.json)
if test "$(jq '.workflow_runs | length' build/successful-performance-runs.json)" -lt 20; then
break
fi
done
if test -z "${expected_run}"; then
echo 'No successful Pages deployment found within 200 successful runs; preserve the existing site and investigate its producer.' >&2
exit 2
fi
python3 tools/ci/daily_performance_site.py fetch \
--base-url "${PAGES_URL}" \
--output build/performance-checkpoint
Expand All @@ -239,6 +273,7 @@ jobs:
exit 2
fi
- name: Build and validate the complete static deployment
if: steps.evidence.outputs.publish == 'true'
env:
CHECKPOINT_SOURCE: ${{ inputs.checkpoint_source || 'pages' }}
run: |
Expand Down Expand Up @@ -271,6 +306,7 @@ jobs:
--site build/performance-site
cat build/ci-evidence/site-summary.md >>"${GITHUB_STEP_SUMMARY}"
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
if: steps.evidence.outputs.publish == 'true'
with:
name: daily-client-performance-checkpoint-${{ github.run_id }}-${{ github.run_attempt }}
path: |
Expand All @@ -281,12 +317,14 @@ jobs:
if-no-files-found: error
retention-days: 90
- uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5.0.0
if: steps.evidence.outputs.publish == 'true'
with:
path: build/performance-site

deploy:
name: Deploy the validated performance site
needs: project
if: needs.project.outputs.publish == 'true'
runs-on: ubuntu-24.04
timeout-minutes: 10
permissions:
Expand All @@ -303,6 +341,7 @@ jobs:
alerts:
name: Reconcile deployed regression alert state
needs: [project, deploy]
if: needs.project.outputs.publish == 'true'
runs-on: ubuntu-24.04
timeout-minutes: 10
permissions:
Expand Down
13 changes: 12 additions & 1 deletion docs/DAILY_CLIENT_PERFORMANCE.md
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,14 @@ SHA, repository, workflow, run, attempt, and any explicit early-failure
diagnostic. A successful benchmark that produces no movement JSON is an
infrastructure failure, not a successful empty observation.

The retired player-view benchmark currently emits the exact
`gpu-hardware-qualification-required` skip record when its implementation is
unavailable. That record is retained in the raw artifact but is not a performance
observation. The workflow reports the unavailable GPU measurements and skips
checkpoint projection, Pages deployment, and alert reconciliation, preserving
the published site and alert state. Only this exact current-schema skip is
accepted; malformed, incomplete, or other skipped evidence still fails closed.

## Public report and durable checkpoint

Successful trusted `main` observations are published at
Expand All @@ -30,7 +38,10 @@ surfaces are:
- `points/run-RUN.json`: one immutable detailed point per retained logical run;
- `reports/run-RUN/index.html`: one readable detailed report per point.

The normal predecessor is the last successfully deployed Pages tree. The
The normal predecessor is the last successfully deployed Pages tree. Successful
no-op runs are excluded when selecting its producer. The lookup checks at most
200 successful workflow runs and fails closed if none has a successful Pages
deployment; it never resets the checkpoint when that history bound is reached. The
publisher downloads only the paths named by its manifest, rejects redirects,
validates every digest and byte count, verifies the repository/workflow
identity, and then constructs a new full tree. A benchmark, predecessor,
Expand Down
23 changes: 22 additions & 1 deletion tools/ci/daily_performance_site.py
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,22 @@ def _read_json(path: Path, context: str) -> Any:
raise SiteError(f"cannot read {context}: {error}") from error


def publication_required(evidence: Any) -> bool:
"""Accept only complete measurements or the producer's exact GPU skip."""
if (
isinstance(evidence, dict)
and type(evidence.get("schema_version")) is int
and evidence == {
"schema_version": report.benchmark_contract.EVIDENCE_SCHEMA_VERSION,
"status": "skipped",
"reason": "gpu-hardware-qualification-required",
}
):
return False
report.validate_evidence(evidence)
return True


def _safe_relative_path(value: str) -> PurePosixPath:
path = PurePosixPath(value)
if (
Expand Down Expand Up @@ -816,6 +832,8 @@ def fetch(relative: str, maximum: int) -> bytes:
def main() -> int:
parser = argparse.ArgumentParser()
subparsers = parser.add_subparsers(dest="command", required=True)
classify = subparsers.add_parser("classify-evidence")
classify.add_argument("--evidence", type=Path, required=True)
build = subparsers.add_parser("build")
build.add_argument("--evidence", type=Path, required=True)
build.add_argument("--output", type=Path, required=True)
Expand All @@ -838,7 +856,10 @@ def main() -> int:
fetch.add_argument("--output", type=Path, required=True)
args = parser.parse_args()
try:
if args.command == "build":
if args.command == "classify-evidence":
required = publication_required(_read_json(args.evidence, "movement evidence"))
print(f"publish={'true' if required else 'false'}")
elif args.command == "build":
if (args.legacy_checkpoint is None) != (args.legacy_final_ref is None):
raise SiteError("legacy checkpoint and final ref must be supplied together")
build_site(
Expand Down
61 changes: 61 additions & 0 deletions tools/tests/test_daily_performance_site.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import importlib.util
import json
from pathlib import Path
import subprocess
import sys
import tempfile
import unittest
Expand Down Expand Up @@ -70,6 +71,66 @@ def setUp(self) -> None:
def tearDown(self) -> None:
self.temporary.cleanup()

def test_exact_gpu_qualification_skip_preserves_publication(self) -> None:
# Observed producer record from daily run 41189141733, attempt 1.
skipped = {
"reason": "gpu-hardware-qualification-required",
"schema_version": 7,
"status": "skipped",
}
self.assertFalse(site.publication_required(skipped))
self.evidence.write_text(json.dumps(skipped))
result = self.classify()
self.assertEqual(result.returncode, 0, result.stderr)
self.assertEqual(result.stdout, "publish=false\n")
self.assertFalse((self.root / "site").exists())
# A skip never becomes a valid complete report.
with self.assertRaises(report.ReportError):
report.validate_evidence(skipped)

def classify(self) -> subprocess.CompletedProcess[str]:
return subprocess.run(
[sys.executable, str(ROOT / "ci" / "daily_performance_site.py"),
"classify-evidence", "--evidence", str(self.evidence)],
capture_output=True, text=True, check=False,
)

def test_complete_measurements_continue_to_publication(self) -> None:
self.assertTrue(site.publication_required(evidence()))
result = self.classify()
self.assertEqual(result.returncode, 0, result.stderr)
self.assertEqual(result.stdout, "publish=true\n")
output = self.root / "site"
manifest = self.build(output)
self.assertEqual(site.validate_site(output), manifest)

def test_unrecognized_or_incomplete_evidence_fails_closed(self) -> None:
skipped = {
"schema_version": 7, "status": "skipped",
"reason": "gpu-hardware-qualification-required",
}
complete = evidence()
invalid = [
{}, None, [],
dict(skipped, reason="no-movement-sensitive-changes"),
dict(skipped, schema_version=6),
dict(skipped, schema_version=7.0),
dict(skipped, extra="untrusted"),
dict(skipped, status="error"),
{key: value for key, value in complete.items() if key != "records"},
]
for value in invalid:
with self.subTest(value=value):
self.evidence.write_text(json.dumps(value))
result = self.classify()
self.assertEqual(result.returncode, 2, result.stderr)
self.assertEqual(result.stdout, "")
self.assertIn("error:", result.stderr)
self.evidence.write_text("{broken")
result = self.classify()
self.assertEqual(result.returncode, 2)
self.assertEqual(result.stdout, "")

def build(self, output: Path, *, run_id: str = "2", attempt: str = "1",
checkpoint: Path | None = None) -> dict[str, object]:
arguments = {
Expand Down
70 changes: 70 additions & 0 deletions tools/tests/test_workflow_contracts.py
Original file line number Diff line number Diff line change
Expand Up @@ -2276,6 +2276,11 @@ def test_daily_report_stages_verified_inputs_and_keeps_timing_informational(
self.assertIn("--legacy-final-ref", project)
self.assertIn("daily_performance_site.py build", project)
self.assertIn("daily_performance_site.py validate", project)
self.assertIn("publish: ${{ steps.evidence.outputs.publish }}", project)
self.assertIn("daily_performance_site.py classify-evidence", project)
self.assertLess(project.index("daily_performance_site.py classify-evidence"),
project.index(" - name: Load the exact predecessor checkpoint"))
self.assertEqual(project.count("if: steps.evidence.outputs.publish == 'true'"), 4)
self.assertRegex(project, r"actions/upload-pages-artifact@[0-9a-f]{40}\b")
self.assertNotIn("benchmark-data:refs", workflow)
self.assertNotIn("HEAD:benchmark-data", workflow)
Expand All @@ -2286,18 +2291,83 @@ def test_daily_report_stages_verified_inputs_and_keeps_timing_informational(
]
self.assertIn("pages: write", deploy)
self.assertIn("id-token: write", deploy)
self.assertIn("if: needs.project.outputs.publish == 'true'", deploy)
self.assertNotIn("issues: write", deploy)
self.assertIn("environment:\n name: github-pages", deploy)
self.assertRegex(deploy, r"actions/deploy-pages@[0-9a-f]{40}\b")

alerts = workflow[workflow.index(" alerts:") :]
self.assertIn("needs: [project, deploy]", alerts)
self.assertIn("if: needs.project.outputs.publish == 'true'", alerts)
self.assertIn("issues: write", alerts)
self.assertNotIn("pages: write", alerts)
self.assertIn("reconcile_performance_alerts.py", alerts)
self.assertIn("daily_performance_site.py validate-artifact", alerts)
self.assertNotIn("pull_request:", workflow[: workflow.index("jobs:")])

@unittest.skipUnless(shutil.which("jq"), "jq is required for workflow shell checks")
def test_daily_predecessor_skips_successful_runs_without_a_deployment(self) -> None:
workflow = self.text("daily-client-performance.yml")
step = workflow.split(" - name: Load the exact predecessor checkpoint\n", 1)[1]
script = textwrap.dedent(step.split(" run: |\n", 1)[1].split(
" - name: Build and validate the complete static deployment\n", 1)[0])
with tempfile.TemporaryDirectory() as temporary:
directory = Path(temporary)
(directory / "build").mkdir()
commands = directory / "bin"
commands.mkdir()
gh = commands / "gh"
gh.write_text(f"#!{shutil.which('python3')}\n" + textwrap.dedent('''\
import json, os, sys
endpoint = sys.argv[-1]
with open(os.environ["READ_LOG"], "a") as log:
log.write(endpoint + "\\n")
if "/workflows/" in endpoint:
if "page=1" in endpoint:
runs = [{"id": number, "event": "schedule", "run_attempt": 1,
"head_sha": "b" * 40} for number in range(22, 2, -1)]
else:
runs = [{"id": 2, "event": "schedule", "run_attempt": 1,
"head_sha": "a" * 40}]
value = {"workflow_runs": runs}
else:
success = "/runs/2/" in endpoint and os.environ["DEPLOY_RESULT"] == "success"
value = {"total_count": 1, "jobs": [{
"name": "Deploy the validated performance site", "status": "completed",
"conclusion": "success" if success else "skipped"}]}
print(json.dumps(value))
'''))
gh.chmod(0o755)
fetch = commands / "python3"
fetch.write_text(f"#!{shutil.which('python3')}\n" + textwrap.dedent('''\
import json
from pathlib import Path
path = Path("build/performance-checkpoint/v1")
path.mkdir(parents=True, exist_ok=True)
(path / "manifest.json").write_text(json.dumps({
"run_id": "2", "run_attempt": "1", "source_sha": "a" * 40}))
'''))
fetch.chmod(0o755)
for outcome in ("success", "skipped"):
with self.subTest(deployed=outcome):
log = directory / "reads.log"
log.unlink(missing_ok=True)
result = subprocess.run(
["bash", "-euo", "pipefail", "-c", script], cwd=directory,
env=dict(os.environ, PATH=f"{commands}:{os.environ['PATH']}",
CHECKPOINT_SOURCE="pages", GITHUB_REPOSITORY="atrinik/classic",
PAGES_URL="https://atrinik.github.io/classic/",
READ_LOG=str(log), DEPLOY_RESULT=outcome),
capture_output=True, text=True, check=False,
)
if outcome == "success":
self.assertEqual(result.returncode, 0, result.stderr)
else:
self.assertNotEqual(result.returncode, 0)
self.assertIn("per_page=20&page=2", log.read_text())
self.assertIn("/runs/3/jobs", log.read_text())
self.assertIn("/runs/2/jobs", log.read_text())

def test_linux_checks_pin_image_and_isolate_compiler_caches(self) -> None:
workflow = self.text("check.yml")
match = re.search(
Expand Down