From 325d9d028a6e6bf41617f6295978ac95ddef75c1 Mon Sep 17 00:00:00 2001 From: jewelkm89 Date: Tue, 30 Jun 2026 18:56:54 +0530 Subject: [PATCH 1/9] Add L1/L2 critical integration tests for CRCR Introduce payload validators, JSON fixtures, and unit tests that run on push/PR to this repo. Enhance the dispatch-receiver and L2 workflows to assert payload shape, PyTorch checkout SHA, and deterministic relay callbacks instead of random pass/fail simulation. Test Plan: ``` python3 -m unittest discover -s tests -p 'test_*.py' -v ``` Authored with an AI assistant. Co-authored-by: Cursor --- .github/workflows/crcr-dispatch-receiver.yml | 79 +++++++---- .github/workflows/crcr-unit-tests.yml | 26 ++++ .github/workflows/oot-l2-ci.yml | 136 +++++++----------- .gitignore | 2 + README.md | 39 ++++++ tests/fixtures/pull_request_closed.json | 19 +++ tests/fixtures/pull_request_opened.json | 20 +++ tests/fixtures/pull_request_synchronize.json | 19 +++ tests/fixtures/push_ciflow_tag.json | 13 ++ tests/test_validators.py | 140 +++++++++++++++++++ tests/validate_callback_payload.py | 69 +++++++++ tests/validate_dispatch_payload.py | 109 +++++++++++++++ tests/validate_pytorch_checkout.py | 65 +++++++++ 13 files changed, 625 insertions(+), 111 deletions(-) create mode 100644 .github/workflows/crcr-unit-tests.yml create mode 100644 .gitignore create mode 100644 tests/fixtures/pull_request_closed.json create mode 100644 tests/fixtures/pull_request_opened.json create mode 100644 tests/fixtures/pull_request_synchronize.json create mode 100644 tests/fixtures/push_ciflow_tag.json create mode 100644 tests/test_validators.py create mode 100644 tests/validate_callback_payload.py create mode 100644 tests/validate_dispatch_payload.py create mode 100644 tests/validate_pytorch_checkout.py diff --git a/.github/workflows/crcr-dispatch-receiver.yml b/.github/workflows/crcr-dispatch-receiver.yml index c777289..5f9ef81 100644 --- a/.github/workflows/crcr-dispatch-receiver.yml +++ b/.github/workflows/crcr-dispatch-receiver.yml @@ -32,42 +32,67 @@ jobs: if: ${{ github.event.client_payload.payload.action == 'closed' }} runs-on: ubuntu-latest steps: - - run: echo "PR closed, canceling older runs in the same concurrency group" - inspect: + - name: Assert closed action skips build job + run: echo "PR closed — cancel-workflow job ran as expected" + + l1-critical: if: ${{ github.event.client_payload.payload.action != 'closed' }} runs-on: ubuntu-latest steps: - - name: Display parsed fields + - name: Checkout crcr-test + uses: actions/checkout@v4 + + - name: L1 — validate dispatch payload structure + env: + CLIENT_PAYLOAD: ${{ toJson(github.event.client_payload) }} + run: | + echo "${CLIENT_PAYLOAD}" | python3 tests/validate_dispatch_payload.py + + - name: L1 — checkout PyTorch at dispatched SHA + uses: actions/checkout@v4 + with: + repository: pytorch/pytorch + ref: >- + ${{ github.event.client_payload.event_type == 'pull_request' && + github.event.client_payload.payload.pull_request.head.sha || + github.event.client_payload.payload.after }} + path: pytorch + + - name: L1 — verify checkout matches dispatch SHA + env: + CLIENT_PAYLOAD: ${{ toJson(github.event.client_payload) }} + run: | + echo "${CLIENT_PAYLOAD}" > /tmp/client_payload.json + python3 tests/validate_pytorch_checkout.py \ + --payload /tmp/client_payload.json \ + --pytorch-dir pytorch + + - name: L1 — assert delivery_id is stable for L2 linkage + run: | + DELIVERY_ID="${{ github.event.client_payload.delivery_id }}" + if [ -z "${DELIVERY_ID}" ]; then + echo "::error::delivery_id is empty" + exit 1 + fi + echo "delivery_id=${DELIVERY_ID}" >> "$GITHUB_OUTPUT" + id: delivery + + - name: Write integration summary + if: always() run: | EVENT_TYPE="${{ github.event.client_payload.event_type }}" - echo "### Dispatch Summary" >> $GITHUB_STEP_SUMMARY + echo "### L1 Critical Tests" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY - echo "| Field | Value |" >> $GITHUB_STEP_SUMMARY - echo "|-------|-------|" >> $GITHUB_STEP_SUMMARY + echo "| Check | Result |" >> $GITHUB_STEP_SUMMARY + echo "|-------|--------|" >> $GITHUB_STEP_SUMMARY + echo "| Payload structure | passed |" >> $GITHUB_STEP_SUMMARY + echo "| PyTorch checkout | passed |" >> $GITHUB_STEP_SUMMARY + echo "| delivery_id | \`${{ steps.delivery.outputs.delivery_id || github.event.client_payload.delivery_id }}\` |" >> $GITHUB_STEP_SUMMARY echo "| Event type | \`${EVENT_TYPE}\` |" >> $GITHUB_STEP_SUMMARY - echo "| Upstream repo | \`${{ github.event.client_payload.payload.repository.full_name }}\` |" >> $GITHUB_STEP_SUMMARY - if [ "$EVENT_TYPE" = "pull_request" ]; then - echo "| PR number | \`${{ github.event.client_payload.payload.pull_request.number }}\` |" >> $GITHUB_STEP_SUMMARY - echo "| PR action | \`${{ github.event.client_payload.payload.action }}\` |" >> $GITHUB_STEP_SUMMARY - echo "| PR title | \`${{ github.event.client_payload.payload.pull_request.title }}\` |" >> $GITHUB_STEP_SUMMARY + echo "| PR | #${{ github.event.client_payload.payload.pull_request.number }} (${{ github.event.client_payload.payload.action }}) |" >> $GITHUB_STEP_SUMMARY echo "| Head SHA | \`${{ github.event.client_payload.payload.pull_request.head.sha }}\` |" >> $GITHUB_STEP_SUMMARY - echo "| Head repo | \`${{ github.event.client_payload.payload.pull_request.head.repo.full_name }}\` |" >> $GITHUB_STEP_SUMMARY - elif [ "$EVENT_TYPE" = "push" ]; then + else echo "| Ref | \`${{ github.event.client_payload.payload.ref }}\` |" >> $GITHUB_STEP_SUMMARY echo "| SHA | \`${{ github.event.client_payload.payload.after }}\` |" >> $GITHUB_STEP_SUMMARY - echo "| Base ref | \`${{ github.event.client_payload.payload.base_ref }}\` |" >> $GITHUB_STEP_SUMMARY - echo "| Deleted | \`${{ github.event.client_payload.payload.deleted }}\` |" >> $GITHUB_STEP_SUMMARY fi - - - name: Log full event to step summary - if: always() - run: | - echo "" >> $GITHUB_STEP_SUMMARY - echo "### Full Event Payload" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - cat <<'EOF' >> $GITHUB_STEP_SUMMARY - ```json - ${{ toJson(github.event) }} - ``` - EOF diff --git a/.github/workflows/crcr-unit-tests.yml b/.github/workflows/crcr-unit-tests.yml new file mode 100644 index 0000000..0c4b258 --- /dev/null +++ b/.github/workflows/crcr-unit-tests.yml @@ -0,0 +1,26 @@ +name: CRCR Unit Tests + +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + +permissions: + contents: read + +jobs: + validators: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Run payload validator unit tests + run: python3 -m unittest discover -s tests -p 'test_*.py' -v + + - name: Validate fixture files individually + run: | + for f in tests/fixtures/*.json; do + echo "Validating $f" + python3 tests/validate_dispatch_payload.py < "$f" + done diff --git a/.github/workflows/oot-l2-ci.yml b/.github/workflows/oot-l2-ci.yml index 4d8e409..606a6c4 100644 --- a/.github/workflows/oot-l2-ci.yml +++ b/.github/workflows/oot-l2-ci.yml @@ -1,4 +1,4 @@ -name: OOT L2 CI +name: OOT L2 Critical on: repository_dispatch: @@ -6,7 +6,7 @@ on: - pull_request run-name: >- - OOT L2 - + OOT L2 Critical - PR #${{ github.event.client_payload.payload.pull_request.number }} (${{ github.event.client_payload.payload.action }}) @@ -25,111 +25,79 @@ jobs: if: ${{ github.event.client_payload.payload.action == 'closed' }} runs-on: ubuntu-latest steps: - - run: echo "PR closed — canceling older runs in the same concurrency group" + - run: echo "PR closed — cancel-workflow job ran as expected" - build-and-test: + l2-critical: if: ${{ github.event.client_payload.payload.action != 'closed' }} runs-on: ubuntu-latest steps: - - name: Report CI started + - name: Checkout crcr-test + uses: actions/checkout@v4 + + - name: L2 — validate dispatch payload before callbacks + env: + CLIENT_PAYLOAD: ${{ toJson(github.event.client_payload) }} + run: echo "${CLIENT_PAYLOAD}" | python3 tests/validate_dispatch_payload.py + + - name: L2 — report in_progress to relay + id: in_progress uses: pytorch/test-infra/.github/actions/cross-repo-ci-relay-callback@main with: status: in_progress - - name: Checkout - uses: actions/checkout@v4 - - - name: Checkout PyTorch at dispatched SHA + - name: L2 — checkout PyTorch at dispatched SHA uses: actions/checkout@v4 with: repository: pytorch/pytorch ref: ${{ github.event.client_payload.payload.pull_request.head.sha }} path: pytorch - - name: Generate random test configuration - id: config + - name: L2 — verify checkout matches dispatch SHA + env: + CLIENT_PAYLOAD: ${{ toJson(github.event.client_payload) }} run: | - SEED=$(( RANDOM ^ ${{ github.run_id }} )) - TOTAL=$(( (SEED % 400) + 100 )) - FAIL_RATE=$(( SEED % 12 )) - FAILED=$(( TOTAL * FAIL_RATE / 100 )) - SKIP_RATE=$(( (SEED / 7) % 5 )) - SKIPPED=$(( TOTAL * SKIP_RATE / 100 )) - PASSED=$(( TOTAL - FAILED - SKIPPED )) - SLEEP_SECS=$(( (SEED % 45) + 15 )) - echo "total=$TOTAL" >> "$GITHUB_OUTPUT" - echo "passed=$PASSED" >> "$GITHUB_OUTPUT" - echo "failed=$FAILED" >> "$GITHUB_OUTPUT" - echo "skipped=$SKIPPED" >> "$GITHUB_OUTPUT" - echo "sleep=$SLEEP_SECS" >> "$GITHUB_OUTPUT" - echo "### Test Configuration" >> $GITHUB_STEP_SUMMARY - echo "| Param | Value |" >> $GITHUB_STEP_SUMMARY - echo "|-------|-------|" >> $GITHUB_STEP_SUMMARY - echo "| Total tests | $TOTAL |" >> $GITHUB_STEP_SUMMARY - echo "| Passed | $PASSED |" >> $GITHUB_STEP_SUMMARY - echo "| Failed | $FAILED |" >> $GITHUB_STEP_SUMMARY - echo "| Skipped | $SKIPPED |" >> $GITHUB_STEP_SUMMARY - echo "| Simulated duration | ${SLEEP_SECS}s |" >> $GITHUB_STEP_SUMMARY + echo "${CLIENT_PAYLOAD}" > /tmp/client_payload.json + python3 tests/validate_pytorch_checkout.py \ + --payload /tmp/client_payload.json \ + --pytorch-dir pytorch - - name: Simulate build + - name: L2 — run deterministic smoke checks + id: smoke run: | - echo "=== Building against PyTorch SHA: ${{ github.event.client_payload.payload.pull_request.head.sha }} ===" - echo "Installing dependencies..." - sleep $(( (RANDOM % 5) + 2 )) - echo "Compiling OOT extensions..." - sleep $(( (RANDOM % 5) + 2 )) - echo "Build complete." + set -euo pipefail + test -f pytorch/README.md + test -d pytorch/.github + python3 -c "import json; print('smoke ok')" + echo "passed=3" >> "$GITHUB_OUTPUT" + echo "failed=0" >> "$GITHUB_OUTPUT" + echo "skipped=0" >> "$GITHUB_OUTPUT" - - name: Simulate test execution - id: tests - run: | - echo "=== Running ${{ steps.config.outputs.total }} tests ===" - - EDGE_ROLL=$(( RANDOM % 20 )) - if [ "$EDGE_ROLL" -eq 0 ]; then - echo "::notice::Simulating edge case: cancelled" - echo "outcome=cancelled" >> "$GITHUB_OUTPUT" - echo "" >> $GITHUB_STEP_SUMMARY - echo "### Test Results" >> $GITHUB_STEP_SUMMARY - echo "- **Outcome:** cancelled (simulated edge case)" >> $GITHUB_STEP_SUMMARY - elif [ "$EDGE_ROLL" -eq 1 ]; then - sleep ${{ steps.config.outputs.sleep }} - echo "::notice::Simulating edge case: timed_out" - echo "outcome=timed_out" >> "$GITHUB_OUTPUT" - echo "" >> $GITHUB_STEP_SUMMARY - echo "### Test Results" >> $GITHUB_STEP_SUMMARY - echo "- **Outcome:** timed_out (simulated edge case)" >> $GITHUB_STEP_SUMMARY - else - sleep ${{ steps.config.outputs.sleep }} - FAILED=${{ steps.config.outputs.failed }} - if [ "$FAILED" -gt 0 ]; then - echo "::warning::$FAILED test(s) failed" - for i in $(seq 1 $FAILED); do - echo "FAIL: test_op_$(( RANDOM % 999 )) (TestOOTBackend)" - done - echo "outcome=failure" >> "$GITHUB_OUTPUT" - else - echo "All tests passed." - echo "outcome=success" >> "$GITHUB_OUTPUT" - fi - echo "" >> $GITHUB_STEP_SUMMARY - echo "### Test Results" >> $GITHUB_STEP_SUMMARY - echo "- **Passed:** ${{ steps.config.outputs.passed }}" >> $GITHUB_STEP_SUMMARY - echo "- **Failed:** ${{ steps.config.outputs.failed }}" >> $GITHUB_STEP_SUMMARY - echo "- **Skipped:** ${{ steps.config.outputs.skipped }}" >> $GITHUB_STEP_SUMMARY - fi - - - name: Report CI completed + - name: L2 — report completed to relay if: always() uses: pytorch/test-infra/.github/actions/cross-repo-ci-relay-callback@main with: status: completed - conclusion: ${{ steps.tests.outputs.outcome || (job.status == 'success' && 'success' || 'failure') }} + conclusion: ${{ job.status == 'success' && 'success' || 'failure' }} test-results: >- { - "passed": ${{ steps.config.outputs.passed || 0 }}, - "failed": ${{ steps.config.outputs.failed || 0 }}, - "skipped": ${{ steps.config.outputs.skipped || 0 }}, - "total": ${{ steps.config.outputs.total || 0 }} + "passed": ${{ steps.smoke.outputs.passed || 0 }}, + "failed": ${{ steps.smoke.outputs.failed || 0 }}, + "skipped": ${{ steps.smoke.outputs.skipped || 0 }} } artifact-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + + - name: Write L2 integration summary + if: always() + run: | + echo "### L2 Critical Tests" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "| Integration point | Status |" >> $GITHUB_STEP_SUMMARY + echo "|-------------------|--------|" >> $GITHUB_STEP_SUMMARY + echo "| Dispatch payload validation | passed |" >> $GITHUB_STEP_SUMMARY + echo "| in_progress callback | ${{ steps.in_progress.outcome }} |" >> $GITHUB_STEP_SUMMARY + echo "| PyTorch checkout | passed |" >> $GITHUB_STEP_SUMMARY + echo "| completed callback | see job status |" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "HUD: [crcr-test dashboard](https://hud.pytorch.org/crcr/pytorch/crcr-test)" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "delivery_id: \`${{ github.event.client_payload.delivery_id }}\`" >> $GITHUB_STEP_SUMMARY diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..7a60b85 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +__pycache__/ +*.pyc diff --git a/README.md b/README.md index 5998544..59614f2 100644 --- a/README.md +++ b/README.md @@ -145,6 +145,45 @@ Once Steps 1–3 are complete, your workflow will start receiving dispatches. Yo 2. Looking for runs triggered by `pytorch-fdn-cross-repo-ci-relay[bot]` 3. Inspecting the step summary for the full event payload +## Critical Tests + +This repository runs automated critical tests on every CRCR dispatch. They validate the L1 and L2 integration points end-to-end. + +| Workflow | Level | Trigger | What it verifies | +|----------|-------|---------|------------------| +| [`crcr-dispatch-receiver.yml`](.github/workflows/crcr-dispatch-receiver.yml) | L1 | `pull_request`, `push` | Dispatch payload shape, `delivery_id`, PyTorch checkout at dispatched SHA | +| [`oot-l2-ci.yml`](.github/workflows/oot-l2-ci.yml) | L2 | `pull_request` | L1 checks + `in_progress`/`completed` relay callbacks, test-results payload, HUD link | +| [`crcr-unit-tests.yml`](.github/workflows/crcr-unit-tests.yml) | Local | push/PR to this repo | Payload validator unit tests against JSON fixtures | + +### L1 integration points + +- `client_payload.event_type` is `pull_request` or `push` +- `client_payload.delivery_id` is present (required for L2 state tracking) +- `client_payload.payload.repository.full_name` is `pytorch/pytorch` +- PR events: `action` in `opened`/`reopened`/`synchronize`/`closed`, valid `pull_request.head.sha` +- Push events: `ref` starts with `refs/`, valid `after` SHA +- PyTorch checkout resolves to the dispatched SHA +- `closed` action runs only the cancel job (no build) + +### L2 integration points + +- OIDC token minting (`id-token: write`) +- `in_progress` callback accepted by relay (state machine: `DISPATCHED → IN_PROGRESS`) +- Deterministic smoke checks (no random pass/fail) +- `completed` callback with `conclusion: success` and `test_results` summary +- `artifact_url` points at the GitHub Actions run page +- Results visible on [hud.pytorch.org/crcr/pytorch/crcr-test](https://hud.pytorch.org/crcr/pytorch/crcr-test) + +### Running validators locally + +```bash +# Validate a fixture or saved client_payload JSON +python3 tests/validate_dispatch_payload.py < tests/fixtures/pull_request_opened.json + +# Run all unit tests +python3 -m unittest discover -s tests -p 'test_*.py' -v +``` + ## Dispatch Payload Structure The relay wraps the original GitHub event inside `client_payload`: diff --git a/tests/fixtures/pull_request_closed.json b/tests/fixtures/pull_request_closed.json new file mode 100644 index 0000000..d9e027d --- /dev/null +++ b/tests/fixtures/pull_request_closed.json @@ -0,0 +1,19 @@ +{ + "event_type": "pull_request", + "delivery_id": "test-delivery-pr-closed-001", + "payload": { + "action": "closed", + "repository": { + "full_name": "pytorch/pytorch" + }, + "pull_request": { + "number": 12345, + "head": { + "sha": "cccccccccccccccccccccccccccccccccccccccc", + "repo": { + "full_name": "contributor/pytorch" + } + } + } + } +} diff --git a/tests/fixtures/pull_request_opened.json b/tests/fixtures/pull_request_opened.json new file mode 100644 index 0000000..18efac7 --- /dev/null +++ b/tests/fixtures/pull_request_opened.json @@ -0,0 +1,20 @@ +{ + "event_type": "pull_request", + "delivery_id": "test-delivery-pr-opened-001", + "payload": { + "action": "opened", + "repository": { + "full_name": "pytorch/pytorch" + }, + "pull_request": { + "number": 12345, + "title": "Test PR", + "head": { + "sha": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "repo": { + "full_name": "contributor/pytorch" + } + } + } + } +} diff --git a/tests/fixtures/pull_request_synchronize.json b/tests/fixtures/pull_request_synchronize.json new file mode 100644 index 0000000..16a623c --- /dev/null +++ b/tests/fixtures/pull_request_synchronize.json @@ -0,0 +1,19 @@ +{ + "event_type": "pull_request", + "delivery_id": "test-delivery-pr-sync-001", + "payload": { + "action": "synchronize", + "repository": { + "full_name": "pytorch/pytorch" + }, + "pull_request": { + "number": 12345, + "head": { + "sha": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "repo": { + "full_name": "contributor/pytorch" + } + } + } + } +} diff --git a/tests/fixtures/push_ciflow_tag.json b/tests/fixtures/push_ciflow_tag.json new file mode 100644 index 0000000..230f84c --- /dev/null +++ b/tests/fixtures/push_ciflow_tag.json @@ -0,0 +1,13 @@ +{ + "event_type": "push", + "delivery_id": "test-delivery-push-001", + "payload": { + "ref": "refs/tags/ciflow/trunk/12345", + "after": "dddddddddddddddddddddddddddddddddddddddd", + "deleted": false, + "base_ref": "refs/heads/main", + "repository": { + "full_name": "pytorch/pytorch" + } + } +} diff --git a/tests/test_validators.py b/tests/test_validators.py new file mode 100644 index 0000000..e030cfc --- /dev/null +++ b/tests/test_validators.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 +"""Unit tests for CRCR payload validators (runs locally without relay).""" + +from __future__ import annotations + +import json +import tempfile +import unittest +from pathlib import Path + +from validate_callback_payload import validate_callback_body +from validate_dispatch_payload import ValidationError, validate_dispatch_payload + +FIXTURES = Path(__file__).parent / "fixtures" + + +def _load(name: str) -> dict: + return json.loads((FIXTURES / name).read_text()) + + +class TestDispatchPayload(unittest.TestCase): + def test_pull_request_opened_fixture(self) -> None: + validate_dispatch_payload(_load("pull_request_opened.json")) + + def test_pull_request_synchronize_fixture(self) -> None: + validate_dispatch_payload(_load("pull_request_synchronize.json")) + + def test_pull_request_closed_fixture(self) -> None: + validate_dispatch_payload(_load("pull_request_closed.json")) + + def test_push_ciflow_tag_fixture(self) -> None: + validate_dispatch_payload(_load("push_ciflow_tag.json")) + + def test_rejects_missing_delivery_id(self) -> None: + payload = _load("pull_request_opened.json") + del payload["delivery_id"] + with self.assertRaises(ValidationError): + validate_dispatch_payload(payload) + + def test_rejects_wrong_upstream_repo(self) -> None: + payload = _load("pull_request_opened.json") + payload["payload"]["repository"]["full_name"] = "other/repo" + with self.assertRaises(ValidationError): + validate_dispatch_payload(payload) + + def test_rejects_invalid_sha(self) -> None: + payload = _load("pull_request_opened.json") + payload["payload"]["pull_request"]["head"]["sha"] = "not-a-sha" + with self.assertRaises(ValidationError): + validate_dispatch_payload(payload) + + +class TestCallbackPayload(unittest.TestCase): + def _completed_body(self) -> dict: + body = _load("pull_request_opened.json") + body["workflow"] = { + "schema_version": "1", + "status": "completed", + "conclusion": "success", + "name": "OOT L2 Critical", + "url": "https://github.com/pytorch/crcr-test/actions/runs/1", + "run_attempt": "1", + "job_name": "l2-critical", + "check_run_id": "12345", + "run_id": "1", + "started_at": "2026-06-29T00:00:00Z", + "completed_at": "2026-06-29T00:01:00Z", + "test_results": {"passed": 3, "failed": 0, "skipped": 0}, + "artifact_url": "https://example.com/artifacts/1", + } + return body + + def test_in_progress_shape(self) -> None: + body = _load("pull_request_opened.json") + body["workflow"] = { + "schema_version": "1", + "status": "in_progress", + "conclusion": None, + "name": "OOT L2 Critical", + "url": "https://github.com/pytorch/crcr-test/actions/runs/1", + "run_attempt": "1", + "job_name": "l2-critical", + "check_run_id": "12345", + "run_id": "1", + "started_at": "2026-06-29T00:00:00Z", + "completed_at": None, + } + validate_callback_body(body, expect_status="in_progress") + + def test_completed_shape(self) -> None: + validate_callback_body(self._completed_body(), expect_status="completed") + + def test_completed_requires_conclusion(self) -> None: + body = self._completed_body() + body["workflow"]["conclusion"] = None + with self.assertRaises(ValidationError): + validate_callback_body(body, expect_status="completed") + + +class TestPytorchCheckout(unittest.TestCase): + def test_checkout_matches_dispatch_sha(self) -> None: + import subprocess + + from validate_pytorch_checkout import _head_sha + + payload = _load("pull_request_opened.json") + expected = _head_sha(payload) + with tempfile.TemporaryDirectory() as tmp: + subprocess.run(["git", "init", tmp], check=True, capture_output=True) + subprocess.run( + ["git", "-C", tmp, "commit", "--allow-empty", "-m", "init"], + check=True, + capture_output=True, + env={ + **__import__("os").environ, + "GIT_AUTHOR_NAME": "test", + "GIT_AUTHOR_EMAIL": "test@test.com", + "GIT_COMMITTER_NAME": "test", + "GIT_COMMITTER_EMAIL": "test@test.com", + }, + ) + # Empty repo won't match arbitrary SHA; just verify script runs. + result = subprocess.run( + [ + "python3", + str(Path(__file__).parent / "validate_pytorch_checkout.py"), + "--payload", + str(FIXTURES / "pull_request_opened.json"), + "--pytorch-dir", + tmp, + ], + capture_output=True, + text=True, + ) + self.assertNotEqual(result.returncode, 0) + self.assertIn("checkout SHA mismatch", result.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/validate_callback_payload.py b/tests/validate_callback_payload.py new file mode 100644 index 0000000..35541da --- /dev/null +++ b/tests/validate_callback_payload.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +"""Validate CRCR L2 callback body shape (downstream -> relay wire format).""" + +from __future__ import annotations + +import json +import sys +from typing import Any + +from validate_dispatch_payload import ValidationError, validate_dispatch_payload + +ALLOWED_STATUSES = frozenset({"in_progress", "completed"}) +ALLOWED_CONCLUSIONS = frozenset({"success", "failure", "cancelled", "timed_out"}) + + +def validate_callback_body(body: dict[str, Any], *, expect_status: str) -> None: + validate_dispatch_payload(body) + + workflow = body.get("workflow") + if not isinstance(workflow, dict): + raise ValidationError("workflow must be an object") + + status = workflow.get("status") + if status != expect_status: + raise ValidationError(f"workflow.status must be {expect_status!r}, got {status!r}") + + for field in ("schema_version", "name", "url", "job_name", "check_run_id", "run_id", "run_attempt"): + if field not in workflow: + raise ValidationError(f"workflow.{field} is required for L2 callbacks") + + if expect_status == "in_progress": + if workflow.get("conclusion") is not None: + raise ValidationError("in_progress callback must not set workflow.conclusion") + if not workflow.get("started_at"): + raise ValidationError("in_progress callback must set workflow.started_at") + else: + conclusion = workflow.get("conclusion") + if conclusion not in ALLOWED_CONCLUSIONS: + raise ValidationError( + f"completed callback conclusion must be one of {sorted(ALLOWED_CONCLUSIONS)}, " + f"got {conclusion!r}" + ) + if not workflow.get("completed_at"): + raise ValidationError("completed callback must set workflow.completed_at") + + +def main() -> int: + if len(sys.argv) != 3: + print("usage: validate_callback_payload.py ", file=sys.stderr) + return 2 + + expect_status = sys.argv[1] + if expect_status not in ALLOWED_STATUSES: + print(f"::error::status arg must be in_progress or completed", file=sys.stderr) + return 2 + + try: + body = json.loads(open(sys.argv[2]).read()) + validate_callback_body(body, expect_status=expect_status) + except (ValidationError, json.JSONDecodeError, OSError) as exc: + print(f"::error::L2 callback validation failed: {exc}", file=sys.stderr) + return 1 + + print(f"L2 {expect_status} callback validation passed") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/validate_dispatch_payload.py b/tests/validate_dispatch_payload.py new file mode 100644 index 0000000..b8ea089 --- /dev/null +++ b/tests/validate_dispatch_payload.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 +"""Validate CRCR repository_dispatch client_payload structure (L1 integration).""" + +from __future__ import annotations + +import json +import re +import sys +from typing import Any + +UPSTREAM_REPO = "pytorch/pytorch" +ALLOWED_EVENT_TYPES = frozenset({"pull_request", "push"}) +ALLOWED_PR_ACTIONS = frozenset({"opened", "reopened", "synchronize", "closed"}) +SHA_RE = re.compile(r"^[0-9a-f]{40}$") + + +class ValidationError(Exception): + pass + + +def _require(obj: dict[str, Any], key: str, label: str) -> Any: + if key not in obj: + raise ValidationError(f"missing {label}.{key}") + return obj[key] + + +def _require_sha(value: str, label: str) -> None: + if not SHA_RE.match(value): + raise ValidationError(f"{label} is not a 40-char hex SHA: {value!r}") + + +def validate_dispatch_payload(client_payload: dict[str, Any]) -> None: + event_type = _require(client_payload, "event_type", "client_payload") + if event_type not in ALLOWED_EVENT_TYPES: + raise ValidationError( + f"client_payload.event_type must be one of {sorted(ALLOWED_EVENT_TYPES)}, " + f"got {event_type!r}" + ) + + delivery_id = _require(client_payload, "delivery_id", "client_payload") + if not isinstance(delivery_id, str) or not delivery_id.strip(): + raise ValidationError("client_payload.delivery_id must be a non-empty string") + + payload = _require(client_payload, "payload", "client_payload") + if not isinstance(payload, dict): + raise ValidationError("client_payload.payload must be an object") + + repo = _require(payload, "repository", "client_payload.payload") + if not isinstance(repo, dict): + raise ValidationError("client_payload.payload.repository must be an object") + full_name = _require(repo, "full_name", "client_payload.payload.repository") + if full_name != UPSTREAM_REPO: + raise ValidationError( + f"upstream repo must be {UPSTREAM_REPO!r}, got {full_name!r}" + ) + + if event_type == "pull_request": + action = _require(payload, "action", "client_payload.payload") + if action not in ALLOWED_PR_ACTIONS: + raise ValidationError( + f"pull_request action must be one of {sorted(ALLOWED_PR_ACTIONS)}, " + f"got {action!r}" + ) + pr = _require(payload, "pull_request", "client_payload.payload") + if not isinstance(pr, dict): + raise ValidationError("client_payload.payload.pull_request must be an object") + number = _require(pr, "number", "client_payload.payload.pull_request") + if not isinstance(number, int) or number <= 0: + raise ValidationError(f"pull_request.number must be a positive int, got {number!r}") + head = _require(pr, "head", "client_payload.payload.pull_request") + if not isinstance(head, dict): + raise ValidationError("pull_request.head must be an object") + head_sha = _require(head, "sha", "pull_request.head") + _require_sha(head_sha, "pull_request.head.sha") + head_repo = _require(head, "repo", "pull_request.head") + if not isinstance(head_repo, dict): + raise ValidationError("pull_request.head.repo must be an object") + _require(head_repo, "full_name", "pull_request.head.repo") + + elif event_type == "push": + ref = _require(payload, "ref", "client_payload.payload") + if not isinstance(ref, str) or not ref.startswith("refs/"): + raise ValidationError(f"push ref must start with refs/, got {ref!r}") + deleted = payload.get("deleted", False) + if not deleted: + after = _require(payload, "after", "client_payload.payload") + _require_sha(after, "payload.after") + + +def main() -> int: + raw = sys.stdin.read() + try: + data = json.loads(raw) + except json.JSONDecodeError as exc: + print(f"::error::client_payload is not valid JSON: {exc}", file=sys.stderr) + return 1 + + try: + validate_dispatch_payload(data) + except ValidationError as exc: + print(f"::error::L1 dispatch payload validation failed: {exc}", file=sys.stderr) + return 1 + + print("L1 dispatch payload validation passed") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/validate_pytorch_checkout.py b/tests/validate_pytorch_checkout.py new file mode 100644 index 0000000..e7fec74 --- /dev/null +++ b/tests/validate_pytorch_checkout.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +"""Verify PyTorch checkout matches the SHA from a CRCR dispatch payload.""" + +from __future__ import annotations + +import argparse +import json +import subprocess +import sys +from pathlib import Path + +from validate_dispatch_payload import ValidationError, validate_dispatch_payload + + +def _head_sha(client_payload: dict) -> str: + event_type = client_payload["event_type"] + payload = client_payload["payload"] + if event_type == "pull_request": + return payload["pull_request"]["head"]["sha"] + return payload["after"] + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--payload", required=True, help="Path to client_payload JSON file") + parser.add_argument("--pytorch-dir", required=True, help="Path to checked-out pytorch tree") + args = parser.parse_args() + + client_payload = json.loads(Path(args.payload).read_text()) + try: + validate_dispatch_payload(client_payload) + except ValidationError as exc: + print(f"::error::{exc}", file=sys.stderr) + return 1 + + expected = _head_sha(client_payload) + pytorch_dir = Path(args.pytorch_dir) + if not pytorch_dir.is_dir(): + print(f"::error::pytorch dir does not exist: {pytorch_dir}", file=sys.stderr) + return 1 + + result = subprocess.run( + ["git", "-C", str(pytorch_dir), "rev-parse", "HEAD"], + check=False, + capture_output=True, + text=True, + ) + if result.returncode != 0: + print(f"::error::git rev-parse failed: {result.stderr.strip()}", file=sys.stderr) + return 1 + + actual = result.stdout.strip() + if actual != expected: + print( + f"::error::checkout SHA mismatch: expected {expected}, got {actual}", + file=sys.stderr, + ) + return 1 + + print(f"PyTorch checkout matches dispatch SHA {expected}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From 627d4f588cb771feb3b8c51a7ab4960b1fcbb315 Mon Sep 17 00:00:00 2001 From: jewelkm89 Date: Mon, 6 Jul 2026 10:15:40 +0530 Subject: [PATCH 2/9] Focus CRCR health probes on real-time repository_dispatch only. Remove any scheduled-monitor direction from docs and strengthen L1/L2 workflows to record actual step outcomes, emit health-report.json artifacts per dispatch run, and assert closed-event cancel behavior. Test Plan: ``` python3 -m unittest discover -s tests -p 'test_*.py' -v ``` Authored with an AI assistant. Co-authored-by: Cursor --- .github/workflows/crcr-dispatch-receiver.yml | 75 +++++++++++++++++--- .github/workflows/oot-l2-ci.yml | 67 +++++++++++++++-- README.md | 19 +++-- tests/test_validators.py | 51 +++++++++++++ tests/write_health_report.py | 62 ++++++++++++++++ 5 files changed, 253 insertions(+), 21 deletions(-) create mode 100644 tests/write_health_report.py diff --git a/.github/workflows/crcr-dispatch-receiver.yml b/.github/workflows/crcr-dispatch-receiver.yml index 5f9ef81..e3d7bcd 100644 --- a/.github/workflows/crcr-dispatch-receiver.yml +++ b/.github/workflows/crcr-dispatch-receiver.yml @@ -7,7 +7,7 @@ on: - push run-name: >- - Dispatch - + L1 Health - ${{ github.event.client_payload.event_type == 'pull_request' && format( @@ -32,8 +32,35 @@ jobs: if: ${{ github.event.client_payload.payload.action == 'closed' }} runs-on: ubuntu-latest steps: + - uses: actions/checkout@v4 + - name: Assert closed action skips build job - run: echo "PR closed — cancel-workflow job ran as expected" + id: closed + run: | + ACTION="${{ github.event.client_payload.payload.action }}" + if [ "${ACTION}" != "closed" ]; then + echo "::error::expected action=closed, got ${ACTION}" + exit 1 + fi + echo "PR closed — cancel-workflow job ran as expected" + + - name: Write closed-event health report + if: always() + run: | + python3 tests/write_health_report.py \ + --probe l1-closed \ + --output health-report.json \ + --delivery-id "${{ github.event.client_payload.delivery_id }}" \ + --run-id "${{ github.run_id }}" \ + --event-type "${{ github.event.client_payload.event_type }}" \ + --pr-number "${{ github.event.client_payload.payload.pull_request.number || '' }}" \ + --check "closed_cancel=${{ steps.closed.outcome }}" + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: crcr-health-l1-closed-${{ github.run_id }} + path: health-report.json l1-critical: if: ${{ github.event.client_payload.payload.action != 'closed' }} @@ -43,12 +70,13 @@ jobs: uses: actions/checkout@v4 - name: L1 — validate dispatch payload structure + id: validate_payload env: CLIENT_PAYLOAD: ${{ toJson(github.event.client_payload) }} - run: | - echo "${CLIENT_PAYLOAD}" | python3 tests/validate_dispatch_payload.py + run: echo "${CLIENT_PAYLOAD}" | python3 tests/validate_dispatch_payload.py - name: L1 — checkout PyTorch at dispatched SHA + id: checkout_pytorch uses: actions/checkout@v4 with: repository: pytorch/pytorch @@ -59,6 +87,7 @@ jobs: path: pytorch - name: L1 — verify checkout matches dispatch SHA + id: verify_checkout env: CLIENT_PAYLOAD: ${{ toJson(github.event.client_payload) }} run: | @@ -68,26 +97,50 @@ jobs: --pytorch-dir pytorch - name: L1 — assert delivery_id is stable for L2 linkage + id: delivery run: | DELIVERY_ID="${{ github.event.client_payload.delivery_id }}" if [ -z "${DELIVERY_ID}" ]; then echo "::error::delivery_id is empty" exit 1 fi - echo "delivery_id=${DELIVERY_ID}" >> "$GITHUB_OUTPUT" - id: delivery + echo "delivery_id=${DELIVERY_ID}" + + - name: Write L1 health report + if: always() + run: | + python3 tests/write_health_report.py \ + --probe l1-critical \ + --output health-report.json \ + --delivery-id "${{ github.event.client_payload.delivery_id }}" \ + --run-id "${{ github.run_id }}" \ + --event-type "${{ github.event.client_payload.event_type }}" \ + --pr-number "${{ github.event.client_payload.payload.pull_request.number || '' }}" \ + --check "validate_payload=${{ steps.validate_payload.outcome }}" \ + --check "checkout_pytorch=${{ steps.checkout_pytorch.outcome }}" \ + --check "verify_checkout=${{ steps.verify_checkout.outcome }}" \ + --check "delivery_id=${{ steps.delivery.outcome }}" + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: crcr-health-l1-${{ github.run_id }} + path: health-report.json - name: Write integration summary if: always() run: | EVENT_TYPE="${{ github.event.client_payload.event_type }}" - echo "### L1 Critical Tests" >> $GITHUB_STEP_SUMMARY + echo "### L1 Real-Time Dispatch Tests" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "Triggered by live \`repository_dispatch\` from CRCR relay." >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo "| Check | Result |" >> $GITHUB_STEP_SUMMARY echo "|-------|--------|" >> $GITHUB_STEP_SUMMARY - echo "| Payload structure | passed |" >> $GITHUB_STEP_SUMMARY - echo "| PyTorch checkout | passed |" >> $GITHUB_STEP_SUMMARY - echo "| delivery_id | \`${{ steps.delivery.outputs.delivery_id || github.event.client_payload.delivery_id }}\` |" >> $GITHUB_STEP_SUMMARY + echo "| Payload structure | ${{ steps.validate_payload.outcome }} |" >> $GITHUB_STEP_SUMMARY + echo "| PyTorch checkout | ${{ steps.checkout_pytorch.outcome }} |" >> $GITHUB_STEP_SUMMARY + echo "| SHA verification | ${{ steps.verify_checkout.outcome }} |" >> $GITHUB_STEP_SUMMARY + echo "| delivery_id | ${{ steps.delivery.outcome }} |" >> $GITHUB_STEP_SUMMARY echo "| Event type | \`${EVENT_TYPE}\` |" >> $GITHUB_STEP_SUMMARY if [ "$EVENT_TYPE" = "pull_request" ]; then echo "| PR | #${{ github.event.client_payload.payload.pull_request.number }} (${{ github.event.client_payload.payload.action }}) |" >> $GITHUB_STEP_SUMMARY @@ -96,3 +149,5 @@ jobs: echo "| Ref | \`${{ github.event.client_payload.payload.ref }}\` |" >> $GITHUB_STEP_SUMMARY echo "| SHA | \`${{ github.event.client_payload.payload.after }}\` |" >> $GITHUB_STEP_SUMMARY fi + echo "" >> $GITHUB_STEP_SUMMARY + echo "delivery_id: \`${{ github.event.client_payload.delivery_id }}\`" >> $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/oot-l2-ci.yml b/.github/workflows/oot-l2-ci.yml index 606a6c4..2f842d3 100644 --- a/.github/workflows/oot-l2-ci.yml +++ b/.github/workflows/oot-l2-ci.yml @@ -6,7 +6,7 @@ on: - pull_request run-name: >- - OOT L2 Critical - + L2 Health - PR #${{ github.event.client_payload.payload.pull_request.number }} (${{ github.event.client_payload.payload.action }}) @@ -25,7 +25,29 @@ jobs: if: ${{ github.event.client_payload.payload.action == 'closed' }} runs-on: ubuntu-latest steps: - - run: echo "PR closed — cancel-workflow job ran as expected" + - uses: actions/checkout@v4 + + - name: Assert closed action skips L2 build job + id: closed + run: echo "PR closed — cancel-workflow job ran as expected" + + - name: Write closed-event health report + if: always() + run: | + python3 tests/write_health_report.py \ + --probe l2-closed \ + --output health-report.json \ + --delivery-id "${{ github.event.client_payload.delivery_id }}" \ + --run-id "${{ github.run_id }}" \ + --event-type pull_request \ + --pr-number "${{ github.event.client_payload.payload.pull_request.number }}" \ + --check "closed_cancel=${{ steps.closed.outcome }}" + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: crcr-health-l2-closed-${{ github.run_id }} + path: health-report.json l2-critical: if: ${{ github.event.client_payload.payload.action != 'closed' }} @@ -35,6 +57,7 @@ jobs: uses: actions/checkout@v4 - name: L2 — validate dispatch payload before callbacks + id: validate_payload env: CLIENT_PAYLOAD: ${{ toJson(github.event.client_payload) }} run: echo "${CLIENT_PAYLOAD}" | python3 tests/validate_dispatch_payload.py @@ -46,6 +69,7 @@ jobs: status: in_progress - name: L2 — checkout PyTorch at dispatched SHA + id: checkout_pytorch uses: actions/checkout@v4 with: repository: pytorch/pytorch @@ -53,6 +77,7 @@ jobs: path: pytorch - name: L2 — verify checkout matches dispatch SHA + id: verify_checkout env: CLIENT_PAYLOAD: ${{ toJson(github.event.client_payload) }} run: | @@ -73,6 +98,7 @@ jobs: echo "skipped=0" >> "$GITHUB_OUTPUT" - name: L2 — report completed to relay + id: completed if: always() uses: pytorch/test-infra/.github/actions/cross-repo-ci-relay-callback@main with: @@ -86,18 +112,45 @@ jobs: } artifact-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + - name: Write L2 health report + if: always() + run: | + python3 tests/write_health_report.py \ + --probe l2-critical \ + --output health-report.json \ + --delivery-id "${{ github.event.client_payload.delivery_id }}" \ + --run-id "${{ github.run_id }}" \ + --event-type pull_request \ + --pr-number "${{ github.event.client_payload.payload.pull_request.number }}" \ + --check "validate_payload=${{ steps.validate_payload.outcome }}" \ + --check "in_progress_callback=${{ steps.in_progress.outcome }}" \ + --check "checkout_pytorch=${{ steps.checkout_pytorch.outcome }}" \ + --check "verify_checkout=${{ steps.verify_checkout.outcome }}" \ + --check "smoke_checks=${{ steps.smoke.outcome }}" \ + --check "completed_callback=${{ steps.completed.outcome }}" + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: crcr-health-l2-${{ github.run_id }} + path: health-report.json + - name: Write L2 integration summary if: always() run: | - echo "### L2 Critical Tests" >> $GITHUB_STEP_SUMMARY + echo "### L2 Real-Time Dispatch Tests" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "Triggered by live \`repository_dispatch\` from CRCR relay." >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo "| Integration point | Status |" >> $GITHUB_STEP_SUMMARY echo "|-------------------|--------|" >> $GITHUB_STEP_SUMMARY - echo "| Dispatch payload validation | passed |" >> $GITHUB_STEP_SUMMARY + echo "| Dispatch payload validation | ${{ steps.validate_payload.outcome }} |" >> $GITHUB_STEP_SUMMARY echo "| in_progress callback | ${{ steps.in_progress.outcome }} |" >> $GITHUB_STEP_SUMMARY - echo "| PyTorch checkout | passed |" >> $GITHUB_STEP_SUMMARY - echo "| completed callback | see job status |" >> $GITHUB_STEP_SUMMARY + echo "| PyTorch checkout | ${{ steps.checkout_pytorch.outcome }} |" >> $GITHUB_STEP_SUMMARY + echo "| SHA verification | ${{ steps.verify_checkout.outcome }} |" >> $GITHUB_STEP_SUMMARY + echo "| Smoke checks | ${{ steps.smoke.outcome }} |" >> $GITHUB_STEP_SUMMARY + echo "| completed callback | ${{ steps.completed.outcome }} |" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY - echo "HUD: [crcr-test dashboard](https://hud.pytorch.org/crcr/pytorch/crcr-test)" >> $GITHUB_STEP_SUMMARY + echo "Verify HUD row: [crcr-test dashboard](https://hud.pytorch.org/crcr/pytorch/crcr-test) (run_id \`${{ github.run_id }}\`)" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo "delivery_id: \`${{ github.event.client_payload.delivery_id }}\`" >> $GITHUB_STEP_SUMMARY diff --git a/README.md b/README.md index 59614f2..51a64d0 100644 --- a/README.md +++ b/README.md @@ -147,13 +147,24 @@ Once Steps 1–3 are complete, your workflow will start receiving dispatches. Yo ## Critical Tests -This repository runs automated critical tests on every CRCR dispatch. They validate the L1 and L2 integration points end-to-end. +This repository is a **downstream CRCR health probe**. It contains no CRCR implementation code — only workflows and validators that run when the relay sends a live `repository_dispatch` event. + +**There is no scheduled cron.** All CRCR health checks run in real time, triggered by upstream PyTorch activity (PR open/sync/close or push/tag events). Each run writes a structured `health-report.json` artifact you can use for metrics. | Workflow | Level | Trigger | What it verifies | |----------|-------|---------|------------------| -| [`crcr-dispatch-receiver.yml`](.github/workflows/crcr-dispatch-receiver.yml) | L1 | `pull_request`, `push` | Dispatch payload shape, `delivery_id`, PyTorch checkout at dispatched SHA | -| [`oot-l2-ci.yml`](.github/workflows/oot-l2-ci.yml) | L2 | `pull_request` | L1 checks + `in_progress`/`completed` relay callbacks, test-results payload, HUD link | -| [`crcr-unit-tests.yml`](.github/workflows/crcr-unit-tests.yml) | Local | push/PR to this repo | Payload validator unit tests against JSON fixtures | +| [`crcr-dispatch-receiver.yml`](.github/workflows/crcr-dispatch-receiver.yml) | L1 | live `repository_dispatch`: `pull_request`, `push` | Dispatch payload shape, `delivery_id`, PyTorch checkout at dispatched SHA | +| [`oot-l2-ci.yml`](.github/workflows/oot-l2-ci.yml) | L2 | live `repository_dispatch`: `pull_request` | L1 checks + `in_progress`/`completed` relay callbacks, test-results payload, HUD link | +| [`crcr-unit-tests.yml`](.github/workflows/crcr-unit-tests.yml) | Offline | push/PR to this repo only | Validator unit tests against JSON fixtures (guards test code, not live CRCR) | + +### When tests run + +| Event | What runs | +|-------|-----------| +| PyTorch PR opened/reopened/synchronized | L1 + L2 health workflows | +| PyTorch PR closed | L1/L2 cancel jobs only (build jobs skipped) | +| PyTorch push / ciflow tag | L1 health workflow only | +| Merge to crcr-test main | `crcr-unit-tests` only (offline contract tests) | ### L1 integration points diff --git a/tests/test_validators.py b/tests/test_validators.py index e030cfc..2f02378 100644 --- a/tests/test_validators.py +++ b/tests/test_validators.py @@ -136,5 +136,56 @@ def test_checkout_matches_dispatch_sha(self) -> None: self.assertIn("checkout SHA mismatch", result.stderr) +class TestHealthReport(unittest.TestCase): + def test_healthy_report(self) -> None: + import subprocess + + with tempfile.TemporaryDirectory() as tmp: + out = Path(tmp) / "report.json" + result = subprocess.run( + [ + "python3", + str(Path(__file__).parent / "write_health_report.py"), + "--probe", + "l1-critical", + "--output", + str(out), + "--check", + "validate_payload=success", + "--check", + "verify_checkout=success", + ], + capture_output=True, + text=True, + ) + self.assertEqual(result.returncode, 0) + report = json.loads(out.read_text()) + self.assertTrue(report["healthy"]) + self.assertEqual(report["trigger"], "repository_dispatch") + + def test_unhealthy_report(self) -> None: + import subprocess + + with tempfile.TemporaryDirectory() as tmp: + out = Path(tmp) / "report.json" + result = subprocess.run( + [ + "python3", + str(Path(__file__).parent / "write_health_report.py"), + "--probe", + "l2-critical", + "--output", + str(out), + "--check", + "in_progress_callback=failure", + ], + capture_output=True, + text=True, + ) + self.assertNotEqual(result.returncode, 0) + report = json.loads(out.read_text()) + self.assertFalse(report["healthy"]) + + if __name__ == "__main__": unittest.main() diff --git a/tests/write_health_report.py b/tests/write_health_report.py new file mode 100644 index 0000000..57c75d0 --- /dev/null +++ b/tests/write_health_report.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +"""Write a structured CRCR health probe result for a repository_dispatch run.""" + +from __future__ import annotations + +import argparse +import json +import sys +from datetime import datetime, timezone +from pathlib import Path + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--probe", required=True, help="Probe name, e.g. l1-critical") + parser.add_argument("--output", required=True, help="Path to write JSON report") + parser.add_argument("--delivery-id", default="") + parser.add_argument("--run-id", default="") + parser.add_argument("--event-type", default="") + parser.add_argument("--pr-number", default="") + parser.add_argument( + "--check", + action="append", + default=[], + help="Check outcome as name=outcome (repeatable)", + ) + args = parser.parse_args() + + checks: dict[str, str] = {} + for item in args.check: + if "=" not in item: + print(f"::error::invalid --check {item!r}, expected name=outcome", file=sys.stderr) + return 2 + name, outcome = item.split("=", 1) + checks[name] = outcome + + report = { + "probe": args.probe, + "trigger": "repository_dispatch", + "timestamp": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"), + "delivery_id": args.delivery_id, + "run_id": args.run_id, + "event_type": args.event_type, + "pr_number": int(args.pr_number) if args.pr_number.isdigit() else None, + "checks": checks, + "healthy": bool(checks) and all(outcome == "success" for outcome in checks.values()), + "hud_url": "https://hud.pytorch.org/crcr/pytorch/crcr-test", + } + + output = Path(args.output) + output.write_text(json.dumps(report, indent=2) + "\n") + print(json.dumps(report)) + + if not report["healthy"]: + failed = [name for name, outcome in checks.items() if outcome != "success"] + print(f"::error::CRCR health probe unhealthy: {', '.join(failed)}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From aae7b37730bc6933c61b4107a76878712d2c45d2 Mon Sep 17 00:00:00 2001 From: jewelkm89 Date: Mon, 6 Jul 2026 11:06:19 +0530 Subject: [PATCH 3/9] Add local test and manual dispatch helper scripts. Test Plan: ``` ./scripts/run-local-tests.sh ``` Authored with an AI assistant. Co-authored-by: Cursor --- scripts/run-local-tests.sh | 17 +++++++ scripts/trigger-test-dispatch.sh | 77 ++++++++++++++++++++++++++++++++ 2 files changed, 94 insertions(+) create mode 100755 scripts/run-local-tests.sh create mode 100755 scripts/trigger-test-dispatch.sh diff --git a/scripts/run-local-tests.sh b/scripts/run-local-tests.sh new file mode 100755 index 0000000..2f51ea8 --- /dev/null +++ b/scripts/run-local-tests.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# Run offline CRCR validator tests (no relay required). +set -euo pipefail +cd "$(dirname "$0")/.." + +echo "=== Unit tests ===" +python3 -m unittest discover -s tests -p 'test_*.py' -v + +echo "" +echo "=== Fixture validation ===" +for f in tests/fixtures/*.json; do + echo "Validating $f" + python3 tests/validate_dispatch_payload.py < "$f" +done + +echo "" +echo "All local tests passed." diff --git a/scripts/trigger-test-dispatch.sh b/scripts/trigger-test-dispatch.sh new file mode 100755 index 0000000..c6c1e2e --- /dev/null +++ b/scripts/trigger-test-dispatch.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +# Send a test repository_dispatch to your fork to exercise L1/L2 workflows. +# Requires: gh CLI, workflows merged to the fork default branch (main). +# +# Usage: +# ./scripts/trigger-test-dispatch.sh # L1+L2 (pull_request/opened) +# ./scripts/trigger-test-dispatch.sh push # L1 only (push event) +set -euo pipefail +cd "$(dirname "$0")/.." + +REPO="${CRCR_TEST_REPO:-jewelkm89/crcr-test}" +MODE="${1:-pr}" + +echo "Fetching latest pytorch/pytorch main SHA..." +PYTORCH_SHA="$(git ls-remote https://github.com/pytorch/pytorch.git HEAD | awk '{print $1}')" +if [[ -z "${PYTORCH_SHA}" ]]; then + echo "error: could not resolve pytorch main SHA" >&2 + exit 1 +fi +echo "Using SHA: ${PYTORCH_SHA}" + +DELIVERY_ID="local-test-$(date -u +%Y%m%dT%H%M%SZ)-$$" + +if [[ "${MODE}" == "push" ]]; then + EVENT_TYPE="push" + PAYLOAD=$(jq -n \ + --arg sha "${PYTORCH_SHA}" \ + --arg did "${DELIVERY_ID}" \ + '{ + event_type: "push", + client_payload: { + event_type: "push", + delivery_id: $did, + payload: { + ref: "refs/heads/main", + after: $sha, + deleted: false, + base_ref: "refs/heads/main", + repository: { full_name: "pytorch/pytorch" } + } + } + }') +else + EVENT_TYPE="pull_request" + PAYLOAD=$(jq -n \ + --arg sha "${PYTORCH_SHA}" \ + --arg did "${DELIVERY_ID}" \ + '{ + event_type: "pull_request", + client_payload: { + event_type: "pull_request", + delivery_id: $did, + payload: { + action: "opened", + repository: { full_name: "pytorch/pytorch" }, + pull_request: { + number: 999999, + title: "CRCR local dispatch test", + head: { + sha: $sha, + repo: { full_name: "pytorch/pytorch" } + } + } + } + } + }') +fi + +echo "Dispatching ${EVENT_TYPE} to ${REPO} (delivery_id=${DELIVERY_ID})..." +echo "${PAYLOAD}" | gh api --method POST "repos/${REPO}/dispatches" --input - + +echo "" +echo "Done. Check workflow runs at:" +echo " https://github.com/${REPO}/actions" +echo "" +echo "Note: L2 callbacks require pytorch/crcr-test on the CRCR allowlist." +echo "On a personal fork, in_progress/completed may fail auth — L1 steps should still pass." From 99d3b7b2dbaa57a48b2a61cfb17c14384f83beea Mon Sep 17 00:00:00 2001 From: jewelkm89 Date: Mon, 6 Jul 2026 13:50:47 +0530 Subject: [PATCH 4/9] Send health probe metrics to HUD via completed callback. Map each integration check to workflow.test_results (passed/failed/total) and post through the existing cross-repo-ci-relay-callback action. L1 now reports in_progress/completed callbacks as well as L2. Test Plan: ``` python3 -m unittest discover -s tests -p 'test_*.py' -v ``` Authored with an AI assistant. Co-authored-by: Cursor --- .github/workflows/crcr-dispatch-receiver.yml | 51 ++++++++- .github/workflows/oot-l2-ci.yml | 45 +++++--- README.md | 17 ++- tests/test_validators.py | 52 ++++++++- tests/write_health_report.py | 109 +++++++++++++++---- 5 files changed, 231 insertions(+), 43 deletions(-) diff --git a/.github/workflows/crcr-dispatch-receiver.yml b/.github/workflows/crcr-dispatch-receiver.yml index e3d7bcd..11de8c8 100644 --- a/.github/workflows/crcr-dispatch-receiver.yml +++ b/.github/workflows/crcr-dispatch-receiver.yml @@ -26,6 +26,7 @@ concurrency: permissions: actions: write + id-token: write jobs: cancel-workflow: @@ -54,7 +55,8 @@ jobs: --run-id "${{ github.run_id }}" \ --event-type "${{ github.event.client_payload.event_type }}" \ --pr-number "${{ github.event.client_payload.payload.pull_request.number || '' }}" \ - --check "closed_cancel=${{ steps.closed.outcome }}" + --check "closed_cancel=${{ steps.closed.outcome }}" \ + --strict - uses: actions/upload-artifact@v4 if: always() @@ -75,6 +77,12 @@ jobs: CLIENT_PAYLOAD: ${{ toJson(github.event.client_payload) }} run: echo "${CLIENT_PAYLOAD}" | python3 tests/validate_dispatch_payload.py + - name: L1 — report in_progress to relay + id: in_progress + uses: pytorch/test-infra/.github/actions/cross-repo-ci-relay-callback@main + with: + status: in_progress + - name: L1 — checkout PyTorch at dispatched SHA id: checkout_pytorch uses: actions/checkout@v4 @@ -106,7 +114,8 @@ jobs: fi echo "delivery_id=${DELIVERY_ID}" - - name: Write L1 health report + - name: L1 — build health metrics for HUD + id: health_metrics if: always() run: | python3 tests/write_health_report.py \ @@ -117,9 +126,39 @@ jobs: --event-type "${{ github.event.client_payload.event_type }}" \ --pr-number "${{ github.event.client_payload.payload.pull_request.number || '' }}" \ --check "validate_payload=${{ steps.validate_payload.outcome }}" \ + --check "in_progress_callback=${{ steps.in_progress.outcome }}" \ --check "checkout_pytorch=${{ steps.checkout_pytorch.outcome }}" \ --check "verify_checkout=${{ steps.verify_checkout.outcome }}" \ - --check "delivery_id=${{ steps.delivery.outcome }}" + --check "delivery_id=${{ steps.delivery.outcome }}" \ + --github-output "$GITHUB_OUTPUT" + + - name: L1 — report completed with health metrics to relay/HUD + id: completed + if: always() + uses: pytorch/test-infra/.github/actions/cross-repo-ci-relay-callback@main + with: + status: completed + conclusion: ${{ steps.health_metrics.outputs.conclusion || (job.status == 'success' && 'success' || 'failure') }} + test-results: ${{ steps.health_metrics.outputs.test_results }} + artifact-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + + - name: L1 — record callback outcome in health report + if: always() + run: | + python3 tests/write_health_report.py \ + --probe l1-critical \ + --output health-report.json \ + --delivery-id "${{ github.event.client_payload.delivery_id }}" \ + --run-id "${{ github.run_id }}" \ + --event-type "${{ github.event.client_payload.event_type }}" \ + --pr-number "${{ github.event.client_payload.payload.pull_request.number || '' }}" \ + --check "validate_payload=${{ steps.validate_payload.outcome }}" \ + --check "in_progress_callback=${{ steps.in_progress.outcome }}" \ + --check "checkout_pytorch=${{ steps.checkout_pytorch.outcome }}" \ + --check "verify_checkout=${{ steps.verify_checkout.outcome }}" \ + --check "delivery_id=${{ steps.delivery.outcome }}" \ + --check "completed_callback=${{ steps.completed.outcome }}" \ + --strict - uses: actions/upload-artifact@v4 if: always() @@ -133,14 +172,16 @@ jobs: EVENT_TYPE="${{ github.event.client_payload.event_type }}" echo "### L1 Real-Time Dispatch Tests" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY - echo "Triggered by live \`repository_dispatch\` from CRCR relay." >> $GITHUB_STEP_SUMMARY + echo "Health metrics sent to HUD via \`completed\` callback \`test_results\`." >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo "| Check | Result |" >> $GITHUB_STEP_SUMMARY echo "|-------|--------|" >> $GITHUB_STEP_SUMMARY echo "| Payload structure | ${{ steps.validate_payload.outcome }} |" >> $GITHUB_STEP_SUMMARY + echo "| in_progress callback | ${{ steps.in_progress.outcome }} |" >> $GITHUB_STEP_SUMMARY echo "| PyTorch checkout | ${{ steps.checkout_pytorch.outcome }} |" >> $GITHUB_STEP_SUMMARY echo "| SHA verification | ${{ steps.verify_checkout.outcome }} |" >> $GITHUB_STEP_SUMMARY echo "| delivery_id | ${{ steps.delivery.outcome }} |" >> $GITHUB_STEP_SUMMARY + echo "| completed callback | ${{ steps.completed.outcome }} |" >> $GITHUB_STEP_SUMMARY echo "| Event type | \`${EVENT_TYPE}\` |" >> $GITHUB_STEP_SUMMARY if [ "$EVENT_TYPE" = "pull_request" ]; then echo "| PR | #${{ github.event.client_payload.payload.pull_request.number }} (${{ github.event.client_payload.payload.action }}) |" >> $GITHUB_STEP_SUMMARY @@ -150,4 +191,6 @@ jobs: echo "| SHA | \`${{ github.event.client_payload.payload.after }}\` |" >> $GITHUB_STEP_SUMMARY fi echo "" >> $GITHUB_STEP_SUMMARY + echo "HUD: [crcr-test dashboard](https://hud.pytorch.org/crcr/pytorch/crcr-test) (run_id \`${{ github.run_id }}\`)" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY echo "delivery_id: \`${{ github.event.client_payload.delivery_id }}\`" >> $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/oot-l2-ci.yml b/.github/workflows/oot-l2-ci.yml index 2f842d3..786dd2c 100644 --- a/.github/workflows/oot-l2-ci.yml +++ b/.github/workflows/oot-l2-ci.yml @@ -41,7 +41,8 @@ jobs: --run-id "${{ github.run_id }}" \ --event-type pull_request \ --pr-number "${{ github.event.client_payload.payload.pull_request.number }}" \ - --check "closed_cancel=${{ steps.closed.outcome }}" + --check "closed_cancel=${{ steps.closed.outcome }}" \ + --strict - uses: actions/upload-artifact@v4 if: always() @@ -93,26 +94,37 @@ jobs: test -f pytorch/README.md test -d pytorch/.github python3 -c "import json; print('smoke ok')" - echo "passed=3" >> "$GITHUB_OUTPUT" - echo "failed=0" >> "$GITHUB_OUTPUT" - echo "skipped=0" >> "$GITHUB_OUTPUT" - - name: L2 — report completed to relay + - name: L2 — build health metrics for HUD + id: health_metrics + if: always() + run: | + python3 tests/write_health_report.py \ + --probe l2-critical \ + --output health-report.json \ + --delivery-id "${{ github.event.client_payload.delivery_id }}" \ + --run-id "${{ github.run_id }}" \ + --event-type pull_request \ + --pr-number "${{ github.event.client_payload.payload.pull_request.number }}" \ + --check "validate_payload=${{ steps.validate_payload.outcome }}" \ + --check "in_progress_callback=${{ steps.in_progress.outcome }}" \ + --check "checkout_pytorch=${{ steps.checkout_pytorch.outcome }}" \ + --check "verify_checkout=${{ steps.verify_checkout.outcome }}" \ + --check "smoke_checks=${{ steps.smoke.outcome }}" \ + --github-output "$GITHUB_OUTPUT" + + - name: L2 — report completed with health metrics to relay/HUD id: completed if: always() uses: pytorch/test-infra/.github/actions/cross-repo-ci-relay-callback@main with: status: completed - conclusion: ${{ job.status == 'success' && 'success' || 'failure' }} - test-results: >- - { - "passed": ${{ steps.smoke.outputs.passed || 0 }}, - "failed": ${{ steps.smoke.outputs.failed || 0 }}, - "skipped": ${{ steps.smoke.outputs.skipped || 0 }} - } + conclusion: ${{ steps.health_metrics.outputs.conclusion || (job.status == 'success' && 'success' || 'failure') }} + test-results: ${{ steps.health_metrics.outputs.test_results }} artifact-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - - name: Write L2 health report + - name: L2 — record callback outcome in health report + id: health_final if: always() run: | python3 tests/write_health_report.py \ @@ -127,7 +139,8 @@ jobs: --check "checkout_pytorch=${{ steps.checkout_pytorch.outcome }}" \ --check "verify_checkout=${{ steps.verify_checkout.outcome }}" \ --check "smoke_checks=${{ steps.smoke.outcome }}" \ - --check "completed_callback=${{ steps.completed.outcome }}" + --check "completed_callback=${{ steps.completed.outcome }}" \ + --strict - uses: actions/upload-artifact@v4 if: always() @@ -140,7 +153,7 @@ jobs: run: | echo "### L2 Real-Time Dispatch Tests" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY - echo "Triggered by live \`repository_dispatch\` from CRCR relay." >> $GITHUB_STEP_SUMMARY + echo "Health metrics sent to HUD via \`completed\` callback \`test_results\`." >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo "| Integration point | Status |" >> $GITHUB_STEP_SUMMARY echo "|-------------------|--------|" >> $GITHUB_STEP_SUMMARY @@ -151,6 +164,6 @@ jobs: echo "| Smoke checks | ${{ steps.smoke.outcome }} |" >> $GITHUB_STEP_SUMMARY echo "| completed callback | ${{ steps.completed.outcome }} |" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY - echo "Verify HUD row: [crcr-test dashboard](https://hud.pytorch.org/crcr/pytorch/crcr-test) (run_id \`${{ github.run_id }}\`)" >> $GITHUB_STEP_SUMMARY + echo "HUD: [crcr-test dashboard](https://hud.pytorch.org/crcr/pytorch/crcr-test) (run_id \`${{ github.run_id }}\`)" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo "delivery_id: \`${{ github.event.client_payload.delivery_id }}\`" >> $GITHUB_STEP_SUMMARY diff --git a/README.md b/README.md index 51a64d0..e2cb55a 100644 --- a/README.md +++ b/README.md @@ -181,10 +181,25 @@ This repository is a **downstream CRCR health probe**. It contains no CRCR imple - OIDC token minting (`id-token: write`) - `in_progress` callback accepted by relay (state machine: `DISPATCHED → IN_PROGRESS`) - Deterministic smoke checks (no random pass/fail) -- `completed` callback with `conclusion: success` and `test_results` summary +- `completed` callback with `conclusion` and `test_results` derived from health checks +- Each health check maps to one HUD test count (`passed` / `failed` / `total`) - `artifact_url` points at the GitHub Actions run page - Results visible on [hud.pytorch.org/crcr/pytorch/crcr-test](https://hud.pytorch.org/crcr/pytorch/crcr-test) +### HUD health metrics + +L1 and L2 workflows send health probe results to the callback lambda on the `completed` +callback. The relay forwards them to HUD as `workflow.test_results`: + +| Health check step outcome | HUD field | +|---------------------------|-----------| +| `success` | counts toward `passed_tests` | +| `failure` / `cancelled` | counts toward `failed_tests` | +| `skipped` | counts toward `skipped_tests` | + +`total_tests` is the number of probe checks. `conclusion` is `success` only when every +check succeeded. Per-check names live in the uploaded `health-report.json` artifact. + ### Running validators locally ```bash diff --git a/tests/test_validators.py b/tests/test_validators.py index 2f02378..71b22f1 100644 --- a/tests/test_validators.py +++ b/tests/test_validators.py @@ -10,6 +10,7 @@ from validate_callback_payload import validate_callback_body from validate_dispatch_payload import ValidationError, validate_dispatch_payload +from write_health_report import build_report, checks_to_test_results FIXTURES = Path(__file__).parent / "fixtures" @@ -137,6 +138,30 @@ def test_checkout_matches_dispatch_sha(self) -> None: class TestHealthReport(unittest.TestCase): + def test_checks_to_test_results(self) -> None: + results = checks_to_test_results( + { + "a": "success", + "b": "success", + "c": "failure", + "d": "skipped", + } + ) + self.assertEqual(results, {"passed": 2, "failed": 1, "skipped": 1, "total": 4}) + + def test_build_report_includes_hud_fields(self) -> None: + report = build_report( + probe="l2-critical", + checks={"validate_payload": "success", "in_progress_callback": "success"}, + delivery_id="d-1", + run_id="99", + event_type="pull_request", + pr_number="123", + ) + self.assertTrue(report["healthy"]) + self.assertEqual(report["conclusion"], "success") + self.assertEqual(report["test_results"], {"passed": 2, "failed": 0, "skipped": 0, "total": 2}) + def test_healthy_report(self) -> None: import subprocess @@ -163,7 +188,31 @@ def test_healthy_report(self) -> None: self.assertTrue(report["healthy"]) self.assertEqual(report["trigger"], "repository_dispatch") - def test_unhealthy_report(self) -> None: + def test_unhealthy_report_without_strict(self) -> None: + import subprocess + + with tempfile.TemporaryDirectory() as tmp: + out = Path(tmp) / "report.json" + result = subprocess.run( + [ + "python3", + str(Path(__file__).parent / "write_health_report.py"), + "--probe", + "l2-critical", + "--output", + str(out), + "--check", + "in_progress_callback=failure", + ], + capture_output=True, + text=True, + ) + self.assertEqual(result.returncode, 0) + report = json.loads(out.read_text()) + self.assertFalse(report["healthy"]) + self.assertEqual(report["test_results"]["failed"], 1) + + def test_unhealthy_report_strict(self) -> None: import subprocess with tempfile.TemporaryDirectory() as tmp: @@ -178,6 +227,7 @@ def test_unhealthy_report(self) -> None: str(out), "--check", "in_progress_callback=failure", + "--strict", ], capture_output=True, text=True, diff --git a/tests/write_health_report.py b/tests/write_health_report.py index 57c75d0..8f375a5 100644 --- a/tests/write_health_report.py +++ b/tests/write_health_report.py @@ -1,15 +1,74 @@ #!/usr/bin/env python3 -"""Write a structured CRCR health probe result for a repository_dispatch run.""" +"""Write CRCR health probe results and map them to HUD test_results.""" from __future__ import annotations import argparse import json +import os import sys from datetime import datetime, timezone from pathlib import Path +def parse_checks(items: list[str]) -> dict[str, str]: + checks: dict[str, str] = {} + for item in items: + if "=" not in item: + raise ValueError(f"invalid --check {item!r}, expected name=outcome") + name, outcome = item.split("=", 1) + checks[name] = outcome + return checks + + +def checks_to_test_results(checks: dict[str, str]) -> dict[str, int]: + passed = sum(1 for outcome in checks.values() if outcome == "success") + skipped = sum(1 for outcome in checks.values() if outcome == "skipped") + failed = len(checks) - passed - skipped + return { + "passed": passed, + "failed": failed, + "skipped": skipped, + "total": len(checks), + } + + +def build_report( + *, + probe: str, + checks: dict[str, str], + delivery_id: str = "", + run_id: str = "", + event_type: str = "", + pr_number: str = "", +) -> dict: + healthy = bool(checks) and all(outcome == "success" for outcome in checks.values()) + test_results = checks_to_test_results(checks) + return { + "probe": probe, + "trigger": "repository_dispatch", + "timestamp": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"), + "delivery_id": delivery_id, + "run_id": run_id, + "event_type": event_type, + "pr_number": int(pr_number) if pr_number.isdigit() else None, + "checks": checks, + "healthy": healthy, + "conclusion": "success" if healthy else "failure", + "test_results": test_results, + "hud_url": "https://hud.pytorch.org/crcr/pytorch/crcr-test", + } + + +def append_github_output(report: dict, github_output: str) -> None: + with open(github_output, "a", encoding="utf-8") as handle: + handle.write(f"healthy={str(report['healthy']).lower()}\n") + handle.write(f"conclusion={report['conclusion']}\n") + handle.write("test_results< int: parser = argparse.ArgumentParser() parser.add_argument("--probe", required=True, help="Probe name, e.g. l1-critical") @@ -24,34 +83,42 @@ def main() -> int: default=[], help="Check outcome as name=outcome (repeatable)", ) + parser.add_argument( + "--github-output", + default="", + help="If set, append healthy/conclusion/test_results to this GITHUB_OUTPUT path", + ) + parser.add_argument( + "--strict", + action="store_true", + default=False, + help="Exit 1 when the probe is unhealthy (default: exit 0 after writing report)", + ) args = parser.parse_args() - checks: dict[str, str] = {} - for item in args.check: - if "=" not in item: - print(f"::error::invalid --check {item!r}, expected name=outcome", file=sys.stderr) - return 2 - name, outcome = item.split("=", 1) - checks[name] = outcome + try: + checks = parse_checks(args.check) + except ValueError as exc: + print(f"::error::{exc}", file=sys.stderr) + return 2 - report = { - "probe": args.probe, - "trigger": "repository_dispatch", - "timestamp": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"), - "delivery_id": args.delivery_id, - "run_id": args.run_id, - "event_type": args.event_type, - "pr_number": int(args.pr_number) if args.pr_number.isdigit() else None, - "checks": checks, - "healthy": bool(checks) and all(outcome == "success" for outcome in checks.values()), - "hud_url": "https://hud.pytorch.org/crcr/pytorch/crcr-test", - } + report = build_report( + probe=args.probe, + checks=checks, + delivery_id=args.delivery_id, + run_id=args.run_id, + event_type=args.event_type, + pr_number=args.pr_number, + ) output = Path(args.output) output.write_text(json.dumps(report, indent=2) + "\n") print(json.dumps(report)) - if not report["healthy"]: + if args.github_output: + append_github_output(report, args.github_output) + + if args.strict and not report["healthy"]: failed = [name for name, outcome in checks.items() if outcome != "success"] print(f"::error::CRCR health probe unhealthy: {', '.join(failed)}", file=sys.stderr) return 1 From a3f71425758b2dc6a401479596f78bbdad50b86a Mon Sep 17 00:00:00 2001 From: jewelkm89 Date: Mon, 6 Jul 2026 15:50:28 +0530 Subject: [PATCH 5/9] Remove personal fork default from trigger-test-dispatch.sh Default to the current gh repo with fallback to pytorch/crcr-test instead of a hardcoded personal fork. CRCR_TEST_REPO still overrides explicitly. Test plan: ``` ./scripts/trigger-test-dispatch.sh --help # N/A; script has no --help git diff scripts/trigger-test-dispatch.sh ``` Authored with an AI assistant. Co-authored-by: Cursor --- scripts/trigger-test-dispatch.sh | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/scripts/trigger-test-dispatch.sh b/scripts/trigger-test-dispatch.sh index c6c1e2e..6075864 100755 --- a/scripts/trigger-test-dispatch.sh +++ b/scripts/trigger-test-dispatch.sh @@ -1,14 +1,16 @@ #!/usr/bin/env bash -# Send a test repository_dispatch to your fork to exercise L1/L2 workflows. -# Requires: gh CLI, workflows merged to the fork default branch (main). +# Send a test repository_dispatch to exercise L1/L2 workflows. +# Requires: gh CLI, workflows merged to the target repo default branch (main). # # Usage: # ./scripts/trigger-test-dispatch.sh # L1+L2 (pull_request/opened) # ./scripts/trigger-test-dispatch.sh push # L1 only (push event) +# +# Target repo: current gh repo, or pytorch/crcr-test. Override with CRCR_TEST_REPO. set -euo pipefail cd "$(dirname "$0")/.." -REPO="${CRCR_TEST_REPO:-jewelkm89/crcr-test}" +REPO="${CRCR_TEST_REPO:-$(gh repo view --json nameWithOwner -q .nameWithOwner 2>/dev/null || echo pytorch/crcr-test)}" MODE="${1:-pr}" echo "Fetching latest pytorch/pytorch main SHA..." @@ -73,5 +75,5 @@ echo "" echo "Done. Check workflow runs at:" echo " https://github.com/${REPO}/actions" echo "" -echo "Note: L2 callbacks require pytorch/crcr-test on the CRCR allowlist." -echo "On a personal fork, in_progress/completed may fail auth — L1 steps should still pass." +echo "Note: callbacks require the target repo on the CRCR allowlist and a relay dispatch record." +echo "Synthetic delivery_id dispatches exercise L1/L2 workflow steps; callbacks may fail locally." From 0605b2c4f81a4b4bfc9b6d7db01f66e32b022854 Mon Sep 17 00:00:00 2001 From: jewelkm89 Date: Mon, 6 Jul 2026 20:49:55 +0530 Subject: [PATCH 6/9] Filter HUD callbacks to opened/reopened to reduce relay 429s L1 runs a light probe (payload + checkout, no callbacks) on synchronize and push. L2 runs only on opened/reopened. Full L1+L2 HUD probes remain on low-volume PR events. Document tiered coverage in README. Test plan: ``` ./scripts/run-local-tests.sh ``` Authored with an AI assistant. Co-authored-by: Cursor --- .github/workflows/crcr-dispatch-receiver.yml | 54 +++++++++++++++++--- .github/workflows/oot-l2-ci.yml | 4 +- README.md | 31 +++++++++-- 3 files changed, 76 insertions(+), 13 deletions(-) diff --git a/.github/workflows/crcr-dispatch-receiver.yml b/.github/workflows/crcr-dispatch-receiver.yml index 11de8c8..b0ef094 100644 --- a/.github/workflows/crcr-dispatch-receiver.yml +++ b/.github/workflows/crcr-dispatch-receiver.yml @@ -71,6 +71,19 @@ jobs: - name: Checkout crcr-test uses: actions/checkout@v4 + - name: L1 — resolve probe mode + id: probe + run: | + ACTION="${{ github.event.client_payload.payload.action }}" + EVENT="${{ github.event.client_payload.event_type }}" + if [ "$EVENT" = "pull_request" ] && { [ "$ACTION" = "opened" ] || [ "$ACTION" = "reopened" ]; }; then + echo "hud_callbacks=true" >> "$GITHUB_OUTPUT" + echo "probe=l1-critical" >> "$GITHUB_OUTPUT" + else + echo "hud_callbacks=false" >> "$GITHUB_OUTPUT" + echo "probe=l1-light" >> "$GITHUB_OUTPUT" + fi + - name: L1 — validate dispatch payload structure id: validate_payload env: @@ -79,6 +92,7 @@ jobs: - name: L1 — report in_progress to relay id: in_progress + if: steps.probe.outputs.hud_callbacks == 'true' uses: pytorch/test-infra/.github/actions/cross-repo-ci-relay-callback@main with: status: in_progress @@ -116,7 +130,7 @@ jobs: - name: L1 — build health metrics for HUD id: health_metrics - if: always() + if: always() && steps.probe.outputs.hud_callbacks == 'true' run: | python3 tests/write_health_report.py \ --probe l1-critical \ @@ -134,7 +148,7 @@ jobs: - name: L1 — report completed with health metrics to relay/HUD id: completed - if: always() + if: always() && steps.probe.outputs.hud_callbacks == 'true' uses: pytorch/test-infra/.github/actions/cross-repo-ci-relay-callback@main with: status: completed @@ -142,8 +156,24 @@ jobs: test-results: ${{ steps.health_metrics.outputs.test_results }} artifact-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - - name: L1 — record callback outcome in health report - if: always() + - name: L1 — write light health report (no HUD callbacks) + if: always() && steps.probe.outputs.hud_callbacks != 'true' + run: | + python3 tests/write_health_report.py \ + --probe l1-light \ + --output health-report.json \ + --delivery-id "${{ github.event.client_payload.delivery_id }}" \ + --run-id "${{ github.run_id }}" \ + --event-type "${{ github.event.client_payload.event_type }}" \ + --pr-number "${{ github.event.client_payload.payload.pull_request.number || '' }}" \ + --check "validate_payload=${{ steps.validate_payload.outcome }}" \ + --check "checkout_pytorch=${{ steps.checkout_pytorch.outcome }}" \ + --check "verify_checkout=${{ steps.verify_checkout.outcome }}" \ + --check "delivery_id=${{ steps.delivery.outcome }}" \ + --strict + + - name: L1 — write full health report with callback outcome + if: always() && steps.probe.outputs.hud_callbacks == 'true' run: | python3 tests/write_health_report.py \ --probe l1-critical \ @@ -170,18 +200,28 @@ jobs: if: always() run: | EVENT_TYPE="${{ github.event.client_payload.event_type }}" + PROBE_MODE="${{ steps.probe.outputs.probe }}" echo "### L1 Real-Time Dispatch Tests" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY - echo "Health metrics sent to HUD via \`completed\` callback \`test_results\`." >> $GITHUB_STEP_SUMMARY + if [ "${PROBE_MODE}" = "l1-critical" ]; then + echo "Probe mode: **full** (HUD callbacks on \`opened\`/\`reopened\`)." >> $GITHUB_STEP_SUMMARY + echo "Health metrics sent to HUD via \`completed\` callback \`test_results\`." >> $GITHUB_STEP_SUMMARY + else + echo "Probe mode: **light** (no HUD callbacks on \`synchronize\`/\`push\`; reduces relay 429 risk)." >> $GITHUB_STEP_SUMMARY + fi echo "" >> $GITHUB_STEP_SUMMARY echo "| Check | Result |" >> $GITHUB_STEP_SUMMARY echo "|-------|--------|" >> $GITHUB_STEP_SUMMARY echo "| Payload structure | ${{ steps.validate_payload.outcome }} |" >> $GITHUB_STEP_SUMMARY - echo "| in_progress callback | ${{ steps.in_progress.outcome }} |" >> $GITHUB_STEP_SUMMARY + if [ "${PROBE_MODE}" = "l1-critical" ]; then + echo "| in_progress callback | ${{ steps.in_progress.outcome }} |" >> $GITHUB_STEP_SUMMARY + fi echo "| PyTorch checkout | ${{ steps.checkout_pytorch.outcome }} |" >> $GITHUB_STEP_SUMMARY echo "| SHA verification | ${{ steps.verify_checkout.outcome }} |" >> $GITHUB_STEP_SUMMARY echo "| delivery_id | ${{ steps.delivery.outcome }} |" >> $GITHUB_STEP_SUMMARY - echo "| completed callback | ${{ steps.completed.outcome }} |" >> $GITHUB_STEP_SUMMARY + if [ "${PROBE_MODE}" = "l1-critical" ]; then + echo "| completed callback | ${{ steps.completed.outcome }} |" >> $GITHUB_STEP_SUMMARY + fi echo "| Event type | \`${EVENT_TYPE}\` |" >> $GITHUB_STEP_SUMMARY if [ "$EVENT_TYPE" = "pull_request" ]; then echo "| PR | #${{ github.event.client_payload.payload.pull_request.number }} (${{ github.event.client_payload.payload.action }}) |" >> $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/oot-l2-ci.yml b/.github/workflows/oot-l2-ci.yml index 786dd2c..85d29b2 100644 --- a/.github/workflows/oot-l2-ci.yml +++ b/.github/workflows/oot-l2-ci.yml @@ -51,7 +51,9 @@ jobs: path: health-report.json l2-critical: - if: ${{ github.event.client_payload.payload.action != 'closed' }} + if: >- + github.event.client_payload.payload.action != 'closed' && + contains(fromJSON('["opened","reopened"]'), github.event.client_payload.payload.action) runs-on: ubuntu-latest steps: - name: Checkout crcr-test diff --git a/README.md b/README.md index e2cb55a..2a55977 100644 --- a/README.md +++ b/README.md @@ -153,17 +153,35 @@ This repository is a **downstream CRCR health probe**. It contains no CRCR imple | Workflow | Level | Trigger | What it verifies | |----------|-------|---------|------------------| -| [`crcr-dispatch-receiver.yml`](.github/workflows/crcr-dispatch-receiver.yml) | L1 | live `repository_dispatch`: `pull_request`, `push` | Dispatch payload shape, `delivery_id`, PyTorch checkout at dispatched SHA | -| [`oot-l2-ci.yml`](.github/workflows/oot-l2-ci.yml) | L2 | live `repository_dispatch`: `pull_request` | L1 checks + `in_progress`/`completed` relay callbacks, test-results payload, HUD link | +| [`crcr-dispatch-receiver.yml`](.github/workflows/crcr-dispatch-receiver.yml) | L1 | live `repository_dispatch`: `pull_request`, `push` | Dispatch payload, checkout SHA, `delivery_id`; HUD callbacks on `opened`/`reopened` only | +| [`oot-l2-ci.yml`](.github/workflows/oot-l2-ci.yml) | L2 | live `repository_dispatch`: `pull_request` (`opened`/`reopened` only) | L1 checks + smoke + `in_progress`/`completed` callbacks, HUD metrics | | [`crcr-unit-tests.yml`](.github/workflows/crcr-unit-tests.yml) | Offline | push/PR to this repo only | Validator unit tests against JSON fixtures (guards test code, not live CRCR) | +### Event filtering (relay rate limits) + +The relay callback Lambda enforces a per-repo sliding-window rate limit (see [crcr-test#8](https://github.com/pytorch/crcr-test/issues/8)). A full L1+L2 probe sends **4 callbacks** (`in_progress` + `completed` x2). PyTorch `synchronize` events are high volume and can exceed the limit during busy periods. + +Health probes use **tiered coverage** by event type: + +| Event | L1 workflow | L2 workflow | Relay callbacks | HUD rows | +|-------|-------------|-------------|-----------------|----------| +| `opened`, `reopened` | Full probe + HUD | Full probe + HUD | **4** | **2** | +| `synchronize` | Light probe (validate + checkout) | Skipped | **0** | **0** | +| `closed` | Cancel job only | Cancel job only | **0** | **0** | +| `push` / ciflow tag | Light probe (validate + checkout) | N/A | **0** | **0** | + +**Light probe** (`l1-light`): validates dispatch payload, checks out PyTorch at the dispatched SHA, verifies the checkout, asserts `delivery_id`, uploads `health-report.json` — no relay callbacks. + +**Full probe** (`l1-critical` / `l2-critical`): light checks plus `in_progress`/`completed` callbacks and HUD `test_results`. Callback mechanics are the same regardless of PR `action`; limiting callbacks to `opened`/`reopened` keeps coverage on low-volume events while `synchronize` still exercises dispatch delivery every upstream push. + ### When tests run | Event | What runs | |-------|-----------| -| PyTorch PR opened/reopened/synchronized | L1 + L2 health workflows | -| PyTorch PR closed | L1/L2 cancel jobs only (build jobs skipped) | -| PyTorch push / ciflow tag | L1 health workflow only | +| PyTorch PR `opened` / `reopened` | L1 full + L2 full (HUD callbacks) | +| PyTorch PR `synchronize` | L1 light only (no callbacks, no L2) | +| PyTorch PR `closed` | L1/L2 cancel jobs only (build jobs skipped) | +| PyTorch push / ciflow tag | L1 light only (no callbacks) | | Merge to crcr-test main | `crcr-unit-tests` only (offline contract tests) | ### L1 integration points @@ -175,9 +193,12 @@ This repository is a **downstream CRCR health probe**. It contains no CRCR imple - Push events: `ref` starts with `refs/`, valid `after` SHA - PyTorch checkout resolves to the dispatched SHA - `closed` action runs only the cancel job (no build) +- **`opened`/`reopened`**: full probe with `in_progress`/`completed` callbacks to HUD +- **`synchronize`/`push`**: light probe only (payload + checkout; no callbacks) ### L2 integration points +- Runs only on PR `opened` and `reopened` (skipped on `synchronize` to limit callback volume) - OIDC token minting (`id-token: write`) - `in_progress` callback accepted by relay (state machine: `DISPATCHED → IN_PROGRESS`) - Deterministic smoke checks (no random pass/fail) From c6cae9afd1f23568067c968df1e56ba9afe17f34 Mon Sep 17 00:00:00 2001 From: jewelkm89 Date: Thu, 9 Jul 2026 14:02:47 +0530 Subject: [PATCH 7/9] Address review feedback from @atalman on PR #9. Factor probe check lists into scripts/health_checks.sh so L1/L2 workflows share one definition per probe mode. HUD metrics still use the pre-callback check set; the artifact adds completed_callback afterward (documented in workflow comments). Add positive-path checkout SHA test, rename L2 workflow to CRCR, drop redundant closed guard, and fix a stray f-string. Test Plan: ``` python3 -m unittest discover -s tests -p 'test_*.py' -v ``` Authored with an AI assistant. Co-authored-by: Cursor --- .github/workflows/crcr-dispatch-receiver.yml | 46 ++++++++++------- .github/workflows/oot-l2-ci.yml | 42 +++++++++------- scripts/health_checks.sh | 46 +++++++++++++++++ tests/test_validators.py | 52 +++++++++++++++++--- tests/validate_callback_payload.py | 2 +- tests/write_health_report.py | 11 +++-- 6 files changed, 154 insertions(+), 45 deletions(-) create mode 100755 scripts/health_checks.sh diff --git a/.github/workflows/crcr-dispatch-receiver.yml b/.github/workflows/crcr-dispatch-receiver.yml index b0ef094..230a177 100644 --- a/.github/workflows/crcr-dispatch-receiver.yml +++ b/.github/workflows/crcr-dispatch-receiver.yml @@ -128,22 +128,29 @@ jobs: fi echo "delivery_id=${DELIVERY_ID}" + - name: L1 — export probe step outcomes + if: always() + run: | + { + echo "VALIDATE_PAYLOAD=${{ steps.validate_payload.outcome }}" + echo "IN_PROGRESS_CALLBACK=${{ steps.in_progress.outcome }}" + echo "CHECKOUT_PYTORCH=${{ steps.checkout_pytorch.outcome }}" + echo "VERIFY_CHECKOUT=${{ steps.verify_checkout.outcome }}" + echo "DELIVERY_ID=${{ steps.delivery.outcome }}" + } >> "$GITHUB_ENV" + - name: L1 — build health metrics for HUD id: health_metrics if: always() && steps.probe.outputs.hud_callbacks == 'true' run: | + mapfile -t CHECK_ARGS < <(bash scripts/health_checks.sh l1-full) python3 tests/write_health_report.py \ --probe l1-critical \ - --output health-report.json \ --delivery-id "${{ github.event.client_payload.delivery_id }}" \ --run-id "${{ github.run_id }}" \ --event-type "${{ github.event.client_payload.event_type }}" \ --pr-number "${{ github.event.client_payload.payload.pull_request.number || '' }}" \ - --check "validate_payload=${{ steps.validate_payload.outcome }}" \ - --check "in_progress_callback=${{ steps.in_progress.outcome }}" \ - --check "checkout_pytorch=${{ steps.checkout_pytorch.outcome }}" \ - --check "verify_checkout=${{ steps.verify_checkout.outcome }}" \ - --check "delivery_id=${{ steps.delivery.outcome }}" \ + "${CHECK_ARGS[@]}" \ --github-output "$GITHUB_OUTPUT" - name: L1 — report completed with health metrics to relay/HUD @@ -159,6 +166,13 @@ jobs: - name: L1 — write light health report (no HUD callbacks) if: always() && steps.probe.outputs.hud_callbacks != 'true' run: | + { + echo "VALIDATE_PAYLOAD=${{ steps.validate_payload.outcome }}" + echo "CHECKOUT_PYTORCH=${{ steps.checkout_pytorch.outcome }}" + echo "VERIFY_CHECKOUT=${{ steps.verify_checkout.outcome }}" + echo "DELIVERY_ID=${{ steps.delivery.outcome }}" + } >> "$GITHUB_ENV" + mapfile -t CHECK_ARGS < <(bash scripts/health_checks.sh l1-light) python3 tests/write_health_report.py \ --probe l1-light \ --output health-report.json \ @@ -166,15 +180,18 @@ jobs: --run-id "${{ github.run_id }}" \ --event-type "${{ github.event.client_payload.event_type }}" \ --pr-number "${{ github.event.client_payload.payload.pull_request.number || '' }}" \ - --check "validate_payload=${{ steps.validate_payload.outcome }}" \ - --check "checkout_pytorch=${{ steps.checkout_pytorch.outcome }}" \ - --check "verify_checkout=${{ steps.verify_checkout.outcome }}" \ - --check "delivery_id=${{ steps.delivery.outcome }}" \ + "${CHECK_ARGS[@]}" \ --strict - - name: L1 — write full health report with callback outcome + # Artifact uses one more check than HUD test_results: completed_callback + # runs after the HUD POST, so its outcome is only known afterward. + - name: L1 — write full health report artifact if: always() && steps.probe.outputs.hud_callbacks == 'true' + env: + INCLUDE_COMPLETED_CALLBACK: "1" + COMPLETED_CALLBACK: ${{ steps.completed.outcome }} run: | + mapfile -t CHECK_ARGS < <(bash scripts/health_checks.sh l1-full) python3 tests/write_health_report.py \ --probe l1-critical \ --output health-report.json \ @@ -182,12 +199,7 @@ jobs: --run-id "${{ github.run_id }}" \ --event-type "${{ github.event.client_payload.event_type }}" \ --pr-number "${{ github.event.client_payload.payload.pull_request.number || '' }}" \ - --check "validate_payload=${{ steps.validate_payload.outcome }}" \ - --check "in_progress_callback=${{ steps.in_progress.outcome }}" \ - --check "checkout_pytorch=${{ steps.checkout_pytorch.outcome }}" \ - --check "verify_checkout=${{ steps.verify_checkout.outcome }}" \ - --check "delivery_id=${{ steps.delivery.outcome }}" \ - --check "completed_callback=${{ steps.completed.outcome }}" \ + "${CHECK_ARGS[@]}" \ --strict - uses: actions/upload-artifact@v4 diff --git a/.github/workflows/oot-l2-ci.yml b/.github/workflows/oot-l2-ci.yml index 85d29b2..0e4efb9 100644 --- a/.github/workflows/oot-l2-ci.yml +++ b/.github/workflows/oot-l2-ci.yml @@ -1,4 +1,4 @@ -name: OOT L2 Critical +name: CRCR L2 Critical on: repository_dispatch: @@ -12,7 +12,7 @@ run-name: >- concurrency: group: >- - oot-l2-${{ github.event.client_payload.payload.repository.full_name }}-${{ + crcr-l2-${{ github.event.client_payload.payload.repository.full_name }}-${{ github.event.client_payload.payload.pull_request.number || github.run_id }} cancel-in-progress: true @@ -51,9 +51,7 @@ jobs: path: health-report.json l2-critical: - if: >- - github.event.client_payload.payload.action != 'closed' && - contains(fromJSON('["opened","reopened"]'), github.event.client_payload.payload.action) + if: contains(fromJSON('["opened","reopened"]'), github.event.client_payload.payload.action) runs-on: ubuntu-latest steps: - name: Checkout crcr-test @@ -97,22 +95,29 @@ jobs: test -d pytorch/.github python3 -c "import json; print('smoke ok')" + - name: L2 — export probe step outcomes + if: always() + run: | + { + echo "VALIDATE_PAYLOAD=${{ steps.validate_payload.outcome }}" + echo "IN_PROGRESS_CALLBACK=${{ steps.in_progress.outcome }}" + echo "CHECKOUT_PYTORCH=${{ steps.checkout_pytorch.outcome }}" + echo "VERIFY_CHECKOUT=${{ steps.verify_checkout.outcome }}" + echo "SMOKE_CHECKS=${{ steps.smoke.outcome }}" + } >> "$GITHUB_ENV" + - name: L2 — build health metrics for HUD id: health_metrics if: always() run: | + mapfile -t CHECK_ARGS < <(bash scripts/health_checks.sh l2-full) python3 tests/write_health_report.py \ --probe l2-critical \ - --output health-report.json \ --delivery-id "${{ github.event.client_payload.delivery_id }}" \ --run-id "${{ github.run_id }}" \ --event-type pull_request \ --pr-number "${{ github.event.client_payload.payload.pull_request.number }}" \ - --check "validate_payload=${{ steps.validate_payload.outcome }}" \ - --check "in_progress_callback=${{ steps.in_progress.outcome }}" \ - --check "checkout_pytorch=${{ steps.checkout_pytorch.outcome }}" \ - --check "verify_checkout=${{ steps.verify_checkout.outcome }}" \ - --check "smoke_checks=${{ steps.smoke.outcome }}" \ + "${CHECK_ARGS[@]}" \ --github-output "$GITHUB_OUTPUT" - name: L2 — report completed with health metrics to relay/HUD @@ -125,10 +130,16 @@ jobs: test-results: ${{ steps.health_metrics.outputs.test_results }} artifact-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - - name: L2 — record callback outcome in health report + # Artifact uses one more check than HUD test_results: completed_callback + # runs after the HUD POST, so its outcome is only known afterward. + - name: L2 — write health report artifact id: health_final if: always() + env: + INCLUDE_COMPLETED_CALLBACK: "1" + COMPLETED_CALLBACK: ${{ steps.completed.outcome }} run: | + mapfile -t CHECK_ARGS < <(bash scripts/health_checks.sh l2-full) python3 tests/write_health_report.py \ --probe l2-critical \ --output health-report.json \ @@ -136,12 +147,7 @@ jobs: --run-id "${{ github.run_id }}" \ --event-type pull_request \ --pr-number "${{ github.event.client_payload.payload.pull_request.number }}" \ - --check "validate_payload=${{ steps.validate_payload.outcome }}" \ - --check "in_progress_callback=${{ steps.in_progress.outcome }}" \ - --check "checkout_pytorch=${{ steps.checkout_pytorch.outcome }}" \ - --check "verify_checkout=${{ steps.verify_checkout.outcome }}" \ - --check "smoke_checks=${{ steps.smoke.outcome }}" \ - --check "completed_callback=${{ steps.completed.outcome }}" \ + "${CHECK_ARGS[@]}" \ --strict - uses: actions/upload-artifact@v4 diff --git a/scripts/health_checks.sh b/scripts/health_checks.sh new file mode 100755 index 0000000..a8a74b7 --- /dev/null +++ b/scripts/health_checks.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# Emit --check name=outcome pairs (one per line) for write_health_report.py. +# Usage: mapfile -t ARGS < <(scripts/health_checks.sh SPEC) +# python3 tests/write_health_report.py "${ARGS[@]}" ... + +set -euo pipefail + +spec="${1:?spec required (l1-light, l1-full, l2-full)}" + +emit_check() { + local name="$1" + local var="$2" + echo --check + echo "${name}=${!var:-skipped}" +} + +case "$spec" in + l1-light) + emit_check validate_payload VALIDATE_PAYLOAD + emit_check checkout_pytorch CHECKOUT_PYTORCH + emit_check verify_checkout VERIFY_CHECKOUT + emit_check delivery_id DELIVERY_ID + ;; + l1-full) + emit_check validate_payload VALIDATE_PAYLOAD + emit_check in_progress_callback IN_PROGRESS_CALLBACK + emit_check checkout_pytorch CHECKOUT_PYTORCH + emit_check verify_checkout VERIFY_CHECKOUT + emit_check delivery_id DELIVERY_ID + ;; + l2-full) + emit_check validate_payload VALIDATE_PAYLOAD + emit_check in_progress_callback IN_PROGRESS_CALLBACK + emit_check checkout_pytorch CHECKOUT_PYTORCH + emit_check verify_checkout VERIFY_CHECKOUT + emit_check smoke_checks SMOKE_CHECKS + ;; + *) + echo "unknown health check spec: $spec" >&2 + exit 2 + ;; +esac + +if [ "${INCLUDE_COMPLETED_CALLBACK:-}" = "1" ]; then + emit_check completed_callback COMPLETED_CALLBACK +fi diff --git a/tests/test_validators.py b/tests/test_validators.py index 71b22f1..a156d73 100644 --- a/tests/test_validators.py +++ b/tests/test_validators.py @@ -58,7 +58,7 @@ def _completed_body(self) -> dict: "schema_version": "1", "status": "completed", "conclusion": "success", - "name": "OOT L2 Critical", + "name": "CRCR L2 Critical", "url": "https://github.com/pytorch/crcr-test/actions/runs/1", "run_attempt": "1", "job_name": "l2-critical", @@ -77,7 +77,7 @@ def test_in_progress_shape(self) -> None: "schema_version": "1", "status": "in_progress", "conclusion": None, - "name": "OOT L2 Critical", + "name": "CRCR L2 Critical", "url": "https://github.com/pytorch/crcr-test/actions/runs/1", "run_attempt": "1", "job_name": "l2-critical", @@ -102,10 +102,51 @@ class TestPytorchCheckout(unittest.TestCase): def test_checkout_matches_dispatch_sha(self) -> None: import subprocess - from validate_pytorch_checkout import _head_sha + with tempfile.TemporaryDirectory() as tmp: + tmp_path = Path(tmp) + subprocess.run(["git", "init", tmp], check=True, capture_output=True) + subprocess.run( + ["git", "-C", tmp, "commit", "--allow-empty", "-m", "init"], + check=True, + capture_output=True, + env={ + **__import__("os").environ, + "GIT_AUTHOR_NAME": "test", + "GIT_AUTHOR_EMAIL": "test@test.com", + "GIT_COMMITTER_NAME": "test", + "GIT_COMMITTER_EMAIL": "test@test.com", + }, + ) + head = subprocess.run( + ["git", "-C", tmp, "rev-parse", "HEAD"], + check=True, + capture_output=True, + text=True, + ) + matching_sha = head.stdout.strip() + + payload = _load("pull_request_opened.json") + payload["payload"]["pull_request"]["head"]["sha"] = matching_sha + payload_path = tmp_path / "payload.json" + payload_path.write_text(json.dumps(payload)) + + result = subprocess.run( + [ + "python3", + str(Path(__file__).parent / "validate_pytorch_checkout.py"), + "--payload", + str(payload_path), + "--pytorch-dir", + tmp, + ], + capture_output=True, + text=True, + ) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_checkout_mismatch_dispatch_sha(self) -> None: + import subprocess - payload = _load("pull_request_opened.json") - expected = _head_sha(payload) with tempfile.TemporaryDirectory() as tmp: subprocess.run(["git", "init", tmp], check=True, capture_output=True) subprocess.run( @@ -120,7 +161,6 @@ def test_checkout_matches_dispatch_sha(self) -> None: "GIT_COMMITTER_EMAIL": "test@test.com", }, ) - # Empty repo won't match arbitrary SHA; just verify script runs. result = subprocess.run( [ "python3", diff --git a/tests/validate_callback_payload.py b/tests/validate_callback_payload.py index 35541da..7632af6 100644 --- a/tests/validate_callback_payload.py +++ b/tests/validate_callback_payload.py @@ -51,7 +51,7 @@ def main() -> int: expect_status = sys.argv[1] if expect_status not in ALLOWED_STATUSES: - print(f"::error::status arg must be in_progress or completed", file=sys.stderr) + print("::error::status arg must be in_progress or completed", file=sys.stderr) return 2 try: diff --git a/tests/write_health_report.py b/tests/write_health_report.py index 8f375a5..317bbaa 100644 --- a/tests/write_health_report.py +++ b/tests/write_health_report.py @@ -72,7 +72,11 @@ def append_github_output(report: dict, github_output: str) -> None: def main() -> int: parser = argparse.ArgumentParser() parser.add_argument("--probe", required=True, help="Probe name, e.g. l1-critical") - parser.add_argument("--output", required=True, help="Path to write JSON report") + parser.add_argument( + "--output", + default="", + help="Path to write JSON report (optional when only using --github-output)", + ) parser.add_argument("--delivery-id", default="") parser.add_argument("--run-id", default="") parser.add_argument("--event-type", default="") @@ -111,8 +115,9 @@ def main() -> int: pr_number=args.pr_number, ) - output = Path(args.output) - output.write_text(json.dumps(report, indent=2) + "\n") + if args.output: + output = Path(args.output) + output.write_text(json.dumps(report, indent=2) + "\n") print(json.dumps(report)) if args.github_output: From b5888370cce718928f0548948c268b8aea3f9ff9 Mon Sep 17 00:00:00 2001 From: jewelkm89 Date: Thu, 9 Jul 2026 14:06:20 +0530 Subject: [PATCH 8/9] Rename oot-l2-ci.yml to crcr-l2-ci.yml and update scripts. Align the L2 workflow filename with CRCR naming used in the workflow title and L1's crcr-dispatch-receiver.yml. Update README and helper scripts to reference the new path. Test Plan: ``` python3 -m unittest discover -s tests -p 'test_*.py' -v ``` Authored with an AI assistant. Co-authored-by: Cursor --- .../{oot-l2-ci.yml => crcr-l2-ci.yml} | 0 README.md | 2 +- agent_space/pr9-body.md | 169 ++++++++++++++++++ scripts/run-local-tests.sh | 1 + scripts/trigger-test-dispatch.sh | 13 +- 5 files changed, 182 insertions(+), 3 deletions(-) rename .github/workflows/{oot-l2-ci.yml => crcr-l2-ci.yml} (100%) create mode 100644 agent_space/pr9-body.md diff --git a/.github/workflows/oot-l2-ci.yml b/.github/workflows/crcr-l2-ci.yml similarity index 100% rename from .github/workflows/oot-l2-ci.yml rename to .github/workflows/crcr-l2-ci.yml diff --git a/README.md b/README.md index 2a55977..2fff1b4 100644 --- a/README.md +++ b/README.md @@ -154,7 +154,7 @@ This repository is a **downstream CRCR health probe**. It contains no CRCR imple | Workflow | Level | Trigger | What it verifies | |----------|-------|---------|------------------| | [`crcr-dispatch-receiver.yml`](.github/workflows/crcr-dispatch-receiver.yml) | L1 | live `repository_dispatch`: `pull_request`, `push` | Dispatch payload, checkout SHA, `delivery_id`; HUD callbacks on `opened`/`reopened` only | -| [`oot-l2-ci.yml`](.github/workflows/oot-l2-ci.yml) | L2 | live `repository_dispatch`: `pull_request` (`opened`/`reopened` only) | L1 checks + smoke + `in_progress`/`completed` callbacks, HUD metrics | +| [`crcr-l2-ci.yml`](.github/workflows/crcr-l2-ci.yml) | L2 | live `repository_dispatch`: `pull_request` (`opened`/`reopened` only) | L1 checks + smoke + `in_progress`/`completed` callbacks, HUD metrics | | [`crcr-unit-tests.yml`](.github/workflows/crcr-unit-tests.yml) | Offline | push/PR to this repo only | Validator unit tests against JSON fixtures (guards test code, not live CRCR) | ### Event filtering (relay rate limits) diff --git a/agent_space/pr9-body.md b/agent_space/pr9-body.md new file mode 100644 index 0000000..ed6ae81 --- /dev/null +++ b/agent_space/pr9-body.md @@ -0,0 +1,169 @@ +## Summary + +Turns `pytorch/crcr-test` into a downstream CRCR test repo. This repo contains test workflows and validators that run on live `repository_dispatch` events from the relay. + +On upstream PyTorch PR `opened`/`reopened`, workflows validate dispatch delivery, PyTorch checkout fidelity, and (for L2) relay callbacks. Health check outcomes are sent to HUD via the existing `completed` callback `test_results` field (each integration check maps to passed/failed/total). + +On `synchronize` and `push`, L1 runs a **light probe** (payload + checkout only, no callbacks). L2 is skipped. This tiered model reduces relay HTTP 429 rate-limit hits ([#8](https://github.com/pytorch/crcr-test/issues/8)) while keeping dispatch coverage on high-volume events. + +```mermaid +flowchart TB + PT[pytorch/pytorch PR or push] --> APP[CRCR GitHub App] + APP --> RD[repository_dispatch to crcr-test] + + RD --> L1C{PR closed?} + RD --> L2C{PR closed?} + + L1C -->|yes| L1X[L1 cancel-workflow] + L1C -->|no| L1A{opened or reopened?} + L1A -->|yes| L1F[L1 full probe + HUD] + L1A -->|no| L1L[L1 light probe no callbacks] + + L2C -->|yes| L2X[L2 cancel-workflow] + L2C -->|no| L2A{opened or reopened?} + L2A -->|yes| L2[L2 full probe + HUD] + L2A -->|no| L2S[L2 skipped] + + L1F --> RELAY[Callback lambda] + L2 --> RELAY + RELAY --> HUD[HUD crcr-test dashboard] + + OFF[Layer 0 offline unit tests] -.-> L1F + OFF -.-> L2 +``` + +## Test layers overview + +| | **L0 — Offline contract tests** | **L1 — Dispatch probe** | **L2 — Callback + HUD probe** | +|---|-------------------------------|-------------------------|-------------------------------| +| **Purpose** | Keep validators/fixtures correct | Prove relay dispatches a usable payload | Prove callbacks work and HUD gets metrics | +| **Workflow** | `crcr-unit-tests.yml` | `crcr-dispatch-receiver.yml` | `oot-l2-ci.yml` | +| **Trigger** | Push / PR to `crcr-test` | Live `repository_dispatch` (`pull_request` + `push`) | Live `repository_dispatch` (`pull_request`, `opened`/`reopened` only) | +| **Touches CRCR relay?** | No | Yes (receives dispatch) | Yes (dispatch + callbacks) | +| **Pushed to HUD?** | No | Yes on `opened`/`reopened` only | Yes on `opened`/`reopened` only | +| **When it runs** | On changes to test code | Every upstream PyTorch PR/push when relay fires | Upstream PR `opened`/`reopened` only | + +## Event filtering (relay rate limits) + +The callback Lambda enforces a per-repo sliding-window limit (~20/min in prod today; 60/min after [test-infra#8203](https://github.com/pytorch/test-infra/pull/8203) deploy). A full L1+L2 probe costs **4 callbacks**. `synchronize` is high volume and was tripping 429s when every event sent full callbacks. + +| Event | L1 | L2 | Callbacks | HUD rows | +|-------|----|----|-----------|----------| +| `opened`, `reopened` | Full + HUD | Full + HUD | **4** | **2** | +| `synchronize` | Light (validate + checkout) | Skipped | **0** | **0** | +| `closed` | Cancel job | Cancel job | **0** | **0** | +| `push` | Light (validate + checkout) | N/A | **0** | **0** | + +Light probe still validates dispatch payload, checks out PyTorch, verifies SHA, and uploads `health-report.json`. Callback coverage remains on low-volume `opened`/`reopened` where the callback code path is identical. + +## Tests and checks by layer + +### L0 — Offline (`crcr-unit-tests.yml`) + +| # | Test | What it validates | +|---|------|-------------------| +| 1 | PR opened fixture | Dispatch payload shape for `opened` | +| 2 | PR synchronize fixture | Payload shape for `synchronize` | +| 3 | PR closed fixture | Payload shape for `closed` | +| 4 | Push / ciflow tag fixture | Payload shape for `push` | +| 5 | Reject missing `delivery_id` | Validator catches bad payload | +| 6 | Reject wrong upstream repo | Must be `pytorch/pytorch` | +| 7 | Reject invalid SHA | 40-char hex required | +| 8 | Callback `in_progress` shape | L2 wire format | +| 9 | Callback `completed` shape | L2 wire format + `conclusion` | +| 10 | Callback rejects missing `conclusion` | Completed must have conclusion | +| 11 | Checkout SHA mismatch detection | `validate_pytorch_checkout.py` logic | +| 12 | Health report to `test_results` mapping | passed/failed/total counts | +| 13 | Healthy / unhealthy report | `healthy`, `conclusion` fields | +| 14 | All fixtures through validator | End-to-end fixture sanity | + +### L1 — Live dispatch probe (`crcr-dispatch-receiver.yml`) + +| # | Check | Full (`opened`/`reopened`) | Light (`synchronize`/`push`) | +|---|-------|---------------------------|------------------------------| +| 1 | `validate_dispatch_payload` | Yes | Yes | +| 2 | `in_progress` callback | Yes | No | +| 3 | Checkout PyTorch @ dispatched SHA | Yes | Yes | +| 4 | `validate_pytorch_checkout` | Yes | Yes | +| 5 | Assert `delivery_id` | Yes | Yes | +| 6 | `completed` callback + HUD metrics | Yes | No | +| — | `cancel-workflow` (PR closed) | Cancel only | Cancel only | + +### L2 — Live callback probe (`oot-l2-ci.yml`) + +Runs only on `opened`/`reopened`. + +| # | Check | What it validates | +|---|-------|-----------------| +| 1 | `validate_dispatch_payload` | Same L1 contract on live payload | +| 2 | `in_progress` callback | State DISPATCHED to IN_PROGRESS | +| 3 | Checkout PyTorch @ PR head SHA | PR ref fetchable | +| 4 | `validate_pytorch_checkout` | Checkout matches head SHA | +| 5 | Smoke checks | README, .github, Python (deterministic) | +| 6 | `write_health_report` + `completed` callback | `test_results` + `conclusion` to HUD | +| 7 | `completed_callback` in final report | Completed callback succeeded | +| — | `cancel-workflow` (PR closed) | L2 build skipped on `closed` | + +## One upstream event — what runs? + +| Upstream event | L0 | L1 | L2 | Callbacks | HUD rows | +|----------------|----|----|-----|-----------|----------| +| PR `opened` / `reopened` | — | `l1-critical` full | `l2-critical` | 4 | 2 | +| PR `synchronize` | — | `l1-light` | — | 0 | 0 | +| PR `closed` | — | `cancel-workflow` | `cancel-workflow` | 0 | 0 | +| Push / ciflow tag | — | `l1-light` | — | 0 | 0 | +| Push/PR to crcr-test | `crcr-unit-tests` | — | — | 0 | 0 | + +## Shared components by layer + +| Component | L0 | L1 | L2 | +|-----------|----|----|-----| +| `validate_dispatch_payload.py` | Fixtures | Live payload | Live payload | +| `validate_pytorch_checkout.py` | Unit test | Live checkout | Live checkout | +| `validate_callback_payload.py` | Unit test | — | — | +| `write_health_report.py` | Unit test | Live to HUD (full only) | Live to HUD | +| `cross-repo-ci-relay-callback` | — | Full probe only | Yes | + +## Key changes + +- **`tests/`** — payload validators, JSON fixtures, `write_health_report.py`, and 16 unit tests +- **`write_health_report.py`** — shared health reporter used by both L1 and L2 probes; aggregates step outcomes into `checks`, maps them to HUD `test_results`, and feeds the `completed` callback (separate report and HUD row per probe) +- **`crcr-dispatch-receiver.yml`** — L1 probe: full on `opened`/`reopened`, light on `synchronize`/`push` +- **`oot-l2-ci.yml`** — L2 probe: `opened`/`reopened` only; deterministic checks + callbacks +- **`crcr-unit-tests.yml`** — offline contract tests on push/PR to this repo +- **`scripts/`** — `run-local-tests.sh` and `trigger-test-dispatch.sh` for local/fork testing +- **README** — documents probe model, event filtering, triggers, and HUD metric mapping + +CRCR health is measured when the relay dispatches; offline unit tests guard validator/fixture drift on merge. + +## HUD health metrics + +L1 and L2 full probes build a health report from step outcomes, then send it on the `completed` callback: + +| Step outcome | HUD `test_results` | +|--------------|-------------------| +| `success` | `passed_tests` | +| `failure` / `cancelled` | `failed_tests` | +| `skipped` | `skipped_tests` | + +`conclusion` is `success` only when all checks pass. Per-check names are in the `health-report.json` workflow artifact. Verify rows at https://hud.pytorch.org/crcr/pytorch/crcr-test + +## Test plan + +Pre-merge (offline): +``` +cd crcr-test +./scripts/run-local-tests.sh +``` + +Pre-merge (CI): +- [ ] `crcr-unit-tests` workflow passes on this PR + +Post-merge (live CRCR): +- [ ] Upstream PyTorch PR `opened` triggers `L1 Health` (full) and `L2 Health` workflows +- [ ] Upstream PyTorch PR `synchronize` triggers L1 light only (no L2, no callbacks) +- [ ] `completed` callback succeeds on `opened`/`reopened` (HTTP 2xx from relay) +- [ ] HUD row shows `passed_tests` / `failed_tests` / `total_tests` matching probe checks +- [ ] `health-report.json` artifact uploaded per run + +Authored with an AI assistant. diff --git a/scripts/run-local-tests.sh b/scripts/run-local-tests.sh index 2f51ea8..b02b31d 100755 --- a/scripts/run-local-tests.sh +++ b/scripts/run-local-tests.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash # Run offline CRCR validator tests (no relay required). +# Live L1/L2 probe workflows: crcr-dispatch-receiver.yml, crcr-l2-ci.yml set -euo pipefail cd "$(dirname "$0")/.." diff --git a/scripts/trigger-test-dispatch.sh b/scripts/trigger-test-dispatch.sh index 6075864..33f0e69 100755 --- a/scripts/trigger-test-dispatch.sh +++ b/scripts/trigger-test-dispatch.sh @@ -1,10 +1,14 @@ #!/usr/bin/env bash -# Send a test repository_dispatch to exercise L1/L2 workflows. +# Send a test repository_dispatch to exercise CRCR probe workflows. # Requires: gh CLI, workflows merged to the target repo default branch (main). # +# Workflows exercised: +# pull_request/opened -> crcr-dispatch-receiver.yml (L1 full) + crcr-l2-ci.yml (L2 full) +# push -> crcr-dispatch-receiver.yml (L1 light) only +# # Usage: # ./scripts/trigger-test-dispatch.sh # L1+L2 (pull_request/opened) -# ./scripts/trigger-test-dispatch.sh push # L1 only (push event) +# ./scripts/trigger-test-dispatch.sh push # L1 light only (push event) # # Target repo: current gh repo, or pytorch/crcr-test. Override with CRCR_TEST_REPO. set -euo pipefail @@ -74,6 +78,11 @@ echo "${PAYLOAD}" | gh api --method POST "repos/${REPO}/dispatches" --input - echo "" echo "Done. Check workflow runs at:" echo " https://github.com/${REPO}/actions" +if [[ "${MODE}" == "push" ]]; then + echo "Expected workflows: crcr-dispatch-receiver.yml (L1 light)" +else + echo "Expected workflows: crcr-dispatch-receiver.yml (L1 full), crcr-l2-ci.yml (L2 full)" +fi echo "" echo "Note: callbacks require the target repo on the CRCR allowlist and a relay dispatch record." echo "Synthetic delivery_id dispatches exercise L1/L2 workflow steps; callbacks may fail locally." From 7bcad41db8f06e16c146235d95a83b9701026d89 Mon Sep 17 00:00:00 2001 From: jewelkm89 Date: Thu, 9 Jul 2026 14:06:45 +0530 Subject: [PATCH 9/9] Remove accidental agent_space file and gitignore the directory. Authored with an AI assistant. Co-authored-by: Cursor --- .gitignore | 1 + agent_space/pr9-body.md | 169 ---------------------------------------- 2 files changed, 1 insertion(+), 169 deletions(-) delete mode 100644 agent_space/pr9-body.md diff --git a/.gitignore b/.gitignore index 7a60b85..fb39099 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ __pycache__/ *.pyc +agent_space/ diff --git a/agent_space/pr9-body.md b/agent_space/pr9-body.md deleted file mode 100644 index ed6ae81..0000000 --- a/agent_space/pr9-body.md +++ /dev/null @@ -1,169 +0,0 @@ -## Summary - -Turns `pytorch/crcr-test` into a downstream CRCR test repo. This repo contains test workflows and validators that run on live `repository_dispatch` events from the relay. - -On upstream PyTorch PR `opened`/`reopened`, workflows validate dispatch delivery, PyTorch checkout fidelity, and (for L2) relay callbacks. Health check outcomes are sent to HUD via the existing `completed` callback `test_results` field (each integration check maps to passed/failed/total). - -On `synchronize` and `push`, L1 runs a **light probe** (payload + checkout only, no callbacks). L2 is skipped. This tiered model reduces relay HTTP 429 rate-limit hits ([#8](https://github.com/pytorch/crcr-test/issues/8)) while keeping dispatch coverage on high-volume events. - -```mermaid -flowchart TB - PT[pytorch/pytorch PR or push] --> APP[CRCR GitHub App] - APP --> RD[repository_dispatch to crcr-test] - - RD --> L1C{PR closed?} - RD --> L2C{PR closed?} - - L1C -->|yes| L1X[L1 cancel-workflow] - L1C -->|no| L1A{opened or reopened?} - L1A -->|yes| L1F[L1 full probe + HUD] - L1A -->|no| L1L[L1 light probe no callbacks] - - L2C -->|yes| L2X[L2 cancel-workflow] - L2C -->|no| L2A{opened or reopened?} - L2A -->|yes| L2[L2 full probe + HUD] - L2A -->|no| L2S[L2 skipped] - - L1F --> RELAY[Callback lambda] - L2 --> RELAY - RELAY --> HUD[HUD crcr-test dashboard] - - OFF[Layer 0 offline unit tests] -.-> L1F - OFF -.-> L2 -``` - -## Test layers overview - -| | **L0 — Offline contract tests** | **L1 — Dispatch probe** | **L2 — Callback + HUD probe** | -|---|-------------------------------|-------------------------|-------------------------------| -| **Purpose** | Keep validators/fixtures correct | Prove relay dispatches a usable payload | Prove callbacks work and HUD gets metrics | -| **Workflow** | `crcr-unit-tests.yml` | `crcr-dispatch-receiver.yml` | `oot-l2-ci.yml` | -| **Trigger** | Push / PR to `crcr-test` | Live `repository_dispatch` (`pull_request` + `push`) | Live `repository_dispatch` (`pull_request`, `opened`/`reopened` only) | -| **Touches CRCR relay?** | No | Yes (receives dispatch) | Yes (dispatch + callbacks) | -| **Pushed to HUD?** | No | Yes on `opened`/`reopened` only | Yes on `opened`/`reopened` only | -| **When it runs** | On changes to test code | Every upstream PyTorch PR/push when relay fires | Upstream PR `opened`/`reopened` only | - -## Event filtering (relay rate limits) - -The callback Lambda enforces a per-repo sliding-window limit (~20/min in prod today; 60/min after [test-infra#8203](https://github.com/pytorch/test-infra/pull/8203) deploy). A full L1+L2 probe costs **4 callbacks**. `synchronize` is high volume and was tripping 429s when every event sent full callbacks. - -| Event | L1 | L2 | Callbacks | HUD rows | -|-------|----|----|-----------|----------| -| `opened`, `reopened` | Full + HUD | Full + HUD | **4** | **2** | -| `synchronize` | Light (validate + checkout) | Skipped | **0** | **0** | -| `closed` | Cancel job | Cancel job | **0** | **0** | -| `push` | Light (validate + checkout) | N/A | **0** | **0** | - -Light probe still validates dispatch payload, checks out PyTorch, verifies SHA, and uploads `health-report.json`. Callback coverage remains on low-volume `opened`/`reopened` where the callback code path is identical. - -## Tests and checks by layer - -### L0 — Offline (`crcr-unit-tests.yml`) - -| # | Test | What it validates | -|---|------|-------------------| -| 1 | PR opened fixture | Dispatch payload shape for `opened` | -| 2 | PR synchronize fixture | Payload shape for `synchronize` | -| 3 | PR closed fixture | Payload shape for `closed` | -| 4 | Push / ciflow tag fixture | Payload shape for `push` | -| 5 | Reject missing `delivery_id` | Validator catches bad payload | -| 6 | Reject wrong upstream repo | Must be `pytorch/pytorch` | -| 7 | Reject invalid SHA | 40-char hex required | -| 8 | Callback `in_progress` shape | L2 wire format | -| 9 | Callback `completed` shape | L2 wire format + `conclusion` | -| 10 | Callback rejects missing `conclusion` | Completed must have conclusion | -| 11 | Checkout SHA mismatch detection | `validate_pytorch_checkout.py` logic | -| 12 | Health report to `test_results` mapping | passed/failed/total counts | -| 13 | Healthy / unhealthy report | `healthy`, `conclusion` fields | -| 14 | All fixtures through validator | End-to-end fixture sanity | - -### L1 — Live dispatch probe (`crcr-dispatch-receiver.yml`) - -| # | Check | Full (`opened`/`reopened`) | Light (`synchronize`/`push`) | -|---|-------|---------------------------|------------------------------| -| 1 | `validate_dispatch_payload` | Yes | Yes | -| 2 | `in_progress` callback | Yes | No | -| 3 | Checkout PyTorch @ dispatched SHA | Yes | Yes | -| 4 | `validate_pytorch_checkout` | Yes | Yes | -| 5 | Assert `delivery_id` | Yes | Yes | -| 6 | `completed` callback + HUD metrics | Yes | No | -| — | `cancel-workflow` (PR closed) | Cancel only | Cancel only | - -### L2 — Live callback probe (`oot-l2-ci.yml`) - -Runs only on `opened`/`reopened`. - -| # | Check | What it validates | -|---|-------|-----------------| -| 1 | `validate_dispatch_payload` | Same L1 contract on live payload | -| 2 | `in_progress` callback | State DISPATCHED to IN_PROGRESS | -| 3 | Checkout PyTorch @ PR head SHA | PR ref fetchable | -| 4 | `validate_pytorch_checkout` | Checkout matches head SHA | -| 5 | Smoke checks | README, .github, Python (deterministic) | -| 6 | `write_health_report` + `completed` callback | `test_results` + `conclusion` to HUD | -| 7 | `completed_callback` in final report | Completed callback succeeded | -| — | `cancel-workflow` (PR closed) | L2 build skipped on `closed` | - -## One upstream event — what runs? - -| Upstream event | L0 | L1 | L2 | Callbacks | HUD rows | -|----------------|----|----|-----|-----------|----------| -| PR `opened` / `reopened` | — | `l1-critical` full | `l2-critical` | 4 | 2 | -| PR `synchronize` | — | `l1-light` | — | 0 | 0 | -| PR `closed` | — | `cancel-workflow` | `cancel-workflow` | 0 | 0 | -| Push / ciflow tag | — | `l1-light` | — | 0 | 0 | -| Push/PR to crcr-test | `crcr-unit-tests` | — | — | 0 | 0 | - -## Shared components by layer - -| Component | L0 | L1 | L2 | -|-----------|----|----|-----| -| `validate_dispatch_payload.py` | Fixtures | Live payload | Live payload | -| `validate_pytorch_checkout.py` | Unit test | Live checkout | Live checkout | -| `validate_callback_payload.py` | Unit test | — | — | -| `write_health_report.py` | Unit test | Live to HUD (full only) | Live to HUD | -| `cross-repo-ci-relay-callback` | — | Full probe only | Yes | - -## Key changes - -- **`tests/`** — payload validators, JSON fixtures, `write_health_report.py`, and 16 unit tests -- **`write_health_report.py`** — shared health reporter used by both L1 and L2 probes; aggregates step outcomes into `checks`, maps them to HUD `test_results`, and feeds the `completed` callback (separate report and HUD row per probe) -- **`crcr-dispatch-receiver.yml`** — L1 probe: full on `opened`/`reopened`, light on `synchronize`/`push` -- **`oot-l2-ci.yml`** — L2 probe: `opened`/`reopened` only; deterministic checks + callbacks -- **`crcr-unit-tests.yml`** — offline contract tests on push/PR to this repo -- **`scripts/`** — `run-local-tests.sh` and `trigger-test-dispatch.sh` for local/fork testing -- **README** — documents probe model, event filtering, triggers, and HUD metric mapping - -CRCR health is measured when the relay dispatches; offline unit tests guard validator/fixture drift on merge. - -## HUD health metrics - -L1 and L2 full probes build a health report from step outcomes, then send it on the `completed` callback: - -| Step outcome | HUD `test_results` | -|--------------|-------------------| -| `success` | `passed_tests` | -| `failure` / `cancelled` | `failed_tests` | -| `skipped` | `skipped_tests` | - -`conclusion` is `success` only when all checks pass. Per-check names are in the `health-report.json` workflow artifact. Verify rows at https://hud.pytorch.org/crcr/pytorch/crcr-test - -## Test plan - -Pre-merge (offline): -``` -cd crcr-test -./scripts/run-local-tests.sh -``` - -Pre-merge (CI): -- [ ] `crcr-unit-tests` workflow passes on this PR - -Post-merge (live CRCR): -- [ ] Upstream PyTorch PR `opened` triggers `L1 Health` (full) and `L2 Health` workflows -- [ ] Upstream PyTorch PR `synchronize` triggers L1 light only (no L2, no callbacks) -- [ ] `completed` callback succeeds on `opened`/`reopened` (HTTP 2xx from relay) -- [ ] HUD row shows `passed_tests` / `failed_tests` / `total_tests` matching probe checks -- [ ] `health-report.json` artifact uploaded per run - -Authored with an AI assistant.