diff --git a/.github/workflows/crcr-dispatch-receiver.yml b/.github/workflows/crcr-dispatch-receiver.yml index c777289..230a177 100644 --- a/.github/workflows/crcr-dispatch-receiver.yml +++ b/.github/workflows/crcr-dispatch-receiver.yml @@ -7,7 +7,7 @@ on: - push run-name: >- - Dispatch - + L1 Health - ${{ github.event.client_payload.event_type == 'pull_request' && format( @@ -26,48 +26,223 @@ concurrency: permissions: actions: write + id-token: write jobs: cancel-workflow: if: ${{ github.event.client_payload.payload.action == 'closed' }} runs-on: ubuntu-latest steps: - - run: echo "PR closed, canceling older runs in the same concurrency group" - inspect: + - uses: actions/checkout@v4 + + - name: Assert closed action skips build job + id: closed + run: | + ACTION="${{ github.event.client_payload.payload.action }}" + if [ "${ACTION}" != "closed" ]; then + echo "::error::expected action=closed, got ${ACTION}" + exit 1 + fi + echo "PR closed — cancel-workflow job ran as expected" + + - name: Write closed-event health report + if: always() + run: | + python3 tests/write_health_report.py \ + --probe l1-closed \ + --output health-report.json \ + --delivery-id "${{ github.event.client_payload.delivery_id }}" \ + --run-id "${{ github.run_id }}" \ + --event-type "${{ github.event.client_payload.event_type }}" \ + --pr-number "${{ github.event.client_payload.payload.pull_request.number || '' }}" \ + --check "closed_cancel=${{ steps.closed.outcome }}" \ + --strict + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: crcr-health-l1-closed-${{ github.run_id }} + path: health-report.json + + l1-critical: if: ${{ github.event.client_payload.payload.action != 'closed' }} runs-on: ubuntu-latest steps: - - name: Display parsed fields + - name: Checkout crcr-test + uses: actions/checkout@v4 + + - name: L1 — resolve probe mode + id: probe + run: | + ACTION="${{ github.event.client_payload.payload.action }}" + EVENT="${{ github.event.client_payload.event_type }}" + if [ "$EVENT" = "pull_request" ] && { [ "$ACTION" = "opened" ] || [ "$ACTION" = "reopened" ]; }; then + echo "hud_callbacks=true" >> "$GITHUB_OUTPUT" + echo "probe=l1-critical" >> "$GITHUB_OUTPUT" + else + echo "hud_callbacks=false" >> "$GITHUB_OUTPUT" + echo "probe=l1-light" >> "$GITHUB_OUTPUT" + fi + + - name: L1 — validate dispatch payload structure + id: validate_payload + env: + CLIENT_PAYLOAD: ${{ toJson(github.event.client_payload) }} + run: echo "${CLIENT_PAYLOAD}" | python3 tests/validate_dispatch_payload.py + + - name: L1 — report in_progress to relay + id: in_progress + if: steps.probe.outputs.hud_callbacks == 'true' + uses: pytorch/test-infra/.github/actions/cross-repo-ci-relay-callback@main + with: + status: in_progress + + - name: L1 — checkout PyTorch at dispatched SHA + id: checkout_pytorch + uses: actions/checkout@v4 + with: + repository: pytorch/pytorch + ref: >- + ${{ github.event.client_payload.event_type == 'pull_request' && + github.event.client_payload.payload.pull_request.head.sha || + github.event.client_payload.payload.after }} + path: pytorch + + - name: L1 — verify checkout matches dispatch SHA + id: verify_checkout + env: + CLIENT_PAYLOAD: ${{ toJson(github.event.client_payload) }} + run: | + echo "${CLIENT_PAYLOAD}" > /tmp/client_payload.json + python3 tests/validate_pytorch_checkout.py \ + --payload /tmp/client_payload.json \ + --pytorch-dir pytorch + + - name: L1 — assert delivery_id is stable for L2 linkage + id: delivery + run: | + DELIVERY_ID="${{ github.event.client_payload.delivery_id }}" + if [ -z "${DELIVERY_ID}" ]; then + echo "::error::delivery_id is empty" + exit 1 + fi + echo "delivery_id=${DELIVERY_ID}" + + - name: L1 — export probe step outcomes + if: always() + run: | + { + echo "VALIDATE_PAYLOAD=${{ steps.validate_payload.outcome }}" + echo "IN_PROGRESS_CALLBACK=${{ steps.in_progress.outcome }}" + echo "CHECKOUT_PYTORCH=${{ steps.checkout_pytorch.outcome }}" + echo "VERIFY_CHECKOUT=${{ steps.verify_checkout.outcome }}" + echo "DELIVERY_ID=${{ steps.delivery.outcome }}" + } >> "$GITHUB_ENV" + + - name: L1 — build health metrics for HUD + id: health_metrics + if: always() && steps.probe.outputs.hud_callbacks == 'true' + run: | + mapfile -t CHECK_ARGS < <(bash scripts/health_checks.sh l1-full) + python3 tests/write_health_report.py \ + --probe l1-critical \ + --delivery-id "${{ github.event.client_payload.delivery_id }}" \ + --run-id "${{ github.run_id }}" \ + --event-type "${{ github.event.client_payload.event_type }}" \ + --pr-number "${{ github.event.client_payload.payload.pull_request.number || '' }}" \ + "${CHECK_ARGS[@]}" \ + --github-output "$GITHUB_OUTPUT" + + - name: L1 — report completed with health metrics to relay/HUD + id: completed + if: always() && steps.probe.outputs.hud_callbacks == 'true' + uses: pytorch/test-infra/.github/actions/cross-repo-ci-relay-callback@main + with: + status: completed + conclusion: ${{ steps.health_metrics.outputs.conclusion || (job.status == 'success' && 'success' || 'failure') }} + test-results: ${{ steps.health_metrics.outputs.test_results }} + artifact-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + + - name: L1 — write light health report (no HUD callbacks) + if: always() && steps.probe.outputs.hud_callbacks != 'true' + run: | + { + echo "VALIDATE_PAYLOAD=${{ steps.validate_payload.outcome }}" + echo "CHECKOUT_PYTORCH=${{ steps.checkout_pytorch.outcome }}" + echo "VERIFY_CHECKOUT=${{ steps.verify_checkout.outcome }}" + echo "DELIVERY_ID=${{ steps.delivery.outcome }}" + } >> "$GITHUB_ENV" + mapfile -t CHECK_ARGS < <(bash scripts/health_checks.sh l1-light) + python3 tests/write_health_report.py \ + --probe l1-light \ + --output health-report.json \ + --delivery-id "${{ github.event.client_payload.delivery_id }}" \ + --run-id "${{ github.run_id }}" \ + --event-type "${{ github.event.client_payload.event_type }}" \ + --pr-number "${{ github.event.client_payload.payload.pull_request.number || '' }}" \ + "${CHECK_ARGS[@]}" \ + --strict + + # Artifact uses one more check than HUD test_results: completed_callback + # runs after the HUD POST, so its outcome is only known afterward. + - name: L1 — write full health report artifact + if: always() && steps.probe.outputs.hud_callbacks == 'true' + env: + INCLUDE_COMPLETED_CALLBACK: "1" + COMPLETED_CALLBACK: ${{ steps.completed.outcome }} + run: | + mapfile -t CHECK_ARGS < <(bash scripts/health_checks.sh l1-full) + python3 tests/write_health_report.py \ + --probe l1-critical \ + --output health-report.json \ + --delivery-id "${{ github.event.client_payload.delivery_id }}" \ + --run-id "${{ github.run_id }}" \ + --event-type "${{ github.event.client_payload.event_type }}" \ + --pr-number "${{ github.event.client_payload.payload.pull_request.number || '' }}" \ + "${CHECK_ARGS[@]}" \ + --strict + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: crcr-health-l1-${{ github.run_id }} + path: health-report.json + + - name: Write integration summary + if: always() run: | EVENT_TYPE="${{ github.event.client_payload.event_type }}" - echo "### Dispatch Summary" >> $GITHUB_STEP_SUMMARY + PROBE_MODE="${{ steps.probe.outputs.probe }}" + echo "### L1 Real-Time Dispatch Tests" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY - echo "| Field | Value |" >> $GITHUB_STEP_SUMMARY - echo "|-------|-------|" >> $GITHUB_STEP_SUMMARY + if [ "${PROBE_MODE}" = "l1-critical" ]; then + echo "Probe mode: **full** (HUD callbacks on \`opened\`/\`reopened\`)." >> $GITHUB_STEP_SUMMARY + echo "Health metrics sent to HUD via \`completed\` callback \`test_results\`." >> $GITHUB_STEP_SUMMARY + else + echo "Probe mode: **light** (no HUD callbacks on \`synchronize\`/\`push\`; reduces relay 429 risk)." >> $GITHUB_STEP_SUMMARY + fi + echo "" >> $GITHUB_STEP_SUMMARY + echo "| Check | Result |" >> $GITHUB_STEP_SUMMARY + echo "|-------|--------|" >> $GITHUB_STEP_SUMMARY + echo "| Payload structure | ${{ steps.validate_payload.outcome }} |" >> $GITHUB_STEP_SUMMARY + if [ "${PROBE_MODE}" = "l1-critical" ]; then + echo "| in_progress callback | ${{ steps.in_progress.outcome }} |" >> $GITHUB_STEP_SUMMARY + fi + echo "| PyTorch checkout | ${{ steps.checkout_pytorch.outcome }} |" >> $GITHUB_STEP_SUMMARY + echo "| SHA verification | ${{ steps.verify_checkout.outcome }} |" >> $GITHUB_STEP_SUMMARY + echo "| delivery_id | ${{ steps.delivery.outcome }} |" >> $GITHUB_STEP_SUMMARY + if [ "${PROBE_MODE}" = "l1-critical" ]; then + echo "| completed callback | ${{ steps.completed.outcome }} |" >> $GITHUB_STEP_SUMMARY + fi echo "| Event type | \`${EVENT_TYPE}\` |" >> $GITHUB_STEP_SUMMARY - echo "| Upstream repo | \`${{ github.event.client_payload.payload.repository.full_name }}\` |" >> $GITHUB_STEP_SUMMARY - if [ "$EVENT_TYPE" = "pull_request" ]; then - echo "| PR number | \`${{ github.event.client_payload.payload.pull_request.number }}\` |" >> $GITHUB_STEP_SUMMARY - echo "| PR action | \`${{ github.event.client_payload.payload.action }}\` |" >> $GITHUB_STEP_SUMMARY - echo "| PR title | \`${{ github.event.client_payload.payload.pull_request.title }}\` |" >> $GITHUB_STEP_SUMMARY + echo "| PR | #${{ github.event.client_payload.payload.pull_request.number }} (${{ github.event.client_payload.payload.action }}) |" >> $GITHUB_STEP_SUMMARY echo "| Head SHA | \`${{ github.event.client_payload.payload.pull_request.head.sha }}\` |" >> $GITHUB_STEP_SUMMARY - echo "| Head repo | \`${{ github.event.client_payload.payload.pull_request.head.repo.full_name }}\` |" >> $GITHUB_STEP_SUMMARY - elif [ "$EVENT_TYPE" = "push" ]; then + else echo "| Ref | \`${{ github.event.client_payload.payload.ref }}\` |" >> $GITHUB_STEP_SUMMARY echo "| SHA | \`${{ github.event.client_payload.payload.after }}\` |" >> $GITHUB_STEP_SUMMARY - echo "| Base ref | \`${{ github.event.client_payload.payload.base_ref }}\` |" >> $GITHUB_STEP_SUMMARY - echo "| Deleted | \`${{ github.event.client_payload.payload.deleted }}\` |" >> $GITHUB_STEP_SUMMARY fi - - - name: Log full event to step summary - if: always() - run: | echo "" >> $GITHUB_STEP_SUMMARY - echo "### Full Event Payload" >> $GITHUB_STEP_SUMMARY + echo "HUD: [crcr-test dashboard](https://hud.pytorch.org/crcr/pytorch/crcr-test) (run_id \`${{ github.run_id }}\`)" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY - cat <<'EOF' >> $GITHUB_STEP_SUMMARY - ```json - ${{ toJson(github.event) }} - ``` - EOF + echo "delivery_id: \`${{ github.event.client_payload.delivery_id }}\`" >> $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/crcr-l2-ci.yml b/.github/workflows/crcr-l2-ci.yml new file mode 100644 index 0000000..0e4efb9 --- /dev/null +++ b/.github/workflows/crcr-l2-ci.yml @@ -0,0 +1,177 @@ +name: CRCR L2 Critical + +on: + repository_dispatch: + types: + - pull_request + +run-name: >- + L2 Health - + PR #${{ github.event.client_payload.payload.pull_request.number }} + (${{ github.event.client_payload.payload.action }}) + +concurrency: + group: >- + crcr-l2-${{ github.event.client_payload.payload.repository.full_name }}-${{ + github.event.client_payload.payload.pull_request.number || github.run_id }} + cancel-in-progress: true + +permissions: + actions: write + id-token: write + +jobs: + cancel-workflow: + if: ${{ github.event.client_payload.payload.action == 'closed' }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Assert closed action skips L2 build job + id: closed + run: echo "PR closed — cancel-workflow job ran as expected" + + - name: Write closed-event health report + if: always() + run: | + python3 tests/write_health_report.py \ + --probe l2-closed \ + --output health-report.json \ + --delivery-id "${{ github.event.client_payload.delivery_id }}" \ + --run-id "${{ github.run_id }}" \ + --event-type pull_request \ + --pr-number "${{ github.event.client_payload.payload.pull_request.number }}" \ + --check "closed_cancel=${{ steps.closed.outcome }}" \ + --strict + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: crcr-health-l2-closed-${{ github.run_id }} + path: health-report.json + + l2-critical: + if: contains(fromJSON('["opened","reopened"]'), github.event.client_payload.payload.action) + runs-on: ubuntu-latest + steps: + - name: Checkout crcr-test + uses: actions/checkout@v4 + + - name: L2 — validate dispatch payload before callbacks + id: validate_payload + env: + CLIENT_PAYLOAD: ${{ toJson(github.event.client_payload) }} + run: echo "${CLIENT_PAYLOAD}" | python3 tests/validate_dispatch_payload.py + + - name: L2 — report in_progress to relay + id: in_progress + uses: pytorch/test-infra/.github/actions/cross-repo-ci-relay-callback@main + with: + status: in_progress + + - name: L2 — checkout PyTorch at dispatched SHA + id: checkout_pytorch + uses: actions/checkout@v4 + with: + repository: pytorch/pytorch + ref: ${{ github.event.client_payload.payload.pull_request.head.sha }} + path: pytorch + + - name: L2 — verify checkout matches dispatch SHA + id: verify_checkout + env: + CLIENT_PAYLOAD: ${{ toJson(github.event.client_payload) }} + run: | + echo "${CLIENT_PAYLOAD}" > /tmp/client_payload.json + python3 tests/validate_pytorch_checkout.py \ + --payload /tmp/client_payload.json \ + --pytorch-dir pytorch + + - name: L2 — run deterministic smoke checks + id: smoke + run: | + set -euo pipefail + test -f pytorch/README.md + test -d pytorch/.github + python3 -c "import json; print('smoke ok')" + + - name: L2 — export probe step outcomes + if: always() + run: | + { + echo "VALIDATE_PAYLOAD=${{ steps.validate_payload.outcome }}" + echo "IN_PROGRESS_CALLBACK=${{ steps.in_progress.outcome }}" + echo "CHECKOUT_PYTORCH=${{ steps.checkout_pytorch.outcome }}" + echo "VERIFY_CHECKOUT=${{ steps.verify_checkout.outcome }}" + echo "SMOKE_CHECKS=${{ steps.smoke.outcome }}" + } >> "$GITHUB_ENV" + + - name: L2 — build health metrics for HUD + id: health_metrics + if: always() + run: | + mapfile -t CHECK_ARGS < <(bash scripts/health_checks.sh l2-full) + python3 tests/write_health_report.py \ + --probe l2-critical \ + --delivery-id "${{ github.event.client_payload.delivery_id }}" \ + --run-id "${{ github.run_id }}" \ + --event-type pull_request \ + --pr-number "${{ github.event.client_payload.payload.pull_request.number }}" \ + "${CHECK_ARGS[@]}" \ + --github-output "$GITHUB_OUTPUT" + + - name: L2 — report completed with health metrics to relay/HUD + id: completed + if: always() + uses: pytorch/test-infra/.github/actions/cross-repo-ci-relay-callback@main + with: + status: completed + conclusion: ${{ steps.health_metrics.outputs.conclusion || (job.status == 'success' && 'success' || 'failure') }} + test-results: ${{ steps.health_metrics.outputs.test_results }} + artifact-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + + # Artifact uses one more check than HUD test_results: completed_callback + # runs after the HUD POST, so its outcome is only known afterward. + - name: L2 — write health report artifact + id: health_final + if: always() + env: + INCLUDE_COMPLETED_CALLBACK: "1" + COMPLETED_CALLBACK: ${{ steps.completed.outcome }} + run: | + mapfile -t CHECK_ARGS < <(bash scripts/health_checks.sh l2-full) + python3 tests/write_health_report.py \ + --probe l2-critical \ + --output health-report.json \ + --delivery-id "${{ github.event.client_payload.delivery_id }}" \ + --run-id "${{ github.run_id }}" \ + --event-type pull_request \ + --pr-number "${{ github.event.client_payload.payload.pull_request.number }}" \ + "${CHECK_ARGS[@]}" \ + --strict + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: crcr-health-l2-${{ github.run_id }} + path: health-report.json + + - name: Write L2 integration summary + if: always() + run: | + echo "### L2 Real-Time Dispatch Tests" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "Health metrics sent to HUD via \`completed\` callback \`test_results\`." >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "| Integration point | Status |" >> $GITHUB_STEP_SUMMARY + echo "|-------------------|--------|" >> $GITHUB_STEP_SUMMARY + echo "| Dispatch payload validation | ${{ steps.validate_payload.outcome }} |" >> $GITHUB_STEP_SUMMARY + echo "| in_progress callback | ${{ steps.in_progress.outcome }} |" >> $GITHUB_STEP_SUMMARY + echo "| PyTorch checkout | ${{ steps.checkout_pytorch.outcome }} |" >> $GITHUB_STEP_SUMMARY + echo "| SHA verification | ${{ steps.verify_checkout.outcome }} |" >> $GITHUB_STEP_SUMMARY + echo "| Smoke checks | ${{ steps.smoke.outcome }} |" >> $GITHUB_STEP_SUMMARY + echo "| completed callback | ${{ steps.completed.outcome }} |" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "HUD: [crcr-test dashboard](https://hud.pytorch.org/crcr/pytorch/crcr-test) (run_id \`${{ github.run_id }}\`)" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "delivery_id: \`${{ github.event.client_payload.delivery_id }}\`" >> $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/crcr-unit-tests.yml b/.github/workflows/crcr-unit-tests.yml new file mode 100644 index 0000000..0c4b258 --- /dev/null +++ b/.github/workflows/crcr-unit-tests.yml @@ -0,0 +1,26 @@ +name: CRCR Unit Tests + +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + +permissions: + contents: read + +jobs: + validators: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Run payload validator unit tests + run: python3 -m unittest discover -s tests -p 'test_*.py' -v + + - name: Validate fixture files individually + run: | + for f in tests/fixtures/*.json; do + echo "Validating $f" + python3 tests/validate_dispatch_payload.py < "$f" + done diff --git a/.github/workflows/oot-l2-ci.yml b/.github/workflows/oot-l2-ci.yml deleted file mode 100644 index 4d8e409..0000000 --- a/.github/workflows/oot-l2-ci.yml +++ /dev/null @@ -1,135 +0,0 @@ -name: OOT L2 CI - -on: - repository_dispatch: - types: - - pull_request - -run-name: >- - OOT L2 - - PR #${{ github.event.client_payload.payload.pull_request.number }} - (${{ github.event.client_payload.payload.action }}) - -concurrency: - group: >- - oot-l2-${{ github.event.client_payload.payload.repository.full_name }}-${{ - github.event.client_payload.payload.pull_request.number || github.run_id }} - cancel-in-progress: true - -permissions: - actions: write - id-token: write - -jobs: - cancel-workflow: - if: ${{ github.event.client_payload.payload.action == 'closed' }} - runs-on: ubuntu-latest - steps: - - run: echo "PR closed — canceling older runs in the same concurrency group" - - build-and-test: - if: ${{ github.event.client_payload.payload.action != 'closed' }} - runs-on: ubuntu-latest - steps: - - name: Report CI started - uses: pytorch/test-infra/.github/actions/cross-repo-ci-relay-callback@main - with: - status: in_progress - - - name: Checkout - uses: actions/checkout@v4 - - - name: Checkout PyTorch at dispatched SHA - uses: actions/checkout@v4 - with: - repository: pytorch/pytorch - ref: ${{ github.event.client_payload.payload.pull_request.head.sha }} - path: pytorch - - - name: Generate random test configuration - id: config - run: | - SEED=$(( RANDOM ^ ${{ github.run_id }} )) - TOTAL=$(( (SEED % 400) + 100 )) - FAIL_RATE=$(( SEED % 12 )) - FAILED=$(( TOTAL * FAIL_RATE / 100 )) - SKIP_RATE=$(( (SEED / 7) % 5 )) - SKIPPED=$(( TOTAL * SKIP_RATE / 100 )) - PASSED=$(( TOTAL - FAILED - SKIPPED )) - SLEEP_SECS=$(( (SEED % 45) + 15 )) - echo "total=$TOTAL" >> "$GITHUB_OUTPUT" - echo "passed=$PASSED" >> "$GITHUB_OUTPUT" - echo "failed=$FAILED" >> "$GITHUB_OUTPUT" - echo "skipped=$SKIPPED" >> "$GITHUB_OUTPUT" - echo "sleep=$SLEEP_SECS" >> "$GITHUB_OUTPUT" - echo "### Test Configuration" >> $GITHUB_STEP_SUMMARY - echo "| Param | Value |" >> $GITHUB_STEP_SUMMARY - echo "|-------|-------|" >> $GITHUB_STEP_SUMMARY - echo "| Total tests | $TOTAL |" >> $GITHUB_STEP_SUMMARY - echo "| Passed | $PASSED |" >> $GITHUB_STEP_SUMMARY - echo "| Failed | $FAILED |" >> $GITHUB_STEP_SUMMARY - echo "| Skipped | $SKIPPED |" >> $GITHUB_STEP_SUMMARY - echo "| Simulated duration | ${SLEEP_SECS}s |" >> $GITHUB_STEP_SUMMARY - - - name: Simulate build - run: | - echo "=== Building against PyTorch SHA: ${{ github.event.client_payload.payload.pull_request.head.sha }} ===" - echo "Installing dependencies..." - sleep $(( (RANDOM % 5) + 2 )) - echo "Compiling OOT extensions..." - sleep $(( (RANDOM % 5) + 2 )) - echo "Build complete." - - - name: Simulate test execution - id: tests - run: | - echo "=== Running ${{ steps.config.outputs.total }} tests ===" - - EDGE_ROLL=$(( RANDOM % 20 )) - if [ "$EDGE_ROLL" -eq 0 ]; then - echo "::notice::Simulating edge case: cancelled" - echo "outcome=cancelled" >> "$GITHUB_OUTPUT" - echo "" >> $GITHUB_STEP_SUMMARY - echo "### Test Results" >> $GITHUB_STEP_SUMMARY - echo "- **Outcome:** cancelled (simulated edge case)" >> $GITHUB_STEP_SUMMARY - elif [ "$EDGE_ROLL" -eq 1 ]; then - sleep ${{ steps.config.outputs.sleep }} - echo "::notice::Simulating edge case: timed_out" - echo "outcome=timed_out" >> "$GITHUB_OUTPUT" - echo "" >> $GITHUB_STEP_SUMMARY - echo "### Test Results" >> $GITHUB_STEP_SUMMARY - echo "- **Outcome:** timed_out (simulated edge case)" >> $GITHUB_STEP_SUMMARY - else - sleep ${{ steps.config.outputs.sleep }} - FAILED=${{ steps.config.outputs.failed }} - if [ "$FAILED" -gt 0 ]; then - echo "::warning::$FAILED test(s) failed" - for i in $(seq 1 $FAILED); do - echo "FAIL: test_op_$(( RANDOM % 999 )) (TestOOTBackend)" - done - echo "outcome=failure" >> "$GITHUB_OUTPUT" - else - echo "All tests passed." - echo "outcome=success" >> "$GITHUB_OUTPUT" - fi - echo "" >> $GITHUB_STEP_SUMMARY - echo "### Test Results" >> $GITHUB_STEP_SUMMARY - echo "- **Passed:** ${{ steps.config.outputs.passed }}" >> $GITHUB_STEP_SUMMARY - echo "- **Failed:** ${{ steps.config.outputs.failed }}" >> $GITHUB_STEP_SUMMARY - echo "- **Skipped:** ${{ steps.config.outputs.skipped }}" >> $GITHUB_STEP_SUMMARY - fi - - - name: Report CI completed - if: always() - uses: pytorch/test-infra/.github/actions/cross-repo-ci-relay-callback@main - with: - status: completed - conclusion: ${{ steps.tests.outputs.outcome || (job.status == 'success' && 'success' || 'failure') }} - test-results: >- - { - "passed": ${{ steps.config.outputs.passed || 0 }}, - "failed": ${{ steps.config.outputs.failed || 0 }}, - "skipped": ${{ steps.config.outputs.skipped || 0 }}, - "total": ${{ steps.config.outputs.total || 0 }} - } - artifact-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..fb39099 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +__pycache__/ +*.pyc +agent_space/ diff --git a/README.md b/README.md index 5998544..2fff1b4 100644 --- a/README.md +++ b/README.md @@ -145,6 +145,92 @@ Once Steps 1–3 are complete, your workflow will start receiving dispatches. Yo 2. Looking for runs triggered by `pytorch-fdn-cross-repo-ci-relay[bot]` 3. Inspecting the step summary for the full event payload +## Critical Tests + +This repository is a **downstream CRCR health probe**. It contains no CRCR implementation code — only workflows and validators that run when the relay sends a live `repository_dispatch` event. + +**There is no scheduled cron.** All CRCR health checks run in real time, triggered by upstream PyTorch activity (PR open/sync/close or push/tag events). Each run writes a structured `health-report.json` artifact you can use for metrics. + +| Workflow | Level | Trigger | What it verifies | +|----------|-------|---------|------------------| +| [`crcr-dispatch-receiver.yml`](.github/workflows/crcr-dispatch-receiver.yml) | L1 | live `repository_dispatch`: `pull_request`, `push` | Dispatch payload, checkout SHA, `delivery_id`; HUD callbacks on `opened`/`reopened` only | +| [`crcr-l2-ci.yml`](.github/workflows/crcr-l2-ci.yml) | L2 | live `repository_dispatch`: `pull_request` (`opened`/`reopened` only) | L1 checks + smoke + `in_progress`/`completed` callbacks, HUD metrics | +| [`crcr-unit-tests.yml`](.github/workflows/crcr-unit-tests.yml) | Offline | push/PR to this repo only | Validator unit tests against JSON fixtures (guards test code, not live CRCR) | + +### Event filtering (relay rate limits) + +The relay callback Lambda enforces a per-repo sliding-window rate limit (see [crcr-test#8](https://github.com/pytorch/crcr-test/issues/8)). A full L1+L2 probe sends **4 callbacks** (`in_progress` + `completed` x2). PyTorch `synchronize` events are high volume and can exceed the limit during busy periods. + +Health probes use **tiered coverage** by event type: + +| Event | L1 workflow | L2 workflow | Relay callbacks | HUD rows | +|-------|-------------|-------------|-----------------|----------| +| `opened`, `reopened` | Full probe + HUD | Full probe + HUD | **4** | **2** | +| `synchronize` | Light probe (validate + checkout) | Skipped | **0** | **0** | +| `closed` | Cancel job only | Cancel job only | **0** | **0** | +| `push` / ciflow tag | Light probe (validate + checkout) | N/A | **0** | **0** | + +**Light probe** (`l1-light`): validates dispatch payload, checks out PyTorch at the dispatched SHA, verifies the checkout, asserts `delivery_id`, uploads `health-report.json` — no relay callbacks. + +**Full probe** (`l1-critical` / `l2-critical`): light checks plus `in_progress`/`completed` callbacks and HUD `test_results`. Callback mechanics are the same regardless of PR `action`; limiting callbacks to `opened`/`reopened` keeps coverage on low-volume events while `synchronize` still exercises dispatch delivery every upstream push. + +### When tests run + +| Event | What runs | +|-------|-----------| +| PyTorch PR `opened` / `reopened` | L1 full + L2 full (HUD callbacks) | +| PyTorch PR `synchronize` | L1 light only (no callbacks, no L2) | +| PyTorch PR `closed` | L1/L2 cancel jobs only (build jobs skipped) | +| PyTorch push / ciflow tag | L1 light only (no callbacks) | +| Merge to crcr-test main | `crcr-unit-tests` only (offline contract tests) | + +### L1 integration points + +- `client_payload.event_type` is `pull_request` or `push` +- `client_payload.delivery_id` is present (required for L2 state tracking) +- `client_payload.payload.repository.full_name` is `pytorch/pytorch` +- PR events: `action` in `opened`/`reopened`/`synchronize`/`closed`, valid `pull_request.head.sha` +- Push events: `ref` starts with `refs/`, valid `after` SHA +- PyTorch checkout resolves to the dispatched SHA +- `closed` action runs only the cancel job (no build) +- **`opened`/`reopened`**: full probe with `in_progress`/`completed` callbacks to HUD +- **`synchronize`/`push`**: light probe only (payload + checkout; no callbacks) + +### L2 integration points + +- Runs only on PR `opened` and `reopened` (skipped on `synchronize` to limit callback volume) +- OIDC token minting (`id-token: write`) +- `in_progress` callback accepted by relay (state machine: `DISPATCHED → IN_PROGRESS`) +- Deterministic smoke checks (no random pass/fail) +- `completed` callback with `conclusion` and `test_results` derived from health checks +- Each health check maps to one HUD test count (`passed` / `failed` / `total`) +- `artifact_url` points at the GitHub Actions run page +- Results visible on [hud.pytorch.org/crcr/pytorch/crcr-test](https://hud.pytorch.org/crcr/pytorch/crcr-test) + +### HUD health metrics + +L1 and L2 workflows send health probe results to the callback lambda on the `completed` +callback. The relay forwards them to HUD as `workflow.test_results`: + +| Health check step outcome | HUD field | +|---------------------------|-----------| +| `success` | counts toward `passed_tests` | +| `failure` / `cancelled` | counts toward `failed_tests` | +| `skipped` | counts toward `skipped_tests` | + +`total_tests` is the number of probe checks. `conclusion` is `success` only when every +check succeeded. Per-check names live in the uploaded `health-report.json` artifact. + +### Running validators locally + +```bash +# Validate a fixture or saved client_payload JSON +python3 tests/validate_dispatch_payload.py < tests/fixtures/pull_request_opened.json + +# Run all unit tests +python3 -m unittest discover -s tests -p 'test_*.py' -v +``` + ## Dispatch Payload Structure The relay wraps the original GitHub event inside `client_payload`: diff --git a/scripts/health_checks.sh b/scripts/health_checks.sh new file mode 100755 index 0000000..a8a74b7 --- /dev/null +++ b/scripts/health_checks.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# Emit --check name=outcome pairs (one per line) for write_health_report.py. +# Usage: mapfile -t ARGS < <(scripts/health_checks.sh SPEC) +# python3 tests/write_health_report.py "${ARGS[@]}" ... + +set -euo pipefail + +spec="${1:?spec required (l1-light, l1-full, l2-full)}" + +emit_check() { + local name="$1" + local var="$2" + echo --check + echo "${name}=${!var:-skipped}" +} + +case "$spec" in + l1-light) + emit_check validate_payload VALIDATE_PAYLOAD + emit_check checkout_pytorch CHECKOUT_PYTORCH + emit_check verify_checkout VERIFY_CHECKOUT + emit_check delivery_id DELIVERY_ID + ;; + l1-full) + emit_check validate_payload VALIDATE_PAYLOAD + emit_check in_progress_callback IN_PROGRESS_CALLBACK + emit_check checkout_pytorch CHECKOUT_PYTORCH + emit_check verify_checkout VERIFY_CHECKOUT + emit_check delivery_id DELIVERY_ID + ;; + l2-full) + emit_check validate_payload VALIDATE_PAYLOAD + emit_check in_progress_callback IN_PROGRESS_CALLBACK + emit_check checkout_pytorch CHECKOUT_PYTORCH + emit_check verify_checkout VERIFY_CHECKOUT + emit_check smoke_checks SMOKE_CHECKS + ;; + *) + echo "unknown health check spec: $spec" >&2 + exit 2 + ;; +esac + +if [ "${INCLUDE_COMPLETED_CALLBACK:-}" = "1" ]; then + emit_check completed_callback COMPLETED_CALLBACK +fi diff --git a/scripts/run-local-tests.sh b/scripts/run-local-tests.sh new file mode 100755 index 0000000..b02b31d --- /dev/null +++ b/scripts/run-local-tests.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# Run offline CRCR validator tests (no relay required). +# Live L1/L2 probe workflows: crcr-dispatch-receiver.yml, crcr-l2-ci.yml +set -euo pipefail +cd "$(dirname "$0")/.." + +echo "=== Unit tests ===" +python3 -m unittest discover -s tests -p 'test_*.py' -v + +echo "" +echo "=== Fixture validation ===" +for f in tests/fixtures/*.json; do + echo "Validating $f" + python3 tests/validate_dispatch_payload.py < "$f" +done + +echo "" +echo "All local tests passed." diff --git a/scripts/trigger-test-dispatch.sh b/scripts/trigger-test-dispatch.sh new file mode 100755 index 0000000..33f0e69 --- /dev/null +++ b/scripts/trigger-test-dispatch.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash +# Send a test repository_dispatch to exercise CRCR probe workflows. +# Requires: gh CLI, workflows merged to the target repo default branch (main). +# +# Workflows exercised: +# pull_request/opened -> crcr-dispatch-receiver.yml (L1 full) + crcr-l2-ci.yml (L2 full) +# push -> crcr-dispatch-receiver.yml (L1 light) only +# +# Usage: +# ./scripts/trigger-test-dispatch.sh # L1+L2 (pull_request/opened) +# ./scripts/trigger-test-dispatch.sh push # L1 light only (push event) +# +# Target repo: current gh repo, or pytorch/crcr-test. Override with CRCR_TEST_REPO. +set -euo pipefail +cd "$(dirname "$0")/.." + +REPO="${CRCR_TEST_REPO:-$(gh repo view --json nameWithOwner -q .nameWithOwner 2>/dev/null || echo pytorch/crcr-test)}" +MODE="${1:-pr}" + +echo "Fetching latest pytorch/pytorch main SHA..." +PYTORCH_SHA="$(git ls-remote https://github.com/pytorch/pytorch.git HEAD | awk '{print $1}')" +if [[ -z "${PYTORCH_SHA}" ]]; then + echo "error: could not resolve pytorch main SHA" >&2 + exit 1 +fi +echo "Using SHA: ${PYTORCH_SHA}" + +DELIVERY_ID="local-test-$(date -u +%Y%m%dT%H%M%SZ)-$$" + +if [[ "${MODE}" == "push" ]]; then + EVENT_TYPE="push" + PAYLOAD=$(jq -n \ + --arg sha "${PYTORCH_SHA}" \ + --arg did "${DELIVERY_ID}" \ + '{ + event_type: "push", + client_payload: { + event_type: "push", + delivery_id: $did, + payload: { + ref: "refs/heads/main", + after: $sha, + deleted: false, + base_ref: "refs/heads/main", + repository: { full_name: "pytorch/pytorch" } + } + } + }') +else + EVENT_TYPE="pull_request" + PAYLOAD=$(jq -n \ + --arg sha "${PYTORCH_SHA}" \ + --arg did "${DELIVERY_ID}" \ + '{ + event_type: "pull_request", + client_payload: { + event_type: "pull_request", + delivery_id: $did, + payload: { + action: "opened", + repository: { full_name: "pytorch/pytorch" }, + pull_request: { + number: 999999, + title: "CRCR local dispatch test", + head: { + sha: $sha, + repo: { full_name: "pytorch/pytorch" } + } + } + } + } + }') +fi + +echo "Dispatching ${EVENT_TYPE} to ${REPO} (delivery_id=${DELIVERY_ID})..." +echo "${PAYLOAD}" | gh api --method POST "repos/${REPO}/dispatches" --input - + +echo "" +echo "Done. Check workflow runs at:" +echo " https://github.com/${REPO}/actions" +if [[ "${MODE}" == "push" ]]; then + echo "Expected workflows: crcr-dispatch-receiver.yml (L1 light)" +else + echo "Expected workflows: crcr-dispatch-receiver.yml (L1 full), crcr-l2-ci.yml (L2 full)" +fi +echo "" +echo "Note: callbacks require the target repo on the CRCR allowlist and a relay dispatch record." +echo "Synthetic delivery_id dispatches exercise L1/L2 workflow steps; callbacks may fail locally." diff --git a/tests/fixtures/pull_request_closed.json b/tests/fixtures/pull_request_closed.json new file mode 100644 index 0000000..d9e027d --- /dev/null +++ b/tests/fixtures/pull_request_closed.json @@ -0,0 +1,19 @@ +{ + "event_type": "pull_request", + "delivery_id": "test-delivery-pr-closed-001", + "payload": { + "action": "closed", + "repository": { + "full_name": "pytorch/pytorch" + }, + "pull_request": { + "number": 12345, + "head": { + "sha": "cccccccccccccccccccccccccccccccccccccccc", + "repo": { + "full_name": "contributor/pytorch" + } + } + } + } +} diff --git a/tests/fixtures/pull_request_opened.json b/tests/fixtures/pull_request_opened.json new file mode 100644 index 0000000..18efac7 --- /dev/null +++ b/tests/fixtures/pull_request_opened.json @@ -0,0 +1,20 @@ +{ + "event_type": "pull_request", + "delivery_id": "test-delivery-pr-opened-001", + "payload": { + "action": "opened", + "repository": { + "full_name": "pytorch/pytorch" + }, + "pull_request": { + "number": 12345, + "title": "Test PR", + "head": { + "sha": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "repo": { + "full_name": "contributor/pytorch" + } + } + } + } +} diff --git a/tests/fixtures/pull_request_synchronize.json b/tests/fixtures/pull_request_synchronize.json new file mode 100644 index 0000000..16a623c --- /dev/null +++ b/tests/fixtures/pull_request_synchronize.json @@ -0,0 +1,19 @@ +{ + "event_type": "pull_request", + "delivery_id": "test-delivery-pr-sync-001", + "payload": { + "action": "synchronize", + "repository": { + "full_name": "pytorch/pytorch" + }, + "pull_request": { + "number": 12345, + "head": { + "sha": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "repo": { + "full_name": "contributor/pytorch" + } + } + } + } +} diff --git a/tests/fixtures/push_ciflow_tag.json b/tests/fixtures/push_ciflow_tag.json new file mode 100644 index 0000000..230f84c --- /dev/null +++ b/tests/fixtures/push_ciflow_tag.json @@ -0,0 +1,13 @@ +{ + "event_type": "push", + "delivery_id": "test-delivery-push-001", + "payload": { + "ref": "refs/tags/ciflow/trunk/12345", + "after": "dddddddddddddddddddddddddddddddddddddddd", + "deleted": false, + "base_ref": "refs/heads/main", + "repository": { + "full_name": "pytorch/pytorch" + } + } +} diff --git a/tests/test_validators.py b/tests/test_validators.py new file mode 100644 index 0000000..a156d73 --- /dev/null +++ b/tests/test_validators.py @@ -0,0 +1,281 @@ +#!/usr/bin/env python3 +"""Unit tests for CRCR payload validators (runs locally without relay).""" + +from __future__ import annotations + +import json +import tempfile +import unittest +from pathlib import Path + +from validate_callback_payload import validate_callback_body +from validate_dispatch_payload import ValidationError, validate_dispatch_payload +from write_health_report import build_report, checks_to_test_results + +FIXTURES = Path(__file__).parent / "fixtures" + + +def _load(name: str) -> dict: + return json.loads((FIXTURES / name).read_text()) + + +class TestDispatchPayload(unittest.TestCase): + def test_pull_request_opened_fixture(self) -> None: + validate_dispatch_payload(_load("pull_request_opened.json")) + + def test_pull_request_synchronize_fixture(self) -> None: + validate_dispatch_payload(_load("pull_request_synchronize.json")) + + def test_pull_request_closed_fixture(self) -> None: + validate_dispatch_payload(_load("pull_request_closed.json")) + + def test_push_ciflow_tag_fixture(self) -> None: + validate_dispatch_payload(_load("push_ciflow_tag.json")) + + def test_rejects_missing_delivery_id(self) -> None: + payload = _load("pull_request_opened.json") + del payload["delivery_id"] + with self.assertRaises(ValidationError): + validate_dispatch_payload(payload) + + def test_rejects_wrong_upstream_repo(self) -> None: + payload = _load("pull_request_opened.json") + payload["payload"]["repository"]["full_name"] = "other/repo" + with self.assertRaises(ValidationError): + validate_dispatch_payload(payload) + + def test_rejects_invalid_sha(self) -> None: + payload = _load("pull_request_opened.json") + payload["payload"]["pull_request"]["head"]["sha"] = "not-a-sha" + with self.assertRaises(ValidationError): + validate_dispatch_payload(payload) + + +class TestCallbackPayload(unittest.TestCase): + def _completed_body(self) -> dict: + body = _load("pull_request_opened.json") + body["workflow"] = { + "schema_version": "1", + "status": "completed", + "conclusion": "success", + "name": "CRCR L2 Critical", + "url": "https://github.com/pytorch/crcr-test/actions/runs/1", + "run_attempt": "1", + "job_name": "l2-critical", + "check_run_id": "12345", + "run_id": "1", + "started_at": "2026-06-29T00:00:00Z", + "completed_at": "2026-06-29T00:01:00Z", + "test_results": {"passed": 3, "failed": 0, "skipped": 0}, + "artifact_url": "https://example.com/artifacts/1", + } + return body + + def test_in_progress_shape(self) -> None: + body = _load("pull_request_opened.json") + body["workflow"] = { + "schema_version": "1", + "status": "in_progress", + "conclusion": None, + "name": "CRCR L2 Critical", + "url": "https://github.com/pytorch/crcr-test/actions/runs/1", + "run_attempt": "1", + "job_name": "l2-critical", + "check_run_id": "12345", + "run_id": "1", + "started_at": "2026-06-29T00:00:00Z", + "completed_at": None, + } + validate_callback_body(body, expect_status="in_progress") + + def test_completed_shape(self) -> None: + validate_callback_body(self._completed_body(), expect_status="completed") + + def test_completed_requires_conclusion(self) -> None: + body = self._completed_body() + body["workflow"]["conclusion"] = None + with self.assertRaises(ValidationError): + validate_callback_body(body, expect_status="completed") + + +class TestPytorchCheckout(unittest.TestCase): + def test_checkout_matches_dispatch_sha(self) -> None: + import subprocess + + with tempfile.TemporaryDirectory() as tmp: + tmp_path = Path(tmp) + subprocess.run(["git", "init", tmp], check=True, capture_output=True) + subprocess.run( + ["git", "-C", tmp, "commit", "--allow-empty", "-m", "init"], + check=True, + capture_output=True, + env={ + **__import__("os").environ, + "GIT_AUTHOR_NAME": "test", + "GIT_AUTHOR_EMAIL": "test@test.com", + "GIT_COMMITTER_NAME": "test", + "GIT_COMMITTER_EMAIL": "test@test.com", + }, + ) + head = subprocess.run( + ["git", "-C", tmp, "rev-parse", "HEAD"], + check=True, + capture_output=True, + text=True, + ) + matching_sha = head.stdout.strip() + + payload = _load("pull_request_opened.json") + payload["payload"]["pull_request"]["head"]["sha"] = matching_sha + payload_path = tmp_path / "payload.json" + payload_path.write_text(json.dumps(payload)) + + result = subprocess.run( + [ + "python3", + str(Path(__file__).parent / "validate_pytorch_checkout.py"), + "--payload", + str(payload_path), + "--pytorch-dir", + tmp, + ], + capture_output=True, + text=True, + ) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_checkout_mismatch_dispatch_sha(self) -> None: + import subprocess + + with tempfile.TemporaryDirectory() as tmp: + subprocess.run(["git", "init", tmp], check=True, capture_output=True) + subprocess.run( + ["git", "-C", tmp, "commit", "--allow-empty", "-m", "init"], + check=True, + capture_output=True, + env={ + **__import__("os").environ, + "GIT_AUTHOR_NAME": "test", + "GIT_AUTHOR_EMAIL": "test@test.com", + "GIT_COMMITTER_NAME": "test", + "GIT_COMMITTER_EMAIL": "test@test.com", + }, + ) + result = subprocess.run( + [ + "python3", + str(Path(__file__).parent / "validate_pytorch_checkout.py"), + "--payload", + str(FIXTURES / "pull_request_opened.json"), + "--pytorch-dir", + tmp, + ], + capture_output=True, + text=True, + ) + self.assertNotEqual(result.returncode, 0) + self.assertIn("checkout SHA mismatch", result.stderr) + + +class TestHealthReport(unittest.TestCase): + def test_checks_to_test_results(self) -> None: + results = checks_to_test_results( + { + "a": "success", + "b": "success", + "c": "failure", + "d": "skipped", + } + ) + self.assertEqual(results, {"passed": 2, "failed": 1, "skipped": 1, "total": 4}) + + def test_build_report_includes_hud_fields(self) -> None: + report = build_report( + probe="l2-critical", + checks={"validate_payload": "success", "in_progress_callback": "success"}, + delivery_id="d-1", + run_id="99", + event_type="pull_request", + pr_number="123", + ) + self.assertTrue(report["healthy"]) + self.assertEqual(report["conclusion"], "success") + self.assertEqual(report["test_results"], {"passed": 2, "failed": 0, "skipped": 0, "total": 2}) + + def test_healthy_report(self) -> None: + import subprocess + + with tempfile.TemporaryDirectory() as tmp: + out = Path(tmp) / "report.json" + result = subprocess.run( + [ + "python3", + str(Path(__file__).parent / "write_health_report.py"), + "--probe", + "l1-critical", + "--output", + str(out), + "--check", + "validate_payload=success", + "--check", + "verify_checkout=success", + ], + capture_output=True, + text=True, + ) + self.assertEqual(result.returncode, 0) + report = json.loads(out.read_text()) + self.assertTrue(report["healthy"]) + self.assertEqual(report["trigger"], "repository_dispatch") + + def test_unhealthy_report_without_strict(self) -> None: + import subprocess + + with tempfile.TemporaryDirectory() as tmp: + out = Path(tmp) / "report.json" + result = subprocess.run( + [ + "python3", + str(Path(__file__).parent / "write_health_report.py"), + "--probe", + "l2-critical", + "--output", + str(out), + "--check", + "in_progress_callback=failure", + ], + capture_output=True, + text=True, + ) + self.assertEqual(result.returncode, 0) + report = json.loads(out.read_text()) + self.assertFalse(report["healthy"]) + self.assertEqual(report["test_results"]["failed"], 1) + + def test_unhealthy_report_strict(self) -> None: + import subprocess + + with tempfile.TemporaryDirectory() as tmp: + out = Path(tmp) / "report.json" + result = subprocess.run( + [ + "python3", + str(Path(__file__).parent / "write_health_report.py"), + "--probe", + "l2-critical", + "--output", + str(out), + "--check", + "in_progress_callback=failure", + "--strict", + ], + capture_output=True, + text=True, + ) + self.assertNotEqual(result.returncode, 0) + report = json.loads(out.read_text()) + self.assertFalse(report["healthy"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/validate_callback_payload.py b/tests/validate_callback_payload.py new file mode 100644 index 0000000..7632af6 --- /dev/null +++ b/tests/validate_callback_payload.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +"""Validate CRCR L2 callback body shape (downstream -> relay wire format).""" + +from __future__ import annotations + +import json +import sys +from typing import Any + +from validate_dispatch_payload import ValidationError, validate_dispatch_payload + +ALLOWED_STATUSES = frozenset({"in_progress", "completed"}) +ALLOWED_CONCLUSIONS = frozenset({"success", "failure", "cancelled", "timed_out"}) + + +def validate_callback_body(body: dict[str, Any], *, expect_status: str) -> None: + validate_dispatch_payload(body) + + workflow = body.get("workflow") + if not isinstance(workflow, dict): + raise ValidationError("workflow must be an object") + + status = workflow.get("status") + if status != expect_status: + raise ValidationError(f"workflow.status must be {expect_status!r}, got {status!r}") + + for field in ("schema_version", "name", "url", "job_name", "check_run_id", "run_id", "run_attempt"): + if field not in workflow: + raise ValidationError(f"workflow.{field} is required for L2 callbacks") + + if expect_status == "in_progress": + if workflow.get("conclusion") is not None: + raise ValidationError("in_progress callback must not set workflow.conclusion") + if not workflow.get("started_at"): + raise ValidationError("in_progress callback must set workflow.started_at") + else: + conclusion = workflow.get("conclusion") + if conclusion not in ALLOWED_CONCLUSIONS: + raise ValidationError( + f"completed callback conclusion must be one of {sorted(ALLOWED_CONCLUSIONS)}, " + f"got {conclusion!r}" + ) + if not workflow.get("completed_at"): + raise ValidationError("completed callback must set workflow.completed_at") + + +def main() -> int: + if len(sys.argv) != 3: + print("usage: validate_callback_payload.py ", file=sys.stderr) + return 2 + + expect_status = sys.argv[1] + if expect_status not in ALLOWED_STATUSES: + print("::error::status arg must be in_progress or completed", file=sys.stderr) + return 2 + + try: + body = json.loads(open(sys.argv[2]).read()) + validate_callback_body(body, expect_status=expect_status) + except (ValidationError, json.JSONDecodeError, OSError) as exc: + print(f"::error::L2 callback validation failed: {exc}", file=sys.stderr) + return 1 + + print(f"L2 {expect_status} callback validation passed") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/validate_dispatch_payload.py b/tests/validate_dispatch_payload.py new file mode 100644 index 0000000..b8ea089 --- /dev/null +++ b/tests/validate_dispatch_payload.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 +"""Validate CRCR repository_dispatch client_payload structure (L1 integration).""" + +from __future__ import annotations + +import json +import re +import sys +from typing import Any + +UPSTREAM_REPO = "pytorch/pytorch" +ALLOWED_EVENT_TYPES = frozenset({"pull_request", "push"}) +ALLOWED_PR_ACTIONS = frozenset({"opened", "reopened", "synchronize", "closed"}) +SHA_RE = re.compile(r"^[0-9a-f]{40}$") + + +class ValidationError(Exception): + pass + + +def _require(obj: dict[str, Any], key: str, label: str) -> Any: + if key not in obj: + raise ValidationError(f"missing {label}.{key}") + return obj[key] + + +def _require_sha(value: str, label: str) -> None: + if not SHA_RE.match(value): + raise ValidationError(f"{label} is not a 40-char hex SHA: {value!r}") + + +def validate_dispatch_payload(client_payload: dict[str, Any]) -> None: + event_type = _require(client_payload, "event_type", "client_payload") + if event_type not in ALLOWED_EVENT_TYPES: + raise ValidationError( + f"client_payload.event_type must be one of {sorted(ALLOWED_EVENT_TYPES)}, " + f"got {event_type!r}" + ) + + delivery_id = _require(client_payload, "delivery_id", "client_payload") + if not isinstance(delivery_id, str) or not delivery_id.strip(): + raise ValidationError("client_payload.delivery_id must be a non-empty string") + + payload = _require(client_payload, "payload", "client_payload") + if not isinstance(payload, dict): + raise ValidationError("client_payload.payload must be an object") + + repo = _require(payload, "repository", "client_payload.payload") + if not isinstance(repo, dict): + raise ValidationError("client_payload.payload.repository must be an object") + full_name = _require(repo, "full_name", "client_payload.payload.repository") + if full_name != UPSTREAM_REPO: + raise ValidationError( + f"upstream repo must be {UPSTREAM_REPO!r}, got {full_name!r}" + ) + + if event_type == "pull_request": + action = _require(payload, "action", "client_payload.payload") + if action not in ALLOWED_PR_ACTIONS: + raise ValidationError( + f"pull_request action must be one of {sorted(ALLOWED_PR_ACTIONS)}, " + f"got {action!r}" + ) + pr = _require(payload, "pull_request", "client_payload.payload") + if not isinstance(pr, dict): + raise ValidationError("client_payload.payload.pull_request must be an object") + number = _require(pr, "number", "client_payload.payload.pull_request") + if not isinstance(number, int) or number <= 0: + raise ValidationError(f"pull_request.number must be a positive int, got {number!r}") + head = _require(pr, "head", "client_payload.payload.pull_request") + if not isinstance(head, dict): + raise ValidationError("pull_request.head must be an object") + head_sha = _require(head, "sha", "pull_request.head") + _require_sha(head_sha, "pull_request.head.sha") + head_repo = _require(head, "repo", "pull_request.head") + if not isinstance(head_repo, dict): + raise ValidationError("pull_request.head.repo must be an object") + _require(head_repo, "full_name", "pull_request.head.repo") + + elif event_type == "push": + ref = _require(payload, "ref", "client_payload.payload") + if not isinstance(ref, str) or not ref.startswith("refs/"): + raise ValidationError(f"push ref must start with refs/, got {ref!r}") + deleted = payload.get("deleted", False) + if not deleted: + after = _require(payload, "after", "client_payload.payload") + _require_sha(after, "payload.after") + + +def main() -> int: + raw = sys.stdin.read() + try: + data = json.loads(raw) + except json.JSONDecodeError as exc: + print(f"::error::client_payload is not valid JSON: {exc}", file=sys.stderr) + return 1 + + try: + validate_dispatch_payload(data) + except ValidationError as exc: + print(f"::error::L1 dispatch payload validation failed: {exc}", file=sys.stderr) + return 1 + + print("L1 dispatch payload validation passed") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/validate_pytorch_checkout.py b/tests/validate_pytorch_checkout.py new file mode 100644 index 0000000..e7fec74 --- /dev/null +++ b/tests/validate_pytorch_checkout.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +"""Verify PyTorch checkout matches the SHA from a CRCR dispatch payload.""" + +from __future__ import annotations + +import argparse +import json +import subprocess +import sys +from pathlib import Path + +from validate_dispatch_payload import ValidationError, validate_dispatch_payload + + +def _head_sha(client_payload: dict) -> str: + event_type = client_payload["event_type"] + payload = client_payload["payload"] + if event_type == "pull_request": + return payload["pull_request"]["head"]["sha"] + return payload["after"] + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--payload", required=True, help="Path to client_payload JSON file") + parser.add_argument("--pytorch-dir", required=True, help="Path to checked-out pytorch tree") + args = parser.parse_args() + + client_payload = json.loads(Path(args.payload).read_text()) + try: + validate_dispatch_payload(client_payload) + except ValidationError as exc: + print(f"::error::{exc}", file=sys.stderr) + return 1 + + expected = _head_sha(client_payload) + pytorch_dir = Path(args.pytorch_dir) + if not pytorch_dir.is_dir(): + print(f"::error::pytorch dir does not exist: {pytorch_dir}", file=sys.stderr) + return 1 + + result = subprocess.run( + ["git", "-C", str(pytorch_dir), "rev-parse", "HEAD"], + check=False, + capture_output=True, + text=True, + ) + if result.returncode != 0: + print(f"::error::git rev-parse failed: {result.stderr.strip()}", file=sys.stderr) + return 1 + + actual = result.stdout.strip() + if actual != expected: + print( + f"::error::checkout SHA mismatch: expected {expected}, got {actual}", + file=sys.stderr, + ) + return 1 + + print(f"PyTorch checkout matches dispatch SHA {expected}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/write_health_report.py b/tests/write_health_report.py new file mode 100644 index 0000000..317bbaa --- /dev/null +++ b/tests/write_health_report.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 +"""Write CRCR health probe results and map them to HUD test_results.""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +from datetime import datetime, timezone +from pathlib import Path + + +def parse_checks(items: list[str]) -> dict[str, str]: + checks: dict[str, str] = {} + for item in items: + if "=" not in item: + raise ValueError(f"invalid --check {item!r}, expected name=outcome") + name, outcome = item.split("=", 1) + checks[name] = outcome + return checks + + +def checks_to_test_results(checks: dict[str, str]) -> dict[str, int]: + passed = sum(1 for outcome in checks.values() if outcome == "success") + skipped = sum(1 for outcome in checks.values() if outcome == "skipped") + failed = len(checks) - passed - skipped + return { + "passed": passed, + "failed": failed, + "skipped": skipped, + "total": len(checks), + } + + +def build_report( + *, + probe: str, + checks: dict[str, str], + delivery_id: str = "", + run_id: str = "", + event_type: str = "", + pr_number: str = "", +) -> dict: + healthy = bool(checks) and all(outcome == "success" for outcome in checks.values()) + test_results = checks_to_test_results(checks) + return { + "probe": probe, + "trigger": "repository_dispatch", + "timestamp": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"), + "delivery_id": delivery_id, + "run_id": run_id, + "event_type": event_type, + "pr_number": int(pr_number) if pr_number.isdigit() else None, + "checks": checks, + "healthy": healthy, + "conclusion": "success" if healthy else "failure", + "test_results": test_results, + "hud_url": "https://hud.pytorch.org/crcr/pytorch/crcr-test", + } + + +def append_github_output(report: dict, github_output: str) -> None: + with open(github_output, "a", encoding="utf-8") as handle: + handle.write(f"healthy={str(report['healthy']).lower()}\n") + handle.write(f"conclusion={report['conclusion']}\n") + handle.write("test_results< int: + parser = argparse.ArgumentParser() + parser.add_argument("--probe", required=True, help="Probe name, e.g. l1-critical") + parser.add_argument( + "--output", + default="", + help="Path to write JSON report (optional when only using --github-output)", + ) + parser.add_argument("--delivery-id", default="") + parser.add_argument("--run-id", default="") + parser.add_argument("--event-type", default="") + parser.add_argument("--pr-number", default="") + parser.add_argument( + "--check", + action="append", + default=[], + help="Check outcome as name=outcome (repeatable)", + ) + parser.add_argument( + "--github-output", + default="", + help="If set, append healthy/conclusion/test_results to this GITHUB_OUTPUT path", + ) + parser.add_argument( + "--strict", + action="store_true", + default=False, + help="Exit 1 when the probe is unhealthy (default: exit 0 after writing report)", + ) + args = parser.parse_args() + + try: + checks = parse_checks(args.check) + except ValueError as exc: + print(f"::error::{exc}", file=sys.stderr) + return 2 + + report = build_report( + probe=args.probe, + checks=checks, + delivery_id=args.delivery_id, + run_id=args.run_id, + event_type=args.event_type, + pr_number=args.pr_number, + ) + + if args.output: + output = Path(args.output) + output.write_text(json.dumps(report, indent=2) + "\n") + print(json.dumps(report)) + + if args.github_output: + append_github_output(report, args.github_output) + + if args.strict and not report["healthy"]: + failed = [name for name, outcome in checks.items() if outcome != "success"] + print(f"::error::CRCR health probe unhealthy: {', '.join(failed)}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main())