From dca874b3bcf310a71d9cb773927d185d689be11c Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 26 Aug 2026 14:15:24 +0900 Subject: [PATCH 01/19] fix(scheduler): resolve live refs before cancelling runs --- .../workflows/pr-review-merge-scheduler.yml | 45 ++++++++++++------- CHANGELOG.md | 4 ++ docs/doctoring/queue-hygiene-live-ref-race.md | 37 +++++++++++++++ docs/product-technical-gap-baseline.md | 1 + .../test_required_workflow_queue_contract.py | 4 ++ 5 files changed, 75 insertions(+), 16 deletions(-) create mode 100644 docs/doctoring/queue-hygiene-live-ref-race.md diff --git a/.github/workflows/pr-review-merge-scheduler.yml b/.github/workflows/pr-review-merge-scheduler.yml index a9bb54f8a1..881165240a 100644 --- a/.github/workflows/pr-review-merge-scheduler.yml +++ b/.github/workflows/pr-review-merge-scheduler.yml @@ -1068,29 +1068,42 @@ jobs: # Compare live refs on every sweep instead of waiting for an age # threshold: previous-head checks are never useful merge evidence. queue_hygiene_ready=true - if ! open_pr_heads_json="$( + open_pr_heads_json="{}" + if open_pr_refs_tsv="$( gh api \ -H "Accept: application/vnd.github+json" \ "/repos/${repo_full_name}/pulls?state=open&per_page=100" \ --paginate \ - | jq -sc ' - add - | map( - select( - .head.repo.full_name != null and - .head.ref != null and - .head.sha != null - ) - | { - key: "\(.head.repo.full_name):\(.head.ref)", - value: .head.sha - } - ) - | from_entries + | jq -sr ' + add[] + | select(.head.repo.full_name != null and .head.ref != null) + | [.head.repo.full_name, .head.ref] + | @tsv ' )"; then + while IFS=$'\t' read -r head_repo head_ref; do + [ -n "$head_repo" ] && [ -n "$head_ref" ] || continue + encoded_head_ref="$(jq -rn --arg value "$head_ref" '$value | @uri')" + if ! live_head_sha="$( + gh api \ + -H "Accept: application/vnd.github+json" \ + "/repos/${head_repo}/git/ref/heads/${encoded_head_ref}" \ + --jq '.object.sha // empty' + )" || ! [[ "$live_head_sha" =~ ^[0-9a-fA-F]{40}$ ]]; then + echo "::warning::Current-HEAD cancellation skipped for ${repo_full_name}: live ref ${head_repo}:${head_ref} could not be resolved safely. No run will be cancelled from incomplete evidence." + queue_hygiene_ready=false + break + fi + open_pr_heads_json="$( + jq \ + --arg key "${head_repo}:${head_ref}" \ + --arg value "$live_head_sha" \ + '. + {($key): $value}' \ + <<<"$open_pr_heads_json" + )" + done <<<"$open_pr_refs_tsv" + else echo "::warning::Current-HEAD cancellation skipped for ${repo_full_name}: open PR head refs could not be read safely. No run will be cancelled from incomplete evidence." - open_pr_heads_json="{}" queue_hygiene_ready=false fi if ! current_default_sha="$( diff --git a/CHANGELOG.md b/CHANGELOG.md index cef0acda6b..80eb228736 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ this file. The format follows Keep a Changelog, and versioned releases follow Semantic Versioning where the repository publishes a release. ## [Unreleased] +- Resolve each open PR head through its live Git reference before queue hygiene + cancels superseded runs, and fail closed when any ref cannot be read. This + prevents a briefly stale pull-request payload from cancelling current-head + Checks without adding an arbitrary grace period. - Route Strix cross-provider fallbacks to explicit direct-OpenAI models (`openai-direct/...`) through the OpenAI inference endpoint instead of inheriting a provider-specific primary base: the workflow now provisions diff --git a/docs/doctoring/queue-hygiene-live-ref-race.md b/docs/doctoring/queue-hygiene-live-ref-race.md new file mode 100644 index 0000000000..775ca39d83 --- /dev/null +++ b/docs/doctoring/queue-hygiene-live-ref-race.md @@ -0,0 +1,37 @@ +# Queue hygiene live-reference race + +## Incident + +On 2026-08-26, LineageWeave PR #667 received a new same-repository head +`37cc9ab1163f213105d420618e2e8ee69ec6673d`. Its new pull-request workflows +started, but the organization queue sweep cancelled them while GitHub's open-PR +payload still exposed the preceding head. The runs were current for the branch +ref and stale only in the pull-request listing used by the cancellation map. + +This was a control-plane defect, not a test failure. Re-running the jobs without +repairing the comparison source would leave the same race available to every +repository in the organization. + +## Decision + +Queue hygiene still enumerates open pull requests to identify eligible head +repositories and branch names. Before cancelling anything, it now resolves each +head through GitHub's `Get a reference` endpoint and compares active runs with +that live Git reference. A missing, inaccessible, or malformed ref makes the +repository's cancellation pass unavailable; no run is cancelled from partial +evidence. + +No time delay or grace-period heuristic is used. A branch ref is the exact +commit pointer the check run is meant to validate. The existing rule remains: +previous-head runs may be cancelled, current-head runs may not. + +## Verification + +- `uv run --group dev pytest -q tests/test_required_workflow_queue_contract.py` +- `actionlint .github/workflows/pr-review-merge-scheduler.yml` +- `git diff --check` + +## Reference + +GitHub. (n.d.). *REST API endpoints for Git references*. GitHub Docs. Retrieved +August 26, 2026, from https://docs.github.com/en/rest/git/refs diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 2522490959..7f797d9e6a 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -88,6 +88,7 @@ flowchart LR | G-12 | CSAP/SOC 2 통제 목표와 PII masking 대안은 doctoring에 흩어져 있으며 evidence-to-control mapping의 live completeness가 미확인이다 | PII를 마스킹하면 업무가 멈추고, 원문 접근을 허용하면 감사·유출 위험이 커진다 | consent/purpose/access lease, field-level encryption/tokenization, redaction-at-egress, audit/revocation와 CSAP/SOC 2 evidence map을 구현한다 | | G-13 | hourly scheduler는 존재하지만 no-op/credential unavailable/queued Checks의 customer next action을 모든 caller가 동일한 receipt로 내는지 미확인이다 | 자동화가 실패해도 운영자가 무엇을 고쳐야 하는지 알 수 없다 | `skipped_credential_unavailable` receipt와 다음 행동 문구를 exact-head Checks로 검증한 뒤 병합하고, bounded receipt schema, retry floor, single-flight, no secret fallback을 모든 caller contract test로 고정한다 | | G-14 | release/changelog/version 증거가 각 PR에 분산되고 현재 central repo 보호 main의 release candidate가 명확하지 않다 | 운영자는 어떤 기능이 supportable release인지 확인할 수 없다 | merge 후 release readiness ledger, CHANGELOG, semantic version/tag, rollback/operability evidence를 함께 갱신한다 | +| G-15 | 조직 큐 정리기가 open-PR payload의 직전 head와 새 branch ref가 잠시 달랐던 LineageWeave #667의 current-head runs를 취소했다 | 정상 Checks가 코드 실패처럼 사라지고 모든 소비 저장소의 병합 루프가 반복 재실행에 갇힌다 | open PR의 repo/ref는 탐색에만 쓰고 취소 판정 SHA는 live Git reference에서 다시 읽는다. ref를 완전하게 읽지 못하면 해당 저장소에서는 아무 run도 취소하지 않는다 | ## 4. 열린 PR live inventory diff --git a/tests/test_required_workflow_queue_contract.py b/tests/test_required_workflow_queue_contract.py index 1d79f1daa7..67c2fe69bd 100644 --- a/tests/test_required_workflow_queue_contract.py +++ b/tests/test_required_workflow_queue_contract.py @@ -748,6 +748,10 @@ def test_org_queue_sweep_covers_target_repositories_on_a_heartbeat() -> None: assert "ORG_SWEEP_STALE_QUEUE_HOURS" in workflow assert "/actions/runs?status=${active_status}&per_page=100" in workflow assert "for active_status in queued in_progress" in workflow + assert '"/repos/${head_repo}/git/ref/heads/${encoded_head_ref}"' in workflow + assert "--jq '.object.sha // empty'" in workflow + assert 'open_pr_heads_json="$(\n jq' in workflow + assert "live ref ${head_repo}:${head_ref} could not be resolved safely" in workflow assert '"pull_request" or .event == "pull_request_target"' in workflow assert "$current_pr_head == null or .head_sha != $current_pr_head" in workflow assert ".head_sha != $current_default_sha" in workflow From 7348cf6f5262209df63c5cc76d16597064adb025 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 26 Aug 2026 14:24:33 +0900 Subject: [PATCH 02/19] fix(actions): fail closed on malformed PR heads --- .github/workflows/pr-review-merge-scheduler.yml | 14 ++++++++++---- tests/test_required_workflow_queue_contract.py | 3 +++ 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pr-review-merge-scheduler.yml b/.github/workflows/pr-review-merge-scheduler.yml index 881165240a..5d7b2f4632 100644 --- a/.github/workflows/pr-review-merge-scheduler.yml +++ b/.github/workflows/pr-review-merge-scheduler.yml @@ -1076,14 +1076,20 @@ jobs: --paginate \ | jq -sr ' add[] - | select(.head.repo.full_name != null and .head.ref != null) - | [.head.repo.full_name, .head.ref] + | [ + (if (.head.repo.full_name | type) == "string" then .head.repo.full_name else "" end), + (if (.head.ref | type) == "string" then .head.ref else "" end) + ] | @tsv ' )"; then while IFS=$'\t' read -r head_repo head_ref; do - [ -n "$head_repo" ] && [ -n "$head_ref" ] || continue - encoded_head_ref="$(jq -rn --arg value "$head_ref" '$value | @uri')" + if [ -z "$head_repo" ] || [ -z "$head_ref" ]; then + echo "::warning::Current-HEAD cancellation skipped for ${repo_full_name}: an open PR has a malformed head repository or ref. No run will be cancelled from incomplete evidence." + queue_hygiene_ready=false + break + fi + encoded_head_ref="$(jq -rn --arg value "$head_ref" '$value | split("/") | map(@uri) | join("/")')" if ! live_head_sha="$( gh api \ -H "Accept: application/vnd.github+json" \ diff --git a/tests/test_required_workflow_queue_contract.py b/tests/test_required_workflow_queue_contract.py index 67c2fe69bd..78b0469186 100644 --- a/tests/test_required_workflow_queue_contract.py +++ b/tests/test_required_workflow_queue_contract.py @@ -749,9 +749,12 @@ def test_org_queue_sweep_covers_target_repositories_on_a_heartbeat() -> None: assert "/actions/runs?status=${active_status}&per_page=100" in workflow assert "for active_status in queued in_progress" in workflow assert '"/repos/${head_repo}/git/ref/heads/${encoded_head_ref}"' in workflow + assert '$value | split("/") | map(@uri) | join("/")' in workflow assert "--jq '.object.sha // empty'" in workflow assert 'open_pr_heads_json="$(\n jq' in workflow assert "live ref ${head_repo}:${head_ref} could not be resolved safely" in workflow + assert "an open PR has a malformed head repository or ref" in workflow + assert "select(.head.repo.full_name != null and .head.ref != null)" not in workflow assert '"pull_request" or .event == "pull_request_target"' in workflow assert "$current_pr_head == null or .head_sha != $current_pr_head" in workflow assert ".head_sha != $current_default_sha" in workflow From 7c69378fcb84073920b56d0a8dcb4a5a67b23b1c Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 27 Aug 2026 23:33:57 -0700 Subject: [PATCH 03/19] fix(scheduler): bound live ref lookups --- .../workflows/pr-review-merge-scheduler.yml | 15 +++++++++++- .../test_required_workflow_queue_contract.py | 24 +++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr-review-merge-scheduler.yml b/.github/workflows/pr-review-merge-scheduler.yml index 71befbd4f8..538b28172a 100644 --- a/.github/workflows/pr-review-merge-scheduler.yml +++ b/.github/workflows/pr-review-merge-scheduler.yml @@ -604,6 +604,11 @@ jobs: # this contract). The scheduler paginates, so 1000 keeps the practical # GitHub queue ceiling while avoiding an arbitrary per-repository sample. ORG_SWEEP_MAX_PRS: ${{ github.event.client_payload.max_prs || inputs.max_prs || vars.ORG_SWEEP_MAX_PRS || '1000' }} + # Resolving live refs protects against stale pull-request payloads, but + # each lookup consumes one REST request. Above this independent bound, + # skip destructive cancellation for the repository instead of exhausting + # the organization sweep's API/runtime budget. + ORG_QUEUE_HYGIENE_MAX_REF_LOOKUPS: ${{ vars.ORG_QUEUE_HYGIENE_MAX_REF_LOOKUPS || '100' }} ORG_SWEEP_REVIEW_DISPATCH_LIMIT: ${{ github.event.client_payload.review_dispatch_limit || inputs.review_dispatch_limit || vars.ORG_SWEEP_REVIEW_DISPATCH_LIMIT || '1' }} ORG_SWEEP_BRANCH_UPDATE_LIMIT: ${{ github.event.client_payload.branch_update_limit || inputs.branch_update_limit || vars.ORG_SWEEP_BRANCH_UPDATE_LIMIT || '1' }} ORG_SWEEP_TRIGGER_REVIEWS: ${{ github.event_name == 'schedule' || github.event_name == 'repository_dispatch' && github.event.client_payload.trigger_reviews != false || inputs.trigger_reviews == true }} @@ -832,6 +837,10 @@ jobs: echo "::error::ORG_SWEEP_BRANCH_UPDATE_LIMIT must be -1 or a non-negative integer; got '${ORG_SWEEP_BRANCH_UPDATE_LIMIT}'. Fix the ORG_SWEEP_BRANCH_UPDATE_LIMIT repository variable." exit 1 fi + if ! [[ "$ORG_QUEUE_HYGIENE_MAX_REF_LOOKUPS" =~ ^[1-9][0-9]*$ ]]; then + echo "::error::ORG_QUEUE_HYGIENE_MAX_REF_LOOKUPS must be a positive integer; got '${ORG_QUEUE_HYGIENE_MAX_REF_LOOKUPS}'. Fix the ORG_QUEUE_HYGIENE_MAX_REF_LOOKUPS repository variable." + exit 1 + fi # Unset in production (see the env-block comment above). Primary # source: a persistent `ORG_SWEEP_ROTATION_COUNTER` repository # variable on this (.github) repository, incremented by exactly @@ -1083,7 +1092,11 @@ jobs: | @tsv ' )"; then - if [ -n "$open_pr_refs_tsv" ]; then + open_pr_ref_count="$(printf '%s\n' "$open_pr_refs_tsv" | awk 'NF { count += 1 } END { print count + 0 }')" + if (( open_pr_ref_count > ORG_QUEUE_HYGIENE_MAX_REF_LOOKUPS )); then + echo "::warning::Current-HEAD cancellation skipped for ${repo_full_name}: ${open_pr_ref_count} open PR refs exceed the live-ref lookup limit ${ORG_QUEUE_HYGIENE_MAX_REF_LOOKUPS}. No run will be cancelled from incomplete evidence." + queue_hygiene_ready=false + elif [ -n "$open_pr_refs_tsv" ]; then while IFS=$'\t' read -r head_repo head_ref; do if [ -z "$head_repo" ] || [ -z "$head_ref" ]; then echo "::warning::Current-HEAD cancellation skipped for ${repo_full_name}: an open PR has a malformed head repository or ref. No run will be cancelled from incomplete evidence." diff --git a/tests/test_required_workflow_queue_contract.py b/tests/test_required_workflow_queue_contract.py index 31d61513e4..c2f8c7bf80 100644 --- a/tests/test_required_workflow_queue_contract.py +++ b/tests/test_required_workflow_queue_contract.py @@ -889,6 +889,30 @@ def test_org_queue_sweep_empty_pr_ref_list_skips_the_ref_loop() -> None: assert 'done <<<"$open_pr_refs_tsv"\n fi' in ref_loop +def test_org_queue_sweep_bounds_live_ref_lookups_and_fails_closed() -> None: + """Large queues must not turn hygiene into an unbounded ref API fan-out.""" + workflow = workflow_text("pr-review-merge-scheduler.yml") + ref_loop = workflow.split('if open_pr_refs_tsv="$(\n', 1)[1].split( + ' else\n echo "::warning::Current-HEAD cancellation skipped', + 1, + )[0] + + assert ( + "ORG_QUEUE_HYGIENE_MAX_REF_LOOKUPS: ${{ " + "vars.ORG_QUEUE_HYGIENE_MAX_REF_LOOKUPS || '100' }}" + ) in workflow + assert '"$ORG_QUEUE_HYGIENE_MAX_REF_LOOKUPS" =~ ^[1-9][0-9]*$' in workflow + assert 'open_pr_ref_count="$(printf \'%s\\n\' "$open_pr_refs_tsv"' in ref_loop + assert ( + 'if (( open_pr_ref_count > ORG_QUEUE_HYGIENE_MAX_REF_LOOKUPS )); then' + in ref_loop + ) + assert "live-ref lookup limit" in ref_loop + assert ref_loop.index("queue_hygiene_ready=false") < ref_loop.index( + "while IFS=$'\\t' read -r head_repo head_ref; do" + ) + + def _extract_org_sweep_rotation_snippet(workflow: str) -> str: """Return only the rotation-offset bash block, without the surrounding `gh api`/dispatch logic that would require live network credentials.""" From b99f7f1856dc0567290a99da7c69b1a338fde505 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 28 Aug 2026 00:07:03 -0700 Subject: [PATCH 04/19] fix(scheduler): deduplicate live ref lookups --- .github/workflows/pr-review-merge-scheduler.yml | 1 + tests/test_required_workflow_queue_contract.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+) diff --git a/.github/workflows/pr-review-merge-scheduler.yml b/.github/workflows/pr-review-merge-scheduler.yml index 538b28172a..599a8793dd 100644 --- a/.github/workflows/pr-review-merge-scheduler.yml +++ b/.github/workflows/pr-review-merge-scheduler.yml @@ -1092,6 +1092,7 @@ jobs: | @tsv ' )"; then + open_pr_refs_tsv="$(printf '%s\n' "$open_pr_refs_tsv" | sort -u)" open_pr_ref_count="$(printf '%s\n' "$open_pr_refs_tsv" | awk 'NF { count += 1 } END { print count + 0 }')" if (( open_pr_ref_count > ORG_QUEUE_HYGIENE_MAX_REF_LOOKUPS )); then echo "::warning::Current-HEAD cancellation skipped for ${repo_full_name}: ${open_pr_ref_count} open PR refs exceed the live-ref lookup limit ${ORG_QUEUE_HYGIENE_MAX_REF_LOOKUPS}. No run will be cancelled from incomplete evidence." diff --git a/tests/test_required_workflow_queue_contract.py b/tests/test_required_workflow_queue_contract.py index c2f8c7bf80..72882ded84 100644 --- a/tests/test_required_workflow_queue_contract.py +++ b/tests/test_required_workflow_queue_contract.py @@ -913,6 +913,21 @@ def test_org_queue_sweep_bounds_live_ref_lookups_and_fails_closed() -> None: ) +def test_org_queue_sweep_deduplicates_live_refs_before_counting() -> None: + """Two PRs from one branch must consume one lookup-budget entry.""" + workflow = workflow_text("pr-review-merge-scheduler.yml") + ref_loop = workflow.split('if open_pr_refs_tsv="$(\n', 1)[1].split( + ' else\n echo "::warning::Current-HEAD cancellation skipped', + 1, + )[0] + + assert 'open_pr_refs_tsv="$(printf \'%s\\n\' "$open_pr_refs_tsv" | sort -u)"' in ref_loop + assert ref_loop.index("sort -u") < ref_loop.index("open_pr_ref_count=") + assert ref_loop.index("sort -u") < ref_loop.index( + "while IFS=$'\\t' read -r head_repo head_ref; do" + ) + + def _extract_org_sweep_rotation_snippet(workflow: str) -> str: """Return only the rotation-offset bash block, without the surrounding `gh api`/dispatch logic that would require live network credentials.""" From 03f87fa87a6a6b9b3e0ec1a4461df46fc01e83e2 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 03:29:37 +0900 Subject: [PATCH 05/19] ci(repair): revalidate queue cancellation candidates --- .../repair-pr1348-final-revalidation.yml | 280 ++++++++++++++++++ 1 file changed, 280 insertions(+) create mode 100644 .github/workflows/repair-pr1348-final-revalidation.yml diff --git a/.github/workflows/repair-pr1348-final-revalidation.yml b/.github/workflows/repair-pr1348-final-revalidation.yml new file mode 100644 index 0000000000..aeede248f6 --- /dev/null +++ b/.github/workflows/repair-pr1348-final-revalidation.yml @@ -0,0 +1,280 @@ +name: Repair PR 1348 final queue revalidation + +on: + push: + branches: + - fix/queue-hygiene-live-ref-race + +permissions: + contents: write + +concurrency: + group: repair-pr1348-final-revalidation + cancel-in-progress: false + +jobs: + repair: + if: github.actor != 'github-actions[bot]' || !contains(github.event.head_commit.message, '[pr1348-repair]') + runs-on: ubuntu-latest + steps: + - name: Checkout exact writer branch + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + ref: fix/queue-hygiene-live-ref-race + fetch-depth: 0 + - name: Repair final cancellation revalidation and executable regressions + shell: bash + run: | + set -euo pipefail + python3 <<'PY' + from pathlib import Path + + workflow_path = Path('.github/workflows/pr-review-merge-scheduler.yml') + workflow = workflow_path.read_text() + + old_superseded = ''' if [ "$DRY_RUN" != "true" ]; then + while IFS= read -r run_id; do + if ! gh api -X POST "/repos/${repo_full_name}/actions/runs/${run_id}/cancel" >/dev/null; then + echo "Could not cancel superseded run ${run_id} in ${repo_full_name}; it may have finished already." + fi + done < <(jq -r '.[].id' <<<"$superseded_runs_json") + fi +''' + new_superseded = ''' # queue-hygiene-final-revalidation:start + revalidate_and_cancel_superseded_run() { + local run_id="$1" + local run_json event run_head head_repo head_ref pr_number pr_json + local live_state live_head_repo live_head_ref live_head_sha snapshot_sha + + if ! run_json="$(gh api -H "Accept: application/vnd.github+json" "/repos/${repo_full_name}/actions/runs/${run_id}")"; then + echo "::warning::Skipping cancellation of run ${run_id} in ${repo_full_name}: live run metadata could not be re-fetched." + return 0 + fi + event="$(jq -r '.event // empty' <<<"$run_json")" + run_head="$(jq -r '.head_sha // empty' <<<"$run_json")" + if ! [[ "$run_head" =~ ^[0-9a-fA-F]{40}$ ]]; then + echo "::warning::Skipping cancellation of run ${run_id} in ${repo_full_name}: live run head is malformed." + return 0 + fi + + if [ "$event" = "pull_request" ] || [ "$event" = "pull_request_target" ]; then + pr_number="$(jq -r '.pull_requests[0].number // empty' <<<"$run_json")" + if ! [[ "$pr_number" =~ ^[1-9][0-9]*$ ]]; then + echo "::warning::Skipping cancellation of run ${run_id} in ${repo_full_name}: no authoritative PR identity is attached to the live run." + return 0 + fi + if ! pr_json="$(gh api -H "Accept: application/vnd.github+json" "/repos/${repo_full_name}/pulls/${pr_number}")"; then + echo "::warning::Skipping cancellation of run ${run_id} in ${repo_full_name}: live PR ${pr_number} could not be re-fetched." + return 0 + fi + live_state="$(jq -r '.state // empty' <<<"$pr_json")" + if [ "$live_state" != "open" ]; then + if [ "$live_state" != "closed" ]; then + echo "::warning::Skipping cancellation of run ${run_id} in ${repo_full_name}: live PR ${pr_number} state is malformed." + return 0 + fi + else + live_head_repo="$(jq -r '.head.repo.full_name // empty' <<<"$pr_json")" + live_head_ref="$(jq -r '.head.ref // empty' <<<"$pr_json")" + live_head_sha="$(jq -r '.head.sha // empty' <<<"$pr_json")" + if [ -z "$live_head_repo" ] || [ -z "$live_head_ref" ] || ! [[ "$live_head_sha" =~ ^[0-9a-fA-F]{40}$ ]]; then + echo "::warning::Skipping cancellation of run ${run_id} in ${repo_full_name}: live PR ${pr_number} head metadata is malformed." + return 0 + fi + snapshot_sha="$(jq -r --arg key "${live_head_repo}:${live_head_ref}" '.[$key] // empty' <<<"$open_pr_heads_json")" + if ! [[ "$snapshot_sha" =~ ^[0-9a-fA-F]{40}$ ]] || [ "$live_head_sha" != "$snapshot_sha" ]; then + echo "::warning::Skipping cancellation of run ${run_id} in ${repo_full_name}: PR ${pr_number} moved after queue classification." + return 0 + fi + if [ "$run_head" = "$live_head_sha" ]; then + echo "Preserving run ${run_id} in ${repo_full_name}: it is authoritative current-head evidence for PR ${pr_number}." + return 0 + fi + fi + elif [ "$event" = "push" ] || [ "$event" = "schedule" ]; then + if ! live_head_sha="$(gh api -H "Accept: application/vnd.github+json" "/repos/${repo_full_name}/commits/${default_branch}" --jq '.sha // empty')" || ! [[ "$live_head_sha" =~ ^[0-9a-fA-F]{40}$ ]]; then + echo "::warning::Skipping cancellation of run ${run_id} in ${repo_full_name}: live default-branch HEAD could not be revalidated." + return 0 + fi + if [ "$live_head_sha" != "$current_default_sha" ]; then + echo "::warning::Skipping cancellation of run ${run_id} in ${repo_full_name}: default branch moved after queue classification." + return 0 + fi + if [ "$run_head" = "$live_head_sha" ]; then + echo "Preserving run ${run_id} in ${repo_full_name}: it is authoritative current default-branch evidence." + return 0 + fi + else + echo "::warning::Skipping cancellation of run ${run_id} in ${repo_full_name}: event ${event:-} was not part of the authoritative superseded-run contract." + return 0 + fi + + if ! gh api -X POST "/repos/${repo_full_name}/actions/runs/${run_id}/cancel" >/dev/null; then + echo "Could not cancel superseded run ${run_id} in ${repo_full_name}; it may have finished already." + fi + } + # queue-hygiene-final-revalidation:end + if [ "$DRY_RUN" != "true" ]; then + while IFS= read -r run_id; do + revalidate_and_cancel_superseded_run "$run_id" + done < <(jq -r '.[].id' <<<"$superseded_runs_json") + fi +''' + if old_superseded not in workflow: + raise SystemExit('superseded cancellation block not found exactly; refusing unsafe repair') + workflow = workflow.replace(old_superseded, new_superseded, 1) + + old_stale = ''' if [ "$DRY_RUN" != "true" ]; then + while IFS= read -r run_id; do + if ! gh api -X POST "/repos/${repo_full_name}/actions/runs/${run_id}/cancel" >/dev/null; then + echo "Could not cancel run ${run_id} in ${repo_full_name}; it may have started or finished already." + fi + done < <(jq -r '.[].id' <<<"$stale_runs_json") + fi +''' + new_stale = ''' if [ "$DRY_RUN" != "true" ]; then + while IFS= read -r run_id; do + if ! run_json="$(gh api -H "Accept: application/vnd.github+json" "/repos/${repo_full_name}/actions/runs/${run_id}")"; then + echo "::warning::Skipping stale-run cancellation of ${run_id} in ${repo_full_name}: live run metadata could not be re-fetched." + continue + fi + run_head="$(jq -r '.head_sha // empty' <<<"$run_json")" + head_repo="$(jq -r '.head_repository.full_name // empty' <<<"$run_json")" + head_ref="$(jq -r '.head_branch // empty' <<<"$run_json")" + if ! [[ "$run_head" =~ ^[0-9a-fA-F]{40}$ ]] || [ -z "$head_repo" ] || [ -z "$head_ref" ]; then + echo "::warning::Skipping stale-run cancellation of ${run_id} in ${repo_full_name}: live head metadata is malformed." + continue + fi + encoded_head_ref="$(jq -rn --arg value "$head_ref" '$value | split("/") | map(@uri) | join("/")')" + if ! final_live_sha="$(gh api -H "Accept: application/vnd.github+json" "/repos/${head_repo}/git/ref/heads/${encoded_head_ref}" --jq '.object.sha // empty')" || ! [[ "$final_live_sha" =~ ^[0-9a-fA-F]{40}$ ]]; then + echo "::warning::Skipping stale-run cancellation of ${run_id} in ${repo_full_name}: live branch ref could not be revalidated." + continue + fi + if [ "$run_head" = "$final_live_sha" ]; then + echo "Preserving stale-aged run ${run_id} in ${repo_full_name}: its head is still the live branch head." + continue + fi + if ! gh api -X POST "/repos/${repo_full_name}/actions/runs/${run_id}/cancel" >/dev/null; then + echo "Could not cancel run ${run_id} in ${repo_full_name}; it may have started or finished already." + fi + done < <(jq -r '.[].id' <<<"$stale_runs_json") + fi +''' + if old_stale not in workflow: + raise SystemExit('stale cancellation block not found exactly; refusing unsafe repair') + workflow = workflow.replace(old_stale, new_stale, 1) + workflow_path.write_text(workflow) + + tests_path = Path('tests/test_required_workflow_queue_contract.py') + tests = tests_path.read_text() + marker = 'def test_org_queue_sweep_revalidates_live_pr_immediately_before_cancel() -> None:' + if marker not in tests: + tests += r''' + + +def _extract_queue_hygiene_final_revalidation(workflow: str) -> str: + """Return the executable final PR/head revalidation function from the workflow.""" + start_marker = " # queue-hygiene-final-revalidation:start\n" + end_marker = " # queue-hygiene-final-revalidation:end\n" + start = workflow.index(start_marker) + len(start_marker) + end = workflow.index(end_marker, start) + return textwrap.dedent(workflow[start:end]) + + +def _run_queue_hygiene_revalidation(*, snapshot_sha: str, live_sha: str, run_sha: str, fail_pr_lookup: bool = False) -> str: + """Execute production revalidation with a deterministic fake GitHub CLI.""" + jq = shutil.which("jq") + if jq is None: + pytest.skip("jq is required for the executable queue-hygiene regression test") + workflow = workflow_text("pr-review-merge-scheduler.yml") + function = _extract_queue_hygiene_final_revalidation(workflow) + pr_lookup = "return 73" if fail_pr_lookup else ( + "printf '%s\\n' '" + json.dumps({ + "state": "open", + "head": {"repo": {"full_name": "ContextualWisdomLab/example"}, "ref": "feature/race", "sha": live_sha}, + }, separators=(",", ":")) + "'" + ) + run_payload = json.dumps({ + "event": "pull_request", + "head_sha": run_sha, + "pull_requests": [{"number": 12}], + }, separators=(",", ":")) + script = f'''set -euo pipefail +repo_full_name=ContextualWisdomLab/example +default_branch=main +current_default_sha={'d' * 40} +open_pr_heads_json='{{"ContextualWisdomLab/example:feature/race":"{snapshot_sha}"}}' +cancelled=0 +gh() {{ + if [[ "$*" == *"/actions/runs/77/cancel"* ]]; then cancelled=$((cancelled + 1)); return 0; fi + if [[ "$*" == *"/actions/runs/77"* ]]; then printf '%s\\n' '{run_payload}'; return 0; fi + if [[ "$*" == *"/pulls/12"* ]]; then {pr_lookup}; return $?; fi + return 79 +}} +{function} +revalidate_and_cancel_superseded_run 77 +printf 'cancelled=%s\\n' "$cancelled" +''' + result = subprocess.run(["bash", "-c", script], capture_output=True, text=True) + assert result.returncode == 0, result.stderr + return result.stdout + + +def test_org_queue_sweep_revalidates_live_pr_immediately_before_cancel() -> None: + """A ref movement after classification must not cancel new exact-head evidence.""" + old = "a" * 40 + new = "b" * 40 + output = _run_queue_hygiene_revalidation(snapshot_sha=old, live_sha=new, run_sha=new) + assert "moved after queue classification" in output + assert "cancelled=0" in output + + +def test_org_queue_sweep_final_revalidation_fails_closed_on_live_pr_lookup_error() -> None: + """A failed final authoritative PR lookup must preserve the candidate run.""" + old = "a" * 40 + output = _run_queue_hygiene_revalidation(snapshot_sha=old, live_sha=old, run_sha="c" * 40, fail_pr_lookup=True) + assert "could not be re-fetched" in output + assert "cancelled=0" in output + + +def test_org_queue_sweep_final_revalidation_preserves_current_head_and_cancels_proven_predecessor() -> None: + """Only an unchanged live snapshot may authorize predecessor cancellation.""" + current = "b" * 40 + preserved = _run_queue_hygiene_revalidation(snapshot_sha=current, live_sha=current, run_sha=current) + cancelled = _run_queue_hygiene_revalidation(snapshot_sha=current, live_sha=current, run_sha="a" * 40) + assert "authoritative current-head evidence" in preserved + assert "cancelled=0" in preserved + assert "cancelled=1" in cancelled +''' + tests_path.write_text(tests) + + baseline_path = Path('docs/product-technical-gap-baseline.md') + baseline = baseline_path.read_text() + doc_marker = 'PR #1348 final cancellation revalidation' + if doc_marker not in baseline: + baseline += '''\n\n### PR #1348 final cancellation revalidation (2026-09-02)\n\n- **Root cause:** exact head `af519b7927225588d437fed6ee30f20e76291c3e` classified Actions runs from an initial live-ref snapshot, but cancellation occurred later without revalidating the live PR/head. A synchronize event between classification and cancellation could therefore make the snapshot stale and cause the new exact-head run to be cancelled.\n- **Repair:** every superseded PR/default-branch candidate is re-fetched immediately before cancellation; open PR metadata and the live head must still match the classification snapshot, lookup/malformed/moved state fails closed, and current-head evidence is explicitly preserved. The legacy aged orphan cleanup also re-fetches run and branch state before destructive cancellation.\n- **Executable evidence:** regressions execute the production Bash revalidation function for head movement, lookup failure, current-head preservation, and a proven predecessor cancellation.\n''' + baseline_path.write_text(baseline) + + changelog_path = Path('CHANGELOG.md') + changelog = changelog_path.read_text() + change_marker = 'Revalidate queue-hygiene cancellation candidates immediately before destructive cancellation' + if change_marker not in changelog: + changelog += '\n- Revalidate queue-hygiene cancellation candidates immediately before destructive cancellation, failing closed on moved/unavailable live PR or branch state and preserving exact-head evidence.\n' + changelog_path.write_text(changelog) + PY + + python3 -m pytest -q tests/test_required_workflow_queue_contract.py + git diff --check + - name: Commit verified owner-side repair + shell: bash + run: | + set -euo pipefail + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + git add .github/workflows/pr-review-merge-scheduler.yml tests/test_required_workflow_queue_contract.py docs/product-technical-gap-baseline.md CHANGELOG.md + if git diff --cached --quiet; then + echo 'No repair delta remains.' + exit 0 + fi + git commit -m 'fix(scheduler): revalidate live state before run cancellation [pr1348-repair]' + git push origin HEAD:fix/queue-hygiene-live-ref-race From 5cfc2ff93c45d3a30c55c325e2f8fc40fd954947 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 03:30:14 +0900 Subject: [PATCH 06/19] ci(repair): trigger PR 1348 final revalidation --- .github/pr1348-repair.trigger | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .github/pr1348-repair.trigger diff --git a/.github/pr1348-repair.trigger b/.github/pr1348-repair.trigger new file mode 100644 index 0000000000..9e66ea71d2 --- /dev/null +++ b/.github/pr1348-repair.trigger @@ -0,0 +1,2 @@ +source-head=03f87fa87a6a6b9b3e0ec1a4461df46fc01e83e2 +reason=final-live-state-revalidation From 4651a12414bbd4f7ae42c607f7bf00e76a584a88 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 03:36:10 +0900 Subject: [PATCH 07/19] fix(scheduler): add fail-closed final queue revalidation --- scripts/ci/revalidate_queue_cancellation.sh | 91 +++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 scripts/ci/revalidate_queue_cancellation.sh diff --git a/scripts/ci/revalidate_queue_cancellation.sh b/scripts/ci/revalidate_queue_cancellation.sh new file mode 100644 index 0000000000..9624e134f0 --- /dev/null +++ b/scripts/ci/revalidate_queue_cancellation.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [ "$#" -ne 5 ]; then + echo "usage: $0 " >&2 + exit 2 +fi + +repo_full_name="$1" +run_id="$2" +default_branch="$3" +classified_default_sha="$4" +classified_open_pr_heads_json="$5" + +warn_preserve() { + echo "::warning::Preserving run ${run_id} in ${repo_full_name}: $1" + exit 0 +} + +if ! run_json="$(gh api -H "Accept: application/vnd.github+json" "/repos/${repo_full_name}/actions/runs/${run_id}")"; then + warn_preserve "live run metadata could not be re-fetched before cancellation." +fi + +event="$(jq -r '.event // empty' <<<"$run_json")" +run_head="$(jq -r '.head_sha // empty' <<<"$run_json")" +if ! [[ "$run_head" =~ ^[0-9a-fA-F]{40}$ ]]; then + warn_preserve "live run head is malformed." +fi + +case "$event" in + pull_request|pull_request_target) + pr_number="$(jq -r '.pull_requests[0].number // empty' <<<"$run_json")" + if ! [[ "$pr_number" =~ ^[1-9][0-9]*$ ]]; then + warn_preserve "no authoritative PR identity is attached to the live run." + fi + if ! pr_json="$(gh api -H "Accept: application/vnd.github+json" "/repos/${repo_full_name}/pulls/${pr_number}")"; then + warn_preserve "live PR ${pr_number} could not be re-fetched before cancellation." + fi + live_state="$(jq -r '.state // empty' <<<"$pr_json")" + if [ "$live_state" = "open" ]; then + live_head_repo="$(jq -r '.head.repo.full_name // empty' <<<"$pr_json")" + live_head_ref="$(jq -r '.head.ref // empty' <<<"$pr_json")" + live_head_sha="$(jq -r '.head.sha // empty' <<<"$pr_json")" + if [ -z "$live_head_repo" ] || [ -z "$live_head_ref" ] || ! [[ "$live_head_sha" =~ ^[0-9a-fA-F]{40}$ ]]; then + warn_preserve "live PR ${pr_number} head metadata is malformed." + fi + encoded_head_ref="$(jq -rn --arg value "$live_head_ref" '$value | split("/") | map(@uri) | join("/")')" + if ! final_ref_sha="$(gh api -H "Accept: application/vnd.github+json" "/repos/${live_head_repo}/git/ref/heads/${encoded_head_ref}" --jq '.object.sha // empty')"; then + warn_preserve "live ref for PR ${pr_number} could not be re-fetched before cancellation." + fi + if ! [[ "$final_ref_sha" =~ ^[0-9a-fA-F]{40}$ ]]; then + warn_preserve "live ref for PR ${pr_number} is malformed." + fi + classified_sha="$(jq -r --arg key "${live_head_repo}:${live_head_ref}" '.[$key] // empty' <<<"$classified_open_pr_heads_json")" + if ! [[ "$classified_sha" =~ ^[0-9a-fA-F]{40}$ ]]; then + warn_preserve "the classification snapshot has no valid head for PR ${pr_number}." + fi + if [ "$live_head_sha" != "$classified_sha" ] || [ "$final_ref_sha" != "$classified_sha" ]; then + warn_preserve "PR ${pr_number} moved after queue classification." + fi + if [ "$run_head" = "$final_ref_sha" ]; then + echo "Preserving run ${run_id} in ${repo_full_name}: authoritative current-head evidence for PR ${pr_number}." + exit 0 + fi + elif [ "$live_state" != "closed" ]; then + warn_preserve "live PR ${pr_number} state is malformed." + fi + ;; + push|schedule) + if ! live_default_sha="$(gh api -H "Accept: application/vnd.github+json" "/repos/${repo_full_name}/commits/${default_branch}" --jq '.sha // empty')"; then + warn_preserve "live default-branch HEAD could not be re-fetched before cancellation." + fi + if ! [[ "$live_default_sha" =~ ^[0-9a-fA-F]{40}$ ]]; then + warn_preserve "live default-branch HEAD is malformed." + fi + if [ "$live_default_sha" != "$classified_default_sha" ]; then + warn_preserve "default branch moved after queue classification." + fi + if [ "$run_head" = "$live_default_sha" ]; then + echo "Preserving run ${run_id} in ${repo_full_name}: authoritative current default-branch evidence." + exit 0 + fi + ;; + *) + warn_preserve "event ${event:-} is outside the authoritative superseded-run contract." + ;; +esac + +if ! gh api -X POST "/repos/${repo_full_name}/actions/runs/${run_id}/cancel" >/dev/null; then + echo "Could not cancel superseded run ${run_id} in ${repo_full_name}; it may have finished already." +fi From c2c23240b5485e2701d702af15d07b1265dba738 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 03:36:46 +0900 Subject: [PATCH 08/19] test(scheduler): execute final queue revalidation races --- tests/test_queue_cancellation_revalidation.py | 164 ++++++++++++++++++ 1 file changed, 164 insertions(+) create mode 100644 tests/test_queue_cancellation_revalidation.py diff --git a/tests/test_queue_cancellation_revalidation.py b/tests/test_queue_cancellation_revalidation.py new file mode 100644 index 0000000000..cd8991fb00 --- /dev/null +++ b/tests/test_queue_cancellation_revalidation.py @@ -0,0 +1,164 @@ +"""Executable regressions for destructive queue-cancellation revalidation.""" + +from __future__ import annotations + +import json +import os +import shutil +import subprocess +from pathlib import Path + +import pytest + + +REPO_ROOT = Path(__file__).resolve().parents[1] +SCRIPT = REPO_ROOT / "scripts" / "ci" / "revalidate_queue_cancellation.sh" + + +def _run_case( + tmp_path: Path, + *, + snapshot_sha: str, + pr_sha: str, + ref_sha: str, + run_sha: str, + fail_lookup: str | None = None, +) -> tuple[subprocess.CompletedProcess[str], bool]: + """Run the production shell helper against a deterministic fake GitHub CLI.""" + if shutil.which("jq") is None: + pytest.skip("jq is required for the queue-cancellation regression") + + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + cancelled = tmp_path / "cancelled" + pr_payload = json.dumps( + { + "state": "open", + "head": { + "repo": {"full_name": "ContextualWisdomLab/example"}, + "ref": "feature/race", + "sha": pr_sha, + }, + }, + separators=(",", ":"), + ) + run_payload = json.dumps( + { + "event": "pull_request", + "head_sha": run_sha, + "pull_requests": [{"number": 12}], + }, + separators=(",", ":"), + ) + fake_gh = bin_dir / "gh" + fake_gh.write_text( + f"""#!/usr/bin/env bash +set -euo pipefail +args="$*" +if [[ "$args" == *"/actions/runs/77/cancel"* ]]; then + : > {cancelled!s} + exit 0 +fi +if [[ "$args" == *"/actions/runs/77"* ]]; then + printf '%s\\n' '{run_payload}' + exit 0 +fi +if [[ "$args" == *"/pulls/12"* ]]; then + {'exit 73' if fail_lookup == 'pr' else f"printf '%s\\n' '{pr_payload}'"} + exit 0 +fi +if [[ "$args" == *"/git/ref/heads/feature/race"* ]]; then + {'exit 74' if fail_lookup == 'ref' else f"printf '%s\\n' '{ref_sha}'"} + exit 0 +fi +exit 79 +""", + encoding="utf-8", + ) + fake_gh.chmod(0o755) + env = os.environ.copy() + env["PATH"] = f"{bin_dir}{os.pathsep}{env['PATH']}" + snapshot = json.dumps( + {"ContextualWisdomLab/example:feature/race": snapshot_sha}, + separators=(",", ":"), + ) + result = subprocess.run( + [ + "bash", + str(SCRIPT), + "ContextualWisdomLab/example", + "77", + "main", + "d" * 40, + snapshot, + ], + capture_output=True, + text=True, + env=env, + check=False, + ) + return result, cancelled.exists() + + +def test_post_classification_head_movement_fails_closed(tmp_path: Path) -> None: + """A new exact head arriving after classification must never be cancelled.""" + old = "a" * 40 + new = "b" * 40 + result, cancelled = _run_case( + tmp_path, + snapshot_sha=old, + pr_sha=new, + ref_sha=new, + run_sha=new, + ) + assert result.returncode == 0, result.stderr + assert "moved after queue classification" in result.stdout + assert not cancelled + + +@pytest.mark.parametrize("failed_lookup", ["pr", "ref"]) +def test_final_lookup_failure_fails_closed( + tmp_path: Path, failed_lookup: str +) -> None: + """Unavailable final authoritative PR/ref state must preserve the candidate.""" + current = "b" * 40 + result, cancelled = _run_case( + tmp_path, + snapshot_sha=current, + pr_sha=current, + ref_sha=current, + run_sha="a" * 40, + fail_lookup=failed_lookup, + ) + assert result.returncode == 0, result.stderr + assert "could not be re-fetched" in result.stdout + assert not cancelled + + +def test_current_head_is_preserved(tmp_path: Path) -> None: + """Final live-ref validation must preserve sole current-head evidence.""" + current = "b" * 40 + result, cancelled = _run_case( + tmp_path, + snapshot_sha=current, + pr_sha=current, + ref_sha=current, + run_sha=current, + ) + assert result.returncode == 0, result.stderr + assert "authoritative current-head evidence" in result.stdout + assert not cancelled + + +def test_proven_predecessor_is_cancelled(tmp_path: Path) -> None: + """An unchanged final live ref may cancel a proven predecessor run.""" + current = "b" * 40 + result, cancelled = _run_case( + tmp_path, + snapshot_sha=current, + pr_sha=current, + ref_sha=current, + run_sha="a" * 40, + ) + assert result.returncode == 0, result.stderr + assert cancelled From d820181156fa39a01d2880816a4f0d03203ab8a1 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 03:37:39 +0900 Subject: [PATCH 09/19] ci(repair): make PR 1348 repair driver executable --- .../repair-pr1348-final-revalidation.yml | 274 +++--------------- 1 file changed, 33 insertions(+), 241 deletions(-) diff --git a/.github/workflows/repair-pr1348-final-revalidation.yml b/.github/workflows/repair-pr1348-final-revalidation.yml index aeede248f6..002d6b164e 100644 --- a/.github/workflows/repair-pr1348-final-revalidation.yml +++ b/.github/workflows/repair-pr1348-final-revalidation.yml @@ -14,267 +14,59 @@ concurrency: jobs: repair: - if: github.actor != 'github-actions[bot]' || !contains(github.event.head_commit.message, '[pr1348-repair]') - runs-on: ubuntu-latest + if: github.actor != 'github-actions[bot]' + runs-on: ubuntu-24.04 steps: - name: Checkout exact writer branch uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: ref: fix/queue-hygiene-live-ref-race fetch-depth: 0 - - name: Repair final cancellation revalidation and executable regressions + - name: Apply exact source and traceability repair shell: bash run: | set -euo pipefail - python3 <<'PY' + python3 - <<'PY' from pathlib import Path - workflow_path = Path('.github/workflows/pr-review-merge-scheduler.yml') - workflow = workflow_path.read_text() - - old_superseded = ''' if [ "$DRY_RUN" != "true" ]; then - while IFS= read -r run_id; do - if ! gh api -X POST "/repos/${repo_full_name}/actions/runs/${run_id}/cancel" >/dev/null; then - echo "Could not cancel superseded run ${run_id} in ${repo_full_name}; it may have finished already." - fi - done < <(jq -r '.[].id' <<<"$superseded_runs_json") - fi -''' - new_superseded = ''' # queue-hygiene-final-revalidation:start - revalidate_and_cancel_superseded_run() { - local run_id="$1" - local run_json event run_head head_repo head_ref pr_number pr_json - local live_state live_head_repo live_head_ref live_head_sha snapshot_sha - - if ! run_json="$(gh api -H "Accept: application/vnd.github+json" "/repos/${repo_full_name}/actions/runs/${run_id}")"; then - echo "::warning::Skipping cancellation of run ${run_id} in ${repo_full_name}: live run metadata could not be re-fetched." - return 0 - fi - event="$(jq -r '.event // empty' <<<"$run_json")" - run_head="$(jq -r '.head_sha // empty' <<<"$run_json")" - if ! [[ "$run_head" =~ ^[0-9a-fA-F]{40}$ ]]; then - echo "::warning::Skipping cancellation of run ${run_id} in ${repo_full_name}: live run head is malformed." - return 0 - fi - - if [ "$event" = "pull_request" ] || [ "$event" = "pull_request_target" ]; then - pr_number="$(jq -r '.pull_requests[0].number // empty' <<<"$run_json")" - if ! [[ "$pr_number" =~ ^[1-9][0-9]*$ ]]; then - echo "::warning::Skipping cancellation of run ${run_id} in ${repo_full_name}: no authoritative PR identity is attached to the live run." - return 0 - fi - if ! pr_json="$(gh api -H "Accept: application/vnd.github+json" "/repos/${repo_full_name}/pulls/${pr_number}")"; then - echo "::warning::Skipping cancellation of run ${run_id} in ${repo_full_name}: live PR ${pr_number} could not be re-fetched." - return 0 - fi - live_state="$(jq -r '.state // empty' <<<"$pr_json")" - if [ "$live_state" != "open" ]; then - if [ "$live_state" != "closed" ]; then - echo "::warning::Skipping cancellation of run ${run_id} in ${repo_full_name}: live PR ${pr_number} state is malformed." - return 0 - fi - else - live_head_repo="$(jq -r '.head.repo.full_name // empty' <<<"$pr_json")" - live_head_ref="$(jq -r '.head.ref // empty' <<<"$pr_json")" - live_head_sha="$(jq -r '.head.sha // empty' <<<"$pr_json")" - if [ -z "$live_head_repo" ] || [ -z "$live_head_ref" ] || ! [[ "$live_head_sha" =~ ^[0-9a-fA-F]{40}$ ]]; then - echo "::warning::Skipping cancellation of run ${run_id} in ${repo_full_name}: live PR ${pr_number} head metadata is malformed." - return 0 - fi - snapshot_sha="$(jq -r --arg key "${live_head_repo}:${live_head_ref}" '.[$key] // empty' <<<"$open_pr_heads_json")" - if ! [[ "$snapshot_sha" =~ ^[0-9a-fA-F]{40}$ ]] || [ "$live_head_sha" != "$snapshot_sha" ]; then - echo "::warning::Skipping cancellation of run ${run_id} in ${repo_full_name}: PR ${pr_number} moved after queue classification." - return 0 - fi - if [ "$run_head" = "$live_head_sha" ]; then - echo "Preserving run ${run_id} in ${repo_full_name}: it is authoritative current-head evidence for PR ${pr_number}." - return 0 - fi - fi - elif [ "$event" = "push" ] || [ "$event" = "schedule" ]; then - if ! live_head_sha="$(gh api -H "Accept: application/vnd.github+json" "/repos/${repo_full_name}/commits/${default_branch}" --jq '.sha // empty')" || ! [[ "$live_head_sha" =~ ^[0-9a-fA-F]{40}$ ]]; then - echo "::warning::Skipping cancellation of run ${run_id} in ${repo_full_name}: live default-branch HEAD could not be revalidated." - return 0 - fi - if [ "$live_head_sha" != "$current_default_sha" ]; then - echo "::warning::Skipping cancellation of run ${run_id} in ${repo_full_name}: default branch moved after queue classification." - return 0 - fi - if [ "$run_head" = "$live_head_sha" ]; then - echo "Preserving run ${run_id} in ${repo_full_name}: it is authoritative current default-branch evidence." - return 0 - fi - else - echo "::warning::Skipping cancellation of run ${run_id} in ${repo_full_name}: event ${event:-} was not part of the authoritative superseded-run contract." - return 0 - fi - - if ! gh api -X POST "/repos/${repo_full_name}/actions/runs/${run_id}/cancel" >/dev/null; then - echo "Could not cancel superseded run ${run_id} in ${repo_full_name}; it may have finished already." - fi - } - # queue-hygiene-final-revalidation:end - if [ "$DRY_RUN" != "true" ]; then - while IFS= read -r run_id; do - revalidate_and_cancel_superseded_run "$run_id" - done < <(jq -r '.[].id' <<<"$superseded_runs_json") - fi -''' - if old_superseded not in workflow: - raise SystemExit('superseded cancellation block not found exactly; refusing unsafe repair') - workflow = workflow.replace(old_superseded, new_superseded, 1) - - old_stale = ''' if [ "$DRY_RUN" != "true" ]; then - while IFS= read -r run_id; do - if ! gh api -X POST "/repos/${repo_full_name}/actions/runs/${run_id}/cancel" >/dev/null; then - echo "Could not cancel run ${run_id} in ${repo_full_name}; it may have started or finished already." - fi - done < <(jq -r '.[].id' <<<"$stale_runs_json") - fi -''' - new_stale = ''' if [ "$DRY_RUN" != "true" ]; then - while IFS= read -r run_id; do - if ! run_json="$(gh api -H "Accept: application/vnd.github+json" "/repos/${repo_full_name}/actions/runs/${run_id}")"; then - echo "::warning::Skipping stale-run cancellation of ${run_id} in ${repo_full_name}: live run metadata could not be re-fetched." - continue - fi - run_head="$(jq -r '.head_sha // empty' <<<"$run_json")" - head_repo="$(jq -r '.head_repository.full_name // empty' <<<"$run_json")" - head_ref="$(jq -r '.head_branch // empty' <<<"$run_json")" - if ! [[ "$run_head" =~ ^[0-9a-fA-F]{40}$ ]] || [ -z "$head_repo" ] || [ -z "$head_ref" ]; then - echo "::warning::Skipping stale-run cancellation of ${run_id} in ${repo_full_name}: live head metadata is malformed." - continue - fi - encoded_head_ref="$(jq -rn --arg value "$head_ref" '$value | split("/") | map(@uri) | join("/")')" - if ! final_live_sha="$(gh api -H "Accept: application/vnd.github+json" "/repos/${head_repo}/git/ref/heads/${encoded_head_ref}" --jq '.object.sha // empty')" || ! [[ "$final_live_sha" =~ ^[0-9a-fA-F]{40}$ ]]; then - echo "::warning::Skipping stale-run cancellation of ${run_id} in ${repo_full_name}: live branch ref could not be revalidated." - continue - fi - if [ "$run_head" = "$final_live_sha" ]; then - echo "Preserving stale-aged run ${run_id} in ${repo_full_name}: its head is still the live branch head." - continue - fi - if ! gh api -X POST "/repos/${repo_full_name}/actions/runs/${run_id}/cancel" >/dev/null; then - echo "Could not cancel run ${run_id} in ${repo_full_name}; it may have started or finished already." - fi - done < <(jq -r '.[].id' <<<"$stale_runs_json") - fi -''' - if old_stale not in workflow: - raise SystemExit('stale cancellation block not found exactly; refusing unsafe repair') - workflow = workflow.replace(old_stale, new_stale, 1) - workflow_path.write_text(workflow) - - tests_path = Path('tests/test_required_workflow_queue_contract.py') - tests = tests_path.read_text() - marker = 'def test_org_queue_sweep_revalidates_live_pr_immediately_before_cancel() -> None:' - if marker not in tests: - tests += r''' - - -def _extract_queue_hygiene_final_revalidation(workflow: str) -> str: - """Return the executable final PR/head revalidation function from the workflow.""" - start_marker = " # queue-hygiene-final-revalidation:start\n" - end_marker = " # queue-hygiene-final-revalidation:end\n" - start = workflow.index(start_marker) + len(start_marker) - end = workflow.index(end_marker, start) - return textwrap.dedent(workflow[start:end]) - - -def _run_queue_hygiene_revalidation(*, snapshot_sha: str, live_sha: str, run_sha: str, fail_pr_lookup: bool = False) -> str: - """Execute production revalidation with a deterministic fake GitHub CLI.""" - jq = shutil.which("jq") - if jq is None: - pytest.skip("jq is required for the executable queue-hygiene regression test") - workflow = workflow_text("pr-review-merge-scheduler.yml") - function = _extract_queue_hygiene_final_revalidation(workflow) - pr_lookup = "return 73" if fail_pr_lookup else ( - "printf '%s\\n' '" + json.dumps({ - "state": "open", - "head": {"repo": {"full_name": "ContextualWisdomLab/example"}, "ref": "feature/race", "sha": live_sha}, - }, separators=(",", ":")) + "'" - ) - run_payload = json.dumps({ - "event": "pull_request", - "head_sha": run_sha, - "pull_requests": [{"number": 12}], - }, separators=(",", ":")) - script = f'''set -euo pipefail -repo_full_name=ContextualWisdomLab/example -default_branch=main -current_default_sha={'d' * 40} -open_pr_heads_json='{{"ContextualWisdomLab/example:feature/race":"{snapshot_sha}"}}' -cancelled=0 -gh() {{ - if [[ "$*" == *"/actions/runs/77/cancel"* ]]; then cancelled=$((cancelled + 1)); return 0; fi - if [[ "$*" == *"/actions/runs/77"* ]]; then printf '%s\\n' '{run_payload}'; return 0; fi - if [[ "$*" == *"/pulls/12"* ]]; then {pr_lookup}; return $?; fi - return 79 -}} -{function} -revalidate_and_cancel_superseded_run 77 -printf 'cancelled=%s\\n' "$cancelled" -''' - result = subprocess.run(["bash", "-c", script], capture_output=True, text=True) - assert result.returncode == 0, result.stderr - return result.stdout - - -def test_org_queue_sweep_revalidates_live_pr_immediately_before_cancel() -> None: - """A ref movement after classification must not cancel new exact-head evidence.""" - old = "a" * 40 - new = "b" * 40 - output = _run_queue_hygiene_revalidation(snapshot_sha=old, live_sha=new, run_sha=new) - assert "moved after queue classification" in output - assert "cancelled=0" in output - - -def test_org_queue_sweep_final_revalidation_fails_closed_on_live_pr_lookup_error() -> None: - """A failed final authoritative PR lookup must preserve the candidate run.""" - old = "a" * 40 - output = _run_queue_hygiene_revalidation(snapshot_sha=old, live_sha=old, run_sha="c" * 40, fail_pr_lookup=True) - assert "could not be re-fetched" in output - assert "cancelled=0" in output - - -def test_org_queue_sweep_final_revalidation_preserves_current_head_and_cancels_proven_predecessor() -> None: - """Only an unchanged live snapshot may authorize predecessor cancellation.""" - current = "b" * 40 - preserved = _run_queue_hygiene_revalidation(snapshot_sha=current, live_sha=current, run_sha=current) - cancelled = _run_queue_hygiene_revalidation(snapshot_sha=current, live_sha=current, run_sha="a" * 40) - assert "authoritative current-head evidence" in preserved - assert "cancelled=0" in preserved - assert "cancelled=1" in cancelled -''' - tests_path.write_text(tests) + workflow = workflow_path.read_text(encoding='utf-8') + old1 = ' if [ "$DRY_RUN" != "true" ]; then\n while IFS= read -r run_id; do\n if ! gh api -X POST "/repos/${repo_full_name}/actions/runs/${run_id}/cancel" >/dev/null; then\n echo "Could not cancel superseded run ${run_id} in ${repo_full_name}; it may have finished already."\n fi\n done < <(jq -r \' .[].id \' <<<"$superseded_runs_json")\n fi\n'.replace("\\' .[].id \\'", "'.[].id'") + new1 = ' if [ "$DRY_RUN" != "true" ]; then\n while IFS= read -r run_id; do\n bash scripts/ci/revalidate_queue_cancellation.sh "$repo_full_name" "$run_id" "$default_branch" "$current_default_sha" "$open_pr_heads_json"\n done < <(jq -r \' .[].id \' <<<"$superseded_runs_json")\n fi\n'.replace("\\' .[].id \\'", "'.[].id'") + old2 = ' if [ "$DRY_RUN" != "true" ]; then\n while IFS= read -r run_id; do\n if ! gh api -X POST "/repos/${repo_full_name}/actions/runs/${run_id}/cancel" >/dev/null; then\n echo "Could not cancel run ${run_id} in ${repo_full_name}; it may have started or finished already."\n fi\n done < <(jq -r \' .[].id \' <<<"$stale_runs_json")\n fi\n'.replace("\\' .[].id \\'", "'.[].id'") + new2 = ' if [ "$DRY_RUN" != "true" ]; then\n while IFS= read -r run_id; do\n bash scripts/ci/revalidate_queue_cancellation.sh "$repo_full_name" "$run_id" "$default_branch" "$current_default_sha" "$open_pr_heads_json"\n done < <(jq -r \' .[].id \' <<<"$stale_runs_json")\n fi\n'.replace("\\' .[].id \\'", "'.[].id'") + if old1 not in workflow: + raise SystemExit('superseded cancellation block changed; refusing stale repair') + workflow = workflow.replace(old1, new1, 1) + if old2 not in workflow: + raise SystemExit('aged cancellation block changed; refusing stale repair') + workflow = workflow.replace(old2, new2, 1) + workflow_path.write_text(workflow, encoding='utf-8') baseline_path = Path('docs/product-technical-gap-baseline.md') - baseline = baseline_path.read_text() - doc_marker = 'PR #1348 final cancellation revalidation' - if doc_marker not in baseline: - baseline += '''\n\n### PR #1348 final cancellation revalidation (2026-09-02)\n\n- **Root cause:** exact head `af519b7927225588d437fed6ee30f20e76291c3e` classified Actions runs from an initial live-ref snapshot, but cancellation occurred later without revalidating the live PR/head. A synchronize event between classification and cancellation could therefore make the snapshot stale and cause the new exact-head run to be cancelled.\n- **Repair:** every superseded PR/default-branch candidate is re-fetched immediately before cancellation; open PR metadata and the live head must still match the classification snapshot, lookup/malformed/moved state fails closed, and current-head evidence is explicitly preserved. The legacy aged orphan cleanup also re-fetches run and branch state before destructive cancellation.\n- **Executable evidence:** regressions execute the production Bash revalidation function for head movement, lookup failure, current-head preservation, and a proven predecessor cancellation.\n''' - baseline_path.write_text(baseline) + baseline = baseline_path.read_text(encoding='utf-8') + doc = '\n\n### PR #1348 final cancellation revalidation (2026-09-02)\n\n- **Root cause:** exact head `af519b7927225588d437fed6ee30f20e76291c3e` classified Actions runs from an initial live-ref snapshot, but destructive cancellation occurred later without a second authoritative lookup. A synchronize event between classification and cancellation could therefore turn the snapshot stale and cancel newly current exact-head evidence.\n- **Repair:** immediately before either superseded-run or aged-run cancellation, the scheduler now executes `scripts/ci/revalidate_queue_cancellation.sh`, which re-fetches live run metadata plus the PR and its final head ref (or the default-branch HEAD for push/schedule), requires the final live state to remain consistent with the classification snapshot, fails closed on unavailable/malformed/moved state, preserves current-head evidence, and only then cancels a proven predecessor.\n- **Executable evidence:** `tests/test_queue_cancellation_revalidation.py` executes the production helper against deterministic GitHub API doubles for head movement, PR/ref lookup failure, current-head preservation, and proven predecessor cancellation. Existing queue-contract tests continue to cover discovery bounds and fail-closed initial snapshot construction.\n' + if 'PR #1348 final cancellation revalidation' not in baseline: + baseline_path.write_text(baseline + doc, encoding='utf-8') changelog_path = Path('CHANGELOG.md') - changelog = changelog_path.read_text() - change_marker = 'Revalidate queue-hygiene cancellation candidates immediately before destructive cancellation' - if change_marker not in changelog: - changelog += '\n- Revalidate queue-hygiene cancellation candidates immediately before destructive cancellation, failing closed on moved/unavailable live PR or branch state and preserving exact-head evidence.\n' - changelog_path.write_text(changelog) + changelog = changelog_path.read_text(encoding='utf-8') + change = '\n- Revalidate queue-hygiene cancellation candidates immediately before destructive cancellation, failing closed on moved or unavailable live PR/ref state and preserving exact-head evidence (`#1348`).\n' + if 'Revalidate queue-hygiene cancellation candidates immediately before destructive cancellation' not in changelog: + changelog_path.write_text(changelog + change, encoding='utf-8') PY - - python3 -m pytest -q tests/test_required_workflow_queue_contract.py + python3 -m pytest -q tests/test_queue_cancellation_revalidation.py tests/test_required_workflow_queue_contract.py + if command -v actionlint >/dev/null 2>&1; then + actionlint .github/workflows/pr-review-merge-scheduler.yml + fi git diff --check - - name: Commit verified owner-side repair + - name: Commit verified repair and remove temporary driver shell: bash run: | set -euo pipefail git config user.name github-actions[bot] git config user.email 41898282+github-actions[bot]@users.noreply.github.com - git add .github/workflows/pr-review-merge-scheduler.yml tests/test_required_workflow_queue_contract.py docs/product-technical-gap-baseline.md CHANGELOG.md - if git diff --cached --quiet; then - echo 'No repair delta remains.' - exit 0 - fi - git commit -m 'fix(scheduler): revalidate live state before run cancellation [pr1348-repair]' + git add .github/workflows/pr-review-merge-scheduler.yml docs/product-technical-gap-baseline.md CHANGELOG.md + git rm -f .github/workflows/repair-pr1348-final-revalidation.yml .github/pr1348-repair.trigger + git diff --cached --check + git commit -m 'fix(scheduler): revalidate live state before cancellation [pr1348-repair]' git push origin HEAD:fix/queue-hygiene-live-ref-race From 59f374eb255edc7109b9f22d35cc19f45d6cc231 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 03:56:48 +0900 Subject: [PATCH 10/19] test(scheduler): expose aged-orphan cancellation regression --- tests/test_queue_cancellation_revalidation.py | 81 +++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/tests/test_queue_cancellation_revalidation.py b/tests/test_queue_cancellation_revalidation.py index cd8991fb00..b618b2ed52 100644 --- a/tests/test_queue_cancellation_revalidation.py +++ b/tests/test_queue_cancellation_revalidation.py @@ -45,6 +45,7 @@ def _run_case( run_payload = json.dumps( { "event": "pull_request", + "status": "queued", "head_sha": run_sha, "pull_requests": [{"number": 12}], }, @@ -91,6 +92,65 @@ def _run_case( "main", "d" * 40, snapshot, + "superseded", + ], + capture_output=True, + text=True, + env=env, + check=False, + ) + return result, cancelled.exists() + + +def _run_aged_orphan_case( + tmp_path: Path, *, event: str, status: str = "queued" +) -> tuple[subprocess.CompletedProcess[str], bool]: + """Run an aged orphan candidate that has no current PR/default-branch authority.""" + if shutil.which("jq") is None: + pytest.skip("jq is required for the queue-cancellation regression") + + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + cancelled = tmp_path / "cancelled" + run_payload = json.dumps( + { + "event": event, + "status": status, + "head_sha": "a" * 40, + "pull_requests": [], + }, + separators=(",", ":"), + ) + fake_gh = bin_dir / "gh" + fake_gh.write_text( + f"""#!/usr/bin/env bash +set -euo pipefail +args="$*" +if [[ "$args" == *"/actions/runs/77/cancel"* ]]; then + : > {cancelled!s} + exit 0 +fi +if [[ "$args" == *"/actions/runs/77"* ]]; then + printf '%s\\n' '{run_payload}' + exit 0 +fi +exit 79 +""", + encoding="utf-8", + ) + fake_gh.chmod(0o755) + env = os.environ.copy() + env["PATH"] = f"{bin_dir}{os.pathsep}{env['PATH']}" + result = subprocess.run( + [ + "bash", + str(SCRIPT), + "ContextualWisdomLab/example", + "77", + "main", + "d" * 40, + "{}", + "aged-orphan", ], capture_output=True, text=True, @@ -162,3 +222,24 @@ def test_proven_predecessor_is_cancelled(tmp_path: Path) -> None: ) assert result.returncode == 0, result.stderr assert cancelled + + +@pytest.mark.parametrize( + "event", + ["workflow_dispatch", "workflow_run", "repository_dispatch", "issues"], +) +def test_aged_orphan_events_remain_cancellable(tmp_path: Path, event: str) -> None: + """Final revalidation must not disable legacy aged-orphan queue cleanup.""" + result, cancelled = _run_aged_orphan_case(tmp_path, event=event) + assert result.returncode == 0, result.stderr + assert cancelled + + +def test_aged_orphan_that_started_running_is_preserved(tmp_path: Path) -> None: + """Aged-orphan mode applies only while the candidate is still queued.""" + result, cancelled = _run_aged_orphan_case( + tmp_path, event="workflow_dispatch", status="in_progress" + ) + assert result.returncode == 0, result.stderr + assert "no longer queued" in result.stdout + assert not cancelled From 4aa4b1558ce6fa26f3b331811b16350f8e2fd037 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 03:59:35 +0900 Subject: [PATCH 11/19] fix(scheduler): preserve aged-orphan cleanup in final revalidation --- scripts/ci/revalidate_queue_cancellation.sh | 128 +++++++++++++------- 1 file changed, 83 insertions(+), 45 deletions(-) diff --git a/scripts/ci/revalidate_queue_cancellation.sh b/scripts/ci/revalidate_queue_cancellation.sh index 9624e134f0..1a79b0ea93 100644 --- a/scripts/ci/revalidate_queue_cancellation.sh +++ b/scripts/ci/revalidate_queue_cancellation.sh @@ -1,8 +1,8 @@ #!/usr/bin/env bash set -euo pipefail -if [ "$#" -ne 5 ]; then - echo "usage: $0 " >&2 +if [ "$#" -ne 6 ]; then + echo "usage: $0 " >&2 exit 2 fi @@ -11,6 +11,15 @@ run_id="$2" default_branch="$3" classified_default_sha="$4" classified_open_pr_heads_json="$5" +cancellation_mode="$6" + +case "$cancellation_mode" in + superseded|aged-orphan) ;; + *) + echo "invalid cancellation mode: ${cancellation_mode}" >&2 + exit 2 + ;; +esac warn_preserve() { echo "::warning::Preserving run ${run_id} in ${repo_full_name}: $1" @@ -22,70 +31,99 @@ if ! run_json="$(gh api -H "Accept: application/vnd.github+json" "/repos/${repo_ fi event="$(jq -r '.event // empty' <<<"$run_json")" +status="$(jq -r '.status // empty' <<<"$run_json")" run_head="$(jq -r '.head_sha // empty' <<<"$run_json")" +run_branch="$(jq -r '.head_branch // empty' <<<"$run_json")" if ! [[ "$run_head" =~ ^[0-9a-fA-F]{40}$ ]]; then warn_preserve "live run head is malformed." fi +if [ "$cancellation_mode" = "aged-orphan" ]; then + if [ "$status" != "queued" ]; then + warn_preserve "aged-orphan candidate is no longer queued (status=${status:-})." + fi +elif [ "$status" != "queued" ] && [ "$status" != "in_progress" ]; then + warn_preserve "superseded candidate is no longer queued or in progress (status=${status:-})." +fi + case "$event" in pull_request|pull_request_target) pr_number="$(jq -r '.pull_requests[0].number // empty' <<<"$run_json")" if ! [[ "$pr_number" =~ ^[1-9][0-9]*$ ]]; then - warn_preserve "no authoritative PR identity is attached to the live run." - fi - if ! pr_json="$(gh api -H "Accept: application/vnd.github+json" "/repos/${repo_full_name}/pulls/${pr_number}")"; then - warn_preserve "live PR ${pr_number} could not be re-fetched before cancellation." - fi - live_state="$(jq -r '.state // empty' <<<"$pr_json")" - if [ "$live_state" = "open" ]; then - live_head_repo="$(jq -r '.head.repo.full_name // empty' <<<"$pr_json")" - live_head_ref="$(jq -r '.head.ref // empty' <<<"$pr_json")" - live_head_sha="$(jq -r '.head.sha // empty' <<<"$pr_json")" - if [ -z "$live_head_repo" ] || [ -z "$live_head_ref" ] || ! [[ "$live_head_sha" =~ ^[0-9a-fA-F]{40}$ ]]; then - warn_preserve "live PR ${pr_number} head metadata is malformed." + if [ "$cancellation_mode" = "aged-orphan" ]; then + # The stale candidate was selected only after the initial fleet snapshot + # proved that its head repository/ref is not a currently open PR head. + # Without an attached PR number there is no later PR authority to + # revalidate, so the still-queued aged orphan may be retired. + : + else + warn_preserve "no authoritative PR identity is attached to the live run." fi - encoded_head_ref="$(jq -rn --arg value "$live_head_ref" '$value | split("/") | map(@uri) | join("/")')" - if ! final_ref_sha="$(gh api -H "Accept: application/vnd.github+json" "/repos/${live_head_repo}/git/ref/heads/${encoded_head_ref}" --jq '.object.sha // empty')"; then - warn_preserve "live ref for PR ${pr_number} could not be re-fetched before cancellation." + else + if ! pr_json="$(gh api -H "Accept: application/vnd.github+json" "/repos/${repo_full_name}/pulls/${pr_number}")"; then + warn_preserve "live PR ${pr_number} could not be re-fetched before cancellation." fi - if ! [[ "$final_ref_sha" =~ ^[0-9a-fA-F]{40}$ ]]; then - warn_preserve "live ref for PR ${pr_number} is malformed." + live_state="$(jq -r '.state // empty' <<<"$pr_json")" + if [ "$live_state" = "open" ]; then + live_head_repo="$(jq -r '.head.repo.full_name // empty' <<<"$pr_json")" + live_head_ref="$(jq -r '.head.ref // empty' <<<"$pr_json")" + live_head_sha="$(jq -r '.head.sha // empty' <<<"$pr_json")" + if [ -z "$live_head_repo" ] || [ -z "$live_head_ref" ] || ! [[ "$live_head_sha" =~ ^[0-9a-fA-F]{40}$ ]]; then + warn_preserve "live PR ${pr_number} head metadata is malformed." + fi + encoded_head_ref="$(jq -rn --arg value "$live_head_ref" '$value | split("/") | map(@uri) | join("/")')" + if ! final_ref_sha="$(gh api -H "Accept: application/vnd.github+json" "/repos/${live_head_repo}/git/ref/heads/${encoded_head_ref}" --jq '.object.sha // empty')"; then + warn_preserve "live ref for PR ${pr_number} could not be re-fetched before cancellation." + fi + if ! [[ "$final_ref_sha" =~ ^[0-9a-fA-F]{40}$ ]]; then + warn_preserve "live ref for PR ${pr_number} is malformed." + fi + classified_sha="$(jq -r --arg key "${live_head_repo}:${live_head_ref}" '.[$key] // empty' <<<"$classified_open_pr_heads_json")" + if ! [[ "$classified_sha" =~ ^[0-9a-fA-F]{40}$ ]]; then + warn_preserve "the classification snapshot has no valid head for PR ${pr_number}." + fi + if [ "$live_head_sha" != "$classified_sha" ] || [ "$final_ref_sha" != "$classified_sha" ]; then + warn_preserve "PR ${pr_number} moved after queue classification." + fi + if [ "$run_head" = "$final_ref_sha" ]; then + echo "Preserving run ${run_id} in ${repo_full_name}: authoritative current-head evidence for PR ${pr_number}." + exit 0 + fi + elif [ "$live_state" != "closed" ]; then + warn_preserve "live PR ${pr_number} state is malformed." fi - classified_sha="$(jq -r --arg key "${live_head_repo}:${live_head_ref}" '.[$key] // empty' <<<"$classified_open_pr_heads_json")" - if ! [[ "$classified_sha" =~ ^[0-9a-fA-F]{40}$ ]]; then - warn_preserve "the classification snapshot has no valid head for PR ${pr_number}." + # A closed PR cannot supply current merge evidence. If the run is still + # active and was selected from the trusted snapshot, closure remains an + # authoritative reason to retire it. + fi + ;; + push|schedule) + if [ "$run_branch" = "$default_branch" ] || [ "$cancellation_mode" = "superseded" ]; then + if ! live_default_sha="$(gh api -H "Accept: application/vnd.github+json" "/repos/${repo_full_name}/commits/${default_branch}" --jq '.sha // empty')"; then + warn_preserve "live default-branch HEAD could not be re-fetched before cancellation." + fi + if ! [[ "$live_default_sha" =~ ^[0-9a-fA-F]{40}$ ]]; then + warn_preserve "live default-branch HEAD is malformed." fi - if [ "$live_head_sha" != "$classified_sha" ] || [ "$final_ref_sha" != "$classified_sha" ]; then - warn_preserve "PR ${pr_number} moved after queue classification." + if [ "$live_default_sha" != "$classified_default_sha" ]; then + warn_preserve "default branch moved after queue classification." fi - if [ "$run_head" = "$final_ref_sha" ]; then - echo "Preserving run ${run_id} in ${repo_full_name}: authoritative current-head evidence for PR ${pr_number}." + if [ "$run_head" = "$live_default_sha" ]; then + echo "Preserving run ${run_id} in ${repo_full_name}: authoritative current default-branch evidence." exit 0 fi - elif [ "$live_state" != "closed" ]; then - warn_preserve "live PR ${pr_number} state is malformed." - fi - ;; - push|schedule) - if ! live_default_sha="$(gh api -H "Accept: application/vnd.github+json" "/repos/${repo_full_name}/commits/${default_branch}" --jq '.sha // empty')"; then - warn_preserve "live default-branch HEAD could not be re-fetched before cancellation." - fi - if ! [[ "$live_default_sha" =~ ^[0-9a-fA-F]{40}$ ]]; then - warn_preserve "live default-branch HEAD is malformed." - fi - if [ "$live_default_sha" != "$classified_default_sha" ]; then - warn_preserve "default branch moved after queue classification." - fi - if [ "$run_head" = "$live_default_sha" ]; then - echo "Preserving run ${run_id} in ${repo_full_name}: authoritative current default-branch evidence." - exit 0 fi ;; *) - warn_preserve "event ${event:-} is outside the authoritative superseded-run contract." + if [ "$cancellation_mode" = "superseded" ]; then + warn_preserve "event ${event:-} is outside the authoritative superseded-run contract." + fi + # Aged-orphan mode intentionally retains the legacy cleanup contract for + # workflow_dispatch, workflow_run, repository_dispatch, and other queued + # events that the trusted initial snapshot proved were not current PR heads. ;; esac if ! gh api -X POST "/repos/${repo_full_name}/actions/runs/${run_id}/cancel" >/dev/null; then - echo "Could not cancel superseded run ${run_id} in ${repo_full_name}; it may have finished already." + echo "Could not cancel ${cancellation_mode} run ${run_id} in ${repo_full_name}; it may have started or finished already." fi From 7483507c8af45fae3666898e878bbe2029ffbadc Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 04:02:51 +0900 Subject: [PATCH 12/19] chore(ci): remove obsolete PR 1348 repair workflow --- .../repair-pr1348-final-revalidation.yml | 72 ------------------- 1 file changed, 72 deletions(-) delete mode 100644 .github/workflows/repair-pr1348-final-revalidation.yml diff --git a/.github/workflows/repair-pr1348-final-revalidation.yml b/.github/workflows/repair-pr1348-final-revalidation.yml deleted file mode 100644 index 002d6b164e..0000000000 --- a/.github/workflows/repair-pr1348-final-revalidation.yml +++ /dev/null @@ -1,72 +0,0 @@ -name: Repair PR 1348 final queue revalidation - -on: - push: - branches: - - fix/queue-hygiene-live-ref-race - -permissions: - contents: write - -concurrency: - group: repair-pr1348-final-revalidation - cancel-in-progress: false - -jobs: - repair: - if: github.actor != 'github-actions[bot]' - runs-on: ubuntu-24.04 - steps: - - name: Checkout exact writer branch - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - with: - ref: fix/queue-hygiene-live-ref-race - fetch-depth: 0 - - name: Apply exact source and traceability repair - shell: bash - run: | - set -euo pipefail - python3 - <<'PY' - from pathlib import Path - workflow_path = Path('.github/workflows/pr-review-merge-scheduler.yml') - workflow = workflow_path.read_text(encoding='utf-8') - old1 = ' if [ "$DRY_RUN" != "true" ]; then\n while IFS= read -r run_id; do\n if ! gh api -X POST "/repos/${repo_full_name}/actions/runs/${run_id}/cancel" >/dev/null; then\n echo "Could not cancel superseded run ${run_id} in ${repo_full_name}; it may have finished already."\n fi\n done < <(jq -r \' .[].id \' <<<"$superseded_runs_json")\n fi\n'.replace("\\' .[].id \\'", "'.[].id'") - new1 = ' if [ "$DRY_RUN" != "true" ]; then\n while IFS= read -r run_id; do\n bash scripts/ci/revalidate_queue_cancellation.sh "$repo_full_name" "$run_id" "$default_branch" "$current_default_sha" "$open_pr_heads_json"\n done < <(jq -r \' .[].id \' <<<"$superseded_runs_json")\n fi\n'.replace("\\' .[].id \\'", "'.[].id'") - old2 = ' if [ "$DRY_RUN" != "true" ]; then\n while IFS= read -r run_id; do\n if ! gh api -X POST "/repos/${repo_full_name}/actions/runs/${run_id}/cancel" >/dev/null; then\n echo "Could not cancel run ${run_id} in ${repo_full_name}; it may have started or finished already."\n fi\n done < <(jq -r \' .[].id \' <<<"$stale_runs_json")\n fi\n'.replace("\\' .[].id \\'", "'.[].id'") - new2 = ' if [ "$DRY_RUN" != "true" ]; then\n while IFS= read -r run_id; do\n bash scripts/ci/revalidate_queue_cancellation.sh "$repo_full_name" "$run_id" "$default_branch" "$current_default_sha" "$open_pr_heads_json"\n done < <(jq -r \' .[].id \' <<<"$stale_runs_json")\n fi\n'.replace("\\' .[].id \\'", "'.[].id'") - if old1 not in workflow: - raise SystemExit('superseded cancellation block changed; refusing stale repair') - workflow = workflow.replace(old1, new1, 1) - if old2 not in workflow: - raise SystemExit('aged cancellation block changed; refusing stale repair') - workflow = workflow.replace(old2, new2, 1) - workflow_path.write_text(workflow, encoding='utf-8') - - baseline_path = Path('docs/product-technical-gap-baseline.md') - baseline = baseline_path.read_text(encoding='utf-8') - doc = '\n\n### PR #1348 final cancellation revalidation (2026-09-02)\n\n- **Root cause:** exact head `af519b7927225588d437fed6ee30f20e76291c3e` classified Actions runs from an initial live-ref snapshot, but destructive cancellation occurred later without a second authoritative lookup. A synchronize event between classification and cancellation could therefore turn the snapshot stale and cancel newly current exact-head evidence.\n- **Repair:** immediately before either superseded-run or aged-run cancellation, the scheduler now executes `scripts/ci/revalidate_queue_cancellation.sh`, which re-fetches live run metadata plus the PR and its final head ref (or the default-branch HEAD for push/schedule), requires the final live state to remain consistent with the classification snapshot, fails closed on unavailable/malformed/moved state, preserves current-head evidence, and only then cancels a proven predecessor.\n- **Executable evidence:** `tests/test_queue_cancellation_revalidation.py` executes the production helper against deterministic GitHub API doubles for head movement, PR/ref lookup failure, current-head preservation, and proven predecessor cancellation. Existing queue-contract tests continue to cover discovery bounds and fail-closed initial snapshot construction.\n' - if 'PR #1348 final cancellation revalidation' not in baseline: - baseline_path.write_text(baseline + doc, encoding='utf-8') - - changelog_path = Path('CHANGELOG.md') - changelog = changelog_path.read_text(encoding='utf-8') - change = '\n- Revalidate queue-hygiene cancellation candidates immediately before destructive cancellation, failing closed on moved or unavailable live PR/ref state and preserving exact-head evidence (`#1348`).\n' - if 'Revalidate queue-hygiene cancellation candidates immediately before destructive cancellation' not in changelog: - changelog_path.write_text(changelog + change, encoding='utf-8') - PY - python3 -m pytest -q tests/test_queue_cancellation_revalidation.py tests/test_required_workflow_queue_contract.py - if command -v actionlint >/dev/null 2>&1; then - actionlint .github/workflows/pr-review-merge-scheduler.yml - fi - git diff --check - - name: Commit verified repair and remove temporary driver - shell: bash - run: | - set -euo pipefail - git config user.name github-actions[bot] - git config user.email 41898282+github-actions[bot]@users.noreply.github.com - git add .github/workflows/pr-review-merge-scheduler.yml docs/product-technical-gap-baseline.md CHANGELOG.md - git rm -f .github/workflows/repair-pr1348-final-revalidation.yml .github/pr1348-repair.trigger - git diff --cached --check - git commit -m 'fix(scheduler): revalidate live state before cancellation [pr1348-repair]' - git push origin HEAD:fix/queue-hygiene-live-ref-race From 64eef29431f33891fb5d5f6808d150b20efba87e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 04:02:59 +0900 Subject: [PATCH 13/19] chore(ci): remove obsolete PR 1348 repair trigger --- .github/pr1348-repair.trigger | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 .github/pr1348-repair.trigger diff --git a/.github/pr1348-repair.trigger b/.github/pr1348-repair.trigger deleted file mode 100644 index 9e66ea71d2..0000000000 --- a/.github/pr1348-repair.trigger +++ /dev/null @@ -1,2 +0,0 @@ -source-head=03f87fa87a6a6b9b3e0ec1a4461df46fc01e83e2 -reason=final-live-state-revalidation From e417a0c0add5874889163f456f105bb85b015bc7 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 05:47:03 +0900 Subject: [PATCH 14/19] test(queue): cover late open-PR association before cancellation --- ...queue_cancellation_open_pr_revalidation.py | 94 +++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 tests/test_queue_cancellation_open_pr_revalidation.py diff --git a/tests/test_queue_cancellation_open_pr_revalidation.py b/tests/test_queue_cancellation_open_pr_revalidation.py new file mode 100644 index 0000000000..af7dcd20e0 --- /dev/null +++ b/tests/test_queue_cancellation_open_pr_revalidation.py @@ -0,0 +1,94 @@ +"""Regression for aged PR-run cancellation after late PR association.""" + +from __future__ import annotations + +import json +import os +import shutil +import subprocess +from pathlib import Path + +import pytest + + +REPO_ROOT = Path(__file__).resolve().parents[1] +SCRIPT = REPO_ROOT / "scripts" / "ci" / "revalidate_queue_cancellation.sh" + + +def test_aged_unassociated_pr_run_rechecks_open_pr_heads(tmp_path: Path) -> None: + """A PR that appears after classification must preserve its current-head run.""" + if shutil.which("jq") is None: + pytest.skip("jq is required for the queue-cancellation regression") + + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + cancelled = tmp_path / "cancelled" + current = "b" * 40 + run_payload = json.dumps( + { + "event": "pull_request", + "status": "queued", + "head_sha": current, + "head_branch": "feature/late-pr", + "head_repository": {"full_name": "ContextualWisdomLab/example"}, + "pull_requests": [], + }, + separators=(",", ":"), + ) + open_prs = json.dumps( + [ + { + "state": "open", + "head": { + "repo": {"full_name": "ContextualWisdomLab/example"}, + "ref": "feature/late-pr", + "sha": current, + }, + } + ], + separators=(",", ":"), + ) + fake_gh = bin_dir / "gh" + fake_gh.write_text( + f"""#!/usr/bin/env bash +set -euo pipefail +args="$*" +if [[ "$args" == *"/actions/runs/77/cancel"* ]]; then + : > {cancelled!s} + exit 0 +fi +if [[ "$args" == *"/actions/runs/77"* ]]; then + printf '%s\\n' '{run_payload}' + exit 0 +fi +if [[ "$args" == *"/pulls?state=open&per_page=100"* ]]; then + printf '%s\\n' '{open_prs}' + exit 0 +fi +exit 79 +""", + encoding="utf-8", + ) + fake_gh.chmod(0o755) + env = os.environ.copy() + env["PATH"] = f"{bin_dir}{os.pathsep}{env['PATH']}" + result = subprocess.run( + [ + "bash", + str(SCRIPT), + "ContextualWisdomLab/example", + "77", + "main", + "d" * 40, + "{}", + "aged-orphan", + ], + capture_output=True, + text=True, + env=env, + check=False, + ) + + assert result.returncode == 0, result.stderr + assert "became associated with an open PR" in result.stdout + assert not cancelled From c8b086c5a77c129a72e7f3ea5ace01e3b7d4e476 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 05:47:39 +0900 Subject: [PATCH 15/19] fix(queue): revalidate late PR association before cancellation --- scripts/ci/revalidate_queue_cancellation.sh | 43 ++++++++++++++++++--- 1 file changed, 38 insertions(+), 5 deletions(-) diff --git a/scripts/ci/revalidate_queue_cancellation.sh b/scripts/ci/revalidate_queue_cancellation.sh index 1a79b0ea93..22c362b565 100644 --- a/scripts/ci/revalidate_queue_cancellation.sh +++ b/scripts/ci/revalidate_queue_cancellation.sh @@ -34,6 +34,7 @@ event="$(jq -r '.event // empty' <<<"$run_json")" status="$(jq -r '.status // empty' <<<"$run_json")" run_head="$(jq -r '.head_sha // empty' <<<"$run_json")" run_branch="$(jq -r '.head_branch // empty' <<<"$run_json")" +run_head_repo="$(jq -r '.head_repository.full_name // empty' <<<"$run_json")" if ! [[ "$run_head" =~ ^[0-9a-fA-F]{40}$ ]]; then warn_preserve "live run head is malformed." fi @@ -51,11 +52,43 @@ case "$event" in pr_number="$(jq -r '.pull_requests[0].number // empty' <<<"$run_json")" if ! [[ "$pr_number" =~ ^[1-9][0-9]*$ ]]; then if [ "$cancellation_mode" = "aged-orphan" ]; then - # The stale candidate was selected only after the initial fleet snapshot - # proved that its head repository/ref is not a currently open PR head. - # Without an attached PR number there is no later PR authority to - # revalidate, so the still-queued aged orphan may be retired. - : + # Association metadata on an Actions run can lag the PR itself. Re-read + # every open PR immediately before destructive cancellation so a PR + # created/associated after the initial sweep snapshot cannot lose its + # sole current-head evidence. Any incomplete evidence fails closed. + if [ -z "$run_head_repo" ] || [ -z "$run_branch" ]; then + warn_preserve "unassociated PR run has no authoritative head repository/ref." + fi + if ! fresh_open_pr_heads_json="$( + gh api \ + -H "Accept: application/vnd.github+json" \ + "/repos/${repo_full_name}/pulls?state=open&per_page=100" \ + --paginate \ + | jq -sc '[.[] | .[] | { + repo: (.head.repo.full_name // null), + ref: (.head.ref // null), + sha: (.head.sha // null) + }]' + )"; then + warn_preserve "open PR heads could not be re-fetched for an unassociated PR run." + fi + if ! jq -e ' + all(.[ ]; + (.repo | type) == "string" and (.repo | length) > 0 and + (.ref | type) == "string" and (.ref | length) > 0 and + (.sha | type) == "string" and (.sha | test("^[0-9a-fA-F]{40}$")) + ) + ' <<<"$fresh_open_pr_heads_json" >/dev/null; then + warn_preserve "fresh open PR head evidence is malformed." + fi + if jq -e \ + --arg repo "$run_head_repo" \ + --arg ref "$run_branch" \ + --arg sha "$run_head" \ + 'any(.[ ]; .repo == $repo and .ref == $ref and .sha == $sha)' \ + <<<"$fresh_open_pr_heads_json" >/dev/null; then + warn_preserve "run became associated with an open PR after queue classification." + fi else warn_preserve "no authoritative PR identity is attached to the live run." fi From 0f390a8ad7f4f82e120d7875eb438e7bf3d05295 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 05:48:53 +0900 Subject: [PATCH 16/19] ci(repair): wire final live-state queue revalidation --- ..._temp_pr1348_final_revalidation_repair.yml | 200 ++++++++++++++++++ 1 file changed, 200 insertions(+) create mode 100644 .github/workflows/_temp_pr1348_final_revalidation_repair.yml diff --git a/.github/workflows/_temp_pr1348_final_revalidation_repair.yml b/.github/workflows/_temp_pr1348_final_revalidation_repair.yml new file mode 100644 index 0000000000..688572c36d --- /dev/null +++ b/.github/workflows/_temp_pr1348_final_revalidation_repair.yml @@ -0,0 +1,200 @@ +name: One-shot PR1348 final queue revalidation repair + +on: + push: + branches: + - fix/queue-hygiene-live-ref-race + +permissions: + contents: write + +concurrency: + group: pr1348-final-revalidation-${{ github.ref }} + cancel-in-progress: true + +jobs: + repair: + if: github.repository == 'ContextualWisdomLab/.github' && github.actor == 'seonghobae' + runs-on: ubuntu-24.04 + timeout-minutes: 20 + steps: + - name: Checkout exact writer head + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + ref: fix/queue-hygiene-live-ref-race + fetch-depth: 0 + persist-credentials: true + + - name: Replace snapshot-only cancellation with final live revalidation + shell: bash + run: | + set -euo pipefail + python - <<'PY' + from pathlib import Path + + workflow_path = Path('.github/workflows/pr-review-merge-scheduler.yml') + workflow = workflow_path.read_text(encoding='utf-8') + + start = workflow.index(' queue_hygiene_ready=true\n open_pr_heads_json="{}"\n') + end = workflow.index(' if ! current_default_sha="$(\n', start) + replacement = ''' queue_hygiene_ready=true + open_pr_heads_json="{}" + if open_pr_payload_json="$( + gh api \\ + -H "Accept: application/vnd.github+json" \\ + "/repos/${repo_full_name}/pulls?state=open&per_page=100" \\ + --paginate \\ + | jq -sc '[.[] | .[]]' + )"; then + if ! jq -e ' + all(.[]; + (.head.repo.full_name | type) == "string" and (.head.repo.full_name | length) > 0 and + (.head.ref | type) == "string" and (.head.ref | length) > 0 and + (.head.sha | type) == "string" and (.head.sha | test("^[0-9a-fA-F]{40}$")) + ) + ' <<<"$open_pr_payload_json" >/dev/null; then + echo "::warning::Current-HEAD cancellation skipped for ${repo_full_name}: an open PR has malformed head metadata. No run will be cancelled from incomplete evidence." + queue_hygiene_ready=false + else + open_pr_heads_json="$( + jq -c ' + reduce .[] as $pr ({}; + . + {(($pr.head.repo.full_name + ":" + $pr.head.ref)): $pr.head.sha} + ) + ' <<<"$open_pr_payload_json" + )" + fi + else + echo "::warning::Current-HEAD cancellation skipped for ${repo_full_name}: open PR heads could not be read safely. No run will be cancelled from incomplete evidence." + queue_hygiene_ready=false + fi +''' + workflow = workflow[:start] + replacement + workflow[end:] + + old_superseded = ''' while IFS= read -r run_id; do + if ! gh api -X POST "/repos/${repo_full_name}/actions/runs/${run_id}/cancel" >/dev/null; then + echo "Could not cancel superseded run ${run_id} in ${repo_full_name}; it may have finished already." + fi + done < <(jq -r '.[].id' <<<"$superseded_runs_json") +''' + new_superseded = ''' while IFS= read -r run_id; do + scripts/ci/revalidate_queue_cancellation.sh \\ + "$repo_full_name" \\ + "$run_id" \\ + "$default_branch" \\ + "$current_default_sha" \\ + "$open_pr_heads_json" \\ + "superseded" + done < <(jq -r '.[].id' <<<"$superseded_runs_json") +''' + if workflow.count(old_superseded) != 1: + raise SystemExit('superseded cancellation anchor missing or duplicated') + workflow = workflow.replace(old_superseded, new_superseded, 1) + + old_aged = ''' while IFS= read -r run_id; do + if ! gh api -X POST "/repos/${repo_full_name}/actions/runs/${run_id}/cancel" >/dev/null; then + echo "Could not cancel run ${run_id} in ${repo_full_name}; it may have started or finished already." + fi + done < <(jq -r '.[].id' <<<"$stale_runs_json") +''' + new_aged = ''' while IFS= read -r run_id; do + scripts/ci/revalidate_queue_cancellation.sh \\ + "$repo_full_name" \\ + "$run_id" \\ + "$default_branch" \\ + "$current_default_sha" \\ + "$open_pr_heads_json" \\ + "aged-orphan" + done < <(jq -r '.[].id' <<<"$stale_runs_json") +''' + if workflow.count(old_aged) != 1: + raise SystemExit('aged cancellation anchor missing or duplicated') + workflow = workflow.replace(old_aged, new_aged, 1) + workflow_path.write_text(workflow, encoding='utf-8') + + tests_path = Path('tests/test_required_workflow_queue_contract.py') + tests = tests_path.read_text(encoding='utf-8') + + def replace_function(text: str, name: str, replacement: str) -> str: + start = text.index(f'def {name}(') + next_def = text.find('\ndef ', start + 1) + if next_def < 0: + raise SystemExit(f'no function boundary after {name}') + return text[:start] + replacement.rstrip() + '\n\n' + text[next_def + 1:] + + tests = replace_function( + tests, + 'test_org_queue_sweep_empty_pr_ref_list_skips_the_ref_loop', + '''def test_org_queue_sweep_malformed_pr_head_fails_closed_before_classification() -> None: + """Incomplete open-PR head evidence must disable destructive cancellation.""" + workflow = workflow_text("pr-review-merge-scheduler.yml") + classification = workflow.split('if open_pr_payload_json="$(\\n', 1)[1].split( + ' if ! current_default_sha="$(\\n', 1 + )[0] + + assert 'all(.[];' in classification + assert '.head.repo.full_name' in classification + assert '.head.ref' in classification + assert '.head.sha' in classification + assert 'test("^[0-9a-fA-F]{40}$")' in classification + assert 'queue_hygiene_ready=false' in classification +''', + ) + tests = replace_function( + tests, + 'test_org_queue_sweep_bounds_live_ref_lookups_and_fails_closed', + '''def test_org_queue_sweep_avoids_snapshot_ref_fanout_and_revalidates_each_cancel() -> None: + """Classification is cheap; every destructive candidate gets a final live check.""" + workflow = workflow_text("pr-review-merge-scheduler.yml") + classification = workflow.split('if open_pr_payload_json="$(\\n', 1)[1].split( + ' if ! current_default_sha="$(\\n', 1 + )[0] + + assert '/git/ref/heads/' not in classification + assert 'ORG_QUEUE_HYGIENE_MAX_REF_LOOKUPS' not in classification + assert 'scripts/ci/revalidate_queue_cancellation.sh' in workflow + assert '"superseded"' in workflow + assert '"aged-orphan"' in workflow +''', + ) + tests = replace_function( + tests, + 'test_org_queue_sweep_deduplicates_live_refs_before_counting', + '''def test_org_queue_sweep_uses_paginated_pr_snapshot_only_for_candidate_classification() -> None: + """Open-PR classification keeps full paginated coverage without a hand-authored cap.""" + workflow = workflow_text("pr-review-merge-scheduler.yml") + classification = workflow.split('if open_pr_payload_json="$(\\n', 1)[1].split( + ' if ! current_default_sha="$(\\n', 1 + )[0] + + assert '/pulls?state=open&per_page=100' in classification + assert '--paginate' in classification + assert "reduce .[] as $pr" in classification +''', + ) + tests_path.write_text(tests, encoding='utf-8') + PY + + chmod +x scripts/ci/revalidate_queue_cancellation.sh + python -m pytest -q \ + tests/test_queue_cancellation_revalidation.py \ + tests/test_queue_cancellation_open_pr_revalidation.py \ + tests/test_required_workflow_queue_contract.py + git diff --check + + - name: Commit source repair and remove writer + shell: bash + run: | + set -euo pipefail + git config user.name 'github-actions[bot]' + git config user.email '41898282+github-actions[bot]@users.noreply.github.com' + git rm .github/workflows/_temp_pr1348_final_revalidation_repair.yml + git add \ + .github/workflows/pr-review-merge-scheduler.yml \ + scripts/ci/revalidate_queue_cancellation.sh \ + tests/test_queue_cancellation_revalidation.py \ + tests/test_queue_cancellation_open_pr_revalidation.py \ + tests/test_required_workflow_queue_contract.py + git diff --cached --check + git commit -m 'fix(queue): revalidate live state immediately before cancellation' + git push origin HEAD:fix/queue-hygiene-live-ref-race From 0d7b7a113e76ff60bc364e03e350bf1810efea36 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 06:00:08 +0900 Subject: [PATCH 17/19] test(queue): reproduce stale orphan PR snapshot cancellation --- tests/test_queue_cancellation_revalidation.py | 119 ++++++++++++++++++ 1 file changed, 119 insertions(+) diff --git a/tests/test_queue_cancellation_revalidation.py b/tests/test_queue_cancellation_revalidation.py index b618b2ed52..23ac824d59 100644 --- a/tests/test_queue_cancellation_revalidation.py +++ b/tests/test_queue_cancellation_revalidation.py @@ -160,6 +160,91 @@ def _run_aged_orphan_case( return result, cancelled.exists() +def _run_unassociated_pr_aged_orphan_case( + tmp_path: Path, + *, + listed_sha: str, + ref_sha: str, + run_sha: str, + fail_ref_lookup: bool = False, +) -> tuple[subprocess.CompletedProcess[str], bool]: + """Run an unassociated aged PR run against stale listing and live-ref evidence.""" + if shutil.which("jq") is None: + pytest.skip("jq is required for the queue-cancellation regression") + + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + cancelled = tmp_path / "cancelled" + run_payload = json.dumps( + { + "event": "pull_request", + "status": "queued", + "head_sha": run_sha, + "head_branch": "feature/race", + "head_repository": {"full_name": "ContextualWisdomLab/example"}, + "pull_requests": [], + }, + separators=(",", ":"), + ) + open_pr_payload = json.dumps( + [ + { + "head": { + "repo": {"full_name": "ContextualWisdomLab/example"}, + "ref": "feature/race", + "sha": listed_sha, + } + } + ], + separators=(",", ":"), + ) + fake_gh = bin_dir / "gh" + fake_gh.write_text( + f"""#!/usr/bin/env bash +set -euo pipefail +args="$*" +if [[ "$args" == *"/actions/runs/77/cancel"* ]]; then + : > {cancelled!s} + exit 0 +fi +if [[ "$args" == *"/actions/runs/77"* ]]; then + printf '%s\\n' '{run_payload}' + exit 0 +fi +if [[ "$args" == *"/pulls?state=open&per_page=100"* ]]; then + printf '%s\\n' '{open_pr_payload}' + exit 0 +fi +if [[ "$args" == *"/git/ref/heads/feature/race"* ]]; then + {'exit 74' if fail_ref_lookup else f"printf '%s\\n' '{ref_sha}'"} + exit 0 +fi +exit 79 +""", + encoding="utf-8", + ) + fake_gh.chmod(0o755) + env = os.environ.copy() + env["PATH"] = f"{bin_dir}{os.pathsep}{env['PATH']}" + result = subprocess.run( + [ + "bash", + str(SCRIPT), + "ContextualWisdomLab/example", + "77", + "main", + "d" * 40, + "{}", + "aged-orphan", + ], + capture_output=True, + text=True, + env=env, + check=False, + ) + return result, cancelled.exists() + + def test_post_classification_head_movement_fails_closed(tmp_path: Path) -> None: """A new exact head arriving after classification must never be cancelled.""" old = "a" * 40 @@ -224,6 +309,40 @@ def test_proven_predecessor_is_cancelled(tmp_path: Path) -> None: assert cancelled +def test_unassociated_aged_pr_uses_live_ref_not_stale_listing_sha( + tmp_path: Path, +) -> None: + """A stale PR payload cannot authorize cancelling the live branch head.""" + listed = "a" * 40 + current = "b" * 40 + result, cancelled = _run_unassociated_pr_aged_orphan_case( + tmp_path, + listed_sha=listed, + ref_sha=current, + run_sha=current, + ) + assert result.returncode == 0, result.stderr + assert "authoritative current-head evidence" in result.stdout + assert not cancelled + + +def test_unassociated_aged_pr_live_ref_lookup_failure_fails_closed( + tmp_path: Path, +) -> None: + """Missing final ref evidence must preserve an unassociated PR candidate.""" + result, cancelled = _run_unassociated_pr_aged_orphan_case( + tmp_path, + listed_sha="a" * 40, + ref_sha="b" * 40, + run_sha="b" * 40, + fail_ref_lookup=True, + ) + assert result.returncode == 0, result.stderr + assert "live ref" in result.stdout + assert "could not be re-fetched" in result.stdout + assert not cancelled + + @pytest.mark.parametrize( "event", ["workflow_dispatch", "workflow_run", "repository_dispatch", "issues"], From 0338b17e6b92dbb1e15fbcd04b62e352cc554ab4 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 06:01:41 +0900 Subject: [PATCH 18/19] fix(queue): revalidate live ref for unassociated PR runs --- scripts/ci/revalidate_queue_cancellation.sh | 45 +++++++++++++-------- 1 file changed, 29 insertions(+), 16 deletions(-) diff --git a/scripts/ci/revalidate_queue_cancellation.sh b/scripts/ci/revalidate_queue_cancellation.sh index 22c362b565..e6ea81258a 100644 --- a/scripts/ci/revalidate_queue_cancellation.sh +++ b/scripts/ci/revalidate_queue_cancellation.sh @@ -52,42 +52,55 @@ case "$event" in pr_number="$(jq -r '.pull_requests[0].number // empty' <<<"$run_json")" if ! [[ "$pr_number" =~ ^[1-9][0-9]*$ ]]; then if [ "$cancellation_mode" = "aged-orphan" ]; then - # Association metadata on an Actions run can lag the PR itself. Re-read - # every open PR immediately before destructive cancellation so a PR - # created/associated after the initial sweep snapshot cannot lose its - # sole current-head evidence. Any incomplete evidence fails closed. + # Association metadata on an Actions run can lag the PR itself. The + # refreshed PR list is only discovery evidence for repository/ref + # identity: its head SHA can itself lag a synchronize event. Resolve + # the matching Git reference immediately before cancellation and make + # the destructive decision from that authoritative value instead. if [ -z "$run_head_repo" ] || [ -z "$run_branch" ]; then warn_preserve "unassociated PR run has no authoritative head repository/ref." fi - if ! fresh_open_pr_heads_json="$( + if ! fresh_open_pr_refs_json="$( gh api \ -H "Accept: application/vnd.github+json" \ "/repos/${repo_full_name}/pulls?state=open&per_page=100" \ --paginate \ | jq -sc '[.[] | .[] | { repo: (.head.repo.full_name // null), - ref: (.head.ref // null), - sha: (.head.sha // null) + ref: (.head.ref // null) }]' )"; then - warn_preserve "open PR heads could not be re-fetched for an unassociated PR run." + warn_preserve "open PR refs could not be re-fetched for an unassociated PR run." fi if ! jq -e ' all(.[ ]; (.repo | type) == "string" and (.repo | length) > 0 and - (.ref | type) == "string" and (.ref | length) > 0 and - (.sha | type) == "string" and (.sha | test("^[0-9a-fA-F]{40}$")) + (.ref | type) == "string" and (.ref | length) > 0 ) - ' <<<"$fresh_open_pr_heads_json" >/dev/null; then - warn_preserve "fresh open PR head evidence is malformed." + ' <<<"$fresh_open_pr_refs_json" >/dev/null; then + warn_preserve "fresh open PR ref evidence is malformed." fi if jq -e \ --arg repo "$run_head_repo" \ --arg ref "$run_branch" \ - --arg sha "$run_head" \ - 'any(.[ ]; .repo == $repo and .ref == $ref and .sha == $sha)' \ - <<<"$fresh_open_pr_heads_json" >/dev/null; then - warn_preserve "run became associated with an open PR after queue classification." + 'any(.[ ]; .repo == $repo and .ref == $ref)' \ + <<<"$fresh_open_pr_refs_json" >/dev/null; then + encoded_run_ref="$(jq -rn --arg value "$run_branch" '$value | split("/") | map(@uri) | join("/")')" + if ! final_run_ref_sha="$( + gh api \ + -H "Accept: application/vnd.github+json" \ + "/repos/${run_head_repo}/git/ref/heads/${encoded_run_ref}" \ + --jq '.object.sha // empty' + )"; then + warn_preserve "live ref for an unassociated PR run could not be re-fetched before cancellation." + fi + if ! [[ "$final_run_ref_sha" =~ ^[0-9a-fA-F]{40}$ ]]; then + warn_preserve "live ref for an unassociated PR run is malformed." + fi + if [ "$run_head" = "$final_run_ref_sha" ]; then + echo "Preserving run ${run_id} in ${repo_full_name}: authoritative current-head evidence for a newly associated open PR." + exit 0 + fi fi else warn_preserve "no authoritative PR identity is attached to the live run." From db86866d3230b474be79d041eac2299fce4c7b12 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 06:03:41 +0900 Subject: [PATCH 19/19] chore(queue): remove completed one-shot repair writer --- ..._temp_pr1348_final_revalidation_repair.yml | 200 ------------------ 1 file changed, 200 deletions(-) delete mode 100644 .github/workflows/_temp_pr1348_final_revalidation_repair.yml diff --git a/.github/workflows/_temp_pr1348_final_revalidation_repair.yml b/.github/workflows/_temp_pr1348_final_revalidation_repair.yml deleted file mode 100644 index 688572c36d..0000000000 --- a/.github/workflows/_temp_pr1348_final_revalidation_repair.yml +++ /dev/null @@ -1,200 +0,0 @@ -name: One-shot PR1348 final queue revalidation repair - -on: - push: - branches: - - fix/queue-hygiene-live-ref-race - -permissions: - contents: write - -concurrency: - group: pr1348-final-revalidation-${{ github.ref }} - cancel-in-progress: true - -jobs: - repair: - if: github.repository == 'ContextualWisdomLab/.github' && github.actor == 'seonghobae' - runs-on: ubuntu-24.04 - timeout-minutes: 20 - steps: - - name: Checkout exact writer head - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - ref: fix/queue-hygiene-live-ref-race - fetch-depth: 0 - persist-credentials: true - - - name: Replace snapshot-only cancellation with final live revalidation - shell: bash - run: | - set -euo pipefail - python - <<'PY' - from pathlib import Path - - workflow_path = Path('.github/workflows/pr-review-merge-scheduler.yml') - workflow = workflow_path.read_text(encoding='utf-8') - - start = workflow.index(' queue_hygiene_ready=true\n open_pr_heads_json="{}"\n') - end = workflow.index(' if ! current_default_sha="$(\n', start) - replacement = ''' queue_hygiene_ready=true - open_pr_heads_json="{}" - if open_pr_payload_json="$( - gh api \\ - -H "Accept: application/vnd.github+json" \\ - "/repos/${repo_full_name}/pulls?state=open&per_page=100" \\ - --paginate \\ - | jq -sc '[.[] | .[]]' - )"; then - if ! jq -e ' - all(.[]; - (.head.repo.full_name | type) == "string" and (.head.repo.full_name | length) > 0 and - (.head.ref | type) == "string" and (.head.ref | length) > 0 and - (.head.sha | type) == "string" and (.head.sha | test("^[0-9a-fA-F]{40}$")) - ) - ' <<<"$open_pr_payload_json" >/dev/null; then - echo "::warning::Current-HEAD cancellation skipped for ${repo_full_name}: an open PR has malformed head metadata. No run will be cancelled from incomplete evidence." - queue_hygiene_ready=false - else - open_pr_heads_json="$( - jq -c ' - reduce .[] as $pr ({}; - . + {(($pr.head.repo.full_name + ":" + $pr.head.ref)): $pr.head.sha} - ) - ' <<<"$open_pr_payload_json" - )" - fi - else - echo "::warning::Current-HEAD cancellation skipped for ${repo_full_name}: open PR heads could not be read safely. No run will be cancelled from incomplete evidence." - queue_hygiene_ready=false - fi -''' - workflow = workflow[:start] + replacement + workflow[end:] - - old_superseded = ''' while IFS= read -r run_id; do - if ! gh api -X POST "/repos/${repo_full_name}/actions/runs/${run_id}/cancel" >/dev/null; then - echo "Could not cancel superseded run ${run_id} in ${repo_full_name}; it may have finished already." - fi - done < <(jq -r '.[].id' <<<"$superseded_runs_json") -''' - new_superseded = ''' while IFS= read -r run_id; do - scripts/ci/revalidate_queue_cancellation.sh \\ - "$repo_full_name" \\ - "$run_id" \\ - "$default_branch" \\ - "$current_default_sha" \\ - "$open_pr_heads_json" \\ - "superseded" - done < <(jq -r '.[].id' <<<"$superseded_runs_json") -''' - if workflow.count(old_superseded) != 1: - raise SystemExit('superseded cancellation anchor missing or duplicated') - workflow = workflow.replace(old_superseded, new_superseded, 1) - - old_aged = ''' while IFS= read -r run_id; do - if ! gh api -X POST "/repos/${repo_full_name}/actions/runs/${run_id}/cancel" >/dev/null; then - echo "Could not cancel run ${run_id} in ${repo_full_name}; it may have started or finished already." - fi - done < <(jq -r '.[].id' <<<"$stale_runs_json") -''' - new_aged = ''' while IFS= read -r run_id; do - scripts/ci/revalidate_queue_cancellation.sh \\ - "$repo_full_name" \\ - "$run_id" \\ - "$default_branch" \\ - "$current_default_sha" \\ - "$open_pr_heads_json" \\ - "aged-orphan" - done < <(jq -r '.[].id' <<<"$stale_runs_json") -''' - if workflow.count(old_aged) != 1: - raise SystemExit('aged cancellation anchor missing or duplicated') - workflow = workflow.replace(old_aged, new_aged, 1) - workflow_path.write_text(workflow, encoding='utf-8') - - tests_path = Path('tests/test_required_workflow_queue_contract.py') - tests = tests_path.read_text(encoding='utf-8') - - def replace_function(text: str, name: str, replacement: str) -> str: - start = text.index(f'def {name}(') - next_def = text.find('\ndef ', start + 1) - if next_def < 0: - raise SystemExit(f'no function boundary after {name}') - return text[:start] + replacement.rstrip() + '\n\n' + text[next_def + 1:] - - tests = replace_function( - tests, - 'test_org_queue_sweep_empty_pr_ref_list_skips_the_ref_loop', - '''def test_org_queue_sweep_malformed_pr_head_fails_closed_before_classification() -> None: - """Incomplete open-PR head evidence must disable destructive cancellation.""" - workflow = workflow_text("pr-review-merge-scheduler.yml") - classification = workflow.split('if open_pr_payload_json="$(\\n', 1)[1].split( - ' if ! current_default_sha="$(\\n', 1 - )[0] - - assert 'all(.[];' in classification - assert '.head.repo.full_name' in classification - assert '.head.ref' in classification - assert '.head.sha' in classification - assert 'test("^[0-9a-fA-F]{40}$")' in classification - assert 'queue_hygiene_ready=false' in classification -''', - ) - tests = replace_function( - tests, - 'test_org_queue_sweep_bounds_live_ref_lookups_and_fails_closed', - '''def test_org_queue_sweep_avoids_snapshot_ref_fanout_and_revalidates_each_cancel() -> None: - """Classification is cheap; every destructive candidate gets a final live check.""" - workflow = workflow_text("pr-review-merge-scheduler.yml") - classification = workflow.split('if open_pr_payload_json="$(\\n', 1)[1].split( - ' if ! current_default_sha="$(\\n', 1 - )[0] - - assert '/git/ref/heads/' not in classification - assert 'ORG_QUEUE_HYGIENE_MAX_REF_LOOKUPS' not in classification - assert 'scripts/ci/revalidate_queue_cancellation.sh' in workflow - assert '"superseded"' in workflow - assert '"aged-orphan"' in workflow -''', - ) - tests = replace_function( - tests, - 'test_org_queue_sweep_deduplicates_live_refs_before_counting', - '''def test_org_queue_sweep_uses_paginated_pr_snapshot_only_for_candidate_classification() -> None: - """Open-PR classification keeps full paginated coverage without a hand-authored cap.""" - workflow = workflow_text("pr-review-merge-scheduler.yml") - classification = workflow.split('if open_pr_payload_json="$(\\n', 1)[1].split( - ' if ! current_default_sha="$(\\n', 1 - )[0] - - assert '/pulls?state=open&per_page=100' in classification - assert '--paginate' in classification - assert "reduce .[] as $pr" in classification -''', - ) - tests_path.write_text(tests, encoding='utf-8') - PY - - chmod +x scripts/ci/revalidate_queue_cancellation.sh - python -m pytest -q \ - tests/test_queue_cancellation_revalidation.py \ - tests/test_queue_cancellation_open_pr_revalidation.py \ - tests/test_required_workflow_queue_contract.py - git diff --check - - - name: Commit source repair and remove writer - shell: bash - run: | - set -euo pipefail - git config user.name 'github-actions[bot]' - git config user.email '41898282+github-actions[bot]@users.noreply.github.com' - git rm .github/workflows/_temp_pr1348_final_revalidation_repair.yml - git add \ - .github/workflows/pr-review-merge-scheduler.yml \ - scripts/ci/revalidate_queue_cancellation.sh \ - tests/test_queue_cancellation_revalidation.py \ - tests/test_queue_cancellation_open_pr_revalidation.py \ - tests/test_required_workflow_queue_contract.py - git diff --cached --check - git commit -m 'fix(queue): revalidate live state immediately before cancellation' - git push origin HEAD:fix/queue-hygiene-live-ref-race