diff --git a/.github/workflows/codeql-scan-dispatch.yml b/.github/workflows/codeql-scan-dispatch.yml index afb51676be..b698f6154e 100644 --- a/.github/workflows/codeql-scan-dispatch.yml +++ b/.github/workflows/codeql-scan-dispatch.yml @@ -589,7 +589,7 @@ jobs: | select(.event == "pull_request") | select(.path == ".github/workflows/codeql-pr.yml") | select(.head_sha == $head) - | select(.status == "completed") + | select(.status == "completed" and .conclusion == "failure") | select([ .pull_requests[]? | select(.number == $pr_number and .head.sha == $head and .base.ref == $base_ref and .base.sha == $base) @@ -601,22 +601,22 @@ jobs: exit 1 fi - while IFS= read -r required_job; do - required_language="$(jq -r '.language' <<<"$required_job")" - required_job_id="$(jq -r '.job_id | tostring' <<<"$required_job")" - expected_name="CodeQL compatibility analysis (${required_language})" - job="$(gh api "repos/${TARGET_REPOSITORY}/actions/jobs/${required_job_id}")" - job_identity="$(printf '%s' "$job" | jq -r --arg head "$HEAD_SHA" --arg name "$expected_name" --argjson run_id "$REQUIRED_RUN_ID" --argjson job_id "$required_job_id" ' - select(.id == $job_id and .run_id == $run_id and .head_sha == $head) - | select(.name == $name) - | select(.status == "completed" and .conclusion == "failure") - | .id // empty - ')" - if [ "$job_identity" != "$required_job_id" ]; then - echo "::error::CodeQL wake rejected missing or ambiguous required job identity." - exit 1 - fi - done < <(printf '%s' "$REQUIRED_JOBS" | jq -c '.[]') + run_jobs="$(gh api --paginate "repos/${TARGET_REPOSITORY}/actions/runs/${REQUIRED_RUN_ID}/jobs" | jq -s '{jobs: map(.jobs[]?)}')" + expected_failed_ids="$(printf '%s' "$REQUIRED_JOBS" | jq -c ' + [ .[] + | select((.language | type) == "string") + | select(.language | test("^[a-z0-9-]+$")) + | select((.job_id | tostring) | test("^[1-9][0-9]*$")) + | .job_id | tonumber + ] | sort + ')" + observed_failed_ids="$(printf '%s' "$run_jobs" | jq -c ' + [ .jobs[] | select(.status == "completed" and .conclusion == "failure") | .id ] | sort + ')" + if [ "$expected_failed_ids" != "$observed_failed_ids" ]; then + echo "::error::CodeQL wake rejected a run whose bound failed CodeQL jobs do not exactly match its failed jobs." + exit 1 + fi gh api -X POST "repos/${TARGET_REPOSITORY}/actions/runs/${REQUIRED_RUN_ID}/rerun-failed-jobs" >/dev/null echo "Re-ran failed jobs in exact CodeQL run ${REQUIRED_RUN_ID} for ${HEAD_SHA} on base ${BASE_REF}@${BASE_SHA} after all dispatch shards completed." diff --git a/AGENTS.md b/AGENTS.md index ff01a0aecf..eef7ed2e34 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -60,6 +60,10 @@ The materialization contract is also covered by [`docs/doctoring/exact-artifact- head. If a current-head dispatch is cancelled while deduplicating, enqueue exactly one replacement for that PR and workflow and verify the replacement carries the same live target head. +- A `repository_dispatch` handler executes its default-branch workflow, not an + open control-plane PR's file. Do not add `workflow_dispatch` merely to test a + privileged branch implementation. Use fixture-backed contracts before merge, + then verify the first default-branch dispatch after protected integration. - CodeQL language shards do not wake required jobs independently. After the complete `scan` matrix terminates, one `wake-required-codeql` coordinator revalidates the live PR and exact required run before one run-level @@ -202,6 +206,11 @@ them alone proves succession. for every literal you touched — event-type strings, cron expressions, environment-variable names, tuple members, pinned digests — not only the obviously named sibling test. A change can satisfy one oracle and still leave a second, independent one stale. +- For a multi-language CodeQL dispatch, do not rerun the required jobs from + matrix shards. After every shard has produced its terminal gate outcome, + verify that the original run's complete failed-job set exactly matches the + authenticated CodeQL binding, then issue one run-level failed-jobs rerun. + GitHub rejects the second concurrent job rerun with HTTP 403. - Read a stale pull request's own changes with a three-dot diff — `git diff ...` — or with `gh pr diff`, which is already three-dot. A two-dot `git diff ` renders everything the base gained since the fork point as though @@ -225,7 +234,9 @@ them alone proves succession. - A successful rerun of one matrix job does not rerun its sibling matrix jobs. Therefore an `already running` response from a second per-job rerun must remain a failure: even if the shared run is active, that sibling can still retain its old failed verdict. Coordinate the wake only - after all dispatch shards publish, then rerun the exact run's failed jobs as one operation. + after all dispatch shards terminate, including a scan failure whose exact verdict must be + reflected by the required check; exclude cancellation because it has no complete evidence. + Then rerun the exact run's failed jobs as one operation. - A `codeql-dispatch/` commit status is head-scoped and carries neither the PR base nor the required-run identity. Keep it as diagnostic output only. Shards and the coordinator may accept a terminal verdict only from a completed central dispatch run named with diff --git a/CHANGELOG.md b/CHANGELOG.md index f2775e20ce..6f3ce3c2d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,9 @@ while a sibling language scan is still running. The coordinator now keeps an active run with the same immutable repository, PR, head, base, and required run identity, avoiding same-PR cancellation of valid evidence. +- The run-level wake starts after a terminal scan matrix whether its verdict is + success or failure, so exact failure evidence reaches the required check; a + cancelled matrix remains excluded because it has no complete verdict. ### Failed-check finding names the Strix sandbox instead of the gateway @@ -69,6 +72,13 @@ ### CodeQL scan dispatch matrix serialisation +- Documented the CodeQL wake bootstrap boundary: a `repository_dispatch` run + executes the default-branch handler, so an open control-plane PR cannot + exercise its proposed privileged workflow by selecting its branch. The + pre-merge evidence is fixture-backed contracts and actionlint; the first + protected default-branch dispatch is the required live proof. No + `workflow_dispatch` bypass was added. + - Serialised the dispatched CodeQL matrix with `toJSON()` in `codeql-scan-dispatch.yml`. `codeql-pr.yml` sends `client_payload.matrix` as an array and the handler assigned it straight into `env:`, where a value must be a scalar, so GitHub rejected the step with "A sequence was not expected" and the dispatched scan never ran -- 0 successes against 136 failures since the handler was added in #1776. The validate step already consumes the value through `jq`, so JSON text is the shape it was written for and no consumer changes. Added a string contract test, because neither `yaml.safe_load` nor `actionlint` 1.7.12 flags this: it is an Actions template rule, so only GitHub's own validator rejects it and no local gate catches the class. ### Contextual-orchestrator pin refresh @@ -84,6 +94,11 @@ - Raised `hourly-review-repair.yml`'s discovery ceiling from 50 to 200 while rotating deterministic 50-PR deep-inspection windows by hourly run number. The scheduler hydrates only the selected window and stops immediately after its single dispatch, preserving access to newer PRs without quadrupling expensive review/check/comment work. See `docs/doctoring/hourly-review-repair-single-file-consolidation.md`'s 2026-09-03 follow-up. ## [Unreleased] +- Serialize multi-language CodeQL dispatch wakeups after the scan matrix has + completed. The owner now proves that the original exact-head run's entire + failed-job set equals the authenticated CodeQL job binding before issuing one + failed-jobs rerun, preventing the second shard from receiving GitHub's + `workflow run already running` 403. - Include merge-scheduler entrypoint, core, and regression-test changes in the existing runtime-quality workflow's trigger and suite selector. Scheduler workflow edits retain queue checks and also select the full review-repair diff --git a/CLAUDE.md b/CLAUDE.md index 115456763f..2e39d3fd13 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -18,6 +18,9 @@ configuring any such loop. The repo/Project — not private agent memory — is the source of truth. This file complements those documents; it does not replace them. +`repository_dispatch` executes the default-branch workflow. An open central workflow PR therefore +needs fixture-backed contracts before merge and a fresh default-branch dispatch after protected +integration; never add branch-selected `workflow_dispatch` to bypass that boundary. For CodeQL's dispatch-and-wake loop, wait for the complete scan matrix, then let one `wake-required-codeql` coordinator revalidate the live PR and exact required run before one run-level failed-job rerun. The wake identity includes PR number, @@ -144,6 +147,10 @@ repeatable compile command. needs the same evidence. - **100% coverage and 100% docstrings on `scripts/ci/`** are hard gates, not aspirations. New helper code needs matching tests and docstrings. +- **Multi-language CodeQL dispatch wakeups are run-level, not shard-level.** Wait for every + matrix shard, prove the failed-job set equals the authenticated binding, then rerun failed jobs + once. Concurrent per-job reruns make the first request reactivate the run and GitHub rejects the + second with HTTP 403. - **Product hourly callers** stay thin. Do not hard-code OriginWeave, aFIPC, naruon, or Keyverse into `pr-review-fix-scheduler.yml`. The model credential remains `NVIDIA_NIM_API_KEY` on the worker, never `COPILOT_GITHUB_TOKEN`. @@ -229,7 +236,8 @@ repeatable compile command. branch or from the autofix flow's conflict-marker resolution. - **Per-job reruns do not cover matrix siblings.** Do not accept a second shard's `already running` response merely because the shared run is active. Coordinate after every dispatch shard has - published its verdict and wake the exact run's failed jobs once, so no sibling retains a stale + terminated and wake the exact run's failed jobs once, so a scan failure is reflected rather than + suppressed and no sibling retains a stale failed required check. The wake is valid only while PR number, head SHA, and base SHA still match both live PR metadata and the exact required run's `pull_requests[]` association. - **Head-only CodeQL statuses are diagnostic, not terminal authority.** They cannot distinguish diff --git a/docs/adr/0025-codeql-required-workflow-dispatch-architecture.md b/docs/adr/0025-codeql-required-workflow-dispatch-architecture.md index 4a84c800f5..518e666212 100644 --- a/docs/adr/0025-codeql-required-workflow-dispatch-architecture.md +++ b/docs/adr/0025-codeql-required-workflow-dispatch-architecture.md @@ -191,8 +191,9 @@ forbidden. The 2026-09-05 rejection of "full matrix in one dispatch" is therefore superseded. Siblings are jobs in one run, not runs in one concurrency group. `required_jobs` remains a 1:1 map of language to canonical job id; the -post-matrix wake validates the whole map and every exact job before one -run-level rerun. A missing, stale, or mismatched identity fails closed. The +post-matrix wake requires the whole map to equal the run's complete failed-job +id set before one run-level rerun. A missing, stale, or mismatched identity +fails closed. The old scalar `required_job_id`/`required_language` payload is retained only as a bounded queued-payload compatibility path where the matrix has exactly one language; it is not the current producer contract. diff --git a/docs/doctoring/codeql-dispatch-multi-language-wake-race.md b/docs/doctoring/codeql-dispatch-multi-language-wake-race.md new file mode 100644 index 0000000000..822dbc47fa --- /dev/null +++ b/docs/doctoring/codeql-dispatch-multi-language-wake-race.md @@ -0,0 +1,51 @@ +# CodeQL dispatch multi-language wake race + +## Incident evidence + +On 2026-09-09, `.github` PR #1857 at +`afeffe3b6a7a5494be1dae12322a0fc2a78c6efe` dispatched run `34320386978` +after both initial CodeQL required jobs reported pending. Its `actions` scan +completed successfully, but the subsequent Python wake failed with GitHub's +`The workflow run containing this job is already running` HTTP 403. The first +per-language rerun had already reactivated the shared required-workflow run. + +The scan itself was not the failing evidence: both the CodeQL analysis and the +Medium-or-higher SARIF gate completed. A status publication 403 is separately +recorded as non-terminal for a clean scan, as required by the existing status +publication contract. + +## Corrective action + +`codeql-scan-dispatch.yml` now waits for the complete matrix and uses one +run-level `rerun-failed-jobs` call. Before that mutation, it revalidates the +open PR and exact head, validates the required workflow run identity, fetches +all jobs, and refuses unless the full failed-job identifier set exactly equals +the authenticated CodeQL binding. This preserves the no-unrelated-job +invariant without relying on concurrent per-job reruns. + +## Verification and recovery + +`tests/test_codeql_scan_dispatch_workflow_contract.py` executes the wake +block against fixture-backed GitHub responses. It covers the bounded job set, +stale and closed pull requests, a mismatched or nonfailed job set, and an +already-running required workflow. Hosted exact-head evidence remains required +before any PR or consumer result is treated as successful. If the binding does +not match, do not retry a job manually; inspect the exact run and dispatch a +new current-head scan only through the owner workflow. + +### Bootstrap boundary + +On 2026-09-09, PR `#2056` at +`aad55ed864db3466d52d24dcd88a80d15e84996d` triggered dispatch run +`34322210652` for required run `34321725703`. The run executed the protected +default-branch handler, as GitHub defines for `repository_dispatch`; its log +therefore used the old per-job wake and reproduced the same HTTP 403 when the +second shard tried to rerun an already-running required workflow. This does +not execute or disprove #2056's proposed run-level wake. + +The branch cannot use `workflow_dispatch` as a substitute: that would let a +caller select an unprotected workflow ref while minting privileged cross-repo +credentials, and the central queue contract forbids it. The pre-merge proof is +the fixture-backed contract plus actionlint. After protected integration, a +fresh default-branch dispatch must prove the one-call wake against an exact +current head before a consumer adopts the owner. diff --git a/docs/doctoring/codeql-partial-shard-wake-duplicate-dispatch.md b/docs/doctoring/codeql-partial-shard-wake-duplicate-dispatch.md index f5e5267e84..6bce1e3b9d 100644 --- a/docs/doctoring/codeql-partial-shard-wake-duplicate-dispatch.md +++ b/docs/doctoring/codeql-partial-shard-wake-duplicate-dispatch.md @@ -41,7 +41,7 @@ wake block을 fixture-backed `gh api`로 실행하면 두 경우 모두 return c - live PR은 open이고 `base.sha == BASE_SHA`, `head.sha == HEAD_SHA`여야 한다. - exact `REQUIRED_RUN_ID`는 pull_request event의 `codeql-pr.yml` completed run이며 `pull_requests[]` 안에 같은 PR number/head/base ref/base SHA tuple이 정확히 하나 있어야 한다. -- 그 뒤에만 기존 failed-job id/name/run/head 검증과 run-level +- 그 뒤에만 complete bound failed-job-id 집합 검증과 run-level `rerun-failed-jobs`가 실행된다. 같은 fixture를 repaired block에 적용하면 두 changed-base 경로 모두 return diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 27a5539de9..688b8c312c 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -10,9 +10,10 @@ 같은 exact head의 CodeQL dispatch `34316388553`에서는 Python shard가 성공한 뒤 required job을 깨웠고, Actions shard가 분석 중일 때 동일 제목의 dispatch `34317266381`가 생성됐다. 같은 PR concurrency가 첫 실행을 취소해 -Actions SARIF가 사라졌다. 중앙 coordinator는 이제 repository·PR·head·base· -required run id가 모두 같은 queued/running dispatch를 찾으면 재전송하지 -않는다. focused RED→GREEN 증거와 실행 시각은 +Actions SARIF가 사라졌다. wake는 모든 shard가 성공한 뒤 exact failed run을 +한 번만 깨우며, 중앙 coordinator는 repository·PR·head·base·required run +id가 모두 같은 queued/running dispatch를 찾으면 재전송하지 않는다. +focused RED→GREEN 증거와 실행 시각은 `docs/doctoring/codeql-partial-shard-wake-duplicate-dispatch.md`에 남긴다. ## 1. 근거와 범위 diff --git a/tests/test_codeql_scan_dispatch_workflow_contract.py b/tests/test_codeql_scan_dispatch_workflow_contract.py index 0c8c88f462..e90421b535 100644 --- a/tests/test_codeql_scan_dispatch_workflow_contract.py +++ b/tests/test_codeql_scan_dispatch_workflow_contract.py @@ -561,17 +561,17 @@ def test_dispatch_wakes_failed_jobs_once_after_all_language_shards() -> None: assert "BASE_SHA: ${{ needs.validate-dispatch.outputs.base_sha }}" in wake_job assert 'gh api "repos/${TARGET_REPOSITORY}/pulls/${PR_NUMBER}"' in wake assert 'gh api "repos/${TARGET_REPOSITORY}/actions/runs/${REQUIRED_RUN_ID}"' in wake - assert 'gh api "repos/${TARGET_REPOSITORY}/actions/jobs/${required_job_id}"' in wake + assert 'actions/runs/${REQUIRED_RUN_ID}/jobs' in wake assert 'select(.event == "pull_request")' in wake assert 'select(.path == ".github/workflows/codeql-pr.yml")' in wake assert "select(.head_sha == $head)" in wake assert ".base.ref == $base_ref" in wake assert ".base.sha == $base" in wake - assert ".run_id == $run_id" in wake - assert "select(.name == $name)" in wake assert 'select(.status == "completed" and .conclusion == "failure")' in wake + assert "bound failed CodeQL jobs" in wake assert 'actions/runs/${REQUIRED_RUN_ID}/rerun-failed-jobs' in wake assert 'actions/jobs/${required_job_id}/rerun"' not in wake + assert "while " not in wake assert "sleep " not in wake @@ -656,13 +656,13 @@ def _run_wake_step( ' fi\n' " exit 0\n" "fi\n" + 'if [ "${2:-}" = "--paginate" ]; then\n' + ' printf \'%s\\n\' "$FAKE_JOBS_PAGE"\n' + " exit 0\n" + "fi\n" 'case "$2" in\n' ' */pulls/*) printf \'%s\\n\' "$FAKE_PULL_JSON" ;;\n' ' */actions/runs/*) printf \'%s\\n\' "$FAKE_RUN_JSON" ;;\n' - ' */actions/jobs/*)\n' - ' job_id="${2##*/}"\n' - ' jq -c --argjson job_id "$job_id" \'map(select(.id == $job_id)) | first // empty\' <<<"$FAKE_JOBS_JSON"\n' - ' ;;\n' " *) exit 1 ;;\n" "esac\n", encoding="utf-8", @@ -673,7 +673,7 @@ def _run_wake_step( "PATH": f"{fake_bin}:{os.environ['PATH']}", "FAKE_PULL_JSON": json.dumps(pull), "FAKE_RUN_JSON": json.dumps(run), - "FAKE_JOBS_JSON": json.dumps(jobs), + "FAKE_JOBS_PAGE": json.dumps({"jobs": jobs}), "FAKE_POST_LOG": str(post_log), "FAKE_RERUN_ERROR": rerun_error or "", "GH_TOKEN": "fake-token", @@ -732,7 +732,7 @@ def test_dispatch_wake_rejects_stale_head_and_closed_pr(tmp_path: Path) -> None: assert not closed_log.exists() -def test_dispatch_wake_rejects_ambiguous_or_nonfailed_job_identity(tmp_path: Path) -> None: +def test_dispatch_wake_rejects_unbound_or_nonfailed_job_set(tmp_path: Path) -> None: wrong_job_result, wrong_job_log = _run_wake_step( tmp_path / "wrong-job", jobs=[ @@ -762,12 +762,13 @@ def test_dispatch_wake_rejects_ambiguous_or_nonfailed_job_identity(tmp_path: Pat assert wrong_job_result.returncode == 1 assert successful_job_result.returncode == 1 - assert "missing or ambiguous required job identity" in wrong_job_result.stdout + assert "bound failed CodeQL jobs" in wrong_job_result.stdout assert not wrong_job_log.exists() assert not successful_job_log.exists() def test_dispatch_wake_rejects_nonterminal_required_run(tmp_path: Path) -> None: + """The single wake runs only after the matrix has completed.""" result, post_log = _run_wake_step( tmp_path, run={