diff --git a/.github/workflows/codeql-pr.yml b/.github/workflows/codeql-pr.yml index c21c8446df..ed71c26040 100644 --- a/.github/workflows/codeql-pr.yml +++ b/.github/workflows/codeql-pr.yml @@ -6,8 +6,9 @@ # runner, then one coordinator POSTs repository_dispatch to # codeql-scan-dispatch.yml (native, unrestricted, in # ContextualWisdomLab/.github) with the remaining language matrix. The -# handler publishes codeql-dispatch/ and reruns only that exact -# failed job. On rerun the shard reads the terminal status once. Design: +# handler may publish codeql-dispatch/ for observability, but +# required verdict admission is bound to the exact completed dispatch run +# (repository, PR, head, live base, required run, and language job). Design: # docs/adr/0025-codeql-required-workflow-dispatch-architecture.md. The # merge-preview scan (analyze-merge) is required nowhere (PR #1766) and was # dropped, not migrated. @@ -157,10 +158,11 @@ jobs: matrix: ${{ fromJSON(needs.detect-languages.outputs.matrix) }} steps: - name: Read current-head CodeQL dispatch verdict - # Shards never dispatch. They re-check the live head, consume an - # authenticated codeql-dispatch/ verdict when one exists, - # and otherwise fail pending so the runner is released. One - # coordinator job POSTs the remaining language matrix after every + # Shards never dispatch. They re-check the live head/base and consume + # only an exact completed dispatch-run language job. Commit statuses + # published by the handler are observability only: a same-head base + # retarget must never inherit an old-base status as required evidence. + # One coordinator POSTs the remaining language matrix after every # shard has a job id. id: dispatch if: needs.detect-languages.outputs.code == 'true' @@ -176,6 +178,7 @@ jobs: set -euo pipefail live_pr="$(gh api "repos/${TARGET_REPOSITORY}/pulls/${PR_NUMBER}")" live_head="$(printf '%s' "$live_pr" | jq -r '.head.sha // empty')" + live_base_ref="$(printf '%s' "$live_pr" | jq -r '.base.ref // empty')" live_base="$(printf '%s' "$live_pr" | jq -r '.base.sha // empty')" live_state="$(printf '%s' "$live_pr" | jq -r 'if (.state | type) == "string" then .state else empty end')" if [ -z "$live_head" ] || [ -z "$live_state" ]; then @@ -190,7 +193,7 @@ jobs: echo "Pull request head moved on the live open PR; a fresh dispatch will fire for the current head." exit 0 fi - if ! [[ "$live_base" =~ ^[0-9a-fA-F]{40}$ ]]; then + if [ -z "$live_base_ref" ] || ! [[ "$live_base" =~ ^[0-9a-fA-F]{40}$ ]]; then echo "::error::Could not validate live pull request base SHA before CodeQL verdict read." exit 1 fi @@ -199,41 +202,26 @@ jobs: exit 1 fi - statuses="$(gh api "repos/${TARGET_REPOSITORY}/commits/${PR_HEAD_SHA}/statuses")" - verdict_state="$(printf '%s' "$statuses" | jq -r --arg ctx "codeql-dispatch/${LANGUAGE}" ' - [ - .[] - | select(.context == $ctx) - | select( - (.creator.login // "" | ascii_downcase) as $creator - | $creator == "opencode-agent" or $creator == "opencode-agent[bot]" - ) - ] - | first // {} | .state // empty - ')" - case "$verdict_state" in - success|failure|error) - echo "verdict=${verdict_state}" >>"$GITHUB_OUTPUT" - echo "Found authenticated current-head CodeQL verdict for ${LANGUAGE}: ${verdict_state}." - exit 0 - ;; - esac - - expected_title="CodeQL Scan Dispatch ${TARGET_REPOSITORY}#${PR_NUMBER}@${PR_HEAD_SHA}/${live_base}/${REQUIRED_RUN_ID}" + expected_title="CodeQL Scan Dispatch ${TARGET_REPOSITORY}#${PR_NUMBER}@${PR_HEAD_SHA}/${live_base_ref}@${live_base}/${REQUIRED_RUN_ID}" expected_job="CodeQL dispatch scan (${LANGUAGE})" runs_json="$(gh api --paginate --slurp "repos/ContextualWisdomLab/.github/actions/workflows/codeql-scan-dispatch.yml/runs")" - run_id="$(printf '%s' "$runs_json" | jq -r --arg title "$expected_title" --arg path ".github/workflows/codeql-scan-dispatch.yml" ' + run_ids="$(printf '%s' "$runs_json" | jq -r --arg title "$expected_title" --arg path ".github/workflows/codeql-scan-dispatch.yml" ' [ .[] | .workflow_runs[] | select(.path == $path) | select(.event == "repository_dispatch") | select(.status == "completed") | select(.display_title == $title or .name == $title) + | .id ] - | first - | .id // empty + | .[] ')" - if [[ "$run_id" =~ ^[1-9][0-9]*$ ]]; then + while IFS= read -r run_id; do + [ -n "$run_id" ] || continue + if ! [[ "$run_id" =~ ^[1-9][0-9]*$ ]]; then + echo "::error::Exact CodeQL dispatch lookup returned a malformed run id." + exit 1 + fi jobs_json="$(gh api --paginate --slurp "repos/ContextualWisdomLab/.github/actions/runs/${run_id}/jobs")" job_conclusion="$(printf '%s' "$jobs_json" | jq -r --arg name "$expected_job" ' [.[] | .jobs[] | select(.name == $name)] @@ -242,14 +230,14 @@ jobs: case "$job_conclusion" in success|failure) echo "verdict=${job_conclusion}" >>"$GITHUB_OUTPUT" - echo "Found completed CodeQL dispatch scan job for ${LANGUAGE}: ${job_conclusion}." + echo "Found exact completed CodeQL dispatch scan job for ${LANGUAGE}: ${job_conclusion} (run_id=${run_id})." exit 0 ;; esac - fi + done <<<"$run_ids" if [ "$RUN_ATTEMPT" != "1" ]; then - echo "::error::Exact CodeQL job was rerun without an authenticated terminal verdict." + echo "::error::Exact CodeQL job was rerun without an exact terminal dispatch verdict." exit 1 fi echo "verdict=pending" >>"$GITHUB_OUTPUT" @@ -279,7 +267,7 @@ jobs: exit 1 ;; *) - echo "::error::CodeQL shard has no authenticated current-head verdict or dispatch receipt." + echo "::error::CodeQL shard has no exact current-head/base/run verdict or dispatch receipt." exit 1 ;; esac @@ -370,33 +358,54 @@ jobs: )" done < <(printf '%s' "$include_json" | jq -c '.[]') - statuses="$(gh api "repos/${TARGET_REPOSITORY}/commits/${PR_HEAD_SHA}/statuses")" + expected_title="CodeQL Scan Dispatch ${TARGET_REPOSITORY}#${PR_NUMBER}@${live_head}/${live_base_ref}@${live_base}/${REQUIRED_RUN_ID}" + runs_json="$(gh api --paginate --slurp "repos/ContextualWisdomLab/.github/actions/workflows/codeql-scan-dispatch.yml/runs?per_page=100")" + completed_run_ids="$(printf '%s' "$runs_json" | jq -r --arg title "$expected_title" --arg path ".github/workflows/codeql-scan-dispatch.yml" ' + [ + .[] | .workflow_runs[] + | select(.path == $path) + | select(.event == "repository_dispatch") + | select(.status == "completed") + | select(.display_title == $title or .name == $title) + | .id + ] + | .[] + ')" + pending_matrix='[]' while IFS= read -r entry; do language="$(printf '%s' "$entry" | jq -r '.language // empty')" - verdict_state="$(printf '%s' "$statuses" | jq -r --arg ctx "codeql-dispatch/${language}" ' - [ - .[] - | select(.context == $ctx) - | select( - (.creator.login // "" | ascii_downcase) as $creator - | $creator == "opencode-agent" or $creator == "opencode-agent[bot]" - ) - ] - | first // {} | .state // empty - ')" - case "$verdict_state" in - success|failure|error) - echo "Found authenticated current-head CodeQL verdict for ${language}: ${verdict_state}." - ;; - *) - pending_matrix="$(jq -c --argjson entry "$entry" '. + [$entry]' <<<"$pending_matrix")" - ;; - esac + expected_job="CodeQL dispatch scan (${language})" + terminal_conclusion="" + terminal_run_id="" + while IFS= read -r run_id; do + [ -n "$run_id" ] || continue + if ! [[ "$run_id" =~ ^[1-9][0-9]*$ ]]; then + echo "::error::Exact CodeQL dispatch lookup returned a malformed run id." + exit 1 + fi + dispatch_jobs_json="$(gh api --paginate --slurp "repos/ContextualWisdomLab/.github/actions/runs/${run_id}/jobs")" + job_conclusion="$(printf '%s' "$dispatch_jobs_json" | jq -r --arg name "$expected_job" ' + [.[] | .jobs[] | select(.name == $name)] + | if length == 1 then .[0].conclusion else empty end + ')" + case "$job_conclusion" in + success|failure) + terminal_conclusion="$job_conclusion" + terminal_run_id="$run_id" + break + ;; + esac + done <<<"$completed_run_ids" + if [ -n "$terminal_conclusion" ]; then + echo "Found exact terminal CodeQL dispatch job for ${language}: ${terminal_conclusion} (run_id=${terminal_run_id})." + else + pending_matrix="$(jq -c --argjson entry "$entry" '. + [$entry]' <<<"$pending_matrix")" + fi done < <(printf '%s' "$include_json" | jq -c '.[]') if [ "$(printf '%s' "$pending_matrix" | jq 'length')" -eq 0 ]; then - echo "All detected CodeQL languages already have authenticated terminal verdicts; skipping dispatch." + echo "All detected CodeQL languages already have exact terminal dispatch verdicts; skipping dispatch." exit 0 fi @@ -411,6 +420,22 @@ jobs: exit 1 fi + active_run_id="$(printf '%s' "$runs_json" | jq -r --arg title "$expected_title" --arg path ".github/workflows/codeql-scan-dispatch.yml" ' + [ + .[] | .workflow_runs[] + | select(.path == $path) + | select(.event == "repository_dispatch") + | select(.status == "queued" or .status == "in_progress" or .status == "waiting" or .status == "requested" or .status == "pending") + | select(.display_title == $title or .name == $title) + ] + | first + | .id // empty + ')" + if [[ "$active_run_id" =~ ^[1-9][0-9]*$ ]]; then + echo "Identical CodeQL dispatch is already active (run_id=${active_run_id}); preserving it." + exit 0 + fi + if [ -z "${ACTIONS_ID_TOKEN_REQUEST_TOKEN:-}" ] || [ -z "${ACTIONS_ID_TOKEN_REQUEST_URL:-}" ]; then echo "::error::CodeQL scan dispatch requires GitHub OIDC." exit 1 diff --git a/.github/workflows/codeql-scan-dispatch.yml b/.github/workflows/codeql-scan-dispatch.yml index c94fdf55c2..afb51676be 100644 --- a/.github/workflows/codeql-scan-dispatch.yml +++ b/.github/workflows/codeql-scan-dispatch.yml @@ -17,6 +17,7 @@ run-name: >- github.repository }}#${{ github.event.client_payload.pr_number || 'event' }}@${{ github.event.client_payload.pr_head_sha || github.sha }}/${{ + github.event.client_payload.pr_base_ref || 'none' }}@${{ github.event.client_payload.pr_base_sha || 'none' }}/${{ github.event.client_payload.required_run_id || github.run_id }} @@ -513,23 +514,41 @@ jobs: echo "::error::Could not publish the CodeQL dispatch status after all configured credentials failed; the exact required job will remain failed and will not be woken with stale or missing evidence." exit 1 - - name: Wake exact CodeQL required job - if: >- - always() - && steps.publish_status.outcome == 'success' - && needs.validate-dispatch.outputs.target_repository != '' - && needs.validate-dispatch.outputs.pr_number != '' - && needs.validate-dispatch.outputs.head_sha != '' - && needs.validate-dispatch.outputs.required_run_id != '' - && needs.validate-dispatch.outputs.required_jobs != '' + wake-required-codeql: + name: Wake exact CodeQL required run + needs: [validate-dispatch, scan] + if: >- + always() + && needs.validate-dispatch.result == 'success' + && needs.scan.result != 'cancelled' + && needs.validate-dispatch.outputs.target_repository != '' + && needs.validate-dispatch.outputs.pr_number != '' + && needs.validate-dispatch.outputs.base_ref != '' + && needs.validate-dispatch.outputs.base_sha != '' + && needs.validate-dispatch.outputs.head_sha != '' + && needs.validate-dispatch.outputs.required_run_id != '' + && needs.validate-dispatch.outputs.required_jobs != '' + runs-on: ubuntu-24.04 + timeout-minutes: 5 + permissions: + actions: write + contents: read + steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@b09bb98e06d4d774595224525879c09bc6e98c40 # v2.20.1 + with: + egress-policy: audit + + - name: Wake exact CodeQL required run env: GH_TOKEN: ${{ needs.validate-dispatch.outputs.target_repository == github.repository && github.token || secrets.PR_REVIEW_MERGE_TOKEN || secrets.OPENCODE_APPROVE_TOKEN }} TARGET_REPOSITORY: ${{ needs.validate-dispatch.outputs.target_repository }} PR_NUMBER: ${{ needs.validate-dispatch.outputs.pr_number }} + BASE_REF: ${{ needs.validate-dispatch.outputs.base_ref }} + BASE_SHA: ${{ needs.validate-dispatch.outputs.base_sha }} HEAD_SHA: ${{ needs.validate-dispatch.outputs.head_sha }} REQUIRED_RUN_ID: ${{ needs.validate-dispatch.outputs.required_run_id }} REQUIRED_JOBS: ${{ needs.validate-dispatch.outputs.required_jobs }} - REQUIRED_LANGUAGE: ${{ matrix.language }} WAKE_TOKEN_SOURCE: ${{ needs.validate-dispatch.outputs.target_repository == github.repository && 'github-token' || secrets.PR_REVIEW_MERGE_TOKEN != '' && 'PR_REVIEW_MERGE_TOKEN' || secrets.OPENCODE_APPROVE_TOKEN != '' && 'OPENCODE_APPROVE_TOKEN' || 'unavailable' }} run: | set -euo pipefail @@ -537,48 +556,67 @@ jobs: echo "::error::Actions-capable CodeQL wake credential is unavailable." exit 1 fi - REQUIRED_JOB_ID="$(printf '%s' "$REQUIRED_JOBS" | jq -r --arg lang "$REQUIRED_LANGUAGE" ' - [.[] | select(.language == $lang) | .job_id | tostring] - | if length == 1 and (.[0] | test("^[1-9][0-9]*$")) then .[0] else empty end - ')" - if ! [[ "$REQUIRED_RUN_ID" =~ ^[1-9][0-9]*$ ]] || - ! [[ "$REQUIRED_JOB_ID" =~ ^[1-9][0-9]*$ ]] || - ! [[ "$REQUIRED_LANGUAGE" =~ ^[a-z0-9-]+$ ]]; then + if [ -z "$BASE_REF" ] || + ! [[ "$BASE_SHA" =~ ^[0-9a-fA-F]{40}$ ]] || + ! [[ "$REQUIRED_RUN_ID" =~ ^[1-9][0-9]*$ ]] || + ! printf '%s' "$REQUIRED_JOBS" | jq -e ' + type == "array" and length > 0 + and all(.[]; (.language | type == "string" and test("^[a-z0-9-]+$")) + and (.job_id | type == "number" and . > 0 and floor == .)) + and (([.[].language] | length) == ([.[].language] | unique | length)) + and (([.[].job_id] | length) == ([.[].job_id] | unique | length)) + ' >/dev/null; then echo "::error::CodeQL wake identity is non-canonical." exit 1 fi pull="$(gh api "repos/${TARGET_REPOSITORY}/pulls/${PR_NUMBER}")" live_state="$(printf '%s' "$pull" | jq -r '.state // empty')" + live_base_ref="$(printf '%s' "$pull" | jq -r '.base.ref // empty')" + live_base="$(printf '%s' "$pull" | jq -r '.base.sha // empty')" live_head="$(printf '%s' "$pull" | jq -r '.head.sha // empty')" - if [ "$live_state" != "open" ] || [ "$live_head" != "$HEAD_SHA" ]; then - echo "::error::CodeQL wake rejected a closed PR or stale head." + if [ "$live_state" != "open" ] || + [ "$live_base_ref" != "$BASE_REF" ] || + [ "$live_base" != "$BASE_SHA" ] || + [ "$live_head" != "$HEAD_SHA" ]; then + echo "::error::CodeQL wake rejected a closed PR or stale base/head identity." exit 1 fi run="$(gh api "repos/${TARGET_REPOSITORY}/actions/runs/${REQUIRED_RUN_ID}")" - run_identity="$(printf '%s' "$run" | jq -r --arg head "$HEAD_SHA" --argjson run_id "$REQUIRED_RUN_ID" ' + run_identity="$(printf '%s' "$run" | jq -r --arg base_ref "$BASE_REF" --arg base "$BASE_SHA" --arg head "$HEAD_SHA" --argjson pr_number "$PR_NUMBER" --argjson run_id "$REQUIRED_RUN_ID" ' select(.id == $run_id) | select(.event == "pull_request") | select(.path == ".github/workflows/codeql-pr.yml") | select(.head_sha == $head) + | select(.status == "completed") + | select([ + .pull_requests[]? + | select(.number == $pr_number and .head.sha == $head and .base.ref == $base_ref and .base.sha == $base) + ] | length == 1) | .id // empty ')" - expected_name="CodeQL compatibility analysis (${REQUIRED_LANGUAGE})" - job="$(gh api "repos/${TARGET_REPOSITORY}/actions/jobs/${REQUIRED_JOB_ID}")" - job_identity="$(printf '%s' "$job" | jq -r --arg head "$HEAD_SHA" --arg name "$expected_name" --argjson run_id "$REQUIRED_RUN_ID" --argjson job_id "$REQUIRED_JOB_ID" ' - select(.id == $job_id) - | select(.run_id == $run_id) - | select(.head_sha == $head) - | select(.name == $name) - | select(.status == "completed" and .conclusion == "failure") - | .id // empty - ')" - if [ "$run_identity" != "$REQUIRED_RUN_ID" ] || - [ "$job_identity" != "$REQUIRED_JOB_ID" ]; then - echo "::error::CodeQL wake rejected missing or ambiguous exact run/job identity." + if [ "$run_identity" != "$REQUIRED_RUN_ID" ]; then + echo "::error::CodeQL wake rejected missing or ambiguous exact run/base identity." exit 1 fi - gh api -X POST "repos/${TARGET_REPOSITORY}/actions/jobs/${REQUIRED_JOB_ID}/rerun" >/dev/null - echo "Re-ran exact failed CodeQL job ${REQUIRED_JOB_ID} for ${REQUIRED_LANGUAGE} on ${HEAD_SHA}." + while IFS= read -r required_job; do + required_language="$(jq -r '.language' <<<"$required_job")" + required_job_id="$(jq -r '.job_id | tostring' <<<"$required_job")" + expected_name="CodeQL compatibility analysis (${required_language})" + job="$(gh api "repos/${TARGET_REPOSITORY}/actions/jobs/${required_job_id}")" + job_identity="$(printf '%s' "$job" | jq -r --arg head "$HEAD_SHA" --arg name "$expected_name" --argjson run_id "$REQUIRED_RUN_ID" --argjson job_id "$required_job_id" ' + select(.id == $job_id and .run_id == $run_id and .head_sha == $head) + | select(.name == $name) + | select(.status == "completed" and .conclusion == "failure") + | .id // empty + ')" + if [ "$job_identity" != "$required_job_id" ]; then + echo "::error::CodeQL wake rejected missing or ambiguous required job identity." + exit 1 + fi + done < <(printf '%s' "$REQUIRED_JOBS" | jq -c '.[]') + + gh api -X POST "repos/${TARGET_REPOSITORY}/actions/runs/${REQUIRED_RUN_ID}/rerun-failed-jobs" >/dev/null + echo "Re-ran failed jobs in exact CodeQL run ${REQUIRED_RUN_ID} for ${HEAD_SHA} on base ${BASE_REF}@${BASE_SHA} after all dispatch shards completed." diff --git a/AGENTS.md b/AGENTS.md index e955f8b36a..ff01a0aecf 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -60,6 +60,16 @@ The materialization contract is also covered by [`docs/doctoring/exact-artifact- head. If a current-head dispatch is cancelled while deduplicating, enqueue exactly one replacement for that PR and workflow and verify the replacement carries the same live target head. +- CodeQL language shards do not wake required jobs independently. After the + complete `scan` matrix terminates, one `wake-required-codeql` coordinator + revalidates the live PR and exact required run before one run-level + `rerun-failed-jobs`. Bind that wake to PR number, head **and base SHA**: the + live PR base/head and the required run's `pull_requests[]` base/head tuple + must all match the validated identity. If the required-workflow rerun would + issue a duplicate central dispatch, preserve a queued/running dispatch whose + immutable title matches repository, PR, head, base, and required run id. + Missing or ambiguous base provenance fails closed. See + `docs/doctoring/codeql-partial-shard-wake-duplicate-dispatch.md`. - Before every review, retry, push, or merge claim, re-fetch the PR's exact head SHA, base SHA, review threads, required checks, and ruleset result. A push invalidates earlier checks and reviews. Never self-approve, dismiss reviews, @@ -212,3 +222,13 @@ them alone proves succession. variable in CI, so a failure class exists that cannot reproduce locally. Before calling a scheduler change clean, run the affected tests both ways, including `GITHUB_ACTIONS=true python3 -m pytest `. +- A successful rerun of one matrix job does not rerun its sibling matrix jobs. Therefore an + `already running` response from a second per-job rerun must remain a failure: even if the shared + run is active, that sibling can still retain its old failed verdict. Coordinate the wake only + after all dispatch shards publish, then rerun the exact run's failed jobs as one operation. +- A `codeql-dispatch/` commit status is head-scoped and carries neither the PR base + nor the required-run identity. Keep it as diagnostic output only. Shards and the coordinator + may accept a terminal verdict only from a completed central dispatch run named with + `{repository}#{PR}@{head}/{base_ref}@{base_sha}/{required_run_id}` and its unique language + job. Otherwise + remain pending and dispatch fresh base-bound work. diff --git a/CHANGELOG.md b/CHANGELOG.md index bf192f6a9e..f2775e20ce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,19 @@ +### CodeQL terminal verdicts bind the PR base and required run + +- Removed head-only `codeql-dispatch/` commit statuses from the + required shard and coordinator decision paths. Those statuses cannot tell a + same-head base retarget or two required runs apart. Terminal authority now + comes from a completed central dispatch run named with repository, PR, head, + base ref, base SHA, and required run plus its unique language job; absent exact evidence, + the coordinator dispatches fresh work. + +### CodeQL partial-shard wake preserves the active dispatch + +- Prevented a required-workflow rerun from posting an identical CodeQL dispatch + while a sibling language scan is still running. The coordinator now keeps an + active run with the same immutable repository, PR, head, base, and required + run identity, avoiding same-PR cancellation of valid evidence. + ### Failed-check finding names the Strix sandbox instead of the gateway - `opencode-review-dispatch.yml`'s `emit_strix_provider_failure_finding` rendered one fixed finding for every `STRIX_PROVIDER_UNAVAILABLE` line, whose Root cause read "The contextual-orchestrator gateway or its discovered provider pool was unavailable for this run". `#1953` had just given the Strix sandbox bootstrap failure its own second verdict token (`STRIX_SANDBOX_UNAVAILABLE`) precisely because that attribution is wrong for it -- the sandbox container never reaches its Caido proxy, so the run dies before the gateway serves anything -- and this consumer re-applied the wrong attribution one step downstream, into the review findings and the failure census. The emitter now branches on the second token: a sandbox verdict gets a finding that names Strix's sandbox, says the verdict does not name the gateway, and tells the reader not to change gateway or provider configuration on its strength. A `STRIX_PROVIDER_UNAVAILABLE` line without the token keeps its existing text verbatim, so the gateway class has no regression surface. No test covered this finding text at all before (`gateway or its discovered provider pool` matched nothing under `tests/`); `tests/test_opencode_dispatch_strix_sandbox_finding.py` now runs the production emitter from the published run block and pins both directions plus the no-signal case. Refs #1953, #1935. diff --git a/CLAUDE.md b/CLAUDE.md index 30db1fc23b..115456763f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -18,6 +18,13 @@ configuring any such loop. The repo/Project — not private agent memory — is the source of truth. This file complements those documents; it does not replace them. +For CodeQL's dispatch-and-wake loop, wait for the complete scan matrix, then let +one `wake-required-codeql` coordinator revalidate the live PR and exact required +run before one run-level failed-job rerun. The wake identity includes PR number, +head SHA, and base SHA; the live PR and required run `pull_requests[]` tuple must +agree. The required-workflow coordinator separately preserves an already-active +dispatch with the same repository/PR/head/base/required-run identity. + ## What this repository is This is the ContextualWisdomLab **organization-wide `.github` special repository**. It has three roles: @@ -220,3 +227,12 @@ repeatable compile command. fence. Do not check by counting fences — a split leaves four where there were two, so an even count proves nothing. The damage can also arrive inherited, from an earlier commit on the same branch or from the autofix flow's conflict-marker resolution. +- **Per-job reruns do not cover matrix siblings.** Do not accept a second shard's `already running` + response merely because the shared run is active. Coordinate after every dispatch shard has + published its verdict and wake the exact run's failed jobs once, so no sibling retains a stale + failed required check. The wake is valid only while PR number, head SHA, and base SHA still match + both live PR metadata and the exact required run's `pull_requests[]` association. +- **Head-only CodeQL statuses are diagnostic, not terminal authority.** They cannot distinguish + two required runs or a same-head base retarget. Read the completed central dispatch run named + with repository, PR, head, base ref, base SHA, and required-run id plus its unique language job; absent that + exact evidence, keep the language pending. diff --git a/docs/adr/0025-codeql-required-workflow-dispatch-architecture.md b/docs/adr/0025-codeql-required-workflow-dispatch-architecture.md index 5a11894767..4a84c800f5 100644 --- a/docs/adr/0025-codeql-required-workflow-dispatch-architecture.md +++ b/docs/adr/0025-codeql-required-workflow-dispatch-architecture.md @@ -1,6 +1,6 @@ # 0025 — Restore central CodeQL as a required workflow via repository_dispatch -**Status:** Proposed, amended 2026-09-07 (one dispatch per pull request; language independence is the handler job matrix) · **Date:** 2026-09-03 · **Owner intent recorded:** loop-brief item 41 +**Status:** Proposed, amended 2026-09-09 (one dispatch per pull request; post-matrix run-level wake; exact PR head/base binding) · **Date:** 2026-09-03 · **Owner intent recorded:** loop-brief item 41 ## Problem @@ -97,72 +97,56 @@ codeql-pr.yml (required workflow, runs in target repo context) analyze-head (matrix) -- SAME REQUIRED-CHECK NAME: "CodeQL compatibility analysis (${{ matrix.language }})". No codeql-action reference and no - repository_dispatch. On attempt one it - re-checks the live head, consumes an - authenticated codeql-dispatch/ - status when one exists, and otherwise fails - pending to release the runner. The trusted - handler publishes the terminal status and - reruns only that failed job. On the woken - attempt the shard reads the authenticated - current-head status once and reflects it as - this job's own exit code. - dispatch-current-head -- NEW: needs analyze-head, runs on attempt one - of an open current-head PR after the shards - have job ids. Collects those ids from this - run's jobs API, POSTs event_type codeql-scan - once with the remaining language matrix and - required_jobs: [{language, job_id}, ...], and - fails closed if any shard job id is missing. - Skips the POST when every language already - has a terminal verdict. github.run_attempt == 1 - is required: a single-job wake re-runs - dependents, and a second POST would cancel - the in-flight multi-language handler. - -.github/workflows/codeql-scan-dispatch.yml (NEW, runs natively in .github, + repository_dispatch. It re-checks the live + head/base and consumes only the matching + completed central dispatch run's unique + language-job conclusion. Without that exact + run evidence it fails pending to release the + runner; a later run-level failed-job rerun + reads the same exact-run evidence once and + reflects it as this job's own exit code. + dispatch-current-head -- needs analyze-head, runs on attempt one of an + open current-head PR after the shards have job + ids. Collects those ids from this run's jobs + API, POSTs event_type codeql-scan once with the + remaining language matrix and required_jobs: + [{language, job_id}, ...], and fails closed if + any shard job id is missing. Skips the POST + when every language already has a terminal + verdict. + +.github/workflows/codeql-scan-dispatch.yml (runs natively in .github, NOT admitted through the ruleset, so codeql-action is unrestricted here) on: repository_dispatch: types: [codeql-scan] validate-dispatch -- Re-validate the payload against the LIVE pull - request in the target repository (identical - pattern to strix.yml's "Validate repository - dispatch against live pull request metadata": - reject if state/base/head don't match exactly). + request in the target repository; state, + repository, base ref/SHA and head ref/SHA must + match exactly. Export the validated base/head + identity and exact required run/job map. scan (matrix over payload languages) -- Exchange OIDC for a target-repo-scoped - OpenCode app token (identical exchange used - by strix.yml's target_app_token step). - Checkout the target repository's PR head at - the exact validated SHA (harden-runner - audited, matching strix.yml's checkout - posture). Run codeql-action/init + - codeql-action/analyze with upload: false - (same as today). Apply the Medium+ SARIF gate - (extracted to scripts/ci/codeql_sarif_gate.py - with its own unit tests, replacing the - current inline-Python duplicated between - analyze-head and analyze-merge -- one script, - one test file, used from both the merge - preview path if it returns and this dispatch - handler). - -- Publish the result as a commit status on the - TARGET repository at context - "codeql-dispatch/" using the - target-scoped token (identical mechanism to - strix.yml's "Publish same-head manual Strix - status" multi-token fallback chain), state - success/failure, description carrying a short - finding count, target_url pointing at this - .github run's own log for full evidence. - -- Upload the SARIF as an artifact on this - .github-side run for audit trail (mirrors - strix.yml's "Preserve CodeQL SARIF evidence" - / artifact retention today). - -- Re-fetch the open PR, exact required workflow - run, and exact failed language job; - require matching path/head/run/job/name before - calling the single-job rerun endpoint. Missing, - stale, closed, or mismatched identity fails + OpenCode app token. Re-read the live PR before + the privileged scan and require the same + validated base/head identity. Checkout the PR + head at the exact validated SHA. Run + codeql-action/init + codeql-action/analyze with + upload: false, apply the shared Medium+ SARIF + gate, preserve SARIF, and publish + codeql-dispatch/ when credentials + permit. + wake-required-codeql -- Needs validate-dispatch + the complete scan + matrix and runs only after every language shard + terminates. Re-fetch the live PR and require + state=open plus exact validated base/head. + Re-fetch the exact required workflow run and + require id/event/path/head/status plus exactly + one pull_requests[] association whose PR + number, head SHA, and base SHA equal the + validated tuple. Revalidate every supplied + failed language job's id/run/head/name/status, + then call the exact run's + rerun-failed-jobs endpoint once. Missing, + stale, retargeted, or ambiguous identity fails closed and leaves the required job failed. ``` @@ -177,9 +161,11 @@ still-pending language in a single `codeql-scan` payload (`matrix` plus its predecessor and other repositories or pull requests stay independent. Language independence is `strategy.fail-fast: false` on that one run's job -matrix. Each scan job still publishes `codeql-dispatch/` and wakes -only its own required job. One language's failure cannot cancel or skip a -sibling. +matrix. Each scan job publishes its own `codeql-dispatch/` verdict, +but it does **not** wake the required workflow independently. One +post-matrix coordinator validates the complete required-job map and performs +one run-level failed-job rerun. This avoids both sibling cancellation and a +stale failed sibling left behind by per-job reruns. #### 2026-09-07 amendment: one dispatch per pull request, adopted for the 60-job ceiling @@ -203,12 +189,45 @@ superseded HEAD of the same pull request, and a language suffix is forbidden. The 2026-09-05 rejection of "full matrix in one dispatch" is therefore -superseded. The sibling-cancel failure mode is gone because siblings are -jobs in one run, not runs in one concurrency group. The exact-job wake -contract is preserved: `required_jobs` is a 1:1 map of language to canonical -job id, each scan shard looks up only its own id, and a missing, stale, or -mismatched identity still fails closed. The old scalar -`required_job_id`/`required_language` payload is retired. +superseded. Siblings are jobs in one run, not runs in one concurrency group. +`required_jobs` remains a 1:1 map of language to canonical job id; the +post-matrix wake validates the whole map and every exact job before one +run-level rerun. A missing, stale, or mismatched identity fails closed. The +old scalar `required_job_id`/`required_language` payload is retained only as +a bounded queued-payload compatibility path where the matrix has exactly one +language; it is not the current producer contract. + +#### 2026-09-09 amendment: one post-matrix wake with exact base binding + +PR #2051 exposed two distinct coordination faults. First, a partial-shard +wake could rerun the required workflow while a sibling scan was still +running. The rerun's `dispatch-current-head` then posted an identical native +dispatch, and workflow/repository/PR `cancel-in-progress` cancelled valid +sibling evidence. The native handler now waits for the complete matrix and +uses one `wake-required-codeql` coordinator. The required-workflow +coordinator also preserves a queued or running central dispatch whose +immutable title matches `(repository, PR, head SHA, base SHA, required run +id)`; another head, base, required run, or terminal cancelled run does not +suppress fresh evidence. + +Second, the post-matrix wake initially revalidated only the PR head and the +required run's head. A PR can retain its head while its base is retargeted. +The validated `base_sha` is therefore part of the wake identity: the live PR +must still have that exact base/head, and the required run's +`pull_requests[]` must contain exactly one association with the same PR +number/head/base. Required run `34318639845` demonstrated that GitHub exposes +that base association directly; no inferred base or mutable external state is +needed. Test-only RED `901af9f024836eadd10c6c98affbee037ffecd58` +reproduced both changed-live-base and wrong-run-base cases against the real +wake shell block; before repair each returned success and emitted one rerun +POST. `66a15d856c251f1db2f91cb3d4a2fa66afd8f48c` binds the wake to the exact +base and makes both cases fail closed with no POST. See +`docs/doctoring/codeql-partial-shard-wake-duplicate-dispatch.md`. + +Per-language wake, polling/sleep, broad workflow rerun, and widening the +concurrency key to include head/base were rejected. They either recreate the +sibling-race class, consume runners while waiting, or prevent a genuinely +superseded head from cancelling its predecessor. ## Scope decision: `analyze-merge` is dropped, not migrated @@ -224,11 +243,11 @@ blocker for this one. ## Security considerations (must be resolved during implementation, not assumed) - **Payload forgery / TOCTOU:** the dispatch handler must re-fetch the live - PR from the API and refuse to scan or publish anything if the dispatched - `pr_head_sha` no longer matches the live head, exactly like `strix.yml`'s - existing `Validate repository dispatch against live pull request metadata` - step and the exact-job wake-time revalidation. A forged or stale - dispatch must never be able to make an unrelated head appear scanned. + PR and refuse scan, publication, or wake when either validated head **or + base SHA** no longer matches. At wake time the exact required workflow run + must also carry exactly one matching PR-number/head/base association in + `pull_requests[]`. A forged, stale, or same-head/different-base run must + never authorize a rerun or make an unrelated base appear scanned. - **Cross-repository checkout trust boundary:** the scan step checks out arbitrary target-repository PR-head content into `.github`'s own runner. This is the same trust boundary `strix.yml` already crosses today (its @@ -242,16 +261,13 @@ blocker for this one. on the *target* repository only, following the same per-repository app-token minting `strix.yml` already performs — never a token with broader org access. -- **Verdict target cannot be spoofed by the PR author:** a commit status is - writable by anyone with `statuses:write` on the repository (including, - depending on token scoping, a workflow running with the default - `GITHUB_TOKEN` in some configurations) — confirm during implementation - that the rerun job in `codeql-pr.yml` verifies the status update's - `creator`/`avatar_url`/app identity matches the expected dispatch-handler - app, not merely the context name, so a malicious PR cannot forge its own - passing status. `strix.yml`'s manual-status-publish step already documents - a similar concern; follow its precedent rather than trusting context name - alone. +- **Commit status is diagnostic only:** even a status from the expected app is + scoped only to a commit SHA. It cannot bind a PR base or one required run, + so a same-head base retarget can leave a stale terminal value behind. The + shard and coordinator accept only a completed central dispatch run named + with repository, PR, head, base ref, base SHA, and required-run id, plus exactly one + matching language job. Missing exact-run evidence stays pending and + triggers fresh base-bound work. ## Alternatives considered and rejected @@ -273,36 +289,36 @@ blocker for this one. ## Risks and effects -- Adds one new workflow file and one new `scripts/ci/codeql_sarif_gate.py` +- Adds one native dispatch workflow and one `scripts/ci/codeql_sarif_gate.py` module (with its own test file, contributing to the 100%-coverage - requirement on `scripts/ci/`) to the org's central CI surface — more - surface area to maintain, offset by removing ~70 lines of duplicated - inline Python between `analyze-head`/`analyze-merge` today. - exact run/job wake-up follows the OpenCode runner-release pattern while - avoiding one occupied runner per language for the scan's full duration. + requirement on `scripts/ci/`) to the org's central CI surface. The extra + surface is offset by removing duplicated inline gate logic and by keeping + CodeQL action execution out of the required-workflow file. - A repository and pull request have one active native handler run. Language parallelism is bounded by the detected CodeQL matrix inside that run, and a superseded HEAD of the same pull request cancels the in-flight handler instead of queuing another copy per language. +- Run-level wake is deliberately stricter than commit status identity. A + same-head base retarget invalidates the wake even when old statuses remain + attached to the commit; fresh base-materialized evidence is required. - Re-admitting `codeql-pr.yml` to ruleset `18156473` must happen only after - this design is implemented, tested, and its `detect-languages`/ - `dispatch-analysis`/`analyze-head` jobs are confirmed free of any - `codeql-action` reference (grep the final file for `codeql-action` and - assert zero matches, as a permanent contract test) — re-adding it with - the bug still present would recreate the exact org-wide 100%-startup_failure - incident this ADR exists to prevent. + this design is implemented, tested, and its required workflow is confirmed + free of every `codeql-action` reference. Re-adding it with the admission bug + still present would recreate the org-wide startup-failure incident this ADR + exists to prevent. ## Follow-up -1. Implement `scripts/ci/codeql_sarif_gate.py` + its test, extracted from - the current inline gate in `codeql-pr.yml`. -2. Implement `codeql-scan-dispatch.yml` per the design above. -3. Rewrite `codeql-pr.yml`'s `analyze-head` job into the dispatch+exact-job-wake shape; - delete `analyze-merge` (tracked as future work, not silently lost — this - ADR is the record). -4. Add a permanent contract test asserting no `codeql-action` reference - exists anywhere in `codeql-pr.yml`. -5. Only then, re-add `.github/workflows/codeql-pr.yml` to ruleset `18156473`'s - required `workflows` list (admin:org PUT, same mechanism used to remove - it) and verify a real PR observes a successful, correctly-named required - check before declaring this ADR's status Accepted. +1. Keep `scripts/ci/codeql_sarif_gate.py` and its tests as the single Medium+ + gate used by the native handler. +2. Verify the final `codeql-scan-dispatch.yml` contract, including full-matrix + independence, exact PR/head/base run binding, and no polling/manual escape + path, on the exact candidate head. +3. Verify `codeql-pr.yml` contains no `codeql-action` reference and preserves + the exact required check names while using one current-head dispatch. +4. Merge the corrected central owner normally only after exact-head hosted + checks and qualifying independent review. +5. After protected-main integration, require a fresh real downstream PR to + demonstrate base-materialized CodeQL dispatch, one post-matrix wake, and + terminal required verdicts before treating consumer bootstrap as GREEN or + declaring this ADR Accepted. diff --git a/docs/doctoring/codeql-partial-shard-wake-duplicate-dispatch.md b/docs/doctoring/codeql-partial-shard-wake-duplicate-dispatch.md new file mode 100644 index 0000000000..f5e5267e84 --- /dev/null +++ b/docs/doctoring/codeql-partial-shard-wake-duplicate-dispatch.md @@ -0,0 +1,64 @@ +# CodeQL dispatch wake coordination and base-binding RCA + +## 관찰 + +2026-09-09 중앙 PR #2052의 required run `34316109112`가 exact head +`4833e6c202aaa02817b5b241178adb1facc6bf2a`와 base +`7fd571dbcdbae6acf29d8f4ee704d7ba6297e4db`를 대상으로 dispatch +`34316388553`을 만들었다. Python job `102353967729`는 05:58:54Z에 +성공했다. Actions job `102353967703`은 06:01:09Z에 시작했지만 같은 +immutable run title을 가진 두 번째 dispatch `34317266381`가 06:01:38Z에 +생성됐고 첫 실행은 06:02:08Z에 `cancelled`로 끝났다. Actions의 CodeQL +analysis step도 `cancelled`였으므로 terminal success나 SARIF 증거로 +계산하지 않는다. + +초기 수리는 언어별 wake를 없애고 `validate-dispatch`와 전체 `scan` matrix가 +종료된 뒤 `wake-required-codeql` coordinator 하나가 exact required run의 +`rerun-failed-jobs`를 한 번 호출하도록 바꿨다. required workflow의 +coordinator는 같은 repository, PR, head, base, required-run tuple을 가진 +queued/running dispatch가 이미 있으면 새 POST를 만들지 않는다. 다른 +identity와 terminal-cancelled run은 fresh dispatch를 막지 않는다. + +## 동일 head의 base retarget gap + +PR #2051의 `927a9e35ed5c5e115a6c9d9b9f0035c7a0c0917e`에서 post-matrix wake는 +live PR state와 head SHA, required run id/event/path/head/status, failed job +identity를 다시 검증했지만 base SHA를 wake identity에 포함하지 않았다. +GitHub의 실제 required run `34318639845`는 `pull_requests[]`에 PR number와 +head/base ref/base SHA를 함께 제공하므로 base provenance를 별도 추정할 필요가 없다. +같은 head를 유지한 채 PR base만 retarget하면 이전 base의 completed run이 +새 base의 wake를 승인할 수 있는 TOCTOU가 남아 있었다. + +Test-only RED `901af9f024836eadd10c6c98affbee037ffecd58`은 두 경로를 고정한다. +하나는 validate 뒤 live PR base만 바뀌는 경우, 다른 하나는 live PR은 현재 +base지만 supplied required run이 다른 base에서 만들어진 경우다. 기존 exact +wake block을 fixture-backed `gh api`로 실행하면 두 경우 모두 return code 0과 +`rerun-failed-jobs` POST 1건을 남겼다. + +`66a15d856c251f1db2f91cb3d4a2fa66afd8f48c`는 `validate-dispatch.outputs.base_sha` +를 wake job에 전달하고 다음을 모두 fail-closed로 결속한다. + +- live PR은 open이고 `base.sha == BASE_SHA`, `head.sha == HEAD_SHA`여야 한다. +- exact `REQUIRED_RUN_ID`는 pull_request event의 `codeql-pr.yml` completed run이며 + `pull_requests[]` 안에 같은 PR number/head/base ref/base SHA tuple이 정확히 하나 있어야 한다. +- 그 뒤에만 기존 failed-job id/name/run/head 검증과 run-level + `rerun-failed-jobs`가 실행된다. + +같은 fixture를 repaired block에 적용하면 두 changed-base 경로 모두 return +code 1, POST 0건이다. 기존 wake fixture도 같은 base-aware payload를 사용하도록 +`f9d46984e1ef35341e9535af245da8e6ab9c061e`에서 보강했다. 이 검증은 hosted +required-check GREEN이나 protected merge를 대신하지 않는다. + +## 경계와 기각한 대안 + +- head SHA만으로 current identity를 정의하지 않는다. GitHub PR은 head를 + 유지한 채 base가 바뀔 수 있다. +- old-base run을 허용한 뒤 새 scan이 언젠가 덮을 것이라고 가정하지 않는다. + required status는 exact base provenance를 잃으면 즉시 fail-closed해야 한다. +- polling, `sleep`, broad workflow rerun, manual/no-op trigger를 추가하지 않는다. +- concurrency key를 head/base까지 확장하지 않는다. genuinely superseded PR head는 + 기존 workflow/repository/PR cancellation contract로 계속 정리한다. +- `pull_requests[]` base metadata가 없거나 모호하면 wake를 허용하지 않는다. + +Hosted checks, independent review, protected-main integration과 실제 downstream +consumer rerun은 이 문서와 별도로 exact successor에서 검증한다. diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 1cc9e20313..27a5539de9 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -7,6 +7,14 @@ 이 문서는 제품·기술·운영 Gap을 현재 문서와 현재 GitHub 상태에 묶어 두는 기준선이다. 새 작업은 먼저 이 문서의 Gap ID를 PR 설명과 테스트 증거에 연결하고, PR의 정확한 exact HEAD·Checks·리뷰를 다시 수집한 뒤 구현한다. 표의 상태는 작성 시점의 관측값이므로, 병합 판단에는 재사용하지 않는다. 이 인벤토리는 스냅샷이며 merge authorization이 아니다. +같은 exact head의 CodeQL dispatch `34316388553`에서는 Python shard가 +성공한 뒤 required job을 깨웠고, Actions shard가 분석 중일 때 동일 제목의 +dispatch `34317266381`가 생성됐다. 같은 PR concurrency가 첫 실행을 취소해 +Actions SARIF가 사라졌다. 중앙 coordinator는 이제 repository·PR·head·base· +required run id가 모두 같은 queued/running dispatch를 찾으면 재전송하지 +않는다. focused RED→GREEN 증거와 실행 시각은 +`docs/doctoring/codeql-partial-shard-wake-duplicate-dispatch.md`에 남긴다. + ## 1. 근거와 범위 ### 1.1 우선순위가 높은 근거 @@ -3353,3 +3361,36 @@ queries the check-runs API at its own time, order-independently. The implementin their change was safe because they had scoped it narrowly, not because they had checked for the name collision — which is the more useful lesson: **a job name is unique only within one workflow file, and the same name in another file can carry the opposite safety property.** + +## 2026-09-09 CodeQL dispatch wake sibling-shard rerun race (targeted by #2051) + +- Live evidence: `ContextualWisdomLab/.github#1563` dispatch run `34297767440` + (for head `20913979589d86ad1e2d26705ffb2c4a675409bd`): the `actions` + matrix shard's `POST .../actions/jobs/{id}/rerun` moved the shared CodeQL + PR run `34297581323` back to in_progress, so the `python` shard's own + rerun POST was rejected with `gh: The workflow run containing this job is + already running (HTTP 403)` and that dispatch shard failed. The required + rerun POST was rejected with `gh: The workflow run containing this job is + already running (HTTP 403)`. A per-job rerun covers that job and its + dependents, not a failed matrix sibling, so the second language kept its + stale failure. Root cause class: parallel per-language wake operations + competing for one shared run while each operation covered only one job. +- Fix (`ContextualWisdomLab/.github#2051`, branch + `fix/codeql-wake-sibling-rerun-race`): wake responsibility moves out of + the language matrix into one coordinator that starts only after every + dispatch shard terminates. It revalidates the live PR, exact completed + CodeQL run, and every supplied failed job, then calls the exact run's + `rerun-failed-jobs` endpoint once. No polling, retry loop, or sleep is + introduced. +- Acceptance remains open until a fresh hosted dispatch run with two failed + shards shows the single coordinator green + and the required CodeQL PR shards reaching terminal verdicts. +- Follow-up exact-identity repair (`70e8c1fc`): a same-head base retarget left + `codeql-dispatch/` statuses attached to the commit, and those + statuses carry neither base SHA nor required-run id. The required shard and + coordinator now use the completed central dispatch identity + `{repository}#{PR}@{head}/{base_ref}@{base_sha}/{run}` and one unique language job instead. + The head-only and same-SHA/different-base-ref contract tests were RED on the + inherited paths; after repair, 52 focused tests and the full repository suite + (`3000 passed, 1 skipped, 21 subtests`) passed. Hosted exact-head evidence and + independent approval remain open acceptance gates. diff --git a/tests/test_codeql_pr_workflow_contract.py b/tests/test_codeql_pr_workflow_contract.py index dc67eef258..94f7b8be95 100644 --- a/tests/test_codeql_pr_workflow_contract.py +++ b/tests/test_codeql_pr_workflow_contract.py @@ -27,14 +27,6 @@ def test_codeql_pr_workflow_structure() -> None: assert "name: CodeQL PR" in workflow assert "branches: [main, master, develop]" not in workflow - # Stronger than the literal-string check above: reject ANY `branches:` - # filter on the pull_request trigger, not just the specific old list -- - # a fixed branch-name list of any shape silently never fires for a - # repository whose default branch isn't in that list, leaving its - # org-required CodeQL check permanently absent rather than passing or - # failing (confirmed live: a repository defaulting to gh-pages received - # every other required check but no CodeQL check at all; caught by Devin - # Review on .github#1661's gap-baseline entry for backlog item 38). trigger_start = workflow.index("on:\n pull_request:") trigger_end = workflow.index("\n\n", trigger_start) trigger_lines = workflow[trigger_start:trigger_end].splitlines() @@ -46,28 +38,19 @@ def test_codeql_pr_workflow_structure() -> None: assert "-name '*.java'" in workflow assert "-name '*.kt'" in workflow assert "analyze-head:" in workflow - # analyze-merge is required nowhere (PR #1766) and is dropped, not - # migrated, per the ADR's explicit scope decision. assert "analyze-merge:" not in workflow assert "CodeQL merge preview" not in workflow assert "refs/pull/{0}/merge" not in workflow assert "event_type:\"codeql-scan\"" in workflow assert "repos/ContextualWisdomLab/.github/dispatches" in workflow - # Reads the authenticated context codeql-scan-dispatch.yml publishes; it - # never publishes that status from the required workflow. - assert '--arg ctx "codeql-dispatch/${LANGUAGE}"' in workflow - assert "commits/${PR_HEAD_SHA}/statuses" in workflow + # Commit statuses remain a handler observability surface, but this required + # workflow must derive acceptance from the exact dispatch-run identity. + assert "commits/${PR_HEAD_SHA}/statuses" not in workflow + assert "expected_title=\"CodeQL Scan Dispatch ${TARGET_REPOSITORY}#${PR_NUMBER}@${PR_HEAD_SHA}/${live_base_ref}@${live_base}/${REQUIRED_RUN_ID}\"" in workflow def test_codeql_pr_shards_do_not_dispatch_and_coordinator_sends_the_full_matrix_once() -> None: - """Shards consume verdicts; one coordinator POSTs the remaining language matrix. - - Per-language repository_dispatch runs were the 60-job ceiling: live - 2026-09-07 queued ~149 ``codeql-scan-dispatch.yml`` runs across 60 PR@SHA - tuples because each analyze-head shard POSTed its own ``codeql-scan``. - Language independence now lives in the handler's job matrix, so the - required workflow may send every still-pending language in one payload. - """ + """Shards consume exact-run verdicts; one coordinator POSTs pending languages.""" workflow = WORKFLOW_PATH.read_text(encoding="utf-8") analyze_head = workflow.split(" analyze-head:\n", 1)[1].split( " dispatch-current-head:\n", 1 @@ -93,14 +76,7 @@ def test_codeql_pr_shards_do_not_dispatch_and_coordinator_sends_the_full_matrix_ def test_codeql_coordinator_dispatches_later_attempts_when_no_terminal_verdict() -> None: - """A rerun must still POST codeql-scan if attempt 1 never dispatched. - - Live ContextualWisdomLab/.github#2028 run 34175742278 was attempt 2. - ``github.run_attempt == 1`` skipped Dispatch current-head, so no - codeql-scan-dispatch.yml run existed and compatibility stayed pending. - The coordinator script already skips when every language has a terminal - opencode-agent verdict, so later attempts are safe. - """ + """A rerun must still POST when no exact terminal dispatch job exists.""" workflow = WORKFLOW_PATH.read_text(encoding="utf-8") coordinator_if = workflow.split(" dispatch-current-head:\n", 1)[1].split( "\n runs-on:", 1 @@ -108,7 +84,7 @@ def test_codeql_coordinator_dispatches_later_attempts_when_no_terminal_verdict() coordinator = workflow.split(" dispatch-current-head:\n", 1)[1] assert "github.run_attempt == 1" not in coordinator_if - assert "All detected CodeQL languages already have authenticated terminal verdicts" in coordinator + assert "exact terminal dispatch verdicts" in coordinator assert 'event_type:"codeql-scan"' in coordinator assert "required_jobs:$required_jobs" in coordinator assert "required_run_id:$required_run_id" in coordinator @@ -126,7 +102,7 @@ def test_codeql_coordinator_dispatches_later_attempts_when_no_terminal_verdict() def test_codeql_pr_dispatch_and_release_run_blocks_are_valid_bash() -> None: - """Both run: blocks in analyze-head must be syntactically valid Bash.""" + """All required-workflow CodeQL run blocks remain valid Bash.""" workflow_text = WORKFLOW_PATH.read_text(encoding="utf-8") if sys.platform == "win32": @@ -158,13 +134,14 @@ def test_codeql_pr_dispatch_and_release_run_blocks_are_valid_bash() -> None: def _dispatch_scan_title( *, head_sha: str = _TEST_HEAD_SHA, + base_ref: str = "main", base_sha: str = _TEST_BASE_SHA, required_run_id: str = _TEST_REQUIRED_RUN_ID, ) -> str: """Return the immutable CodeQL dispatch run-name for one required shard.""" return ( "CodeQL Scan Dispatch ContextualWisdomLab/naruon#42@" - f"{head_sha}/{base_sha}/{required_run_id}" + f"{head_sha}/{base_ref}@{base_sha}/{required_run_id}" ) @@ -192,7 +169,7 @@ def _run_verdict_read( dispatch_jobs: dict | list[dict] | None = None, run_attempt: str = "2", ) -> tuple[subprocess.CompletedProcess[str], subprocess.CompletedProcess[str]]: - """Execute the real one-shot status read and verdict enforcement blocks.""" + """Execute the real one-shot exact-run read and verdict enforcement blocks.""" bash = shutil.which("bash") jq = shutil.which("jq") assert bash is not None and jq is not None, "bash and jq are required to run this test" @@ -204,7 +181,7 @@ def _run_verdict_read( head_sha = _TEST_HEAD_SHA live_pr = { "head": {"sha": head_sha}, - "base": {"sha": _TEST_BASE_SHA}, + "base": {"ref": "main", "sha": _TEST_BASE_SHA}, "state": "open", } @@ -284,18 +261,8 @@ def _run_verdict_read( return dispatch_result, verdict_result -def test_codeql_pr_one_shot_read_ignores_status_forged_by_non_opencode_creator(tmp_path: Path) -> None: - """A PR-forged 'codeql-dispatch/: success' status must not stand in for the real verdict. - - Only a status published by codeql-scan-dispatch.yml's own app identity - (opencode-agent[bot], minted via the same OIDC exchange - opencode-review-dispatch.yml uses) may satisfy the verdict read -- matching the - context string alone is not enough, since anyone with statuses:write on - the repository can publish an arbitrary context (ADR 0025, "Poll target - cannot be spoofed by the PR author"). This proves the forged success is - skipped in favor of the legitimate (here, failing) verdict rather than - accepted. - """ +def test_codeql_pr_one_shot_read_ignores_all_head_only_statuses(tmp_path: Path) -> None: + """Creator-authenticated statuses cannot replace exact base/run evidence.""" dispatch_result, verdict_result = _run_verdict_read( tmp_path, statuses=[ @@ -306,14 +273,18 @@ def test_codeql_pr_one_shot_read_ignores_status_forged_by_non_opencode_creator(t "creator": {"login": "opencode-agent[bot]"}, }, ], + run_attempt="1", ) assert dispatch_result.returncode == 0, dispatch_result.stderr assert verdict_result.returncode == 1, verdict_result.stderr - assert "did not pass (state=failure)" in verdict_result.stdout + assert "CodeQL scan dispatched" in verdict_result.stdout + assert "state=failure" not in verdict_result.stdout -def test_codeql_pr_one_shot_read_accepts_the_opencode_agent_creator(tmp_path: Path) -> None: - """The legitimate handler's own success status is accepted once creator identity matches.""" +def test_codeql_pr_one_shot_read_does_not_accept_trusted_status_without_exact_run( + tmp_path: Path, +) -> None: + """Even the handler's own status is observability-only without exact run evidence.""" dispatch_result, verdict_result = _run_verdict_read( tmp_path, statuses=[ @@ -323,22 +294,18 @@ def test_codeql_pr_one_shot_read_accepts_the_opencode_agent_creator(tmp_path: Pa "creator": {"login": "opencode-agent[bot]"}, } ], + run_attempt="1", ) assert dispatch_result.returncode == 0, dispatch_result.stderr - assert verdict_result.returncode == 0, verdict_result.stderr - assert "Current-head CodeQL dispatch verdict for python: success." in verdict_result.stdout + assert verdict_result.returncode == 1, verdict_result.stderr + assert "CodeQL scan dispatched" in verdict_result.stdout + assert "Current-head CodeQL dispatch verdict for python: success." not in verdict_result.stdout def test_codeql_pr_one_shot_read_accepts_completed_dispatch_scan_job_when_status_unpublishable( tmp_path: Path, ) -> None: - """A completed dispatch scan job is terminal evidence when statuses:write 403s. - - Live 2026-09-08 naruon#1596 dispatch run 34173910106 scanned clean, then - POST /statuses returned HTTP 403 for opencode-agent (statuses:read only) - and github.token (cross-repo). The required shard must consume that - completed scan job instead of staying fail-closed on a missing status. - """ + """A completed exact dispatch scan job is terminal evidence when status publish 403s.""" head_sha = _TEST_HEAD_SHA title = _dispatch_scan_title(head_sha=head_sha) dispatch_result, verdict_result = _run_verdict_read( @@ -356,7 +323,7 @@ def test_codeql_pr_one_shot_read_accepts_completed_dispatch_scan_job_when_status ) assert dispatch_result.returncode == 0, dispatch_result.stderr + dispatch_result.stdout assert verdict_result.returncode == 0, verdict_result.stderr + verdict_result.stdout - assert "completed CodeQL dispatch scan job for python: success" in dispatch_result.stdout + assert "exact completed CodeQL dispatch scan job for python: success" in dispatch_result.stdout assert "Current-head CodeQL dispatch verdict for python: success." in verdict_result.stdout @@ -388,18 +355,13 @@ def test_codeql_pr_finds_completed_dispatch_scan_beyond_first_results_page( assert dispatch_result.returncode == 0, dispatch_result.stderr + dispatch_result.stdout assert verdict_result.returncode == 0, verdict_result.stderr + verdict_result.stdout - assert "completed CodeQL dispatch scan job for python: success" in dispatch_result.stdout + assert "exact completed CodeQL dispatch scan job for python: success" in dispatch_result.stdout def test_codeql_pr_rejects_completed_dispatch_scan_from_a_stale_base( tmp_path: Path, ) -> None: - """Same head and language after a base retarget must not reuse the prior scan. - - A PR can keep its head SHA while the base moves. The native handler already - binds receipts to the live base SHA; the required shard must not accept a - completed dispatch whose run-name still names the predecessor base. - """ + """Same head and language after a base retarget must not reuse the prior scan.""" stale_title = _dispatch_scan_title(base_sha="c" * 40) dispatch_result, _verdict_result = _run_verdict_read( tmp_path, @@ -416,20 +378,14 @@ def test_codeql_pr_rejects_completed_dispatch_scan_from_a_stale_base( ) assert dispatch_result.returncode == 1, dispatch_result.stderr + dispatch_result.stdout - assert "without an authenticated terminal verdict" in dispatch_result.stdout - assert "completed CodeQL dispatch scan job for python: success" not in dispatch_result.stdout + assert "without an exact terminal dispatch verdict" in dispatch_result.stdout + assert "exact completed CodeQL dispatch scan job for python: success" not in dispatch_result.stdout def test_codeql_pr_rejects_completed_dispatch_scan_from_a_different_required_run( tmp_path: Path, ) -> None: - """A same-PR/head/language scan for another required run cannot wake this shard. - - Language plus repository/PR/head is not enough: each waiting required job - lives in one required-workflow run. Binding required_run_id in the - dispatch run-name, together with the language job name, is the job - identity the shard can observe without reading client_payload. - """ + """A same-PR/head/language scan for another required run cannot wake this shard.""" other_run_title = _dispatch_scan_title(required_run_id="99") dispatch_result, _verdict_result = _run_verdict_read( tmp_path, @@ -448,8 +404,8 @@ def test_codeql_pr_rejects_completed_dispatch_scan_from_a_different_required_run ) assert dispatch_result.returncode == 1, dispatch_result.stderr + dispatch_result.stdout - assert "without an authenticated terminal verdict" in dispatch_result.stdout - assert "completed CodeQL dispatch scan job for python: success" not in dispatch_result.stdout + assert "without an exact terminal dispatch verdict" in dispatch_result.stdout + assert "exact completed CodeQL dispatch scan job for python: success" not in dispatch_result.stdout def test_codeql_pr_fallback_binds_live_base_and_required_run_identity() -> None: @@ -460,12 +416,14 @@ def test_codeql_pr_fallback_binds_live_base_and_required_run_identity() -> None: )[0] assert "REQUIRED_RUN_ID: ${{ github.run_id }}" in shard + assert 'live_base_ref="$(printf' in shard assert 'live_base="$(printf' in shard assert ( 'expected_title="CodeQL Scan Dispatch ${TARGET_REPOSITORY}#${PR_NUMBER}' - '@${PR_HEAD_SHA}/${live_base}/${REQUIRED_RUN_ID}"' + '@${PR_HEAD_SHA}/${live_base_ref}@${live_base}/${REQUIRED_RUN_ID}"' ) in shard assert "Could not validate live pull request base SHA before CodeQL verdict read." in shard + assert "commits/${PR_HEAD_SHA}/statuses" not in shard def test_codeql_action_steps_use_one_version_per_workflow() -> None: @@ -496,7 +454,7 @@ def test_codeql_shard_releases_runner_and_reads_exact_head_verdict() -> None: assert "required_job_id:$required_job_id" not in shard assert "required_language:$required_language" not in shard assert "The dispatch workflow will rerun this exact failed CodeQL job" in shard - assert "commits/${PR_HEAD_SHA}/statuses" in shard + assert "commits/${PR_HEAD_SHA}/statuses" not in shard assert "repos/ContextualWisdomLab/.github/dispatches" not in shard @@ -518,7 +476,7 @@ def test_codeql_required_workflow_does_not_gain_actions_write() -> None: def test_codeql_pr_attempt_one_without_verdict_fails_pending_without_dispatch( tmp_path: Path, ) -> None: - """Attempt 1 with no authenticated status releases the runner and does not POST.""" + """Attempt 1 with no exact terminal job releases the runner and does not POST.""" bash = shutil.which("bash") jq = shutil.which("jq") assert bash is not None and jq is not None, "bash and jq are required to run this test" @@ -557,7 +515,7 @@ def test_codeql_pr_attempt_one_without_verdict_fails_pending_without_dispatch( "FAKE_PULL_JSON": json.dumps( { "head": {"sha": head_sha}, - "base": {"sha": _TEST_BASE_SHA}, + "base": {"ref": "main", "sha": _TEST_BASE_SHA}, "state": "open", } ), @@ -606,6 +564,7 @@ def _write_coordinator_fakes( pull: dict, jobs: dict, statuses: list[dict], + dispatch_runs: dict, ) -> tuple[Path, Path, Path]: """Install fake gh/curl binaries and return (bin, post_log, post_body).""" fake_bin = tmp_path / "bin" @@ -621,12 +580,14 @@ def _write_coordinator_fakes( "method=GET\n" "path=\n" "jq_filter=\n" + "slurp=false\n" "while [ $# -gt 0 ]; do\n" ' case "$1" in\n' " -X) shift; method=$1 ;;\n" " --input) shift; input=$1 ;;\n" " --jq|-q) shift; jq_filter=$1 ;;\n" " --paginate) ;;\n" + " --slurp) slurp=true ;;\n" ' repos/*) path=$1 ;;\n' " esac\n" " shift || true\n" @@ -640,9 +601,11 @@ def _write_coordinator_fakes( 'case "$path" in\n' " */pulls/*) body=$FAKE_PULL_JSON ;;\n" " */statuses) body=$FAKE_STATUSES_JSON ;;\n" + " */codeql-scan-dispatch.yml/runs*) body=$FAKE_DISPATCH_RUNS_JSON ;;\n" " */actions/runs/*/jobs) body=$FAKE_JOBS_JSON ;;\n" " *) exit 1 ;;\n" "esac\n" + 'if [ "$slurp" = true ]; then body="[$body]"; fi\n' 'if [ -n "${jq_filter}" ]; then printf \'%s\\n\' "$body" | jq -c "$jq_filter"; else printf \'%s\\n\' "$body"; fi\n', encoding="utf-8", ) @@ -672,6 +635,7 @@ def _run_coordinator( pull: dict | None = None, jobs: dict | None = None, statuses: list[dict] | None = None, + dispatch_runs: dict | None = None, env_overrides: dict[str, str] | None = None, ) -> tuple[subprocess.CompletedProcess[str], Path, Path]: """Execute the coordinator dispatch block against fixture-backed APIs.""" @@ -703,8 +667,13 @@ def _run_coordinator( ], } statuses = statuses if statuses is not None else [] + dispatch_runs = dispatch_runs or {"workflow_runs": []} fake_bin, post_log, post_body = _write_coordinator_fakes( - tmp_path, pull=pull, jobs=jobs, statuses=statuses + tmp_path, + pull=pull, + jobs=jobs, + statuses=statuses, + dispatch_runs=dispatch_runs, ) script = _extract_run_block( WORKFLOW_PATH.read_text(encoding="utf-8"), COORDINATOR_STEP_NAME @@ -715,6 +684,7 @@ def _run_coordinator( "FAKE_PULL_JSON": json.dumps(pull), "FAKE_JOBS_JSON": json.dumps(jobs), "FAKE_STATUSES_JSON": json.dumps(statuses), + "FAKE_DISPATCH_RUNS_JSON": json.dumps(dispatch_runs), "FAKE_POST_LOG": str(post_log), "FAKE_POST_BODY": str(post_body), "FAKE_CURL_LOG": str(tmp_path / "curl.log"), @@ -774,12 +744,69 @@ def test_codeql_coordinator_posts_one_dispatch_for_every_pending_language( assert jobs_by_language == {"python": 101, "actions": 102} -def test_codeql_coordinator_skips_dispatch_when_every_language_has_a_verdict( +def test_codeql_coordinator_does_not_cancel_an_identical_active_dispatch( + tmp_path: Path, +) -> None: + """A partial-shard wake must not replace its still-running exact dispatch.""" + title = _dispatch_scan_title(required_run_id="99") + result, post_log, post_body = _run_coordinator( + tmp_path, + dispatch_runs={ + "workflow_runs": [ + { + "id": 123, + "event": "repository_dispatch", + "path": ".github/workflows/codeql-scan-dispatch.yml", + "status": "in_progress", + "display_title": title, + "name": title, + } + ] + }, + ) + + assert result.returncode == 0, result.stderr + result.stdout + assert not post_log.exists() + assert not post_body.exists() or post_body.read_text(encoding="utf-8") == "" + assert "Identical CodeQL dispatch is already active" in result.stdout + + +def test_codeql_coordinator_skips_dispatch_when_exact_run_has_terminal_jobs( tmp_path: Path, ) -> None: - """A rerun that already has terminal statuses must not enqueue another scan.""" + """A rerun with exact terminal language jobs must not enqueue another scan.""" + title = _dispatch_scan_title(required_run_id="99") result, post_log, post_body = _run_coordinator( tmp_path, + jobs={ + "total_count": 4, + "jobs": [ + { + "id": 101, + "name": "CodeQL compatibility analysis (python)", + "status": "completed", + "conclusion": "failure", + }, + { + "id": 102, + "name": "CodeQL compatibility analysis (actions)", + "status": "completed", + "conclusion": "failure", + }, + { + "id": 201, + "name": "CodeQL dispatch scan (python)", + "status": "completed", + "conclusion": "success", + }, + { + "id": 202, + "name": "CodeQL dispatch scan (actions)", + "status": "completed", + "conclusion": "failure", + }, + ], + }, statuses=[ { "context": "codeql-dispatch/python", @@ -792,12 +819,15 @@ def test_codeql_coordinator_skips_dispatch_when_every_language_has_a_verdict( "creator": {"login": "opencode-agent[bot]"}, }, ], + dispatch_runs={ + "workflow_runs": [_completed_dispatch_run(title=title, run_id=123)] + }, ) assert result.returncode == 0, result.stderr + result.stdout assert not post_log.exists() assert not post_body.exists() or post_body.read_text(encoding="utf-8") == "" - assert "already have authenticated terminal verdicts" in result.stdout + assert "exact terminal dispatch verdicts" in result.stdout def test_codeql_coordinator_fails_closed_when_a_shard_job_id_is_missing( diff --git a/tests/test_codeql_scan_dispatch_workflow_contract.py b/tests/test_codeql_scan_dispatch_workflow_contract.py index dd30c8506d..0c8c88f462 100644 --- a/tests/test_codeql_scan_dispatch_workflow_contract.py +++ b/tests/test_codeql_scan_dispatch_workflow_contract.py @@ -36,7 +36,7 @@ "Fetch the pinned CodeQL SARIF gate script", "Materialize pull request head for CodeQL scan", "Publish CodeQL dispatch status", - "Wake exact CodeQL required job", + "Wake exact CodeQL required run", ) @@ -518,6 +518,7 @@ def test_codeql_scan_dispatch_run_name_binds_base_and_required_run() -> None: group_value = workflow_level_concurrency_group(workflow) assert "github.event.client_payload.pr_head_sha" in header + assert "github.event.client_payload.pr_base_ref" in header assert "github.event.client_payload.pr_base_sha" in header assert "github.event.client_payload.required_run_id" in header assert "github.event.client_payload.pr_base_sha" not in group_value @@ -534,7 +535,7 @@ def test_dispatch_publish_keeps_successful_scan_when_status_write_is_denied() -> """ workflow = WORKFLOW_PATH.read_text(encoding="utf-8") publish = workflow.split(" - name: Publish CodeQL dispatch status\n", 1)[1].split( - "\n - name: Wake exact CodeQL required job\n", 1 + "\n wake-required-codeql:\n", 1 )[0] assert "GATE_OUTCOME" in publish @@ -544,39 +545,48 @@ def test_dispatch_publish_keeps_successful_scan_when_status_write_is_denied() -> assert "cancel-in-progress: true" not in publish -def test_dispatch_wakes_only_the_exact_failed_codeql_job() -> None: +def test_dispatch_wakes_failed_jobs_once_after_all_language_shards() -> None: workflow = WORKFLOW_PATH.read_text(encoding="utf-8") - wake = workflow.split(" - name: Wake exact CodeQL required job\n", 1)[1].split( + wake_job = workflow.split(" wake-required-codeql:\n", 1)[1] + wake = wake_job.split(" - name: Wake exact CodeQL required run\n", 1)[1].split( "\n\n - name:", 1 )[0] - assert "steps.publish_status.outcome == 'success'" in wake + assert "needs: [validate-dispatch, scan]" in wake_job + assert "always()" in wake_job + assert "needs.scan.result != 'cancelled'" in wake_job + assert "needs.validate-dispatch.outputs.base_ref != ''" in wake_job + assert "needs.validate-dispatch.outputs.base_sha != ''" in wake_job + assert "BASE_REF: ${{ needs.validate-dispatch.outputs.base_ref }}" in wake_job + assert "BASE_SHA: ${{ needs.validate-dispatch.outputs.base_sha }}" in wake_job assert 'gh api "repos/${TARGET_REPOSITORY}/pulls/${PR_NUMBER}"' in wake assert 'gh api "repos/${TARGET_REPOSITORY}/actions/runs/${REQUIRED_RUN_ID}"' in wake - assert 'gh api "repos/${TARGET_REPOSITORY}/actions/jobs/${REQUIRED_JOB_ID}"' in wake + assert 'gh api "repos/${TARGET_REPOSITORY}/actions/jobs/${required_job_id}"' in wake assert 'select(.event == "pull_request")' in wake assert 'select(.path == ".github/workflows/codeql-pr.yml")' in wake assert "select(.head_sha == $head)" in wake - assert "select(.run_id == $run_id)" in wake + assert ".base.ref == $base_ref" in wake + assert ".base.sha == $base" in wake + assert ".run_id == $run_id" in wake assert "select(.name == $name)" in wake assert 'select(.status == "completed" and .conclusion == "failure")' in wake - assert 'actions/jobs/${REQUIRED_JOB_ID}/rerun' in wake - assert "rerun-failed-jobs" not in wake - assert "while " not in wake + assert 'actions/runs/${REQUIRED_RUN_ID}/rerun-failed-jobs' in wake + assert 'actions/jobs/${required_job_id}/rerun"' not in wake assert "sleep " not in wake def test_dispatch_wake_has_only_trusted_actions_write_boundary() -> None: workflow = WORKFLOW_PATH.read_text(encoding="utf-8") - scan = workflow.split(" scan:\n", 1)[1] - scan_permissions = scan.split(" strategy:\n", 1)[0] + wake_job = workflow.split(" wake-required-codeql:\n", 1)[1] + wake_permissions = wake_job.split(" steps:\n", 1)[0] - assert "actions: write" in scan_permissions + assert "actions: write" in wake_permissions assert "pull_request:" not in workflow assert "pull_request_target:" not in workflow - assert "needs.validate-dispatch.outputs.required_run_id != ''" in scan - assert "needs.validate-dispatch.outputs.required_jobs != ''" in scan - assert "github.event.client_payload.required_job_id" not in scan + assert "needs.validate-dispatch.outputs.base_sha != ''" in wake_job + assert "needs.validate-dispatch.outputs.required_run_id != ''" in wake_job + assert "needs.validate-dispatch.outputs.required_jobs != ''" in wake_job + assert "github.event.client_payload.required_job_id" not in wake_job def _run_wake_step( @@ -584,15 +594,22 @@ def _run_wake_step( *, pull: dict | None = None, run: dict | None = None, - job: dict | None = None, + jobs: list[dict] | None = None, + rerun_error: str | None = None, ) -> tuple[subprocess.CompletedProcess[str], Path]: """Execute the exact wake block against fixture-backed GitHub API responses.""" bash = shutil.which("bash") jq = shutil.which("jq") assert bash is not None and jq is not None, "bash and jq are required to run this test" + base_sha = "a" * 40 head_sha = "b" * 40 - pull = pull or {"state": "open", "head": {"sha": head_sha}} + pull = pull or { + "state": "open", + "number": 42, + "base": {"ref": "main", "sha": base_sha}, + "head": {"sha": head_sha}, + } run = run or { "id": 42, "event": "pull_request", @@ -600,17 +617,27 @@ def _run_wake_step( "head_sha": head_sha, "status": "completed", "conclusion": "failure", + "pull_requests": [ + { + "number": 42, + "head": {"sha": head_sha}, + "base": {"ref": "main", "sha": base_sha}, + } + ], } - job = job or { - "id": 43, - "run_id": 42, - "head_sha": head_sha, - "name": "CodeQL compatibility analysis (python)", - "status": "completed", - "conclusion": "failure", - } + jobs = jobs or [ + { + "id": job_id, + "run_id": 42, + "head_sha": head_sha, + "name": f"CodeQL compatibility analysis ({language})", + "status": "completed", + "conclusion": "failure", + } + for language, job_id in (("python", 43), ("actions", 44)) + ] script = _extract_run_block( - WORKFLOW_PATH.read_text(encoding="utf-8"), "Wake exact CodeQL required job" + WORKFLOW_PATH.read_text(encoding="utf-8"), "Wake exact CodeQL required run" ) fake_bin = tmp_path / "bin" fake_bin.mkdir(parents=True) @@ -623,12 +650,19 @@ def _run_wake_step( 'if [ "${2:-}" = "-X" ]; then\n' ' test "$3" = POST\n' ' printf \'%s\\n\' "$4" >>"$FAKE_POST_LOG"\n' + ' if [ -n "$FAKE_RERUN_ERROR" ]; then\n' + ' printf \'%s\\n\' "$FAKE_RERUN_ERROR" >&2\n' + ' exit 1\n' + ' fi\n' " exit 0\n" "fi\n" 'case "$2" in\n' ' */pulls/*) printf \'%s\\n\' "$FAKE_PULL_JSON" ;;\n' ' */actions/runs/*) printf \'%s\\n\' "$FAKE_RUN_JSON" ;;\n' - ' */actions/jobs/*) printf \'%s\\n\' "$FAKE_JOB_JSON" ;;\n' + ' */actions/jobs/*)\n' + ' job_id="${2##*/}"\n' + ' jq -c --argjson job_id "$job_id" \'map(select(.id == $job_id)) | first // empty\' <<<"$FAKE_JOBS_JSON"\n' + ' ;;\n' " *) exit 1 ;;\n" "esac\n", encoding="utf-8", @@ -639,12 +673,15 @@ def _run_wake_step( "PATH": f"{fake_bin}:{os.environ['PATH']}", "FAKE_PULL_JSON": json.dumps(pull), "FAKE_RUN_JSON": json.dumps(run), - "FAKE_JOB_JSON": json.dumps(job), + "FAKE_JOBS_JSON": json.dumps(jobs), "FAKE_POST_LOG": str(post_log), + "FAKE_RERUN_ERROR": rerun_error or "", "GH_TOKEN": "fake-token", "WAKE_TOKEN_SOURCE": "PR_REVIEW_MERGE_TOKEN", "TARGET_REPOSITORY": "ContextualWisdomLab/naruon", "PR_NUMBER": "42", + "BASE_REF": "main", + "BASE_SHA": base_sha, "HEAD_SHA": head_sha, "REQUIRED_RUN_ID": "42", "REQUIRED_JOBS": json.dumps( @@ -653,7 +690,6 @@ def _run_wake_step( {"language": "actions", "job_id": 44}, ] ), - "REQUIRED_LANGUAGE": "python", } result = subprocess.run( [bash], input=script, text=True, capture_output=True, check=False, env=env @@ -661,12 +697,24 @@ def _run_wake_step( return result, post_log -def test_dispatch_wake_reruns_only_fixture_bound_exact_job(tmp_path: Path) -> None: +def test_dispatch_wake_fails_closed_when_run_rerun_is_rejected(tmp_path: Path) -> None: + result, post_log = _run_wake_step( + tmp_path, + rerun_error="gh: Resource not accessible by integration (HTTP 403)", + ) + + assert result.returncode == 1 + assert post_log.read_text(encoding="utf-8").splitlines() == [ + "repos/ContextualWisdomLab/naruon/actions/runs/42/rerun-failed-jobs" + ] + + +def test_dispatch_wake_reruns_exact_runs_failed_jobs_once(tmp_path: Path) -> None: result, post_log = _run_wake_step(tmp_path) assert result.returncode == 0, result.stderr assert post_log.read_text(encoding="utf-8").splitlines() == [ - "repos/ContextualWisdomLab/naruon/actions/jobs/43/rerun" + "repos/ContextualWisdomLab/naruon/actions/runs/42/rerun-failed-jobs" ] @@ -687,36 +735,39 @@ def test_dispatch_wake_rejects_stale_head_and_closed_pr(tmp_path: Path) -> None: def test_dispatch_wake_rejects_ambiguous_or_nonfailed_job_identity(tmp_path: Path) -> None: wrong_job_result, wrong_job_log = _run_wake_step( tmp_path / "wrong-job", - job={ - "id": 43, - "run_id": 999, - "head_sha": "b" * 40, - "name": "CodeQL compatibility analysis (python)", - "status": "completed", - "conclusion": "failure", - }, + jobs=[ + { + "id": 43, + "run_id": 999, + "head_sha": "b" * 40, + "name": "CodeQL compatibility analysis (python)", + "status": "completed", + "conclusion": "failure", + } + ], ) successful_job_result, successful_job_log = _run_wake_step( tmp_path / "successful-job", - job={ - "id": 43, - "run_id": 42, - "head_sha": "b" * 40, - "name": "CodeQL compatibility analysis (python)", - "status": "completed", - "conclusion": "success", - }, + jobs=[ + { + "id": 43, + "run_id": 42, + "head_sha": "b" * 40, + "name": "CodeQL compatibility analysis (python)", + "status": "completed", + "conclusion": "success", + } + ], ) assert wrong_job_result.returncode == 1 assert successful_job_result.returncode == 1 - assert "missing or ambiguous exact run/job identity" in wrong_job_result.stdout + assert "missing or ambiguous required job identity" in wrong_job_result.stdout assert not wrong_job_log.exists() assert not successful_job_log.exists() -def test_dispatch_wake_allows_parallel_language_rerun_on_same_exact_run(tmp_path: Path) -> None: - """Another language may already have moved the shared run back to in_progress.""" +def test_dispatch_wake_rejects_nonterminal_required_run(tmp_path: Path) -> None: result, post_log = _run_wake_step( tmp_path, run={ @@ -729,8 +780,8 @@ def test_dispatch_wake_allows_parallel_language_rerun_on_same_exact_run(tmp_path }, ) - assert result.returncode == 0, result.stderr - assert post_log.exists() + assert result.returncode == 1 + assert not post_log.exists() def test_codeql_scan_dispatch_serialises_the_matrix_payload() -> None: diff --git a/tests/test_codeql_verdict_exact_run_binding.py b/tests/test_codeql_verdict_exact_run_binding.py new file mode 100644 index 0000000000..176542b137 --- /dev/null +++ b/tests/test_codeql_verdict_exact_run_binding.py @@ -0,0 +1,108 @@ +"""Fail-closed tests for CodeQL terminal verdict identity across PR base changes.""" + +from __future__ import annotations + +import json +import os +import shutil +import subprocess +from pathlib import Path + +from tests.test_opencode_workflow_shell_syntax import _extract_run_block + +REPO_ROOT = Path(__file__).resolve().parents[1] +WORKFLOW_PATH = REPO_ROOT / ".github/workflows/codeql-pr.yml" + + +def _run_shard_verdict_reader_with_stale_status( + tmp_path: Path, +) -> tuple[subprocess.CompletedProcess[str], Path]: + """Run the production shard reader with only a head-scoped old-base status.""" + bash = shutil.which("bash") + assert bash is not None, "bash is required to run this test" + + head_sha = "b" * 40 + live_base_sha = "c" * 40 + pull = { + "state": "open", + "number": 42, + "base": {"ref": "main", "sha": live_base_sha}, + "head": {"sha": head_sha}, + } + # GitHub commit statuses have no PR-base or required-run identity. This + # terminal status represents evidence left on the same head by an earlier + # base/run and therefore must not authorize the current shard by itself. + statuses = [ + { + "context": "codeql-dispatch/python", + "state": "success", + "creator": {"login": "opencode-agent[bot]"}, + } + ] + + fake_bin = tmp_path / "bin" + fake_bin.mkdir(parents=True) + output = tmp_path / "github-output" + fake_gh = fake_bin / "gh" + fake_gh.write_text( + "#!/usr/bin/env bash\n" + "set -euo pipefail\n" + 'test "$1" = api\n' + "shift\n" + 'case "$*" in\n' + ' *"pulls/42"*) printf \'%s\\n\' "$FAKE_PULL_JSON" ;;\n' + ' *"statuses"*) printf \'%s\\n\' "$FAKE_STATUSES_JSON" ;;\n' + ' *"codeql-scan-dispatch.yml/runs"*) printf \'%s\\n\' \'[{"workflow_runs":[]}]\' ;;\n' + " *) exit 1 ;;\n" + "esac\n", + encoding="utf-8", + ) + fake_gh.chmod(0o755) + + script = _extract_run_block( + WORKFLOW_PATH.read_text(encoding="utf-8"), + "Read current-head CodeQL dispatch verdict", + ) + env = { + **os.environ, + "PATH": f"{fake_bin}:{os.environ['PATH']}", + "FAKE_PULL_JSON": json.dumps(pull), + "FAKE_STATUSES_JSON": json.dumps(statuses), + "GH_TOKEN": "fake-token", + "TARGET_REPOSITORY": "ContextualWisdomLab/example", + "PR_NUMBER": "42", + "PR_HEAD_SHA": head_sha, + "LANGUAGE": "python", + "RUN_ATTEMPT": "1", + "REQUIRED_RUN_ID": "99", + "GITHUB_OUTPUT": str(output), + } + result = subprocess.run( + [bash], + input=script, + text=True, + capture_output=True, + check=False, + env=env, + timeout=60, + ) + return result, output + + +def test_shard_rejects_terminal_status_without_exact_base_run_binding( + tmp_path: Path, +) -> None: + """A same-head status from another base/run is not terminal evidence.""" + result, output = _run_shard_verdict_reader_with_stale_status(tmp_path) + + assert result.returncode == 0, result.stderr + result.stdout + assert output.read_text(encoding="utf-8").splitlines() == ["verdict=pending"] + assert "Found authenticated current-head CodeQL verdict" not in result.stdout + + +def test_coordinator_does_not_suppress_dispatch_from_head_only_statuses() -> None: + """Pending-language admission must be derived from exact dispatch-run identity.""" + workflow = WORKFLOW_PATH.read_text(encoding="utf-8") + coordinator = workflow.split(" dispatch-current-head:\n", 1)[1] + + assert 'commits/${PR_HEAD_SHA}/statuses' not in coordinator diff --git a/tests/test_codeql_wake_base_binding.py b/tests/test_codeql_wake_base_binding.py new file mode 100644 index 0000000000..f682054c19 --- /dev/null +++ b/tests/test_codeql_wake_base_binding.py @@ -0,0 +1,148 @@ +"""Fail-closed contract for CodeQL wake identity across pull-request base changes.""" + +from __future__ import annotations + +import json +import os +import shutil +import subprocess +from pathlib import Path + +from tests.test_opencode_workflow_shell_syntax import _extract_run_block + +REPO_ROOT = Path(__file__).resolve().parents[1] +WORKFLOW_PATH = REPO_ROOT / ".github/workflows/codeql-scan-dispatch.yml" + + +def _run_wake( + tmp_path: Path, + *, + live_base_sha: str, + run_base_sha: str, + live_base_ref: str = "main", + run_base_ref: str = "main", +) -> tuple[subprocess.CompletedProcess[str], Path]: + """Execute the production wake block against base-aware GitHub API fixtures.""" + bash = shutil.which("bash") + jq = shutil.which("jq") + assert bash is not None and jq is not None + + expected_base_sha = "a" * 40 + head_sha = "b" * 40 + pull = { + "state": "open", + "number": 42, + "base": {"sha": live_base_sha, "ref": live_base_ref}, + "head": {"sha": head_sha}, + } + run = { + "id": 42, + "event": "pull_request", + "path": ".github/workflows/codeql-pr.yml", + "head_sha": head_sha, + "status": "completed", + "conclusion": "failure", + "pull_requests": [ + { + "number": 42, + "head": {"sha": head_sha}, + "base": {"sha": run_base_sha, "ref": run_base_ref}, + } + ], + } + jobs = [ + { + "id": 43, + "run_id": 42, + "head_sha": head_sha, + "name": "CodeQL compatibility analysis (python)", + "status": "completed", + "conclusion": "failure", + } + ] + + fake_bin = tmp_path / "bin" + fake_bin.mkdir(parents=True) + post_log = tmp_path / "posts" + fake_gh = fake_bin / "gh" + fake_gh.write_text( + "#!/usr/bin/env bash\n" + "set -euo pipefail\n" + 'test "$1" = api\n' + 'if [ "${2:-}" = "-X" ]; then\n' + ' test "$3" = POST\n' + ' printf \'%s\\n\' "$4" >>"$FAKE_POST_LOG"\n' + " exit 0\n" + "fi\n" + 'case "$2" in\n' + ' */pulls/*) printf \'%s\\n\' "$FAKE_PULL_JSON" ;;\n' + ' */actions/runs/*) printf \'%s\\n\' "$FAKE_RUN_JSON" ;;\n' + ' */actions/jobs/*) printf \'%s\\n\' "$FAKE_JOB_JSON" ;;\n' + " *) exit 1 ;;\n" + "esac\n", + encoding="utf-8", + ) + fake_gh.chmod(0o755) + + script = _extract_run_block( + WORKFLOW_PATH.read_text(encoding="utf-8"), "Wake exact CodeQL required run" + ) + env = { + **os.environ, + "PATH": f"{fake_bin}:{os.environ['PATH']}", + "FAKE_PULL_JSON": json.dumps(pull), + "FAKE_RUN_JSON": json.dumps(run), + "FAKE_JOB_JSON": json.dumps(jobs[0]), + "FAKE_POST_LOG": str(post_log), + "GH_TOKEN": "fake-token", + "WAKE_TOKEN_SOURCE": "PR_REVIEW_MERGE_TOKEN", + "TARGET_REPOSITORY": "ContextualWisdomLab/naruon", + "PR_NUMBER": "42", + "BASE_REF": "main", + "BASE_SHA": expected_base_sha, + "HEAD_SHA": head_sha, + "REQUIRED_RUN_ID": "42", + "REQUIRED_JOBS": json.dumps([{"language": "python", "job_id": 43}]), + } + result = subprocess.run( + [bash], input=script, text=True, capture_output=True, check=False, env=env + ) + return result, post_log + + +def test_wake_rejects_same_head_after_live_base_change(tmp_path: Path) -> None: + """Retargeting only the PR base invalidates an earlier validated wake identity.""" + result, post_log = _run_wake( + tmp_path, + live_base_sha="c" * 40, + run_base_sha="a" * 40, + ) + + assert result.returncode == 1 + assert not post_log.exists() + + +def test_wake_rejects_required_run_created_for_other_base(tmp_path: Path) -> None: + """An exact-head run from another base cannot authorize the current PR wake.""" + result, post_log = _run_wake( + tmp_path, + live_base_sha="a" * 40, + run_base_sha="c" * 40, + ) + + assert result.returncode == 1 + assert not post_log.exists() + + +def test_wake_rejects_same_base_sha_under_another_base_ref(tmp_path: Path) -> None: + """A same-SHA retarget to another branch invalidates the wake identity.""" + result, post_log = _run_wake( + tmp_path, + live_base_sha="a" * 40, + run_base_sha="a" * 40, + live_base_ref="release", + run_base_ref="release", + ) + + assert result.returncode == 1 + assert not post_log.exists() diff --git a/tests/test_scheduler_and_codeql_dispatch_runner_image_contract.py b/tests/test_scheduler_and_codeql_dispatch_runner_image_contract.py index ba0b2598a9..709c5342ef 100644 --- a/tests/test_scheduler_and_codeql_dispatch_runner_image_contract.py +++ b/tests/test_scheduler_and_codeql_dispatch_runner_image_contract.py @@ -51,10 +51,10 @@ def test_codeql_pr_uses_explicit_supported_image(self) -> None: self.assertEqual(workflow.count("runs-on: ubuntu-24.04"), 3) def test_codeql_scan_dispatch_uses_explicit_supported_image(self) -> None: - """Require both CodeQL Scan Dispatch jobs to pin Ubuntu 24.04.""" + """Require all CodeQL Scan Dispatch jobs to pin Ubuntu 24.04.""" workflow = CODEQL_SCAN_DISPATCH.read_text(encoding="utf-8") self.assertNotIn("runs-on: ubuntu-latest", workflow) - self.assertEqual(workflow.count("runs-on: ubuntu-24.04"), 2) + self.assertEqual(workflow.count("runs-on: ubuntu-24.04"), 3) def test_python_security_uses_explicit_supported_image(self) -> None: """Require all three Python Security jobs to pin Ubuntu 24.04."""