diff --git a/.github/workflows/codeql-pr.yml b/.github/workflows/codeql-pr.yml index c21c8446df..8c305ad1f8 100644 --- a/.github/workflows/codeql-pr.yml +++ b/.github/workflows/codeql-pr.yml @@ -227,7 +227,7 @@ jobs: .[] | .workflow_runs[] | select(.path == $path) | select(.event == "repository_dispatch") - | select(.status == "completed") + | select(.status == "completed" or .status == "in_progress") | select(.display_title == $title or .name == $title) ] | first @@ -236,7 +236,9 @@ jobs: if [[ "$run_id" =~ ^[1-9][0-9]*$ ]]; then jobs_json="$(gh api --paginate --slurp "repos/ContextualWisdomLab/.github/actions/runs/${run_id}/jobs")" job_conclusion="$(printf '%s' "$jobs_json" | jq -r --arg name "$expected_job" ' - [.[] | .jobs[] | select(.name == $name)] + [.[] | .jobs[]] as $jobs + | select([$jobs[] | select(.name == "validate-dispatch" and .status == "completed" and .conclusion == "success")] | length == 1) + | [$jobs[] | select(.name == $name and .status == "completed")] | if length == 1 then .[0].conclusion else empty end ')" case "$job_conclusion" in @@ -371,6 +373,18 @@ jobs: done < <(printf '%s' "$include_json" | jq -c '.[]') statuses="$(gh api "repos/${TARGET_REPOSITORY}/commits/${PR_HEAD_SHA}/statuses")" + expected_title="CodeQL Scan Dispatch ${TARGET_REPOSITORY}#${PR_NUMBER}@${PR_HEAD_SHA}/${live_base}/${REQUIRED_RUN_ID}" + dispatch_runs="$(gh api --paginate --slurp "repos/ContextualWisdomLab/.github/actions/workflows/codeql-scan-dispatch.yml/runs")" + dispatch_run_id="$(printf '%s' "$dispatch_runs" | jq -r --arg title "$expected_title" ' + [.[] | .workflow_runs[] + | select(.path == ".github/workflows/codeql-scan-dispatch.yml" and .event == "repository_dispatch") + | select(.status == "completed" or .status == "in_progress") + | select(.display_title == $title or .name == $title)] | first | .id // empty + ')" + dispatch_jobs='[]' + if [[ "$dispatch_run_id" =~ ^[1-9][0-9]*$ ]]; then + dispatch_jobs="$(gh api --paginate --slurp "repos/ContextualWisdomLab/.github/actions/runs/${dispatch_run_id}/jobs")" + fi pending_matrix='[]' while IFS= read -r entry; do language="$(printf '%s' "$entry" | jq -r '.language // empty')" @@ -385,6 +399,14 @@ jobs: ] | first // {} | .state // empty ')" + if ! [[ "$verdict_state" =~ ^(success|failure|error)$ ]]; then + verdict_state="$(printf '%s' "$dispatch_jobs" | jq -r --arg name "CodeQL dispatch scan (${language})" ' + [.[] | .jobs[]] as $jobs + | select([$jobs[] | select(.name == "validate-dispatch" and .status == "completed" and .conclusion == "success")] | length == 1) + | [$jobs[] | select(.name == $name and .status == "completed")] + | if length == 1 then .[0].conclusion else empty end + ')" + fi case "$verdict_state" in success|failure|error) echo "Found authenticated current-head CodeQL verdict for ${language}: ${verdict_state}." diff --git a/.github/workflows/codeql-scan-dispatch.yml b/.github/workflows/codeql-scan-dispatch.yml index c94fdf55c2..03c340bac5 100644 --- a/.github/workflows/codeql-scan-dispatch.yml +++ b/.github/workflows/codeql-scan-dispatch.yml @@ -277,7 +277,7 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 30 permissions: - actions: write + actions: read contents: read security-events: read id-token: write @@ -513,10 +513,23 @@ jobs: echo "::error::Could not publish the CodeQL dispatch status after all configured credentials failed; the exact required job will remain failed and will not be woken with stale or missing evidence." exit 1 + wake-required: + name: Wake verified CodeQL required jobs + needs: [validate-dispatch, scan] + if: >- + always() + && needs.validate-dispatch.result == 'success' + && needs.scan.result != 'cancelled' + && needs.scan.result != 'skipped' + runs-on: ubuntu-24.04 + timeout-minutes: 8 + permissions: + actions: write + contents: read + steps: - name: Wake exact CodeQL required job if: >- always() - && steps.publish_status.outcome == 'success' && needs.validate-dispatch.outputs.target_repository != '' && needs.validate-dispatch.outputs.pr_number != '' && needs.validate-dispatch.outputs.head_sha != '' @@ -529,7 +542,6 @@ jobs: HEAD_SHA: ${{ needs.validate-dispatch.outputs.head_sha }} REQUIRED_RUN_ID: ${{ needs.validate-dispatch.outputs.required_run_id }} REQUIRED_JOBS: ${{ needs.validate-dispatch.outputs.required_jobs }} - REQUIRED_LANGUAGE: ${{ matrix.language }} WAKE_TOKEN_SOURCE: ${{ needs.validate-dispatch.outputs.target_repository == github.repository && 'github-token' || secrets.PR_REVIEW_MERGE_TOKEN != '' && 'PR_REVIEW_MERGE_TOKEN' || secrets.OPENCODE_APPROVE_TOKEN != '' && 'OPENCODE_APPROVE_TOKEN' || 'unavailable' }} run: | set -euo pipefail @@ -537,13 +549,7 @@ jobs: echo "::error::Actions-capable CodeQL wake credential is unavailable." exit 1 fi - REQUIRED_JOB_ID="$(printf '%s' "$REQUIRED_JOBS" | jq -r --arg lang "$REQUIRED_LANGUAGE" ' - [.[] | select(.language == $lang) | .job_id | tostring] - | if length == 1 and (.[0] | test("^[1-9][0-9]*$")) then .[0] else empty end - ')" - if ! [[ "$REQUIRED_RUN_ID" =~ ^[1-9][0-9]*$ ]] || - ! [[ "$REQUIRED_JOB_ID" =~ ^[1-9][0-9]*$ ]] || - ! [[ "$REQUIRED_LANGUAGE" =~ ^[a-z0-9-]+$ ]]; then + if ! [[ "$REQUIRED_RUN_ID" =~ ^[1-9][0-9]*$ ]]; then echo "::error::CodeQL wake identity is non-canonical." exit 1 fi @@ -562,23 +568,28 @@ jobs: | select(.event == "pull_request") | select(.path == ".github/workflows/codeql-pr.yml") | select(.head_sha == $head) - | .id // empty - ')" - expected_name="CodeQL compatibility analysis (${REQUIRED_LANGUAGE})" - job="$(gh api "repos/${TARGET_REPOSITORY}/actions/jobs/${REQUIRED_JOB_ID}")" - job_identity="$(printf '%s' "$job" | jq -r --arg head "$HEAD_SHA" --arg name "$expected_name" --argjson run_id "$REQUIRED_RUN_ID" --argjson job_id "$REQUIRED_JOB_ID" ' - select(.id == $job_id) - | select(.run_id == $run_id) - | select(.head_sha == $head) - | select(.name == $name) | select(.status == "completed" and .conclusion == "failure") | .id // empty ')" + jobs="$(gh api --paginate --slurp "repos/${TARGET_REPOSITORY}/actions/runs/${REQUIRED_RUN_ID}/jobs")" + jobs_verified="$(printf '%s' "$jobs" | jq -r --arg head "$HEAD_SHA" --argjson run_id "$REQUIRED_RUN_ID" --argjson required "$REQUIRED_JOBS" ' + [.[] | .jobs[]] as $jobs + | [$jobs[] | select(.conclusion == "failure")] as $failed + | ($required | length > 0) + and ($failed | length > 0) + and ($jobs | all(.status == "completed")) + and ($required | all(. as $expected | + [$jobs[] | select(.id == $expected.job_id) + | select(.run_id == $run_id and .head_sha == $head) + | select(.name == ("CodeQL compatibility analysis (" + $expected.language + ")")) + | select(.conclusion == "failure")] | length == 1)) + and (($failed | map(.id) | sort) == ($required | map(.job_id) | sort)) + ')" if [ "$run_identity" != "$REQUIRED_RUN_ID" ] || - [ "$job_identity" != "$REQUIRED_JOB_ID" ]; then + [ "$jobs_verified" != "true" ]; then echo "::error::CodeQL wake rejected missing or ambiguous exact run/job identity." exit 1 fi - gh api -X POST "repos/${TARGET_REPOSITORY}/actions/jobs/${REQUIRED_JOB_ID}/rerun" >/dev/null - echo "Re-ran exact failed CodeQL job ${REQUIRED_JOB_ID} for ${REQUIRED_LANGUAGE} on ${HEAD_SHA}." + gh api -X POST "repos/${TARGET_REPOSITORY}/actions/runs/${REQUIRED_RUN_ID}/rerun-failed-jobs" >/dev/null + echo "Re-ran verified failed CodeQL jobs once for run ${REQUIRED_RUN_ID} on ${HEAD_SHA}." diff --git a/AGENTS.md b/AGENTS.md index e955f8b36a..7cf37ad836 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -171,6 +171,12 @@ them alone proves succession. ## Test-gate regressions and stale-PR merges +- Queue measurements must label timestamp, repository scope, and units. Workflow + runs, check runs, and executing jobs are not interchangeable; a repository's + in-progress run count cannot establish organization-wide job-ceiling utilization. + Distinguish runner-admission wait from execution time. Before consolidating + dynamic and central scanners, verify query coverage and target SARIF publication + equivalence; additional job fan-out alone does not prove duplicate security coverage. - A red `tests`, coverage, or `interrogate` gate on your pull request is not proof that your diff caused it. Full-suite execution on a push to `main` is not guaranteed: the workflows that run `pytest tests` on push are `paths:`-filtered, so a pairing broken outside their @@ -212,3 +218,17 @@ them alone proves succession. variable in CI, so a failure class exists that cannot reproduce locally. Before calling a scheduler change clean, run the affected tests both ways, including `GITHUB_ACTIONS=true python3 -m pytest `. +- CodeQL wake test doubles must reproduce GitHub rejecting a job rerun while + its containing run is already running. A fake POST that always succeeds hides + the language-shard race observed in dispatch run `34178442472`. Keep scan, + status publication, authenticated verdict consumption, and required-job + recovery as separate outcomes; a published `success` from an unaccepted + creator is not a passing required check. Do not mask this with blanket 403 + suppression, sleep polling, or a wider unauthenticated creator allowlist. +- Finish the language matrix before one trusted coordinator requests recovery. + Before a native failed-jobs rerun, verify the current PR head, completed failed + run, and the complete failed-job set against the supplied CodeQL identities. + Keep verdict readers and dispatch admission aligned: completed validated scan + jobs must stop redispatch after both success and real findings, even while + the recovery coordinator is still finishing. Otherwise recovery starts a new + scan loop or waits for its own containing workflow to finish. diff --git a/CHANGELOG.md b/CHANGELOG.md index bf192f6a9e..c36b0d5cc0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +### CodeQL failed-job recovery coordinates multilingual wakes + +- Restacked ContextualWisdomLab/.github#2032 onto protected main after live dispatch run `34182987578` reproduced the multilingual wake race: both CodeQL scans completed cleanly and published exact-head success, but the actions shard's second per-job rerun was rejected with HTTP 403 because the python shard had already moved required run `34181386094` back to `in_progress`. Recovery now waits for the complete scan matrix, validates the full failed-job set, and calls the native failed-jobs endpoint once. The merge preserves #2028's base-SHA and required-run identity binding; no status-creator trust or polling is widened. + ### Failed-check finding names the Strix sandbox instead of the gateway - `opencode-review-dispatch.yml`'s `emit_strix_provider_failure_finding` rendered one fixed finding for every `STRIX_PROVIDER_UNAVAILABLE` line, whose Root cause read "The contextual-orchestrator gateway or its discovered provider pool was unavailable for this run". `#1953` had just given the Strix sandbox bootstrap failure its own second verdict token (`STRIX_SANDBOX_UNAVAILABLE`) precisely because that attribution is wrong for it -- the sandbox container never reaches its Caido proxy, so the run dies before the gateway serves anything -- and this consumer re-applied the wrong attribution one step downstream, into the review findings and the failure census. The emitter now branches on the second token: a sandbox verdict gets a finding that names Strix's sandbox, says the verdict does not name the gateway, and tells the reader not to change gateway or provider configuration on its strength. A `STRIX_PROVIDER_UNAVAILABLE` line without the token keeps its existing text verbatim, so the gateway class has no regression surface. No test covered this finding text at all before (`gateway or its discovered provider pool` matched nothing under `tests/`); `tests/test_opencode_dispatch_strix_sandbox_finding.py` now runs the production emitter from the published run block and pins both directions plus the no-signal case. Refs #1953, #1935. diff --git a/docs/doctoring/codeql-matrix-wake-race.md b/docs/doctoring/codeql-matrix-wake-race.md new file mode 100644 index 0000000000..d68b613c8b --- /dev/null +++ b/docs/doctoring/codeql-matrix-wake-race.md @@ -0,0 +1,66 @@ +# CodeQL matrix recovery race + +Status: local implementation under verification; not merged or deployed. + +## Evidence and failure boundary + +On 2026-09-08, `.github` PR #2029 head +`eb79481bc1696c63273b6c2ca22b5e34f68d0208` produced two successful scan +statuses in dispatch run `34178442472`. Python job `101912523347` then failed +at the wake step with `The workflow run containing this job is already running +(HTTP 403)`. The actions shard had already requested recovery of required run +`34177963535`. This was not a CodeQL finding or a status publication denial. +Both published statuses were authored by `github-actions[bot]`, which the +required consumer deliberately does not trust as the OpenCode app identity. + +The existing test double accepted every POST. On base +`c99d49a86a2728d136d9e3a34a6b82a2c0e84a90`, all 24 dispatch tests passed. +Reproducing GitHub's observed active-run rejection made the parallel-language +test fail while completed-run recovery still passed (1 failed, 1 passed). + +## Proposed repair and alternatives + +One coordinator follows the complete scan matrix and requests one native +failed-jobs rerun. It validates the live open PR head, exact workflow/run, +terminal run state, and equality of the entire failed-job set with the supplied +CodeQL job identities. An unrelated failure, stale identity, or active run +fails closed without a POST. Scan jobs retain read-only Actions access; +recovery alone needs write access. No polling or extra model invocation is added. + +The consumer can read a completed scan job only after the same dispatch run's +validation job succeeds. The coordinator may still be finishing; requiring +the entire dispatch run to finish would introduce a race with the jobs it +just restarted. Admission uses the same terminal-evidence conditions to stop +rescanning both clean results and real findings. Findings remain failed checks. + +Rejected alternatives: ignoring 403 loses recovery; repeating per-language +requests races again; sleeping occupies runners without establishing identity; +trusting every status creator weakens the verification boundary. A separate +completion-event workflow adds another event and evidence-transfer contract. + +## Independent reproduction + +On 2026-09-08, .github PR #2033 head +`de96b8b46143fe63d8fec1929b5739a4babee8c4` reproduced the same boundary in +dispatch run `34182987578`. Both language scans completed cleanly and published +success statuses. Python woke failed required job `101922706890` first; while +required run `34181386094` was consequently running, the actions wake for job +`101922706920` was rejected with GitHub HTTP 403. This independently confirms +that per-language POSTs race on the shared run and that one post-matrix +failed-jobs request is the causal repair. + +## Verification still required + +Run both CodeQL contract modules, broader workflow admission/image contracts, +and current-head hosted checks. Inspect the rendered guidance. Prove one +recovery request for a real multilingual PR and terminal required checks before +claiming rollout. Local fake APIs cannot establish hosted convergence or relief +of the organization-wide 60-job ceiling. + +## Reference + +GitHub. (n.d.). *REST API endpoints for workflow runs*. Retrieved September 8, +2026, from https://docs.github.com/en/rest/actions/workflow-runs + +The documented failed-jobs endpoint restarts failed jobs and their dependent +jobs; therefore the full failed-job set must be validated before calling it. diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 1cc9e20313..9b94fc9c92 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -3214,18 +3214,32 @@ requesting `ubuntu-latest`. Pinned all six to `ubuntu-24.04` (10 total job occur `tests/test_scheduler_and_codeql_dispatch_runner_image_contract.py` covering all six. **This does not, by itself, explain today's stall**: a direct query of `.github`'s own queued-run backlog (307 queued, confirmed via `actions/runs?status=queued`, cross-checked against `status=in_progress` returning only -5-6 -- itself anomalous against the documented 60-job Team-plan ceiling, since 5-6 is far below 60) showed +5-6 workflow runs in this repository, not organization-wide executing jobs) showed the dominant contributors by far were `Required PR Review Merge Scheduler` (~32 of a ~300-run sample), `Python Security` (~29), `CodeQL PR` (~25), `Security Scan` (~23), `SAST Semgrep` (~20), and `Agent Review Runtime Quality CI` (~16) -- and four of those six (`pr-review-merge-scheduler.yml`, `security-scan.yml`, `sast-semgrep.yml`, `agent-review-runtime-quality-ci.yml`) were *already* pinned to `ubuntu-24.04` before this pass, per their own existing contract tests, and equally stuck. GitHub's own status page showed no -active incident at the time. The 5-6-vs-60 in-progress gap therefore remains unexplained -- not resolved -by this fix, not attributable to a known starved image, and not (per prior explicit ruling; see +active incident at the time. These observations do not establish a 5-6-vs-60 capacity gap: +a workflow run can contain multiple matrix jobs, and this repository-only sample excludes +other organization repositories. The congestion remains unresolved by this fix, is not +attributable to a known starved image, and is not (per prior explicit ruling; see `project_actions_plan_concurrency_ceiling.md`) a case for proposing paid additional capacity. Flagging for whoever investigates next: check org-level Actions settings (a policy-level concurrent-job cap below 60), a spending/usage limit (though billing access was unavailable to verify), or a GitHub-side runner -provisioning degradation not severe enough to reach the public status page. +provisioning degradation not severe enough to reach the public status page. First count actual +executing jobs across the organization at a recorded timestamp and distinguish runner admission +wait from execution duration; do not infer either a lower cap or spare capacity from run counts. + +**Measurement correction, 2026-09-08:** The historical 307 queued / 5-6 in-progress observations +above are retained as run-level evidence, not job-ceiling utilization. Current Naruon head +`64bf6c766e315b86eaa180fbd1a82f9087202e66` also exposed two dynamic Code Quality jobs and three +dynamic default-CodeQL jobs alongside the required central CodeQL lane. This is additional +fan-out, not proof of redundant coverage: default setup selects extended queries while the +inspected central source preserves SARIF as artifacts with `upload: false`. Coverage and target +publication equivalence must precede consolidation; see the verified +[queue finding](https://github.com/ContextualWisdomLab/.github/issues/712#issuecomment-5579207955) +and [coverage comparison](https://github.com/ContextualWisdomLab/.github/issues/712#issuecomment-5579235044). **Separately found while validating this fix, not yet fixed:** `tests/test_pr_review_autofix_nvidia_nim_contract.py::test_review_fix_caller_runs_once_each_hour` fails on a clean `origin/main` checkout, independent of this fix — `hourly-review-repair.yml` was renamed to @@ -3353,3 +3367,11 @@ queries the check-runs API at its own time, order-independently. The implementin their change was safe because they had scoped it narrowly, not because they had checked for the name collision — which is the more useful lesson: **a job name is unique only within one workflow file, and the same name in another file can carry the opposite safety property.** + +### 2026-09-08 — CodeQL multilingual wake race (Proposed) + +- **Context / owner:** CI/review/security/release control plane; canonical owner ContextualWisdomLab/.github, PR #2032. +- **Exact evidence:** dispatch run `34182987578` for .github#2033@`de96b8b46143fe63d8fec1929b5739a4babee8c4` completed both language scans and published `codeql-dispatch/python=success` and `codeql-dispatch/actions=success`. The actions wake then failed with GitHub HTTP 403 because the python wake had already restarted required run `34181386094`. +- **Gap / failure scene:** matrix shards independently mutated one shared required run. The first job rerun made the run active, so the second job could not wake; a clean security result remained a failed required check. +- **Action:** #2032 moves Actions write to one post-matrix coordinator, validates the exact open PR/head/base/run and complete failed-job set, then requests one `rerun-failed-jobs`. It retains #2028's immutable base-SHA and required-run binding and admits completed scan evidence only after dispatch validation succeeds. +- **Status:** Proposed. Non-force restacked onto protected main; a cross-PR RED then proved the coordinator still used the pre-#2028 title, and GREEN aligned it with `head/base/required_run`. Final local evidence is 52 focused contract cases plus Python/YAML syntax checks; exact-head hosted Checks remain authoritative before merge. diff --git a/tests/test_codeql_pr_workflow_contract.py b/tests/test_codeql_pr_workflow_contract.py index dc67eef258..cfa1d3837b 100644 --- a/tests/test_codeql_pr_workflow_contract.py +++ b/tests/test_codeql_pr_workflow_contract.py @@ -6,6 +6,8 @@ import sys from pathlib import Path +import pytest + from tests.test_opencode_workflow_shell_syntax import _extract_run_block @@ -172,13 +174,14 @@ def _completed_dispatch_run( *, title: str, run_id: int = 34173910106, + status: str = "completed", ) -> dict: """Return one completed central CodeQL dispatch workflow-run fixture.""" return { "id": run_id, "event": "repository_dispatch", "path": ".github/workflows/codeql-scan-dispatch.yml", - "status": "completed", + "status": status, "display_title": title, "name": title, } @@ -274,8 +277,8 @@ def _run_verdict_read( verdict_env = { **os.environ, "LANGUAGE": "python", - "DISPATCH_OUTCOME": "success", - "VERDICT_STATE": output_values["verdict"], + "DISPATCH_OUTCOME": "success" if dispatch_result.returncode == 0 else "failure", + "VERDICT_STATE": output_values.get("verdict", ""), } verdict_result = subprocess.run( [bash], input=verdict_script, text=True, capture_output=True, check=False, @@ -329,8 +332,15 @@ def test_codeql_pr_one_shot_read_accepts_the_opencode_agent_creator(tmp_path: Pa assert "Current-head CodeQL dispatch verdict for python: success." in verdict_result.stdout +@pytest.mark.parametrize("run_status,validation_status,scan_status,expected", [ + ("completed", "success", "completed", 0), + ("in_progress", "success", "completed", 0), + ("in_progress", "failure", "completed", 1), + ("in_progress", "success", "in_progress", 1), +]) def test_codeql_pr_one_shot_read_accepts_completed_dispatch_scan_job_when_status_unpublishable( tmp_path: Path, + run_status: str, validation_status: str, scan_status: str, expected: int, ) -> None: """A completed dispatch scan job is terminal evidence when statuses:write 403s. @@ -344,20 +354,31 @@ def test_codeql_pr_one_shot_read_accepts_completed_dispatch_scan_job_when_status dispatch_result, verdict_result = _run_verdict_read( tmp_path, statuses=[], - dispatch_runs={"workflow_runs": [_completed_dispatch_run(title=title)]}, + dispatch_runs={ + "workflow_runs": [ + _completed_dispatch_run(title=title, status=run_status) + ] + }, dispatch_jobs={ "jobs": [ + { + "name": "validate-dispatch", + "status": "completed", + "conclusion": validation_status, + }, { "name": "CodeQL dispatch scan (python)", + "status": scan_status, "conclusion": "success", } ] }, ) - assert dispatch_result.returncode == 0, dispatch_result.stderr + dispatch_result.stdout - assert verdict_result.returncode == 0, verdict_result.stderr + verdict_result.stdout - assert "completed CodeQL dispatch scan job for python: success" in dispatch_result.stdout - assert "Current-head CodeQL dispatch verdict for python: success." in verdict_result.stdout + assert dispatch_result.returncode == expected, dispatch_result.stderr + dispatch_result.stdout + assert verdict_result.returncode == expected, verdict_result.stderr + verdict_result.stdout + if expected == 0: + assert "completed CodeQL dispatch scan job for python: success" in dispatch_result.stdout + assert "Current-head CodeQL dispatch verdict for python: success." in verdict_result.stdout def test_codeql_pr_finds_completed_dispatch_scan_beyond_first_results_page( @@ -377,8 +398,10 @@ def test_codeql_pr_finds_completed_dispatch_scan_beyond_first_results_page( {"jobs": []}, { "jobs": [ + {"name": "validate-dispatch", "status": "completed", "conclusion": "success"}, { "name": "CodeQL dispatch scan (python)", + "status": "completed", "conclusion": "success", } ] @@ -407,6 +430,11 @@ def test_codeql_pr_rejects_completed_dispatch_scan_from_a_stale_base( dispatch_runs={"workflow_runs": [_completed_dispatch_run(title=stale_title)]}, dispatch_jobs={ "jobs": [ + { + "name": "validate-dispatch", + "status": "completed", + "conclusion": "success", + }, { "name": "CodeQL dispatch scan (python)", "conclusion": "success", @@ -439,6 +467,11 @@ def test_codeql_pr_rejects_completed_dispatch_scan_from_a_different_required_run }, dispatch_jobs={ "jobs": [ + { + "name": "validate-dispatch", + "status": "completed", + "conclusion": "success", + }, { "name": "CodeQL dispatch scan (python)", "conclusion": "success", @@ -468,6 +501,17 @@ def test_codeql_pr_fallback_binds_live_base_and_required_run_identity() -> None: assert "Could not validate live pull request base SHA before CodeQL verdict read." in shard +def test_codeql_coordinator_fallback_binds_live_base_and_required_run_identity() -> None: + """Coordinator lookup must use the exact dispatch run identity from #2028.""" + workflow = WORKFLOW_PATH.read_text(encoding="utf-8") + coordinator = workflow.split(" dispatch-current-head:\n", 1)[1] + + assert ( + 'expected_title="CodeQL Scan Dispatch ${TARGET_REPOSITORY}#${PR_NUMBER}' + '@${PR_HEAD_SHA}/${live_base}/${REQUIRED_RUN_ID}"' + ) in coordinator + + def test_codeql_action_steps_use_one_version_per_workflow() -> None: """Prevent CodeQL init/analyze version splits from failing the scheduled scan.""" workflow = (REPO_ROOT / ".github/workflows/scheduled-security-scan.yml").read_text( @@ -640,6 +684,8 @@ def _write_coordinator_fakes( 'case "$path" in\n' " */pulls/*) body=$FAKE_PULL_JSON ;;\n" " */statuses) body=$FAKE_STATUSES_JSON ;;\n" + " */codeql-scan-dispatch.yml/runs) body=$FAKE_DISPATCH_RUNS_JSON ;;\n" + " repos/ContextualWisdomLab/.github/actions/runs/*/jobs) body=$FAKE_DISPATCH_JOBS_JSON ;;\n" " */actions/runs/*/jobs) body=$FAKE_JOBS_JSON ;;\n" " *) exit 1 ;;\n" "esac\n" @@ -715,6 +761,8 @@ def _run_coordinator( "FAKE_PULL_JSON": json.dumps(pull), "FAKE_JOBS_JSON": json.dumps(jobs), "FAKE_STATUSES_JSON": json.dumps(statuses), + "FAKE_DISPATCH_RUNS_JSON": json.dumps([{"workflow_runs": []}]), + "FAKE_DISPATCH_JOBS_JSON": json.dumps([{"jobs": []}]), "FAKE_POST_LOG": str(post_log), "FAKE_POST_BODY": str(post_body), "FAKE_CURL_LOG": str(tmp_path / "curl.log"), @@ -800,6 +848,27 @@ def test_codeql_coordinator_skips_dispatch_when_every_language_has_a_verdict( assert "already have authenticated terminal verdicts" in result.stdout +@pytest.mark.parametrize("scan_conclusion", ["success", "failure"]) +def test_codeql_coordinator_does_not_redispatch_completed_scan_jobs( + tmp_path: Path, scan_conclusion: str, +) -> None: + """Terminal fallback evidence stops rescan loops, including real findings.""" + title = _dispatch_scan_title(required_run_id="99") + result, post_log, _ = _run_coordinator(tmp_path, env_overrides={ + "FAKE_DISPATCH_RUNS_JSON": json.dumps([{"workflow_runs": [{ + "id": 123, "path": ".github/workflows/codeql-scan-dispatch.yml", + "event": "repository_dispatch", "status": "in_progress", "display_title": title, + }]}]), + "FAKE_DISPATCH_JOBS_JSON": json.dumps([{"jobs": [ + {"name": "validate-dispatch", "status": "completed", "conclusion": "success"}, + *[{"name": f"CodeQL dispatch scan ({language})", "status": "completed", + "conclusion": scan_conclusion} for language in ("python", "actions")], + ]}]), + }) + assert result.returncode == 0, result.stderr + assert not post_log.exists() + + def test_codeql_coordinator_fails_closed_when_a_shard_job_id_is_missing( tmp_path: Path, ) -> None: diff --git a/tests/test_codeql_scan_dispatch_workflow_contract.py b/tests/test_codeql_scan_dispatch_workflow_contract.py index dd30c8506d..e0b87f7d71 100644 --- a/tests/test_codeql_scan_dispatch_workflow_contract.py +++ b/tests/test_codeql_scan_dispatch_workflow_contract.py @@ -525,7 +525,6 @@ def test_codeql_scan_dispatch_run_name_binds_base_and_required_run() -> None: assert "github.event.client_payload.target_repository" in group_value assert "github.event.client_payload.pr_number" in group_value - def test_dispatch_publish_keeps_successful_scan_when_status_write_is_denied() -> None: """A clean SARIF gate must not fail the handler solely because POST /statuses 403s. @@ -534,7 +533,7 @@ def test_dispatch_publish_keeps_successful_scan_when_status_write_is_denied() -> """ workflow = WORKFLOW_PATH.read_text(encoding="utf-8") publish = workflow.split(" - name: Publish CodeQL dispatch status\n", 1)[1].split( - "\n - name: Wake exact CodeQL required job\n", 1 + "\n wake-required:\n", 1 )[0] assert "GATE_OUTCOME" in publish @@ -550,18 +549,21 @@ def test_dispatch_wakes_only_the_exact_failed_codeql_job() -> None: "\n\n - name:", 1 )[0] - assert "steps.publish_status.outcome == 'success'" in wake + coordinator = workflow.split(" wake-required:\n", 1)[1] + assert "needs: [validate-dispatch, scan]" in coordinator + assert "matrix:" not in coordinator + assert "steps.publish_status.outcome" not in wake assert 'gh api "repos/${TARGET_REPOSITORY}/pulls/${PR_NUMBER}"' in wake assert 'gh api "repos/${TARGET_REPOSITORY}/actions/runs/${REQUIRED_RUN_ID}"' in wake - assert 'gh api "repos/${TARGET_REPOSITORY}/actions/jobs/${REQUIRED_JOB_ID}"' in wake + assert 'actions/runs/${REQUIRED_RUN_ID}/jobs' in wake assert 'select(.event == "pull_request")' in wake assert 'select(.path == ".github/workflows/codeql-pr.yml")' in wake assert "select(.head_sha == $head)" in wake - assert "select(.run_id == $run_id)" in wake - assert "select(.name == $name)" in wake + assert "select(.run_id == $run_id and .head_sha == $head)" in wake + assert '$expected.language' in wake assert 'select(.status == "completed" and .conclusion == "failure")' in wake - assert 'actions/jobs/${REQUIRED_JOB_ID}/rerun' in wake - assert "rerun-failed-jobs" not in wake + assert wake.count('gh api -X POST') == 1 + assert 'actions/runs/${REQUIRED_RUN_ID}/rerun-failed-jobs' in wake assert "while " not in wake assert "sleep " not in wake @@ -571,7 +573,8 @@ def test_dispatch_wake_has_only_trusted_actions_write_boundary() -> None: scan = workflow.split(" scan:\n", 1)[1] scan_permissions = scan.split(" strategy:\n", 1)[0] - assert "actions: write" in scan_permissions + assert "actions: write" not in scan_permissions + assert "actions: write" in workflow.split(" wake-required:\n", 1)[1] assert "pull_request:" not in workflow assert "pull_request_target:" not in workflow assert "needs.validate-dispatch.outputs.required_run_id != ''" in scan @@ -585,6 +588,7 @@ def _run_wake_step( pull: dict | None = None, run: dict | None = None, job: dict | None = None, + extra_jobs: list[dict] | None = None, ) -> tuple[subprocess.CompletedProcess[str], Path]: """Execute the exact wake block against fixture-backed GitHub API responses.""" bash = shutil.which("bash") @@ -623,9 +627,14 @@ def _run_wake_step( 'if [ "${2:-}" = "-X" ]; then\n' ' test "$3" = POST\n' ' printf \'%s\\n\' "$4" >>"$FAKE_POST_LOG"\n' + ' if [ "$(printf \'%s\' "$FAKE_RUN_JSON" | jq -r .status)" != completed ]; then\n' + ' echo "gh: The workflow run containing this job is already running (HTTP 403)" >&2\n' + ' exit 1\n' + ' fi\n' " exit 0\n" "fi\n" 'case "$2" in\n' + ' --paginate) printf \'%s\\n\' "$FAKE_JOBS_JSON" ;;\n' ' */pulls/*) printf \'%s\\n\' "$FAKE_PULL_JSON" ;;\n' ' */actions/runs/*) printf \'%s\\n\' "$FAKE_RUN_JSON" ;;\n' ' */actions/jobs/*) printf \'%s\\n\' "$FAKE_JOB_JSON" ;;\n' @@ -640,6 +649,11 @@ def _run_wake_step( "FAKE_PULL_JSON": json.dumps(pull), "FAKE_RUN_JSON": json.dumps(run), "FAKE_JOB_JSON": json.dumps(job), + "FAKE_JOBS_JSON": json.dumps([{"jobs": [job, { + "id": 44, "run_id": 42, "head_sha": head_sha, + "name": "CodeQL compatibility analysis (actions)", + "status": "completed", "conclusion": "failure", + }, *(extra_jobs or [])]}]), "FAKE_POST_LOG": str(post_log), "GH_TOKEN": "fake-token", "WAKE_TOKEN_SOURCE": "PR_REVIEW_MERGE_TOKEN", @@ -666,7 +680,7 @@ def test_dispatch_wake_reruns_only_fixture_bound_exact_job(tmp_path: Path) -> No assert result.returncode == 0, result.stderr assert post_log.read_text(encoding="utf-8").splitlines() == [ - "repos/ContextualWisdomLab/naruon/actions/jobs/43/rerun" + "repos/ContextualWisdomLab/naruon/actions/runs/42/rerun-failed-jobs" ] @@ -715,8 +729,8 @@ def test_dispatch_wake_rejects_ambiguous_or_nonfailed_job_identity(tmp_path: Pat assert not successful_job_log.exists() -def test_dispatch_wake_allows_parallel_language_rerun_on_same_exact_run(tmp_path: Path) -> None: - """Another language may already have moved the shared run back to in_progress.""" +def test_dispatch_wake_rejects_a_run_already_in_progress(tmp_path: Path) -> None: + """Never POST another rerun while the parent run is already active.""" result, post_log = _run_wake_step( tmp_path, run={ @@ -729,8 +743,18 @@ def test_dispatch_wake_allows_parallel_language_rerun_on_same_exact_run(tmp_path }, ) - assert result.returncode == 0, result.stderr - assert post_log.exists() + assert result.returncode == 1, result.stderr + assert not post_log.exists() + + +def test_dispatch_wake_rejects_unrelated_failed_jobs(tmp_path: Path) -> None: + """The native failed-jobs endpoint must not retry unvalidated jobs.""" + result, post_log = _run_wake_step(tmp_path, extra_jobs=[{ + "id": 45, "run_id": 42, "head_sha": "b" * 40, + "name": "Other failed job", "status": "completed", "conclusion": "failure", + }]) + assert result.returncode == 1 + assert not post_log.exists() def test_codeql_scan_dispatch_serialises_the_matrix_payload() -> None: diff --git a/tests/test_scheduler_and_codeql_dispatch_runner_image_contract.py b/tests/test_scheduler_and_codeql_dispatch_runner_image_contract.py index ba0b2598a9..38750871b5 100644 --- a/tests/test_scheduler_and_codeql_dispatch_runner_image_contract.py +++ b/tests/test_scheduler_and_codeql_dispatch_runner_image_contract.py @@ -54,7 +54,10 @@ def test_codeql_scan_dispatch_uses_explicit_supported_image(self) -> None: """Require both CodeQL Scan Dispatch jobs to pin Ubuntu 24.04.""" workflow = CODEQL_SCAN_DISPATCH.read_text(encoding="utf-8") self.assertNotIn("runs-on: ubuntu-latest", workflow) - self.assertEqual(workflow.count("runs-on: ubuntu-24.04"), 2) + self.assertEqual(workflow.count("runs-on: ubuntu-24.04"), 3) + recovery = workflow.split(" wake-required:\n", 1)[1] + self.assertIn("needs: [validate-dispatch, scan]", recovery) + self.assertNotIn("matrix:", recovery) def test_python_security_uses_explicit_supported_image(self) -> None: """Require all three Python Security jobs to pin Ubuntu 24.04."""