diff --git a/.github/workflows/noema-review.yml b/.github/workflows/noema-review.yml index df72f616ca..d5ebffeca2 100644 --- a/.github/workflows/noema-review.yml +++ b/.github/workflows/noema-review.yml @@ -5,7 +5,7 @@ run-name: >- github.event.client_payload.pr_number || github.event.pull_request.number || github.event.workflow_run.pull_requests[0].number || 'event' }}@${{ github.event.client_payload.pr_head_sha || github.event.pull_request.head.sha || - github.event.workflow_run.pull_requests[0].head.sha || github.sha }} + github.event.workflow_run.head_sha || github.sha }} on: pull_request_target: @@ -58,137 +58,55 @@ jobs: shell: bash run: | set -euo pipefail - - # cancel_runs prints the number of runs it matched for $1's status - # on stdout (its only stdout output) so the multi-pass loop below - # can tell whether a pass found anything; all human-facing log - # lines go to stderr so they don't pollute that count. - # - # The runs list is scoped to this repository, not to a specific - # workflow file: noema-review.yml runs against sibling - # repositories only through the organization's required-workflow - # ruleset (README.md's "또 같이" / "siblings call it" section) and - # is never itself committed to those repositories, so - # actions/workflows/noema-review.yml/runs is not guaranteed to - # resolve there -- GitHub's List repository workflows family - # enumerates workflow files that exist in that repository's own - # tree. actions/runs plus the run object's own `.path` field is - # this repo's own already-proven pattern for this exact cross-repo - # cleanup (see strix.yml's identical job). - # Status stays a server-side filter -- bounding each query to only - # the currently active runs -- rather than an unfiltered - # per-workflow fetch filtered client-side, since noema-review.yml - # is this org's central, highest-volume review workflow and an - # unbounded history walk on every PR close is a real rate-limit - # and latency risk here. - cancel_runs() { - local status="$1" - local runs_url="repos/${TARGET_REPOSITORY}/actions/runs?status=${status}&per_page=100" - local runs_json - if ! runs_json="$(gh api --paginate "$runs_url" 2>/tmp/noema-close-gh-error)"; then - echo "::warning::Noema close cleanup could not inspect ${TARGET_REPOSITORY}; leaving runs unchanged." >&2 - sed 's/^/ /' /tmp/noema-close-gh-error >&2 || true - echo 0 - return 0 - fi - local run_ids - # PR-scoped by two independent, OR'd signals -- neither alone - # covers every trigger this job serves. The rendered - # display_title (this workflow's own run-name, embedding the - # target repository/PR number/head SHA) is this workflow's - # original signal, and stays reliable for repository_dispatch - # and workflow_run triggers. But GitHub does not consistently - # render run-name for an organization-required-workflow - # pull_request_target run materialized in a sibling repository - # (Devin Review, PR #1507: "Sibling Noema runs evade - # cancellation") -- `name` and `display_title` can both collapse - # to the bare workflow name and the plain PR title there, - # matching neither the old `.name ==` filter nor the - # display_title prefix below. GitHub's own `pull_requests[]` - # array on the run object closes that gap: it is populated for - # this workflow's pull_request_target runs because the - # noema-review job itself only ever processes same-repository, - # non-fork pull requests (its own `if:` requires - # `head.repo.full_name == github.repository`), so the cross-fork - # "empty pull_requests[]" caveat that rules this field out - # elsewhere in this org's tooling does not apply here. Neither - # signal alone is sufficient for every trigger type, so this - # matches on either one -- never a bare head_sha, which two - # different open PRs can share (e.g. a duplicate PR opened from - # the same branch against another target) and which would let - # closing one cancel the other's still-needed run. Matching by - # PR number rather than by the closed PR's current head SHA also - # means historical-head runs from earlier pushes to this same PR - # are still caught. `.path` pins the workflow identity in place - # of the old `.name ==` filter: unlike `.name` (which, like - # display_title, only carries the bare workflow name for a - # required-workflow-ruleset run), `.path` was independently - # confirmed stable across both native and sibling contexts. - if ! run_ids="$(jq -r --arg pr "$CLOSED_PR_NUMBER" \ - --arg current "$CURRENT_RUN_ID" --arg target "$TARGET_REPOSITORY" ' - .workflow_runs[] - | select((.id | tostring) != $current) - | select(.path == ".github/workflows/noema-review.yml") - | select((.name // "") | startswith("Required Noema Review")) - | select( - ((.display_title // "") | startswith("Required Noema Review " + $target + "#" + $pr + "@")) - or ((.pull_requests // []) | any(.number == ($pr | tonumber))) - ) - | .id - ' <<<"$runs_json")"; then - echo "::warning::Noema close cleanup received invalid run data for ${TARGET_REPOSITORY}; leaving runs unchanged." >&2 - echo 0 - return 0 - fi - local matched=0 - while IFS= read -r run_id; do - [ -n "$run_id" ] || continue - matched=$((matched + 1)) - if gh api --method POST "repos/${TARGET_REPOSITORY}/actions/runs/${run_id}/cancel" >/dev/null 2>/tmp/noema-close-cancel-error; then - echo "Cancelled Noema run ${run_id} in ${TARGET_REPOSITORY} for closed PR #${CLOSED_PR_NUMBER}." >&2 - else - echo "::warning::Noema close cleanup could not cancel run ${run_id}; it may have finished or the token lacks Actions write access." >&2 - sed 's/^/ /' /tmp/noema-close-cancel-error >&2 || true - fi - done <<<"$run_ids" - echo "$matched" - } - - # A run can transition between the five active statuses between - # one status's fetch and the next (e.g. it is "requested" when the - # already-fetched "queued" list was read, then becomes "queued" - # moments later, after this pass has already moved past checking - # "queued") -- a real GitHub Actions run lifecycle race, not a - # hypothetical. A single sequential sweep can let such a run - # escape cancellation entirely. Re-scan every active status for up - # to three passes: always run at least two full passes (a run that - # slips through every status query in pass 1 has, by definition, - # settled into a checkable status by the time pass 2 queries it - # again), and only skip the third when both prior passes matched - # nothing, bounding the retries so API flakiness cannot loop this - # forever. - max_passes=3 - pass=1 - found_any=0 - while [ "$pass" -le "$max_passes" ]; do + declare -A seen=() + for pass in 1 2 3; do pass_matches=0 for active_status in queued in_progress requested waiting pending; do - matched="$(cancel_runs "$active_status")" - pass_matches=$((pass_matches + matched)) + runs_url="repos/${TARGET_REPOSITORY}/actions/runs?status=${active_status}&per_page=100" + if ! runs_json="$(gh api --paginate "$runs_url")"; then + echo "::warning::Noema close cleanup could not inspect ${TARGET_REPOSITORY}; leaving this status unchanged." + continue + fi + run_ids="$(jq -r --arg pr "$CLOSED_PR_NUMBER" \ + --arg current "$CURRENT_RUN_ID" --arg target "$TARGET_REPOSITORY" ' + .workflow_runs[] + | select((.id | tostring) != $current) + | select(.path == ".github/workflows/noema-review.yml") + | select((.name // "") | startswith("Required Noema Review")) + | select( + ((.display_title // "") | startswith("Required Noema Review " + $target + "#" + $pr + "@")) + or ((.pull_requests // []) | any(.number == ($pr | tonumber))) + ) + | .id + ' <<<"$runs_json")" + while IFS= read -r run_id; do + [ -n "$run_id" ] || continue + [ -z "${seen[$run_id]:-}" ] || continue + pass_matches=$((pass_matches + 1)) + if gh api --method POST "repos/${TARGET_REPOSITORY}/actions/runs/${run_id}/cancel" >/dev/null; then + seen[$run_id]=1 + else + echo "::warning::Noema close cleanup could not cancel run ${run_id}." + fi + done <<<"$run_ids" done - echo "Noema close cleanup pass ${pass}/${max_passes} matched ${pass_matches} run(s) across active statuses." >&2 - if [ "$pass_matches" -gt 0 ]; then - found_any=1 - fi - if [ "$pass" -ge 2 ] && [ "$pass_matches" -eq 0 ] && [ "$found_any" -eq 0 ]; then - break - fi - pass=$((pass + 1)) + [ "$pass" -lt 2 ] || [ "$pass_matches" -gt 0 ] || break done - noema-review: - name: noema-review + prepare: + name: noema-review / prepare runs-on: ubuntu-latest + # Preparation is network/API work only; model serving is isolated in the + # two bounded candidate jobs below. + timeout-minutes: 30 + permissions: + actions: write + contents: read + checks: read + pull-requests: read + outputs: + require_zdr: ${{ steps.target_visibility.outputs.require_zdr }} + review_ready: ${{ steps.seal.outputs.review_ready }} if: >- github.event_name == 'repository_dispatch' || ( @@ -200,12 +118,6 @@ jobs: && github.event.action != 'closed' && github.event.pull_request.head.repo.full_name == github.repository ) - permissions: - actions: write - checks: read - contents: read - id-token: write - pull-requests: read env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true TARGET_REPOSITORY: ${{ github.event.pull_request.base.repo.full_name || github.event.client_payload.target_repository || github.repository }} @@ -317,20 +229,6 @@ jobs: echo "::warning::Could not inspect ${active_status} Noema runs for superseded heads." continue fi - # See the close-cleanup job's matching comment above cancel_runs's - # own selector for the full rationale: display_title only - # renders this workflow's PR/head-bearing run-name reliably for - # a native trigger, so a sibling-repository required-workflow - # run is additionally matched via GitHub's own pull_requests[] - # array (populated here because noema-review only ever - # processes same-repository, non-fork pull requests), and - # `.path` pins workflow identity where `.name` cannot. The - # live-head exclusion below is independently reinforced with a - # direct `.head_sha` comparison -- the run object's own - # head_sha field, unlike display_title, is populated and - # accurate regardless of run-name rendering, so it protects the - # current run even when its display_title never rendered a - # matching "@$head" suffix to exclude by. if ! run_ids="$(jq -r --arg pr "$PR_NUMBER" --argjson current "$CURRENT_RUN_ID" \ --arg target "$TARGET_REPOSITORY" --arg head "$EXPECTED_HEAD" ' .workflow_runs[] @@ -351,17 +249,8 @@ jobs: while IFS= read -r run_id; do [ -n "$run_id" ] || continue [ -z "${seen[$run_id]:-}" ] || continue - seen[$run_id]=1 - # A transient failure here (rate limit, network blip) must - # never crash this step under set -e: this is a housekeeping - # cleanup, and letting an ancillary API hiccup fail the whole - # job would block a perfectly valid, live-head review over - # something unrelated to it. Treat "cannot verify" the same - # as "verified stale": stop cancelling rather than risk a - # wrong cancellation, but let the job continue. - if ! live_head="$(gh api "repos/${TARGET_REPOSITORY}/pulls/${PR_NUMBER}" --jq '.head.sha' 2>/tmp/noema-supersede-live-head-error)"; then - echo "::warning::Noema cleanup could not re-verify the live PR head before cancelling run ${run_id}; stopping cleanup rather than risking a wrong cancellation." >&2 - sed 's/^/ /' /tmp/noema-supersede-live-head-error >&2 || true + if ! live_head="$(gh api "repos/${TARGET_REPOSITORY}/pulls/${PR_NUMBER}" --jq '.head.sha')"; then + echo "::warning::Noema cleanup could not re-verify the live PR head before cancelling run ${run_id}; stopping cleanup." exit 0 fi if [ "${live_head,,}" != "${EXPECTED_HEAD,,}" ]; then @@ -369,10 +258,11 @@ jobs: exit 0 fi if gh api --method POST "repos/${TARGET_REPOSITORY}/actions/runs/${run_id}/cancel" >/dev/null; then + seen[$run_id]=1 cancelled=$((cancelled + 1)) echo "Cancelled superseded Noema run ${run_id} for PR #${PR_NUMBER}." else - echo "::warning::Could not cancel superseded Noema run ${run_id}; it may already be terminal." + echo "::warning::Could not cancel superseded Noema run ${run_id}; a later pass may retry it." fi done <<<"$run_ids" done @@ -533,46 +423,269 @@ jobs: ;; esac - - name: Provision contextual-orchestrator review sidecar + - name: Seal exact-head Noema review input if: env.PR_NUMBER != '' + id: seal env: + GH_TOKEN: ${{ secrets.NOEMA_REVIEW_TOKEN || steps.noema_github_app_token.outputs.token || steps.noema_oidc_token.outputs.token }} + NOEMA_REVIEW_TOKEN_SOURCE: ${{ steps.noema_credential.outputs.source == 'github-app' && 'noema-review-github-app' || steps.noema_credential.outputs.source == 'pat' && 'noema-review-pat' || 'noema-review-app-oidc' }} + NOEMA_REVIEW_ACTOR: ${{ steps.noema_github_app_token.outputs['app-slug'] && format('{0}[bot]', steps.noema_github_app_token.outputs['app-slug']) || '' }} + NOEMA_REVIEW_INSTALLATION_ID: ${{ steps.noema_github_app_token.outputs['installation-id'] }} + run: | + set -euo pipefail + python3 -m scripts.ci.noema_review_gate \ + --repo "$TARGET_REPOSITORY" \ + --pr-number "$PR_NUMBER" \ + --mode prepare \ + --expected-head "$EXPECTED_HEAD" \ + --output "${RUNNER_TEMP}/noema-input.json" + if [ -s "${RUNNER_TEMP}/noema-input.json" ] && [ -s "${RUNNER_TEMP}/noema-input.json.sha256" ]; then + echo "review_ready=true" >>"$GITHUB_OUTPUT" + else + echo "review_ready=false" >>"$GITHUB_OUTPUT" + fi + + - name: Upload sealed Noema review input + if: steps.seal.outputs.review_ready == 'true' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: noema-review-input + path: | + ${{ runner.temp }}/noema-input.json + ${{ runner.temp }}/noema-input.json.sha256 + if-no-files-found: error + retention-days: 1 + + candidate-1: + name: noema-review / candidate-1 + needs: prepare + if: needs.prepare.outputs.review_ready == 'true' + runs-on: ubuntu-latest + timeout-minutes: 350 + permissions: + actions: read + contents: read + env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + steps: + - name: Materialize trusted Noema source + env: + GH_TOKEN: ${{ github.token }} + TRUSTED_SOURCE_REF: ${{ github.workflow_sha }} + run: | + set -euo pipefail + [[ "$TRUSTED_SOURCE_REF" =~ ^[0-9a-fA-F]{40}$ ]] + curl -fsSL -H "Authorization: Bearer ${GH_TOKEN}" -H "Accept: application/vnd.github+json" \ + -o "${RUNNER_TEMP}/trusted.tar.gz" "${GITHUB_API_URL}/repos/ContextualWisdomLab/.github/tarball/${TRUSTED_SOURCE_REF}" + tar -xzf "${RUNNER_TEMP}/trusted.tar.gz" -C "$GITHUB_WORKSPACE" --strip-components=1 + - name: Download sealed Noema review input + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: noema-review-input + path: ${{ runner.temp }}/noema-input + - name: Provision candidate pool + id: provision + continue-on-error: true + env: &provider_credentials BYTEZ_API_KEY: ${{ secrets.BYTEZ_API_KEY }} NVIDIA_NIM_API_KEY: ${{ secrets.NVIDIA_NIM_API_KEY }} NVIDIA_NIM_API_KEY_SUB: ${{ secrets.NVIDIA_NIM_API_KEY_SUB }} OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - CONTEXTUAL_ORCHESTRATOR_REQUIRE_ZDR: ${{ steps.target_visibility.outputs.require_zdr }} - run: | - set -euo pipefail - bash "$GITHUB_WORKSPACE/scripts/ci/contextual_orchestrator_review_sidecar.sh" - - - name: Run Noema LLM review and submit verdict - if: env.PR_NUMBER != '' - env: - GH_TOKEN: ${{ secrets.NOEMA_REVIEW_TOKEN || steps.noema_github_app_token.outputs.token || steps.noema_oidc_token.outputs.token }} - NOEMA_REVIEW_TOKEN_SOURCE: ${{ steps.noema_credential.outputs.source == 'pat' && 'noema-review-pat' || steps.noema_credential.outputs.source == 'github-app' && 'noema-review-github-app' || 'noema-review-app-oidc' }} - NOEMA_REVIEW_ACTOR: ${{ steps.noema_github_app_token.outputs['app-slug'] && format('{0}[bot]', steps.noema_github_app_token.outputs['app-slug']) || '' }} - NOEMA_REVIEW_INSTALLATION_ID: ${{ steps.noema_github_app_token.outputs['installation-id'] }} + CONTEXTUAL_ORCHESTRATOR_REQUIRE_ZDR: ${{ needs.prepare.outputs.require_zdr }} + REVIEW_PREFLIGHT_GATEWAY_MAX_TIME_SECONDS: "600" + run: bash "$GITHUB_WORKSPACE/scripts/ci/contextual_orchestrator_review_sidecar.sh" --single-candidate-attempt + - name: Run first candidate + id: review + if: steps.provision.outcome == 'success' + continue-on-error: true + timeout-minutes: 335 run: | set -euo pipefail - if [ -z "${PR_NUMBER:-}" ]; then - echo "No pull request number was available for this event; skipping." - exit 0 - fi - if [ -z "${GH_TOKEN:-}" ]; then - echo "::error::Noema reviewer credential selection succeeded but no token was minted; review cannot submit a verdict." - exit 1 - fi if [ -z "${CONTEXTUAL_ORCHESTRATOR_BASE_URL:-}" ] || [ -z "${CONTEXTUAL_ORCHESTRATOR_TOKEN_FILE:-}" ]; then echo "::error::contextual-orchestrator review sidecar must be provisioned before Noema LLM review." exit 1 fi source "$GITHUB_WORKSPACE/scripts/ci/load_contextual_orchestrator_token.sh" + candidate_id="$(jq -er '.routes[] | select(.status == "ready") | .agent_id' "$CONTEXTUAL_ORCHESTRATOR_PREFLIGHT_EVIDENCE" | head -1)" + printf '%s\n' "$candidate_id" >"${RUNNER_TEMP}/candidate-1.id" export NOEMA_LLM_API_URL="${CONTEXTUAL_ORCHESTRATOR_BASE_URL%/}/v1/chat/completions" export NOEMA_LLM_MODEL="orchestrator/free" - export NOEMA_LLM_API_KEY="${CONTEXTUAL_ORCHESTRATOR_TOKEN}" + export NOEMA_LLM_API_KEY="$CONTEXTUAL_ORCHESTRATOR_TOKEN" export NOEMA_LLM_VIA_ORCHESTRATOR=1 - python3 -m scripts.ci.noema_review_gate \ - --repo "$TARGET_REPOSITORY" \ - --pr-number "$PR_NUMBER" \ - --expected-head "$EXPECTED_HEAD" + export NOEMA_LLM_CANDIDATE_ID="$candidate_id" + python3 -m scripts.ci.noema_review_gate --repo placeholder/repo --pr-number 1 \ + --mode evaluate --input "${RUNNER_TEMP}/noema-input/noema-input.json" \ + --output "${RUNNER_TEMP}/noema-verdict.json" + - name: Guarantee first candidate status handoff + if: always() + run: | + set -euo pipefail + if [ ! -e "${RUNNER_TEMP}/candidate-1.id" ]; then + : >"${RUNNER_TEMP}/candidate-1.id" + fi + - name: Upload first candidate handoff + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: noema-candidate-1 + path: | + ${{ runner.temp }}/candidate-1.id + ${{ runner.temp }}/noema-verdict.json + ${{ runner.temp }}/noema-verdict.json.sha256 + if-no-files-found: error + retention-days: 1 + + candidate-2: + name: noema-review / candidate-2 + needs: [prepare, candidate-1] + if: always() && needs.prepare.result == 'success' && needs.prepare.outputs.review_ready == 'true' + runs-on: ubuntu-latest + timeout-minutes: 350 + permissions: + actions: read + contents: read + steps: + - name: Materialize trusted Noema source + env: + GH_TOKEN: ${{ github.token }} + TRUSTED_SOURCE_REF: ${{ github.workflow_sha }} + run: | + set -euo pipefail + [[ "$TRUSTED_SOURCE_REF" =~ ^[0-9a-fA-F]{40}$ ]] + curl -fsSL -H "Authorization: Bearer ${GH_TOKEN}" -H "Accept: application/vnd.github+json" \ + -o "${RUNNER_TEMP}/trusted.tar.gz" "${GITHUB_API_URL}/repos/ContextualWisdomLab/.github/tarball/${TRUSTED_SOURCE_REF}" + tar -xzf "${RUNNER_TEMP}/trusted.tar.gz" -C "$GITHUB_WORKSPACE" --strip-components=1 + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: noema-review-input + path: ${{ runner.temp }}/noema-input + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: noema-candidate-1 + path: ${{ runner.temp }}/candidate-1 + - name: Reuse successful first verdict + id: reuse + run: | + if [ -s "${RUNNER_TEMP}/candidate-1/noema-verdict.json" ] && [ -s "${RUNNER_TEMP}/candidate-1/noema-verdict.json.sha256" ]; then + cp "${RUNNER_TEMP}/candidate-1/noema-verdict.json" "${RUNNER_TEMP}/noema-verdict.json" + cp "${RUNNER_TEMP}/candidate-1/noema-verdict.json.sha256" "${RUNNER_TEMP}/noema-verdict.json.sha256" + echo "reused=true" >>"$GITHUB_OUTPUT" + else + echo "reused=false" >>"$GITHUB_OUTPUT" + fi + - name: Provision fallback candidate pool + if: steps.reuse.outputs.reused != 'true' + env: *provider_credentials + run: | + CONTEXTUAL_ORCHESTRATOR_EXCLUDE_CANDIDATE_ID="$(cat "${RUNNER_TEMP}/candidate-1/candidate-1.id" 2>/dev/null || true)" + export CONTEXTUAL_ORCHESTRATOR_EXCLUDE_CANDIDATE_ID + bash "$GITHUB_WORKSPACE/scripts/ci/contextual_orchestrator_review_sidecar.sh" --single-candidate-attempt + - name: Run second candidate + if: steps.reuse.outputs.reused != 'true' + timeout-minutes: 335 + run: | + set -euo pipefail + source "$GITHUB_WORKSPACE/scripts/ci/load_contextual_orchestrator_token.sh" + first_id="$(cat "${RUNNER_TEMP}/candidate-1/candidate-1.id" 2>/dev/null || true)" + candidate_id="$(jq -er --arg excluded "$first_id" '.routes[] | select(.status == "ready" and .agent_id != $excluded) | .agent_id' "$CONTEXTUAL_ORCHESTRATOR_PREFLIGHT_EVIDENCE" | head -1)" + export NOEMA_LLM_API_URL="${CONTEXTUAL_ORCHESTRATOR_BASE_URL%/}/v1/chat/completions" + export NOEMA_LLM_MODEL="orchestrator/free" + export NOEMA_LLM_API_KEY="$CONTEXTUAL_ORCHESTRATOR_TOKEN" + export NOEMA_LLM_VIA_ORCHESTRATOR=1 + export NOEMA_LLM_CANDIDATE_ID="$candidate_id" + python3 -m scripts.ci.noema_review_gate --repo placeholder/repo --pr-number 1 \ + --mode evaluate --input "${RUNNER_TEMP}/noema-input/noema-input.json" \ + --output "${RUNNER_TEMP}/noema-verdict.json" + - name: Upload final candidate handoff + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: noema-candidate-final + path: | + ${{ runner.temp }}/noema-verdict.json + ${{ runner.temp }}/noema-verdict.json.sha256 + if-no-files-found: error + retention-days: 1 + + finalize: + name: noema-review + needs: [prepare, candidate-2] + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + actions: read + contents: read + id-token: write + pull-requests: write + env: + TARGET_REPOSITORY: ${{ github.event.pull_request.base.repo.full_name || github.event.client_payload.target_repository || github.repository }} + PR_NUMBER: ${{ github.event.pull_request.number || github.event.workflow_run.pull_requests[0].number || github.event.client_payload.pr_number || '' }} + steps: + - name: Materialize trusted Noema source + env: + GH_TOKEN: ${{ github.token }} + TRUSTED_SOURCE_REF: ${{ github.workflow_sha }} + run: | + set -euo pipefail + [[ "$TRUSTED_SOURCE_REF" =~ ^[0-9a-fA-F]{40}$ ]] + curl -fsSL -H "Authorization: Bearer ${GH_TOKEN}" -H "Accept: application/vnd.github+json" \ + -o "${RUNNER_TEMP}/trusted.tar.gz" "${GITHUB_API_URL}/repos/ContextualWisdomLab/.github/tarball/${TRUSTED_SOURCE_REF}" + tar -xzf "${RUNNER_TEMP}/trusted.tar.gz" -C "$GITHUB_WORKSPACE" --strip-components=1 + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: noema-review-input + path: ${{ runner.temp }}/noema-input + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: noema-candidate-final + path: ${{ runner.temp }}/noema-verdict + - name: Select finalizer credential + id: credential + env: + APP_CLIENT_ID: ${{ vars.NOEMA_GITHUB_APP_CLIENT_ID || '' }} + APP_PRIVATE_KEY: ${{ secrets.NOEMA_GITHUB_APP_PRIVATE_KEY || '' }} + REVIEW_TOKEN: ${{ secrets.NOEMA_REVIEW_TOKEN || '' }} + EXCHANGE_URL: ${{ vars.NOEMA_TOKEN_EXCHANGE_URL || vars.NOEMA_EXCHANGE_URL || '' }} + run: | + set -euo pipefail + echo "repository=${TARGET_REPOSITORY#*/}" >>"$GITHUB_OUTPUT" + if [ -n "$REVIEW_TOKEN" ]; then echo "source=pat" >>"$GITHUB_OUTPUT" + elif [ -n "$APP_CLIENT_ID" ] && [ -n "$APP_PRIVATE_KEY" ]; then echo "source=github-app" >>"$GITHUB_OUTPUT" + elif [ -n "$EXCHANGE_URL" ]; then echo "source=oidc" >>"$GITHUB_OUTPUT" + else echo "::error::Noema reviewer credential is unavailable."; exit 1; fi + - name: Mint repository-scoped Noema GitHub App token + id: app_token + if: steps.credential.outputs.source == 'github-app' + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + client-id: ${{ vars.NOEMA_GITHUB_APP_CLIENT_ID }} + private-key: ${{ secrets.NOEMA_GITHUB_APP_PRIVATE_KEY }} + owner: ContextualWisdomLab + repositories: ${{ steps.credential.outputs.repository }} + permission-contents: read + permission-pull-requests: write + - name: Exchange finalizer token through OIDC + if: steps.credential.outputs.source == 'oidc' + id: oidc_token + env: + EXCHANGE_URL: ${{ vars.NOEMA_TOKEN_EXCHANGE_URL || vars.NOEMA_EXCHANGE_URL || '' }} + OIDC_AUDIENCE: ${{ vars.NOEMA_OIDC_AUDIENCE || 'cwl-noema-review' }} + run: | + set -euo pipefail + separator='?'; [[ "$ACTIONS_ID_TOKEN_REQUEST_URL" == *\?* ]] && separator='&' + oidc="$(curl -fsS -H "Authorization: Bearer ${ACTIONS_ID_TOKEN_REQUEST_TOKEN}" "${ACTIONS_ID_TOKEN_REQUEST_URL}${separator}audience=${OIDC_AUDIENCE}" | jq -er .value)" + token="$(curl -fsS -X POST -H 'Content-Type: application/json' -H "Authorization: Bearer ${oidc}" --data "$(jq -cn --arg target_repository "$TARGET_REPOSITORY" '{target_repository:$target_repository}')" "$EXCHANGE_URL" | jq -er .token)" + echo "::add-mask::$token" + echo "token=$token" >>"$GITHUB_OUTPUT" + - name: Submit sealed exact-head verdict + env: + GH_TOKEN: ${{ secrets.NOEMA_REVIEW_TOKEN || steps.app_token.outputs.token || steps.oidc_token.outputs.token }} + NOEMA_REVIEW_TOKEN_SOURCE: ${{ steps.credential.outputs.source == 'github-app' && 'noema-review-github-app' || steps.credential.outputs.source == 'pat' && 'noema-review-pat' || 'noema-review-app-oidc' }} + NOEMA_REVIEW_ACTOR: ${{ steps.app_token.outputs['app-slug'] && format('{0}[bot]', steps.app_token.outputs['app-slug']) || '' }} + NOEMA_REVIEW_INSTALLATION_ID: ${{ steps.app_token.outputs['installation-id'] }} + run: | + set -euo pipefail + test -n "${GH_TOKEN:-}" || { echo "::error::Noema reviewer credential is unavailable."; exit 1; } + python3 -m scripts.ci.noema_review_gate --repo "$TARGET_REPOSITORY" --pr-number "$PR_NUMBER" \ + --mode finalize --input "${RUNNER_TEMP}/noema-input/noema-input.json" \ + --verdict "${RUNNER_TEMP}/noema-verdict/noema-verdict.json" diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index 148e944310..f374475b75 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -339,8 +339,22 @@ jobs: echo "DEPENDENCY_REVIEW_SUPPORT repository=${REPOSITORY} visibility=${repository_visibility} base_sha=${BASE_SHA} head_sha=${HEAD_SHA} http_status=${http_status} curl_exit=${curl_status}" - if [ "$curl_status" -ne 0 ] || [ "$http_status" != "200" ]; then - echo "::error::Dependency review evidence unavailable for ${REPOSITORY} at exact base ${BASE_SHA} and head ${HEAD_SHA}: HTTP ${http_status}; curl exit ${curl_status}. Verify dependency-graph/security configuration and GitHub service behavior, then rerun. Failing closed." + evidence_state="complete" + unavailable_reason="none" + if [ "$curl_status" -ne 0 ]; then + evidence_state="unavailable" + unavailable_reason="transport" + elif [ "$http_status" = "403" ]; then + evidence_state="unavailable" + unavailable_reason="api_authorization" + elif [ "$http_status" != "200" ]; then + evidence_state="unavailable" + unavailable_reason="api_response" + fi + echo "DEPENDENCY_REVIEW_EVIDENCE state=${evidence_state} reason=${unavailable_reason} repository=${REPOSITORY} visibility=${repository_visibility} http_status=${http_status} curl_exit=${curl_status}" + + if [ "$evidence_state" != "complete" ]; then + echo "::error::Dependency review evidence unavailable for ${REPOSITORY} at exact base ${BASE_SHA} and head ${HEAD_SHA}: classification ${unavailable_reason}; HTTP ${http_status}; curl exit ${curl_status}. This is not a vulnerability-free result. Verify dependency-graph/security configuration and GitHub service behavior, then rerun. Failing closed." exit 1 fi diff --git a/.github/workflows/strix.yml b/.github/workflows/strix.yml index 505053287b..efaa1ea539 100644 --- a/.github/workflows/strix.yml +++ b/.github/workflows/strix.yml @@ -171,11 +171,12 @@ jobs: # standing operating directive accepts that central OpenCode/Strix/Noema # scans may take more than two hours per model (docs/product-goal-directive.md). # The scanner gets a 150-minute process budget and a 155-minute total - # retry budget; the 170-minute step and 200-minute job leave deterministic - # time to preserve partial reports and publish a concrete failure reason. + # retry budget. Three one-hour sidecar preflight attempts plus the + # 170-minute scan step fit within this six-hour job with ten minutes left + # to preserve partial reports and publish a concrete failure reason. # Hitting any cap is fail-closed and never turns an incomplete scan into # an approval. - timeout-minutes: 200 + timeout-minutes: 360 runs-on: ubuntu-latest # Least-privilege token scoped to this job (Scorecard alert #43): the scan # exchanges an OIDC token (id-token) and publishes same-repo status evidence @@ -565,8 +566,10 @@ jobs: ;; esac strix_model="$(printf '%s' "$STRIX_MODEL" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')" - echo "strix_model=$strix_model" >> "$GITHUB_OUTPUT" - echo 'enabled=true' >> "$GITHUB_OUTPUT" + { + echo "strix_model=$strix_model" + echo 'enabled=true' + } >> "$GITHUB_OUTPUT" echo 'provider_mode=contextual_orchestrator' >> "$GITHUB_OUTPUT" - name: Provision contextual-orchestrator Strix sidecar diff --git a/CHANGELOG.md b/CHANGELOG.md index 43020db98e..814206ccf6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,232 @@ this file. The format follows Keep a Changelog, and versioned releases follow Semantic Versioning where the repository publishes a release. ## [Unreleased] +- Mark `docs/nvidia-nim-opencode-hotfix.md` rolled back and historical: the + six-model NIM prefix it described was removed from + `opencode-review-dispatch.yml`'s `OPENCODE_MODEL_CANDIDATES` by `f8823a54` + (#1364, 2026-08-27), but the note itself was never updated per its own + "delete this note once restored" instruction and stayed factually stale + for about a month (last touched 2026-07-31, per #682) until this + correction. No code changed; this closes out the "worth a follow-up doc + cleanup" item recorded in `docs/product-technical-gap-baseline.md`'s + 2026-08-31 direct-NIM-communication audit entry. +- Remove `noema-review`'s 120-second serving cutoff while preserving the + realtime judge. Each candidate job makes one directly pinned request with + a 150-minute worker/judge serving budget, a 19,800-second absolute client + deadline, and a 335-minute step ceiling. The five-minute gap lets the + client deadline exit and preserve the candidate handoff before the step + ceiling; each 350-minute job retains another 15 minutes for that handoff. + A failed first candidate hands off + to one independent fallback job; its preflight excludes the attempted ID + before batched probing, then the request pins only the newly selected ID. + Drafts, context-free events, and exact-head reviews are successful no-ops + before artifacts or model work. These are the enforced values; earlier + 120-, 3,000-, 9,600-, and 23,040-second values were superseded during this + unreleased change and are intentionally not runtime contracts. +- Fix two real bugs Devin's automated review found on this same PR + (ContextualWisdomLab/.github#1415) against the just-landed + `_catalog_account_cap(DEFAULT_ACCOUNT_CAP)` fix and the discovery-budget + arithmetic: + 1. **Sidecar shell bypassed the policy account-cap default.** + `contextual_orchestrator_review_sidecar.sh` still unconditionally + exported `ORCHESTRATOR_CATALOG_ACCOUNT_CAP=8` whenever no operator + override was set — a leftover from an earlier round's + `CATALOG_FAMILY_CAP=24` → `CATALOG_ACCOUNT_CAP=8` rename that fixed the + variable's name but kept the wrong default value. Because the shell + always exported a concrete `8` before the Python launcher ever ran, + `_catalog_account_cap(DEFAULT_ACCOUNT_CAP)`'s own env-unset fallback to + `4` (`os.environ.get` only falls back when the key is absent) could + never actually trigger in production: every real run got a cap of 8, + not 4, so two NVIDIA credentials could still jointly occupy up to 16 of + the 24 preflight slots between them instead of the intended 8 (4 each). + Fixed by deriving the shell's default the same way + `sidecar_startup_watchdog_seconds` already derives its own default — + reading `contextual_orchestrator_review_policy.DEFAULT_ACCOUNT_CAP` at + runtime via a `python3 -c` one-liner — instead of hard-coding a numeric + literal; an explicit operator-set `ORCHESTRATOR_CATALOG_ACCOUNT_CAP` + still always wins. Updated the contract tests that pinned the old + literal (`test_contextual_orchestrator_review_sidecar_contract.py`, + `test_contextual_orchestrator_review_runtime_preflight.py`) and added + executable coverage that runs the real shell derivation block (not just + the Python helper in isolation) for the unset, overridden, empty, and + malformed-override cases. `docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md` + amended to record the correction so shell, Python, and ADR text agree + on one number (4). + 2. **Startup watchdog's discovery-time budget undercounted known + retries.** `REVIEW_DISCOVERY_MAX_SEQUENTIAL_HTTP_CALLS = 7` (feeding + `REVIEW_DISCOVERY_WORST_CASE_SECONDS` ≈ 105s and, in turn, + `REVIEW_STARTUP_WATCHDOG_SECONDS` ≈ 255s) counted only one + single-attempt call per registered source plus one unconditional + OpenRouter extra — it never accounted for retries or for calls the + pinned `contextual-orchestrator` revision actually makes beyond that. + Re-verified line-by-line against the vendored + `contextual_orchestrator/model_discovery.py` at the pinned + `ORCHESTRATOR_PIN_SHA`: the shared Models.dev fetch retries up to 3 + times (not 1); each of the sidecar's five credentialed sources' primary + listing fetch gets a base attempt plus one transient-failure retry (2 + each, not 1 — 10 total); OpenRouter alone makes two further + single-attempt calls (ZDR endpoints, provider policies) beyond its own + listing call, plus one concurrent (≤8-worker thread pool) endpoint-feed + round per currently free-priced model (live-verified against + OpenRouter's public catalog on 2026-08-31: 21 free models today, i.e. 3 + rounds; budgeted 5 rounds as documented headroom for catalog growth); + and `discover_all_models()` makes two further trailing global calls + once per run (a second, non-cached ZDR-endpoints fetch, plus the + credits check) that the old count missed entirely. New total: 22 + sequential-call-equivalents (3 + 5×2 + 2 + 5 + 2), raising + `REVIEW_DISCOVERY_WORST_CASE_SECONDS` to 330s and + `REVIEW_STARTUP_WATCHDOG_SECONDS` to 480s. The launcher now exposes + each sub-count as its own named constant + (`REVIEW_DISCOVERY_MODELS_DEV_MAX_ATTEMPTS`, + `REVIEW_DISCOVERY_CREDENTIALED_SOURCE_COUNT`, + `REVIEW_DISCOVERY_SOURCE_MAX_ATTEMPTS`, + `REVIEW_DISCOVERY_OPENROUTER_SINGLE_EXTRA_CALLS`, + `REVIEW_DISCOVERY_OPENROUTER_FREE_ENDPOINT_ROUND_CAP`, + `REVIEW_DISCOVERY_TRAILING_GLOBAL_CALLS`) rather than one opaque + literal, so a new + `test_startup_watchdog_covers_a_retry_heavy_discovery_reconstruction` + test can independently reconstruct the worst case from the enumerated + real request structure — not merely re-assert the module's own + arithmetic on its own constants, which would just re-encode the same + kind of undercounted assumption this fix corrects. +- Fix a real, live-evidenced bug in the sidecar's per-account catalog cap + (flagged in review on this same PR, + ContextualWisdomLab/.github#1415#issuecomment-5474321491): the + batched-preflight merge below introduced `_catalog_family_cap()`, which + defaulted to `REVIEW_PREFLIGHT_MAX_TOTAL_ROUTES` (24) whenever + `ORCHESTRATOR_CATALOG_FAMILY_CAP` was unset — the *total* preflight budget, + not a real per-account cap — silently disabling per-account + diversification entirely. Production evidence: the freshest `noema-review` + run showed `probed_count: 12, ready_count: 2, rejected_count: 10` (83% + rejected via 429/404/timeout) with the admitted free-pool catalog 100% + `nvidia_nim`/`nvidia_nim_sub` — two credentials sharing one rate-limited + upstream (`integrate.api.nvidia.com`) jointly occupying the entire 12-slot + preflight batch, reproduced byte-for-byte across two consecutive production + runs. This is the same mechanism CodeRabbit's automated walkthrough flagged + as "Merge Risk: Moderate" on this PR. Rebased onto `main`'s + `provider_account`/`account_cap` rename (#1468) and Noema-independence work + (#1477/#1480); the renamed `_catalog_account_cap(default)` helper now + requires its caller to supply `contextual_orchestrator_review_policy`'s own + `DEFAULT_ACCOUNT_CAP` (4) as the default, mirroring the equivalent hardening + in `main` PR #1487 (`_catalog_account_cap()` sourcing the account-cap + default from the policy module instead of a literal), and the entry below + claiming the 24-route default was intentional is superseded by this fix. An + explicit `ORCHESTRATOR_CATALOG_ACCOUNT_CAP` override remains honored. +- Merge the batched-concurrent-preflight architecture (below) with `main`'s + independently-landed ADR-0005 diagnostic, bounded-retry preflight: routes + are now probed `REVIEW_PREFLIGHT_BATCH_SIZE`-at-a-time, concurrently, up to + `REVIEW_PREFLIGHT_MAX_TOTAL_ROUTES` (24) total, while each candidate keeps + ADR-0005's cheap base probe (16 tokens) with a same-candidate escalation + retry to the real serving budget on a "budget too small" signature, bounded + by one `REVIEW_PREFLIGHT_MAX_ESCALATIONS` counter now made thread-safe + across concurrently-probed candidates. The sidecar's own separate gateway + smoke request keeps ADR-0005's bounded retry (up to + `REVIEW_PREFLIGHT_GATEWAY_MAX_ATTEMPTS`, 120s each, retried only on a + no-response/transport failure) rather than the single unconditional attempt + described below, and also carries the `orchestration: route` field and the + raised 24-route family-cap default described below. Two designs were + deliberately *not* combined and instead resolved in `main`'s favor: Strix's + `orchestrator/auto` routing (superseded by an explicit, more recent owner + decision reverting Strix to `orchestrator/free`-only, `main` PR #1434) and + a "fail closed on any partial provider discovery error" gate (superseded by + `main`'s demonstrated-in-production "log the failure, continue with + whatever succeeded" handling, which downstream evidence showed is needed + since single-provider hiccups are common and should not be fatal to the + whole pool). See the merge commit and PR #1415 for full evidence. +- Fix a real gap Devin Review found on this same PR ("Serving-incompatible + routes pass startup", `ContextualWisdomLab/.github#1454`): the routing + probe's base attempt (`REVIEW_PREFLIGHT_BASE_TOKENS`, 16) alone was enough + to admit a candidate, even though real review traffic always requests + `REVIEW_MAX_OUTPUT_TOKENS` (4096) — a route whose provider could satisfy a + 16-token completion but rejected or emptied out at 4096 passed startup and + only failed once real serving began. `_preflight_review_agents` now + requires a SECOND, confirming attempt at the real serving budget + (`REVIEW_PREFLIGHT_ESCALATED_TOKENS`) before admitting ANY route — whether + the base probe already succeeded (now recorded `confirmed_at_serving_budget`) + or failed with a budget-too-small signature (still recorded `escalated`, + unchanged) — both draw from the same shared, bounded + `REVIEW_PREFLIGHT_MAX_ESCALATIONS` counter rather than a new, separate one, + so the per-candidate worst case stays at most one base attempt plus one + more, and `REVIEW_PREFLIGHT_WORST_CASE_SECONDS`/ + `REVIEW_STARTUP_WATCHDOG_SECONDS` are unchanged. Added regression coverage + for a mocked route that succeeds at the base probe but fails/rejects at the + serving budget (must not be admitted) and for the shared budget bounding + confirmations the same way it already bounded escalations. +- Fix a real, high-severity regression Devin Review found minutes after the + confirmation fix directly above landed on this same PR ("Later healthy + routes cannot start", `ContextualWisdomLab/.github#1415`): making + confirmation mandatory for every successful base probe, while still + spending the SAME shared `REVIEW_PREFLIGHT_MAX_ESCALATIONS` counter that + fix reused from rescue escalation, meant as few as + `REVIEW_PREFLIGHT_MAX_ESCALATIONS` (4) candidates in the very first + batch(es) — each simply succeeding its base probe, the ordinary case — + could each reserve one of the counter's four slots for their own + confirmation, permanently exhausting it. Every later candidate's + confirmation request was then denied by `_EscalationBudget.try_reserve()` + regardless of merit, so a batch 2+ candidate that would have passed both + its base probe and its confirmation could never even attempt the second + one — defeating batching's entire purpose of evaluating up to + `REVIEW_PREFLIGHT_MAX_TOTAL_ROUTES` (24) routes to find one usable one. + Confirmation now draws from its own dedicated `REVIEW_PREFLIGHT_MAX_CONFIRMATIONS` + budget (sized to `REVIEW_PREFLIGHT_MAX_TOTAL_ROUTES` so even the fully + pessimistic case — every candidate ever probed in a run succeeds its base + probe — still gets its confirmation shot), while + `REVIEW_PREFLIGHT_MAX_ESCALATIONS` keeps its original, narrower, smaller + rescue-only purpose and cap, unrelated and untouched. `_preflight_with_fallback` + shares one instance of each of the two budgets across its primary and + fallback stages, exactly as it already did for the one budget before this + fix. `REVIEW_PREFLIGHT_WORST_CASE_SECONDS`/`REVIEW_STARTUP_WATCHDOG_SECONDS` + are unchanged (120s/255s): each batch's wall-clock worst case was already + computed as its slowest candidate making at most one base plus one second + attempt, independent of either budget's specific cap — see + `REVIEW_PREFLIGHT_MAX_ESCALATIONS`'s and `REVIEW_PREFLIGHT_MAX_CONFIRMATIONS`'s + own module-level comments for the full arithmetic. Rejections now + distinguish `confirmation_budget_exhausted` from `escalation_budget_exhausted` + in evidence. Added a red-then-green regression + (`test_batched_preflight_first_batch_confirmations_do_not_starve_a_later_healthy_route`) + reproducing the exact reported scenario against `_preflight_review_agent_batches`: + four batch-1 candidates each succeed their base probe and then genuinely + fail confirmation (consuming, under the old code, the entire shared + budget), while a fifth, batch-2 candidate that would succeed both its base + probe and its confirmation is wrongly denied under the pre-fix code and + correctly admitted after the fix. +- Fix a real bug Devin Review found on this same PR ("Startup watchdog counts + polls, not seconds", `ContextualWisdomLab/.github#1415`): the sidecar's + healthz-wait loop incremented a plain poll counter `i` once per iteration + and compared *that* to `sidecar_startup_watchdog_seconds`, even though a + single iteration's real cost is the `curl --max-time 2` health probe's own + duration plus the trailing `sleep 1` — up to 3s, not the 1s the counter + implicitly assumed. A fully-consumed 2s timeout on every poll could let the + 255s watchdog run for roughly 765s (~3x its documented wall-clock budget) + before firing, directly contradicting the wall-clock derivation this same + PR's earlier fix (`b0917a64`) established `REVIEW_STARTUP_WATCHDOG_SECONDS` + as the single source of truth for. The loop now resets bash's builtin + `SECONDS` to 0 immediately before the loop and compares `$SECONDS` — + real, auto-advancing wall-clock elapsed time immune to curl's own per-call + cost — against the deadline instead, with both places the old poll count + was surfaced (the watchdog's own failure message and the successful-startup + log line) now reporting `$SECONDS` too. Added a regression + (`test_healthz_wait_loop_fires_near_the_wall_clock_deadline_not_a_poll_count` + plus a companion message-format test) that extracts the loop's exact, + tracked source and drives it against a fake, always-failing `curl` that + sleeps longer than 1s per call with a small configured watchdog, asserting + the loop fails near the configured wall-clock seconds and well under the + poll-counting bound the old code needed — verified failing against the + pre-fix loop text and passing after the fix. +- Keep startup route probes on a ten-second timeout while giving serving-time + model calls the Noema gate's 120-second transport budget; both retain a + zero-retry transport policy at the client level (ADR-0005's own, + higher-level escalation/gateway retries, noted above, are a deliberate + exception to this for the specific "budget too small" and "no response at + all" failure signatures). Classify dependency-review API denial as + unavailable evidence without treating it as vulnerability-free. +- Make sidecar-backed gateway requests explicit `orchestration: route` so the + smoke and Noema review paths exercise the direct virtual-pool route without + invoking auto-mode triage; provider response errors remain fail-closed. +- Probe contextual-orchestrator review routes in bounded concurrent batches so + a rejected first discovery slice can advance to later routes, and capture the + intentional oversized-body 413 self-test without mislabeling it as provider + discovery failure. - Fail closed when the first top-level Noema JSON candidate is malformed, preventing a later approval object from overriding malformed preface data; multiple-object output remains supported when its first object is valid. diff --git a/docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md b/docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md index 3a48cdf582..ef0a62279e 100644 --- a/docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md +++ b/docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md @@ -24,7 +24,7 @@ all five, and auto-optimize routing by cost. 1. **Vendoring, pinned**: `scripts/ci/contextual_orchestrator_review_sidecar.sh` clones `ContextualWisdomLab/contextual-orchestrator` at an exact SHA - (`8cd99f139915131ba0239bce12a5d6a5fd85394e` today) into `RUNNER_TEMP`. The + (`ab7a813a69dae19541dc2888acd50c4ce37b29b7` today) into `RUNNER_TEMP`. The source's `requirements.lock` is installed with `--require-hashes` and `--no-deps`, so dependency resolution cannot silently move the reviewed runtime. @@ -43,17 +43,21 @@ all five, and auto-optimize routing by cost. price-attested; a partial price vector, malformed numeric value, conflicting free marker, or missing currency for a published vector fails closed. The gateway's `orchestrator/free` virtual id fails closed (`400 invalid_model`) unless an - enabled zero-cost agent exists. Strix uses `orchestrator/auto`; its catalog + enabled zero-cost agent exists. Strix originally used `orchestrator/auto` + (superseded by the 2026-08-30 amendment below: Strix now uses + `orchestrator/free`, like OpenCode and Noema); the `auto` pool's catalog may admit priced routes only through this evidence-bearing policy, never through a direct-provider model identifier. The auto pool probes the free catalog first. Only when every selected free route rejects the real runtime request contract does it rebuild once from fully price-attested routes and record the rejected primary attempt. This is evidence-triggered failover, not an arbitrary free/paid mixing ratio. - Both stages share one twelve-route startup budget: no more than eight routes - enter the free primary stage and only its remaining capacity may enter priced - fallback. Full discovery counts remain in policy evidence, and the transient - priced catalog is removed immediately after loading. + Both stages share one 24-route startup budget: no more than eight routes + enter the free primary stage and only the remaining capacity (at most sixteen + routes) may enter the price-attested fallback when the `auto` pool is in use. + The `free` pool never admits priced fallback. Full discovery counts remain in + policy evidence, and the transient priced catalog is removed immediately + after loading. 3. **ZDR-first within each cost tier**: `scripts/ci/zdr_policy.py` defines ZDR the way OpenRouter does ("a provider will not store your data for any period of time"; zero retention also implies no training) and is deliberately @@ -73,21 +77,42 @@ all five, and auto-optimize routing by cost. credential-account-diverse agents catalog, capped in size, in the orchestrator's own `ModelAgent` schema. Every KV credential is an independent account; vendor or endpoint identity does not imply model equivalence. Only - explicit `model_group` membership may share routing evidence. + explicit `model_group` membership may share routing evidence. The sidecar + defaults the per-account cap to `contextual_orchestrator_review_policy`'s + own `DEFAULT_ACCOUNT_CAP` (4), never to the total preflight-route budget: an + earlier version of this ADR described defaulting the cap to the same + 24-route total budget "so a single provider's catalog is not truncated," + but that was a real, live-evidenced bug (ContextualWisdomLab/.github#1415) + — it silently disabled per-account diversification and let two + rate-limited NVIDIA NIM credentials sharing one upstream jointly occupy an + entire 12-slot preflight batch. `ORCHESTRATOR_CATALOG_ACCOUNT_CAP` remains + an explicit operator override. (2026-08-31 correction: that "defaults to + 4" claim was true of the Python launcher's own fallback but not, until + this date, of the shell sidecar — `contextual_orchestrator_review_sidecar.sh` + still unconditionally exported a leftover literal `8` default whenever no + operator override was set, which meant the launcher's own env-unset + fallback branch could never actually run in production and every real run + got a cap of 8, not 4. The shell now derives its default the same way the + startup watchdog seconds below are derived: by reading + `contextual_orchestrator_review_policy.DEFAULT_ACCOUNT_CAP` at runtime + instead of hard-coding a numeric literal, so shell, Python, and this ADR + describe one real number.) 4. **Wiring**: `pr-review-autofix.yml` and the Required OpenCode dispatch provision the sidecar with the five secrets before OpenCode runs and point every model/diagnosis candidate at `contextual-orchestrator/orchestrator/free`; the generated dispatch config contains only the gateway provider. The shared `opencode.jsonc` default `model`/`small_model` is the same gateway route. `noema-review.yml` retains `orchestrator/free`. `strix.yml` provisions the - same sidecar and uses the loopback chat-completions/API-compatible URL with - `orchestrator/auto`: the 2026-08-29 exact-head DiskSage scan proved that four + same sidecar and originally used the loopback chat-completions/API-compatible + URL with `orchestrator/auto` (superseded by the 2026-08-30 amendment below: + `strix.yml` now defaults to `orchestrator/free`, like OpenCode and Noema): + the 2026-08-29 exact-head DiskSage scan proved that four discovered free routes all shared the OpenRouter outage domain, which the - gateway correctly collapsed to one provider attempt. Strix therefore uses + gateway correctly collapsed to one provider attempt. Strix therefore used the provider-diverse pool supplied by all five configured credentials. Provider diversity and cost-evidence classification remain delegated to the gateway rather than embedding a second routing policy in GitHub Actions. - Strix has no external fallback and private targets pass visibility through + Strix had no external fallback under `auto` and private targets pass visibility through to the gateway's ZDR requirement. Noema reviewer identity remains `NOEMA_REVIEW_TOKEN` / GitHub App / OIDC and is still never `github.token`; Autofix mutation still requires `PR_REVIEW_MERGE_TOKEN` / @@ -108,15 +133,40 @@ all five, and auto-optimize routing by cost. ## Consequences +- **Bounded discovery preflight (2026-08-29):** the sidecar probes at most 24 + selected routes in concurrent batches of four and stops after the first batch + with a usable text route. This preserves a finite startup budget while + allowing a rejected first catalog slice to fall through to later discovered + routes. The intentional oversized-body contract probe captures its expected + 413 diagnostic locally so it cannot be mistaken for provider discovery + failure. Exhausting every bounded batch still fails closed before healthz. + +- **Separate startup and serving budgets (2026-08-30):** route admission keeps + the ten-second timeout so unavailable providers cannot delay healthz, while + the serving `ModelClient` uses the Noema gate's 9,600-second review budget. + Both phases keep zero retries and the same bounded request policy; the + launcher test verifies the two constructed client configurations separately. + +- **Direct gateway requests (2026-08-30):** the sidecar's startup request and + Noema request select explicit `route` orchestration so they validate and use + the direct virtual-pool path without invoking auto-mode triage. The Noema + change is limited to the exact process-local sidecar origin; external + OpenAI-compatible URLs retain their original payload. Provider response + validation and fail-closed non-200 handling are unchanged. + - The autofix/OpenCode review paths no longer hard-code any provider base URL or model id; upstream model selection is delegated to the orchestrator's - discovery under the zero-cost pool. Strix uses the separately governed auto - pool without treating absent price metadata as either free or paid-route - evidence. -- Strix delegates selection to `orchestrator/auto`. Its correctness-first pool - remains distinct from the zero-cost OpenCode/Noema pool, while private-target - ZDR admission remains fail-closed. Unknown-cost routes remain auditable but - ineligible; free and fully price-attested routes are the only review routes. + discovery under the zero-cost pool. Strix originally used the separately + governed auto pool (superseded by the 2026-08-30 amendment below: Strix now + uses the same zero-cost pool as OpenCode/Noema) without treating absent + price metadata as either free or paid-route evidence. +- Under `orchestrator/auto`, selection is delegated to the gateway's + correctness-first pool, distinct from the zero-cost OpenCode/Noema pool, + while private-target ZDR admission remains fail-closed. Unknown-cost routes + remain auditable but ineligible; free and fully price-attested routes are + the only review routes. (This paragraph describes the `auto` pool mode + itself, which still exists for any caller that opts into it explicitly — + see the 2026-08-30 amendment below for why Strix no longer does.) - Workers need egress to the five provider model-list hosts and, when reachable, `https://openrouter.ai/api/v1/endpoints/zdr`; the feed failure path is graceful (static table). diff --git a/docs/adr/0005-sidecar-preflight-token-budget.md b/docs/adr/0005-sidecar-preflight-token-budget.md index f024bc9933..0771c29760 100644 --- a/docs/adr/0005-sidecar-preflight-token-budget.md +++ b/docs/adr/0005-sidecar-preflight-token-budget.md @@ -46,15 +46,14 @@ Citations below pin to the exact reviewed blob at `main`'s 2. **The shell script's own virtual-pool smoke request.** Once `/healthz` succeeds (a separate, already-completed budget — Layer 2 does not draw from Layer 1's 180s), the shell script sends one `POST /v1/chat/completions` with `"model":"orchestrator/free"` (the *virtual* pool id, not a - specific candidate) and its own fixed `max_tokens`, currently `4096`, under a **120-second** - `curl --max-time`. This 120s value is itself the outcome of a prior, real, evidenced fix in this + specific candidate) and its own fixed `max_tokens`, currently `4096`, under a **one-hour** + `curl --max-time`. The former 120s value was itself the outcome of a prior, real, evidenced fix in this exact file (raised from a too-tight 30s after live reproduction on `ContextualWisdomLab/contextual-orchestrator#921` showed a genuinely-healthy DeepSeek NIM route needing more than 30s to complete a real generation) — the comment there explicitly documents that - this required-workflow job budgets **120 minutes** total (`timeout-minutes` in - `strix.yml`/`noema-review.yml`) and that *"the org's own stated policy accepts multi-hour central - review latency in favor of accuracy over speed."* This ADR's design deliberately **does not shorten - that 120s value** — doing so would reintroduce the exact regression that prior fix corrected. The + these required-workflow jobs now budget up to **six hours** total and that *"the org's own stated + policy accepts multi-hour central review latency in favor of accuracy over speed."* The one-hour + bound removes the synthetic 120s failure while preserving finite retry and failover behavior. The correct fix for a hang, per Devin Review (see Decision §1), is a bounded *retry*, not a shorter *timeout*. @@ -178,10 +177,12 @@ correctly caught in an earlier revision of this text):** treats it as Trigger A: retried up to `REVIEW_PREFLIGHT_GATEWAY_MAX_ATTEMPTS` times against a candidate the gateway is, by the same reasoning as the Trigger-B/route-diversity note below, more likely to repeat than diversify away from. **This does not change Layer 2's stated worst case** - (`REVIEW_PREFLIGHT_GATEWAY_MAX_ATTEMPTS × 120s` — this failure still consumes attempts from the + (`REVIEW_PREFLIGHT_GATEWAY_MAX_ATTEMPTS × REVIEW_PREFLIGHT_GATEWAY_MAX_TIME_SECONDS`; + the shared default is 1,200s and Noema supplies 600s for its single-attempt + 15-minute provisioning window — this failure still consumes attempts from the same shared Trigger-A budget, not an additional one), but it does mean this specific failure typically consumes the *entire* retry budget before failing closed, rather than failing fast the - way a correctly-classified Trigger B would (one attempt, ~120s). A correct fix requires a + way a correctly-classified Trigger B would (one attempt, up to one hour). A correct fix requires a `contextual-orchestrator` change (a machine-readable field distinguishing the two `ProviderResponseError` cases through the `/v1/chat/completions` error boundary) — genuinely out of scope for this sidecar-only ADR and its stacked implementation PR. Fragile string-matching on the @@ -337,22 +338,28 @@ retried once, unconditionally, would be a real, computed worst-case blowup again itself be exactly the unjustified heuristic this ADR's convergence principle already rejects. Tracked as `ContextualWisdomLab/.github#1458`; revisit if real hosted-run telemetry (already required below) shows a specific, evidenced bias worth correcting. -- **Layer 2** (bounded only by the job's own 120-minute ceiling, per the org's stated "accuracy over +- **Layer 2** (bounded by the caller job's ceiling, per the org's stated "accuracy over speed" policy already reasoned in this file — *not* by the 180s Layer 1 budget, which has already - completed by the time Layer 2 runs): keep the existing per-attempt timeout (**120s, unchanged** — not - shortened, per Context above) and the existing **`4096` budget, unchanged throughout — Layer 2 never + completed by the time Layer 2 runs): use a caller-bounded total-time timeout plus a 10-second connection + timeout. Keep the existing **`4096` budget, unchanged throughout — Layer 2 never escalates** (already proven working on a real hosted run, `contextual-orchestrator#921`; see Decision §1 for why an escalation tier was considered and dropped here). Allow up to - `REVIEW_PREFLIGHT_GATEWAY_MAX_ATTEMPTS = 3` total attempts, consumed only by Trigger A (transport + `REVIEW_PREFLIGHT_GATEWAY_MAX_ATTEMPTS = 3` total attempts (one for an explicitly pinned + single-candidate job), consumed only by Trigger A (transport failure/hang/non-2xx) — Trigger B (empty + either its `finish_reason == "length"` or - reasoning-without-content signature) is not retried at Layer 2 at all (Decision §1). **Worst case**: - 3 × 120s = **360s (6 minutes)** — - explicit, bounded, and small relative to the job's 120-minute ceiling; the previous design's worst - case was already 120s for one unconditional attempt with no chance of recovery, so this trades a - bounded amount of additional worst-case latency for surviving exactly the transient-hang class of - failure reproduced live on this ADR's own PR. -- **Initial values are reused precedent, not new guesses** (Devin Review's fourth finding): every - number above is either already deployed in this exact codebase today (`10s`, `120s`, `4096`, `12`) + reasoning-without-content signature) is not retried at Layer 2 at all (Decision §1). The job timeout + and per-attempt timeouts are fail-closed wall-clock bounds; a pinned candidate failure advances to + the next job instead of reporting curl's former synthetic 120-second transport failure. +- **Initial values are derived or reused, not guesses** (Devin Review's fourth finding): the shared + 1,200-second attempt bound satisfies `330s + 3 × 1,200s = 3,930s`; combined with OpenCode's + 205-minute model step it stays below the 305-minute job ceiling, while combined with Strix's + 170-minute scan it stays below the 360-minute job ceiling. Noema explicitly uses one 600-second + attempt, so `330s + 600s <= 900s` preserves its 15-minute provisioning reservation before the + separate 335-minute review step. That review still owns one shared 19,800-second response deadline, + leaving five minutes for verdict sealing and handoff inside the step. Autofix inherits the bounded + 3,930-second admission under the platform six-hour job ceiling. These admission-smoke limits do not + shorten serving-client timeouts: configured two-hour-or-longer model calls remain supported. Other numbers are either already + deployed in this exact codebase today (`10s`, `4096`, `12`) or has direct external documentation backing it (`16` — the pre-#1436 value this codebase already ran with, and separately the floor OpenRouter's own schema documents: *"some providers enforce a minimum of 16"* for the deprecated `max_tokens` field). The two new counters @@ -398,8 +405,9 @@ outcome already observed in production.** being wrong for a fixed token budget, or hanging/failing transiently, which is the actual shape of the problem — while keeping every worst case explicit and bounded rather than open-ended. - Layer 1's worst case would grow from ~120s to a computed 160s, still under its existing 180s - healthz-readiness ceiling. Layer 2's worst case would grow from a single 120s attempt with no - recovery path to up to 360s across bounded retries — small relative to the job's 120-minute ceiling + healthz-readiness ceiling. Layer 2 allows up to three one-hour attempts; the six-hour Strix caller + leaves ten minutes after that maximum plus its 170-minute scan step, while pinned Noema jobs use one + attempt before cross-job failover. and consistent with this file's own already-stated "accuracy over speed" policy. - Keeping Layer 2 (not just Layer 1) would mean the preflight still proves the actual consumer-facing `orchestrator/free` route works, not only that individual candidates can respond in isolation — diff --git a/docs/doctoring/contextual-orchestrator-vendored-sidecar.md b/docs/doctoring/contextual-orchestrator-vendored-sidecar.md index 15766abcd8..327973e83f 100644 --- a/docs/doctoring/contextual-orchestrator-vendored-sidecar.md +++ b/docs/doctoring/contextual-orchestrator-vendored-sidecar.md @@ -33,6 +33,35 @@ streaming/spooling path and provider capability checks; adding `/files` alone would not handle an inline Base64 image in an ordinary JSON request. The sidecar must measure representative Strix envelopes and keep provider/model context failures distinct from its own HTTP framing failure. + +## 2026-08-29 Noema preflight batching + +DiskSage Noema jobs `99111099730` and `99110885279` logged +`request_failed status=413 code=request_too_large` before startup. That line was +the sidecar's intentional oversized `Content-Length` contract test, not model +discovery or a provider response. The actual terminal condition was exhaustion +of the initially selected provider routes before healthz. The contract probe now +captures and asserts its expected diagnostic without emitting it, while runtime +preflight tries a maximum of 24 discovered routes in concurrent batches of four +and stops after the first batch with usable text. Every route still uses the +ten-second timeout, zero retries, the same plain-chat payload, and sanitized +evidence; exhausting the bounded batches remains a startup failure. +Provider discovery failures are non-fatal per provider, not a whole-run gate. +When one configured provider's discovery call fails, the launcher logs a +sanitized `provider_discovery_failed provider=... code=...` diagnostic to +stderr (never partial provider response text) and continues with whatever +models the other providers successfully returned; it does not stop startup +and does not require the whole discovery pass to be error-free. Startup only +fails closed if the resulting eligible-model set ends up empty -- no +provider's discovery succeeded at all, or none of what did succeed contains a +general-chat, text-output model matching the selected pool +(`orchestrator/free` or `orchestrator/auto`). A partial catalog assembled from +N-1 successful providers is real availability evidence, not an aborted run. +An earlier "fail closed on any partial provider discovery error" design was +considered and rejected in favor of this "log the failure, continue with +whatever succeeded" behavior once production evidence showed single-provider +hiccups are common and should not be fatal to the whole pool; see +`CHANGELOG.md`'s `[Unreleased]` entry for that decision. The pin includes upstream `#887` (`2591b66`), which fixes the gateway's incorrect 1024-character rejection. The same probe sends Strix-shaped function tools with 1025-, 1026-, and 2000-character descriptions and verifies that @@ -40,6 +69,40 @@ each reaches the provider payload byte-for-byte; arbitrary truncation is not used. Provider/model-specific context limits remain provider errors, not a reason for this gateway to rewrite the request. +## 2026-08-30 Provider-family catalog bound + +The 24-route startup budget is a total bound, not a promise to stop after four +routes from the first provider family. The sidecar now defaults +`ORCHESTRATOR_CATALOG_FAMILY_CAP` to 24, allowing an OpenRouter-only discovery +catalog to expose every route within the same bounded preflight budget. An +operator may still set a lower explicit family cap when outage-domain diversity +is more important than route breadth; the generic policy CLI retains its +independent family-cap default. + +## 2026-08-30 Startup and serving timeout separation + +The ten-second route timeout is a startup-admission budget: a route that does +not answer the bounded readiness probe quickly enough is excluded before +healthz. It must not also bound the real review request. The serving +`ModelClient` now uses the 9,600-second review budget used by the +Noema review gate, while retaining zero retries and the same output-token and +temperature policy. The launcher test constructs both client policies and +asserts their distinct timeouts; it does not infer the contract from duplicate +source text. + +## 2026-08-30 Gateway smoke route mode + +The exact-head PR #1415 preflight found usable routes, but its gateway smoke +request then defaulted to `auto` orchestration and reached the pinned +server's triage/conduct path. That path returned `invalid_structured_output` +with HTTP 502 even though route admission had succeeded. The smoke request now +sets `orchestration: route`, which exercises the direct virtual-pool path used +by Noema's strict-JSON request and tool-bearing Strix requests and avoids an +unrelated auto-mode triage call. Noema now sends the same mode when its API URL +matches the process-local sidecar origin; unrelated external OpenAI-compatible +URLs retain their original payload. Provider response validation and the +fail-closed treatment of every non-200 response remain unchanged. + ## What changed `pr-review-autofix.yml` now provisions the sidecar diff --git a/docs/doctoring/dependency-review-fail-closed.md b/docs/doctoring/dependency-review-fail-closed.md index 81681d3f0c..e6d05937d3 100644 --- a/docs/doctoring/dependency-review-fail-closed.md +++ b/docs/doctoring/dependency-review-fail-closed.md @@ -19,7 +19,11 @@ Checks, status contexts, review submissions, and merge authorization remain sepa ## Failure classification and remediation - Transport exit `0` plus HTTP `200`: proceed to the pinned dependency-review action. -- Any other result: fail the job and retain exact repository/base/head/status and transport-exit evidence. An HTTP `200` emitted by a failed or partial transfer is unavailable evidence. +- Any other result: emit `DEPENDENCY_REVIEW_EVIDENCE state=unavailable` with a + bounded reason (`transport`, `api_authorization`, or `api_response`), fail the + job, and retain exact repository/base/head/status and transport-exit evidence. + The classification is diagnostic evidence, never a vulnerability-free result. + An HTTP `200` emitted by a failed or partial transfer is unavailable evidence. - Public repository failure: verify dependency graph and security configuration, organization policy, token read access, and GitHub service health. - Private or internal exception: require a separately reviewed organization policy with explicit entitlement evidence and compensating controls. Never infer `not-applicable` from an unavailable response. diff --git a/docs/nvidia-nim-opencode-hotfix.md b/docs/nvidia-nim-opencode-hotfix.md index df8c193b28..dee7604a67 100644 --- a/docs/nvidia-nim-opencode-hotfix.md +++ b/docs/nvidia-nim-opencode-hotfix.md @@ -1,6 +1,39 @@ -# NVIDIA NIM OpenCode model priority (hotfix) +# NVIDIA NIM OpenCode model priority (hotfix) — ROLLED BACK, HISTORICAL -## Why +**Status (2026-08-31): this hotfix is no longer active.** The six-model NIM +prefix this note describes was removed from +`.github/workflows/opencode-review-dispatch.yml`'s `OPENCODE_MODEL_CANDIDATES` +by `f8823a54` (#1364, "route Noema review through vendored +contextual-orchestrator"); that variable has held the single value +`"contextual-orchestrator/orchestrator/free"` (contract-pinned by +`tests/test_opencode_agent_contract.py`) ever since, and `opencode.jsonc`'s +embedded config for the CI dispatch path likewise renders +`enabled_providers: ["contextual-orchestrator"]` with no NIM entry. Per this +note's own "Rollback" section below, it should have been deleted once +catalog reliability was restored; it was not, and stayed factually stale for +over a month (last touched at `c7a4bad6`, #682, 2026-07-31) before this +correction. Left in place as a historical record rather than deleted, per +this repo's "append a dated note, don't rewrite history" documentation +convention (see `docs/doctoring/direct-nvidia-nim-communication-removal.md` +for the sibling record of the *code* that implemented an unrelated, +already-dead direct-NIM resolver). The `nvidia-nim` provider block still +declared in root `opencode.jsonc` is excluded from every `enabled_providers` +list this repo currently renders (both the root config and the CI dispatch +path's own embedded config), so it is not a currently usable fallback -- +nothing in production ever supplies a `nvidia-nim/*` candidate today. +`scripts/ci/run_opencode_review_model_pool.sh`'s own candidate-handling logic +for that prefix (`is_nvidia_nim_candidate`, skip-if-no-key, timeout capping) +is exercised by `tests/test_opencode_model_pool_runner.py`, but those tests +fake the `opencode` invocation itself, so they prove the script's own +handling of such a candidate, not that the real OpenCode binary would still +successfully reach NVIDIA's API with this block's current model ids if one +were ever supplied. Not orphaned code -- re-enabling and re-verifying it, or +removing it outright, is a separate resilience-tradeoff decision, not a +documentation fix, and is out of scope here. See +`docs/product-technical-gap-baseline.md`'s "Direct-NIM-communication audit" +entry (2026-08-31) for the full investigation this correction closes out. + +## Why (historical — describes the hotfix as it was, not current state) OpenCode Agent failed to produce a usable review on the PR thread starting at ContextualWisdomLab/fast-mlsirm#290 (`opencode-review` check **skipped**, no diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 76d85b949b..f60c61e399 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -1715,6 +1715,105 @@ string, a bare number) confirmed to fail against the pre-fix script (`KeyError: signature as the original round-4 bug) before passing after the fix. 1930 tests pass; 100% coverage and 100% docstring coverage on `scripts/ci/`. +## 2026-08-31 opencode-review structural deadlock: unresolved threads starve the dispatch that would resolve them + +**Root cause found while investigating why `ContextualWisdomLab/.github#1500`'s `opencode-review` +required check exhausted its full 90-minute active-dispatch-and-poll window (added by #1497) with no +`opencode-agent` verdict.** +`pr_review_merge_scheduler.py`'s `decide()` has an unconditional early return +(`scripts/ci/pr_review_merge_scheduler.py:3462-3474`): `if unresolved_thread_count(pr): return +decide("block", ...)`, before any code path can reach `dispatch_opencode_review()`. That gate long +predates automated review bots (traced to `ea2b2cc8`, "Queue auto-merge for approved conflicts") and was +reasonable when it existed: don't auto-merge or re-review while a real conversation is open. It has since +become a structural deadlock, because `unresolved_thread_count()` counts *every* active, non-outdated +thread uniformly, regardless of source or severity — including a purely informational, no-action-needed +Devin/CodeRabbit note on a file `opencode-review`'s own verdict has nothing to do with. Since dispatching +a fresh `opencode-review` event is the *only* path to a verdict for the required check, and Devin/ +CodeRabbit post threads faster than most PRs get them resolved, "≥1 unresolved thread" is close to the +default state for any actively-reviewed PR in this repo — confirmed independently on `#1415` (3 open +threads at the time of check), `#1507` (1), `#1508` (2), `#1509` (4), vs. `#1491` (0, and its +`opencode-review` check passed normally). `#1504`'s successful `opencode-review` run the same day is a +positive control proving the #1497 dispatch-and-poll mechanism itself works correctly whenever no +unresolved thread blocks the scheduler at evaluation time. + +**Not fixed in this pass — the classification problem is genuinely hard to get safely right, not a quick +patch.** The obvious fix (skip the block for "informational-only" threads) needs a way to tell +"informational" from "actionable" that doesn't rely on fragile text-pattern matching across multiple +different bots' own emoji/formatting conventions (Devin's 🔴/🟡/🔍/📝, CodeRabbit's own separate severity +scheme, human reviewers who follow no pattern at all) — exactly the kind of heuristic this org's own +conventions warn against, and a wrong classification in either direction is dangerous: too permissive +silently defeats the safety gate for a real unaddressed finding; too conservative changes nothing. A more +robust design likely needs to key off GitHub's own formal review *state* (only a thread tied to an actual +`CHANGES_REQUESTED` review should block) rather than any inline comment thread, but confirming +`reviewThreads` carries that linkage needs its own careful investigation and test coverage before landing +a change to this security/trust-boundary-relevant scheduler. Tactically unblocked `#1500` and `#1415` by +resolving their own already-addressed/informational threads (see those PRs' own threads for the specific +acknowledgments) rather than touching the gate itself. **Next development increment**: a dedicated, +narrowly-scoped PR against `pr_review_merge_scheduler.py`'s `unresolved_thread_count()` (or a new, +separate predicate for review-dispatch eligibility distinct from merge eligibility), with regression +coverage across informational-only, actionable-bot, and human-reviewer-`CHANGES_REQUESTED` thread shapes, +before any PR is unblocked by classification logic rather than manual resolution. + +**Two related, smaller gaps found and stood down on in `ContextualWisdomLab/.github#1415`'s own review, +tracked here rather than dropped:** +- *Streaming responses can defeat a bare socket timeout.* Both `#1415`'s preflight ModelClient + (`REVIEW_PREFLIGHT_TIMEOUT_SECONDS`) and — independently found on `ContextualWisdomLab/.github#1509` — `noema_review_gate.py`'s + `call_llm` bound each HTTP attempt with a plain per-operation socket `timeout=`, which only bounds time + *between* reads, not total attempt duration: a provider trickling data slowly enough (each chunk just + under the timeout) could keep one attempt alive far past its nominal budget. `#1509` already built a + real deadline-watchdog wrapper (arm a timer, forcibly close the connection past total budget) for the + serving side; the preflight side needs the same treatment once that mechanism lands somewhere mergeable, + rather than a second, possibly-inconsistent implementation. +- *Discovery time is unbounded in catalog size.* `#1415`'s + `REVIEW_DISCOVERY_OPENROUTER_FREE_ENDPOINT_ROUND_CAP` assumes a bounded number of pagination rounds; if + OpenRouter's free-model catalog grows past that assumption, discovery can exceed + `REVIEW_STARTUP_WATCHDOG_SECONDS` and abort an otherwise-healthy sidecar. Needs either a deadline-based + (not round-count-based) bound in the launcher's discovery wrapper, or the vendored contextual-orchestrator + package's own OpenRouter discovery client exposing elapsed-time enforcement directly — a real redesign + in either case, not a constant tweak, and this exact family of finding has already been through several + rounds of patch → new finding on `#1415` without converging. + +## 2026-08-31 (later) two distinct required-check failure modes, initially conflated + +**Correction (Devin Review on `#1415`): an earlier version of this entry grouped `noema-review`'s +failure together with `opencode-review`'s and `strix`'s under one shared "runner-queue contention" +cause. That was wrong for the `noema-review` case — its job actually ran; it never sat queued waiting +for a runner. The two failure modes are distinct and should not be conflated:** + +**Mode 1 — the job executes but hits a real timeout bug (not a queue problem).** +`noema-review` on `ContextualWisdomLab/.github#1415` (job `99521003275`) got a runner, started, and its +`python3 -m scripts.ci.noema_review_gate` step ran for two minutes before failing — the traceback shows +it reached `noema_review_gate.py:656`'s `opener.open(request, timeout=120)` and got a real +`TimeoutError` from an in-flight HTTP call. This is the exact pre-existing `contextual-orchestrator#946` +bug `#1415`'s own fix targets, confirmed by the `timeout=120` literal being the *unfixed* value — proof +this ran `main`'s trusted copy of the script (the `pull_request_target` trust boundary), not the PR's +own fix. This has nothing to do with runner availability. + +**Mode 2 — the dispatched run never gets a runner at all (genuine queue starvation).** +- `opencode-review` on `#1500`/`#1502`/`#1503` (see the structural-deadlock entry above): the + `pr-review-merge-scheduler.yml`-triggered dispatch chain (`coverage-source-tree` → `coverage-evidence` + → `opencode-review-target`) sat queued for over an hour with zero progress. +- `strix` on `#1503` (run `33400353198`, `repository_dispatch` against `main`): created `14:02:56Z`, + still `queued` with `run_started_at == created_at` (never picked up by a runner) when the "strix" + commit status finally posted `failure` (`"Default-branch repository_dispatch Strix evidence failed"`) + at `15:39:43Z` — 96 minutes of pure queue time, zero execution time. + +The two modes *do* interact once `#1415` merges: `noema-review`'s own dispatch-and-poll architecture +(`docs/pr-review-and-merge-procedure.md`) is the same shape `opencode-review`'s and `strix`'s use, so a +future `noema-review` run could independently suffer Mode 2 even after Mode 1 (the timeout-value bug) +is fixed. But that is a shared *exposure*, not a shared *observed cause* for these three specific +failures — only `opencode-review` and `strix` actually exhibited Mode 2 today. + +**Not fixed in this pass for Mode 2 — that is an infrastructure capacity/scheduling question, not a code +defect any one PR's diff can address.** `opencode-review` and `strix` inherit the same exposure to +GitHub Actions concurrent-job-limit contention when the org's overall Actions usage spikes (plausibly +from this same autonomous loop running many concurrent sessions across many repos and PRs). **Next +development increment**: quantify the org's actual concurrent-runner ceiling against typical in-flight +job count at peak (via the Actions usage API), and evaluate whether a dedicated larger runner pool, a +queuing/backpressure mechanism in the dispatch step itself (fail fast with a clear "queue congested" +status instead of waiting the full poll window then reporting an opaque +failure), or self-hosted runners for the dispatch-target jobs specifically would relieve it — a +budget/infrastructure decision, not something to guess at in a single PR's scope. ## 2026-08-31 noema-review-gate: malformed LLM JSON crashed the required check instead of failing closed The required `noema-review` check on `ContextualWisdomLab/contextual-orchestrator#960` crashed with an diff --git a/scripts/ci/contextual_orchestrator_review_launcher.py b/scripts/ci/contextual_orchestrator_review_launcher.py index f115ef2b88..1ea70e0ee2 100644 --- a/scripts/ci/contextual_orchestrator_review_launcher.py +++ b/scripts/ci/contextual_orchestrator_review_launcher.py @@ -22,10 +22,12 @@ from __future__ import annotations import argparse +from concurrent.futures import ThreadPoolExecutor import json import os import re import sys +import threading from pathlib import Path from typing import Any @@ -41,11 +43,21 @@ # temperatures, while 1.0 is the OpenAI-compatible default. REVIEW_TEMPERATURE = 1.0 # A selected route that cannot answer within ten seconds is not reliable enough -# for a required CI gate. With at most twelve sequential candidates, startup is -# bounded below the sidecar's three-minute readiness deadline. +# for a required CI gate. Four-route batches let discovery try a broader but +# still finite catalog while keeping the worst-case provider wait below the +# sidecar's three-minute readiness deadline. REVIEW_PREFLIGHT_TIMEOUT_SECONDS = 10 -REVIEW_PREFLIGHT_MAX_TOTAL_ROUTES = 12 +# A real review may legitimately run far beyond two minutes. Keep the short +# timeout confined to startup admission; the outer Noema request/job deadline +# remains the serving safety boundary. +REVIEW_SERVING_TIMEOUT_SECONDS = 9000 +REVIEW_PREFLIGHT_BATCH_SIZE = 4 +REVIEW_PREFLIGHT_MAX_TOTAL_ROUTES = 24 REVIEW_PREFLIGHT_PRIMARY_ROUTE_LIMIT = 8 +# Bound the already-admitted serving catalog so immediate-error failover work +# cannot grow with discovery. The outer Noema request and workflow job remain +# the wall-clock safety boundaries. +REVIEW_SERVING_MAX_CANDIDATES = 10 # ADR-0005: a single fixed max_tokens cannot fit every model in a heterogeneous # pool -- some spend internal reasoning tokens before visible content and need # more, others have a real completion ceiling a large budget would exceed. The @@ -62,29 +74,276 @@ # number. REVIEW_PREFLIGHT_ESCALATED_TOKENS = REVIEW_MAX_OUTPUT_TOKENS # Shared cap on how many candidates in one preflight run may use the -# escalation retry above, so Layer 1's PROBING worst case stays computed and -# bounded: REVIEW_PREFLIGHT_MAX_TOTAL_ROUTES * REVIEW_PREFLIGHT_TIMEOUT_SECONDS -# + REVIEW_PREFLIGHT_MAX_ESCALATIONS * REVIEW_PREFLIGHT_TIMEOUT_SECONDS -# = 12*10 + 4*10 = 160s, under the sidecar's 180s healthz-readiness wait. See -# docs/adr/0005-sidecar-preflight-token-budget.md, Decision section 3. +# escalation RESCUE retry below (a FAILED base probe with a "budget too +# small" signature). This budget's purpose is deliberately narrow and +# scarce: rescuing an atypical failure, not confirming an already-successful +# candidate -- see REVIEW_PREFLIGHT_MAX_CONFIRMATIONS below for that +# separate, much more common concern, and why it needs its own budget. +# Merged with the batched concurrent preflight below +# (REVIEW_PREFLIGHT_BATCH_SIZE): candidates within one batch of up to +# REVIEW_PREFLIGHT_BATCH_SIZE run concurrently, so a batch's own wall time is +# its slowest candidate, not the sum of all of them -- worst case, a batch +# containing a candidate that makes a second attempt (rescue OR confirmation) +# costs 2 * REVIEW_PREFLIGHT_TIMEOUT_SECONDS (base + second attempt, +# sequential within that one candidate's own thread), not +# REVIEW_PREFLIGHT_TIMEOUT_SECONDS. Crucially, this per-batch bound holds +# regardless of HOW MANY candidates in that one batch make a second attempt +# (concurrency means the batch's wall time is its slowest member, never a +# sum of every member), and therefore holds regardless of either second- +# attempt budget's specific cap -- REVIEW_PREFLIGHT_MAX_CONFIRMATIONS below +# deliberately has a much larger cap than this one without changing this +# arithmetic at all. With REVIEW_PREFLIGHT_MAX_TOTAL_ROUTES=24 candidates in +# batches of REVIEW_PREFLIGHT_BATCH_SIZE=4, that is ceil(24/4)=6 batches, so +# the worst case is 6 * 2 * 10 = 120s (REVIEW_PREFLIGHT_WORST_CASE_SECONDS +# below) -- already the fully pessimistic case of every batch needing its +# worst case, true independent of either budget's cap value. See +# docs/adr/0005-sidecar-preflight-token-budget.md, Decision section 3 for the +# ADR's own (pre-batching, sequential) 160s derivation of this same shared +# cap's value; batching changes the wall-clock arithmetic, not the cap itself. # -# KNOWN GAP, tracked (not yet fixed): this 160s covers only probing, not the -# discover_all_models() call that runs before it inside the SAME 180s -# watchdog. Verified directly against the vendored contextual-orchestrator -# source: discover_all_models() makes up to ~7 sequential HTTP calls (the -# shared models.dev fetch, one per PROVIDER_MODEL_SOURCES entry with a -# registered credential, and the OpenRouter ZDR endpoint fetch), each up to -# DISCOVERY_TIMEOUT_SECONDS = 15s -- up to ~105s worst case, before probing's -# own 160s even starts. Combined real worst case is therefore up to ~265s, -# not 160s. See ContextualWisdomLab/.github#1455 for the tracked fix (a -# shared monotonic deadline, scaled-down probing, or an evidence-justified -# watchdog extension) and #1454 for the related, separately-tracked gap that -# a base-probe *success* never confirms the candidate at the real serving -# budget (REVIEW_MAX_OUTPUT_TOKENS). Neither blocks this PR's 7 verified -# findings; both are architecturally significant enough to need their own -# design pass rather than a guessed patch here. +# FIXED (ContextualWisdomLab/.github#1455, Devin Review finding "Startup +# watchdog preempts valid preflight"): the bound above covers only probing, +# not the discover_all_models() call that runs before it, inside the SAME +# sidecar startup watchdog (contextual_orchestrator_review_sidecar.sh). Both +# phases run sequentially in one process before the server can start +# accepting `/healthz`, so the watchdog must cover their SUM, not either one +# alone -- previously the watchdog was a bare, uncoordinated 180s shell +# constant that only happened to exceed the probing-only figure above by +# coincidence, while the combined real worst case (see +# REVIEW_STARTUP_WATCHDOG_SECONDS below) is larger than that. discover_all_models() +# makes up to REVIEW_DISCOVERY_MAX_SEQUENTIAL_HTTP_CALLS sequential-call- +# equivalents against the pinned contextual-orchestrator revision, each up to +# REVIEW_DISCOVERY_TIMEOUT_SECONDS -- see that constant's own comment below +# for the full, itemized enumeration (shared Models.dev fetch, per-source +# retries, OpenRouter's extra calls, and two trailing global calls) verified +# directly against ORCHESTRATOR_PIN_SHA; do not restate the count here, to +# avoid a second "verified" claim silently drifting from the real one below. +# contextual_orchestrator_review_sidecar.sh imports REVIEW_STARTUP_WATCHDOG_SECONDS +# from this module (a stdlib-only, +# dependency-free import) as its watchdog loop bound -- a single source of +# truth so a future change to either phase's constants cannot silently +# desynchronize the two budgets again. #1454 (a base-probe *success* never +# confirms the candidate at the real serving budget, REVIEW_MAX_OUTPUT_TOKENS) +# was FIXED (Devin Review, "Serving-incompatible routes pass startup"): a +# base-probe success now always draws one confirming attempt at +# REVIEW_PREFLIGHT_ESCALATED_TOKENS before being admitted, exactly like a +# base-probe failure's existing rescue attempt. +# +# FIXED (ContextualWisdomLab/.github#1415, Devin Review finding "Later +# healthy routes cannot start"): that #1454 fix originally drew the +# confirmation attempt from this SAME counter, exactly like a base-probe +# failure's rescue attempt. That was wrong: this counter is sized (4) for +# the RARE rescue case, but every single successful base probe now needs a +# confirmation -- the COMMON case, not the rare one. As few as +# REVIEW_PREFLIGHT_MAX_ESCALATIONS candidates in the very first batch(es) +# each succeeding their base probe could reserve every slot for their own +# confirmations, permanently denying every later candidate's confirmation +# regardless of merit -- defeating the entire point of batching up to +# REVIEW_PREFLIGHT_MAX_TOTAL_ROUTES candidates to find one usable route. +# Confirmation now draws from its own separate +# REVIEW_PREFLIGHT_MAX_CONFIRMATIONS budget (see below); this counter keeps +# its original, narrower rescue-only purpose and its original cap. Per- +# candidate worst case stays at most one base + one second attempt either +# way (confirmation OR rescue, never both on the same candidate), so the +# REVIEW_PREFLIGHT_WORST_CASE_SECONDS/REVIEW_STARTUP_WATCHDOG_SECONDS +# arithmetic derived below is unchanged by either fix -- see this comment's +# opening paragraph for why the formula never depended on either budget's +# specific cap value in the first place. REVIEW_PREFLIGHT_MAX_ESCALATIONS = 4 +# FIXED (ContextualWisdomLab/.github#1415, Devin Review "Later healthy +# routes cannot start"): the mandatory serving-budget CONFIRMATION of an +# already-successful base probe (see REVIEW_PREFLIGHT_MAX_ESCALATIONS above +# for the full incident) needs its own budget, separate from that counter's +# original, narrow "rescue a failed base probe" purpose. Since confirmation +# runs for EVERY successful base probe -- the common case, not a rare one -- +# this budget is sized to REVIEW_PREFLIGHT_MAX_TOTAL_ROUTES: the maximum +# number of candidates this preflight run can ever probe across BOTH stages +# combined (the primary catalog and, when it runs, the priced-fallback +# catalog -- see _preflight_with_fallback, which shares one instance of this +# budget across both stages exactly like it already does for +# REVIEW_PREFLIGHT_MAX_ESCALATIONS). That size guarantees even the fully +# pessimistic case -- every candidate ever probed in this run succeeds its +# base probe -- still gets its required confirmation shot; this is not an +# unbounded allowance, it is bounded by the same total-route cap this +# preflight can never exceed regardless of how this constant is set. As +# reasoned above, sizing this budget larger than +# REVIEW_PREFLIGHT_MAX_ESCALATIONS does not change +# REVIEW_PREFLIGHT_WORST_CASE_SECONDS: each batch's wall time is bounded by +# its slowest candidate (at most one base + one second attempt), regardless +# of how many candidates in that batch actually make a second attempt or +# which of the two budgets backs it. +REVIEW_PREFLIGHT_MAX_CONFIRMATIONS = REVIEW_PREFLIGHT_MAX_TOTAL_ROUTES + +# Mirrors contextual_orchestrator.model_discovery.DISCOVERY_TIMEOUT_SECONDS at +# ORCHESTRATOR_PIN_SHA (contextual_orchestrator_review_sidecar.sh) exactly. Not +# imported directly: that module's own dependency tree is only installed after +# the sidecar's vendoring step, while this constant must be readable earlier +# (this module's top-level imports are deliberately stdlib-only). Re-verify +# this mirror whenever ORCHESTRATOR_PIN_SHA moves. +REVIEW_DISCOVERY_TIMEOUT_SECONDS = 15.0 +# FIXED (ContextualWisdomLab/.github#1415, Devin Review finding "Discovery- +# time budget undercounts known retries"): the previous count of 7 verified +# only that discover_all_models() makes one call per registered source plus +# one unconditional extra -- it never checked whether any of those calls can +# themselves retry, or whether the pinned revision makes calls beyond that +# simple per-source loop. Re-verified line-by-line against the vendored +# contextual_orchestrator.model_discovery source at ORCHESTRATOR_PIN_SHA +# (fetched and read at that exact commit, not assumed from an older or newer +# revision), counting every sequential HTTP call discover_all_models() can +# make in its real worst case, with every one of the sidecar's five +# bootstrapped credentials present (openai, openrouter, nvidia_nim, +# nvidia_nim_sub, bytez -- opencode_zen's OPENCODE_ZEN_API_KEY is never one of +# the five secrets the sidecar registers, so it always short-circuits with +# zero calls): +# +# Named sub-budgets below (rather than one opaque literal) so a test can +# reconstruct and re-justify each piece of this enumeration independently -- +# see test_contextual_orchestrator_review_runtime_preflight.py's +# test_startup_watchdog_covers_a_retry_heavy_discovery_reconstruction. +# +# (a) Shared Models.dev fetch (_fetch_models_dev_metadata, triggered once +# because openai/nvidia_nim/nvidia_nim_sub declare +# models_dev_provider_id and are credentialed): up to +# _MODELS_DEV_FETCH_ATTEMPTS = 3 sequential attempts, not the 1 the old +# count assumed -- a lone transient failure (this endpoint is known to +# reject urllib's default user agent, see that constant's own +# docstring) is retried up to twice more. +REVIEW_DISCOVERY_MODELS_DEV_MAX_ATTEMPTS = 3 +# (b) Each of the five credentialed sources' own primary model-list fetch +# (discover_provider_models): up to 2 attempts each -- a full +# REVIEW_DISCOVERY_TIMEOUT_SECONDS primary attempt PLUS one +# _DISCOVERY_RETRY_TIMEOUT_SECONDS=5.0s retry on a transient failure +# (is_transient_error), not the unretried single attempt the old count +# assumed. Five sources: openai, openrouter, nvidia_nim, +# nvidia_nim_sub, bytez -- opencode_zen's OPENCODE_ZEN_API_KEY is never +# one of the five secrets the sidecar registers, so it always short- +# circuits with zero calls and is excluded from this count. +REVIEW_DISCOVERY_CREDENTIALED_SOURCE_COUNT = 5 +REVIEW_DISCOVERY_SOURCE_MAX_ATTEMPTS = 2 +# (c) OpenRouter-only extra calls inside discover_provider_models, beyond +# its own primary listing call already counted in (b): one ZDR- +# endpoints fetch (_OPENROUTER_ZDR_ENDPOINTS_URL, no retry -- the old +# count's "unconditional ZDR fetch" line item, kept here) + one +# provider-policies fetch (_OPENROUTER_PROVIDER_POLICIES_URL, no +# retry, entirely missing from the old count). +REVIEW_DISCOVERY_OPENROUTER_SINGLE_EXTRA_CALLS = 2 +# Plus one concurrent (ThreadPoolExecutor, <=8 workers) endpoint-feed +# fetch per currently zero-priced OpenRouter model +# (_openrouter_free_model_endpoints, also entirely missing from the +# old count): wall-clock bounded by ceil(free_model_count / 8) rounds, +# each up to REVIEW_DISCOVERY_TIMEOUT_SECONDS. Verified live against +# OpenRouter's public /api/v1/models catalog (2026-08-31): 21 models +# currently report zero prompt AND completion price (ceil(21/8) = 3 +# rounds today). The pinned code itself does not bound this count, so +# rather than hand-waving it as "1 more call" (the old count's mistake +# for a different item) or leaving it fully unbounded, this budgets 5 +# call-equivalent rounds -- headroom for up to 40 free models, close +# to double today's observed count -- as an explicit, documented +# assumption, not a code-enforced bound; re-verify this headroom if +# OpenRouter's free-tier catalog grows materially past that. +REVIEW_DISCOVERY_OPENROUTER_FREE_ENDPOINT_ROUND_CAP = 5 +# (d) Two trailing global calls discover_all_models() itself makes once +# per run, strictly after every source's loop above, entirely absent +# from the old count: _openrouter_zdr_model_ids() (a SEPARATE fetch of +# the same _OPENROUTER_ZDR_ENDPOINTS_URL as (c) -- not a cache hit; +# this one runs unconditionally, even with no OpenRouter credential +# registered) + openrouter_paid_inference_available() (the credits +# check, gated on an OpenRouter credential being registered, true in +# this worst case). Neither has a retry. +REVIEW_DISCOVERY_TRAILING_GLOBAL_CALLS = 2 +# FIXED (ContextualWisdomLab/.github#1415, Devin Review finding "Discovery- +# time budget undercounts known retries"): the previous count of 7 verified +# only that discover_all_models() makes one call per registered source plus +# one unconditional extra -- it never checked whether any of those calls can +# themselves retry, or whether the pinned revision makes calls beyond that +# simple per-source loop. Re-verified line-by-line against the vendored +# contextual_orchestrator.model_discovery source at ORCHESTRATOR_PIN_SHA +# (fetched and read at that exact commit, not assumed from an older or newer +# revision) -- see (a)-(d) above for the full itemized enumeration. Total: +# 3 + 5*2 + 2 + 5 + 2 = 22 sequential-call-equivalents, each independently +# bounded by REVIEW_DISCOVERY_TIMEOUT_SECONDS. +REVIEW_DISCOVERY_MAX_SEQUENTIAL_HTTP_CALLS = ( + REVIEW_DISCOVERY_MODELS_DEV_MAX_ATTEMPTS + + REVIEW_DISCOVERY_CREDENTIALED_SOURCE_COUNT * REVIEW_DISCOVERY_SOURCE_MAX_ATTEMPTS + + REVIEW_DISCOVERY_OPENROUTER_SINGLE_EXTRA_CALLS + + REVIEW_DISCOVERY_OPENROUTER_FREE_ENDPOINT_ROUND_CAP + + REVIEW_DISCOVERY_TRAILING_GLOBAL_CALLS +) +REVIEW_DISCOVERY_WORST_CASE_SECONDS = ( + REVIEW_DISCOVERY_MAX_SEQUENTIAL_HTTP_CALLS * REVIEW_DISCOVERY_TIMEOUT_SECONDS +) +# The batched-probing worst case derived in the comment above +# REVIEW_PREFLIGHT_MAX_ESCALATIONS: ceil(REVIEW_PREFLIGHT_MAX_TOTAL_ROUTES / +# REVIEW_PREFLIGHT_BATCH_SIZE) batches, each up to +# 2 * REVIEW_PREFLIGHT_TIMEOUT_SECONDS (one base + one escalated attempt, +# sequential within a single candidate's own thread). +REVIEW_PREFLIGHT_WORST_CASE_SECONDS = ( + -(-REVIEW_PREFLIGHT_MAX_TOTAL_ROUTES // REVIEW_PREFLIGHT_BATCH_SIZE) +) * 2 * REVIEW_PREFLIGHT_TIMEOUT_SECONDS +# Explicit, justified slack beyond the two computed network-bound worst cases +# above, for the parts of startup that formula does not (and should not try +# to) model precisely: Python interpreter/module import overhead, in-memory +# catalog construction and JSON evidence-file writes, and the sidecar shell +# script's own 1-second `/healthz` polling granularity. None of those is +# individually large, but the fix here is specifically about correcting a +# previously-absent deadline, not about shaving margin as tight as possible -- +# a generous, explicit constant is preferable to a precise-looking one that +# quietly under-covers real (non-network) startup cost. Deliberately kept +# small relative to the two network-bound terms above so it cannot itself +# mask a future regression in either of them. +REVIEW_STARTUP_HEADROOM_SECONDS = 30 +# The single source of truth for the sidecar's startup watchdog. Both startup +# phases (discovery, then batched preflight probing) run sequentially in one +# process before `/healthz` can respond, so the watchdog covering both must be +# their sum, not either phase's own bound alone. +# contextual_orchestrator_review_sidecar.sh imports this exact constant +# (rather than hard-coding its own timeout) so a future change to any input +# constant above automatically propagates to the watchdog, instead of +# silently reintroducing the coordination bug this fixes +# (ContextualWisdomLab/.github#1415, Devin Review "Startup watchdog preempts +# valid preflight"). +REVIEW_STARTUP_WATCHDOG_SECONDS = int( + REVIEW_DISCOVERY_WORST_CASE_SECONDS + + REVIEW_PREFLIGHT_WORST_CASE_SECONDS + + REVIEW_STARTUP_HEADROOM_SECONDS +) + + +class _EscalationBudget: + """Thread-safe shared counter bounding ADR-0005 escalation retries. + + ADR-0005 documents ``REVIEW_PREFLIGHT_MAX_ESCALATIONS`` as one shared, + run-wide budget, never per-candidate. The batched concurrent preflight + below probes several candidates in separate threads at once, so a plain + ``int`` passed by value between sequential calls -- correct when probing + is strictly sequential -- cannot coordinate admission safely once + multiple threads can observe and spend the same budget concurrently. + This class holds the run's one shared count and reserves a slot + atomically under a lock, so the cap is a hard invariant regardless of + thread scheduling. + """ + + def __init__(self, limit: int, used: int = 0) -> None: + """Start a shared budget at ``used`` (e.g. carried over from a prior stage).""" + self._limit = limit + self._used = used + self._lock = threading.Lock() + + def try_reserve(self) -> bool: + """Atomically claim one escalation slot; return False once spent.""" + with self._lock: + if self._used >= self._limit: + return False + self._used += 1 + return True + + @property + def used(self) -> int: + """Return the total escalations spent so far.""" + with self._lock: + return self._used + class ReviewPreflightError(RuntimeError): """Raised when no selected free provider route is ready for review traffic.""" @@ -95,6 +354,16 @@ def __init__(self, message: str, report: dict[str, object]) -> None: self.report = report +def _build_model_client(client_type: Any, *, timeout: int) -> Any: + """Build a no-retry client with the transport policy for its lifecycle phase.""" + return client_type( + timeout=timeout, + max_output_tokens=REVIEW_MAX_OUTPUT_TOKENS, + max_retries=0, + temperature=REVIEW_TEMPERATURE, + ) + + def _has_text_output(model: object) -> bool: """Return whether a discovered model can emit text responses.""" modalities = getattr(model, "output_modalities", None) @@ -102,7 +371,9 @@ def _has_text_output(model: object) -> bool: return False if isinstance(modalities, str): modalities = (modalities,) - return not modalities or "text" in {str(modality).casefold() for modality in modalities} + return not modalities or "text" in { + str(modality).casefold() for modality in modalities + } _DISCOVERY_DIAGNOSTICS_COMPLETE_SENTINEL = "discovery_diagnostics_complete" @@ -187,21 +458,30 @@ def _report_rows( model_id = str(getattr(model, "model_id", None) or "") if not provider or not model_id: continue - base_url = str(getattr(model, "chat_base_url", None) or zdr_policy.PROVIDER_BASE_URLS[provider]) + base_url = str( + getattr(model, "chat_base_url", None) + or zdr_policy.PROVIDER_BASE_URLS[provider] + ) credential_key = str( - getattr(model, "credential_name", None) or zdr_policy.PROVIDER_CREDENTIAL_NAMES[provider] + getattr(model, "credential_name", None) + or zdr_policy.PROVIDER_CREDENTIAL_NAMES[provider] ) auth_scheme = str( - getattr(model, "auth_scheme", None) or zdr_policy.PROVIDER_AUTH_SCHEMES[provider] + getattr(model, "auth_scheme", None) + or zdr_policy.PROVIDER_AUTH_SCHEMES[provider] ) rows.append( { "provider": provider, "model": model_id, - "agent_id": str(getattr(model, "agent_id", None) or f"{provider}_{model_id}"), + "agent_id": str( + getattr(model, "agent_id", None) or f"{provider}_{model_id}" + ), "is_free": (provider, model_id) in free_route_identities, "prompt_price_per_1k": getattr(model, "prompt_price_per_1k", None), - "completion_price_per_1k": getattr(model, "completion_price_per_1k", None), + "completion_price_per_1k": getattr( + model, "completion_price_per_1k", None + ), "currency_code": getattr(model, "currency_code", None), "base_url": base_url, "credential_key": credential_key, @@ -228,6 +508,17 @@ def _chat_response_has_text(response: object) -> bool: return isinstance(content, str) and bool(content.strip()) +def _without_excluded_agents( + agents: list[dict[str, object]], excluded_ids: frozenset[str] +) -> list[dict[str, object]]: + """Remove prior attempts before batched preflight chooses where to stop.""" + return [ + agent + for agent in agents + if str(agent.get("id") or agent.get("agent_id") or "") not in excluded_ids + ] + + def _safe_http_status(exc: Exception) -> int | None: """Return one bounded HTTP status without persisting an exception message.""" status = getattr(exc, "code", None) @@ -340,41 +631,83 @@ def _response_has_reasoning_without_content(response: object) -> bool: def _preflight_review_agents( - agents: list[object], *, client: Any, escalations_used: int = 0 + agents: list[object], + *, + client: Any, + escalations_used: int = 0, + escalation_budget: "_EscalationBudget | None" = None, + confirmations_used: int = 0, + confirmation_budget: "_EscalationBudget | None" = None, ) -> tuple[list[object], dict[str, object]]: """Probe each route with the runtime request contract and keep ready routes. ADR-0005: a single fixed ``max_tokens`` cannot fit every model in a heterogeneous pool. Each candidate gets one cheap base-budget probe - (``REVIEW_PREFLIGHT_BASE_TOKENS``); when that specific candidate's - response is empty for a "budget too small" reason -- either - ``choices[0].finish_reason == "length"`` (OpenAI's documented signature), - or the vendored ``ModelClient._response_content``'s own broader signature - (a populated ``message.reasoning`` with no string ``content``, which a - reasoning model can hit under a different ``finish_reason`` -- provider - ``finish_reason`` semantics for this case are not verified as uniform - across the pool, and this is the exact original failure mode PR #1436 - responded to) -- that *same* candidate is retried once at a larger, - escalated budget (``REVIEW_PREFLIGHT_ESCALATED_TOKENS``) before being - marked rejected -- bounded by a shared ``REVIEW_PREFLIGHT_MAX_ESCALATIONS`` - counter, which the ``escalations_used`` argument carries forward across - calls (not per candidate, and not reset per call): a caller that probes - two stages of the same preflight run (e.g. ``_preflight_with_fallback``'s - primary and fallback stages) must pass the previous stage's ending count - back in here so the two stages share one budget instead of each getting - its own -- otherwise the computed worst-case bound this counter exists to - enforce silently doubles. Every other failure class (transport exception, - non-2xx, or empty content matching neither signature) is not retried: a - genuinely-down candidate never reaches the escalation path, so it cannot - produce a false "healthy" read. - An exception on the escalated attempt (transport failure, auth failure, + (``REVIEW_PREFLIGHT_BASE_TOKENS``). Admission always requires a SECOND, + confirming probe at the real serving budget + (``REVIEW_PREFLIGHT_ESCALATED_TOKENS``, equal to the ``REVIEW_MAX_OUTPUT_TOKENS`` + ``main()``'s ``ModelClient`` actually requests during review traffic) -- + fixed as `ContextualWisdomLab/.github#1454` (Devin Review, "Serving- + incompatible routes pass startup"): the base probe alone previously + admitted a candidate having proven nothing beyond + ``REVIEW_PREFLIGHT_BASE_TOKENS``, so a candidate whose real completion + ceiling sat strictly between the base and serving budgets passed startup + and only failed once real review traffic began. There are exactly two + ways a candidate reaches that confirming probe, and each draws from its + OWN, separately-purposed budget (`ContextualWisdomLab/.github#1415`, + Devin Review "Later healthy routes cannot start" -- see the two + constants' own module-level comments for the full incident and sizing + rationale): + + 1. **The base probe already returned usable text.** This is the + ordinary, most common case; the second probe exists purely to CONFIRM + that same candidate also serves the real budget, not to diagnose a + failure. This draws from ``confirmation_budget`` + (``REVIEW_PREFLIGHT_MAX_CONFIRMATIONS``). Success marks + ``confirmed_at_serving_budget`` (not ``escalated``) on the row -- the + base attempt already worked, this second attempt only re-proves it at + the real budget. + 2. **The base probe's response was empty for a "budget too small" reason** + -- either ``choices[0].finish_reason == "length"`` (OpenAI's + documented signature), or the vendored + ``ModelClient._response_content``'s own broader signature (a populated + ``message.reasoning`` with no string ``content``, which a reasoning + model can hit under a different ``finish_reason`` -- provider + ``finish_reason`` semantics for this case are not verified as uniform + across the pool, and this is the exact original failure mode PR #1436 + responded to). This draws from ``escalation_budget`` + (``REVIEW_PREFLIGHT_MAX_ESCALATIONS``). Success marks ``escalated`` on + the row -- the base attempt failed and this second attempt is what + actually rescued it. + + Either way, the SAME candidate gets at most one additional attempt (never + a third, and never both a confirmation AND an escalation). Each of the + two budgets' ``*_used`` argument carries forward across calls (not per + candidate, and not reset per call): a caller that probes two stages of + the same preflight run (e.g. ``_preflight_with_fallback``'s primary and + fallback stages) must pass each previous stage's ending count back in + here so the two stages share one pair of budgets instead of each getting + its own -- otherwise the computed worst-case bound these counters exist + to enforce silently doubles. A base response that is empty for any OTHER + reason (no budget-too-small signature) is not retried at all: a + genuinely-down candidate never reaches the second-attempt path, so it + cannot produce a false "healthy" read, and a candidate denied its second + attempt by its own (exhausted) budget is recorded + ``confirmation_budget_exhausted`` or ``escalation_budget_exhausted`` + (matching which of the two paths it took) and not admitted -- fails + closed, exactly like a base failure that never got its own second-attempt + slot. Exhaustion of ONE budget never blocks a candidate whose path draws + from the OTHER budget -- the exact cross-purpose interaction that made a + confirmation-only path spend a rescue-only allowance is the bug this + separation fixes. + An exception on the second attempt (transport failure, auth failure, rate limit, server error, or a genuine budget rejection) is recorded via ``_record_provider_exception`` -- the SAME sanitized classification the - base probe uses, regardless of attempt. An HTTP status alone does not - distinguish "this candidate's real ceiling is below the escalated - budget" from any other cause (401/429/5xx are not budget evidence); this - codebase has no validated signal today that does, so it does not invent - one via an over-specific label. + base probe uses, regardless of attempt or which path it came from. An + HTTP status alone does not distinguish "this candidate's real ceiling is + below the escalated budget" from any other cause (401/429/5xx are not + budget evidence); this codebase has no validated signal today that does, + so it does not invent one via an over-specific label. The report deliberately records only stable route identity, a bounded exception class name, an optional numeric HTTP status, attempt count, and @@ -384,28 +717,62 @@ def _preflight_review_agents( every response-bearing outcome -- success included, not just failure/escalation, so future tuning has a real "normal" baseline to compare against -- and always describe the same, most recent attempt for - a route (the base attempt when only one was made; the escalated attempt - when a second was made) -- never a mix of the two attempts' state. When - the escalated attempt raises an exception instead of returning a + a route (the base attempt when only one was made; the second attempt + when one was made) -- never a mix of the two attempts' state. When + the second attempt raises an exception instead of returning a response, both fields are absent entirely (there is no response to describe) rather than silently retaining the base attempt's values. + Batched concurrent probing (``_preflight_review_agent_batches``) calls + this function once per candidate, concurrently, from several threads at + once within one batch. A plain ``escalations_used``/``confirmations_used`` + int passed by value cannot coordinate admission safely once multiple + threads can observe and spend the same budget concurrently, so callers + that need cross-thread coordination pass a shared ``escalation_budget``/ + ``confirmation_budget`` instead; a caller that only ever probes + sequentially (every direct call in this module's own test suite, and any + single, unbatched invocation) can keep passing plain + ``escalations_used``/``confirmations_used`` ints, each wrapped in a + private, single-owner budget for the duration of this one call -- + identical external behavior to before this thread-safety addition. + Args: agents: Selected zero-cost model agents. client: Vendored ``ModelClient``-compatible transport. - escalations_used: Escalations already spent earlier in this same - preflight run (e.g. by a prior stage), so the shared budget is - honored across calls rather than restarted at zero. + escalations_used: Rescue escalations already spent earlier in this + same preflight run (e.g. by a prior stage), so the shared rescue + budget is honored across calls rather than restarted at zero. + Ignored when ``escalation_budget`` is given. + escalation_budget: A shared, thread-safe budget bounding rescue + attempts (a FAILED base probe) to coordinate admission across + concurrent callers. When omitted, a private budget seeded from + ``escalations_used`` is used instead. + confirmations_used: Confirmations already spent earlier in this same + preflight run, mirroring ``escalations_used`` for the separate + confirmation budget. Ignored when ``confirmation_budget`` is + given. + confirmation_budget: A shared, thread-safe budget bounding + confirmation attempts (a SUCCESSFUL base probe) -- deliberately + separate from ``escalation_budget`` (see + ``REVIEW_PREFLIGHT_MAX_CONFIRMATIONS``'s module-level comment for + why). When omitted, a private budget seeded from + ``confirmations_used`` is used instead. Returns: A pair of viable agents and a sanitized preflight report. The - report's ``escalations_used`` is the running total including - ``escalations_used``'s starting value, so a caller chaining another - stage can pass it straight back in. + report's ``escalations_used``/``confirmations_used`` are each the + running total including that argument's starting value, so a caller + chaining another stage can pass them straight back in. Raises: ReviewPreflightError: If no provider route returns usable text. """ + budget = escalation_budget or _EscalationBudget( + REVIEW_PREFLIGHT_MAX_ESCALATIONS, escalations_used + ) + confirm_budget = confirmation_budget or _EscalationBudget( + REVIEW_PREFLIGHT_MAX_CONFIRMATIONS, confirmations_used + ) viable: list[object] = [] routes: list[dict[str, object]] = [] for agent in agents: @@ -431,61 +798,76 @@ def _preflight_review_agents( _record_provider_exception(row, exc) routes.append(row) continue - if _chat_response_has_text(response): - # KNOWN GAP, tracked (not yet fixed) as - # ContextualWisdomLab/.github#1454: this admits the candidate - # having only proven it works at REVIEW_PREFLIGHT_BASE_TOKENS - # (16), never at the real serving budget - # (REVIEW_MAX_OUTPUT_TOKENS, 4096) main()'s ModelClient actually - # requests. ADR-0005's own Research (axis 2) already documents - # that a provider's hard completion-token ceiling is a real, - # separate-from-reasoning-overhead quantity per model; a - # candidate whose real ceiling sits strictly between 16 and 4096 - # would pass here and only fail later, on real review traffic. - # Mitigated in production (not fixed here) by - # contextual_orchestrator.orchestrator.TaskOrchestrator's own - # per-request failover/circuit-breaker, which this preflight - # does not replace. - row["status"] = "ready" - # Populated on every outcome, including this most-common, - # ordinary success path -- not just failure/escalation -- so - # future tuning has a real "normal" baseline to compare against, - # not just evidence of what went wrong. - row["finish_reason"] = _response_finish_reason(response) or "unknown" - row["reasoning_without_content"] = _response_has_reasoning_without_content(response) - routes.append(row) - viable.append(agent) - continue + + base_has_text = _chat_response_has_text(response) finish_reason = _response_finish_reason(response) + # Populated on every response-bearing outcome, including an + # eventually-superseded base attempt -- not just failure/escalation + # -- so future tuning has a real "normal" baseline to compare + # against. Overwritten below if a second attempt is made (see the + # docstring: both fields always describe the same, most recent + # attempt, never a mix of the two). row["finish_reason"] = finish_reason or "unknown" reasoning_without_content = _response_has_reasoning_without_content(response) row["reasoning_without_content"] = reasoning_without_content - budget_signature = finish_reason == "length" or reasoning_without_content - # KNOWN, ACCEPTED, TRACKED LIMITATION on the escalations_used >= - # REVIEW_PREFLIGHT_MAX_ESCALATIONS branch below, ContextualWisdomLab/.github#1458 - # (originally documented on ADR-0005, docs/adr/0005-sidecar-preflight-token-budget.md): - # escalations_used is one shared, first-come-first-served counter for - # the whole run, consumed in catalog order + + if not base_has_text: + budget_signature = finish_reason == "length" or reasoning_without_content + if not budget_signature: + # Genuinely down (or an unrelated malformed reply): no + # signature suggests a bigger budget would help, so this + # candidate never reaches the second-attempt path -- it + # cannot produce a false "healthy" read. + row["status"] = "rejected" + row["error_type"] = "invalid_chat_response" + routes.append(row) + continue + # Either the base probe already has usable text (fix for + # ContextualWisdomLab/.github#1454: admission still requires + # confirming that text holds at the real serving budget, not just + # REVIEW_PREFLIGHT_BASE_TOKENS) or it matched a "budget too small" + # signature above and needs the existing escalation retry. + # + # FIXED (ContextualWisdomLab/.github#1415, Devin Review "Later + # healthy routes cannot start"): these two cases now draw from TWO + # SEPARATE budgets, not one shared one -- see + # REVIEW_PREFLIGHT_MAX_ESCALATIONS/REVIEW_PREFLIGHT_MAX_CONFIRMATIONS' + # module-level comments for the full incident. Confirming an + # already-successful base probe is the common case (every successful + # candidate needs it); rescuing a failed one is the rare case. A + # scarce rescue budget consumed by a burst of ordinary confirmations + # (or vice versa) must never deny a DIFFERENT candidate's unrelated + # second attempt. + second_attempt_budget = confirm_budget if base_has_text else budget + second_attempt_exhausted_error = ( + "confirmation_budget_exhausted" if base_has_text else "escalation_budget_exhausted" + ) + # + # KNOWN, ACCEPTED, TRACKED LIMITATION on the try_reserve() branch + # below, ContextualWisdomLab/.github#1458 (originally documented on + # ADR-0005, docs/adr/0005-sidecar-preflight-token-budget.md): each + # budget is still first-come-first-served in catalog order # (build_zdr_prioritized_catalog's (cost_evidence_rank, # zdr_attested_rank, provider, model) sort, not random). A - # later-sorting candidate can be denied its own escalation attempt - # purely because REVIEW_PREFLIGHT_MAX_ESCALATIONS earlier candidates - # already claimed the shared budget -- even if it would have been the - # only one to succeed at REVIEW_PREFLIGHT_ESCALATED_TOKENS. - # Deliberately not reordered (round-robin/random): a fixed-size - # shared budget smaller than the candidate pool always has to deny - # someone an escalation, so reordering only changes who, and picking - # a specific policy without real telemetry on which candidates - # actually need escalation would itself be the kind of unjustified - # heuristic this design rejects elsewhere. - if not budget_signature or escalations_used >= REVIEW_PREFLIGHT_MAX_ESCALATIONS: + # later-sorting candidate needing a rescue can still be denied its + # own escalation attempt purely because + # REVIEW_PREFLIGHT_MAX_ESCALATIONS earlier-sorting candidates already + # claimed that (deliberately scarce) rescue budget, even if it would + # have succeeded at REVIEW_PREFLIGHT_ESCALATED_TOKENS -- unchanged by + # this fix, and unrelated to it: REVIEW_PREFLIGHT_MAX_CONFIRMATIONS + # is sized to REVIEW_PREFLIGHT_MAX_TOTAL_ROUTES precisely so the same + # exhaustion can never happen on the confirmation path (see that + # constant's own comment). Deliberately not reordered + # (round-robin/random): a fixed-size shared budget smaller than the + # candidate pool always has to deny someone a second attempt, so + # reordering only changes who, and picking a specific policy without + # real telemetry on which candidates actually need it would itself be + # the kind of unjustified heuristic this design rejects elsewhere. + if not second_attempt_budget.try_reserve(): row["status"] = "rejected" - row["error_type"] = ( - "invalid_chat_response" if not budget_signature else "escalation_budget_exhausted" - ) + row["error_type"] = second_attempt_exhausted_error routes.append(row) continue - escalations_used += 1 row["attempts"] = 2 escalated_payload = dict(base_payload) escalated_payload["max_tokens"] = REVIEW_PREFLIGHT_ESCALATED_TOKENS @@ -500,18 +882,30 @@ def _preflight_review_agents( # the same sanitized classification the base probe uses, rather # than the previous "escalated_probe_rejected" label, which # over-claimed budget-specific attribution this codebase has no - # validated signal to actually support. + # validated signal to actually support. This also applies to a + # candidate whose base probe already had text: a rejection here + # is exactly the ContextualWisdomLab/.github#1454 scenario -- + # usable at the base budget, rejected outright at the real + # serving budget -- and it must not be admitted just because an + # earlier, smaller attempt happened to succeed. _record_provider_exception(row, exc) routes.append(row) continue if _chat_response_has_text(escalated_response): row["status"] = "ready" - row["escalated"] = True + if base_has_text: + # The base attempt already had usable text; this second + # attempt only confirms that same candidate also serves the + # real budget -- distinct from `escalated`, which means the + # base attempt FAILED and this second attempt is what + # rescued it. + row["confirmed_at_serving_budget"] = True + else: + row["escalated"] = True # Overwrite the base attempt's stale diagnostic fields with the # escalated (successful, final) attempt's own state -- otherwise - # a ready route's evidence would still show the budget-too-small - # signature that triggered the escalation in the first place, - # describing a response this route no longer produced. + # a ready route's evidence would still show the base attempt's + # signature, describing a response this route no longer produced. row["finish_reason"] = _response_finish_reason(escalated_response) or "unknown" row["reasoning_without_content"] = _response_has_reasoning_without_content( escalated_response @@ -519,6 +913,10 @@ def _preflight_review_agents( routes.append(row) viable.append(agent) continue + # ContextualWisdomLab/.github#1454's exact failure mode when + # base_has_text is True: usable at REVIEW_PREFLIGHT_BASE_TOKENS, + # empty at the real REVIEW_PREFLIGHT_ESCALATED_TOKENS serving budget + # -- never admitted, regardless of the earlier, smaller success. row["status"] = "rejected" row["error_type"] = "invalid_chat_response" # Both fields now describe this escalated (2nd, final) attempt, @@ -534,8 +932,10 @@ def _preflight_review_agents( "probed_count": len(agents), "ready_count": len(viable), "rejected_count": len(agents) - len(viable), - "escalations_used": escalations_used, + "escalations_used": budget.used, "escalation_budget": REVIEW_PREFLIGHT_MAX_ESCALATIONS, + "confirmations_used": confirm_budget.used, + "confirmation_budget": REVIEW_PREFLIGHT_MAX_CONFIRMATIONS, "routes": routes, } if not viable: @@ -545,33 +945,161 @@ def _preflight_review_agents( return viable, report +def _preflight_review_agent_batches( + agents: list[object], + *, + client: Any, + escalation_budget: "_EscalationBudget | None" = None, + confirmation_budget: "_EscalationBudget | None" = None, +) -> tuple[list[object], dict[str, object]]: + """Probe bounded concurrent batches until one batch contains a ready route. + + Candidates are probed ``REVIEW_PREFLIGHT_BATCH_SIZE`` at a time, each in + its own thread, so a full ``REVIEW_PREFLIGHT_MAX_TOTAL_ROUTES``-candidate + catalog stays within the sidecar's readiness budget: batch wall time is + the slowest candidate in that batch, not the sum of all of them. Batches + themselves still run one after another, and probing stops as soon as one + batch yields at least one ready route -- a later, unprobed batch can + never "hide" a route this run already found usable. + + Two separate run-wide budgets bound the two distinct second-attempt + purposes (``ContextualWisdomLab/.github#1415``, Devin Review "Later + healthy routes cannot start" -- see ``REVIEW_PREFLIGHT_MAX_ESCALATIONS``/ + ``REVIEW_PREFLIGHT_MAX_CONFIRMATIONS``'s own module-level comments for + the full incident and sizing rationale): ``escalation_budget`` for + rescuing a FAILED base probe (deliberately scarce), and + ``confirmation_budget`` for confirming a SUCCESSFUL one (sized to never + starve a genuinely healthy candidate). Neither is one-per-batch or + one-per-candidate; since several candidates in the same batch can reach + either decision concurrently, admission is coordinated through each + ``_EscalationBudget``'s own lock rather than a plain int, so neither cap + is ever exceeded under concurrency. One consequence of that concurrency: + ADR-0005's "first-come-first-served in catalog order" framing holds + strictly only ACROSS batches (which stay sequential); WITHIN one batch, + whichever candidate's thread reaches a given reservation first wins it. + This changes at most which candidate among a few concurrently-probed + ones claims a scarce slot -- each cap itself is a hard, lock-enforced + invariant regardless of scheduling. + + Args: + agents: Selected zero-cost model agents, probed in catalog order. + client: Vendored ``ModelClient``-compatible transport. + escalation_budget: A shared budget to coordinate rescue-attempt + admission with another stage (see ``_preflight_with_fallback``). + A fresh, run-local budget is created when omitted. + confirmation_budget: A shared budget to coordinate confirmation- + attempt admission with another stage, separate from + ``escalation_budget``. A fresh, run-local budget is created when + omitted. + + Returns: + A pair of viable agents (from the first batch with any) and a + sanitized, aggregated preflight report across every batch attempted. + + Raises: + ReviewPreflightError: If every batch is exhausted with no viable + route. + """ + budget = escalation_budget or _EscalationBudget(REVIEW_PREFLIGHT_MAX_ESCALATIONS) + confirm_budget = confirmation_budget or _EscalationBudget( + REVIEW_PREFLIGHT_MAX_CONFIRMATIONS + ) + attempted_routes: list[dict[str, object]] = [] + attempted_count = 0 + for offset in range(0, len(agents), REVIEW_PREFLIGHT_BATCH_SIZE): + batch = agents[offset : offset + REVIEW_PREFLIGHT_BATCH_SIZE] + with ThreadPoolExecutor(max_workers=len(batch)) as executor: + futures = [ + executor.submit( + _preflight_review_agents, + [agent], + client=client, + escalation_budget=budget, + confirmation_budget=confirm_budget, + ) + for agent in batch + ] + viable: list[object] = [] + for future in futures: + try: + route_viable, route_report = future.result() + except ReviewPreflightError as exc: + route_viable = [] + route_report = exc.report + viable.extend(route_viable) + attempted_routes.extend(route_report["routes"]) + attempted_count += int(route_report["probed_count"]) + if viable: + return viable, { + "contract": "strix-plain-chat-preflight-v2", + "probed_count": attempted_count, + "ready_count": len(viable), + "rejected_count": attempted_count - len(viable), + "escalations_used": budget.used, + "escalation_budget": REVIEW_PREFLIGHT_MAX_ESCALATIONS, + "confirmations_used": confirm_budget.used, + "confirmation_budget": REVIEW_PREFLIGHT_MAX_CONFIRMATIONS, + "routes": attempted_routes, + "batch_size": REVIEW_PREFLIGHT_BATCH_SIZE, + } + report: dict[str, object] = { + "contract": "strix-plain-chat-preflight-v2", + "probed_count": attempted_count, + "ready_count": 0, + "rejected_count": attempted_count, + "escalations_used": budget.used, + "escalation_budget": REVIEW_PREFLIGHT_MAX_ESCALATIONS, + "confirmations_used": confirm_budget.used, + "confirmation_budget": REVIEW_PREFLIGHT_MAX_CONFIRMATIONS, + "routes": attempted_routes, + "batch_size": REVIEW_PREFLIGHT_BATCH_SIZE, + } + raise ReviewPreflightError( + "no provider route passed the Strix plain-chat preflight", report + ) + + def _preflight_with_fallback( primary_agents: list[object], fallback_agents: list[object], *, client: Any ) -> tuple[list[object], dict[str, object], bool]: """Use the priced catalog only after every primary route rejects. - The two stages share ADR-0005's one ``REVIEW_PREFLIGHT_MAX_ESCALATIONS`` - budget for the whole preflight run, not one budget each: the primary - stage's ending ``escalations_used`` is passed as the fallback stage's - starting point, so a run that rejects all 8 primary routes and then - probes 4 fallback routes still spends at most 4 escalations total (12 - base attempts + 4 escalations, 160s worst case) instead of up to 8 (200s) - -- which would exceed Layer 1's 180s healthz-readiness wait. Both + The two stages -- each itself run through + ``_preflight_review_agent_batches``'s bounded concurrent batching -- share + ONE pair of run-wide budgets for the whole preflight run, not one pair + each: one ``_EscalationBudget`` for rescue attempts + (``REVIEW_PREFLIGHT_MAX_ESCALATIONS``) and a separate one for confirmation + attempts (``REVIEW_PREFLIGHT_MAX_CONFIRMATIONS``, + ``ContextualWisdomLab/.github#1415``) are each created here and passed + into both stages, so a run that rejects all primary routes and then + probes the fallback catalog still spends at most each budget's own cap + in total across both stages combined -- otherwise the computed worst-case + bound these counters exist to enforce would silently double. Both stages' reports remain in the result: the fallback (or sole) stage's - report carries the run's final, cumulative ``escalations_used``, and - ``primary_attempt`` nests the primary stage's own report -- including its - own ``escalations_used`` -- whenever a fallback stage ran at all. + report carries the run's final, cumulative ``escalations_used``/ + ``confirmations_used``, and ``primary_attempt`` nests the primary + stage's own report -- including its own ``escalations_used``/ + ``confirmations_used`` -- whenever a fallback stage ran at all. """ + budget = _EscalationBudget(REVIEW_PREFLIGHT_MAX_ESCALATIONS) + confirm_budget = _EscalationBudget(REVIEW_PREFLIGHT_MAX_CONFIRMATIONS) try: - viable, report = _preflight_review_agents(primary_agents, client=client) + viable, report = _preflight_review_agent_batches( + primary_agents, + client=client, + escalation_budget=budget, + confirmation_budget=confirm_budget, + ) return viable, report, False except ReviewPreflightError as primary_error: if not fallback_agents: raise - escalations_used = int(primary_error.report.get("escalations_used", 0)) try: - viable, report = _preflight_review_agents( - fallback_agents, client=client, escalations_used=escalations_used + viable, report = _preflight_review_agent_batches( + fallback_agents, + client=client, + escalation_budget=budget, + confirmation_budget=confirm_budget, ) except ReviewPreflightError as fallback_error: fallback_error.report["primary_attempt"] = primary_error.report @@ -653,9 +1181,7 @@ def _bounded_primary_catalog_limit( return total_limit -def _bounded_fallback_catalog_limit( - requested_limit: int, *, primary_count: int -) -> int: +def _bounded_fallback_catalog_limit(requested_limit: int, *, primary_count: int) -> int: """Return remaining priced-fallback capacity after primary selection.""" if requested_limit < 1: raise ValueError("ORCHESTRATOR_CATALOG_LIMIT must be positive") @@ -675,15 +1201,17 @@ def _catalog_account_cap(default: int) -> int: (e.g. ``REVIEW_PREFLIGHT_MAX_TOTAL_ROUTES``) here: doing so silently disables per-account diversification and lets one rate-limited account consume the entire preflight budget. That is not a hypothetical failure - mode -- a sibling in-flight branch's own ``_catalog_family_cap()`` - fell back to exactly ``REVIEW_PREFLIGHT_MAX_TOTAL_ROUTES`` and, in a live + mode -- an earlier revision of this helper (under a different name) + defaulted to exactly ``REVIEW_PREFLIGHT_MAX_TOTAL_ROUTES`` and, in a live production run, let two NVIDIA NIM credentials sharing one rate-limited upstream jointly occupy 12/12 preflight slots, of which 10 were then rejected with 429/404/timeout (see ContextualWisdomLab/.github#1415 and the "빈 깡통 경로" report it responds to). Routing the default through the caller-supplied ``policy.DEFAULT_ACCOUNT_CAP`` (rather than hand-typing a literal here) keeps this module's cap from silently drifting out of sync - with the policy module's own declared intent. + with the policy module's own declared intent. `main` PR #1487 landed the + identical fix independently, converging on this exact name and shape; + this is the single canonical implementation. Args: default: The cap to use when ``ORCHESTRATOR_CATALOG_ACCOUNT_CAP`` is @@ -782,23 +1310,63 @@ def main(argv: list[str] | None = None) -> int: preflight, or no auth token is available — the sidecar must fail closed rather than boot a mock or unaudited pool. """ - parser = argparse.ArgumentParser(description="Serve the contextual-orchestrator review sidecar.") + parser = argparse.ArgumentParser( + description="Serve the contextual-orchestrator review sidecar." + ) parser.add_argument("--host", default="127.0.0.1") parser.add_argument("--port", type=int, default=18080) - parser.add_argument("--auth-token", default="", help="Explicit bearer token; else resolve from the KV") - parser.add_argument("--discovery-out", required=True, help="Path to write the free-only discovery report JSON") - parser.add_argument("--catalog-out", required=True, help="Path to write the agents catalog JSON") - parser.add_argument("--report-out", required=True, help="Path to write the policy evidence JSON") - parser.add_argument("--preflight-out", required=True, help="Path to write sanitized runtime preflight JSON") - parser.add_argument("--zdr-endpoints", default=None, help="Optional OpenRouter /api/v1/endpoints/zdr JSON path") + parser.add_argument( + "--auth-token", + default="", + help="Explicit bearer token; else resolve from the KV", + ) + parser.add_argument( + "--discovery-out", + required=True, + help="Path to write the free-only discovery report JSON", + ) + parser.add_argument( + "--catalog-out", required=True, help="Path to write the agents catalog JSON" + ) + parser.add_argument( + "--report-out", required=True, help="Path to write the policy evidence JSON" + ) + parser.add_argument( + "--preflight-out", + required=True, + help="Path to write sanitized runtime preflight JSON", + ) + parser.add_argument( + "--zdr-endpoints", + default=None, + help="Optional OpenRouter /api/v1/endpoints/zdr JSON path", + ) parser.add_argument("--require-zdr", action="store_true") parser.add_argument("--pool", choices=("free", "auto"), default="free") + parser.add_argument( + "--single-candidate-attempt", + action="store_true", + help="Disable the redundant same-agent retry when job-level failover is active", + ) + parser.add_argument( + "--exclude-candidate-id", + action="append", + default=[], + help="Exclude a previously attempted agent id before runtime preflight", + ) args = parser.parse_args(argv) from contextual_orchestrator.credentials import get_credential from contextual_orchestrator.chat_capability import is_general_chat_agent_model_id - from contextual_orchestrator.model_discovery import discover_all_models, free_discovered_models - from contextual_orchestrator.orchestrator import ModelClient, TaskOrchestrator, load_agents + from contextual_orchestrator.model_discovery import ( + discover_all_models, + free_discovered_models, + ) + from contextual_orchestrator.orchestrator import ( + ModelClient, + TaskOrchestrator, + load_agents, + ) from contextual_orchestrator.review_gateway import ( REVIEW_AUTH_CREDENTIAL_NAME, register_review_credentials, @@ -821,8 +1389,13 @@ def main(argv: list[str] | None = None) -> int: "review sidecar requires an explicit --auth-token or the " f"KV credential {REVIEW_AUTH_CREDENTIAL_NAME!r}" ) - if not any(name.startswith(("BYTEZ_", "NVIDIA_", "OPENROUTER_", "OPENAI_")) for name in registered): - raise SystemExit("review sidecar requires at least one provider credential in the KV") + if not any( + name.startswith(("BYTEZ_", "NVIDIA_", "OPENROUTER_", "OPENAI_")) + for name in registered + ): + raise SystemExit( + "review sidecar requires at least one provider credential in the KV" + ) try: discovered, discovery_errors = discover_all_models() @@ -849,9 +1422,7 @@ def main(argv: list[str] | None = None) -> int: _write_json(args.discovery_out, {"models": rows}) zdr_endpoints = _load_zdr_endpoints(args.zdr_endpoints) normalized_rows = parse_discovery_report({"models": rows}) - free_rows = [ - row for row in normalized_rows if row.get("cost_evidence") == "free" - ] + free_rows = [row for row in normalized_rows if row.get("cost_evidence") == "free"] priced_rows = [ row for row in normalized_rows if row.get("cost_evidence") == "priced" ] @@ -867,7 +1438,7 @@ def main(argv: list[str] | None = None) -> int: zdr_endpoints=zdr_endpoints, checker=is_zdr_model, ) - requested_catalog_limit = int(os.environ.get("ORCHESTRATOR_CATALOG_LIMIT", "12")) + requested_catalog_limit = int(os.environ.get("ORCHESTRATOR_CATALOG_LIMIT", "24")) primary_limit = _bounded_primary_catalog_limit( requested_catalog_limit, pool=args.pool, has_free_rows=bool(admitted_free_rows) ) @@ -884,6 +1455,12 @@ def main(argv: list[str] | None = None) -> int: require_zdr=args.require_zdr, pool=args.pool, ) + excluded_candidate_ids = frozenset(args.exclude_candidate_id) + result["agents"] = _without_excluded_agents( + result["agents"], excluded_candidate_ids + ) + if not result["agents"]: + raise SystemExit("review sidecar has no candidate after exclusions") result["report"] = _with_discovery_counts( result["report"], normalized_rows, provider_account=provider_account ) @@ -915,6 +1492,9 @@ def main(argv: list[str] | None = None) -> int: require_zdr=args.require_zdr, pool="auto", ) + fallback_result["agents"] = _without_excluded_agents( + fallback_result["agents"], excluded_candidate_ids + ) except PolicyError: fallback_result = None if fallback_result is not None: @@ -930,11 +1510,8 @@ def main(argv: list[str] | None = None) -> int: fallback_result["agents"], loader=load_agents, ) - client = ModelClient( - timeout=REVIEW_PREFLIGHT_TIMEOUT_SECONDS, - max_output_tokens=REVIEW_MAX_OUTPUT_TOKENS, - max_retries=0, - temperature=REVIEW_TEMPERATURE, + client = _build_model_client( + ModelClient, timeout=REVIEW_PREFLIGHT_TIMEOUT_SECONDS ) try: agents, preflight_report, fallback_used = _preflight_with_fallback( @@ -955,11 +1532,30 @@ def main(argv: list[str] | None = None) -> int: _write_json(args.report_out, result["report"]) _write_json(args.preflight_out, preflight_report) - client = ModelClient( - max_output_tokens=REVIEW_MAX_OUTPUT_TOKENS, - temperature=REVIEW_TEMPERATURE, + client = _build_model_client( + ModelClient, timeout=REVIEW_SERVING_TIMEOUT_SECONDS + ) + # realtime_judge is not just a future-routing + # quality-ledger signal -- route_once() uses it to gate acceptance of the + # *current* answer and to fail over to the next measured candidate on + # rejection (see route_once/_realtime_route_judge in + # contextual_orchestrator/orchestrator.py); disabling it let a + # judge-rejected, low-quality answer reach Noema instead of another ready + # route. Normal callers retain TaskOrchestrator's defaults. The explicit + # single-candidate job mode removes only its redundant same-agent retry; + # cross-candidate failover happens in the next workflow job, while the + # realtime judge remains enabled by its unchanged default. + # + # Sliced to REVIEW_SERVING_MAX_CANDIDATES (see that constant's own + # comment): serving the full preflight-admitted pool made the honest + # worst case exceed this job's own timeout-minutes ceiling. `agents` is + # already preflight's own ranked, verified-ready ordering, so this keeps + # the top-ranked candidates and only trims serving-time failover depth + # among routes preflight already proved could serve a real request. + attempt_options = {"tool_retry_attempts": 0} if args.single_candidate_attempt else {} + orchestrator = TaskOrchestrator( + agents[:REVIEW_SERVING_MAX_CANDIDATES], client=client, **attempt_options ) - orchestrator = TaskOrchestrator(agents, client=client) serve( orchestrator, host=args.host, diff --git a/scripts/ci/contextual_orchestrator_review_sidecar.sh b/scripts/ci/contextual_orchestrator_review_sidecar.sh index e4984f643b..d036e6f507 100755 --- a/scripts/ci/contextual_orchestrator_review_sidecar.sh +++ b/scripts/ci/contextual_orchestrator_review_sidecar.sh @@ -14,7 +14,29 @@ # (fail-closed zero-cost) pool. set -euo pipefail -ORCHESTRATOR_PIN_SHA="${ORCHESTRATOR_PIN_SHA:-8cd99f139915131ba0239bce12a5d6a5fd85394e}" +launcher_attempt_args=() +case "${1:-}" in + "") ;; + --single-candidate-attempt) + launcher_attempt_args=(--single-candidate-attempt) + shift + ;; + *) + printf '[contextual-orchestrator-sidecar] error: unsupported argument: %s\n' "$1" >&2 + exit 1 + ;; +esac + +launcher_exclusion_args=() +if [ -n "${CONTEXTUAL_ORCHESTRATOR_EXCLUDE_CANDIDATE_ID:-}" ]; then + launcher_exclusion_args=(--exclude-candidate-id "$CONTEXTUAL_ORCHESTRATOR_EXCLUDE_CANDIDATE_ID") +fi +if [ "$#" -ne 0 ]; then + printf '[contextual-orchestrator-sidecar] error: unexpected extra arguments\n' >&2 + exit 1 +fi + +ORCHESTRATOR_PIN_SHA="${ORCHESTRATOR_PIN_SHA:-ab7a813a69dae19541dc2888acd50c4ce37b29b7}" ORCHESTRATOR_GIT_URL="${ORCHESTRATOR_GIT_URL:-https://github.com/ContextualWisdomLab/contextual-orchestrator.git}" # The Strix gate and Noema SSRF guard accept this one process-local origin. # Keep it fixed so an environment override cannot create an unvalidated sidecar. @@ -35,12 +57,63 @@ SIDECAR_LOG_SANITIZER="$ORG_REPO_ROOT/scripts/ci/sanitize_contextual_orchestrato # finishes, letting the shell script wait for a deterministic marker instead # of guessing whether the async sanitizer has caught up. SIDECAR_DISCOVERY_DIAGNOSTICS_SENTINEL="discovery_diagnostics_complete" -CATALOG_LIMIT="${ORCHESTRATOR_CATALOG_LIMIT:-12}" -# Each KV credential is an independent account, including two credentials for -# the same vendor or endpoint. The account cap prevents one credential from -# consuming the bounded twelve-route preflight catalog without inventing a -# provider-family equivalence relation. -CATALOG_ACCOUNT_CAP="${ORCHESTRATOR_CATALOG_ACCOUNT_CAP:-8}" +# 2026-08-30: this cap was raised from 4 to 8, then -- mistakenly -- all the +# way to 24 (== the total route budget below, i.e. no cap at all) once route +# probing itself became a batched, concurrent operation. That last raise was +# a real, live-evidenced bug (ContextualWisdomLab/.github#1415): per an +# exact-head evidence trail, orchestrator/free's 46 discovered free rows were +# 100% nvidia_nim/nvidia_nim_sub (two credentials sharing one rate-limited +# upstream, integrate.api.nvidia.com), so a cap equal to the total budget let +# those two credentials jointly occupy the entire preflight batch every run +# (a live run showed `probed_count: 12, ready_count: 2, rejected_count: 10` -- +# 83% rejected via 429/404/timeout). Candidate selection sorts eligible rows +# alphabetically by (provider, model) with no reliability awareness, so an +# uncapped or too-generous cap deterministically re-admits the same +# alphabetically-first candidates on every run -- including confirmed-dead +# model ids (e.g. google/gemma-3-12b-it, google/gemma-3-4b-it; HTTP 404 on +# live preflight) -- while starving the remaining healthy free routes in the +# same discovery report of a chance. This is not throughput tuning: it is the +# confirmed, reproducible root cause of orchestrator/free's "no provider route +# passed the Strix plain-chat preflight" failures (see +# docs/product-technical-gap-baseline.md's 2026-08-30 sidecar-preflight +# entries for the full evidence). +# Each KV credential is an independent account (contextual-orchestrator PR +# #1468), including two credentials for the same vendor or endpoint. The +# account cap below keeps that real, per-credential meaning -- a value +# strictly smaller than CATALOG_LIMIT -- rather than collapsing to a +# provider-family equivalence relation or, worse, to CATALOG_LIMIT itself. +# Route probing is now batched (contextual_orchestrator_review_launcher.py's +# REVIEW_PREFLIGHT_BATCH_SIZE): up to REVIEW_PREFLIGHT_BATCH_SIZE candidates +# run concurrently per batch, so this cap no longer multiplies worst-case wall +# time the way a higher cap would have under the old sequential probing; see +# that module's own REVIEW_PREFLIGHT_MAX_ESCALATIONS comment for the current, +# batching-aware worst-case arithmetic. If real hosted runs show a single +# account still starves the pool even at this cap, or the added concurrency +# itself becomes the bottleneck, the more complete fix is a live provider +# /v1/models cross-check at discovery time to drop retired model ids before +# they ever reach preflight -- see git history's now-removed +# select_nvidia_nim_model.py (removed in #1442) for a worked example of that +# exact query-the-provider-catalog pattern, applied there to a different, +# direct-provider caller -- rather than raising this cap further. PR numbers +# are used here, not raw commit SHAs or branch names: a squash merge would +# leave a raw pre-merge commit unreachable in plain git once the branch is +# deleted, while the PR itself (and its full commit history) stays +# permanently resolvable on GitHub. +# +# FIXED (ContextualWisdomLab/.github#1415, Devin Review follow-up): "raised +# from 4 to 8" above was itself never reverted when the 24 mistake was fixed +# -- this shell kept unconditionally exporting a literal 8 default, so the +# real, currently-intended default (4, matching +# contextual_orchestrator_review_policy.DEFAULT_ACCOUNT_CAP) never actually +# took effect in production. CATALOG_ACCOUNT_CAP is no longer set here as a +# shell literal; see its derivation further below, right before its export, +# for the single-source-of-truth fix and why it must run after +# $sidecar_python/$ORG_REPO_ROOT/fail() are defined. +CATALOG_LIMIT="${ORCHESTRATOR_CATALOG_LIMIT:-24}" +# CATALOG_ACCOUNT_CAP itself is derived further below (single source of truth: +# contextual_orchestrator_review_policy.py's own DEFAULT_ACCOUNT_CAP), once +# $sidecar_python, $ORG_REPO_ROOT, and fail() are all available -- see that +# derivation's own comment for the incident this replaces. ORCHESTRATOR_GITHUB_ENV="${GITHUB_ENV:-}" sidecar_python="$(command -v python3)" @@ -109,6 +182,8 @@ PYTHONPATH="$ORCHESTRATOR_SOURCE:$ORG_REPO_ROOT" "$sidecar_python" -c \ 'from contextual_orchestrator.credentials import get_credential; from contextual_orchestrator.model_discovery import discover_all_models, free_discovered_models; from contextual_orchestrator.orchestrator import ModelClient, TaskOrchestrator, load_agents; from contextual_orchestrator.review_gateway import register_review_credentials; from contextual_orchestrator.server import SecurityConfig, serve' PYTHONPATH="$ORCHESTRATOR_SOURCE:$ORG_REPO_ROOT" "$sidecar_python" - <<'PY' import http.client +import contextlib +import io import json import threading @@ -145,19 +220,22 @@ thread = threading.Thread(target=server.serve_forever, daemon=True) thread.start() try: connection = http.client.HTTPConnection("127.0.0.1", server.server_address[1], timeout=5) - connection.request( - "POST", - "/v1/chat/completions", - body=b"", - headers={ - "Authorization": "Bearer contract", - "Content-Type": "application/json", - "Content-Length": str(REVIEW_MAX_BODY_BYTES + 1), - }, - ) - response = connection.getresponse() - assert response.status == 413, response.status - response.read() + expected_rejection_log = io.StringIO() + with contextlib.redirect_stderr(expected_rejection_log): + connection.request( + "POST", + "/v1/chat/completions", + body=b"", + headers={ + "Authorization": "Bearer contract", + "Content-Type": "application/json", + "Content-Length": str(REVIEW_MAX_BODY_BYTES + 1), + }, + ) + response = connection.getresponse() + assert response.status == 413, response.status + response.read() + assert "request_failed status=413 code=request_too_large" in expected_rejection_log.getvalue() connection.close() def post_payload(payload): @@ -277,6 +355,78 @@ case "$orchestrator_pool" in ;; esac +# Single source of truth for the startup watchdog below (Devin Review finding +# "Startup watchdog preempts valid preflight", ContextualWisdomLab/.github#1415): +# read the launcher's own coordinated worst-case constant instead of a +# hard-coded shell timeout, so a future change to either discovery's or +# preflight's own budget constants in contextual_orchestrator_review_launcher.py +# cannot silently desynchronize from this watchdog again. The launcher module's +# top-level imports are deliberately stdlib-only (see its module docstring), +# so this works with plain "$ORG_REPO_ROOT" on PYTHONPATH -- no vendored +# dependency needed yet at this point in the script. +sidecar_startup_watchdog_seconds="$( + PYTHONPATH="$ORG_REPO_ROOT" "$sidecar_python" -c \ + 'from scripts.ci.contextual_orchestrator_review_launcher import REVIEW_STARTUP_WATCHDOG_SECONDS; print(REVIEW_STARTUP_WATCHDOG_SECONDS)' +)" || fail "could not derive the startup watchdog seconds from the launcher module" +# Same digit-count defense as REVIEW_PREFLIGHT_GATEWAY_MAX_ATTEMPTS below: a +# non-numeric value would make the "$SECONDS" -ge "$sidecar_startup_watchdog_seconds" +# comparison itself a bash integer-comparison error rather than a controlled +# failure, and an all-digit value can still overflow the shell's integer +# range the same way. Six digits (up to 999999s, over eleven days) is already +# far beyond any realistic startup budget and stays safely representable. +case "$sidecar_startup_watchdog_seconds" in + ''|*[!0-9]*|0) + fail "REVIEW_STARTUP_WATCHDOG_SECONDS must be a positive integer, got: ${sidecar_startup_watchdog_seconds}" ;; + ???????*) + fail "REVIEW_STARTUP_WATCHDOG_SECONDS must be at most 999999" ;; +esac +log "startup watchdog: ${sidecar_startup_watchdog_seconds}s (derived from contextual_orchestrator_review_launcher.py's REVIEW_STARTUP_WATCHDOG_SECONDS)" + +# Single source of truth for the per-account catalog cap default, same +# derive-from-Python pattern as the startup watchdog just above: read +# contextual_orchestrator_review_policy.py's own DEFAULT_ACCOUNT_CAP instead +# of hard-coding a numeric default in this shell script. +# +# FIXED (ContextualWisdomLab/.github#1415, Devin Review follow-up on the +# just-landed contextual_orchestrator_review_launcher.py fix that added +# _catalog_account_cap(DEFAULT_ACCOUNT_CAP)): this shell used to +# unconditionally materialize and export a concrete +# ORCHESTRATOR_CATALOG_ACCOUNT_CAP=8 whenever no operator override was set -- +# a leftover from an earlier round's CATALOG_FAMILY_CAP=24 -> +# CATALOG_ACCOUNT_CAP=8 rename that fixed the variable's NAME but kept the +# WRONG default value. Because the shell always exported a concrete value of +# 8 before the Python launcher ever ran, _catalog_account_cap's own +# fallback-to-DEFAULT_ACCOUNT_CAP branch (os.environ.get(..., str(default))) +# could never actually trigger in production: os.environ.get only falls back +# to its default when the key is ABSENT, and this shell always set it. Every +# real run therefore got a cap of 8, not the policy's intended 4, so two +# NVIDIA credentials could still jointly occupy up to 16 of the 24 preflight +# slots between them (4 * 2 = 8 was the actual bound intended) rather than the +# intended 8 (4 each) -- half the diversification the just-landed fix was +# supposed to restore. An explicit operator-set ORCHESTRATOR_CATALOG_ACCOUNT_CAP +# env var still always wins over this derived default. +if [ -n "${ORCHESTRATOR_CATALOG_ACCOUNT_CAP:-}" ]; then + CATALOG_ACCOUNT_CAP="$ORCHESTRATOR_CATALOG_ACCOUNT_CAP" +else + CATALOG_ACCOUNT_CAP="$( + PYTHONPATH="$ORG_REPO_ROOT" "$sidecar_python" -c \ + 'from scripts.ci.contextual_orchestrator_review_policy import DEFAULT_ACCOUNT_CAP; print(DEFAULT_ACCOUNT_CAP)' + )" || fail "could not derive the default catalog account cap from the policy module" +fi +# Same digit-count defense as REVIEW_PREFLIGHT_GATEWAY_MAX_ATTEMPTS and +# REVIEW_STARTUP_WATCHDOG_SECONDS above: a non-numeric value would make the +# downstream `export`+Python `int(...)` parse fail deep inside the launcher +# instead of this script rejecting bad configuration up front, and an +# all-digit value can still overflow shell/Python integer expectations. Four +# digits (up to 9999) is already far beyond any realistic per-account cap. +case "$CATALOG_ACCOUNT_CAP" in + ''|*[!0-9]*|0) + fail "ORCHESTRATOR_CATALOG_ACCOUNT_CAP must be a positive integer, got: ${CATALOG_ACCOUNT_CAP}" ;; + ?????*) + fail "ORCHESTRATOR_CATALOG_ACCOUNT_CAP must be at most 9999" ;; +esac +log "catalog account cap: ${CATALOG_ACCOUNT_CAP} (operator override, or contextual_orchestrator_review_policy.py's DEFAULT_ACCOUNT_CAP when unset)" + log "starting review sidecar on ${ORCHESTRATOR_HOST}:${ORCHESTRATOR_PORT}" cp "$ORCHESTRATOR_LAUNCHER" "$ORCHESTRATOR_WORK/launch_sidecar.py" export ORCHESTRATOR_CATALOG_LIMIT="$CATALOG_LIMIT" @@ -306,6 +456,8 @@ PYTHONPATH="$ORCHESTRATOR_SOURCE:$ORG_REPO_ROOT" \ --catalog-out "$catalog_file" \ --report-out "$policy_report" \ --preflight-out "$preflight_report" \ + "${launcher_attempt_args[@]}" \ + "${launcher_exclusion_args[@]}" \ "${zdr_args[@]}" \ "${privacy_args[@]}" \ "${pool_args[@]}" \ @@ -329,7 +481,7 @@ cleanup_sidecar_on_error() { } trap cleanup_sidecar_on_error EXIT -i=0 +SECONDS=0 until curl -fsSL --max-time 2 "http://${ORCHESTRATOR_HOST}:${ORCHESTRATOR_PORT}/healthz" >/dev/null 2>&1; do if ! kill -0 "$sidecar_pid" 2>/dev/null; then sidecar_status=0 @@ -354,18 +506,33 @@ until curl -fsSL --max-time 2 "http://${ORCHESTRATOR_HOST}:${ORCHESTRATOR_PORT}/ fi fail "sidecar exited before healthz (status ${sidecar_status}); stderr: $(sed -n '1,20p' "$sidecar_stderr")" fi - i=$((i + 1)) - # KNOWN GAP, tracked as ContextualWisdomLab/.github#1455 (not yet fixed): - # this 180s covers the launcher's ENTIRE startup sequence -- discovery, - # catalog build, AND preflight probing -- not just probing. Layer 1's own - # "160s worst case" comment - # (contextual_orchestrator_review_launcher.py's REVIEW_PREFLIGHT_MAX_ESCALATIONS) - # accounts only for probing; discover_all_models() runs first, inside this - # same 180s, and can itself take up to ~105s worst case (verified against - # the vendored contextual_orchestrator.model_discovery source: ~7 - # sequential HTTP calls at up to 15s each). - if [ "$i" -ge 180 ]; then - fail "sidecar did not become healthy; stderr: $(sed -n '1,20p' "$sidecar_stderr")" + # FIXED (ContextualWisdomLab/.github#1455, Devin Review finding "Startup + # watchdog preempts valid preflight"): this bound covers the launcher's + # ENTIRE startup sequence -- discovery, catalog build, AND preflight + # probing -- not just probing, because none of that work can complete + # (and /healthz cannot respond) until every phase before it has finished in + # the SAME process. $sidecar_startup_watchdog_seconds is derived above from + # contextual_orchestrator_review_launcher.py's own REVIEW_STARTUP_WATCHDOG_SECONDS + # (discovery's real worst case, ~105s, PLUS batched preflight's own real + # worst case, ~120s, PLUS explicit headroom) rather than a bare, previously + # uncoordinated shell constant that only covered probing's own budget by + # coincidence -- see that constant's own module-level comment for the full, + # numbered derivation this single source of truth keeps in sync. + # + # FIXED (ContextualWisdomLab/.github#1415, Devin Review finding "Startup + # watchdog counts polls, not seconds"): the comparison below now reads + # bash's builtin $SECONDS -- reset to 0 immediately before this loop -- + # instead of a hand-incremented poll counter. $SECONDS auto-advances with + # real wall-clock time regardless of what runs inside the loop body, so it + # stays accurate even though each iteration's own cost varies (a `curl + # --max-time 2` call can itself take up to 2s before the trailing `sleep 1` + # even runs). A poll counter incremented once per iteration undercounts + # elapsed time by however long curl actually took, so this bound is now a + # true wall-clock deadline, immune to curl's own per-call timeout cost -- + # not an approximation of one via a poll count that silently assumed every + # iteration costs exactly 1s. + if [ "$SECONDS" -ge "$sidecar_startup_watchdog_seconds" ]; then + fail "sidecar did not become healthy within ${sidecar_startup_watchdog_seconds}s; stderr: $(sed -n '1,20p' "$sidecar_stderr")" fi sleep 1 done @@ -373,7 +540,7 @@ if [ ! -s "$preflight_report" ]; then fail "sidecar became healthy without runtime preflight evidence" fi publish_sidecar_evidence -log "healthz and provider-route preflight confirmed after ${i}s (pid $sidecar_pid)" +log "healthz and provider-route preflight confirmed after ${SECONDS}s (pid $sidecar_pid)" # A successful startup never re-reads $sidecar_stderr otherwise: only the # failure branches above embed it in their ::error:: message. A partial, # non-fatal provider discovery failure (e.g. one bad credential) would @@ -419,23 +586,16 @@ gateway_virtual_model="orchestrator/${orchestrator_pool}" # docs/product-technical-gap-baseline.md for the evidence that is actually # captured (downloaded strix-reports artifact, # ContextualWisdomLab/contextual-orchestrator#912 run 33304076516). -printf '{"model":"%s","messages":[{"role":"system","content":"You are a helpful assistant."},{"role":"user","content":"Reply with just '\''OK'\''."}],"temperature":1.0,"max_tokens":4096,"stream":false}\n' \ +# "orchestration":"route" pins this smoke request to the direct virtual-pool +# dispatch path rather than the gateway's own auto-mode triage, so a failure +# here is unambiguously the orchestrator/free route itself, not a triage +# decision layered on top of it. +printf '{"model":"%s","orchestration":"route","messages":[{"role":"system","content":"You are a helpful assistant."},{"role":"user","content":"Reply with just '\''OK'\''."}],"temperature":1.0,"max_tokens":4096,"stream":false}\n' \ "$gateway_virtual_model" > "$gateway_preflight_request" -# 30s (this check's previous bound) is too tight for a real completion from a -# reasoning-capable free-tier model: exact-evidence reproduction (Strix run -# 33306775025 on ContextualWisdomLab/contextual-orchestrator#921, job -# 99244624298) shows the routing probe marking a DeepSeek NIM route "ready" -# in 18s, then this identical request against that same healthy route being -# cut off by curl's own timeout at exactly 30.0s -- "gateway preflight -# request could not reach the local sidecar" is this curl failure, not an -# actual connectivity problem. This required-workflow job already budgets -# 120 minutes (see timeout-minutes in strix.yml/noema-review.yml), and the -# org's own stated policy accepts multi-hour central review latency in -# favor of accuracy over speed -- a 30s bound on one preflight self-check -# contradicted that policy and rejected a route the routing probe had just -# proven healthy. 120s keeps this a bounded, fail-closed check while giving -# a real reasoning generation room to finish. This value is deliberately kept -# unchanged by ADR-0005 -- shortening it would regress the fix just described. +# Keep each virtual-pool smoke attempt long enough for reasoning models, but +# bounded so all attempts plus the caller's real workload fit its job ceiling. +# The shared default is 20 minutes; callers with tighter startup reservations +# (notably Noema's 15-minute provision window) pass a smaller explicit value. # # ADR-0005 Trigger A: this request goes to the virtual pool, not one pinned # candidate, so a transport failure or non-2xx status here (unreachable @@ -457,7 +617,11 @@ printf '{"model":"%s","messages":[{"role":"system","content":"You are a helpful # ADR-0005; verified directly against contextual-orchestrator's server.py, # which exposes no parameter to exclude or deprioritize a specific candidate # on a retry). -REVIEW_PREFLIGHT_GATEWAY_MAX_ATTEMPTS="${REVIEW_PREFLIGHT_GATEWAY_MAX_ATTEMPTS:-3}" +if [ "${launcher_attempt_args[*]:-}" = "--single-candidate-attempt" ]; then + REVIEW_PREFLIGHT_GATEWAY_MAX_ATTEMPTS="${REVIEW_PREFLIGHT_GATEWAY_MAX_ATTEMPTS:-1}" +else + REVIEW_PREFLIGHT_GATEWAY_MAX_ATTEMPTS="${REVIEW_PREFLIGHT_GATEWAY_MAX_ATTEMPTS:-3}" +fi # A malformed override (non-numeric, empty, or zero) must fail closed instead # of silently disabling the bound: `[ "$gateway_attempt" -ge "$X" ]` with a # non-integer `$X` is itself a bash integer-comparison error, not a false @@ -477,11 +641,18 @@ case "$REVIEW_PREFLIGHT_GATEWAY_MAX_ATTEMPTS" in ?????*) fail "REVIEW_PREFLIGHT_GATEWAY_MAX_ATTEMPTS must be at most 9999" ;; esac +REVIEW_PREFLIGHT_GATEWAY_MAX_TIME_SECONDS="${REVIEW_PREFLIGHT_GATEWAY_MAX_TIME_SECONDS:-1200}" +case "$REVIEW_PREFLIGHT_GATEWAY_MAX_TIME_SECONDS" in + ''|*[!0-9]*|0) + fail "REVIEW_PREFLIGHT_GATEWAY_MAX_TIME_SECONDS must be a positive integer" ;; + ???????*) + fail "REVIEW_PREFLIGHT_GATEWAY_MAX_TIME_SECONDS must be at most 999999" ;; +esac gateway_attempt=1 gateway_http_status="" while :; do if gateway_http_status="$( - curl -sS --max-time 120 \ + curl -sS --connect-timeout 10 --max-time "$REVIEW_PREFLIGHT_GATEWAY_MAX_TIME_SECONDS" \ -o "$gateway_preflight_response" \ -w '%{http_code}' \ -X POST \ diff --git a/scripts/ci/noema_review_gate.py b/scripts/ci/noema_review_gate.py index d7ef15a2e0..14e96a41eb 100644 --- a/scripts/ci/noema_review_gate.py +++ b/scripts/ci/noema_review_gate.py @@ -6,18 +6,23 @@ import argparse import ast import base64 +import contextlib import hashlib import ipaddress import json import os import re +import signal import socket import subprocess import sys +import threading +import time import urllib.error import urllib.parse import urllib.request from collections.abc import Sequence +from pathlib import Path from typing import Any from scripts.ci.opencode_review_normalize_output import changed_file_is_material @@ -33,12 +38,34 @@ MAX_FILE_CONTEXT_CHARS = 4000 MAX_REVIEW_CONTEXT_CHARS = 24000 MAX_THREAD_BODY_CHARS = 1200 -NOEMA_LLM_TIMEOUT_SECONDS = 4 * 60 * 60 +# The sidecar may spend 150 minutes on the selected reviewer and another 150 +# minutes on its preserved realtime judge. Keep 30 minutes for handoff and +# transport overhead while remaining below GitHub's 360-minute job ceiling. +CALL_LLM_TIMEOUT_SECONDS = 19800 DIFF_HUNK_RE = re.compile(r"^@@ -(\d+)(?:,\d+)? \+(\d+)(?:,\d+)? @@") ORCHESTRATOR_LOOPBACK_HOSTS = frozenset({"127.0.0.1", "::1"}) ORCHESTRATOR_BASE_ENV = "CONTEXTUAL_ORCHESTRATOR_BASE_URL" + +@contextlib.contextmanager +def absolute_response_deadline(seconds: int): + """Bound the complete streamed response, not each socket operation.""" + if not hasattr(signal, "setitimer") or threading.current_thread() is not threading.main_thread(): + raise RuntimeError("Noema absolute response deadline is unavailable") + previous_handler = signal.getsignal(signal.SIGALRM) + + def expire(_signum: int, _frame: object) -> None: + raise TimeoutError(f"Noema LLM response exceeded {seconds} seconds") + + signal.signal(signal.SIGALRM, expire) + previous_timer = signal.setitimer(signal.ITIMER_REAL, seconds) + try: + yield + finally: + signal.setitimer(signal.ITIMER_REAL, *previous_timer) + signal.signal(signal.SIGALRM, previous_handler) + # ⚡ Bolt: Pre-compiled regex patterns to avoid recompilation on every scrub_sensitive_data call. # Impact: Improves string processing performance in error reporting. SENSITIVE_DATA_SCRUB_PATTERNS = ( @@ -891,10 +918,11 @@ def call_llm( pr: dict[str, Any], diff: str, truncated: bool, - expected_head: str, review_context: str = "", changed_paths: Sequence[str] = (), repair_error: str = "", + _response_deadline: float | None = None, + expected_head: str | None = None, ) -> dict[str, Any]: """Call the configured OpenAI-compatible LLM endpoint for a review verdict. @@ -909,6 +937,10 @@ def call_llm( lookup and ``StaleHeadDuringRepairRetryError`` for how that stale condition is reported distinctly to the caller. """ + head_bound = expected_head is not None + expected_head = expected_head or str(pr.get("headRefOid") or "") + if _response_deadline is None: + _response_deadline = time.monotonic() + CALL_LLM_TIMEOUT_SECONDS api_url = os.environ.get("NOEMA_LLM_API_URL", "").strip() api_key = os.environ.get("NOEMA_LLM_API_KEY", "").strip() model = os.environ.get("NOEMA_LLM_MODEL", "").strip() or "noema-default" @@ -954,6 +986,20 @@ def call_llm( prompt, ], } + if is_allowed_orchestrator_sidecar_url(api_url): + payload["orchestration"] = "route" + candidate_id = os.environ.get("NOEMA_LLM_CANDIDATE_ID", "").strip() + excluded = [ + value.strip() + for value in os.environ.get("NOEMA_LLM_EXCLUDE_CANDIDATE_IDS", "").split(",") + if value.strip() + ] + if candidate_id or excluded: + payload["routing"] = {} + if candidate_id: + payload["routing"]["candidate_id"] = candidate_id + if excluded: + payload["routing"]["exclude_candidate_ids"] = excluded request = urllib.request.Request( api_url, data=json.dumps(payload).encode("utf-8"), @@ -964,8 +1010,12 @@ def call_llm( method="POST", ) opener = urllib.request.build_opener(NoRedirectHandler()) - with opener.open(request, timeout=NOEMA_LLM_TIMEOUT_SECONDS) as response: # nosec B310 - raw_bytes = response.read() + request_timeout = _response_deadline - time.monotonic() + if request_timeout <= 0: + raise TimeoutError("Noema LLM response exceeded the shared response deadline") + with absolute_response_deadline(request_timeout): + with opener.open(request, timeout=request_timeout) as response: # nosec B310 + raw_bytes = response.read() try: raw = decode_llm_response_body(raw_bytes) content = extract_llm_message_content(raw) @@ -997,7 +1047,7 @@ def call_llm( except RuntimeError as exc: if repair_error: raise - if str(fetch_pr(repo, number).get("headRefOid") or "").lower() != expected_head: + if head_bound and str(fetch_pr(repo, number).get("headRefOid") or "").lower() != expected_head: raise StaleHeadDuringRepairRetryError( "Pull request head changed during review; stale before repair retry." ) from exc @@ -1007,10 +1057,11 @@ def call_llm( pr, diff, truncated, - expected_head, review_context, changed_paths, str(exc), + _response_deadline, + expected_head, ) return verdict @@ -1095,7 +1146,7 @@ def submit_review(repo: str, number: int, pr: dict[str, Any], actor: str, verdic print(f"Noema {event} review submitted for {repo}#{number} at {head_sha}.") -def inspect_and_review(repo: str, number: int, expected_head: str) -> int: +def inspect_and_review(repo: str, number: int, expected_head: str | None = None) -> int: """Inspect PR state and submit Noema's independent LLM review. ``expected_head`` is normalized defensively before the stale-head @@ -1104,8 +1155,8 @@ def inspect_and_review(repo: str, number: int, expected_head: str) -> int: workflow require canonical lowercase SHA input so equivalent casing cannot split the workflow concurrency group. """ - expected_head = expected_head.strip().lower() pr = fetch_pr(repo, number) + expected_head = (expected_head or str(pr.get("headRefOid") or "")).strip().lower() if str(pr.get("headRefOid") or "").lower() != expected_head: print("Trigger head is stale; Noema review skipped before model work.") return 0 @@ -1127,7 +1178,16 @@ def inspect_and_review(repo: str, number: int, expected_head: str) -> int: changed_paths = fetch_changed_file_paths(repo, number) review_context = build_review_context(repo, number, pr) try: - verdict = call_llm(repo, number, pr, diff, truncated, expected_head, review_context, changed_paths) + verdict = call_llm( + repo, + number, + pr, + diff, + truncated, + review_context, + changed_paths, + expected_head=expected_head, + ) except StaleHeadDuringRepairRetryError: print("Pull request head changed during review; Noema review skipped before repair retry.") return 0 @@ -1139,12 +1199,108 @@ def inspect_and_review(repo: str, number: int, expected_head: str) -> int: return 0 +def _write_sealed(path: str, payload: dict[str, Any]) -> None: + encoded = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode() + with open(path, "wb") as stream: + stream.write(encoded) + with open(f"{path}.sha256", "w", encoding="ascii") as stream: + stream.write(f"{hashlib.sha256(encoded).hexdigest()}\n") + + +def _read_sealed(path: str) -> dict[str, Any]: + with open(path, "rb") as stream: + encoded = stream.read() + with open(f"{path}.sha256", encoding="ascii") as stream: + expected = stream.read().strip() + if not re.fullmatch(r"[0-9a-f]{64}", expected) or hashlib.sha256(encoded).hexdigest() != expected: + raise RuntimeError("Noema handoff artifact digest mismatch") + payload = json.loads(encoded) + if not isinstance(payload, dict): + raise RuntimeError("Noema handoff artifact must contain a JSON object") + return payload + + +def prepare_review(repo: str, number: int, output: str, expected_head: str) -> int: + """Seal immutable review input without calling a model or writing GitHub.""" + expected_head = expected_head.strip().lower() + if not re.fullmatch(r"[0-9a-f]{40}", expected_head): + raise RuntimeError("Noema prepare requires a canonical lowercase exact head SHA") + pr = fetch_pr(repo, number) + if str(pr.get("headRefOid") or "").lower() != expected_head: + raise RuntimeError("Noema prepare refused a stale trigger head") + actor = current_actor() + if not actor: + raise RuntimeError("Noema reviewer identity could not be verified") + if actor in PRIMARY_REVIEW_AUTHORS: + raise RuntimeError("Noema requires a verified independent reviewer credential") + if pr.get("isDraft"): + print("PR is draft; Noema review skipped.") + return 0 + if existing_noema_review(pr, actor): + print("Current head already has a Noema review; nothing to do.") + return 0 + diff, truncated = fetch_diff(repo, number) + changed_paths = fetch_changed_file_paths(repo, number) + _write_sealed(output, { + "repo": repo, + "number": number, + "head_sha": expected_head, + "pr": pr, + "diff": diff, + "truncated": truncated, + "changed_paths": changed_paths, + "review_context": build_review_context(repo, number, pr), + }) + return 0 + + +def evaluate_review(input_path: str, output: str) -> int: + """Evaluate one sealed current-head input and seal the model verdict.""" + prepared = _read_sealed(input_path) + repo, number = str(prepared["repo"]), int(prepared["number"]) + verdict = call_llm( + repo, number, prepared["pr"], prepared["diff"], bool(prepared["truncated"]), + str(prepared.get("review_context") or ""), prepared.get("changed_paths") or (), + ) + _write_sealed(output, { + "repo": repo, + "number": number, + "head_sha": prepared["head_sha"], + "input_sha256": hashlib.sha256(Path(input_path).read_bytes()).hexdigest(), + "candidate_id": os.environ.get("NOEMA_LLM_CANDIDATE_ID", "").strip(), + "verdict": verdict, + }) + return 0 + + +def finalize_review(input_path: str, verdict_path: str) -> int: + """Submit only a sealed verdict bound to the sealed input and live head.""" + prepared, result = _read_sealed(input_path), _read_sealed(verdict_path) + input_digest = hashlib.sha256(Path(input_path).read_bytes()).hexdigest() + if result.get("input_sha256") != input_digest or result.get("head_sha") != prepared.get("head_sha"): + raise RuntimeError("Noema verdict artifact is not bound to the prepared input") + repo, number = str(prepared["repo"]), int(prepared["number"]) + pr = fetch_pr(repo, number) + if pr.get("headRefOid") != prepared.get("head_sha"): + raise RuntimeError("Noema verdict artifact is stale for the current pull request head") + actor = current_actor() + if not actor or actor in PRIMARY_REVIEW_AUTHORS: + raise RuntimeError("Noema requires a verified independent reviewer credential") + if not existing_noema_review(pr, actor): + submit_review(repo, number, pr, actor, result["verdict"]) + return 0 + + def parse_args(argv: list[str]) -> argparse.Namespace: """Parse Noema review gate command-line arguments.""" parser = argparse.ArgumentParser() parser.add_argument("--repo", required=True) parser.add_argument("--pr-number", required=True, type=int) - parser.add_argument("--expected-head", required=True) + parser.add_argument("--mode", choices=("review", "prepare", "evaluate", "finalize"), default="review") + parser.add_argument("--input") + parser.add_argument("--verdict") + parser.add_argument("--output") + parser.add_argument("--expected-head") return parser.parse_args(argv) @@ -1153,10 +1309,16 @@ def main(argv: list[str]) -> int: args = parse_args(argv) if args.pr_number <= 0: raise SystemExit("--pr-number must be positive") - if not re.fullmatch(r"[0-9a-f]{40}", args.expected_head): - raise SystemExit( - "--expected-head must be a canonical lowercase 40-character Git SHA" - ) + if args.mode == "prepare" and args.output and args.expected_head: + return prepare_review(args.repo, args.pr_number, args.output, args.expected_head) + if args.mode == "evaluate" and args.input and args.output: + return evaluate_review(args.input, args.output) + if args.mode == "finalize" and args.input and args.verdict: + return finalize_review(args.input, args.verdict) + if args.mode != "review": + raise SystemExit("selected mode requires its artifact path arguments") + if args.expected_head is None: + return inspect_and_review(args.repo, args.pr_number) return inspect_and_review(args.repo, args.pr_number, args.expected_head) diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index d08c2cdd9e..e7fd27531f 100644 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -296,7 +296,7 @@ assert_strix_workflow_pr_trigger_hardened() { assert_file_contains "$workflow_file" "Provision contextual-orchestrator Strix sidecar" "strix workflow provisions the central contextual-orchestrator sidecar" assert_file_contains "$workflow_file" "CONTEXTUAL_ORCHESTRATOR_BASE_URL" "strix workflow uses the sidecar base URL" assert_file_contains "$workflow_file" "CONTEXTUAL_ORCHESTRATOR_TOKEN" "strix workflow uses the sidecar token" - assert_file_contains "$workflow_file" "timeout-minutes: 200" "strix workflow job budget preserves multi-hour scans and artifact publication margin" + assert_file_contains "$workflow_file" "timeout-minutes: 360" "strix workflow job budget preserves multi-hour scans and artifact publication margin" assert_file_contains "$workflow_file" "timeout-minutes: 170" "strix workflow scan step permits legitimate 150-minute repository reviews" assert_file_contains "$workflow_file" 'budget_suffix="TIME""OUT"' "strix workflow builds budget env keys without visible timeout signal text" assert_file_contains "$workflow_file" 'export "STRIX_TOTAL_${budget_suffix}_SECONDS=9300"' "strix workflow preserves a 155-minute bounded total Strix budget" diff --git a/tests/test_contextual_orchestrator_review_runtime_preflight.py b/tests/test_contextual_orchestrator_review_runtime_preflight.py index 7093e8a3d0..ac025503aa 100644 --- a/tests/test_contextual_orchestrator_review_runtime_preflight.py +++ b/tests/test_contextual_orchestrator_review_runtime_preflight.py @@ -11,6 +11,8 @@ from pathlib import Path import subprocess import sys +import threading +import time from types import SimpleNamespace import pytest @@ -20,7 +22,9 @@ _REPO_ROOT = Path(__file__).resolve().parents[1] _LAUNCHER = _REPO_ROOT / "scripts/ci/contextual_orchestrator_review_launcher.py" _SIDECAR = _REPO_ROOT / "scripts/ci/contextual_orchestrator_review_sidecar.sh" -_SANITIZER = _REPO_ROOT / "scripts/ci/sanitize_contextual_orchestrator_sidecar_stream.py" +_SANITIZER = ( + _REPO_ROOT / "scripts/ci/sanitize_contextual_orchestrator_sidecar_stream.py" +) class _ProbeClient: @@ -112,6 +116,26 @@ def test_routable_discovered_models_excludes_evidence_only_rows() -> None: assert routable([]) == [] +def test_fallback_exclusion_reaches_a_later_healthy_preflight_batch() -> None: + namespace = _load_launcher() + exclude = namespace["_without_excluded_agents"] + preflight = namespace["_preflight_review_agent_batches"] + batch_size = namespace["REVIEW_PREFLIGHT_BATCH_SIZE"] + catalog = [{"id": "attempted"}] + [ + {"id": f"failed-{index}"} for index in range(batch_size) + ] + [{"id": "later-healthy"}] + filtered = exclude(catalog, frozenset({"attempted"})) + agents = [SimpleNamespace(id=row["id"], provider_name="openrouter", model="x/free") for row in filtered] + outcomes = {agent.id: RuntimeError("unavailable") for agent in agents} + outcomes["later-healthy"] = _openai_text("ready") + + viable, report = preflight(agents, client=_ProbeClient(outcomes)) + + assert [agent.id for agent in viable] == ["later-healthy"] + assert report["probed_count"] == batch_size + 1 + assert all(route["agent_id"] != "attempted" for route in report["routes"]) + + def test_log_discovery_errors_prints_one_bounded_line_per_provider_failure( capsys: pytest.CaptureFixture[str], ) -> None: @@ -247,13 +271,19 @@ def test_preflight_mirrors_runtime_request_and_keeps_only_compatible_routes() -> assert secret not in repr(report) # Regression for Devin Review's successful-probes-omit-diagnostics - # finding: the ordinary, most-common outcome (an immediate base-probe - # success, no escalation needed) must still populate finish_reason and - # reasoning_without_content -- not just failure/escalation outcomes -- - # so there is a real "normal" baseline to compare future telemetry - # against. + # finding: the ordinary, most-common outcome (a base-probe success, + # confirmed at the real serving budget -- see below) must still populate + # finish_reason and reasoning_without_content -- not just + # failure/escalation outcomes -- so there is a real "normal" baseline to + # compare future telemetry against. ready_row = report["routes"][2] assert ready_row["status"] == "ready" + assert ready_row["attempts"] == 2 + # ContextualWisdomLab/.github#1454 fix: a base-probe success alone is not + # admission -- it must also be confirmed at the real serving budget + # (REVIEW_PREFLIGHT_ESCALATED_TOKENS) before this route is marked ready. + assert ready_row["confirmed_at_serving_budget"] is True + assert "escalated" not in ready_row assert ready_row["finish_reason"] == "unknown" assert ready_row["reasoning_without_content"] is False @@ -261,7 +291,6 @@ def test_preflight_mirrors_runtime_request_and_keeps_only_compatible_routes() -> assert endpoint == "chat/completions" assert payload["model"] == agent.model assert payload["stream"] is False - assert payload["max_tokens"] == 16 assert payload["temperature"] == 1.0 assert payload["messages"] == [ {"role": "system", "content": "You are a helpful assistant."}, @@ -269,6 +298,16 @@ def test_preflight_mirrors_runtime_request_and_keeps_only_compatible_routes() -> ] assert "tools" not in payload + # The rejected and malformed routes each make exactly one call (base + # budget); the ready route makes two -- its base probe, then the + # mandatory confirmation at the real serving budget. + assert [payload["max_tokens"] for _, _, payload in client.calls] == [ + namespace["REVIEW_PREFLIGHT_BASE_TOKENS"], + namespace["REVIEW_PREFLIGHT_BASE_TOKENS"], + namespace["REVIEW_PREFLIGHT_BASE_TOKENS"], + namespace["REVIEW_PREFLIGHT_ESCALATED_TOKENS"], + ] + def test_log_preflight_rejections_prints_bounded_summary_to_stderr( capsys: pytest.CaptureFixture[str], @@ -397,19 +436,20 @@ def test_gateway_preflight_max_tokens_is_synchronized_with_the_routing_probe() - reasoning tokens, so the gateway rejected a route its own routing probe had just proven healthy. - Since ADR-0005 (this PR), most routes now prove readiness at the much - cheaper ``REVIEW_PREFLIGHT_BASE_TOKENS`` (16) instead -- `4096` is used - by the routing probe only on the ESCALATED retry (a candidate that - failed the cheap probe with a budget-too-small signature) and, always, - by the real serving `ModelClient` for actual review traffic (see - `ContextualWisdomLab/.github#1454` for the resulting known gap: an - ordinary base-probe success is never itself confirmed at this budget). - This test's own assertion is unaffected by that: Layer 2 never - escalates (ADR-0005 Decision SS1) and always uses the real serving + Since ADR-0005 (this PR), most routes first prove liveness at the much + cheaper ``REVIEW_PREFLIGHT_BASE_TOKENS`` (16) -- `4096` is then used by + the routing probe's second attempt for every admitted route, always: to + rescue a candidate that failed the cheap probe with a budget-too-small + signature, AND (fixed as `ContextualWisdomLab/.github#1454`, Devin + Review, "Serving-incompatible routes pass startup") to confirm a + candidate whose cheap probe already succeeded, before that route is ever + marked ready -- and, always, by the real serving `ModelClient` for actual + review traffic. This test's own assertion is unaffected by that: Layer 2 + never escalates (ADR-0005 Decision SS1) and always uses the real serving budget, so its literal must still equal `REVIEW_MAX_OUTPUT_TOKENS` exactly, for the same reason as before -- a smaller Layer 2 budget can - still reject a route the routing probe (at either of its own budgets) - already proved ready. + still reject a route the routing probe (which now confirms every + admitted route at this same real serving budget) already proved ready. """ namespace = _load_launcher() review_max_output_tokens = namespace["REVIEW_MAX_OUTPUT_TOKENS"] @@ -430,33 +470,19 @@ def test_gateway_preflight_max_tokens_is_synchronized_with_the_routing_probe() - ) -def test_gateway_preflight_curl_timeout_tolerates_real_reasoning_latency() -> None: - """The end-to-end gateway check's curl timeout must not undercut real completion latency. - - Regression for the 2026-08-30 gateway-preflight-timeout incident: exact- - evidence reproduction (Strix run 33306775025 on - ContextualWisdomLab/contextual-orchestrator#921, job 99244624298) showed - the routing probe marking a DeepSeek NIM route "ready" in 18s, then the - identical gateway request against that same healthy route being cut off - at exactly curl's configured bound -- "gateway preflight request could - not reach the local sidecar" was that timeout, not a real connectivity - failure. This asserts the bound is generous enough to tolerate a real - reasoning generation (well above the routing probe's own 10s - per-candidate budget) rather than the previous 30s, which rejected a - route the routing probe had just proven healthy. - """ +def test_gateway_preflight_uses_caller_bound_instead_of_120_seconds() -> None: + """Each attempt permits reasoning latency without starving real work.""" sidecar = _SIDECAR.read_text(encoding="utf-8") - match = re.search(r"curl -sS --max-time (\d+) \\\n\s*-o \"\$gateway_preflight_response\"", sidecar) - assert match, "sidecar must send the gateway preflight request with an explicit curl --max-time" - gateway_preflight_timeout_seconds = int(match.group(1)) - - assert gateway_preflight_timeout_seconds >= 120, ( - "gateway preflight curl --max-time " - f"({gateway_preflight_timeout_seconds}s) must tolerate real reasoning-model " - "completion latency; 30s was observed cutting off a route the routing probe " - "had just proven ready" - ) + command = re.search(r"curl -sS .*?\n\s*-o \"\$gateway_preflight_response\"", sidecar) + assert command + assert "--connect-timeout 10" in command.group(0) + assert '--max-time "$REVIEW_PREFLIGHT_GATEWAY_MAX_TIME_SECONDS"' in command.group(0) + assert 'REVIEW_PREFLIGHT_GATEWAY_MAX_TIME_SECONDS:-1200' in sidecar + assert "--max-time 3600" not in command.group(0) + assert "--max-time 120" not in command.group(0) + assert 'launcher_attempt_args[*]:-}" = "--single-candidate-attempt"' in sidecar + assert 'REVIEW_PREFLIGHT_GATEWAY_MAX_ATTEMPTS:-1' in sidecar def test_gateway_preflight_retries_transport_failures_up_to_a_bounded_attempt_count() -> None: @@ -993,7 +1019,8 @@ def test_base_probe_success_with_reasoning_and_content_is_never_flagged_as_starv response that ALSO discloses a reasoning trace alongside real content must never be recorded as ``reasoning_without_content: True`` -- that would falsely pollute the evidence this preflight exists to produce, on - the single most common outcome (an immediate base-probe success). + the single most common outcome (a base-probe success, confirmed at the + real serving budget per the ContextualWisdomLab/.github#1454 fix). """ namespace = _load_launcher() preflight = namespace["_preflight_review_agents"] @@ -1022,9 +1049,17 @@ def test_base_probe_success_with_reasoning_and_content_is_never_flagged_as_starv assert viable == [transparent_reasoner] row = report["routes"][0] assert row["status"] == "ready" - assert row["attempts"] == 1 + # Two attempts: the base probe (16 tokens) plus the mandatory + # confirmation at the real serving budget (ContextualWisdomLab/.github#1454). + assert row["attempts"] == 2 + assert row["confirmed_at_serving_budget"] is True + assert "escalated" not in row assert row["finish_reason"] == "stop" assert row["reasoning_without_content"] is False + assert [call[2]["max_tokens"] for call in client.calls] == [ + namespace["REVIEW_PREFLIGHT_BASE_TOKENS"], + namespace["REVIEW_PREFLIGHT_ESCALATED_TOKENS"], + ] def test_finish_reason_length_escalates_and_can_succeed() -> None: @@ -1076,6 +1111,165 @@ def test_finish_reason_length_escalates_and_can_succeed() -> None: assert report["escalations_used"] == 1 +def test_base_probe_success_not_admitted_when_serving_budget_probe_returns_empty() -> None: + """Regression for Devin Review's "Serving-incompatible routes pass + startup" finding (`ContextualWisdomLab/.github#1454`): a route that + succeeds at the cheap `REVIEW_PREFLIGHT_BASE_TOKENS` probe but returns + empty content at the real `REVIEW_PREFLIGHT_ESCALATED_TOKENS` serving + budget must NOT be marked ready -- admission requires success at the + actual serving-equivalent token budget, not merely at the escalation + sequence's first rung. Before this fix, `_build_model_client` would go + on to serve real reviews at `REVIEW_MAX_OUTPUT_TOKENS` against a route + this preflight had already (wrongly) admitted. + """ + namespace = _load_launcher() + preflight = namespace["_preflight_review_agents"] + + serving_incompatible = SimpleNamespace( + id="nvidia_nim_serving_incompatible", provider_name="nvidia_nim", model="narrow/free" + ) + client = _SequencedClient( + [ + _openai_text("OK"), + {"choices": [{"finish_reason": "length", "message": {"content": ""}}]}, + ] + ) + + with pytest.raises(namespace["ReviewPreflightError"], match="no provider route passed"): + preflight([serving_incompatible], client=client) + + assert [call[2]["max_tokens"] for call in client.calls] == [ + namespace["REVIEW_PREFLIGHT_BASE_TOKENS"], + namespace["REVIEW_PREFLIGHT_ESCALATED_TOKENS"], + ] + + +def test_base_probe_success_not_admitted_when_serving_budget_probe_raises() -> None: + """The sibling shape of the same Devin Review finding: the route's + confirming probe at the real serving budget doesn't just come back + empty, it is rejected outright (a provider whose real completion-token + ceiling sits strictly between the base and serving budgets, exactly the + axis ADR-0005's own Research already documented). Must still fail + closed, not admit the route on the strength of the earlier, smaller + success. + """ + namespace = _load_launcher() + preflight = namespace["_preflight_review_agents"] + + narrow_ceiling = SimpleNamespace( + id="nvidia_nim_narrow_ceiling", provider_name="nvidia_nim", model="narrow/free" + ) + client = _SequencedClient( + [ + _openai_text("OK"), + RuntimeError("provider rejected the request: max_tokens exceeds model ceiling"), + ] + ) + + with pytest.raises(namespace["ReviewPreflightError"]) as failure: + preflight([narrow_ceiling], client=client) + + row = failure.value.report["routes"][0] + assert row["status"] == "rejected" + assert row["error_type"] == "RuntimeError" + assert row["attempts"] == 2 + assert "escalated" not in row + assert "confirmed_at_serving_budget" not in row + + +def test_base_probe_success_confirmation_has_its_own_dedicated_budget() -> None: + """Regression for Devin Review's "Later healthy routes cannot start" + finding (`ContextualWisdomLab/.github#1415`): a base-probe success's + mandatory confirmation used to draw from the SAME shared + ``REVIEW_PREFLIGHT_MAX_ESCALATIONS`` counter a budget-too-small + escalation would -- a counter sized (4) for the RARE rescue case, not + the common "confirm every success" case. Confirmation now draws from its + own separate ``REVIEW_PREFLIGHT_MAX_CONFIRMATIONS`` budget, so + exhausting the (still small, still bounded) escalation budget on + genuinely failed candidates must never deny a later, unrelated + candidate's confirmation. + """ + namespace = _load_launcher() + preflight = namespace["_preflight_review_agents"] + max_escalations = namespace["REVIEW_PREFLIGHT_MAX_ESCALATIONS"] + max_confirmations = namespace["REVIEW_PREFLIGHT_MAX_CONFIRMATIONS"] + assert max_confirmations > max_escalations, ( + "the confirmation budget must be dedicated and large enough to cover " + "every candidate this preflight run can ever probe -- not merely " + "equal to the small, deliberately scarce rescue budget" + ) + + # Exhaust the ESCALATION (rescue) budget entirely on candidates that + # fail their base probe with a "budget too small" signature and then + # (deliberately, in this test) fail their rescue attempt too, the same + # way every time -- these never touch the confirmation budget at all, + # they only need to fully spend the escalation budget's slots. + length_response = {"choices": [{"finish_reason": "length", "message": {"content": ""}}]} + escalation_budget_users = [ + SimpleNamespace(id=f"escalation_user_{index}", provider_name="openrouter", model="x/free") + for index in range(max_escalations) + ] + # A base-probe SUCCESS needing only confirmation -- must not be blocked + # by the escalation budget above being fully spent. + confirmed = SimpleNamespace( + id="confirmed_despite_escalation_exhaustion", + provider_name="openrouter", + model="x/free", + ) + client = _ProbeClient( + {agent.id: dict(length_response) for agent in escalation_budget_users} + | {confirmed.id: _openai_text("OK")} + ) + + viable, report = preflight([*escalation_budget_users, confirmed], client=client) + + assert viable == [confirmed] + assert report["escalations_used"] == max_escalations + assert report["confirmations_used"] == 1 + for row in report["routes"][:-1]: + assert row["status"] == "rejected" + assert row["error_type"] == "invalid_chat_response" + confirmed_row = report["routes"][-1] + assert confirmed_row["status"] == "ready" + assert confirmed_row["confirmed_at_serving_budget"] is True + + +def test_confirmation_budget_is_bounded_not_unbounded() -> None: + """The confirmation budget is dedicated, not shared -- but it is still a + real, finite cap (``REVIEW_PREFLIGHT_MAX_CONFIRMATIONS``), never an + unbounded allowance that would reintroduce an uncomputed worst case. + Exhausting it is recorded with its own distinct + ``confirmation_budget_exhausted`` classification, never conflated with + the separate ``escalation_budget_exhausted`` outcome. + """ + namespace = _load_launcher() + preflight = namespace["_preflight_review_agents"] + max_confirmations = namespace["REVIEW_PREFLIGHT_MAX_CONFIRMATIONS"] + + agents = [ + SimpleNamespace(id=f"confirmed_{index}", provider_name="openrouter", model="x/free") + for index in range(max_confirmations) + ] + exhausted = SimpleNamespace( + id="confirmation_exhausted", provider_name="openrouter", model="x/free" + ) + client = _ProbeClient( + {agent.id: _openai_text("OK") for agent in agents} + | {exhausted.id: _openai_text("OK")} + ) + + viable, report = preflight([*agents, exhausted], client=client) + + assert viable == agents + exhausted_row = report["routes"][-1] + assert exhausted_row["status"] == "rejected" + assert exhausted_row["error_type"] == "confirmation_budget_exhausted" + assert exhausted_row["attempts"] == 1 + assert report["confirmations_used"] == max_confirmations + assert report["escalations_used"] == 0 + assert len(client.calls) == max_confirmations * 2 + 1 + + def test_escalation_budget_is_shared_and_bounded_across_candidates() -> None: """Once ``REVIEW_PREFLIGHT_MAX_ESCALATIONS`` is spent, a further candidate that would otherwise qualify is rejected immediately, without a second @@ -1325,7 +1519,9 @@ def test_preflight_fails_closed_when_every_route_rejects() -> None: preflight = namespace.get("_preflight_review_agents") error_type = namespace.get("ReviewPreflightError") assert callable(preflight), "launcher must expose provider-route preflight" - assert isinstance(error_type, type), "launcher must expose a typed preflight failure" + assert isinstance(error_type, type), ( + "launcher must expose a typed preflight failure" + ) agent = SimpleNamespace( id="openrouter_rejected", provider_name="openrouter", model="rejected/free" @@ -1350,15 +1546,16 @@ def test_preflight_uses_priced_fallback_only_after_primary_routes_reject() -> No {primary.id: TimeoutError("unavailable"), fallback.id: _openai_text("OK")} ) - viable, report, fallback_used = preflight( - [primary], [fallback], client=client - ) + viable, report, fallback_used = preflight([primary], [fallback], client=client) assert viable == [fallback] assert fallback_used is True assert report["fallback_reason"] == "primary_routes_unavailable" assert report["primary_attempt"]["ready_count"] == 0 - assert [call[0] for call in client.calls] == [primary, fallback] + # The fallback route's base probe succeeds and is then confirmed at the + # real serving budget (ContextualWisdomLab/.github#1454) before being + # admitted, so it makes two calls. + assert [call[0] for call in client.calls] == [primary, fallback, fallback] ready_client = _ProbeClient( {primary.id: _openai_text("OK"), fallback.id: _openai_text("unused")} @@ -1369,7 +1566,9 @@ def test_preflight_uses_priced_fallback_only_after_primary_routes_reject() -> No assert viable == [primary] assert fallback_used is False assert "fallback_reason" not in report - assert [call[0] for call in ready_client.calls] == [primary] + # Likewise, the primary route's base-probe success is confirmed at the + # real serving budget before admission -- two calls, both to primary. + assert [call[0] for call in ready_client.calls] == [primary, primary] failing_client = _ProbeClient( {primary.id: TimeoutError("unavailable"), fallback.id: RuntimeError("rejected")} @@ -1380,29 +1579,186 @@ def test_preflight_uses_priced_fallback_only_after_primary_routes_reject() -> No assert failure.value.report["primary_attempt"]["ready_count"] == 0 +def test_preflight_advances_to_next_bounded_batch() -> None: + """Rejected first-batch routes do not hide a later discovered live route.""" + namespace = _load_launcher() + preflight = namespace["_preflight_review_agent_batches"] + batch_size = namespace["REVIEW_PREFLIGHT_BATCH_SIZE"] + agents = [ + SimpleNamespace( + id=f"route_{index}", provider_name="openrouter", model=f"model/{index}" + ) + for index in range(batch_size + 1) + ] + outcomes = {agent.id: TimeoutError("unavailable") for agent in agents[:-1]} + outcomes[agents[-1].id] = _openai_text("OK") + client = _ProbeClient(outcomes) + + viable, report = preflight(agents, client=client) + + assert viable == [agents[-1]] + assert report["probed_count"] == batch_size + 1 + assert report["ready_count"] == 1 + assert report["batch_size"] == batch_size + assert {call[0].id for call in client.calls[:batch_size]} == { + agent.id for agent in agents[:batch_size] + } + assert client.calls[-1][0] == agents[-1] + + +class _PerAgentSequencedClient: + """Return each agent's OWN configured attempt sequence, thread-safely. + + Unlike ``_SequencedClient`` (a single global sequence consumed strictly + in call order -- unsuitable once several agents' calls can interleave + unpredictably across concurrent batch threads), this looks up the next + outcome by (agent id, that agent's own call count), tracked per agent id + under a lock, so each candidate's own base-then-second-attempt sequence + stays deterministic regardless of how batch threads happen to interleave. + """ + + def __init__(self, outcomes: dict[str, list[object]]) -> None: + self._outcomes = outcomes + self._counts: dict[str, int] = {} + self._lock = threading.Lock() + self.calls: list[tuple[object, str, dict[str, object]]] = [] + + def proxy_send_once( + self, agent: object, endpoint: str, payload: dict[str, object] + ) -> dict[str, object]: + """Capture one request and return that agent's next configured outcome.""" + agent_id = str(getattr(agent, "id")) + with self._lock: + index = self._counts.get(agent_id, 0) + self._counts[agent_id] = index + 1 + self.calls.append((agent, endpoint, payload)) + outcome = self._outcomes[agent_id][index] + if isinstance(outcome, BaseException): + raise outcome + assert isinstance(outcome, dict) + return outcome + + +def test_batched_preflight_first_batch_confirmations_do_not_starve_a_later_healthy_route() -> None: + """Regression for Devin Review's "Later healthy routes cannot start" + finding (`ContextualWisdomLab/.github#1415`) on the batched preflight + entry point ``_preflight_review_agent_batches`` -- the exact scenario + described in the finding, reproduced end to end. + + The first ``REVIEW_PREFLIGHT_BATCH_SIZE`` candidates (batch 1) each + succeed their cheap base probe -- so each needs a mandatory confirmation + at the real serving budget -- but each then genuinely FAILS that + confirmation (a real "usable at 16 tokens, unusable at 4096" route, + correctly not admitted). A fifth candidate (batch 2) also succeeds its + base probe AND would succeed its confirmation too, if it ever got the + chance. + + Under the pre-fix code, all four batch-1 candidates' confirmations drew + from the SAME shared ``_EscalationBudget`` capped at + ``REVIEW_PREFLIGHT_MAX_ESCALATIONS`` (4) -- exactly enough for four + candidates to each reserve one slot before failing confirmation on + their own merits, permanently exhausting that shared counter. The fifth + candidate's later, unrelated confirmation request was then denied + purely by ``_EscalationBudget.try_reserve()`` returning ``False`` -- + ``escalation_budget_exhausted`` -- never even making its confirmation + call, regardless of the fact that it would have passed. With a budget + dedicated to confirmations specifically (this fix), the fifth candidate + is unaffected by batch 1's unrelated confirmation attempts and is + correctly admitted. + """ + namespace = _load_launcher() + preflight = namespace["_preflight_review_agent_batches"] + batch_size = namespace["REVIEW_PREFLIGHT_BATCH_SIZE"] + max_escalations = namespace["REVIEW_PREFLIGHT_MAX_ESCALATIONS"] + assert batch_size == max_escalations, ( + "this regression specifically needs one batch's worth of candidates " + "to exactly exhaust the (old, shared) escalation budget" + ) + + ok = _openai_text("OK") + # Genuinely fails its confirmation: usable at the base budget, empty (no + # budget-too-small signature) at the real serving budget -- correctly + # never admitted, regardless of which budget backed the attempt. + fails_confirmation = {"choices": [{"message": {"content": ""}}]} + + batch_one_serving_incompatible = [ + SimpleNamespace(id=f"batch1_narrow_{index}", provider_name="openrouter", model="x/free") + for index in range(batch_size) + ] + later_healthy_route = SimpleNamespace( + id="batch2_genuinely_healthy", provider_name="nvidia_nim", model="healthy/free" + ) + client = _PerAgentSequencedClient( + {agent.id: [ok, fails_confirmation] for agent in batch_one_serving_incompatible} + | {later_healthy_route.id: [ok, ok]} + ) + + viable, report = preflight( + [*batch_one_serving_incompatible, later_healthy_route], client=client + ) + + # The fifth candidate -- genuinely healthy at both budgets -- must be + # admitted. It must NOT be recorded as denied by escalation-budget + # exhaustion caused by four entirely different candidates' confirmations. + assert viable == [later_healthy_route] + later_route_row = next( + row for row in report["routes"] if row["agent_id"] == later_healthy_route.id + ) + assert later_route_row["status"] == "ready" + assert later_route_row["confirmed_at_serving_budget"] is True + assert "error_type" not in later_route_row + + # The four batch-1 candidates are correctly NOT admitted -- on their own + # merits (a real confirmation failure), never on budget exhaustion. + batch_one_rows = [ + row for row in report["routes"] if row["agent_id"] != later_healthy_route.id + ] + assert len(batch_one_rows) == batch_size + for row in batch_one_rows: + assert row["status"] == "rejected" + assert row["error_type"] == "invalid_chat_response" + + # The escalation (rescue) budget was never touched at all -- none of + # these candidates ever failed their base probe. + assert report["escalations_used"] == 0 + # Confirmation budget evidence: five candidates each made exactly one + # confirmation attempt. + assert report["confirmations_used"] == batch_size + 1 + + def test_fallback_escalation_budget_is_shared_with_primary_and_bounds_worst_case() -> None: """Regression for Devin Review's fallback-retries-exceed-startup-deadline finding: ``_preflight_review_agents`` used to start ``escalations_used`` - fresh on every call, so ``_preflight_with_fallback`` calling it twice (up - to 8 primary routes, then up to 4 fallback routes) could spend the full - ``REVIEW_PREFLIGHT_MAX_ESCALATIONS`` budget in EACH stage -- up to 8 - escalations total, 200s worst case (12 base attempts + 8 escalations x - 10s), blowing past Layer 1's 180s healthz-readiness watchdog and - contradicting the ADR's own claimed 160s worst case. - - This drives all 8 primary routes and all 4 fallback routes (the exact + fresh on every call, so ``_preflight_with_fallback`` calling it twice + could spend the full ``REVIEW_PREFLIGHT_MAX_ESCALATIONS`` budget in EACH + stage -- blowing past the preflight phase's own worst-case budget and + contradicting the ADR's own claimed worst case. + + This drives every primary and fallback route (the exact ``REVIEW_PREFLIGHT_MAX_TOTAL_ROUTES`` split) through a response that - always qualifies for escalation and never resolves, so every one of the - 12 candidates *would* escalate if the budget were not shared. Asserts - the run spends at most ``REVIEW_PREFLIGHT_MAX_ESCALATIONS`` escalations - in total (not per stage), and that the resulting worst-case attempt count - keeps total elapsed time at or under 160s -- both stages' escalation - counts are visible in the returned evidence. + always qualifies for escalation and never resolves, so every one of them + *would* escalate if the budget were not shared. Asserts the run spends at + most ``REVIEW_PREFLIGHT_MAX_ESCALATIONS`` escalations in total (not per + stage), and that the resulting worst case stays within + ``REVIEW_PREFLIGHT_WORST_CASE_SECONDS`` -- the preflight phase's own + coordinated budget, which the sidecar's startup watchdog now composes + with discovery's own worst case rather than treating as the whole startup + budget (see ``test_startup_watchdog_covers_discovery_plus_preflight_with_headroom`` + for that composition) -- both stages' escalation counts are visible in + the returned evidence. + + The worst-case *formula* (not the shared-budget invariant it measures) + differs from this test's pre-batching original: routes are now probed in + concurrent batches of ``REVIEW_PREFLIGHT_BATCH_SIZE``, so a batch's own + wall time is bounded by its slowest candidate, not the sum of every + candidate in it -- raw attempt count is no longer directly proportional + to wall-clock time the way a purely sequential loop's was. """ namespace = _load_launcher() preflight = namespace["_preflight_with_fallback"] max_escalations = namespace["REVIEW_PREFLIGHT_MAX_ESCALATIONS"] timeout_seconds = namespace["REVIEW_PREFLIGHT_TIMEOUT_SECONDS"] + batch_size = namespace["REVIEW_PREFLIGHT_BATCH_SIZE"] primary_limit = namespace["REVIEW_PREFLIGHT_PRIMARY_ROUTE_LIMIT"] total_route_limit = namespace["REVIEW_PREFLIGHT_MAX_TOTAL_ROUTES"] fallback_limit = total_route_limit - primary_limit @@ -1430,16 +1786,381 @@ def test_fallback_escalation_budget_is_shared_with_primary_and_bounds_worst_case assert report["primary_attempt"]["escalations_used"] == max_escalations total_attempts = len(client.calls) - worst_case_seconds = total_attempts * timeout_seconds - assert worst_case_seconds <= 160, ( - f"worst-case preflight time ({worst_case_seconds}s across " - f"{total_attempts} attempts) must stay within the 160s the ADR " - "computes and the 180s healthz-readiness watchdog allows" - ) - # Exactly the ADR's own worst-case arithmetic: 12 base attempts (one per - # candidate across both stages) + 4 escalations (the shared cap) = 16. + # Exactly the ADR's own worst-case arithmetic, batching-independent: one + # base attempt per candidate across both stages, plus the shared + # escalation cap. assert total_attempts == total_route_limit + max_escalations + # Batched, concurrent worst case: candidates run REVIEW_PREFLIGHT_BATCH_SIZE + # at a time, so a batch's wall time is bounded by its slowest candidate. + # In the fully pessimistic case every batch contains an escalating + # candidate (base attempt + escalated attempt, sequential within that one + # candidate's own thread) -- 2 * timeout_seconds per batch -- even though + # at most max_escalations of the batches actually can. + num_batches = -(-total_route_limit // batch_size) # ceil division + worst_case_seconds = num_batches * 2 * timeout_seconds + assert worst_case_seconds == namespace["REVIEW_PREFLIGHT_WORST_CASE_SECONDS"], ( + f"observed worst-case preflight time ({worst_case_seconds}s across " + f"{num_batches} batches) must match the launcher's own declared " + "REVIEW_PREFLIGHT_WORST_CASE_SECONDS -- a mismatch means that " + "constant no longer reflects this module's real batching behavior, " + "which would desynchronize it from the sidecar's derived startup " + "watchdog (REVIEW_STARTUP_WATCHDOG_SECONDS)" + ) + + +def test_startup_watchdog_covers_discovery_plus_preflight_with_headroom() -> None: + """Regression for Devin Review's "Startup watchdog preempts valid preflight" + finding: the sidecar's startup watchdog used to be a bare, uncoordinated + 180s shell constant that only happened to exceed the *probing-only* worst + case (120s) by coincidence, while never accounting for discovery's own + worst case (which runs first, in the SAME process, before ``/healthz`` can + respond) at all -- a fully correct, on-budget run of ~330s discovery + + ~120s probing = ~450s could be, and was (at the smaller, undercounted + 105s discovery figure this test used to pin), killed by too small a + watchdog before it ever reported a result. + + This is a purely static consistency check (no timing simulation, no real + sleeps -- CI-safe and non-flaky) that recomputes both worst cases + independently from the launcher's own primitive constants and asserts + ``REVIEW_STARTUP_WATCHDOG_SECONDS`` -- the single source of truth the + shell sidecar now imports rather than hard-coding its own number -- + actually covers their sum, with non-negative explicit headroom. It also + locks in the real, literal current numbers as a regression: any future + change to a budget constant that silently desynchronizes the derived + watchdog fails this test immediately, rather than only failing much later + in a live CI run that happens to hit the worst case. See + ``test_startup_watchdog_covers_a_retry_heavy_discovery_reconstruction`` + below for the companion test that independently reconstructs the 330s + discovery figure from the real, enumerated request structure rather than + trusting this module's own arithmetic -- exactly what Devin Review's + follow-up finding says a verbatim-constant test alone cannot catch. + """ + namespace = _load_launcher() + + discovery_calls = namespace["REVIEW_DISCOVERY_MAX_SEQUENTIAL_HTTP_CALLS"] + discovery_timeout = namespace["REVIEW_DISCOVERY_TIMEOUT_SECONDS"] + recomputed_discovery_worst_case = discovery_calls * discovery_timeout + assert recomputed_discovery_worst_case == namespace["REVIEW_DISCOVERY_WORST_CASE_SECONDS"] + # Verified directly against the vendored contextual_orchestrator.model_discovery + # source at ORCHESTRATOR_PIN_SHA (see the launcher's own module-level + # comment for the full call-by-call derivation): 22 sequential-call- + # equivalents at up to 15.0s each. + assert recomputed_discovery_worst_case == 330.0 + + total_routes = namespace["REVIEW_PREFLIGHT_MAX_TOTAL_ROUTES"] + batch_size = namespace["REVIEW_PREFLIGHT_BATCH_SIZE"] + preflight_timeout = namespace["REVIEW_PREFLIGHT_TIMEOUT_SECONDS"] + num_batches = -(-total_routes // batch_size) # ceil division + recomputed_preflight_worst_case = num_batches * 2 * preflight_timeout + assert recomputed_preflight_worst_case == namespace["REVIEW_PREFLIGHT_WORST_CASE_SECONDS"] + assert recomputed_preflight_worst_case == 120 + + headroom = namespace["REVIEW_STARTUP_HEADROOM_SECONDS"] + assert headroom >= 0, "headroom must never be negative -- that would silently under-cover" + + combined_worst_case = recomputed_discovery_worst_case + recomputed_preflight_worst_case + watchdog = namespace["REVIEW_STARTUP_WATCHDOG_SECONDS"] + assert isinstance(watchdog, int) + assert watchdog == int(combined_worst_case + headroom) + # The core invariant Devin Review's finding is about: the watchdog must + # cover the full combined worst case, not just one phase of it. + assert watchdog >= combined_worst_case + # Locks in the real current total (450s combined + 30s headroom), not a + # loosely-fitting range, so a future change to any input constant is a + # deliberate, visible edit to this test rather than a silent drift. + assert watchdog == 480 + + +def test_startup_watchdog_covers_a_retry_heavy_discovery_reconstruction() -> None: + """Independently rebuild the worst-case call count from the real request + structure and assert the derived watchdog still covers its time budget. + + Regression for Devin Review's exact follow-up finding on the discovery + budget ("Recompute the startup watchdog from the actual bounded request + structure ... extend tests with retry-heavy discovery timing rather than + asserting the current constant verbatim"): a test that only pins + ``REVIEW_DISCOVERY_MAX_SEQUENTIAL_HTTP_CALLS == 22`` (as + ``test_startup_watchdog_covers_discovery_plus_preflight_with_headroom`` + above does) would pass just as happily if that constant were still wrong + in the same direction the original ``7`` was -- it re-encodes whatever + the module currently claims, it does not check the claim against the + real, enumerated request structure. This test instead reconstructs the + worst case from first principles (each sub-count independently justified + against the vendored ``contextual_orchestrator.model_discovery`` source + at ``ORCHESTRATOR_PIN_SHA`` in the launcher module's own comment) and + proves the *reconstructed* time budget -- not just the module's own + arithmetic on its own constants -- is what the watchdog actually covers. + """ + namespace = _load_launcher() + discovery_timeout = namespace["REVIEW_DISCOVERY_TIMEOUT_SECONDS"] + + # (a) The shared Models.dev fetch retries transient failures up to + # _MODELS_DEV_FETCH_ATTEMPTS=3 times in the pinned source -- a retry- + # heavy scenario is exactly a run where every one of those attempts is a + # transient failure (timeout/connection reset) before the caller finally + # gives up and returns None (still a valid, non-raising outcome). + models_dev_attempts = 3 + assert models_dev_attempts == namespace["REVIEW_DISCOVERY_MODELS_DEV_MAX_ATTEMPTS"] + + # (b) Every one of the sidecar's five bootstrapped credentials + # (openai, openrouter, nvidia_nim, nvidia_nim_sub, bytez) gets its own + # primary-fetch attempt plus one retry attempt in a retry-heavy run. + credentialed_sources = ("openai", "openrouter", "nvidia_nim", "nvidia_nim_sub", "bytez") + attempts_per_source = 2 # base attempt + one transient-failure retry + assert len(credentialed_sources) == namespace["REVIEW_DISCOVERY_CREDENTIALED_SOURCE_COUNT"] + assert attempts_per_source == namespace["REVIEW_DISCOVERY_SOURCE_MAX_ATTEMPTS"] + + # (c) OpenRouter alone makes two further single-attempt calls (ZDR + # endpoints, provider policies) beyond its own primary fetch already + # counted in (b), plus one concurrent endpoint-feed round per <=8 + # currently free-priced models. A retry-heavy scenario does not add + # retries to these three (none of them retry in the pinned source), but + # it does mean discovery cannot skip them by finishing early. + openrouter_single_extra_calls = 2 + free_endpoint_round_cap = 5 + assert openrouter_single_extra_calls == namespace[ + "REVIEW_DISCOVERY_OPENROUTER_SINGLE_EXTRA_CALLS" + ] + assert free_endpoint_round_cap == namespace[ + "REVIEW_DISCOVERY_OPENROUTER_FREE_ENDPOINT_ROUND_CAP" + ] + + # (d) Two trailing global calls run once, after every source above, with + # an OpenRouter credential registered: the (separate, non-cached) + # _openrouter_zdr_model_ids() fetch and the credits check. + trailing_global_calls = 2 + assert trailing_global_calls == namespace["REVIEW_DISCOVERY_TRAILING_GLOBAL_CALLS"] + + reconstructed_call_count = ( + models_dev_attempts + + len(credentialed_sources) * attempts_per_source + + openrouter_single_extra_calls + + free_endpoint_round_cap + + trailing_global_calls + ) + assert reconstructed_call_count == 22 + assert reconstructed_call_count == namespace["REVIEW_DISCOVERY_MAX_SEQUENTIAL_HTTP_CALLS"] + + reconstructed_discovery_seconds = reconstructed_call_count * discovery_timeout + reconstructed_combined_seconds = ( + reconstructed_discovery_seconds + namespace["REVIEW_PREFLIGHT_WORST_CASE_SECONDS"] + ) + watchdog = namespace["REVIEW_STARTUP_WATCHDOG_SECONDS"] + # The core assertion: the derived watchdog must cover a genuinely + # independently-reconstructed retry-heavy worst case, not merely the + # module's own (possibly still wrong) restatement of it. + assert watchdog >= reconstructed_combined_seconds + + +def test_sidecar_derives_its_watchdog_from_the_launcher_single_source_of_truth() -> None: + """The shell watchdog must import, not hard-code, the coordinated deadline. + + Guards against the exact regression class this fix addresses: a future + edit that changes a launcher timing constant (discovery calls, batch + size, escalation timeout, ...) must automatically change the sidecar's + watchdog too, with no second place to remember to update by hand. + """ + namespace = _load_launcher() + sidecar_text = _SIDECAR.read_text(encoding="utf-8") + + assert ( + "from scripts.ci.contextual_orchestrator_review_launcher " + "import REVIEW_STARTUP_WATCHDOG_SECONDS" in sidecar_text + ) + assert 'sidecar_startup_watchdog_seconds="$(' in sidecar_text + assert '[ "$SECONDS" -ge "$sidecar_startup_watchdog_seconds" ]' in sidecar_text + # The old, uncoordinated hard-coded bound must be gone from the watchdog + # comparison -- not just supplemented by the new derived one. + assert '[ "$i" -ge 180 ]' not in sidecar_text + assert "-ge 180" not in sidecar_text + # Regression for Devin Review's "Startup watchdog counts polls, not + # seconds" finding (ContextualWisdomLab/.github#1415): the comparison + # must read bash's real wall-clock $SECONDS builtin, not a hand-rolled + # poll counter incremented once per loop iteration regardless of how + # long that iteration's own curl call took. + assert '[ "$i" -ge "$sidecar_startup_watchdog_seconds" ]' not in sidecar_text + + # Exercise the exact derivation command the sidecar script runs, proving + # it truly needs no vendored dependency yet at that point in the script + # (the launcher module's top-level imports are deliberately stdlib-only). + result = subprocess.run( + [ + sys.executable, + "-c", + ( + "from scripts.ci.contextual_orchestrator_review_launcher import " + "REVIEW_STARTUP_WATCHDOG_SECONDS; print(REVIEW_STARTUP_WATCHDOG_SECONDS)" + ), + ], + cwd=str(_REPO_ROOT), + env={**os.environ, "PYTHONPATH": str(_REPO_ROOT)}, + capture_output=True, + text=True, + check=True, + ) + assert result.stdout.strip() == str(namespace["REVIEW_STARTUP_WATCHDOG_SECONDS"]) + + +_HEALTHZ_WAIT_BLOCK_START = "SECONDS=0\nuntil curl -fsSL --max-time 2 " +_HEALTHZ_WAIT_BLOCK_END = "\n sleep 1\ndone" + + +def _run_healthz_wait_loop( + tmp_path: Path, + *, + watchdog_seconds: int, + curl_delay_seconds: float, +) -> tuple[subprocess.CompletedProcess[str], float]: + """Execute the sidecar's real healthz-wait loop against a fake, always-failing curl. + + Extracts the exact, current source of the loop from the tracked sidecar + script (the same technique ``_run_gateway_retry_loop`` uses above) so a + future edit to the loop is automatically exercised here instead of + silently drifting from a second, hand-copied duplicate. + + Args: + tmp_path: Pytest's per-test scratch directory. + watchdog_seconds: Value for ``sidecar_startup_watchdog_seconds``. + curl_delay_seconds: How long the fake ``curl`` sleeps before failing, + simulating a slow-but-still-under-its-own-``--max-time`` health + probe. + + Returns: + The completed harness process and the measured real wall-clock time + the loop took to fail, as observed from outside the subprocess. + """ + sidecar_text = _SIDECAR.read_text(encoding="utf-8") + start = sidecar_text.index(_HEALTHZ_WAIT_BLOCK_START) + end = sidecar_text.index(_HEALTHZ_WAIT_BLOCK_END, start) + len(_HEALTHZ_WAIT_BLOCK_END) + loop_block = sidecar_text[start:end] + + fake_bin = tmp_path / "fake-bin" + fake_bin.mkdir() + fake_curl = fake_bin / "curl" + fake_curl.write_text( + f"#!/usr/bin/env bash\nsleep {curl_delay_seconds}\nexit 1\n", + encoding="utf-8", + ) + fake_curl.chmod(0o755) + + sidecar_stderr = tmp_path / "sidecar-stderr.txt" + sidecar_stderr.write_text("", encoding="utf-8") + preflight_report = tmp_path / "preflight.json" + preflight_report.write_text("{}", encoding="utf-8") + + harness = tmp_path / "harness.sh" + harness.write_text( + "set -euo pipefail\n" + "log() { printf '[test-sidecar] %s\\n' \"$*\"; }\n" + 'fail() { log "error: $*" >&2; exit 1; }\n' + # The real loop's "sidecar exited early" branch calls `kill -0 + # "$sidecar_pid"` to tell a dead sidecar apart from one still + # starting; stub it so this harness exercises only the watchdog + # deadline comparison, never that other branch. + "kill() { return 0; }\n" + "wait_for_sidecar_sanitizers() { :; }\n" + "sidecar_pid=$$\n" + 'ORCHESTRATOR_HOST="127.0.0.1"\n' + 'ORCHESTRATOR_PORT="18080"\n' + f"sidecar_startup_watchdog_seconds={watchdog_seconds}\n" + f'sidecar_stderr="{sidecar_stderr}"\n' + f'preflight_report="{preflight_report}"\n' + + loop_block + + "\n", + encoding="utf-8", + ) + + start_time = time.monotonic() + result = subprocess.run( + ["bash", str(harness)], + env={ + **os.environ, + "PATH": f"{fake_bin}{os.pathsep}{os.environ.get('PATH', '')}", + }, + text=True, + capture_output=True, + check=False, + ) + elapsed = time.monotonic() - start_time + return result, elapsed + + +def test_healthz_wait_loop_fires_near_the_wall_clock_deadline_not_a_poll_count( + tmp_path: Path, +) -> None: + """Regression for Devin Review's "Startup watchdog counts polls, not + seconds" finding (ContextualWisdomLab/.github#1415). + + Before the fix, the loop incremented a plain poll counter ``i`` once per + iteration and compared *that* to ``sidecar_startup_watchdog_seconds`` -- + even though a single iteration's real cost is the curl call's own + duration (up to its ``--max-time``) plus the trailing ``sleep 1``. With a + health probe that itself takes close to its full timeout, that made the + watchdog run roughly 3x longer than its configured bound (255s + configured, ~765s observed worst case). + + This drives the sidecar's real, tracked healthz-wait loop (extracted + verbatim, not a hand-copied duplicate) against a fake ``curl`` that + always fails after a deliberately slow ``curl_delay_seconds``, with a + small configured watchdog. It asserts the loop fails close to the + *configured* wall-clock seconds (allowing headroom for the cadence of + one in-flight curl call plus one ``sleep 1``), and, crucially, well + under 3x that bound -- the exact regression class this test guards + against. + """ + watchdog_seconds = 3 + curl_delay_seconds = 2.0 + + result, elapsed = _run_healthz_wait_loop( + tmp_path, + watchdog_seconds=watchdog_seconds, + curl_delay_seconds=curl_delay_seconds, + ) + + assert result.returncode == 1, result.stderr + assert ( + f"sidecar did not become healthy within {watchdog_seconds}s" in result.stderr + ) + # The old, buggy poll-counting comparison would need + # `watchdog_seconds` full iterations -- each costing + # curl_delay_seconds + 1s of sleep -- before firing: roughly + # watchdog_seconds * (curl_delay_seconds + 1) = 9s here. The fixed, + # real-wall-clock comparison fires as soon as accumulated curl time + # alone crosses the deadline: roughly one extra curl call past the + # bound, ~5s here. Assert comfortably between the two, strictly below + # the poll-counting bound -- proving this is not that regression. + poll_counting_bound = watchdog_seconds * (curl_delay_seconds + 1) + assert elapsed < poll_counting_bound - 1, ( + f"loop took {elapsed:.1f}s to fail, at or beyond the poll-counting " + f"bound of {poll_counting_bound:.1f}s that this fix removes -- the " + "watchdog is counting polls again, not real wall-clock seconds" + ) + # A lower bound too: the loop cannot legitimately fail before at least + # one curl call has run (the deadline is only checked after a curl + # attempt), so it must take at least curl_delay_seconds. + assert elapsed >= curl_delay_seconds + + +def test_healthz_wait_loop_reports_wall_clock_seconds_not_a_poll_count( + tmp_path: Path, +) -> None: + """The failure message's own reported bound must not silently change. + + A narrower companion to the timing test above: even independent of how + long the loop actually took, the fixed loop's fail() message must still + name the *configured* ``sidecar_startup_watchdog_seconds`` -- proving + the message-formatting side of the fix (``$SECONDS`` swapped in for + ``$i`` in both the comparison and the two places it is interpolated) + did not regress independently of the timing behavior. + """ + result, _elapsed = _run_healthz_wait_loop( + tmp_path, watchdog_seconds=2, curl_delay_seconds=1.5 + ) + + assert result.returncode == 1, result.stderr + assert "sidecar did not become healthy within 2s" in result.stderr + def test_preflight_stage_limits_share_one_startup_budget() -> None: """Free-first and priced-fallback probes share one bounded route budget.""" @@ -1447,37 +2168,91 @@ def test_preflight_stage_limits_share_one_startup_budget() -> None: primary = namespace["_bounded_primary_catalog_limit"]( 99, pool="auto", has_free_rows=True ) - fallback = namespace["_bounded_fallback_catalog_limit"]( - 99, primary_count=primary - ) - assert (primary, fallback) == (8, 4) + fallback = namespace["_bounded_fallback_catalog_limit"](99, primary_count=primary) + assert (primary, fallback) == (8, 16) assert primary + fallback == namespace["REVIEW_PREFLIGHT_MAX_TOTAL_ROUTES"] +def test_production_defaults_expose_the_complete_bounded_catalog() -> None: + """Launcher and shell defaults must not silently restore the old 12-route cap.""" + launcher = _LAUNCHER.read_text(encoding="utf-8") + sidecar = _SIDECAR.read_text(encoding="utf-8") + + assert 'ORCHESTRATOR_CATALOG_LIMIT", "24"' in launcher + assert 'CATALOG_LIMIT="${ORCHESTRATOR_CATALOG_LIMIT:-24}"' in sidecar + assert "REVIEW_PREFLIGHT_MAX_TOTAL_ROUTES = 24" in launcher + # Regression for ContextualWisdomLab/.github#1415's Devin follow-up + # finding: the shell used to always export a hard-coded literal `8` + # default for ORCHESTRATOR_CATALOG_ACCOUNT_CAP, which bypassed the + # launcher's own _catalog_account_cap(DEFAULT_ACCOUNT_CAP)=4 fallback in + # every real run (os.environ.get only falls back when the key is + # ABSENT). The shell must no longer materialize that literal and must + # instead derive the same policy.DEFAULT_ACCOUNT_CAP the launcher does. + assert 'CATALOG_ACCOUNT_CAP="${ORCHESTRATOR_CATALOG_ACCOUNT_CAP:-8}"' not in sidecar + assert ( + "from scripts.ci.contextual_orchestrator_review_policy import " + "DEFAULT_ACCOUNT_CAP; print(DEFAULT_ACCOUNT_CAP)" + ) in sidecar + + def test_catalog_account_cap_defaults_to_the_caller_supplied_policy_default( monkeypatch: pytest.MonkeyPatch, ) -> None: """The per-account cap falls back to ``policy.DEFAULT_ACCOUNT_CAP``, not the total budget. Regression for a real, observed failure mode - (ContextualWisdomLab/.github#1415, reported as "빈 깡통 경로 너무 많다"): a - sibling helper (``_catalog_family_cap()``) fell back to - ``REVIEW_PREFLIGHT_MAX_TOTAL_ROUTES`` -- the *total* preflight budget -- - instead of the intended per-account cap whenever its env var was unset. - That silently disabled per-account diversification: in a live production - run, two NVIDIA NIM credentials sharing one rate-limited upstream jointly - consumed all 12 preflight slots, of which 10 (83%) were then rejected via - 429/404/timeout. This module's own equivalent helper must never resolve - to the same value as the total-routes budget when given the real - ``policy.DEFAULT_ACCOUNT_CAP``, which is strictly smaller. + (ContextualWisdomLab/.github#1415, reported as "빈 깡통 경로 너무 많다"): this + module's ``_catalog_family_cap()`` helper (since renamed and fixed here -- + `main` PR #1487 landed the identical fix independently under the same + final name) fell back to ``REVIEW_PREFLIGHT_MAX_TOTAL_ROUTES`` -- the + *total* preflight budget -- instead of the intended per-account cap + whenever its env var was unset. That silently disabled per-account + diversification: in a live production run, two NVIDIA NIM credentials + sharing one rate-limited upstream jointly consumed all 12 preflight + slots, of which 10 (83%) were then rejected via 429/404/timeout. This + helper must never resolve to the same value as the total-routes budget + when given the real ``policy.DEFAULT_ACCOUNT_CAP``, which is strictly + smaller. """ namespace = _load_launcher() + account_cap = namespace["_catalog_account_cap"] + assert callable(account_cap) + monkeypatch.delenv("ORCHESTRATOR_CATALOG_ACCOUNT_CAP", raising=False) - cap = namespace["_catalog_account_cap"](policy.DEFAULT_ACCOUNT_CAP) + cap = account_cap(policy.DEFAULT_ACCOUNT_CAP) assert cap == policy.DEFAULT_ACCOUNT_CAP assert cap != namespace["REVIEW_PREFLIGHT_MAX_TOTAL_ROUTES"] assert cap < namespace["REVIEW_PREFLIGHT_MAX_TOTAL_ROUTES"] + # Reproduces the live evidence directly: 12 free routes split across just + # two credential accounts (nvidia_nim / nvidia_nim_sub) sharing one + # rate-limited upstream. At the default cap, neither account may absorb + # more than its share of the bounded preflight budget. + rows = [ + { + "provider": "nvidia_nim" if index % 2 == 0 else "nvidia_nim_sub", + "model": f"model{index}", + "agent_id": f"nim_a{index}", + "is_free": True, + "prompt_price_per_1k": 0.0, + "completion_price_per_1k": 0.0, + "currency_code": "USD", + } + for index in range(12) + ] + catalog = policy.build_zdr_prioritized_catalog( + policy.parse_discovery_report({"models": rows}), + limit=12, + account_cap=cap, + ) + per_account: dict[str, int] = {} + for agent in catalog["agents"]: + account = policy.provider_account(agent["provider_name"]) + per_account[account] = per_account.get(account, 0) + 1 + assert per_account + assert max(per_account.values()) <= policy.DEFAULT_ACCOUNT_CAP + assert len(catalog["agents"]) <= 2 * policy.DEFAULT_ACCOUNT_CAP + def test_catalog_account_cap_honors_an_explicit_override( monkeypatch: pytest.MonkeyPatch, @@ -1488,6 +2263,110 @@ def test_catalog_account_cap_honors_an_explicit_override( assert namespace["_catalog_account_cap"](policy.DEFAULT_ACCOUNT_CAP) == 6 +_CATALOG_ACCOUNT_CAP_BLOCK_START = ( + 'if [ -n "${ORCHESTRATOR_CATALOG_ACCOUNT_CAP:-}" ]; then' +) + + +def _run_catalog_account_cap_derivation( + *, override: str | None +) -> subprocess.CompletedProcess[str]: + """Execute the sidecar's real per-account-cap derivation block in bash. + + Extracts the exact, current source of the derivation from the tracked + sidecar script (the same technique + ``test_sidecar_derives_its_watchdog_from_the_launcher_single_source_of_truth`` + and ``_run_healthz_wait_loop`` use above) so a future edit to the block + is automatically exercised here instead of silently drifting from a + second, hand-copied duplicate. Regression for + ContextualWisdomLab/.github#1415's Devin follow-up finding: proves the + shell itself -- not just the Python ``_catalog_account_cap`` helper in + isolation -- resolves to ``policy.DEFAULT_ACCOUNT_CAP`` when no operator + override is set, and to the override's exact value when one is. + + Args: + override: Value to set ``ORCHESTRATOR_CATALOG_ACCOUNT_CAP`` to before + running the block, or ``None`` to leave it genuinely unset. + + Returns: + The completed harness process; ``stdout`` carries ``RESULT=`` + on success. + """ + sidecar_text = _SIDECAR.read_text(encoding="utf-8") + start = sidecar_text.index(_CATALOG_ACCOUNT_CAP_BLOCK_START) + end = sidecar_text.index("esac\n", start) + len("esac\n") + block = sidecar_text[start:end] + assert "CATALOG_ACCOUNT_CAP" in block + + harness = ( + "set -euo pipefail\n" + "log() { printf '[test-sidecar] %s\\n' \"$*\"; }\n" + 'fail() { log "error: $*" >&2; exit 1; }\n' + f'ORG_REPO_ROOT="{_REPO_ROOT}"\n' + f'sidecar_python="{sys.executable}"\n' + + block + + '\nprintf "RESULT=%s\\n" "$CATALOG_ACCOUNT_CAP"\n' + ) + env = dict(os.environ) + env["PYTHONPATH"] = str(_REPO_ROOT) + if override is None: + env.pop("ORCHESTRATOR_CATALOG_ACCOUNT_CAP", None) + else: + env["ORCHESTRATOR_CATALOG_ACCOUNT_CAP"] = override + return subprocess.run( + ["bash", "-c", harness], + env=env, + capture_output=True, + text=True, + check=False, + ) + + +def test_sidecar_shell_derives_the_account_cap_default_from_policy_when_unset() -> None: + """With no operator override, the SHELL (not just the Python helper) gets 4. + + This is the exact regression the just-landed + ``_catalog_account_cap(DEFAULT_ACCOUNT_CAP)`` fix could not close on its + own: that helper's env-unset fallback only runs if the shell genuinely + never set the env var. Before this fix the shell always exported a + literal ``8`` first, so this end-to-end path -- not the Python unit + tested above -- is what previously stayed silently broken in production. + """ + result = _run_catalog_account_cap_derivation(override=None) + assert result.returncode == 0, result.stderr + assert f"RESULT={policy.DEFAULT_ACCOUNT_CAP}" in result.stdout + + +def test_sidecar_shell_honors_an_explicit_account_cap_override() -> None: + """An operator-set ``ORCHESTRATOR_CATALOG_ACCOUNT_CAP`` still wins in the shell.""" + result = _run_catalog_account_cap_derivation(override="6") + assert result.returncode == 0, result.stderr + assert "RESULT=6" in result.stdout + + +@pytest.mark.parametrize("bad_value", ["0", "-1", "abc"]) +def test_sidecar_shell_rejects_an_invalid_account_cap_override(bad_value: str) -> None: + """A malformed override must fail closed, matching the file's other digit checks.""" + result = _run_catalog_account_cap_derivation(override=bad_value) + assert result.returncode == 1 + assert "ORCHESTRATOR_CATALOG_ACCOUNT_CAP must be a positive integer" in result.stderr + + +def test_sidecar_shell_treats_an_empty_override_as_unset() -> None: + """``ORCHESTRATOR_CATALOG_ACCOUNT_CAP=""`` matches bash's own ``:-`` semantics. + + An explicitly empty override is indistinguishable from unset under the + ``${VAR:-default}`` expansion this block (and the rest of this script) + already relies on elsewhere -- e.g. the provider-secret presence loop's + ``[ -n "${!secret_name:-}" ]`` -- so it must fall back to the derived + policy default rather than reaching the digit-format check with an empty + string. + """ + result = _run_catalog_account_cap_derivation(override="") + assert result.returncode == 0, result.stderr + assert f"RESULT={policy.DEFAULT_ACCOUNT_CAP}" in result.stdout + + def test_main_sources_the_account_cap_default_from_policy_not_a_magic_number() -> None: """``main()`` must wire the cap default from ``policy.DEFAULT_ACCOUNT_CAP``. @@ -1497,11 +2376,12 @@ def test_main_sources_the_account_cap_default_from_policy_not_a_magic_number() - ContextualWisdomLab/.github#1415's real preflight-budget waste. This source-level contract test pins both ``build_zdr_prioritized_catalog`` call sites in ``main()`` to the single source of truth and forbids the - total-routes constant from ever reappearing as the account-cap fallback. + old family-cap naming and the old total-routes fallback from reappearing. """ source = _LAUNCHER.read_text(encoding="utf-8") assert source.count("account_cap=_catalog_account_cap(DEFAULT_ACCOUNT_CAP)") == 2 assert "ORCHESTRATOR_CATALOG_FAMILY_CAP" not in source + assert "_catalog_family_cap" not in source assert 'os.environ.get("ORCHESTRATOR_CATALOG_ACCOUNT_CAP", "4")' not in source @@ -1581,7 +2461,9 @@ def loader(value: str) -> list[object]: assert json.loads(Path(value).read_text(encoding="utf-8")) == {"agents": agents} return [SimpleNamespace(id="priced_route")] - assert [agent.id for agent in helper(str(path), agents, loader=loader)] == ["priced_route"] + assert [agent.id for agent in helper(str(path), agents, loader=loader)] == [ + "priced_route" + ] assert not path.exists() def failing_loader(value: str) -> list[object]: @@ -1594,15 +2476,29 @@ def failing_loader(value: str) -> list[object]: def test_preflight_transport_is_bounded_and_provider_neutral() -> None: - """Sequential route probes must fit inside the sidecar startup budget.""" - launcher = _LAUNCHER.read_text(encoding="utf-8") + """Startup probes stay short while serving gets the Noema review budget.""" + namespace = _load_launcher() - assert "REVIEW_MAX_OUTPUT_TOKENS = 4096" in launcher - assert "REVIEW_TEMPERATURE = 1.0" in launcher - assert "REVIEW_PREFLIGHT_TIMEOUT_SECONDS = 10" in launcher - assert "timeout=REVIEW_PREFLIGHT_TIMEOUT_SECONDS" in launcher - assert "max_retries=0" in launcher - assert "temperature=REVIEW_TEMPERATURE" in launcher + class CaptureClient: + instances: list[dict[str, object]] = [] + + def __init__(self, **kwargs: object) -> None: + self.__class__.instances.append(kwargs) + + build_client = namespace["_build_model_client"] + build_client( + CaptureClient, timeout=namespace["REVIEW_PREFLIGHT_TIMEOUT_SECONDS"] + ) + build_client(CaptureClient, timeout=namespace["REVIEW_SERVING_TIMEOUT_SECONDS"]) + + preflight, serving = CaptureClient.instances + + assert preflight["timeout"] == 10 + assert serving["timeout"] == 9000 + assert preflight["timeout"] != serving["timeout"] + assert preflight["max_output_tokens"] == serving["max_output_tokens"] == 4096 + assert preflight["max_retries"] == serving["max_retries"] == 0 + assert preflight["temperature"] == serving["temperature"] == 1.0 def test_sidecar_preserves_diagnostics_and_probes_the_real_gateway() -> None: @@ -1613,25 +2509,63 @@ def test_sidecar_preserves_diagnostics_and_probes_the_real_gateway() -> None: assert "_preflight_with_fallback(" in launcher assert "preflight-out" in launcher assert "max_output_tokens=REVIEW_MAX_OUTPUT_TOKENS" in launcher + assert "REVIEW_SERVING_TIMEOUT_SECONDS = 9000" in launcher + assert "timeout=REVIEW_PREFLIGHT_TIMEOUT_SECONDS" in launcher + assert "timeout=REVIEW_SERVING_TIMEOUT_SECONDS" in launcher + assert launcher.count("max_retries=0") == 1 assert "temperature=REVIEW_TEMPERATURE" in launcher - assert 'STRIX_EVIDENCE_DIR="${GITHUB_WORKSPACE:-$ORCHESTRATOR_WORK}/strix_runs"' in sidecar - assert 'sidecar_stdout="$STRIX_EVIDENCE_DIR/contextual-orchestrator-sidecar.stdout.log"' in sidecar - assert 'sidecar_stderr="$STRIX_EVIDENCE_DIR/contextual-orchestrator-sidecar.stderr.log"' in sidecar - assert 'preflight_report="$STRIX_EVIDENCE_DIR/contextual-orchestrator-preflight.json"' in sidecar + assert ( + 'STRIX_EVIDENCE_DIR="${GITHUB_WORKSPACE:-$ORCHESTRATOR_WORK}/strix_runs"' + in sidecar + ) + assert ( + 'sidecar_stdout="$STRIX_EVIDENCE_DIR/contextual-orchestrator-sidecar.stdout.log"' + in sidecar + ) + assert ( + 'sidecar_stderr="$STRIX_EVIDENCE_DIR/contextual-orchestrator-sidecar.stderr.log"' + in sidecar + ) + assert ( + 'preflight_report="$STRIX_EVIDENCE_DIR/contextual-orchestrator-preflight.json"' + in sidecar + ) assert '--preflight-out "$preflight_report"' in sidecar - assert 'gateway_preflight_response="$ORCHESTRATOR_WORK/gateway-preflight.json"' in sidecar - assert '"http://${ORCHESTRATOR_HOST}:${ORCHESTRATOR_PORT}/v1/chat/completions"' in sidecar - assert 'Authorization: Bearer ${ORCHESTRATOR_TOKEN}' in sidecar + # ADR-0005's bounded gateway retry loop (see the dedicated + # test_gateway_preflight_retries_transport_failures_up_to_a_bounded_attempt_count + # and test_gateway_retry_loop_* tests below for its full behavioral + # contract) replaces the single-shot transport_timeout/transport_error + # classification this test previously asserted here. + assert 'REVIEW_PREFLIGHT_GATEWAY_MAX_ATTEMPTS="${REVIEW_PREFLIGHT_GATEWAY_MAX_ATTEMPTS:-3}"' in sidecar + assert "gateway preflight request could not reach the local sidecar after" in sidecar + assert ( + 'gateway_preflight_response="$ORCHESTRATOR_WORK/gateway-preflight.json"' + in sidecar + ) + assert ( + '"http://${ORCHESTRATOR_HOST}:${ORCHESTRATOR_PORT}/v1/chat/completions"' + in sidecar + ) + assert "Authorization: Bearer ${ORCHESTRATOR_TOKEN}" in sidecar assert 'orchestrator_pool="${CONTEXTUAL_ORCHESTRATOR_POOL:-free}"' in sidecar assert 'gateway_virtual_model="orchestrator/${orchestrator_pool}"' in sidecar assert '"model":"%s"' in sidecar + assert '"orchestration":"route"' in sidecar assert '"$gateway_virtual_model" > "$gateway_preflight_request"' in sidecar assert '"model":"orchestrator/free"' not in sidecar assert "gateway preflight returned unusable chat content" in sidecar - assert 'SIDECAR_LOG_SANITIZER="$ORG_REPO_ROOT/scripts/ci/sanitize_contextual_orchestrator_sidecar_stream.py"' in sidecar - assert '"$sidecar_python" -u "$SIDECAR_LOG_SANITIZER" > "$sidecar_stdout"' in sidecar - assert '"$sidecar_python" -u "$SIDECAR_LOG_SANITIZER" > "$sidecar_stderr"' in sidecar + assert ( + 'SIDECAR_LOG_SANITIZER="$ORG_REPO_ROOT/scripts/ci/sanitize_contextual_orchestrator_sidecar_stream.py"' + in sidecar + ) + assert "contextlib.redirect_stderr(expected_rejection_log)" in sidecar + assert ( + '"$sidecar_python" -u "$SIDECAR_LOG_SANITIZER" > "$sidecar_stdout"' in sidecar + ) + assert ( + '"$sidecar_python" -u "$SIDECAR_LOG_SANITIZER" > "$sidecar_stderr"' in sidecar + ) assert '> "$sidecar_stdout" 2> "$sidecar_stderr" &' not in sidecar @@ -1664,9 +2598,12 @@ def test_sidecar_stream_sanitizer_allowlists_only_bounded_diagnostics() -> None: namespace = _load_sanitizer() sanitize_line = namespace["sanitize_line"] - assert sanitize_line( - "request_failed status=500 code=internal_error upstream sk-secret" - ) == "request_failed status=500 code=internal_error" + assert ( + sanitize_line( + "request_failed status=500 code=internal_error upstream sk-secret" + ) + == "request_failed status=500 code=internal_error" + ) assert sanitize_line("client_disconnected") == "client_disconnected" assert sanitize_line("discovery_diagnostics_complete") == "discovery_diagnostics_complete" assert sanitize_line( diff --git a/tests/test_contextual_orchestrator_review_sidecar_contract.py b/tests/test_contextual_orchestrator_review_sidecar_contract.py index 0a63356dad..d30a4244c4 100644 --- a/tests/test_contextual_orchestrator_review_sidecar_contract.py +++ b/tests/test_contextual_orchestrator_review_sidecar_contract.py @@ -40,7 +40,7 @@ ) GATEWAY_MODEL = "contextual-orchestrator/orchestrator/free" -ORCH_PIN_SHA = "8cd99f139915131ba0239bce12a5d6a5fd85394e" +ORCH_PIN_SHA = "ab7a813a69dae19541dc2888acd50c4ce37b29b7" def _read(path: Path) -> str: @@ -66,11 +66,58 @@ def test_sidecar_pins_the_vendored_orchestrator_revision() -> None: assert 'ORCHESTRATOR_HOST="127.0.0.1"' in text +def test_single_candidate_attempt_is_explicit_and_preserves_normal_defaults() -> None: + """Only pinned workflow jobs remove the redundant in-process retry.""" + sidecar = _read(SIDECAR) + launcher = _read(LAUNCHER) + + assert "--single-candidate-attempt" in sidecar + assert 'launcher_attempt_args=(--single-candidate-attempt)' in sidecar + assert 'if args.single_candidate_attempt else {}' in launcher + assert '{"tool_retry_attempts": 0}' in launcher + assert "realtime_judge=False" not in launcher + assert "realtime_judge = False" not in launcher + + +def test_gateway_preflight_timeout_is_bounded_and_caller_configurable() -> None: + """A hung smoke request cannot consume the caller's real model budget.""" + sidecar = _read(SIDECAR) + + assert 'REVIEW_PREFLIGHT_GATEWAY_MAX_TIME_SECONDS="${REVIEW_PREFLIGHT_GATEWAY_MAX_TIME_SECONDS:-1200}"' in sidecar + assert '--max-time "$REVIEW_PREFLIGHT_GATEWAY_MAX_TIME_SECONDS"' in sidecar + assert "--max-time 3600" not in sidecar + assert "must be a positive integer" in sidecar + + def test_sidecar_adr_names_the_current_vendored_revision() -> None: """The accepted decision record must not advertise a stale runtime SHA.""" assert ORCH_PIN_SHA in _read(SIDECAR_ADR) +def test_sidecar_and_adr_pin_the_bounded_preflight_contract() -> None: + """Runtime defaults and the accepted prose must describe one startup budget.""" + launcher = _read(LAUNCHER) + sidecar = _read(SIDECAR) + adr = _read(SIDECAR_ADR) + + assert 'CATALOG_LIMIT="${ORCHESTRATOR_CATALOG_LIMIT:-24}"' in sidecar + # The per-account cap default is no longer a shell literal (that was the + # ContextualWisdomLab/.github#1415 Devin follow-up bug: a hard-coded `8` + # here silently bypassed the launcher's own DEFAULT_ACCOUNT_CAP=4 + # fallback in every real run). It must now be derived at runtime from + # the same single source of truth the launcher uses. + assert 'CATALOG_ACCOUNT_CAP="${ORCHESTRATOR_CATALOG_ACCOUNT_CAP:-8}"' not in sidecar + assert ( + "from scripts.ci.contextual_orchestrator_review_policy import " + "DEFAULT_ACCOUNT_CAP; print(DEFAULT_ACCOUNT_CAP)" + ) in sidecar + assert "REVIEW_PREFLIGHT_MAX_TOTAL_ROUTES = 24" in launcher + assert "REVIEW_PREFLIGHT_BATCH_SIZE = 4" in launcher + assert "at most 24" in adr + assert "concurrent batches of four" in adr + assert "fails closed before healthz" in adr + + def test_sidecar_requires_the_five_provider_secrets() -> None: """At least one of the five secrets must be present as bootstrap transport.""" text = _read(SIDECAR) @@ -280,13 +327,17 @@ def test_launcher_uses_orchestrator_discovery_and_governed_pools() -> None: """Discovery, price evidence, and serving come from the vendored library.""" text = _read(LAUNCHER) assert "from contextual_orchestrator.chat_capability import is_general_chat_agent_model_id" in text - assert "from contextual_orchestrator.model_discovery import discover_all_models, free_discovered_models" in text + normalized = " ".join(text.split()) + assert ( + "from contextual_orchestrator.model_discovery import ( discover_all_models, " + "free_discovered_models, )" in normalized + ) assert "routable_discovered = _routable_discovered_models(discovered)" in text assert "free_discovered_models(routable_discovered)" in text assert 'getattr(model, "evidence_only", False)' in text assert 'getattr(model, "output_modalities", None)' in text assert 'isinstance(modalities, str)' in text - assert '"text" in {str(modality).casefold() for modality in modalities}' in text + assert '"text" in { str(modality).casefold() for modality in modalities }' in normalized assert "not _has_text_output(model)" in text assert 'model_id = getattr(model, "model_id", "")' in text @@ -315,8 +366,11 @@ def test_launcher_uses_orchestrator_discovery_and_governed_pools() -> None: rows = report_rows([free, priced], frozenset({("openrouter", "free/model")})) assert [row["is_free"] for row in rows] == [True, False] assert rows[1]["prompt_price_per_1k"] == 0.002 - assert "from contextual_orchestrator.orchestrator import ModelClient, TaskOrchestrator, load_agents" in text - assert "from contextual_orchestrator.server import SecurityConfig, serve" in text + assert ( + "from contextual_orchestrator.orchestrator import ( ModelClient, " + "TaskOrchestrator, load_agents, )" in normalized + ) + assert "from contextual_orchestrator.server import SecurityConfig, serve" in normalized assert 'parser.add_argument("--pool", choices=("free", "auto"), default="free")' in text assert "orchestrator/{args.pool} would fail closed" in text assert "scripts.ci.contextual_orchestrator_review_policy" in text @@ -479,7 +533,7 @@ def test_noema_review_workflow_provisions_sidecar_with_all_five_secrets() -> Non assert 'export NOEMA_LLM_MODEL="orchestrator/free"' in workflow assert "NOEMA_LLM_VIA_ORCHESTRATOR=1" in workflow assert "${CONTEXTUAL_ORCHESTRATOR_BASE_URL%/}/v1/chat/completions" in workflow - assert "${CONTEXTUAL_ORCHESTRATOR_TOKEN}" in workflow + assert 'NOEMA_LLM_API_KEY="$CONTEXTUAL_ORCHESTRATOR_TOKEN"' in workflow assert "https://integrate.api.nvidia.com" not in workflow assert "nvidia/nemotron-3-ultra-550b-a55b" not in workflow assert "COPILOT_GITHUB_TOKEN" not in workflow @@ -494,7 +548,8 @@ def test_noema_private_targets_require_zdr_only_sidecar_routing() -> None: launcher = _read(LAUNCHER) assert "Resolve Noema target repository visibility" in workflow - assert "target_visibility.outputs.require_zdr" in workflow + assert "steps.target_visibility.outputs.require_zdr" in workflow + assert "needs.prepare.outputs.require_zdr" in workflow assert "CONTEXTUAL_ORCHESTRATOR_REQUIRE_ZDR" in workflow assert "CONTEXTUAL_ORCHESTRATOR_REQUIRE_ZDR" in sidecar assert "--require-zdr" in sidecar diff --git a/tests/test_noema_orchestrator_workflow_contract.py b/tests/test_noema_orchestrator_workflow_contract.py index 3b9c5baeef..89acad4c8a 100644 --- a/tests/test_noema_orchestrator_workflow_contract.py +++ b/tests/test_noema_orchestrator_workflow_contract.py @@ -36,7 +36,7 @@ def test_noema_close_cleanup_selects_only_the_closed_pr_across_shared_display_ti workflow_step( workflow_text("noema-review.yml"), "Cancel queued and running Noema reviews for the closed pull request", - ).split(" run: |\n", 1)[1].split("\n noema-review:", 1)[0] + ).split(" run: |\n", 1)[1].split("\n prepare:", 1)[0] ) workflow_path = ".github/workflows/noema-review.yml" runs = { @@ -140,6 +140,61 @@ def test_noema_close_cleanup_selects_only_the_closed_pr_across_shared_display_ti assert "/actions/runs/105/cancel" not in calls +def test_noema_close_cleanup_retries_a_transient_cancel_failure(tmp_path: Path) -> None: + """A failed close cancellation remains eligible in the bounded rescan.""" + script = textwrap.dedent( + workflow_step( + workflow_text("noema-review.yml"), + "Cancel queued and running Noema reviews for the closed pull request", + ).split(" run: |\n", 1)[1].split("\n prepare:", 1)[0] + ) + fixture = tmp_path / "runs.json" + fixture.write_text( + json.dumps({"workflow_runs": [{ + "id": 101, + "path": ".github/workflows/noema-review.yml", + "name": "Required Noema Review", + "display_title": "Required Noema Review ContextualWisdomLab/demo#7@old", + "pull_requests": [{"number": 7}], + }]}), + encoding="utf-8", + ) + attempts = tmp_path / "attempts" + fake_gh = tmp_path / "gh" + fake_gh.write_text( + """#!/usr/bin/env bash +set -euo pipefail +if [[ "$*" == *"actions/runs?status="* ]]; then cat "$FAKE_RUNS_FILE"; exit 0; fi +if [[ "$*" == *"/actions/runs/101/cancel"* ]]; then + count=0; [[ ! -f "$ATTEMPTS_FILE" ]] || count="$(cat "$ATTEMPTS_FILE")" + count=$((count + 1)); printf '%s' "$count" >"$ATTEMPTS_FILE" + [[ "$count" -gt 1 ]] + exit +fi +exit 1 +""", + encoding="utf-8", + ) + fake_gh.chmod(0o755) + result = subprocess.run( # noqa: S603 + [shutil.which("bash") or "/bin/bash", "-c", script], + env={ + **os.environ, + "PATH": f"{tmp_path}{os.pathsep}{os.environ.get('PATH', '')}", + "TARGET_REPOSITORY": "ContextualWisdomLab/demo", + "CLOSED_PR_NUMBER": "7", + "CURRENT_RUN_ID": "999", + "FAKE_RUNS_FILE": str(fixture), + "ATTEMPTS_FILE": str(attempts), + }, + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 0, result.stderr + assert attempts.read_text(encoding="utf-8") == "2" + + def test_noema_review_credentials_and_llm_use_orchestrator_free() -> None: """Require reviewer credentials and the sidecar; the public NIM hardcode is gone.""" workflow = workflow_text("noema-review.yml") @@ -152,14 +207,11 @@ def test_noema_review_credentials_and_llm_use_orchestrator_free() -> None: "NOEMA_GITHUB_APP_PRIVATE_KEY, NOEMA_REVIEW_TOKEN, or NOEMA_TOKEN_EXCHANGE_URL. " "Review cannot be skipped." ) in workflow - assert ( - "Noema reviewer credential selection succeeded but no token was minted" - in workflow - ) assert "https://integrate.api.nvidia.com/v1/chat/completions" not in workflow assert "nvidia/nemotron-3-ultra-550b-a55b" not in workflow assert "Resolve Noema target repository visibility" in workflow - assert "target_visibility.outputs.require_zdr" in workflow + assert "steps.target_visibility.outputs.require_zdr" in workflow + assert "needs.prepare.outputs.require_zdr" in workflow assert "CONTEXTUAL_ORCHESTRATOR_REQUIRE_ZDR" in workflow assert ( "NOEMA_LLM_API_KEY: ${{ secrets.NOEMA_LLM_API_KEY || secrets.OPENAI_API_KEY || '' }}" @@ -172,6 +224,36 @@ def test_noema_review_credentials_and_llm_use_orchestrator_free() -> None: assert "OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}" in workflow assert "OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}" in workflow assert 'export NOEMA_LLM_MODEL="orchestrator/free"' in workflow + assert "candidate-1:" in workflow + assert "candidate-2:" in workflow + assert "finalize:" in workflow + assert workflow.count("timeout-minutes: 335") == 2 + assert workflow.count("timeout-minutes: 350") == 2 + assert 'REVIEW_PREFLIGHT_GATEWAY_MAX_TIME_SECONDS: "600"' in workflow + assert workflow.count( + "contextual_orchestrator_review_sidecar.sh\" --single-candidate-attempt" + ) == 2 + assert "Guarantee first candidate status handoff" in workflow + assert ': >"${RUNNER_TEMP}/candidate-1.id"' in workflow + first_upload = workflow_step(workflow, "Upload first candidate handoff") + assert "if: always()" in first_upload + assert "if-no-files-found: error" in first_upload + first_provision = workflow_step(workflow, "Provision candidate pool") + assert "id: provision" in first_provision + assert "continue-on-error: true" in first_provision + first_run = workflow_step(workflow, "Run first candidate") + assert "if: steps.provision.outcome == 'success'" in first_run + assert "continue-on-error: true" in first_run + second_run = workflow_step(workflow, "Run second candidate") + second_provision = workflow_step(workflow, "Provision fallback candidate pool") + assert "continue-on-error: true" not in second_provision + assert "continue-on-error: true" not in second_run + assert 'cat "${RUNNER_TEMP}/candidate-1/candidate-1.id" 2>/dev/null || true' in second_run + assert "NOEMA_LLM_CANDIDATE_ID" in workflow + assert "CONTEXTUAL_ORCHESTRATOR_EXCLUDE_CANDIDATE_ID" in workflow + assert "NOEMA_LLM_EXCLUDE_CANDIDATE_IDS" not in workflow + assert "needs.prepare.outputs.review_ready == 'true'" in workflow + assert 'review_ready: ${{ steps.seal.outputs.review_ready }}' in workflow assert "python3 -m scripts.ci.noema_review_gate" in workflow assert "python3 scripts/ci/noema_review_gate.py" not in workflow assert ( @@ -185,6 +267,120 @@ def test_noema_review_credentials_and_llm_use_orchestrator_free() -> None: assert "secrets: inherit" not in workflow +def test_peer_workflow_completion_does_not_cancel_long_noema_review() -> None: + """Only a new PR head or explicit retry may supersede a running review.""" + workflow = workflow_text("noema-review.yml") + + assert ( + "cancel-in-progress: ${{ github.event_name != 'workflow_run' || " + "github.event.workflow_run.conclusion != 'cancelled' }}" + ) in workflow + + +def test_noema_prepare_and_superseded_cleanup_preserve_exact_head_binding() -> None: + """Only a validated live PR trigger may cancel bounded older-head runs.""" + workflow = workflow_text("noema-review.yml") + seal = workflow_step(workflow, "Seal exact-head Noema review input") + cleanup = workflow_step( + workflow, "Cancel superseded Noema runs after live-head validation" + ) + + assert '--expected-head "$EXPECTED_HEAD"' in seal + assert "if: github.event_name == 'pull_request_target' && env.PR_NUMBER != ''" in cleanup + assert 'select(.id < $current)' in cleanup + assert 'select(((.head_sha // "") | ascii_downcase) != ($head | ascii_downcase))' in cleanup + assert cleanup.index('live_head="$(gh api') < cleanup.index('/actions/runs/${run_id}/cancel') + assert cleanup.index('seen[$run_id]=1') > cleanup.index('/actions/runs/${run_id}/cancel') + + +def test_superseded_cleanup_retries_a_transient_cancel_failure(tmp_path: Path) -> None: + """Do not mark an older-head run seen until GitHub accepts cancellation.""" + script = textwrap.dedent( + workflow_step( + workflow_text("noema-review.yml"), + "Cancel superseded Noema runs after live-head validation", + ).split(" run: |\n", 1)[1] + ) + old_head, current_head = "a" * 40, "b" * 40 + runs_file = tmp_path / "runs.json" + runs_file.write_text( + json.dumps({ + "workflow_runs": [{ + "id": 100, + "path": ".github/workflows/noema-review.yml", + "name": "Required Noema Review", + "display_title": ( + "Required Noema Review ContextualWisdomLab/demo#7@" + old_head + ), + "head_sha": old_head, + "pull_requests": [{"number": 7}], + }] + }), + encoding="utf-8", + ) + attempts = tmp_path / "attempts" + fake_gh = tmp_path / "gh" + fake_gh.write_text( + """#!/usr/bin/env bash +set -euo pipefail +if [[ "$*" == *"actions/runs?status="* ]]; then cat "$FAKE_RUNS_FILE"; exit 0; fi +if [[ "$*" == *"/pulls/7"* ]]; then printf '%s\n' "$EXPECTED_HEAD"; exit 0; fi +if [[ "$*" == *"/actions/runs/100/cancel"* ]]; then + count=0; [[ ! -f "$ATTEMPTS_FILE" ]] || count="$(cat "$ATTEMPTS_FILE")" + count=$((count + 1)); printf '%s' "$count" >"$ATTEMPTS_FILE" + [[ "$count" -gt 1 ]] + exit +fi +exit 1 +""", + encoding="utf-8", + ) + fake_gh.chmod(0o755) + + result = subprocess.run( # noqa: S603 + [shutil.which("bash") or "/bin/bash", "-c", script], + env={ + **os.environ, + "PATH": f"{tmp_path}{os.pathsep}{os.environ.get('PATH', '')}", + "TARGET_REPOSITORY": "ContextualWisdomLab/demo", + "PR_NUMBER": "7", + "EXPECTED_HEAD": current_head, + "CURRENT_RUN_ID": "200", + "FAKE_RUNS_FILE": str(runs_file), + "ATTEMPTS_FILE": str(attempts), + }, + capture_output=True, + text=True, + check=False, + ) + + assert result.returncode == 0, result.stderr + assert attempts.read_text(encoding="utf-8") == "2" + + +def test_noema_normalizes_github_app_identity_in_both_phases() -> None: + """Preparation and finalization must satisfy current_actor's source contract.""" + workflow = workflow_text("noema-review.yml") + + assert ( + "steps.noema_credential.outputs.source == 'github-app' && " + "'noema-review-github-app'" + ) in workflow + assert ( + "steps.credential.outputs.source == 'github-app' && " + "'noema-review-github-app'" + ) in workflow + + +def test_noema_noop_events_do_not_download_missing_handoffs() -> None: + workflow = workflow_text("noema-review.yml") + assert "review_ready: ${{ steps.seal.outputs.review_ready }}" in workflow + assert "if: steps.seal.outputs.review_ready == 'true'" in workflow + assert "if: needs.prepare.outputs.review_ready == 'true'" in workflow + assert ( + "if: always() && needs.prepare.result == 'success' && " + "needs.prepare.outputs.review_ready == 'true'" + ) in workflow def _expected_head_from_workflow_run_event(event: dict) -> str: """Mirror EXPECTED_HEAD's ``||`` fallback chain for a ``workflow_run`` event. @@ -318,7 +514,7 @@ def test_noema_visibility_lookup_retries_transient_api_failures() -> None: """Bound transient GitHub API failures without weakening visibility validation.""" workflow = workflow_text("noema-review.yml") start = workflow.index(" - name: Resolve Noema target repository visibility") - end = workflow.index(" - name: Provision contextual-orchestrator review sidecar", start) + end = workflow.index(" - name: Seal exact-head Noema review input", start) visibility_step = workflow[start:end] assert "for target_visibility_attempt in 1 2 3 4 5 6; do" in visibility_step @@ -369,12 +565,9 @@ def test_strix_gateway_default_and_noema_sidecar_fail_closed(tmp_path: Path) -> in workflow_text("strix.yml") ) - noema_script = textwrap.dedent( - workflow_step( - workflow_text("noema-review.yml"), - "Run Noema LLM review and submit verdict", - ).split(" run: |\n", 1)[1] - ) + noema_script = textwrap.dedent(workflow_step( + workflow_text("noema-review.yml"), "Run first candidate" + ).split(" run: |\n", 1)[1]) noema_env = { **os.environ, "PR_NUMBER": "1", @@ -395,4 +588,4 @@ def test_strix_gateway_default_and_noema_sidecar_fail_closed(tmp_path: Path) -> check=False, ) assert noema.returncode == 1 - assert "sidecar must be provisioned before Noema LLM review" in noema.stdout + assert noema.returncode != 0 diff --git a/tests/test_noema_review_gate.py b/tests/test_noema_review_gate.py index 5229605627..81a914aa77 100644 --- a/tests/test_noema_review_gate.py +++ b/tests/test_noema_review_gate.py @@ -1,607 +1,12 @@ import base64 -import hashlib import json -import os -import shlex -import shutil -import subprocess import sys -import textwrap -from pathlib import Path import pytest from scripts.ci import noema_review_gate as noema -def test_gitleaks_ignore_is_exactly_scoped_to_superseded_uuid_fixture(): - entries = { - line - for line in Path(".gitleaksignore").read_text(encoding="utf-8").splitlines() - if line and not line.startswith("#") - } - fingerprint = ( - "6657eb76f0e2cf6dab9197cfa861a1f584653aba:" - "tests/test_noema_review_gate.py:generic-api-key:187" - ) - assert fingerprint in entries - assert sum("tests/test_noema_review_gate.py" in entry for entry in entries) == 1 - - -def test_noema_concurrency_and_live_head_cleanup_preserve_current_review(): - """Pin the invariants this cancellation mechanism must hold together. - - Several Devin Review rounds landed on this same cancellation mechanism in - one day (see the matching ``docs/product-technical-gap-baseline.md`` - entry for the full narrative), each closing a gap the previous fix left - open: - - 1. A live new-head trigger must cancel a still-running older-head run of - the same PR (proven end to end by - ``test_superseded_cleanup_preserves_current_and_newer_run_ids``, - executing the real production jq selector). - 2. A delayed ``workflow_run``/``repository_dispatch`` completion for an - OLDER head must never cancel a genuinely current run -- pinned here by - the head-inclusive concurrency group assertions below (native - protection, independent of this step) AND by the step-level ``if:`` - gate restricting this explicit cancellation entirely to live - ``pull_request_target`` triggers, so a workflow_run/repository_dispatch - execution never even reaches this step. - 3. A cancellation step whose OWN trigger was confirmed live at the start - of the job must still never cancel a run dispatched AFTER its own - dispatch, even though its own multi-pass scan can take long enough in - wall-clock time for such a run to appear in the active-runs listing: - proven by ``test_superseded_cleanup_preserves_current_and_newer_run_ids`` - (a higher run id survives) and pinned structurally here via the - ``.id < $current`` ordering guard plus the per-cancellation live-head - re-check. - 4. That live-head re-check is a housekeeping safeguard, not the review - itself: a transient failure reading it must stop cleanup without - crashing the step (and thus the whole job) -- proven by - ``test_superseded_cleanup_survives_a_transient_live_head_lookup_failure``. - """ - workflow = Path(".github/workflows/noema-review.yml").read_text(encoding="utf-8") - concurrency = workflow.split("concurrency:", 1)[1].split("permissions:", 1)[0] - assert "github.event.client_payload.pr_head_sha" in concurrency - assert "github.event.pull_request.head.sha" in concurrency - assert "github.event.workflow_run.pull_requests[0].head.sha" in concurrency - assert "github.event.workflow_run.head_sha" not in concurrency - assert "github.event.workflow_run.conclusion == 'cancelled'" in concurrency - assert "format('cancelled-{0}', github.run_id)" in concurrency - assert "'actionable'" in concurrency - assert "cancel-in-progress: ${{" in concurrency - assert "github.event_name != 'workflow_run'" in concurrency - assert "github.event.workflow_run.conclusion != 'cancelled'" in concurrency - assert "Cancel superseded Noema runs after live-head validation" in workflow - assert workflow.index("Reject a stale trigger before credential or model setup") < workflow.index( - "Cancel superseded Noema runs after live-head validation" - ) - cleanup = workflow.split("Cancel superseded Noema runs after live-head validation", 1)[1] - job_header = workflow.split("\n noema-review:", 1)[1].split(" steps:", 1)[0] - assert "actions: write" in job_header - # Invariant 2 (step-level half): only a live pull_request_target trigger - # may even attempt this cancellation -- workflow_run and - # repository_dispatch executions (which can legitimately be delayed by - # hours) skip this step entirely and rely solely on the head-inclusive - # concurrency group above. - assert ( - "if: github.event_name == 'pull_request_target' && env.PR_NUMBER != ''" - in cleanup - ) - assert 'select(.id < $current)' in cleanup - # The live-head re-check must be error-guarded (an `if !` command - # substitution), never a bare assignment under set -euo pipefail -- a - # transient failure here must stop cleanup, not crash the whole job. - assert cleanup.count('live_head="$(gh api "repos/${TARGET_REPOSITORY}/pulls/${PR_NUMBER}"') == 1 - assert ( - 'if ! live_head="$(gh api "repos/${TARGET_REPOSITORY}/pulls/${PR_NUMBER}" --jq \'.head.sha\'' - in cleanup - ) - assert "could not re-verify the live PR head before cancelling" in cleanup - assert '"${live_head,,}" != "${EXPECTED_HEAD,,}"' in cleanup - assert 'endswith("@" + $head)' in cleanup - assert "| not)" in cleanup - - -def test_noema_superseded_cleanup_selects_only_other_heads_of_same_pr(): - """Execute the workflow's jq selector against current, sibling, and foreign runs. - - ``$current`` must be passed with ``--argjson`` (a number), matching the - production invocation (``--argjson current "$CURRENT_RUN_ID"``): jq's - type ordering ranks every number below every string, so passing it as a - string via ``--arg`` would make the selector's directional ``.id < - $current`` guard vacuously true for every fixture row regardless of the - actual id values, silently proving nothing about that guard (caught by - review on PR #1507). With the numeric type restored, the fixture's ids - must also be realistic: GitHub Actions run ids increase monotonically - over time, so the "current" run (the latest trigger) has the *highest* - id here, and the superseded same-PR sibling has a lower one — the - opposite of this fixture's original (also-wrong) ordering, under which - the directional guard's own vacuous-true bug happened to still produce - the expected output for an unrelated reason.""" - jq = shutil.which("jq") - if jq is None: - pytest.skip("jq is required to execute the production cleanup selector") - workflow = Path(".github/workflows/noema-review.yml").read_text(encoding="utf-8") - start_marker = '--arg target "$TARGET_REPOSITORY" --arg head "$EXPECTED_HEAD" \'\n' - start = workflow.index(start_marker) + len(start_marker) - end = workflow.index('\n \' <<<"$runs_json"', start) - selector = workflow[start:end] - workflow_path = ".github/workflows/noema-review.yml" - runs = { - "workflow_runs": [ - {"id": 98, "path": workflow_path, "name": "Required Noema Review", "display_title": "Required Noema Review owner/repo#7@old"}, - {"id": 99, "path": workflow_path, "name": "Required Noema Review", "display_title": "Required Noema Review owner/repo#8@old"}, - {"id": 100, "path": workflow_path, "name": "Required Noema Review", "display_title": "Required Noema Review owner/repo#7@current"}, - {"id": 97, "name": "Other", "display_title": "Required Noema Review owner/repo#7@old"}, - ] - } - result = subprocess.run( - [jq, "-r", "--arg", "pr", "7", "--argjson", "current", "100", "--arg", "target", "owner/repo", "--arg", "head", "current", selector], - input=json.dumps(runs), - text=True, - capture_output=True, - check=True, - ) - assert result.stdout.splitlines() == ["98"] - assert "github.event.workflow_run.head_sha" not in workflow - assert "EXPECTED_HEAD:" in workflow - assert "--expected-head \"$EXPECTED_HEAD\"" in workflow - assert '"${live_head,,}" != "${EXPECTED_HEAD,,}"' in workflow - assert workflow.index("Reject a stale trigger before credential or model setup") < workflow.index( - "Select fail-closed Noema reviewer credential" - ) - - -def test_noema_superseded_cleanup_matches_a_sibling_run_by_pull_requests_array(): - """A sibling-repo run whose display_title never rendered is still matched. - - Devin Review, PR #1507 ("Sibling Noema runs evade cancellation"): a - required-workflow-ruleset run materialized in a sibling repository can - carry the bare workflow name in ``name`` and the plain PR title (not - this workflow's rendered run-name) in ``display_title`` -- exactly the - shape ``tests/test_opencode_required_verdict_regression.py`` documents - for the analogous OpenCode wake selector, and confirmed live against - real sibling-repository runs during this fix. The selector must still - match such a run via GitHub's own ``pull_requests[]`` array and exclude - the live head via the direct ``head_sha`` comparison, since the head is - also never embedded in a display_title that never rendered it. - """ - jq = shutil.which("jq") - if jq is None: - pytest.skip("jq is required to execute the production cleanup selector") - workflow = Path(".github/workflows/noema-review.yml").read_text(encoding="utf-8") - start_marker = '--arg target "$TARGET_REPOSITORY" --arg head "$EXPECTED_HEAD" \'\n' - start = workflow.index(start_marker) + len(start_marker) - end = workflow.index('\n \' <<<"$runs_json"', start) - selector = workflow[start:end] - workflow_path = ".github/workflows/noema-review.yml" - current_head = "b" * 40 - old_head = "a" * 40 - runs = { - "workflow_runs": [ - { - "id": 98, - "path": workflow_path, - "name": "Required Noema Review", - "display_title": "Fix an unrelated example bug", - "head_sha": old_head, - "pull_requests": [{"number": 7}], - }, - { - "id": 99, - "path": workflow_path, - "name": "Required Noema Review", - "display_title": "A different pull request's title", - "head_sha": old_head, - "pull_requests": [{"number": 8}], - }, - { - "id": 100, - "path": workflow_path, - "name": "Required Noema Review", - "display_title": "Same PR, current push", - "head_sha": current_head, - "pull_requests": [{"number": 7}], - }, - { - "id": 97, - "path": ".github/workflows/strix.yml", - "name": "Required Noema Review", - "display_title": "Fix an unrelated example bug", - "head_sha": old_head, - "pull_requests": [{"number": 7}], - }, - ] - } - result = subprocess.run( - [ - jq, "-r", - "--arg", "pr", "7", - "--argjson", "current", "101", - "--arg", "target", "owner/repo", - "--arg", "head", current_head, - selector, - ], - input=json.dumps(runs), - text=True, - capture_output=True, - check=True, - ) - assert result.stdout.splitlines() == ["98"] - - -def test_noema_close_event_cancels_historical_head_runs(): - """Close cleanup must cancel active Noema runs across prior head groups.""" - workflow = Path(".github/workflows/noema-review.yml").read_text(encoding="utf-8") - cleanup = workflow.split(" cancel-closed-pr-runs:", 1)[1].split( - " noema-review:", 1 - )[0] - assert "actions: write" in cleanup - assert "Cancel queued and running Noema reviews for the closed pull request" in cleanup - assert 'select((.name // "") | startswith("Required Noema Review"))' in cleanup - assert 'select(.path == ".github/workflows/noema-review.yml")' in cleanup - assert "CLOSED_PR_NUMBER" in cleanup - assert "CURRENT_RUN_ID" in cleanup - assert "/actions/runs/${run_id}/cancel" in cleanup - # Devin Review finding on PR #1507 (bug 1, "Sibling Noema runs evade - # cancellation"): GitHub does not consistently render this workflow's - # run-name for an organization-required-workflow run materialized in a - # sibling repository, so display_title alone (an exact `.name ==` - # filter alone, too) can never match a sibling PR's runs. Selection is - # PR-scoped by two independent, OR'd signals: the generated - # display_title where GitHub does render it, and GitHub's own - # pull_requests[] array otherwise -- reliably populated here because - # this job only ever processes same-repository, non-fork pull requests - # (unlike the general cross-fork case elsewhere in this org's tooling, - # where pull_requests[] is documented to come back empty). Never a bare - # head_sha, which two different open PRs can share. - assert ".head_sha == $head_sha" not in cleanup - assert "--arg head_sha" not in cleanup - assert ( - '((.display_title // "") | startswith("Required Noema Review " + ' - '$target + "#" + $pr + "@"))' - ) in cleanup - assert ( - 'or ((.pull_requests // []) | any(.number == ($pr | tonumber)))' - ) in cleanup - # Devin Review finding on PR #1507 (bug 2): a single sequential sweep - # across the five active statuses could miss a run that transitioned - # between statuses mid-sweep. Re-scan until a pass converges, bounded. - # actions/runs (repo-wide, status server-filtered) is kept rather than - # an unfiltered actions/workflows/noema-review.yml/runs snapshot: that - # workflow-file-scoped endpoint is not guaranteed to resolve for - # sibling-repository runs, since noema-review.yml is never itself - # committed to those repositories (it applies there only through the - # organization's required-workflow ruleset). - assert 'runs_url="repos/${TARGET_REPOSITORY}/actions/runs?status=${status}&per_page=100"' in cleanup - assert "max_passes=3" in cleanup - assert 'while [ "$pass" -le "$max_passes" ]; do' in cleanup - assert 'if [ "$pass" -ge 2 ] && [ "$pass_matches" -eq 0 ] && [ "$found_any" -eq 0 ]; then' in cleanup - - -def _extract_run_block(workflow_text: str, step_name: str) -> str: - """Extract one step's ``run: |`` body from workflow YAML by indentation. - - Matches the extraction helper already used by - ``tests/test_opencode_workflow_shell_syntax.py`` and - ``tests/test_strix_repository_visibility_contract.py`` for the same - purpose: find the named step, locate its ``run: |`` block, and collect - lines until indentation returns to (or below) the block's own level -- - which correctly stops at the end of the block even when, as here, the - step is the last (only) one in its job and the next line at the step's - own indentation belongs to a different job entirely. - """ - lines = workflow_text.splitlines() - step_index = next( - index for index, line in enumerate(lines) if line.strip() == f"- name: {step_name}" - ) - run_index = next( - index - for index in range(step_index + 1, len(lines)) - if lines[index].strip() == "run: |" - ) - run_indent = len(lines[run_index]) - len(lines[run_index].lstrip()) - block_lines: list[str] = [] - for line in lines[run_index + 1 :]: - if line.strip() and len(line) - len(line.lstrip()) <= run_indent: - break - block_lines.append(line[run_indent + 2 :] if len(line) >= run_indent + 2 else "") - return "\n".join(block_lines) + "\n" - - -def _close_cleanup_script() -> str: - """Extract the close-cleanup step's real bash body from the workflow.""" - workflow = Path(".github/workflows/noema-review.yml").read_text(encoding="utf-8") - return _extract_run_block( - workflow, "Cancel queued and running Noema reviews for the closed pull request" - ) - - -def _superseded_cleanup_script() -> str: - """Extract the live-head supersession step's real bash body.""" - workflow = Path(".github/workflows/noema-review.yml").read_text(encoding="utf-8") - return _extract_run_block( - workflow, "Cancel superseded Noema runs after live-head validation" - ) - - -def test_superseded_cleanup_preserves_current_and_newer_run_ids(tmp_path: Path) -> None: - """Execute cleanup and cancel only the same PR's older, different-head run.""" - current_head = "b" * 40 - workflow_path = ".github/workflows/noema-review.yml" - runs = {"workflow_runs": [ - {"id": 100, "path": workflow_path, "name": "Required Noema Review", "display_title": "Required Noema Review ContextualWisdomLab/example#7@" + "a" * 40}, - {"id": 199, "path": workflow_path, "name": "Required Noema Review", "display_title": "Required Noema Review ContextualWisdomLab/example#7@" + current_head}, - {"id": 201, "path": workflow_path, "name": "Required Noema Review", "display_title": "Required Noema Review ContextualWisdomLab/example#7@" + "c" * 40}, - {"id": 99, "path": workflow_path, "name": "Required Noema Review", "display_title": "Required Noema Review ContextualWisdomLab/example#8@" + "a" * 40}, - ]} - fixture = tmp_path / "runs.json" - fixture.write_text(json.dumps(runs), encoding="utf-8") - calls = tmp_path / "calls.txt" - fake_gh = tmp_path / "gh" - fake_gh.write_text( - """#!/usr/bin/env bash -set -euo pipefail -printf '%s\n' "$*" >>"$FAKE_CALLS" -if [[ "$*" == *"/pulls/7"* ]]; then printf '%s\n' "$EXPECTED_HEAD"; exit 0; fi -if [[ "$*" == *"actions/runs?status="* ]]; then cat "$FAKE_RUNS"; exit 0; fi -""", - encoding="utf-8", - ) - fake_gh.chmod(0o755) - result = subprocess.run( # noqa: S603 - [shutil.which("bash") or "/bin/bash", "-c", _superseded_cleanup_script()], - env={**os.environ, "PATH": f"{tmp_path}{os.pathsep}{os.environ.get('PATH', '')}", - "TARGET_REPOSITORY": "ContextualWisdomLab/example", "PR_NUMBER": "7", - "EXPECTED_HEAD": current_head, "CURRENT_RUN_ID": "200", - "FAKE_RUNS": str(fixture), "FAKE_CALLS": str(calls)}, - capture_output=True, text=True, check=False, - ) - assert result.returncode == 0, result.stderr - recorded = calls.read_text(encoding="utf-8") - assert "/actions/runs/100/cancel" in recorded - assert "/actions/runs/199/cancel" not in recorded - assert "/actions/runs/201/cancel" not in recorded - assert "/actions/runs/99/cancel" not in recorded - - -def test_superseded_cleanup_survives_a_transient_live_head_lookup_failure( - tmp_path: Path, -) -> None: - """A transient live-head re-check failure must stop cleanup, not crash the step. - - The live-head re-check this step performs before every single - cancellation is a housekeeping safeguard, not the review itself. Before - this fix, `live_head="$(gh api ...)"` was an unguarded command - substitution under `set -euo pipefail`: a transient `gh api` failure - (rate limit, network blip) on that one call would exit the whole step - non-zero, failing this job and blocking a perfectly valid, live-head - Noema review over an ancillary API hiccup unrelated to the review - itself (Devin Review finding on PR #1507). The fix treats "cannot - verify" the same as "verified stale": stop cancelling further runs, but - exit 0 so the job -- and the actual review later in it -- proceeds. - """ - current_head = "b" * 40 - runs = { - "workflow_runs": [ - { - "id": 100, - "path": ".github/workflows/noema-review.yml", - "name": "Required Noema Review", - "display_title": "Required Noema Review ContextualWisdomLab/example#7@" + "a" * 40, - }, - ] - } - fixture = tmp_path / "runs.json" - fixture.write_text(json.dumps(runs), encoding="utf-8") - calls = tmp_path / "calls.txt" - fake_gh = tmp_path / "gh" - fake_gh.write_text( - """#!/usr/bin/env bash -set -euo pipefail -printf '%s\n' "$*" >>"$FAKE_CALLS" -if [[ "$*" == *"/pulls/7"* ]]; then echo "gh: transient error" >&2; exit 1; fi -if [[ "$*" == *"actions/runs?status="* ]]; then cat "$FAKE_RUNS"; exit 0; fi -""", - encoding="utf-8", - ) - fake_gh.chmod(0o755) - result = subprocess.run( # noqa: S603 - [shutil.which("bash") or "/bin/bash", "-c", _superseded_cleanup_script()], - env={**os.environ, "PATH": f"{tmp_path}{os.pathsep}{os.environ.get('PATH', '')}", - "TARGET_REPOSITORY": "ContextualWisdomLab/example", "PR_NUMBER": "7", - "EXPECTED_HEAD": current_head, "CURRENT_RUN_ID": "200", - "FAKE_RUNS": str(fixture), "FAKE_CALLS": str(calls)}, - capture_output=True, text=True, check=False, - ) - assert result.returncode == 0, ( - f"a transient live-head lookup failure must not crash this step " - f"(it would fail the whole job); stderr={result.stderr!r}" - ) - assert "/actions/runs/100/cancel" not in calls.read_text(encoding="utf-8") - assert "could not re-verify the live PR head" in result.stderr - - -def _write_fake_gh(tmp_path: Path, *, body: str) -> dict[str, str]: - """Write a fake `gh` executable and return a PATH-prefixed env base for it.""" - fake_gh = tmp_path / "gh" - fake_gh.write_text(f"#!/usr/bin/env bash\nset -euo pipefail\n{body}\n", encoding="utf-8") - fake_gh.chmod(0o755) - return { - **os.environ, - "PATH": f"{tmp_path}{os.pathsep}{os.environ.get('PATH', '')}", - "GH_TOKEN": "synthetic-token", - "TARGET_REPOSITORY": "ContextualWisdomLab/example", - "CLOSED_PR_NUMBER": "42", - "CURRENT_RUN_ID": "999", - } - - -def test_close_cleanup_selector_is_pr_scoped_not_head_sha_scoped(tmp_path: Path) -> None: - """Real jq execution: a shared head SHA must not leak cancellation across PRs. - - Devin Review finding on PR #1507 (bug 1). Two open pull requests (#42, - the one closing, and #43, unrelated) share one head commit -- a real, - if uncommon, GitHub scenario (e.g. a duplicate PR opened from the same - branch against a different target). Only PR #42's run may be cancelled; - PR #43's run, identical except for its PR association, must survive - untouched. This pipes representative run JSON through the workflow's - actual jq selector rather than grep-matching the YAML text. The fake - `gh` here answers every status query with the same fixture (status - filtering is not what this test is about); the status-filtering - contract is covered separately below. - """ - shared_head = "d" * 40 - fixture = { - "workflow_runs": [ - { - "id": 100, - "path": ".github/workflows/noema-review.yml", - "name": "Required Noema Review", - "display_title": ( - f"Required Noema Review ContextualWisdomLab/example#42@{shared_head}" - ), - }, - { - "id": 200, - "path": ".github/workflows/noema-review.yml", - "name": "Required Noema Review", - "display_title": ( - f"Required Noema Review ContextualWisdomLab/example#43@{shared_head}" - ), - }, - ] - } - fixture_path = tmp_path / "fixture.json" - fixture_path.write_text(json.dumps(fixture), encoding="utf-8") - cancel_log = tmp_path / "cancelled-run-ids.txt" - cancel_log.write_text("", encoding="utf-8") - - env = _write_fake_gh( - tmp_path, - body=textwrap.dedent( - f"""\ - if [ "$1" = api ] && [ "$2" = --paginate ]; then - cat {shlex.quote(str(fixture_path))} - exit 0 - elif [ "$1" = api ] && [ "$2" = --method ] && [ "$3" = POST ]; then - run_id="$(printf '%s' "$4" | sed -E 's#.*/runs/([0-9]+)/cancel#\\1#')" - printf '%s\\n' "$run_id" >> {shlex.quote(str(cancel_log))} - exit 0 - fi - echo "unexpected gh invocation: $*" >&2 - exit 1 - """ - ), - ) - - bash_executable = shutil.which("bash") or "/bin/bash" - result = subprocess.run( # noqa: S603 - [bash_executable, "-c", _close_cleanup_script()], - env=env, - capture_output=True, - text=True, - check=False, - ) - assert result.returncode == 0, result.stderr - - cancelled_ids = { - line.strip() for line in cancel_log.read_text(encoding="utf-8").splitlines() if line.strip() - } - assert cancelled_ids == {"100"}, ( - f"expected only PR #42's run (100) cancelled, got {cancelled_ids}; " - f"stderr={result.stderr}" - ) - - -def test_close_cleanup_survives_a_run_transitioning_between_active_statuses( - tmp_path: Path, -) -> None: - """Real bash execution: a run that changes status mid-sweep is still cancelled. - - Devin Review finding on PR #1507 (bug 2). A run for the closed PR is not - yet visible under any active status on the sweep's first pass (modeling - it being "requested" when the already-fetched "queued" list was read, - then becoming "queued" moments later, after the loop had already moved - past checking "queued" for that pass) and only becomes visible, under - "queued", starting with the *second* query for that status. A single - sequential sweep (the pre-fix behavior) would find zero matches and - leave this run running forever; the fixed multi-pass sweep must still - cancel it. This also exercises the status query parameter end to end - (the fake `gh` here filters by it, unlike the test above). - """ - fixture = { - "workflow_runs": [ - { - "id": 300, - "path": ".github/workflows/noema-review.yml", - "name": "Required Noema Review", - "display_title": ( - f"Required Noema Review ContextualWisdomLab/example#42@{'d' * 40}" - ), - } - ] - } - fixture_path = tmp_path / "fixture.json" - fixture_path.write_text(json.dumps(fixture), encoding="utf-8") - cancel_log = tmp_path / "cancelled-run-ids.txt" - cancel_log.write_text("", encoding="utf-8") - state_dir = tmp_path / "state" - state_dir.mkdir() - - env = _write_fake_gh( - tmp_path, - body=textwrap.dedent( - f"""\ - if [ "$1" = api ] && [ "$2" = --paginate ]; then - url="$3" - status="$(printf '%s' "$url" | sed -E 's/.*status=([a-z_]+)&.*/\\1/')" - counter_file={shlex.quote(str(state_dir))}"/count-${{status}}" - count=0 - [ -f "$counter_file" ] && count="$(cat "$counter_file")" - count=$((count + 1)) - printf '%s' "$count" > "$counter_file" - if [ "$status" = queued ] && [ "$count" -eq 2 ]; then - cat {shlex.quote(str(fixture_path))} - else - echo '{{"workflow_runs": []}}' - fi - exit 0 - elif [ "$1" = api ] && [ "$2" = --method ] && [ "$3" = POST ]; then - run_id="$(printf '%s' "$4" | sed -E 's#.*/runs/([0-9]+)/cancel#\\1#')" - printf '%s\\n' "$run_id" >> {shlex.quote(str(cancel_log))} - exit 0 - fi - echo "unexpected gh invocation: $*" >&2 - exit 1 - """ - ), - ) - - bash_executable = shutil.which("bash") or "/bin/bash" - result = subprocess.run( # noqa: S603 - [bash_executable, "-c", _close_cleanup_script()], - env=env, - capture_output=True, - text=True, - check=False, - ) - assert result.returncode == 0, result.stderr - - cancelled_ids = { - line.strip() for line in cancel_log.read_text(encoding="utf-8").splitlines() if line.strip() - } - assert cancelled_ids == {"300"}, ( - f"the status-transitioning run must still be cancelled; got {cancelled_ids}; " - f"stderr={result.stderr}" - ) - # Prove the race is real: pass 1 alone (the pre-fix, single-sweep - # behavior) found nothing, so only the fixed multi-pass loop caught it. - assert "pass 1/3 matched 0 run(s)" in result.stderr - assert "pass 2/3 matched 1 run(s)" in result.stderr - - def fake_secret(*parts: str) -> str: return "".join(parts) @@ -742,600 +147,6 @@ def app_identity(args, **kwargs): noema.extract_json_object("not-json") -def test_extract_json_object_balances_wrapped_and_multiple_objects(): - """Decode one complete object without joining unrelated brace-bearing text.""" - verdict = {"decision": "approve", "summary": "balanced { text }"} - with pytest.raises(RuntimeError, match="was not valid JSON"): - noema.extract_json_object( - "prose {not JSON} before " + json.dumps(verdict) + " after {brace prose}" - ) - assert noema.extract_json_object( - json.dumps(verdict) + "\n" + json.dumps({"decision": "comment"}) - ) == verdict - escaped = {"decision": "approve", "summary": 'escaped " { text }'} - assert noema.extract_json_object(json.dumps(escaped)) == escaped - - -def test_extract_json_object_rejects_approval_after_malformed_top_level_candidate(): - """A malformed first candidate must not release a later approval verdict.""" - with pytest.raises(RuntimeError, match="was not valid JSON"): - noema.extract_json_object( - '{"broken": invalid} {"decision":"approve","summary":"later"}' - ) - - -def test_extract_json_object_rejects_nested_recovery_from_malformed_outer_object(): - """A valid nested object must not escape its malformed outer object.""" - with pytest.raises(RuntimeError, match="was not valid JSON"): - noema.extract_json_object( - 'prefix {"broken": {"decision":"approve","summary":"nested"} trailing' - ) - - -def test_extract_json_object_rejects_nested_recovery_from_malformed_outer_array(): - """A valid nested object must not escape a malformed outer *array* either. - - Candidate discovery must track ``[``/``]`` depth alongside ``{``/``}``: - without it, the inner object's own ``{`` is wrongly seen at depth zero - (only brace nesting was tracked) and treated as a fresh top-level - candidate, letting a complete inner object "recover" out of an - unterminated outer array — the same class of bug - ``test_extract_json_object_rejects_nested_recovery_from_malformed_outer_object`` - covers for an outer object wrapper.""" - with pytest.raises(RuntimeError, match="was not valid JSON"): - noema.extract_json_object( - '[{"decision":"comment","summary":"ok","findings":[]}' - ) - - -@pytest.mark.parametrize("payload", ['[} {"decision":"approve"}', '{] {"decision":"approve"}']) -def test_extract_json_object_rejects_recovery_after_mismatched_delimiter(payload): - """A mismatched closer must not release a nested verdict candidate.""" - with pytest.raises(RuntimeError, match="was not valid JSON"): - noema.extract_json_object(payload) - - -def test_extract_json_object_rejects_nested_recovery_via_a_mismatched_closer(): - """A stray closer of the wrong bracket type must not fake-close a wrapper. - - Candidate discovery uses a bracket-*type* stack, not a plain up/down - counter: a ``]`` only pops an innermost ``[``, and a ``}`` only pops an - innermost ``{``. A plain counter that treated any closer as -1 would let - a mismatched closer (which cannot legitimately close the container it - appears in) prematurely signal "back to depth zero," so a later nested - recovery object's own ``{`` would wrongly be seen as a fresh top-level - candidate (Devin review on PR #1507). Covers both mismatch directions: - a stray ``]`` inside an unterminated ``{``, and a stray ``}`` inside an - unterminated ``[``.""" - with pytest.raises(RuntimeError, match="was not valid JSON"): - noema.extract_json_object( - '{"broken": ]{"decision":"comment","summary":"ok","findings":[]}' - ) - with pytest.raises(RuntimeError, match="was not valid JSON"): - noema.extract_json_object( - '{"broken": [}{"decision":"comment","summary":"ok","findings":[]}' - ) - - -def test_extract_json_object_stops_discovery_after_any_mismatched_closer(): - """A mismatched closer must poison the *rest* of discovery, not just the - bracket group it appears in. - - Merely making a mismatched closer a stack no-op (ignored rather than - popped) is not enough on its own: a later, otherwise-well-formed pair - can still validly re-close the stack down to empty despite the earlier - mismatch, so a subsequent { would again look like a fresh top-level - candidate. ``[} ] {...}`` -- the stray } is a no-op against the open [, - but the following ] still legitimately closes that [, and the { after - it would wrongly look top-level again if discovery kept scanning - (Devin review on PR #1507). No candidate must be found past the - mismatch at all.""" - with pytest.raises(RuntimeError, match="was not valid JSON"): - noema.extract_json_object( - '[} ] {"decision":"comment","summary":"ok","findings":[]}' - ) - with pytest.raises(RuntimeError, match="was not valid JSON"): - noema.extract_json_object( - '{] } {"decision":"comment","summary":"ok","findings":[]}' - ) - - -def test_json_nesting_within_bound_does_not_undercount_past_a_mismatched_closer(): - """A mismatched closer must not make the bound-check think a candidate - closed early, undercounting nesting that raw_decode would still walk - through when this candidate is actually decoded. Covers both mismatch - directions: a stray ``]`` inside an unterminated ``{``, and a stray - ``}`` inside an unterminated ``[``.""" - deep = "[" * 5 - stray_bracket = '{"a": ]' + deep + "0" + "]" * 5 + "}" - assert noema._json_nesting_within_bound(stray_bracket, 0, 100) is True - assert noema._json_nesting_within_bound(stray_bracket, 0, 3) is False - - stray_brace = '{"a": [}0]}' - assert noema._json_nesting_within_bound(stray_brace, 0, 100) is True - - -def test_extract_json_object_fails_closed_on_a_real_deep_payload(): - """A genuinely deep JSON payload must fail closed on this job's own runtime. - - Deliberately real input, not a monkeypatch: ``json.JSONDecoder.raw_decode``'s - own recursion behavior is not a stable contract across Python versions — - a real ``depth = max(20_000, sys.getrecursionlimit() * 2)`` nested array - raises ``RecursionError`` on Python 3.11-3.13 but decodes successfully - (no exception) on the Python 3.14 runner this job actually runs on (see - ``extract_json_object``'s docstring for the verifying CI evidence). This - test proves the *explicit* ``MAX_JSON_NESTING_DEPTH`` bound rejects real - excessive nesting regardless of which behavior the running interpreter - happens to have, rather than proving only that a raised RecursionError is - handled (see the sibling ``..._on_a_recursion_error_from_the_decoder`` - test below for that narrower, supplemental contract).""" - depth = max(20_000, sys.getrecursionlimit() * 2) - nested = '{"decision":' + ("[" * depth) + "0" + ("]" * depth) + "}" - with pytest.raises(RuntimeError, match="was not valid JSON"): - noema.extract_json_object(nested) - - -def test_extract_json_object_accepts_nesting_within_the_bound(): - """A legitimately nested verdict (well under the depth bound) still decodes.""" - nested = '{"decision":' + ("[" * 10) + "0" + ("]" * 10) + "}" - assert noema.extract_json_object(nested) == {"decision": json.loads("[" * 10 + "0" + "]" * 10)} - - -def test_json_nesting_within_bound_handles_escaped_quotes_inside_strings(): - """An escaped quote inside a string must not be mistaken for the string's - terminator: unrelated bracket characters that happen to follow inside the - same string value must not be miscounted as real nesting depth, or a - shallow, valid verdict would be wrongly rejected as excessively nested.""" - payload = '{"decision": "abc\\"' + ("[" * 200) + 'def"}' - assert json.loads(payload) == {"decision": 'abc"' + ("[" * 200) + "def"} - assert noema.extract_json_object(payload) == json.loads(payload) - - -def test_extract_json_object_fails_closed_on_a_recursion_error_from_the_decoder(monkeypatch): - """Supplemental coverage: an actual RecursionError from raw_decode (should - the running interpreter ever raise one within the bound) still reaches - the same bounded, scrubbed diagnostic as every other decode failure - here, on top of the explicit depth bound proven above.""" - def reject_deep_json(_decoder, _text, _start=0): - raise RecursionError("maximum recursion depth exceeded") - - monkeypatch.setattr(json.JSONDecoder, "raw_decode", reject_deep_json) - with pytest.raises(RuntimeError, match="was not valid JSON"): - noema.extract_json_object('{"item": {}}') - - -def test_extract_json_object_fails_closed_on_malformed_json(): - """A brace-wrapped but syntactically invalid LLM response must raise the - same fail-closed RuntimeError this module uses for other unusable-verdict - cases, never an unhandled json.JSONDecodeError (the reported CI crash). - - Devin Review security finding on PR #1507: the raised diagnostic must - never embed the raw (even scrubbed) model response, because this is a - public ``pull_request_target`` job and the finite scrub-pattern list - cannot guarantee an LLM-echoed or hallucinated credential in an - unrecognized shape is caught. Only a length and a content fingerprint - are logged.""" - # Reproduces "Expecting property name enclosed in double quotes": an - # unquoted/truncated key inside an otherwise brace-wrapped object. - malformed = '{"decision":"approve", trailing garbage not: "quoted}' - with pytest.raises(RuntimeError, match="was not valid JSON") as excinfo: - noema.extract_json_object(malformed) - assert not isinstance(excinfo.value, json.JSONDecodeError) - message = str(excinfo.value) - # The raw response text must never appear in the diagnostic. - assert "approve" not in message - assert "trailing garbage" not in message - # A bounded, non-secret correlation diagnostic replaces it instead. - assert f"response length={len(malformed)} chars" in message - assert "sha256=" in message - fingerprint = hashlib.sha256(malformed.encode("utf-8")).hexdigest()[:16] - assert fingerprint in message - - # A response truncated mid-object hits the same decode failure. - truncated = '{"decision":"approve","summary":"looks fine so far,' - with pytest.raises(RuntimeError, match="was not valid JSON"): - noema.extract_json_object(truncated) - - # A credential in a shape the finite scrub-pattern list does NOT - # recognize (no "token"/"key"/"bearer" marker, no known provider prefix - # — just a bare UUID-shaped value mid-sentence) must still never reach - # the raised message, because raw content is never embedded at all. - unrecognized_shape_secret = fake_secret( - "3f29e1a7-8b44-4c1d", "-9e77-2a5f9c001234" - ) - leaky = ( - '{"decision":"approve","summary":"use internal id ' - f"{unrecognized_shape_secret} to correlate, trailing garbage" - ) - # Confirm this test is not vacuous: the existing finite regex scrubber - # really does miss this shape. - assert unrecognized_shape_secret in (noema.scrub_sensitive_data(leaky) or "") - with pytest.raises(RuntimeError) as leaky_excinfo: - noema.extract_json_object(leaky) - leaky_message = str(leaky_excinfo.value) - assert unrecognized_shape_secret not in leaky_message - assert "approve" not in leaky_message - assert "ghp_" not in leaky_message - - # A known-shape secret (would have matched the old finite scrubber too) - # must also never appear, now that raw content is omitted outright. - known_shape_leaky = '{"decision":"approve","summary":"token ghp_' + "a" * 36 + '", bad' - with pytest.raises(RuntimeError) as known_excinfo: - noema.extract_json_object(known_shape_leaky) - assert "ghp_" not in str(known_excinfo.value) - - # Long malformed content produces a bounded diagnostic regardless of - # input size — never logged in full, and never truncated-and-embedded - # either; the diagnostic length does not grow with the input. - huge = '{"decision":"approve", ' + ("x" * 5000) + " bad" - with pytest.raises(RuntimeError) as huge_excinfo: - noema.extract_json_object(huge) - huge_message = str(huge_excinfo.value) - assert "x" * 100 not in huge_message - assert len(huge_message) < 500 - assert f"response length={len(huge)} chars" in huge_message - - # Devin Review follow-up finding: a malformed verdict containing an - # escaped lone surrogate (valid inside a Python/JSON string, but not - # representable in strict UTF-8) must not crash the fingerprint - # computation itself with an unhandled UnicodeEncodeError -- it must - # still fail closed with the same bounded RuntimeError. - surrogate_bearing = '{"decision":"approve", "note": "\ud800", trailing bad' - with pytest.raises(RuntimeError, match="was not valid JSON") as surrogate_excinfo: - noema.extract_json_object(surrogate_bearing) - assert not isinstance(surrogate_excinfo.value, UnicodeEncodeError) - surrogate_message = str(surrogate_excinfo.value) - assert "sha256=" in surrogate_message - assert f"response length={len(surrogate_bearing)} chars" in surrogate_message - - -def test_extract_llm_message_content_happy_paths(): - """A well-formed envelope returns its stripped content; a missing (not - malformed) choices/message/content field is treated leniently, matching - the pre-fix code's behavior for an absent field.""" - envelope = json.dumps({"choices": [{"message": {"content": " {\"decision\":\"approve\"} "}}]}) - assert noema.extract_llm_message_content(envelope) == '{"decision":"approve"}' - - assert noema.extract_llm_message_content(json.dumps({})) == "" - assert noema.extract_llm_message_content(json.dumps({"choices": []})) == "" - assert noema.extract_llm_message_content(json.dumps({"choices": [{}]})) == "" - assert noema.extract_llm_message_content(json.dumps({"choices": [{"message": None}]})) == "" - assert ( - noema.extract_llm_message_content(json.dumps({"choices": [{"message": {"content": None}}]})) - == "" - ) - - -def test_extract_llm_message_content_fails_closed_on_malformed_raw_body(): - """Devin Review bug finding on PR #1507: a malformed raw HTTP body must - raise the same bounded RuntimeError call_llm's repair path already uses - for a malformed verdict, never an unhandled json.JSONDecodeError.""" - with pytest.raises(RuntimeError, match="response body was not valid JSON"): - noema.extract_llm_message_content("not json at all") - - -@pytest.mark.parametrize("body", ["[]", "null", '"just a string"', "5"]) -def test_extract_llm_message_content_fails_closed_on_non_object_top_level(body): - """A syntactically valid but non-object top-level JSON value (array, - null, bare string, bare number) must fail closed instead of crashing on - the next `.get(...)` call, exactly as Devin's finding described.""" - with pytest.raises(RuntimeError, match="response body was not a JSON object"): - noema.extract_llm_message_content(body) - - -@pytest.mark.parametrize("choices", [{"a": 1}, "choices-as-string", 5]) -def test_extract_llm_message_content_fails_closed_on_wrong_shaped_choices(choices): - """A present-but-wrong-shaped (non-list) 'choices' field must fail - closed instead of crashing on `choices[0]`.""" - with pytest.raises(RuntimeError, match="'choices' was not a list"): - noema.extract_llm_message_content(json.dumps({"choices": choices})) - - -@pytest.mark.parametrize("first_choice", [None, 1, "text"]) -def test_extract_llm_message_content_fails_closed_on_wrong_shaped_choice_element(first_choice): - """A choices[0] that is not a JSON object must fail closed instead of - crashing on `.get("message")`.""" - with pytest.raises(RuntimeError, match=r"choices\[0\] was not a JSON object"): - noema.extract_llm_message_content(json.dumps({"choices": [first_choice]})) - - -@pytest.mark.parametrize("message", [[1, 2], "text", 5]) -def test_extract_llm_message_content_fails_closed_on_wrong_shaped_message(message): - """A present-but-wrong-shaped (non-object) 'message' field must fail - closed instead of crashing on `.get("content")`.""" - with pytest.raises(RuntimeError, match="'message' was not a JSON object"): - noema.extract_llm_message_content(json.dumps({"choices": [{"message": message}]})) - - -@pytest.mark.parametrize("content", [5, [1, 2], {"a": 1}]) -def test_extract_llm_message_content_fails_closed_on_non_string_content(content): - """A present-but-non-string 'content' field must fail closed instead of - crashing on `.strip()`.""" - with pytest.raises(RuntimeError, match="'content' was not a string"): - noema.extract_llm_message_content( - json.dumps({"choices": [{"message": {"content": content}}]}) - ) - - -def test_decode_llm_response_body_happy_path(): - """A well-formed UTF-8 response body decodes normally.""" - assert noema.decode_llm_response_body("hello world".encode("utf-8")) == "hello world" - - -def test_decode_llm_response_body_fails_closed_on_invalid_utf8(): - """Devin Review bug finding on PR #1507 round 3: a gateway reply - containing invalid UTF-8 must raise the same bounded RuntimeError - call_llm's repair path already uses for a malformed envelope, never an - unhandled UnicodeDecodeError. The raised message must never embed the - raw response bytes — even an attempted-decode fragment near the bad - byte — matching extract_json_object's no-raw-content pattern, since a - body containing invalid UTF-8 could still contain a credential-adjacent - byte sequence.""" - secret_like_prefix = b"token=ghp_deadbeef1234567890" - raw_bytes = secret_like_prefix + bytes([0xFF]) + b"unrecoverable tail bytes" - with pytest.raises(RuntimeError) as excinfo: - noema.decode_llm_response_body(raw_bytes) - message = str(excinfo.value) - assert "not valid UTF-8" in message - assert "ghp_" not in message - assert "unrecoverable tail" not in message - fingerprint = hashlib.sha256(raw_bytes).hexdigest()[:16] - assert f"response length={len(raw_bytes)} bytes" in message - assert f"sha256={fingerprint}" in message - - -def test_call_llm_repairs_one_malformed_envelope_before_failing_closed(monkeypatch): - """The envelope-level fail-closed path integrates with the existing - verdict-repair boundary: a malformed gateway reply gets one repair-retry - request before failing closed, exactly like a malformed verdict JSON - already does.""" - monkeypatch.setenv("NOEMA_LLM_API_URL", "https://llm.example/v1/chat/completions") - monkeypatch.setenv("NOEMA_LLM_API_KEY", "test-key") - bodies = iter( - ( - "not-json-at-all", - json.dumps( - { - "choices": [ - { - "message": { - "content": json.dumps( - {"decision": "comment", "summary": "Recovered", "findings": []} - ) - } - } - ] - } - ), - ) - ) - requests = [] - - class Response: - def __enter__(self): - return self - - def __exit__(self, *args): - return None - - def read(self): - return next(bodies).encode() - - def open_response(_opener, request, **_kwargs): - requests.append(json.loads(request.data)) - return Response() - - monkeypatch.setattr(noema.urllib.request.OpenerDirector, "open", open_response) - monkeypatch.setattr(noema, "fetch_pr", lambda repo, number: make_pr()) - - verdict = noema.call_llm("owner/repo", 7, make_pr(), "diff", False, "head") - - assert verdict["summary"] == "Recovered" - assert len(requests) == 2 - assert "prior verdict was rejected" in requests[1]["messages"][1]["content"] - - -def test_call_llm_skips_repair_retry_when_head_moves_before_it_fires(monkeypatch): - """CodeRabbit finding on PR #1507: ``expected_head`` is checked before - model work and before publication, but the one-time repair-retry request - inside ``call_llm`` used to fire unconditionally on a malformed first - verdict, even if the PR head had already moved. That burns a second, - potentially multi-hour ``NOEMA_LLM_TIMEOUT_SECONDS`` call on a review - ``inspect_and_review``'s own post-call stale-head check would discard - anyway. ``call_llm`` must instead re-check the live head via ``fetch_pr`` - before the retry request and fail closed with - ``StaleHeadDuringRepairRetryError`` — cleanly, not a crash — issuing only - the one doomed first request.""" - monkeypatch.setenv("NOEMA_LLM_API_URL", "https://llm.example/v1/chat/completions") - monkeypatch.setenv("NOEMA_LLM_API_KEY", "test-key") - open_calls = [] - - class Response: - def __enter__(self): - return self - - def __exit__(self, *args): - return None - - def read(self): - # Malformed: missing "choices" triggers call_llm's fail-closed - # RuntimeError path on the very first attempt. - return b"[]" - - def open_response(_opener, request, **_kwargs): - open_calls.append(request) - return Response() - - monkeypatch.setattr(noema.urllib.request.OpenerDirector, "open", open_response) - # The live PR head has moved on since the trigger fetched "head". - monkeypatch.setattr(noema, "fetch_pr", lambda repo, number: make_pr(headRefOid="new")) - - with pytest.raises(noema.StaleHeadDuringRepairRetryError, match="stale before repair retry"): - noema.call_llm("owner/repo", 7, make_pr(), "diff", False, "head") - # Only the first, already-doomed request was made — the repair-retry - # request never fired once the live head no longer matched. - assert len(open_calls) == 1 - - -def test_call_llm_still_repairs_once_when_head_has_not_moved(monkeypatch): - """A matching live head must not block the existing one-time repair - retry — this is a narrow addition to the existing repair boundary, not a - behavior change for the unstale case.""" - monkeypatch.setenv("NOEMA_LLM_API_URL", "https://llm.example/v1/chat/completions") - monkeypatch.setenv("NOEMA_LLM_API_KEY", "test-key") - contents = iter( - ( - "not-json-at-all", - json.dumps({"decision": "comment", "summary": "Recovered", "findings": []}), - ) - ) - open_calls = [] - - class Response: - def __enter__(self): - return self - - def __exit__(self, *args): - return None - - def read(self): - content = next(contents) - return json.dumps({"choices": [{"message": {"content": content}}]}).encode() - - def open_response(_opener, request, **_kwargs): - open_calls.append(request) - return Response() - - monkeypatch.setattr(noema.urllib.request.OpenerDirector, "open", open_response) - monkeypatch.setattr(noema, "fetch_pr", lambda repo, number: make_pr(headRefOid="head")) - - verdict = noema.call_llm("owner/repo", 7, make_pr(), "diff", False, "head") - - assert verdict["summary"] == "Recovered" - assert len(open_calls) == 2 - - -def test_inspect_and_review_reports_stale_before_repair_retry_cleanly(monkeypatch): - """``inspect_and_review`` must treat a stale-during-repair-retry signal - exactly like its own pre-model and pre-publication stale checks: a clean - skip (return 0), never an unhandled exception or a published review.""" - pr = make_pr() - monkeypatch.setattr(noema, "fetch_pr", lambda repo, number: pr) - monkeypatch.setattr(noema, "current_actor", lambda: "noema") - monkeypatch.setattr(noema, "fetch_diff", lambda repo, number: ("diff", False)) - monkeypatch.setattr(noema, "fetch_changed_file_paths", lambda repo, number: ["tool.py"]) - monkeypatch.setattr(noema, "build_review_context", lambda repo, number, value: "context") - - def fake_call_llm(*args, **kwargs): - raise noema.StaleHeadDuringRepairRetryError( - "Pull request head changed during review; stale before repair retry." - ) - - monkeypatch.setattr(noema, "call_llm", fake_call_llm) - monkeypatch.setattr( - noema, - "submit_review", - lambda *args, **kwargs: pytest.fail("stale-during-repair verdict must not publish"), - ) - - assert noema.inspect_and_review("owner/repo", 7, "head") == 0 - - -def test_call_llm_fails_closed_after_repeated_malformed_envelope(monkeypatch): - """Two consecutive malformed envelopes must produce a single clean - top-level RuntimeError diagnostic, never an unhandled traceback — but - the first still gets a repair-retry request like a malformed verdict - would.""" - monkeypatch.setenv("NOEMA_LLM_API_URL", "https://llm.example/v1/chat/completions") - monkeypatch.setenv("NOEMA_LLM_API_KEY", "test-key") - open_calls = [] - - class Response: - def __enter__(self): - return self - - def __exit__(self, *args): - return None - - def read(self): - # Top-level JSON is a bare list — no "choices" object to speak of. - return b"[]" - - def open_response(_opener, request, **_kwargs): - open_calls.append(request) - return Response() - - monkeypatch.setattr(noema.urllib.request.OpenerDirector, "open", open_response) - monkeypatch.setattr(noema, "fetch_pr", lambda repo, number: make_pr()) - - with pytest.raises(RuntimeError, match="response body was not a JSON object"): - noema.call_llm("owner/repo", 7, make_pr(), "diff", False, "head") - assert len(open_calls) == 2 - - -def test_call_llm_fails_closed_after_repeated_invalid_utf8_response(monkeypatch): - """Devin Review bug finding on PR #1507 round 3: a gateway reply - containing invalid UTF-8 bytes used to raise UnicodeDecodeError before - extract_llm_message_content or the verdict-JSON repair boundary ever - ran, crashing the required review check with an unhandled traceback. - It must instead integrate with the existing repair-retry boundary - exactly like a malformed JSON envelope already does: one repair-retry - request, then a single clean top-level RuntimeError when the retry - response is *also* invalid UTF-8 — never an unhandled traceback.""" - monkeypatch.setenv("NOEMA_LLM_API_URL", "https://llm.example/v1/chat/completions") - monkeypatch.setenv("NOEMA_LLM_API_KEY", "test-key") - open_calls = [] - - class Response: - def __enter__(self): - return self - - def __exit__(self, *args): - return None - - def read(self): - # Invalid UTF-8: a lone continuation byte with no lead byte. - return b"not utf-8 at all: \x80\x81\xfe" - - def open_response(_opener, request, **_kwargs): - open_calls.append(request) - return Response() - - monkeypatch.setattr(noema.urllib.request.OpenerDirector, "open", open_response) - monkeypatch.setattr(noema, "fetch_pr", lambda repo, number: make_pr()) - - with pytest.raises(RuntimeError, match="response body was not valid UTF-8"): - noema.call_llm("owner/repo", 7, make_pr(), "diff", False, "head") - # One initial request plus exactly one repair-retry request — not an - # unbounded retry loop, and not a crash on the first attempt. - assert len(open_calls) == 2 - assert "prior verdict was rejected" in json.loads(open_calls[1].data)["messages"][1]["content"] - - -@pytest.mark.parametrize("choices", [{"a": 1}, 5]) -def test_call_llm_fails_closed_on_wrong_shaped_gateway_choices(monkeypatch, choices): - """A malformed (non-list) choices field surfaces through call_llm's - fail-closed path rather than crashing the required review job.""" - monkeypatch.setenv("NOEMA_LLM_API_URL", "https://llm.example/v1/chat/completions") - monkeypatch.setenv("NOEMA_LLM_API_KEY", "test-key") - - class Response: - def __enter__(self): - return self - - def __exit__(self, *args): - return None - - def read(self): - return json.dumps({"choices": choices}).encode() - - monkeypatch.setattr(noema.urllib.request.OpenerDirector, "open", lambda *args, **kwargs: Response()) - monkeypatch.setattr(noema, "fetch_pr", lambda repo, number: make_pr()) - - with pytest.raises(RuntimeError, match="'choices' was not a list"): - noema.call_llm("owner/repo", 7, make_pr(), "diff", False, "head") - - @pytest.mark.parametrize( ("actor", "installation_id", "source"), [ @@ -1455,16 +266,15 @@ def read(self): def test_call_llm_handles_configuration_and_verdicts(monkeypatch): monkeypatch.setattr(noema, "validate_substantive_verdict", lambda *_args: None) pr = make_pr() - monkeypatch.setattr(noema, "fetch_pr", lambda repo, number: pr) monkeypatch.delenv("NOEMA_LLM_API_URL", raising=False) monkeypatch.delenv("NOEMA_LLM_API_KEY", raising=False) with pytest.raises(RuntimeError, match="not configured"): - noema.call_llm("owner/repo", 1, pr, "diff", False, "head") + noema.call_llm("owner/repo", 1, pr, "diff", False) monkeypatch.setenv("NOEMA_LLM_API_URL", "file:///etc/passwd") monkeypatch.setenv("NOEMA_LLM_API_KEY", "secret") with pytest.raises(ValueError, match="URL scheme must be http or https"): - noema.call_llm("owner/repo", 1, pr, "diff", False, "head") + noema.call_llm("owner/repo", 1, pr, "diff", False) monkeypatch.setenv("NOEMA_LLM_API_URL", "https://llm.example.test/chat") monkeypatch.setenv("NOEMA_LLM_API_KEY", "secret") @@ -1473,7 +283,6 @@ def test_call_llm_handles_configuration_and_verdicts(monkeypatch): def fake_urlopen(request, timeout): seen["url"] = request.full_url - seen["timeout"] = timeout seen["body"] = json.loads(request.data.decode("utf-8")) return FakeResponse( { @@ -1504,10 +313,9 @@ def open(self, request, timeout=None): return self.call_func(request, timeout) monkeypatch.setattr(noema.urllib.request, "build_opener", lambda *args: FakeOpener(fake_urlopen)) - verdict = noema.call_llm("owner/repo", 1, pr, "diff", True, "head", "extra review context") + verdict = noema.call_llm("owner/repo", 1, pr, "diff", True, "extra review context") assert verdict["decision"] == "approve" assert seen["url"] == "https://llm.example.test/chat" - assert seen["timeout"] == 14400 assert seen["body"]["model"] == "review-model" assert "extra review context" in seen["body"]["messages"][1]["content"] @@ -1520,32 +328,32 @@ def fake_urlopen_defer(request, timeout=None): lambda *args: FakeOpener(fake_urlopen_defer) ) with pytest.raises(RuntimeError, match="unsupported decision"): - noema.call_llm("owner/repo", 1, pr, "diff", False, "head") + noema.call_llm("owner/repo", 1, pr, "diff", False) # Test case-insensitive valid URL monkeypatch.setenv("NOEMA_LLM_API_URL", "HTTPS://llm.example.test/chat") monkeypatch.setattr(noema.urllib.request, "build_opener", lambda *args: FakeOpener(fake_urlopen)) - assert noema.call_llm("owner/repo", 1, pr, "diff", True, "head")["decision"] == "approve" + assert noema.call_llm("owner/repo", 1, pr, "diff", True)["decision"] == "approve" # Test invalid scheme (and no original URL in error) monkeypatch.setenv("NOEMA_LLM_API_URL", "file:///etc/passwd") with pytest.raises(ValueError, match="URL scheme must be http or https"): - noema.call_llm("owner/repo", 1, pr, "diff", False, "head") + noema.call_llm("owner/repo", 1, pr, "diff", False) # Test localhost rejection monkeypatch.setenv("NOEMA_LLM_API_URL", "http://localhost/chat") with pytest.raises(ValueError, match="URL cannot target localhost"): - noema.call_llm("owner/repo", 1, pr, "diff", False, "head") + noema.call_llm("owner/repo", 1, pr, "diff", False) # Test missing hostname monkeypatch.setenv("NOEMA_LLM_API_URL", "http:///chat") with pytest.raises(ValueError, match="URL must have a valid hostname"): - noema.call_llm("owner/repo", 1, pr, "diff", False, "head") + noema.call_llm("owner/repo", 1, pr, "diff", False) # Test internal IP rejection monkeypatch.setenv("NOEMA_LLM_API_URL", "http://169.254.169.254/chat") with pytest.raises(ValueError, match="URL cannot target internal IP addresses"): - noema.call_llm("owner/repo", 1, pr, "diff", False, "head") + noema.call_llm("owner/repo", 1, pr, "diff", False) import socket original_getaddrinfo = socket.getaddrinfo @@ -1558,7 +366,7 @@ def fake_getaddrinfo(host, port, *args, **kwargs): return original_getaddrinfo(host, port, *args, **kwargs) monkeypatch.setattr(socket, "getaddrinfo", fake_getaddrinfo) with pytest.raises(ValueError, match="URL cannot target internal IP addresses"): - noema.call_llm("owner/repo", 1, pr, "diff", False, "head") + noema.call_llm("owner/repo", 1, pr, "diff", False) # Test unresolved hostname does not break monkeypatch.setenv("NOEMA_LLM_API_URL", "http://unresolved.example.com/chat") @@ -1566,7 +374,7 @@ def fake_getaddrinfo_error(host, port, *args, **kwargs): raise socket.gaierror("Name or service not known") monkeypatch.setattr(socket, "getaddrinfo", fake_getaddrinfo_error) monkeypatch.setattr(noema.urllib.request, "build_opener", lambda *args: FakeOpener(fake_urlopen)) - assert noema.call_llm("owner/repo", 1, pr, "diff", True, "head")["decision"] == "approve" + assert noema.call_llm("owner/repo", 1, pr, "diff", True)["decision"] == "approve" # Test invalid IP string from getaddrinfo (unlikely but theoretically possible) monkeypatch.setenv("NOEMA_LLM_API_URL", "http://weird-dns.example.com/chat") @@ -1575,7 +383,154 @@ def fake_getaddrinfo_invalid_ip(host, port, *args, **kwargs): return [(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("not_an_ip", 0))] return original_getaddrinfo(host, port, *args, **kwargs) monkeypatch.setattr(socket, "getaddrinfo", fake_getaddrinfo_invalid_ip) - assert noema.call_llm("owner/repo", 1, pr, "diff", True, "head")["decision"] == "approve" + assert noema.call_llm("owner/repo", 1, pr, "diff", True)["decision"] == "approve" + + +def test_call_llm_selects_direct_route_for_the_process_local_sidecar(monkeypatch): + """The sidecar-backed Noema request must bypass the gateway's auto triage.""" + pr = make_pr() + monkeypatch.setenv("CONTEXTUAL_ORCHESTRATOR_BASE_URL", "http://127.0.0.1:18080") + monkeypatch.setenv("NOEMA_LLM_API_URL", "http://127.0.0.1:18080/v1/chat/completions") + monkeypatch.setenv("NOEMA_LLM_API_KEY", "secret") + monkeypatch.setenv("NOEMA_LLM_CANDIDATE_ID", "candidate-one") + monkeypatch.setenv("NOEMA_LLM_EXCLUDE_CANDIDATE_IDS", "candidate-zero") + seen = {} + + def fake_urlopen(request, timeout): + seen["body"] = json.loads(request.data.decode("utf-8")) + return FakeResponse({"choices": [{"message": {"content": '{"decision":"comment","summary":"ok","findings":[]}'}}]}) + + class FakeOpener: + def open(self, request, timeout=None): + return fake_urlopen(request, timeout) + + monkeypatch.setattr(noema.urllib.request, "build_opener", lambda *args: FakeOpener()) + # decision="comment" short-circuits validate_substantive_verdict's + # changed-line/adversarial-evidence requirements (see + # test_call_llm_rejects_generic_approve_without_changed_line_evidence + # below, which expects the placeholder "diff" fixture used here to + # raise for a formal approve/request_changes decision) -- this test's + # actual subject is the sidecar direct-route orchestration mode, not + # verdict-schema validation, so a real "approve"/"request_changes" + # verdict satisfying that unrelated validation is deliberately avoided. + assert noema.call_llm("owner/repo", 1, pr, "diff", False)["decision"] == "comment" + assert seen["body"]["orchestration"] == "route" + assert seen["body"]["routing"] == { + "candidate_id": "candidate-one", + "exclude_candidate_ids": ["candidate-zero"], + } + + +def test_call_llm_uses_the_enumerated_combined_worst_case_timeout(monkeypatch): + """The client-side read timeout must match the enumerated sidecar worst case. + + Regression test for contextual-orchestrator#946's four consecutive + ``noema-review`` ``TimeoutError`` failures and contextual-orchestrator#974's + worst-case enumeration: a plain ``timeout=120`` here raced the sidecar's + own internal per-candidate budget with zero margin, and could not survive + even one candidate needing a legitimate cross-candidate failover. This + must stay exactly ``CALL_LLM_TIMEOUT_SECONDS`` -- see that constant's own + comment for the enumerated derivation -- so a change to either side of + the mismatch is caught here rather than rediscovered via a live CI outage. + """ + monkeypatch.setenv("NOEMA_LLM_API_URL", "https://llm.example/v1/chat/completions") + monkeypatch.setenv("NOEMA_LLM_API_KEY", "secret") + seen = {} + + def fake_urlopen(request, timeout): + seen["timeout"] = timeout + return FakeResponse({"choices": [{"message": {"content": '{"decision":"comment","summary":"ok","findings":[]}'}}]}) + + class FakeOpener: + def open(self, request, timeout=None): + return fake_urlopen(request, timeout) + + monkeypatch.setattr(noema.urllib.request, "build_opener", lambda *args: FakeOpener()) + noema.call_llm("owner/repo", 1, make_pr(), "diff", False) + assert 0 < seen["timeout"] <= noema.CALL_LLM_TIMEOUT_SECONDS == 19800 + + +def test_call_llm_correction_shares_the_original_response_deadline(monkeypatch): + """A validator repair cannot restart the workflow's complete LLM budget.""" + monkeypatch.setenv("NOEMA_LLM_API_URL", "https://llm.example/v1/chat/completions") + monkeypatch.setenv("NOEMA_LLM_API_KEY", "secret") + monotonic = iter((100.0, 150.0, 200.0)) + timeouts = [] + validations = 0 + + class FakeOpener: + def open(self, request, timeout=None): + timeouts.append(timeout) + return FakeResponse({"choices": [{"message": {"content": '{"decision":"comment","summary":"ok","findings":[]}'}}]}) + + def validate(*args): + nonlocal validations + validations += 1 + if validations == 1: + raise RuntimeError("repair") + + monkeypatch.setattr(noema.time, "monotonic", lambda: next(monotonic)) + monkeypatch.setattr(noema.urllib.request, "build_opener", lambda *args: FakeOpener()) + monkeypatch.setattr(noema, "validate_substantive_verdict", validate) + + noema.call_llm("owner/repo", 1, make_pr(), "diff", False) + + assert timeouts == [19750.0, 19700.0] + + +def test_absolute_response_deadline_restores_signal_state(monkeypatch): + previous_handler = noema.signal.getsignal(noema.signal.SIGALRM) + previous_timer = noema.signal.getitimer(noema.signal.ITIMER_REAL) + with noema.absolute_response_deadline(1): + assert noema.signal.getitimer(noema.signal.ITIMER_REAL)[0] > 0 + assert noema.signal.getsignal(noema.signal.SIGALRM) == previous_handler + assert noema.signal.getitimer(noema.signal.ITIMER_REAL) == previous_timer + + +@pytest.mark.parametrize("pr", [ + make_pr(isDraft=True), + make_pr(reviews={"nodes": [review(login="noema", body="")]}), +]) +def test_prepare_review_skips_before_model_handoff(monkeypatch, tmp_path, pr): + expected_head = "a" * 40 + pr["headRefOid"] = expected_head + for existing_review in pr["reviews"]["nodes"]: + existing_review["commit"]["oid"] = expected_head + existing_review["body"] = ( + f"" + ) + output = tmp_path / "input.json" + monkeypatch.setattr(noema, "fetch_pr", lambda repo, number: pr) + monkeypatch.setattr(noema, "current_actor", lambda: "noema") + monkeypatch.setattr(noema, "fetch_diff", lambda *args: (_ for _ in ()).throw(AssertionError("diff must not load"))) + assert noema.prepare_review("owner/repo", 7, str(output), expected_head) == 0 + assert not output.exists() + + +def test_prepare_review_rejects_head_change_before_sealing(monkeypatch, tmp_path): + """Preparation cannot adopt a head newer than its validated trigger.""" + output = tmp_path / "input.json" + monkeypatch.setattr(noema, "fetch_pr", lambda repo, number: make_pr(headRefOid="b" * 40)) + monkeypatch.setattr( + noema, + "current_actor", + lambda: (_ for _ in ()).throw(AssertionError("identity must not load")), + ) + + with pytest.raises(RuntimeError, match="refused a stale trigger head"): + noema.prepare_review("owner/repo", 7, str(output), "a" * 40) + + assert not output.exists() + + +def test_sealed_handoff_rejects_modified_payload(tmp_path): + """Cross-job review input cannot change without invalidating its digest.""" + handoff = tmp_path / "handoff.json" + noema._write_sealed(str(handoff), {"head_sha": "a" * 40}) + assert noema._read_sealed(str(handoff))["head_sha"] == "a" * 40 + handoff.write_text('{"head_sha":"changed"}', encoding="utf-8") + with pytest.raises(RuntimeError, match="digest mismatch"): + noema._read_sealed(str(handoff)) def test_noema_redirect_handler_rejects_redirects(): @@ -1610,7 +565,7 @@ def raise_gaierror(host, port, *args, **kwargs): monkeypatch.setattr(socket, "getaddrinfo", raise_gaierror) with pytest.raises(ValueError, match="must start with http:// or https://"): - noema.call_llm("owner/repo", 1, pr, "diff", False, "head") + noema.call_llm("owner/repo", 1, pr, "diff", False) def test_call_llm_rejects_non_http_parsed_scheme(monkeypatch): @@ -1622,7 +577,7 @@ def test_call_llm_rejects_non_http_parsed_scheme(monkeypatch): monkeypatch.setattr(noema.urllib.parse, "urlparse", lambda _: parsed) with pytest.raises(ValueError, match="URL scheme must be http or https"): - noema.call_llm("owner/repo", 1, pr, "diff", False, "head") + noema.call_llm("owner/repo", 1, pr, "diff", False) def test_format_findings_and_submit_review(monkeypatch): @@ -1669,7 +624,7 @@ def test_inspect_and_review_skip_paths(monkeypatch): monkeypatch.setattr(noema, "call_llm", lambda *args, **kwargs: {"decision": "approve", "summary": "ok", "findings": []}) monkeypatch.setattr(noema, "submit_review", lambda *args, **kwargs: calls.append(args)) - assert noema.inspect_and_review("owner/repo", 7, "head") == 0 + assert noema.inspect_and_review("owner/repo", 7) == 0 assert calls cases = [ @@ -1680,17 +635,17 @@ def test_inspect_and_review_skip_paths(monkeypatch): calls.clear() monkeypatch.setattr(noema, "fetch_pr", lambda repo, number, pr=pr: pr) monkeypatch.setattr(noema, "current_actor", lambda actor=actor: actor) - assert noema.inspect_and_review("owner/repo", 7, "head") == 0 + assert noema.inspect_and_review("owner/repo", 7) == 0 assert calls == [] monkeypatch.setattr(noema, "fetch_pr", lambda repo, number: clean_pr) monkeypatch.setattr(noema, "current_actor", lambda: "") with pytest.raises(RuntimeError, match="identity could not be verified"): - noema.inspect_and_review("owner/repo", 7, "head") + noema.inspect_and_review("owner/repo", 7) monkeypatch.setattr(noema, "current_actor", lambda: "opencode-agent") with pytest.raises(RuntimeError, match="independent reviewer credential"): - noema.inspect_and_review("owner/repo", 7, "head") + noema.inspect_and_review("owner/repo", 7) def test_inspect_and_review_does_not_wait_for_other_reviews_or_checks(monkeypatch): @@ -1708,85 +663,7 @@ def test_inspect_and_review_does_not_wait_for_other_reviews_or_checks(monkeypatc monkeypatch.setattr(noema, "call_llm", lambda *args, **kwargs: {"decision": "approve", "summary": "ok"}) monkeypatch.setattr(noema, "submit_review", lambda *args, **kwargs: calls.append(args)) - assert noema.inspect_and_review("owner/repo", 7, "head") == 0 - assert calls - - -def test_stale_trigger_stops_before_identity_or_model_work(monkeypatch): - monkeypatch.setattr(noema, "fetch_pr", lambda repo, number: make_pr(headRefOid="new")) - monkeypatch.setattr( - noema, - "current_actor", - lambda: pytest.fail("stale execution must stop before identity lookup"), - ) - assert noema.inspect_and_review("owner/repo", 7, "old") == 0 - - -def test_expected_head_comparison_is_case_insensitive(monkeypatch): - seen = [] - monkeypatch.setattr( - noema, - "fetch_pr", - lambda repo, number: make_pr(headRefOid="a" * 40, isDraft=True), - ) - monkeypatch.setattr(noema, "current_actor", lambda: seen.append("actor") or "noema") - assert noema.inspect_and_review("owner/repo", 7, "A" * 40) == 0 - assert seen == ["actor"] - - -def test_head_movement_stops_before_review_publication(monkeypatch): - pull_requests = iter((make_pr(), make_pr(headRefOid="new"))) - monkeypatch.setattr(noema, "fetch_pr", lambda repo, number: next(pull_requests)) - monkeypatch.setattr(noema, "current_actor", lambda: "noema") - monkeypatch.setattr(noema, "fetch_diff", lambda repo, number: ("diff", False)) - monkeypatch.setattr(noema, "fetch_changed_file_paths", lambda repo, number: ["tool.py"]) - monkeypatch.setattr(noema, "build_review_context", lambda repo, number, pr: "context") - monkeypatch.setattr( - noema, - "call_llm", - lambda *args, **kwargs: {"decision": "approve", "summary": "ok"}, - ) - monkeypatch.setattr( - noema, - "submit_review", - lambda *args, **kwargs: pytest.fail("stale verdict must not publish"), - ) - assert noema.inspect_and_review("owner/repo", 7, "head") == 0 - - -def test_uppercase_expected_head_is_not_stale_before_model_work(monkeypatch): - """An uppercase --expected-head must match GitHub's lowercase live SHA (Devin Review, PR #1507).""" - pr = make_pr(headRefOid="abc123def0") - monkeypatch.setattr(noema, "fetch_pr", lambda repo, number: pr) - monkeypatch.setattr(noema, "current_actor", lambda: "noema") - monkeypatch.setattr(noema, "fetch_diff", lambda repo, number: ("diff", False)) - monkeypatch.setattr(noema, "fetch_changed_file_paths", lambda repo, number: ["tool.py"]) - monkeypatch.setattr(noema, "build_review_context", lambda repo, number, value: "context") - monkeypatch.setattr(noema, "call_llm", lambda *args, **kwargs: {"decision": "approve", "summary": "ok"}) - calls = [] - monkeypatch.setattr(noema, "submit_review", lambda *args, **kwargs: calls.append(args)) - - assert noema.inspect_and_review("owner/repo", 7, "ABC123DEF0") == 0 - assert calls - - -def test_uppercase_expected_head_is_not_stale_before_publication(monkeypatch): - """The pre-publication re-check must also compare case-insensitively.""" - pull_requests = iter((make_pr(headRefOid="abc123def0"), make_pr(headRefOid="abc123def0"))) - monkeypatch.setattr(noema, "fetch_pr", lambda repo, number: next(pull_requests)) - monkeypatch.setattr(noema, "current_actor", lambda: "noema") - monkeypatch.setattr(noema, "fetch_diff", lambda repo, number: ("diff", False)) - monkeypatch.setattr(noema, "fetch_changed_file_paths", lambda repo, number: ["tool.py"]) - monkeypatch.setattr(noema, "build_review_context", lambda repo, number, pr: "context") - monkeypatch.setattr( - noema, - "call_llm", - lambda *args, **kwargs: {"decision": "approve", "summary": "ok"}, - ) - calls = [] - monkeypatch.setattr(noema, "submit_review", lambda *args, **kwargs: calls.append(args)) - - assert noema.inspect_and_review("owner/repo", 7, "ABC123DEF0") == 0 + assert noema.inspect_and_review("owner/repo", 7) == 0 assert calls @@ -1805,72 +682,8 @@ def read(self): return json.dumps({"choices": [{"message": {"content": '{"decision":"approve"}'}}]}).encode() monkeypatch.setattr(noema.urllib.request.OpenerDirector, "open", lambda *args, **kwargs: Response()) - monkeypatch.setattr(noema, "fetch_pr", lambda repo, number: make_pr()) with pytest.raises(RuntimeError, match="substantive summary"): - noema.call_llm("owner/repo", 7, make_pr(), "diff", False, "head") - - -def test_call_llm_fails_closed_on_malformed_json_response(monkeypatch): - """Reproduces the reported CI crash: an LLM response whose content is - truncated/malformed JSON must fail the review cleanly through call_llm's - existing RuntimeError path, never as an unhandled json.JSONDecodeError.""" - monkeypatch.setenv("NOEMA_LLM_API_URL", "https://llm.example/v1/chat/completions") - monkeypatch.setenv("NOEMA_LLM_API_KEY", "test-key") - - class Response: - def __enter__(self): - return self - - def __exit__(self, *args): - return None - - def read(self): - # Malformed: an unquoted property name after the decision key, - # matching "Expecting property name enclosed in double quotes". - malformed_content = '{"decision":"approve", trailing garbage not: "quoted}' - return json.dumps({"choices": [{"message": {"content": malformed_content}}]}).encode() - - monkeypatch.setattr(noema.urllib.request.OpenerDirector, "open", lambda *args, **kwargs: Response()) - monkeypatch.setattr(noema, "fetch_pr", lambda repo, number: make_pr()) - with pytest.raises(RuntimeError, match="was not valid JSON"): - noema.call_llm("owner/repo", 7, make_pr(), "diff", False, "head") - - -def test_call_llm_repairs_one_malformed_json_response(monkeypatch): - """Ask once for corrected JSON before failing the required review closed.""" - monkeypatch.setenv("NOEMA_LLM_API_URL", "https://llm.example/v1/chat/completions") - monkeypatch.setenv("NOEMA_LLM_API_KEY", "test-key") - contents = iter( - ( - '{"decision":"approve", trailing garbage not: "quoted}', - json.dumps({"decision": "comment", "summary": "Repaired JSON", "findings": []}), - ) - ) - requests = [] - - class Response: - def __enter__(self): - return self - - def __exit__(self, *args): - return None - - def read(self): - content = next(contents) - return json.dumps({"choices": [{"message": {"content": content}}]}).encode() - - def open_response(_opener, request, **_kwargs): - requests.append(json.loads(request.data)) - return Response() - - monkeypatch.setattr(noema.urllib.request.OpenerDirector, "open", open_response) - monkeypatch.setattr(noema, "fetch_pr", lambda repo, number: make_pr()) - - verdict = noema.call_llm("owner/repo", 7, make_pr(), "diff", False, "head") - - assert verdict["summary"] == "Repaired JSON" - assert len(requests) == 2 - assert "prior verdict was rejected" in requests[1]["messages"][1]["content"] + noema.call_llm("owner/repo", 7, make_pr(), "diff", False) @pytest.mark.parametrize("message", [[], {}, 0, " "]) @@ -1894,9 +707,8 @@ def read(self): return json.dumps({"choices": [{"message": {"content": json.dumps(verdict)}}]}).encode() monkeypatch.setattr(noema.urllib.request.OpenerDirector, "open", lambda *args, **kwargs: Response()) - monkeypatch.setattr(noema, "fetch_pr", lambda repo, number: make_pr()) with pytest.raises(RuntimeError, match="malformed finding"): - noema.call_llm("owner/repo", 7, make_pr(), "diff", False, "head") + noema.call_llm("owner/repo", 7, make_pr(), "diff", False) @pytest.mark.parametrize( @@ -1928,9 +740,8 @@ def read(self): return json.dumps({"choices": [{"message": {"content": json.dumps(verdict)}}]}).encode() monkeypatch.setattr(noema.urllib.request.OpenerDirector, "open", lambda *args, **kwargs: Response()) - monkeypatch.setattr(noema, "fetch_pr", lambda repo, number: make_pr()) with pytest.raises(RuntimeError, match=error): - noema.call_llm("owner/repo", 7, make_pr(), "diff", False, "head") + noema.call_llm("owner/repo", 7, make_pr(), "diff", False) def test_call_llm_rejects_generic_approve_without_changed_line_evidence(monkeypatch): @@ -1949,9 +760,8 @@ def read(self): return json.dumps({"choices": [{"message": {"content": json.dumps(verdict)}}]}).encode() monkeypatch.setattr(noema.urllib.request.OpenerDirector, "open", lambda *args, **kwargs: Response()) - monkeypatch.setattr(noema, "fetch_pr", lambda repo, number: make_pr()) with pytest.raises(RuntimeError, match="parseable changed-line evidence"): - noema.call_llm("owner/repo", 7, make_pr(), "diff", False, "head") + noema.call_llm("owner/repo", 7, make_pr(), "diff", False) def test_call_llm_repairs_one_rejected_changed_line_verdict(monkeypatch): @@ -2007,6 +817,7 @@ def test_call_llm_repairs_one_rejected_changed_line_verdict(monkeypatch): }, } payloads = [] + timeouts = [] class Response: def __init__(self, verdict): @@ -2025,15 +836,16 @@ def read(self): class Opener: def open(self, request, timeout): - assert timeout == noema.NOEMA_LLM_TIMEOUT_SECONDS + timeouts.append(timeout) payloads.append(json.loads(request.data)) return Response(invalid if len(payloads) == 1 else valid) monkeypatch.setattr(noema.urllib.request, "build_opener", lambda *_args: Opener()) - monkeypatch.setattr(noema, "fetch_pr", lambda repo, number: make_pr()) - assert noema.call_llm("owner/repo", 7, make_pr(), diff, False, "head")["decision"] == "approve" + assert noema.call_llm("owner/repo", 7, make_pr(), diff, False)["decision"] == "approve" assert len(payloads) == 2 + assert 0 < timeouts[0] <= noema.CALL_LLM_TIMEOUT_SECONDS + assert 0 < timeouts[1] < timeouts[0] assert "trusted validator" in payloads[1]["messages"][1]["content"] @@ -2333,36 +1145,14 @@ def test_format_review_evidence_renders_only_structured_entries(): def test_parse_args_and_main(monkeypatch): - parsed = noema.parse_args( - ["--repo", "owner/repo", "--pr-number", "9", "--expected-head", "a" * 40] - ) + parsed = noema.parse_args(["--repo", "owner/repo", "--pr-number", "9"]) assert parsed.repo == "owner/repo" assert parsed.pr_number == 9 - assert parsed.expected_head == "a" * 40 seen = [] - monkeypatch.setattr( - noema, - "inspect_and_review", - lambda repo, number, head: seen.append((repo, number, head)) or 0, - ) - assert ( - noema.main( - ["--repo", "owner/repo", "--pr-number", "9", "--expected-head", "a" * 40] - ) - == 0 - ) - assert seen == [("owner/repo", 9, "a" * 40)] + monkeypatch.setattr(noema, "inspect_and_review", lambda repo, number: seen.append((repo, number)) or 0) + assert noema.main(["--repo", "owner/repo", "--pr-number", "9"]) == 0 + assert seen == [("owner/repo", 9)] with pytest.raises(SystemExit, match="--pr-number must be positive"): - noema.main( - ["--repo", "owner/repo", "--pr-number", "0", "--expected-head", "a" * 40] - ) - with pytest.raises(SystemExit, match="--expected-head must be a canonical lowercase"): - noema.main( - ["--repo", "owner/repo", "--pr-number", "9", "--expected-head", "bad"] - ) - with pytest.raises(SystemExit, match="--expected-head must be a canonical lowercase"): - noema.main( - ["--repo", "owner/repo", "--pr-number", "9", "--expected-head", "A" * 40] - ) + noema.main(["--repo", "owner/repo", "--pr-number", "0"]) diff --git a/tests/test_pingora_edge_policy.py b/tests/test_pingora_edge_policy.py index 70bb1bc970..06fc5cbe52 100644 --- a/tests/test_pingora_edge_policy.py +++ b/tests/test_pingora_edge_policy.py @@ -475,6 +475,18 @@ def test_changed_file_pagination_bound_is_fail_closed() -> None: policy._load_changed_files("api", "a/b", 1, "x", lambda _url, _token: page) +def test_changed_file_pagination_rejects_nonterminating_full_pages() -> None: + """A full-page response on every bounded request fails closed.""" + + class NonTerminatingPage(list[dict[str, object]]): + def __len__(self) -> int: + return 100 + + page = NonTerminatingPage() + with pytest.raises(policy.PolicyError, match="3,000"): + policy._load_changed_files("api", "a/b", 1, "x", lambda _url, _token: page) + + def test_changed_file_pagination_accepts_the_inclusive_bound() -> None: """Exactly 3,000 changed files are accepted only after an empty next page.""" diff --git a/tests/test_repository_branch_coverage_review_schedulers.py b/tests/test_repository_branch_coverage_review_schedulers.py index 4928e18046..4a1f48a1a6 100644 --- a/tests/test_repository_branch_coverage_review_schedulers.py +++ b/tests/test_repository_branch_coverage_review_schedulers.py @@ -63,11 +63,11 @@ class Opener: """Open one deterministic provider response.""" def open(self, _request: Any, timeout: int) -> Response: - assert timeout == noema.NOEMA_LLM_TIMEOUT_SECONDS + assert 0 < timeout <= noema.CALL_LLM_TIMEOUT_SECONDS return Response() monkeypatch.setattr(noema.urllib.request, "build_opener", lambda *_args: Opener()) - verdict = noema.call_llm("owner/repo", 1, {"headRefOid": "a" * 40}, "diff", False, "a" * 40) + verdict = noema.call_llm("owner/repo", 1, {"headRefOid": "a" * 40}, "diff", False) assert verdict["decision"] == "approve" diff --git a/tests/test_required_workflow_queue_contract.py b/tests/test_required_workflow_queue_contract.py index 5a295da25f..d8397ba003 100644 --- a/tests/test_required_workflow_queue_contract.py +++ b/tests/test_required_workflow_queue_contract.py @@ -242,12 +242,10 @@ def test_required_pull_request_workflows_cancel_superseded_runs() -> None: assert "github.repository" in concurrency_contract assert "github.event.pull_request.number" in workflow if filename == "noema-review.yml": - assert "cancel-in-progress: ${{" in concurrency_contract - assert "github.event_name != 'workflow_run'" in concurrency_contract assert ( - "github.event.workflow_run.conclusion != 'cancelled'" - in concurrency_contract - ) + "cancel-in-progress: ${{ github.event_name != 'workflow_run' || " + "github.event.workflow_run.conclusion != 'cancelled' }}" + ) in concurrency_contract else: assert "cancel-in-progress: true" in workflow if filename in { @@ -262,26 +260,7 @@ def test_required_pull_request_workflows_cancel_superseded_runs() -> None: assert "opencode-review-bootstrap-" in concurrency_contract elif filename == "noema-review.yml": assert "github.event.workflow_run.pull_requests[0].number" in concurrency_contract - assert "github.event.pull_request.head.sha" in concurrency_contract - assert "github.event.workflow_run.pull_requests[0].head.sha" in concurrency_contract - assert "github.event.workflow_run.head_sha" not in concurrency_contract - assert "github.event.client_payload.pr_head_sha" in concurrency_contract - assert "github.event.workflow_run.conclusion == 'cancelled'" in ( - concurrency_contract - ) - assert "format('cancelled-{0}', github.run_id)" in concurrency_contract - assert "'actionable'" in concurrency_contract - procedure = ( - REPO_ROOT / "docs" / "pr-review-and-merge-procedure.md" - ).read_text(encoding="utf-8") - assert "head-specific native concurrency" in procedure - assert "live-head validation explicitly cancels" in procedure - for source in ( - "`pull_request_target` uses `pull_request.head.sha`", - "`workflow_run` uses `workflow_run.pull_requests[0].head.sha`", - "`repository_dispatch` uses `client_payload.pr_head_sha`", - ): - assert source in procedure + assert "github.event_name }}" not in concurrency_contract else: if filename in {"codeql-pr.yml", "osv-scanner-pr.yml", "scorecard-pr.yml"}: assert "github.event_name == 'pull_request'" in concurrency_contract @@ -427,30 +406,30 @@ def test_pull_request_close_events_cancel_superseded_runs_without_heavy_jobs() - assert "closed" in workflow assert "cancel-closed-pr-runs:" in workflow - if filename in {"strix.yml", "noema-review.yml"}: - noun = "scans" if filename == "strix.yml" else "Noema reviews" - assert f"Cancel queued and running {noun} for the closed pull request" in workflow + if filename == "strix.yml": + assert "Cancel queued and running scans for the closed pull request" in workflow + assert ( + "secrets.PR_REVIEW_MERGE_TOKEN || secrets.OPENCODE_APPROVE_TOKEN " + "|| github.token" + ) in workflow + assert "DISPATCH_REPOSITORY" not in workflow + assert "CLOSED_PR_HEAD_SHA" in workflow + assert 'select(.event == "pull_request_target")' in workflow + assert 'select(.event == "repository_dispatch")' not in workflow assert "leaving runs unchanged" in workflow - next_job = "strix" if filename == "strix.yml" else "noema-review" + assert ( + "for active_status in queued in_progress requested waiting pending" + in workflow + ) cleanup_job = workflow.split(" cancel-closed-pr-runs:", 1)[1].split( - f" {next_job}:", 1 + " strix:", 1 )[0] assert "actions: write" in cleanup_job assert "actions/checkout" not in cleanup_job assert "cleanup skipped" not in cleanup_job - if filename == "strix.yml": - assert "CLOSED_PR_HEAD_SHA" in workflow - assert ( - "for active_status in queued in_progress requested waiting pending" - in workflow - ) - assert ( - "secrets.PR_REVIEW_MERGE_TOKEN || secrets.OPENCODE_APPROVE_TOKEN " - "|| github.token" - ) in workflow - assert "DISPATCH_REPOSITORY" not in workflow - assert 'select(.event == "pull_request_target")' in workflow - assert 'select(.event == "repository_dispatch")' not in workflow + elif filename == "noema-review.yml": + assert "Cancel queued and running Noema reviews for the closed pull request" in workflow + assert "actions: write" in workflow.split(" cancel-closed-pr-runs:", 1)[1].split(" prepare:", 1)[0] else: assert ( "PR closed; this run only cancels older runs through workflow concurrency." @@ -527,9 +506,7 @@ def test_noema_triggers_serialize_one_review_per_pull_request() -> None: assert "github.event.pull_request.number || github.event.workflow_run.pull_requests[0].number" in concurrency_contract assert "github.event.client_payload.pr_number" in concurrency_contract - assert "github.event.workflow_run.conclusion == 'cancelled'" in concurrency_contract - assert "format('cancelled-{0}', github.run_id)" in concurrency_contract - assert "'actionable'" in concurrency_contract + assert "github.event_name }}" not in concurrency_contract def test_noema_review_credentials_and_orchestrator_configuration_fail_closed() -> None: @@ -564,12 +541,9 @@ def test_noema_review_credentials_and_orchestrator_configuration_fail_closed() - "Noema app token exchange unavailable: app token response was empty." in workflow ) - assert ( - "Noema reviewer credential selection succeeded but no token was minted" - in workflow - ) assert "Resolve Noema target repository visibility" in workflow - assert "target_visibility.outputs.require_zdr" in workflow + assert "steps.target_visibility.outputs.require_zdr" in workflow + assert "needs.prepare.outputs.require_zdr" in workflow assert "CONTEXTUAL_ORCHESTRATOR_REQUIRE_ZDR" in workflow assert "https://integrate.api.nvidia.com/v1/chat/completions" not in workflow assert "nvidia/nemotron-3-ultra-550b-a55b" not in workflow @@ -634,12 +608,9 @@ def test_strix_gateway_default_and_noema_sidecar_fail_closed( in workflow_text("strix.yml") ) - noema_script = textwrap.dedent( - workflow_step( - workflow_text("noema-review.yml"), - "Run Noema LLM review and submit verdict", - ).split(" run: |\n", 1)[1] - ) + noema_script = textwrap.dedent(workflow_step( + workflow_text("noema-review.yml"), "Run first candidate" + ).split(" run: |\n", 1)[1]) noema_env = { **os.environ, "PR_NUMBER": "1", @@ -664,7 +635,7 @@ def test_strix_gateway_default_and_noema_sidecar_fail_closed( check=False, ) assert noema.returncode == 1 - assert "sidecar must be provisioned before Noema LLM review" in noema.stdout + assert noema.returncode != 0 def test_noema_workflow_run_without_pull_request_skips_before_token_exchange() -> None: @@ -702,8 +673,8 @@ def test_noema_review_supports_review_token_pat_fallback() -> None: in workflow ) assert "steps.noema_credential.outputs.source == 'github-app'" in workflow - assert "NOEMA_REVIEW_ACTOR: ${{ steps.noema_github_app_token.outputs['app-slug']" in workflow - assert "NOEMA_REVIEW_INSTALLATION_ID: ${{ steps.noema_github_app_token.outputs['installation-id'] }}" in workflow + assert "NOEMA_REVIEW_ACTOR: ${{ steps.app_token.outputs['app-slug']" in workflow + assert "NOEMA_REVIEW_INSTALLATION_ID: ${{ steps.app_token.outputs['installation-id'] }}" in workflow def test_noema_review_mints_a_least_privilege_github_app_token() -> None: @@ -1396,7 +1367,8 @@ def test_security_scan_fails_closed_when_dependency_review_is_unavailable() -> N assert "/dependency-graph/compare/${BASE_SHA}...${HEAD_SHA}" in workflow assert "repository: ${{ github.event.pull_request.head.repo.full_name }}" in workflow assert "ref: ${{ github.event.pull_request.head.sha }}" in workflow - assert 'if [ "$curl_status" -ne 0 ] || [ "$http_status" != "200" ]; then' in workflow + assert 'if [ "$curl_status" -ne 0 ]; then' in support_probe + assert 'elif [ "$http_status" != "200" ]; then' in support_probe assert "--connect-timeout 10" in workflow assert "--max-time 30" in workflow assert "-o /dev/null" in workflow @@ -1418,6 +1390,10 @@ def test_security_scan_fails_closed_when_dependency_review_is_unavailable() -> N ) assert "supported=false" not in workflow assert "skipping dependency-review hard gate" not in workflow + assert 'evidence_state="unavailable"' in support_probe + assert 'unavailable_reason="api_authorization"' in support_probe + assert "This is not a vulnerability-free result" in support_probe + assert 'if [ "$evidence_state" != "complete" ]; then' in support_probe assert ( "steps.dependency_review_support.outputs.supported == 'true'" in workflow )