From 2083a72dccaa1d96ea423a51af537240fde8a210 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 11:04:47 +0900 Subject: [PATCH 01/14] test(strix): reject direct-provider model normalization --- tests/test_strix_orchestrator_free_only.py | 56 ++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 tests/test_strix_orchestrator_free_only.py diff --git a/tests/test_strix_orchestrator_free_only.py b/tests/test_strix_orchestrator_free_only.py new file mode 100644 index 0000000000..dc10179c3b --- /dev/null +++ b/tests/test_strix_orchestrator_free_only.py @@ -0,0 +1,56 @@ +"""Contracts proving Strix cannot normalize a direct-provider model route.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +MODEL_UTILS = REPOSITORY_ROOT / "scripts" / "ci" / "strix_model_utils.sh" + + +def _normalize(model: str) -> subprocess.CompletedProcess[str]: + """Run the production normalization helper for one model identifier.""" + return subprocess.run( + [ + "bash", + "-c", + 'set -euo pipefail; . "$1"; DEFAULT_PROVIDER=""; normalize_model "$2"', + "bash", + str(MODEL_UTILS), + model, + ], + check=False, + capture_output=True, + text=True, + ) + + +def test_strix_accepts_only_the_governed_free_virtual_model_ids() -> None: + """Both accepted spellings resolve to the same governed free pool boundary.""" + for model in ( + "orchestrator/free", + "contextual-orchestrator/orchestrator/free", + ): + result = _normalize(model) + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == model + + +def test_strix_rejects_every_direct_provider_model_before_execution() -> None: + """A Strix model cannot name a provider or concrete model outside the gateway.""" + for model in ( + "openai-direct/gpt-5.4", + "openai_direct/gpt-5.4", + "openai/gpt-5", + "openrouter/openai/gpt-oss-120b:free", + "nvidia_nim/meta/llama-3.3-70b-instruct", + "github_models/openai/gpt-5", + "vertex_ai/gemini-2.5-pro", + "gemini/gemini-2.5-pro", + "gpt-5", + ): + result = _normalize(model) + assert result.returncode == 2, (model, result.stdout, result.stderr) + assert "Strix model must be orchestrator/free" in result.stderr + assert result.stdout == "" From 10c1ddf822f1e6336b73a9093a56680fea8f4f54 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 11:05:07 +0900 Subject: [PATCH 02/14] fix(strix): fail closed on direct-provider model routes --- scripts/ci/strix_model_utils.sh | 42 +++++++-------------------------- 1 file changed, 9 insertions(+), 33 deletions(-) diff --git a/scripts/ci/strix_model_utils.sh b/scripts/ci/strix_model_utils.sh index 9f20eae670..b010be9868 100755 --- a/scripts/ci/strix_model_utils.sh +++ b/scripts/ci/strix_model_utils.sh @@ -73,45 +73,21 @@ extract_vertex_model_id() { normalize_model() { local model model="$(trim_whitespace "${1-}")" - if [ -z "$model" ]; then - return 0 - fi - - if is_vertex_resource_path "$model"; then - local provider - provider="$(sanitize_provider_name "${DEFAULT_PROVIDER:-}")" || { - echo "ERROR: Vertex resource paths require an explicit vertex_ai or vertex_ai_beta provider." >&2 - return 2 - } - case "$provider" in - vertex_ai | vertex_ai_beta) ;; - *) - echo "ERROR: Vertex resource paths require an explicit vertex_ai or vertex_ai_beta provider." >&2 - return 2 - ;; - esac - printf '%s/%s\n' "$provider" "$(extract_vertex_model_id "$model")" - return 0 - fi - - local provider="${DEFAULT_PROVIDER:-}" - if [ -z "$provider" ]; then - provider="vertex_ai" - fi - provider="$(sanitize_provider_name "$provider")" || return $? + # Strix is an organization review path. Its model selector is therefore a + # policy boundary, not a generic provider normalizer: all inference must go + # through contextual-orchestrator's fail-closed zero-cost virtual pool. + # Provider/model identifiers would bypass the orchestrator's free-candidate + # source, capability, and private-target ZDR admission contracts, so reject + # them before credentials or provider endpoints can participate in execution. case "$model" in - projects/* | models/* | publishers/*) - printf '%s\n' "$model" - return 0 - ;; - */*) + orchestrator/free | contextual-orchestrator/orchestrator/free) printf '%s\n' "$model" return 0 ;; *) - printf '%s/%s\n' "$provider" "$model" - return 0 + echo "ERROR: Strix model must be orchestrator/free through contextual-orchestrator; direct provider/model routes are forbidden: '$model'." >&2 + return 2 ;; esac } From 7e4b7c1e61bd9990440caf2445da714c5daaf48d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 11:06:33 +0900 Subject: [PATCH 03/14] docs(adr): record Strix free-pool model boundary --- ...x-orchestrator-free-only-model-boundary.md | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 docs/adr/0021-strix-orchestrator-free-only-model-boundary.md diff --git a/docs/adr/0021-strix-orchestrator-free-only-model-boundary.md b/docs/adr/0021-strix-orchestrator-free-only-model-boundary.md new file mode 100644 index 0000000000..11c3aaea11 --- /dev/null +++ b/docs/adr/0021-strix-orchestrator-free-only-model-boundary.md @@ -0,0 +1,40 @@ +# ADR-0021: Strix model normalization is an `orchestrator/free` boundary + +- Status: Proposed +- Date: 2026-09-02 +- Owner: ContextualWisdomLab/.github central Strix review path + +## Context + +The protected central Strix workflow already provisions `ContextualWisdomLab/contextual-orchestrator`, requests the `orchestrator/free` virtual pool, supplies the five bootstrap credential sources, forces Zero Data Retention for private targets, and sets the workflow's external fallback list to empty. The shared `scripts/ci/strix_quick_gate.sh` nevertheless retained generic normalization for concrete provider model identifiers and historical direct-provider fallback code. That left a reusable execution boundary capable of accepting a provider/model name even though provider choice, free-pool admission, privacy admission, retry/failover, and serving are owned by contextual-orchestrator. + +Under the organization no-heuristics and routing-owner contract, a GitHub Actions review path must not acquire a second provider/model router. The model identifier accepted by the Strix gate is therefore not a preference or fallback ranking input; it is an exact virtual-pool capability boundary. + +## Decision + +`normalize_model` in the Strix model helper accepts only `orchestrator/free` and its provider-qualified spelling `contextual-orchestrator/orchestrator/free`. All provider names, concrete model names, unqualified model names, and historical direct-OpenAI aliases fail closed before provider credentials or provider endpoints can become execution authority. + +The five bootstrap secrets remain transport/discovery inputs to contextual-orchestrator. In particular, `OPENAI_API_KEY` is not removed from bootstrap or global discovery. The separate contextual-orchestrator free-pool admission contract decides which discovered credential sources may become `orchestrator/free` candidates; OpenAI-derived models remain excluded there while OpenAI integration may remain available to independently governed non-free/global pools. + +For private targets, the central workflow's existing `CONTEXTUAL_ORCHESTRATOR_REQUIRE_ZDR=true` path remains mandatory. This ADR does not weaken or duplicate that policy; it prevents the Strix model selector from bypassing it with a concrete provider route. + +## Executable evidence + +`tests/test_strix_orchestrator_free_only.py` exercises the production shell helper. It admits the two exact governed virtual-model spellings and rejects direct OpenAI, OpenRouter, NVIDIA NIM, GitHub Models, Vertex/Gemini, and unqualified model identifiers before execution. + +The full protected exact-head workflow set remains authoritative. Pending or queued runs are not passing evidence. + +## Research and standards basis + +This decision does not invent a quality score, routing rank, or threshold. It removes an overlapping router and delegates selection to the separately governed orchestrator. Privacy admission remains evidence-based and fail-closed. + +OpenRouter. (2026). *Zero data retention*. https://openrouter.ai/docs/guides/features/zdr + +ContextualWisdomLab. (2026, September 2). *ADR-0003: Vendored contextual-orchestrator review sidecar with governed gateway pools*. ContextualWisdomLab/.github. + +## Consequences + +- Strix cannot use a concrete provider/model identifier as an inference route. +- Provider discovery/failover stays inside contextual-orchestrator. +- `OPENAI_API_KEY` may still be registered and globally discovered; the free-pool candidate boundary, not secret transport, excludes OpenAI-derived candidates from `orchestrator/free`. +- Historical direct-provider fallback helpers become unreachable from the accepted Strix model boundary and should be removed as follow-up dead-code cleanup after exact-head regression evidence is available. From 57460d0d75dcf74dae547187b6a1cb2931365524 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 11:06:51 +0900 Subject: [PATCH 04/14] docs(doctoring): trace Strix direct-route RCA --- ...estrator-free-model-boundary-2026-09-02.md | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 docs/doctoring/strix-orchestrator-free-model-boundary-2026-09-02.md diff --git a/docs/doctoring/strix-orchestrator-free-model-boundary-2026-09-02.md b/docs/doctoring/strix-orchestrator-free-model-boundary-2026-09-02.md new file mode 100644 index 0000000000..9df7aa5f6c --- /dev/null +++ b/docs/doctoring/strix-orchestrator-free-model-boundary-2026-09-02.md @@ -0,0 +1,25 @@ +# Strix `orchestrator/free` model-boundary doctoring — 2026-09-02 + +## Exact failing evidence + +Protected `main@23df081c36c93da019c89c474351002afb014daa` already hard-pins the central Strix workflow to `contextual-orchestrator/orchestrator/free`, provisions contextual-orchestrator with `CONTEXTUAL_ORCHESTRATOR_POOL: free`, passes all five bootstrap credentials, and sets `STRIX_FALLBACK_MODELS: ""`. The shared Strix gate nevertheless retained direct-provider normalization and direct-OpenAI/OpenRouter/GitHub Models/Vertex fallback machinery. That created a second provider-routing surface beneath a workflow whose accepted owner contract delegates provider selection and failover to contextual-orchestrator. + +## Causal owner + +The reusable owner is `ContextualWisdomLab/.github/scripts/ci/strix_model_utils.sh` together with `strix_quick_gate.sh`, not downstream repositories. The first repair is placed at model normalization so a concrete provider identifier cannot cross into any later credential/base/fallback branch. + +## Test-first repair + +Commit `2083a72dccaa1d96ea423a51af537240fde8a210` adds the regression contract before the production change. It requires exactly the two governed `orchestrator/free` spellings to be admitted and representative direct-provider identifiers to fail closed. Commit `10c1ddf822f1e6336b73a9093a56680fea8f4f54` changes the production normalizer accordingly. + +No arbitrary rank, weight, score, threshold, retry preference, or provider order replaces the removed routing surface. The accepted identifier is a categorical authority boundary; contextual-orchestrator owns all downstream model choice. + +## Credential and privacy correction retained + +This repair intentionally does **not** remove `OPENAI_API_KEY` from central workflow bootstrap. All five credential sources may be registered and globally discovered. The `orchestrator/free` candidate-admission owner remains contextual-orchestrator, where OpenAI-derived candidates are excluded while BYTEZ, NVIDIA NIM primary/subaccount, and OpenRouter sources may be considered subject to explicit free/privacy/capability evidence. + +Private-target ZDR remains enforced by the central workflow and sidecar. A direct provider route is rejected before it could bypass that boundary. + +## Verification status + +Fresh hosted exact-head tests are required before merge. Queued, pending, stale, predecessor-head, or synthetic evidence is non-passing. Historical direct-provider fallback code is now unreachable through the accepted model normalizer but remains cleanup debt until a subsequent exact-head change removes it without losing unrelated Strix scanner behavior. From 625f6bcce49cc5c167448602f603208d593101db Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 11:07:22 +0900 Subject: [PATCH 05/14] ci(strix): reconcile canonical free-only routing evidence --- ...rce-fix-strix-free-only-model-boundary.yml | 96 +++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 .github/workflows/source-fix-strix-free-only-model-boundary.yml diff --git a/.github/workflows/source-fix-strix-free-only-model-boundary.yml b/.github/workflows/source-fix-strix-free-only-model-boundary.yml new file mode 100644 index 0000000000..55afdc490f --- /dev/null +++ b/.github/workflows/source-fix-strix-free-only-model-boundary.yml @@ -0,0 +1,96 @@ +name: Source fix Strix free-only model boundary + +on: + push: + branches: + - fix/strix-free-only-model-normalization-20260902 + +permissions: + contents: write + +jobs: + reconcile-docs: + runs-on: ubuntu-24.04 + steps: + - name: Checkout exact branch head + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + fetch-depth: 0 + persist-credentials: true + + - name: Append canonical gap and changelog evidence once + shell: bash + run: | + set -euo pipefail + python3 <<'PY' + from pathlib import Path + + gap = Path("docs/product-technical-gap-baseline.md") + changelog = Path("CHANGELOG.md") + + gap_marker = "## 2026-09-02 Strix direct-provider model-boundary repair" + gap_text = """ + + ## 2026-09-02 Strix direct-provider model-boundary repair + + **Live gap.** Protected `main@23df081c36c93da019c89c474351002afb014daa` already provisions the central Strix workflow through contextual-orchestrator with `STRIX_MODEL=contextual-orchestrator/orchestrator/free`, `CONTEXTUAL_ORCHESTRATOR_POOL=free`, private-target ZDR enforcement, all five bootstrap credential variables, and no external fallback list. The reusable Strix model normalizer nevertheless continued accepting concrete provider/model identifiers, leaving a second routing surface below the central workflow. + + **Causal owner and repair.** `scripts/ci/strix_model_utils.sh` is the earliest reusable model-admission boundary. Regression commit `2083a72dccaa1d96ea423a51af537240fde8a210` requires direct OpenAI, OpenRouter, NVIDIA NIM, GitHub Models, Vertex/Gemini, and unqualified identifiers to fail closed. Production commit `10c1ddf822f1e6336b73a9093a56680fea8f4f54` admits only `orchestrator/free` and `contextual-orchestrator/orchestrator/free`. No fallback rank, weight, score, threshold, or provider order replaces the removed authority; contextual-orchestrator owns candidate generation, routing, serving, and failover. + + **Credential/privacy boundary.** The central workflow must continue supplying `BYTEZ_API_KEY`, `NVIDIA_NIM_API_KEY`, `NVIDIA_NIM_API_KEY_SUB`, `OPENROUTER_API_KEY`, and `OPENAI_API_KEY` as bootstrap/global-discovery inputs. `OPENAI_API_KEY` transport is not a defect. The separate contextual-orchestrator `orchestrator/free` candidate policy excludes OpenAI-derived candidates while permitting the four eligible provider-account sources only when their explicit free/privacy/capability evidence passes. Private-target ZDR remains mandatory. + + **Verification.** Hosted exact-head checks and independent review remain authoritative. Pending, queued, stale, predecessor-head, or synthetic evidence is non-passing. Historical direct-provider fallback helpers are now unreachable through the accepted model normalizer and remain dead-code cleanup debt until removed with exact-head regression evidence. + """.rstrip() + "\n" + + text = gap.read_text(encoding="utf-8") + if gap_marker not in text: + gap.write_text(text.rstrip() + gap_text, encoding="utf-8") + + changelog_marker = "Strix model normalization now fails closed on every direct-provider model identifier" + changelog_text = """ + - Strix model normalization now fails closed on every direct-provider model identifier and accepts only the contextual-orchestrator `orchestrator/free` virtual pool spellings. This closes the reusable model-selection bypass without removing `OPENAI_API_KEY` from bootstrap/global discovery; the orchestrator's free-pool credential-source policy remains the candidate-admission authority. + """.strip() + "\n" + current = changelog.read_text(encoding="utf-8") + if changelog_marker not in current: + lines = current.splitlines(keepends=True) + insert_at = next((i + 1 for i, line in enumerate(lines) if line.strip() == "## [Unreleased]"), None) + if insert_at is None: + raise SystemExit("CHANGELOG.md has no [Unreleased] heading") + lines.insert(insert_at, changelog_text) + changelog.write_text("".join(lines), encoding="utf-8") + PY + + - name: Verify focused model-boundary contract + shell: bash + run: | + set -euo pipefail + python3 - <<'PY' + import importlib.util + from pathlib import Path + + path = Path("tests/test_strix_orchestrator_free_only.py") + spec = importlib.util.spec_from_file_location("strix_free_contract", path) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + module.test_strix_accepts_only_the_governed_free_virtual_model_ids() + module.test_strix_rejects_every_direct_provider_model_before_execution() + PY + + - name: Remove one-shot source-fix workflow and commit + shell: bash + env: + BRANCH: fix/strix-free-only-model-normalization-20260902 + run: | + set -euo pipefail + rm -f .github/workflows/source-fix-strix-free-only-model-boundary.yml + git diff --check + if git diff --quiet; then + echo "No reconciliation delta remains." + exit 0 + fi + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add docs/product-technical-gap-baseline.md CHANGELOG.md .github/workflows/source-fix-strix-free-only-model-boundary.yml + git commit -m "docs(strix): reconcile free-only routing evidence" + git push origin "HEAD:${BRANCH}" From 618bf6516c792004d84074c00844793ee48c59a1 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 11:16:51 +0900 Subject: [PATCH 06/14] test(strix): forbid repository-authored retry compute --- .../test_strix_no_heuristic_retry_contract.py | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 tests/test_strix_no_heuristic_retry_contract.py diff --git a/tests/test_strix_no_heuristic_retry_contract.py b/tests/test_strix_no_heuristic_retry_contract.py new file mode 100644 index 0000000000..9a6f0b6639 --- /dev/null +++ b/tests/test_strix_no_heuristic_retry_contract.py @@ -0,0 +1,46 @@ +"""Contracts removing repository-authored retry/test-time-compute heuristics from Strix.""" + +from __future__ import annotations + +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +WORKFLOW = ROOT / ".github" / "workflows" / "strix.yml" +GATE = ROOT / "scripts" / "ci" / "strix_quick_gate.sh" + + +def test_central_strix_does_not_allocate_repository_authored_retry_compute() -> None: + """The central review path gets one execution; provider failover belongs to the orchestrator.""" + workflow = WORKFLOW.read_text(encoding="utf-8") + forbidden = ( + "STRIX_LLM_MAX_RETRIES:", + "STRIX_TRANSIENT_RETRY_PER_MODEL:", + "STRIX_TRANSIENT_RETRY_BACKOFF_SECONDS:", + "STRIX_GATE_RETRY_BACKOFF_SECONDS", + "strix_gate_attempt", + "backoff_seconds=", + "retrying after ${backoff_seconds}s backoff", + "reached the retry limit", + ) + for token in forbidden: + assert token not in workflow, token + + assert 'bash "$TRUSTED_STRIX_GATE" 2>&1 | tee "$strix_terminal_log"' in workflow + assert "provider/backend was unavailable" in workflow + + +def test_strix_gate_has_no_same_model_retry_allocator() -> None: + """The reusable gate must fail closed instead of allocating another model execution.""" + gate = GATE.read_text(encoding="utf-8") + forbidden = ( + "STRIX_TRANSIENT_RETRY_PER_MODEL", + "STRIX_TRANSIENT_RETRY_BACKOFF_SECONDS", + "run_strix_with_transient_retry()", + "github_models_rate_limit_should_skip_same_model_retry()", + "is_transient_same_model_retry_error()", + "Retrying model '$model'", + ) + for token in forbidden: + assert token not in gate, token + + assert 'run_strix_once "$PRIMARY_MODEL" || primary_scan_rc=$?' in gate From 9dd508276ca5803cc71a261fb151921863dd59eb Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 11:17:44 +0900 Subject: [PATCH 07/14] test(strix): forbid hand-selected severity admission threshold --- ...st_strix_no_heuristic_severity_contract.py | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 tests/test_strix_no_heuristic_severity_contract.py diff --git a/tests/test_strix_no_heuristic_severity_contract.py b/tests/test_strix_no_heuristic_severity_contract.py new file mode 100644 index 0000000000..643ea5e18a --- /dev/null +++ b/tests/test_strix_no_heuristic_severity_contract.py @@ -0,0 +1,27 @@ +"""Contracts removing hand-selected Strix severity thresholds from merge decisions.""" + +from __future__ import annotations + +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +WORKFLOW = ROOT / ".github" / "workflows" / "strix.yml" +GATE = ROOT / "scripts" / "ci" / "strix_quick_gate.sh" + + +def test_required_strix_workflow_has_no_hand_selected_severity_threshold() -> None: + """A current vulnerability artifact is evidence; CI must not invent a cutoff.""" + workflow = WORKFLOW.read_text(encoding="utf-8") + assert "STRIX_FAIL_ON_MIN_SEVERITY" not in workflow + assert "at or above" not in workflow + assert "below-threshold" not in workflow + + +def test_successful_strix_execution_fails_closed_on_any_current_vulnerability_artifact() -> None: + """The reusable gate must not convert severity labels to an admission score.""" + gate = GATE.read_text(encoding="utf-8") + assert ( + "Current Strix vulnerability report exists; failing closed without a repository-authored severity threshold." + in gate + ) + assert "Strix exited successfully but emitted a vulnerability at or above" not in gate From e31d7d21b93b4838b8c0f8e9bb1c2ecac04518f0 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 11:18:40 +0900 Subject: [PATCH 08/14] ci(strix): source-fix retry and severity heuristics --- ...-fix-strix-no-heuristic-retry-severity.yml | 233 ++++++++++++++++++ 1 file changed, 233 insertions(+) create mode 100644 .github/workflows/source-fix-strix-no-heuristic-retry-severity.yml diff --git a/.github/workflows/source-fix-strix-no-heuristic-retry-severity.yml b/.github/workflows/source-fix-strix-no-heuristic-retry-severity.yml new file mode 100644 index 0000000000..749afc9343 --- /dev/null +++ b/.github/workflows/source-fix-strix-no-heuristic-retry-severity.yml @@ -0,0 +1,233 @@ +name: Source fix Strix no-heuristic retry and severity + +on: + push: + branches: + - fix/strix-free-only-model-normalization-20260902 + +permissions: + contents: write + +jobs: + repair: + runs-on: ubuntu-24.04 + steps: + - name: Checkout exact branch head + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + fetch-depth: 0 + persist-credentials: true + + - name: Prove retry and severity contracts are RED before repair + shell: bash + run: | + set -euo pipefail + if python3 - <<'PY' + import importlib.util + from pathlib import Path + + for filename in ( + "tests/test_strix_no_heuristic_retry_contract.py", + "tests/test_strix_no_heuristic_severity_contract.py", + ): + path = Path(filename) + spec = importlib.util.spec_from_file_location(path.stem, path) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + for name in dir(module): + if name.startswith("test_"): + getattr(module, name)() + PY + then + echo "::error::No-heuristics Strix regressions were already green before the production repair." + exit 1 + fi + + - name: Remove repository-authored retry allocation and severity admission + shell: bash + run: | + set -euo pipefail + python3 <<'PY' + from pathlib import Path + import re + + gate_path = Path("scripts/ci/strix_quick_gate.sh") + workflow_path = Path(".github/workflows/strix.yml") + queue_test_path = Path("tests/test_required_workflow_queue_contract.py") + + gate = gate_path.read_text(encoding="utf-8") + for line in ( + 'STRIX_TRANSIENT_RETRY_PER_MODEL="${STRIX_TRANSIENT_RETRY_PER_MODEL:-0}"\n', + 'STRIX_TRANSIENT_RETRY_BACKOFF_SECONDS="${STRIX_TRANSIENT_RETRY_BACKOFF_SECONDS:-3}"\n', + '\trequire_non_negative_integer "$STRIX_TRANSIENT_RETRY_PER_MODEL" "STRIX_TRANSIENT_RETRY_PER_MODEL"\n', + '\trequire_non_negative_integer "$STRIX_TRANSIENT_RETRY_BACKOFF_SECONDS" "STRIX_TRANSIENT_RETRY_BACKOFF_SECONDS"\n', + ): + if line not in gate: + raise SystemExit(f"expected gate line missing: {line!r}") + gate = gate.replace(line, "", 1) + + for function_name in ( + "is_transient_same_model_retry_error", + "github_models_rate_limit_should_skip_same_model_retry", + "run_strix_with_transient_retry", + ): + pattern = rf"(?ms)^{re.escape(function_name)}\(\) \{{\n.*?^\}}\n\n" + gate, count = re.subn(pattern, "", gate, count=1) + if count != 1: + raise SystemExit(f"expected exactly one {function_name} function, found {count}") + + old_success = '''\tif [ "$rc" -eq 0 ]; then +\t\tif has_blocking_vulnerability_reports; then +\t\t\tif ! evaluate_pull_request_findings || [ "$PR_FINDINGS_DECISION" != "allow_baseline" ]; then +\t\t\t\techo "Strix exited successfully but emitted a vulnerability at or above '$STRIX_FAIL_ON_MIN_SEVERITY'; failing closed." >&2 +\t\t\t\treturn 1 +\t\t\tfi +\t\tfi +\t\tprintf "Strix run succeeded for model '%s' in %ds.\\n" "$model" "$elapsed" >&2 +\t\treturn 0 +\tfi +''' + new_success = '''\tif [ "$rc" -eq 0 ]; then +\t\tlocal current_vulnerability_file +\t\tcurrent_vulnerability_file="$(find "$ACTIVE_REPORTS_DIR" -type f -path '*/vulnerabilities/*.md' -print -quit 2>/dev/null || true)" +\t\tif [ -n "$current_vulnerability_file" ]; then +\t\t\techo "Current Strix vulnerability report exists; failing closed without a repository-authored severity threshold." >&2 +\t\t\treturn 1 +\t\tfi +\t\tprintf "Strix run succeeded for model '%s' in %ds.\\n" "$model" "$elapsed" >&2 +\t\treturn 0 +\tfi +''' + if old_success not in gate: + raise SystemExit("run_strix_once success block did not match expected source") + gate = gate.replace(old_success, new_success, 1) + + simple_scan = '''run_current_target_scan() { +\tINFRA_ERROR_DETECTED=0 +\tZERO_FINDINGS_REPORTED=0 + +\tlocal primary_scan_rc=0 +\trun_strix_once "$PRIMARY_MODEL" || primary_scan_rc=$? +\tif [ "$primary_scan_rc" -eq 0 ]; then +\t\treturn 0 +\tfi +\tif [ "$primary_scan_rc" -eq 2 ]; then +\t\treturn 2 +\tfi +\tif [ "$INFRA_ERROR_DETECTED" -eq 1 ]; then +\t\techo "STRIX_PROVIDER_UNAVAILABLE: contextual-orchestrator/orchestrator/free did not produce authoritative scan evidence; failing closed without repository-authored retry or fallback allocation." >&2 +\telse +\t\techo "Strix quick scan failed; failing closed without repository-authored retry or fallback allocation." >&2 +\tfi +\treturn 1 +} + +''' + gate, count = re.subn( + r"(?ms)^run_current_target_scan\(\) \{\n.*?^\}\n\n(?=prepare_pull_request_scan_scope)", + simple_scan, + gate, + count=1, + ) + if count != 1: + raise SystemExit(f"expected one run_current_target_scan block, found {count}") + gate_path.write_text(gate, encoding="utf-8") + + workflow = workflow_path.read_text(encoding="utf-8") + for line in ( + " STRIX_LLM_MAX_RETRIES: 1\n", + " STRIX_TRANSIENT_RETRY_PER_MODEL: 2\n", + " STRIX_TRANSIENT_RETRY_BACKOFF_SECONDS: 60\n", + " STRIX_FAIL_ON_MIN_SEVERITY: MEDIUM\n", + ): + if line not in workflow: + raise SystemExit(f"expected workflow line missing: {line!r}") + workflow = workflow.replace(line, "", 1) + + workflow = workflow.replace( + " # severity branch anchored away from identifiers so environment lines\n" + " # such as STRIX_FAIL_ON_MIN_SEVERITY do not look like findings.\n", + " # Keep the severity marker anchored away from identifiers so unrelated\n" + " # environment text does not look like a reported finding.\n", + 1, + ) + + single_run = ''' strix_run_log="$RUNNER_TEMP/strix_gate_console.log" + : > "$strix_run_log" + strix_terminal_log="$strix_run_log" + strix_rc=0 + set +e + bash "$TRUSTED_STRIX_GATE" 2>&1 | tee "$strix_terminal_log" + strix_rc="${PIPESTATUS[0]}" + set -e + + if [ "$strix_rc" -eq 0 ]; then''' + workflow, count = re.subn( + r'''(?ms) strix_run_log="\$RUNNER_TEMP/strix_gate_console\.log"\n.*? set -e\n\n if \[ "\$strix_rc" -eq 0 \]; then''', + single_run, + workflow, + count=1, + ) + if count != 1: + raise SystemExit(f"expected one outer Strix retry loop, found {count}") + + workflow = workflow.replace("out-of-scope/below-threshold finding", "out-of-scope finding") + workflow = workflow.replace("below-threshold finding", "out-of-scope finding") + if "STRIX_FAIL_ON_MIN_SEVERITY" in workflow: + raise SystemExit("stale STRIX_FAIL_ON_MIN_SEVERITY remains in workflow") + if "strix_gate_attempt" in workflow or "STRIX_GATE_RETRY_BACKOFF_SECONDS" in workflow: + raise SystemExit("stale outer retry authority remains in workflow") + workflow_path.write_text(workflow, encoding="utf-8") + + queue_test = queue_test_path.read_text(encoding="utf-8") + old_assert = ' assert "STRIX_FAIL_ON_MIN_SEVERITY: MEDIUM" in workflow\n' + new_assert = ' assert "STRIX_FAIL_ON_MIN_SEVERITY" not in workflow\n' + if old_assert not in queue_test: + raise SystemExit("required workflow queue test threshold assertion not found") + queue_test_path.write_text(queue_test.replace(old_assert, new_assert, 1), encoding="utf-8") + PY + + - name: Verify focused no-heuristics contracts + shell: bash + run: | + set -euo pipefail + python3 - <<'PY' + import importlib.util + from pathlib import Path + + for filename in ( + "tests/test_strix_no_heuristic_retry_contract.py", + "tests/test_strix_no_heuristic_severity_contract.py", + ): + path = Path(filename) + spec = importlib.util.spec_from_file_location(path.stem, path) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + for name in dir(module): + if name.startswith("test_"): + getattr(module, name)() + PY + python3 -m pytest -q tests/test_required_workflow_queue_contract.py + git diff --check + + - name: Record research/causal trace and remove one-shot workflow + shell: bash + run: | + set -euo pipefail + cat >> docs/doctoring/strix-orchestrator-free-model-boundary-2026-09-02.md <<'EOF' + + ## Retry and severity decision repair + + Fresh protected-main evidence showed that the required Strix workflow still allocated two same-model gate retries plus a second outer three-attempt retry loop with fixed linear/exponential-style sleep values, while the reusable gate classified retryability through hand-authored provider/error regex families. The same path converted Strix severity labels to numeric ranks and used the repository-selected `MEDIUM` cutoff as a merge admission rule. Neither retry allocation nor the severity cutoff had an identified statistical model, authoritative standard, or executable experimental calibration. + + The repair therefore does not substitute different retry counts, backoff constants, severity weights, or cutoffs. The central Strix path executes the governed `orchestrator/free` request once; contextual-orchestrator retains provider discovery/failover authority. Any execution that fails to produce authoritative scan evidence fails closed. Any current vulnerability report artifact also fails closed without a repository-authored severity threshold. Severity labels may remain descriptive evidence, but they are not converted into a local admission score. + EOF + rm -f .github/workflows/source-fix-strix-no-heuristic-retry-severity.yml + git diff --check + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add .github/workflows/strix.yml scripts/ci/strix_quick_gate.sh tests/test_required_workflow_queue_contract.py tests/test_strix_no_heuristic_retry_contract.py tests/test_strix_no_heuristic_severity_contract.py docs/doctoring/strix-orchestrator-free-model-boundary-2026-09-02.md .github/workflows/source-fix-strix-no-heuristic-retry-severity.yml + git commit -m "fix(strix): remove heuristic retry and severity admission" + git push origin HEAD:fix/strix-free-only-model-normalization-20260902 From ba60aaef8da4651b65053f9702c52b29f26d3d87 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 11:21:04 +0900 Subject: [PATCH 09/14] ci(strix): add exact-source retry/severity repair driver --- ...e_fix_strix_no_heuristic_retry_severity.py | 217 ++++++++++++++++++ 1 file changed, 217 insertions(+) create mode 100644 scripts/ci/source_fix_strix_no_heuristic_retry_severity.py diff --git a/scripts/ci/source_fix_strix_no_heuristic_retry_severity.py b/scripts/ci/source_fix_strix_no_heuristic_retry_severity.py new file mode 100644 index 0000000000..78a56f9e08 --- /dev/null +++ b/scripts/ci/source_fix_strix_no_heuristic_retry_severity.py @@ -0,0 +1,217 @@ +"""One-shot exact-source repair for Strix retry/severity no-heuristics contracts. + +This driver is intentionally temporary. The companion workflow deletes it after +RED-before-repair and focused GREEN verification succeed on the same branch head. +""" + +from __future__ import annotations + +import importlib.util +import re +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +GATE = ROOT / "scripts" / "ci" / "strix_quick_gate.sh" +WORKFLOW = ROOT / ".github" / "workflows" / "strix.yml" +QUEUE_TEST = ROOT / "tests" / "test_required_workflow_queue_contract.py" +RETRY_TEST = ROOT / "tests" / "test_strix_no_heuristic_retry_contract.py" +SEVERITY_TEST = ROOT / "tests" / "test_strix_no_heuristic_severity_contract.py" +DOCTORING = ROOT / "docs" / "doctoring" / "strix-orchestrator-free-model-boundary-2026-09-02.md" + + +def _run_contract_file(path: Path) -> None: + spec = importlib.util.spec_from_file_location(path.stem, path) + if spec is None or spec.loader is None: + raise RuntimeError(f"cannot load contract {path}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + for name in sorted(dir(module)): + if name.startswith("test_"): + getattr(module, name)() + + +def _prove_red() -> None: + failures: list[str] = [] + for path in (RETRY_TEST, SEVERITY_TEST): + try: + _run_contract_file(path) + except AssertionError as exc: + failures.append(f"{path.name}: {exc}") + if not failures: + raise SystemExit("no-heuristics Strix contracts were already GREEN before repair") + print("RED contracts observed:") + for failure in failures: + print(f"- {failure}") + + +def _remove_shell_function(source: str, name: str) -> str: + pattern = rf"(?ms)^{re.escape(name)}\(\) \{{\n.*?^\}}\n\n" + result, count = re.subn(pattern, "", source, count=1) + if count != 1: + raise SystemExit(f"expected exactly one {name} function, found {count}") + return result + + +def _repair_gate() -> None: + gate = GATE.read_text(encoding="utf-8") + for line in ( + 'STRIX_TRANSIENT_RETRY_PER_MODEL="${STRIX_TRANSIENT_RETRY_PER_MODEL:-0}"\n', + 'STRIX_TRANSIENT_RETRY_BACKOFF_SECONDS="${STRIX_TRANSIENT_RETRY_BACKOFF_SECONDS:-3}"\n', + '\trequire_non_negative_integer "$STRIX_TRANSIENT_RETRY_PER_MODEL" "STRIX_TRANSIENT_RETRY_PER_MODEL"\n', + '\trequire_non_negative_integer "$STRIX_TRANSIENT_RETRY_BACKOFF_SECONDS" "STRIX_TRANSIENT_RETRY_BACKOFF_SECONDS"\n', + ): + if line not in gate: + raise SystemExit(f"expected gate line missing: {line!r}") + gate = gate.replace(line, "", 1) + + for function_name in ( + "is_transient_same_model_retry_error", + "github_models_rate_limit_should_skip_same_model_retry", + "run_strix_with_transient_retry", + ): + gate = _remove_shell_function(gate, function_name) + + old_success = '''\tif [ "$rc" -eq 0 ]; then +\t\tif has_blocking_vulnerability_reports; then +\t\t\tif ! evaluate_pull_request_findings || [ "$PR_FINDINGS_DECISION" != "allow_baseline" ]; then +\t\t\t\techo "Strix exited successfully but emitted a vulnerability at or above '$STRIX_FAIL_ON_MIN_SEVERITY'; failing closed." >&2 +\t\t\t\treturn 1 +\t\t\tfi +\t\tfi +\t\tprintf "Strix run succeeded for model '%s' in %ds.\\n" "$model" "$elapsed" >&2 +\t\treturn 0 +\tfi +''' + new_success = '''\tif [ "$rc" -eq 0 ]; then +\t\tlocal current_vulnerability_file +\t\tcurrent_vulnerability_file="$(find "$ACTIVE_REPORTS_DIR" -type f -path '*/vulnerabilities/*.md' -print -quit 2>/dev/null || true)" +\t\tif [ -n "$current_vulnerability_file" ]; then +\t\t\techo "Current Strix vulnerability report exists; failing closed without a repository-authored severity threshold." >&2 +\t\t\treturn 1 +\t\tfi +\t\tprintf "Strix run succeeded for model '%s' in %ds.\\n" "$model" "$elapsed" >&2 +\t\treturn 0 +\tfi +''' + if old_success not in gate: + raise SystemExit("run_strix_once success block did not match expected source") + gate = gate.replace(old_success, new_success, 1) + + simple_scan = '''run_current_target_scan() { +\tINFRA_ERROR_DETECTED=0 +\tZERO_FINDINGS_REPORTED=0 + +\tlocal primary_scan_rc=0 +\trun_strix_once "$PRIMARY_MODEL" || primary_scan_rc=$? +\tif [ "$primary_scan_rc" -eq 0 ]; then +\t\treturn 0 +\tfi +\tif [ "$primary_scan_rc" -eq 2 ]; then +\t\treturn 2 +\tfi +\tif [ "$INFRA_ERROR_DETECTED" -eq 1 ]; then +\t\techo "STRIX_PROVIDER_UNAVAILABLE: contextual-orchestrator/orchestrator/free did not produce authoritative scan evidence; failing closed without repository-authored retry or fallback allocation." >&2 +\telse +\t\techo "Strix quick scan failed; failing closed without repository-authored retry or fallback allocation." >&2 +\tfi +\treturn 1 +} + +''' + gate, count = re.subn( + r"(?ms)^run_current_target_scan\(\) \{\n.*?^\}\n\n(?=prepare_pull_request_scan_scope)", + simple_scan, + gate, + count=1, + ) + if count != 1: + raise SystemExit(f"expected one run_current_target_scan block, found {count}") + GATE.write_text(gate, encoding="utf-8") + + +def _repair_workflow() -> None: + workflow = WORKFLOW.read_text(encoding="utf-8") + for line in ( + " STRIX_LLM_MAX_RETRIES: 1\n", + " STRIX_TRANSIENT_RETRY_PER_MODEL: 2\n", + " STRIX_TRANSIENT_RETRY_BACKOFF_SECONDS: 60\n", + " STRIX_FAIL_ON_MIN_SEVERITY: MEDIUM\n", + ): + if line not in workflow: + raise SystemExit(f"expected workflow line missing: {line!r}") + workflow = workflow.replace(line, "", 1) + + workflow = workflow.replace( + " # severity branch anchored away from identifiers so environment lines\n" + " # such as STRIX_FAIL_ON_MIN_SEVERITY do not look like findings.\n", + " # Keep the severity marker anchored away from identifiers so unrelated\n" + " # environment text does not look like a reported finding.\n", + 1, + ) + + single_run = ''' strix_run_log="$RUNNER_TEMP/strix_gate_console.log" + : > "$strix_run_log" + strix_terminal_log="$strix_run_log" + strix_rc=0 + set +e + bash "$TRUSTED_STRIX_GATE" 2>&1 | tee "$strix_terminal_log" + strix_rc="${PIPESTATUS[0]}" + set -e + + if [ "$strix_rc" -eq 0 ]; then''' + workflow, count = re.subn( + r'''(?ms) strix_run_log="\$RUNNER_TEMP/strix_gate_console\.log"\n.*? set -e\n\n if \[ "\$strix_rc" -eq 0 \]; then''', + single_run, + workflow, + count=1, + ) + if count != 1: + raise SystemExit(f"expected one outer Strix retry loop, found {count}") + + workflow = workflow.replace("out-of-scope/below-threshold finding", "out-of-scope finding") + workflow = workflow.replace("below-threshold finding", "out-of-scope finding") + if "STRIX_FAIL_ON_MIN_SEVERITY" in workflow: + raise SystemExit("stale STRIX_FAIL_ON_MIN_SEVERITY remains in workflow") + if "strix_gate_attempt" in workflow or "STRIX_GATE_RETRY_BACKOFF_SECONDS" in workflow: + raise SystemExit("stale outer retry authority remains in workflow") + WORKFLOW.write_text(workflow, encoding="utf-8") + + +def _repair_existing_test() -> None: + source = QUEUE_TEST.read_text(encoding="utf-8") + old = ' assert "STRIX_FAIL_ON_MIN_SEVERITY: MEDIUM" in workflow\n' + new = ' assert "STRIX_FAIL_ON_MIN_SEVERITY" not in workflow\n' + if old not in source: + raise SystemExit("required workflow queue test threshold assertion not found") + QUEUE_TEST.write_text(source.replace(old, new, 1), encoding="utf-8") + + +def _append_trace() -> None: + marker = "## Retry and severity decision repair" + current = DOCTORING.read_text(encoding="utf-8") + if marker in current: + return + addition = ''' + +## Retry and severity decision repair + +Fresh protected-main evidence showed that the required Strix workflow still allocated two same-model gate retries plus a second outer three-attempt retry loop with fixed backoff values, while the reusable gate classified retryability through hand-authored provider/error regex families. The same path converted Strix severity labels to numeric ranks and used the repository-selected `MEDIUM` cutoff as a merge admission rule. Neither retry allocation nor the severity cutoff had an identified statistical model, authoritative standard, or executable experimental calibration. + +The repair therefore does not substitute different retry counts, backoff constants, severity weights, or cutoffs. The central Strix path executes the governed `orchestrator/free` request once; contextual-orchestrator retains provider discovery/failover authority. Any execution that fails to produce authoritative scan evidence fails closed. Any current vulnerability report artifact also fails closed without a repository-authored severity threshold. Severity labels may remain descriptive evidence, but they are not converted into a local admission score. +''' + DOCTORING.write_text(current.rstrip() + addition, encoding="utf-8") + + +def main() -> None: + _prove_red() + _repair_gate() + _repair_workflow() + _repair_existing_test() + _append_trace() + _run_contract_file(RETRY_TEST) + _run_contract_file(SEVERITY_TEST) + print("Focused no-heuristics Strix contracts are GREEN after repair.") + + +if __name__ == "__main__": + main() From ffc0383ab9a86a34a90b88b3c5a5ab8d167ec522 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 11:21:15 +0900 Subject: [PATCH 10/14] fix(ci): simplify Strix no-heuristics source repair workflow --- ...-fix-strix-no-heuristic-retry-severity.yml | 211 ++---------------- 1 file changed, 13 insertions(+), 198 deletions(-) diff --git a/.github/workflows/source-fix-strix-no-heuristic-retry-severity.yml b/.github/workflows/source-fix-strix-no-heuristic-retry-severity.yml index 749afc9343..b7bd63991f 100644 --- a/.github/workflows/source-fix-strix-no-heuristic-retry-severity.yml +++ b/.github/workflows/source-fix-strix-no-heuristic-retry-severity.yml @@ -18,216 +18,31 @@ jobs: fetch-depth: 0 persist-credentials: true - - name: Prove retry and severity contracts are RED before repair + - name: Apply RED-before-GREEN exact-source repair shell: bash - run: | - set -euo pipefail - if python3 - <<'PY' - import importlib.util - from pathlib import Path - - for filename in ( - "tests/test_strix_no_heuristic_retry_contract.py", - "tests/test_strix_no_heuristic_severity_contract.py", - ): - path = Path(filename) - spec = importlib.util.spec_from_file_location(path.stem, path) - module = importlib.util.module_from_spec(spec) - assert spec.loader is not None - spec.loader.exec_module(module) - for name in dir(module): - if name.startswith("test_"): - getattr(module, name)() - PY - then - echo "::error::No-heuristics Strix regressions were already green before the production repair." - exit 1 - fi - - - name: Remove repository-authored retry allocation and severity admission - shell: bash - run: | - set -euo pipefail - python3 <<'PY' - from pathlib import Path - import re - - gate_path = Path("scripts/ci/strix_quick_gate.sh") - workflow_path = Path(".github/workflows/strix.yml") - queue_test_path = Path("tests/test_required_workflow_queue_contract.py") - - gate = gate_path.read_text(encoding="utf-8") - for line in ( - 'STRIX_TRANSIENT_RETRY_PER_MODEL="${STRIX_TRANSIENT_RETRY_PER_MODEL:-0}"\n', - 'STRIX_TRANSIENT_RETRY_BACKOFF_SECONDS="${STRIX_TRANSIENT_RETRY_BACKOFF_SECONDS:-3}"\n', - '\trequire_non_negative_integer "$STRIX_TRANSIENT_RETRY_PER_MODEL" "STRIX_TRANSIENT_RETRY_PER_MODEL"\n', - '\trequire_non_negative_integer "$STRIX_TRANSIENT_RETRY_BACKOFF_SECONDS" "STRIX_TRANSIENT_RETRY_BACKOFF_SECONDS"\n', - ): - if line not in gate: - raise SystemExit(f"expected gate line missing: {line!r}") - gate = gate.replace(line, "", 1) - - for function_name in ( - "is_transient_same_model_retry_error", - "github_models_rate_limit_should_skip_same_model_retry", - "run_strix_with_transient_retry", - ): - pattern = rf"(?ms)^{re.escape(function_name)}\(\) \{{\n.*?^\}}\n\n" - gate, count = re.subn(pattern, "", gate, count=1) - if count != 1: - raise SystemExit(f"expected exactly one {function_name} function, found {count}") - - old_success = '''\tif [ "$rc" -eq 0 ]; then -\t\tif has_blocking_vulnerability_reports; then -\t\t\tif ! evaluate_pull_request_findings || [ "$PR_FINDINGS_DECISION" != "allow_baseline" ]; then -\t\t\t\techo "Strix exited successfully but emitted a vulnerability at or above '$STRIX_FAIL_ON_MIN_SEVERITY'; failing closed." >&2 -\t\t\t\treturn 1 -\t\t\tfi -\t\tfi -\t\tprintf "Strix run succeeded for model '%s' in %ds.\\n" "$model" "$elapsed" >&2 -\t\treturn 0 -\tfi -''' - new_success = '''\tif [ "$rc" -eq 0 ]; then -\t\tlocal current_vulnerability_file -\t\tcurrent_vulnerability_file="$(find "$ACTIVE_REPORTS_DIR" -type f -path '*/vulnerabilities/*.md' -print -quit 2>/dev/null || true)" -\t\tif [ -n "$current_vulnerability_file" ]; then -\t\t\techo "Current Strix vulnerability report exists; failing closed without a repository-authored severity threshold." >&2 -\t\t\treturn 1 -\t\tfi -\t\tprintf "Strix run succeeded for model '%s' in %ds.\\n" "$model" "$elapsed" >&2 -\t\treturn 0 -\tfi -''' - if old_success not in gate: - raise SystemExit("run_strix_once success block did not match expected source") - gate = gate.replace(old_success, new_success, 1) - - simple_scan = '''run_current_target_scan() { -\tINFRA_ERROR_DETECTED=0 -\tZERO_FINDINGS_REPORTED=0 + run: python3 scripts/ci/source_fix_strix_no_heuristic_retry_severity.py -\tlocal primary_scan_rc=0 -\trun_strix_once "$PRIMARY_MODEL" || primary_scan_rc=$? -\tif [ "$primary_scan_rc" -eq 0 ]; then -\t\treturn 0 -\tfi -\tif [ "$primary_scan_rc" -eq 2 ]; then -\t\treturn 2 -\tfi -\tif [ "$INFRA_ERROR_DETECTED" -eq 1 ]; then -\t\techo "STRIX_PROVIDER_UNAVAILABLE: contextual-orchestrator/orchestrator/free did not produce authoritative scan evidence; failing closed without repository-authored retry or fallback allocation." >&2 -\telse -\t\techo "Strix quick scan failed; failing closed without repository-authored retry or fallback allocation." >&2 -\tfi -\treturn 1 -} - -''' - gate, count = re.subn( - r"(?ms)^run_current_target_scan\(\) \{\n.*?^\}\n\n(?=prepare_pull_request_scan_scope)", - simple_scan, - gate, - count=1, - ) - if count != 1: - raise SystemExit(f"expected one run_current_target_scan block, found {count}") - gate_path.write_text(gate, encoding="utf-8") - - workflow = workflow_path.read_text(encoding="utf-8") - for line in ( - " STRIX_LLM_MAX_RETRIES: 1\n", - " STRIX_TRANSIENT_RETRY_PER_MODEL: 2\n", - " STRIX_TRANSIENT_RETRY_BACKOFF_SECONDS: 60\n", - " STRIX_FAIL_ON_MIN_SEVERITY: MEDIUM\n", - ): - if line not in workflow: - raise SystemExit(f"expected workflow line missing: {line!r}") - workflow = workflow.replace(line, "", 1) - - workflow = workflow.replace( - " # severity branch anchored away from identifiers so environment lines\n" - " # such as STRIX_FAIL_ON_MIN_SEVERITY do not look like findings.\n", - " # Keep the severity marker anchored away from identifiers so unrelated\n" - " # environment text does not look like a reported finding.\n", - 1, - ) - - single_run = ''' strix_run_log="$RUNNER_TEMP/strix_gate_console.log" - : > "$strix_run_log" - strix_terminal_log="$strix_run_log" - strix_rc=0 - set +e - bash "$TRUSTED_STRIX_GATE" 2>&1 | tee "$strix_terminal_log" - strix_rc="${PIPESTATUS[0]}" - set -e - - if [ "$strix_rc" -eq 0 ]; then''' - workflow, count = re.subn( - r'''(?ms) strix_run_log="\$RUNNER_TEMP/strix_gate_console\.log"\n.*? set -e\n\n if \[ "\$strix_rc" -eq 0 \]; then''', - single_run, - workflow, - count=1, - ) - if count != 1: - raise SystemExit(f"expected one outer Strix retry loop, found {count}") - - workflow = workflow.replace("out-of-scope/below-threshold finding", "out-of-scope finding") - workflow = workflow.replace("below-threshold finding", "out-of-scope finding") - if "STRIX_FAIL_ON_MIN_SEVERITY" in workflow: - raise SystemExit("stale STRIX_FAIL_ON_MIN_SEVERITY remains in workflow") - if "strix_gate_attempt" in workflow or "STRIX_GATE_RETRY_BACKOFF_SECONDS" in workflow: - raise SystemExit("stale outer retry authority remains in workflow") - workflow_path.write_text(workflow, encoding="utf-8") - - queue_test = queue_test_path.read_text(encoding="utf-8") - old_assert = ' assert "STRIX_FAIL_ON_MIN_SEVERITY: MEDIUM" in workflow\n' - new_assert = ' assert "STRIX_FAIL_ON_MIN_SEVERITY" not in workflow\n' - if old_assert not in queue_test: - raise SystemExit("required workflow queue test threshold assertion not found") - queue_test_path.write_text(queue_test.replace(old_assert, new_assert, 1), encoding="utf-8") - PY - - - name: Verify focused no-heuristics contracts + - name: Verify focused contracts and syntax shell: bash run: | set -euo pipefail - python3 - <<'PY' - import importlib.util - from pathlib import Path - - for filename in ( - "tests/test_strix_no_heuristic_retry_contract.py", - "tests/test_strix_no_heuristic_severity_contract.py", - ): - path = Path(filename) - spec = importlib.util.spec_from_file_location(path.stem, path) - module = importlib.util.module_from_spec(spec) - assert spec.loader is not None - spec.loader.exec_module(module) - for name in dir(module): - if name.startswith("test_"): - getattr(module, name)() - PY - python3 -m pytest -q tests/test_required_workflow_queue_contract.py + python3 -m pytest -q \ + tests/test_strix_no_heuristic_retry_contract.py \ + tests/test_strix_no_heuristic_severity_contract.py \ + tests/test_required_workflow_queue_contract.py + bash -n scripts/ci/strix_quick_gate.sh git diff --check - - name: Record research/causal trace and remove one-shot workflow + - name: Remove one-shot repair artifacts and publish non-destructively shell: bash run: | set -euo pipefail - cat >> docs/doctoring/strix-orchestrator-free-model-boundary-2026-09-02.md <<'EOF' - - ## Retry and severity decision repair - - Fresh protected-main evidence showed that the required Strix workflow still allocated two same-model gate retries plus a second outer three-attempt retry loop with fixed linear/exponential-style sleep values, while the reusable gate classified retryability through hand-authored provider/error regex families. The same path converted Strix severity labels to numeric ranks and used the repository-selected `MEDIUM` cutoff as a merge admission rule. Neither retry allocation nor the severity cutoff had an identified statistical model, authoritative standard, or executable experimental calibration. - - The repair therefore does not substitute different retry counts, backoff constants, severity weights, or cutoffs. The central Strix path executes the governed `orchestrator/free` request once; contextual-orchestrator retains provider discovery/failover authority. Any execution that fails to produce authoritative scan evidence fails closed. Any current vulnerability report artifact also fails closed without a repository-authored severity threshold. Severity labels may remain descriptive evidence, but they are not converted into a local admission score. - EOF - rm -f .github/workflows/source-fix-strix-no-heuristic-retry-severity.yml + rm -f \ + .github/workflows/source-fix-strix-no-heuristic-retry-severity.yml \ + scripts/ci/source_fix_strix_no_heuristic_retry_severity.py git diff --check git config user.name "github-actions[bot]" git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add .github/workflows/strix.yml scripts/ci/strix_quick_gate.sh tests/test_required_workflow_queue_contract.py tests/test_strix_no_heuristic_retry_contract.py tests/test_strix_no_heuristic_severity_contract.py docs/doctoring/strix-orchestrator-free-model-boundary-2026-09-02.md .github/workflows/source-fix-strix-no-heuristic-retry-severity.yml + git add -A git commit -m "fix(strix): remove heuristic retry and severity admission" git push origin HEAD:fix/strix-free-only-model-normalization-20260902 From cbbab94537cb574f6df898771a1c060d09a40e5f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 2 Sep 2026 11:36:57 +0000 Subject: [PATCH 11/14] docs(strix): reconcile free-only routing evidence --- ...rce-fix-strix-free-only-model-boundary.yml | 96 ------------------- CHANGELOG.md | 1 + docs/product-technical-gap-baseline.md | 10 ++ 3 files changed, 11 insertions(+), 96 deletions(-) delete mode 100644 .github/workflows/source-fix-strix-free-only-model-boundary.yml diff --git a/.github/workflows/source-fix-strix-free-only-model-boundary.yml b/.github/workflows/source-fix-strix-free-only-model-boundary.yml deleted file mode 100644 index 55afdc490f..0000000000 --- a/.github/workflows/source-fix-strix-free-only-model-boundary.yml +++ /dev/null @@ -1,96 +0,0 @@ -name: Source fix Strix free-only model boundary - -on: - push: - branches: - - fix/strix-free-only-model-normalization-20260902 - -permissions: - contents: write - -jobs: - reconcile-docs: - runs-on: ubuntu-24.04 - steps: - - name: Checkout exact branch head - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - with: - fetch-depth: 0 - persist-credentials: true - - - name: Append canonical gap and changelog evidence once - shell: bash - run: | - set -euo pipefail - python3 <<'PY' - from pathlib import Path - - gap = Path("docs/product-technical-gap-baseline.md") - changelog = Path("CHANGELOG.md") - - gap_marker = "## 2026-09-02 Strix direct-provider model-boundary repair" - gap_text = """ - - ## 2026-09-02 Strix direct-provider model-boundary repair - - **Live gap.** Protected `main@23df081c36c93da019c89c474351002afb014daa` already provisions the central Strix workflow through contextual-orchestrator with `STRIX_MODEL=contextual-orchestrator/orchestrator/free`, `CONTEXTUAL_ORCHESTRATOR_POOL=free`, private-target ZDR enforcement, all five bootstrap credential variables, and no external fallback list. The reusable Strix model normalizer nevertheless continued accepting concrete provider/model identifiers, leaving a second routing surface below the central workflow. - - **Causal owner and repair.** `scripts/ci/strix_model_utils.sh` is the earliest reusable model-admission boundary. Regression commit `2083a72dccaa1d96ea423a51af537240fde8a210` requires direct OpenAI, OpenRouter, NVIDIA NIM, GitHub Models, Vertex/Gemini, and unqualified identifiers to fail closed. Production commit `10c1ddf822f1e6336b73a9093a56680fea8f4f54` admits only `orchestrator/free` and `contextual-orchestrator/orchestrator/free`. No fallback rank, weight, score, threshold, or provider order replaces the removed authority; contextual-orchestrator owns candidate generation, routing, serving, and failover. - - **Credential/privacy boundary.** The central workflow must continue supplying `BYTEZ_API_KEY`, `NVIDIA_NIM_API_KEY`, `NVIDIA_NIM_API_KEY_SUB`, `OPENROUTER_API_KEY`, and `OPENAI_API_KEY` as bootstrap/global-discovery inputs. `OPENAI_API_KEY` transport is not a defect. The separate contextual-orchestrator `orchestrator/free` candidate policy excludes OpenAI-derived candidates while permitting the four eligible provider-account sources only when their explicit free/privacy/capability evidence passes. Private-target ZDR remains mandatory. - - **Verification.** Hosted exact-head checks and independent review remain authoritative. Pending, queued, stale, predecessor-head, or synthetic evidence is non-passing. Historical direct-provider fallback helpers are now unreachable through the accepted model normalizer and remain dead-code cleanup debt until removed with exact-head regression evidence. - """.rstrip() + "\n" - - text = gap.read_text(encoding="utf-8") - if gap_marker not in text: - gap.write_text(text.rstrip() + gap_text, encoding="utf-8") - - changelog_marker = "Strix model normalization now fails closed on every direct-provider model identifier" - changelog_text = """ - - Strix model normalization now fails closed on every direct-provider model identifier and accepts only the contextual-orchestrator `orchestrator/free` virtual pool spellings. This closes the reusable model-selection bypass without removing `OPENAI_API_KEY` from bootstrap/global discovery; the orchestrator's free-pool credential-source policy remains the candidate-admission authority. - """.strip() + "\n" - current = changelog.read_text(encoding="utf-8") - if changelog_marker not in current: - lines = current.splitlines(keepends=True) - insert_at = next((i + 1 for i, line in enumerate(lines) if line.strip() == "## [Unreleased]"), None) - if insert_at is None: - raise SystemExit("CHANGELOG.md has no [Unreleased] heading") - lines.insert(insert_at, changelog_text) - changelog.write_text("".join(lines), encoding="utf-8") - PY - - - name: Verify focused model-boundary contract - shell: bash - run: | - set -euo pipefail - python3 - <<'PY' - import importlib.util - from pathlib import Path - - path = Path("tests/test_strix_orchestrator_free_only.py") - spec = importlib.util.spec_from_file_location("strix_free_contract", path) - module = importlib.util.module_from_spec(spec) - assert spec.loader is not None - spec.loader.exec_module(module) - module.test_strix_accepts_only_the_governed_free_virtual_model_ids() - module.test_strix_rejects_every_direct_provider_model_before_execution() - PY - - - name: Remove one-shot source-fix workflow and commit - shell: bash - env: - BRANCH: fix/strix-free-only-model-normalization-20260902 - run: | - set -euo pipefail - rm -f .github/workflows/source-fix-strix-free-only-model-boundary.yml - git diff --check - if git diff --quiet; then - echo "No reconciliation delta remains." - exit 0 - fi - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add docs/product-technical-gap-baseline.md CHANGELOG.md .github/workflows/source-fix-strix-free-only-model-boundary.yml - git commit -m "docs(strix): reconcile free-only routing evidence" - git push origin "HEAD:${BRANCH}" diff --git a/CHANGELOG.md b/CHANGELOG.md index 552966c233..d360454307 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ this file. The format follows Keep a Changelog, and versioned releases follow Semantic Versioning where the repository publishes a release. ## [Unreleased] +- Strix model normalization now fails closed on every direct-provider model identifier and accepts only the contextual-orchestrator `orchestrator/free` virtual pool spellings. This closes the reusable model-selection bypass without removing `OPENAI_API_KEY` from bootstrap/global discovery; the orchestrator's free-pool credential-source policy remains the candidate-admission authority. - **Fix stale test assertions and dead-code gaps left by `#1654`, `#1656`, and `#1658`.** Reproduced all failures on a fresh unmodified `main` clone before attributing blame. `#1654` (introducing `scripts/ci/current_head_run_coalescer.py` and hardening several diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 29acdfeecc..81d8cd64fb 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -2613,3 +2613,13 @@ Higgins, S. S., Crepalde, N., & Fernandes, L. (2021). Segmented multiplexity: A **Expected effect.** No observable change to any current GitHub Actions review run (every current invocation already resolves to `free`). The effect is structural: it is no longer possible for a future workflow edit or manual dispatch override to admit priced-model spend into a required review check without an explicit, reviewed code change to this one `case` statement (and its now-locked-in regression test) first. **Follow-up.** If the organization later solves free+ZDR routing robustly enough to deliberately widen required-review CI to `orchestrator/auto` (e.g. once a spend ceiling and reviewer-visible cost evidence exist for that path), the change is exactly one `case` arm plus the corresponding assertions in `test_sidecar_pins_the_pool_to_free_for_github_actions` — this entry is the record of *why* it was narrowed, not a permanent prohibition. + +## 2026-09-02 Strix direct-provider model-boundary repair + +**Live gap.** Protected `main@23df081c36c93da019c89c474351002afb014daa` already provisions the central Strix workflow through contextual-orchestrator with `STRIX_MODEL=contextual-orchestrator/orchestrator/free`, `CONTEXTUAL_ORCHESTRATOR_POOL=free`, private-target ZDR enforcement, all five bootstrap credential variables, and no external fallback list. The reusable Strix model normalizer nevertheless continued accepting concrete provider/model identifiers, leaving a second routing surface below the central workflow. + +**Causal owner and repair.** `scripts/ci/strix_model_utils.sh` is the earliest reusable model-admission boundary. Regression commit `2083a72dccaa1d96ea423a51af537240fde8a210` requires direct OpenAI, OpenRouter, NVIDIA NIM, GitHub Models, Vertex/Gemini, and unqualified identifiers to fail closed. Production commit `10c1ddf822f1e6336b73a9093a56680fea8f4f54` admits only `orchestrator/free` and `contextual-orchestrator/orchestrator/free`. No fallback rank, weight, score, threshold, or provider order replaces the removed authority; contextual-orchestrator owns candidate generation, routing, serving, and failover. + +**Credential/privacy boundary.** The central workflow must continue supplying `BYTEZ_API_KEY`, `NVIDIA_NIM_API_KEY`, `NVIDIA_NIM_API_KEY_SUB`, `OPENROUTER_API_KEY`, and `OPENAI_API_KEY` as bootstrap/global-discovery inputs. `OPENAI_API_KEY` transport is not a defect. The separate contextual-orchestrator `orchestrator/free` candidate policy excludes OpenAI-derived candidates while permitting the four eligible provider-account sources only when their explicit free/privacy/capability evidence passes. Private-target ZDR remains mandatory. + +**Verification.** Hosted exact-head checks and independent review remain authoritative. Pending, queued, stale, predecessor-head, or synthetic evidence is non-passing. Historical direct-provider fallback helpers are now unreachable through the accepted model normalizer and remain dead-code cleanup debt until removed with exact-head regression evidence. From 15027c1970aebc2049732681029e98deb83e2510 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 09:54:10 +0900 Subject: [PATCH 12/14] fix(strix): complete the no-heuristics retry/severity repair by hand main had moved substantially since this branch's RED-before-repair commits and its one-shot source_fix_strix_no_heuristic_retry_severity.py driver were authored: run_current_target_scan had grown a full cross-model fallback loop (FALLBACK_MODELS_RAW, per-candidate retry, PR_FINDINGS_DECISION-gated blocking, a severity_rank/ STRIX_MAX_SEVERITY_RANK threshold check) the driver's simple_scan template didn't anticipate matching text for, so running it failed loudly rather than corrupting anything. Reconciled by hand instead of patching the driver's brittle string matches: confirmed the fallback-model system was already present at this branch's own base commit (not a later regression -- gutting it entirely was always the intended scope), then applied the equivalent repair directly -- strix_quick_gate.sh's run_current_target_scan now issues one governed request via run_strix_once and fails closed on anything else, with no repository-authored retry, model fallback, or severity-threshold admission; strix.yml drops the outer 3-attempt backoff loop and the STRIX_LLM_MAX_RETRIES/STRIX_TRANSIENT_RETRY_*/ STRIX_FAIL_ON_MIN_SEVERITY env vars; test_required_workflow_queue_ contract.py's stale assertion is fixed to match. Fixed the resulting ripple in six tests across four files that exercised the removed is_transient_same_model_retry_error/ github_models_rate_limit_should_skip_same_model_retry/ run_strix_with_transient_retry orchestration directly: three (Caido bootstrap timing, ModelBehaviorError, NVIDIA NIM 404) were narrowed to assert only that each signal classifier still feeds has_detected_infrastructure_error, since same-model retry and cross-model fallback no longer exist to wire into; two multi-attempt-specific assertions in the exempted-finding regression were removed as testing behavior that no longer exists, keeping the still-valid single-attempt/never-retries coverage; one now-stale comment reference to a removed function was corrected. Removed source_fix_strix_no_heuristic_retry_severity.py and its companion workflow per the driver's own documented one-shot lifecycle ("deletes it after RED-before-repair and focused GREEN verification succeed") now that the repair it existed to apply is complete. Recorded the reconciliation in this doctoring entry's addendum. Full local triad: 2815 passed (22 pre-existing failures unrelated to this PR -- the same main-side admission-controller-burst/cron-cadence gap tracked and fixed in .github#1871, unaffected by this diff); coverage and docstring gaps match that same pre-existing baseline exactly, with zero new gaps from this change. Co-Authored-By: Claude Sonnet 5 --- ...-fix-strix-no-heuristic-retry-severity.yml | 48 --- .github/workflows/strix.yml | 56 +--- ...estrator-free-model-boundary-2026-09-02.md | 8 + ...e_fix_strix_no_heuristic_retry_severity.py | 217 ------------- scripts/ci/strix_quick_gate.sh | 307 +----------------- .../test_required_workflow_queue_contract.py | 2 +- ...kend_unavailable_after_exempted_finding.py | 26 +- ...test_strix_caido_bootstrap_timing_retry.py | 37 +-- tests/test_strix_model_behavior_error.py | 16 +- ...est_strix_nvidia_nim_not_found_fallback.py | 27 +- 10 files changed, 61 insertions(+), 683 deletions(-) delete mode 100644 .github/workflows/source-fix-strix-no-heuristic-retry-severity.yml delete mode 100644 scripts/ci/source_fix_strix_no_heuristic_retry_severity.py diff --git a/.github/workflows/source-fix-strix-no-heuristic-retry-severity.yml b/.github/workflows/source-fix-strix-no-heuristic-retry-severity.yml deleted file mode 100644 index b7bd63991f..0000000000 --- a/.github/workflows/source-fix-strix-no-heuristic-retry-severity.yml +++ /dev/null @@ -1,48 +0,0 @@ -name: Source fix Strix no-heuristic retry and severity - -on: - push: - branches: - - fix/strix-free-only-model-normalization-20260902 - -permissions: - contents: write - -jobs: - repair: - runs-on: ubuntu-24.04 - steps: - - name: Checkout exact branch head - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - with: - fetch-depth: 0 - persist-credentials: true - - - name: Apply RED-before-GREEN exact-source repair - shell: bash - run: python3 scripts/ci/source_fix_strix_no_heuristic_retry_severity.py - - - name: Verify focused contracts and syntax - shell: bash - run: | - set -euo pipefail - python3 -m pytest -q \ - tests/test_strix_no_heuristic_retry_contract.py \ - tests/test_strix_no_heuristic_severity_contract.py \ - tests/test_required_workflow_queue_contract.py - bash -n scripts/ci/strix_quick_gate.sh - git diff --check - - - name: Remove one-shot repair artifacts and publish non-destructively - shell: bash - run: | - set -euo pipefail - rm -f \ - .github/workflows/source-fix-strix-no-heuristic-retry-severity.yml \ - scripts/ci/source_fix_strix_no_heuristic_retry_severity.py - git diff --check - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add -A - git commit -m "fix(strix): remove heuristic retry and severity admission" - git push origin HEAD:fix/strix-free-only-model-normalization-20260902 diff --git a/.github/workflows/strix.yml b/.github/workflows/strix.yml index 97ed60e320..68d8f7be5b 100644 --- a/.github/workflows/strix.yml +++ b/.github/workflows/strix.yml @@ -922,9 +922,6 @@ jobs: # The gateway auto pool is provider-diverse. Strix function tools # must not send a provider-specific reasoning setting to every route. STRIX_REASONING_EFFORT: none - STRIX_LLM_MAX_RETRIES: 1 - STRIX_TRANSIENT_RETRY_PER_MODEL: 2 - STRIX_TRANSIENT_RETRY_BACKOFF_SECONDS: 60 # The gateway owns discovery and provider failover; Strix must not # bypass its ZDR/privacy policy with an external fallback model. STRIX_FALLBACK_MODELS: "" @@ -933,7 +930,6 @@ jobs: PNPM_CONFIG_IGNORE_SCRIPTS: "true" YARN_ENABLE_SCRIPTS: "false" BUN_CONFIG_IGNORE_SCRIPTS: "true" - STRIX_FAIL_ON_MIN_SEVERITY: MEDIUM STRIX_DISABLE_PR_SCOPING: ${{ (github.event_name == 'pull_request_target' || github.event.client_payload.pr_number != '') && '0' || '1' }} # A repository_dispatch executes in this central repository, so its # github.token cannot read the target repository's PR. Reuse the @@ -957,8 +953,8 @@ jobs: model_behavior_error_signal='(^|[^A-Za-z0-9_])(agents|pydantic_ai|strix)(\.[A-Za-z_][A-Za-z0-9_]*)*\.ModelBehaviorError([^A-Za-z0-9_]|$)' # Any evidence that a vulnerability was actually reported. Its presence # forces a hard failure so real findings are NEVER downgraded. Keep the - # severity branch anchored away from identifiers so environment lines - # such as STRIX_FAIL_ON_MIN_SEVERITY do not look like findings. + # Keep the severity marker anchored away from identifiers so unrelated + # environment text does not look like a reported finding. reported_vulnerability_signal='Vulnerabilities[[:space:]]+[1-9]|(^|[^A-Za-z0-9_])severity[[:space:]]*:' # Capture the gate exit code plus its console output. The gate returns @@ -979,51 +975,9 @@ jobs: : > "$strix_run_log" strix_terminal_log="$strix_run_log" strix_rc=0 - strix_gate_attempt=1 set +e - while : ; do - strix_attempt_log="$RUNNER_TEMP/strix_gate_console_attempt_${strix_gate_attempt}.log" - : > "$strix_attempt_log" - bash "$TRUSTED_STRIX_GATE" 2>&1 | tee "$strix_attempt_log" - strix_rc="${PIPESTATUS[0]}" - cat "$strix_attempt_log" >> "$strix_run_log" - strix_terminal_log="$strix_attempt_log" - if [ "$strix_rc" -eq 0 ]; then - break - fi - # Only exit-code 1 scan failures can be infrastructure outcomes. - if [ "$strix_rc" -ne 1 ]; then - break - fi - # Scope this attempt's retry decision to the log tail after the - # last pipeline-continuation marker, exactly like the terminal - # classification below: an already-exempted finding before the - # marker must not mask a retryable outage after it. - strix_retry_scope_log="$strix_terminal_log" - if grep -Fq 'allowing pipeline continuation' "$strix_terminal_log"; then - strix_retry_scope_log="$RUNNER_TEMP/strix_gate_console_tail.log" - awk '/allowing pipeline continuation/{buf=""; next} {buf=buf $0 "\n"} END{printf "%s", buf}' \ - "$strix_terminal_log" > "$strix_retry_scope_log" - fi - # A reported vulnerability is authoritative evidence: never retry - # and never risk downgrading it. - if grep -Eiq "$reported_vulnerability_signal" "$strix_retry_scope_log"; then - break - fi - # Retry only recognized provider-outage / model-behavior classes. - if ! grep -Eiq "$backend_unavailable_signal" "$strix_retry_scope_log" \ - && ! grep -Eq "$model_behavior_error_signal" "$strix_retry_scope_log"; then - break - fi - backoff_seconds=$(( ${STRIX_GATE_RETRY_BACKOFF_SECONDS:-90} * strix_gate_attempt )) - if [ "$strix_gate_attempt" -ge 3 ]; then - echo "Provider-unavailable Strix attempt ${strix_gate_attempt} reached the retry limit; failing closed." >&2 - break - fi - echo "Strix provider outage on attempt ${strix_gate_attempt}; retrying after ${backoff_seconds}s backoff." >&2 - sleep "$backoff_seconds" - strix_gate_attempt=$(( strix_gate_attempt + 1 )) - done + bash "$TRUSTED_STRIX_GATE" 2>&1 | tee "$strix_terminal_log" + strix_rc="${PIPESTATUS[0]}" set -e if [ "$strix_rc" -eq 0 ]; then @@ -1037,7 +991,7 @@ jobs: exit "$strix_rc" fi - # An earlier out-of-scope/below-threshold finding may already have + # An earlier out-of-scope finding may already have # been exempted by the trusted gate. Classify a later provider # outage from the tail after the last continuation marker, but keep # that incomplete later scan non-passing. diff --git a/docs/doctoring/strix-orchestrator-free-model-boundary-2026-09-02.md b/docs/doctoring/strix-orchestrator-free-model-boundary-2026-09-02.md index 9df7aa5f6c..b0d2d43d00 100644 --- a/docs/doctoring/strix-orchestrator-free-model-boundary-2026-09-02.md +++ b/docs/doctoring/strix-orchestrator-free-model-boundary-2026-09-02.md @@ -23,3 +23,11 @@ Private-target ZDR remains enforced by the central workflow and sidecar. A direc ## Verification status Fresh hosted exact-head tests are required before merge. Queued, pending, stale, predecessor-head, or synthetic evidence is non-passing. Historical direct-provider fallback code is now unreachable through the accepted model normalizer but remains cleanup debt until a subsequent exact-head change removes it without losing unrelated Strix scanner behavior. + +## Retry and severity decision repair + +Fresh protected-main evidence showed that the required Strix workflow still allocated two same-model gate retries plus a second outer three-attempt retry loop with fixed backoff values, while the reusable gate classified retryability through hand-authored provider/error regex families. The same path converted Strix severity labels to numeric ranks and used the repository-selected `MEDIUM` cutoff as a merge admission rule. Neither retry allocation nor the severity cutoff had an identified statistical model, authoritative standard, or executable experimental calibration. + +The repair therefore does not substitute different retry counts, backoff constants, severity weights, or cutoffs. The central Strix path executes the governed `orchestrator/free` request once; contextual-orchestrator retains provider discovery/failover authority. Any execution that fails to produce authoritative scan evidence fails closed. Any current vulnerability report artifact also fails closed without a repository-authored severity threshold. Severity labels may remain descriptive evidence, but they are not converted into a local admission score. + +**2026-09-05 addendum: applied by hand, not by the one-shot driver.** `main` had moved substantially since this doctoring entry and the `2083a72d`/`10c1ddf8` commits were authored: `run_current_target_scan` had grown a full cross-model fallback loop (`FALLBACK_MODELS_RAW`, per-candidate retry, `PR_FINDINGS_DECISION`-gated blocking, a `severity_rank`/`STRIX_MAX_SEVERITY_RANK` threshold check) that the driver script's own `simple_scan` template did not anticipate matching text for. Reconciled by hand: replaced the entire function with the single-governed-request form the driver always intended, confirmed the fallback-model system was already present at this PR's own base commit (not a later regression), and fixed the resulting ripple in six tests across four files that exercised the now-removed `is_transient_same_model_retry_error`/`github_models_rate_limit_should_skip_same_model_retry`/`run_strix_with_transient_retry` orchestration directly — three were rewritten to assert only that each underlying signal classifier (Caido bootstrap timing, ModelBehaviorError, NVIDIA NIM 404) still feeds `has_detected_infrastructure_error`, and three multi-attempt-specific assertions were removed as testing behavior that no longer exists. `scripts/ci/source_fix_strix_no_heuristic_retry_severity.py` and its companion workflow are removed here, matching the driver's own documented one-shot lifecycle, since the repair they existed to apply is now complete. diff --git a/scripts/ci/source_fix_strix_no_heuristic_retry_severity.py b/scripts/ci/source_fix_strix_no_heuristic_retry_severity.py deleted file mode 100644 index 78a56f9e08..0000000000 --- a/scripts/ci/source_fix_strix_no_heuristic_retry_severity.py +++ /dev/null @@ -1,217 +0,0 @@ -"""One-shot exact-source repair for Strix retry/severity no-heuristics contracts. - -This driver is intentionally temporary. The companion workflow deletes it after -RED-before-repair and focused GREEN verification succeed on the same branch head. -""" - -from __future__ import annotations - -import importlib.util -import re -from pathlib import Path - -ROOT = Path(__file__).resolve().parents[2] -GATE = ROOT / "scripts" / "ci" / "strix_quick_gate.sh" -WORKFLOW = ROOT / ".github" / "workflows" / "strix.yml" -QUEUE_TEST = ROOT / "tests" / "test_required_workflow_queue_contract.py" -RETRY_TEST = ROOT / "tests" / "test_strix_no_heuristic_retry_contract.py" -SEVERITY_TEST = ROOT / "tests" / "test_strix_no_heuristic_severity_contract.py" -DOCTORING = ROOT / "docs" / "doctoring" / "strix-orchestrator-free-model-boundary-2026-09-02.md" - - -def _run_contract_file(path: Path) -> None: - spec = importlib.util.spec_from_file_location(path.stem, path) - if spec is None or spec.loader is None: - raise RuntimeError(f"cannot load contract {path}") - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - for name in sorted(dir(module)): - if name.startswith("test_"): - getattr(module, name)() - - -def _prove_red() -> None: - failures: list[str] = [] - for path in (RETRY_TEST, SEVERITY_TEST): - try: - _run_contract_file(path) - except AssertionError as exc: - failures.append(f"{path.name}: {exc}") - if not failures: - raise SystemExit("no-heuristics Strix contracts were already GREEN before repair") - print("RED contracts observed:") - for failure in failures: - print(f"- {failure}") - - -def _remove_shell_function(source: str, name: str) -> str: - pattern = rf"(?ms)^{re.escape(name)}\(\) \{{\n.*?^\}}\n\n" - result, count = re.subn(pattern, "", source, count=1) - if count != 1: - raise SystemExit(f"expected exactly one {name} function, found {count}") - return result - - -def _repair_gate() -> None: - gate = GATE.read_text(encoding="utf-8") - for line in ( - 'STRIX_TRANSIENT_RETRY_PER_MODEL="${STRIX_TRANSIENT_RETRY_PER_MODEL:-0}"\n', - 'STRIX_TRANSIENT_RETRY_BACKOFF_SECONDS="${STRIX_TRANSIENT_RETRY_BACKOFF_SECONDS:-3}"\n', - '\trequire_non_negative_integer "$STRIX_TRANSIENT_RETRY_PER_MODEL" "STRIX_TRANSIENT_RETRY_PER_MODEL"\n', - '\trequire_non_negative_integer "$STRIX_TRANSIENT_RETRY_BACKOFF_SECONDS" "STRIX_TRANSIENT_RETRY_BACKOFF_SECONDS"\n', - ): - if line not in gate: - raise SystemExit(f"expected gate line missing: {line!r}") - gate = gate.replace(line, "", 1) - - for function_name in ( - "is_transient_same_model_retry_error", - "github_models_rate_limit_should_skip_same_model_retry", - "run_strix_with_transient_retry", - ): - gate = _remove_shell_function(gate, function_name) - - old_success = '''\tif [ "$rc" -eq 0 ]; then -\t\tif has_blocking_vulnerability_reports; then -\t\t\tif ! evaluate_pull_request_findings || [ "$PR_FINDINGS_DECISION" != "allow_baseline" ]; then -\t\t\t\techo "Strix exited successfully but emitted a vulnerability at or above '$STRIX_FAIL_ON_MIN_SEVERITY'; failing closed." >&2 -\t\t\t\treturn 1 -\t\t\tfi -\t\tfi -\t\tprintf "Strix run succeeded for model '%s' in %ds.\\n" "$model" "$elapsed" >&2 -\t\treturn 0 -\tfi -''' - new_success = '''\tif [ "$rc" -eq 0 ]; then -\t\tlocal current_vulnerability_file -\t\tcurrent_vulnerability_file="$(find "$ACTIVE_REPORTS_DIR" -type f -path '*/vulnerabilities/*.md' -print -quit 2>/dev/null || true)" -\t\tif [ -n "$current_vulnerability_file" ]; then -\t\t\techo "Current Strix vulnerability report exists; failing closed without a repository-authored severity threshold." >&2 -\t\t\treturn 1 -\t\tfi -\t\tprintf "Strix run succeeded for model '%s' in %ds.\\n" "$model" "$elapsed" >&2 -\t\treturn 0 -\tfi -''' - if old_success not in gate: - raise SystemExit("run_strix_once success block did not match expected source") - gate = gate.replace(old_success, new_success, 1) - - simple_scan = '''run_current_target_scan() { -\tINFRA_ERROR_DETECTED=0 -\tZERO_FINDINGS_REPORTED=0 - -\tlocal primary_scan_rc=0 -\trun_strix_once "$PRIMARY_MODEL" || primary_scan_rc=$? -\tif [ "$primary_scan_rc" -eq 0 ]; then -\t\treturn 0 -\tfi -\tif [ "$primary_scan_rc" -eq 2 ]; then -\t\treturn 2 -\tfi -\tif [ "$INFRA_ERROR_DETECTED" -eq 1 ]; then -\t\techo "STRIX_PROVIDER_UNAVAILABLE: contextual-orchestrator/orchestrator/free did not produce authoritative scan evidence; failing closed without repository-authored retry or fallback allocation." >&2 -\telse -\t\techo "Strix quick scan failed; failing closed without repository-authored retry or fallback allocation." >&2 -\tfi -\treturn 1 -} - -''' - gate, count = re.subn( - r"(?ms)^run_current_target_scan\(\) \{\n.*?^\}\n\n(?=prepare_pull_request_scan_scope)", - simple_scan, - gate, - count=1, - ) - if count != 1: - raise SystemExit(f"expected one run_current_target_scan block, found {count}") - GATE.write_text(gate, encoding="utf-8") - - -def _repair_workflow() -> None: - workflow = WORKFLOW.read_text(encoding="utf-8") - for line in ( - " STRIX_LLM_MAX_RETRIES: 1\n", - " STRIX_TRANSIENT_RETRY_PER_MODEL: 2\n", - " STRIX_TRANSIENT_RETRY_BACKOFF_SECONDS: 60\n", - " STRIX_FAIL_ON_MIN_SEVERITY: MEDIUM\n", - ): - if line not in workflow: - raise SystemExit(f"expected workflow line missing: {line!r}") - workflow = workflow.replace(line, "", 1) - - workflow = workflow.replace( - " # severity branch anchored away from identifiers so environment lines\n" - " # such as STRIX_FAIL_ON_MIN_SEVERITY do not look like findings.\n", - " # Keep the severity marker anchored away from identifiers so unrelated\n" - " # environment text does not look like a reported finding.\n", - 1, - ) - - single_run = ''' strix_run_log="$RUNNER_TEMP/strix_gate_console.log" - : > "$strix_run_log" - strix_terminal_log="$strix_run_log" - strix_rc=0 - set +e - bash "$TRUSTED_STRIX_GATE" 2>&1 | tee "$strix_terminal_log" - strix_rc="${PIPESTATUS[0]}" - set -e - - if [ "$strix_rc" -eq 0 ]; then''' - workflow, count = re.subn( - r'''(?ms) strix_run_log="\$RUNNER_TEMP/strix_gate_console\.log"\n.*? set -e\n\n if \[ "\$strix_rc" -eq 0 \]; then''', - single_run, - workflow, - count=1, - ) - if count != 1: - raise SystemExit(f"expected one outer Strix retry loop, found {count}") - - workflow = workflow.replace("out-of-scope/below-threshold finding", "out-of-scope finding") - workflow = workflow.replace("below-threshold finding", "out-of-scope finding") - if "STRIX_FAIL_ON_MIN_SEVERITY" in workflow: - raise SystemExit("stale STRIX_FAIL_ON_MIN_SEVERITY remains in workflow") - if "strix_gate_attempt" in workflow or "STRIX_GATE_RETRY_BACKOFF_SECONDS" in workflow: - raise SystemExit("stale outer retry authority remains in workflow") - WORKFLOW.write_text(workflow, encoding="utf-8") - - -def _repair_existing_test() -> None: - source = QUEUE_TEST.read_text(encoding="utf-8") - old = ' assert "STRIX_FAIL_ON_MIN_SEVERITY: MEDIUM" in workflow\n' - new = ' assert "STRIX_FAIL_ON_MIN_SEVERITY" not in workflow\n' - if old not in source: - raise SystemExit("required workflow queue test threshold assertion not found") - QUEUE_TEST.write_text(source.replace(old, new, 1), encoding="utf-8") - - -def _append_trace() -> None: - marker = "## Retry and severity decision repair" - current = DOCTORING.read_text(encoding="utf-8") - if marker in current: - return - addition = ''' - -## Retry and severity decision repair - -Fresh protected-main evidence showed that the required Strix workflow still allocated two same-model gate retries plus a second outer three-attempt retry loop with fixed backoff values, while the reusable gate classified retryability through hand-authored provider/error regex families. The same path converted Strix severity labels to numeric ranks and used the repository-selected `MEDIUM` cutoff as a merge admission rule. Neither retry allocation nor the severity cutoff had an identified statistical model, authoritative standard, or executable experimental calibration. - -The repair therefore does not substitute different retry counts, backoff constants, severity weights, or cutoffs. The central Strix path executes the governed `orchestrator/free` request once; contextual-orchestrator retains provider discovery/failover authority. Any execution that fails to produce authoritative scan evidence fails closed. Any current vulnerability report artifact also fails closed without a repository-authored severity threshold. Severity labels may remain descriptive evidence, but they are not converted into a local admission score. -''' - DOCTORING.write_text(current.rstrip() + addition, encoding="utf-8") - - -def main() -> None: - _prove_red() - _repair_gate() - _repair_workflow() - _repair_existing_test() - _append_trace() - _run_contract_file(RETRY_TEST) - _run_contract_file(SEVERITY_TEST) - print("Focused no-heuristics Strix contracts are GREEN after repair.") - - -if __name__ == "__main__": - main() diff --git a/scripts/ci/strix_quick_gate.sh b/scripts/ci/strix_quick_gate.sh index 3a563d7020..f2d232437d 100755 --- a/scripts/ci/strix_quick_gate.sh +++ b/scripts/ci/strix_quick_gate.sh @@ -1,10 +1,11 @@ #!/usr/bin/env bash -# strix_quick_gate.sh — CI gate that runs Strix security scans with -# automatic model fallback, transient-error retry, and severity-based -# pass/fail decisions. +# strix_quick_gate.sh — CI gate that runs one governed Strix security scan +# request. No repository-authored retry, model fallback, or severity +# threshold decides admission: contextual-orchestrator's `orchestrator/free` +# gateway owns provider discovery and failover, and any current vulnerability +# report artifact fails the gate closed. # -# STRIX_LOG is a per-attempt temp file consumed only by -# is_transient_same_model_retry_error(); cumulative report dirs in +# STRIX_LOG is a per-attempt temp file; cumulative report dirs in # STRIX_REPORTS_DIR are never overwritten. Refer to ARCHITECTURE.md # for the 3-tier timeout classification hierarchy. set -euo pipefail @@ -44,8 +45,6 @@ STRIX_INPUT_FILE_ROOT="${STRIX_INPUT_FILE_ROOT:-${RUNNER_TEMP:-}}" STRIX_EXECUTABLE_PATH="${STRIX_EXECUTABLE_PATH:-}" STRIX_EXECUTABLE_ROOT="${STRIX_EXECUTABLE_ROOT:-}" STRIX_EXECUTABLE_SHA256="${STRIX_EXECUTABLE_SHA256:-}" -STRIX_TRANSIENT_RETRY_PER_MODEL="${STRIX_TRANSIENT_RETRY_PER_MODEL:-0}" -STRIX_TRANSIENT_RETRY_BACKOFF_SECONDS="${STRIX_TRANSIENT_RETRY_BACKOFF_SECONDS:-3}" STRIX_FAIL_ON_MIN_SEVERITY="${STRIX_FAIL_ON_MIN_SEVERITY:-MEDIUM}" STRIX_FAIL_ON_PROVIDER_SIGNAL="${STRIX_FAIL_ON_PROVIDER_SIGNAL:-0}" RUN_START_EPOCH=0 @@ -884,8 +883,6 @@ if is_github_models_model "$PRIMARY_MODEL" && [ -z "$LLM_API_BASE_FILE" ]; then exit 2 fi -require_non_negative_integer "$STRIX_TRANSIENT_RETRY_PER_MODEL" "STRIX_TRANSIENT_RETRY_PER_MODEL" -require_non_negative_integer "$STRIX_TRANSIENT_RETRY_BACKOFF_SECONDS" "STRIX_TRANSIENT_RETRY_BACKOFF_SECONDS" require_non_negative_integer "$STRIX_PROCESS_TIMEOUT_SECONDS" "STRIX_PROCESS_TIMEOUT_SECONDS" require_non_negative_integer "$STRIX_TOTAL_TIMEOUT_SECONDS" "STRIX_TOTAL_TIMEOUT_SECONDS" case "$STRIX_FAIL_ON_PROVIDER_SIGNAL" in @@ -2929,11 +2926,11 @@ PY fi if [ "$rc" -eq 0 ]; then - if has_blocking_vulnerability_reports; then - if ! evaluate_pull_request_findings || [ "$PR_FINDINGS_DECISION" != "allow_baseline" ]; then - echo "Strix exited successfully but emitted a vulnerability at or above '$STRIX_FAIL_ON_MIN_SEVERITY'; failing closed." >&2 - return 1 - fi + local current_vulnerability_file + current_vulnerability_file="$(find "$ACTIVE_REPORTS_DIR" -type f -path '*/vulnerabilities/*.md' -print -quit 2>/dev/null || true)" + if [ -n "$current_vulnerability_file" ]; then + echo "Current Strix vulnerability report exists; failing closed without a repository-authored severity threshold." >&2 + return 1 fi printf "Strix run succeeded for model '%s' in %ds.\n" "$model" "$elapsed" >&2 return 0 @@ -3043,108 +3040,6 @@ is_model_behavior_error() { return 1 } -## Determines whether the last strix failure is a transient error eligible -## for same-model retry (up to STRIX_TRANSIENT_RETRY_PER_MODEL times). -## Five error families qualify: -## - RateLimit / RESOURCE_EXHAUSTED / HTTP 429 -## - litellm API connection failures with LLM-provider evidence -## - litellm service-unavailable / high-demand provider failures -## - MidStreamFallbackError (litellm mid-stream provider switch) -## - Caido bootstrap timing failures (guest login before the local proxy is up) -## Timeouts are infrastructure failures. In strict CI mode they fail closed; -## otherwise the caller may still move to fallback model evaluation. -is_transient_same_model_retry_error() { - local model="${1-}" - if is_timeout_error; then - return 1 - fi - if is_llm_api_connection_error; then - return 0 - fi - if is_llm_service_unavailable_error; then - return 0 - fi - if is_rate_limit_error; then - return 0 - fi - if is_midstream_fallback_error; then - return 0 - fi - if is_caido_bootstrap_timing_error; then - return 0 - fi - return 1 -} - -github_models_rate_limit_should_skip_same_model_retry() { - local model="$1" - - if ! is_rate_limit_error; then - return 1 - fi - if ! is_github_models_api_compatible_model "$model"; then - return 1 - fi - github_models_api_base_is_active -} - -run_strix_with_transient_retry() { - local model="$1" - local max_attempts=$((STRIX_TRANSIENT_RETRY_PER_MODEL + 1)) - local attempt=1 - - while [ "$attempt" -le "$max_attempts" ]; do - local run_rc=0 - run_strix_once "$model" || run_rc=$? - if [ "$run_rc" -eq 0 ]; then - return 0 - fi - if [ "$run_rc" -eq 2 ]; then - return 2 - fi - if [ "$TOTAL_TIMEOUT_EXCEEDED" -eq 1 ]; then - return 1 - fi - - if [ "$attempt" -ge "$max_attempts" ]; then - return 1 - fi - - if [ "$STRIX_TOTAL_TIMEOUT_SECONDS" -gt 0 ] && [ "$(remaining_total_budget)" -le 0 ]; then - TOTAL_TIMEOUT_EXCEEDED=1 - printf "Strix quick scan exceeded total timeout of %ss.\n" "$STRIX_TOTAL_TIMEOUT_SECONDS" | tee "$STRIX_LOG" >&2 - return 1 - fi - - if github_models_rate_limit_should_skip_same_model_retry "$model"; then - echo "GitHub Models rate limit detected for model '$model'; skipping same-model retry and moving directly to fallback models or current-head neutral classification." >&2 - return 1 - fi - - if ! is_transient_same_model_retry_error "$model"; then - return 1 - fi - - local retry_reason="transient error" - if is_rate_limit_error; then - retry_reason="rate limit" - elif is_llm_api_connection_error; then - retry_reason="LLM API connection" - elif is_llm_service_unavailable_error; then - retry_reason="LLM service unavailable" - elif is_midstream_fallback_error; then - retry_reason="midstream fallback" - elif is_caido_bootstrap_timing_error; then - retry_reason="Caido sandbox bootstrap timing" - fi - echo "Retrying model '$model' due to $retry_reason (attempt $((attempt + 1))/$max_attempts)." >&2 - sleep "$STRIX_TRANSIENT_RETRY_BACKOFF_SECONDS" - attempt=$((attempt + 1)) - done - - return 1 -} - is_vertex_not_found_error() { # Match Vertex/LiteLLM model-not-found errors. # These functions are only called within the Vertex fallback path @@ -4321,189 +4216,17 @@ run_current_target_scan() { ZERO_FINDINGS_REPORTED=0 local primary_scan_rc=0 - run_strix_with_transient_retry "$PRIMARY_MODEL" || primary_scan_rc=$? + run_strix_once "$PRIMARY_MODEL" || primary_scan_rc=$? if [ "$primary_scan_rc" -eq 0 ]; then return 0 fi if [ "$primary_scan_rc" -eq 2 ]; then return 2 fi - if [ "$TOTAL_TIMEOUT_EXCEEDED" -eq 1 ]; then - return 1 - fi - - local strict_primary_provider_fallback=0 - if [ "$INFRA_ERROR_DETECTED" -eq 1 ] && provider_signal_fail_closed_enabled; then - if is_contextual_orchestrator_model "$PRIMARY_MODEL"; then - echo "STRIX_PROVIDER_UNAVAILABLE: contextual-orchestrator/orchestrator/free exhausted; the gateway owns provider discovery and failover." >&2 - return 1 - elif is_model_retryable_error "$PRIMARY_MODEL" && has_distinct_fallback_model_for_model "$PRIMARY_MODEL"; then - strict_primary_provider_fallback=1 - else - echo "Strix scan failed after provider infrastructure or failure-signal output; failing closed." >&2 - return 1 - fi - fi - - if has_only_below_threshold_vulnerabilities; then - return 0 - fi - - if evaluate_pull_request_findings; then - if [ "$strict_primary_provider_fallback" -eq 0 ]; then - return 0 - fi - fi - - case "$PR_FINDINGS_DECISION" in - block_changed | block_unmapped | block_manifest_unverified) - if [ "$strict_primary_provider_fallback" -eq 1 ] && fail_reported_vulnerabilities_before_fallback_success; then - return 1 - fi - echo "Strix quick scan failed with a non-recoverable error." >&2 - return 1 - ;; - esac - if fail_unmapped_threshold_report; then - return 1 - fi - - if [ "$strict_primary_provider_fallback" -eq 1 ] && fail_reported_vulnerabilities_before_fallback_success; then - return 1 - fi - - if ! is_model_retryable_error "$PRIMARY_MODEL"; then - echo "Strix quick scan failed with a non-recoverable error." >&2 - return 1 - fi - if is_contextual_orchestrator_model "$PRIMARY_MODEL"; then - echo "STRIX_PROVIDER_UNAVAILABLE: contextual-orchestrator/orchestrator/free exhausted; no external fallback is permitted." >&2 - return 1 - fi - - FALLBACK_MODELS_RAW="$(fallback_models_raw_for_model "$PRIMARY_MODEL")" - FALLBACK_MODELS_RAW="${FALLBACK_MODELS_RAW//$'\r'/ }" - FALLBACK_MODELS_RAW="${FALLBACK_MODELS_RAW//$'\n'/ }" - read -r -a FALLBACK_MODELS <<<"$FALLBACK_MODELS_RAW" - - fallback_tried=0 - for candidate_raw in "${FALLBACK_MODELS[@]}"; do - candidate="$(normalize_model "$candidate_raw")" - if [ -z "$candidate" ] || [ "$candidate" = "$PRIMARY_MODEL" ]; then - if [ -n "$candidate" ]; then - echo "Skipping fallback model '$candidate' — same as primary model." >&2 - fi - continue - fi - if [ "$TOTAL_TIMEOUT_EXCEEDED" -eq 1 ]; then - return 1 - fi - - fallback_tried=1 - if is_vertex_model "$PRIMARY_MODEL"; then - echo "Primary Vertex model unavailable; retrying with fallback '$candidate'." - else - echo "Primary model unavailable; retrying with fallback '$candidate'." - fi - local fallback_scan_rc=0 - local fallback_start_epoch - fallback_start_epoch="$(date +%s)" - run_strix_with_transient_retry "$candidate" || fallback_scan_rc=$? - local fallback_elapsed=$(( $(date +%s) - fallback_start_epoch )) - if [ "$fallback_scan_rc" -eq 0 ]; then - if fail_reported_vulnerabilities_before_fallback_success; then - return 1 - fi - echo "Strix quick scan succeeded with fallback model '$candidate' in ${fallback_elapsed}s." >&2 - return 0 - fi - if [ "$fallback_scan_rc" -eq 2 ]; then - return 2 - fi - - local strict_fallback_provider_signal=0 - if [ "$INFRA_ERROR_DETECTED" -eq 1 ] && provider_signal_fail_closed_enabled; then - strict_fallback_provider_signal=1 - fi - - if has_only_below_threshold_vulnerabilities; then - return 0 - fi - - if evaluate_pull_request_findings; then - if [ "$strict_fallback_provider_signal" -eq 0 ]; then - return 0 - fi - fi - - case "$PR_FINDINGS_DECISION" in - block_changed | block_unmapped | block_manifest_unverified) - if [ "$strict_fallback_provider_signal" -eq 1 ] && fail_reported_vulnerabilities_before_fallback_success; then - return 1 - fi - echo "Strix quick scan failed with a non-recoverable error." >&2 - return 1 - ;; - esac - if fail_unmapped_threshold_report; then - return 1 - fi - - if fail_reported_vulnerabilities_before_fallback_success; then - return 1 - fi - - if [ "$strict_fallback_provider_signal" -eq 1 ]; then - if is_model_retryable_error "$candidate"; then - continue - fi - echo "Strix fallback model '$candidate' emitted provider infrastructure or failure-signal output; trying next configured fallback if available." >&2 - continue - fi - - if ! is_model_retryable_error "$candidate"; then - echo "Strix quick scan failed with a non-recoverable error." >&2 - return 1 - fi - done - - if should_fail_pull_request_infra_zero_findings; then - return 1 - fi - - if [ "$fallback_tried" -eq 0 ]; then - local fallback_config_name - fallback_config_name="$(fallback_models_config_name_for_model "$PRIMARY_MODEL")" - local configured_fallback_count=0 - for candidate_raw in "${FALLBACK_MODELS[@]}"; do - candidate="$(normalize_model "$candidate_raw")" - [ -n "$candidate" ] && configured_fallback_count=$((configured_fallback_count + 1)) - done - if [ "$configured_fallback_count" -eq 0 ]; then - echo "ERROR: No fallback models configured ($fallback_config_name is empty). Configure distinct models." >&2 - else - echo "ERROR: All configured fallback models are the same as the primary model" >&2 - fi - return 1 - fi - - if [ "$INFRA_ERROR_DETECTED" -eq 1 ] && - [ "$PR_FINDINGS_DECISION" = "allow_baseline" ]; then - echo "STRIX_PROVIDER_UNAVAILABLE: provider models were exhausted after incomplete scan evidence." >&2 - return 1 - fi - - local threshold_rank - threshold_rank="$(severity_rank "$STRIX_FAIL_ON_MIN_SEVERITY")" - if [ "${STRIX_MAX_SEVERITY_RANK:--1}" -ge "$threshold_rank" ]; then - echo "Strix quick scan failed with a non-recoverable error." >&2 - return 1 - fi - - if is_vertex_model "$PRIMARY_MODEL"; then - echo "Configured Vertex model and fallback models were unavailable." >&2 + if [ "$INFRA_ERROR_DETECTED" -eq 1 ]; then + echo "STRIX_PROVIDER_UNAVAILABLE: contextual-orchestrator/orchestrator/free did not produce authoritative scan evidence; failing closed without repository-authored retry or fallback allocation." >&2 else - echo "Configured model and fallback models were unavailable." >&2 + echo "Strix quick scan failed; failing closed without repository-authored retry or fallback allocation." >&2 fi return 1 } diff --git a/tests/test_required_workflow_queue_contract.py b/tests/test_required_workflow_queue_contract.py index 803d43ab59..a2c1fdca3b 100644 --- a/tests/test_required_workflow_queue_contract.py +++ b/tests/test_required_workflow_queue_contract.py @@ -1421,7 +1421,7 @@ def test_strix_provider_outage_without_findings_is_typed_non_passing() -> None: assert "zero_vulnerabilities_signal" not in workflow assert "Vulnerabilities[[:space:]]+[1-9]" in workflow assert "(^|[^A-Za-z0-9_])severity[[:space:]]*:" in workflow - assert "STRIX_FAIL_ON_MIN_SEVERITY: MEDIUM" in workflow + assert "STRIX_FAIL_ON_MIN_SEVERITY" not in workflow assert "::error title=STRIX_PROVIDER_UNAVAILABLE::" in workflow assert 'exit "$strix_rc"' in workflow assert "Treating as a neutral skip" not in workflow diff --git a/tests/test_strix_backend_unavailable_after_exempted_finding.py b/tests/test_strix_backend_unavailable_after_exempted_finding.py index f9b75e313d..eb10cfe7c7 100644 --- a/tests/test_strix_backend_unavailable_after_exempted_finding.py +++ b/tests/test_strix_backend_unavailable_after_exempted_finding.py @@ -240,26 +240,6 @@ def test_bare_backend_outage_with_no_finding_is_non_passing( self.assertEqual(_run_gate_tail(GITHUB_MODELS_BROWNOUT), 1) - def test_exempted_finding_then_outage_recovers_on_second_attempt(self) -> None: - """An exempt finding before continuation must not block outage retry.""" - - gate = r"""#!/usr/bin/env bash -calls=$(( $(cat __COUNTER__) + 1 )) -echo "$calls" > __COUNTER__ -if [ "$calls" -le 1 ]; then - printf '%s\n' \ - "Strix findings are limited to unchanged files in this pull request; allowing pipeline continuation." \ - "LLM CONNECTION FAILED" \ - "Configured model and fallback models were unavailable." - exit 1 -fi -echo "scan complete" -exit 0 -""" - returncode, calls = _run_gate_retry(gate) - self.assertEqual(returncode, 0) - self.assertEqual(calls, 2) - def test_real_finding_after_continuation_never_retries(self) -> None: """A tail-scoped real finding is authoritative: zero retries, fail closed.""" @@ -276,12 +256,10 @@ def test_real_finding_after_continuation_never_retries(self) -> None: self.assertEqual(returncode, 1) self.assertEqual(calls, 1) - def test_retry_contract_preserves_logs_without_wall_clock_budget(self) -> None: - """Retries retain every attempt without imposing an inference deadline.""" + def test_workflow_imposes_no_wall_clock_inference_deadline(self) -> None: + """The single governed request is not cut short by a repository-authored deadline.""" workflow = STRIX_WORKFLOW.read_text(encoding="utf-8") - self.assertIn('strix_attempt_log="$RUNNER_TEMP/strix_gate_console_attempt_', workflow) - self.assertIn('cat "$strix_attempt_log" >> "$strix_run_log"', workflow) self.assertNotIn("strix_gate_attempt_budget_seconds", workflow) self.assertNotIn("STRIX_PROCESS_TIMEOUT_SECONDS:", workflow) self.assertNotIn("STRIX_TOTAL_TIMEOUT_SECONDS:", workflow) diff --git a/tests/test_strix_caido_bootstrap_timing_retry.py b/tests/test_strix_caido_bootstrap_timing_retry.py index a60b9d801b..32b6bacd97 100644 --- a/tests/test_strix_caido_bootstrap_timing_retry.py +++ b/tests/test_strix_caido_bootstrap_timing_retry.py @@ -5,9 +5,9 @@ 10-attempt loginAsGuest retry budget. A slow CI runner can exceed that budget before the local proxy is reachable, even though the penetration test itself never started and no security evidence was produced or lost. -This is local sandbox/container boot timing, not tied to any one LLM model, -so it must be retried same-model rather than treated as grounds to switch -models or as a genuine, non-backend scan failure. +This is local sandbox/container boot timing, not a genuine, non-backend scan +failure: it must classify as an infrastructure error so the gate fails +closed without repository-authored retry or fallback allocation. """ from __future__ import annotations @@ -74,7 +74,7 @@ def _classifies_as_caido_bootstrap_timing(log_text: str) -> bool: class StrixCaidoBootstrapTimingRetryTests(unittest.TestCase): - """Protect same-model retry for the upstream sandbox boot race.""" + """Protect infrastructure classification for the upstream sandbox boot race.""" def test_observed_caido_login_failure_is_retryable(self) -> None: """Recognize the exact loginAsGuest failure observed in required CI.""" @@ -103,14 +103,13 @@ def test_login_failure_without_the_connect_evidence_is_not_retryable(self) -> No log = "loginAsGuest failed after 10 attempts: unknown reason\n" self.assertFalse(_classifies_as_caido_bootstrap_timing(log)) - def test_wired_into_same_model_retry_and_infrastructure_not_cross_model( - self, - ) -> None: - """Retry the same model; do not treat this as a reason to switch models. + def test_wired_into_infrastructure_detection(self) -> None: + """The classifier feeds the fail-closed infrastructure-error signal. - Switching LLM models cannot change how long the local sandbox - container takes to boot, so cross-model fallback (`is_model_retryable_error`) - must stay untouched by this classifier. + No repository-authored retry or cross-model fallback exists anymore: + contextual-orchestrator's `orchestrator/free` gateway owns provider + discovery and failover, so this classifier only needs to be wired + into infrastructure-error detection. """ gate_source = STRIX_GATE.read_text(encoding="utf-8") @@ -118,24 +117,8 @@ def test_wired_into_same_model_retry_and_infrastructure_not_cross_model( gate_source, "has_detected_infrastructure_error", ) - same_model_retry = _function_block( - gate_source, - "is_transient_same_model_retry_error", - ) - cross_model_retry = _function_block(gate_source, "is_model_retryable_error") self.assertIn("is_caido_bootstrap_timing_error", infrastructure) - self.assertIn("is_caido_bootstrap_timing_error", same_model_retry) - self.assertNotIn("is_caido_bootstrap_timing_error", cross_model_retry) - - def test_retry_reason_is_logged_for_operators(self) -> None: - """Keep the diagnostic retry-reason message in sync with the classifier.""" - - gate_source = STRIX_GATE.read_text(encoding="utf-8") - self.assertIn( - 'retry_reason="Caido sandbox bootstrap timing"', - gate_source, - ) if __name__ == "__main__": diff --git a/tests/test_strix_model_behavior_error.py b/tests/test_strix_model_behavior_error.py index 3d0fd0bc42..8956d152a6 100644 --- a/tests/test_strix_model_behavior_error.py +++ b/tests/test_strix_model_behavior_error.py @@ -152,23 +152,21 @@ def test_agents_sdk_tool_protocol_failure_is_retryable(self) -> None: ) self.assertTrue(_classifies_as_model_behavior_error(log)) - def test_behavior_error_skips_same_model_and_enters_fallback(self) -> None: - """Wire the classifier into infrastructure and cross-model fallback.""" + def test_behavior_error_wired_into_infrastructure_detection(self) -> None: + """The classifier feeds the fail-closed infrastructure-error signal. + + No repository-authored retry or cross-model fallback exists anymore: + contextual-orchestrator's `orchestrator/free` gateway owns provider + discovery and failover. + """ gate_source = STRIX_GATE.read_text(encoding="utf-8") infrastructure = _function_block( gate_source, "has_detected_infrastructure_error", ) - retryable = _function_block(gate_source, "is_model_retryable_error") - same_model_retry = _function_block( - gate_source, - "is_transient_same_model_retry_error", - ) self.assertIn("is_model_behavior_error", infrastructure) - self.assertIn("is_model_behavior_error", retryable) - self.assertNotIn("is_model_behavior_error", same_model_retry) def test_outer_workflow_classifies_zero_finding_protocol_flake(self) -> None: """Empty scans that hit ModelBehaviorError receive typed diagnostics.""" diff --git a/tests/test_strix_nvidia_nim_not_found_fallback.py b/tests/test_strix_nvidia_nim_not_found_fallback.py index ba8344455b..4444e12f42 100644 --- a/tests/test_strix_nvidia_nim_not_found_fallback.py +++ b/tests/test_strix_nvidia_nim_not_found_fallback.py @@ -1,9 +1,10 @@ -"""Regression contract for NVIDIA NIM model retirement and hosted 404 fallback. +"""Regression contract for NVIDIA NIM model retirement and hosted 404 classification. -The central Strix workflow must not turn a provider-side model-catalog 404 into a -security finding or retry the same unavailable model. It must move to another -approved free NVIDIA NIM candidate before using the existing GitHub Models -fallbacks, while ordinary application 404 output remains non-retryable. +The central Strix workflow must not turn a provider-side model-catalog 404 +into a security finding: it must classify as an infrastructure error and +fail closed, delegating provider discovery and failover to +contextual-orchestrator's `orchestrator/free` gateway, while ordinary +application 404 output remains non-retryable. """ from __future__ import annotations @@ -167,23 +168,21 @@ def test_provider_literal_without_litellm_error_is_not_retryable(self) -> None: log = "source literal: Nvidia_nimException Error code: 404\n" self.assertFalse(_classifies_as_nvidia_not_found(log)) - def test_not_found_skips_same_model_and_enters_cross_model_fallback(self) -> None: - """Wire the classifier only into infrastructure and model fallback.""" + def test_not_found_wired_into_infrastructure_detection(self) -> None: + """The classifier feeds the fail-closed infrastructure-error signal. + + No repository-authored retry or cross-model fallback exists anymore: + contextual-orchestrator's `orchestrator/free` gateway owns provider + discovery and failover. + """ gate_source = STRIX_GATE.read_text(encoding="utf-8") infrastructure = _function_block( gate_source, "has_detected_infrastructure_error", ) - retryable = _function_block(gate_source, "is_model_retryable_error") - same_model_retry = _function_block( - gate_source, - "is_transient_same_model_retry_error", - ) self.assertIn("is_nvidia_nim_not_found_error", infrastructure) - self.assertIn("is_nvidia_nim_not_found_error", retryable) - self.assertNotIn("is_nvidia_nim_not_found_error", same_model_retry) def test_workflow_routes_all_scans_through_contextual_orchestrator(self) -> None: """The central workflow owns one gateway route and no provider override.""" From 17c5fb876985c9595f35a789edb3e0bfcbf42ba0 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 20:15:03 +0900 Subject: [PATCH 13/14] test(strix): rewrite the quick-gate self-test for single-governed-request routing The bash self-test still described the pre-migration architecture and reported 949 failures on this branch's head. Drive it to zero without letting the rewrite bless behaviour the migration did not intend to remove. Retargeted, not deleted, wherever the subject survives: - 119 invocations moved to the gateway contract (orchestrator/free, the pinned loopback base, and the single-attempt model/api-base sequences). - 16 provider-routing scenarios now assert the policy boundary itself: a direct provider route exits 2 with "direct provider/model routes are forbidden". These cover the new boundary, which nothing else tested. - preserve-existing-api-base and the untrusted-api-base case had their expected outcome inverted, because the gate got stricter: it now pins the loopback rather than keeping a caller-supplied base, and refuses an api-base file outside the trusted root rather than ignoring it. - Six new fail-closed scenarios replace the deleted fallback cluster's signal coverage (rate limit, resource-exhausted, quota, token limit, timeout), plus a negative control so the provider-unavailable and generic-failure branches cannot silently collapse into one. Deleted only where the mechanism is gone: 86 multi-attempt scenarios, 11 severity-threshold and fallback-config scenarios, the GitHub Models HTTP 410 cluster (unreachable behind is_github_models_api_compatible_model), and five assert_model_requires_vertex_auth calls whose helper normalises internally and can no longer return 0. Also restores three pull-request-scope calls in run_current_target_scan. 15027c19 rewrote that function by hand right after a merge from main introduced evaluate_pull_request_findings, and dropped all of its call sites along with fail_unmapped_threshold_report and should_fail_pull_request_infra_zero_findings. None of those belong to the "repository-authored retry, model fallback, or severity-threshold admission" this change set out to remove; without them the gate fails a pull request on findings confined to files it never modified, and a zero-findings result produced while the provider was failing reads as a clean scan. load_pull_request_head_sha is left alone: it is orphaned on main too. Verified against an unmodified clone of this same branch head: the self-test goes 949 -> 0, and the rest of the suite is unchanged at 22 failed / 2815 passed, coverage 99%, interrogate 98.3%. Those pre-existing failures are branch staleness (34 commits behind main, missing #1883's coverage and docstring repair) and are not touched here. Co-Authored-By: Claude Opus 5 --- scripts/ci/strix_quick_gate.sh | 32 +- scripts/ci/test_strix_quick_gate.sh | 4660 ++++++--------------------- 2 files changed, 1058 insertions(+), 3634 deletions(-) diff --git a/scripts/ci/strix_quick_gate.sh b/scripts/ci/strix_quick_gate.sh index f2d232437d..6fdfc4ad4f 100755 --- a/scripts/ci/strix_quick_gate.sh +++ b/scripts/ci/strix_quick_gate.sh @@ -4217,12 +4217,38 @@ run_current_target_scan() { local primary_scan_rc=0 run_strix_once "$PRIMARY_MODEL" || primary_scan_rc=$? - if [ "$primary_scan_rc" -eq 0 ]; then - return 0 - fi if [ "$primary_scan_rc" -eq 2 ]; then return 2 fi + + # Pull-request scope is orthogonal to the retry/fallback/severity admission + # this change removed. It answers a different question: do the findings this + # one governed request produced intersect the pull request at all? Skipping it + # fails the pull request on findings confined to files it never modified. + if evaluate_pull_request_findings; then + return 0 + fi + + case "$PR_FINDINGS_DECISION" in + block_changed | block_unmapped | block_manifest_finding) + echo "Strix quick scan failed with a non-recoverable error." >&2 + return 1 + ;; + esac + + if fail_unmapped_threshold_report; then + return 1 + fi + + # A zero-findings result produced while the provider was failing is not + # clean scan evidence, so it must not pass the pull request. + if should_fail_pull_request_infra_zero_findings; then + return 1 + fi + + if [ "$primary_scan_rc" -eq 0 ]; then + return 0 + fi if [ "$INFRA_ERROR_DETECTED" -eq 1 ]; then echo "STRIX_PROVIDER_UNAVAILABLE: contextual-orchestrator/orchestrator/free did not produce authoritative scan evidence; failing closed without repository-authored retry or fallback allocation." >&2 else diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index e6b0ab11a0..8d4f84fedb 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -201,7 +201,8 @@ assert_strix_workflow_pr_trigger_hardened() { assert_file_contains "$workflow_file" "pull_request_target:" "strix workflow uses trusted PR trigger" assert_file_contains "$workflow_file" "admit-current-head:" "strix workflow admits the live pull request head before provider concurrency" assert_file_contains "$workflow_file" "needs: [changed-scope, admit-current-head]" "strix provider queue waits for live-head admission" - assert_file_contains "$workflow_file" 'strix-security-scan-${{ needs.admit-current-head.outputs.target_repository }}-${{' "strix workflow defines one admitted repository and PR concurrency group" + assert_file_contains "$workflow_file" 'strix-security-scan-${{' "strix workflow coalesces by repository and PR before job admission" + assert_file_not_contains "$workflow_file" 'strix-security-scan-${{ needs.admit-current-head.outputs.target_repository }}-${{' "strix concurrency is not delayed until job admission" assert_file_contains "$workflow_file" "cancel-superseded-pr-runs:" "strix workflow runs superseded-head cleanup outside the provider scan queue" assert_file_not_contains "$workflow_file" "format('closed-pr-{0}-{1}'" "strix cleanup does not need a second concurrency queue" assert_file_contains "$workflow_file" 'echo "pr_number=${GITHUB_RUN_ID}"' "strix workflow preserves independent push and schedule evidence" @@ -3352,6 +3353,30 @@ printf '%s\n' "$target_path" >> "${FAKE_STRIX_TARGET_LOG:?}" STRIX_REPORTS_DIR="${STRIX_REPORTS_DIR:-strix_runs}" case "${FAKE_STRIX_SCENARIO:?}" in + provider-ratelimit-fails-closed) + echo "Penetration test failed: LLM request failed: litellm.RateLimitError" + exit 1 + ;; + provider-resource-exhausted-fails-closed) + echo '{"error":{"status":"RESOURCE_EXHAUSTED"}}' + exit 1 + ;; + provider-quota-fails-closed) + echo "openai.agents: Error streaming response: You exceeded your current quota, please check your plan and billing details." + exit 1 + ;; + provider-token-limit-fails-closed) + echo "litellm.APIStatusError: Error code: 413 - Request body too large" + exit 1 + ;; + provider-timeout-fails-closed) + echo "litellm.exceptions.Timeout: provider request timed out" + exit 1 + ;; + provider-plain-failure-fails-closed-without-infra-marker) + echo "penetration test aborted by an unclassified condition" + exit 1 + ;; success|runtime-env-forwarding|custom-openai-compatible-preserves-effort|vertex-primary-success-timing-message|direct-openai-gpt-does-not-require-github-models-api-base|pr-executable-integrity-mismatch|pr-executable-group-writable) echo "scan ok" exit 0 @@ -3391,109 +3416,12 @@ REPORT echo "Vulnerabilities 1" exit 0 ;; - slow-timeout) - sleep "${FAKE_STRIX_TIMEOUT_SLEEP_SECONDS:?}" - exit 0 - ;; timeout-disabled-success) sleep 1 echo "scan ok with timeout disabled" exit 0 ;; vertex-primary-notfound-fallback-success|github-models-fallback-success|github-models-fallback-success-deepseek-v3|github-models-token-limit-fallback-success|github-models-fallback-requires-api-base|github-models-model-prefix-with-api-base-succeeds|github-models-meta-prefix-with-api-base-succeeds|github-models-mistral-prefix-with-api-base-succeeds) - case "${STRIX_LLM:-}" in - vertex_ai/missing-primary) - echo "Error: litellm.NotFoundError: Vertex_aiException - x" - echo '"status": "NOT_FOUND"' - exit 1 - ;; - vertex_ai/fallback-one) - echo "scan ok with fallback" - exit 0 - ;; - openai/gpt-5|openai/openai/gpt-5.4|openai/meta/test-github-model|openai/mistral-ai/test-github-model) - if [ "${FAKE_STRIX_SCENARIO:?}" = "github-models-token-limit-fallback-success" ]; then - echo "openai.APIStatusError: Error code: 413 - {'error': {'code': 'tokens_limit_reached', 'message': 'Request body too large for gpt-5 model. Max size: 4000 tokens.'}}" - exit 1 - fi - echo "scan ok with GitHub Models fallback" - exit 0 - ;; - openai/deepseek/deepseek-r1-0528) - if [ "${FAKE_STRIX_SCENARIO:?}" = "github-models-fallback-success-deepseek-v3" ]; then - echo "LLM CONNECTION FAILED" - echo "Could not establish connection to the language model." - echo "Error: litellm.BadRequestError: OpenAIException - Unavailable model: deepseek-r1-0528" - exit 1 - fi - echo "scan ok with GitHub Models fallback" - exit 0 - ;; - openai/deepseek/deepseek-v3-0324) - echo "scan ok with GitHub Models fallback" - exit 0 - ;; - *) - echo "unexpected model ${STRIX_LLM:-}" >&2 - exit 9 - ;; - esac - ;; - nvidia-rate-limit-openai-direct-fallback-clears-api-base) - case "${STRIX_LLM:-}" in - nvidia_nim/nvidia/rate-limited-primary) - echo "LLM CONNECTION FAILED" - echo "Error: litellm.RateLimitError: Nvidia_nimException - Error code: 429 Too Many Requests" - exit 1 - ;; - openai/gpt-5.4) - if [ "${STRIX_REASONING_EFFORT:-}" != "none" ]; then - echo "direct OpenAI function-tools fallback requires reasoning effort none" >&2 - exit 29 - fi - if [ "${LLM_API_KEY:-}" != "openai-fallback-token" ]; then - echo "unexpected direct-OpenAI fallback key (${LLM_API_KEY:-})" >&2 - exit 26 - fi - if [ -n "${LLM_API_BASE:-}" ]; then - echo "direct OpenAI fallback inherited foreign API base ${LLM_API_BASE}" >&2 - exit 27 - fi - echo "scan ok after direct-OpenAI fallback" - exit 0 - ;; - *) - echo "unexpected cross-provider model ${STRIX_LLM:-}" >&2 - exit 28 - ;; - esac - ;; - openai-direct-quota-github-models-fallback-success) - case "${STRIX_LLM:-}" in - openai/gpt-5.4) - if [ "${LLM_API_KEY:-}" != "dummy" ]; then - echo "unexpected direct-OpenAI key for primary (${LLM_API_KEY:-})" >&2 - exit 15 - fi - echo "Error getting response: Error code: 429 - {'error': {'message': 'You exceeded your current quota, please check your plan and billing details.', 'type': 'insufficient_quota', 'code': 'insufficient_quota'}}" - echo "openai.RateLimitError: Error code: 429" - exit 1 - ;; - openai/o3) - if [ "${LLM_API_KEY:-}" != "github-models-fallback-token" ]; then - echo "unexpected GitHub Models key for fallback (${LLM_API_KEY:-})" >&2 - exit 16 - fi - echo "scan ok with GitHub Models fallback" - exit 0 - ;; - *) - echo "unexpected model ${STRIX_LLM:-}" >&2 - exit 9 - ;; - esac - ;; - vertex-all-notfound) echo "Error: litellm.NotFoundError: Vertex_aiException - x" echo '"status": "NOT_FOUND"' exit 1 @@ -3510,23 +3438,6 @@ REPORT echo "Error: provider prefix not normalized (${STRIX_LLM:-})" >&2 exit 10 ;; - provider-prefix-fallback-normalization) - case "${STRIX_LLM:-}" in - vertex_ai/missing-primary) - echo "Error: litellm.NotFoundError: Vertex_aiException - x" - echo '"status": "NOT_FOUND"' - exit 1 - ;; - vertex_ai/fallback-one) - echo "scan ok after fallback normalization" - exit 0 - ;; - *) - echo "Error: fallback provider prefix not normalized (${STRIX_LLM:-})" >&2 - exit 11 - ;; - esac - ;; provider-prefix-required-resource-path-primary-implicit-default-provider | provider-prefix-required-resource-path-primary-explicit-empty-default-provider) if [ "${STRIX_LLM:-}" = "vertex_ai/gemini-2.5-pro" ]; then echo "scan ok with resource-path normalization" @@ -3535,23 +3446,6 @@ REPORT echo "Error: resource-path model not normalized (${STRIX_LLM:-})" >&2 exit 12 ;; - provider-prefix-resource-path-primary-notfound-fallback-success) - case "${STRIX_LLM:-}" in - vertex_ai/missing-primary) - echo "Error: litellm.NotFoundError: Vertex_aiException - x" - echo '"status": "NOT_FOUND"' - exit 1 - ;; - vertex_ai/fallback-one) - echo "scan ok after resource-path fallback" - exit 0 - ;; - *) - echo "Error: resource-path fallback model not normalized (${STRIX_LLM:-})" >&2 - exit 13 - ;; - esac - ;; vertex-custom-model-resource-path) # projects/

/locations//models/ (no publishers/ segment) if [ "${STRIX_LLM:-}" = "vertex_ai/my-custom-model-123" ]; then @@ -3561,39 +3455,6 @@ REPORT echo "Error: custom model resource-path not normalized (${STRIX_LLM:-})" >&2 exit 40 ;; - vertex-notfound-without-status-fallback-success) - case "${STRIX_LLM:-}" in - vertex_ai/missing-primary) - echo "Error: litellm.NotFoundError: Vertex_aiException - x" - exit 1 - ;; - vertex_ai/fallback-one) - echo "scan ok after status-less not found fallback" - exit 0 - ;; - *) - echo "Error: status-less fallback model not normalized (${STRIX_LLM:-})" >&2 - exit 14 - ;; - esac - ;; - vertex-notfound-compact-status-fallback-success) - case "${STRIX_LLM:-}" in - vertex_ai/missing-primary) - echo 'litellm.exceptions.NotFoundError: VertexAI error' - echo '{"error":{"status":"NOT_FOUND"}}' - exit 1 - ;; - vertex_ai/fallback-one) - echo "scan ok after compact-status not found fallback" - exit 0 - ;; - *) - echo "Error: compact-status fallback model not normalized (${STRIX_LLM:-})" >&2 - exit 17 - ;; - esac - ;; nonvertex-slash-model-passthrough) if [ "${STRIX_LLM:-}" = "foo/bar" ]; then echo "scan ok with non-vertex slash model passthrough" @@ -3602,310 +3463,52 @@ REPORT echo "Error: non-vertex slash model was rewritten (${STRIX_LLM:-})" >&2 exit 18 ;; - primary-duplicate-in-fallback) - case "${STRIX_LLM:-}" in - vertex_ai/missing-primary) - echo "Error: litellm.NotFoundError: Vertex_aiException - x" - echo '"status": "NOT_FOUND"' - exit 1 - ;; - vertex_ai/fallback-one) - echo "scan ok after duplicate-primary skip" - exit 0 - ;; - *) - echo "Error: duplicate-primary path unexpected (${STRIX_LLM:-})" >&2 - exit 15 - ;; - esac - ;; - multiline-fallback-success) - case "${STRIX_LLM:-}" in - vertex_ai/missing-primary) - echo "Error: litellm.NotFoundError: Vertex_aiException - x" - echo '"status": "NOT_FOUND"' - exit 1 - ;; - vertex_ai/fallback-one) - echo "Error: litellm.NotFoundError: Vertex_aiException - x" - echo '"status": "NOT_FOUND"' - exit 1 - ;; - vertex_ai/fallback-two) - echo "scan ok after multiline fallback parsing" - exit 0 - ;; - *) - echo "Error: multiline fallback path unexpected (${STRIX_LLM:-})" >&2 - exit 19 - ;; - esac - ;; - vertex-primary-ratelimit-fallback-success) - case "${STRIX_LLM:-}" in - vertex_ai/ratelimit-primary) - echo "Penetration test failed: LLM request failed: RateLimitError" - exit 1 - ;; - vertex_ai/fallback-one) - echo "scan ok after rate-limit fallback" - exit 0 - ;; - *) - echo "Error: ratelimit fallback path unexpected (${STRIX_LLM:-})" >&2 - exit 21 - ;; - esac - ;; - vertex-primary-resource-exhausted-fallback-success) - case "${STRIX_LLM:-}" in - vertex_ai/resource-exhausted-primary) - echo '{"error":{"status":"RESOURCE_EXHAUSTED"}}' - exit 1 - ;; - vertex_ai/fallback-one) - echo "scan ok after resource exhausted fallback" - exit 0 - ;; - *) - echo "Error: resource exhausted fallback path unexpected (${STRIX_LLM:-})" >&2 - exit 23 - ;; - esac - ;; - openai-primary-quota-fallback-success) - case "${STRIX_LLM:-}" in - openai/quota-primary) - echo "openai.agents: Error streaming response: You exceeded your current quota, please check your plan and billing details." - exit 1 - ;; - openai/fallback-one) - echo "scan ok after quota fallback" - exit 0 - ;; - *) - echo "Error: quota fallback path unexpected (${STRIX_LLM:-})" >&2 - exit 24 - ;; - esac - ;; - vertex-primary-429-fallback-success) - case "${STRIX_LLM:-}" in - vertex_ai/http429-primary) - echo "litellm: HTTP 429 Too Many Requests" - exit 1 - ;; - vertex_ai/fallback-one) - echo "scan ok after 429 fallback" - exit 0 - ;; - *) - echo "Error: 429 fallback path unexpected (${STRIX_LLM:-})" >&2 - exit 24 - ;; - esac - ;; - vertex-primary-midstream-fallback-success) - case "${STRIX_LLM:-}" in - vertex_ai/midstream-primary) - echo "Penetration test failed: LLM request failed: MidStreamFallbackError" - exit 1 - ;; - vertex_ai/fallback-one) - echo "scan ok after midstream fallback" - exit 0 - ;; - *) - echo "Error: midstream fallback path unexpected (${STRIX_LLM:-})" >&2 - exit 25 - ;; - esac - ;; - vertex-primary-midstream-retry-same-model-success) - case "${STRIX_LLM:-}" in - vertex_ai/retry-midstream-primary) - attempt="0" - if [ -f "${FAKE_STRIX_STATE_FILE:?}" ]; then - attempt="$(cat "${FAKE_STRIX_STATE_FILE:?}")" - fi - attempt="$((attempt + 1))" - echo "$attempt" > "${FAKE_STRIX_STATE_FILE:?}" - if [ "$attempt" -eq 1 ]; then - echo "Penetration test failed: LLM request failed: MidStreamFallbackError" - exit 1 - fi - echo "scan ok after same-model retry" - exit 0 - ;; - vertex_ai/fallback-one) - echo "Error: fallback should not be needed for same-model retry scenario" >&2 - exit 30 - ;; - *) - echo "Error: midstream fallback path unexpected (${STRIX_LLM:-})" >&2 - exit 30 - ;; - esac - ;; - vertex-primary-ratelimit-retry-same-model-success|vertex-primary-ratelimit-retry-reason-message) - case "${STRIX_LLM:-}" in - vertex_ai/retry-ratelimit-primary) - attempt="0" - if [ -f "${FAKE_STRIX_STATE_FILE:?}" ]; then - attempt="$(cat "${FAKE_STRIX_STATE_FILE:?}")" - fi - attempt="$((attempt + 1))" - echo "$attempt" > "${FAKE_STRIX_STATE_FILE:?}" - if [ "$attempt" -eq 1 ]; then - echo "Penetration test failed: LLM request failed: RateLimitError" - exit 1 - fi - echo "scan ok after same-model rate-limit retry" - exit 0 - ;; - vertex_ai/fallback-one) - echo "Error: fallback should not be needed for same-model rate-limit retry scenario" >&2 - exit 31 - ;; - *) - echo "Error: rate-limit fallback path unexpected (${STRIX_LLM:-})" >&2 - exit 31 - ;; - esac - ;; vertex-primary-api-connection-retry-same-model-success|github-models-internal-server-connection-retry-same-model-success|internal-server-error-unrelated-output-nonretryable|internal-server-error-many-blocks-retry-same-model-success) - case "${STRIX_LLM:-}" in - gemini/retry-api-connection-primary|vertex_ai/retry-api-connection-primary|openai/openai/retry-api-connection-primary) - attempt="0" - if [ -f "${FAKE_STRIX_STATE_FILE:?}" ]; then - attempt="$(cat "${FAKE_STRIX_STATE_FILE:?}")" - fi - attempt="$((attempt + 1))" - echo "$attempt" > "${FAKE_STRIX_STATE_FILE:?}" - if [ "$attempt" -eq 1 ]; then - if [ "${STRIX_LLM:-}" = "openai/openai/retry-api-connection-primary" ]; then - if [ "${FAKE_STRIX_SCENARIO:?}" = "internal-server-error-unrelated-output-nonretryable" ]; then + attempt="0" + if [ -f "${FAKE_STRIX_STATE_FILE:?}" ]; then + attempt="$(cat "${FAKE_STRIX_STATE_FILE:?}")" + fi + attempt="$((attempt + 1))" + echo "$attempt" > "${FAKE_STRIX_STATE_FILE:?}" + if [ "$attempt" -eq 1 ]; then + if [ "${STRIX_LLM:-}" = "openai/openai/retry-api-connection-primary" ]; then + if [ "${FAKE_STRIX_SCENARIO:?}" = "internal-server-error-unrelated-output-nonretryable" ]; then + echo "Error: litellm.InternalServerError: upstream request failed" + for filler in 1 2 3 4 5 6; do + echo "target application diagnostic $filler" + done + echo "Internal Server Error" + exit 1 + fi + if [ "${FAKE_STRIX_SCENARIO:?}" = "internal-server-error-many-blocks-retry-same-model-success" ]; then + # Regression for the SIGPIPE race (Devin finding on + # PR #1394): emit enough matching + # litellm.InternalServerError blocks that the bounded + # awk scan's piped output exceeds a single pipe + # buffer, so a `grep -q` that stops reading at the + # first match cannot SIGPIPE the still-writing awk + # producer into a false non-match under + # `set -o pipefail`. + for _ in $(seq 1 2000); do + echo "line filler some unrelated target application output padding padding padding" echo "Error: litellm.InternalServerError: upstream request failed" - for filler in 1 2 3 4 5 6; do - echo "target application diagnostic $filler" - done echo "Internal Server Error" - exit 1 - fi - if [ "${FAKE_STRIX_SCENARIO:?}" = "internal-server-error-many-blocks-retry-same-model-success" ]; then - # Regression for the SIGPIPE race (Devin finding on - # PR #1394): emit enough matching - # litellm.InternalServerError blocks that the bounded - # awk scan's piped output exceeds a single pipe - # buffer, so a `grep -q` that stops reading at the - # first match cannot SIGPIPE the still-writing awk - # producer into a false non-match under - # `set -o pipefail`. - for _ in $(seq 1 2000); do - echo "line filler some unrelated target application output padding padding padding" - echo "Error: litellm.InternalServerError: upstream request failed" - echo "Internal Server Error" - echo "more filler after context one" - echo "more filler after context two" - done - exit 1 - fi - echo "LLM CONNECTION FAILED" - echo "Could not establish connection to the language model." - echo "Error: litellm.InternalServerError: upstream request failed" - else - echo "LLM CONNECTION FAILED" - echo "litellm.APIConnectionError: GeminiException - Server disconnected without sending a response." + echo "more filler after context one" + echo "more filler after context two" + done + exit 1 fi - exit 1 - fi - echo "scan ok after same-model api connection retry" - exit 0 - ;; - vertex_ai/fallback-one) - echo "Error: fallback should not be needed for API connection retry scenario" >&2 - exit 36 - ;; - *) - echo "Error: API connection retry path unexpected (${STRIX_LLM:-})" >&2 - exit 36 - ;; - esac - ;; - openrouter-502-fallback-retry-same-model-success) - case "${STRIX_LLM:-}" in - vertex_ai/missing-primary) - echo "Error: litellm.NotFoundError: Vertex_aiException - x" - echo '"status": "NOT_FOUND"' - exit 1 - ;; - openrouter/free) - attempt="0" - if [ -f "${FAKE_STRIX_STATE_FILE:?}" ]; then - attempt="$(cat "${FAKE_STRIX_STATE_FILE:?}")" - fi - attempt="$((attempt + 1))" - echo "$attempt" > "${FAKE_STRIX_STATE_FILE:?}" - if [ "$attempt" -eq 1 ]; then - echo "Error: litellm.APIError: APIError:" - echo "OpenrouterException -" - echo '{"error":{"message":"Invalid URL:' - echo '","code":502,"metadata":{"provider_name":"Stealth"}}}' - exit 1 - fi - echo "scan ok after OpenRouter 502 same-model retry" - exit 0 - ;; - vertex_ai/fallback-two) - echo "Error: second fallback should not be needed after transient OpenRouter 502" >&2 - exit 38 - ;; - *) - echo "Error: OpenRouter 502 fallback path unexpected (${STRIX_LLM:-})" >&2 - exit 38 - ;; - esac - ;; - openrouter-502-distant-target-output-nonretryable) - case "${STRIX_LLM:-}" in - vertex_ai/missing-primary) - echo "Error: litellm.NotFoundError: Vertex_aiException - x" - echo '"status": "NOT_FOUND"' - exit 1 - ;; - openrouter/free) - echo "Error: litellm.APIError: APIError: OpenrouterException -" - printf 'target output\n%.0s' 1 2 3 4 5 6 - echo '{"code":502,"metadata":{"provider_name":"spoof"}}' - exit 1 - ;; - vertex_ai/fallback-two) - echo "scan ok after distant target output" - exit 0 - ;; - esac - ;; - github-models-primary-unavailable-fallback-success|github-models-primary-denied-fallback-success) - case "${STRIX_LLM:-}" in - openai/gpt-5) - echo "LLM CONNECTION FAILED" - echo "Could not establish connection to the language model." - if [ "${FAKE_STRIX_SCENARIO:?}" = "github-models-primary-denied-fallback-success" ]; then - echo "openai.PermissionDeniedError: Error code: 403" + echo "LLM CONNECTION FAILED" + echo "Could not establish connection to the language model." + echo "Error: litellm.InternalServerError: upstream request failed" else - echo "Error: litellm.BadRequestError: OpenAIException - Unavailable model: gpt-5" + echo "LLM CONNECTION FAILED" + echo "litellm.APIConnectionError: GeminiException - Server disconnected without sending a response." fi exit 1 - ;; - openai/deepseek/deepseek-r1-0528) - echo "scan ok after GitHub Models unavailable fallback" - exit 0 - ;; - *) - echo "Error: GitHub Models unavailable fallback path unexpected (${STRIX_LLM:-})" >&2 - exit 37 - ;; - esac + fi + echo "scan ok after same-model api connection retry" + exit 0 ;; github-models-http410-authenticated-fallback-success | github-models-http410-missing-http-token | github-models-http410-missing-provider-error | github-models-http410-numeric-continuation-4100 | github-models-http410-numeric-continuation-4104 | github-models-http410-target-output-spoof | github-models-retirement-brownout-phrase-only) case "${STRIX_LLM:-}" in @@ -3945,30 +3548,12 @@ REPORT ;; esac ;; - github-models-primary-ratelimit-fallback-success) + github-models-fallback-provider-signal-tries-next | github-models-fallback-baseline-vulnerability-before-next-success-continues | github-models-exhausted-after-baseline-vulnerability-fails-closed | github-models-fallback-changed-vulnerability-before-next-success-blocks | github-models-fallback-dockerfile-test-baseline-before-next-success-continues) case "${STRIX_LLM:-}" in openai/gpt-5) echo "LLM CONNECTION FAILED" echo "Could not establish connection to the language model." - echo "Error: litellm.RateLimitError: RateLimitError: OpenAIException - Too many requests. For more on scraping GitHub and how it may affect your rights, please review our Terms of Service." - exit 1 - ;; - openai/deepseek/deepseek-r1-0528) - echo "scan ok after GitHub Models rate-limit fallback" - exit 0 - ;; - *) - echo "Error: GitHub Models rate-limit fallback path unexpected (${STRIX_LLM:-})" >&2 - exit 38 - ;; - esac - ;; - github-models-fallback-provider-signal-tries-next | github-models-fallback-baseline-vulnerability-before-next-success-continues | github-models-exhausted-after-baseline-vulnerability-fails-closed | github-models-fallback-changed-vulnerability-before-next-success-blocks | github-models-fallback-dockerfile-test-baseline-before-next-success-continues) - case "${STRIX_LLM:-}" in - openai/gpt-5) - echo "LLM CONNECTION FAILED" - echo "Could not establish connection to the language model." - echo "Error: litellm.RateLimitError: RateLimitError: OpenAIException - Too many requests." + echo "Error: litellm.RateLimitError: RateLimitError: OpenAIException - Too many requests." exit 1 ;; openai/deepseek/deepseek-r1-0528) @@ -4017,95 +3602,6 @@ EOS ;; esac ;; - gemini-high-demand-retry-same-model-success) - case "${STRIX_LLM:-}" in - gemini/retry-high-demand-primary) - attempt="0" - if [ -f "${FAKE_STRIX_STATE_FILE:?}" ]; then - attempt="$(cat "${FAKE_STRIX_STATE_FILE:?}")" - fi - attempt="$((attempt + 1))" - echo "$attempt" > "${FAKE_STRIX_STATE_FILE:?}" - if [ "$attempt" -eq 1 ]; then - echo "LLM CONNECTION FAILED" - echo 'litellm.ServiceUnavailableError: GeminiException - {"error":{"code":503,"message":"This model is currently experiencing high demand. Spikes in demand are usually temporary. Please try again later.","status":"UNAVAILABLE"}}' - exit 1 - fi - echo "scan ok after same-model high-demand retry" - exit 0 - ;; - *) - echo "Error: high-demand retry path unexpected (${STRIX_LLM:-})" >&2 - exit 37 - ;; - esac - ;; - nvidia-overloaded-direct-fallback-success) - case "${STRIX_LLM:-}" in - nvidia_nim/nvidia/overloaded-primary) - echo "LLM CONNECTION FAILED" - echo "Could not establish connection to the language model." - echo "Error: litellm.ServiceUnavailableError: Nvidia_nimException - Service temporarily overloaded" - exit 1 - ;; - nvidia_nim/nvidia/fallback-one) - echo "scan ok after NVIDIA overload fallback" - exit 0 - ;; - *) - echo "Error: NVIDIA overload fallback path unexpected (${STRIX_LLM:-})" >&2 - exit 37 - ;; - esac - ;; - gemini-timeout-direct-fallback-success) - case "${STRIX_LLM:-}" in - gemini/retry-timeout-primary) - echo "LLM CONNECTION FAILED" - echo "Error: litellm.Timeout: Connection timed out after None seconds." - exit 1 - ;; - gemini/fallback-one) - echo "scan ok after timeout fallback" - exit 0 - ;; - *) - echo "Error: gemini timeout fallback path unexpected (${STRIX_LLM:-})" >&2 - exit 38 - ;; - esac - ;; - gemini-timeout-fallback-success|gemini-generic-fallback-success) - case "${STRIX_LLM:-}" in - gemini/timeout-fallback-primary) - echo "LLM CONNECTION FAILED" - echo "Error: litellm.Timeout: Connection timed out after None seconds." - exit 1 - ;; - gemini/fallback-one) - echo "scan ok after gemini fallback" - exit 0 - ;; - *) - echo "Error: gemini timeout fallback path unexpected (${STRIX_LLM:-})" >&2 - exit 39 - ;; - esac - ;; - gemini-zero-findings-timeout-fallback-allows-pr) - case "${STRIX_LLM:-}" in - gemini/zero-timeout-primary|gemini/fallback-one) - echo "Vulnerabilities 0" - echo "LLM CONNECTION FAILED" - echo "Error: litellm.Timeout: Connection timed out after None seconds." - exit 1 - ;; - *) - echo "Error: gemini zero-finding fallback path unexpected (${STRIX_LLM:-})" >&2 - exit 40 - ;; - esac - ;; pr-scope-zero-finding-does-not-leak) if [ -f "$target_path/sync-module-system/smart-crawling-biz/src/main/java/org/empasy/sync/modules/system/controller/SysPositionController.java" ]; then echo "Vulnerabilities 0" @@ -4131,36 +3627,18 @@ EOS echo "ConnectionError: Server disconnected without sending a response." exit 1 ;; - vertex-all-ratelimited) - echo "Penetration test failed: LLM request failed: RateLimitError" - exit 1 - ;; vertex-primary-hallucinated-endpoint-fallback-success|target-path-src-default-source-dirs) - case "${STRIX_LLM:-}" in - vertex_ai/hallucination-primary) - mkdir -p "$STRIX_REPORTS_DIR/fake-hallucinated/vulnerabilities" - cat >"$STRIX_REPORTS_DIR/fake-hallucinated/vulnerabilities/vuln-0001.md" <<'EOS' + mkdir -p "$STRIX_REPORTS_DIR/fake-hallucinated/vulnerabilities" + cat >"$STRIX_REPORTS_DIR/fake-hallucinated/vulnerabilities/vuln-0001.md" <<'EOS' **Severity:** CRITICAL **Endpoint:** /api/ghost-admin EOS - echo "Penetration test failed: CRITICAL finding on /api/ghost-admin" - exit 1 - ;; - vertex_ai/fallback-one) - echo "scan ok after hallucinated-endpoint fallback" - exit 0 - ;; - *) - echo "Error: hallucinated-endpoint fallback path unexpected (${STRIX_LLM:-})" >&2 - exit 26 - ;; - esac + echo "Penetration test failed: CRITICAL finding on /api/ghost-admin" + exit 1 ;; opencode-documented-env-api-key-fallback-success) - case "${STRIX_LLM:-}" in - vertex_ai/opencode-env-primary) - mkdir -p "$STRIX_REPORTS_DIR/fake-opencode-env/vulnerabilities" - cat >"$STRIX_REPORTS_DIR/fake-opencode-env/vulnerabilities/vuln-0001.md" <"$STRIX_REPORTS_DIR/fake-opencode-env/vulnerabilities/vuln-0001.md" <&2 - exit 27 - ;; - esac + echo "Penetration test failed: documented OpenCode env apiKey reference" + exit 1 ;; generic-github-actions-workflow-fallback-success) - case "${STRIX_LLM:-}" in - vertex_ai/generic-actions-primary) - mkdir -p "$STRIX_REPORTS_DIR/fake-generic-actions/vulnerabilities" - cat >"$STRIX_REPORTS_DIR/fake-generic-actions/vulnerabilities/vuln-0001.md" <<'EOS' + mkdir -p "$STRIX_REPORTS_DIR/fake-generic-actions/vulnerabilities" + cat >"$STRIX_REPORTS_DIR/fake-generic-actions/vulnerabilities/vuln-0001.md" <<'EOS' # Insecure Configurations in GitHub Actions Workflows **Severity:** CRITICAL @@ -4219,68 +3685,32 @@ The GitHub Actions configuration contains several security weaknesses: + Secured version ``` EOS - echo "Penetration test failed: generic GitHub Actions workflow finding" - exit 1 - ;; - vertex_ai/fallback-one) - echo "scan ok after generic GitHub Actions workflow false positive" - exit 0 - ;; - *) - echo "Error: generic GitHub Actions workflow fallback path unexpected (${STRIX_LLM:-})" >&2 - exit 37 - ;; - esac + echo "Penetration test failed: generic GitHub Actions workflow finding" + exit 1 ;; vertex-primary-existing-endpoint-nonrecoverable|multi-source-dirs-existing-endpoint) - case "${STRIX_LLM:-}" in - vertex_ai/existing-endpoint-primary|vertex_ai/multi-dir-primary) - mkdir -p "$STRIX_REPORTS_DIR/fake-existing-endpoint/vulnerabilities" - cat >"$STRIX_REPORTS_DIR/fake-existing-endpoint/vulnerabilities/vuln-0001.md" <<'EOS' + mkdir -p "$STRIX_REPORTS_DIR/fake-existing-endpoint/vulnerabilities" + cat >"$STRIX_REPORTS_DIR/fake-existing-endpoint/vulnerabilities/vuln-0001.md" <<'EOS' **Endpoint:** /api/status EOS - echo "Penetration test failed: CRITICAL finding on /api/status" - exit 1 - ;; - vertex_ai/fallback-one|vertex_ai/fallback-two) - echo "Error: existing endpoint findings must remain non-recoverable (${STRIX_LLM:-})" >&2 - exit 27 - ;; - *) - echo "Error: existing-endpoint scenario unexpected model (${STRIX_LLM:-})" >&2 - exit 28 - ;; - esac + echo "Penetration test failed: CRITICAL finding on /api/status" + exit 1 ;; pr-stale-source-claim-fallback-success) - case "${STRIX_LLM:-}" in - vertex_ai/stale-source-primary) - mkdir -p "$STRIX_REPORTS_DIR/fake-stale-source/vulnerabilities" - cat >"$STRIX_REPORTS_DIR/fake-stale-source/vulnerabilities/vuln-0001.md" <<'EOS' + mkdir -p "$STRIX_REPORTS_DIR/fake-stale-source/vulnerabilities" + cat >"$STRIX_REPORTS_DIR/fake-stale-source/vulnerabilities/vuln-0001.md" <<'EOS' **Severity:** HIGH **Target:** backend/db/models.py The `WorkspaceRunnerConfig.registration_token` field stores the token as plain text. The vulnerable line is `registration_token: Mapped[str | None] = mapped_column(String, nullable=True)`. EOS - echo "Penetration test failed: stale HIGH finding on backend/db/models.py" - exit 1 - ;; - vertex_ai/fallback-one) - echo "scan ok after stale-source fallback" - exit 0 - ;; - *) - echo "Error: stale-source scenario unexpected model (${STRIX_LLM:-})" >&2 - exit 30 - ;; - esac + echo "Penetration test failed: stale HIGH finding on backend/db/models.py" + exit 1 ;; pr-stale-snapshot-snippet-fallback-success) - case "${STRIX_LLM:-}" in - vertex_ai/stale-snapshot-primary) - mkdir -p "$STRIX_REPORTS_DIR/fake-stale-snapshot/vulnerabilities" - cat >"$STRIX_REPORTS_DIR/fake-stale-snapshot/vulnerabilities/vuln-0001.md" <<'EOS' + mkdir -p "$STRIX_REPORTS_DIR/fake-stale-snapshot/vulnerabilities" + cat >"$STRIX_REPORTS_DIR/fake-stale-snapshot/vulnerabilities/vuln-0001.md" <<'EOS' # IDOR in /api/snapshots endpoint allows unauthorized access to database schemas **Severity:** MEDIUM @@ -4312,118 +3742,60 @@ return snapshot + return snapshot ``` EOS - echo "Penetration test failed: stale MEDIUM snapshot snippet" - exit 1 - ;; - vertex_ai/fallback-one) - echo "scan ok after stale snapshot snippet fallback" - exit 0 - ;; - *) - echo "Error: stale-snapshot scenario unexpected model (${STRIX_LLM:-})" >&2 - exit 38 - ;; - esac + echo "Penetration test failed: stale MEDIUM snapshot snippet" + exit 1 ;; pr-stale-source-plus-real-finding-blocks) - case "${STRIX_LLM:-}" in - vertex_ai/stale-source-primary) - mkdir -p "$STRIX_REPORTS_DIR/fake-mixed-findings/vulnerabilities" - cat >"$STRIX_REPORTS_DIR/fake-mixed-findings/vulnerabilities/vuln-0001.md" <<'EOS' + mkdir -p "$STRIX_REPORTS_DIR/fake-mixed-findings/vulnerabilities" + cat >"$STRIX_REPORTS_DIR/fake-mixed-findings/vulnerabilities/vuln-0001.md" <<'EOS' **Severity:** HIGH **Target:** backend/db/models.py The `WorkspaceRunnerConfig.registration_token` field stores the token as plain text. The vulnerable line is `registration_token: Mapped[str | None] = mapped_column(String, nullable=True)`. EOS - cat >"$STRIX_REPORTS_DIR/fake-mixed-findings/vulnerabilities/vuln-0002.md" <<'EOS' + cat >"$STRIX_REPORTS_DIR/fake-mixed-findings/vulnerabilities/vuln-0002.md" <<'EOS' **Severity:** HIGH **Target:** backend/api/emails.py This is a concrete changed-file finding that must remain blocking. EOS - echo "Penetration test failed: mixed stale and real HIGH findings" - exit 1 - ;; - vertex_ai/fallback-one) - echo "Error: mixed real findings must not reach fallback" >&2 - exit 31 - ;; - *) - echo "Error: mixed-findings scenario unexpected model (${STRIX_LLM:-})" >&2 - exit 32 - ;; - esac + echo "Penetration test failed: mixed stale and real HIGH findings" + exit 1 ;; pr-changed-finding-with-retry-marker-blocks) - case "${STRIX_LLM:-}" in - vertex_ai/changed-finding-primary) - mkdir -p "$STRIX_REPORTS_DIR/fake-changed-retry-marker/vulnerabilities" - cat >"$STRIX_REPORTS_DIR/fake-changed-retry-marker/vulnerabilities/vuln-0001.md" <<'EOS' + mkdir -p "$STRIX_REPORTS_DIR/fake-changed-retry-marker/vulnerabilities" + cat >"$STRIX_REPORTS_DIR/fake-changed-retry-marker/vulnerabilities/vuln-0001.md" <<'EOS' **Severity:** HIGH **Target:** backend/api/emails.py This changed-file finding must remain blocking even when the model log also contains retryable provider text. EOS - echo "litellm.exceptions.Timeout: provider timed out after writing a HIGH changed-file finding" - exit 1 - ;; - vertex_ai/fallback-one) - echo "Error: changed-file findings with retry markers must not reach fallback" >&2 - exit 33 - ;; - *) - echo "Error: changed-retry-marker scenario unexpected model (${STRIX_LLM:-})" >&2 - exit 34 - ;; - esac + echo "litellm.exceptions.Timeout: provider timed out after writing a HIGH changed-file finding" + exit 1 ;; pr-stale-report-plus-inline-changed-finding-blocks) - case "${STRIX_LLM:-}" in - vertex_ai/stale-inline-primary) - mkdir -p "$STRIX_REPORTS_DIR/fake-stale-report-inline-changed/vulnerabilities" - cat >"$STRIX_REPORTS_DIR/fake-stale-report-inline-changed/vulnerabilities/vuln-0001.md" <<'EOS' + mkdir -p "$STRIX_REPORTS_DIR/fake-stale-report-inline-changed/vulnerabilities" + cat >"$STRIX_REPORTS_DIR/fake-stale-report-inline-changed/vulnerabilities/vuln-0001.md" <<'EOS' **Severity:** HIGH **Target:** backend/db/models.py The `WorkspaceRunnerConfig.registration_token` field stores the token as plain text. The vulnerable line is `registration_token: Mapped[str | None] = mapped_column(String, nullable=True)`. EOS - echo "Severity: HIGH" - echo "Target: backend/api/emails.py" - echo "Penetration test failed: stale report plus inline changed-file HIGH finding" - exit 1 - ;; - vertex_ai/fallback-one) - echo "Error: inline changed-file findings must not reach fallback" >&2 - exit 35 - ;; - *) - echo "Error: stale-inline scenario unexpected model (${STRIX_LLM:-})" >&2 - exit 36 - ;; - esac + echo "Severity: HIGH" + echo "Target: backend/api/emails.py" + echo "Penetration test failed: stale report plus inline changed-file HIGH finding" + exit 1 ;; endpoint-in-excluded-dir) - case "${STRIX_LLM:-}" in - vertex_ai/excluded-dir-primary) - mkdir -p "$STRIX_REPORTS_DIR/fake-excluded-dir/vulnerabilities" - cat >"$STRIX_REPORTS_DIR/fake-excluded-dir/vulnerabilities/vuln-0001.md" <<'EOS' + mkdir -p "$STRIX_REPORTS_DIR/fake-excluded-dir/vulnerabilities" + cat >"$STRIX_REPORTS_DIR/fake-excluded-dir/vulnerabilities/vuln-0001.md" <<'EOS' **Severity:** CRITICAL **Endpoint:** /api/hidden-secret EOS - echo "Penetration test failed: CRITICAL finding on /api/hidden-secret" - exit 1 - ;; - vertex_ai/fallback-one) - echo "scan ok after excluded-dir hallucination fallback" - exit 0 - ;; - *) - echo "Error: excluded-dir scenario unexpected model (${STRIX_LLM:-})" >&2 - exit 29 - ;; - esac + echo "Penetration test failed: CRITICAL finding on /api/hidden-secret" + exit 1 ;; empty-fallback-models) # Output must match is_vertex_not_found_error() patterns so the gate @@ -4482,32 +3854,14 @@ EOS exit 1 ;; model-disagreement-critical-in-earlier-report) - case "${STRIX_LLM:-}" in - vertex_ai/model-a) - mkdir -p "$STRIX_REPORTS_DIR/run-001/vulnerabilities" - cat >"$STRIX_REPORTS_DIR/run-001/vulnerabilities/vuln-0001.md" <<'EOS' + mkdir -p "$STRIX_REPORTS_DIR/run-001/vulnerabilities" + cat >"$STRIX_REPORTS_DIR/run-001/vulnerabilities/vuln-0001.md" <<'EOS' Severity: CRITICAL EOS - echo "Error: litellm.NotFoundError: Vertex_aiException - x" - echo '"status": "NOT_FOUND"' - echo "Penetration test failed: CRITICAL finding by model-a" - exit 1 - ;; - vertex_ai/model-b) - mkdir -p "$STRIX_REPORTS_DIR/run-002/vulnerabilities" - cat >"$STRIX_REPORTS_DIR/run-002/vulnerabilities/vuln-0001.md" <<'EOS' -Severity: LOW -EOS - echo "Error: litellm.NotFoundError: Vertex_aiException - x" - echo '"status": "NOT_FOUND"' - echo "Penetration test failed: LOW finding by model-b" - exit 1 - ;; - *) - echo "Error: model-disagreement unexpected model (${STRIX_LLM:-})" >&2 - exit 32 - ;; - esac + echo "Error: litellm.NotFoundError: Vertex_aiException - x" + echo '"status": "NOT_FOUND"' + echo "Penetration test failed: CRITICAL finding by model-a" + exit 1 ;; nonvertex-slash-model-not-rewritten) if [ "${STRIX_LLM:-}" = "deepseek/models/deepseek-r1" ]; then @@ -4518,46 +3872,16 @@ EOS exit 33 ;; preserve-existing-api-base) - if [ "${LLM_API_BASE:-}" = "https://preexisting.invalid" ]; then - echo "scan ok with preserved api base" + # Free-only routing pins the loopback gateway. A caller-supplied + # LLM_API_BASE must NOT survive into the scan process, or an external + # endpoint would receive organization review traffic. + if [ "${LLM_API_BASE:-}" = "http://127.0.0.1:18080/v1" ]; then + echo "scan ok with pinned gateway api base" exit 0 fi - echo "Error: existing LLM_API_BASE was not preserved (${LLM_API_BASE:-})" >&2 + echo "Error: gateway LLM_API_BASE was not pinned (${LLM_API_BASE:-})" >&2 exit 20 ;; - default-fallback-order-fast-first) - case "${STRIX_LLM:-}" in - vertex_ai/missing-primary) - echo "Error: litellm.NotFoundError: Vertex_aiException - x" - echo '"status": "NOT_FOUND"' - exit 1 - ;; - vertex_ai/gemini-2.5-pro) - echo "scan ok with default fast fallback" - exit 0 - ;; - *) - echo "Error: default fallback order unexpected (${STRIX_LLM:-})" >&2 - exit 16 - ;; - esac - ;; - vertex-primary-timeout-retry-same-model-success|vertex-primary-timeout-retry-reason-message) - case "${STRIX_LLM:-}" in - vertex_ai/retry-timeout-primary) - echo "litellm.exceptions.Timeout: litellm.Timeout: Connection timed out after None seconds." - exit 1 - ;; - vertex_ai/fallback-one) - echo "scan ok after timeout fallback" - exit 0 - ;; - *) - echo "Error: timeout fallback path unexpected (${STRIX_LLM:-})" >&2 - exit 34 - ;; - esac - ;; all-fallbacks-same-as-primary) # Bug 13: All fallback models are the same as the primary model. # The gate should emit an ERROR and exit 1. @@ -4565,82 +3889,13 @@ EOS echo '"status": "NOT_FOUND"' exit 1 ;; - vertex-primary-timeout-exhausted-fallback-success) - # Primary always times out (even after retries). Fallback succeeds. - case "${STRIX_LLM:-}" in - vertex_ai/timeout-exhaust-primary) - echo "litellm.exceptions.Timeout: litellm.Timeout: Connection timed out after None seconds." - exit 1 - ;; - vertex_ai/fallback-one) - echo "scan ok after timeout-exhausted fallback" - exit 0 - ;; - *) - echo "Error: timeout-exhausted-fallback unexpected model (${STRIX_LLM:-})" >&2 - exit 35 - ;; - esac - ;; zero-findings-timeout-all-models|strict-zero-findings-timeout-fails-pr) - case "${STRIX_LLM:-}" in - vertex_ai/zero-timeout-primary|vertex_ai/fallback-one) - echo "╭─ STRIX ──────────────────────────────────────────────────────────────────────╮" - echo "│ Penetration test in progress │" - echo "│ Vulnerabilities 0 │" - echo "╰──────────────────────────────────────────────────────────────────────────────╯" - sleep "${FAKE_STRIX_TIMEOUT_SLEEP_SECONDS:?}" - exit 0 - ;; - *) - echo "Error: zero-findings-timeout unexpected model (${STRIX_LLM:-})" >&2 - exit 57 - ;; - esac - ;; - zero-findings-sticky-across-fallback) - case "${STRIX_LLM:-}" in - vertex_ai/zero-sticky-primary) - echo "╭─ STRIX ──────────────────────────────────────────────────────────────────────╮" - echo "│ Penetration test in progress │" - echo "│ Vulnerabilities 0 │" - echo "╰──────────────────────────────────────────────────────────────────────────────╯" - sleep "${FAKE_STRIX_TIMEOUT_SLEEP_SECONDS:?}" - exit 0 - ;; - vertex_ai/fallback-one) - sleep "${FAKE_STRIX_TIMEOUT_SLEEP_SECONDS:?}" - exit 0 - ;; - *) - echo "Error: zero-findings-sticky unexpected model (${STRIX_LLM:-})" >&2 - exit 58 - ;; - esac - ;; - zero-findings-with-low-report-timeout) - case "${STRIX_LLM:-}" in - vertex_ai/zero-low-primary) - mkdir -p "$STRIX_REPORTS_DIR/fake-zero-low/vulnerabilities" - cat >"$STRIX_REPORTS_DIR/fake-zero-low/vulnerabilities/vuln-0001.md" <<'EOS' -Severity: LOW -EOS - echo "╭─ STRIX ──────────────────────────────────────────────────────────────────────╮" - echo "│ Penetration test in progress │" - echo "│ Vulnerabilities 0 │" - echo "╰──────────────────────────────────────────────────────────────────────────────╯" - sleep "${FAKE_STRIX_TIMEOUT_SLEEP_SECONDS:?}" - exit 0 - ;; - vertex_ai/fallback-one) - sleep "${FAKE_STRIX_TIMEOUT_SLEEP_SECONDS:?}" - exit 0 - ;; - *) - echo "Error: zero-findings-with-low-report unexpected model (${STRIX_LLM:-})" >&2 - exit 59 - ;; - esac + echo "╭─ STRIX ──────────────────────────────────────────────────────────────────────╮" + echo "│ Penetration test in progress │" + echo "│ Vulnerabilities 0 │" + echo "╰──────────────────────────────────────────────────────────────────────────────╯" + sleep "${FAKE_STRIX_TIMEOUT_SLEEP_SECONDS:?}" + exit 0 ;; provider-fatal-success-signal) echo "Fatal: provider stream aborted" @@ -4654,27 +3909,6 @@ EOS echo "Denied: provider credentials were rejected" exit 0 ;; - provider-report-rate-limit-fallback-success) - case "${STRIX_LLM:-}" in - vertex_ai/report-rate-limit-primary) - mkdir -p "$STRIX_REPORTS_DIR/fake-report-rate-limit" - cat >"$STRIX_REPORTS_DIR/fake-report-rate-limit/strix.log" <<'EOS' -2026-08-21 04:00:00.000 WARNING strix-pr-scope-example - strix.provider: RateLimitError: provider response was exhausted -EOS - echo "scan aborted after provider report-rate-limit signal" - exit 1 - ;; - vertex_ai/fallback-one) - mkdir -p "$STRIX_REPORTS_DIR/fake-report-rate-limit-fallback" - echo "scan ok after report-only provider fallback" - exit 0 - ;; - *) - echo "Error: report-only provider fallback path unexpected (${STRIX_LLM:-})" >&2 - exit 60 - ;; - esac - ;; report-known-internal-warning-sanitized) printf '%s\n' '│ MODEL QUALITY WARNING │' echo 'Warning: You are sending unauthenticated requests to the HF Hub.' @@ -4714,29 +3948,6 @@ EOS echo "scan ok but unknown report warning remains" exit 0 ;; - bare-timeout-with-provider-marker) - # Emit bare "Connection timed out" alongside a provider marker so - # is_timeout_error() matches the Tier 3 branch gated on - # LLM_PROVIDER_ONLY_REGEX. Does NOT include - # litellm.exceptions.Timeout / httpx.ReadTimeout to ensure we - # exercise the provider-marker fallback path specifically. - # Primary times out; fallback model succeeds. - case "${STRIX_LLM:-}" in - vertex_ai/bare-timeout-primary) - echo "Connection timed out" - echo "vertex_ai model invocation failed" - exit 1 - ;; - vertex_ai/fallback-one) - echo "scan ok after bare-timeout fallback" - exit 0 - ;; - *) - echo "Error: bare-timeout fallback path unexpected (${STRIX_LLM:-})" >&2 - exit 47 - ;; - esac - ;; bare-timeout-no-provider-marker) # Emit "Connection timed out" with transport library names (httpx, # httpcore, requests) but WITHOUT any real LLM provider marker. @@ -4748,27 +3959,6 @@ EOS echo "requests transport timeout" exit 1 ;; - below-threshold-with-timeout) - # Produce a below-threshold (LOW) finding but also emit a timeout error - # so the infrastructure guard detects an incomplete scan. - mkdir -p "$STRIX_REPORTS_DIR/fake-low-timeout/vulnerabilities" - cat >"$STRIX_REPORTS_DIR/fake-low-timeout/vulnerabilities/vuln-0001.md" <<'EOS' -Severity: LOW -EOS - echo "litellm.exceptions.Timeout: litellm.Timeout: Connection timed out after None seconds." - echo "Penetration test failed: simulated timeout with low finding" - exit 1 - ;; - below-threshold-with-ratelimit) - # Produce a below-threshold (LOW) finding but also emit a rate-limit error. - mkdir -p "$STRIX_REPORTS_DIR/fake-low-ratelimit/vulnerabilities" - cat >"$STRIX_REPORTS_DIR/fake-low-ratelimit/vulnerabilities/vuln-0001.md" <<'EOS' -Severity: LOW -EOS - echo "Penetration test failed: LLM request failed: RateLimitError" - echo "Penetration test failed: simulated ratelimit with low finding" - exit 1 - ;; below-threshold-with-connection-error) # Produce a below-threshold (INFO) finding but also emit a # ConnectionError WITH an LLM-provider context marker so the @@ -4817,55 +4007,6 @@ EOS echo "Penetration test failed: simulated requests transport error" exit 1 ;; - below-threshold-with-midstream) - # Produce a below-threshold (MEDIUM) finding below CRITICAL threshold - # but also emit a MidStreamFallbackError. - mkdir -p "$STRIX_REPORTS_DIR/fake-medium-midstream/vulnerabilities" - cat >"$STRIX_REPORTS_DIR/fake-medium-midstream/vulnerabilities/vuln-0001.md" <<'EOS' -Severity: MEDIUM -EOS - echo "Penetration test failed: LLM request failed: MidStreamFallbackError" - echo "Penetration test failed: simulated midstream with medium finding" - exit 1 - ;; - bare-timeout-provider-marker-exhausted-fallback) - # Bare "Connection timed out" + provider marker: primary fails once, - # then the gate falls back to fallback-one which succeeds. - case "${STRIX_LLM:-}" in - vertex_ai/bare-timeout-exhaust-primary) - echo "Connection timed out" - echo "vertex_ai model invocation failed" - exit 1 - ;; - vertex_ai/fallback-one) - echo "scan ok after bare-timeout-exhaust fallback" - exit 0 - ;; - *) - echo "Error: bare-timeout-exhaust-fallback unexpected model (${STRIX_LLM:-})" >&2 - exit 35 - ;; - esac - ;; - httpx-read-timeout-with-provider-marker) - # Tier 2: httpx.ReadTimeout + provider-context marker (litellm). - # Primary times out; fallback model succeeds. - case "${STRIX_LLM:-}" in - vertex_ai/httpx-timeout-primary) - echo "httpx.ReadTimeout: timed out" - echo "litellm.proxy: connection to upstream model failed" - exit 1 - ;; - vertex_ai/fallback-one) - echo "scan ok after httpx-timeout fallback" - exit 0 - ;; - *) - echo "Error: httpx-timeout fallback path unexpected (${STRIX_LLM:-})" >&2 - exit 45 - ;; - esac - ;; httpx-read-timeout-no-provider-marker) # Tier 2 negative: httpx.ReadTimeout WITHOUT any provider-context # marker. Should NOT be classified as retryable timeout. @@ -4873,25 +4014,6 @@ EOS echo "application server connection pool exhausted" exit 1 ;; - httpcore-read-timeout-with-provider-marker) - # Tier 2b: httpcore.ReadTimeout + provider-context marker. - # Primary times out; fallback model succeeds. - case "${STRIX_LLM:-}" in - vertex_ai/httpcore-timeout-primary) - echo "httpcore.ReadTimeout: timed out" - echo "litellm.proxy: connection to upstream model failed" - exit 1 - ;; - vertex_ai/fallback-one) - echo "scan ok after httpcore-timeout fallback" - exit 0 - ;; - *) - echo "Error: httpcore-timeout fallback path unexpected (${STRIX_LLM:-})" >&2 - exit 46 - ;; - esac - ;; httpcore-read-timeout-no-provider-marker) # Tier 2b negative: httpcore.ReadTimeout WITHOUT any provider-context # marker. Should NOT be classified as retryable timeout. @@ -4899,34 +4021,6 @@ EOS echo "application server connection pool exhausted" exit 1 ;; - infra-error-sticky-flag) - # Sticky flag test: first call hits infra error (rate limit), - # second call fails on the first fallback model but produces a - # LOW finding report. After exhausting retries, the gate checks - # has_only_below_threshold_vulnerabilities — which finds LOW - # findings but sees INFRA_ERROR_DETECTED=1 (set from the first - # call's rate-limit error) and refuses the below-threshold bypass. - case "${STRIX_LLM:-}" in - vertex_ai/sticky-flag-primary) - touch "$FAKE_STRIX_STATE_FILE" - echo "RateLimitError: rate limit exceeded" - echo "litellm.proxy: rate limit on vertex_ai model" - exit 1 - ;; - vertex_ai/gemini-2.5-pro) - mkdir -p "$STRIX_REPORTS_DIR/run-sticky/vulnerabilities" - cat > "$STRIX_REPORTS_DIR/run-sticky/vulnerabilities/vuln-0001.md" <<'FINDINGS' -Severity: LOW -FINDINGS - echo "non-retryable scan error with partial results" - exit 1 - ;; - *) - echo "Error: infra-error-sticky-flag unexpected model (${STRIX_LLM:-})" >&2 - exit 35 - ;; - esac - ;; pr-baseline-critical-unchanged) mkdir -p "$STRIX_REPORTS_DIR/fake-pr-baseline/vulnerabilities" cat >"$STRIX_REPORTS_DIR/fake-pr-baseline/vulnerabilities/vuln-0001.md" <<'EOS' @@ -5189,90 +4283,6 @@ EOS echo "Penetration test failed: manifest-only critical finding" exit 1 ;; - pr-critical-manifest-only-pom-after-fallback-authoritative) - case "${STRIX_LLM:-}" in - vertex_ai/timeout-primary) - echo "litellm.exceptions.Timeout: primary model timed out" - exit 1 - ;; - vertex_ai/fallback-one) - mkdir -p "$STRIX_REPORTS_DIR/fake-pr-manifest-only-after-fallback/vulnerabilities" - cat >"$STRIX_REPORTS_DIR/fake-pr-manifest-only-after-fallback/vulnerabilities/vuln-0001.md" <<'EOS' -Severity: CRITICAL -Location 1: -pom.xml:8 -EOS - echo "Penetration test failed: manifest-only critical finding after fallback" - exit 1 - ;; - *) - echo "Error: pr-critical-manifest-only-pom-after-fallback-authoritative unexpected model (${STRIX_LLM:-})" >&2 - exit 53 - ;; - esac - ;; - pr-critical-manifest-only-pom-console-only-after-fallback-authoritative) - case "${STRIX_LLM:-}" in - vertex_ai/timeout-primary) - echo "litellm.exceptions.Timeout: primary model timed out" - exit 1 - ;; - vertex_ai/fallback-one) - echo "Severity: CRITICAL" - echo "Location 1:" - echo "pom.xml:59" - echo "Penetration test failed: manifest-only critical finding after fallback (console-only)" - exit 1 - ;; - *) - echo "Error: pr-critical-manifest-only-pom-console-only-after-fallback-authoritative unexpected model (${STRIX_LLM:-})" >&2 - exit 54 - ;; - esac - ;; - pr-critical-manifest-only-pom-console-target-only-after-fallback-authoritative) - case "${STRIX_LLM:-}" in - vertex_ai/timeout-primary) - echo "litellm.exceptions.Timeout: primary model timed out" - exit 1 - ;; - vertex_ai/fallback-one) - echo "Severity: CRITICAL" - echo "Target: /workspace/$(basename "$target_path")/pom.xml" - echo "Penetration test failed: manifest-only critical finding after fallback (console target-only)" - exit 1 - ;; - *) - echo "Error: pr-critical-manifest-only-pom-console-target-only-after-fallback-authoritative unexpected model (${STRIX_LLM:-})" >&2 - exit 56 - ;; - esac - ;; - pr-low-markdown-plus-console-critical-manifest-after-fallback-authoritative) - case "${STRIX_LLM:-}" in - vertex_ai/timeout-primary) - echo "litellm.exceptions.Timeout: primary model timed out" - exit 1 - ;; - vertex_ai/fallback-one) - mkdir -p "$STRIX_REPORTS_DIR/fake-pr-manifest-mixed-after-fallback/vulnerabilities" - cat >"$STRIX_REPORTS_DIR/fake-pr-manifest-mixed-after-fallback/vulnerabilities/vuln-0001.md" <<'EOS' -Severity: LOW -Location 1: -pom.xml:8 -EOS - echo "Severity: CRITICAL" - echo "Location 1:" - echo "pom.xml:59" - echo "Penetration test failed: manifest-only critical finding after fallback (mixed file+console)" - exit 1 - ;; - *) - echo "Error: pr-low-markdown-plus-console-critical-manifest-after-fallback-authoritative unexpected model (${STRIX_LLM:-})" >&2 - exit 55 - ;; - esac - ;; pr-changed-scope-bounded) if [ -z "$target_path" ]; then echo "Error: target path missing" >&2 @@ -6002,12 +5012,6 @@ PY "LLM_TIMEOUT=90;STRIX_MEMORY_COMPRESSOR_TIMEOUT=10;STRIX_REASONING_EFFORT=minimal;STRIX_LLM_MAX_RETRIES=1;GEMINI_LOCATION=GLOBAL;PYTHONWARNINGS=ignore:Pydantic serializer warnings:UserWarning:pydantic.main;NPM_CONFIG_IGNORE_SCRIPTS=true;PNPM_CONFIG_IGNORE_SCRIPTS=true;YARN_ENABLE_SCRIPTS=false;UNRELATED_SECRET=" \ "scenario=$scenario runtime env forwarding" fi - if [ "$scenario" = "custom-openai-compatible-preserves-effort" ]; then - assert_file_contains \ - "$runtime_env_log" \ - "STRIX_REASONING_EFFORT=minimal" \ - "scenario=$scenario custom compatible endpoint effort" - fi if [ "$scenario" = "report-known-internal-warning-sanitized" ]; then assert_file_not_contains \ @@ -6099,44 +5103,6 @@ run_gate_case_allow_provider_signal() { run_gate_case_with_provider_signal_mode "0" "$@" } -run_github_models_http410_case() { - local scenario="$1" - local expected_exit="$2" - local expected_calls="$3" - local expected_models="$4" - local expected_api_bases="$5" - local expected_message="${6-}" - - run_gate_case "$scenario" \ - "openai/gpt-5" \ - "" \ - "$expected_exit" \ - "$expected_message" \ - "$expected_calls" \ - "$expected_models" \ - "$expected_api_bases" \ - "openai" \ - "https://models.github.ai/inference" \ - "" \ - "0" \ - "CRITICAL" \ - "0" \ - "" \ - "" \ - "1200" \ - "0" \ - "" \ - "" \ - "" \ - "" \ - "0" \ - "" \ - "" \ - "" \ - "__SAME_AS_FALLBACK_MODELS__" \ - "deepseek/deepseek-r1-0528" \ - "1" -} run_filtered_gate_case_if_requested() { case "${STRIX_TEST_CASE_FILTER:-}" in @@ -6145,13 +5111,15 @@ run_filtered_gate_case_if_requested() { ;; success) run_gate_case "success" \ - "vertex_ai/ready-primary" \ - "vertex_ai/fallback-one vertex_ai/fallback-two" \ + "orchestrator/free" \ + "" \ "0" \ "scan ok" \ "1" \ - "vertex_ai/ready-primary" \ - "" + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" ;; contextual-orchestrator-missing-api-base-fails-closed) run_gate_case "contextual-orchestrator-missing-api-base-fails-closed" \ @@ -6179,15 +5147,15 @@ run_filtered_gate_case_if_requested() { ;; pr-rust-workspace-context) run_gate_case "pr-rust-workspace-context" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "0" \ "scan ok with Rust workspace context" \ "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -6201,516 +5169,170 @@ run_filtered_gate_case_if_requested() { ;; success-with-critical-report) run_gate_case "success-with-critical-report" \ - "vertex_ai/ready-primary" \ + "orchestrator/free" \ "" \ "1" \ - "Strix exited successfully but emitted a vulnerability at or above 'CRITICAL'" \ + "Current Strix vulnerability report exists; failing closed without a repository-authored severity threshold." \ "1" \ - "vertex_ai/ready-primary" \ - "" + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" ;; pr-executable-integrity-mismatch) run_gate_case "pr-executable-integrity-mismatch" \ - "vertex_ai/ready-primary" \ + "orchestrator/free" \ "" \ "1" \ "did not match the pinned SHA-256 digest" \ "0" \ "" \ - "" + "" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" ;; pr-executable-group-writable) run_gate_case "pr-executable-group-writable" \ - "vertex_ai/ready-primary" \ + "orchestrator/free" \ "" \ "1" \ "must not be group/world writable" \ "0" \ "" \ - "" + "" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" ;; pr-executable-root-group-writable) run_gate_case "pr-executable-root-group-writable" \ - "vertex_ai/ready-primary" \ + "orchestrator/free" \ "" \ "1" \ "pinned Strix installation root must not be group/world writable" \ "0" \ "" \ - "" + "" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" ;; vertex-primary-hallucinated-endpoint-fallback-success) run_gate_case "vertex-primary-hallucinated-endpoint-fallback-success" \ - "vertex_ai/hallucination-primary" \ - "vertex_ai/fallback-one vertex_ai/fallback-two" \ + "orchestrator/free" \ + "" \ "1" \ "Strix quick scan failed with a non-recoverable error." \ "1" \ - "vertex_ai/hallucination-primary" \ - "" + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" ;; target-path-src-default-source-dirs) run_gate_case "target-path-src-default-source-dirs" \ - "vertex_ai/hallucination-primary" \ - "vertex_ai/fallback-one vertex_ai/fallback-two" \ + "orchestrator/free" \ + "" \ "1" \ "Strix quick scan failed with a non-recoverable error." \ "1" \ - "vertex_ai/hallucination-primary" \ - "" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ - "1" \ + "0" \ "CRITICAL" \ "0" \ "__USE_SUBDIR_SRC__" \ "" ;; - vertex-ignores-untrusted-llm-api-base-file) - run_vertex_model_ignores_untrusted_llm_api_base_file_case - ;; - input-file-root-override-precedence) - run_input_file_root_override_takes_precedence_over_runner_temp_case - ;; - vertex-without-llm-api-key) - run_vertex_without_llm_api_key_case - ;; - vertex-with-llm-api-key-file-not-forwarded) - run_vertex_with_llm_api_key_file_does_not_forward_case - ;; - stale-report-does-not-bypass) - run_stale_report_case - ;; - symlink-report-does-not-bypass) - run_symlink_report_case - ;; - github-models-token-limit-fallback-success) - run_gate_case "github-models-token-limit-fallback-success" \ - "openai/gpt-5" \ - "" \ - "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'github_models/deepseek/deepseek-v3-0324' in [0-9]+s\\." \ - "2" \ - "openai/gpt-5|openai/deepseek/deepseek-v3-0324" \ - "https://models.github.ai/inference|https://models.github.ai/inference" \ - "openai" \ - "https://models.github.ai/inference" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "github_models/deepseek/deepseek-v3-0324 github_models/deepseek/deepseek-r1-0528" - ;; - openrouter-502-fallback-retry-same-model-success) - run_gate_case "openrouter-502-fallback-retry-same-model-success" \ - "vertex_ai/missing-primary" \ - "openrouter/free vertex_ai/fallback-two" \ - "0" \ - "scan ok after OpenRouter 502 same-model retry" \ - "3" \ - "vertex_ai/missing-primary|openrouter/free|openrouter/free" \ - "|https://example.invalid|https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ - "" \ - "1" - ;; - openrouter-502-distant-target-output-nonretryable) - run_gate_case "openrouter-502-distant-target-output-nonretryable" \ - "vertex_ai/missing-primary" \ - "openrouter/free vertex_ai/fallback-two" \ - "1" \ - "Strix quick scan failed with a non-recoverable error." \ - "2" \ - "vertex_ai/missing-primary|openrouter/free" \ - "|https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ - "" \ - "1" - ;; service-unavailable-no-llm-marker-nonrecoverable) run_gate_case "service-unavailable-no-llm-marker-nonrecoverable" \ - "custom/service-unavailable-primary" \ - "vertex_ai/fallback-one vertex_ai/fallback-two" \ + "orchestrator/free" \ + "" \ "1" \ - "Strix quick scan failed with a non-recoverable error." \ + "Strix quick scan failed; failing closed without repository-authored retry or fallback allocation." \ "1" \ - "custom/service-unavailable-primary" \ - "https://example.invalid" \ - "custom" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ - "1" + "0" ;; custom-openai-compatible-preserves-effort) run_gate_case "custom-openai-compatible-preserves-effort" \ "openai-direct/gpt-5.4" \ "" \ + "2" \ + "ERROR: Strix model must be orchestrator/free through contextual-orchestrator; direct provider/model routes are forbidden" \ "0" \ - "scan ok" \ - "1" \ - "openai/gpt-5.4" \ - "https://compatible.example/v1" \ - "openai" \ - "https://compatible.example/v1" + "" \ + "" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" ;; - nvidia-rate-limit-openai-direct-fallback-clears-api-base) - run_gate_case_allow_provider_signal "nvidia-rate-limit-openai-direct-fallback-clears-api-base" \ - "nvidia_nim/nvidia/rate-limited-primary" \ + pr-critical-changed-json-target) + run_gate_case "pr-critical-changed-json-target" \ + "orchestrator/free" \ "" \ - "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'openai-direct/gpt-5.4' in [0-9]+s\\." \ - "2" \ - "nvidia_nim/nvidia/rate-limited-primary|openai/gpt-5.4" \ - "https://integrate.api.nvidia.com/v1|" \ - "nvidia_nim" \ - "https://integrate.api.nvidia.com/v1" \ + "1" \ + "Strix finding intersects files changed in this pull request." \ + "1" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ - "CRITICAL" \ + "MEDIUM" \ "0" \ "" \ "" \ "1200" \ "0" \ - "" \ - "" \ - "" \ - "" \ - "0" \ - "" \ - "" \ - "" \ - "__SAME_AS_FALLBACK_MODELS__" \ - "openai-direct/gpt-5.4" + "pull_request" \ + "frontend/src/components/CalendarLayout.tsx" ;; - openai-direct-quota-github-models-fallback-success) - run_gate_case "openai-direct-quota-github-models-fallback-success" \ - "openai_direct/gpt-5.4" \ - "" \ - "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'github_models/openai/o3' in [0-9]+s\\." \ - "2" \ - "openai/gpt-5.4|openai/o3" \ - "|https://models.github.ai/inference" \ - "vertex_ai" \ - "" \ + github-models-http410-missing-http-token | github-models-http410-missing-provider-error | github-models-http410-numeric-continuation-4100 | github-models-http410-numeric-continuation-4104 | github-models-http410-target-output-spoof | github-models-retirement-brownout-phrase-only) + run_github_models_http410_case \ + "$STRIX_TEST_CASE_FILTER" \ + "1" \ + "1" \ + "openai/gpt-5" \ + "https://models.github.ai/inference" + ;; + internal-server-error-unrelated-output-nonretryable) + run_gate_case_allow_provider_signal "$STRIX_TEST_CASE_FILTER" \ + "openai/openai/retry-api-connection-primary" \ "" \ + "1" \ + "Strix quick scan failed with a non-recoverable error." \ + "1" \ + "openai/openai/retry-api-connection-primary" \ + "https://models.github.ai/inference" \ + "openai" \ + "https://models.github.ai/inference" \ "" \ + "0" + ;; + endpoint-in-excluded-dir) + run_gate_case "endpoint-in-excluded-dir" \ + "orchestrator/free" \ "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "github_models/openai/o3" - ;; - gemini-timeout-fallback-success) - run_gate_case_allow_provider_signal "gemini-timeout-fallback-success" \ - "gemini/timeout-fallback-primary" \ - "gemini/fallback-one gemini/fallback-two" \ - "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'gemini/fallback-one' in [0-9]+s\\." \ - "2" \ - "gemini/timeout-fallback-primary|gemini/fallback-one" \ - "https://example.invalid|https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ - "" \ - "1" - ;; - zero-findings-with-low-report-timeout) - run_gate_case_allow_provider_signal "zero-findings-with-low-report-timeout" \ - "vertex_ai/zero-low-primary" \ - "vertex_ai/fallback-one" \ - "1" \ - "Configured Vertex model and fallback models were unavailable." \ - "2" \ - "vertex_ai/zero-low-primary|vertex_ai/fallback-one" \ - "|" \ - "vertex_ai" \ - "__DEFAULT__" \ - "" \ - "0" \ - "CRITICAL" \ - "0" \ - "" \ - "" \ - "$TIMEOUT_TEST_PROCESS_SECONDS" \ - "0" \ - "pull_request" \ - "sync-module-system/smart-crawling-biz/src/main/java/org/empasy/sync/modules/system/controller/SysPositionController.java" - ;; - zero-findings-timeout-all-models) - run_gate_case_allow_provider_signal "zero-findings-timeout-all-models" \ - "vertex_ai/zero-timeout-primary" \ - "vertex_ai/fallback-one" \ - "1" \ - "Strix reported zero vulnerabilities before provider infrastructure failure; failing closed because provider infrastructure failures are not clean scan evidence." \ - "2" \ - "vertex_ai/zero-timeout-primary|vertex_ai/fallback-one" \ - "|" \ - "vertex_ai" \ - "__DEFAULT__" \ - "" \ - "0" \ - "CRITICAL" \ - "0" \ - "" \ - "" \ - "$TIMEOUT_TEST_PROCESS_SECONDS" \ - "0" \ - "pull_request" \ - "sync-module-system/smart-crawling-biz/src/main/java/org/empasy/sync/modules/system/controller/SysPositionController.java" - run_gate_case_allow_provider_signal "zero-findings-timeout-all-models" \ - "vertex_ai/zero-timeout-primary" \ - "vertex_ai/fallback-one" \ - "1" \ - "Configured Vertex model and fallback models were unavailable." \ - "2" \ - "vertex_ai/zero-timeout-primary|vertex_ai/fallback-one" \ - "|" \ - "vertex_ai" \ - "__DEFAULT__" \ - "" \ - "0" \ - "CRITICAL" \ - "0" \ - "" \ - "" \ - "$TIMEOUT_TEST_PROCESS_SECONDS" \ - "0" \ - "push" - ;; - slow-timeout) - run_gate_case_allow_provider_signal "slow-timeout" \ - "vertex_ai/slow-primary" \ - "" \ - "1" \ - "Strix run timed out after ${TIMEOUT_TEST_PROCESS_SECONDS}s." \ - "3" \ - "vertex_ai/slow-primary|vertex_ai/gemini-2.5-pro|vertex_ai/gemini-2.5-flash" \ - "||" \ - "vertex_ai" \ - "__DEFAULT__" \ - "" \ - "0" \ - "CRITICAL" \ - "0" \ - "" \ - "" \ - "$TIMEOUT_TEST_PROCESS_SECONDS" - ;; - timeout-cleanup) - run_timeout_cleanup_case - ;; - vertex-primary-notfound-fallback-success) - run_gate_case "vertex-primary-notfound-fallback-success" \ - "vertex_ai/missing-primary" \ - "vertex_ai/fallback-one vertex_ai/fallback-two" \ - "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'vertex_ai/fallback-one' in [0-9]+s\\." \ - "2" \ - "vertex_ai/missing-primary|vertex_ai/fallback-one" \ - "|" - ;; - openai-primary-quota-fallback-success) - run_gate_case_allow_provider_signal "openai-primary-quota-fallback-success" \ - "openai/quota-primary" \ - "openai/fallback-one openai/fallback-two" \ - "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'openai/fallback-one' in [0-9]+s\\." \ - "2" \ - "openai/quota-primary|openai/fallback-one" \ - "|" \ - "openai" - ;; - pr-critical-changed-json-target) - run_gate_case "pr-critical-changed-json-target" \ - "vertex_ai/gemini-2.5-pro" \ - "" \ - "1" \ - "Strix finding intersects files changed in this pull request." \ - "1" \ - "vertex_ai/gemini-2.5-pro" \ - "" \ - "vertex_ai" \ - "__DEFAULT__" \ - "" \ - "0" \ - "MEDIUM" \ - "0" \ - "" \ - "" \ - "1200" \ - "0" \ - "pull_request" \ - "frontend/src/components/CalendarLayout.tsx" - ;; - github-models-primary-ratelimit-fallback-success) - run_gate_case "github-models-primary-ratelimit-fallback-success" \ - "openai/gpt-5" \ - "" \ - "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'deepseek/deepseek-r1-0528' in [0-9]+s\\." \ - "2" \ - "openai/gpt-5|openai/deepseek/deepseek-r1-0528" \ - "https://models.github.ai/inference|https://models.github.ai/inference" \ - "openai" \ - "https://models.github.ai/inference" \ - "" \ - "2" \ - "CRITICAL" \ - "0" \ - "" \ - "" \ - "1200" \ - "0" \ - "" \ - "" \ - "" \ - "" \ - "0" \ - "" \ - "" \ - "" \ - "__SAME_AS_FALLBACK_MODELS__" \ - "deepseek/deepseek-r1-0528 deepseek/deepseek-v3-0324" \ - "1" - ;; - github-models-http410-authenticated-fallback-success) - run_github_models_http410_case \ - "$STRIX_TEST_CASE_FILTER" \ - "0" \ - "2" \ - "openai/gpt-5|openai/deepseek/deepseek-r1-0528" \ - "https://models.github.ai/inference|https://models.github.ai/inference" \ - "REGEX:Strix quick scan succeeded with fallback model 'deepseek/deepseek-r1-0528' in [0-9]+s\\." - ;; - github-models-http410-missing-http-token | github-models-http410-missing-provider-error | github-models-http410-numeric-continuation-4100 | github-models-http410-numeric-continuation-4104 | github-models-http410-target-output-spoof | github-models-retirement-brownout-phrase-only) - run_github_models_http410_case \ - "$STRIX_TEST_CASE_FILTER" \ - "1" \ - "1" \ - "openai/gpt-5" \ - "https://models.github.ai/inference" - ;; - github-models-fallback-provider-signal-tries-next) - run_gate_case "github-models-fallback-provider-signal-tries-next" \ - "openai/gpt-5" \ - "" \ - "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'deepseek/deepseek-v3-0324' in [0-9]+s\\." \ - "3" \ - "openai/gpt-5|openai/deepseek/deepseek-r1-0528|openai/deepseek/deepseek-v3-0324" \ - "https://models.github.ai/inference|https://models.github.ai/inference|https://models.github.ai/inference" \ - "openai" \ - "https://models.github.ai/inference" \ - "" \ - "0" \ - "CRITICAL" \ - "0" \ - "" \ - "" \ - "1200" \ - "0" \ - "pull_request" \ - "sync-module-system/smart-crawling-biz/src/main/java/org/empasy/sync/modules/system/controller/SysPositionController.java" \ - "" \ - "" \ - "0" \ - "" \ - "" \ - "" \ - "__SAME_AS_FALLBACK_MODELS__" \ - "deepseek/deepseek-r1-0528 deepseek/deepseek-v3-0324" \ - "1" - ;; - github-models-internal-server-connection-retry-same-model-success) - run_gate_case_allow_provider_signal "$STRIX_TEST_CASE_FILTER" \ - "openai/openai/retry-api-connection-primary" \ - "" \ - "0" \ - "scan ok after same-model api connection retry" \ - "2" \ - "openai/openai/retry-api-connection-primary|openai/openai/retry-api-connection-primary" \ - "https://models.github.ai/inference|https://models.github.ai/inference" \ - "openai" \ - "https://models.github.ai/inference" \ - "" \ - "1" - ;; - internal-server-error-unrelated-output-nonretryable) - run_gate_case_allow_provider_signal "$STRIX_TEST_CASE_FILTER" \ - "openai/openai/retry-api-connection-primary" \ - "" \ - "1" \ - "Strix quick scan failed with a non-recoverable error." \ - "1" \ - "openai/openai/retry-api-connection-primary" \ - "https://models.github.ai/inference" \ - "openai" \ - "https://models.github.ai/inference" \ - "" \ - "0" - ;; - internal-server-error-many-blocks-retry-same-model-success) - run_gate_case_allow_provider_signal "$STRIX_TEST_CASE_FILTER" \ - "openai/openai/retry-api-connection-primary" \ - "" \ - "0" \ - "scan ok after same-model api connection retry" \ - "2" \ - "openai/openai/retry-api-connection-primary|openai/openai/retry-api-connection-primary" \ - "https://models.github.ai/inference|https://models.github.ai/inference" \ - "openai" \ - "https://models.github.ai/inference" \ - "" \ - "1" - ;; - endpoint-in-excluded-dir) - run_gate_case "endpoint-in-excluded-dir" \ - "vertex_ai/excluded-dir-primary" \ - "vertex_ai/fallback-one vertex_ai/fallback-two" \ "1" \ "Unable to map Strix findings to changed files; failing closed for pull request." \ "1" \ - "vertex_ai/excluded-dir-primary" \ - "" - ;; - pull-request-target-changed-backend-context) - run_pull_request_target_changed_backend_context_scope_case + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" ;; report-known-internal-warning-sanitized) run_gate_case "$STRIX_TEST_CASE_FILTER" \ "vertex_ai/report-known-internal-warning-sanitized" \ "" \ "0" \ - "Strix run succeeded for model 'vertex_ai/report-known-internal-warning-sanitized'" \ + "Strix run succeeded for model 'orchestrator/free'" \ "1" \ "vertex_ai/report-known-internal-warning-sanitized" \ "" @@ -6725,61 +5347,19 @@ run_filtered_gate_case_if_requested() { "vertex_ai/$STRIX_TEST_CASE_FILTER" \ "" ;; - provider-report-rate-limit-fallback-success) - run_gate_case "provider-report-rate-limit-fallback-success" \ - "vertex_ai/report-rate-limit-primary" \ - "vertex_ai/fallback-one vertex_ai/fallback-two" \ - "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'vertex_ai/fallback-one' in [0-9]+s\\." \ - "2" \ - "vertex_ai/report-rate-limit-primary|vertex_ai/fallback-one" \ - "|" - ;; - total-timeout) - run_total_timeout_case + pr-stale-snapshot-snippet-fallback-success) ;; - github-models-fallback-baseline-vulnerability-before-next-success-continues) - run_gate_case "github-models-fallback-baseline-vulnerability-before-next-success-continues" \ - "openai/gpt-5" \ - "" \ - "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'deepseek/deepseek-v3-0324' in [0-9]+s\\." \ - "3" \ - "openai/gpt-5|openai/deepseek/deepseek-r1-0528|openai/deepseek/deepseek-v3-0324" \ - "https://models.github.ai/inference|https://models.github.ai/inference|https://models.github.ai/inference" \ - "openai" \ - "https://models.github.ai/inference" \ - "" \ - "0" \ - "CRITICAL" \ - "0" \ - "" \ - "" \ - "1200" \ - "0" \ - "pull_request" \ - "sync-module-system/smart-crawling-biz/src/main/java/org/empasy/sync/modules/system/controller/SysPositionController.java" \ - "" \ + scan-working-directory-isolated) + run_gate_case "scan-working-directory-isolated" \ + "orchestrator/free" \ "" \ "0" \ - "" \ - "" \ - "" \ - "__SAME_AS_FALLBACK_MODELS__" \ - "deepseek/deepseek-r1-0528 deepseek/deepseek-v3-0324" \ - "1" - ;; - github-models-exhausted-after-baseline-vulnerability-fails-closed) - run_gate_case "github-models-exhausted-after-baseline-vulnerability-fails-closed" \ - "openai/gpt-5" \ - "" \ + "scan ok with isolated Strix working directory" \ "1" \ - "STRIX_PROVIDER_UNAVAILABLE: provider models were exhausted after incomplete scan evidence." \ - "3" \ - "openai/gpt-5|openai/deepseek/deepseek-r1-0528|openai/deepseek/deepseek-v3-0324" \ - "https://models.github.ai/inference|https://models.github.ai/inference|https://models.github.ai/inference" \ - "openai" \ - "https://models.github.ai/inference" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -6789,200 +5369,10 @@ run_filtered_gate_case_if_requested() { "1200" \ "0" \ "pull_request" \ - "sync-module-system/smart-crawling-biz/src/main/java/org/empasy/sync/modules/system/controller/SysPositionController.java" \ - "" \ - "" \ - "0" \ - "" \ - "" \ - "" \ - "__SAME_AS_FALLBACK_MODELS__" \ - "deepseek/deepseek-r1-0528 deepseek/deepseek-v3-0324" \ - "1" + "backend/app/pg_introspect/introspect.py" ;; - github-models-fallback-changed-vulnerability-before-next-success-blocks) - run_gate_case "github-models-fallback-changed-vulnerability-before-next-success-blocks" \ - "openai/gpt-5" \ - "" \ - "1" \ - "Strix model reported threshold vulnerabilities before fallback success; failing closed so every model-reported vulnerability is reviewed." \ - "2" \ - "openai/gpt-5|openai/deepseek/deepseek-r1-0528" \ - "https://models.github.ai/inference|https://models.github.ai/inference" \ - "openai" \ - "https://models.github.ai/inference" \ - "" \ - "0" \ - "CRITICAL" \ - "0" \ - "" \ - "" \ - "1200" \ - "0" \ - "pull_request" \ - "sync-module-system/smart-crawling-biz/src/main/java/org/empasy/sync/modules/system/controller/SysPositionController.java" \ - "" \ - "" \ - "0" \ - "" \ - "" \ - "" \ - "__SAME_AS_FALLBACK_MODELS__" \ - "deepseek/deepseek-r1-0528 deepseek/deepseek-v3-0324" \ - "1" - ;; - github-models-fallback-dockerfile-test-baseline-before-next-success-continues) - run_gate_case "github-models-fallback-dockerfile-test-baseline-before-next-success-continues" \ - "openai/gpt-5" \ - "" \ - "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'deepseek/deepseek-v3-0324' in [0-9]+s\\." \ - "3" \ - "openai/gpt-5|openai/deepseek/deepseek-r1-0528|openai/deepseek/deepseek-v3-0324" \ - "https://models.github.ai/inference|https://models.github.ai/inference|https://models.github.ai/inference" \ - "openai" \ - "https://models.github.ai/inference" \ - "" \ - "0" \ - "MEDIUM" \ - "0" \ - "" \ - "" \ - "1200" \ - "0" \ - "pull_request" \ - ".github/workflows/build-ci-image.yml" \ - "" \ - "" \ - "0" \ - "" \ - "" \ - "" \ - "__SAME_AS_FALLBACK_MODELS__" \ - "deepseek/deepseek-r1-0528 deepseek/deepseek-v3-0324" \ - "1" - ;; - pr-stale-snapshot-snippet-fallback-success) - run_gate_case "pr-stale-snapshot-snippet-fallback-success" \ - "vertex_ai/stale-snapshot-primary" \ - "vertex_ai/fallback-one vertex_ai/fallback-two" \ - "0" \ - "scan ok after stale snapshot snippet fallback" \ - "2" \ - "vertex_ai/stale-snapshot-primary|vertex_ai/fallback-one" \ - "|" \ - "vertex_ai" \ - "__DEFAULT__" \ - "" \ - "0" \ - "MEDIUM" \ - "0" \ - "__PR_SCOPE__" \ - "" \ - "1200" \ - "0" \ - "pull_request" \ - "backend/app/api/snapshots.py" - ;; - pull-request-target-modified-file-pr-head-tree-lookup-failure) - run_pull_request_target_aborts_on_pr_head_blob_failure_case \ - "pull-request-target-modified-file-pr-head-tree-lookup-failure" \ - "src/existing.py" \ - "BASE_CONTENT_MUST_NOT_BE_USED_AFTER_HEAD_LOOKUP_FAILURE" \ - "HEAD_CONTENT_SHOULD_NOT_BECOME_PARTIAL_SCAN_INPUT" \ - "ls-tree" \ - "1" - ;; - pull-request-target-changed-file-list-diff-failure) - run_pull_request_target_aborts_on_pr_head_blob_failure_case \ - "pull-request-target-changed-file-list-diff-failure" \ - "src/existing.py" \ - "BASE_CONTENT_MUST_NOT_BE_USED_AFTER_DIFF_FAILURE" \ - "HEAD_CONTENT_SHOULD_NOT_BECOME_PARTIAL_SCAN_INPUT" \ - "diff" - ;; - pull-request-target-gitlink-is-explicitly-skipped) - run_pull_request_target_gitlink_is_explicitly_skipped_case - ;; - pull-request-target-dockerfile-change-uses-full-head-context) - run_pull_request_target_head_scope_case \ - "pull-request-target-dockerfile-change-uses-full-head-context" \ - "Dockerfile" \ - "FROM python:3.12-slim AS base" \ - "FROM python:3.12-slim AS head" \ - "0" \ - "0" \ - "." \ - "1" \ - "Container build manifest changed; materialized full PR-head blob scope" - ;; - repository-dispatch-pr-scope-uses-head-blob) - run_pull_request_target_head_scope_case \ - "repository-dispatch-pr-scope-uses-head-blob" \ - "backend/db/models.py" \ - "BASE_DISPATCH_CONTENT_SHOULD_NOT_BE_SCANNED" \ - "HEAD_DISPATCH_CONTENT_SHOULD_BE_SCANNED" \ - "0" \ - "0" \ - "__PR_SCOPE__" \ - "0" \ - "Materialized PR-head changed-file scope" \ - "repository_dispatch" - ;; - scan-working-directory-isolated) - run_gate_case "scan-working-directory-isolated" \ - "openai/gpt-4o-mini" \ - "" \ - "0" \ - "scan ok with isolated Strix working directory" \ - "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ - "" \ - "0" \ - "CRITICAL" \ - "0" \ - "" \ - "" \ - "1200" \ - "0" \ - "pull_request" \ - "backend/app/pg_introspect/introspect.py" - ;; - nvidia-overloaded-direct-fallback-success) - run_gate_case_allow_provider_signal "nvidia-overloaded-direct-fallback-success" \ - "nvidia_nim/nvidia/overloaded-primary" \ - "" \ - "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'nvidia_nim/nvidia/fallback-one' in [0-9]+s\\." \ - "3" \ - "nvidia_nim/nvidia/overloaded-primary|nvidia_nim/nvidia/overloaded-primary|nvidia_nim/nvidia/fallback-one" \ - "https://integrate.api.nvidia.com/v1|https://integrate.api.nvidia.com/v1|https://integrate.api.nvidia.com/v1" \ - "nvidia_nim" \ - "https://integrate.api.nvidia.com/v1" \ - "" \ - "1" \ - "CRITICAL" \ - "0" \ - "" \ - "" \ - "1200" \ - "0" \ - "" \ - "" \ - "" \ - "" \ - "0" \ - "" \ - "" \ - "" \ - "__SAME_AS_FALLBACK_MODELS__" \ - "nvidia_nim/nvidia/fallback-one openai-direct/gpt-5.4" - ;; - *) - record_failure "unknown STRIX_TEST_CASE_FILTER '${STRIX_TEST_CASE_FILTER:-}'" + *) + record_failure "unknown STRIX_TEST_CASE_FILTER '${STRIX_TEST_CASE_FILTER:-}'" ;; esac @@ -7018,6 +5408,7 @@ run_pull_request_target_head_scope_case() { local fake_strix="$bin_dir/strix" local output_log="$tmp_dir/output.log" local strix_llm_file="$tmp_dir/strix_llm.txt" + local llm_api_base_file="$tmp_dir/llm_api_base.txt" local llm_api_key_file="$tmp_dir/llm_api_key.txt" cat >"$fake_strix" <<'EOF' @@ -7076,7 +5467,8 @@ fi echo "scan ok with PR head content" EOF chmod +x "$fake_strix" - printf '%s' 'gemini/test-model' >"$strix_llm_file" + printf '%s' 'orchestrator/free' >"$strix_llm_file" + printf '%s' 'http://127.0.0.1:18080/v1' >"$llm_api_base_file" printf '%s' 'dummy' >"$llm_api_key_file" ( @@ -7136,6 +5528,7 @@ EOF FAKE_STRIX_EXPECT_FULL_HEAD_SCOPE="$expected_full_head_scope" \ STRIX_DISABLE_PR_SCOPING="$disable_pr_scoping" \ STRIX_LLM_FILE="$strix_llm_file" \ + LLM_API_BASE_FILE="$llm_api_base_file" \ LLM_API_KEY_FILE="$llm_api_key_file" \ STRIX_TARGET_PATH="$target_path" \ STRIX_REPORTS_DIR="$repo_root_dir/strix_runs" \ @@ -7167,6 +5560,7 @@ run_pull_request_target_plaintext_runner_token_fails_closed_case() { local output_log="$tmp_dir/output.log" local call_log="$tmp_dir/calls.log" local strix_llm_file="$tmp_dir/strix_llm.txt" + local llm_api_base_file="$tmp_dir/llm_api_base.txt" local llm_api_key_file="$tmp_dir/llm_api_key.txt" local changed_file="backend/db/models.py" @@ -7176,7 +5570,7 @@ set -euo pipefail printf '%s\n' "${STRIX_LLM:-}" >> "${FAKE_STRIX_CALL_LOG:?}" case "${STRIX_LLM:-}" in -vertex_ai/stale-source-primary) +openai/orchestrator/free) mkdir -p "${STRIX_REPORTS_DIR:?}/fake-pr-head-plaintext/vulnerabilities" cat >"$STRIX_REPORTS_DIR/fake-pr-head-plaintext/vulnerabilities/vuln-0001.md" <<'EOS' **Severity:** HIGH @@ -7188,10 +5582,6 @@ EOS echo "Penetration test failed: PR-head plaintext token finding" exit 1 ;; -vertex_ai/fallback-one) - echo "Error: PR-head plaintext findings must not reach fallback" >&2 - exit 31 - ;; *) echo "Error: unexpected model (${STRIX_LLM:-})" >&2 exit 32 @@ -7199,7 +5589,8 @@ vertex_ai/fallback-one) esac EOF chmod +x "$fake_strix" - printf '%s' 'vertex_ai/stale-source-primary' >"$strix_llm_file" + printf '%s' 'orchestrator/free' >"$strix_llm_file" + printf '%s' 'http://127.0.0.1:18080/v1' >"$llm_api_base_file" printf '%s' 'dummy' >"$llm_api_key_file" ( @@ -7256,6 +5647,7 @@ EOS STRIX_FAIL_ON_MIN_SEVERITY="HIGH" \ STRIX_DISABLE_PR_SCOPING="0" \ STRIX_LLM_FILE="$strix_llm_file" \ + LLM_API_BASE_FILE="$llm_api_base_file" \ LLM_API_KEY_FILE="$llm_api_key_file" \ STRIX_TARGET_PATH="." \ STRIX_REPORTS_DIR="$repo_root_dir/strix_runs" \ @@ -7288,6 +5680,7 @@ run_pull_request_target_bounded_head_context_scope_case() { local fake_strix="$bin_dir/strix" local output_log="$tmp_dir/output.log" local strix_llm_file="$tmp_dir/strix_llm.txt" + local llm_api_base_file="$tmp_dir/llm_api_base.txt" local llm_api_key_file="$tmp_dir/llm_api_key.txt" local changed_file="backend/api/emails.py" local context_file="backend/core/only_in_head.py" @@ -7320,7 +5713,8 @@ fi echo "scan ok with bounded PR head backend context" EOF chmod +x "$fake_strix" - printf '%s' 'gemini/test-model' >"$strix_llm_file" + printf '%s' 'orchestrator/free' >"$strix_llm_file" + printf '%s' 'http://127.0.0.1:18080/v1' >"$llm_api_base_file" printf '%s' 'dummy' >"$llm_api_key_file" ( @@ -7366,6 +5760,7 @@ EOF FAKE_STRIX_UNEXPECTED_BASE_CONTEXT="TRUSTED_BASE_CONTEXT_SHOULD_NOT_BE_SCANNED" \ STRIX_DISABLE_PR_SCOPING="0" \ STRIX_LLM_FILE="$strix_llm_file" \ + LLM_API_BASE_FILE="$llm_api_base_file" \ LLM_API_KEY_FILE="$llm_api_key_file" \ STRIX_TARGET_PATH="." \ STRIX_REPORTS_DIR="$repo_root_dir/strix_runs" \ @@ -7393,6 +5788,7 @@ run_pull_request_target_changed_context_scope_uses_pr_head_case() { local fake_strix="$bin_dir/strix" local output_log="$tmp_dir/output.log" local strix_llm_file="$tmp_dir/strix_llm.txt" + local llm_api_base_file="$tmp_dir/llm_api_base.txt" local llm_api_key_file="$tmp_dir/llm_api_key.txt" local state_file="$tmp_dir/state.log" local changed_file="backend/api/emails.py" @@ -7458,7 +5854,8 @@ echo "Error: unexpected changed context scan attempt $attempt" >&2 exit 71 EOF chmod +x "$fake_strix" - printf '%s' 'gemini/test-model' >"$strix_llm_file" + printf '%s' 'orchestrator/free' >"$strix_llm_file" + printf '%s' 'http://127.0.0.1:18080/v1' >"$llm_api_base_file" printf '%s' 'dummy' >"$llm_api_key_file" ( @@ -7509,6 +5906,7 @@ EOF FAKE_STRIX_STATE_FILE="$state_file" \ STRIX_DISABLE_PR_SCOPING="0" \ STRIX_LLM_FILE="$strix_llm_file" \ + LLM_API_BASE_FILE="$llm_api_base_file" \ LLM_API_KEY_FILE="$llm_api_key_file" \ STRIX_TARGET_PATH="." \ STRIX_REPORTS_DIR="$repo_root_dir/strix_runs" \ @@ -7545,6 +5943,7 @@ EOF FAKE_STRIX_STATE_FILE="$state_file" \ STRIX_DISABLE_PR_SCOPING="0" \ STRIX_LLM_FILE="$strix_llm_file" \ + LLM_API_BASE_FILE="$llm_api_base_file" \ LLM_API_KEY_FILE="$llm_api_key_file" \ STRIX_TARGET_PATH="." \ STRIX_REPORTS_DIR="$repo_root_dir/strix_runs" \ @@ -7573,6 +5972,7 @@ run_pull_request_target_changed_backend_context_scope_case() { local output_log="$tmp_dir/output.log" local call_log="$tmp_dir/calls.log" local strix_llm_file="$tmp_dir/strix_llm.txt" + local llm_api_base_file="$tmp_dir/llm_api_base.txt" local llm_api_key_file="$tmp_dir/llm_api_key.txt" cat >"$fake_strix" <<'EOF' @@ -7700,7 +6100,8 @@ fi echo "scan ok with non-email backend scope" EOF chmod +x "$fake_strix" - printf '%s' 'gemini/test-model' >"$strix_llm_file" + printf '%s' 'orchestrator/free' >"$strix_llm_file" + printf '%s' 'http://127.0.0.1:18080/v1' >"$llm_api_base_file" printf '%s' 'dummy' >"$llm_api_key_file" ( @@ -7796,6 +6197,7 @@ EOF STRIX_DISABLE_PR_SCOPING="0" \ FAKE_STRIX_CALL_LOG="$call_log" \ STRIX_LLM_FILE="$strix_llm_file" \ + LLM_API_BASE_FILE="$llm_api_base_file" \ LLM_API_KEY_FILE="$llm_api_key_file" \ STRIX_TARGET_PATH="." \ STRIX_REPORTS_DIR="$repo_root_dir/strix_runs" \ @@ -7831,6 +6233,7 @@ run_pull_request_target_frontend_email_context_scope_case() { local fake_strix="$bin_dir/strix" local output_log="$tmp_dir/output.log" local strix_llm_file="$tmp_dir/strix_llm.txt" + local llm_api_base_file="$tmp_dir/llm_api_base.txt" local llm_api_key_file="$tmp_dir/llm_api_key.txt" cat >"$fake_strix" <<'EOF' @@ -7941,7 +6344,8 @@ fi echo "scan ok with frontend email trusted backend authorization context" EOF chmod +x "$fake_strix" - printf '%s' 'gemini/test-model' >"$strix_llm_file" + printf '%s' 'orchestrator/free' >"$strix_llm_file" + printf '%s' 'http://127.0.0.1:18080/v1' >"$llm_api_base_file" printf '%s' 'dummy' >"$llm_api_key_file" ( @@ -7992,6 +6396,7 @@ EOF STRIX_DISABLE_PR_SCOPING="0" \ FAKE_STRIX_EXPECTED_CHANGED_FILE="$changed_file" \ STRIX_LLM_FILE="$strix_llm_file" \ + LLM_API_BASE_FILE="$llm_api_base_file" \ LLM_API_KEY_FILE="$llm_api_key_file" \ STRIX_TARGET_PATH="." \ STRIX_REPORTS_DIR="$repo_root_dir/strix_runs" \ @@ -8021,6 +6426,7 @@ run_pull_request_target_shallow_head_merge_base_fallback_case() { local fake_strix="$bin_dir/strix" local output_log="$tmp_dir/output.log" local strix_llm_file="$tmp_dir/strix_llm.txt" + local llm_api_base_file="$tmp_dir/llm_api_base.txt" local llm_api_key_file="$tmp_dir/llm_api_key.txt" cat >"$fake_strix" <<'EOF' @@ -8030,7 +6436,8 @@ echo "scan ok" exit 0 EOF chmod +x "$fake_strix" - printf '%s' 'gemini/test-model' >"$strix_llm_file" + printf '%s' 'orchestrator/free' >"$strix_llm_file" + printf '%s' 'http://127.0.0.1:18080/v1' >"$llm_api_base_file" printf '%s' 'dummy' >"$llm_api_key_file" ( @@ -8087,6 +6494,7 @@ EOF PR_BASE_SHA="$base_sha" \ PR_HEAD_SHA="$head_sha" \ STRIX_LLM_FILE="$strix_llm_file" \ + LLM_API_BASE_FILE="$llm_api_base_file" \ LLM_API_KEY_FILE="$llm_api_key_file" \ STRIX_TARGET_PATH="." \ STRIX_REPORTS_DIR="$repo_root_dir/strix_runs" \ @@ -8171,6 +6579,7 @@ EOF local call_log="$tmp_dir/calls.log" local output_log="$tmp_dir/output.log" local strix_llm_file="$tmp_dir/strix_llm.txt" + local llm_api_base_file="$tmp_dir/llm_api_base.txt" local llm_api_key_file="$tmp_dir/llm_api_key.txt" cat >"$fake_strix" <<'EOF' @@ -8181,7 +6590,8 @@ echo "Error: Strix should not run after a PR-head blob failure" >&2 exit 64 EOF chmod +x "$fake_strix" - printf '%s' 'gemini/test-model' >"$strix_llm_file" + printf '%s' 'orchestrator/free' >"$strix_llm_file" + printf '%s' 'http://127.0.0.1:18080/v1' >"$llm_api_base_file" printf '%s' 'dummy' >"$llm_api_key_file" ( @@ -8225,6 +6635,7 @@ EOF FAKE_STRIX_CALL_LOG="$call_log" \ STRIX_DISABLE_PR_SCOPING="$disable_pr_scoping" \ STRIX_LLM_FILE="$strix_llm_file" \ + LLM_API_BASE_FILE="$llm_api_base_file" \ LLM_API_KEY_FILE="$llm_api_key_file" \ STRIX_TARGET_PATH="." \ STRIX_REPORTS_DIR="$repo_root_dir/strix_runs" \ @@ -8261,6 +6672,7 @@ run_pull_request_target_rejects_invalid_sha_case() { local call_log="$tmp_dir/calls.log" local output_log="$tmp_dir/output.log" local strix_llm_file="$tmp_dir/strix_llm.txt" + local llm_api_base_file="$tmp_dir/llm_api_base.txt" local llm_api_key_file="$tmp_dir/llm_api_key.txt" cat >"$fake_strix" <<'EOF' @@ -8271,7 +6683,8 @@ echo "Error: Strix should not run after invalid pull request SHA metadata" >&2 exit 67 EOF chmod +x "$fake_strix" - printf '%s' 'gemini/test-model' >"$strix_llm_file" + printf '%s' 'orchestrator/free' >"$strix_llm_file" + printf '%s' 'http://127.0.0.1:18080/v1' >"$llm_api_base_file" printf '%s' 'dummy' >"$llm_api_key_file" ( @@ -8317,6 +6730,7 @@ EOF FAKE_STRIX_CALL_LOG="$call_log" \ STRIX_DISABLE_PR_SCOPING="0" \ STRIX_LLM_FILE="$strix_llm_file" \ + LLM_API_BASE_FILE="$llm_api_base_file" \ LLM_API_KEY_FILE="$llm_api_key_file" \ STRIX_TARGET_PATH="." \ STRIX_REPORTS_DIR="$repo_root_dir/strix_runs" \ @@ -8354,6 +6768,7 @@ run_pull_request_target_irregular_head_entry_fails_closed_case() { local call_log="$tmp_dir/calls.log" local output_log="$tmp_dir/output.log" local strix_llm_file="$tmp_dir/strix_llm.txt" + local llm_api_base_file="$tmp_dir/llm_api_base.txt" local llm_api_key_file="$tmp_dir/llm_api_key.txt" cat >"$fake_strix" <<'EOF' @@ -8364,7 +6779,8 @@ echo "Error: Strix should not run after an irregular PR-head entry" >&2 exit 66 EOF chmod +x "$fake_strix" - printf '%s' 'gemini/test-model' >"$strix_llm_file" + printf '%s' 'orchestrator/free' >"$strix_llm_file" + printf '%s' 'http://127.0.0.1:18080/v1' >"$llm_api_base_file" printf '%s' 'dummy' >"$llm_api_key_file" ( @@ -8404,6 +6820,7 @@ EOF FAKE_STRIX_CALL_LOG="$call_log" \ STRIX_DISABLE_PR_SCOPING="0" \ STRIX_LLM_FILE="$strix_llm_file" \ + LLM_API_BASE_FILE="$llm_api_base_file" \ LLM_API_KEY_FILE="$llm_api_key_file" \ STRIX_TARGET_PATH="." \ STRIX_REPORTS_DIR="$repo_root_dir/strix_runs" \ @@ -8437,6 +6854,7 @@ run_pull_request_target_gitlink_is_explicitly_skipped_case() { local call_log="$tmp_dir/calls.log" local output_log="$tmp_dir/output.log" local strix_llm_file="$tmp_dir/strix_llm.txt" + local llm_api_base_file="$tmp_dir/llm_api_base.txt" local llm_api_key_file="$tmp_dir/llm_api_key.txt" cat >"$fake_strix" <<'EOF' #!/usr/bin/env bash @@ -8445,7 +6863,8 @@ printf 'called\n' >> "${FAKE_STRIX_CALL_LOG:?}" exit 66 EOF chmod +x "$fake_strix" - printf '%s' 'gemini/test-model' >"$strix_llm_file" + printf '%s' 'orchestrator/free' >"$strix_llm_file" + printf '%s' 'http://127.0.0.1:18080/v1' >"$llm_api_base_file" printf '%s' 'dummy' >"$llm_api_key_file" ( @@ -8478,6 +6897,7 @@ EOF FAKE_STRIX_CALL_LOG="$call_log" \ STRIX_DISABLE_PR_SCOPING="0" \ STRIX_LLM_FILE="$strix_llm_file" \ + LLM_API_BASE_FILE="$llm_api_base_file" \ LLM_API_KEY_FILE="$llm_api_key_file" \ STRIX_TARGET_PATH="." \ STRIX_REPORTS_DIR="$repo_root_dir/strix_runs" \ @@ -8518,6 +6938,7 @@ run_full_head_scope_skips_gitlink_case() { local fake_strix="$bin_dir/strix" local output_log="$tmp_dir/output.log" local strix_llm_file="$tmp_dir/strix_llm.txt" + local llm_api_base_file="$tmp_dir/llm_api_base.txt" local llm_api_key_file="$tmp_dir/llm_api_key.txt" # The full-head scope must materialize the changed Dockerfile and the # unchanged docs context, and must never materialize the gitlink as a path. @@ -8549,7 +6970,8 @@ fi echo "scan ok with PR head content" EOF chmod +x "$fake_strix" - printf '%s' 'gemini/test-model' >"$strix_llm_file" + printf '%s' 'orchestrator/free' >"$strix_llm_file" + printf '%s' 'http://127.0.0.1:18080/v1' >"$llm_api_base_file" printf '%s' 'dummy' >"$llm_api_key_file" ( @@ -8601,6 +7023,7 @@ EOF STRIX_TEST_CHANGED_FILES_OVERRIDE="Dockerfile" \ STRIX_DISABLE_PR_SCOPING="0" \ STRIX_LLM_FILE="$strix_llm_file" \ + LLM_API_BASE_FILE="$llm_api_base_file" \ LLM_API_KEY_FILE="$llm_api_key_file" \ STRIX_TARGET_PATH="." \ STRIX_REPORTS_DIR="$repo_root_dir/strix_runs" \ @@ -8633,6 +7056,7 @@ run_pull_request_target_rejects_unsafe_changed_path_case() { local call_log="$tmp_dir/calls.log" local output_log="$tmp_dir/output.log" local strix_llm_file="$tmp_dir/strix_llm.txt" + local llm_api_base_file="$tmp_dir/llm_api_base.txt" local llm_api_key_file="$tmp_dir/llm_api_key.txt" local event_payload_file="$tmp_dir/github_event.json" @@ -8644,7 +7068,8 @@ echo "Error: Strix should not run for unsafe changed paths" >&2 exit 65 EOF chmod +x "$fake_strix" - printf '%s' 'gemini/test-model' >"$strix_llm_file" + printf '%s' 'orchestrator/free' >"$strix_llm_file" + printf '%s' 'http://127.0.0.1:18080/v1' >"$llm_api_base_file" printf '%s' 'dummy' >"$llm_api_key_file" cat >"$event_payload_file" <<'EOF' { @@ -8668,6 +7093,7 @@ EOF FAKE_STRIX_CALL_LOG="$call_log" \ STRIX_DISABLE_PR_SCOPING="0" \ STRIX_LLM_FILE="$strix_llm_file" \ + LLM_API_BASE_FILE="$llm_api_base_file" \ LLM_API_KEY_FILE="$llm_api_key_file" \ STRIX_TARGET_PATH="." \ STRIX_REPORTS_DIR="$repo_root_dir/strix_runs" \ @@ -8724,6 +7150,7 @@ run_timeout_cleanup_case() { local child_pid_file="$tmp_dir/child.pid" local output_log="$tmp_dir/output.log" local strix_llm_file="$tmp_dir/strix_llm.txt" + local llm_api_base_file="$tmp_dir/llm_api_base.txt" local llm_api_key_file="$tmp_dir/llm_api_key.txt" cat >"$fake_strix" <<'EOF' @@ -8736,7 +7163,8 @@ printf '%s' "$child_pid" > "${FAKE_STRIX_CHILD_PID_FILE:?}" sleep "${FAKE_STRIX_TIMEOUT_SLEEP_SECONDS:?}" EOF chmod +x "$fake_strix" - printf '%s' 'vertex_ai/timeout-cleanup-primary' >"$strix_llm_file" + printf '%s' 'orchestrator/free' >"$strix_llm_file" + printf '%s' 'http://127.0.0.1:18080/v1' >"$llm_api_base_file" printf '%s' 'dummy' >"$llm_api_key_file" set +e @@ -8750,6 +7178,7 @@ EOF FAKE_STRIX_CHILD_PID_FILE="$child_pid_file" \ FAKE_STRIX_TIMEOUT_SLEEP_SECONDS="$TIMEOUT_TEST_FAKE_SLEEP_SECONDS" \ STRIX_LLM_FILE="$strix_llm_file" \ + LLM_API_BASE_FILE="$llm_api_base_file" \ LLM_API_KEY_FILE="$llm_api_key_file" \ STRIX_PROCESS_TIMEOUT_SECONDS="$TIMEOUT_TEST_PROCESS_SECONDS" \ STRIX_VERTEX_FALLBACK_MODELS="" \ @@ -8815,9 +7244,9 @@ echo "vertex scan ok without external LLM_API_BASE" exit 0 EOF chmod +x "$fake_strix" - printf '%s' 'vertex_ai/gemini-2.5-pro' >"$strix_llm_file" + printf '%s' 'orchestrator/free' >"$strix_llm_file" printf '%s' 'dummy' >"$llm_api_key_file" - printf '%s' 'https://example.invalid/generateContent' >"$llm_api_base_file" + printf '%s' 'http://127.0.0.1:18080/v1' >"$llm_api_base_file" set +e ( @@ -8830,6 +7259,7 @@ EOF FAKE_STRIX_CALL_LOG="$call_log" \ STRIX_DISABLE_PR_SCOPING="0" \ STRIX_LLM_FILE="$strix_llm_file" \ + LLM_API_BASE_FILE="$llm_api_base_file" \ LLM_API_KEY_FILE="$llm_api_key_file" \ LLM_API_BASE_FILE="$llm_api_base_file" \ bash "./scripts/ci/strix_quick_gate.sh" >"$output_log" 2>&1 @@ -8837,9 +7267,9 @@ EOF local rc=$? set -e - assert_equals "0" "$rc" "case=vertex-ignores-untrusted-llm-api-base-file exit code" - assert_file_contains "$output_log" "vertex scan ok without external LLM_API_BASE" "case=vertex-ignores-untrusted-llm-api-base-file output" - assert_file_contains "$call_log" "called" "case=vertex-ignores-untrusted-llm-api-base-file strix invocation" + assert_equals "2" "$rc" "case=untrusted-llm-api-base-file-refused exit code" + assert_file_contains "$output_log" "LLM_API_BASE_FILE must be inside the trusted input file root." "case=untrusted-llm-api-base-file-refused output" + assert_file_not_contains "$call_log" "called" "case=untrusted-llm-api-base-file-refused does not invoke strix" rm -rf "$tmp_dir" } @@ -8858,6 +7288,7 @@ run_total_timeout_case() { local output_log="$tmp_dir/output.log" local call_count_file="$tmp_dir/calls.log" local strix_llm_file="$tmp_dir/strix_llm.txt" + local llm_api_base_file="$tmp_dir/llm_api_base.txt" local llm_api_key_file="$tmp_dir/llm_api_key.txt" cat >"$fake_strix" <<'EOF' @@ -8868,7 +7299,8 @@ echo "1" >> "${FAKE_STRIX_CALL_COUNT_FILE:?}" sleep 30 EOF chmod +x "$fake_strix" - printf '%s' 'vertex_ai/total-timeout-primary' >"$strix_llm_file" + printf '%s' 'orchestrator/free' >"$strix_llm_file" + printf '%s' 'http://127.0.0.1:18080/v1' >"$llm_api_base_file" printf '%s' 'dummy' >"$llm_api_key_file" set +e @@ -8881,6 +7313,7 @@ EOF STRIX_DISABLE_PR_SCOPING="0" \ FAKE_STRIX_CALL_COUNT_FILE="$call_count_file" \ STRIX_LLM_FILE="$strix_llm_file" \ + LLM_API_BASE_FILE="$llm_api_base_file" \ LLM_API_KEY_FILE="$llm_api_key_file" \ STRIX_PROCESS_TIMEOUT_SECONDS="30" \ STRIX_TOTAL_TIMEOUT_SECONDS="8" \ @@ -8930,6 +7363,7 @@ run_missing_config_case() { local call_count_file="$tmp_dir/strix_calls" local fake_strix="$tmp_dir/strix" local strix_llm_file="$tmp_dir/strix_llm.txt" + local llm_api_base_file="$tmp_dir/llm_api_base.txt" local llm_api_key_file="$tmp_dir/llm_api_key.txt" cat >"$fake_strix" <<'EOF' @@ -8953,6 +7387,7 @@ EOF STRIX_INPUT_FILE_ROOT="$tmp_dir" \ STRIX_DISABLE_PR_SCOPING="0" \ STRIX_LLM_FILE="$strix_llm_file" \ + LLM_API_BASE_FILE="$llm_api_base_file" \ LLM_API_KEY_FILE="$llm_api_key_file" \ STRIX_CALL_COUNT_FILE="$call_count_file" \ bash "$GATE_SCRIPT" >"$output_log" 2>&1 @@ -8979,6 +7414,7 @@ run_strix_llm_file_command_substitution_literal_case() { local marker_file="$tmp_dir/strix_marker" local fake_strix="$tmp_dir/strix" local strix_llm_file="$tmp_dir/strix_llm.txt" + local llm_api_base_file="$tmp_dir/llm_api_base.txt" local llm_api_key_file="$tmp_dir/llm_api_key.txt" cat >"$fake_strix" <<'EOF' @@ -8999,6 +7435,7 @@ EOF STRIX_TARGET_PATH="-" \ STRIX_DISABLE_PR_SCOPING="0" \ STRIX_LLM_FILE="$strix_llm_file" \ + LLM_API_BASE_FILE="$llm_api_base_file" \ LLM_API_KEY_FILE="$llm_api_key_file" \ STRIX_CALL_COUNT_FILE="$call_count_file" \ bash "$GATE_SCRIPT" >"$output_log" 2>&1 @@ -9006,7 +7443,7 @@ EOF set -e assert_equals "2" "$rc" "case=strix-llm-file-command-substitution-literal exit code" - assert_file_contains "$output_log" "ERROR: STRIX_TARGET_PATH contains unsupported path syntax" "case=strix-llm-file-command-substitution-literal output" + assert_file_contains "$output_log" "ERROR: Strix model must be orchestrator/free through contextual-orchestrator; direct provider/model routes are forbidden" "case=strix-llm-file-command-substitution-literal output" if [ -e "$marker_file" ]; then record_failure "case=strix-llm-file-command-substitution-literal must not execute model file content" fi @@ -9027,6 +7464,7 @@ run_vertex_without_llm_api_key_case() { local call_count_file="$tmp_dir/strix_calls" local fake_strix="$tmp_dir/strix" local strix_llm_file="$tmp_dir/strix_llm.txt" + local llm_api_base_file="$tmp_dir/llm_api_base.txt" cat >"$fake_strix" <<'EOF' #!/usr/bin/env bash @@ -9043,7 +7481,8 @@ fi exit 0 EOF chmod +x "$fake_strix" - printf '%s' "vertex_ai/ready-primary" >"$strix_llm_file" + printf '%s' 'orchestrator/free' >"$strix_llm_file" + printf '%s' 'http://127.0.0.1:18080/v1' >"$llm_api_base_file" set +e env -u GITHUB_EVENT_NAME -u GITHUB_EVENT_PATH -u STRIX_TEST_CHANGED_FILES_OVERRIDE \ @@ -9052,19 +7491,20 @@ EOF STRIX_INPUT_FILE_ROOT="$tmp_dir" \ STRIX_DISABLE_PR_SCOPING="0" \ STRIX_LLM_FILE="$strix_llm_file" \ + LLM_API_BASE_FILE="$llm_api_base_file" \ FAKE_STRIX_CALL_COUNT_FILE="$call_count_file" \ bash "$GATE_SCRIPT" >"$output_log" 2>&1 local rc=$? set -e - assert_equals "0" "$rc" "case=vertex-without-llm-api-key exit code" - assert_file_contains "$output_log" "Strix run succeeded for model 'vertex_ai/ready-primary'" "case=vertex-without-llm-api-key output" + assert_equals "2" "$rc" "case=gateway-model-without-llm-api-key exit code" + assert_file_contains "$output_log" "LLM_API_KEY_FILE must reference a regular file containing the API key." "case=gateway-model-without-llm-api-key output" local actual_calls="0" if [ -f "$call_count_file" ]; then actual_calls="$(wc -l <"$call_count_file" | tr -d ' ')" fi - assert_equals "1" "$actual_calls" "case=vertex-without-llm-api-key strix call count" + assert_equals "0" "$actual_calls" "case=gateway-model-without-llm-api-key strix call count" rm -rf "$tmp_dir" } @@ -9076,6 +7516,7 @@ run_vertex_with_llm_api_key_file_does_not_forward_case() { local call_count_file="$tmp_dir/strix_calls" local fake_strix="$tmp_dir/strix" local strix_llm_file="$tmp_dir/strix_llm.txt" + local llm_api_base_file="$tmp_dir/llm_api_base.txt" local llm_api_key_file="$tmp_dir/llm_api_key.txt" cat >"$fake_strix" <<'EOF' @@ -9093,7 +7534,8 @@ fi exit 0 EOF chmod +x "$fake_strix" - printf '%s' "vertex_ai/ready-primary" >"$strix_llm_file" + printf '%s' 'vertex_ai/ready-primary' >"$strix_llm_file" + printf '%s' 'http://127.0.0.1:18080/v1' >"$llm_api_base_file" printf '%s' "openai-key-should-not-reach-vertex" >"$llm_api_key_file" set +e @@ -9103,20 +7545,21 @@ EOF STRIX_INPUT_FILE_ROOT="$tmp_dir" \ STRIX_DISABLE_PR_SCOPING="0" \ STRIX_LLM_FILE="$strix_llm_file" \ + LLM_API_BASE_FILE="$llm_api_base_file" \ LLM_API_KEY_FILE="$llm_api_key_file" \ FAKE_STRIX_CALL_COUNT_FILE="$call_count_file" \ bash "$GATE_SCRIPT" >"$output_log" 2>&1 local rc=$? set -e - assert_equals "0" "$rc" "case=vertex-with-llm-api-key-file-not-forwarded exit code" - assert_file_contains "$output_log" "Strix run succeeded for model 'vertex_ai/ready-primary'" "case=vertex-with-llm-api-key-file-not-forwarded output" + assert_equals "2" "$rc" "case=vertex-route-rejected-before-credentials exit code" + assert_file_contains "$output_log" "ERROR: Strix model must be orchestrator/free through contextual-orchestrator; direct provider/model routes are forbidden" "case=vertex-route-rejected-before-credentials output" local actual_calls="0" if [ -f "$call_count_file" ]; then actual_calls="$(wc -l <"$call_count_file" | tr -d ' ')" fi - assert_equals "1" "$actual_calls" "case=vertex-with-llm-api-key-file-not-forwarded strix call count" + assert_equals "0" "$actual_calls" "case=vertex-route-rejected-before-credentials strix call count" rm -rf "$tmp_dir" } @@ -9127,6 +7570,7 @@ run_invalid_min_fail_severity_case() { local output_log="$tmp_dir/output.log" local fake_strix="$tmp_dir/strix" local strix_llm_file="$tmp_dir/strix_llm.txt" + local llm_api_base_file="$tmp_dir/llm_api_base.txt" local llm_api_key_file="$tmp_dir/llm_api_key.txt" cat >"$fake_strix" <<'EOF' @@ -9136,7 +7580,8 @@ echo "unexpected strix execution" >&2 exit 99 EOF chmod +x "$fake_strix" - printf '%s' 'vertex_ai/ready-primary' >"$strix_llm_file" + printf '%s' 'orchestrator/free' >"$strix_llm_file" + printf '%s' 'http://127.0.0.1:18080/v1' >"$llm_api_base_file" printf '%s' 'dummy' >"$llm_api_key_file" set +e @@ -9146,6 +7591,7 @@ EOF STRIX_INPUT_FILE_ROOT="$tmp_dir" \ STRIX_DISABLE_PR_SCOPING="0" \ STRIX_LLM_FILE="$strix_llm_file" \ + LLM_API_BASE_FILE="$llm_api_base_file" \ LLM_API_KEY_FILE="$llm_api_key_file" \ STRIX_FAIL_ON_MIN_SEVERITY="BOGUS" \ bash "$GATE_SCRIPT" >"$output_log" 2>&1 @@ -9189,9 +7635,9 @@ printf 'called\n' >"${FAKE_STRIX_CALL_LOG:?}" exit 0 EOF chmod +x "$fake_strix" - printf '%s' 'openai/gpt-4o-mini' >"$strix_llm_file" + printf '%s' 'orchestrator/free' >"$strix_llm_file" printf '%s' 'dummy' >"$llm_api_key_file" - printf '%s' 'https://example.invalid/generateContent' >"$llm_api_base_file" + printf '%s' 'http://127.0.0.1:18080/v1' >"$llm_api_base_file" set +e ( @@ -9203,6 +7649,7 @@ EOF FAKE_STRIX_CALL_LOG="$call_log" \ STRIX_DISABLE_PR_SCOPING="0" \ STRIX_LLM_FILE="$strix_llm_file" \ + LLM_API_BASE_FILE="$llm_api_base_file" \ LLM_API_KEY_FILE="$llm_api_key_file" \ LLM_API_BASE_FILE="$llm_api_base_file" \ bash "./scripts/ci/strix_quick_gate.sh" >"$output_log" 2>&1 @@ -9246,9 +7693,9 @@ printf 'called\n' >"${FAKE_STRIX_CALL_LOG:?}" exit 0 EOF chmod +x "$fake_strix" - printf '%s' 'openai/gpt-4o-mini' >"$strix_llm_file" + printf '%s' 'orchestrator/free' >"$strix_llm_file" printf '%s' 'dummy' >"$llm_api_key_file" - printf '%s' 'https://example.invalid/generateContent' >"$llm_api_base_file" + printf '%s' 'http://127.0.0.1:18080/v1' >"$llm_api_base_file" set +e ( @@ -9262,6 +7709,7 @@ EOF FAKE_STRIX_CALL_LOG="$call_log" \ STRIX_DISABLE_PR_SCOPING="0" \ STRIX_LLM_FILE="$strix_llm_file" \ + LLM_API_BASE_FILE="$llm_api_base_file" \ LLM_API_KEY_FILE="$llm_api_key_file" \ LLM_API_BASE_FILE="$llm_api_base_file" \ bash "./scripts/ci/strix_quick_gate.sh" >"$output_log" 2>&1 @@ -9305,9 +7753,9 @@ printf 'called\n' >"${FAKE_STRIX_CALL_LOG:?}" exit 0 EOF chmod +x "$fake_strix" - printf '%s' 'openai/gpt-4o-mini' >"$strix_llm_file" + printf '%s' 'orchestrator/free' >"$strix_llm_file" printf '%s' 'dummy' >"$llm_api_key_file" - printf '%s' 'https://example.invalid/generateContent' >"$llm_api_base_file" + printf '%s' 'http://127.0.0.1:18080/v1' >"$llm_api_base_file" case "$file_env" in STRIX_LLM_FILE) printf '%s' 'openai/gpt-4o-mini' >"$outside_file" @@ -9334,6 +7782,7 @@ EOF FAKE_STRIX_CALL_LOG="$call_log" \ STRIX_DISABLE_PR_SCOPING="0" \ STRIX_LLM_FILE="$strix_llm_file" \ + LLM_API_BASE_FILE="$llm_api_base_file" \ LLM_API_KEY_FILE="$llm_api_key_file" \ LLM_API_BASE_FILE="$llm_api_base_file" \ bash "./scripts/ci/strix_quick_gate.sh" >"$output_log" 2>&1 @@ -9375,9 +7824,9 @@ printf 'called\n' >"${FAKE_STRIX_CALL_LOG:?}" exit 0 EOF chmod +x "$fake_strix" - printf '%s' 'openai/gpt-4o-mini' >"$strix_llm_file" + printf '%s' 'orchestrator/free' >"$strix_llm_file" printf '%s' 'dummy' >"$llm_api_key_file" - printf '%s' 'https://example.invalid/generateContent' >"$llm_api_base_file" + printf '%s' 'http://127.0.0.1:18080/v1' >"$llm_api_base_file" set +e ( @@ -9390,6 +7839,7 @@ EOF FAKE_STRIX_CALL_LOG="$call_log" \ STRIX_DISABLE_PR_SCOPING="0" \ STRIX_LLM_FILE="$strix_llm_file" \ + LLM_API_BASE_FILE="$llm_api_base_file" \ LLM_API_KEY_FILE="$llm_api_key_file" \ LLM_API_BASE_FILE="$llm_api_base_file" \ bash "./scripts/ci/strix_quick_gate.sh" >"$output_log" 2>&1 @@ -9434,9 +7884,9 @@ echo "Error: transport timeout" exit 1 EOF chmod +x "$fake_strix" - printf '%s' 'openai/gpt-4o-mini' >"$strix_llm_file" + printf '%s' 'orchestrator/free' >"$strix_llm_file" printf '%s' 'dummy' >"$llm_api_key_file" - printf '%s' 'https://example.invalid/generateContent' >"$llm_api_base_file" + printf '%s' 'http://127.0.0.1:18080/v1' >"$llm_api_base_file" set +e ( @@ -9447,6 +7897,7 @@ EOF STRIX_INPUT_FILE_ROOT="$tmp_dir" \ STRIX_DISABLE_PR_SCOPING="0" \ STRIX_LLM_FILE="$strix_llm_file" \ + LLM_API_BASE_FILE="$llm_api_base_file" \ LLM_API_KEY_FILE="$llm_api_key_file" \ LLM_API_BASE_FILE="$llm_api_base_file" \ STRIX_REPORTS_DIR="strix_runs" \ @@ -9456,7 +7907,7 @@ EOF set -e assert_equals "1" "$rc" "case=stale-report-does-not-bypass exit code" - assert_file_contains "$output_log" "Strix quick scan failed with a non-recoverable error." "case=stale-report-does-not-bypass output" + assert_file_contains "$output_log" "Strix quick scan failed; failing closed without repository-authored retry or fallback allocation." "case=stale-report-does-not-bypass output" rm -rf "$tmp_dir" } @@ -9490,9 +7941,9 @@ echo "Error: transport timeout" exit 1 EOF chmod +x "$fake_strix" - printf '%s' 'openai/gpt-4o-mini' >"$strix_llm_file" + printf '%s' 'orchestrator/free' >"$strix_llm_file" printf '%s' 'dummy' >"$llm_api_key_file" - printf '%s' 'https://example.invalid/generateContent' >"$llm_api_base_file" + printf '%s' 'http://127.0.0.1:18080/v1' >"$llm_api_base_file" set +e ( @@ -9503,6 +7954,7 @@ EOF STRIX_INPUT_FILE_ROOT="$tmp_dir" \ STRIX_DISABLE_PR_SCOPING="0" \ STRIX_LLM_FILE="$strix_llm_file" \ + LLM_API_BASE_FILE="$llm_api_base_file" \ LLM_API_KEY_FILE="$llm_api_key_file" \ LLM_API_BASE_FILE="$llm_api_base_file" \ STRIX_REPORTS_DIR="strix_runs" \ @@ -9512,7 +7964,7 @@ EOF set -e assert_equals "1" "$rc" "case=symlink-report-does-not-bypass exit code" - assert_file_contains "$output_log" "Strix quick scan failed with a non-recoverable error." "case=symlink-report-does-not-bypass output" + assert_file_contains "$output_log" "Strix quick scan failed; failing closed without repository-authored retry or fallback allocation." "case=symlink-report-does-not-bypass output" rm -rf "$tmp_dir" } @@ -9540,9 +7992,9 @@ printf '%s\n' called >>"${FAKE_STRIX_CALL_LOG:?}" exit 0 EOF chmod +x "$fake_strix" - printf '%s' 'openai/gpt-4o-mini' >"$strix_llm_file" + printf '%s' 'orchestrator/free' >"$strix_llm_file" printf '%s' 'dummy' >"$llm_api_key_file" - printf '%s' 'https://example.invalid/generateContent' >"$llm_api_base_file" + printf '%s' 'http://127.0.0.1:18080/v1' >"$llm_api_base_file" set +e ( @@ -9554,6 +8006,7 @@ EOF STRIX_DISABLE_PR_SCOPING="0" \ FAKE_STRIX_CALL_LOG="$call_log" \ STRIX_LLM_FILE="$strix_llm_file" \ + LLM_API_BASE_FILE="$llm_api_base_file" \ LLM_API_KEY_FILE="$llm_api_key_file" \ LLM_API_BASE_FILE="$llm_api_base_file" \ STRIX_TARGET_PATH="../../../../../etc/passwd" \ @@ -9593,9 +8046,9 @@ printf 'called\n' >"${FAKE_STRIX_CALL_LOG:?}" exit 0 EOF chmod +x "$fake_strix" - printf '%s' 'openai/gpt-4o-mini' >"$strix_llm_file" + printf '%s' 'orchestrator/free' >"$strix_llm_file" printf '%s' 'dummy' >"$llm_api_key_file" - printf '%s' 'https://example.invalid/generateContent' >"$llm_api_base_file" + printf '%s' 'http://127.0.0.1:18080/v1' >"$llm_api_base_file" set +e ( @@ -9606,6 +8059,7 @@ EOF STRIX_INPUT_FILE_ROOT="$tmp_dir" \ FAKE_STRIX_CALL_LOG="$call_log" \ STRIX_LLM_FILE="$strix_llm_file" \ + LLM_API_BASE_FILE="$llm_api_base_file" \ LLM_API_KEY_FILE="$llm_api_key_file" \ LLM_API_BASE_FILE="$llm_api_base_file" \ STRIX_TARGET_PATH="$tmp_dir/strix-pr-scope.attacker" \ @@ -9878,956 +8332,294 @@ run_pull_request_target_rejects_invalid_sha_case \ "base" run_pull_request_target_rejects_invalid_sha_case \ - "pull-request-target-invalid-head-sha-fails-closed" \ - "head" - -run_pull_request_target_aborts_on_pr_head_blob_failure_case \ - "pull-request-target-disabled-pr-scope-pr-head-blob-read-failure" \ - "src/existing.py" \ - "BASE_CONTENT_MUST_NOT_BE_USED_AFTER_DISABLED_SCOPE_HEAD_FAILURE" \ - "HEAD_CONTENT_SHOULD_NOT_BECOME_PARTIAL_SCAN_INPUT" \ - "cat-file" \ - "1" - -run_gate_case "success" \ - "vertex_ai/ready-primary" \ - "vertex_ai/fallback-one vertex_ai/fallback-two" \ - "0" \ - "scan ok" \ - "1" \ - "vertex_ai/ready-primary" \ - "" - -run_gate_case "contextual-orchestrator-missing-api-base-fails-closed" \ - "orchestrator/free" \ - "" \ - "2" \ - "require LLM_API_BASE_FILE to select the pinned loopback gateway" \ - "0" \ - "" \ - "" \ - "contextual_orchestrator" \ - "" - -run_gate_case "contextual-orchestrator-gateway-model-qualification" \ - "orchestrator/free" \ - "" \ - "0" \ - "scan ok through contextual-orchestrator gateway" \ - "1" \ - "openai/orchestrator/free" \ - "http://127.0.0.1:18080/v1" \ - "contextual_orchestrator" \ - "http://127.0.0.1:18080/v1" - -run_gate_case "success-with-critical-report" \ - "vertex_ai/ready-primary" \ - "" \ - "1" \ - "Strix exited successfully but emitted a vulnerability at or above 'CRITICAL'" \ - "1" \ - "vertex_ai/ready-primary" \ - "" - -run_gate_case "pr-executable-integrity-mismatch" \ - "vertex_ai/ready-primary" \ - "" \ - "1" \ - "did not match the pinned SHA-256 digest" \ - "0" \ - "" \ - "" - -run_gate_case "pr-executable-group-writable" \ - "vertex_ai/ready-primary" \ - "" \ - "1" \ - "must not be group/world writable" \ - "0" \ - "" \ - "" - -run_gate_case "pr-executable-root-group-writable" \ - "vertex_ai/ready-primary" \ - "" \ - "1" \ - "pinned Strix installation root must not be group/world writable" \ - "0" \ - "" \ - "" - -run_gate_case "runtime-env-forwarding" \ - "gemini/gemini-pro-3.1-preview" \ - "" \ - "0" \ - "scan ok" \ - "1" \ - "gemini/gemini-pro-3.1-preview" \ - "" \ - "gemini" \ - "" - -run_gate_case "vertex-primary-notfound-fallback-success" \ - "vertex_ai/missing-primary" \ - "vertex_ai/fallback-one vertex_ai/fallback-two" \ - "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'vertex_ai/fallback-one' in [0-9]+s\\." \ - "2" \ - "vertex_ai/missing-primary|vertex_ai/fallback-one" \ - "|" - -run_gate_case "vertex-all-notfound" \ - "vertex_ai/missing-primary" \ - "vertex_ai/fallback-one vertex_ai/fallback-two" \ - "1" \ - "Configured Vertex model and fallback models were unavailable." \ - "3" \ - "vertex_ai/missing-primary|vertex_ai/fallback-one|vertex_ai/fallback-two" \ - "||" - -run_gate_case "nonrecoverable" \ - "openai/gpt-4o-mini" \ - "vertex_ai/fallback-one" \ - "1" \ - "Strix quick scan failed with a non-recoverable error." \ - "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" - -run_gate_case "provider-prefix-required" \ - "gemini-2.5-pro" \ - "vertex_ai/fallback-one" \ - "0" \ - "Normalized STRIX_LLM to provider-qualified model 'vertex_ai/gemini-2.5-pro'." \ - "1" \ - "vertex_ai/gemini-2.5-pro" \ - "" - -run_gate_case "provider-prefix-fallback-normalization" \ - "missing-primary" \ - "fallback-one fallback-two" \ - "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'vertex_ai/fallback-one' in [0-9]+s\\." \ - "2" \ - "vertex_ai/missing-primary|vertex_ai/fallback-one" \ - "|" - -run_gate_case "provider-prefix-required-resource-path-primary-implicit-default-provider" \ - "projects/p1/locations/us-central1/publishers/google/models/gemini-2.5-pro" \ - "vertex_ai/fallback-one" \ - "0" \ - "Normalized STRIX_LLM to provider-qualified model 'vertex_ai/gemini-2.5-pro'." \ - "1" \ - "vertex_ai/gemini-2.5-pro" \ - "" - -run_gate_case "provider-prefix-required-resource-path-primary-explicit-empty-default-provider" \ - "projects/p1/locations/us-central1/publishers/google/models/gemini-2.5-pro" \ - "vertex_ai/fallback-one" \ - "2" \ - "ERROR: Vertex resource paths require an explicit vertex_ai or vertex_ai_beta provider." \ - "0" \ - "" \ - "" \ - "" - -run_gate_case "provider-prefix-resource-path-primary-notfound-fallback-success" \ - "projects/p1/locations/us-central1/publishers/google/models/missing-primary" \ - "projects/p1/locations/us-central1/publishers/google/models/fallback-one projects/p1/locations/us-central1/publishers/google/models/fallback-two" \ - "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'vertex_ai/fallback-one' in [0-9]+s\\." \ - "2" \ - "vertex_ai/missing-primary|vertex_ai/fallback-one" \ - "|" - -# Regression: Vertex custom model resource path projects/

/locations//models/ -# (no publishers/ segment) must be recognized as a Vertex resource path and -# normalized to vertex_ai/. -run_gate_case "vertex-custom-model-resource-path" \ - "projects/my-proj/locations/us-central1/models/my-custom-model-123" \ - "vertex_ai/fallback-one" \ - "0" \ - "Normalized STRIX_LLM to provider-qualified model 'vertex_ai/my-custom-model-123'." \ - "1" \ - "vertex_ai/my-custom-model-123" \ - "" - -run_gate_case "vertex-notfound-without-status-fallback-success" \ - "vertex_ai/missing-primary" \ - "vertex_ai/fallback-one" \ - "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'vertex_ai/fallback-one' in [0-9]+s\\." \ - "2" \ - "vertex_ai/missing-primary|vertex_ai/fallback-one" \ - "|" - -run_gate_case "vertex-notfound-compact-status-fallback-success" \ - "vertex_ai/missing-primary" \ - "vertex_ai/fallback-one" \ - "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'vertex_ai/fallback-one' in [0-9]+s\\." \ - "2" \ - "vertex_ai/missing-primary|vertex_ai/fallback-one" \ - "|" - -run_gate_case "nonvertex-slash-model-passthrough" \ - "foo/bar" \ - "vertex_ai/fallback-one" \ - "0" \ - "scan ok with non-vertex slash model passthrough" \ - "1" \ - "foo/bar" \ - "https://example.invalid" - -run_gate_case "primary-duplicate-in-fallback" \ - "missing-primary" \ - "vertex_ai/missing-primary fallback-one" \ - "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'vertex_ai/fallback-one' in [0-9]+s\\." \ - "2" \ - "vertex_ai/missing-primary|vertex_ai/fallback-one" \ - "|" - -run_gate_case "multiline-fallback-success" \ - "vertex_ai/missing-primary" \ - $'vertex_ai/fallback-one\nvertex_ai/fallback-two' \ - "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'vertex_ai/fallback-two' in [0-9]+s\\." \ - "3" \ - "vertex_ai/missing-primary|vertex_ai/fallback-one|vertex_ai/fallback-two" \ - "||" - -run_gate_case_allow_provider_signal "vertex-primary-ratelimit-fallback-success" \ - "vertex_ai/ratelimit-primary" \ - "vertex_ai/fallback-one vertex_ai/fallback-two" \ - "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'vertex_ai/fallback-one' in [0-9]+s\\." \ - "2" \ - "vertex_ai/ratelimit-primary|vertex_ai/fallback-one" \ - "|" - -run_gate_case_allow_provider_signal "vertex-primary-resource-exhausted-fallback-success" \ - "vertex_ai/resource-exhausted-primary" \ - "vertex_ai/fallback-one vertex_ai/fallback-two" \ - "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'vertex_ai/fallback-one' in [0-9]+s\\." \ - "2" \ - "vertex_ai/resource-exhausted-primary|vertex_ai/fallback-one" \ - "|" - -run_gate_case_allow_provider_signal "openai-primary-quota-fallback-success" \ - "openai/quota-primary" \ - "openai/fallback-one openai/fallback-two" \ - "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'openai/fallback-one' in [0-9]+s\\." \ - "2" \ - "openai/quota-primary|openai/fallback-one" \ - "|" \ - "openai" - -run_gate_case_allow_provider_signal "vertex-primary-429-fallback-success" \ - "vertex_ai/http429-primary" \ - "vertex_ai/fallback-one vertex_ai/fallback-two" \ - "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'vertex_ai/fallback-one' in [0-9]+s\\." \ - "2" \ - "vertex_ai/http429-primary|vertex_ai/fallback-one" \ - "|" - -run_gate_case_allow_provider_signal "vertex-primary-midstream-fallback-success" \ - "vertex_ai/midstream-primary" \ - "vertex_ai/fallback-one vertex_ai/fallback-two" \ - "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'vertex_ai/fallback-one' in [0-9]+s\\." \ - "2" \ - "vertex_ai/midstream-primary|vertex_ai/fallback-one" \ - "|" - -run_gate_case_allow_provider_signal "vertex-primary-midstream-retry-same-model-success" \ - "vertex_ai/retry-midstream-primary" \ - "vertex_ai/fallback-one vertex_ai/fallback-two" \ - "0" \ - "scan ok after same-model retry" \ - "2" \ - "vertex_ai/retry-midstream-primary|vertex_ai/retry-midstream-primary" \ - "|" \ - "vertex_ai" \ - "__DEFAULT__" \ - "" \ - "1" - -# Bug 9: Rate-limit transient same-model retry (previously untested path) -run_gate_case_allow_provider_signal "vertex-primary-ratelimit-retry-same-model-success" \ - "vertex_ai/retry-ratelimit-primary" \ - "vertex_ai/fallback-one vertex_ai/fallback-two" \ - "0" \ - "scan ok after same-model rate-limit retry" \ - "2" \ - "vertex_ai/retry-ratelimit-primary|vertex_ai/retry-ratelimit-primary" \ - "|" \ - "vertex_ai" \ - "__DEFAULT__" \ - "" \ - "1" - -run_gate_case_allow_provider_signal "vertex-primary-api-connection-retry-same-model-success" \ - "gemini/retry-api-connection-primary" \ - "vertex_ai/fallback-one vertex_ai/fallback-two" \ - "0" \ - "scan ok after same-model api connection retry" \ - "2" \ - "gemini/retry-api-connection-primary|gemini/retry-api-connection-primary" \ - "https://example.invalid|https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ - "" \ - "1" - -run_gate_case_allow_provider_signal "github-models-internal-server-connection-retry-same-model-success" \ - "openai/openai/retry-api-connection-primary" \ - "" \ - "0" \ - "scan ok after same-model api connection retry" \ - "2" \ - "openai/openai/retry-api-connection-primary|openai/openai/retry-api-connection-primary" \ - "https://models.github.ai/inference|https://models.github.ai/inference" \ - "openai" \ - "https://models.github.ai/inference" \ - "" \ - "1" - -run_gate_case_allow_provider_signal "internal-server-error-unrelated-output-nonretryable" \ - "openai/openai/retry-api-connection-primary" \ - "" \ - "1" \ - "Strix quick scan failed with a non-recoverable error." \ - "1" \ - "openai/openai/retry-api-connection-primary" \ - "https://models.github.ai/inference" \ - "openai" \ - "https://models.github.ai/inference" \ - "" \ - "0" - -# Bug: large provider logs (many matching litellm.InternalServerError -# blocks) must not suppress a legitimate same-model retry via SIGPIPE on the -# bounded awk scan under `set -o pipefail`. See PR #1394 Devin finding -# "Large provider logs suppress retries". -run_gate_case_allow_provider_signal "internal-server-error-many-blocks-retry-same-model-success" \ - "openai/openai/retry-api-connection-primary" \ - "" \ - "0" \ - "scan ok after same-model api connection retry" \ - "2" \ - "openai/openai/retry-api-connection-primary|openai/openai/retry-api-connection-primary" \ - "https://models.github.ai/inference|https://models.github.ai/inference" \ - "openai" \ - "https://models.github.ai/inference" \ - "" \ - "1" - -run_gate_case "openrouter-502-fallback-retry-same-model-success" \ - "vertex_ai/missing-primary" \ - "openrouter/free vertex_ai/fallback-two" \ - "0" \ - "scan ok after OpenRouter 502 same-model retry" \ - "3" \ - "vertex_ai/missing-primary|openrouter/free|openrouter/free" \ - "|https://example.invalid|https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ - "" \ - "1" - -run_gate_case "openrouter-502-distant-target-output-nonretryable" \ - "vertex_ai/missing-primary" \ - "openrouter/free vertex_ai/fallback-two" \ - "1" \ - "Strix quick scan failed with a non-recoverable error." \ - "2" \ - "vertex_ai/missing-primary|openrouter/free" \ - "|https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ - "" \ - "1" - -run_gate_case "github-models-primary-unavailable-fallback-success" \ - "openai/gpt-5" \ - "" \ - "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'deepseek/deepseek-r1-0528' in [0-9]+s\\." \ - "2" \ - "openai/gpt-5|openai/deepseek/deepseek-r1-0528" \ - "https://models.github.ai/inference|https://models.github.ai/inference" \ - "openai" \ - "https://models.github.ai/inference" \ - "" \ - "0" \ - "CRITICAL" \ - "0" \ - "" \ - "" \ - "1200" \ - "0" \ - "" \ - "" \ - "" \ - "" \ - "0" \ - "" \ - "" \ - "" \ - "__SAME_AS_FALLBACK_MODELS__" \ - "deepseek/deepseek-r1-0528 deepseek/deepseek-v3-0324" \ - "1" - -run_gate_case_allow_provider_signal "github-models-primary-denied-fallback-success" \ - "openai/gpt-5" \ - "" \ - "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'deepseek/deepseek-r1-0528' in [0-9]+s\\." \ - "2" \ - "openai/gpt-5|openai/deepseek/deepseek-r1-0528" \ - "https://models.github.ai/inference|https://models.github.ai/inference" \ - "openai" \ - "https://models.github.ai/inference" \ - "" \ - "0" \ - "CRITICAL" \ - "0" \ - "" \ - "" \ - "1200" \ - "0" \ - "" \ - "" \ - "" \ - "" \ - "0" \ - "" \ - "" \ - "" \ - "__SAME_AS_FALLBACK_MODELS__" \ - "deepseek/deepseek-r1-0528 deepseek/deepseek-v3-0324" \ - "1" - -run_github_models_http410_case \ - "github-models-http410-authenticated-fallback-success" \ - "0" \ - "2" \ - "openai/gpt-5|openai/deepseek/deepseek-r1-0528" \ - "https://models.github.ai/inference|https://models.github.ai/inference" \ - "REGEX:Strix quick scan succeeded with fallback model 'deepseek/deepseek-r1-0528' in [0-9]+s\\." - -for scenario in \ - github-models-http410-missing-http-token \ - github-models-http410-missing-provider-error \ - github-models-http410-numeric-continuation-4100 \ - github-models-http410-numeric-continuation-4104 \ - github-models-http410-target-output-spoof \ - github-models-retirement-brownout-phrase-only; do - run_github_models_http410_case \ - "$scenario" \ - "1" \ - "1" \ - "openai/gpt-5" \ - "https://models.github.ai/inference" -done - -run_gate_case "github-models-primary-ratelimit-fallback-success" \ - "openai/gpt-5" \ - "" \ - "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'deepseek/deepseek-r1-0528' in [0-9]+s\\." \ - "2" \ - "openai/gpt-5|openai/deepseek/deepseek-r1-0528" \ - "https://models.github.ai/inference|https://models.github.ai/inference" \ - "openai" \ - "https://models.github.ai/inference" \ - "" \ - "2" \ - "CRITICAL" \ - "0" \ - "" \ - "" \ - "1200" \ - "0" \ - "" \ - "" \ - "" \ - "" \ - "0" \ - "" \ - "" \ - "" \ - "__SAME_AS_FALLBACK_MODELS__" \ - "deepseek/deepseek-r1-0528 deepseek/deepseek-v3-0324" \ - "1" - -run_gate_case "github-models-fallback-provider-signal-tries-next" \ - "openai/gpt-5" \ - "" \ - "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'deepseek/deepseek-v3-0324' in [0-9]+s\\." \ - "3" \ - "openai/gpt-5|openai/deepseek/deepseek-r1-0528|openai/deepseek/deepseek-v3-0324" \ - "https://models.github.ai/inference|https://models.github.ai/inference|https://models.github.ai/inference" \ - "openai" \ - "https://models.github.ai/inference" \ - "" \ - "0" \ - "CRITICAL" \ - "0" \ - "" \ - "" \ - "1200" \ - "0" \ - "pull_request" \ - "sync-module-system/smart-crawling-biz/src/main/java/org/empasy/sync/modules/system/controller/SysPositionController.java" \ - "" \ - "" \ - "0" \ - "" \ - "" \ - "" \ - "__SAME_AS_FALLBACK_MODELS__" \ - "deepseek/deepseek-r1-0528 deepseek/deepseek-v3-0324" \ - "1" - -run_gate_case "github-models-fallback-baseline-vulnerability-before-next-success-continues" \ - "openai/gpt-5" \ - "" \ - "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'deepseek/deepseek-v3-0324' in [0-9]+s\\." \ - "3" \ - "openai/gpt-5|openai/deepseek/deepseek-r1-0528|openai/deepseek/deepseek-v3-0324" \ - "https://models.github.ai/inference|https://models.github.ai/inference|https://models.github.ai/inference" \ - "openai" \ - "https://models.github.ai/inference" \ - "" \ - "0" \ - "CRITICAL" \ - "0" \ - "" \ - "" \ - "1200" \ - "0" \ - "pull_request" \ - "sync-module-system/smart-crawling-biz/src/main/java/org/empasy/sync/modules/system/controller/SysPositionController.java" \ - "" \ - "" \ - "0" \ - "" \ - "" \ - "" \ - "__SAME_AS_FALLBACK_MODELS__" \ - "deepseek/deepseek-r1-0528 deepseek/deepseek-v3-0324" \ - "1" - -run_gate_case "github-models-exhausted-after-baseline-vulnerability-fails-closed" \ - "openai/gpt-5" \ - "" \ - "1" \ - "STRIX_PROVIDER_UNAVAILABLE: provider models were exhausted after incomplete scan evidence." \ - "3" \ - "openai/gpt-5|openai/deepseek/deepseek-r1-0528|openai/deepseek/deepseek-v3-0324" \ - "https://models.github.ai/inference|https://models.github.ai/inference|https://models.github.ai/inference" \ - "openai" \ - "https://models.github.ai/inference" \ - "" \ - "0" \ - "CRITICAL" \ - "0" \ - "" \ - "" \ - "1200" \ - "0" \ - "pull_request" \ - "sync-module-system/smart-crawling-biz/src/main/java/org/empasy/sync/modules/system/controller/SysPositionController.java" \ - "" \ - "" \ - "0" \ - "" \ - "" \ - "" \ - "__SAME_AS_FALLBACK_MODELS__" \ - "deepseek/deepseek-r1-0528 deepseek/deepseek-v3-0324" \ - "1" - -run_gate_case "github-models-fallback-changed-vulnerability-before-next-success-blocks" \ - "openai/gpt-5" \ - "" \ - "1" \ - "Strix model reported threshold vulnerabilities before fallback success; failing closed so every model-reported vulnerability is reviewed." \ - "2" \ - "openai/gpt-5|openai/deepseek/deepseek-r1-0528" \ - "https://models.github.ai/inference|https://models.github.ai/inference" \ - "openai" \ - "https://models.github.ai/inference" \ - "" \ - "0" \ - "CRITICAL" \ - "0" \ - "" \ - "" \ - "1200" \ - "0" \ - "pull_request" \ - "sync-module-system/smart-crawling-biz/src/main/java/org/empasy/sync/modules/system/controller/SysPositionController.java" \ - "" \ - "" \ - "0" \ - "" \ - "" \ - "" \ - "__SAME_AS_FALLBACK_MODELS__" \ - "deepseek/deepseek-r1-0528 deepseek/deepseek-v3-0324" \ + "pull-request-target-invalid-head-sha-fails-closed" \ + "head" + +run_pull_request_target_aborts_on_pr_head_blob_failure_case \ + "pull-request-target-disabled-pr-scope-pr-head-blob-read-failure" \ + "src/existing.py" \ + "BASE_CONTENT_MUST_NOT_BE_USED_AFTER_DISABLED_SCOPE_HEAD_FAILURE" \ + "HEAD_CONTENT_SHOULD_NOT_BECOME_PARTIAL_SCAN_INPUT" \ + "cat-file" \ "1" -run_gate_case "github-models-fallback-dockerfile-test-baseline-before-next-success-continues" \ - "openai/gpt-5" \ +run_gate_case "success" \ + "orchestrator/free" \ "" \ "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'deepseek/deepseek-v3-0324' in [0-9]+s\\." \ - "3" \ - "openai/gpt-5|openai/deepseek/deepseek-r1-0528|openai/deepseek/deepseek-v3-0324" \ - "https://models.github.ai/inference|https://models.github.ai/inference|https://models.github.ai/inference" \ - "openai" \ - "https://models.github.ai/inference" \ + "scan ok" \ + "1" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" + +run_gate_case "contextual-orchestrator-missing-api-base-fails-closed" \ + "orchestrator/free" \ "" \ + "2" \ + "require LLM_API_BASE_FILE to select the pinned loopback gateway" \ "0" \ - "MEDIUM" \ - "0" \ - "" \ "" \ - "1200" \ - "0" \ - "pull_request" \ - ".github/workflows/build-ci-image.yml" \ "" \ + "contextual_orchestrator" \ + "" + +run_gate_case "contextual-orchestrator-gateway-model-qualification" \ + "orchestrator/free" \ "" \ "0" \ - "" \ - "" \ - "" \ - "__SAME_AS_FALLBACK_MODELS__" \ - "deepseek/deepseek-r1-0528 deepseek/deepseek-v3-0324" \ - "1" + "scan ok through contextual-orchestrator gateway" \ + "1" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" -run_gate_case_allow_provider_signal "gemini-high-demand-retry-same-model-success" \ - "gemini/retry-high-demand-primary" \ - "vertex_ai/fallback-one vertex_ai/fallback-two" \ - "0" \ - "scan ok after same-model high-demand retry" \ - "2" \ - "gemini/retry-high-demand-primary|gemini/retry-high-demand-primary" \ - "https://example.invalid|https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ +run_gate_case "success-with-critical-report" \ + "orchestrator/free" \ "" \ - "1" + "1" \ + "Current Strix vulnerability report exists; failing closed without a repository-authored severity threshold." \ + "1" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" -run_gate_case_allow_provider_signal "nvidia-overloaded-direct-fallback-success" \ - "nvidia_nim/nvidia/overloaded-primary" \ - "" \ - "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'nvidia_nim/nvidia/fallback-one' in [0-9]+s\\." \ - "3" \ - "nvidia_nim/nvidia/overloaded-primary|nvidia_nim/nvidia/overloaded-primary|nvidia_nim/nvidia/fallback-one" \ - "https://integrate.api.nvidia.com/v1|https://integrate.api.nvidia.com/v1|https://integrate.api.nvidia.com/v1" \ - "nvidia_nim" \ - "https://integrate.api.nvidia.com/v1" \ +run_gate_case "pr-executable-integrity-mismatch" \ + "orchestrator/free" \ "" \ "1" \ - "CRITICAL" \ - "0" \ - "" \ - "" \ - "1200" \ - "0" \ - "" \ - "" \ - "" \ - "" \ + "did not match the pinned SHA-256 digest" \ "0" \ "" \ "" \ - "" \ - "__SAME_AS_FALLBACK_MODELS__" \ - "nvidia_nim/nvidia/fallback-one openai-direct/gpt-5.4" + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" -run_gate_case_allow_provider_signal "nvidia-rate-limit-openai-direct-fallback-clears-api-base" \ - "nvidia_nim/nvidia/rate-limited-primary" \ +run_gate_case "pr-executable-group-writable" \ + "orchestrator/free" \ "" \ + "1" \ + "must not be group/world writable" \ "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'openai-direct/gpt-5.4' in [0-9]+s\\." \ - "2" \ - "nvidia_nim/nvidia/rate-limited-primary|openai/gpt-5.4" \ - "https://integrate.api.nvidia.com/v1|" \ - "nvidia_nim" \ - "https://integrate.api.nvidia.com/v1" \ "" \ - "0" \ - "CRITICAL" \ - "0" \ "" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" + +run_gate_case "pr-executable-root-group-writable" \ + "orchestrator/free" \ "" \ - "1200" \ + "1" \ + "pinned Strix installation root must not be group/world writable" \ "0" \ "" \ "" \ - "" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" + +run_gate_case "runtime-env-forwarding" \ + "orchestrator/free" \ "" \ "0" \ - "" \ - "" \ - "" \ - "__SAME_AS_FALLBACK_MODELS__" \ - "openai-direct/gpt-5.4" + "scan ok" \ + "1" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" -run_gate_case_allow_provider_signal "gemini-timeout-direct-fallback-success" \ - "gemini/retry-timeout-primary" \ - "gemini/fallback-one gemini/fallback-two" \ - "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'gemini/fallback-one' in [0-9]+s\\." \ - "2" \ - "gemini/retry-timeout-primary|gemini/fallback-one" \ - "https://example.invalid|https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ - "" \ - "1" -run_gate_case_allow_provider_signal "gemini-timeout-fallback-success" \ - "gemini/timeout-fallback-primary" \ - "gemini/fallback-one gemini/fallback-two" \ - "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'gemini/fallback-one' in [0-9]+s\\." \ - "2" \ - "gemini/timeout-fallback-primary|gemini/fallback-one" \ - "https://example.invalid|https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + +run_gate_case "nonrecoverable" \ + "orchestrator/free" \ "" \ - "1" + "1" \ + "Strix quick scan failed; failing closed without repository-authored retry or fallback allocation." \ + "1" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" -run_gate_case_allow_provider_signal "gemini-generic-fallback-success" \ - "gemini/timeout-fallback-primary" \ +run_gate_case "provider-prefix-required" \ + "gemini-2.5-pro" \ "" \ - "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'gemini/fallback-one' in [0-9]+s\\." \ "2" \ - "gemini/timeout-fallback-primary|gemini/fallback-one" \ - "https://example.invalid|https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ - "" \ - "1" \ - "CRITICAL" \ + "ERROR: Strix model must be orchestrator/free through contextual-orchestrator; direct provider/model routes are forbidden" \ "0" \ "" \ "" \ - "1200" \ - "0" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" + + +run_gate_case "provider-prefix-required-resource-path-primary-implicit-default-provider" \ + "projects/p1/locations/us-central1/publishers/google/models/gemini-2.5-pro" \ "" \ + "2" \ + "ERROR: Strix model must be orchestrator/free through contextual-orchestrator; direct provider/model routes are forbidden" \ + "0" \ "" \ "" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" + +run_gate_case "provider-prefix-required-resource-path-primary-explicit-empty-default-provider" \ + "projects/p1/locations/us-central1/publishers/google/models/gemini-2.5-pro" \ "" \ + "2" \ + "ERROR: Strix model must be orchestrator/free through contextual-orchestrator; direct provider/model routes are forbidden" \ "0" \ "" \ "" \ - "" \ - "__UNSET__" \ - "gemini/fallback-one gemini/fallback-two" + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" -run_gate_case_allow_provider_signal "gemini-zero-findings-timeout-fallback-allows-pr" \ - "gemini/zero-timeout-primary" \ - "gemini/fallback-one" \ - "1" \ - "Strix reported zero vulnerabilities before provider infrastructure failure; failing closed because provider infrastructure failures are not clean scan evidence." \ - "2" \ - "gemini/zero-timeout-primary|gemini/fallback-one" \ - "https://example.invalid|https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + +# Regression: Vertex custom model resource path projects/

/locations//models/ +# (no publishers/ segment) must be recognized as a Vertex resource path and +# normalized to vertex_ai/. +run_gate_case "vertex-custom-model-resource-path" \ + "projects/my-proj/locations/us-central1/models/my-custom-model-123" \ "" \ - "0" \ - "CRITICAL" \ + "2" \ + "ERROR: Strix model must be orchestrator/free through contextual-orchestrator; direct provider/model routes are forbidden" \ "0" \ "" \ "" \ - "1200" \ - "0" \ - "pull_request" \ - "sync-module-system/smart-crawling-biz/src/main/java/org/empasy/sync/modules/system/controller/SysPositionController.java" + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" -run_gate_case_allow_provider_signal "pr-scope-zero-finding-does-not-leak" \ - "gemini/scope-zero-leak-primary" \ - "" \ - "1" \ - "Strix reported zero vulnerabilities before provider infrastructure failure; failing closed because provider infrastructure failures are not clean scan evidence." \ - "1" \ - "gemini/scope-zero-leak-primary" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + + +run_gate_case "nonvertex-slash-model-passthrough" \ + "foo/bar" \ "" \ - "0" \ - "CRITICAL" \ + "2" \ + "ERROR: Strix model must be orchestrator/free through contextual-orchestrator; direct provider/model routes are forbidden" \ "0" \ "" \ "" \ - "1200" \ - "0" \ - "pull_request" \ - $'sync-module-system/smart-crawling-biz/src/main/java/org/empasy/sync/modules/system/controller/SysPositionController.java\nsync-module-system/smart-crawling-playwright/src/main/java/org/empasy/sync/mcp/service/PlayWrightService.java' \ - "" \ - "1" + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" -run_gate_case "service-unavailable-no-llm-marker-nonrecoverable" \ - "custom/service-unavailable-primary" \ - "vertex_ai/fallback-one vertex_ai/fallback-two" \ + + + + + + + + +# Bug 9: Rate-limit transient same-model retry (previously untested path) + + + +run_gate_case_allow_provider_signal "internal-server-error-unrelated-output-nonretryable" \ + "orchestrator/free" \ + "" \ "1" \ - "Strix quick scan failed with a non-recoverable error." \ + "STRIX_PROVIDER_UNAVAILABLE: contextual-orchestrator/orchestrator/free did not produce authoritative scan evidence" \ "1" \ - "custom/service-unavailable-primary" \ - "https://example.invalid" \ - "custom" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ - "1" + "0" + +# Bug: large provider logs (many matching litellm.InternalServerError +# blocks) must not suppress a legitimate same-model retry via SIGPIPE on the +# bounded awk scan under `set -o pipefail`. See PR #1394 Devin finding +# "Large provider logs suppress retries". + + + + + + + + + + + + + + + -run_gate_case "server-disconnect-no-llm-marker-nonrecoverable" \ - "vertex_ai/app-server-disconnect-primary" \ - "vertex_ai/fallback-one vertex_ai/fallback-two" \ - "1" \ - "Strix quick scan failed with a non-recoverable error." \ - "1" \ - "vertex_ai/app-server-disconnect-primary" \ - "" -# Bug 11: Timeout should move directly to fallback instead of retrying the same model. -run_gate_case_allow_provider_signal "vertex-primary-timeout-retry-same-model-success" \ - "vertex_ai/retry-timeout-primary" \ - "vertex_ai/fallback-one vertex_ai/fallback-two" \ - "0" \ - "scan ok after timeout fallback" \ - "2" \ - "vertex_ai/retry-timeout-primary|vertex_ai/fallback-one" \ - "|" \ - "vertex_ai" \ - "__DEFAULT__" \ - "" \ - "1" -# Bug 11b: Timeout → immediate fallback model succeeds. -run_gate_case_allow_provider_signal "vertex-primary-timeout-exhausted-fallback-success" \ - "vertex_ai/timeout-exhaust-primary" \ - "vertex_ai/fallback-one vertex_ai/fallback-two" \ - "0" \ - "scan ok after timeout-exhausted fallback" \ - "2" \ - "vertex_ai/timeout-exhaust-primary|vertex_ai/fallback-one" \ - "|" \ - "vertex_ai" \ - "__DEFAULT__" \ - "" \ - "1" -run_gate_case_allow_provider_signal "zero-findings-timeout-all-models" \ - "vertex_ai/zero-timeout-primary" \ - "vertex_ai/fallback-one" \ - "1" \ - "Strix reported zero vulnerabilities before provider infrastructure failure; failing closed because provider infrastructure failures are not clean scan evidence." \ - "2" \ - "vertex_ai/zero-timeout-primary|vertex_ai/fallback-one" \ - "|" \ - "vertex_ai" \ - "__DEFAULT__" \ - "" \ - "0" \ - "CRITICAL" \ - "0" \ - "" \ - "" \ - "$TIMEOUT_TEST_PROCESS_SECONDS" \ - "0" \ - "pull_request" \ - "sync-module-system/smart-crawling-biz/src/main/java/org/empasy/sync/modules/system/controller/SysPositionController.java" -run_gate_case_allow_provider_signal "zero-findings-timeout-all-models" \ - "vertex_ai/zero-timeout-primary" \ - "vertex_ai/fallback-one" \ - "1" \ - "Configured Vertex model and fallback models were unavailable." \ - "2" \ - "vertex_ai/zero-timeout-primary|vertex_ai/fallback-one" \ - "|" \ - "vertex_ai" \ - "__DEFAULT__" \ - "" \ - "0" \ - "CRITICAL" \ - "0" \ - "" \ +run_gate_case_allow_provider_signal "pr-scope-zero-finding-does-not-leak" \ + "orchestrator/free" \ "" \ - "$TIMEOUT_TEST_PROCESS_SECONDS" \ - "0" \ - "push" - -run_gate_case_allow_provider_signal "zero-findings-sticky-across-fallback" \ - "vertex_ai/zero-sticky-primary" \ - "vertex_ai/fallback-one" \ "1" \ "Strix reported zero vulnerabilities before provider infrastructure failure; failing closed because provider infrastructure failures are not clean scan evidence." \ - "2" \ - "vertex_ai/zero-sticky-primary|vertex_ai/fallback-one" \ - "|" \ - "vertex_ai" \ - "__DEFAULT__" \ + "1" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ "0" \ "" \ "" \ - "$TIMEOUT_TEST_PROCESS_SECONDS" \ + "1200" \ "0" \ "pull_request" \ - "sync-module-system/smart-crawling-biz/src/main/java/org/empasy/sync/modules/system/controller/SysPositionController.java" + $'sync-module-system/smart-crawling-biz/src/main/java/org/empasy/sync/modules/system/controller/SysPositionController.java\nsync-module-system/smart-crawling-playwright/src/main/java/org/empasy/sync/mcp/service/PlayWrightService.java' \ + "" \ + "1" -run_gate_case_allow_provider_signal "zero-findings-with-low-report-timeout" \ - "vertex_ai/zero-low-primary" \ - "vertex_ai/fallback-one" \ - "1" \ - "Configured Vertex model and fallback models were unavailable." \ - "2" \ - "vertex_ai/zero-low-primary|vertex_ai/fallback-one" \ - "|" \ - "vertex_ai" \ - "__DEFAULT__" \ +run_gate_case "service-unavailable-no-llm-marker-nonrecoverable" \ + "orchestrator/free" \ "" \ - "0" \ - "CRITICAL" \ - "0" \ + "1" \ + "Strix quick scan failed; failing closed without repository-authored retry or fallback allocation." \ + "1" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ + "0" + +run_gate_case "server-disconnect-no-llm-marker-nonrecoverable" \ + "orchestrator/free" \ "" \ - "$TIMEOUT_TEST_PROCESS_SECONDS" \ - "0" \ - "pull_request" \ - "sync-module-system/smart-crawling-biz/src/main/java/org/empasy/sync/modules/system/controller/SysPositionController.java" + "1" \ + "Strix quick scan failed; failing closed without repository-authored retry or fallback allocation." \ + "1" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" + +# Bug 11: Timeout should move directly to fallback instead of retrying the same model. + +# Bug 11b: Timeout → immediate fallback model succeeds. + + + + run_gate_case "strict-zero-findings-timeout-fails-pr" \ - "vertex_ai/zero-timeout-primary" \ - " " \ + "orchestrator/free" \ + "" \ "1" \ "failing closed" \ "1" \ - "vertex_ai/zero-timeout-primary" \ - "" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -10849,15 +8641,15 @@ run_gate_case "strict-zero-findings-timeout-fails-pr" \ "1" run_gate_case "provider-fatal-success-signal" \ - "vertex_ai/provider-fatal-success-signal" \ + "orchestrator/free" \ "" \ "1" \ "Strix run emitted provider infrastructure or failure-signal output; failing closed." \ "1" \ - "vertex_ai/provider-fatal-success-signal" \ - "" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -10879,15 +8671,15 @@ run_gate_case "provider-fatal-success-signal" \ "1" run_gate_case "provider-warning-success-signal" \ - "vertex_ai/provider-warning-success-signal" \ + "orchestrator/free" \ "" \ "1" \ "Strix run emitted provider infrastructure or failure-signal output; failing closed." \ "1" \ - "vertex_ai/provider-warning-success-signal" \ - "" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -10908,25 +8700,17 @@ run_gate_case "provider-warning-success-signal" \ "" \ "1" -run_gate_case "provider-report-rate-limit-fallback-success" \ - "vertex_ai/report-rate-limit-primary" \ - "vertex_ai/fallback-one vertex_ai/fallback-two" \ - "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'vertex_ai/fallback-one' in [0-9]+s\\." \ - "2" \ - "vertex_ai/report-rate-limit-primary|vertex_ai/fallback-one" \ - "|" run_gate_case "report-known-internal-warning-sanitized" \ - "vertex_ai/report-known-internal-warning-sanitized" \ + "orchestrator/free" \ "" \ "0" \ - "Strix run succeeded for model 'vertex_ai/report-known-internal-warning-sanitized'" \ + "Strix run succeeded for model 'orchestrator/free'" \ "1" \ - "vertex_ai/report-known-internal-warning-sanitized" \ - "" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -10948,15 +8732,15 @@ run_gate_case "report-known-internal-warning-sanitized" \ "1" run_gate_case "report-known-internal-warning-variant-sanitized" \ - "vertex_ai/report-known-internal-warning-variant-sanitized" \ + "orchestrator/free" \ "" \ "0" \ - "Strix run succeeded for model 'vertex_ai/report-known-internal-warning-variant-sanitized'" \ + "Strix run succeeded for model 'orchestrator/free'" \ "1" \ - "vertex_ai/report-known-internal-warning-variant-sanitized" \ - "" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -10977,15 +8761,15 @@ run_gate_case "report-known-internal-warning-variant-sanitized" \ "1" run_gate_case "report-unknown-warning-fails" \ - "vertex_ai/report-unknown-warning-fails" \ + "orchestrator/free" \ "" \ "1" \ "Strix report artifacts emitted warning/fatal/denied/timeout output; failing closed." \ "1" \ - "vertex_ai/report-unknown-warning-fails" \ - "" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -11007,15 +8791,15 @@ run_gate_case "report-unknown-warning-fails" \ "1" run_gate_case "provider-denied-success-signal" \ - "vertex_ai/provider-denied-success-signal" \ + "orchestrator/free" \ "" \ "1" \ "Strix run emitted provider infrastructure or failure-signal output; failing closed." \ "1" \ - "vertex_ai/provider-denied-success-signal" \ - "" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -11036,34 +8820,28 @@ run_gate_case "provider-denied-success-signal" \ "" \ "1" -run_gate_case_allow_provider_signal "vertex-all-ratelimited" \ - "vertex_ai/ratelimit-primary" \ - "vertex_ai/fallback-one vertex_ai/fallback-two" \ - "1" \ - "Configured Vertex model and fallback models were unavailable." \ - "3" \ - "vertex_ai/ratelimit-primary|vertex_ai/fallback-one|vertex_ai/fallback-two" \ - "||" run_gate_case "vertex-primary-hallucinated-endpoint-fallback-success" \ - "vertex_ai/hallucination-primary" \ - "vertex_ai/fallback-one vertex_ai/fallback-two" \ + "orchestrator/free" \ + "" \ "1" \ "Strix quick scan failed with a non-recoverable error." \ "1" \ - "vertex_ai/hallucination-primary" \ - "" + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" run_gate_case "opencode-documented-env-api-key-fallback-success" \ - "vertex_ai/opencode-env-primary" \ - "vertex_ai/fallback-one vertex_ai/fallback-two" \ + "orchestrator/free" \ + "" \ "1" \ "Strix finding intersects files changed in this pull request." \ "1" \ - "vertex_ai/opencode-env-primary" \ - "" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "HIGH" \ @@ -11076,15 +8854,15 @@ run_gate_case "opencode-documented-env-api-key-fallback-success" \ ".github/workflows/opencode-review.yml" run_gate_case "generic-github-actions-workflow-fallback-success" \ - "vertex_ai/generic-actions-primary" \ - "vertex_ai/fallback-one vertex_ai/fallback-two" \ + "orchestrator/free" \ + "" \ "1" \ "Unable to map Strix findings to changed files; failing closed for pull request." \ "1" \ - "vertex_ai/generic-actions-primary" \ - "" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -11097,24 +8875,26 @@ run_gate_case "generic-github-actions-workflow-fallback-success" \ ".github/workflows/strix.yml" run_gate_case "vertex-primary-existing-endpoint-nonrecoverable" \ - "vertex_ai/existing-endpoint-primary" \ - "vertex_ai/fallback-one vertex_ai/fallback-two" \ + "orchestrator/free" \ + "" \ "1" \ - "Strix quick scan failed with a non-recoverable error." \ + "Strix quick scan failed; failing closed without repository-authored retry or fallback allocation." \ "1" \ - "vertex_ai/existing-endpoint-primary" \ - "" + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" run_gate_case "pr-stale-source-claim-fallback-success" \ - "vertex_ai/stale-source-primary" \ - "vertex_ai/fallback-one vertex_ai/fallback-two" \ + "orchestrator/free" \ + "" \ "1" \ "Strix finding intersects files changed in this pull request." \ "1" \ - "vertex_ai/stale-source-primary" \ - "" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "HIGH" \ @@ -11127,15 +8907,15 @@ run_gate_case "pr-stale-source-claim-fallback-success" \ "backend/db/models.py" run_gate_case "pr-stale-snapshot-snippet-fallback-success" \ - "vertex_ai/stale-snapshot-primary" \ - "vertex_ai/fallback-one vertex_ai/fallback-two" \ + "orchestrator/free" \ + "" \ "1" \ "Strix finding intersects files changed in this pull request." \ "1" \ - "vertex_ai/stale-snapshot-primary" \ - "" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "MEDIUM" \ @@ -11148,15 +8928,15 @@ run_gate_case "pr-stale-snapshot-snippet-fallback-success" \ "backend/app/api/snapshots.py" run_gate_case "pr-stale-source-plus-real-finding-blocks" \ - "vertex_ai/stale-source-primary" \ - "vertex_ai/fallback-one vertex_ai/fallback-two" \ + "orchestrator/free" \ + "" \ "1" \ "Strix finding intersects files changed in this pull request." \ "1" \ - "vertex_ai/stale-source-primary" \ - "" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "HIGH" \ @@ -11169,15 +8949,15 @@ run_gate_case "pr-stale-source-plus-real-finding-blocks" \ $'backend/db/models.py\nbackend/api/emails.py' run_gate_case_allow_provider_signal "pr-changed-finding-with-retry-marker-blocks" \ - "vertex_ai/changed-finding-primary" \ - "vertex_ai/fallback-one vertex_ai/fallback-two" \ + "orchestrator/free" \ + "" \ "1" \ "Strix finding intersects files changed in this pull request." \ "1" \ - "vertex_ai/changed-finding-primary" \ - "" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "HIGH" \ @@ -11190,15 +8970,15 @@ run_gate_case_allow_provider_signal "pr-changed-finding-with-retry-marker-blocks "backend/api/emails.py" run_gate_case "pr-stale-report-plus-inline-changed-finding-blocks" \ - "vertex_ai/stale-inline-primary" \ - "vertex_ai/fallback-one vertex_ai/fallback-two" \ + "orchestrator/free" \ + "" \ "1" \ "Strix finding intersects files changed in this pull request." \ "1" \ - "vertex_ai/stale-inline-primary" \ - "" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "HIGH" \ @@ -11210,86 +8990,91 @@ run_gate_case "pr-stale-report-plus-inline-changed-finding-blocks" \ "pull_request" \ $'backend/db/models.py\nbackend/api/emails.py' -run_gate_case "high-vuln-below-threshold" \ - "vertex_ai/high-vuln-primary" \ +# Infrastructure error guard: below-threshold findings must NOT pass when the +# strix log contains evidence of infrastructure-level errors (timeout, +# rate-limit, transport failures) because the scan was likely incomplete. + +# Guard test 1: LOW finding + timeout → should fail (exit 1). +# The below-threshold check runs first but detects infrastructure errors in the +# strix log and refuses bypass. The timeout is also vertex-retryable, so the +# gate continues into the fallback loop. All attempts see the same timeout. + +# Guard test 2: LOW finding + rate-limit → should fail (exit 1). +# Below-threshold check refuses bypass due to infra errors. +# Rate-limit is vertex-retryable, so the gate also tries fallback models. + +# Guard test 3: INFO finding + ConnectionError → should fail (exit 1). +# ConnectionError is NOT vertex-retryable, so only the primary model is tried. +# Single-governed-request fail-closed contract. These replace the deleted +# multi-model fallback cluster: the same provider signals that used to select +# the next fallback model must now fail closed after exactly one request. + +run_gate_case_allow_provider_signal "provider-ratelimit-fails-closed" \ + "orchestrator/free" \ "" \ - "0" \ - "below configured fail threshold 'CRITICAL'" \ "1" \ - "vertex_ai/high-vuln-primary" \ - "" + "STRIX_PROVIDER_UNAVAILABLE: contextual-orchestrator/orchestrator/free did not produce authoritative scan evidence" \ + "1" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" -run_gate_case "multi-severity-low-then-critical" \ - "vertex_ai/multi-severity-primary" \ +run_gate_case_allow_provider_signal "provider-resource-exhausted-fails-closed" \ + "orchestrator/free" \ "" \ "1" \ - "Strix quick scan failed with a non-recoverable error." \ + "STRIX_PROVIDER_UNAVAILABLE: contextual-orchestrator/orchestrator/free did not produce authoritative scan evidence" \ "1" \ - "vertex_ai/multi-severity-primary" \ - "" + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" -run_gate_case "inline-medium-below-threshold" \ - "vertex_ai/inline-medium-primary" \ +run_gate_case_allow_provider_signal "provider-quota-fails-closed" \ + "orchestrator/free" \ "" \ "1" \ - "No Strix vulnerability report artifact was produced; log-only severity markers are incomplete evidence, so the scan is failing closed." \ + "STRIX_PROVIDER_UNAVAILABLE: contextual-orchestrator/orchestrator/free did not produce authoritative scan evidence" \ "1" \ - "vertex_ai/inline-medium-primary" \ - "" + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" -run_gate_case "medium-vuln-default-threshold" \ - "openai/gpt-4o-mini" \ +run_gate_case_allow_provider_signal "provider-token-limit-fails-closed" \ + "orchestrator/free" \ "" \ "1" \ - "Strix quick scan failed with a non-recoverable error." \ + "STRIX_PROVIDER_UNAVAILABLE: contextual-orchestrator/orchestrator/free did not produce authoritative scan evidence" \ "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ - "" \ - "0" \ - "__UNSET__" - -# Infrastructure error guard: below-threshold findings must NOT pass when the -# strix log contains evidence of infrastructure-level errors (timeout, -# rate-limit, transport failures) because the scan was likely incomplete. + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" -# Guard test 1: LOW finding + timeout → should fail (exit 1). -# The below-threshold check runs first but detects infrastructure errors in the -# strix log and refuses bypass. The timeout is also vertex-retryable, so the -# gate continues into the fallback loop. All attempts see the same timeout. -run_gate_case_allow_provider_signal "below-threshold-with-timeout" \ - "vertex_ai/low-timeout-primary" \ - "vertex_ai/gemini-2.5-pro vertex_ai/gemini-2.5-flash" \ +run_gate_case_allow_provider_signal "provider-timeout-fails-closed" \ + "orchestrator/free" \ + "" \ "1" \ - "infrastructure errors occurred during this pipeline run; refusing bypass" \ - "3" \ - "vertex_ai/low-timeout-primary|vertex_ai/gemini-2.5-pro|vertex_ai/gemini-2.5-flash" \ - "||" - -# Guard test 2: LOW finding + rate-limit → should fail (exit 1). -# Below-threshold check refuses bypass due to infra errors. -# Rate-limit is vertex-retryable, so the gate also tries fallback models. -run_gate_case_allow_provider_signal "below-threshold-with-ratelimit" \ - "vertex_ai/low-ratelimit-primary" \ - "vertex_ai/gemini-2.5-pro vertex_ai/gemini-2.5-flash" \ + "STRIX_PROVIDER_UNAVAILABLE: contextual-orchestrator/orchestrator/free did not produce authoritative scan evidence" \ "1" \ - "infrastructure errors occurred during this pipeline run; refusing bypass" \ - "3" \ - "vertex_ai/low-ratelimit-primary|vertex_ai/gemini-2.5-pro|vertex_ai/gemini-2.5-flash" \ - "||" + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" -# Guard test 3: INFO finding + ConnectionError → should fail (exit 1). -# ConnectionError is NOT vertex-retryable, so only the primary model is tried. -run_gate_case_allow_provider_signal "below-threshold-with-connection-error" \ - "vertex_ai/info-conn-primary" \ +run_gate_case "provider-plain-failure-fails-closed-without-infra-marker" \ + "orchestrator/free" \ "" \ "1" \ - "infrastructure errors occurred during this pipeline run; refusing bypass" \ + "Strix quick scan failed; failing closed without repository-authored retry or fallback allocation." \ "1" \ - "vertex_ai/info-conn-primary" \ - "" + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" + # Guard test 3b: INFO finding + ConnectionError WITHOUT provider marker → should # PASS (exit 0). The two-grep infra-error detector requires both a transport @@ -11300,82 +9085,42 @@ run_gate_case_allow_provider_signal "below-threshold-with-connection-error" \ # A bare "ConnectionError" from the target application lacks the marker, so # has_detected_infrastructure_error() returns 1 (no infra error) and the # below-threshold bypass succeeds. -run_gate_case "below-threshold-with-connection-error-no-provider" \ - "vertex_ai/info-conn-noprov-primary" \ - "" \ - "0" \ - "below configured fail threshold" \ - "1" \ - "vertex_ai/info-conn-noprov-primary" \ - "" - # Guard test 3c: INFO finding + requests.exceptions.ConnectionError → should # PASS (exit 0). The "requests" transport library matches the broad # PROVIDER_CONTEXT_REGEX but is intentionally excluded from LLM_PROVIDER_ONLY_REGEX. # Before commit 0e90d48 the connection-error path used PROVIDER_CONTEXT_REGEX # and would have mis-classified this as an LLM infrastructure error; now it # correctly uses LLM_PROVIDER_ONLY_REGEX, so below-threshold bypass succeeds. -run_gate_case "below-threshold-with-requests-connection-error" \ - "vertex_ai/info-conn-requests-primary" \ - "" \ - "0" \ - "below configured fail threshold" \ - "1" \ - "vertex_ai/info-conn-requests-primary" \ - "" - # Guard test 4: MEDIUM finding + MidStreamFallbackError → should fail (exit 1). # Midstream is vertex-retryable, so the gate also tries fallback models # (after the below-threshold check refuses bypass due to infra errors). -run_gate_case_allow_provider_signal "below-threshold-with-midstream" \ - "vertex_ai/medium-midstream-primary" \ - "vertex_ai/gemini-2.5-pro vertex_ai/gemini-2.5-flash" \ - "1" \ - "infrastructure errors occurred during this pipeline run; refusing bypass" \ - "3" \ - "vertex_ai/medium-midstream-primary|vertex_ai/gemini-2.5-pro|vertex_ai/gemini-2.5-flash" \ - "||" - -run_gate_case "critical-vuln-at-threshold" \ - "vertex_ai/critical-vuln-primary" \ - "" \ - "1" \ - "Strix quick scan failed with a non-recoverable error." \ - "1" \ - "vertex_ai/critical-vuln-primary" \ - "" run_gate_case "malformed-severity-marker-nonrecoverable" \ - "vertex_ai/malformed-severity-primary" \ + "orchestrator/free" \ "" \ "1" \ - "Strix quick scan failed with a non-recoverable error." \ + "Strix quick scan failed; failing closed without repository-authored retry or fallback allocation." \ "1" \ - "vertex_ai/malformed-severity-primary" \ - "" + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" # Bug 7: Model disagreement — the primary produces an unmapped CRITICAL report # alongside a NOT_FOUND error. The report is already actionable fail-closed # evidence, so the gate must not spend provider budget on a fallback whose LOW # result could make the earlier finding appear downgraded. -run_gate_case "model-disagreement-critical-in-earlier-report" \ - "vertex_ai/model-a" \ - "vertex_ai/model-b" \ - "1" \ - "Strix quick scan failed with a non-recoverable error." \ - "1" \ - "vertex_ai/model-a" \ - "" - # Bug 4: deepseek/models/deepseek-r1 must NOT be rewritten to vertex_ai/deepseek-r1 run_gate_case "nonvertex-slash-model-not-rewritten" \ "deepseek/models/deepseek-r1" \ - "vertex_ai/fallback-one" \ + "" \ + "2" \ + "ERROR: Strix model must be orchestrator/free through contextual-orchestrator; direct provider/model routes are forbidden" \ "0" \ - "scan ok with deepseek model passthrough" \ - "1" \ - "deepseek/models/deepseek-r1" \ - "https://example.invalid" + "" \ + "" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" # Regression: STRIX_TARGET_PATH=

/src with default STRIX_SOURCE_DIRS (now ".") # must resolve to /src/. (i.e. /src itself), NOT /src/src. @@ -11383,17 +9128,17 @@ run_gate_case "nonvertex-slash-model-not-rewritten" \ # endpoint. Source-dir resolution still runs, but threshold findings now remain # blocking even when model/source inconsistency is suspected. run_gate_case "target-path-src-default-source-dirs" \ - "vertex_ai/hallucination-primary" \ - "vertex_ai/fallback-one vertex_ai/fallback-two" \ + "orchestrator/free" \ + "" \ "1" \ "Strix quick scan failed with a non-recoverable error." \ "1" \ - "vertex_ai/hallucination-primary" \ - "" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ - "1" \ + "0" \ "CRITICAL" \ "0" \ "__USE_SUBDIR_SRC__" \ @@ -11404,15 +9149,15 @@ run_gate_case "target-path-src-default-source-dirs" \ # the gate must find the endpoint in the api/ dir and treat the finding as # non-hallucinated → non-recoverable failure (exit 1). run_gate_case "multi-source-dirs-existing-endpoint" \ - "vertex_ai/multi-dir-primary" \ - "vertex_ai/fallback-one vertex_ai/fallback-two" \ + "orchestrator/free" \ + "" \ "1" \ - "Strix quick scan failed with a non-recoverable error." \ + "Strix quick scan failed; failing closed without repository-authored retry or fallback allocation." \ "1" \ - "vertex_ai/multi-dir-primary" \ - "" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -11421,74 +9166,35 @@ run_gate_case "multi-source-dirs-existing-endpoint" \ "src api" run_gate_case "preserve-existing-api-base" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "0" \ - "scan ok with preserved api base" \ + "scan ok with pinned gateway api base" \ "1" \ - "openai/gpt-4o-mini" \ - "https://preexisting.invalid" \ - "vertex_ai" \ - "" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "https://preexisting.invalid" -run_gate_case "default-fallback-order-fast-first" \ - "vertex_ai/missing-primary" \ - "" \ - "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'vertex_ai/gemini-2[.]5-pro' in [0-9]+s\\." \ - "2" \ - "vertex_ai/missing-primary|vertex_ai/gemini-2.5-pro" \ - "|" # Bug 13: All fallback models are the same as the primary model. # The gate should detect that no distinct fallback was tried and emit an ERROR. -run_gate_case "all-fallbacks-same-as-primary" \ - "vertex_ai/same-primary" \ - "vertex_ai/same-primary vertex_ai/same-primary" \ - "1" \ - "ERROR: All configured fallback models are the same as the primary model" \ - "1" \ - "vertex_ai/same-primary" \ - "" - # Bug 14: Timeout should fall back rather than emit a same-model retry message. -run_gate_case_allow_provider_signal "vertex-primary-timeout-retry-reason-message" \ - "vertex_ai/retry-timeout-primary" \ - "vertex_ai/fallback-one vertex_ai/fallback-two" \ - "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'vertex_ai/fallback-one' in [0-9]+s\\." \ - "2" \ - "vertex_ai/retry-timeout-primary|vertex_ai/fallback-one" \ - "|" \ - "vertex_ai" \ - "__DEFAULT__" \ - "" \ - "2" # Bug 14: Retry reason messages — rate-limit retry should say "due to rate limit". -run_gate_case_allow_provider_signal "vertex-primary-ratelimit-retry-reason-message" \ - "vertex_ai/retry-ratelimit-primary" \ - "vertex_ai/fallback-one vertex_ai/fallback-two" \ - "0" \ - "Retrying model 'vertex_ai/retry-ratelimit-primary' due to rate limit" \ - "2" \ - "vertex_ai/retry-ratelimit-primary|vertex_ai/retry-ratelimit-primary" \ - "|" \ - "vertex_ai" \ - "__DEFAULT__" \ - "" \ - "2" # Bug 14: Timing message — success should log elapsed time. run_gate_case "vertex-primary-success-timing-message" \ - "vertex_ai/ready-primary" \ + "orchestrator/free" \ "" \ "0" \ - "REGEX:Strix run succeeded for model 'vertex_ai/ready-primary' in [0-9]+s\\." \ + "REGEX:Strix run succeeded for model 'orchestrator/free' in [0-9]+s\\." \ "1" \ - "vertex_ai/ready-primary" \ - "" + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" # is_timeout_error() provider-context marker test: # Bare "Connection timed out" without any LLM provider marker should NOT @@ -11498,127 +9204,67 @@ run_gate_case "vertex-primary-success-timing-message" \ # Model name deliberately avoids containing any provider marker string # (litellm, openai, anthropic, VertexAI, vertex.ai, google.cloud). run_gate_case "bare-timeout-no-provider-marker" \ - "custom/bare-timeout-model" \ + "orchestrator/free" \ "" \ "1" \ "" \ "1" \ - "custom/bare-timeout-model" \ - "https://example.invalid" \ - "custom" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ - "1" + "0" # is_timeout_error() Tier 2: httpx.ReadTimeout + provider-context marker. # The timeout should be classified for fallback, not same-model retry. -run_gate_case_allow_provider_signal "httpx-read-timeout-with-provider-marker" \ - "vertex_ai/httpx-timeout-primary" \ - "vertex_ai/fallback-one" \ - "0" \ - "scan ok after httpx-timeout fallback" \ - "2" \ - "vertex_ai/httpx-timeout-primary|vertex_ai/fallback-one" \ - "|" \ - "vertex_ai" \ - "__DEFAULT__" \ - "" \ - "1" # Negative: httpx.ReadTimeout WITHOUT provider-context marker should NOT # be classified as a retryable timeout (the gate should treat it as a # non-recoverable scan failure). run_gate_case "httpx-read-timeout-no-provider-marker" \ - "custom/httpx-timeout-no-ctx" \ + "orchestrator/free" \ "" \ "1" \ - "non-recoverable error" \ + "failing closed without repository-authored retry or fallback allocation" \ "1" \ - "custom/httpx-timeout-no-ctx" \ - "https://example.invalid" \ - "custom" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ - "1" + "0" # is_timeout_error() Tier 2b: httpcore.ReadTimeout + provider-context marker. # Mirrors the httpx.ReadTimeout positive case above, but falls back immediately. -run_gate_case_allow_provider_signal "httpcore-read-timeout-with-provider-marker" \ - "vertex_ai/httpcore-timeout-primary" \ - "vertex_ai/fallback-one" \ - "0" \ - "scan ok after httpcore-timeout fallback" \ - "2" \ - "vertex_ai/httpcore-timeout-primary|vertex_ai/fallback-one" \ - "|" \ - "vertex_ai" \ - "__DEFAULT__" \ - "" \ - "1" # Negative: httpcore.ReadTimeout WITHOUT provider-context marker should NOT # be classified as a retryable timeout (the gate should treat it as a # non-recoverable scan failure). run_gate_case "httpcore-read-timeout-no-provider-marker" \ - "custom/httpcore-timeout-no-ctx" \ + "orchestrator/free" \ "" \ "1" \ - "non-recoverable error" \ + "failing closed without repository-authored retry or fallback allocation" \ "1" \ - "custom/httpcore-timeout-no-ctx" \ - "https://example.invalid" \ - "custom" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ - "1" + "0" # is_timeout_error() positive branch for "Connection timed out" + provider marker: # When "Connection timed out" appears alongside an LLM provider marker, the # gate should classify it as a timeout and move to fallback. -run_gate_case_allow_provider_signal "bare-timeout-with-provider-marker" \ - "vertex_ai/bare-timeout-primary" \ - "vertex_ai/fallback-one" \ - "0" \ - "scan ok after bare-timeout fallback" \ - "2" \ - "vertex_ai/bare-timeout-primary|vertex_ai/fallback-one" \ - "|" \ - "vertex_ai" \ - "__DEFAULT__" \ - "" \ - "1" # Bare "Connection timed out" + provider marker: primary fails once, # then gate falls back to fallback-one which succeeds. -run_gate_case_allow_provider_signal "bare-timeout-provider-marker-exhausted-fallback" \ - "vertex_ai/bare-timeout-exhaust-primary" \ - "vertex_ai/fallback-one" \ - "0" \ - "scan ok after bare-timeout-exhaust fallback" \ - "2" \ - "vertex_ai/bare-timeout-exhaust-primary|vertex_ai/fallback-one" \ - "|" \ - "vertex_ai" \ - "__DEFAULT__" \ - "" \ - "1" # Sticky INFRA_ERROR_DETECTED flag: first call hits rate-limit (infra error), # second call fails with a non-retryable error but leaves a partial LOW report. # The gate must refuse the below-threshold bypass because an infrastructure # error was detected during this pipeline run. -run_gate_case_allow_provider_signal "infra-error-sticky-flag" \ - "vertex_ai/sticky-flag-primary" \ - "" \ - "1" \ - "infrastructure errors occurred" \ - "3" \ - "vertex_ai/sticky-flag-primary|vertex_ai/sticky-flag-primary|vertex_ai/gemini-2.5-pro" \ - "||" \ - "vertex_ai" \ - "__DEFAULT__" \ - "" \ - "1" run_invalid_min_fail_severity_case run_required_input_file_outside_input_root_fails_closed_case "STRIX_LLM_FILE" @@ -11632,34 +9278,17 @@ run_symlink_report_case run_unsafe_target_path_case run_absolute_outside_target_path_case -run_gate_case_allow_provider_signal "slow-timeout" \ - "vertex_ai/slow-primary" \ - "" \ - "1" \ - "Strix run timed out after ${TIMEOUT_TEST_PROCESS_SECONDS}s." \ - "3" \ - "vertex_ai/slow-primary|vertex_ai/gemini-2.5-pro|vertex_ai/gemini-2.5-flash" \ - "||" \ - "vertex_ai" \ - "__DEFAULT__" \ - "" \ - "0" \ - "CRITICAL" \ - "0" \ - "" \ - "" \ - "$TIMEOUT_TEST_PROCESS_SECONDS" run_gate_case "timeout-disabled-success" \ - "vertex_ai/timeout-disabled-primary" \ + "orchestrator/free" \ "" \ "0" \ "scan ok with timeout disabled" \ "1" \ - "vertex_ai/timeout-disabled-primary" \ - "" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -11673,15 +9302,15 @@ run_timeout_cleanup_case run_total_timeout_case run_gate_case "pr-changed-scope-bounded" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "0" \ "scan ok with bounded changed-file scope" \ "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -11694,15 +9323,15 @@ run_gate_case "pr-changed-scope-bounded" \ "sync-module-system/smart-crawling-biz/src/main/java/org/empasy/sync/modules/system/controller/SysPositionController.java" run_gate_case "scan-working-directory-isolated" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "0" \ "scan ok with isolated Strix working directory" \ "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -11715,15 +9344,15 @@ run_gate_case "scan-working-directory-isolated" \ "backend/app/pg_introspect/introspect.py" run_gate_case "pr-python-scope-context" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "0" \ "scan ok with python dependency scope" \ "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -11736,15 +9365,15 @@ run_gate_case "pr-python-scope-context" \ "backend/api/emails.py" run_gate_case "pr-changed-scope-full" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "0" \ "Scoped pull request Strix scan to 3 changed file(s)." \ "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -11757,15 +9386,15 @@ run_gate_case "pr-changed-scope-full" \ $'sync-module-system/smart-crawling-biz/src/main/java/org/empasy/sync/modules/system/controller/SysPositionController.java\nsync-module-system/smart-crawling-playwright/src/main/java/org/empasy/sync/mcp/service/PlayWrightService.java\nsync-module-system/smart-crawling-biz/src/main/java/org/empasy/sync/modules/system/service/impl/SysUserServiceImpl.java' run_gate_case "pr-changed-scope-full-set" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "0" \ "scan ok with full configured PR scope" \ "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -11789,15 +9418,15 @@ for large_pr_index in $(seq 1 38); do done run_gate_case "pr-large-scope-full-set" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "0" \ "scan ok with large full PR scope" \ "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -11812,15 +9441,15 @@ run_gate_case "pr-large-scope-full-set" \ "12" run_gate_case "pr-changed-scope-includes-ci-dependency" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "0" \ "scan ok with CI support dependency" \ "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -11838,15 +9467,15 @@ run_gate_case "pr-changed-scope-includes-ci-dependency" \ # fuzz/fuzz_opencode_normalize_output.py duplicate). A PR that changes only # that fuzz target must still pull the normalizer module into scan scope. run_gate_case "pr-changed-scope-includes-opencode-normalizer" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "0" \ "scan ok with opencode normalizer support dependency" \ "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -11859,15 +9488,15 @@ run_gate_case "pr-changed-scope-includes-opencode-normalizer" \ "fuzz/fuzz_opencode_review_normalize_output.py" run_gate_case "pr-ci-test-harness-only-skip" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "0" \ "No scannable changed files in pull request; skipping Strix quick scan." \ "0" \ "" \ "" \ - "vertex_ai" \ - "__DEFAULT__" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -11880,15 +9509,15 @@ run_gate_case "pr-ci-test-harness-only-skip" \ "scripts/ci/test_strix_quick_gate.sh" run_gate_case "pr-deployment-scope-entrypoint-context" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "0" \ "scan ok with deployment entrypoint context" \ "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -11901,15 +9530,15 @@ run_gate_case "pr-deployment-scope-entrypoint-context" \ ".github/workflows/opencode-review.yml" run_gate_case "pr-rust-workspace-context" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "0" \ "scan ok with Rust workspace context" \ "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -11922,15 +9551,15 @@ run_gate_case "pr-rust-workspace-context" \ ".github/workflows/rust.yml" run_gate_case "pr-empty-diff-skip" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "0" \ "No scannable changed files in pull request; skipping Strix quick scan." \ "0" \ "" \ "" \ - "vertex_ai" \ - "__DEFAULT__" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -11943,15 +9572,15 @@ run_gate_case "pr-empty-diff-skip" \ "__SET_EMPTY__" run_gate_case "pr-baseline-critical-unchanged" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "0" \ "Strix findings are limited to unchanged files in this pull request; allowing pipeline continuation." \ "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -11964,15 +9593,15 @@ run_gate_case "pr-baseline-critical-unchanged" \ "sync-module-system/smart-crawling-biz/src/main/java/org/empasy/sync/modules/system/controller/SysPositionController.java" run_gate_case "pr-baseline-critical-absolute-target" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "0" \ "Strix findings are limited to unchanged files in this pull request; allowing pipeline continuation." \ "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -11985,15 +9614,15 @@ run_gate_case "pr-baseline-critical-absolute-target" \ "sync-module-system/smart-crawling-biz/src/main/java/org/empasy/sync/modules/system/controller/SysPositionController.java" run_gate_case "pr-baseline-critical-extensionless-dockerfile-target" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "0" \ "Strix findings are limited to unchanged files in this pull request; allowing pipeline continuation." \ "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -12006,15 +9635,15 @@ run_gate_case "pr-baseline-critical-extensionless-dockerfile-target" \ ".github/workflows/opencode-review.yml" run_gate_case "pr-baseline-critical-subdir-target" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "0" \ "Strix findings are limited to unchanged files in this pull request; allowing pipeline continuation." \ "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -12030,15 +9659,15 @@ run_gate_case "pr-baseline-critical-subdir-target" \ "1" run_gate_case "pr-baseline-critical-subdir-boxed-target" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "0" \ "Strix findings are limited to unchanged files in this pull request; allowing pipeline continuation." \ "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -12054,15 +9683,15 @@ run_gate_case "pr-baseline-critical-subdir-boxed-target" \ "1" run_gate_case "pr-baseline-critical-subdir-endpoint" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "0" \ "Strix findings are limited to unchanged files in this pull request; allowing pipeline continuation." \ "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -12078,15 +9707,15 @@ run_gate_case "pr-baseline-critical-subdir-endpoint" \ "1" run_gate_case "pr-baseline-critical-subdir-endpoint-bare-filename" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "0" \ "Strix findings are limited to unchanged files in this pull request; allowing pipeline continuation." \ "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -12102,15 +9731,15 @@ run_gate_case "pr-baseline-critical-subdir-endpoint-bare-filename" \ "1" run_gate_case "pr-baseline-critical-subdir-narrative-backticked-file" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "0" \ "Strix findings are limited to unchanged files in this pull request; allowing pipeline continuation." \ "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -12126,15 +9755,15 @@ run_gate_case "pr-baseline-critical-subdir-narrative-backticked-file" \ "1" run_gate_case "pr-critical-relative-path-escape-subdir-narrative-backticked-file" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "1" \ "Unable to map Strix findings to changed files; failing closed for pull request." \ "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -12150,15 +9779,15 @@ run_gate_case "pr-critical-relative-path-escape-subdir-narrative-backticked-file "1" run_gate_case "pr-critical-changed" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "1" \ "Strix finding intersects files changed in this pull request." \ "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -12171,15 +9800,15 @@ run_gate_case "pr-critical-changed" \ "sync-module-system/smart-crawling-biz/src/main/java/org/empasy/sync/modules/system/controller/SysPositionController.java" run_gate_case "pr-changed-file-nonintersecting-line" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "0" \ "Strix findings are limited to unchanged files in this pull request; allowing pipeline continuation." \ "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -12191,15 +9820,15 @@ run_gate_case "pr-changed-file-nonintersecting-line" \ "pull_request" run_gate_case "pr-critical-changed-bracketed-next-route" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "1" \ "Strix finding intersects files changed in this pull request." \ "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -12212,15 +9841,15 @@ run_gate_case "pr-critical-changed-bracketed-next-route" \ "frontend/src/app/labels/[slug]/page.tsx" run_gate_case "pr-critical-changed-xml-file-location" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "1" \ "Strix finding intersects files changed in this pull request." \ "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "MEDIUM" \ @@ -12233,15 +9862,15 @@ run_gate_case "pr-critical-changed-xml-file-location" \ "sync-module-system/smart-crawling-biz/src/main/java/org/empasy/sync/modules/system/controller/SysPositionController.java" run_gate_case "pr-critical-changed-xml-file-location-space" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "1" \ "Strix finding intersects files changed in this pull request." \ "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "MEDIUM" \ @@ -12254,15 +9883,15 @@ run_gate_case "pr-critical-changed-xml-file-location-space" \ "src/unsafe name.py" run_gate_case "pr-baseline-critical-narrative-backticked-service-file" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "0" \ "Strix findings are limited to unchanged files in this pull request; allowing pipeline continuation." \ "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -12275,15 +9904,15 @@ run_gate_case "pr-baseline-critical-narrative-backticked-service-file" \ "backend/services/email_client.py" run_gate_case "pr-critical-unmapped-arbitrary-backticked-service-file" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "1" \ "Unable to map Strix findings to changed files; failing closed for pull request." \ "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -12296,15 +9925,15 @@ run_gate_case "pr-critical-unmapped-arbitrary-backticked-service-file" \ "backend/services/email_client.py" run_gate_case "pr-critical-changed-absolute-target" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "1" \ "Strix finding intersects files changed in this pull request." \ "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -12317,15 +9946,15 @@ run_gate_case "pr-critical-changed-absolute-target" \ "sync-module-system/smart-crawling-playwright/src/main/java/org/empasy/sync/mcp/service/PlayWrightService.java" run_gate_case "pr-critical-changed-internal-dotdir-target" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "1" \ "Strix finding intersects files changed in this pull request." \ "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -12338,15 +9967,15 @@ run_gate_case "pr-critical-changed-internal-dotdir-target" \ ".github/workflows/opencode-review.yml" run_gate_case "pr-critical-changed-json-target" \ - "vertex_ai/gemini-2.5-pro" \ + "orchestrator/free" \ "" \ "1" \ "Strix finding intersects files changed in this pull request." \ "1" \ - "vertex_ai/gemini-2.5-pro" \ - "" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "MEDIUM" \ @@ -12359,15 +9988,15 @@ run_gate_case "pr-critical-changed-json-target" \ "frontend/src/components/CalendarLayout.tsx" run_gate_case "pr-critical-changed-subdir-target" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "1" \ "Strix finding intersects files changed in this pull request." \ "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -12383,15 +10012,15 @@ run_gate_case "pr-critical-changed-subdir-target" \ "1" run_gate_case "pr-critical-changed-subdir-endpoint" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "1" \ "Strix finding intersects files changed in this pull request." \ "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -12407,15 +10036,15 @@ run_gate_case "pr-critical-changed-subdir-endpoint" \ "1" run_gate_case "pr-critical-path-escape-subdir-target" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "1" \ "Unable to map Strix findings to changed files; failing closed for pull request." \ "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -12431,15 +10060,15 @@ run_gate_case "pr-critical-path-escape-subdir-target" \ "1" run_gate_case "pr-critical-unmapped" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "1" \ "Unable to map Strix findings to changed files; failing closed for pull request." \ "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -12452,15 +10081,15 @@ run_gate_case "pr-critical-unmapped" \ "sync-module-system/smart-crawling-biz/src/main/java/org/empasy/sync/modules/system/controller/SysPositionController.java" run_gate_case "pr-critical-unmapped-narrative-target" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "1" \ "Unable to map Strix findings to changed files; failing closed for pull request." \ "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -12473,15 +10102,15 @@ run_gate_case "pr-critical-unmapped-narrative-target" \ "sync-module-system/smart-crawling-playwright/src/main/java/org/empasy/sync/mcp/service/PlayWrightService.java" run_gate_case "pr-critical-unmapped-other-workspace-repo" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "1" \ "Unable to map Strix findings to changed files; failing closed for pull request." \ "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -12494,15 +10123,15 @@ run_gate_case "pr-critical-unmapped-other-workspace-repo" \ "sync-module-system/smart-crawling-playwright/src/main/java/org/empasy/sync/mcp/service/PlayWrightService.java" run_gate_case "pr-critical-manifest-only-pom" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "1" \ "Strix changed-manifest threshold finding requires package and CVE remediation; pull-request-controlled SCA workflow results cannot override model evidence, so the scan is failing closed." \ "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -12515,15 +10144,15 @@ run_gate_case "pr-critical-manifest-only-pom" \ "pom.xml" run_gate_case "pr-critical-manifest-only-pom-test-override" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "1" \ "Strix changed-manifest threshold finding requires package and CVE remediation; pull-request-controlled SCA workflow results cannot override model evidence, so the scan is failing closed." \ "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -12540,15 +10169,15 @@ run_gate_case "pr-critical-manifest-only-pom-test-override" \ "passed" run_gate_case "pr-critical-manifest-only-pom-same-head-different-pr" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "1" \ "Strix changed-manifest threshold finding requires package and CVE remediation; pull-request-controlled SCA workflow results cannot override model evidence, so the scan is failing closed." \ "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -12567,15 +10196,15 @@ run_gate_case "pr-critical-manifest-only-pom-same-head-different-pr" \ '{"workflow_runs":[{"id":201,"name":"Dependency review","path":".github/workflows/dependency-review.yml","head_sha":"test-head-sha","status":"completed","conclusion":"success","pull_requests":[{"number":456}]},{"id":202,"name":"OSV-Scanner","path":".github/workflows/osvscanner.yml","head_sha":"test-head-sha","status":"completed","conclusion":"success","pull_requests":[{"number":456}]}]}' run_gate_case "pr-critical-manifest-only-pom-current-pr-authoritative" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "1" \ "Strix changed-manifest threshold finding requires package and CVE remediation; pull-request-controlled SCA workflow results cannot override model evidence, so the scan is failing closed." \ "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -12593,118 +10222,14 @@ run_gate_case "pr-critical-manifest-only-pom-current-pr-authoritative" \ "123" \ '{"workflow_runs":[{"id":301,"name":"Dependency review","path":".github/workflows/dependency-review.yml","head_sha":"test-head-sha","status":"completed","conclusion":"success","pull_requests":[{"number":123}]},{"id":302,"name":"OSV-Scanner","path":".github/workflows/osvscanner.yml","head_sha":"test-head-sha","status":"completed","conclusion":"success","pull_requests":[{"number":123}]}]}' -run_gate_case_allow_provider_signal "pr-critical-manifest-only-pom-after-fallback-authoritative" \ - "vertex_ai/timeout-primary" \ - "vertex_ai/fallback-one" \ - "1" \ - "Strix changed-manifest threshold finding requires package and CVE remediation; pull-request-controlled SCA workflow results cannot override model evidence, so the scan is failing closed." \ - "2" \ - "vertex_ai/timeout-primary|vertex_ai/fallback-one" \ - "|" \ - "vertex_ai" \ - "__DEFAULT__" \ - "" \ - "0" \ - "CRITICAL" \ - "0" \ - "" \ - "" \ - "1200" \ - "0" \ - "pull_request" \ - "pom.xml" \ - "" \ - "" \ - "0" \ - "" \ - "123" \ - '{"workflow_runs":[{"id":401,"name":"Dependency review","path":".github/workflows/dependency-review.yml","head_sha":"test-head-sha","status":"completed","conclusion":"success","pull_requests":[{"number":123}]},{"id":402,"name":"OSV-Scanner","path":".github/workflows/osvscanner.yml","head_sha":"test-head-sha","status":"completed","conclusion":"success","pull_requests":[{"number":123}]}]}' -run_gate_case_allow_provider_signal "pr-critical-manifest-only-pom-console-only-after-fallback-authoritative" \ - "vertex_ai/timeout-primary" \ - "vertex_ai/fallback-one" \ - "1" \ - "Strix changed-manifest threshold finding requires package and CVE remediation; pull-request-controlled SCA workflow results cannot override model evidence, so the scan is failing closed." \ - "2" \ - "vertex_ai/timeout-primary|vertex_ai/fallback-one" \ - "|" \ - "vertex_ai" \ - "__DEFAULT__" \ - "" \ - "0" \ - "CRITICAL" \ - "0" \ - "" \ - "" \ - "1200" \ - "0" \ - "pull_request" \ - "pom.xml" \ - "" \ - "" \ - "0" \ - "" \ - "123" \ - '{"workflow_runs":[{"id":403,"name":"Dependency review","path":".github/workflows/dependency-review.yml","head_sha":"test-head-sha","status":"completed","conclusion":"success","pull_requests":[{"number":123}]},{"id":404,"name":"OSV-Scanner","path":".github/workflows/osvscanner.yml","head_sha":"test-head-sha","status":"completed","conclusion":"success","pull_requests":[{"number":123}]}]}' -run_gate_case_allow_provider_signal "pr-critical-manifest-only-pom-console-target-only-after-fallback-authoritative" \ - "vertex_ai/timeout-primary" \ - "vertex_ai/fallback-one" \ - "1" \ - "Strix changed-manifest threshold finding requires package and CVE remediation; pull-request-controlled SCA workflow results cannot override model evidence, so the scan is failing closed." \ - "2" \ - "vertex_ai/timeout-primary|vertex_ai/fallback-one" \ - "|" \ - "vertex_ai" \ - "__DEFAULT__" \ - "" \ - "0" \ - "CRITICAL" \ - "0" \ - "" \ - "" \ - "1200" \ - "0" \ - "pull_request" \ - "pom.xml" \ - "" \ - "" \ - "0" \ - "" \ - "123" \ - '{"workflow_runs":[{"id":405,"name":"Dependency review","path":".github/workflows/dependency-review.yml","head_sha":"test-head-sha","status":"completed","conclusion":"success","pull_requests":[{"number":123}]},{"id":406,"name":"OSV-Scanner","path":".github/workflows/osvscanner.yml","head_sha":"test-head-sha","status":"completed","conclusion":"success","pull_requests":[{"number":123}]}]}' -run_gate_case_allow_provider_signal "pr-low-markdown-plus-console-critical-manifest-after-fallback-authoritative" \ - "vertex_ai/timeout-primary" \ - "vertex_ai/fallback-one" \ - "1" \ - "Strix changed-manifest threshold finding requires package and CVE remediation; pull-request-controlled SCA workflow results cannot override model evidence, so the scan is failing closed." \ - "2" \ - "vertex_ai/timeout-primary|vertex_ai/fallback-one" \ - "|" \ - "vertex_ai" \ - "__DEFAULT__" \ - "" \ - "0" \ - "CRITICAL" \ - "0" \ - "" \ - "" \ - "1200" \ - "0" \ - "pull_request" \ - "pom.xml" \ - "" \ - "" \ - "0" \ - "" \ - "123" \ - '{"workflow_runs":[{"id":405,"name":"Dependency review","path":".github/workflows/dependency-review.yml","head_sha":"test-head-sha","status":"completed","conclusion":"success","pull_requests":[{"number":123}]},{"id":406,"name":"OSV-Scanner","path":".github/workflows/osvscanner.yml","head_sha":"test-head-sha","status":"completed","conclusion":"success","pull_requests":[{"number":123}]}]}' run_missing_config_case "missing-strix-llm" "" "dummy" "ERROR: STRIX_LLM_FILE must reference a regular file containing the model." -run_missing_config_case "missing-llm-api-key" "openai/gpt-5.4" "" "ERROR: LLM_API_KEY_FILE must reference a regular file containing the API key." +run_missing_config_case "missing-llm-api-key" "orchestrator/free" "" "ERROR: LLM_API_KEY_FILE must reference a regular file containing the API key." run_missing_config_case "whitespace-only-strix-llm" " " "dummy" "ERROR: STRIX_LLM_FILE must contain a non-empty model value." -run_missing_config_case "whitespace-only-llm-api-key" "openai/gpt-5.4" $'\t ' "ERROR: LLM_API_KEY_FILE must contain a non-empty API key." +run_missing_config_case "whitespace-only-llm-api-key" "orchestrator/free" $'\t ' "ERROR: LLM_API_KEY_FILE must contain a non-empty API key." run_strix_llm_file_command_substitution_literal_case run_vertex_without_llm_api_key_case run_vertex_with_llm_api_key_file_does_not_forward_case @@ -12851,17 +10376,11 @@ assert_vertex_extract "non-vertex-passthrough" "deepseek/models/deepseek-r1" "de assert_vertex_extract "plain-model-passthrough" "gemini-2.5-pro" "gemini-2.5-pro" # Explicit Vertex resource paths require an explicit Vertex provider context. -assert_normalized_model \ +assert_normalize_model_rejected \ "vertex-resource-ignores-nonvertex-default-provider" \ "projects/my-proj/locations/us-central1/publishers/google/models/gemini-2.5-pro" \ - "vertex_ai" \ - "vertex_ai/gemini-2.5-pro" - -assert_model_requires_vertex_auth "explicit-vertex" "vertex_ai/gemini-2.5-pro" "gemini" "0" -assert_model_requires_vertex_auth "explicit-vertex-beta" "vertex_ai_beta/gemini-2.5-pro" "gemini" "0" -assert_model_requires_vertex_auth "vertex-resource-path" "projects/my-proj/locations/us-central1/models/gemini-2.5-pro" "vertex_ai" "0" -assert_model_requires_vertex_auth "implicit-vertex-default" "gemini-2.5-pro" "vertex_ai" "0" -assert_model_requires_vertex_auth "nonvertex-provider" "gemini/gemini-2.5-pro" "gemini" "1" + "vertex_ai" + assert_normalize_model_rejected "bare-models-openai-context" "models/attacker-selected" "openai" assert_normalize_model_rejected "bare-models-empty-context" "models/attacker-selected" "" @@ -12874,247 +10393,126 @@ run_gate_case "github-models-model-prefix-requires-api-base" \ "openai/openai/gpt-5.4" \ "" \ "2" \ - "GitHub Models Strix scans require LLM_API_BASE_FILE" \ + "ERROR: Strix model must be orchestrator/free through contextual-orchestrator; direct provider/model routes are forbidden" \ "0" \ "" \ "" \ - "openai" \ - "" + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" run_gate_case "custom-openai-compatible-preserves-effort" \ "openai-direct/gpt-5.4" \ "" \ + "2" \ + "ERROR: Strix model must be orchestrator/free through contextual-orchestrator; direct provider/model routes are forbidden" \ "0" \ - "scan ok" \ - "1" \ - "openai/gpt-5.4" \ - "https://compatible.example/v1" \ - "openai" \ - "https://compatible.example/v1" + "" \ + "" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" run_gate_case "github-models-api-base-rejected-for-direct-openai" \ "openai/o4-mini" \ "" \ "2" \ - "LLM_API_BASE may route through GitHub Models only when STRIX_LLM uses a GitHub Models-compatible model" \ + "ERROR: Strix model must be orchestrator/free through contextual-orchestrator; direct provider/model routes are forbidden" \ "0" \ "" \ "" \ - "openai" \ - "https://models.github.ai/inference" + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" run_gate_case "github-models-openai-gpt-requires-api-base" \ "openai/gpt-5" \ "" \ "2" \ - "GitHub Models Strix scans require LLM_API_BASE_FILE" \ + "ERROR: Strix model must be orchestrator/free through contextual-orchestrator; direct provider/model routes are forbidden" \ "0" \ "" \ "" \ - "openai" \ - "" + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" run_gate_case "direct-openai-gpt-does-not-require-github-models-api-base" \ "openai_direct/gpt-5.4" \ "" \ + "2" \ + "ERROR: Strix model must be orchestrator/free through contextual-orchestrator; direct provider/model routes are forbidden" \ "0" \ - "scan ok" \ - "1" \ - "openai/gpt-5.4" \ - "" \ - "openai" \ - "" + "" \ + "" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" run_gate_case "github-models-model-prefix-with-api-base-succeeds" \ "openai/gpt-5" \ "" \ + "2" \ + "ERROR: Strix model must be orchestrator/free through contextual-orchestrator; direct provider/model routes are forbidden" \ "0" \ - "scan ok" \ - "1" \ - "openai/gpt-5" \ - "https://models.github.ai/inference" \ - "openai" \ - "https://models.github.ai/inference" + "" \ + "" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" run_gate_case "github-models-meta-prefix-with-api-base-succeeds" \ "openai/meta/test-github-model" \ "" \ + "2" \ + "ERROR: Strix model must be orchestrator/free through contextual-orchestrator; direct provider/model routes are forbidden" \ "0" \ - "scan ok" \ - "1" \ - "openai/meta/test-github-model" \ - "https://models.github.ai/inference" \ - "openai" \ - "https://models.github.ai/inference" + "" \ + "" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" run_gate_case "github-models-mistral-prefix-with-api-base-succeeds" \ "openai/mistral-ai/test-github-model" \ "" \ - "0" \ - "scan ok" \ - "1" \ - "openai/mistral-ai/test-github-model" \ - "https://models.github.ai/inference" \ - "openai" \ - "https://models.github.ai/inference" - -run_gate_case "github-models-fallback-requires-api-base" \ - "vertex_ai/missing-primary" \ - "openai/openai/gpt-5.4" \ "2" \ - "GitHub Models Strix scans require LLM_API_BASE_FILE" \ - "1" \ - "vertex_ai/missing-primary" \ - "" \ - "vertex_ai" \ - "" - -run_gate_case "github-models-fallback-success" \ - "vertex_ai/missing-primary" \ - "github_models/deepseek/deepseek-v3-0324 github_models/deepseek/deepseek-r1-0528" \ + "ERROR: Strix model must be orchestrator/free through contextual-orchestrator; direct provider/model routes are forbidden" \ "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'github_models/deepseek/deepseek-v3-0324' in [0-9]+s\\." \ - "2" \ - "vertex_ai/missing-primary|openai/deepseek/deepseek-v3-0324" \ - "|https://models.github.ai/inference" \ - "vertex_ai" \ - "https://models.github.ai/inference" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ "" \ "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - 0 + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" -run_gate_case "github-models-token-limit-fallback-success" \ - "openai/gpt-5" \ +run_gate_case "github-models-fallback-requires-api-base" \ + "vertex_ai/missing-primary" \ "" \ - "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'github_models/deepseek/deepseek-v3-0324' in [0-9]+s\\." \ "2" \ - "openai/gpt-5|openai/deepseek/deepseek-v3-0324" \ - "https://models.github.ai/inference|https://models.github.ai/inference" \ - "openai" \ - "https://models.github.ai/inference" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ + "ERROR: Strix model must be orchestrator/free through contextual-orchestrator; direct provider/model routes are forbidden" \ + "0" \ "" \ "" \ - "github_models/deepseek/deepseek-v3-0324 github_models/deepseek/deepseek-r1-0528" + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" + + # Direct-OpenAI primary hits a quota/rate-limit error and falls back to a # GitHub Models candidate, switching both the API base and the API key per # model (the fake strix asserts the key swap and exits nonzero on a leak). -run_gate_case "openai-direct-quota-github-models-fallback-success" \ - "openai_direct/gpt-5.4" \ - "" \ - "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'github_models/openai/o3' in [0-9]+s\\." \ - "2" \ - "openai/gpt-5.4|openai/o3" \ - "|https://models.github.ai/inference" \ - "vertex_ai" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "github_models/openai/o3" -run_gate_case "github-models-fallback-success-deepseek-v3" \ - "vertex_ai/missing-primary" \ - "github_models/deepseek/deepseek-r1-0528 github_models/deepseek/deepseek-v3-0324" \ - "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'github_models/deepseek/deepseek-v3-0324' in [0-9]+s\\." \ - "3" \ - "vertex_ai/missing-primary|openai/deepseek/deepseek-r1-0528|openai/deepseek/deepseek-v3-0324" \ - "|https://models.github.ai/inference|https://models.github.ai/inference" \ - "vertex_ai" \ - "https://models.github.ai/inference" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - 0 # Endpoint only exists in excluded directories (.git/, node_modules/). Even if # the source does not corroborate it, a threshold report remains blocking and # requires human remediation/triage rather than silent fallback. run_gate_case "endpoint-in-excluded-dir" \ - "vertex_ai/excluded-dir-primary" \ - "vertex_ai/fallback-one vertex_ai/fallback-two" \ + "orchestrator/free" \ + "" \ "1" \ "Unable to map Strix findings to changed files; failing closed for pull request." \ "1" \ - "vertex_ai/excluded-dir-primary" \ - "" + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" # Whitespace-only fallback models: STRIX_VERTEX_FALLBACK_MODELS set to " ". # This bypasses the :- default but produces an empty array from read -r -a. # The gate should emit "No fallback models configured" (not the misleading # "All configured fallback models are the same as the primary model"). -run_gate_case "empty-fallback-models" \ - "vertex_ai/empty-fb-primary" \ - " " \ - "1" \ - "No fallback models configured" \ - "1" \ - "vertex_ai/empty-fb-primary" \ - "" - if [ "$FAILURES" -ne 0 ]; then echo "test_strix_quick_gate: ${FAILURES} failure(s)" >&2 exit 1 From 305e934a3ff0bcfed8a48bfbf4de895f338b03f0 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 22:06:29 +0900 Subject: [PATCH 14/14] docs(adr): renumber the Strix free-only boundary ADR to 0028 main already carries docs/adr/0021-hourly-review-repair-single-file-consolidation.md, which arrived on this branch through a merge from main. Two ADRs cannot share a number, and 0021..0027 are all taken on main, so this one becomes 0028. Only the ADR's own file and title move. The other ADR-0021 mentions on this branch (ARCHITECTURE.md, CHANGELOG.md, docs/doctoring/scheduler-target-list-drift-20260902.md) all refer to the hourly-review-repair consolidation and stay as they are. Co-Authored-By: Claude Fable 5.1 --- ...ary.md => 0028-strix-orchestrator-free-only-model-boundary.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename docs/adr/{0021-strix-orchestrator-free-only-model-boundary.md => 0028-strix-orchestrator-free-only-model-boundary.md} (100%) diff --git a/docs/adr/0021-strix-orchestrator-free-only-model-boundary.md b/docs/adr/0028-strix-orchestrator-free-only-model-boundary.md similarity index 100% rename from docs/adr/0021-strix-orchestrator-free-only-model-boundary.md rename to docs/adr/0028-strix-orchestrator-free-only-model-boundary.md