diff --git a/.github/workflows/strix.yml b/.github/workflows/strix.yml index 58ed3dab8d..c0bf1a459e 100644 --- a/.github/workflows/strix.yml +++ b/.github/workflows/strix.yml @@ -928,7 +928,6 @@ jobs: PNPM_CONFIG_IGNORE_SCRIPTS: "true" YARN_ENABLE_SCRIPTS: "false" BUN_CONFIG_IGNORE_SCRIPTS: "true" - STRIX_FAIL_ON_MIN_SEVERITY: MEDIUM STRIX_DISABLE_PR_SCOPING: ${{ (github.event_name == 'pull_request_target' || github.event.client_payload.pr_number != '') && '0' || '1' }} # A repository_dispatch executes in this central repository, so its # github.token cannot read the target repository's PR. Reuse the @@ -952,8 +951,8 @@ jobs: model_behavior_error_signal='(^|[^A-Za-z0-9_])(agents|pydantic_ai|strix)(\.[A-Za-z_][A-Za-z0-9_]*)*\.ModelBehaviorError([^A-Za-z0-9_]|$)' # Any evidence that a vulnerability was actually reported. Its presence # forces a hard failure so real findings are NEVER downgraded. Keep the - # severity branch anchored away from identifiers so environment lines - # such as STRIX_FAIL_ON_MIN_SEVERITY do not look like findings. + # Keep the severity marker anchored away from identifiers so unrelated + # environment text does not look like a reported finding. reported_vulnerability_signal='Vulnerabilities[[:space:]]+[1-9]|(^|[^A-Za-z0-9_])severity[[:space:]]*:' # Capture the gate exit code plus its console output. The gate returns @@ -988,7 +987,7 @@ jobs: exit "$strix_rc" fi - # An earlier out-of-scope/below-threshold finding may already have + # An earlier out-of-scope finding may already have # been exempted by the trusted gate. Classify a later provider # outage from the tail after the last continuation marker, but keep # that incomplete later scan non-passing. diff --git a/CHANGELOG.md b/CHANGELOG.md index ca81dcea1d..9a51230bff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -118,6 +118,7 @@ this file. The format follows Keep a Changelog, and versioned releases follow Semantic Versioning where the repository publishes a release. ## [Unreleased] +- Strix model normalization now fails closed on every direct-provider model identifier and accepts only the contextual-orchestrator `orchestrator/free` virtual pool spellings. This closes the reusable model-selection bypass without removing `OPENAI_API_KEY` from bootstrap/global discovery; the orchestrator's free-pool credential-source policy remains the candidate-admission authority. - **Pin `opencode-review-dispatch.yml` off the starved floating `ubuntu-latest` image.** The 2026-09-01 floating-image fix (see that entry below) pinned `strix.yml`, `opencode-review.yml`, and `noema-review.yml` -- the three required-check diff --git a/docs/adr/0028-strix-orchestrator-free-only-model-boundary.md b/docs/adr/0028-strix-orchestrator-free-only-model-boundary.md new file mode 100644 index 0000000000..11c3aaea11 --- /dev/null +++ b/docs/adr/0028-strix-orchestrator-free-only-model-boundary.md @@ -0,0 +1,40 @@ +# ADR-0021: Strix model normalization is an `orchestrator/free` boundary + +- Status: Proposed +- Date: 2026-09-02 +- Owner: ContextualWisdomLab/.github central Strix review path + +## Context + +The protected central Strix workflow already provisions `ContextualWisdomLab/contextual-orchestrator`, requests the `orchestrator/free` virtual pool, supplies the five bootstrap credential sources, forces Zero Data Retention for private targets, and sets the workflow's external fallback list to empty. The shared `scripts/ci/strix_quick_gate.sh` nevertheless retained generic normalization for concrete provider model identifiers and historical direct-provider fallback code. That left a reusable execution boundary capable of accepting a provider/model name even though provider choice, free-pool admission, privacy admission, retry/failover, and serving are owned by contextual-orchestrator. + +Under the organization no-heuristics and routing-owner contract, a GitHub Actions review path must not acquire a second provider/model router. The model identifier accepted by the Strix gate is therefore not a preference or fallback ranking input; it is an exact virtual-pool capability boundary. + +## Decision + +`normalize_model` in the Strix model helper accepts only `orchestrator/free` and its provider-qualified spelling `contextual-orchestrator/orchestrator/free`. All provider names, concrete model names, unqualified model names, and historical direct-OpenAI aliases fail closed before provider credentials or provider endpoints can become execution authority. + +The five bootstrap secrets remain transport/discovery inputs to contextual-orchestrator. In particular, `OPENAI_API_KEY` is not removed from bootstrap or global discovery. The separate contextual-orchestrator free-pool admission contract decides which discovered credential sources may become `orchestrator/free` candidates; OpenAI-derived models remain excluded there while OpenAI integration may remain available to independently governed non-free/global pools. + +For private targets, the central workflow's existing `CONTEXTUAL_ORCHESTRATOR_REQUIRE_ZDR=true` path remains mandatory. This ADR does not weaken or duplicate that policy; it prevents the Strix model selector from bypassing it with a concrete provider route. + +## Executable evidence + +`tests/test_strix_orchestrator_free_only.py` exercises the production shell helper. It admits the two exact governed virtual-model spellings and rejects direct OpenAI, OpenRouter, NVIDIA NIM, GitHub Models, Vertex/Gemini, and unqualified model identifiers before execution. + +The full protected exact-head workflow set remains authoritative. Pending or queued runs are not passing evidence. + +## Research and standards basis + +This decision does not invent a quality score, routing rank, or threshold. It removes an overlapping router and delegates selection to the separately governed orchestrator. Privacy admission remains evidence-based and fail-closed. + +OpenRouter. (2026). *Zero data retention*. https://openrouter.ai/docs/guides/features/zdr + +ContextualWisdomLab. (2026, September 2). *ADR-0003: Vendored contextual-orchestrator review sidecar with governed gateway pools*. ContextualWisdomLab/.github. + +## Consequences + +- Strix cannot use a concrete provider/model identifier as an inference route. +- Provider discovery/failover stays inside contextual-orchestrator. +- `OPENAI_API_KEY` may still be registered and globally discovered; the free-pool candidate boundary, not secret transport, excludes OpenAI-derived candidates from `orchestrator/free`. +- Historical direct-provider fallback helpers become unreachable from the accepted Strix model boundary and should be removed as follow-up dead-code cleanup after exact-head regression evidence is available. diff --git a/docs/doctoring/strix-orchestrator-free-model-boundary-2026-09-02.md b/docs/doctoring/strix-orchestrator-free-model-boundary-2026-09-02.md new file mode 100644 index 0000000000..b0d2d43d00 --- /dev/null +++ b/docs/doctoring/strix-orchestrator-free-model-boundary-2026-09-02.md @@ -0,0 +1,33 @@ +# Strix `orchestrator/free` model-boundary doctoring — 2026-09-02 + +## Exact failing evidence + +Protected `main@23df081c36c93da019c89c474351002afb014daa` already hard-pins the central Strix workflow to `contextual-orchestrator/orchestrator/free`, provisions contextual-orchestrator with `CONTEXTUAL_ORCHESTRATOR_POOL: free`, passes all five bootstrap credentials, and sets `STRIX_FALLBACK_MODELS: ""`. The shared Strix gate nevertheless retained direct-provider normalization and direct-OpenAI/OpenRouter/GitHub Models/Vertex fallback machinery. That created a second provider-routing surface beneath a workflow whose accepted owner contract delegates provider selection and failover to contextual-orchestrator. + +## Causal owner + +The reusable owner is `ContextualWisdomLab/.github/scripts/ci/strix_model_utils.sh` together with `strix_quick_gate.sh`, not downstream repositories. The first repair is placed at model normalization so a concrete provider identifier cannot cross into any later credential/base/fallback branch. + +## Test-first repair + +Commit `2083a72dccaa1d96ea423a51af537240fde8a210` adds the regression contract before the production change. It requires exactly the two governed `orchestrator/free` spellings to be admitted and representative direct-provider identifiers to fail closed. Commit `10c1ddf822f1e6336b73a9093a56680fea8f4f54` changes the production normalizer accordingly. + +No arbitrary rank, weight, score, threshold, retry preference, or provider order replaces the removed routing surface. The accepted identifier is a categorical authority boundary; contextual-orchestrator owns all downstream model choice. + +## Credential and privacy correction retained + +This repair intentionally does **not** remove `OPENAI_API_KEY` from central workflow bootstrap. All five credential sources may be registered and globally discovered. The `orchestrator/free` candidate-admission owner remains contextual-orchestrator, where OpenAI-derived candidates are excluded while BYTEZ, NVIDIA NIM primary/subaccount, and OpenRouter sources may be considered subject to explicit free/privacy/capability evidence. + +Private-target ZDR remains enforced by the central workflow and sidecar. A direct provider route is rejected before it could bypass that boundary. + +## Verification status + +Fresh hosted exact-head tests are required before merge. Queued, pending, stale, predecessor-head, or synthetic evidence is non-passing. Historical direct-provider fallback code is now unreachable through the accepted model normalizer but remains cleanup debt until a subsequent exact-head change removes it without losing unrelated Strix scanner behavior. + +## Retry and severity decision repair + +Fresh protected-main evidence showed that the required Strix workflow still allocated two same-model gate retries plus a second outer three-attempt retry loop with fixed backoff values, while the reusable gate classified retryability through hand-authored provider/error regex families. The same path converted Strix severity labels to numeric ranks and used the repository-selected `MEDIUM` cutoff as a merge admission rule. Neither retry allocation nor the severity cutoff had an identified statistical model, authoritative standard, or executable experimental calibration. + +The repair therefore does not substitute different retry counts, backoff constants, severity weights, or cutoffs. The central Strix path executes the governed `orchestrator/free` request once; contextual-orchestrator retains provider discovery/failover authority. Any execution that fails to produce authoritative scan evidence fails closed. Any current vulnerability report artifact also fails closed without a repository-authored severity threshold. Severity labels may remain descriptive evidence, but they are not converted into a local admission score. + +**2026-09-05 addendum: applied by hand, not by the one-shot driver.** `main` had moved substantially since this doctoring entry and the `2083a72d`/`10c1ddf8` commits were authored: `run_current_target_scan` had grown a full cross-model fallback loop (`FALLBACK_MODELS_RAW`, per-candidate retry, `PR_FINDINGS_DECISION`-gated blocking, a `severity_rank`/`STRIX_MAX_SEVERITY_RANK` threshold check) that the driver script's own `simple_scan` template did not anticipate matching text for. Reconciled by hand: replaced the entire function with the single-governed-request form the driver always intended, confirmed the fallback-model system was already present at this PR's own base commit (not a later regression), and fixed the resulting ripple in six tests across four files that exercised the now-removed `is_transient_same_model_retry_error`/`github_models_rate_limit_should_skip_same_model_retry`/`run_strix_with_transient_retry` orchestration directly — three were rewritten to assert only that each underlying signal classifier (Caido bootstrap timing, ModelBehaviorError, NVIDIA NIM 404) still feeds `has_detected_infrastructure_error`, and three multi-attempt-specific assertions were removed as testing behavior that no longer exists. `scripts/ci/source_fix_strix_no_heuristic_retry_severity.py` and its companion workflow are removed here, matching the driver's own documented one-shot lifecycle, since the repair they existed to apply is now complete. diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 1cc9e20313..7b012b31a2 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -2649,6 +2649,16 @@ Higgins, S. S., Crepalde, N., & Fernandes, L. (2021). Segmented multiplexity: A **Follow-up.** If the organization later solves free+ZDR routing robustly enough to deliberately widen required-review CI to `orchestrator/auto` (e.g. once a spend ceiling and reviewer-visible cost evidence exist for that path), the change is exactly one `case` arm plus the corresponding assertions in `test_sidecar_pins_the_pool_to_free_for_github_actions` — this entry is the record of *why* it was narrowed, not a permanent prohibition. +## 2026-09-02 Strix direct-provider model-boundary repair + +**Live gap.** Protected `main@23df081c36c93da019c89c474351002afb014daa` already provisions the central Strix workflow through contextual-orchestrator with `STRIX_MODEL=contextual-orchestrator/orchestrator/free`, `CONTEXTUAL_ORCHESTRATOR_POOL=free`, private-target ZDR enforcement, all five bootstrap credential variables, and no external fallback list. The reusable Strix model normalizer nevertheless continued accepting concrete provider/model identifiers, leaving a second routing surface below the central workflow. + +**Causal owner and repair.** `scripts/ci/strix_model_utils.sh` is the earliest reusable model-admission boundary. Regression commit `2083a72dccaa1d96ea423a51af537240fde8a210` requires direct OpenAI, OpenRouter, NVIDIA NIM, GitHub Models, Vertex/Gemini, and unqualified identifiers to fail closed. Production commit `10c1ddf822f1e6336b73a9093a56680fea8f4f54` admits only `orchestrator/free` and `contextual-orchestrator/orchestrator/free`. No fallback rank, weight, score, threshold, or provider order replaces the removed authority; contextual-orchestrator owns candidate generation, routing, serving, and failover. + +**Credential/privacy boundary.** The central workflow must continue supplying `BYTEZ_API_KEY`, `NVIDIA_NIM_API_KEY`, `NVIDIA_NIM_API_KEY_SUB`, `OPENROUTER_API_KEY`, and `OPENAI_API_KEY` as bootstrap/global-discovery inputs. `OPENAI_API_KEY` transport is not a defect. The separate contextual-orchestrator `orchestrator/free` candidate policy excludes OpenAI-derived candidates while permitting the four eligible provider-account sources only when their explicit free/privacy/capability evidence passes. Private-target ZDR remains mandatory. + +**Verification.** Hosted exact-head checks and independent review remain authoritative. Pending, queued, stale, predecessor-head, or synthetic evidence is non-passing. Historical direct-provider fallback helpers are now unreachable through the accepted model normalizer and remain dead-code cleanup debt until removed with exact-head regression evidence. + ## 2026-09-02 org-queue-sweep investigation: historical conclusion superseded by PR #1821 **Current status (2026-09-04).** The conclusion below was invalidated by live queue evidence. PR #1821 removed the organization-wide Actions-run inventory and cancellation block from `org-queue-sweep` and merged as `11bb6a7871f4d95ab8a3eab616b4264d02327010`. Native per-PR concurrency and the current-head coalescer now own stale-run cancellation; the scheduled sweep retains only missed review, merge, and branch-update recovery. Focused ownership contracts passed 78 tests before merge. This preserves the event-gap recovery described below without paying the repository-wide run-listing and cancellation API cost. diff --git a/scripts/ci/strix_model_utils.sh b/scripts/ci/strix_model_utils.sh old mode 100755 new mode 100644 index 9f20eae670..b010be9868 --- a/scripts/ci/strix_model_utils.sh +++ b/scripts/ci/strix_model_utils.sh @@ -73,45 +73,21 @@ extract_vertex_model_id() { normalize_model() { local model model="$(trim_whitespace "${1-}")" - if [ -z "$model" ]; then - return 0 - fi - - if is_vertex_resource_path "$model"; then - local provider - provider="$(sanitize_provider_name "${DEFAULT_PROVIDER:-}")" || { - echo "ERROR: Vertex resource paths require an explicit vertex_ai or vertex_ai_beta provider." >&2 - return 2 - } - case "$provider" in - vertex_ai | vertex_ai_beta) ;; - *) - echo "ERROR: Vertex resource paths require an explicit vertex_ai or vertex_ai_beta provider." >&2 - return 2 - ;; - esac - printf '%s/%s\n' "$provider" "$(extract_vertex_model_id "$model")" - return 0 - fi - - local provider="${DEFAULT_PROVIDER:-}" - if [ -z "$provider" ]; then - provider="vertex_ai" - fi - provider="$(sanitize_provider_name "$provider")" || return $? + # Strix is an organization review path. Its model selector is therefore a + # policy boundary, not a generic provider normalizer: all inference must go + # through contextual-orchestrator's fail-closed zero-cost virtual pool. + # Provider/model identifiers would bypass the orchestrator's free-candidate + # source, capability, and private-target ZDR admission contracts, so reject + # them before credentials or provider endpoints can participate in execution. case "$model" in - projects/* | models/* | publishers/*) - printf '%s\n' "$model" - return 0 - ;; - */*) + orchestrator/free | contextual-orchestrator/orchestrator/free) printf '%s\n' "$model" return 0 ;; *) - printf '%s/%s\n' "$provider" "$model" - return 0 + echo "ERROR: Strix model must be orchestrator/free through contextual-orchestrator; direct provider/model routes are forbidden: '$model'." >&2 + return 2 ;; esac } diff --git a/scripts/ci/strix_quick_gate.sh b/scripts/ci/strix_quick_gate.sh index 3a563d7020..6fdfc4ad4f 100755 --- a/scripts/ci/strix_quick_gate.sh +++ b/scripts/ci/strix_quick_gate.sh @@ -1,10 +1,11 @@ #!/usr/bin/env bash -# strix_quick_gate.sh — CI gate that runs Strix security scans with -# automatic model fallback, transient-error retry, and severity-based -# pass/fail decisions. +# strix_quick_gate.sh — CI gate that runs one governed Strix security scan +# request. No repository-authored retry, model fallback, or severity +# threshold decides admission: contextual-orchestrator's `orchestrator/free` +# gateway owns provider discovery and failover, and any current vulnerability +# report artifact fails the gate closed. # -# STRIX_LOG is a per-attempt temp file consumed only by -# is_transient_same_model_retry_error(); cumulative report dirs in +# STRIX_LOG is a per-attempt temp file; cumulative report dirs in # STRIX_REPORTS_DIR are never overwritten. Refer to ARCHITECTURE.md # for the 3-tier timeout classification hierarchy. set -euo pipefail @@ -44,8 +45,6 @@ STRIX_INPUT_FILE_ROOT="${STRIX_INPUT_FILE_ROOT:-${RUNNER_TEMP:-}}" STRIX_EXECUTABLE_PATH="${STRIX_EXECUTABLE_PATH:-}" STRIX_EXECUTABLE_ROOT="${STRIX_EXECUTABLE_ROOT:-}" STRIX_EXECUTABLE_SHA256="${STRIX_EXECUTABLE_SHA256:-}" -STRIX_TRANSIENT_RETRY_PER_MODEL="${STRIX_TRANSIENT_RETRY_PER_MODEL:-0}" -STRIX_TRANSIENT_RETRY_BACKOFF_SECONDS="${STRIX_TRANSIENT_RETRY_BACKOFF_SECONDS:-3}" STRIX_FAIL_ON_MIN_SEVERITY="${STRIX_FAIL_ON_MIN_SEVERITY:-MEDIUM}" STRIX_FAIL_ON_PROVIDER_SIGNAL="${STRIX_FAIL_ON_PROVIDER_SIGNAL:-0}" RUN_START_EPOCH=0 @@ -884,8 +883,6 @@ if is_github_models_model "$PRIMARY_MODEL" && [ -z "$LLM_API_BASE_FILE" ]; then exit 2 fi -require_non_negative_integer "$STRIX_TRANSIENT_RETRY_PER_MODEL" "STRIX_TRANSIENT_RETRY_PER_MODEL" -require_non_negative_integer "$STRIX_TRANSIENT_RETRY_BACKOFF_SECONDS" "STRIX_TRANSIENT_RETRY_BACKOFF_SECONDS" require_non_negative_integer "$STRIX_PROCESS_TIMEOUT_SECONDS" "STRIX_PROCESS_TIMEOUT_SECONDS" require_non_negative_integer "$STRIX_TOTAL_TIMEOUT_SECONDS" "STRIX_TOTAL_TIMEOUT_SECONDS" case "$STRIX_FAIL_ON_PROVIDER_SIGNAL" in @@ -2929,11 +2926,11 @@ PY fi if [ "$rc" -eq 0 ]; then - if has_blocking_vulnerability_reports; then - if ! evaluate_pull_request_findings || [ "$PR_FINDINGS_DECISION" != "allow_baseline" ]; then - echo "Strix exited successfully but emitted a vulnerability at or above '$STRIX_FAIL_ON_MIN_SEVERITY'; failing closed." >&2 - return 1 - fi + local current_vulnerability_file + current_vulnerability_file="$(find "$ACTIVE_REPORTS_DIR" -type f -path '*/vulnerabilities/*.md' -print -quit 2>/dev/null || true)" + if [ -n "$current_vulnerability_file" ]; then + echo "Current Strix vulnerability report exists; failing closed without a repository-authored severity threshold." >&2 + return 1 fi printf "Strix run succeeded for model '%s' in %ds.\n" "$model" "$elapsed" >&2 return 0 @@ -3043,108 +3040,6 @@ is_model_behavior_error() { return 1 } -## Determines whether the last strix failure is a transient error eligible -## for same-model retry (up to STRIX_TRANSIENT_RETRY_PER_MODEL times). -## Five error families qualify: -## - RateLimit / RESOURCE_EXHAUSTED / HTTP 429 -## - litellm API connection failures with LLM-provider evidence -## - litellm service-unavailable / high-demand provider failures -## - MidStreamFallbackError (litellm mid-stream provider switch) -## - Caido bootstrap timing failures (guest login before the local proxy is up) -## Timeouts are infrastructure failures. In strict CI mode they fail closed; -## otherwise the caller may still move to fallback model evaluation. -is_transient_same_model_retry_error() { - local model="${1-}" - if is_timeout_error; then - return 1 - fi - if is_llm_api_connection_error; then - return 0 - fi - if is_llm_service_unavailable_error; then - return 0 - fi - if is_rate_limit_error; then - return 0 - fi - if is_midstream_fallback_error; then - return 0 - fi - if is_caido_bootstrap_timing_error; then - return 0 - fi - return 1 -} - -github_models_rate_limit_should_skip_same_model_retry() { - local model="$1" - - if ! is_rate_limit_error; then - return 1 - fi - if ! is_github_models_api_compatible_model "$model"; then - return 1 - fi - github_models_api_base_is_active -} - -run_strix_with_transient_retry() { - local model="$1" - local max_attempts=$((STRIX_TRANSIENT_RETRY_PER_MODEL + 1)) - local attempt=1 - - while [ "$attempt" -le "$max_attempts" ]; do - local run_rc=0 - run_strix_once "$model" || run_rc=$? - if [ "$run_rc" -eq 0 ]; then - return 0 - fi - if [ "$run_rc" -eq 2 ]; then - return 2 - fi - if [ "$TOTAL_TIMEOUT_EXCEEDED" -eq 1 ]; then - return 1 - fi - - if [ "$attempt" -ge "$max_attempts" ]; then - return 1 - fi - - if [ "$STRIX_TOTAL_TIMEOUT_SECONDS" -gt 0 ] && [ "$(remaining_total_budget)" -le 0 ]; then - TOTAL_TIMEOUT_EXCEEDED=1 - printf "Strix quick scan exceeded total timeout of %ss.\n" "$STRIX_TOTAL_TIMEOUT_SECONDS" | tee "$STRIX_LOG" >&2 - return 1 - fi - - if github_models_rate_limit_should_skip_same_model_retry "$model"; then - echo "GitHub Models rate limit detected for model '$model'; skipping same-model retry and moving directly to fallback models or current-head neutral classification." >&2 - return 1 - fi - - if ! is_transient_same_model_retry_error "$model"; then - return 1 - fi - - local retry_reason="transient error" - if is_rate_limit_error; then - retry_reason="rate limit" - elif is_llm_api_connection_error; then - retry_reason="LLM API connection" - elif is_llm_service_unavailable_error; then - retry_reason="LLM service unavailable" - elif is_midstream_fallback_error; then - retry_reason="midstream fallback" - elif is_caido_bootstrap_timing_error; then - retry_reason="Caido sandbox bootstrap timing" - fi - echo "Retrying model '$model' due to $retry_reason (attempt $((attempt + 1))/$max_attempts)." >&2 - sleep "$STRIX_TRANSIENT_RETRY_BACKOFF_SECONDS" - attempt=$((attempt + 1)) - done - - return 1 -} - is_vertex_not_found_error() { # Match Vertex/LiteLLM model-not-found errors. # These functions are only called within the Vertex fallback path @@ -4321,189 +4216,43 @@ run_current_target_scan() { ZERO_FINDINGS_REPORTED=0 local primary_scan_rc=0 - run_strix_with_transient_retry "$PRIMARY_MODEL" || primary_scan_rc=$? - if [ "$primary_scan_rc" -eq 0 ]; then - return 0 - fi + run_strix_once "$PRIMARY_MODEL" || primary_scan_rc=$? if [ "$primary_scan_rc" -eq 2 ]; then return 2 fi - if [ "$TOTAL_TIMEOUT_EXCEEDED" -eq 1 ]; then - return 1 - fi - - local strict_primary_provider_fallback=0 - if [ "$INFRA_ERROR_DETECTED" -eq 1 ] && provider_signal_fail_closed_enabled; then - if is_contextual_orchestrator_model "$PRIMARY_MODEL"; then - echo "STRIX_PROVIDER_UNAVAILABLE: contextual-orchestrator/orchestrator/free exhausted; the gateway owns provider discovery and failover." >&2 - return 1 - elif is_model_retryable_error "$PRIMARY_MODEL" && has_distinct_fallback_model_for_model "$PRIMARY_MODEL"; then - strict_primary_provider_fallback=1 - else - echo "Strix scan failed after provider infrastructure or failure-signal output; failing closed." >&2 - return 1 - fi - fi - - if has_only_below_threshold_vulnerabilities; then - return 0 - fi + # Pull-request scope is orthogonal to the retry/fallback/severity admission + # this change removed. It answers a different question: do the findings this + # one governed request produced intersect the pull request at all? Skipping it + # fails the pull request on findings confined to files it never modified. if evaluate_pull_request_findings; then - if [ "$strict_primary_provider_fallback" -eq 0 ]; then - return 0 - fi + return 0 fi case "$PR_FINDINGS_DECISION" in - block_changed | block_unmapped | block_manifest_unverified) - if [ "$strict_primary_provider_fallback" -eq 1 ] && fail_reported_vulnerabilities_before_fallback_success; then - return 1 - fi + block_changed | block_unmapped | block_manifest_finding) echo "Strix quick scan failed with a non-recoverable error." >&2 return 1 ;; esac - if fail_unmapped_threshold_report; then - return 1 - fi - - if [ "$strict_primary_provider_fallback" -eq 1 ] && fail_reported_vulnerabilities_before_fallback_success; then - return 1 - fi - if ! is_model_retryable_error "$PRIMARY_MODEL"; then - echo "Strix quick scan failed with a non-recoverable error." >&2 - return 1 - fi - if is_contextual_orchestrator_model "$PRIMARY_MODEL"; then - echo "STRIX_PROVIDER_UNAVAILABLE: contextual-orchestrator/orchestrator/free exhausted; no external fallback is permitted." >&2 + if fail_unmapped_threshold_report; then return 1 fi - FALLBACK_MODELS_RAW="$(fallback_models_raw_for_model "$PRIMARY_MODEL")" - FALLBACK_MODELS_RAW="${FALLBACK_MODELS_RAW//$'\r'/ }" - FALLBACK_MODELS_RAW="${FALLBACK_MODELS_RAW//$'\n'/ }" - read -r -a FALLBACK_MODELS <<<"$FALLBACK_MODELS_RAW" - - fallback_tried=0 - for candidate_raw in "${FALLBACK_MODELS[@]}"; do - candidate="$(normalize_model "$candidate_raw")" - if [ -z "$candidate" ] || [ "$candidate" = "$PRIMARY_MODEL" ]; then - if [ -n "$candidate" ]; then - echo "Skipping fallback model '$candidate' — same as primary model." >&2 - fi - continue - fi - if [ "$TOTAL_TIMEOUT_EXCEEDED" -eq 1 ]; then - return 1 - fi - - fallback_tried=1 - if is_vertex_model "$PRIMARY_MODEL"; then - echo "Primary Vertex model unavailable; retrying with fallback '$candidate'." - else - echo "Primary model unavailable; retrying with fallback '$candidate'." - fi - local fallback_scan_rc=0 - local fallback_start_epoch - fallback_start_epoch="$(date +%s)" - run_strix_with_transient_retry "$candidate" || fallback_scan_rc=$? - local fallback_elapsed=$(( $(date +%s) - fallback_start_epoch )) - if [ "$fallback_scan_rc" -eq 0 ]; then - if fail_reported_vulnerabilities_before_fallback_success; then - return 1 - fi - echo "Strix quick scan succeeded with fallback model '$candidate' in ${fallback_elapsed}s." >&2 - return 0 - fi - if [ "$fallback_scan_rc" -eq 2 ]; then - return 2 - fi - - local strict_fallback_provider_signal=0 - if [ "$INFRA_ERROR_DETECTED" -eq 1 ] && provider_signal_fail_closed_enabled; then - strict_fallback_provider_signal=1 - fi - - if has_only_below_threshold_vulnerabilities; then - return 0 - fi - - if evaluate_pull_request_findings; then - if [ "$strict_fallback_provider_signal" -eq 0 ]; then - return 0 - fi - fi - - case "$PR_FINDINGS_DECISION" in - block_changed | block_unmapped | block_manifest_unverified) - if [ "$strict_fallback_provider_signal" -eq 1 ] && fail_reported_vulnerabilities_before_fallback_success; then - return 1 - fi - echo "Strix quick scan failed with a non-recoverable error." >&2 - return 1 - ;; - esac - if fail_unmapped_threshold_report; then - return 1 - fi - - if fail_reported_vulnerabilities_before_fallback_success; then - return 1 - fi - - if [ "$strict_fallback_provider_signal" -eq 1 ]; then - if is_model_retryable_error "$candidate"; then - continue - fi - echo "Strix fallback model '$candidate' emitted provider infrastructure or failure-signal output; trying next configured fallback if available." >&2 - continue - fi - - if ! is_model_retryable_error "$candidate"; then - echo "Strix quick scan failed with a non-recoverable error." >&2 - return 1 - fi - done - + # A zero-findings result produced while the provider was failing is not + # clean scan evidence, so it must not pass the pull request. if should_fail_pull_request_infra_zero_findings; then return 1 fi - if [ "$fallback_tried" -eq 0 ]; then - local fallback_config_name - fallback_config_name="$(fallback_models_config_name_for_model "$PRIMARY_MODEL")" - local configured_fallback_count=0 - for candidate_raw in "${FALLBACK_MODELS[@]}"; do - candidate="$(normalize_model "$candidate_raw")" - [ -n "$candidate" ] && configured_fallback_count=$((configured_fallback_count + 1)) - done - if [ "$configured_fallback_count" -eq 0 ]; then - echo "ERROR: No fallback models configured ($fallback_config_name is empty). Configure distinct models." >&2 - else - echo "ERROR: All configured fallback models are the same as the primary model" >&2 - fi - return 1 - fi - - if [ "$INFRA_ERROR_DETECTED" -eq 1 ] && - [ "$PR_FINDINGS_DECISION" = "allow_baseline" ]; then - echo "STRIX_PROVIDER_UNAVAILABLE: provider models were exhausted after incomplete scan evidence." >&2 - return 1 - fi - - local threshold_rank - threshold_rank="$(severity_rank "$STRIX_FAIL_ON_MIN_SEVERITY")" - if [ "${STRIX_MAX_SEVERITY_RANK:--1}" -ge "$threshold_rank" ]; then - echo "Strix quick scan failed with a non-recoverable error." >&2 - return 1 + if [ "$primary_scan_rc" -eq 0 ]; then + return 0 fi - - if is_vertex_model "$PRIMARY_MODEL"; then - echo "Configured Vertex model and fallback models were unavailable." >&2 + if [ "$INFRA_ERROR_DETECTED" -eq 1 ]; then + echo "STRIX_PROVIDER_UNAVAILABLE: contextual-orchestrator/orchestrator/free did not produce authoritative scan evidence; failing closed without repository-authored retry or fallback allocation." >&2 else - echo "Configured model and fallback models were unavailable." >&2 + echo "Strix quick scan failed; failing closed without repository-authored retry or fallback allocation." >&2 fi return 1 } diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index b9b1c43de3..171497ec15 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -3353,6 +3353,30 @@ printf '%s\n' "$target_path" >> "${FAKE_STRIX_TARGET_LOG:?}" STRIX_REPORTS_DIR="${STRIX_REPORTS_DIR:-strix_runs}" case "${FAKE_STRIX_SCENARIO:?}" in + provider-ratelimit-fails-closed) + echo "Penetration test failed: LLM request failed: litellm.RateLimitError" + exit 1 + ;; + provider-resource-exhausted-fails-closed) + echo '{"error":{"status":"RESOURCE_EXHAUSTED"}}' + exit 1 + ;; + provider-quota-fails-closed) + echo "openai.agents: Error streaming response: You exceeded your current quota, please check your plan and billing details." + exit 1 + ;; + provider-token-limit-fails-closed) + echo "litellm.APIStatusError: Error code: 413 - Request body too large" + exit 1 + ;; + provider-timeout-fails-closed) + echo "litellm.exceptions.Timeout: provider request timed out" + exit 1 + ;; + provider-plain-failure-fails-closed-without-infra-marker) + echo "penetration test aborted by an unclassified condition" + exit 1 + ;; success|runtime-env-forwarding|custom-openai-compatible-preserves-effort|vertex-primary-success-timing-message|direct-openai-gpt-does-not-require-github-models-api-base|pr-executable-integrity-mismatch|pr-executable-group-writable) echo "scan ok" exit 0 @@ -3392,109 +3416,12 @@ REPORT echo "Vulnerabilities 1" exit 0 ;; - slow-timeout) - sleep "${FAKE_STRIX_TIMEOUT_SLEEP_SECONDS:?}" - exit 0 - ;; timeout-disabled-success) sleep 1 echo "scan ok with timeout disabled" exit 0 ;; vertex-primary-notfound-fallback-success|github-models-fallback-success|github-models-fallback-success-deepseek-v3|github-models-token-limit-fallback-success|github-models-fallback-requires-api-base|github-models-model-prefix-with-api-base-succeeds|github-models-meta-prefix-with-api-base-succeeds|github-models-mistral-prefix-with-api-base-succeeds) - case "${STRIX_LLM:-}" in - vertex_ai/missing-primary) - echo "Error: litellm.NotFoundError: Vertex_aiException - x" - echo '"status": "NOT_FOUND"' - exit 1 - ;; - vertex_ai/fallback-one) - echo "scan ok with fallback" - exit 0 - ;; - openai/gpt-5|openai/openai/gpt-5.4|openai/meta/test-github-model|openai/mistral-ai/test-github-model) - if [ "${FAKE_STRIX_SCENARIO:?}" = "github-models-token-limit-fallback-success" ]; then - echo "openai.APIStatusError: Error code: 413 - {'error': {'code': 'tokens_limit_reached', 'message': 'Request body too large for gpt-5 model. Max size: 4000 tokens.'}}" - exit 1 - fi - echo "scan ok with GitHub Models fallback" - exit 0 - ;; - openai/deepseek/deepseek-r1-0528) - if [ "${FAKE_STRIX_SCENARIO:?}" = "github-models-fallback-success-deepseek-v3" ]; then - echo "LLM CONNECTION FAILED" - echo "Could not establish connection to the language model." - echo "Error: litellm.BadRequestError: OpenAIException - Unavailable model: deepseek-r1-0528" - exit 1 - fi - echo "scan ok with GitHub Models fallback" - exit 0 - ;; - openai/deepseek/deepseek-v3-0324) - echo "scan ok with GitHub Models fallback" - exit 0 - ;; - *) - echo "unexpected model ${STRIX_LLM:-}" >&2 - exit 9 - ;; - esac - ;; - nvidia-rate-limit-openai-direct-fallback-clears-api-base) - case "${STRIX_LLM:-}" in - nvidia_nim/nvidia/rate-limited-primary) - echo "LLM CONNECTION FAILED" - echo "Error: litellm.RateLimitError: Nvidia_nimException - Error code: 429 Too Many Requests" - exit 1 - ;; - openai/gpt-5.4) - if [ "${STRIX_REASONING_EFFORT:-}" != "none" ]; then - echo "direct OpenAI function-tools fallback requires reasoning effort none" >&2 - exit 29 - fi - if [ "${LLM_API_KEY:-}" != "openai-fallback-token" ]; then - echo "unexpected direct-OpenAI fallback key (${LLM_API_KEY:-})" >&2 - exit 26 - fi - if [ -n "${LLM_API_BASE:-}" ]; then - echo "direct OpenAI fallback inherited foreign API base ${LLM_API_BASE}" >&2 - exit 27 - fi - echo "scan ok after direct-OpenAI fallback" - exit 0 - ;; - *) - echo "unexpected cross-provider model ${STRIX_LLM:-}" >&2 - exit 28 - ;; - esac - ;; - openai-direct-quota-github-models-fallback-success) - case "${STRIX_LLM:-}" in - openai/gpt-5.4) - if [ "${LLM_API_KEY:-}" != "dummy" ]; then - echo "unexpected direct-OpenAI key for primary (${LLM_API_KEY:-})" >&2 - exit 15 - fi - echo "Error getting response: Error code: 429 - {'error': {'message': 'You exceeded your current quota, please check your plan and billing details.', 'type': 'insufficient_quota', 'code': 'insufficient_quota'}}" - echo "openai.RateLimitError: Error code: 429" - exit 1 - ;; - openai/o3) - if [ "${LLM_API_KEY:-}" != "github-models-fallback-token" ]; then - echo "unexpected GitHub Models key for fallback (${LLM_API_KEY:-})" >&2 - exit 16 - fi - echo "scan ok with GitHub Models fallback" - exit 0 - ;; - *) - echo "unexpected model ${STRIX_LLM:-}" >&2 - exit 9 - ;; - esac - ;; - vertex-all-notfound) echo "Error: litellm.NotFoundError: Vertex_aiException - x" echo '"status": "NOT_FOUND"' exit 1 @@ -3511,23 +3438,6 @@ REPORT echo "Error: provider prefix not normalized (${STRIX_LLM:-})" >&2 exit 10 ;; - provider-prefix-fallback-normalization) - case "${STRIX_LLM:-}" in - vertex_ai/missing-primary) - echo "Error: litellm.NotFoundError: Vertex_aiException - x" - echo '"status": "NOT_FOUND"' - exit 1 - ;; - vertex_ai/fallback-one) - echo "scan ok after fallback normalization" - exit 0 - ;; - *) - echo "Error: fallback provider prefix not normalized (${STRIX_LLM:-})" >&2 - exit 11 - ;; - esac - ;; provider-prefix-required-resource-path-primary-implicit-default-provider | provider-prefix-required-resource-path-primary-explicit-empty-default-provider) if [ "${STRIX_LLM:-}" = "vertex_ai/gemini-2.5-pro" ]; then echo "scan ok with resource-path normalization" @@ -3536,23 +3446,6 @@ REPORT echo "Error: resource-path model not normalized (${STRIX_LLM:-})" >&2 exit 12 ;; - provider-prefix-resource-path-primary-notfound-fallback-success) - case "${STRIX_LLM:-}" in - vertex_ai/missing-primary) - echo "Error: litellm.NotFoundError: Vertex_aiException - x" - echo '"status": "NOT_FOUND"' - exit 1 - ;; - vertex_ai/fallback-one) - echo "scan ok after resource-path fallback" - exit 0 - ;; - *) - echo "Error: resource-path fallback model not normalized (${STRIX_LLM:-})" >&2 - exit 13 - ;; - esac - ;; vertex-custom-model-resource-path) # projects/

/locations//models/ (no publishers/ segment) if [ "${STRIX_LLM:-}" = "vertex_ai/my-custom-model-123" ]; then @@ -3562,39 +3455,6 @@ REPORT echo "Error: custom model resource-path not normalized (${STRIX_LLM:-})" >&2 exit 40 ;; - vertex-notfound-without-status-fallback-success) - case "${STRIX_LLM:-}" in - vertex_ai/missing-primary) - echo "Error: litellm.NotFoundError: Vertex_aiException - x" - exit 1 - ;; - vertex_ai/fallback-one) - echo "scan ok after status-less not found fallback" - exit 0 - ;; - *) - echo "Error: status-less fallback model not normalized (${STRIX_LLM:-})" >&2 - exit 14 - ;; - esac - ;; - vertex-notfound-compact-status-fallback-success) - case "${STRIX_LLM:-}" in - vertex_ai/missing-primary) - echo 'litellm.exceptions.NotFoundError: VertexAI error' - echo '{"error":{"status":"NOT_FOUND"}}' - exit 1 - ;; - vertex_ai/fallback-one) - echo "scan ok after compact-status not found fallback" - exit 0 - ;; - *) - echo "Error: compact-status fallback model not normalized (${STRIX_LLM:-})" >&2 - exit 17 - ;; - esac - ;; nonvertex-slash-model-passthrough) if [ "${STRIX_LLM:-}" = "foo/bar" ]; then echo "scan ok with non-vertex slash model passthrough" @@ -3603,310 +3463,52 @@ REPORT echo "Error: non-vertex slash model was rewritten (${STRIX_LLM:-})" >&2 exit 18 ;; - primary-duplicate-in-fallback) - case "${STRIX_LLM:-}" in - vertex_ai/missing-primary) - echo "Error: litellm.NotFoundError: Vertex_aiException - x" - echo '"status": "NOT_FOUND"' - exit 1 - ;; - vertex_ai/fallback-one) - echo "scan ok after duplicate-primary skip" - exit 0 - ;; - *) - echo "Error: duplicate-primary path unexpected (${STRIX_LLM:-})" >&2 - exit 15 - ;; - esac - ;; - multiline-fallback-success) - case "${STRIX_LLM:-}" in - vertex_ai/missing-primary) - echo "Error: litellm.NotFoundError: Vertex_aiException - x" - echo '"status": "NOT_FOUND"' - exit 1 - ;; - vertex_ai/fallback-one) - echo "Error: litellm.NotFoundError: Vertex_aiException - x" - echo '"status": "NOT_FOUND"' - exit 1 - ;; - vertex_ai/fallback-two) - echo "scan ok after multiline fallback parsing" - exit 0 - ;; - *) - echo "Error: multiline fallback path unexpected (${STRIX_LLM:-})" >&2 - exit 19 - ;; - esac - ;; - vertex-primary-ratelimit-fallback-success) - case "${STRIX_LLM:-}" in - vertex_ai/ratelimit-primary) - echo "Penetration test failed: LLM request failed: RateLimitError" - exit 1 - ;; - vertex_ai/fallback-one) - echo "scan ok after rate-limit fallback" - exit 0 - ;; - *) - echo "Error: ratelimit fallback path unexpected (${STRIX_LLM:-})" >&2 - exit 21 - ;; - esac - ;; - vertex-primary-resource-exhausted-fallback-success) - case "${STRIX_LLM:-}" in - vertex_ai/resource-exhausted-primary) - echo '{"error":{"status":"RESOURCE_EXHAUSTED"}}' - exit 1 - ;; - vertex_ai/fallback-one) - echo "scan ok after resource exhausted fallback" - exit 0 - ;; - *) - echo "Error: resource exhausted fallback path unexpected (${STRIX_LLM:-})" >&2 - exit 23 - ;; - esac - ;; - openai-primary-quota-fallback-success) - case "${STRIX_LLM:-}" in - openai/quota-primary) - echo "openai.agents: Error streaming response: You exceeded your current quota, please check your plan and billing details." - exit 1 - ;; - openai/fallback-one) - echo "scan ok after quota fallback" - exit 0 - ;; - *) - echo "Error: quota fallback path unexpected (${STRIX_LLM:-})" >&2 - exit 24 - ;; - esac - ;; - vertex-primary-429-fallback-success) - case "${STRIX_LLM:-}" in - vertex_ai/http429-primary) - echo "litellm: HTTP 429 Too Many Requests" - exit 1 - ;; - vertex_ai/fallback-one) - echo "scan ok after 429 fallback" - exit 0 - ;; - *) - echo "Error: 429 fallback path unexpected (${STRIX_LLM:-})" >&2 - exit 24 - ;; - esac - ;; - vertex-primary-midstream-fallback-success) - case "${STRIX_LLM:-}" in - vertex_ai/midstream-primary) - echo "Penetration test failed: LLM request failed: MidStreamFallbackError" - exit 1 - ;; - vertex_ai/fallback-one) - echo "scan ok after midstream fallback" - exit 0 - ;; - *) - echo "Error: midstream fallback path unexpected (${STRIX_LLM:-})" >&2 - exit 25 - ;; - esac - ;; - vertex-primary-midstream-retry-same-model-success) - case "${STRIX_LLM:-}" in - vertex_ai/retry-midstream-primary) - attempt="0" - if [ -f "${FAKE_STRIX_STATE_FILE:?}" ]; then - attempt="$(cat "${FAKE_STRIX_STATE_FILE:?}")" - fi - attempt="$((attempt + 1))" - echo "$attempt" > "${FAKE_STRIX_STATE_FILE:?}" - if [ "$attempt" -eq 1 ]; then - echo "Penetration test failed: LLM request failed: MidStreamFallbackError" - exit 1 - fi - echo "scan ok after same-model retry" - exit 0 - ;; - vertex_ai/fallback-one) - echo "Error: fallback should not be needed for same-model retry scenario" >&2 - exit 30 - ;; - *) - echo "Error: midstream fallback path unexpected (${STRIX_LLM:-})" >&2 - exit 30 - ;; - esac - ;; - vertex-primary-ratelimit-retry-same-model-success|vertex-primary-ratelimit-retry-reason-message) - case "${STRIX_LLM:-}" in - vertex_ai/retry-ratelimit-primary) - attempt="0" - if [ -f "${FAKE_STRIX_STATE_FILE:?}" ]; then - attempt="$(cat "${FAKE_STRIX_STATE_FILE:?}")" - fi - attempt="$((attempt + 1))" - echo "$attempt" > "${FAKE_STRIX_STATE_FILE:?}" - if [ "$attempt" -eq 1 ]; then - echo "Penetration test failed: LLM request failed: RateLimitError" - exit 1 - fi - echo "scan ok after same-model rate-limit retry" - exit 0 - ;; - vertex_ai/fallback-one) - echo "Error: fallback should not be needed for same-model rate-limit retry scenario" >&2 - exit 31 - ;; - *) - echo "Error: rate-limit fallback path unexpected (${STRIX_LLM:-})" >&2 - exit 31 - ;; - esac - ;; vertex-primary-api-connection-retry-same-model-success|github-models-internal-server-connection-retry-same-model-success|internal-server-error-unrelated-output-nonretryable|internal-server-error-many-blocks-retry-same-model-success) - case "${STRIX_LLM:-}" in - gemini/retry-api-connection-primary|vertex_ai/retry-api-connection-primary|openai/openai/retry-api-connection-primary) - attempt="0" - if [ -f "${FAKE_STRIX_STATE_FILE:?}" ]; then - attempt="$(cat "${FAKE_STRIX_STATE_FILE:?}")" - fi - attempt="$((attempt + 1))" - echo "$attempt" > "${FAKE_STRIX_STATE_FILE:?}" - if [ "$attempt" -eq 1 ]; then - if [ "${STRIX_LLM:-}" = "openai/openai/retry-api-connection-primary" ]; then - if [ "${FAKE_STRIX_SCENARIO:?}" = "internal-server-error-unrelated-output-nonretryable" ]; then + attempt="0" + if [ -f "${FAKE_STRIX_STATE_FILE:?}" ]; then + attempt="$(cat "${FAKE_STRIX_STATE_FILE:?}")" + fi + attempt="$((attempt + 1))" + echo "$attempt" > "${FAKE_STRIX_STATE_FILE:?}" + if [ "$attempt" -eq 1 ]; then + if [ "${STRIX_LLM:-}" = "openai/openai/retry-api-connection-primary" ]; then + if [ "${FAKE_STRIX_SCENARIO:?}" = "internal-server-error-unrelated-output-nonretryable" ]; then + echo "Error: litellm.InternalServerError: upstream request failed" + for filler in 1 2 3 4 5 6; do + echo "target application diagnostic $filler" + done + echo "Internal Server Error" + exit 1 + fi + if [ "${FAKE_STRIX_SCENARIO:?}" = "internal-server-error-many-blocks-retry-same-model-success" ]; then + # Regression for the SIGPIPE race (Devin finding on + # PR #1394): emit enough matching + # litellm.InternalServerError blocks that the bounded + # awk scan's piped output exceeds a single pipe + # buffer, so a `grep -q` that stops reading at the + # first match cannot SIGPIPE the still-writing awk + # producer into a false non-match under + # `set -o pipefail`. + for _ in $(seq 1 2000); do + echo "line filler some unrelated target application output padding padding padding" echo "Error: litellm.InternalServerError: upstream request failed" - for filler in 1 2 3 4 5 6; do - echo "target application diagnostic $filler" - done echo "Internal Server Error" - exit 1 - fi - if [ "${FAKE_STRIX_SCENARIO:?}" = "internal-server-error-many-blocks-retry-same-model-success" ]; then - # Regression for the SIGPIPE race (Devin finding on - # PR #1394): emit enough matching - # litellm.InternalServerError blocks that the bounded - # awk scan's piped output exceeds a single pipe - # buffer, so a `grep -q` that stops reading at the - # first match cannot SIGPIPE the still-writing awk - # producer into a false non-match under - # `set -o pipefail`. - for _ in $(seq 1 2000); do - echo "line filler some unrelated target application output padding padding padding" - echo "Error: litellm.InternalServerError: upstream request failed" - echo "Internal Server Error" - echo "more filler after context one" - echo "more filler after context two" - done - exit 1 - fi - echo "LLM CONNECTION FAILED" - echo "Could not establish connection to the language model." - echo "Error: litellm.InternalServerError: upstream request failed" - else - echo "LLM CONNECTION FAILED" - echo "litellm.APIConnectionError: GeminiException - Server disconnected without sending a response." + echo "more filler after context one" + echo "more filler after context two" + done + exit 1 fi - exit 1 - fi - echo "scan ok after same-model api connection retry" - exit 0 - ;; - vertex_ai/fallback-one) - echo "Error: fallback should not be needed for API connection retry scenario" >&2 - exit 36 - ;; - *) - echo "Error: API connection retry path unexpected (${STRIX_LLM:-})" >&2 - exit 36 - ;; - esac - ;; - openrouter-502-fallback-retry-same-model-success) - case "${STRIX_LLM:-}" in - vertex_ai/missing-primary) - echo "Error: litellm.NotFoundError: Vertex_aiException - x" - echo '"status": "NOT_FOUND"' - exit 1 - ;; - openrouter/free) - attempt="0" - if [ -f "${FAKE_STRIX_STATE_FILE:?}" ]; then - attempt="$(cat "${FAKE_STRIX_STATE_FILE:?}")" - fi - attempt="$((attempt + 1))" - echo "$attempt" > "${FAKE_STRIX_STATE_FILE:?}" - if [ "$attempt" -eq 1 ]; then - echo "Error: litellm.APIError: APIError:" - echo "OpenrouterException -" - echo '{"error":{"message":"Invalid URL:' - echo '","code":502,"metadata":{"provider_name":"Stealth"}}}' - exit 1 - fi - echo "scan ok after OpenRouter 502 same-model retry" - exit 0 - ;; - vertex_ai/fallback-two) - echo "Error: second fallback should not be needed after transient OpenRouter 502" >&2 - exit 38 - ;; - *) - echo "Error: OpenRouter 502 fallback path unexpected (${STRIX_LLM:-})" >&2 - exit 38 - ;; - esac - ;; - openrouter-502-distant-target-output-nonretryable) - case "${STRIX_LLM:-}" in - vertex_ai/missing-primary) - echo "Error: litellm.NotFoundError: Vertex_aiException - x" - echo '"status": "NOT_FOUND"' - exit 1 - ;; - openrouter/free) - echo "Error: litellm.APIError: APIError: OpenrouterException -" - printf 'target output\n%.0s' 1 2 3 4 5 6 - echo '{"code":502,"metadata":{"provider_name":"spoof"}}' - exit 1 - ;; - vertex_ai/fallback-two) - echo "scan ok after distant target output" - exit 0 - ;; - esac - ;; - github-models-primary-unavailable-fallback-success|github-models-primary-denied-fallback-success) - case "${STRIX_LLM:-}" in - openai/gpt-5) - echo "LLM CONNECTION FAILED" - echo "Could not establish connection to the language model." - if [ "${FAKE_STRIX_SCENARIO:?}" = "github-models-primary-denied-fallback-success" ]; then - echo "openai.PermissionDeniedError: Error code: 403" + echo "LLM CONNECTION FAILED" + echo "Could not establish connection to the language model." + echo "Error: litellm.InternalServerError: upstream request failed" else - echo "Error: litellm.BadRequestError: OpenAIException - Unavailable model: gpt-5" + echo "LLM CONNECTION FAILED" + echo "litellm.APIConnectionError: GeminiException - Server disconnected without sending a response." fi exit 1 - ;; - openai/deepseek/deepseek-r1-0528) - echo "scan ok after GitHub Models unavailable fallback" - exit 0 - ;; - *) - echo "Error: GitHub Models unavailable fallback path unexpected (${STRIX_LLM:-})" >&2 - exit 37 - ;; - esac + fi + echo "scan ok after same-model api connection retry" + exit 0 ;; github-models-http410-authenticated-fallback-success | github-models-http410-missing-http-token | github-models-http410-missing-provider-error | github-models-http410-numeric-continuation-4100 | github-models-http410-numeric-continuation-4104 | github-models-http410-target-output-spoof | github-models-retirement-brownout-phrase-only) case "${STRIX_LLM:-}" in @@ -3946,30 +3548,12 @@ REPORT ;; esac ;; - github-models-primary-ratelimit-fallback-success) + github-models-fallback-provider-signal-tries-next | github-models-fallback-baseline-vulnerability-before-next-success-continues | github-models-exhausted-after-baseline-vulnerability-fails-closed | github-models-fallback-changed-vulnerability-before-next-success-blocks | github-models-fallback-dockerfile-test-baseline-before-next-success-continues) case "${STRIX_LLM:-}" in openai/gpt-5) echo "LLM CONNECTION FAILED" echo "Could not establish connection to the language model." - echo "Error: litellm.RateLimitError: RateLimitError: OpenAIException - Too many requests. For more on scraping GitHub and how it may affect your rights, please review our Terms of Service." - exit 1 - ;; - openai/deepseek/deepseek-r1-0528) - echo "scan ok after GitHub Models rate-limit fallback" - exit 0 - ;; - *) - echo "Error: GitHub Models rate-limit fallback path unexpected (${STRIX_LLM:-})" >&2 - exit 38 - ;; - esac - ;; - github-models-fallback-provider-signal-tries-next | github-models-fallback-baseline-vulnerability-before-next-success-continues | github-models-exhausted-after-baseline-vulnerability-fails-closed | github-models-fallback-changed-vulnerability-before-next-success-blocks | github-models-fallback-dockerfile-test-baseline-before-next-success-continues) - case "${STRIX_LLM:-}" in - openai/gpt-5) - echo "LLM CONNECTION FAILED" - echo "Could not establish connection to the language model." - echo "Error: litellm.RateLimitError: RateLimitError: OpenAIException - Too many requests." + echo "Error: litellm.RateLimitError: RateLimitError: OpenAIException - Too many requests." exit 1 ;; openai/deepseek/deepseek-r1-0528) @@ -4018,95 +3602,6 @@ EOS ;; esac ;; - gemini-high-demand-retry-same-model-success) - case "${STRIX_LLM:-}" in - gemini/retry-high-demand-primary) - attempt="0" - if [ -f "${FAKE_STRIX_STATE_FILE:?}" ]; then - attempt="$(cat "${FAKE_STRIX_STATE_FILE:?}")" - fi - attempt="$((attempt + 1))" - echo "$attempt" > "${FAKE_STRIX_STATE_FILE:?}" - if [ "$attempt" -eq 1 ]; then - echo "LLM CONNECTION FAILED" - echo 'litellm.ServiceUnavailableError: GeminiException - {"error":{"code":503,"message":"This model is currently experiencing high demand. Spikes in demand are usually temporary. Please try again later.","status":"UNAVAILABLE"}}' - exit 1 - fi - echo "scan ok after same-model high-demand retry" - exit 0 - ;; - *) - echo "Error: high-demand retry path unexpected (${STRIX_LLM:-})" >&2 - exit 37 - ;; - esac - ;; - nvidia-overloaded-direct-fallback-success) - case "${STRIX_LLM:-}" in - nvidia_nim/nvidia/overloaded-primary) - echo "LLM CONNECTION FAILED" - echo "Could not establish connection to the language model." - echo "Error: litellm.ServiceUnavailableError: Nvidia_nimException - Service temporarily overloaded" - exit 1 - ;; - nvidia_nim/nvidia/fallback-one) - echo "scan ok after NVIDIA overload fallback" - exit 0 - ;; - *) - echo "Error: NVIDIA overload fallback path unexpected (${STRIX_LLM:-})" >&2 - exit 37 - ;; - esac - ;; - gemini-timeout-direct-fallback-success) - case "${STRIX_LLM:-}" in - gemini/retry-timeout-primary) - echo "LLM CONNECTION FAILED" - echo "Error: litellm.Timeout: Connection timed out after None seconds." - exit 1 - ;; - gemini/fallback-one) - echo "scan ok after timeout fallback" - exit 0 - ;; - *) - echo "Error: gemini timeout fallback path unexpected (${STRIX_LLM:-})" >&2 - exit 38 - ;; - esac - ;; - gemini-timeout-fallback-success|gemini-generic-fallback-success) - case "${STRIX_LLM:-}" in - gemini/timeout-fallback-primary) - echo "LLM CONNECTION FAILED" - echo "Error: litellm.Timeout: Connection timed out after None seconds." - exit 1 - ;; - gemini/fallback-one) - echo "scan ok after gemini fallback" - exit 0 - ;; - *) - echo "Error: gemini timeout fallback path unexpected (${STRIX_LLM:-})" >&2 - exit 39 - ;; - esac - ;; - gemini-zero-findings-timeout-fallback-allows-pr) - case "${STRIX_LLM:-}" in - gemini/zero-timeout-primary|gemini/fallback-one) - echo "Vulnerabilities 0" - echo "LLM CONNECTION FAILED" - echo "Error: litellm.Timeout: Connection timed out after None seconds." - exit 1 - ;; - *) - echo "Error: gemini zero-finding fallback path unexpected (${STRIX_LLM:-})" >&2 - exit 40 - ;; - esac - ;; pr-scope-zero-finding-does-not-leak) if [ -f "$target_path/sync-module-system/smart-crawling-biz/src/main/java/org/empasy/sync/modules/system/controller/SysPositionController.java" ]; then echo "Vulnerabilities 0" @@ -4132,36 +3627,18 @@ EOS echo "ConnectionError: Server disconnected without sending a response." exit 1 ;; - vertex-all-ratelimited) - echo "Penetration test failed: LLM request failed: RateLimitError" - exit 1 - ;; vertex-primary-hallucinated-endpoint-fallback-success|target-path-src-default-source-dirs) - case "${STRIX_LLM:-}" in - vertex_ai/hallucination-primary) - mkdir -p "$STRIX_REPORTS_DIR/fake-hallucinated/vulnerabilities" - cat >"$STRIX_REPORTS_DIR/fake-hallucinated/vulnerabilities/vuln-0001.md" <<'EOS' + mkdir -p "$STRIX_REPORTS_DIR/fake-hallucinated/vulnerabilities" + cat >"$STRIX_REPORTS_DIR/fake-hallucinated/vulnerabilities/vuln-0001.md" <<'EOS' **Severity:** CRITICAL **Endpoint:** /api/ghost-admin EOS - echo "Penetration test failed: CRITICAL finding on /api/ghost-admin" - exit 1 - ;; - vertex_ai/fallback-one) - echo "scan ok after hallucinated-endpoint fallback" - exit 0 - ;; - *) - echo "Error: hallucinated-endpoint fallback path unexpected (${STRIX_LLM:-})" >&2 - exit 26 - ;; - esac + echo "Penetration test failed: CRITICAL finding on /api/ghost-admin" + exit 1 ;; opencode-documented-env-api-key-fallback-success) - case "${STRIX_LLM:-}" in - vertex_ai/opencode-env-primary) - mkdir -p "$STRIX_REPORTS_DIR/fake-opencode-env/vulnerabilities" - cat >"$STRIX_REPORTS_DIR/fake-opencode-env/vulnerabilities/vuln-0001.md" <"$STRIX_REPORTS_DIR/fake-opencode-env/vulnerabilities/vuln-0001.md" <&2 - exit 27 - ;; - esac + echo "Penetration test failed: documented OpenCode env apiKey reference" + exit 1 ;; generic-github-actions-workflow-fallback-success) - case "${STRIX_LLM:-}" in - vertex_ai/generic-actions-primary) - mkdir -p "$STRIX_REPORTS_DIR/fake-generic-actions/vulnerabilities" - cat >"$STRIX_REPORTS_DIR/fake-generic-actions/vulnerabilities/vuln-0001.md" <<'EOS' + mkdir -p "$STRIX_REPORTS_DIR/fake-generic-actions/vulnerabilities" + cat >"$STRIX_REPORTS_DIR/fake-generic-actions/vulnerabilities/vuln-0001.md" <<'EOS' # Insecure Configurations in GitHub Actions Workflows **Severity:** CRITICAL @@ -4220,68 +3685,32 @@ The GitHub Actions configuration contains several security weaknesses: + Secured version ``` EOS - echo "Penetration test failed: generic GitHub Actions workflow finding" - exit 1 - ;; - vertex_ai/fallback-one) - echo "scan ok after generic GitHub Actions workflow false positive" - exit 0 - ;; - *) - echo "Error: generic GitHub Actions workflow fallback path unexpected (${STRIX_LLM:-})" >&2 - exit 37 - ;; - esac + echo "Penetration test failed: generic GitHub Actions workflow finding" + exit 1 ;; vertex-primary-existing-endpoint-nonrecoverable|multi-source-dirs-existing-endpoint) - case "${STRIX_LLM:-}" in - vertex_ai/existing-endpoint-primary|vertex_ai/multi-dir-primary) - mkdir -p "$STRIX_REPORTS_DIR/fake-existing-endpoint/vulnerabilities" - cat >"$STRIX_REPORTS_DIR/fake-existing-endpoint/vulnerabilities/vuln-0001.md" <<'EOS' + mkdir -p "$STRIX_REPORTS_DIR/fake-existing-endpoint/vulnerabilities" + cat >"$STRIX_REPORTS_DIR/fake-existing-endpoint/vulnerabilities/vuln-0001.md" <<'EOS' **Endpoint:** /api/status EOS - echo "Penetration test failed: CRITICAL finding on /api/status" - exit 1 - ;; - vertex_ai/fallback-one|vertex_ai/fallback-two) - echo "Error: existing endpoint findings must remain non-recoverable (${STRIX_LLM:-})" >&2 - exit 27 - ;; - *) - echo "Error: existing-endpoint scenario unexpected model (${STRIX_LLM:-})" >&2 - exit 28 - ;; - esac + echo "Penetration test failed: CRITICAL finding on /api/status" + exit 1 ;; pr-stale-source-claim-fallback-success) - case "${STRIX_LLM:-}" in - vertex_ai/stale-source-primary) - mkdir -p "$STRIX_REPORTS_DIR/fake-stale-source/vulnerabilities" - cat >"$STRIX_REPORTS_DIR/fake-stale-source/vulnerabilities/vuln-0001.md" <<'EOS' + mkdir -p "$STRIX_REPORTS_DIR/fake-stale-source/vulnerabilities" + cat >"$STRIX_REPORTS_DIR/fake-stale-source/vulnerabilities/vuln-0001.md" <<'EOS' **Severity:** HIGH **Target:** backend/db/models.py The `WorkspaceRunnerConfig.registration_token` field stores the token as plain text. The vulnerable line is `registration_token: Mapped[str | None] = mapped_column(String, nullable=True)`. EOS - echo "Penetration test failed: stale HIGH finding on backend/db/models.py" - exit 1 - ;; - vertex_ai/fallback-one) - echo "scan ok after stale-source fallback" - exit 0 - ;; - *) - echo "Error: stale-source scenario unexpected model (${STRIX_LLM:-})" >&2 - exit 30 - ;; - esac + echo "Penetration test failed: stale HIGH finding on backend/db/models.py" + exit 1 ;; pr-stale-snapshot-snippet-fallback-success) - case "${STRIX_LLM:-}" in - vertex_ai/stale-snapshot-primary) - mkdir -p "$STRIX_REPORTS_DIR/fake-stale-snapshot/vulnerabilities" - cat >"$STRIX_REPORTS_DIR/fake-stale-snapshot/vulnerabilities/vuln-0001.md" <<'EOS' + mkdir -p "$STRIX_REPORTS_DIR/fake-stale-snapshot/vulnerabilities" + cat >"$STRIX_REPORTS_DIR/fake-stale-snapshot/vulnerabilities/vuln-0001.md" <<'EOS' # IDOR in /api/snapshots endpoint allows unauthorized access to database schemas **Severity:** MEDIUM @@ -4313,118 +3742,60 @@ return snapshot + return snapshot ``` EOS - echo "Penetration test failed: stale MEDIUM snapshot snippet" - exit 1 - ;; - vertex_ai/fallback-one) - echo "scan ok after stale snapshot snippet fallback" - exit 0 - ;; - *) - echo "Error: stale-snapshot scenario unexpected model (${STRIX_LLM:-})" >&2 - exit 38 - ;; - esac + echo "Penetration test failed: stale MEDIUM snapshot snippet" + exit 1 ;; pr-stale-source-plus-real-finding-blocks) - case "${STRIX_LLM:-}" in - vertex_ai/stale-source-primary) - mkdir -p "$STRIX_REPORTS_DIR/fake-mixed-findings/vulnerabilities" - cat >"$STRIX_REPORTS_DIR/fake-mixed-findings/vulnerabilities/vuln-0001.md" <<'EOS' + mkdir -p "$STRIX_REPORTS_DIR/fake-mixed-findings/vulnerabilities" + cat >"$STRIX_REPORTS_DIR/fake-mixed-findings/vulnerabilities/vuln-0001.md" <<'EOS' **Severity:** HIGH **Target:** backend/db/models.py The `WorkspaceRunnerConfig.registration_token` field stores the token as plain text. The vulnerable line is `registration_token: Mapped[str | None] = mapped_column(String, nullable=True)`. EOS - cat >"$STRIX_REPORTS_DIR/fake-mixed-findings/vulnerabilities/vuln-0002.md" <<'EOS' + cat >"$STRIX_REPORTS_DIR/fake-mixed-findings/vulnerabilities/vuln-0002.md" <<'EOS' **Severity:** HIGH **Target:** backend/api/emails.py This is a concrete changed-file finding that must remain blocking. EOS - echo "Penetration test failed: mixed stale and real HIGH findings" - exit 1 - ;; - vertex_ai/fallback-one) - echo "Error: mixed real findings must not reach fallback" >&2 - exit 31 - ;; - *) - echo "Error: mixed-findings scenario unexpected model (${STRIX_LLM:-})" >&2 - exit 32 - ;; - esac + echo "Penetration test failed: mixed stale and real HIGH findings" + exit 1 ;; pr-changed-finding-with-retry-marker-blocks) - case "${STRIX_LLM:-}" in - vertex_ai/changed-finding-primary) - mkdir -p "$STRIX_REPORTS_DIR/fake-changed-retry-marker/vulnerabilities" - cat >"$STRIX_REPORTS_DIR/fake-changed-retry-marker/vulnerabilities/vuln-0001.md" <<'EOS' + mkdir -p "$STRIX_REPORTS_DIR/fake-changed-retry-marker/vulnerabilities" + cat >"$STRIX_REPORTS_DIR/fake-changed-retry-marker/vulnerabilities/vuln-0001.md" <<'EOS' **Severity:** HIGH **Target:** backend/api/emails.py This changed-file finding must remain blocking even when the model log also contains retryable provider text. EOS - echo "litellm.exceptions.Timeout: provider timed out after writing a HIGH changed-file finding" - exit 1 - ;; - vertex_ai/fallback-one) - echo "Error: changed-file findings with retry markers must not reach fallback" >&2 - exit 33 - ;; - *) - echo "Error: changed-retry-marker scenario unexpected model (${STRIX_LLM:-})" >&2 - exit 34 - ;; - esac + echo "litellm.exceptions.Timeout: provider timed out after writing a HIGH changed-file finding" + exit 1 ;; pr-stale-report-plus-inline-changed-finding-blocks) - case "${STRIX_LLM:-}" in - vertex_ai/stale-inline-primary) - mkdir -p "$STRIX_REPORTS_DIR/fake-stale-report-inline-changed/vulnerabilities" - cat >"$STRIX_REPORTS_DIR/fake-stale-report-inline-changed/vulnerabilities/vuln-0001.md" <<'EOS' + mkdir -p "$STRIX_REPORTS_DIR/fake-stale-report-inline-changed/vulnerabilities" + cat >"$STRIX_REPORTS_DIR/fake-stale-report-inline-changed/vulnerabilities/vuln-0001.md" <<'EOS' **Severity:** HIGH **Target:** backend/db/models.py The `WorkspaceRunnerConfig.registration_token` field stores the token as plain text. The vulnerable line is `registration_token: Mapped[str | None] = mapped_column(String, nullable=True)`. EOS - echo "Severity: HIGH" - echo "Target: backend/api/emails.py" - echo "Penetration test failed: stale report plus inline changed-file HIGH finding" - exit 1 - ;; - vertex_ai/fallback-one) - echo "Error: inline changed-file findings must not reach fallback" >&2 - exit 35 - ;; - *) - echo "Error: stale-inline scenario unexpected model (${STRIX_LLM:-})" >&2 - exit 36 - ;; - esac + echo "Severity: HIGH" + echo "Target: backend/api/emails.py" + echo "Penetration test failed: stale report plus inline changed-file HIGH finding" + exit 1 ;; endpoint-in-excluded-dir) - case "${STRIX_LLM:-}" in - vertex_ai/excluded-dir-primary) - mkdir -p "$STRIX_REPORTS_DIR/fake-excluded-dir/vulnerabilities" - cat >"$STRIX_REPORTS_DIR/fake-excluded-dir/vulnerabilities/vuln-0001.md" <<'EOS' + mkdir -p "$STRIX_REPORTS_DIR/fake-excluded-dir/vulnerabilities" + cat >"$STRIX_REPORTS_DIR/fake-excluded-dir/vulnerabilities/vuln-0001.md" <<'EOS' **Severity:** CRITICAL **Endpoint:** /api/hidden-secret EOS - echo "Penetration test failed: CRITICAL finding on /api/hidden-secret" - exit 1 - ;; - vertex_ai/fallback-one) - echo "scan ok after excluded-dir hallucination fallback" - exit 0 - ;; - *) - echo "Error: excluded-dir scenario unexpected model (${STRIX_LLM:-})" >&2 - exit 29 - ;; - esac + echo "Penetration test failed: CRITICAL finding on /api/hidden-secret" + exit 1 ;; empty-fallback-models) # Output must match is_vertex_not_found_error() patterns so the gate @@ -4483,32 +3854,14 @@ EOS exit 1 ;; model-disagreement-critical-in-earlier-report) - case "${STRIX_LLM:-}" in - vertex_ai/model-a) - mkdir -p "$STRIX_REPORTS_DIR/run-001/vulnerabilities" - cat >"$STRIX_REPORTS_DIR/run-001/vulnerabilities/vuln-0001.md" <<'EOS' + mkdir -p "$STRIX_REPORTS_DIR/run-001/vulnerabilities" + cat >"$STRIX_REPORTS_DIR/run-001/vulnerabilities/vuln-0001.md" <<'EOS' Severity: CRITICAL EOS - echo "Error: litellm.NotFoundError: Vertex_aiException - x" - echo '"status": "NOT_FOUND"' - echo "Penetration test failed: CRITICAL finding by model-a" - exit 1 - ;; - vertex_ai/model-b) - mkdir -p "$STRIX_REPORTS_DIR/run-002/vulnerabilities" - cat >"$STRIX_REPORTS_DIR/run-002/vulnerabilities/vuln-0001.md" <<'EOS' -Severity: LOW -EOS - echo "Error: litellm.NotFoundError: Vertex_aiException - x" - echo '"status": "NOT_FOUND"' - echo "Penetration test failed: LOW finding by model-b" - exit 1 - ;; - *) - echo "Error: model-disagreement unexpected model (${STRIX_LLM:-})" >&2 - exit 32 - ;; - esac + echo "Error: litellm.NotFoundError: Vertex_aiException - x" + echo '"status": "NOT_FOUND"' + echo "Penetration test failed: CRITICAL finding by model-a" + exit 1 ;; nonvertex-slash-model-not-rewritten) if [ "${STRIX_LLM:-}" = "deepseek/models/deepseek-r1" ]; then @@ -4519,46 +3872,16 @@ EOS exit 33 ;; preserve-existing-api-base) - if [ "${LLM_API_BASE:-}" = "https://preexisting.invalid" ]; then - echo "scan ok with preserved api base" + # Free-only routing pins the loopback gateway. A caller-supplied + # LLM_API_BASE must NOT survive into the scan process, or an external + # endpoint would receive organization review traffic. + if [ "${LLM_API_BASE:-}" = "http://127.0.0.1:18080/v1" ]; then + echo "scan ok with pinned gateway api base" exit 0 fi - echo "Error: existing LLM_API_BASE was not preserved (${LLM_API_BASE:-})" >&2 + echo "Error: gateway LLM_API_BASE was not pinned (${LLM_API_BASE:-})" >&2 exit 20 ;; - default-fallback-order-fast-first) - case "${STRIX_LLM:-}" in - vertex_ai/missing-primary) - echo "Error: litellm.NotFoundError: Vertex_aiException - x" - echo '"status": "NOT_FOUND"' - exit 1 - ;; - vertex_ai/gemini-2.5-pro) - echo "scan ok with default fast fallback" - exit 0 - ;; - *) - echo "Error: default fallback order unexpected (${STRIX_LLM:-})" >&2 - exit 16 - ;; - esac - ;; - vertex-primary-timeout-retry-same-model-success|vertex-primary-timeout-retry-reason-message) - case "${STRIX_LLM:-}" in - vertex_ai/retry-timeout-primary) - echo "litellm.exceptions.Timeout: litellm.Timeout: Connection timed out after None seconds." - exit 1 - ;; - vertex_ai/fallback-one) - echo "scan ok after timeout fallback" - exit 0 - ;; - *) - echo "Error: timeout fallback path unexpected (${STRIX_LLM:-})" >&2 - exit 34 - ;; - esac - ;; all-fallbacks-same-as-primary) # Bug 13: All fallback models are the same as the primary model. # The gate should emit an ERROR and exit 1. @@ -4566,82 +3889,13 @@ EOS echo '"status": "NOT_FOUND"' exit 1 ;; - vertex-primary-timeout-exhausted-fallback-success) - # Primary always times out (even after retries). Fallback succeeds. - case "${STRIX_LLM:-}" in - vertex_ai/timeout-exhaust-primary) - echo "litellm.exceptions.Timeout: litellm.Timeout: Connection timed out after None seconds." - exit 1 - ;; - vertex_ai/fallback-one) - echo "scan ok after timeout-exhausted fallback" - exit 0 - ;; - *) - echo "Error: timeout-exhausted-fallback unexpected model (${STRIX_LLM:-})" >&2 - exit 35 - ;; - esac - ;; zero-findings-timeout-all-models|strict-zero-findings-timeout-fails-pr) - case "${STRIX_LLM:-}" in - vertex_ai/zero-timeout-primary|vertex_ai/fallback-one) - echo "╭─ STRIX ──────────────────────────────────────────────────────────────────────╮" - echo "│ Penetration test in progress │" - echo "│ Vulnerabilities 0 │" - echo "╰──────────────────────────────────────────────────────────────────────────────╯" - sleep "${FAKE_STRIX_TIMEOUT_SLEEP_SECONDS:?}" - exit 0 - ;; - *) - echo "Error: zero-findings-timeout unexpected model (${STRIX_LLM:-})" >&2 - exit 57 - ;; - esac - ;; - zero-findings-sticky-across-fallback) - case "${STRIX_LLM:-}" in - vertex_ai/zero-sticky-primary) - echo "╭─ STRIX ──────────────────────────────────────────────────────────────────────╮" - echo "│ Penetration test in progress │" - echo "│ Vulnerabilities 0 │" - echo "╰──────────────────────────────────────────────────────────────────────────────╯" - sleep "${FAKE_STRIX_TIMEOUT_SLEEP_SECONDS:?}" - exit 0 - ;; - vertex_ai/fallback-one) - sleep "${FAKE_STRIX_TIMEOUT_SLEEP_SECONDS:?}" - exit 0 - ;; - *) - echo "Error: zero-findings-sticky unexpected model (${STRIX_LLM:-})" >&2 - exit 58 - ;; - esac - ;; - zero-findings-with-low-report-timeout) - case "${STRIX_LLM:-}" in - vertex_ai/zero-low-primary) - mkdir -p "$STRIX_REPORTS_DIR/fake-zero-low/vulnerabilities" - cat >"$STRIX_REPORTS_DIR/fake-zero-low/vulnerabilities/vuln-0001.md" <<'EOS' -Severity: LOW -EOS - echo "╭─ STRIX ──────────────────────────────────────────────────────────────────────╮" - echo "│ Penetration test in progress │" - echo "│ Vulnerabilities 0 │" - echo "╰──────────────────────────────────────────────────────────────────────────────╯" - sleep "${FAKE_STRIX_TIMEOUT_SLEEP_SECONDS:?}" - exit 0 - ;; - vertex_ai/fallback-one) - sleep "${FAKE_STRIX_TIMEOUT_SLEEP_SECONDS:?}" - exit 0 - ;; - *) - echo "Error: zero-findings-with-low-report unexpected model (${STRIX_LLM:-})" >&2 - exit 59 - ;; - esac + echo "╭─ STRIX ──────────────────────────────────────────────────────────────────────╮" + echo "│ Penetration test in progress │" + echo "│ Vulnerabilities 0 │" + echo "╰──────────────────────────────────────────────────────────────────────────────╯" + sleep "${FAKE_STRIX_TIMEOUT_SLEEP_SECONDS:?}" + exit 0 ;; provider-fatal-success-signal) echo "Fatal: provider stream aborted" @@ -4655,27 +3909,6 @@ EOS echo "Denied: provider credentials were rejected" exit 0 ;; - provider-report-rate-limit-fallback-success) - case "${STRIX_LLM:-}" in - vertex_ai/report-rate-limit-primary) - mkdir -p "$STRIX_REPORTS_DIR/fake-report-rate-limit" - cat >"$STRIX_REPORTS_DIR/fake-report-rate-limit/strix.log" <<'EOS' -2026-08-21 04:00:00.000 WARNING strix-pr-scope-example - strix.provider: RateLimitError: provider response was exhausted -EOS - echo "scan aborted after provider report-rate-limit signal" - exit 1 - ;; - vertex_ai/fallback-one) - mkdir -p "$STRIX_REPORTS_DIR/fake-report-rate-limit-fallback" - echo "scan ok after report-only provider fallback" - exit 0 - ;; - *) - echo "Error: report-only provider fallback path unexpected (${STRIX_LLM:-})" >&2 - exit 60 - ;; - esac - ;; report-known-internal-warning-sanitized) printf '%s\n' '│ MODEL QUALITY WARNING │' echo 'Warning: You are sending unauthenticated requests to the HF Hub.' @@ -4715,29 +3948,6 @@ EOS echo "scan ok but unknown report warning remains" exit 0 ;; - bare-timeout-with-provider-marker) - # Emit bare "Connection timed out" alongside a provider marker so - # is_timeout_error() matches the Tier 3 branch gated on - # LLM_PROVIDER_ONLY_REGEX. Does NOT include - # litellm.exceptions.Timeout / httpx.ReadTimeout to ensure we - # exercise the provider-marker fallback path specifically. - # Primary times out; fallback model succeeds. - case "${STRIX_LLM:-}" in - vertex_ai/bare-timeout-primary) - echo "Connection timed out" - echo "vertex_ai model invocation failed" - exit 1 - ;; - vertex_ai/fallback-one) - echo "scan ok after bare-timeout fallback" - exit 0 - ;; - *) - echo "Error: bare-timeout fallback path unexpected (${STRIX_LLM:-})" >&2 - exit 47 - ;; - esac - ;; bare-timeout-no-provider-marker) # Emit "Connection timed out" with transport library names (httpx, # httpcore, requests) but WITHOUT any real LLM provider marker. @@ -4749,27 +3959,6 @@ EOS echo "requests transport timeout" exit 1 ;; - below-threshold-with-timeout) - # Produce a below-threshold (LOW) finding but also emit a timeout error - # so the infrastructure guard detects an incomplete scan. - mkdir -p "$STRIX_REPORTS_DIR/fake-low-timeout/vulnerabilities" - cat >"$STRIX_REPORTS_DIR/fake-low-timeout/vulnerabilities/vuln-0001.md" <<'EOS' -Severity: LOW -EOS - echo "litellm.exceptions.Timeout: litellm.Timeout: Connection timed out after None seconds." - echo "Penetration test failed: simulated timeout with low finding" - exit 1 - ;; - below-threshold-with-ratelimit) - # Produce a below-threshold (LOW) finding but also emit a rate-limit error. - mkdir -p "$STRIX_REPORTS_DIR/fake-low-ratelimit/vulnerabilities" - cat >"$STRIX_REPORTS_DIR/fake-low-ratelimit/vulnerabilities/vuln-0001.md" <<'EOS' -Severity: LOW -EOS - echo "Penetration test failed: LLM request failed: RateLimitError" - echo "Penetration test failed: simulated ratelimit with low finding" - exit 1 - ;; below-threshold-with-connection-error) # Produce a below-threshold (INFO) finding but also emit a # ConnectionError WITH an LLM-provider context marker so the @@ -4818,55 +4007,6 @@ EOS echo "Penetration test failed: simulated requests transport error" exit 1 ;; - below-threshold-with-midstream) - # Produce a below-threshold (MEDIUM) finding below CRITICAL threshold - # but also emit a MidStreamFallbackError. - mkdir -p "$STRIX_REPORTS_DIR/fake-medium-midstream/vulnerabilities" - cat >"$STRIX_REPORTS_DIR/fake-medium-midstream/vulnerabilities/vuln-0001.md" <<'EOS' -Severity: MEDIUM -EOS - echo "Penetration test failed: LLM request failed: MidStreamFallbackError" - echo "Penetration test failed: simulated midstream with medium finding" - exit 1 - ;; - bare-timeout-provider-marker-exhausted-fallback) - # Bare "Connection timed out" + provider marker: primary fails once, - # then the gate falls back to fallback-one which succeeds. - case "${STRIX_LLM:-}" in - vertex_ai/bare-timeout-exhaust-primary) - echo "Connection timed out" - echo "vertex_ai model invocation failed" - exit 1 - ;; - vertex_ai/fallback-one) - echo "scan ok after bare-timeout-exhaust fallback" - exit 0 - ;; - *) - echo "Error: bare-timeout-exhaust-fallback unexpected model (${STRIX_LLM:-})" >&2 - exit 35 - ;; - esac - ;; - httpx-read-timeout-with-provider-marker) - # Tier 2: httpx.ReadTimeout + provider-context marker (litellm). - # Primary times out; fallback model succeeds. - case "${STRIX_LLM:-}" in - vertex_ai/httpx-timeout-primary) - echo "httpx.ReadTimeout: timed out" - echo "litellm.proxy: connection to upstream model failed" - exit 1 - ;; - vertex_ai/fallback-one) - echo "scan ok after httpx-timeout fallback" - exit 0 - ;; - *) - echo "Error: httpx-timeout fallback path unexpected (${STRIX_LLM:-})" >&2 - exit 45 - ;; - esac - ;; httpx-read-timeout-no-provider-marker) # Tier 2 negative: httpx.ReadTimeout WITHOUT any provider-context # marker. Should NOT be classified as retryable timeout. @@ -4874,25 +4014,6 @@ EOS echo "application server connection pool exhausted" exit 1 ;; - httpcore-read-timeout-with-provider-marker) - # Tier 2b: httpcore.ReadTimeout + provider-context marker. - # Primary times out; fallback model succeeds. - case "${STRIX_LLM:-}" in - vertex_ai/httpcore-timeout-primary) - echo "httpcore.ReadTimeout: timed out" - echo "litellm.proxy: connection to upstream model failed" - exit 1 - ;; - vertex_ai/fallback-one) - echo "scan ok after httpcore-timeout fallback" - exit 0 - ;; - *) - echo "Error: httpcore-timeout fallback path unexpected (${STRIX_LLM:-})" >&2 - exit 46 - ;; - esac - ;; httpcore-read-timeout-no-provider-marker) # Tier 2b negative: httpcore.ReadTimeout WITHOUT any provider-context # marker. Should NOT be classified as retryable timeout. @@ -4900,34 +4021,6 @@ EOS echo "application server connection pool exhausted" exit 1 ;; - infra-error-sticky-flag) - # Sticky flag test: first call hits infra error (rate limit), - # second call fails on the first fallback model but produces a - # LOW finding report. After exhausting retries, the gate checks - # has_only_below_threshold_vulnerabilities — which finds LOW - # findings but sees INFRA_ERROR_DETECTED=1 (set from the first - # call's rate-limit error) and refuses the below-threshold bypass. - case "${STRIX_LLM:-}" in - vertex_ai/sticky-flag-primary) - touch "$FAKE_STRIX_STATE_FILE" - echo "RateLimitError: rate limit exceeded" - echo "litellm.proxy: rate limit on vertex_ai model" - exit 1 - ;; - vertex_ai/gemini-2.5-pro) - mkdir -p "$STRIX_REPORTS_DIR/run-sticky/vulnerabilities" - cat > "$STRIX_REPORTS_DIR/run-sticky/vulnerabilities/vuln-0001.md" <<'FINDINGS' -Severity: LOW -FINDINGS - echo "non-retryable scan error with partial results" - exit 1 - ;; - *) - echo "Error: infra-error-sticky-flag unexpected model (${STRIX_LLM:-})" >&2 - exit 35 - ;; - esac - ;; pr-baseline-critical-unchanged) mkdir -p "$STRIX_REPORTS_DIR/fake-pr-baseline/vulnerabilities" cat >"$STRIX_REPORTS_DIR/fake-pr-baseline/vulnerabilities/vuln-0001.md" <<'EOS' @@ -5190,90 +4283,6 @@ EOS echo "Penetration test failed: manifest-only critical finding" exit 1 ;; - pr-critical-manifest-only-pom-after-fallback-authoritative) - case "${STRIX_LLM:-}" in - vertex_ai/timeout-primary) - echo "litellm.exceptions.Timeout: primary model timed out" - exit 1 - ;; - vertex_ai/fallback-one) - mkdir -p "$STRIX_REPORTS_DIR/fake-pr-manifest-only-after-fallback/vulnerabilities" - cat >"$STRIX_REPORTS_DIR/fake-pr-manifest-only-after-fallback/vulnerabilities/vuln-0001.md" <<'EOS' -Severity: CRITICAL -Location 1: -pom.xml:8 -EOS - echo "Penetration test failed: manifest-only critical finding after fallback" - exit 1 - ;; - *) - echo "Error: pr-critical-manifest-only-pom-after-fallback-authoritative unexpected model (${STRIX_LLM:-})" >&2 - exit 53 - ;; - esac - ;; - pr-critical-manifest-only-pom-console-only-after-fallback-authoritative) - case "${STRIX_LLM:-}" in - vertex_ai/timeout-primary) - echo "litellm.exceptions.Timeout: primary model timed out" - exit 1 - ;; - vertex_ai/fallback-one) - echo "Severity: CRITICAL" - echo "Location 1:" - echo "pom.xml:59" - echo "Penetration test failed: manifest-only critical finding after fallback (console-only)" - exit 1 - ;; - *) - echo "Error: pr-critical-manifest-only-pom-console-only-after-fallback-authoritative unexpected model (${STRIX_LLM:-})" >&2 - exit 54 - ;; - esac - ;; - pr-critical-manifest-only-pom-console-target-only-after-fallback-authoritative) - case "${STRIX_LLM:-}" in - vertex_ai/timeout-primary) - echo "litellm.exceptions.Timeout: primary model timed out" - exit 1 - ;; - vertex_ai/fallback-one) - echo "Severity: CRITICAL" - echo "Target: /workspace/$(basename "$target_path")/pom.xml" - echo "Penetration test failed: manifest-only critical finding after fallback (console target-only)" - exit 1 - ;; - *) - echo "Error: pr-critical-manifest-only-pom-console-target-only-after-fallback-authoritative unexpected model (${STRIX_LLM:-})" >&2 - exit 56 - ;; - esac - ;; - pr-low-markdown-plus-console-critical-manifest-after-fallback-authoritative) - case "${STRIX_LLM:-}" in - vertex_ai/timeout-primary) - echo "litellm.exceptions.Timeout: primary model timed out" - exit 1 - ;; - vertex_ai/fallback-one) - mkdir -p "$STRIX_REPORTS_DIR/fake-pr-manifest-mixed-after-fallback/vulnerabilities" - cat >"$STRIX_REPORTS_DIR/fake-pr-manifest-mixed-after-fallback/vulnerabilities/vuln-0001.md" <<'EOS' -Severity: LOW -Location 1: -pom.xml:8 -EOS - echo "Severity: CRITICAL" - echo "Location 1:" - echo "pom.xml:59" - echo "Penetration test failed: manifest-only critical finding after fallback (mixed file+console)" - exit 1 - ;; - *) - echo "Error: pr-low-markdown-plus-console-critical-manifest-after-fallback-authoritative unexpected model (${STRIX_LLM:-})" >&2 - exit 55 - ;; - esac - ;; pr-changed-scope-bounded) if [ -z "$target_path" ]; then echo "Error: target path missing" >&2 @@ -6003,12 +5012,6 @@ PY "LLM_TIMEOUT=90;STRIX_MEMORY_COMPRESSOR_TIMEOUT=10;STRIX_REASONING_EFFORT=minimal;STRIX_LLM_MAX_RETRIES=1;GEMINI_LOCATION=GLOBAL;PYTHONWARNINGS=ignore:Pydantic serializer warnings:UserWarning:pydantic.main;NPM_CONFIG_IGNORE_SCRIPTS=true;PNPM_CONFIG_IGNORE_SCRIPTS=true;YARN_ENABLE_SCRIPTS=false;UNRELATED_SECRET=" \ "scenario=$scenario runtime env forwarding" fi - if [ "$scenario" = "custom-openai-compatible-preserves-effort" ]; then - assert_file_contains \ - "$runtime_env_log" \ - "STRIX_REASONING_EFFORT=minimal" \ - "scenario=$scenario custom compatible endpoint effort" - fi if [ "$scenario" = "report-known-internal-warning-sanitized" ]; then assert_file_not_contains \ @@ -6100,44 +5103,6 @@ run_gate_case_allow_provider_signal() { run_gate_case_with_provider_signal_mode "0" "$@" } -run_github_models_http410_case() { - local scenario="$1" - local expected_exit="$2" - local expected_calls="$3" - local expected_models="$4" - local expected_api_bases="$5" - local expected_message="${6-}" - - run_gate_case "$scenario" \ - "openai/gpt-5" \ - "" \ - "$expected_exit" \ - "$expected_message" \ - "$expected_calls" \ - "$expected_models" \ - "$expected_api_bases" \ - "openai" \ - "https://models.github.ai/inference" \ - "" \ - "0" \ - "CRITICAL" \ - "0" \ - "" \ - "" \ - "1200" \ - "0" \ - "" \ - "" \ - "" \ - "" \ - "0" \ - "" \ - "" \ - "" \ - "__SAME_AS_FALLBACK_MODELS__" \ - "deepseek/deepseek-r1-0528" \ - "1" -} run_filtered_gate_case_if_requested() { case "${STRIX_TEST_CASE_FILTER:-}" in @@ -6146,13 +5111,15 @@ run_filtered_gate_case_if_requested() { ;; success) run_gate_case "success" \ - "vertex_ai/ready-primary" \ - "vertex_ai/fallback-one vertex_ai/fallback-two" \ + "orchestrator/free" \ + "" \ "0" \ "scan ok" \ "1" \ - "vertex_ai/ready-primary" \ - "" + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" ;; contextual-orchestrator-missing-api-base-fails-closed) run_gate_case "contextual-orchestrator-missing-api-base-fails-closed" \ @@ -6180,15 +5147,15 @@ run_filtered_gate_case_if_requested() { ;; pr-rust-workspace-context) run_gate_case "pr-rust-workspace-context" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "0" \ "scan ok with Rust workspace context" \ "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -6202,516 +5169,170 @@ run_filtered_gate_case_if_requested() { ;; success-with-critical-report) run_gate_case "success-with-critical-report" \ - "vertex_ai/ready-primary" \ + "orchestrator/free" \ "" \ "1" \ - "Strix exited successfully but emitted a vulnerability at or above 'CRITICAL'" \ + "Current Strix vulnerability report exists; failing closed without a repository-authored severity threshold." \ "1" \ - "vertex_ai/ready-primary" \ - "" + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" ;; pr-executable-integrity-mismatch) run_gate_case "pr-executable-integrity-mismatch" \ - "vertex_ai/ready-primary" \ + "orchestrator/free" \ "" \ "1" \ "did not match the pinned SHA-256 digest" \ "0" \ "" \ - "" + "" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" ;; pr-executable-group-writable) run_gate_case "pr-executable-group-writable" \ - "vertex_ai/ready-primary" \ + "orchestrator/free" \ "" \ "1" \ "must not be group/world writable" \ "0" \ "" \ - "" + "" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" ;; pr-executable-root-group-writable) run_gate_case "pr-executable-root-group-writable" \ - "vertex_ai/ready-primary" \ + "orchestrator/free" \ "" \ "1" \ "pinned Strix installation root must not be group/world writable" \ "0" \ "" \ - "" + "" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" ;; vertex-primary-hallucinated-endpoint-fallback-success) run_gate_case "vertex-primary-hallucinated-endpoint-fallback-success" \ - "vertex_ai/hallucination-primary" \ - "vertex_ai/fallback-one vertex_ai/fallback-two" \ + "orchestrator/free" \ + "" \ "1" \ "Strix quick scan failed with a non-recoverable error." \ "1" \ - "vertex_ai/hallucination-primary" \ - "" + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" ;; target-path-src-default-source-dirs) run_gate_case "target-path-src-default-source-dirs" \ - "vertex_ai/hallucination-primary" \ - "vertex_ai/fallback-one vertex_ai/fallback-two" \ + "orchestrator/free" \ + "" \ "1" \ "Strix quick scan failed with a non-recoverable error." \ "1" \ - "vertex_ai/hallucination-primary" \ - "" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ - "1" \ + "0" \ "CRITICAL" \ "0" \ "__USE_SUBDIR_SRC__" \ "" ;; - vertex-ignores-untrusted-llm-api-base-file) - run_vertex_model_ignores_untrusted_llm_api_base_file_case - ;; - input-file-root-override-precedence) - run_input_file_root_override_takes_precedence_over_runner_temp_case - ;; - vertex-without-llm-api-key) - run_vertex_without_llm_api_key_case - ;; - vertex-with-llm-api-key-file-not-forwarded) - run_vertex_with_llm_api_key_file_does_not_forward_case - ;; - stale-report-does-not-bypass) - run_stale_report_case - ;; - symlink-report-does-not-bypass) - run_symlink_report_case - ;; - github-models-token-limit-fallback-success) - run_gate_case "github-models-token-limit-fallback-success" \ - "openai/gpt-5" \ - "" \ - "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'github_models/deepseek/deepseek-v3-0324' in [0-9]+s\\." \ - "2" \ - "openai/gpt-5|openai/deepseek/deepseek-v3-0324" \ - "https://models.github.ai/inference|https://models.github.ai/inference" \ - "openai" \ - "https://models.github.ai/inference" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "github_models/deepseek/deepseek-v3-0324 github_models/deepseek/deepseek-r1-0528" - ;; - openrouter-502-fallback-retry-same-model-success) - run_gate_case "openrouter-502-fallback-retry-same-model-success" \ - "vertex_ai/missing-primary" \ - "openrouter/free vertex_ai/fallback-two" \ - "0" \ - "scan ok after OpenRouter 502 same-model retry" \ - "3" \ - "vertex_ai/missing-primary|openrouter/free|openrouter/free" \ - "|https://example.invalid|https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ - "" \ - "1" - ;; - openrouter-502-distant-target-output-nonretryable) - run_gate_case "openrouter-502-distant-target-output-nonretryable" \ - "vertex_ai/missing-primary" \ - "openrouter/free vertex_ai/fallback-two" \ - "1" \ - "Strix quick scan failed with a non-recoverable error." \ - "2" \ - "vertex_ai/missing-primary|openrouter/free" \ - "|https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ - "" \ - "1" - ;; service-unavailable-no-llm-marker-nonrecoverable) run_gate_case "service-unavailable-no-llm-marker-nonrecoverable" \ - "custom/service-unavailable-primary" \ - "vertex_ai/fallback-one vertex_ai/fallback-two" \ + "orchestrator/free" \ + "" \ "1" \ - "Strix quick scan failed with a non-recoverable error." \ + "Strix quick scan failed; failing closed without repository-authored retry or fallback allocation." \ "1" \ - "custom/service-unavailable-primary" \ - "https://example.invalid" \ - "custom" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ - "1" + "0" ;; custom-openai-compatible-preserves-effort) run_gate_case "custom-openai-compatible-preserves-effort" \ "openai-direct/gpt-5.4" \ "" \ + "2" \ + "ERROR: Strix model must be orchestrator/free through contextual-orchestrator; direct provider/model routes are forbidden" \ "0" \ - "scan ok" \ - "1" \ - "openai/gpt-5.4" \ - "https://compatible.example/v1" \ - "openai" \ - "https://compatible.example/v1" + "" \ + "" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" ;; - nvidia-rate-limit-openai-direct-fallback-clears-api-base) - run_gate_case_allow_provider_signal "nvidia-rate-limit-openai-direct-fallback-clears-api-base" \ - "nvidia_nim/nvidia/rate-limited-primary" \ + pr-critical-changed-json-target) + run_gate_case "pr-critical-changed-json-target" \ + "orchestrator/free" \ "" \ - "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'openai-direct/gpt-5.4' in [0-9]+s\\." \ - "2" \ - "nvidia_nim/nvidia/rate-limited-primary|openai/gpt-5.4" \ - "https://integrate.api.nvidia.com/v1|" \ - "nvidia_nim" \ - "https://integrate.api.nvidia.com/v1" \ + "1" \ + "Strix finding intersects files changed in this pull request." \ + "1" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ - "CRITICAL" \ + "MEDIUM" \ "0" \ "" \ "" \ "1200" \ "0" \ - "" \ - "" \ - "" \ - "" \ - "0" \ - "" \ - "" \ - "" \ - "__SAME_AS_FALLBACK_MODELS__" \ - "openai-direct/gpt-5.4" + "pull_request" \ + "frontend/src/components/CalendarLayout.tsx" ;; - openai-direct-quota-github-models-fallback-success) - run_gate_case "openai-direct-quota-github-models-fallback-success" \ - "openai_direct/gpt-5.4" \ - "" \ - "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'github_models/openai/o3' in [0-9]+s\\." \ - "2" \ - "openai/gpt-5.4|openai/o3" \ - "|https://models.github.ai/inference" \ - "vertex_ai" \ - "" \ + github-models-http410-missing-http-token | github-models-http410-missing-provider-error | github-models-http410-numeric-continuation-4100 | github-models-http410-numeric-continuation-4104 | github-models-http410-target-output-spoof | github-models-retirement-brownout-phrase-only) + run_github_models_http410_case \ + "$STRIX_TEST_CASE_FILTER" \ + "1" \ + "1" \ + "openai/gpt-5" \ + "https://models.github.ai/inference" + ;; + internal-server-error-unrelated-output-nonretryable) + run_gate_case_allow_provider_signal "$STRIX_TEST_CASE_FILTER" \ + "openai/openai/retry-api-connection-primary" \ "" \ + "1" \ + "Strix quick scan failed with a non-recoverable error." \ + "1" \ + "openai/openai/retry-api-connection-primary" \ + "https://models.github.ai/inference" \ + "openai" \ + "https://models.github.ai/inference" \ "" \ + "0" + ;; + endpoint-in-excluded-dir) + run_gate_case "endpoint-in-excluded-dir" \ + "orchestrator/free" \ "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "github_models/openai/o3" - ;; - gemini-timeout-fallback-success) - run_gate_case_allow_provider_signal "gemini-timeout-fallback-success" \ - "gemini/timeout-fallback-primary" \ - "gemini/fallback-one gemini/fallback-two" \ - "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'gemini/fallback-one' in [0-9]+s\\." \ - "2" \ - "gemini/timeout-fallback-primary|gemini/fallback-one" \ - "https://example.invalid|https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ - "" \ - "1" - ;; - zero-findings-with-low-report-timeout) - run_gate_case_allow_provider_signal "zero-findings-with-low-report-timeout" \ - "vertex_ai/zero-low-primary" \ - "vertex_ai/fallback-one" \ - "1" \ - "Configured Vertex model and fallback models were unavailable." \ - "2" \ - "vertex_ai/zero-low-primary|vertex_ai/fallback-one" \ - "|" \ - "vertex_ai" \ - "__DEFAULT__" \ - "" \ - "0" \ - "CRITICAL" \ - "0" \ - "" \ - "" \ - "$TIMEOUT_TEST_PROCESS_SECONDS" \ - "0" \ - "pull_request" \ - "sync-module-system/smart-crawling-biz/src/main/java/org/empasy/sync/modules/system/controller/SysPositionController.java" - ;; - zero-findings-timeout-all-models) - run_gate_case_allow_provider_signal "zero-findings-timeout-all-models" \ - "vertex_ai/zero-timeout-primary" \ - "vertex_ai/fallback-one" \ - "1" \ - "Strix reported zero vulnerabilities before provider infrastructure failure; failing closed because provider infrastructure failures are not clean scan evidence." \ - "2" \ - "vertex_ai/zero-timeout-primary|vertex_ai/fallback-one" \ - "|" \ - "vertex_ai" \ - "__DEFAULT__" \ - "" \ - "0" \ - "CRITICAL" \ - "0" \ - "" \ - "" \ - "$TIMEOUT_TEST_PROCESS_SECONDS" \ - "0" \ - "pull_request" \ - "sync-module-system/smart-crawling-biz/src/main/java/org/empasy/sync/modules/system/controller/SysPositionController.java" - run_gate_case_allow_provider_signal "zero-findings-timeout-all-models" \ - "vertex_ai/zero-timeout-primary" \ - "vertex_ai/fallback-one" \ - "1" \ - "Configured Vertex model and fallback models were unavailable." \ - "2" \ - "vertex_ai/zero-timeout-primary|vertex_ai/fallback-one" \ - "|" \ - "vertex_ai" \ - "__DEFAULT__" \ - "" \ - "0" \ - "CRITICAL" \ - "0" \ - "" \ - "" \ - "$TIMEOUT_TEST_PROCESS_SECONDS" \ - "0" \ - "push" - ;; - slow-timeout) - run_gate_case_allow_provider_signal "slow-timeout" \ - "vertex_ai/slow-primary" \ - "" \ - "1" \ - "Strix run timed out after ${TIMEOUT_TEST_PROCESS_SECONDS}s." \ - "3" \ - "vertex_ai/slow-primary|vertex_ai/gemini-2.5-pro|vertex_ai/gemini-2.5-flash" \ - "||" \ - "vertex_ai" \ - "__DEFAULT__" \ - "" \ - "0" \ - "CRITICAL" \ - "0" \ - "" \ - "" \ - "$TIMEOUT_TEST_PROCESS_SECONDS" - ;; - timeout-cleanup) - run_timeout_cleanup_case - ;; - vertex-primary-notfound-fallback-success) - run_gate_case "vertex-primary-notfound-fallback-success" \ - "vertex_ai/missing-primary" \ - "vertex_ai/fallback-one vertex_ai/fallback-two" \ - "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'vertex_ai/fallback-one' in [0-9]+s\\." \ - "2" \ - "vertex_ai/missing-primary|vertex_ai/fallback-one" \ - "|" - ;; - openai-primary-quota-fallback-success) - run_gate_case_allow_provider_signal "openai-primary-quota-fallback-success" \ - "openai/quota-primary" \ - "openai/fallback-one openai/fallback-two" \ - "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'openai/fallback-one' in [0-9]+s\\." \ - "2" \ - "openai/quota-primary|openai/fallback-one" \ - "|" \ - "openai" - ;; - pr-critical-changed-json-target) - run_gate_case "pr-critical-changed-json-target" \ - "vertex_ai/gemini-2.5-pro" \ - "" \ - "1" \ - "Strix finding intersects files changed in this pull request." \ - "1" \ - "vertex_ai/gemini-2.5-pro" \ - "" \ - "vertex_ai" \ - "__DEFAULT__" \ - "" \ - "0" \ - "MEDIUM" \ - "0" \ - "" \ - "" \ - "1200" \ - "0" \ - "pull_request" \ - "frontend/src/components/CalendarLayout.tsx" - ;; - github-models-primary-ratelimit-fallback-success) - run_gate_case "github-models-primary-ratelimit-fallback-success" \ - "openai/gpt-5" \ - "" \ - "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'deepseek/deepseek-r1-0528' in [0-9]+s\\." \ - "2" \ - "openai/gpt-5|openai/deepseek/deepseek-r1-0528" \ - "https://models.github.ai/inference|https://models.github.ai/inference" \ - "openai" \ - "https://models.github.ai/inference" \ - "" \ - "2" \ - "CRITICAL" \ - "0" \ - "" \ - "" \ - "1200" \ - "0" \ - "" \ - "" \ - "" \ - "" \ - "0" \ - "" \ - "" \ - "" \ - "__SAME_AS_FALLBACK_MODELS__" \ - "deepseek/deepseek-r1-0528 deepseek/deepseek-v3-0324" \ - "1" - ;; - github-models-http410-authenticated-fallback-success) - run_github_models_http410_case \ - "$STRIX_TEST_CASE_FILTER" \ - "0" \ - "2" \ - "openai/gpt-5|openai/deepseek/deepseek-r1-0528" \ - "https://models.github.ai/inference|https://models.github.ai/inference" \ - "REGEX:Strix quick scan succeeded with fallback model 'deepseek/deepseek-r1-0528' in [0-9]+s\\." - ;; - github-models-http410-missing-http-token | github-models-http410-missing-provider-error | github-models-http410-numeric-continuation-4100 | github-models-http410-numeric-continuation-4104 | github-models-http410-target-output-spoof | github-models-retirement-brownout-phrase-only) - run_github_models_http410_case \ - "$STRIX_TEST_CASE_FILTER" \ - "1" \ - "1" \ - "openai/gpt-5" \ - "https://models.github.ai/inference" - ;; - github-models-fallback-provider-signal-tries-next) - run_gate_case "github-models-fallback-provider-signal-tries-next" \ - "openai/gpt-5" \ - "" \ - "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'deepseek/deepseek-v3-0324' in [0-9]+s\\." \ - "3" \ - "openai/gpt-5|openai/deepseek/deepseek-r1-0528|openai/deepseek/deepseek-v3-0324" \ - "https://models.github.ai/inference|https://models.github.ai/inference|https://models.github.ai/inference" \ - "openai" \ - "https://models.github.ai/inference" \ - "" \ - "0" \ - "CRITICAL" \ - "0" \ - "" \ - "" \ - "1200" \ - "0" \ - "pull_request" \ - "sync-module-system/smart-crawling-biz/src/main/java/org/empasy/sync/modules/system/controller/SysPositionController.java" \ - "" \ - "" \ - "0" \ - "" \ - "" \ - "" \ - "__SAME_AS_FALLBACK_MODELS__" \ - "deepseek/deepseek-r1-0528 deepseek/deepseek-v3-0324" \ - "1" - ;; - github-models-internal-server-connection-retry-same-model-success) - run_gate_case_allow_provider_signal "$STRIX_TEST_CASE_FILTER" \ - "openai/openai/retry-api-connection-primary" \ - "" \ - "0" \ - "scan ok after same-model api connection retry" \ - "2" \ - "openai/openai/retry-api-connection-primary|openai/openai/retry-api-connection-primary" \ - "https://models.github.ai/inference|https://models.github.ai/inference" \ - "openai" \ - "https://models.github.ai/inference" \ - "" \ - "1" - ;; - internal-server-error-unrelated-output-nonretryable) - run_gate_case_allow_provider_signal "$STRIX_TEST_CASE_FILTER" \ - "openai/openai/retry-api-connection-primary" \ - "" \ - "1" \ - "Strix quick scan failed with a non-recoverable error." \ - "1" \ - "openai/openai/retry-api-connection-primary" \ - "https://models.github.ai/inference" \ - "openai" \ - "https://models.github.ai/inference" \ - "" \ - "0" - ;; - internal-server-error-many-blocks-retry-same-model-success) - run_gate_case_allow_provider_signal "$STRIX_TEST_CASE_FILTER" \ - "openai/openai/retry-api-connection-primary" \ - "" \ - "0" \ - "scan ok after same-model api connection retry" \ - "2" \ - "openai/openai/retry-api-connection-primary|openai/openai/retry-api-connection-primary" \ - "https://models.github.ai/inference|https://models.github.ai/inference" \ - "openai" \ - "https://models.github.ai/inference" \ - "" \ - "1" - ;; - endpoint-in-excluded-dir) - run_gate_case "endpoint-in-excluded-dir" \ - "vertex_ai/excluded-dir-primary" \ - "vertex_ai/fallback-one vertex_ai/fallback-two" \ "1" \ "Unable to map Strix findings to changed files; failing closed for pull request." \ "1" \ - "vertex_ai/excluded-dir-primary" \ - "" - ;; - pull-request-target-changed-backend-context) - run_pull_request_target_changed_backend_context_scope_case + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" ;; report-known-internal-warning-sanitized) run_gate_case "$STRIX_TEST_CASE_FILTER" \ "vertex_ai/report-known-internal-warning-sanitized" \ "" \ "0" \ - "Strix run succeeded for model 'vertex_ai/report-known-internal-warning-sanitized'" \ + "Strix run succeeded for model 'orchestrator/free'" \ "1" \ "vertex_ai/report-known-internal-warning-sanitized" \ "" @@ -6726,61 +5347,19 @@ run_filtered_gate_case_if_requested() { "vertex_ai/$STRIX_TEST_CASE_FILTER" \ "" ;; - provider-report-rate-limit-fallback-success) - run_gate_case "provider-report-rate-limit-fallback-success" \ - "vertex_ai/report-rate-limit-primary" \ - "vertex_ai/fallback-one vertex_ai/fallback-two" \ - "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'vertex_ai/fallback-one' in [0-9]+s\\." \ - "2" \ - "vertex_ai/report-rate-limit-primary|vertex_ai/fallback-one" \ - "|" - ;; - total-timeout) - run_total_timeout_case + pr-stale-snapshot-snippet-fallback-success) ;; - github-models-fallback-baseline-vulnerability-before-next-success-continues) - run_gate_case "github-models-fallback-baseline-vulnerability-before-next-success-continues" \ - "openai/gpt-5" \ - "" \ - "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'deepseek/deepseek-v3-0324' in [0-9]+s\\." \ - "3" \ - "openai/gpt-5|openai/deepseek/deepseek-r1-0528|openai/deepseek/deepseek-v3-0324" \ - "https://models.github.ai/inference|https://models.github.ai/inference|https://models.github.ai/inference" \ - "openai" \ - "https://models.github.ai/inference" \ - "" \ - "0" \ - "CRITICAL" \ - "0" \ - "" \ - "" \ - "1200" \ - "0" \ - "pull_request" \ - "sync-module-system/smart-crawling-biz/src/main/java/org/empasy/sync/modules/system/controller/SysPositionController.java" \ - "" \ + scan-working-directory-isolated) + run_gate_case "scan-working-directory-isolated" \ + "orchestrator/free" \ "" \ "0" \ - "" \ - "" \ - "" \ - "__SAME_AS_FALLBACK_MODELS__" \ - "deepseek/deepseek-r1-0528 deepseek/deepseek-v3-0324" \ - "1" - ;; - github-models-exhausted-after-baseline-vulnerability-fails-closed) - run_gate_case "github-models-exhausted-after-baseline-vulnerability-fails-closed" \ - "openai/gpt-5" \ - "" \ + "scan ok with isolated Strix working directory" \ "1" \ - "STRIX_PROVIDER_UNAVAILABLE: provider models were exhausted after incomplete scan evidence." \ - "3" \ - "openai/gpt-5|openai/deepseek/deepseek-r1-0528|openai/deepseek/deepseek-v3-0324" \ - "https://models.github.ai/inference|https://models.github.ai/inference|https://models.github.ai/inference" \ - "openai" \ - "https://models.github.ai/inference" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -6790,200 +5369,10 @@ run_filtered_gate_case_if_requested() { "1200" \ "0" \ "pull_request" \ - "sync-module-system/smart-crawling-biz/src/main/java/org/empasy/sync/modules/system/controller/SysPositionController.java" \ - "" \ - "" \ - "0" \ - "" \ - "" \ - "" \ - "__SAME_AS_FALLBACK_MODELS__" \ - "deepseek/deepseek-r1-0528 deepseek/deepseek-v3-0324" \ - "1" + "backend/app/pg_introspect/introspect.py" ;; - github-models-fallback-changed-vulnerability-before-next-success-blocks) - run_gate_case "github-models-fallback-changed-vulnerability-before-next-success-blocks" \ - "openai/gpt-5" \ - "" \ - "1" \ - "Strix model reported threshold vulnerabilities before fallback success; failing closed so every model-reported vulnerability is reviewed." \ - "2" \ - "openai/gpt-5|openai/deepseek/deepseek-r1-0528" \ - "https://models.github.ai/inference|https://models.github.ai/inference" \ - "openai" \ - "https://models.github.ai/inference" \ - "" \ - "0" \ - "CRITICAL" \ - "0" \ - "" \ - "" \ - "1200" \ - "0" \ - "pull_request" \ - "sync-module-system/smart-crawling-biz/src/main/java/org/empasy/sync/modules/system/controller/SysPositionController.java" \ - "" \ - "" \ - "0" \ - "" \ - "" \ - "" \ - "__SAME_AS_FALLBACK_MODELS__" \ - "deepseek/deepseek-r1-0528 deepseek/deepseek-v3-0324" \ - "1" - ;; - github-models-fallback-dockerfile-test-baseline-before-next-success-continues) - run_gate_case "github-models-fallback-dockerfile-test-baseline-before-next-success-continues" \ - "openai/gpt-5" \ - "" \ - "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'deepseek/deepseek-v3-0324' in [0-9]+s\\." \ - "3" \ - "openai/gpt-5|openai/deepseek/deepseek-r1-0528|openai/deepseek/deepseek-v3-0324" \ - "https://models.github.ai/inference|https://models.github.ai/inference|https://models.github.ai/inference" \ - "openai" \ - "https://models.github.ai/inference" \ - "" \ - "0" \ - "MEDIUM" \ - "0" \ - "" \ - "" \ - "1200" \ - "0" \ - "pull_request" \ - ".github/workflows/build-ci-image.yml" \ - "" \ - "" \ - "0" \ - "" \ - "" \ - "" \ - "__SAME_AS_FALLBACK_MODELS__" \ - "deepseek/deepseek-r1-0528 deepseek/deepseek-v3-0324" \ - "1" - ;; - pr-stale-snapshot-snippet-fallback-success) - run_gate_case "pr-stale-snapshot-snippet-fallback-success" \ - "vertex_ai/stale-snapshot-primary" \ - "vertex_ai/fallback-one vertex_ai/fallback-two" \ - "0" \ - "scan ok after stale snapshot snippet fallback" \ - "2" \ - "vertex_ai/stale-snapshot-primary|vertex_ai/fallback-one" \ - "|" \ - "vertex_ai" \ - "__DEFAULT__" \ - "" \ - "0" \ - "MEDIUM" \ - "0" \ - "__PR_SCOPE__" \ - "" \ - "1200" \ - "0" \ - "pull_request" \ - "backend/app/api/snapshots.py" - ;; - pull-request-target-modified-file-pr-head-tree-lookup-failure) - run_pull_request_target_aborts_on_pr_head_blob_failure_case \ - "pull-request-target-modified-file-pr-head-tree-lookup-failure" \ - "src/existing.py" \ - "BASE_CONTENT_MUST_NOT_BE_USED_AFTER_HEAD_LOOKUP_FAILURE" \ - "HEAD_CONTENT_SHOULD_NOT_BECOME_PARTIAL_SCAN_INPUT" \ - "ls-tree" \ - "1" - ;; - pull-request-target-changed-file-list-diff-failure) - run_pull_request_target_aborts_on_pr_head_blob_failure_case \ - "pull-request-target-changed-file-list-diff-failure" \ - "src/existing.py" \ - "BASE_CONTENT_MUST_NOT_BE_USED_AFTER_DIFF_FAILURE" \ - "HEAD_CONTENT_SHOULD_NOT_BECOME_PARTIAL_SCAN_INPUT" \ - "diff" - ;; - pull-request-target-gitlink-is-explicitly-skipped) - run_pull_request_target_gitlink_is_explicitly_skipped_case - ;; - pull-request-target-dockerfile-change-uses-full-head-context) - run_pull_request_target_head_scope_case \ - "pull-request-target-dockerfile-change-uses-full-head-context" \ - "Dockerfile" \ - "FROM python:3.12-slim AS base" \ - "FROM python:3.12-slim AS head" \ - "0" \ - "0" \ - "." \ - "1" \ - "Container build manifest changed; materialized full PR-head blob scope" - ;; - repository-dispatch-pr-scope-uses-head-blob) - run_pull_request_target_head_scope_case \ - "repository-dispatch-pr-scope-uses-head-blob" \ - "backend/db/models.py" \ - "BASE_DISPATCH_CONTENT_SHOULD_NOT_BE_SCANNED" \ - "HEAD_DISPATCH_CONTENT_SHOULD_BE_SCANNED" \ - "0" \ - "0" \ - "__PR_SCOPE__" \ - "0" \ - "Materialized PR-head changed-file scope" \ - "repository_dispatch" - ;; - scan-working-directory-isolated) - run_gate_case "scan-working-directory-isolated" \ - "openai/gpt-4o-mini" \ - "" \ - "0" \ - "scan ok with isolated Strix working directory" \ - "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ - "" \ - "0" \ - "CRITICAL" \ - "0" \ - "" \ - "" \ - "1200" \ - "0" \ - "pull_request" \ - "backend/app/pg_introspect/introspect.py" - ;; - nvidia-overloaded-direct-fallback-success) - run_gate_case_allow_provider_signal "nvidia-overloaded-direct-fallback-success" \ - "nvidia_nim/nvidia/overloaded-primary" \ - "" \ - "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'nvidia_nim/nvidia/fallback-one' in [0-9]+s\\." \ - "3" \ - "nvidia_nim/nvidia/overloaded-primary|nvidia_nim/nvidia/overloaded-primary|nvidia_nim/nvidia/fallback-one" \ - "https://integrate.api.nvidia.com/v1|https://integrate.api.nvidia.com/v1|https://integrate.api.nvidia.com/v1" \ - "nvidia_nim" \ - "https://integrate.api.nvidia.com/v1" \ - "" \ - "1" \ - "CRITICAL" \ - "0" \ - "" \ - "" \ - "1200" \ - "0" \ - "" \ - "" \ - "" \ - "" \ - "0" \ - "" \ - "" \ - "" \ - "__SAME_AS_FALLBACK_MODELS__" \ - "nvidia_nim/nvidia/fallback-one openai-direct/gpt-5.4" - ;; - *) - record_failure "unknown STRIX_TEST_CASE_FILTER '${STRIX_TEST_CASE_FILTER:-}'" + *) + record_failure "unknown STRIX_TEST_CASE_FILTER '${STRIX_TEST_CASE_FILTER:-}'" ;; esac @@ -7019,6 +5408,7 @@ run_pull_request_target_head_scope_case() { local fake_strix="$bin_dir/strix" local output_log="$tmp_dir/output.log" local strix_llm_file="$tmp_dir/strix_llm.txt" + local llm_api_base_file="$tmp_dir/llm_api_base.txt" local llm_api_key_file="$tmp_dir/llm_api_key.txt" cat >"$fake_strix" <<'EOF' @@ -7077,7 +5467,8 @@ fi echo "scan ok with PR head content" EOF chmod +x "$fake_strix" - printf '%s' 'gemini/test-model' >"$strix_llm_file" + printf '%s' 'orchestrator/free' >"$strix_llm_file" + printf '%s' 'http://127.0.0.1:18080/v1' >"$llm_api_base_file" printf '%s' 'dummy' >"$llm_api_key_file" ( @@ -7137,6 +5528,7 @@ EOF FAKE_STRIX_EXPECT_FULL_HEAD_SCOPE="$expected_full_head_scope" \ STRIX_DISABLE_PR_SCOPING="$disable_pr_scoping" \ STRIX_LLM_FILE="$strix_llm_file" \ + LLM_API_BASE_FILE="$llm_api_base_file" \ LLM_API_KEY_FILE="$llm_api_key_file" \ STRIX_TARGET_PATH="$target_path" \ STRIX_REPORTS_DIR="$repo_root_dir/strix_runs" \ @@ -7168,6 +5560,7 @@ run_pull_request_target_plaintext_runner_token_fails_closed_case() { local output_log="$tmp_dir/output.log" local call_log="$tmp_dir/calls.log" local strix_llm_file="$tmp_dir/strix_llm.txt" + local llm_api_base_file="$tmp_dir/llm_api_base.txt" local llm_api_key_file="$tmp_dir/llm_api_key.txt" local changed_file="backend/db/models.py" @@ -7177,7 +5570,7 @@ set -euo pipefail printf '%s\n' "${STRIX_LLM:-}" >> "${FAKE_STRIX_CALL_LOG:?}" case "${STRIX_LLM:-}" in -vertex_ai/stale-source-primary) +openai/orchestrator/free) mkdir -p "${STRIX_REPORTS_DIR:?}/fake-pr-head-plaintext/vulnerabilities" cat >"$STRIX_REPORTS_DIR/fake-pr-head-plaintext/vulnerabilities/vuln-0001.md" <<'EOS' **Severity:** HIGH @@ -7189,10 +5582,6 @@ EOS echo "Penetration test failed: PR-head plaintext token finding" exit 1 ;; -vertex_ai/fallback-one) - echo "Error: PR-head plaintext findings must not reach fallback" >&2 - exit 31 - ;; *) echo "Error: unexpected model (${STRIX_LLM:-})" >&2 exit 32 @@ -7200,7 +5589,8 @@ vertex_ai/fallback-one) esac EOF chmod +x "$fake_strix" - printf '%s' 'vertex_ai/stale-source-primary' >"$strix_llm_file" + printf '%s' 'orchestrator/free' >"$strix_llm_file" + printf '%s' 'http://127.0.0.1:18080/v1' >"$llm_api_base_file" printf '%s' 'dummy' >"$llm_api_key_file" ( @@ -7257,6 +5647,7 @@ EOS STRIX_FAIL_ON_MIN_SEVERITY="HIGH" \ STRIX_DISABLE_PR_SCOPING="0" \ STRIX_LLM_FILE="$strix_llm_file" \ + LLM_API_BASE_FILE="$llm_api_base_file" \ LLM_API_KEY_FILE="$llm_api_key_file" \ STRIX_TARGET_PATH="." \ STRIX_REPORTS_DIR="$repo_root_dir/strix_runs" \ @@ -7289,6 +5680,7 @@ run_pull_request_target_bounded_head_context_scope_case() { local fake_strix="$bin_dir/strix" local output_log="$tmp_dir/output.log" local strix_llm_file="$tmp_dir/strix_llm.txt" + local llm_api_base_file="$tmp_dir/llm_api_base.txt" local llm_api_key_file="$tmp_dir/llm_api_key.txt" local changed_file="backend/api/emails.py" local context_file="backend/core/only_in_head.py" @@ -7321,7 +5713,8 @@ fi echo "scan ok with bounded PR head backend context" EOF chmod +x "$fake_strix" - printf '%s' 'gemini/test-model' >"$strix_llm_file" + printf '%s' 'orchestrator/free' >"$strix_llm_file" + printf '%s' 'http://127.0.0.1:18080/v1' >"$llm_api_base_file" printf '%s' 'dummy' >"$llm_api_key_file" ( @@ -7367,6 +5760,7 @@ EOF FAKE_STRIX_UNEXPECTED_BASE_CONTEXT="TRUSTED_BASE_CONTEXT_SHOULD_NOT_BE_SCANNED" \ STRIX_DISABLE_PR_SCOPING="0" \ STRIX_LLM_FILE="$strix_llm_file" \ + LLM_API_BASE_FILE="$llm_api_base_file" \ LLM_API_KEY_FILE="$llm_api_key_file" \ STRIX_TARGET_PATH="." \ STRIX_REPORTS_DIR="$repo_root_dir/strix_runs" \ @@ -7394,6 +5788,7 @@ run_pull_request_target_changed_context_scope_uses_pr_head_case() { local fake_strix="$bin_dir/strix" local output_log="$tmp_dir/output.log" local strix_llm_file="$tmp_dir/strix_llm.txt" + local llm_api_base_file="$tmp_dir/llm_api_base.txt" local llm_api_key_file="$tmp_dir/llm_api_key.txt" local state_file="$tmp_dir/state.log" local changed_file="backend/api/emails.py" @@ -7459,7 +5854,8 @@ echo "Error: unexpected changed context scan attempt $attempt" >&2 exit 71 EOF chmod +x "$fake_strix" - printf '%s' 'gemini/test-model' >"$strix_llm_file" + printf '%s' 'orchestrator/free' >"$strix_llm_file" + printf '%s' 'http://127.0.0.1:18080/v1' >"$llm_api_base_file" printf '%s' 'dummy' >"$llm_api_key_file" ( @@ -7510,6 +5906,7 @@ EOF FAKE_STRIX_STATE_FILE="$state_file" \ STRIX_DISABLE_PR_SCOPING="0" \ STRIX_LLM_FILE="$strix_llm_file" \ + LLM_API_BASE_FILE="$llm_api_base_file" \ LLM_API_KEY_FILE="$llm_api_key_file" \ STRIX_TARGET_PATH="." \ STRIX_REPORTS_DIR="$repo_root_dir/strix_runs" \ @@ -7546,6 +5943,7 @@ EOF FAKE_STRIX_STATE_FILE="$state_file" \ STRIX_DISABLE_PR_SCOPING="0" \ STRIX_LLM_FILE="$strix_llm_file" \ + LLM_API_BASE_FILE="$llm_api_base_file" \ LLM_API_KEY_FILE="$llm_api_key_file" \ STRIX_TARGET_PATH="." \ STRIX_REPORTS_DIR="$repo_root_dir/strix_runs" \ @@ -7574,6 +5972,7 @@ run_pull_request_target_changed_backend_context_scope_case() { local output_log="$tmp_dir/output.log" local call_log="$tmp_dir/calls.log" local strix_llm_file="$tmp_dir/strix_llm.txt" + local llm_api_base_file="$tmp_dir/llm_api_base.txt" local llm_api_key_file="$tmp_dir/llm_api_key.txt" cat >"$fake_strix" <<'EOF' @@ -7701,7 +6100,8 @@ fi echo "scan ok with non-email backend scope" EOF chmod +x "$fake_strix" - printf '%s' 'gemini/test-model' >"$strix_llm_file" + printf '%s' 'orchestrator/free' >"$strix_llm_file" + printf '%s' 'http://127.0.0.1:18080/v1' >"$llm_api_base_file" printf '%s' 'dummy' >"$llm_api_key_file" ( @@ -7797,6 +6197,7 @@ EOF STRIX_DISABLE_PR_SCOPING="0" \ FAKE_STRIX_CALL_LOG="$call_log" \ STRIX_LLM_FILE="$strix_llm_file" \ + LLM_API_BASE_FILE="$llm_api_base_file" \ LLM_API_KEY_FILE="$llm_api_key_file" \ STRIX_TARGET_PATH="." \ STRIX_REPORTS_DIR="$repo_root_dir/strix_runs" \ @@ -7832,6 +6233,7 @@ run_pull_request_target_frontend_email_context_scope_case() { local fake_strix="$bin_dir/strix" local output_log="$tmp_dir/output.log" local strix_llm_file="$tmp_dir/strix_llm.txt" + local llm_api_base_file="$tmp_dir/llm_api_base.txt" local llm_api_key_file="$tmp_dir/llm_api_key.txt" cat >"$fake_strix" <<'EOF' @@ -7942,7 +6344,8 @@ fi echo "scan ok with frontend email trusted backend authorization context" EOF chmod +x "$fake_strix" - printf '%s' 'gemini/test-model' >"$strix_llm_file" + printf '%s' 'orchestrator/free' >"$strix_llm_file" + printf '%s' 'http://127.0.0.1:18080/v1' >"$llm_api_base_file" printf '%s' 'dummy' >"$llm_api_key_file" ( @@ -7993,6 +6396,7 @@ EOF STRIX_DISABLE_PR_SCOPING="0" \ FAKE_STRIX_EXPECTED_CHANGED_FILE="$changed_file" \ STRIX_LLM_FILE="$strix_llm_file" \ + LLM_API_BASE_FILE="$llm_api_base_file" \ LLM_API_KEY_FILE="$llm_api_key_file" \ STRIX_TARGET_PATH="." \ STRIX_REPORTS_DIR="$repo_root_dir/strix_runs" \ @@ -8022,6 +6426,7 @@ run_pull_request_target_shallow_head_merge_base_fallback_case() { local fake_strix="$bin_dir/strix" local output_log="$tmp_dir/output.log" local strix_llm_file="$tmp_dir/strix_llm.txt" + local llm_api_base_file="$tmp_dir/llm_api_base.txt" local llm_api_key_file="$tmp_dir/llm_api_key.txt" cat >"$fake_strix" <<'EOF' @@ -8031,7 +6436,8 @@ echo "scan ok" exit 0 EOF chmod +x "$fake_strix" - printf '%s' 'gemini/test-model' >"$strix_llm_file" + printf '%s' 'orchestrator/free' >"$strix_llm_file" + printf '%s' 'http://127.0.0.1:18080/v1' >"$llm_api_base_file" printf '%s' 'dummy' >"$llm_api_key_file" ( @@ -8088,6 +6494,7 @@ EOF PR_BASE_SHA="$base_sha" \ PR_HEAD_SHA="$head_sha" \ STRIX_LLM_FILE="$strix_llm_file" \ + LLM_API_BASE_FILE="$llm_api_base_file" \ LLM_API_KEY_FILE="$llm_api_key_file" \ STRIX_TARGET_PATH="." \ STRIX_REPORTS_DIR="$repo_root_dir/strix_runs" \ @@ -8172,6 +6579,7 @@ EOF local call_log="$tmp_dir/calls.log" local output_log="$tmp_dir/output.log" local strix_llm_file="$tmp_dir/strix_llm.txt" + local llm_api_base_file="$tmp_dir/llm_api_base.txt" local llm_api_key_file="$tmp_dir/llm_api_key.txt" cat >"$fake_strix" <<'EOF' @@ -8182,7 +6590,8 @@ echo "Error: Strix should not run after a PR-head blob failure" >&2 exit 64 EOF chmod +x "$fake_strix" - printf '%s' 'gemini/test-model' >"$strix_llm_file" + printf '%s' 'orchestrator/free' >"$strix_llm_file" + printf '%s' 'http://127.0.0.1:18080/v1' >"$llm_api_base_file" printf '%s' 'dummy' >"$llm_api_key_file" ( @@ -8226,6 +6635,7 @@ EOF FAKE_STRIX_CALL_LOG="$call_log" \ STRIX_DISABLE_PR_SCOPING="$disable_pr_scoping" \ STRIX_LLM_FILE="$strix_llm_file" \ + LLM_API_BASE_FILE="$llm_api_base_file" \ LLM_API_KEY_FILE="$llm_api_key_file" \ STRIX_TARGET_PATH="." \ STRIX_REPORTS_DIR="$repo_root_dir/strix_runs" \ @@ -8262,6 +6672,7 @@ run_pull_request_target_rejects_invalid_sha_case() { local call_log="$tmp_dir/calls.log" local output_log="$tmp_dir/output.log" local strix_llm_file="$tmp_dir/strix_llm.txt" + local llm_api_base_file="$tmp_dir/llm_api_base.txt" local llm_api_key_file="$tmp_dir/llm_api_key.txt" cat >"$fake_strix" <<'EOF' @@ -8272,7 +6683,8 @@ echo "Error: Strix should not run after invalid pull request SHA metadata" >&2 exit 67 EOF chmod +x "$fake_strix" - printf '%s' 'gemini/test-model' >"$strix_llm_file" + printf '%s' 'orchestrator/free' >"$strix_llm_file" + printf '%s' 'http://127.0.0.1:18080/v1' >"$llm_api_base_file" printf '%s' 'dummy' >"$llm_api_key_file" ( @@ -8318,6 +6730,7 @@ EOF FAKE_STRIX_CALL_LOG="$call_log" \ STRIX_DISABLE_PR_SCOPING="0" \ STRIX_LLM_FILE="$strix_llm_file" \ + LLM_API_BASE_FILE="$llm_api_base_file" \ LLM_API_KEY_FILE="$llm_api_key_file" \ STRIX_TARGET_PATH="." \ STRIX_REPORTS_DIR="$repo_root_dir/strix_runs" \ @@ -8355,6 +6768,7 @@ run_pull_request_target_irregular_head_entry_fails_closed_case() { local call_log="$tmp_dir/calls.log" local output_log="$tmp_dir/output.log" local strix_llm_file="$tmp_dir/strix_llm.txt" + local llm_api_base_file="$tmp_dir/llm_api_base.txt" local llm_api_key_file="$tmp_dir/llm_api_key.txt" cat >"$fake_strix" <<'EOF' @@ -8365,7 +6779,8 @@ echo "Error: Strix should not run after an irregular PR-head entry" >&2 exit 66 EOF chmod +x "$fake_strix" - printf '%s' 'gemini/test-model' >"$strix_llm_file" + printf '%s' 'orchestrator/free' >"$strix_llm_file" + printf '%s' 'http://127.0.0.1:18080/v1' >"$llm_api_base_file" printf '%s' 'dummy' >"$llm_api_key_file" ( @@ -8405,6 +6820,7 @@ EOF FAKE_STRIX_CALL_LOG="$call_log" \ STRIX_DISABLE_PR_SCOPING="0" \ STRIX_LLM_FILE="$strix_llm_file" \ + LLM_API_BASE_FILE="$llm_api_base_file" \ LLM_API_KEY_FILE="$llm_api_key_file" \ STRIX_TARGET_PATH="." \ STRIX_REPORTS_DIR="$repo_root_dir/strix_runs" \ @@ -8438,6 +6854,7 @@ run_pull_request_target_gitlink_is_explicitly_skipped_case() { local call_log="$tmp_dir/calls.log" local output_log="$tmp_dir/output.log" local strix_llm_file="$tmp_dir/strix_llm.txt" + local llm_api_base_file="$tmp_dir/llm_api_base.txt" local llm_api_key_file="$tmp_dir/llm_api_key.txt" cat >"$fake_strix" <<'EOF' #!/usr/bin/env bash @@ -8446,7 +6863,8 @@ printf 'called\n' >> "${FAKE_STRIX_CALL_LOG:?}" exit 66 EOF chmod +x "$fake_strix" - printf '%s' 'gemini/test-model' >"$strix_llm_file" + printf '%s' 'orchestrator/free' >"$strix_llm_file" + printf '%s' 'http://127.0.0.1:18080/v1' >"$llm_api_base_file" printf '%s' 'dummy' >"$llm_api_key_file" ( @@ -8479,6 +6897,7 @@ EOF FAKE_STRIX_CALL_LOG="$call_log" \ STRIX_DISABLE_PR_SCOPING="0" \ STRIX_LLM_FILE="$strix_llm_file" \ + LLM_API_BASE_FILE="$llm_api_base_file" \ LLM_API_KEY_FILE="$llm_api_key_file" \ STRIX_TARGET_PATH="." \ STRIX_REPORTS_DIR="$repo_root_dir/strix_runs" \ @@ -8519,6 +6938,7 @@ run_full_head_scope_skips_gitlink_case() { local fake_strix="$bin_dir/strix" local output_log="$tmp_dir/output.log" local strix_llm_file="$tmp_dir/strix_llm.txt" + local llm_api_base_file="$tmp_dir/llm_api_base.txt" local llm_api_key_file="$tmp_dir/llm_api_key.txt" # The full-head scope must materialize the changed Dockerfile and the # unchanged docs context, and must never materialize the gitlink as a path. @@ -8550,7 +6970,8 @@ fi echo "scan ok with PR head content" EOF chmod +x "$fake_strix" - printf '%s' 'gemini/test-model' >"$strix_llm_file" + printf '%s' 'orchestrator/free' >"$strix_llm_file" + printf '%s' 'http://127.0.0.1:18080/v1' >"$llm_api_base_file" printf '%s' 'dummy' >"$llm_api_key_file" ( @@ -8602,6 +7023,7 @@ EOF STRIX_TEST_CHANGED_FILES_OVERRIDE="Dockerfile" \ STRIX_DISABLE_PR_SCOPING="0" \ STRIX_LLM_FILE="$strix_llm_file" \ + LLM_API_BASE_FILE="$llm_api_base_file" \ LLM_API_KEY_FILE="$llm_api_key_file" \ STRIX_TARGET_PATH="." \ STRIX_REPORTS_DIR="$repo_root_dir/strix_runs" \ @@ -8634,6 +7056,7 @@ run_pull_request_target_rejects_unsafe_changed_path_case() { local call_log="$tmp_dir/calls.log" local output_log="$tmp_dir/output.log" local strix_llm_file="$tmp_dir/strix_llm.txt" + local llm_api_base_file="$tmp_dir/llm_api_base.txt" local llm_api_key_file="$tmp_dir/llm_api_key.txt" local event_payload_file="$tmp_dir/github_event.json" @@ -8645,7 +7068,8 @@ echo "Error: Strix should not run for unsafe changed paths" >&2 exit 65 EOF chmod +x "$fake_strix" - printf '%s' 'gemini/test-model' >"$strix_llm_file" + printf '%s' 'orchestrator/free' >"$strix_llm_file" + printf '%s' 'http://127.0.0.1:18080/v1' >"$llm_api_base_file" printf '%s' 'dummy' >"$llm_api_key_file" cat >"$event_payload_file" <<'EOF' { @@ -8669,6 +7093,7 @@ EOF FAKE_STRIX_CALL_LOG="$call_log" \ STRIX_DISABLE_PR_SCOPING="0" \ STRIX_LLM_FILE="$strix_llm_file" \ + LLM_API_BASE_FILE="$llm_api_base_file" \ LLM_API_KEY_FILE="$llm_api_key_file" \ STRIX_TARGET_PATH="." \ STRIX_REPORTS_DIR="$repo_root_dir/strix_runs" \ @@ -8725,6 +7150,7 @@ run_timeout_cleanup_case() { local child_pid_file="$tmp_dir/child.pid" local output_log="$tmp_dir/output.log" local strix_llm_file="$tmp_dir/strix_llm.txt" + local llm_api_base_file="$tmp_dir/llm_api_base.txt" local llm_api_key_file="$tmp_dir/llm_api_key.txt" cat >"$fake_strix" <<'EOF' @@ -8737,7 +7163,8 @@ printf '%s' "$child_pid" > "${FAKE_STRIX_CHILD_PID_FILE:?}" sleep "${FAKE_STRIX_TIMEOUT_SLEEP_SECONDS:?}" EOF chmod +x "$fake_strix" - printf '%s' 'vertex_ai/timeout-cleanup-primary' >"$strix_llm_file" + printf '%s' 'orchestrator/free' >"$strix_llm_file" + printf '%s' 'http://127.0.0.1:18080/v1' >"$llm_api_base_file" printf '%s' 'dummy' >"$llm_api_key_file" set +e @@ -8751,6 +7178,7 @@ EOF FAKE_STRIX_CHILD_PID_FILE="$child_pid_file" \ FAKE_STRIX_TIMEOUT_SLEEP_SECONDS="$TIMEOUT_TEST_FAKE_SLEEP_SECONDS" \ STRIX_LLM_FILE="$strix_llm_file" \ + LLM_API_BASE_FILE="$llm_api_base_file" \ LLM_API_KEY_FILE="$llm_api_key_file" \ STRIX_PROCESS_TIMEOUT_SECONDS="$TIMEOUT_TEST_PROCESS_SECONDS" \ STRIX_VERTEX_FALLBACK_MODELS="" \ @@ -8816,9 +7244,9 @@ echo "vertex scan ok without external LLM_API_BASE" exit 0 EOF chmod +x "$fake_strix" - printf '%s' 'vertex_ai/gemini-2.5-pro' >"$strix_llm_file" + printf '%s' 'orchestrator/free' >"$strix_llm_file" printf '%s' 'dummy' >"$llm_api_key_file" - printf '%s' 'https://example.invalid/generateContent' >"$llm_api_base_file" + printf '%s' 'http://127.0.0.1:18080/v1' >"$llm_api_base_file" set +e ( @@ -8831,6 +7259,7 @@ EOF FAKE_STRIX_CALL_LOG="$call_log" \ STRIX_DISABLE_PR_SCOPING="0" \ STRIX_LLM_FILE="$strix_llm_file" \ + LLM_API_BASE_FILE="$llm_api_base_file" \ LLM_API_KEY_FILE="$llm_api_key_file" \ LLM_API_BASE_FILE="$llm_api_base_file" \ bash "./scripts/ci/strix_quick_gate.sh" >"$output_log" 2>&1 @@ -8838,9 +7267,9 @@ EOF local rc=$? set -e - assert_equals "0" "$rc" "case=vertex-ignores-untrusted-llm-api-base-file exit code" - assert_file_contains "$output_log" "vertex scan ok without external LLM_API_BASE" "case=vertex-ignores-untrusted-llm-api-base-file output" - assert_file_contains "$call_log" "called" "case=vertex-ignores-untrusted-llm-api-base-file strix invocation" + assert_equals "2" "$rc" "case=untrusted-llm-api-base-file-refused exit code" + assert_file_contains "$output_log" "LLM_API_BASE_FILE must be inside the trusted input file root." "case=untrusted-llm-api-base-file-refused output" + assert_file_not_contains "$call_log" "called" "case=untrusted-llm-api-base-file-refused does not invoke strix" rm -rf "$tmp_dir" } @@ -8859,6 +7288,7 @@ run_total_timeout_case() { local output_log="$tmp_dir/output.log" local call_count_file="$tmp_dir/calls.log" local strix_llm_file="$tmp_dir/strix_llm.txt" + local llm_api_base_file="$tmp_dir/llm_api_base.txt" local llm_api_key_file="$tmp_dir/llm_api_key.txt" cat >"$fake_strix" <<'EOF' @@ -8869,7 +7299,8 @@ echo "1" >> "${FAKE_STRIX_CALL_COUNT_FILE:?}" sleep 30 EOF chmod +x "$fake_strix" - printf '%s' 'vertex_ai/total-timeout-primary' >"$strix_llm_file" + printf '%s' 'orchestrator/free' >"$strix_llm_file" + printf '%s' 'http://127.0.0.1:18080/v1' >"$llm_api_base_file" printf '%s' 'dummy' >"$llm_api_key_file" set +e @@ -8882,6 +7313,7 @@ EOF STRIX_DISABLE_PR_SCOPING="0" \ FAKE_STRIX_CALL_COUNT_FILE="$call_count_file" \ STRIX_LLM_FILE="$strix_llm_file" \ + LLM_API_BASE_FILE="$llm_api_base_file" \ LLM_API_KEY_FILE="$llm_api_key_file" \ STRIX_PROCESS_TIMEOUT_SECONDS="30" \ STRIX_TOTAL_TIMEOUT_SECONDS="8" \ @@ -8931,6 +7363,7 @@ run_missing_config_case() { local call_count_file="$tmp_dir/strix_calls" local fake_strix="$tmp_dir/strix" local strix_llm_file="$tmp_dir/strix_llm.txt" + local llm_api_base_file="$tmp_dir/llm_api_base.txt" local llm_api_key_file="$tmp_dir/llm_api_key.txt" cat >"$fake_strix" <<'EOF' @@ -8954,6 +7387,7 @@ EOF STRIX_INPUT_FILE_ROOT="$tmp_dir" \ STRIX_DISABLE_PR_SCOPING="0" \ STRIX_LLM_FILE="$strix_llm_file" \ + LLM_API_BASE_FILE="$llm_api_base_file" \ LLM_API_KEY_FILE="$llm_api_key_file" \ STRIX_CALL_COUNT_FILE="$call_count_file" \ bash "$GATE_SCRIPT" >"$output_log" 2>&1 @@ -8980,6 +7414,7 @@ run_strix_llm_file_command_substitution_literal_case() { local marker_file="$tmp_dir/strix_marker" local fake_strix="$tmp_dir/strix" local strix_llm_file="$tmp_dir/strix_llm.txt" + local llm_api_base_file="$tmp_dir/llm_api_base.txt" local llm_api_key_file="$tmp_dir/llm_api_key.txt" cat >"$fake_strix" <<'EOF' @@ -9000,6 +7435,7 @@ EOF STRIX_TARGET_PATH="-" \ STRIX_DISABLE_PR_SCOPING="0" \ STRIX_LLM_FILE="$strix_llm_file" \ + LLM_API_BASE_FILE="$llm_api_base_file" \ LLM_API_KEY_FILE="$llm_api_key_file" \ STRIX_CALL_COUNT_FILE="$call_count_file" \ bash "$GATE_SCRIPT" >"$output_log" 2>&1 @@ -9007,7 +7443,7 @@ EOF set -e assert_equals "2" "$rc" "case=strix-llm-file-command-substitution-literal exit code" - assert_file_contains "$output_log" "ERROR: STRIX_TARGET_PATH contains unsupported path syntax" "case=strix-llm-file-command-substitution-literal output" + assert_file_contains "$output_log" "ERROR: Strix model must be orchestrator/free through contextual-orchestrator; direct provider/model routes are forbidden" "case=strix-llm-file-command-substitution-literal output" if [ -e "$marker_file" ]; then record_failure "case=strix-llm-file-command-substitution-literal must not execute model file content" fi @@ -9028,6 +7464,7 @@ run_vertex_without_llm_api_key_case() { local call_count_file="$tmp_dir/strix_calls" local fake_strix="$tmp_dir/strix" local strix_llm_file="$tmp_dir/strix_llm.txt" + local llm_api_base_file="$tmp_dir/llm_api_base.txt" cat >"$fake_strix" <<'EOF' #!/usr/bin/env bash @@ -9044,7 +7481,8 @@ fi exit 0 EOF chmod +x "$fake_strix" - printf '%s' "vertex_ai/ready-primary" >"$strix_llm_file" + printf '%s' 'orchestrator/free' >"$strix_llm_file" + printf '%s' 'http://127.0.0.1:18080/v1' >"$llm_api_base_file" set +e env -u GITHUB_EVENT_NAME -u GITHUB_EVENT_PATH -u STRIX_TEST_CHANGED_FILES_OVERRIDE \ @@ -9053,19 +7491,20 @@ EOF STRIX_INPUT_FILE_ROOT="$tmp_dir" \ STRIX_DISABLE_PR_SCOPING="0" \ STRIX_LLM_FILE="$strix_llm_file" \ + LLM_API_BASE_FILE="$llm_api_base_file" \ FAKE_STRIX_CALL_COUNT_FILE="$call_count_file" \ bash "$GATE_SCRIPT" >"$output_log" 2>&1 local rc=$? set -e - assert_equals "0" "$rc" "case=vertex-without-llm-api-key exit code" - assert_file_contains "$output_log" "Strix run succeeded for model 'vertex_ai/ready-primary'" "case=vertex-without-llm-api-key output" + assert_equals "2" "$rc" "case=gateway-model-without-llm-api-key exit code" + assert_file_contains "$output_log" "LLM_API_KEY_FILE must reference a regular file containing the API key." "case=gateway-model-without-llm-api-key output" local actual_calls="0" if [ -f "$call_count_file" ]; then actual_calls="$(wc -l <"$call_count_file" | tr -d ' ')" fi - assert_equals "1" "$actual_calls" "case=vertex-without-llm-api-key strix call count" + assert_equals "0" "$actual_calls" "case=gateway-model-without-llm-api-key strix call count" rm -rf "$tmp_dir" } @@ -9077,6 +7516,7 @@ run_vertex_with_llm_api_key_file_does_not_forward_case() { local call_count_file="$tmp_dir/strix_calls" local fake_strix="$tmp_dir/strix" local strix_llm_file="$tmp_dir/strix_llm.txt" + local llm_api_base_file="$tmp_dir/llm_api_base.txt" local llm_api_key_file="$tmp_dir/llm_api_key.txt" cat >"$fake_strix" <<'EOF' @@ -9094,7 +7534,8 @@ fi exit 0 EOF chmod +x "$fake_strix" - printf '%s' "vertex_ai/ready-primary" >"$strix_llm_file" + printf '%s' 'vertex_ai/ready-primary' >"$strix_llm_file" + printf '%s' 'http://127.0.0.1:18080/v1' >"$llm_api_base_file" printf '%s' "openai-key-should-not-reach-vertex" >"$llm_api_key_file" set +e @@ -9104,20 +7545,21 @@ EOF STRIX_INPUT_FILE_ROOT="$tmp_dir" \ STRIX_DISABLE_PR_SCOPING="0" \ STRIX_LLM_FILE="$strix_llm_file" \ + LLM_API_BASE_FILE="$llm_api_base_file" \ LLM_API_KEY_FILE="$llm_api_key_file" \ FAKE_STRIX_CALL_COUNT_FILE="$call_count_file" \ bash "$GATE_SCRIPT" >"$output_log" 2>&1 local rc=$? set -e - assert_equals "0" "$rc" "case=vertex-with-llm-api-key-file-not-forwarded exit code" - assert_file_contains "$output_log" "Strix run succeeded for model 'vertex_ai/ready-primary'" "case=vertex-with-llm-api-key-file-not-forwarded output" + assert_equals "2" "$rc" "case=vertex-route-rejected-before-credentials exit code" + assert_file_contains "$output_log" "ERROR: Strix model must be orchestrator/free through contextual-orchestrator; direct provider/model routes are forbidden" "case=vertex-route-rejected-before-credentials output" local actual_calls="0" if [ -f "$call_count_file" ]; then actual_calls="$(wc -l <"$call_count_file" | tr -d ' ')" fi - assert_equals "1" "$actual_calls" "case=vertex-with-llm-api-key-file-not-forwarded strix call count" + assert_equals "0" "$actual_calls" "case=vertex-route-rejected-before-credentials strix call count" rm -rf "$tmp_dir" } @@ -9128,6 +7570,7 @@ run_invalid_min_fail_severity_case() { local output_log="$tmp_dir/output.log" local fake_strix="$tmp_dir/strix" local strix_llm_file="$tmp_dir/strix_llm.txt" + local llm_api_base_file="$tmp_dir/llm_api_base.txt" local llm_api_key_file="$tmp_dir/llm_api_key.txt" cat >"$fake_strix" <<'EOF' @@ -9137,7 +7580,8 @@ echo "unexpected strix execution" >&2 exit 99 EOF chmod +x "$fake_strix" - printf '%s' 'vertex_ai/ready-primary' >"$strix_llm_file" + printf '%s' 'orchestrator/free' >"$strix_llm_file" + printf '%s' 'http://127.0.0.1:18080/v1' >"$llm_api_base_file" printf '%s' 'dummy' >"$llm_api_key_file" set +e @@ -9147,6 +7591,7 @@ EOF STRIX_INPUT_FILE_ROOT="$tmp_dir" \ STRIX_DISABLE_PR_SCOPING="0" \ STRIX_LLM_FILE="$strix_llm_file" \ + LLM_API_BASE_FILE="$llm_api_base_file" \ LLM_API_KEY_FILE="$llm_api_key_file" \ STRIX_FAIL_ON_MIN_SEVERITY="BOGUS" \ bash "$GATE_SCRIPT" >"$output_log" 2>&1 @@ -9190,9 +7635,9 @@ printf 'called\n' >"${FAKE_STRIX_CALL_LOG:?}" exit 0 EOF chmod +x "$fake_strix" - printf '%s' 'openai/gpt-4o-mini' >"$strix_llm_file" + printf '%s' 'orchestrator/free' >"$strix_llm_file" printf '%s' 'dummy' >"$llm_api_key_file" - printf '%s' 'https://example.invalid/generateContent' >"$llm_api_base_file" + printf '%s' 'http://127.0.0.1:18080/v1' >"$llm_api_base_file" set +e ( @@ -9204,6 +7649,7 @@ EOF FAKE_STRIX_CALL_LOG="$call_log" \ STRIX_DISABLE_PR_SCOPING="0" \ STRIX_LLM_FILE="$strix_llm_file" \ + LLM_API_BASE_FILE="$llm_api_base_file" \ LLM_API_KEY_FILE="$llm_api_key_file" \ LLM_API_BASE_FILE="$llm_api_base_file" \ bash "./scripts/ci/strix_quick_gate.sh" >"$output_log" 2>&1 @@ -9247,9 +7693,9 @@ printf 'called\n' >"${FAKE_STRIX_CALL_LOG:?}" exit 0 EOF chmod +x "$fake_strix" - printf '%s' 'openai/gpt-4o-mini' >"$strix_llm_file" + printf '%s' 'orchestrator/free' >"$strix_llm_file" printf '%s' 'dummy' >"$llm_api_key_file" - printf '%s' 'https://example.invalid/generateContent' >"$llm_api_base_file" + printf '%s' 'http://127.0.0.1:18080/v1' >"$llm_api_base_file" set +e ( @@ -9263,6 +7709,7 @@ EOF FAKE_STRIX_CALL_LOG="$call_log" \ STRIX_DISABLE_PR_SCOPING="0" \ STRIX_LLM_FILE="$strix_llm_file" \ + LLM_API_BASE_FILE="$llm_api_base_file" \ LLM_API_KEY_FILE="$llm_api_key_file" \ LLM_API_BASE_FILE="$llm_api_base_file" \ bash "./scripts/ci/strix_quick_gate.sh" >"$output_log" 2>&1 @@ -9306,9 +7753,9 @@ printf 'called\n' >"${FAKE_STRIX_CALL_LOG:?}" exit 0 EOF chmod +x "$fake_strix" - printf '%s' 'openai/gpt-4o-mini' >"$strix_llm_file" + printf '%s' 'orchestrator/free' >"$strix_llm_file" printf '%s' 'dummy' >"$llm_api_key_file" - printf '%s' 'https://example.invalid/generateContent' >"$llm_api_base_file" + printf '%s' 'http://127.0.0.1:18080/v1' >"$llm_api_base_file" case "$file_env" in STRIX_LLM_FILE) printf '%s' 'openai/gpt-4o-mini' >"$outside_file" @@ -9335,6 +7782,7 @@ EOF FAKE_STRIX_CALL_LOG="$call_log" \ STRIX_DISABLE_PR_SCOPING="0" \ STRIX_LLM_FILE="$strix_llm_file" \ + LLM_API_BASE_FILE="$llm_api_base_file" \ LLM_API_KEY_FILE="$llm_api_key_file" \ LLM_API_BASE_FILE="$llm_api_base_file" \ bash "./scripts/ci/strix_quick_gate.sh" >"$output_log" 2>&1 @@ -9376,9 +7824,9 @@ printf 'called\n' >"${FAKE_STRIX_CALL_LOG:?}" exit 0 EOF chmod +x "$fake_strix" - printf '%s' 'openai/gpt-4o-mini' >"$strix_llm_file" + printf '%s' 'orchestrator/free' >"$strix_llm_file" printf '%s' 'dummy' >"$llm_api_key_file" - printf '%s' 'https://example.invalid/generateContent' >"$llm_api_base_file" + printf '%s' 'http://127.0.0.1:18080/v1' >"$llm_api_base_file" set +e ( @@ -9391,6 +7839,7 @@ EOF FAKE_STRIX_CALL_LOG="$call_log" \ STRIX_DISABLE_PR_SCOPING="0" \ STRIX_LLM_FILE="$strix_llm_file" \ + LLM_API_BASE_FILE="$llm_api_base_file" \ LLM_API_KEY_FILE="$llm_api_key_file" \ LLM_API_BASE_FILE="$llm_api_base_file" \ bash "./scripts/ci/strix_quick_gate.sh" >"$output_log" 2>&1 @@ -9435,9 +7884,9 @@ echo "Error: transport timeout" exit 1 EOF chmod +x "$fake_strix" - printf '%s' 'openai/gpt-4o-mini' >"$strix_llm_file" + printf '%s' 'orchestrator/free' >"$strix_llm_file" printf '%s' 'dummy' >"$llm_api_key_file" - printf '%s' 'https://example.invalid/generateContent' >"$llm_api_base_file" + printf '%s' 'http://127.0.0.1:18080/v1' >"$llm_api_base_file" set +e ( @@ -9448,6 +7897,7 @@ EOF STRIX_INPUT_FILE_ROOT="$tmp_dir" \ STRIX_DISABLE_PR_SCOPING="0" \ STRIX_LLM_FILE="$strix_llm_file" \ + LLM_API_BASE_FILE="$llm_api_base_file" \ LLM_API_KEY_FILE="$llm_api_key_file" \ LLM_API_BASE_FILE="$llm_api_base_file" \ STRIX_REPORTS_DIR="strix_runs" \ @@ -9457,7 +7907,7 @@ EOF set -e assert_equals "1" "$rc" "case=stale-report-does-not-bypass exit code" - assert_file_contains "$output_log" "Strix quick scan failed with a non-recoverable error." "case=stale-report-does-not-bypass output" + assert_file_contains "$output_log" "Strix quick scan failed; failing closed without repository-authored retry or fallback allocation." "case=stale-report-does-not-bypass output" rm -rf "$tmp_dir" } @@ -9491,9 +7941,9 @@ echo "Error: transport timeout" exit 1 EOF chmod +x "$fake_strix" - printf '%s' 'openai/gpt-4o-mini' >"$strix_llm_file" + printf '%s' 'orchestrator/free' >"$strix_llm_file" printf '%s' 'dummy' >"$llm_api_key_file" - printf '%s' 'https://example.invalid/generateContent' >"$llm_api_base_file" + printf '%s' 'http://127.0.0.1:18080/v1' >"$llm_api_base_file" set +e ( @@ -9504,6 +7954,7 @@ EOF STRIX_INPUT_FILE_ROOT="$tmp_dir" \ STRIX_DISABLE_PR_SCOPING="0" \ STRIX_LLM_FILE="$strix_llm_file" \ + LLM_API_BASE_FILE="$llm_api_base_file" \ LLM_API_KEY_FILE="$llm_api_key_file" \ LLM_API_BASE_FILE="$llm_api_base_file" \ STRIX_REPORTS_DIR="strix_runs" \ @@ -9513,7 +7964,7 @@ EOF set -e assert_equals "1" "$rc" "case=symlink-report-does-not-bypass exit code" - assert_file_contains "$output_log" "Strix quick scan failed with a non-recoverable error." "case=symlink-report-does-not-bypass output" + assert_file_contains "$output_log" "Strix quick scan failed; failing closed without repository-authored retry or fallback allocation." "case=symlink-report-does-not-bypass output" rm -rf "$tmp_dir" } @@ -9541,9 +7992,9 @@ printf '%s\n' called >>"${FAKE_STRIX_CALL_LOG:?}" exit 0 EOF chmod +x "$fake_strix" - printf '%s' 'openai/gpt-4o-mini' >"$strix_llm_file" + printf '%s' 'orchestrator/free' >"$strix_llm_file" printf '%s' 'dummy' >"$llm_api_key_file" - printf '%s' 'https://example.invalid/generateContent' >"$llm_api_base_file" + printf '%s' 'http://127.0.0.1:18080/v1' >"$llm_api_base_file" set +e ( @@ -9555,6 +8006,7 @@ EOF STRIX_DISABLE_PR_SCOPING="0" \ FAKE_STRIX_CALL_LOG="$call_log" \ STRIX_LLM_FILE="$strix_llm_file" \ + LLM_API_BASE_FILE="$llm_api_base_file" \ LLM_API_KEY_FILE="$llm_api_key_file" \ LLM_API_BASE_FILE="$llm_api_base_file" \ STRIX_TARGET_PATH="../../../../../etc/passwd" \ @@ -9594,9 +8046,9 @@ printf 'called\n' >"${FAKE_STRIX_CALL_LOG:?}" exit 0 EOF chmod +x "$fake_strix" - printf '%s' 'openai/gpt-4o-mini' >"$strix_llm_file" + printf '%s' 'orchestrator/free' >"$strix_llm_file" printf '%s' 'dummy' >"$llm_api_key_file" - printf '%s' 'https://example.invalid/generateContent' >"$llm_api_base_file" + printf '%s' 'http://127.0.0.1:18080/v1' >"$llm_api_base_file" set +e ( @@ -9607,6 +8059,7 @@ EOF STRIX_INPUT_FILE_ROOT="$tmp_dir" \ FAKE_STRIX_CALL_LOG="$call_log" \ STRIX_LLM_FILE="$strix_llm_file" \ + LLM_API_BASE_FILE="$llm_api_base_file" \ LLM_API_KEY_FILE="$llm_api_key_file" \ LLM_API_BASE_FILE="$llm_api_base_file" \ STRIX_TARGET_PATH="$tmp_dir/strix-pr-scope.attacker" \ @@ -9879,956 +8332,294 @@ run_pull_request_target_rejects_invalid_sha_case \ "base" run_pull_request_target_rejects_invalid_sha_case \ - "pull-request-target-invalid-head-sha-fails-closed" \ - "head" - -run_pull_request_target_aborts_on_pr_head_blob_failure_case \ - "pull-request-target-disabled-pr-scope-pr-head-blob-read-failure" \ - "src/existing.py" \ - "BASE_CONTENT_MUST_NOT_BE_USED_AFTER_DISABLED_SCOPE_HEAD_FAILURE" \ - "HEAD_CONTENT_SHOULD_NOT_BECOME_PARTIAL_SCAN_INPUT" \ - "cat-file" \ - "1" - -run_gate_case "success" \ - "vertex_ai/ready-primary" \ - "vertex_ai/fallback-one vertex_ai/fallback-two" \ - "0" \ - "scan ok" \ - "1" \ - "vertex_ai/ready-primary" \ - "" - -run_gate_case "contextual-orchestrator-missing-api-base-fails-closed" \ - "orchestrator/free" \ - "" \ - "2" \ - "require LLM_API_BASE_FILE to select the pinned loopback gateway" \ - "0" \ - "" \ - "" \ - "contextual_orchestrator" \ - "" - -run_gate_case "contextual-orchestrator-gateway-model-qualification" \ - "orchestrator/free" \ - "" \ - "0" \ - "scan ok through contextual-orchestrator gateway" \ - "1" \ - "openai/orchestrator/free" \ - "http://127.0.0.1:18080/v1" \ - "contextual_orchestrator" \ - "http://127.0.0.1:18080/v1" - -run_gate_case "success-with-critical-report" \ - "vertex_ai/ready-primary" \ - "" \ - "1" \ - "Strix exited successfully but emitted a vulnerability at or above 'CRITICAL'" \ - "1" \ - "vertex_ai/ready-primary" \ - "" - -run_gate_case "pr-executable-integrity-mismatch" \ - "vertex_ai/ready-primary" \ - "" \ - "1" \ - "did not match the pinned SHA-256 digest" \ - "0" \ - "" \ - "" - -run_gate_case "pr-executable-group-writable" \ - "vertex_ai/ready-primary" \ - "" \ - "1" \ - "must not be group/world writable" \ - "0" \ - "" \ - "" - -run_gate_case "pr-executable-root-group-writable" \ - "vertex_ai/ready-primary" \ - "" \ - "1" \ - "pinned Strix installation root must not be group/world writable" \ - "0" \ - "" \ - "" - -run_gate_case "runtime-env-forwarding" \ - "gemini/gemini-pro-3.1-preview" \ - "" \ - "0" \ - "scan ok" \ - "1" \ - "gemini/gemini-pro-3.1-preview" \ - "" \ - "gemini" \ - "" - -run_gate_case "vertex-primary-notfound-fallback-success" \ - "vertex_ai/missing-primary" \ - "vertex_ai/fallback-one vertex_ai/fallback-two" \ - "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'vertex_ai/fallback-one' in [0-9]+s\\." \ - "2" \ - "vertex_ai/missing-primary|vertex_ai/fallback-one" \ - "|" - -run_gate_case "vertex-all-notfound" \ - "vertex_ai/missing-primary" \ - "vertex_ai/fallback-one vertex_ai/fallback-two" \ - "1" \ - "Configured Vertex model and fallback models were unavailable." \ - "3" \ - "vertex_ai/missing-primary|vertex_ai/fallback-one|vertex_ai/fallback-two" \ - "||" - -run_gate_case "nonrecoverable" \ - "openai/gpt-4o-mini" \ - "vertex_ai/fallback-one" \ - "1" \ - "Strix quick scan failed with a non-recoverable error." \ - "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" - -run_gate_case "provider-prefix-required" \ - "gemini-2.5-pro" \ - "vertex_ai/fallback-one" \ - "0" \ - "Normalized STRIX_LLM to provider-qualified model 'vertex_ai/gemini-2.5-pro'." \ - "1" \ - "vertex_ai/gemini-2.5-pro" \ - "" - -run_gate_case "provider-prefix-fallback-normalization" \ - "missing-primary" \ - "fallback-one fallback-two" \ - "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'vertex_ai/fallback-one' in [0-9]+s\\." \ - "2" \ - "vertex_ai/missing-primary|vertex_ai/fallback-one" \ - "|" - -run_gate_case "provider-prefix-required-resource-path-primary-implicit-default-provider" \ - "projects/p1/locations/us-central1/publishers/google/models/gemini-2.5-pro" \ - "vertex_ai/fallback-one" \ - "0" \ - "Normalized STRIX_LLM to provider-qualified model 'vertex_ai/gemini-2.5-pro'." \ - "1" \ - "vertex_ai/gemini-2.5-pro" \ - "" - -run_gate_case "provider-prefix-required-resource-path-primary-explicit-empty-default-provider" \ - "projects/p1/locations/us-central1/publishers/google/models/gemini-2.5-pro" \ - "vertex_ai/fallback-one" \ - "2" \ - "ERROR: Vertex resource paths require an explicit vertex_ai or vertex_ai_beta provider." \ - "0" \ - "" \ - "" \ - "" - -run_gate_case "provider-prefix-resource-path-primary-notfound-fallback-success" \ - "projects/p1/locations/us-central1/publishers/google/models/missing-primary" \ - "projects/p1/locations/us-central1/publishers/google/models/fallback-one projects/p1/locations/us-central1/publishers/google/models/fallback-two" \ - "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'vertex_ai/fallback-one' in [0-9]+s\\." \ - "2" \ - "vertex_ai/missing-primary|vertex_ai/fallback-one" \ - "|" - -# Regression: Vertex custom model resource path projects/

/locations//models/ -# (no publishers/ segment) must be recognized as a Vertex resource path and -# normalized to vertex_ai/. -run_gate_case "vertex-custom-model-resource-path" \ - "projects/my-proj/locations/us-central1/models/my-custom-model-123" \ - "vertex_ai/fallback-one" \ - "0" \ - "Normalized STRIX_LLM to provider-qualified model 'vertex_ai/my-custom-model-123'." \ - "1" \ - "vertex_ai/my-custom-model-123" \ - "" - -run_gate_case "vertex-notfound-without-status-fallback-success" \ - "vertex_ai/missing-primary" \ - "vertex_ai/fallback-one" \ - "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'vertex_ai/fallback-one' in [0-9]+s\\." \ - "2" \ - "vertex_ai/missing-primary|vertex_ai/fallback-one" \ - "|" - -run_gate_case "vertex-notfound-compact-status-fallback-success" \ - "vertex_ai/missing-primary" \ - "vertex_ai/fallback-one" \ - "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'vertex_ai/fallback-one' in [0-9]+s\\." \ - "2" \ - "vertex_ai/missing-primary|vertex_ai/fallback-one" \ - "|" - -run_gate_case "nonvertex-slash-model-passthrough" \ - "foo/bar" \ - "vertex_ai/fallback-one" \ - "0" \ - "scan ok with non-vertex slash model passthrough" \ - "1" \ - "foo/bar" \ - "https://example.invalid" - -run_gate_case "primary-duplicate-in-fallback" \ - "missing-primary" \ - "vertex_ai/missing-primary fallback-one" \ - "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'vertex_ai/fallback-one' in [0-9]+s\\." \ - "2" \ - "vertex_ai/missing-primary|vertex_ai/fallback-one" \ - "|" - -run_gate_case "multiline-fallback-success" \ - "vertex_ai/missing-primary" \ - $'vertex_ai/fallback-one\nvertex_ai/fallback-two' \ - "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'vertex_ai/fallback-two' in [0-9]+s\\." \ - "3" \ - "vertex_ai/missing-primary|vertex_ai/fallback-one|vertex_ai/fallback-two" \ - "||" - -run_gate_case_allow_provider_signal "vertex-primary-ratelimit-fallback-success" \ - "vertex_ai/ratelimit-primary" \ - "vertex_ai/fallback-one vertex_ai/fallback-two" \ - "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'vertex_ai/fallback-one' in [0-9]+s\\." \ - "2" \ - "vertex_ai/ratelimit-primary|vertex_ai/fallback-one" \ - "|" - -run_gate_case_allow_provider_signal "vertex-primary-resource-exhausted-fallback-success" \ - "vertex_ai/resource-exhausted-primary" \ - "vertex_ai/fallback-one vertex_ai/fallback-two" \ - "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'vertex_ai/fallback-one' in [0-9]+s\\." \ - "2" \ - "vertex_ai/resource-exhausted-primary|vertex_ai/fallback-one" \ - "|" - -run_gate_case_allow_provider_signal "openai-primary-quota-fallback-success" \ - "openai/quota-primary" \ - "openai/fallback-one openai/fallback-two" \ - "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'openai/fallback-one' in [0-9]+s\\." \ - "2" \ - "openai/quota-primary|openai/fallback-one" \ - "|" \ - "openai" - -run_gate_case_allow_provider_signal "vertex-primary-429-fallback-success" \ - "vertex_ai/http429-primary" \ - "vertex_ai/fallback-one vertex_ai/fallback-two" \ - "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'vertex_ai/fallback-one' in [0-9]+s\\." \ - "2" \ - "vertex_ai/http429-primary|vertex_ai/fallback-one" \ - "|" - -run_gate_case_allow_provider_signal "vertex-primary-midstream-fallback-success" \ - "vertex_ai/midstream-primary" \ - "vertex_ai/fallback-one vertex_ai/fallback-two" \ - "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'vertex_ai/fallback-one' in [0-9]+s\\." \ - "2" \ - "vertex_ai/midstream-primary|vertex_ai/fallback-one" \ - "|" - -run_gate_case_allow_provider_signal "vertex-primary-midstream-retry-same-model-success" \ - "vertex_ai/retry-midstream-primary" \ - "vertex_ai/fallback-one vertex_ai/fallback-two" \ - "0" \ - "scan ok after same-model retry" \ - "2" \ - "vertex_ai/retry-midstream-primary|vertex_ai/retry-midstream-primary" \ - "|" \ - "vertex_ai" \ - "__DEFAULT__" \ - "" \ - "1" - -# Bug 9: Rate-limit transient same-model retry (previously untested path) -run_gate_case_allow_provider_signal "vertex-primary-ratelimit-retry-same-model-success" \ - "vertex_ai/retry-ratelimit-primary" \ - "vertex_ai/fallback-one vertex_ai/fallback-two" \ - "0" \ - "scan ok after same-model rate-limit retry" \ - "2" \ - "vertex_ai/retry-ratelimit-primary|vertex_ai/retry-ratelimit-primary" \ - "|" \ - "vertex_ai" \ - "__DEFAULT__" \ - "" \ - "1" - -run_gate_case_allow_provider_signal "vertex-primary-api-connection-retry-same-model-success" \ - "gemini/retry-api-connection-primary" \ - "vertex_ai/fallback-one vertex_ai/fallback-two" \ - "0" \ - "scan ok after same-model api connection retry" \ - "2" \ - "gemini/retry-api-connection-primary|gemini/retry-api-connection-primary" \ - "https://example.invalid|https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ - "" \ - "1" - -run_gate_case_allow_provider_signal "github-models-internal-server-connection-retry-same-model-success" \ - "openai/openai/retry-api-connection-primary" \ - "" \ - "0" \ - "scan ok after same-model api connection retry" \ - "2" \ - "openai/openai/retry-api-connection-primary|openai/openai/retry-api-connection-primary" \ - "https://models.github.ai/inference|https://models.github.ai/inference" \ - "openai" \ - "https://models.github.ai/inference" \ - "" \ - "1" - -run_gate_case_allow_provider_signal "internal-server-error-unrelated-output-nonretryable" \ - "openai/openai/retry-api-connection-primary" \ - "" \ - "1" \ - "Strix quick scan failed with a non-recoverable error." \ - "1" \ - "openai/openai/retry-api-connection-primary" \ - "https://models.github.ai/inference" \ - "openai" \ - "https://models.github.ai/inference" \ - "" \ - "0" - -# Bug: large provider logs (many matching litellm.InternalServerError -# blocks) must not suppress a legitimate same-model retry via SIGPIPE on the -# bounded awk scan under `set -o pipefail`. See PR #1394 Devin finding -# "Large provider logs suppress retries". -run_gate_case_allow_provider_signal "internal-server-error-many-blocks-retry-same-model-success" \ - "openai/openai/retry-api-connection-primary" \ - "" \ - "0" \ - "scan ok after same-model api connection retry" \ - "2" \ - "openai/openai/retry-api-connection-primary|openai/openai/retry-api-connection-primary" \ - "https://models.github.ai/inference|https://models.github.ai/inference" \ - "openai" \ - "https://models.github.ai/inference" \ - "" \ - "1" - -run_gate_case "openrouter-502-fallback-retry-same-model-success" \ - "vertex_ai/missing-primary" \ - "openrouter/free vertex_ai/fallback-two" \ - "0" \ - "scan ok after OpenRouter 502 same-model retry" \ - "3" \ - "vertex_ai/missing-primary|openrouter/free|openrouter/free" \ - "|https://example.invalid|https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ - "" \ - "1" - -run_gate_case "openrouter-502-distant-target-output-nonretryable" \ - "vertex_ai/missing-primary" \ - "openrouter/free vertex_ai/fallback-two" \ - "1" \ - "Strix quick scan failed with a non-recoverable error." \ - "2" \ - "vertex_ai/missing-primary|openrouter/free" \ - "|https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ - "" \ - "1" - -run_gate_case "github-models-primary-unavailable-fallback-success" \ - "openai/gpt-5" \ - "" \ - "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'deepseek/deepseek-r1-0528' in [0-9]+s\\." \ - "2" \ - "openai/gpt-5|openai/deepseek/deepseek-r1-0528" \ - "https://models.github.ai/inference|https://models.github.ai/inference" \ - "openai" \ - "https://models.github.ai/inference" \ - "" \ - "0" \ - "CRITICAL" \ - "0" \ - "" \ - "" \ - "1200" \ - "0" \ - "" \ - "" \ - "" \ - "" \ - "0" \ - "" \ - "" \ - "" \ - "__SAME_AS_FALLBACK_MODELS__" \ - "deepseek/deepseek-r1-0528 deepseek/deepseek-v3-0324" \ - "1" - -run_gate_case_allow_provider_signal "github-models-primary-denied-fallback-success" \ - "openai/gpt-5" \ - "" \ - "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'deepseek/deepseek-r1-0528' in [0-9]+s\\." \ - "2" \ - "openai/gpt-5|openai/deepseek/deepseek-r1-0528" \ - "https://models.github.ai/inference|https://models.github.ai/inference" \ - "openai" \ - "https://models.github.ai/inference" \ - "" \ - "0" \ - "CRITICAL" \ - "0" \ - "" \ - "" \ - "1200" \ - "0" \ - "" \ - "" \ - "" \ - "" \ - "0" \ - "" \ - "" \ - "" \ - "__SAME_AS_FALLBACK_MODELS__" \ - "deepseek/deepseek-r1-0528 deepseek/deepseek-v3-0324" \ - "1" - -run_github_models_http410_case \ - "github-models-http410-authenticated-fallback-success" \ - "0" \ - "2" \ - "openai/gpt-5|openai/deepseek/deepseek-r1-0528" \ - "https://models.github.ai/inference|https://models.github.ai/inference" \ - "REGEX:Strix quick scan succeeded with fallback model 'deepseek/deepseek-r1-0528' in [0-9]+s\\." - -for scenario in \ - github-models-http410-missing-http-token \ - github-models-http410-missing-provider-error \ - github-models-http410-numeric-continuation-4100 \ - github-models-http410-numeric-continuation-4104 \ - github-models-http410-target-output-spoof \ - github-models-retirement-brownout-phrase-only; do - run_github_models_http410_case \ - "$scenario" \ - "1" \ - "1" \ - "openai/gpt-5" \ - "https://models.github.ai/inference" -done - -run_gate_case "github-models-primary-ratelimit-fallback-success" \ - "openai/gpt-5" \ - "" \ - "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'deepseek/deepseek-r1-0528' in [0-9]+s\\." \ - "2" \ - "openai/gpt-5|openai/deepseek/deepseek-r1-0528" \ - "https://models.github.ai/inference|https://models.github.ai/inference" \ - "openai" \ - "https://models.github.ai/inference" \ - "" \ - "2" \ - "CRITICAL" \ - "0" \ - "" \ - "" \ - "1200" \ - "0" \ - "" \ - "" \ - "" \ - "" \ - "0" \ - "" \ - "" \ - "" \ - "__SAME_AS_FALLBACK_MODELS__" \ - "deepseek/deepseek-r1-0528 deepseek/deepseek-v3-0324" \ - "1" - -run_gate_case "github-models-fallback-provider-signal-tries-next" \ - "openai/gpt-5" \ - "" \ - "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'deepseek/deepseek-v3-0324' in [0-9]+s\\." \ - "3" \ - "openai/gpt-5|openai/deepseek/deepseek-r1-0528|openai/deepseek/deepseek-v3-0324" \ - "https://models.github.ai/inference|https://models.github.ai/inference|https://models.github.ai/inference" \ - "openai" \ - "https://models.github.ai/inference" \ - "" \ - "0" \ - "CRITICAL" \ - "0" \ - "" \ - "" \ - "1200" \ - "0" \ - "pull_request" \ - "sync-module-system/smart-crawling-biz/src/main/java/org/empasy/sync/modules/system/controller/SysPositionController.java" \ - "" \ - "" \ - "0" \ - "" \ - "" \ - "" \ - "__SAME_AS_FALLBACK_MODELS__" \ - "deepseek/deepseek-r1-0528 deepseek/deepseek-v3-0324" \ - "1" - -run_gate_case "github-models-fallback-baseline-vulnerability-before-next-success-continues" \ - "openai/gpt-5" \ - "" \ - "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'deepseek/deepseek-v3-0324' in [0-9]+s\\." \ - "3" \ - "openai/gpt-5|openai/deepseek/deepseek-r1-0528|openai/deepseek/deepseek-v3-0324" \ - "https://models.github.ai/inference|https://models.github.ai/inference|https://models.github.ai/inference" \ - "openai" \ - "https://models.github.ai/inference" \ - "" \ - "0" \ - "CRITICAL" \ - "0" \ - "" \ - "" \ - "1200" \ - "0" \ - "pull_request" \ - "sync-module-system/smart-crawling-biz/src/main/java/org/empasy/sync/modules/system/controller/SysPositionController.java" \ - "" \ - "" \ - "0" \ - "" \ - "" \ - "" \ - "__SAME_AS_FALLBACK_MODELS__" \ - "deepseek/deepseek-r1-0528 deepseek/deepseek-v3-0324" \ - "1" - -run_gate_case "github-models-exhausted-after-baseline-vulnerability-fails-closed" \ - "openai/gpt-5" \ - "" \ - "1" \ - "STRIX_PROVIDER_UNAVAILABLE: provider models were exhausted after incomplete scan evidence." \ - "3" \ - "openai/gpt-5|openai/deepseek/deepseek-r1-0528|openai/deepseek/deepseek-v3-0324" \ - "https://models.github.ai/inference|https://models.github.ai/inference|https://models.github.ai/inference" \ - "openai" \ - "https://models.github.ai/inference" \ - "" \ - "0" \ - "CRITICAL" \ - "0" \ - "" \ - "" \ - "1200" \ - "0" \ - "pull_request" \ - "sync-module-system/smart-crawling-biz/src/main/java/org/empasy/sync/modules/system/controller/SysPositionController.java" \ - "" \ - "" \ - "0" \ - "" \ - "" \ - "" \ - "__SAME_AS_FALLBACK_MODELS__" \ - "deepseek/deepseek-r1-0528 deepseek/deepseek-v3-0324" \ - "1" - -run_gate_case "github-models-fallback-changed-vulnerability-before-next-success-blocks" \ - "openai/gpt-5" \ - "" \ - "1" \ - "Strix model reported threshold vulnerabilities before fallback success; failing closed so every model-reported vulnerability is reviewed." \ - "2" \ - "openai/gpt-5|openai/deepseek/deepseek-r1-0528" \ - "https://models.github.ai/inference|https://models.github.ai/inference" \ - "openai" \ - "https://models.github.ai/inference" \ - "" \ - "0" \ - "CRITICAL" \ - "0" \ - "" \ - "" \ - "1200" \ - "0" \ - "pull_request" \ - "sync-module-system/smart-crawling-biz/src/main/java/org/empasy/sync/modules/system/controller/SysPositionController.java" \ - "" \ - "" \ - "0" \ - "" \ - "" \ - "" \ - "__SAME_AS_FALLBACK_MODELS__" \ - "deepseek/deepseek-r1-0528 deepseek/deepseek-v3-0324" \ + "pull-request-target-invalid-head-sha-fails-closed" \ + "head" + +run_pull_request_target_aborts_on_pr_head_blob_failure_case \ + "pull-request-target-disabled-pr-scope-pr-head-blob-read-failure" \ + "src/existing.py" \ + "BASE_CONTENT_MUST_NOT_BE_USED_AFTER_DISABLED_SCOPE_HEAD_FAILURE" \ + "HEAD_CONTENT_SHOULD_NOT_BECOME_PARTIAL_SCAN_INPUT" \ + "cat-file" \ "1" -run_gate_case "github-models-fallback-dockerfile-test-baseline-before-next-success-continues" \ - "openai/gpt-5" \ +run_gate_case "success" \ + "orchestrator/free" \ "" \ "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'deepseek/deepseek-v3-0324' in [0-9]+s\\." \ - "3" \ - "openai/gpt-5|openai/deepseek/deepseek-r1-0528|openai/deepseek/deepseek-v3-0324" \ - "https://models.github.ai/inference|https://models.github.ai/inference|https://models.github.ai/inference" \ - "openai" \ - "https://models.github.ai/inference" \ + "scan ok" \ + "1" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" + +run_gate_case "contextual-orchestrator-missing-api-base-fails-closed" \ + "orchestrator/free" \ "" \ + "2" \ + "require LLM_API_BASE_FILE to select the pinned loopback gateway" \ "0" \ - "MEDIUM" \ - "0" \ - "" \ "" \ - "1200" \ - "0" \ - "pull_request" \ - ".github/workflows/build-ci-image.yml" \ "" \ + "contextual_orchestrator" \ + "" + +run_gate_case "contextual-orchestrator-gateway-model-qualification" \ + "orchestrator/free" \ "" \ "0" \ - "" \ - "" \ - "" \ - "__SAME_AS_FALLBACK_MODELS__" \ - "deepseek/deepseek-r1-0528 deepseek/deepseek-v3-0324" \ - "1" + "scan ok through contextual-orchestrator gateway" \ + "1" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" -run_gate_case_allow_provider_signal "gemini-high-demand-retry-same-model-success" \ - "gemini/retry-high-demand-primary" \ - "vertex_ai/fallback-one vertex_ai/fallback-two" \ - "0" \ - "scan ok after same-model high-demand retry" \ - "2" \ - "gemini/retry-high-demand-primary|gemini/retry-high-demand-primary" \ - "https://example.invalid|https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ +run_gate_case "success-with-critical-report" \ + "orchestrator/free" \ "" \ - "1" + "1" \ + "Current Strix vulnerability report exists; failing closed without a repository-authored severity threshold." \ + "1" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" -run_gate_case_allow_provider_signal "nvidia-overloaded-direct-fallback-success" \ - "nvidia_nim/nvidia/overloaded-primary" \ - "" \ - "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'nvidia_nim/nvidia/fallback-one' in [0-9]+s\\." \ - "3" \ - "nvidia_nim/nvidia/overloaded-primary|nvidia_nim/nvidia/overloaded-primary|nvidia_nim/nvidia/fallback-one" \ - "https://integrate.api.nvidia.com/v1|https://integrate.api.nvidia.com/v1|https://integrate.api.nvidia.com/v1" \ - "nvidia_nim" \ - "https://integrate.api.nvidia.com/v1" \ +run_gate_case "pr-executable-integrity-mismatch" \ + "orchestrator/free" \ "" \ "1" \ - "CRITICAL" \ - "0" \ - "" \ - "" \ - "1200" \ - "0" \ - "" \ - "" \ - "" \ - "" \ + "did not match the pinned SHA-256 digest" \ "0" \ "" \ "" \ - "" \ - "__SAME_AS_FALLBACK_MODELS__" \ - "nvidia_nim/nvidia/fallback-one openai-direct/gpt-5.4" + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" -run_gate_case_allow_provider_signal "nvidia-rate-limit-openai-direct-fallback-clears-api-base" \ - "nvidia_nim/nvidia/rate-limited-primary" \ +run_gate_case "pr-executable-group-writable" \ + "orchestrator/free" \ "" \ + "1" \ + "must not be group/world writable" \ "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'openai-direct/gpt-5.4' in [0-9]+s\\." \ - "2" \ - "nvidia_nim/nvidia/rate-limited-primary|openai/gpt-5.4" \ - "https://integrate.api.nvidia.com/v1|" \ - "nvidia_nim" \ - "https://integrate.api.nvidia.com/v1" \ "" \ - "0" \ - "CRITICAL" \ - "0" \ "" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" + +run_gate_case "pr-executable-root-group-writable" \ + "orchestrator/free" \ "" \ - "1200" \ + "1" \ + "pinned Strix installation root must not be group/world writable" \ "0" \ "" \ "" \ - "" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" + +run_gate_case "runtime-env-forwarding" \ + "orchestrator/free" \ "" \ "0" \ - "" \ - "" \ - "" \ - "__SAME_AS_FALLBACK_MODELS__" \ - "openai-direct/gpt-5.4" + "scan ok" \ + "1" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" -run_gate_case_allow_provider_signal "gemini-timeout-direct-fallback-success" \ - "gemini/retry-timeout-primary" \ - "gemini/fallback-one gemini/fallback-two" \ - "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'gemini/fallback-one' in [0-9]+s\\." \ - "2" \ - "gemini/retry-timeout-primary|gemini/fallback-one" \ - "https://example.invalid|https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ - "" \ - "1" -run_gate_case_allow_provider_signal "gemini-timeout-fallback-success" \ - "gemini/timeout-fallback-primary" \ - "gemini/fallback-one gemini/fallback-two" \ - "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'gemini/fallback-one' in [0-9]+s\\." \ - "2" \ - "gemini/timeout-fallback-primary|gemini/fallback-one" \ - "https://example.invalid|https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + +run_gate_case "nonrecoverable" \ + "orchestrator/free" \ "" \ - "1" + "1" \ + "Strix quick scan failed; failing closed without repository-authored retry or fallback allocation." \ + "1" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" -run_gate_case_allow_provider_signal "gemini-generic-fallback-success" \ - "gemini/timeout-fallback-primary" \ +run_gate_case "provider-prefix-required" \ + "gemini-2.5-pro" \ "" \ - "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'gemini/fallback-one' in [0-9]+s\\." \ "2" \ - "gemini/timeout-fallback-primary|gemini/fallback-one" \ - "https://example.invalid|https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ - "" \ - "1" \ - "CRITICAL" \ + "ERROR: Strix model must be orchestrator/free through contextual-orchestrator; direct provider/model routes are forbidden" \ "0" \ "" \ "" \ - "1200" \ - "0" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" + + +run_gate_case "provider-prefix-required-resource-path-primary-implicit-default-provider" \ + "projects/p1/locations/us-central1/publishers/google/models/gemini-2.5-pro" \ "" \ + "2" \ + "ERROR: Strix model must be orchestrator/free through contextual-orchestrator; direct provider/model routes are forbidden" \ + "0" \ "" \ "" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" + +run_gate_case "provider-prefix-required-resource-path-primary-explicit-empty-default-provider" \ + "projects/p1/locations/us-central1/publishers/google/models/gemini-2.5-pro" \ "" \ + "2" \ + "ERROR: Strix model must be orchestrator/free through contextual-orchestrator; direct provider/model routes are forbidden" \ "0" \ "" \ "" \ - "" \ - "__UNSET__" \ - "gemini/fallback-one gemini/fallback-two" + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" -run_gate_case_allow_provider_signal "gemini-zero-findings-timeout-fallback-allows-pr" \ - "gemini/zero-timeout-primary" \ - "gemini/fallback-one" \ - "1" \ - "Strix reported zero vulnerabilities before provider infrastructure failure; failing closed because provider infrastructure failures are not clean scan evidence." \ - "2" \ - "gemini/zero-timeout-primary|gemini/fallback-one" \ - "https://example.invalid|https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + +# Regression: Vertex custom model resource path projects/

/locations//models/ +# (no publishers/ segment) must be recognized as a Vertex resource path and +# normalized to vertex_ai/. +run_gate_case "vertex-custom-model-resource-path" \ + "projects/my-proj/locations/us-central1/models/my-custom-model-123" \ "" \ - "0" \ - "CRITICAL" \ + "2" \ + "ERROR: Strix model must be orchestrator/free through contextual-orchestrator; direct provider/model routes are forbidden" \ "0" \ "" \ "" \ - "1200" \ - "0" \ - "pull_request" \ - "sync-module-system/smart-crawling-biz/src/main/java/org/empasy/sync/modules/system/controller/SysPositionController.java" + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" -run_gate_case_allow_provider_signal "pr-scope-zero-finding-does-not-leak" \ - "gemini/scope-zero-leak-primary" \ - "" \ - "1" \ - "Strix reported zero vulnerabilities before provider infrastructure failure; failing closed because provider infrastructure failures are not clean scan evidence." \ - "1" \ - "gemini/scope-zero-leak-primary" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + + +run_gate_case "nonvertex-slash-model-passthrough" \ + "foo/bar" \ "" \ - "0" \ - "CRITICAL" \ + "2" \ + "ERROR: Strix model must be orchestrator/free through contextual-orchestrator; direct provider/model routes are forbidden" \ "0" \ "" \ "" \ - "1200" \ - "0" \ - "pull_request" \ - $'sync-module-system/smart-crawling-biz/src/main/java/org/empasy/sync/modules/system/controller/SysPositionController.java\nsync-module-system/smart-crawling-playwright/src/main/java/org/empasy/sync/mcp/service/PlayWrightService.java' \ - "" \ - "1" + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" -run_gate_case "service-unavailable-no-llm-marker-nonrecoverable" \ - "custom/service-unavailable-primary" \ - "vertex_ai/fallback-one vertex_ai/fallback-two" \ + + + + + + + + +# Bug 9: Rate-limit transient same-model retry (previously untested path) + + + +run_gate_case_allow_provider_signal "internal-server-error-unrelated-output-nonretryable" \ + "orchestrator/free" \ + "" \ "1" \ - "Strix quick scan failed with a non-recoverable error." \ + "STRIX_PROVIDER_UNAVAILABLE: contextual-orchestrator/orchestrator/free did not produce authoritative scan evidence" \ "1" \ - "custom/service-unavailable-primary" \ - "https://example.invalid" \ - "custom" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ - "1" + "0" + +# Bug: large provider logs (many matching litellm.InternalServerError +# blocks) must not suppress a legitimate same-model retry via SIGPIPE on the +# bounded awk scan under `set -o pipefail`. See PR #1394 Devin finding +# "Large provider logs suppress retries". + + + + + + + + + + + + + + + -run_gate_case "server-disconnect-no-llm-marker-nonrecoverable" \ - "vertex_ai/app-server-disconnect-primary" \ - "vertex_ai/fallback-one vertex_ai/fallback-two" \ - "1" \ - "Strix quick scan failed with a non-recoverable error." \ - "1" \ - "vertex_ai/app-server-disconnect-primary" \ - "" -# Bug 11: Timeout should move directly to fallback instead of retrying the same model. -run_gate_case_allow_provider_signal "vertex-primary-timeout-retry-same-model-success" \ - "vertex_ai/retry-timeout-primary" \ - "vertex_ai/fallback-one vertex_ai/fallback-two" \ - "0" \ - "scan ok after timeout fallback" \ - "2" \ - "vertex_ai/retry-timeout-primary|vertex_ai/fallback-one" \ - "|" \ - "vertex_ai" \ - "__DEFAULT__" \ - "" \ - "1" -# Bug 11b: Timeout → immediate fallback model succeeds. -run_gate_case_allow_provider_signal "vertex-primary-timeout-exhausted-fallback-success" \ - "vertex_ai/timeout-exhaust-primary" \ - "vertex_ai/fallback-one vertex_ai/fallback-two" \ - "0" \ - "scan ok after timeout-exhausted fallback" \ - "2" \ - "vertex_ai/timeout-exhaust-primary|vertex_ai/fallback-one" \ - "|" \ - "vertex_ai" \ - "__DEFAULT__" \ - "" \ - "1" -run_gate_case_allow_provider_signal "zero-findings-timeout-all-models" \ - "vertex_ai/zero-timeout-primary" \ - "vertex_ai/fallback-one" \ - "1" \ - "Strix reported zero vulnerabilities before provider infrastructure failure; failing closed because provider infrastructure failures are not clean scan evidence." \ - "2" \ - "vertex_ai/zero-timeout-primary|vertex_ai/fallback-one" \ - "|" \ - "vertex_ai" \ - "__DEFAULT__" \ - "" \ - "0" \ - "CRITICAL" \ - "0" \ - "" \ - "" \ - "$TIMEOUT_TEST_PROCESS_SECONDS" \ - "0" \ - "pull_request" \ - "sync-module-system/smart-crawling-biz/src/main/java/org/empasy/sync/modules/system/controller/SysPositionController.java" -run_gate_case_allow_provider_signal "zero-findings-timeout-all-models" \ - "vertex_ai/zero-timeout-primary" \ - "vertex_ai/fallback-one" \ - "1" \ - "Configured Vertex model and fallback models were unavailable." \ - "2" \ - "vertex_ai/zero-timeout-primary|vertex_ai/fallback-one" \ - "|" \ - "vertex_ai" \ - "__DEFAULT__" \ - "" \ - "0" \ - "CRITICAL" \ - "0" \ - "" \ +run_gate_case_allow_provider_signal "pr-scope-zero-finding-does-not-leak" \ + "orchestrator/free" \ "" \ - "$TIMEOUT_TEST_PROCESS_SECONDS" \ - "0" \ - "push" - -run_gate_case_allow_provider_signal "zero-findings-sticky-across-fallback" \ - "vertex_ai/zero-sticky-primary" \ - "vertex_ai/fallback-one" \ "1" \ "Strix reported zero vulnerabilities before provider infrastructure failure; failing closed because provider infrastructure failures are not clean scan evidence." \ - "2" \ - "vertex_ai/zero-sticky-primary|vertex_ai/fallback-one" \ - "|" \ - "vertex_ai" \ - "__DEFAULT__" \ + "1" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ "0" \ "" \ "" \ - "$TIMEOUT_TEST_PROCESS_SECONDS" \ + "1200" \ "0" \ "pull_request" \ - "sync-module-system/smart-crawling-biz/src/main/java/org/empasy/sync/modules/system/controller/SysPositionController.java" + $'sync-module-system/smart-crawling-biz/src/main/java/org/empasy/sync/modules/system/controller/SysPositionController.java\nsync-module-system/smart-crawling-playwright/src/main/java/org/empasy/sync/mcp/service/PlayWrightService.java' \ + "" \ + "1" -run_gate_case_allow_provider_signal "zero-findings-with-low-report-timeout" \ - "vertex_ai/zero-low-primary" \ - "vertex_ai/fallback-one" \ - "1" \ - "Configured Vertex model and fallback models were unavailable." \ - "2" \ - "vertex_ai/zero-low-primary|vertex_ai/fallback-one" \ - "|" \ - "vertex_ai" \ - "__DEFAULT__" \ +run_gate_case "service-unavailable-no-llm-marker-nonrecoverable" \ + "orchestrator/free" \ "" \ - "0" \ - "CRITICAL" \ - "0" \ + "1" \ + "Strix quick scan failed; failing closed without repository-authored retry or fallback allocation." \ + "1" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ + "0" + +run_gate_case "server-disconnect-no-llm-marker-nonrecoverable" \ + "orchestrator/free" \ "" \ - "$TIMEOUT_TEST_PROCESS_SECONDS" \ - "0" \ - "pull_request" \ - "sync-module-system/smart-crawling-biz/src/main/java/org/empasy/sync/modules/system/controller/SysPositionController.java" + "1" \ + "Strix quick scan failed; failing closed without repository-authored retry or fallback allocation." \ + "1" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" + +# Bug 11: Timeout should move directly to fallback instead of retrying the same model. + +# Bug 11b: Timeout → immediate fallback model succeeds. + + + + run_gate_case "strict-zero-findings-timeout-fails-pr" \ - "vertex_ai/zero-timeout-primary" \ - " " \ + "orchestrator/free" \ + "" \ "1" \ "failing closed" \ "1" \ - "vertex_ai/zero-timeout-primary" \ - "" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -10850,15 +8641,15 @@ run_gate_case "strict-zero-findings-timeout-fails-pr" \ "1" run_gate_case "provider-fatal-success-signal" \ - "vertex_ai/provider-fatal-success-signal" \ + "orchestrator/free" \ "" \ "1" \ "Strix run emitted provider infrastructure or failure-signal output; failing closed." \ "1" \ - "vertex_ai/provider-fatal-success-signal" \ - "" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -10880,15 +8671,15 @@ run_gate_case "provider-fatal-success-signal" \ "1" run_gate_case "provider-warning-success-signal" \ - "vertex_ai/provider-warning-success-signal" \ + "orchestrator/free" \ "" \ "1" \ "Strix run emitted provider infrastructure or failure-signal output; failing closed." \ "1" \ - "vertex_ai/provider-warning-success-signal" \ - "" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -10909,25 +8700,17 @@ run_gate_case "provider-warning-success-signal" \ "" \ "1" -run_gate_case "provider-report-rate-limit-fallback-success" \ - "vertex_ai/report-rate-limit-primary" \ - "vertex_ai/fallback-one vertex_ai/fallback-two" \ - "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'vertex_ai/fallback-one' in [0-9]+s\\." \ - "2" \ - "vertex_ai/report-rate-limit-primary|vertex_ai/fallback-one" \ - "|" run_gate_case "report-known-internal-warning-sanitized" \ - "vertex_ai/report-known-internal-warning-sanitized" \ + "orchestrator/free" \ "" \ "0" \ - "Strix run succeeded for model 'vertex_ai/report-known-internal-warning-sanitized'" \ + "Strix run succeeded for model 'orchestrator/free'" \ "1" \ - "vertex_ai/report-known-internal-warning-sanitized" \ - "" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -10949,15 +8732,15 @@ run_gate_case "report-known-internal-warning-sanitized" \ "1" run_gate_case "report-known-internal-warning-variant-sanitized" \ - "vertex_ai/report-known-internal-warning-variant-sanitized" \ + "orchestrator/free" \ "" \ "0" \ - "Strix run succeeded for model 'vertex_ai/report-known-internal-warning-variant-sanitized'" \ + "Strix run succeeded for model 'orchestrator/free'" \ "1" \ - "vertex_ai/report-known-internal-warning-variant-sanitized" \ - "" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -10978,15 +8761,15 @@ run_gate_case "report-known-internal-warning-variant-sanitized" \ "1" run_gate_case "report-unknown-warning-fails" \ - "vertex_ai/report-unknown-warning-fails" \ + "orchestrator/free" \ "" \ "1" \ "Strix report artifacts emitted warning/fatal/denied/timeout output; failing closed." \ "1" \ - "vertex_ai/report-unknown-warning-fails" \ - "" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -11008,15 +8791,15 @@ run_gate_case "report-unknown-warning-fails" \ "1" run_gate_case "provider-denied-success-signal" \ - "vertex_ai/provider-denied-success-signal" \ + "orchestrator/free" \ "" \ "1" \ "Strix run emitted provider infrastructure or failure-signal output; failing closed." \ "1" \ - "vertex_ai/provider-denied-success-signal" \ - "" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -11037,34 +8820,28 @@ run_gate_case "provider-denied-success-signal" \ "" \ "1" -run_gate_case_allow_provider_signal "vertex-all-ratelimited" \ - "vertex_ai/ratelimit-primary" \ - "vertex_ai/fallback-one vertex_ai/fallback-two" \ - "1" \ - "Configured Vertex model and fallback models were unavailable." \ - "3" \ - "vertex_ai/ratelimit-primary|vertex_ai/fallback-one|vertex_ai/fallback-two" \ - "||" run_gate_case "vertex-primary-hallucinated-endpoint-fallback-success" \ - "vertex_ai/hallucination-primary" \ - "vertex_ai/fallback-one vertex_ai/fallback-two" \ + "orchestrator/free" \ + "" \ "1" \ "Strix quick scan failed with a non-recoverable error." \ "1" \ - "vertex_ai/hallucination-primary" \ - "" + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" run_gate_case "opencode-documented-env-api-key-fallback-success" \ - "vertex_ai/opencode-env-primary" \ - "vertex_ai/fallback-one vertex_ai/fallback-two" \ + "orchestrator/free" \ + "" \ "1" \ "Strix finding intersects files changed in this pull request." \ "1" \ - "vertex_ai/opencode-env-primary" \ - "" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "HIGH" \ @@ -11077,15 +8854,15 @@ run_gate_case "opencode-documented-env-api-key-fallback-success" \ ".github/workflows/opencode-review.yml" run_gate_case "generic-github-actions-workflow-fallback-success" \ - "vertex_ai/generic-actions-primary" \ - "vertex_ai/fallback-one vertex_ai/fallback-two" \ + "orchestrator/free" \ + "" \ "1" \ "Unable to map Strix findings to changed files; failing closed for pull request." \ "1" \ - "vertex_ai/generic-actions-primary" \ - "" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -11098,24 +8875,26 @@ run_gate_case "generic-github-actions-workflow-fallback-success" \ ".github/workflows/strix.yml" run_gate_case "vertex-primary-existing-endpoint-nonrecoverable" \ - "vertex_ai/existing-endpoint-primary" \ - "vertex_ai/fallback-one vertex_ai/fallback-two" \ + "orchestrator/free" \ + "" \ "1" \ - "Strix quick scan failed with a non-recoverable error." \ + "Strix quick scan failed; failing closed without repository-authored retry or fallback allocation." \ "1" \ - "vertex_ai/existing-endpoint-primary" \ - "" + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" run_gate_case "pr-stale-source-claim-fallback-success" \ - "vertex_ai/stale-source-primary" \ - "vertex_ai/fallback-one vertex_ai/fallback-two" \ + "orchestrator/free" \ + "" \ "1" \ "Strix finding intersects files changed in this pull request." \ "1" \ - "vertex_ai/stale-source-primary" \ - "" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "HIGH" \ @@ -11128,15 +8907,15 @@ run_gate_case "pr-stale-source-claim-fallback-success" \ "backend/db/models.py" run_gate_case "pr-stale-snapshot-snippet-fallback-success" \ - "vertex_ai/stale-snapshot-primary" \ - "vertex_ai/fallback-one vertex_ai/fallback-two" \ + "orchestrator/free" \ + "" \ "1" \ "Strix finding intersects files changed in this pull request." \ "1" \ - "vertex_ai/stale-snapshot-primary" \ - "" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "MEDIUM" \ @@ -11149,15 +8928,15 @@ run_gate_case "pr-stale-snapshot-snippet-fallback-success" \ "backend/app/api/snapshots.py" run_gate_case "pr-stale-source-plus-real-finding-blocks" \ - "vertex_ai/stale-source-primary" \ - "vertex_ai/fallback-one vertex_ai/fallback-two" \ + "orchestrator/free" \ + "" \ "1" \ "Strix finding intersects files changed in this pull request." \ "1" \ - "vertex_ai/stale-source-primary" \ - "" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "HIGH" \ @@ -11170,15 +8949,15 @@ run_gate_case "pr-stale-source-plus-real-finding-blocks" \ $'backend/db/models.py\nbackend/api/emails.py' run_gate_case_allow_provider_signal "pr-changed-finding-with-retry-marker-blocks" \ - "vertex_ai/changed-finding-primary" \ - "vertex_ai/fallback-one vertex_ai/fallback-two" \ + "orchestrator/free" \ + "" \ "1" \ "Strix finding intersects files changed in this pull request." \ "1" \ - "vertex_ai/changed-finding-primary" \ - "" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "HIGH" \ @@ -11191,15 +8970,15 @@ run_gate_case_allow_provider_signal "pr-changed-finding-with-retry-marker-blocks "backend/api/emails.py" run_gate_case "pr-stale-report-plus-inline-changed-finding-blocks" \ - "vertex_ai/stale-inline-primary" \ - "vertex_ai/fallback-one vertex_ai/fallback-two" \ + "orchestrator/free" \ + "" \ "1" \ "Strix finding intersects files changed in this pull request." \ "1" \ - "vertex_ai/stale-inline-primary" \ - "" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "HIGH" \ @@ -11211,86 +8990,91 @@ run_gate_case "pr-stale-report-plus-inline-changed-finding-blocks" \ "pull_request" \ $'backend/db/models.py\nbackend/api/emails.py' -run_gate_case "high-vuln-below-threshold" \ - "vertex_ai/high-vuln-primary" \ +# Infrastructure error guard: below-threshold findings must NOT pass when the +# strix log contains evidence of infrastructure-level errors (timeout, +# rate-limit, transport failures) because the scan was likely incomplete. + +# Guard test 1: LOW finding + timeout → should fail (exit 1). +# The below-threshold check runs first but detects infrastructure errors in the +# strix log and refuses bypass. The timeout is also vertex-retryable, so the +# gate continues into the fallback loop. All attempts see the same timeout. + +# Guard test 2: LOW finding + rate-limit → should fail (exit 1). +# Below-threshold check refuses bypass due to infra errors. +# Rate-limit is vertex-retryable, so the gate also tries fallback models. + +# Guard test 3: INFO finding + ConnectionError → should fail (exit 1). +# ConnectionError is NOT vertex-retryable, so only the primary model is tried. +# Single-governed-request fail-closed contract. These replace the deleted +# multi-model fallback cluster: the same provider signals that used to select +# the next fallback model must now fail closed after exactly one request. + +run_gate_case_allow_provider_signal "provider-ratelimit-fails-closed" \ + "orchestrator/free" \ "" \ - "0" \ - "below configured fail threshold 'CRITICAL'" \ "1" \ - "vertex_ai/high-vuln-primary" \ - "" + "STRIX_PROVIDER_UNAVAILABLE: contextual-orchestrator/orchestrator/free did not produce authoritative scan evidence" \ + "1" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" -run_gate_case "multi-severity-low-then-critical" \ - "vertex_ai/multi-severity-primary" \ +run_gate_case_allow_provider_signal "provider-resource-exhausted-fails-closed" \ + "orchestrator/free" \ "" \ "1" \ - "Strix quick scan failed with a non-recoverable error." \ + "STRIX_PROVIDER_UNAVAILABLE: contextual-orchestrator/orchestrator/free did not produce authoritative scan evidence" \ "1" \ - "vertex_ai/multi-severity-primary" \ - "" + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" -run_gate_case "inline-medium-below-threshold" \ - "vertex_ai/inline-medium-primary" \ +run_gate_case_allow_provider_signal "provider-quota-fails-closed" \ + "orchestrator/free" \ "" \ "1" \ - "No Strix vulnerability report artifact was produced; log-only severity markers are incomplete evidence, so the scan is failing closed." \ + "STRIX_PROVIDER_UNAVAILABLE: contextual-orchestrator/orchestrator/free did not produce authoritative scan evidence" \ "1" \ - "vertex_ai/inline-medium-primary" \ - "" + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" -run_gate_case "medium-vuln-default-threshold" \ - "openai/gpt-4o-mini" \ +run_gate_case_allow_provider_signal "provider-token-limit-fails-closed" \ + "orchestrator/free" \ "" \ "1" \ - "Strix quick scan failed with a non-recoverable error." \ + "STRIX_PROVIDER_UNAVAILABLE: contextual-orchestrator/orchestrator/free did not produce authoritative scan evidence" \ "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ - "" \ - "0" \ - "__UNSET__" - -# Infrastructure error guard: below-threshold findings must NOT pass when the -# strix log contains evidence of infrastructure-level errors (timeout, -# rate-limit, transport failures) because the scan was likely incomplete. + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" -# Guard test 1: LOW finding + timeout → should fail (exit 1). -# The below-threshold check runs first but detects infrastructure errors in the -# strix log and refuses bypass. The timeout is also vertex-retryable, so the -# gate continues into the fallback loop. All attempts see the same timeout. -run_gate_case_allow_provider_signal "below-threshold-with-timeout" \ - "vertex_ai/low-timeout-primary" \ - "vertex_ai/gemini-2.5-pro vertex_ai/gemini-2.5-flash" \ +run_gate_case_allow_provider_signal "provider-timeout-fails-closed" \ + "orchestrator/free" \ + "" \ "1" \ - "infrastructure errors occurred during this pipeline run; refusing bypass" \ - "3" \ - "vertex_ai/low-timeout-primary|vertex_ai/gemini-2.5-pro|vertex_ai/gemini-2.5-flash" \ - "||" - -# Guard test 2: LOW finding + rate-limit → should fail (exit 1). -# Below-threshold check refuses bypass due to infra errors. -# Rate-limit is vertex-retryable, so the gate also tries fallback models. -run_gate_case_allow_provider_signal "below-threshold-with-ratelimit" \ - "vertex_ai/low-ratelimit-primary" \ - "vertex_ai/gemini-2.5-pro vertex_ai/gemini-2.5-flash" \ + "STRIX_PROVIDER_UNAVAILABLE: contextual-orchestrator/orchestrator/free did not produce authoritative scan evidence" \ "1" \ - "infrastructure errors occurred during this pipeline run; refusing bypass" \ - "3" \ - "vertex_ai/low-ratelimit-primary|vertex_ai/gemini-2.5-pro|vertex_ai/gemini-2.5-flash" \ - "||" + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" -# Guard test 3: INFO finding + ConnectionError → should fail (exit 1). -# ConnectionError is NOT vertex-retryable, so only the primary model is tried. -run_gate_case_allow_provider_signal "below-threshold-with-connection-error" \ - "vertex_ai/info-conn-primary" \ +run_gate_case "provider-plain-failure-fails-closed-without-infra-marker" \ + "orchestrator/free" \ "" \ "1" \ - "infrastructure errors occurred during this pipeline run; refusing bypass" \ + "Strix quick scan failed; failing closed without repository-authored retry or fallback allocation." \ "1" \ - "vertex_ai/info-conn-primary" \ - "" + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" + # Guard test 3b: INFO finding + ConnectionError WITHOUT provider marker → should # PASS (exit 0). The two-grep infra-error detector requires both a transport @@ -11301,82 +9085,42 @@ run_gate_case_allow_provider_signal "below-threshold-with-connection-error" \ # A bare "ConnectionError" from the target application lacks the marker, so # has_detected_infrastructure_error() returns 1 (no infra error) and the # below-threshold bypass succeeds. -run_gate_case "below-threshold-with-connection-error-no-provider" \ - "vertex_ai/info-conn-noprov-primary" \ - "" \ - "0" \ - "below configured fail threshold" \ - "1" \ - "vertex_ai/info-conn-noprov-primary" \ - "" - # Guard test 3c: INFO finding + requests.exceptions.ConnectionError → should # PASS (exit 0). The "requests" transport library matches the broad # PROVIDER_CONTEXT_REGEX but is intentionally excluded from LLM_PROVIDER_ONLY_REGEX. # Before commit 0e90d48 the connection-error path used PROVIDER_CONTEXT_REGEX # and would have mis-classified this as an LLM infrastructure error; now it # correctly uses LLM_PROVIDER_ONLY_REGEX, so below-threshold bypass succeeds. -run_gate_case "below-threshold-with-requests-connection-error" \ - "vertex_ai/info-conn-requests-primary" \ - "" \ - "0" \ - "below configured fail threshold" \ - "1" \ - "vertex_ai/info-conn-requests-primary" \ - "" - # Guard test 4: MEDIUM finding + MidStreamFallbackError → should fail (exit 1). # Midstream is vertex-retryable, so the gate also tries fallback models # (after the below-threshold check refuses bypass due to infra errors). -run_gate_case_allow_provider_signal "below-threshold-with-midstream" \ - "vertex_ai/medium-midstream-primary" \ - "vertex_ai/gemini-2.5-pro vertex_ai/gemini-2.5-flash" \ - "1" \ - "infrastructure errors occurred during this pipeline run; refusing bypass" \ - "3" \ - "vertex_ai/medium-midstream-primary|vertex_ai/gemini-2.5-pro|vertex_ai/gemini-2.5-flash" \ - "||" - -run_gate_case "critical-vuln-at-threshold" \ - "vertex_ai/critical-vuln-primary" \ - "" \ - "1" \ - "Strix quick scan failed with a non-recoverable error." \ - "1" \ - "vertex_ai/critical-vuln-primary" \ - "" run_gate_case "malformed-severity-marker-nonrecoverable" \ - "vertex_ai/malformed-severity-primary" \ + "orchestrator/free" \ "" \ "1" \ - "Strix quick scan failed with a non-recoverable error." \ + "Strix quick scan failed; failing closed without repository-authored retry or fallback allocation." \ "1" \ - "vertex_ai/malformed-severity-primary" \ - "" + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" # Bug 7: Model disagreement — the primary produces an unmapped CRITICAL report # alongside a NOT_FOUND error. The report is already actionable fail-closed # evidence, so the gate must not spend provider budget on a fallback whose LOW # result could make the earlier finding appear downgraded. -run_gate_case "model-disagreement-critical-in-earlier-report" \ - "vertex_ai/model-a" \ - "vertex_ai/model-b" \ - "1" \ - "Strix quick scan failed with a non-recoverable error." \ - "1" \ - "vertex_ai/model-a" \ - "" - # Bug 4: deepseek/models/deepseek-r1 must NOT be rewritten to vertex_ai/deepseek-r1 run_gate_case "nonvertex-slash-model-not-rewritten" \ "deepseek/models/deepseek-r1" \ - "vertex_ai/fallback-one" \ + "" \ + "2" \ + "ERROR: Strix model must be orchestrator/free through contextual-orchestrator; direct provider/model routes are forbidden" \ "0" \ - "scan ok with deepseek model passthrough" \ - "1" \ - "deepseek/models/deepseek-r1" \ - "https://example.invalid" + "" \ + "" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" # Regression: STRIX_TARGET_PATH=

/src with default STRIX_SOURCE_DIRS (now ".") # must resolve to /src/. (i.e. /src itself), NOT /src/src. @@ -11384,17 +9128,17 @@ run_gate_case "nonvertex-slash-model-not-rewritten" \ # endpoint. Source-dir resolution still runs, but threshold findings now remain # blocking even when model/source inconsistency is suspected. run_gate_case "target-path-src-default-source-dirs" \ - "vertex_ai/hallucination-primary" \ - "vertex_ai/fallback-one vertex_ai/fallback-two" \ + "orchestrator/free" \ + "" \ "1" \ "Strix quick scan failed with a non-recoverable error." \ "1" \ - "vertex_ai/hallucination-primary" \ - "" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ - "1" \ + "0" \ "CRITICAL" \ "0" \ "__USE_SUBDIR_SRC__" \ @@ -11405,15 +9149,15 @@ run_gate_case "target-path-src-default-source-dirs" \ # the gate must find the endpoint in the api/ dir and treat the finding as # non-hallucinated → non-recoverable failure (exit 1). run_gate_case "multi-source-dirs-existing-endpoint" \ - "vertex_ai/multi-dir-primary" \ - "vertex_ai/fallback-one vertex_ai/fallback-two" \ + "orchestrator/free" \ + "" \ "1" \ - "Strix quick scan failed with a non-recoverable error." \ + "Strix quick scan failed; failing closed without repository-authored retry or fallback allocation." \ "1" \ - "vertex_ai/multi-dir-primary" \ - "" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -11422,74 +9166,35 @@ run_gate_case "multi-source-dirs-existing-endpoint" \ "src api" run_gate_case "preserve-existing-api-base" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "0" \ - "scan ok with preserved api base" \ + "scan ok with pinned gateway api base" \ "1" \ - "openai/gpt-4o-mini" \ - "https://preexisting.invalid" \ - "vertex_ai" \ - "" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "https://preexisting.invalid" -run_gate_case "default-fallback-order-fast-first" \ - "vertex_ai/missing-primary" \ - "" \ - "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'vertex_ai/gemini-2[.]5-pro' in [0-9]+s\\." \ - "2" \ - "vertex_ai/missing-primary|vertex_ai/gemini-2.5-pro" \ - "|" # Bug 13: All fallback models are the same as the primary model. # The gate should detect that no distinct fallback was tried and emit an ERROR. -run_gate_case "all-fallbacks-same-as-primary" \ - "vertex_ai/same-primary" \ - "vertex_ai/same-primary vertex_ai/same-primary" \ - "1" \ - "ERROR: All configured fallback models are the same as the primary model" \ - "1" \ - "vertex_ai/same-primary" \ - "" - # Bug 14: Timeout should fall back rather than emit a same-model retry message. -run_gate_case_allow_provider_signal "vertex-primary-timeout-retry-reason-message" \ - "vertex_ai/retry-timeout-primary" \ - "vertex_ai/fallback-one vertex_ai/fallback-two" \ - "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'vertex_ai/fallback-one' in [0-9]+s\\." \ - "2" \ - "vertex_ai/retry-timeout-primary|vertex_ai/fallback-one" \ - "|" \ - "vertex_ai" \ - "__DEFAULT__" \ - "" \ - "2" # Bug 14: Retry reason messages — rate-limit retry should say "due to rate limit". -run_gate_case_allow_provider_signal "vertex-primary-ratelimit-retry-reason-message" \ - "vertex_ai/retry-ratelimit-primary" \ - "vertex_ai/fallback-one vertex_ai/fallback-two" \ - "0" \ - "Retrying model 'vertex_ai/retry-ratelimit-primary' due to rate limit" \ - "2" \ - "vertex_ai/retry-ratelimit-primary|vertex_ai/retry-ratelimit-primary" \ - "|" \ - "vertex_ai" \ - "__DEFAULT__" \ - "" \ - "2" # Bug 14: Timing message — success should log elapsed time. run_gate_case "vertex-primary-success-timing-message" \ - "vertex_ai/ready-primary" \ + "orchestrator/free" \ "" \ "0" \ - "REGEX:Strix run succeeded for model 'vertex_ai/ready-primary' in [0-9]+s\\." \ + "REGEX:Strix run succeeded for model 'orchestrator/free' in [0-9]+s\\." \ "1" \ - "vertex_ai/ready-primary" \ - "" + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" # is_timeout_error() provider-context marker test: # Bare "Connection timed out" without any LLM provider marker should NOT @@ -11499,127 +9204,67 @@ run_gate_case "vertex-primary-success-timing-message" \ # Model name deliberately avoids containing any provider marker string # (litellm, openai, anthropic, VertexAI, vertex.ai, google.cloud). run_gate_case "bare-timeout-no-provider-marker" \ - "custom/bare-timeout-model" \ + "orchestrator/free" \ "" \ "1" \ "" \ "1" \ - "custom/bare-timeout-model" \ - "https://example.invalid" \ - "custom" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ - "1" + "0" # is_timeout_error() Tier 2: httpx.ReadTimeout + provider-context marker. # The timeout should be classified for fallback, not same-model retry. -run_gate_case_allow_provider_signal "httpx-read-timeout-with-provider-marker" \ - "vertex_ai/httpx-timeout-primary" \ - "vertex_ai/fallback-one" \ - "0" \ - "scan ok after httpx-timeout fallback" \ - "2" \ - "vertex_ai/httpx-timeout-primary|vertex_ai/fallback-one" \ - "|" \ - "vertex_ai" \ - "__DEFAULT__" \ - "" \ - "1" # Negative: httpx.ReadTimeout WITHOUT provider-context marker should NOT # be classified as a retryable timeout (the gate should treat it as a # non-recoverable scan failure). run_gate_case "httpx-read-timeout-no-provider-marker" \ - "custom/httpx-timeout-no-ctx" \ + "orchestrator/free" \ "" \ "1" \ - "non-recoverable error" \ + "failing closed without repository-authored retry or fallback allocation" \ "1" \ - "custom/httpx-timeout-no-ctx" \ - "https://example.invalid" \ - "custom" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ - "1" + "0" # is_timeout_error() Tier 2b: httpcore.ReadTimeout + provider-context marker. # Mirrors the httpx.ReadTimeout positive case above, but falls back immediately. -run_gate_case_allow_provider_signal "httpcore-read-timeout-with-provider-marker" \ - "vertex_ai/httpcore-timeout-primary" \ - "vertex_ai/fallback-one" \ - "0" \ - "scan ok after httpcore-timeout fallback" \ - "2" \ - "vertex_ai/httpcore-timeout-primary|vertex_ai/fallback-one" \ - "|" \ - "vertex_ai" \ - "__DEFAULT__" \ - "" \ - "1" # Negative: httpcore.ReadTimeout WITHOUT provider-context marker should NOT # be classified as a retryable timeout (the gate should treat it as a # non-recoverable scan failure). run_gate_case "httpcore-read-timeout-no-provider-marker" \ - "custom/httpcore-timeout-no-ctx" \ + "orchestrator/free" \ "" \ "1" \ - "non-recoverable error" \ + "failing closed without repository-authored retry or fallback allocation" \ "1" \ - "custom/httpcore-timeout-no-ctx" \ - "https://example.invalid" \ - "custom" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ - "1" + "0" # is_timeout_error() positive branch for "Connection timed out" + provider marker: # When "Connection timed out" appears alongside an LLM provider marker, the # gate should classify it as a timeout and move to fallback. -run_gate_case_allow_provider_signal "bare-timeout-with-provider-marker" \ - "vertex_ai/bare-timeout-primary" \ - "vertex_ai/fallback-one" \ - "0" \ - "scan ok after bare-timeout fallback" \ - "2" \ - "vertex_ai/bare-timeout-primary|vertex_ai/fallback-one" \ - "|" \ - "vertex_ai" \ - "__DEFAULT__" \ - "" \ - "1" # Bare "Connection timed out" + provider marker: primary fails once, # then gate falls back to fallback-one which succeeds. -run_gate_case_allow_provider_signal "bare-timeout-provider-marker-exhausted-fallback" \ - "vertex_ai/bare-timeout-exhaust-primary" \ - "vertex_ai/fallback-one" \ - "0" \ - "scan ok after bare-timeout-exhaust fallback" \ - "2" \ - "vertex_ai/bare-timeout-exhaust-primary|vertex_ai/fallback-one" \ - "|" \ - "vertex_ai" \ - "__DEFAULT__" \ - "" \ - "1" # Sticky INFRA_ERROR_DETECTED flag: first call hits rate-limit (infra error), # second call fails with a non-retryable error but leaves a partial LOW report. # The gate must refuse the below-threshold bypass because an infrastructure # error was detected during this pipeline run. -run_gate_case_allow_provider_signal "infra-error-sticky-flag" \ - "vertex_ai/sticky-flag-primary" \ - "" \ - "1" \ - "infrastructure errors occurred" \ - "3" \ - "vertex_ai/sticky-flag-primary|vertex_ai/sticky-flag-primary|vertex_ai/gemini-2.5-pro" \ - "||" \ - "vertex_ai" \ - "__DEFAULT__" \ - "" \ - "1" run_invalid_min_fail_severity_case run_required_input_file_outside_input_root_fails_closed_case "STRIX_LLM_FILE" @@ -11633,34 +9278,17 @@ run_symlink_report_case run_unsafe_target_path_case run_absolute_outside_target_path_case -run_gate_case_allow_provider_signal "slow-timeout" \ - "vertex_ai/slow-primary" \ - "" \ - "1" \ - "Strix run timed out after ${TIMEOUT_TEST_PROCESS_SECONDS}s." \ - "3" \ - "vertex_ai/slow-primary|vertex_ai/gemini-2.5-pro|vertex_ai/gemini-2.5-flash" \ - "||" \ - "vertex_ai" \ - "__DEFAULT__" \ - "" \ - "0" \ - "CRITICAL" \ - "0" \ - "" \ - "" \ - "$TIMEOUT_TEST_PROCESS_SECONDS" run_gate_case "timeout-disabled-success" \ - "vertex_ai/timeout-disabled-primary" \ + "orchestrator/free" \ "" \ "0" \ "scan ok with timeout disabled" \ "1" \ - "vertex_ai/timeout-disabled-primary" \ - "" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -11674,15 +9302,15 @@ run_timeout_cleanup_case run_total_timeout_case run_gate_case "pr-changed-scope-bounded" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "0" \ "scan ok with bounded changed-file scope" \ "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -11695,15 +9323,15 @@ run_gate_case "pr-changed-scope-bounded" \ "sync-module-system/smart-crawling-biz/src/main/java/org/empasy/sync/modules/system/controller/SysPositionController.java" run_gate_case "scan-working-directory-isolated" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "0" \ "scan ok with isolated Strix working directory" \ "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -11716,15 +9344,15 @@ run_gate_case "scan-working-directory-isolated" \ "backend/app/pg_introspect/introspect.py" run_gate_case "pr-python-scope-context" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "0" \ "scan ok with python dependency scope" \ "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -11737,15 +9365,15 @@ run_gate_case "pr-python-scope-context" \ "backend/api/emails.py" run_gate_case "pr-changed-scope-full" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "0" \ "Scoped pull request Strix scan to 3 changed file(s)." \ "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -11758,15 +9386,15 @@ run_gate_case "pr-changed-scope-full" \ $'sync-module-system/smart-crawling-biz/src/main/java/org/empasy/sync/modules/system/controller/SysPositionController.java\nsync-module-system/smart-crawling-playwright/src/main/java/org/empasy/sync/mcp/service/PlayWrightService.java\nsync-module-system/smart-crawling-biz/src/main/java/org/empasy/sync/modules/system/service/impl/SysUserServiceImpl.java' run_gate_case "pr-changed-scope-full-set" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "0" \ "scan ok with full configured PR scope" \ "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -11790,15 +9418,15 @@ for large_pr_index in $(seq 1 38); do done run_gate_case "pr-large-scope-full-set" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "0" \ "scan ok with large full PR scope" \ "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -11813,15 +9441,15 @@ run_gate_case "pr-large-scope-full-set" \ "12" run_gate_case "pr-changed-scope-includes-ci-dependency" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "0" \ "scan ok with CI support dependency" \ "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -11839,15 +9467,15 @@ run_gate_case "pr-changed-scope-includes-ci-dependency" \ # fuzz/fuzz_opencode_normalize_output.py duplicate). A PR that changes only # that fuzz target must still pull the normalizer module into scan scope. run_gate_case "pr-changed-scope-includes-opencode-normalizer" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "0" \ "scan ok with opencode normalizer support dependency" \ "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -11860,15 +9488,15 @@ run_gate_case "pr-changed-scope-includes-opencode-normalizer" \ "fuzz/fuzz_opencode_review_normalize_output.py" run_gate_case "pr-ci-test-harness-only-skip" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "0" \ "No scannable changed files in pull request; skipping Strix quick scan." \ "0" \ "" \ "" \ - "vertex_ai" \ - "__DEFAULT__" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -11881,15 +9509,15 @@ run_gate_case "pr-ci-test-harness-only-skip" \ "scripts/ci/test_strix_quick_gate.sh" run_gate_case "pr-deployment-scope-entrypoint-context" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "0" \ "scan ok with deployment entrypoint context" \ "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -11902,15 +9530,15 @@ run_gate_case "pr-deployment-scope-entrypoint-context" \ ".github/workflows/opencode-review.yml" run_gate_case "pr-rust-workspace-context" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "0" \ "scan ok with Rust workspace context" \ "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -11923,15 +9551,15 @@ run_gate_case "pr-rust-workspace-context" \ ".github/workflows/rust.yml" run_gate_case "pr-empty-diff-skip" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "0" \ "No scannable changed files in pull request; skipping Strix quick scan." \ "0" \ "" \ "" \ - "vertex_ai" \ - "__DEFAULT__" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -11944,15 +9572,15 @@ run_gate_case "pr-empty-diff-skip" \ "__SET_EMPTY__" run_gate_case "pr-baseline-critical-unchanged" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "0" \ "Strix findings are limited to unchanged files in this pull request; allowing pipeline continuation." \ "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -11965,15 +9593,15 @@ run_gate_case "pr-baseline-critical-unchanged" \ "sync-module-system/smart-crawling-biz/src/main/java/org/empasy/sync/modules/system/controller/SysPositionController.java" run_gate_case "pr-baseline-critical-absolute-target" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "0" \ "Strix findings are limited to unchanged files in this pull request; allowing pipeline continuation." \ "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -11986,15 +9614,15 @@ run_gate_case "pr-baseline-critical-absolute-target" \ "sync-module-system/smart-crawling-biz/src/main/java/org/empasy/sync/modules/system/controller/SysPositionController.java" run_gate_case "pr-baseline-critical-extensionless-dockerfile-target" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "0" \ "Strix findings are limited to unchanged files in this pull request; allowing pipeline continuation." \ "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -12007,15 +9635,15 @@ run_gate_case "pr-baseline-critical-extensionless-dockerfile-target" \ ".github/workflows/opencode-review.yml" run_gate_case "pr-baseline-critical-subdir-target" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "0" \ "Strix findings are limited to unchanged files in this pull request; allowing pipeline continuation." \ "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -12031,15 +9659,15 @@ run_gate_case "pr-baseline-critical-subdir-target" \ "1" run_gate_case "pr-baseline-critical-subdir-boxed-target" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "0" \ "Strix findings are limited to unchanged files in this pull request; allowing pipeline continuation." \ "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -12055,15 +9683,15 @@ run_gate_case "pr-baseline-critical-subdir-boxed-target" \ "1" run_gate_case "pr-baseline-critical-subdir-endpoint" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "0" \ "Strix findings are limited to unchanged files in this pull request; allowing pipeline continuation." \ "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -12079,15 +9707,15 @@ run_gate_case "pr-baseline-critical-subdir-endpoint" \ "1" run_gate_case "pr-baseline-critical-subdir-endpoint-bare-filename" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "0" \ "Strix findings are limited to unchanged files in this pull request; allowing pipeline continuation." \ "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -12103,15 +9731,15 @@ run_gate_case "pr-baseline-critical-subdir-endpoint-bare-filename" \ "1" run_gate_case "pr-baseline-critical-subdir-narrative-backticked-file" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "0" \ "Strix findings are limited to unchanged files in this pull request; allowing pipeline continuation." \ "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -12127,15 +9755,15 @@ run_gate_case "pr-baseline-critical-subdir-narrative-backticked-file" \ "1" run_gate_case "pr-critical-relative-path-escape-subdir-narrative-backticked-file" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "1" \ "Unable to map Strix findings to changed files; failing closed for pull request." \ "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -12151,15 +9779,15 @@ run_gate_case "pr-critical-relative-path-escape-subdir-narrative-backticked-file "1" run_gate_case "pr-critical-changed" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "1" \ "Strix finding intersects files changed in this pull request." \ "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -12172,15 +9800,15 @@ run_gate_case "pr-critical-changed" \ "sync-module-system/smart-crawling-biz/src/main/java/org/empasy/sync/modules/system/controller/SysPositionController.java" run_gate_case "pr-changed-file-nonintersecting-line" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "0" \ "Strix findings are limited to unchanged files in this pull request; allowing pipeline continuation." \ "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -12192,15 +9820,15 @@ run_gate_case "pr-changed-file-nonintersecting-line" \ "pull_request" run_gate_case "pr-critical-changed-bracketed-next-route" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "1" \ "Strix finding intersects files changed in this pull request." \ "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -12213,15 +9841,15 @@ run_gate_case "pr-critical-changed-bracketed-next-route" \ "frontend/src/app/labels/[slug]/page.tsx" run_gate_case "pr-critical-changed-xml-file-location" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "1" \ "Strix finding intersects files changed in this pull request." \ "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "MEDIUM" \ @@ -12234,15 +9862,15 @@ run_gate_case "pr-critical-changed-xml-file-location" \ "sync-module-system/smart-crawling-biz/src/main/java/org/empasy/sync/modules/system/controller/SysPositionController.java" run_gate_case "pr-critical-changed-xml-file-location-space" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "1" \ "Strix finding intersects files changed in this pull request." \ "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "MEDIUM" \ @@ -12255,15 +9883,15 @@ run_gate_case "pr-critical-changed-xml-file-location-space" \ "src/unsafe name.py" run_gate_case "pr-baseline-critical-narrative-backticked-service-file" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "0" \ "Strix findings are limited to unchanged files in this pull request; allowing pipeline continuation." \ "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -12276,15 +9904,15 @@ run_gate_case "pr-baseline-critical-narrative-backticked-service-file" \ "backend/services/email_client.py" run_gate_case "pr-critical-unmapped-arbitrary-backticked-service-file" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "1" \ "Unable to map Strix findings to changed files; failing closed for pull request." \ "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -12297,15 +9925,15 @@ run_gate_case "pr-critical-unmapped-arbitrary-backticked-service-file" \ "backend/services/email_client.py" run_gate_case "pr-critical-changed-absolute-target" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "1" \ "Strix finding intersects files changed in this pull request." \ "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -12318,15 +9946,15 @@ run_gate_case "pr-critical-changed-absolute-target" \ "sync-module-system/smart-crawling-playwright/src/main/java/org/empasy/sync/mcp/service/PlayWrightService.java" run_gate_case "pr-critical-changed-internal-dotdir-target" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "1" \ "Strix finding intersects files changed in this pull request." \ "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -12339,15 +9967,15 @@ run_gate_case "pr-critical-changed-internal-dotdir-target" \ ".github/workflows/opencode-review.yml" run_gate_case "pr-critical-changed-json-target" \ - "vertex_ai/gemini-2.5-pro" \ + "orchestrator/free" \ "" \ "1" \ "Strix finding intersects files changed in this pull request." \ "1" \ - "vertex_ai/gemini-2.5-pro" \ - "" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "MEDIUM" \ @@ -12360,15 +9988,15 @@ run_gate_case "pr-critical-changed-json-target" \ "frontend/src/components/CalendarLayout.tsx" run_gate_case "pr-critical-changed-subdir-target" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "1" \ "Strix finding intersects files changed in this pull request." \ "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -12384,15 +10012,15 @@ run_gate_case "pr-critical-changed-subdir-target" \ "1" run_gate_case "pr-critical-changed-subdir-endpoint" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "1" \ "Strix finding intersects files changed in this pull request." \ "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -12408,15 +10036,15 @@ run_gate_case "pr-critical-changed-subdir-endpoint" \ "1" run_gate_case "pr-critical-path-escape-subdir-target" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "1" \ "Unable to map Strix findings to changed files; failing closed for pull request." \ "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -12432,15 +10060,15 @@ run_gate_case "pr-critical-path-escape-subdir-target" \ "1" run_gate_case "pr-critical-unmapped" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "1" \ "Unable to map Strix findings to changed files; failing closed for pull request." \ "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -12453,15 +10081,15 @@ run_gate_case "pr-critical-unmapped" \ "sync-module-system/smart-crawling-biz/src/main/java/org/empasy/sync/modules/system/controller/SysPositionController.java" run_gate_case "pr-critical-unmapped-narrative-target" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "1" \ "Unable to map Strix findings to changed files; failing closed for pull request." \ "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -12474,15 +10102,15 @@ run_gate_case "pr-critical-unmapped-narrative-target" \ "sync-module-system/smart-crawling-playwright/src/main/java/org/empasy/sync/mcp/service/PlayWrightService.java" run_gate_case "pr-critical-unmapped-other-workspace-repo" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "1" \ "Unable to map Strix findings to changed files; failing closed for pull request." \ "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -12495,15 +10123,15 @@ run_gate_case "pr-critical-unmapped-other-workspace-repo" \ "sync-module-system/smart-crawling-playwright/src/main/java/org/empasy/sync/mcp/service/PlayWrightService.java" run_gate_case "pr-critical-manifest-only-pom" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "1" \ "Strix changed-manifest threshold finding requires package and CVE remediation; pull-request-controlled SCA workflow results cannot override model evidence, so the scan is failing closed." \ "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -12516,15 +10144,15 @@ run_gate_case "pr-critical-manifest-only-pom" \ "pom.xml" run_gate_case "pr-critical-manifest-only-pom-test-override" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "1" \ "Strix changed-manifest threshold finding requires package and CVE remediation; pull-request-controlled SCA workflow results cannot override model evidence, so the scan is failing closed." \ "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -12541,15 +10169,15 @@ run_gate_case "pr-critical-manifest-only-pom-test-override" \ "passed" run_gate_case "pr-critical-manifest-only-pom-same-head-different-pr" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "1" \ "Strix changed-manifest threshold finding requires package and CVE remediation; pull-request-controlled SCA workflow results cannot override model evidence, so the scan is failing closed." \ "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -12568,15 +10196,15 @@ run_gate_case "pr-critical-manifest-only-pom-same-head-different-pr" \ '{"workflow_runs":[{"id":201,"name":"Dependency review","path":".github/workflows/dependency-review.yml","head_sha":"test-head-sha","status":"completed","conclusion":"success","pull_requests":[{"number":456}]},{"id":202,"name":"OSV-Scanner","path":".github/workflows/osvscanner.yml","head_sha":"test-head-sha","status":"completed","conclusion":"success","pull_requests":[{"number":456}]}]}' run_gate_case "pr-critical-manifest-only-pom-current-pr-authoritative" \ - "openai/gpt-4o-mini" \ + "orchestrator/free" \ "" \ "1" \ "Strix changed-manifest threshold finding requires package and CVE remediation; pull-request-controlled SCA workflow results cannot override model evidence, so the scan is failing closed." \ "1" \ - "openai/gpt-4o-mini" \ - "https://example.invalid" \ - "vertex_ai" \ - "__DEFAULT__" \ + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" \ "" \ "0" \ "CRITICAL" \ @@ -12594,118 +10222,14 @@ run_gate_case "pr-critical-manifest-only-pom-current-pr-authoritative" \ "123" \ '{"workflow_runs":[{"id":301,"name":"Dependency review","path":".github/workflows/dependency-review.yml","head_sha":"test-head-sha","status":"completed","conclusion":"success","pull_requests":[{"number":123}]},{"id":302,"name":"OSV-Scanner","path":".github/workflows/osvscanner.yml","head_sha":"test-head-sha","status":"completed","conclusion":"success","pull_requests":[{"number":123}]}]}' -run_gate_case_allow_provider_signal "pr-critical-manifest-only-pom-after-fallback-authoritative" \ - "vertex_ai/timeout-primary" \ - "vertex_ai/fallback-one" \ - "1" \ - "Strix changed-manifest threshold finding requires package and CVE remediation; pull-request-controlled SCA workflow results cannot override model evidence, so the scan is failing closed." \ - "2" \ - "vertex_ai/timeout-primary|vertex_ai/fallback-one" \ - "|" \ - "vertex_ai" \ - "__DEFAULT__" \ - "" \ - "0" \ - "CRITICAL" \ - "0" \ - "" \ - "" \ - "1200" \ - "0" \ - "pull_request" \ - "pom.xml" \ - "" \ - "" \ - "0" \ - "" \ - "123" \ - '{"workflow_runs":[{"id":401,"name":"Dependency review","path":".github/workflows/dependency-review.yml","head_sha":"test-head-sha","status":"completed","conclusion":"success","pull_requests":[{"number":123}]},{"id":402,"name":"OSV-Scanner","path":".github/workflows/osvscanner.yml","head_sha":"test-head-sha","status":"completed","conclusion":"success","pull_requests":[{"number":123}]}]}' -run_gate_case_allow_provider_signal "pr-critical-manifest-only-pom-console-only-after-fallback-authoritative" \ - "vertex_ai/timeout-primary" \ - "vertex_ai/fallback-one" \ - "1" \ - "Strix changed-manifest threshold finding requires package and CVE remediation; pull-request-controlled SCA workflow results cannot override model evidence, so the scan is failing closed." \ - "2" \ - "vertex_ai/timeout-primary|vertex_ai/fallback-one" \ - "|" \ - "vertex_ai" \ - "__DEFAULT__" \ - "" \ - "0" \ - "CRITICAL" \ - "0" \ - "" \ - "" \ - "1200" \ - "0" \ - "pull_request" \ - "pom.xml" \ - "" \ - "" \ - "0" \ - "" \ - "123" \ - '{"workflow_runs":[{"id":403,"name":"Dependency review","path":".github/workflows/dependency-review.yml","head_sha":"test-head-sha","status":"completed","conclusion":"success","pull_requests":[{"number":123}]},{"id":404,"name":"OSV-Scanner","path":".github/workflows/osvscanner.yml","head_sha":"test-head-sha","status":"completed","conclusion":"success","pull_requests":[{"number":123}]}]}' -run_gate_case_allow_provider_signal "pr-critical-manifest-only-pom-console-target-only-after-fallback-authoritative" \ - "vertex_ai/timeout-primary" \ - "vertex_ai/fallback-one" \ - "1" \ - "Strix changed-manifest threshold finding requires package and CVE remediation; pull-request-controlled SCA workflow results cannot override model evidence, so the scan is failing closed." \ - "2" \ - "vertex_ai/timeout-primary|vertex_ai/fallback-one" \ - "|" \ - "vertex_ai" \ - "__DEFAULT__" \ - "" \ - "0" \ - "CRITICAL" \ - "0" \ - "" \ - "" \ - "1200" \ - "0" \ - "pull_request" \ - "pom.xml" \ - "" \ - "" \ - "0" \ - "" \ - "123" \ - '{"workflow_runs":[{"id":405,"name":"Dependency review","path":".github/workflows/dependency-review.yml","head_sha":"test-head-sha","status":"completed","conclusion":"success","pull_requests":[{"number":123}]},{"id":406,"name":"OSV-Scanner","path":".github/workflows/osvscanner.yml","head_sha":"test-head-sha","status":"completed","conclusion":"success","pull_requests":[{"number":123}]}]}' -run_gate_case_allow_provider_signal "pr-low-markdown-plus-console-critical-manifest-after-fallback-authoritative" \ - "vertex_ai/timeout-primary" \ - "vertex_ai/fallback-one" \ - "1" \ - "Strix changed-manifest threshold finding requires package and CVE remediation; pull-request-controlled SCA workflow results cannot override model evidence, so the scan is failing closed." \ - "2" \ - "vertex_ai/timeout-primary|vertex_ai/fallback-one" \ - "|" \ - "vertex_ai" \ - "__DEFAULT__" \ - "" \ - "0" \ - "CRITICAL" \ - "0" \ - "" \ - "" \ - "1200" \ - "0" \ - "pull_request" \ - "pom.xml" \ - "" \ - "" \ - "0" \ - "" \ - "123" \ - '{"workflow_runs":[{"id":405,"name":"Dependency review","path":".github/workflows/dependency-review.yml","head_sha":"test-head-sha","status":"completed","conclusion":"success","pull_requests":[{"number":123}]},{"id":406,"name":"OSV-Scanner","path":".github/workflows/osvscanner.yml","head_sha":"test-head-sha","status":"completed","conclusion":"success","pull_requests":[{"number":123}]}]}' run_missing_config_case "missing-strix-llm" "" "dummy" "ERROR: STRIX_LLM_FILE must reference a regular file containing the model." -run_missing_config_case "missing-llm-api-key" "openai/gpt-5.4" "" "ERROR: LLM_API_KEY_FILE must reference a regular file containing the API key." +run_missing_config_case "missing-llm-api-key" "orchestrator/free" "" "ERROR: LLM_API_KEY_FILE must reference a regular file containing the API key." run_missing_config_case "whitespace-only-strix-llm" " " "dummy" "ERROR: STRIX_LLM_FILE must contain a non-empty model value." -run_missing_config_case "whitespace-only-llm-api-key" "openai/gpt-5.4" $'\t ' "ERROR: LLM_API_KEY_FILE must contain a non-empty API key." +run_missing_config_case "whitespace-only-llm-api-key" "orchestrator/free" $'\t ' "ERROR: LLM_API_KEY_FILE must contain a non-empty API key." run_strix_llm_file_command_substitution_literal_case run_vertex_without_llm_api_key_case run_vertex_with_llm_api_key_file_does_not_forward_case @@ -12852,17 +10376,11 @@ assert_vertex_extract "non-vertex-passthrough" "deepseek/models/deepseek-r1" "de assert_vertex_extract "plain-model-passthrough" "gemini-2.5-pro" "gemini-2.5-pro" # Explicit Vertex resource paths require an explicit Vertex provider context. -assert_normalized_model \ +assert_normalize_model_rejected \ "vertex-resource-ignores-nonvertex-default-provider" \ "projects/my-proj/locations/us-central1/publishers/google/models/gemini-2.5-pro" \ - "vertex_ai" \ - "vertex_ai/gemini-2.5-pro" - -assert_model_requires_vertex_auth "explicit-vertex" "vertex_ai/gemini-2.5-pro" "gemini" "0" -assert_model_requires_vertex_auth "explicit-vertex-beta" "vertex_ai_beta/gemini-2.5-pro" "gemini" "0" -assert_model_requires_vertex_auth "vertex-resource-path" "projects/my-proj/locations/us-central1/models/gemini-2.5-pro" "vertex_ai" "0" -assert_model_requires_vertex_auth "implicit-vertex-default" "gemini-2.5-pro" "vertex_ai" "0" -assert_model_requires_vertex_auth "nonvertex-provider" "gemini/gemini-2.5-pro" "gemini" "1" + "vertex_ai" + assert_normalize_model_rejected "bare-models-openai-context" "models/attacker-selected" "openai" assert_normalize_model_rejected "bare-models-empty-context" "models/attacker-selected" "" @@ -12875,247 +10393,126 @@ run_gate_case "github-models-model-prefix-requires-api-base" \ "openai/openai/gpt-5.4" \ "" \ "2" \ - "GitHub Models Strix scans require LLM_API_BASE_FILE" \ + "ERROR: Strix model must be orchestrator/free through contextual-orchestrator; direct provider/model routes are forbidden" \ "0" \ "" \ "" \ - "openai" \ - "" + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" run_gate_case "custom-openai-compatible-preserves-effort" \ "openai-direct/gpt-5.4" \ "" \ + "2" \ + "ERROR: Strix model must be orchestrator/free through contextual-orchestrator; direct provider/model routes are forbidden" \ "0" \ - "scan ok" \ - "1" \ - "openai/gpt-5.4" \ - "https://compatible.example/v1" \ - "openai" \ - "https://compatible.example/v1" + "" \ + "" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" run_gate_case "github-models-api-base-rejected-for-direct-openai" \ "openai/o4-mini" \ "" \ "2" \ - "LLM_API_BASE may route through GitHub Models only when STRIX_LLM uses a GitHub Models-compatible model" \ + "ERROR: Strix model must be orchestrator/free through contextual-orchestrator; direct provider/model routes are forbidden" \ "0" \ "" \ "" \ - "openai" \ - "https://models.github.ai/inference" + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" run_gate_case "github-models-openai-gpt-requires-api-base" \ "openai/gpt-5" \ "" \ "2" \ - "GitHub Models Strix scans require LLM_API_BASE_FILE" \ + "ERROR: Strix model must be orchestrator/free through contextual-orchestrator; direct provider/model routes are forbidden" \ "0" \ "" \ "" \ - "openai" \ - "" + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" run_gate_case "direct-openai-gpt-does-not-require-github-models-api-base" \ "openai_direct/gpt-5.4" \ "" \ + "2" \ + "ERROR: Strix model must be orchestrator/free through contextual-orchestrator; direct provider/model routes are forbidden" \ "0" \ - "scan ok" \ - "1" \ - "openai/gpt-5.4" \ - "" \ - "openai" \ - "" + "" \ + "" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" run_gate_case "github-models-model-prefix-with-api-base-succeeds" \ "openai/gpt-5" \ "" \ + "2" \ + "ERROR: Strix model must be orchestrator/free through contextual-orchestrator; direct provider/model routes are forbidden" \ "0" \ - "scan ok" \ - "1" \ - "openai/gpt-5" \ - "https://models.github.ai/inference" \ - "openai" \ - "https://models.github.ai/inference" + "" \ + "" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" run_gate_case "github-models-meta-prefix-with-api-base-succeeds" \ "openai/meta/test-github-model" \ "" \ + "2" \ + "ERROR: Strix model must be orchestrator/free through contextual-orchestrator; direct provider/model routes are forbidden" \ "0" \ - "scan ok" \ - "1" \ - "openai/meta/test-github-model" \ - "https://models.github.ai/inference" \ - "openai" \ - "https://models.github.ai/inference" + "" \ + "" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" run_gate_case "github-models-mistral-prefix-with-api-base-succeeds" \ "openai/mistral-ai/test-github-model" \ "" \ - "0" \ - "scan ok" \ - "1" \ - "openai/mistral-ai/test-github-model" \ - "https://models.github.ai/inference" \ - "openai" \ - "https://models.github.ai/inference" - -run_gate_case "github-models-fallback-requires-api-base" \ - "vertex_ai/missing-primary" \ - "openai/openai/gpt-5.4" \ "2" \ - "GitHub Models Strix scans require LLM_API_BASE_FILE" \ - "1" \ - "vertex_ai/missing-primary" \ - "" \ - "vertex_ai" \ - "" - -run_gate_case "github-models-fallback-success" \ - "vertex_ai/missing-primary" \ - "github_models/deepseek/deepseek-v3-0324 github_models/deepseek/deepseek-r1-0528" \ + "ERROR: Strix model must be orchestrator/free through contextual-orchestrator; direct provider/model routes are forbidden" \ "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'github_models/deepseek/deepseek-v3-0324' in [0-9]+s\\." \ - "2" \ - "vertex_ai/missing-primary|openai/deepseek/deepseek-v3-0324" \ - "|https://models.github.ai/inference" \ - "vertex_ai" \ - "https://models.github.ai/inference" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ "" \ "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - 0 + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" -run_gate_case "github-models-token-limit-fallback-success" \ - "openai/gpt-5" \ +run_gate_case "github-models-fallback-requires-api-base" \ + "vertex_ai/missing-primary" \ "" \ - "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'github_models/deepseek/deepseek-v3-0324' in [0-9]+s\\." \ "2" \ - "openai/gpt-5|openai/deepseek/deepseek-v3-0324" \ - "https://models.github.ai/inference|https://models.github.ai/inference" \ - "openai" \ - "https://models.github.ai/inference" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ + "ERROR: Strix model must be orchestrator/free through contextual-orchestrator; direct provider/model routes are forbidden" \ + "0" \ "" \ "" \ - "github_models/deepseek/deepseek-v3-0324 github_models/deepseek/deepseek-r1-0528" + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" + + # Direct-OpenAI primary hits a quota/rate-limit error and falls back to a # GitHub Models candidate, switching both the API base and the API key per # model (the fake strix asserts the key swap and exits nonzero on a leak). -run_gate_case "openai-direct-quota-github-models-fallback-success" \ - "openai_direct/gpt-5.4" \ - "" \ - "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'github_models/openai/o3' in [0-9]+s\\." \ - "2" \ - "openai/gpt-5.4|openai/o3" \ - "|https://models.github.ai/inference" \ - "vertex_ai" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "github_models/openai/o3" -run_gate_case "github-models-fallback-success-deepseek-v3" \ - "vertex_ai/missing-primary" \ - "github_models/deepseek/deepseek-r1-0528 github_models/deepseek/deepseek-v3-0324" \ - "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'github_models/deepseek/deepseek-v3-0324' in [0-9]+s\\." \ - "3" \ - "vertex_ai/missing-primary|openai/deepseek/deepseek-r1-0528|openai/deepseek/deepseek-v3-0324" \ - "|https://models.github.ai/inference|https://models.github.ai/inference" \ - "vertex_ai" \ - "https://models.github.ai/inference" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - "" \ - 0 # Endpoint only exists in excluded directories (.git/, node_modules/). Even if # the source does not corroborate it, a threshold report remains blocking and # requires human remediation/triage rather than silent fallback. run_gate_case "endpoint-in-excluded-dir" \ - "vertex_ai/excluded-dir-primary" \ - "vertex_ai/fallback-one vertex_ai/fallback-two" \ + "orchestrator/free" \ + "" \ "1" \ "Unable to map Strix findings to changed files; failing closed for pull request." \ "1" \ - "vertex_ai/excluded-dir-primary" \ - "" + "openai/orchestrator/free" \ + "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" \ + "http://127.0.0.1:18080/v1" # Whitespace-only fallback models: STRIX_VERTEX_FALLBACK_MODELS set to " ". # This bypasses the :- default but produces an empty array from read -r -a. # The gate should emit "No fallback models configured" (not the misleading # "All configured fallback models are the same as the primary model"). -run_gate_case "empty-fallback-models" \ - "vertex_ai/empty-fb-primary" \ - " " \ - "1" \ - "No fallback models configured" \ - "1" \ - "vertex_ai/empty-fb-primary" \ - "" - if [ "$FAILURES" -ne 0 ]; then echo "test_strix_quick_gate: ${FAILURES} failure(s)" >&2 exit 1 diff --git a/tests/test_required_workflow_queue_contract.py b/tests/test_required_workflow_queue_contract.py index 803d43ab59..a2c1fdca3b 100644 --- a/tests/test_required_workflow_queue_contract.py +++ b/tests/test_required_workflow_queue_contract.py @@ -1421,7 +1421,7 @@ def test_strix_provider_outage_without_findings_is_typed_non_passing() -> None: assert "zero_vulnerabilities_signal" not in workflow assert "Vulnerabilities[[:space:]]+[1-9]" in workflow assert "(^|[^A-Za-z0-9_])severity[[:space:]]*:" in workflow - assert "STRIX_FAIL_ON_MIN_SEVERITY: MEDIUM" in workflow + assert "STRIX_FAIL_ON_MIN_SEVERITY" not in workflow assert "::error title=STRIX_PROVIDER_UNAVAILABLE::" in workflow assert 'exit "$strix_rc"' in workflow assert "Treating as a neutral skip" not in workflow diff --git a/tests/test_strix_caido_bootstrap_timing_retry.py b/tests/test_strix_caido_bootstrap_timing_retry.py index a60b9d801b..32b6bacd97 100644 --- a/tests/test_strix_caido_bootstrap_timing_retry.py +++ b/tests/test_strix_caido_bootstrap_timing_retry.py @@ -5,9 +5,9 @@ 10-attempt loginAsGuest retry budget. A slow CI runner can exceed that budget before the local proxy is reachable, even though the penetration test itself never started and no security evidence was produced or lost. -This is local sandbox/container boot timing, not tied to any one LLM model, -so it must be retried same-model rather than treated as grounds to switch -models or as a genuine, non-backend scan failure. +This is local sandbox/container boot timing, not a genuine, non-backend scan +failure: it must classify as an infrastructure error so the gate fails +closed without repository-authored retry or fallback allocation. """ from __future__ import annotations @@ -74,7 +74,7 @@ def _classifies_as_caido_bootstrap_timing(log_text: str) -> bool: class StrixCaidoBootstrapTimingRetryTests(unittest.TestCase): - """Protect same-model retry for the upstream sandbox boot race.""" + """Protect infrastructure classification for the upstream sandbox boot race.""" def test_observed_caido_login_failure_is_retryable(self) -> None: """Recognize the exact loginAsGuest failure observed in required CI.""" @@ -103,14 +103,13 @@ def test_login_failure_without_the_connect_evidence_is_not_retryable(self) -> No log = "loginAsGuest failed after 10 attempts: unknown reason\n" self.assertFalse(_classifies_as_caido_bootstrap_timing(log)) - def test_wired_into_same_model_retry_and_infrastructure_not_cross_model( - self, - ) -> None: - """Retry the same model; do not treat this as a reason to switch models. + def test_wired_into_infrastructure_detection(self) -> None: + """The classifier feeds the fail-closed infrastructure-error signal. - Switching LLM models cannot change how long the local sandbox - container takes to boot, so cross-model fallback (`is_model_retryable_error`) - must stay untouched by this classifier. + No repository-authored retry or cross-model fallback exists anymore: + contextual-orchestrator's `orchestrator/free` gateway owns provider + discovery and failover, so this classifier only needs to be wired + into infrastructure-error detection. """ gate_source = STRIX_GATE.read_text(encoding="utf-8") @@ -118,24 +117,8 @@ def test_wired_into_same_model_retry_and_infrastructure_not_cross_model( gate_source, "has_detected_infrastructure_error", ) - same_model_retry = _function_block( - gate_source, - "is_transient_same_model_retry_error", - ) - cross_model_retry = _function_block(gate_source, "is_model_retryable_error") self.assertIn("is_caido_bootstrap_timing_error", infrastructure) - self.assertIn("is_caido_bootstrap_timing_error", same_model_retry) - self.assertNotIn("is_caido_bootstrap_timing_error", cross_model_retry) - - def test_retry_reason_is_logged_for_operators(self) -> None: - """Keep the diagnostic retry-reason message in sync with the classifier.""" - - gate_source = STRIX_GATE.read_text(encoding="utf-8") - self.assertIn( - 'retry_reason="Caido sandbox bootstrap timing"', - gate_source, - ) if __name__ == "__main__": diff --git a/tests/test_strix_model_behavior_error.py b/tests/test_strix_model_behavior_error.py index 3d0fd0bc42..8956d152a6 100644 --- a/tests/test_strix_model_behavior_error.py +++ b/tests/test_strix_model_behavior_error.py @@ -152,23 +152,21 @@ def test_agents_sdk_tool_protocol_failure_is_retryable(self) -> None: ) self.assertTrue(_classifies_as_model_behavior_error(log)) - def test_behavior_error_skips_same_model_and_enters_fallback(self) -> None: - """Wire the classifier into infrastructure and cross-model fallback.""" + def test_behavior_error_wired_into_infrastructure_detection(self) -> None: + """The classifier feeds the fail-closed infrastructure-error signal. + + No repository-authored retry or cross-model fallback exists anymore: + contextual-orchestrator's `orchestrator/free` gateway owns provider + discovery and failover. + """ gate_source = STRIX_GATE.read_text(encoding="utf-8") infrastructure = _function_block( gate_source, "has_detected_infrastructure_error", ) - retryable = _function_block(gate_source, "is_model_retryable_error") - same_model_retry = _function_block( - gate_source, - "is_transient_same_model_retry_error", - ) self.assertIn("is_model_behavior_error", infrastructure) - self.assertIn("is_model_behavior_error", retryable) - self.assertNotIn("is_model_behavior_error", same_model_retry) def test_outer_workflow_classifies_zero_finding_protocol_flake(self) -> None: """Empty scans that hit ModelBehaviorError receive typed diagnostics.""" diff --git a/tests/test_strix_no_heuristic_retry_contract.py b/tests/test_strix_no_heuristic_retry_contract.py new file mode 100644 index 0000000000..9a6f0b6639 --- /dev/null +++ b/tests/test_strix_no_heuristic_retry_contract.py @@ -0,0 +1,46 @@ +"""Contracts removing repository-authored retry/test-time-compute heuristics from Strix.""" + +from __future__ import annotations + +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +WORKFLOW = ROOT / ".github" / "workflows" / "strix.yml" +GATE = ROOT / "scripts" / "ci" / "strix_quick_gate.sh" + + +def test_central_strix_does_not_allocate_repository_authored_retry_compute() -> None: + """The central review path gets one execution; provider failover belongs to the orchestrator.""" + workflow = WORKFLOW.read_text(encoding="utf-8") + forbidden = ( + "STRIX_LLM_MAX_RETRIES:", + "STRIX_TRANSIENT_RETRY_PER_MODEL:", + "STRIX_TRANSIENT_RETRY_BACKOFF_SECONDS:", + "STRIX_GATE_RETRY_BACKOFF_SECONDS", + "strix_gate_attempt", + "backoff_seconds=", + "retrying after ${backoff_seconds}s backoff", + "reached the retry limit", + ) + for token in forbidden: + assert token not in workflow, token + + assert 'bash "$TRUSTED_STRIX_GATE" 2>&1 | tee "$strix_terminal_log"' in workflow + assert "provider/backend was unavailable" in workflow + + +def test_strix_gate_has_no_same_model_retry_allocator() -> None: + """The reusable gate must fail closed instead of allocating another model execution.""" + gate = GATE.read_text(encoding="utf-8") + forbidden = ( + "STRIX_TRANSIENT_RETRY_PER_MODEL", + "STRIX_TRANSIENT_RETRY_BACKOFF_SECONDS", + "run_strix_with_transient_retry()", + "github_models_rate_limit_should_skip_same_model_retry()", + "is_transient_same_model_retry_error()", + "Retrying model '$model'", + ) + for token in forbidden: + assert token not in gate, token + + assert 'run_strix_once "$PRIMARY_MODEL" || primary_scan_rc=$?' in gate diff --git a/tests/test_strix_no_heuristic_severity_contract.py b/tests/test_strix_no_heuristic_severity_contract.py new file mode 100644 index 0000000000..643ea5e18a --- /dev/null +++ b/tests/test_strix_no_heuristic_severity_contract.py @@ -0,0 +1,27 @@ +"""Contracts removing hand-selected Strix severity thresholds from merge decisions.""" + +from __future__ import annotations + +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +WORKFLOW = ROOT / ".github" / "workflows" / "strix.yml" +GATE = ROOT / "scripts" / "ci" / "strix_quick_gate.sh" + + +def test_required_strix_workflow_has_no_hand_selected_severity_threshold() -> None: + """A current vulnerability artifact is evidence; CI must not invent a cutoff.""" + workflow = WORKFLOW.read_text(encoding="utf-8") + assert "STRIX_FAIL_ON_MIN_SEVERITY" not in workflow + assert "at or above" not in workflow + assert "below-threshold" not in workflow + + +def test_successful_strix_execution_fails_closed_on_any_current_vulnerability_artifact() -> None: + """The reusable gate must not convert severity labels to an admission score.""" + gate = GATE.read_text(encoding="utf-8") + assert ( + "Current Strix vulnerability report exists; failing closed without a repository-authored severity threshold." + in gate + ) + assert "Strix exited successfully but emitted a vulnerability at or above" not in gate diff --git a/tests/test_strix_nvidia_nim_not_found_fallback.py b/tests/test_strix_nvidia_nim_not_found_fallback.py index ba8344455b..4444e12f42 100644 --- a/tests/test_strix_nvidia_nim_not_found_fallback.py +++ b/tests/test_strix_nvidia_nim_not_found_fallback.py @@ -1,9 +1,10 @@ -"""Regression contract for NVIDIA NIM model retirement and hosted 404 fallback. +"""Regression contract for NVIDIA NIM model retirement and hosted 404 classification. -The central Strix workflow must not turn a provider-side model-catalog 404 into a -security finding or retry the same unavailable model. It must move to another -approved free NVIDIA NIM candidate before using the existing GitHub Models -fallbacks, while ordinary application 404 output remains non-retryable. +The central Strix workflow must not turn a provider-side model-catalog 404 +into a security finding: it must classify as an infrastructure error and +fail closed, delegating provider discovery and failover to +contextual-orchestrator's `orchestrator/free` gateway, while ordinary +application 404 output remains non-retryable. """ from __future__ import annotations @@ -167,23 +168,21 @@ def test_provider_literal_without_litellm_error_is_not_retryable(self) -> None: log = "source literal: Nvidia_nimException Error code: 404\n" self.assertFalse(_classifies_as_nvidia_not_found(log)) - def test_not_found_skips_same_model_and_enters_cross_model_fallback(self) -> None: - """Wire the classifier only into infrastructure and model fallback.""" + def test_not_found_wired_into_infrastructure_detection(self) -> None: + """The classifier feeds the fail-closed infrastructure-error signal. + + No repository-authored retry or cross-model fallback exists anymore: + contextual-orchestrator's `orchestrator/free` gateway owns provider + discovery and failover. + """ gate_source = STRIX_GATE.read_text(encoding="utf-8") infrastructure = _function_block( gate_source, "has_detected_infrastructure_error", ) - retryable = _function_block(gate_source, "is_model_retryable_error") - same_model_retry = _function_block( - gate_source, - "is_transient_same_model_retry_error", - ) self.assertIn("is_nvidia_nim_not_found_error", infrastructure) - self.assertIn("is_nvidia_nim_not_found_error", retryable) - self.assertNotIn("is_nvidia_nim_not_found_error", same_model_retry) def test_workflow_routes_all_scans_through_contextual_orchestrator(self) -> None: """The central workflow owns one gateway route and no provider override.""" diff --git a/tests/test_strix_orchestrator_free_only.py b/tests/test_strix_orchestrator_free_only.py new file mode 100644 index 0000000000..dc10179c3b --- /dev/null +++ b/tests/test_strix_orchestrator_free_only.py @@ -0,0 +1,56 @@ +"""Contracts proving Strix cannot normalize a direct-provider model route.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +MODEL_UTILS = REPOSITORY_ROOT / "scripts" / "ci" / "strix_model_utils.sh" + + +def _normalize(model: str) -> subprocess.CompletedProcess[str]: + """Run the production normalization helper for one model identifier.""" + return subprocess.run( + [ + "bash", + "-c", + 'set -euo pipefail; . "$1"; DEFAULT_PROVIDER=""; normalize_model "$2"', + "bash", + str(MODEL_UTILS), + model, + ], + check=False, + capture_output=True, + text=True, + ) + + +def test_strix_accepts_only_the_governed_free_virtual_model_ids() -> None: + """Both accepted spellings resolve to the same governed free pool boundary.""" + for model in ( + "orchestrator/free", + "contextual-orchestrator/orchestrator/free", + ): + result = _normalize(model) + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == model + + +def test_strix_rejects_every_direct_provider_model_before_execution() -> None: + """A Strix model cannot name a provider or concrete model outside the gateway.""" + for model in ( + "openai-direct/gpt-5.4", + "openai_direct/gpt-5.4", + "openai/gpt-5", + "openrouter/openai/gpt-oss-120b:free", + "nvidia_nim/meta/llama-3.3-70b-instruct", + "github_models/openai/gpt-5", + "vertex_ai/gemini-2.5-pro", + "gemini/gemini-2.5-pro", + "gpt-5", + ): + result = _normalize(model) + assert result.returncode == 2, (model, result.stdout, result.stderr) + assert "Strix model must be orchestrator/free" in result.stderr + assert result.stdout == ""