From 2cd1d34e2027fec9f93a5dc522046d507276ef7a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 06:23:54 +0000 Subject: [PATCH 1/3] fix(strix): classify ModelBehaviorError as retryable protocol evidence Treat the exact PascalCase agent-protocol exception as cross-model fallback evidence when the scan reports no vulnerabilities. Keep Vulnerabilities [1-9] and severity markers fail-closed. --- .../strix-changed-path-quality-ci.yml | 6 +- .github/workflows/strix.yml | 2 +- CHANGELOG.md | 3 + docs/doctoring/strix-model-behavior-error.md | 50 +++++ scripts/ci/strix_quick_gate.sh | 21 ++ tests/test_strix_model_behavior_error.py | 192 ++++++++++++++++++ ...st_strix_quality_timeout_fixture_budget.py | 2 + 7 files changed, 274 insertions(+), 2 deletions(-) create mode 100644 docs/doctoring/strix-model-behavior-error.md create mode 100644 tests/test_strix_model_behavior_error.py diff --git a/.github/workflows/strix-changed-path-quality-ci.yml b/.github/workflows/strix-changed-path-quality-ci.yml index 75e9b7d8e2..31924910a3 100644 --- a/.github/workflows/strix-changed-path-quality-ci.yml +++ b/.github/workflows/strix-changed-path-quality-ci.yml @@ -5,12 +5,16 @@ on: branches: [main] paths: - ".github/workflows/strix-changed-path-quality-ci.yml" + - ".github/workflows/strix.yml" - "CHANGELOG.md" - "docs/doctoring/strix-legal-git-paths.md" + - "docs/doctoring/strix-model-behavior-error.md" - "docs/doctoring/strix-quality-timeout-fixtures.md" - "scripts/ci/strix_quick_gate.sh" - "scripts/ci/test_strix_quick_gate.sh" - "tests/test_strix_changed_path_policy.py" + - "tests/test_strix_model_behavior_error.py" + - "tests/test_strix_nvidia_nim_not_found_fallback.py" - "tests/test_strix_workflow_dependency_hashes.py" - "tests/test_strix_quality_timeout_fixture_budget.py" @@ -66,6 +70,6 @@ jobs: test "$(git rev-parse HEAD)" = "${{ github.event.pull_request.head.sha || github.sha }}" python -m coverage run -m pytest tests -q bash scripts/ci/test_strix_quick_gate.sh - python -m compileall -q tests/test_strix_changed_path_policy.py tests/test_strix_workflow_dependency_hashes.py tests/test_strix_quality_timeout_fixture_budget.py + python -m compileall -q tests/test_strix_changed_path_policy.py tests/test_strix_model_behavior_error.py tests/test_strix_nvidia_nim_not_found_fallback.py tests/test_strix_workflow_dependency_hashes.py tests/test_strix_quality_timeout_fixture_budget.py bash -n scripts/ci/strix_quick_gate.sh git diff --exit-code diff --git a/.github/workflows/strix.yml b/.github/workflows/strix.yml index 4155c73461..4223e9b18e 100644 --- a/.github/workflows/strix.yml +++ b/.github/workflows/strix.yml @@ -866,7 +866,7 @@ jobs: fi # Recognized signals that the LLM backend was unavailable / starved. - backend_unavailable_signal='RateLimitError|Too many requests\. For more on scraping GitHub|exceeded your current quota|insufficient_quota|billing details|"status"[[:space:]]*:[[:space:]]*"RESOURCE_EXHAUSTED"|tokens_limit_reached|Request body too large|Max size:[[:space:]]*[0-9]+[[:space:]]+tokens|Error code:[[:space:]]*413|LLM CONNECTION FAILED|Could not establish connection to the language model|LLM warm-up failed|Configured model and fallback models were unavailable|Configured Vertex model and fallback models were unavailable|emitted provider infrastructure or failure-signal output|before provider infrastructure failure|litellm(\.exceptions)?\.NotFoundError[^[:cntrl:]]*Nvidia_nimException[^[:cntrl:]]*Error code:[[:space:]]*404' + backend_unavailable_signal='RateLimitError|Too many requests\. For more on scraping GitHub|exceeded your current quota|insufficient_quota|billing details|"status"[[:space:]]*:[[:space:]]*"RESOURCE_EXHAUSTED"|tokens_limit_reached|Request body too large|Max size:[[:space:]]*[0-9]+[[:space:]]+tokens|Error code:[[:space:]]*413|LLM CONNECTION FAILED|Could not establish connection to the language model|LLM warm-up failed|Configured model and fallback models were unavailable|Configured Vertex model and fallback models were unavailable|emitted provider infrastructure or failure-signal output|before provider infrastructure failure|litellm(\.exceptions)?\.NotFoundError[^[:cntrl:]]*Nvidia_nimException[^[:cntrl:]]*Error code:[[:space:]]*404|(^|[^A-Za-z])ModelBehaviorError([^A-Za-z]|$)' # Any evidence that a vulnerability was actually reported. Its presence # forces a hard failure so real findings are NEVER downgraded. Keep the # severity branch anchored away from identifiers so environment lines diff --git a/CHANGELOG.md b/CHANGELOG.md index f4903c2f3f..3b879b2fbb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,9 @@ Semantic Versioning where the repository publishes a release. ### Fixed +- Classify Strix `ModelBehaviorError` with zero reported vulnerabilities as + retryable model-protocol evidence, while keeping `Vulnerabilities [1-9]` and + other severity signals fail-closed. - Parsed `opencode.jsonc` as JSONC (stripping `//` and `/* */` comments outside string literals) in the reasoning-effort guard and its contract tests, instead of raw `json.loads`, which rejected the file the moment it carried its first explanatory comment (added for the `contextual-orchestrator` provider block) with `Expecting property name enclosed in double quotes`. Comment markers inside string values, such as the `$schema` URL, are left untouched. - Download the pinned `uv` 0.12.1 exporter from the official GitHub Releases URL instead of `releases.astral.sh`, which now returns HTTP 403 and blocks org-wide OpenCode `coverage-evidence`. The SHA-256 pin is unchanged. The opener may follow one hop onto `release-assets.githubusercontent.com` or `objects.githubusercontent.com` and still rejects every other host, userinfo, non-HTTPS scheme, and nondefault port (ContextualWisdomLab/.github#1109). - Compared the trusted `uv` executable's post-install `--version` output against the real GitHub Releases build's full string, `uv 0.12.1 (x86_64-unknown-linux-gnu)`, instead of the bare `uv 0.12.1` the prior check required; the genuine release binary always prints the target triple, so every installation was failing the pin check immediately after the archive download itself was fixed (ContextualWisdomLab/.github#1109). diff --git a/docs/doctoring/strix-model-behavior-error.md b/docs/doctoring/strix-model-behavior-error.md new file mode 100644 index 0000000000..7333404fe9 --- /dev/null +++ b/docs/doctoring/strix-model-behavior-error.md @@ -0,0 +1,50 @@ +# Strix ModelBehaviorError classifier + +기준일: **2026-08-21** + +## Incident + +Required Strix scans can fail closed after the agent runtime raises +`ModelBehaviorError` even when the log reports `Vulnerabilities 0`. The +exception means the selected model did not follow Strix's tool-calling +protocol. Treating that protocol failure as a security finding blocked +current-head progress on otherwise empty scans. + +## Decision + +`scripts/ci/strix_quick_gate.sh` recognizes the exact PascalCase runtime +identifier `ModelBehaviorError` as retryable model evidence. The gate moves to +the configured fallback sequence and does not retry the same model. The outer +`.github/workflows/strix.yml` neutralization path may skip only when that +signal is present **and** the log contains no vulnerability evidence. + +`Vulnerabilities[[:space:]]+[1-9]` and `severity:` markers remain blocking. +Generic warnings, timeouts, provider failures, and MEDIUM-or-higher findings +are unchanged. + +## Verification contract + +`tests/test_strix_model_behavior_error.py` executes the production classifier +and the outer workflow neutralization condition against bounded synthetic +logs. It proves: + +1. `ModelBehaviorError` plus `Vulnerabilities 0` is retryable and may + neutralize; +2. `ModelBehaviorError` plus `Vulnerabilities 1` stays fail-closed; +3. lowercase application prose is not classified as the runtime exception; +4. the identifier is wired into infrastructure detection and cross-model + fallback, never same-model retry. + +## Rollback + +If a future Strix release renames the exception, add the exact new identifier +and a matching regression. Do not remove the vulnerability fail-closed guard. + +## References (APA 7th) + +GitHub. (n.d.). *Workflow syntax for GitHub Actions*. GitHub Docs. Retrieved +August 21, 2026, from +https://docs.github.com/en/actions/reference/workflows-and-actions/workflow-syntax + +GitHub. (n.d.). *Using workflow run logs*. GitHub Docs. Retrieved August 21, +2026, from https://docs.github.com/en/actions/how-tos/monitor-workflows/use-workflow-run-logs diff --git a/scripts/ci/strix_quick_gate.sh b/scripts/ci/strix_quick_gate.sh index 0f37f34605..3794e2e440 100755 --- a/scripts/ci/strix_quick_gate.sh +++ b/scripts/ci/strix_quick_gate.sh @@ -2660,6 +2660,19 @@ is_nvidia_nim_not_found_error() { return 1 } +is_model_behavior_error() { + # Classify only Strix's exact agent-protocol exception. The PascalCase + # identifier is the runtime class name; lowercase application prose is + # not retryable. Cross-model fallback may continue; same-model retry + # does not, because repeating the same protocol failure is not a + # transient transport flake. + if grep -Eq '(^|[^A-Za-z])ModelBehaviorError([^A-Za-z]|$)' "$STRIX_LOG"; then + return 0 + fi + + return 1 +} + ## Determines whether the last strix failure is a transient error eligible ## for same-model retry (up to STRIX_TRANSIENT_RETRY_PER_MODEL times). ## Four error families qualify: @@ -2976,6 +2989,10 @@ has_detected_infrastructure_error() { return 0 fi + if is_model_behavior_error; then + return 0 + fi + # Generic strix non-zero exit with known transport/connection errors # that don't fall into the specific categories above. # Use LLM_PROVIDER_ONLY_REGEX (not PROVIDER_CONTEXT_REGEX) to avoid @@ -3826,6 +3843,10 @@ is_model_retryable_error() { return 0 fi + if is_model_behavior_error; then + return 0 + fi + if is_github_models_api_compatible_model "$model" && is_github_models_unavailable_model_error; then return 0 fi diff --git a/tests/test_strix_model_behavior_error.py b/tests/test_strix_model_behavior_error.py new file mode 100644 index 0000000000..ffad8d4078 --- /dev/null +++ b/tests/test_strix_model_behavior_error.py @@ -0,0 +1,192 @@ +"""Regression contract for Strix ModelBehaviorError protocol flakes. + +A ModelBehaviorError with zero reported vulnerabilities is retryable model +evidence. Real vulnerability counts remain fail-closed. +""" + +from __future__ import annotations + +import re +import subprocess +import tempfile +import unittest +from pathlib import Path + + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +STRIX_GATE = REPOSITORY_ROOT / "scripts" / "ci" / "strix_quick_gate.sh" +STRIX_WORKFLOW = REPOSITORY_ROOT / ".github" / "workflows" / "strix.yml" +QUALITY_WORKFLOW = ( + REPOSITORY_ROOT / ".github" / "workflows" / "strix-changed-path-quality-ci.yml" +) + + +def _function_block(source: str, function_name: str) -> str: + """Return one top-level Bash function, including its closing brace.""" + + match = re.search( + rf"(?ms)^{re.escape(function_name)}\(\) {{\n.*?^}}\n", + source, + ) + if match is None: + raise AssertionError(f"missing Bash function: {function_name}") + return match.group(0) + + +def _classifies_as_model_behavior_error(log_text: str) -> bool: + """Execute the production classifier against a bounded synthetic log.""" + + gate_source = STRIX_GATE.read_text(encoding="utf-8") + function_source = _function_block(gate_source, "is_model_behavior_error") + with tempfile.TemporaryDirectory(prefix="strix-model-behavior-") as temp_dir: + log_path = Path(temp_dir) / "strix.log" + log_path.write_text(log_text, encoding="utf-8") + script = "\n".join( + ( + "set -euo pipefail", + 'STRIX_LOG="$1"', + function_source, + "is_model_behavior_error", + ) + ) + completed = subprocess.run( + ["bash", "-c", script, "strix-classifier", str(log_path)], + check=False, + capture_output=True, + text=True, + ) + if completed.returncode not in {0, 1}: + raise AssertionError(completed.stderr) + return completed.returncode == 0 + + +def _workflow_signal_pattern(workflow: str, variable_name: str) -> str: + """Extract one single-quoted POSIX ERE assigned in the Strix workflow.""" + + match = re.search( + rf"(?m)^\s+{re.escape(variable_name)}='([^']+)'$", + workflow, + ) + if match is None: + raise AssertionError(f"missing workflow signal: {variable_name}") + return match.group(1) + + +def _workflow_neutralizes(log_text: str) -> bool: + """Execute the outer workflow's backend-neutralization condition.""" + + workflow = STRIX_WORKFLOW.read_text(encoding="utf-8") + backend_pattern = _workflow_signal_pattern( + workflow, + "backend_unavailable_signal", + ) + vulnerability_pattern = _workflow_signal_pattern( + workflow, + "reported_vulnerability_signal", + ) + with tempfile.TemporaryDirectory(prefix="strix-workflow-mbe-") as temp_dir: + log_path = Path(temp_dir) / "strix.log" + log_path.write_text(log_text, encoding="utf-8") + backend = subprocess.run( + ["grep", "-Eiq", backend_pattern, str(log_path)], + check=False, + capture_output=True, + text=True, + ) + vulnerability = subprocess.run( + ["grep", "-Eiq", vulnerability_pattern, str(log_path)], + check=False, + capture_output=True, + text=True, + ) + if backend.returncode not in {0, 1}: + raise AssertionError(backend.stderr) + if vulnerability.returncode not in {0, 1}: + raise AssertionError(vulnerability.stderr) + return backend.returncode == 0 and vulnerability.returncode == 1 + + +class StrixModelBehaviorErrorTests(unittest.TestCase): + """Protect protocol flakes without weakening vulnerability fail-closed.""" + + def test_runtime_model_behavior_error_is_retryable(self) -> None: + """Recognize the exact PascalCase Strix agent-protocol exception.""" + + log = ( + "strix.agents.base.ModelBehaviorError: tool protocol mismatch\n" + "Vulnerabilities 0\n" + ) + self.assertTrue(_classifies_as_model_behavior_error(log)) + + def test_lowercase_application_prose_is_not_retryable(self) -> None: + """Reject target-application text that only resembles the exception.""" + + log = "the model behavior error was logged by the scanned service\n" + self.assertFalse(_classifies_as_model_behavior_error(log)) + + def test_behavior_error_skips_same_model_and_enters_fallback(self) -> None: + """Wire the classifier into infrastructure and cross-model fallback.""" + + gate_source = STRIX_GATE.read_text(encoding="utf-8") + infrastructure = _function_block( + gate_source, + "has_detected_infrastructure_error", + ) + retryable = _function_block(gate_source, "is_model_retryable_error") + same_model_retry = _function_block( + gate_source, + "is_transient_same_model_retry_error", + ) + + self.assertIn("is_model_behavior_error", infrastructure) + self.assertIn("is_model_behavior_error", retryable) + self.assertNotIn("is_model_behavior_error", same_model_retry) + + def test_outer_workflow_neutralizes_zero_finding_protocol_flake(self) -> None: + """Empty scans that only hit ModelBehaviorError may skip.""" + + self.assertTrue( + _workflow_neutralizes( + "strix.agents.base.ModelBehaviorError: tool protocol mismatch\n" + "Vulnerabilities 0\n" + ) + ) + + def test_outer_workflow_never_neutralizes_reported_vulnerabilities(self) -> None: + """Keep a real vulnerability signal blocking despite protocol failure.""" + + self.assertFalse( + _workflow_neutralizes( + "strix.agents.base.ModelBehaviorError: tool protocol mismatch\n" + "Vulnerabilities 1\n" + ) + ) + self.assertFalse( + _workflow_neutralizes( + "strix.agents.base.ModelBehaviorError: tool protocol mismatch\n" + "Vulnerabilities 9\n" + ) + ) + + def test_workflow_keeps_fail_closed_vulnerability_contract(self) -> None: + """Retain the static fail-closed vulnerability evidence contract.""" + + workflow = STRIX_WORKFLOW.read_text(encoding="utf-8") + self.assertIn("ModelBehaviorError", workflow) + self.assertIn("reported_vulnerability_signal", workflow) + self.assertIn("Vulnerabilities[[:space:]]+[1-9]", workflow) + self.assertIn( + '! grep -Eiq "$reported_vulnerability_signal"', + workflow, + ) + + def test_quality_trigger_includes_model_behavior_contracts(self) -> None: + """Keep classifier, doctoring, and workflow edits on the quality path.""" + + workflow = QUALITY_WORKFLOW.read_text(encoding="utf-8") + self.assertIn(' - "docs/doctoring/strix-model-behavior-error.md"', workflow) + self.assertIn(' - "tests/test_strix_model_behavior_error.py"', workflow) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_strix_quality_timeout_fixture_budget.py b/tests/test_strix_quality_timeout_fixture_budget.py index 78fcc8a7aa..0ea4e3b37d 100644 --- a/tests/test_strix_quality_timeout_fixture_budget.py +++ b/tests/test_strix_quality_timeout_fixture_budget.py @@ -33,6 +33,8 @@ def test_strix_quality_trigger_includes_fixture_contract_paths() -> None: assert "docs/doctoring/strix-quality-timeout-fixtures.md" in trigger assert "tests/test_strix_quality_timeout_fixture_budget.py" in trigger + assert "docs/doctoring/strix-model-behavior-error.md" in trigger + assert "tests/test_strix_model_behavior_error.py" in trigger def test_strix_quality_keeps_real_scanner_budgets_out_of_fixture_overrides() -> None: From d8d05463d5ca1e220f31ef490b3c268ddb957669 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 06:25:21 +0000 Subject: [PATCH 2/3] fix(strix): require module-qualified ModelBehaviorError context Do not treat a bare source-file token as protocol failure. Match only agents, pydantic_ai, or strix exception paths so Vulnerabilities [1-9] stay fail-closed. --- .github/workflows/strix.yml | 2 +- docs/doctoring/strix-model-behavior-error.md | 14 ++++++++------ scripts/ci/strix_quick_gate.sh | 10 ++++------ tests/test_strix_model_behavior_error.py | 13 +++++++++++++ 4 files changed, 26 insertions(+), 13 deletions(-) diff --git a/.github/workflows/strix.yml b/.github/workflows/strix.yml index 4223e9b18e..d531384ef8 100644 --- a/.github/workflows/strix.yml +++ b/.github/workflows/strix.yml @@ -866,7 +866,7 @@ jobs: fi # Recognized signals that the LLM backend was unavailable / starved. - backend_unavailable_signal='RateLimitError|Too many requests\. For more on scraping GitHub|exceeded your current quota|insufficient_quota|billing details|"status"[[:space:]]*:[[:space:]]*"RESOURCE_EXHAUSTED"|tokens_limit_reached|Request body too large|Max size:[[:space:]]*[0-9]+[[:space:]]+tokens|Error code:[[:space:]]*413|LLM CONNECTION FAILED|Could not establish connection to the language model|LLM warm-up failed|Configured model and fallback models were unavailable|Configured Vertex model and fallback models were unavailable|emitted provider infrastructure or failure-signal output|before provider infrastructure failure|litellm(\.exceptions)?\.NotFoundError[^[:cntrl:]]*Nvidia_nimException[^[:cntrl:]]*Error code:[[:space:]]*404|(^|[^A-Za-z])ModelBehaviorError([^A-Za-z]|$)' + backend_unavailable_signal='RateLimitError|Too many requests\. For more on scraping GitHub|exceeded your current quota|insufficient_quota|billing details|"status"[[:space:]]*:[[:space:]]*"RESOURCE_EXHAUSTED"|tokens_limit_reached|Request body too large|Max size:[[:space:]]*[0-9]+[[:space:]]+tokens|Error code:[[:space:]]*413|LLM CONNECTION FAILED|Could not establish connection to the language model|LLM warm-up failed|Configured model and fallback models were unavailable|Configured Vertex model and fallback models were unavailable|emitted provider infrastructure or failure-signal output|before provider infrastructure failure|litellm(\.exceptions)?\.NotFoundError[^[:cntrl:]]*Nvidia_nimException[^[:cntrl:]]*Error code:[[:space:]]*404|(^|[^A-Za-z0-9_])(agents|pydantic_ai|strix)(\.[A-Za-z_][A-Za-z0-9_]*)*\.ModelBehaviorError([^A-Za-z0-9_]|$)' # Any evidence that a vulnerability was actually reported. Its presence # forces a hard failure so real findings are NEVER downgraded. Keep the # severity branch anchored away from identifiers so environment lines diff --git a/docs/doctoring/strix-model-behavior-error.md b/docs/doctoring/strix-model-behavior-error.md index 7333404fe9..0bb06be357 100644 --- a/docs/doctoring/strix-model-behavior-error.md +++ b/docs/doctoring/strix-model-behavior-error.md @@ -12,8 +12,9 @@ current-head progress on otherwise empty scans. ## Decision -`scripts/ci/strix_quick_gate.sh` recognizes the exact PascalCase runtime -identifier `ModelBehaviorError` as retryable model evidence. The gate moves to +`scripts/ci/strix_quick_gate.sh` recognizes a **module-qualified** +`ModelBehaviorError` from `agents`, `pydantic_ai`, or `strix` as retryable +model evidence. A bare source-file mention is not enough. The gate moves to the configured fallback sequence and does not retry the same model. The outer `.github/workflows/strix.yml` neutralization path may skip only when that signal is present **and** the log contains no vulnerability evidence. @@ -28,10 +29,11 @@ are unchanged. and the outer workflow neutralization condition against bounded synthetic logs. It proves: -1. `ModelBehaviorError` plus `Vulnerabilities 0` is retryable and may - neutralize; -2. `ModelBehaviorError` plus `Vulnerabilities 1` stays fail-closed; -3. lowercase application prose is not classified as the runtime exception; +1. a module-qualified `agents`/`pydantic_ai`/`strix` `ModelBehaviorError` + plus `Vulnerabilities 0` is retryable and may neutralize; +2. the same exception plus `Vulnerabilities 1` stays fail-closed; +3. lowercase application prose or a bare `ModelBehaviorError` token is not + classified as the runtime exception; 4. the identifier is wired into infrastructure detection and cross-model fallback, never same-model retry. diff --git a/scripts/ci/strix_quick_gate.sh b/scripts/ci/strix_quick_gate.sh index 3794e2e440..07e459db04 100755 --- a/scripts/ci/strix_quick_gate.sh +++ b/scripts/ci/strix_quick_gate.sh @@ -2661,12 +2661,10 @@ is_nvidia_nim_not_found_error() { } is_model_behavior_error() { - # Classify only Strix's exact agent-protocol exception. The PascalCase - # identifier is the runtime class name; lowercase application prose is - # not retryable. Cross-model fallback may continue; same-model retry - # does not, because repeating the same protocol failure is not a - # transient transport flake. - if grep -Eq '(^|[^A-Za-z])ModelBehaviorError([^A-Za-z]|$)' "$STRIX_LOG"; then + # Classify only a module-qualified Strix/Agents SDK protocol exception. + # A bare source-file mention of ModelBehaviorError is not retryable. + # Cross-model fallback may continue; same-model retry does not. + if grep -Eq '(^|[^A-Za-z0-9_])(agents|pydantic_ai|strix)(\.[A-Za-z_][A-Za-z0-9_]*)*\.ModelBehaviorError([^A-Za-z0-9_]|$)' "$STRIX_LOG"; then return 0 fi diff --git a/tests/test_strix_model_behavior_error.py b/tests/test_strix_model_behavior_error.py index ffad8d4078..336f160098 100644 --- a/tests/test_strix_model_behavior_error.py +++ b/tests/test_strix_model_behavior_error.py @@ -123,6 +123,16 @@ def test_lowercase_application_prose_is_not_retryable(self) -> None: log = "the model behavior error was logged by the scanned service\n" self.assertFalse(_classifies_as_model_behavior_error(log)) + self.assertFalse(_classifies_as_model_behavior_error("ModelBehaviorError\n")) + + def test_agents_sdk_tool_protocol_failure_is_retryable(self) -> None: + """Recognize the OpenAI Agents SDK exception observed in required CI.""" + + log = ( + "agents.exceptions.ModelBehaviorError: Tool ls not found in agent strix\n" + "Vulnerabilities 0\n" + ) + self.assertTrue(_classifies_as_model_behavior_error(log)) def test_behavior_error_skips_same_model_and_enters_fallback(self) -> None: """Wire the classifier into infrastructure and cross-model fallback.""" @@ -151,6 +161,9 @@ def test_outer_workflow_neutralizes_zero_finding_protocol_flake(self) -> None: "Vulnerabilities 0\n" ) ) + self.assertFalse( + _workflow_neutralizes("ModelBehaviorError\nVulnerabilities 0\n") + ) def test_outer_workflow_never_neutralizes_reported_vulnerabilities(self) -> None: """Keep a real vulnerability signal blocking despite protocol failure.""" From 48e5b610e1405e10fa6f32ea8c8c4f2c54880532 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 16:44:49 +0900 Subject: [PATCH 3/3] fix(strix): preserve behavior error case sensitivity --- .github/workflows/strix.yml | 6 ++++-- tests/test_strix_model_behavior_error.py | 23 ++++++++++++++++++++++- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/.github/workflows/strix.yml b/.github/workflows/strix.yml index d531384ef8..f5bb1fb9b6 100644 --- a/.github/workflows/strix.yml +++ b/.github/workflows/strix.yml @@ -866,7 +866,8 @@ jobs: fi # Recognized signals that the LLM backend was unavailable / starved. - backend_unavailable_signal='RateLimitError|Too many requests\. For more on scraping GitHub|exceeded your current quota|insufficient_quota|billing details|"status"[[:space:]]*:[[:space:]]*"RESOURCE_EXHAUSTED"|tokens_limit_reached|Request body too large|Max size:[[:space:]]*[0-9]+[[:space:]]+tokens|Error code:[[:space:]]*413|LLM CONNECTION FAILED|Could not establish connection to the language model|LLM warm-up failed|Configured model and fallback models were unavailable|Configured Vertex model and fallback models were unavailable|emitted provider infrastructure or failure-signal output|before provider infrastructure failure|litellm(\.exceptions)?\.NotFoundError[^[:cntrl:]]*Nvidia_nimException[^[:cntrl:]]*Error code:[[:space:]]*404|(^|[^A-Za-z0-9_])(agents|pydantic_ai|strix)(\.[A-Za-z_][A-Za-z0-9_]*)*\.ModelBehaviorError([^A-Za-z0-9_]|$)' + backend_unavailable_signal='RateLimitError|Too many requests\. For more on scraping GitHub|exceeded your current quota|insufficient_quota|billing details|"status"[[:space:]]*:[[:space:]]*"RESOURCE_EXHAUSTED"|tokens_limit_reached|Request body too large|Max size:[[:space:]]*[0-9]+[[:space:]]+tokens|Error code:[[:space:]]*413|LLM CONNECTION FAILED|Could not establish connection to the language model|LLM warm-up failed|Configured model and fallback models were unavailable|Configured Vertex model and fallback models were unavailable|emitted provider infrastructure or failure-signal output|before provider infrastructure failure|litellm(\.exceptions)?\.NotFoundError[^[:cntrl:]]*Nvidia_nimException[^[:cntrl:]]*Error code:[[:space:]]*404' + model_behavior_error_signal='(^|[^A-Za-z0-9_])(agents|pydantic_ai|strix)(\.[A-Za-z_][A-Za-z0-9_]*)*\.ModelBehaviorError([^A-Za-z0-9_]|$)' # Any evidence that a vulnerability was actually reported. Its presence # forces a hard failure so real findings are NEVER downgraded. Keep the # severity branch anchored away from identifiers so environment lines @@ -877,7 +878,8 @@ jobs: # present and no vulnerability was reported anywhere. This preserves # real security gating while keeping uncontrollable provider outages # from blocking current-head merge progress. - if grep -Eiq "$backend_unavailable_signal" "$strix_run_log" \ + if ( grep -Eiq "$backend_unavailable_signal" "$strix_run_log" \ + || grep -Eq "$model_behavior_error_signal" "$strix_run_log" ) \ && ! grep -Eiq "$reported_vulnerability_signal" "$strix_run_log"; then echo "::warning title=Strix backend unavailable::Strix could not complete because its LLM backend was unavailable (rate limit / token cap / connection or warm-up failure) before producing a vulnerability report. Treating as a neutral skip so an infrastructure outage does not block merges; genuine findings still fail the check. See the strix-reports artifact and the run log." exit 0 diff --git a/tests/test_strix_model_behavior_error.py b/tests/test_strix_model_behavior_error.py index 336f160098..4ebee77681 100644 --- a/tests/test_strix_model_behavior_error.py +++ b/tests/test_strix_model_behavior_error.py @@ -80,6 +80,10 @@ def _workflow_neutralizes(log_text: str) -> bool: workflow, "backend_unavailable_signal", ) + model_behavior_pattern = _workflow_signal_pattern( + workflow, + "model_behavior_error_signal", + ) vulnerability_pattern = _workflow_signal_pattern( workflow, "reported_vulnerability_signal", @@ -93,6 +97,12 @@ def _workflow_neutralizes(log_text: str) -> bool: capture_output=True, text=True, ) + model_behavior = subprocess.run( + ["grep", "-Eq", model_behavior_pattern, str(log_path)], + check=False, + capture_output=True, + text=True, + ) vulnerability = subprocess.run( ["grep", "-Eiq", vulnerability_pattern, str(log_path)], check=False, @@ -101,9 +111,14 @@ def _workflow_neutralizes(log_text: str) -> bool: ) if backend.returncode not in {0, 1}: raise AssertionError(backend.stderr) + if model_behavior.returncode not in {0, 1}: + raise AssertionError(model_behavior.stderr) if vulnerability.returncode not in {0, 1}: raise AssertionError(vulnerability.stderr) - return backend.returncode == 0 and vulnerability.returncode == 1 + return ( + (backend.returncode == 0 or model_behavior.returncode == 0) + and vulnerability.returncode == 1 + ) class StrixModelBehaviorErrorTests(unittest.TestCase): @@ -164,6 +179,11 @@ def test_outer_workflow_neutralizes_zero_finding_protocol_flake(self) -> None: self.assertFalse( _workflow_neutralizes("ModelBehaviorError\nVulnerabilities 0\n") ) + self.assertFalse( + _workflow_neutralizes( + "agents.foo.modelbehaviorerror\nVulnerabilities 0\n" + ) + ) def test_outer_workflow_never_neutralizes_reported_vulnerabilities(self) -> None: """Keep a real vulnerability signal blocking despite protocol failure.""" @@ -186,6 +206,7 @@ def test_workflow_keeps_fail_closed_vulnerability_contract(self) -> None: workflow = STRIX_WORKFLOW.read_text(encoding="utf-8") self.assertIn("ModelBehaviorError", workflow) + self.assertIn("model_behavior_error_signal", workflow) self.assertIn("reported_vulnerability_signal", workflow) self.assertIn("Vulnerabilities[[:space:]]+[1-9]", workflow) self.assertIn(