diff --git a/docs/doctoring/strix-report-failure-attribution.md b/docs/doctoring/strix-report-failure-attribution.md new file mode 100644 index 0000000000..5f304410f2 --- /dev/null +++ b/docs/doctoring/strix-report-failure-attribution.md @@ -0,0 +1,33 @@ +# Strix report 경고의 원인 분류 + +## 관측과 범위 + +ContextualWisdomLab/naruon#1244의 run `34039160285`, job `101508385806`, +artifact `9993763655`는 trusted gate `dd0b96feded94f66ecf59b25a5a9b58cfc8b4f69`를 사용했다. +`strix.log:442`의 `web_search invoked without PERPLEXITY_API_KEY configured` +WARNING이 report 실패 패턴에 매칭됐다. 같은 로그의 867–889행은 모델 요청 재시도 뒤 +완료를 기록했고, child attempt는 rc0, run.json은 completed/scan_completed=true, +SARIF 결과는 0개였다. 이 완료 기록은 경고 없는 보안 증거를 뜻하지 않는다. + +`main@c9052e607e5f3cc76e73207e7786b21500721b79`의 gate는 당시 gate와 동일했다. +기존 #1668 `82b19c4144d10550fb35145b2dc24cdb2db6f27a`는 pool 소진 단정을 이미 +제거했다. 이 후속 개발 stack은 그 변경을 복제하지 않고, 남은 report-only 경고의 +provider 장애 오표시만 분리한다. 소비자 배포나 미출시 의존성 채택이 아니다. + +## 수리 계약 + +알려진 좁은 경고 예외와 Warn/Fatal/Denied/Timeout 실패 패턴은 바꾸지 않는다. +원래 console 신호와 기존 report-provider 분류기를 먼저 확인하고, 독립된 provider +신호가 없는 report 실패는 `STRIX_REPORT_FAILURE`로 반환한다. gate가 스스로 붙인 +실패 문구를 provider 원인 증거로 재사용하지 않는다. 실제 provider 신호가 함께 있으면 +기존 provider 실패 경로를 유지한다. 이는 원인 분류이며 report의 안전성 승인도, +모든 provider 오류 형식을 완전히 식별한다는 주장도 아니다. + +기존 `test_strix_quick_gate.sh`의 fake Strix harness를 확장했다. +`report-web-search-warning-fails`는 실제 경고 형식, rc0, completed run.json, +빈 SARIF를 만들고도 gate exit 1과 report 진단을 요구한다. +`report-web-search-warning-with-provider-failure`는 같은 report에 RateLimitError를 +추가해 provider 진단 보존을 확인한다. 두 경우 모두 호출은 1회이며 원래 경고와 +child rc0 기록은 artifact에 남는다. 실제 스캔, API key 추가, 경고 무시, 유료 우회는 없다. + +로컬 mock 회귀는 hosted 실행이나 보안 분석 결과의 정확성을 입증하지 않는다. diff --git a/scripts/ci/strix_quick_gate.sh b/scripts/ci/strix_quick_gate.sh index 6fdfc4ad4f..1214438eef 100755 --- a/scripts/ci/strix_quick_gate.sh +++ b/scripts/ci/strix_quick_gate.sh @@ -61,6 +61,7 @@ REPO_NAME="${REPO_ROOT##*/}" # from masking scan incompleteness — a successful strix run (exit 0) ignores # this flag because the scan itself produced a complete result set. INFRA_ERROR_DETECTED=0 +REPORT_ONLY_FAILURE_DETECTED=0 ZERO_FINDINGS_REPORTED=0 PR_FINDINGS_DECISION="not_applicable" CHANGED_FILES=() @@ -2914,13 +2915,21 @@ PY local report_failure_signal=0 if has_strix_report_failure_signal "$ACTIVE_REPORTS_DIR" "${resolved_target_path%/}/strix_runs"; then report_failure_signal=1 + # Classify before appending our own failure message to the console log. + # An unknown report warning is not evidence of a provider outage. + if ! has_detected_infrastructure_error && + ! has_strix_report_provider_failure_signal "$ACTIVE_REPORTS_DIR" "${resolved_target_path%/}/strix_runs"; then + REPORT_ONLY_FAILURE_DETECTED=1 + fi echo "Strix report artifacts emitted warning/fatal/denied/timeout output; failing closed." | tee -a "$STRIX_LOG" >&2 fi if [ "$report_failure_signal" -eq 1 ] || has_detected_infrastructure_error; then INFRA_ERROR_DETECTED=1 if [ "$rc" -eq 0 ] && provider_signal_fail_closed_enabled; then - echo "Strix run emitted provider infrastructure or failure-signal output; failing closed." >&2 + if [ "$REPORT_ONLY_FAILURE_DETECTED" -ne 1 ]; then + echo "Strix run emitted provider infrastructure or failure-signal output; failing closed." >&2 + fi return 1 fi fi @@ -4213,6 +4222,7 @@ is_model_retryable_error() { run_current_target_scan() { INFRA_ERROR_DETECTED=0 + REPORT_ONLY_FAILURE_DETECTED=0 ZERO_FINDINGS_REPORTED=0 local primary_scan_rc=0 @@ -4220,6 +4230,10 @@ run_current_target_scan() { if [ "$primary_scan_rc" -eq 2 ]; then return 2 fi + if [ "$primary_scan_rc" -ne 0 ] && [ "$REPORT_ONLY_FAILURE_DETECTED" -eq 1 ]; then + echo "STRIX_REPORT_FAILURE: Strix report artifacts contain failure signals; failing closed without attributing them to the model provider." >&2 + return 1 + fi # Pull-request scope is orthogonal to the retry/fallback/severity admission # this change removed. It answers a different question: do the findings this diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index 171497ec15..e822ad4b2a 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -3940,6 +3940,20 @@ EOS echo "scan ok with sanitized internal Strix report notice variant" exit 0 ;; + report-web-search-warning-fails | report-web-search-warning-with-provider-failure) + mkdir -p "$STRIX_REPORTS_DIR/fake-web-search-warning" + cat >"$STRIX_REPORTS_DIR/fake-web-search-warning/strix.log" <<'EOS' +2026-09-06 16:48:30.660 WARNING strix-pr-scope-mbokzb_498a - strix.tools.web_search.tool: web_search invoked without PERPLEXITY_API_KEY configured +2026-09-06 17:20:22.750 INFO strix-pr-scope-mbokzb_498a - strix.tools.finish.tool: finish_scan: completed scan with 0 vulnerability report(s) +EOS + printf '%s\n' '{"status":"completed","scan_results":{"scan_completed":true}}' >"$STRIX_REPORTS_DIR/fake-web-search-warning/run.json" + printf '%s\n' '{"version":"2.1.0","runs":[{"tool":{"driver":{"name":"Strix"}},"results":[]}]}' >"$STRIX_REPORTS_DIR/fake-web-search-warning/findings.sarif" + if [ "$FAKE_STRIX_SCENARIO" = "report-web-search-warning-with-provider-failure" ]; then + echo 'RateLimitError: provider exhausted' >>"$STRIX_REPORTS_DIR/fake-web-search-warning/strix.log" + fi + echo 'Vulnerabilities 0' + exit 0 + ;; report-unknown-warning-fails) mkdir -p "$STRIX_REPORTS_DIR/fake-unknown-warning" cat >"$STRIX_REPORTS_DIR/fake-unknown-warning/strix.log" <<'EOS' @@ -4739,6 +4753,8 @@ EOS for large_scope_index in $(seq 1 38); do printf 'file %s\n' "$large_scope_index" >"$repo_root_dir/backend/large-scope/file-$large_scope_index.py" done + elif [[ "$scenario" = report-web-search-warning-* ]]; then + printf '%s\n' 'print("scan fixture")' >"$repo_root_dir/src/app.py" elif [ "$scenario" = "scan-working-directory-isolated" ]; then mkdir -p "$repo_root_dir/backend/app/pg_introspect" printf '%s\n' 'HEAD_INTROSPECT_SHOULD_BE_SCANNED' >"$repo_root_dir/backend/app/pg_introspect/introspect.py" @@ -5047,6 +5063,20 @@ PY "scenario=$scenario keeps non-warning Strix report evidence" fi + if [[ "$scenario" = report-web-search-warning-* ]]; then + assert_file_contains "$repo_root_dir/strix_runs/fake-web-search-warning/strix.log" \ + "web_search invoked without PERPLEXITY_API_KEY configured" "report warning remains published" + assert_file_contains "$repo_root_dir/strix_runs/gate-attempts/001-orchestrator_free-rc0.log" \ + "Vulnerabilities 0" "child rc0 is distinct from the gate verdict" + if [ "$scenario" = "report-web-search-warning-fails" ]; then + assert_file_contains "$output_log" "STRIX_REPORT_FAILURE" "report-only failure is classified" + assert_file_not_contains "$output_log" "provider infrastructure failure" "report warning does not prove a provider failure" + assert_file_not_contains "$output_log" "STRIX_PROVIDER_UNAVAILABLE" "report warning does not prove provider unavailability" + else + assert_file_contains "$output_log" "provider infrastructure failure" "independent provider signal keeps its diagnosis" + fi + fi + if [ "$scenario" = "github-models-primary-ratelimit-fallback-success" ]; then assert_file_contains \ "$output_log" \ @@ -5106,6 +5136,13 @@ run_gate_case_allow_provider_signal() { run_filtered_gate_case_if_requested() { case "${STRIX_TEST_CASE_FILTER:-}" in + report-web-search-warning-fails | report-web-search-warning-with-provider-failure) + run_gate_case "$STRIX_TEST_CASE_FILTER" "orchestrator/free" "" "1" \ + "Strix report artifacts emitted warning/fatal/denied/timeout output; failing closed." \ + "1" "openai/orchestrator/free" "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" "http://127.0.0.1:18080/v1" "" "0" "CRITICAL" "0" "" "" "1200" "0" \ + "pull_request" "src/app.py" + ;; "") return 0 ;; @@ -8760,6 +8797,14 @@ run_gate_case "report-known-internal-warning-variant-sanitized" \ "" \ "1" +for report_warning_case in report-web-search-warning-fails report-web-search-warning-with-provider-failure; do + run_gate_case "$report_warning_case" "orchestrator/free" "" "1" \ + "Strix report artifacts emitted warning/fatal/denied/timeout output; failing closed." \ + "1" "openai/orchestrator/free" "http://127.0.0.1:18080/v1" \ + "contextual_orchestrator" "http://127.0.0.1:18080/v1" "" "0" "CRITICAL" "0" "" "" "1200" "0" \ + "pull_request" "src/app.py" +done + run_gate_case "report-unknown-warning-fails" \ "orchestrator/free" \ "" \