From 53added67244ea7158c627f8c206d415b7f1f71a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 17:00:39 +0900 Subject: [PATCH] =?UTF-8?q?fix(opencode):=20=EC=83=8C=EB=93=9C=EB=B0=95?= =?UTF-8?q?=EC=8A=A4=20=EC=8B=9C=EC=9E=91=20=EC=98=A4=EB=A5=98=EC=9D=98=20?= =?UTF-8?q?=EC=A7=84=EB=8B=A8=EC=9D=84=20=EA=B5=AC=EB=B6=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 샌드박스 판정과 공급자 오류를 구분하고 독립 취약점 보고를 보존한다. 기존 CI 작업에 실행 회귀 검증과 AGENTS 재발 방지 절차를 연결한다. Commit-Message-Assisted-by: Codex (OpenAI) Signed-off-by: Seongho Bae --- .../agent-review-runtime-quality-ci.yml | 8 +- .../workflows/opencode-review-dispatch.yml | 10 +- AGENTS.md | 9 ++ CHANGELOG.md | 4 + ...opencode_failed_check_fallback_findings.sh | 13 +- ...t_pr_review_autofix_nvidia_nim_contract.py | 2 +- tests/test_strix_fallback_attribution.py | 135 ++++++++++++++++++ 7 files changed, 176 insertions(+), 5 deletions(-) create mode 100644 tests/test_strix_fallback_attribution.py diff --git a/.github/workflows/agent-review-runtime-quality-ci.yml b/.github/workflows/agent-review-runtime-quality-ci.yml index 3680da8778..e863a0af0f 100644 --- a/.github/workflows/agent-review-runtime-quality-ci.yml +++ b/.github/workflows/agent-review-runtime-quality-ci.yml @@ -15,6 +15,8 @@ on: - "docs/product-technical-gap-baseline.md" - ".github/workflows/opencode-review-dispatch.yml" - "scripts/ci/ensure_rust_llvm19.sh" + - "scripts/ci/emit_opencode_failed_check_fallback_findings.sh" + - "tests/test_strix_fallback_attribution.py" - "tests/test_opencode_rust_coverage_toolchain_contract.py" - "tests/test_pr_review_autofix_nvidia_nim_contract.py" - "docs/doctoring/opencode-rust-coverage-runtime-boundary.md" @@ -184,6 +186,8 @@ jobs: ;; .github/workflows/opencode-review-dispatch.yml|\ scripts/ci/ensure_rust_llvm19.sh|\ + scripts/ci/emit_opencode_failed_check_fallback_findings.sh|\ + tests/test_strix_fallback_attribution.py|\ tests/test_opencode_rust_coverage_toolchain_contract.py|\ docs/doctoring/opencode-rust-coverage-runtime-boundary.md) opencode_suite=true @@ -343,8 +347,8 @@ jobs: if: steps.affected_suites.outputs.opencode == 'true' run: | set -euo pipefail - python -m pytest -q tests/test_opencode_rust_coverage_toolchain_contract.py - python -m compileall -q tests/test_opencode_rust_coverage_toolchain_contract.py + python -m pytest -q tests/test_opencode_rust_coverage_toolchain_contract.py tests/test_strix_fallback_attribution.py + python -m compileall -q tests/test_opencode_rust_coverage_toolchain_contract.py tests/test_strix_fallback_attribution.py - name: Verify exact-head path policy and syntax if: steps.affected_suites.outputs.strix == 'true' diff --git a/.github/workflows/opencode-review-dispatch.yml b/.github/workflows/opencode-review-dispatch.yml index 26e8555967..a3c7c4fa7b 100644 --- a/.github/workflows/opencode-review-dispatch.yml +++ b/.github/workflows/opencode-review-dispatch.yml @@ -5602,7 +5602,15 @@ jobs: local path=".github/workflows/strix.yml" local line="1" - if ! grep -Eq "LLM CONNECTION FAILED|RateLimitError|Too many requests|budget limit|Configured model and fallback models were unavailable|provider infrastructure|STRIX_PROVIDER_UNAVAILABLE" "$strix_evidence_file"; then + if grep -Eq '(^|[[:space:]])STRIX_PROVIDER_UNAVAILABLE:[[:space:]]+STRIX_SANDBOX_UNAVAILABLE:' "$strix_evidence_file"; then + printf 'Non-source-backed Strix sandbox bootstrap failure: this signal alone does not establish a gateway or provider-pool failure. Do not approve from this incomplete current-head scan. Inspect sandbox startup, repair it, and rerun the intended current-head scan; review any independently reported vulnerabilities. No application source edit is justified by this signal alone.\n' >&2 + fi + + if ! awk ' + { gsub(/(^|[[:space:]])STRIX_PROVIDER_UNAVAILABLE:[[:space:]]+STRIX_SANDBOX_UNAVAILABLE:/, " ") } + /LLM CONNECTION FAILED|RateLimitError|Too many requests|budget limit|Configured model and fallback models were unavailable|provider infrastructure|STRIX_PROVIDER_UNAVAILABLE/ { found = 1 } + END { exit !found } + ' "$strix_evidence_file"; then return 0 fi diff --git a/AGENTS.md b/AGENTS.md index e955f8b36a..a8d9fcf20e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -68,6 +68,15 @@ The materialization contract is also covered by [`docs/doctoring/exact-artifact- ## Verification discipline +OpenCode failed-check diagnostics must distinguish a Strix sandbox bootstrap +failure from gateway availability. The paired `STRIX_PROVIDER_UNAVAILABLE: +STRIX_SANDBOX_UNAVAILABLE:` marker is not a source-backed vulnerability or proof +of provider failure. Keep both the canonical helper and missing-helper workflow +fallback aligned; exclude only that paired marker from provider classification, +preserve independent provider signals and mapped/unmapped vulnerability reports, +and keep the current-head scan incomplete. Run +`python -m pytest -q -W error tests/test_strix_fallback_attribution.py`. + Many agent sessions work this organization concurrently under the same standing brief. Silence is not evidence: "I have not touched X" describes one session's history, never the organization's actual state. diff --git a/CHANGELOG.md b/CHANGELOG.md index efd2771e96..3104fae59a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,6 +48,10 @@ - Raised `hourly-review-repair.yml`'s discovery ceiling from 50 to 200 while rotating deterministic 50-PR deep-inspection windows by hourly run number. The scheduler hydrates only the selected window and stops immediately after its single dispatch, preserving access to newer PRs without quadrupling expensive review/check/comment work. See `docs/doctoring/hourly-review-repair-single-file-consolidation.md`'s 2026-09-03 follow-up. ## [Unreleased] +- Failed security-scan diagnostics now identify sandbox startup failures without + blaming the model gateway or proposing an unrelated source edit. Independently + reported provider errors and vulnerabilities remain visible; the failed scan + still cannot support approval. - Include merge-scheduler entrypoint, core, and regression-test changes in the existing runtime-quality workflow's trigger and suite selector. Scheduler workflow edits retain queue checks and also select the full review-repair diff --git a/scripts/ci/emit_opencode_failed_check_fallback_findings.sh b/scripts/ci/emit_opencode_failed_check_fallback_findings.sh index 6ea3706dfb..aeb65ab396 100755 --- a/scripts/ci/emit_opencode_failed_check_fallback_findings.sh +++ b/scripts/ci/emit_opencode_failed_check_fallback_findings.sh @@ -650,7 +650,18 @@ emit_strix_provider_failure_finding() { local path=".github/workflows/strix.yml" local line="1" - if ! grep -Eq "LLM CONNECTION FAILED|RateLimitError|Too many requests|HTTPStatusError|401 Unauthorized|api\\.deepseek\\.com|Authentication Fails|DeepseekException|budget limit|Configured model and fallback models were unavailable|provider infrastructure|Below-threshold findings detected|Unable to map Strix findings" "$strix_evidence_file"; then + if grep -Eq '(^|[[:space:]])STRIX_PROVIDER_UNAVAILABLE:[[:space:]]+STRIX_SANDBOX_UNAVAILABLE:' "$strix_evidence_file"; then + printf 'Non-source-backed Strix sandbox bootstrap failure: this signal alone does not establish a gateway or provider-pool failure. Do not approve from this incomplete current-head scan. Inspect sandbox startup, repair it, and rerun the intended current-head scan; review any independently reported vulnerabilities. No application source edit is justified by this signal alone.\n' >&2 + if [ -s "$unmapped_strix_reports_file" ]; then + awk -F '\t' '{ printf "Unmapped Strix report: %s reported \"%s\" (%s; %s). Obtain a repository Code Location before proposing a source edit.\n", $1, $2, $3, $4 }' "$unmapped_strix_reports_file" >&2 + fi + fi + + if ! awk ' + { gsub(/(^|[[:space:]])STRIX_PROVIDER_UNAVAILABLE:[[:space:]]+STRIX_SANDBOX_UNAVAILABLE:/, " ") } + /LLM CONNECTION FAILED|RateLimitError|Too many requests|HTTPStatusError|401 Unauthorized|api\.deepseek\.com|Authentication Fails|DeepseekException|budget limit|Configured model and fallback models were unavailable|provider infrastructure|Below-threshold findings detected|Unable to map Strix findings/ { found = 1 } + END { exit !found } + ' "$strix_evidence_file"; then return 0 fi diff --git a/tests/test_pr_review_autofix_nvidia_nim_contract.py b/tests/test_pr_review_autofix_nvidia_nim_contract.py index 2d2304aaf1..fcb55528d4 100644 --- a/tests/test_pr_review_autofix_nvidia_nim_contract.py +++ b/tests/test_pr_review_autofix_nvidia_nim_contract.py @@ -17,7 +17,7 @@ DOCTORING_RECORD = Path("docs/doctoring/hourly-nvidia-nim-autofix.md") CHANGELOG = Path("CHANGELOG.md") REVIEW_DISPATCH_WORKFLOW = Path(".github/workflows/opencode-review-dispatch.yml") -REVIEW_DISPATCH_BLOB_SHA = "26e8555967171a5f3974602ac05700c27bddebf1" +REVIEW_DISPATCH_BLOB_SHA = "a3c7c4fa7b389549fb58a2aefa5f6cbb0592f760" def _workflow_text(path: Path) -> str: diff --git a/tests/test_strix_fallback_attribution.py b/tests/test_strix_fallback_attribution.py new file mode 100644 index 0000000000..7e2e2bbd84 --- /dev/null +++ b/tests/test_strix_fallback_attribution.py @@ -0,0 +1,135 @@ +"""Execute both failed-check emitters without GitHub or model access.""" + +import os +import subprocess +import textwrap +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[1] +SANDBOX_SIGNAL = ( + "STRIX_PROVIDER_UNAVAILABLE: STRIX_SANDBOX_UNAVAILABLE: bootstrap failed" +) + + +@pytest.mark.parametrize("changed_path", [ + "scripts/ci/emit_opencode_failed_check_fallback_findings.sh", + "tests/test_strix_fallback_attribution.py", +]) +def test_either_owned_path_selects_the_existing_opencode_contract_job(changed_path): + """Helper-only and test-only edits must execute this check without another job.""" + workflow_text = (REPO_ROOT / ".github/workflows/agent-review-runtime-quality-ci.yml").read_text( + encoding="utf-8" + ) + trigger_text = workflow_text.split("on:\n", 1)[1].split("\nconcurrency:\n", 1)[0] + assert f' - "{changed_path}"' in trigger_text + selector_text = workflow_text.split(' case "$changed_path" in\n', 1)[1].split( + " esac", 1 + )[0] + result = subprocess.run([ + "bash", "-c", + 'set -euo pipefail\nopencode_suite=false\nchanged_path="$1"\ncase "$changed_path" in\n' + + selector_text + 'esac\nprintf "%s" "$opencode_suite"', + "selector-test", changed_path, + ], capture_output=True, text=True, check=True, env={"PATH": os.defpath}) + assert result.stdout == "true" + step_text = workflow_text.split("- name: Verify OpenCode Rust coverage toolchain contract", 1)[1].split( + "\n - name:", 1 + )[0] + assert "tests/test_strix_fallback_attribution.py" in step_text + + +def run_emitter(tmp_path: Path, route_name: str, evidence_text: str) -> subprocess.CompletedProcess: + """Run the real helper or its actual missing-helper workflow fallback.""" + evidence_file = tmp_path / "failed-checks.md" + evidence_file.write_text(evidence_text, encoding="utf-8") + fixture_repo = tmp_path / "repository" + fixture_repo.mkdir() + (fixture_repo / "README.md").write_text("Fixture source line.\n", encoding="utf-8") + helper_file = REPO_ROOT / "scripts/ci/emit_opencode_failed_check_fallback_findings.sh" + command_args = ["bash", str(helper_file), str(evidence_file), str(fixture_repo)] + if route_name == "inline": + workflow_text = (REPO_ROOT / ".github/workflows/opencode-review-dispatch.yml").read_text( + encoding="utf-8" + ) + start_index = workflow_text.index(" emit_line_specific_fallback_findings() {") + end_index = workflow_text.index("\n }", start_index) + len("\n }") + function_text = textwrap.dedent(workflow_text[start_index:end_index]) + command_args = [ + "bash", "-c", + 'set -euo pipefail\n' + function_text + '\nemit_line_specific_fallback_findings "$1"', + "fallback-test", str(evidence_file), + ] + return subprocess.run( + command_args, cwd=fixture_repo, capture_output=True, text=True, check=False, + env={"PATH": os.defpath, "GITHUB_WORKSPACE": str(fixture_repo)}, + ) + + +@pytest.mark.parametrize("route_name", ["helper", "inline"]) +def test_sandbox_bootstrap_does_not_invent_a_provider_or_source_finding(tmp_path, route_name): + """A sandbox-only failed scan remains incomplete without a fabricated finding.""" + result = run_emitter(tmp_path, route_name, f"## Failed check: Strix Security Scan/strix\n{SANDBOX_SIGNAL}\n") + assert result.returncode == 1 + assert "Strix sandbox bootstrap failure" in result.stderr + assert "Do not approve" in result.stderr + assert "No PR review was posted" in result.stderr + assert "### 1." not in result.stdout + assert "gateway or its discovered provider pool was unavailable" not in result.stdout + + +@pytest.mark.parametrize("route_name", ["helper", "inline"]) +@pytest.mark.parametrize("evidence_text", [ + "## Failed check: Strix Security Scan/strix\nSTRIX_SANDBOX_UNAVAILABLE: bootstrap failed\n", + f"## Failed check: Strix Security Scan/strix\nUnknown failure\n## Failed check: Unit tests\n{SANDBOX_SIGNAL}\n", +]) +def test_sandbox_signal_requires_the_paired_marker_in_the_strix_check(tmp_path, route_name, evidence_text): + """Unrelated checks and a bare substring are not authoritative sandbox evidence.""" + result = run_emitter(tmp_path, route_name, evidence_text) + assert result.returncode == 1 + assert "Strix sandbox bootstrap failure" not in result.stderr + assert "### 1." not in result.stdout + + +@pytest.mark.parametrize("route_name", ["helper", "inline"]) +@pytest.mark.parametrize("sandbox_line", ["", SANDBOX_SIGNAL + "\n", SANDBOX_SIGNAL + "; "]) +def test_provider_evidence_keeps_its_existing_classification(tmp_path, route_name, sandbox_line): + """Adding a sandbox distinction must not remove independent provider evidence.""" + result = run_emitter(tmp_path, route_name, f"## Failed check: Strix Security Scan/strix\n{sandbox_line}RateLimitError\n") + assert result.returncode == 0 + assert "### 1. HIGH" in result.stdout + assert "provider" in result.stdout + assert ("Strix sandbox bootstrap failure" in result.stderr) == bool(sandbox_line) + + +@pytest.mark.parametrize("source_path", ["README.md", "missing_source.py"]) +def test_sandbox_signal_preserves_a_separately_reported_vulnerability(tmp_path, source_path): + """Emitted findings are not a passing scan, even when the helper exits zero.""" + evidence_text = f"""## Failed check: Strix Security Scan/strix +{SANDBOX_SIGNAL} + +### Strix vulnerability report window 1 +Model openai/example-model Vulnerabilities 1 +│ Vulnerability Report +│ Title: Fixture vulnerability +│ Severity: HIGH +│ Endpoint: /example +│ Code Locations +│ Location 1: {source_path}:1 +""" + result = run_emitter(tmp_path, "helper", evidence_text) + if source_path == "README.md": + assert result.returncode == 0 + assert "HIGH README.md:1 - Strix report from openai/example-model: Fixture vulnerability" in result.stdout + else: + assert result.returncode == 1 + assert "Unmapped Strix report" in result.stderr + assert "openai/example-model" in result.stderr + assert "Fixture vulnerability" in result.stderr + assert "HIGH" in result.stderr + assert "did not include a mappable Code Location" in result.stderr + assert "### 1." not in result.stdout + assert "### 2." not in result.stdout + assert "Strix sandbox bootstrap failure" in result.stderr + assert "no Strix Vulnerability Report window was produced" not in result.stdout