From 8155d504e56939b0b2bef7eb008a81d708d958e3 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 1 Sep 2026 23:44:34 +0900 Subject: [PATCH 01/24] test(noema): reproduce malformed verdict failure classification --- ...ema_model_output_failure_classification.py | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 tests/test_noema_model_output_failure_classification.py diff --git a/tests/test_noema_model_output_failure_classification.py b/tests/test_noema_model_output_failure_classification.py new file mode 100644 index 0000000000..cb29fc10ad --- /dev/null +++ b/tests/test_noema_model_output_failure_classification.py @@ -0,0 +1,64 @@ +"""Regression for #1611: malformed model verdicts are infrastructure/model evidence. + +A schema-valid JSON envelope whose adversarial probe uses an out-of-domain +outcome is not a consumer repository defect. The deterministic validator must +still reject it, but with a typed model-output error so the retry/control plane +can preserve the distinction from source findings and provider exhaustion. +""" + +import pytest + +from scripts.ci import noema_review_gate as gate + + +DIFF = """diff --git a/README.md b/README.md +index 1111111..2222222 100644 +--- a/README.md ++++ b/README.md +@@ -1 +1 @@ +-old ++new +""" + + +def _verdict() -> dict: + return { + "decision": "approve", + "summary": "The changed line was reviewed.", + "reviewed_lines": [ + { + "path": "README.md", + "line": 1, + "side": "RIGHT", + "analysis": "The replacement is bounded and reviewable.", + } + ], + "adversarial_validation": { + "status": "passed", + "residual_risk": "No additional risk identified.", + "probes": [ + { + "path": "README.md", + "line": 1, + "side": "RIGHT", + "hypothesis": "The replacement could be wrong.", + "attack_or_counterexample": "Compare the exact changed line.", + "evidence": "Observed the exact replacement in the diff.", + "outcome": "passed", # real #1611 failure shape + } + ], + }, + "findings": [], + } + + +def test_invalid_probe_outcome_is_typed_model_output_failure() -> None: + """Reject malformed LLM evidence without reclassifying it as source failure.""" + error_type = getattr(gate, "NoemaModelOutputError", None) + assert error_type is not None, ( + "Noema must expose a typed model-output/schema failure so malformed " + "LLM evidence cannot collapse into an opaque generic RuntimeError" + ) + + with pytest.raises(error_type, match="outcome must be falsified or confirmed"): + gate.validate_substantive_verdict(_verdict(), DIFF, ["README.md"]) From 35a0f4b4628b8cdc35636926d24b1b4d38cfbe30 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 00:04:13 +0900 Subject: [PATCH 02/24] chore(noema): stage exact #1617 source repair --- scripts/ci/repair_noema_model_output_1617.py | 291 +++++++++++++++++++ 1 file changed, 291 insertions(+) create mode 100644 scripts/ci/repair_noema_model_output_1617.py diff --git a/scripts/ci/repair_noema_model_output_1617.py b/scripts/ci/repair_noema_model_output_1617.py new file mode 100644 index 0000000000..609dada1ad --- /dev/null +++ b/scripts/ci/repair_noema_model_output_1617.py @@ -0,0 +1,291 @@ +#!/usr/bin/env python3 +"""Apply the one-shot, test-first Noema model-output repair for PR #1617. + +This helper exists only to make an exact, reviewable transformation on the +single-writer PR branch. The workflow that invokes it deletes this helper and +itself before committing the production repair. +""" + +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] +SOURCE = ROOT / "scripts/ci/noema_review_gate.py" +TEST = ROOT / "tests/test_noema_model_output_failure_classification.py" +CHANGELOG = ROOT / "CHANGELOG.md" +BASELINE = ROOT / "docs/product-technical-gap-baseline.md" +ARCHITECTURE = ROOT / "ARCHITECTURE.md" +DOCTORING = ROOT / "docs/doctoring/noema-model-output-repair-boundary.md" + + +def replace_once(text: str, old: str, new: str, label: str) -> str: + """Replace one exact source fragment and fail closed on drift.""" + count = text.count(old) + if count != 1: + raise RuntimeError(f"{label}: expected exactly one match, found {count}") + return text.replace(old, new, 1) + + +def replace_raises_between(text: str, start: str, end: str) -> str: + """Retype model-output validation errors within one bounded source span.""" + start_index = text.index(start) + end_index = text.index(end, start_index) + span = text[start_index:end_index] + if "raise RuntimeError(" not in span: + raise RuntimeError(f"{start.strip()}: no RuntimeError raises found") + span = span.replace("raise RuntimeError(", "raise NoemaModelOutputError(") + return text[:start_index] + span + text[end_index:] + + +def update_source() -> None: + """Implement typed model-output failures and a bounded one-time repair call.""" + text = SOURCE.read_text(encoding="utf-8") + text = replace_once( + text, + 'ORCHESTRATOR_BASE_ENV = "CONTEXTUAL_ORCHESTRATOR_BASE_URL"\n', + 'ORCHESTRATOR_BASE_ENV = "CONTEXTUAL_ORCHESTRATOR_BASE_URL"\n' + '# A repair request corrects an already-completed model verdict; it is not a\n' + '# second unbounded full review. Fifteen minutes is the hard client-side\n' + '# ceiling for that one corrective HTTP request. The primary review remains\n' + '# governed by contextual-orchestrator rather than a fixed inference timeout.\n' + 'NOEMA_REPAIR_TIMEOUT_SECONDS = 15 * 60\n\n\n' + 'class NoemaModelOutputError(RuntimeError):\n' + ' """Raised when untrusted model output violates the trusted verdict contract."""\n\n\n' + 'class NoemaTransportError(RuntimeError):\n' + ' """Raised when the bounded review transport cannot produce usable evidence."""\n', + "typed Noema error classes", + ) + + text = replace_raises_between( + text, + "def validate_substantive_verdict(\n", + "\ndef truncate_text(", + ) + text = replace_raises_between(text, "def extract_json_object(", "\ndef extract_llm_message_content(") + text = replace_raises_between( + text, + "def extract_llm_message_content(", + "\ndef decode_llm_response_body(", + ) + text = replace_raises_between( + text, + "def decode_llm_response_body(", + "\ndef _truthy_env(", + ) + + # Retype the immediate post-response verdict-shape checks. These are all + # model-output/schema failures, not GitHub/source or transport failures. + for old, new in ( + ( + 'raise RuntimeError(f"Noema LLM returned unsupported decision: {decision!r}")', + 'raise NoemaModelOutputError(f"Noema LLM returned unsupported decision: {decision!r}")', + ), + ( + 'raise RuntimeError("Noema LLM response did not contain a substantive summary")', + 'raise NoemaModelOutputError("Noema LLM response did not contain a substantive summary")', + ), + ( + 'raise RuntimeError("Noema LLM response findings must be a list of objects")', + 'raise NoemaModelOutputError("Noema LLM response findings must be a list of objects")', + ), + ( + 'raise RuntimeError("Noema LLM response contained a malformed finding")', + 'raise NoemaModelOutputError("Noema LLM response contained a malformed finding")', + ), + ( + 'raise RuntimeError("Noema LLM request_changes response did not contain a substantive finding")', + 'raise NoemaModelOutputError("Noema LLM request_changes response did not contain a substantive finding")', + ), + ): + text = replace_once(text, old, new, old) + + text = replace_once( + text, + """ with opener.open(request) as response: # nosec B310\n raw_bytes = response.read()\n""", + """ if is_retry:\n response_context = opener.open( # nosec B310\n request, timeout=NOEMA_REPAIR_TIMEOUT_SECONDS\n )\n else:\n response_context = opener.open(request) # nosec B310\n with response_context as response:\n raw_bytes = response.read()\n""", + "bounded repair HTTP timeout", + ) + + text = replace_once( + text, + """ except (RuntimeError, urllib.error.URLError, http.client.HTTPException, OSError) as exc:\n if is_retry:\n if isinstance(exc, RuntimeError):\n raise\n raise RuntimeError(str(exc)) from exc\n if str(fetch_pr(repo, number).get(\"headRefOid\") or \"\").lower() != expected_head:\n raise StaleHeadDuringRepairRetryError(\n \"Pull request head changed during review; stale before repair retry.\"\n ) from exc\n return call_llm(\n repo,\n number,\n pr,\n diff,\n truncated,\n expected_head,\n review_context,\n changed_paths,\n str(exc),\n is_retry=True,\n )\n""", + """ except (RuntimeError, urllib.error.URLError, http.client.HTTPException, OSError) as exc:\n current_failure = scrub_sensitive_data(str(exc)) or type(exc).__name__\n if is_retry:\n initial_failure = (\n scrub_sensitive_data(repair_error)\n or \"no diagnostic message was available\"\n )\n if isinstance(exc, NoemaModelOutputError):\n raise NoemaModelOutputError(\n \"Noema model-output repair remained invalid; \"\n f\"initial failure: {initial_failure}; repair failure: {current_failure}\"\n ) from exc\n if isinstance(\n exc, (urllib.error.URLError, http.client.HTTPException, OSError)\n ):\n raise NoemaTransportError(\n \"Noema bounded repair transport was exhausted; \"\n f\"initial failure: {initial_failure}; repair failure: \"\n f\"{type(exc).__name__}: {current_failure}\"\n ) from exc\n raise RuntimeError(\n \"Noema repair failed closed; \"\n f\"initial failure: {initial_failure}; repair failure: {current_failure}\"\n ) from exc\n if str(fetch_pr(repo, number).get(\"headRefOid\") or \"\").lower() != expected_head:\n raise StaleHeadDuringRepairRetryError(\n \"Pull request head changed during review; stale before repair retry.\"\n ) from exc\n return call_llm(\n repo,\n number,\n pr,\n diff,\n truncated,\n expected_head,\n review_context,\n changed_paths,\n current_failure,\n is_retry=True,\n )\n""", + "typed repair exhaustion", + ) + + text = text.replace( + "Fails closed with ``RuntimeError``", + "Fails closed with ``NoemaModelOutputError``", + ) + SOURCE.write_text(text, encoding="utf-8") + + +def update_tests() -> None: + """Extend the pre-existing RED with timeout and evidence-preservation coverage.""" + text = TEST.read_text(encoding="utf-8") + marker = "def test_bounded_repair_preserves_initial_schema_and_transport_evidence" + if marker in text: + raise RuntimeError("#1617 repair tests already present") + text += r''' + + +def test_bounded_repair_preserves_initial_schema_and_transport_evidence(monkeypatch) -> None: + """A malformed verdict followed by 502 keeps both typed evidence classes.""" + import json + import urllib.error + + monkeypatch.setenv("NOEMA_LLM_API_URL", "https://llm.example/v1/chat/completions") + monkeypatch.setenv("NOEMA_LLM_API_KEY", "test-key") + head_sha = "a" * 40 + requests: list[tuple[object, dict]] = [] + + class Response: + def __enter__(self): + return self + + def __exit__(self, *_args): + return None + + def read(self): + return json.dumps( + {"choices": [{"message": {"content": json.dumps(_verdict())}}]} + ).encode() + + def open_response(_opener, request, **kwargs): + requests.append((request, kwargs)) + if len(requests) == 1: + return Response() + raise urllib.error.HTTPError(request.full_url, 502, "Bad Gateway", {}, None) + + monkeypatch.setattr(gate.urllib.request.OpenerDirector, "open", open_response) + monkeypatch.setattr( + gate, + "fetch_pr", + lambda _repo, _number: {"headRefOid": head_sha}, + ) + + with pytest.raises(gate.NoemaTransportError) as exc_info: + gate.call_llm( + "owner/repo", + 7, + {"title": "test", "headRefOid": head_sha}, + DIFF, + False, + head_sha, + changed_paths=("README.md",), + ) + + message = str(exc_info.value) + assert "outcome must be falsified or confirmed" in message + assert "HTTPError" in message + assert "502" in message + assert len(requests) == 2 + assert requests[0][1] == {} + assert requests[1][1]["timeout"] == gate.NOEMA_REPAIR_TIMEOUT_SECONDS + + +def test_repeated_model_output_failure_remains_typed(monkeypatch) -> None: + """A second malformed verdict fails closed as model-output evidence.""" + import json + + monkeypatch.setenv("NOEMA_LLM_API_URL", "https://llm.example/v1/chat/completions") + monkeypatch.setenv("NOEMA_LLM_API_KEY", "test-key") + head_sha = "b" * 40 + + class Response: + def __enter__(self): + return self + + def __exit__(self, *_args): + return None + + def read(self): + return json.dumps( + {"choices": [{"message": {"content": json.dumps(_verdict())}}]} + ).encode() + + monkeypatch.setattr( + gate.urllib.request.OpenerDirector, + "open", + lambda *_args, **_kwargs: Response(), + ) + monkeypatch.setattr( + gate, + "fetch_pr", + lambda _repo, _number: {"headRefOid": head_sha}, + ) + + with pytest.raises(gate.NoemaModelOutputError) as exc_info: + gate.call_llm( + "owner/repo", + 7, + {"title": "test", "headRefOid": head_sha}, + DIFF, + False, + head_sha, + changed_paths=("README.md",), + ) + + assert "initial failure" in str(exc_info.value) + assert "repair failure" in str(exc_info.value) +''' + TEST.write_text(text, encoding="utf-8") + + +def update_docs() -> None: + """Record the RCA, bounded contract, and architecture consequence.""" + changelog = CHANGELOG.read_text(encoding="utf-8") + entry = """- **Classify and bound Noema malformed-verdict repair failures (#1611/#1617).** A schema-invalid model verdict now raises typed `NoemaModelOutputError` evidence instead of an undifferentiated runtime failure. The one corrective HTTP request has a 15-minute client ceiling while the primary contextual-orchestrator review remains under its no-fixed-inference-timeout contract. If the repair then fails at transport, `NoemaTransportError` preserves the first validator diagnostic plus the later transport class/status without logging raw model output or secrets.\n""" + changelog = replace_once(changelog, "## [Unreleased]\n", "## [Unreleased]\n" + entry, "changelog unreleased") + CHANGELOG.write_text(changelog, encoding="utf-8") + + architecture = ARCHITECTURE.read_text(encoding="utf-8") + architecture_note = """ + +### Noema model-output and repair boundary + +Noema separates deterministic model-output/schema failures from GitHub/source +findings and provider transport exhaustion. A malformed verdict remains +non-passing and is represented by `NoemaModelOutputError`. Its single corrective +request still routes only through the loopback contextual-orchestrator +`orchestrator/free` gateway, but is capped at 15 minutes because it repairs an +already-completed verdict rather than performing a second unbounded full +review. If that corrective request encounters transport exhaustion, the typed +transport error retains both the first trusted-validator diagnostic and the +later transport class/status while omitting raw model content and secrets. +""" + if "### Noema model-output and repair boundary" not in architecture: + architecture += architecture_note + ARCHITECTURE.write_text(architecture, encoding="utf-8") + + baseline = BASELINE.read_text(encoding="utf-8") + baseline_note = """ + +## 2026-09-01 Noema malformed-verdict retry classification and wall-clock bound (#1611/#1617) + +- **Observed consumer evidence:** `ContextualWisdomLab/naruon#1505@7da2a242e463f59d4580cb38e7591f1ba4b4049e`, Required Noema run `33460498090` / job `99742587317`. The first response reached the trusted semantic validator but used an out-of-domain adversarial-probe `outcome`; the generic repair attempt later ended as HTTP 502 after roughly 88 minutes. +- **Root cause:** model-output/schema rejection, repair transport exhaustion, and consumer-source findings shared an undifferentiated `RuntimeError` boundary. The corrective HTTP request also had no client-side repair-specific ceiling, so a malformed first verdict could initiate another effectively full-duration request. +- **Repair:** model-output/schema rejection is typed as `NoemaModelOutputError`; the one corrective request has a 900-second hard client ceiling; repair transport exhaustion is typed as `NoemaTransportError`; and the final fail-closed diagnostic preserves the sanitized first validator error plus the later typed transport evidence. Primary review inference remains governed by contextual-orchestrator `orchestrator/free` and is not given a new fixed model-inference timeout. +- **Security/operability invariant:** raw model content, credentials, and provider secrets are never included in the combined diagnostic. Exact-head revalidation still occurs before retry and before publication. No direct-provider fallback or GitHub authority change is introduced. +- **Verification contract:** deterministic tests cover the original invalid `outcome`, malformed-then-502 evidence preservation and the repair-only timeout, and repeated malformed model output remaining typed and non-passing. The affected Naruon head must be re-run after protected integration; predecessor review/check evidence does not transfer. +""" + if "## 2026-09-01 Noema malformed-verdict retry classification" not in baseline: + baseline += baseline_note + BASELINE.write_text(baseline, encoding="utf-8") + + DOCTORING.parent.mkdir(parents=True, exist_ok=True) + DOCTORING.write_text( + """# Noema model-output repair boundary\n\n## Incident\n\nOn 2026-09-01 the required Noema review for `ContextualWisdomLab/naruon#1505` reached deterministic verdict validation, rejected an adversarial-probe `outcome` outside the closed `falsified|confirmed` domain, then spent the repair path on a long second model call that ultimately surfaced only `HTTP 502 Bad Gateway`. That final transport symptom erased the more informative first trusted-validator failure from the top-level diagnostic.\n\n## Decision\n\n1. Model-produced JSON/envelope/schema/semantic-contract failures are `NoemaModelOutputError`; they remain fail-closed and are not consumer-source findings.\n2. The primary review keeps the accepted contextual-orchestrator no-fixed-inference-timeout contract. The *single corrective request* is different: it repairs an already-completed verdict and therefore has a hard 900-second `urllib` client timeout.\n3. A corrective transport failure is `NoemaTransportError` and carries the sanitized first validator diagnostic plus the later transport exception class/status. Raw model output is never copied into public Actions diagnostics.\n4. Exact-head validation before retry and before publication remains mandatory. All model traffic remains on contextual-orchestrator `orchestrator/free`.\n\n## Verification\n\nThe #1617 regression first proved RED because `NoemaModelOutputError` did not exist. The repair adds focused cases for malformed-verdict typing, malformed-then-502 evidence preservation with the 900-second repair-only timeout, and repeated malformed output remaining typed and non-passing. The repository full coverage/docstring gate is run before the one-shot repair workflow commits the result.\n\n## References\n\nFielding, R., Nottingham, M., & Reschke, J. (2022). *HTTP semantics* (RFC 9110). Internet Engineering Task Force.\n\nPython Software Foundation. (2026). *urllib.request — Extensible library for opening URLs*. Python 3 documentation.\n""", + encoding="utf-8", + ) + + +def main() -> None: + """Apply all production, regression, and traceability changes.""" + update_source() + update_tests() + update_docs() + + +if __name__ == "__main__": + main() From 145cc5f7e81391bc90163fdc9b32cde00c4dbf73 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 00:05:16 +0900 Subject: [PATCH 03/24] chore(noema): run and retire #1617 source repair --- .../repair-noema-model-output-1617.yml | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 .github/workflows/repair-noema-model-output-1617.yml diff --git a/.github/workflows/repair-noema-model-output-1617.yml b/.github/workflows/repair-noema-model-output-1617.yml new file mode 100644 index 0000000000..f5ab8c3639 --- /dev/null +++ b/.github/workflows/repair-noema-model-output-1617.yml @@ -0,0 +1,61 @@ +name: Repair Noema model-output boundary 1617 + +on: + push: + branches: [fix/noema-model-output-retry-20260901] + +concurrency: + group: repair-noema-model-output-1617 + cancel-in-progress: true + +permissions: + contents: write + +jobs: + repair: + if: github.repository == 'ContextualWisdomLab/.github' && github.actor == 'seonghobae' + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Checkout exact repair branch + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + ref: fix/noema-model-output-retry-20260901 + fetch-depth: 0 + persist-credentials: true + + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.14' + + - name: Install repository-pinned quality tools + run: python -m pip install --require-hashes -r requirements-opencode-review-ci-hashes.txt + + - name: Apply exact source repair + run: PYTHONPATH=. python scripts/ci/repair_noema_model_output_1617.py + + - name: Verify focused RED-to-GREEN regressions + run: PYTHONPATH=. python -m pytest -q tests/test_noema_model_output_failure_classification.py tests/test_noema_review_gate.py + + - name: Verify complete suite and 100 percent source coverage + run: | + PYTHONPATH=. coverage run -m pytest tests -q + coverage report --show-missing + interrogate + git diff --check + + - name: Remove purpose-complete repair machinery + run: | + rm scripts/ci/repair_noema_model_output_1617.py + rm .github/workflows/repair-noema-model-output-1617.yml + + - name: Commit verified production repair + run: | + set -euo pipefail + git config user.name 'github-actions[bot]' + git config user.email '41898282+github-actions[bot]@users.noreply.github.com' + git add -A + test -n "$(git status --porcelain)" + git commit -m 'fix(noema): classify and bound malformed-verdict repair failures' + git push origin HEAD:fix/noema-model-output-retry-20260901 From adcf7dac4e8606f2305f641762bef3da262d1dac Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 00:07:46 +0900 Subject: [PATCH 04/24] chore(noema): use explicit 24.04 repair runner --- .github/workflows/repair-noema-model-output-1617.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/repair-noema-model-output-1617.yml b/.github/workflows/repair-noema-model-output-1617.yml index f5ab8c3639..9940718e50 100644 --- a/.github/workflows/repair-noema-model-output-1617.yml +++ b/.github/workflows/repair-noema-model-output-1617.yml @@ -14,7 +14,7 @@ permissions: jobs: repair: if: github.repository == 'ContextualWisdomLab/.github' && github.actor == 'seonghobae' - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 30 steps: - name: Checkout exact repair branch From 161f06ec955dbbd617ee74f35ea3bc74793afea1 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 00:10:37 +0900 Subject: [PATCH 05/24] test(noema): align repair timeout fixture with new contract --- .../ci/repair_noema_timeout_fixture_1617.py | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 scripts/ci/repair_noema_timeout_fixture_1617.py diff --git a/scripts/ci/repair_noema_timeout_fixture_1617.py b/scripts/ci/repair_noema_timeout_fixture_1617.py new file mode 100644 index 0000000000..8a32c1879f --- /dev/null +++ b/scripts/ci/repair_noema_timeout_fixture_1617.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +"""Update the existing Noema repair fixture for the repair-only timeout contract.""" + +from pathlib import Path + + +TEST = Path(__file__).resolve().parents[2] / "tests/test_noema_review_gate.py" + + +def main() -> None: + """Require no primary timeout and the bounded timeout on the one repair call.""" + text = TEST.read_text(encoding="utf-8") + old = ''' def open(self, request, timeout=None): + assert timeout is None + payloads.append(json.loads(request.data)) + return Response(invalid if len(payloads) == 1 else valid) +''' + new = ''' def open(self, request, timeout=None): + if payloads: + assert timeout == noema.NOEMA_REPAIR_TIMEOUT_SECONDS + else: + assert timeout is None + payloads.append(json.loads(request.data)) + return Response(invalid if len(payloads) == 1 else valid) +''' + count = text.count(old) + if count != 1: + raise RuntimeError(f"expected one repair-timeout fixture, found {count}") + TEST.write_text(text.replace(old, new, 1), encoding="utf-8") + + +if __name__ == "__main__": + main() From b4ec3b029a795dd5ca24d14149f036928691054b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 00:11:03 +0900 Subject: [PATCH 06/24] test(noema): execute repair-timeout fixture update --- .github/workflows/repair-noema-model-output-1617.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/repair-noema-model-output-1617.yml b/.github/workflows/repair-noema-model-output-1617.yml index 9940718e50..39944322ac 100644 --- a/.github/workflows/repair-noema-model-output-1617.yml +++ b/.github/workflows/repair-noema-model-output-1617.yml @@ -32,8 +32,10 @@ jobs: - name: Install repository-pinned quality tools run: python -m pip install --require-hashes -r requirements-opencode-review-ci-hashes.txt - - name: Apply exact source repair - run: PYTHONPATH=. python scripts/ci/repair_noema_model_output_1617.py + - name: Apply exact source and fixture repair + run: | + PYTHONPATH=. python scripts/ci/repair_noema_model_output_1617.py + PYTHONPATH=. python scripts/ci/repair_noema_timeout_fixture_1617.py - name: Verify focused RED-to-GREEN regressions run: PYTHONPATH=. python -m pytest -q tests/test_noema_model_output_failure_classification.py tests/test_noema_review_gate.py @@ -48,6 +50,7 @@ jobs: - name: Remove purpose-complete repair machinery run: | rm scripts/ci/repair_noema_model_output_1617.py + rm scripts/ci/repair_noema_timeout_fixture_1617.py rm .github/workflows/repair-noema-model-output-1617.yml - name: Commit verified production repair From 7a068659ac44b5b70947e2e119bc7d00b46a9ca9 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 00:18:25 +0900 Subject: [PATCH 07/24] fix(noema): preserve no-timeout repair contract --- .../repair-noema-model-output-1617.yml | 154 +++++++++++++++++- 1 file changed, 146 insertions(+), 8 deletions(-) diff --git a/.github/workflows/repair-noema-model-output-1617.yml b/.github/workflows/repair-noema-model-output-1617.yml index 39944322ac..898d940379 100644 --- a/.github/workflows/repair-noema-model-output-1617.yml +++ b/.github/workflows/repair-noema-model-output-1617.yml @@ -32,11 +32,155 @@ jobs: - name: Install repository-pinned quality tools run: python -m pip install --require-hashes -r requirements-opencode-review-ci-hashes.txt - - name: Apply exact source and fixture repair + - name: Apply exact typed-evidence repair run: | PYTHONPATH=. python scripts/ci/repair_noema_model_output_1617.py PYTHONPATH=. python scripts/ci/repair_noema_timeout_fixture_1617.py + - name: Preserve no-fixed-inference-timeout policy and remove repair machinery + run: | + PYTHONPATH=. python - <<'PY' + from pathlib import Path + + root = Path('.') + source_path = root / 'scripts/ci/noema_review_gate.py' + source = source_path.read_text(encoding='utf-8') + timeout_block = '''# A repair request corrects an already-completed model verdict; it is not a + # second unbounded full review. Fifteen minutes is the hard client-side + # ceiling for that one corrective HTTP request. The primary review remains + # governed by contextual-orchestrator rather than a fixed inference timeout. + NOEMA_REPAIR_TIMEOUT_SECONDS = 15 * 60 + + + ''' + if source.count(timeout_block) != 1: + raise RuntimeError('expected exactly one repair-timeout policy block') + source = source.replace(timeout_block, '', 1) + bounded_open = ''' if is_retry: + response_context = opener.open( # nosec B310 + request, timeout=NOEMA_REPAIR_TIMEOUT_SECONDS + ) + else: + response_context = opener.open(request) # nosec B310 + with response_context as response: + raw_bytes = response.read() + ''' + unbounded_open = ''' with opener.open(request) as response: # nosec B310 + raw_bytes = response.read() + ''' + if source.count(bounded_open) != 1: + raise RuntimeError('expected exactly one bounded repair opener block') + source = source.replace(bounded_open, unbounded_open, 1) + source = source.replace('Noema bounded repair transport was exhausted', 'Noema repair transport was exhausted') + source_path.write_text(source, encoding='utf-8') + + test_path = root / 'tests/test_noema_model_output_failure_classification.py' + tests = test_path.read_text(encoding='utf-8') + old_timeout_assert = ' assert requests[1][1]["timeout"] == gate.NOEMA_REPAIR_TIMEOUT_SECONDS\n' + if tests.count(old_timeout_assert) != 1: + raise RuntimeError('expected exactly one repair-timeout assertion') + tests = tests.replace(old_timeout_assert, ' assert requests[1][1] == {}\n', 1) + generic_marker = 'def test_retry_runtime_failure_preserves_initial_model_output_evidence' + if generic_marker not in tests: + tests += r''' + + +def test_retry_runtime_failure_preserves_initial_model_output_evidence(monkeypatch) -> None: + """An unexpected retry RuntimeError keeps the first validator diagnostic.""" + import json + + monkeypatch.setenv("NOEMA_LLM_API_URL", "https://llm.example/v1/chat/completions") + monkeypatch.setenv("NOEMA_LLM_API_KEY", "test-key") + head_sha = "c" * 40 + calls = 0 + + class Response: + def __enter__(self): + return self + + def __exit__(self, *_args): + return None + + def read(self): + return json.dumps( + {"choices": [{"message": {"content": json.dumps(_verdict())}}]} + ).encode() + + def open_response(_opener, _request, **_kwargs): + nonlocal calls + calls += 1 + if calls == 1: + return Response() + raise RuntimeError("synthetic retry runtime failure") + + monkeypatch.setattr(gate.urllib.request.OpenerDirector, "open", open_response) + monkeypatch.setattr(gate, "fetch_pr", lambda _repo, _number: {"headRefOid": head_sha}) + + with pytest.raises(RuntimeError) as exc_info: + gate.call_llm( + "owner/repo", + 7, + {"title": "test", "headRefOid": head_sha}, + DIFF, + False, + head_sha, + changed_paths=("README.md",), + ) + + message = str(exc_info.value) + assert "initial failure" in message + assert "outcome must be falsified or confirmed" in message + assert "synthetic retry runtime failure" in message + assert calls == 2 +''' + test_path.write_text(tests, encoding='utf-8') + + replacements = { + root / 'CHANGELOG.md': ( + 'The one corrective HTTP request has a 15-minute client ceiling while the primary contextual-orchestrator review remains under its no-fixed-inference-timeout contract.', + 'The single corrective request preserves the contextual-orchestrator no-fixed-inference-timeout contract while retaining the first validator diagnostic if later transport fails.', + ), + root / 'ARCHITECTURE.md': ( + 'but is capped at 15 minutes because it repairs an\nalready-completed verdict rather than performing a second unbounded full\nreview.', + 'and preserves the same no-fixed-inference-timeout policy as the primary\nreview; the retry is bounded by attempt count rather than an inference wall clock.', + ), + root / 'docs/product-technical-gap-baseline.md': ( + 'the one corrective request has a 900-second hard client ceiling;', + 'the one corrective request preserves the no-fixed-inference-timeout contract and is bounded to one repair attempt;', + ), + root / 'docs/doctoring/noema-model-output-repair-boundary.md': ( + 'The *single corrective request* is different: it repairs an already-completed verdict and therefore has a hard 900-second `urllib` client timeout.', + 'The single corrective request keeps the same no-fixed-inference-timeout contract and is bounded by allowing only one repair attempt.', + ), + } + for path, (old, new) in replacements.items(): + text = path.read_text(encoding='utf-8') + if old not in text: + raise RuntimeError(f'expected policy text missing from {path}') + path.write_text(text.replace(old, new, 1), encoding='utf-8') + + baseline = (root / 'docs/product-technical-gap-baseline.md') + text = baseline.read_text(encoding='utf-8') + text = text.replace( + 'The corrective HTTP request also had no client-side repair-specific ceiling, so a malformed first verdict could initiate another effectively full-duration request.', + 'A malformed first verdict could initiate one corrective request whose later transport failure replaced the first validator evidence in the top-level diagnostic.', + ) + text = text.replace('malformed-then-502 evidence preservation and the repair-only timeout', 'malformed-then-502 evidence preservation without a fixed inference timeout') + baseline.write_text(text, encoding='utf-8') + + doctor = root / 'docs/doctoring/noema-model-output-repair-boundary.md' + text = doctor.read_text(encoding='utf-8') + text = text.replace('with the 900-second repair-only timeout', 'while preserving the no-fixed-inference-timeout policy') + doctor.write_text(text, encoding='utf-8') + + for temporary in ( + root / 'scripts/ci/repair_noema_model_output_1617.py', + root / 'scripts/ci/repair_noema_timeout_fixture_1617.py', + root / '.github/workflows/repair-noema-model-output-1617.yml', + ): + temporary.unlink(missing_ok=True) + PY + - name: Verify focused RED-to-GREEN regressions run: PYTHONPATH=. python -m pytest -q tests/test_noema_model_output_failure_classification.py tests/test_noema_review_gate.py @@ -47,12 +191,6 @@ jobs: interrogate git diff --check - - name: Remove purpose-complete repair machinery - run: | - rm scripts/ci/repair_noema_model_output_1617.py - rm scripts/ci/repair_noema_timeout_fixture_1617.py - rm .github/workflows/repair-noema-model-output-1617.yml - - name: Commit verified production repair run: | set -euo pipefail @@ -60,5 +198,5 @@ jobs: git config user.email '41898282+github-actions[bot]@users.noreply.github.com' git add -A test -n "$(git status --porcelain)" - git commit -m 'fix(noema): classify and bound malformed-verdict repair failures' + git commit -m 'fix(noema): classify malformed verdict repair evidence' git push origin HEAD:fix/noema-model-output-retry-20260901 From 859491ef3983daac62931718930ab86400f23b22 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 00:27:31 +0900 Subject: [PATCH 08/24] test(noema): add true wall-clock repair deadline regression --- scripts/ci/repair_noema_wall_clock_1617.py | 247 +++++++++++++++++++++ 1 file changed, 247 insertions(+) create mode 100644 scripts/ci/repair_noema_wall_clock_1617.py diff --git a/scripts/ci/repair_noema_wall_clock_1617.py b/scripts/ci/repair_noema_wall_clock_1617.py new file mode 100644 index 0000000000..ae8ba72b00 --- /dev/null +++ b/scripts/ci/repair_noema_wall_clock_1617.py @@ -0,0 +1,247 @@ +#!/usr/bin/env python3 +"""Finish PR #1617 with a true repair wall-clock deadline. + +Temporary one-shot branch repair helper. The repair workflow removes this file +before committing the production change. +""" + +from pathlib import Path +import textwrap + + +ROOT = Path(__file__).resolve().parents[2] +SOURCE = ROOT / "scripts/ci/noema_review_gate.py" +TEST = ROOT / "tests/test_noema_model_output_failure_classification.py" +CHANGELOG = ROOT / "CHANGELOG.md" +BASELINE = ROOT / "docs/product-technical-gap-baseline.md" +ARCHITECTURE = ROOT / "ARCHITECTURE.md" +DOCTORING = ROOT / "docs/doctoring/noema-model-output-repair-boundary.md" + + +def replace_once(text: str, old: str, new: str, label: str) -> str: + count = text.count(old) + if count != 1: + raise RuntimeError(f"{label}: expected exactly one match, found {count}") + return text.replace(old, new, 1) + + +def update_source() -> None: + text = SOURCE.read_text(encoding="utf-8") + text = replace_once(text, "import base64\n", "import base64\nimport contextlib\n", "contextlib import") + text = replace_once(text, "import re\n", "import re\nimport signal\n", "signal import") + text = replace_once( + text, + "# A repair request corrects an already-completed model verdict; it is not a\n" + "# second unbounded full review. Fifteen minutes is the hard client-side\n" + "# ceiling for that one corrective HTTP request. The primary review remains\n" + "# governed by contextual-orchestrator rather than a fixed inference timeout.\n" + "NOEMA_REPAIR_TIMEOUT_SECONDS = 15 * 60\n", + "# A repair request corrects an already-completed model verdict; it is not a\n" + "# second unbounded full review. Fifteen minutes is an absolute wall-clock\n" + "# deadline for the complete corrective attempt (open/read/decode/validate),\n" + "# not a socket inactivity timeout. The primary review remains governed by\n" + "# contextual-orchestrator rather than a fixed inference timeout.\n" + "NOEMA_REPAIR_DEADLINE_SECONDS = 15 * 60\n", + "repair deadline constant", + ) + marker = '''class NoemaTransportError(RuntimeError): + """Raised when the bounded review transport cannot produce usable evidence.""" +''' + addition = marker + '''\n\nclass NoemaRepairDeadlineExceeded(TimeoutError): + """Raised when the corrective attempt exceeds its total wall-clock budget.""" +''' + text = replace_once(text, marker, addition, "deadline error class") + + stale_marker = '''class StaleHeadDuringRepairRetryError(RuntimeError): + """Raised when the PR head moves before ``call_llm``'s repair-retry request fires.""" +''' + deadline_helper = '''@contextlib.contextmanager +def _repair_wall_clock_deadline(seconds: float): + """Interrupt the entire corrective attempt after ``seconds`` of wall time. + + ``urllib``'s timeout is a socket-operation timeout and can be extended by + trickling bytes. Required Noema Review runs on Linux, so ITIMER_REAL gives + the repair attempt one process-level wall-clock budget across open, read, + decode, and deterministic validation. An existing process alarm is not + overwritten; that condition fails closed instead. + """ + if seconds <= 0: + raise ValueError("repair wall-clock deadline must be positive") + if not hasattr(signal, "setitimer") or not hasattr(signal, "ITIMER_REAL"): + raise RuntimeError("repair wall-clock deadline requires POSIX setitimer support") + previous_remaining, previous_interval = signal.getitimer(signal.ITIMER_REAL) + if previous_remaining > 0 or previous_interval > 0: + raise RuntimeError("repair wall-clock deadline refused to overwrite an active process alarm") + previous_handler = signal.getsignal(signal.SIGALRM) + + def expire(_signum, _frame): + raise NoemaRepairDeadlineExceeded( + f"Noema repair exceeded {seconds:g}-second absolute wall-clock deadline" + ) + + try: + signal.signal(signal.SIGALRM, expire) + except ValueError as exc: + raise RuntimeError("repair wall-clock deadline must run on the process main thread") from exc + signal.setitimer(signal.ITIMER_REAL, seconds) + try: + yield + finally: + signal.setitimer(signal.ITIMER_REAL, 0) + signal.signal(signal.SIGALRM, previous_handler) + + +''' + stale_marker + text = replace_once(text, stale_marker, deadline_helper, "deadline helper") + + old_open = ''' if is_retry: + response_context = opener.open( # nosec B310 + request, timeout=NOEMA_REPAIR_TIMEOUT_SECONDS + ) + else: + response_context = opener.open(request) # nosec B310 + with response_context as response: + raw_bytes = response.read() +''' + plain_open = ''' with opener.open(request) as response: # nosec B310 + raw_bytes = response.read() +''' + text = replace_once(text, old_open, plain_open, "remove socket timeout") + + try_marker = " try:\n with opener.open(request) as response: # nosec B310\n" + start = text.index(try_marker) + body_start = start + len(" try:\n") + except_marker = " except (RuntimeError, urllib.error.URLError, http.client.HTTPException, OSError) as exc:\n" + end = text.index(except_marker, body_start) + body = text[body_start:end] + wrapped = ( + " deadline_context = (\n" + " _repair_wall_clock_deadline(NOEMA_REPAIR_DEADLINE_SECONDS)\n" + " if is_retry\n" + " else contextlib.nullcontext()\n" + " )\n" + " with deadline_context:\n" + + textwrap.indent(body, " ") + ) + text = text[:body_start] + wrapped + text[end:] + SOURCE.write_text(text, encoding="utf-8") + + +def update_tests() -> None: + text = TEST.read_text(encoding="utf-8") + text = replace_once( + text, + ' assert requests[1][1]["timeout"] == gate.NOEMA_REPAIR_TIMEOUT_SECONDS\n', + ' assert requests[1][1] == {}\n', + "socket-timeout assertion", + ) + marker = "def test_total_repair_wall_clock_deadline_interrupts_slow_read" + if marker in text: + raise RuntimeError("wall-clock regression already present") + text += r''' + + +def test_total_repair_wall_clock_deadline_interrupts_slow_read(monkeypatch) -> None: + """Trickling/slow response activity cannot extend the one repair budget.""" + import json + import signal + import time + + if not hasattr(signal, "setitimer"): + pytest.skip("POSIX process timer is required by the Linux review runner") + + monkeypatch.setenv("NOEMA_LLM_API_URL", "https://llm.example/v1/chat/completions") + monkeypatch.setenv("NOEMA_LLM_API_KEY", "test-key") + monkeypatch.setattr(gate, "NOEMA_REPAIR_DEADLINE_SECONDS", 0.05) + head_sha = "d" * 40 + calls = 0 + + class FirstResponse: + def __enter__(self): + return self + + def __exit__(self, *_args): + return None + + def read(self): + return json.dumps( + {"choices": [{"message": {"content": json.dumps(_verdict())}}]} + ).encode() + + class SlowRepairResponse: + def __enter__(self): + return self + + def __exit__(self, *_args): + return None + + def read(self): + time.sleep(2) + return b"{}" + + def open_response(_opener, _request, **kwargs): + nonlocal calls + calls += 1 + assert kwargs == {} + return FirstResponse() if calls == 1 else SlowRepairResponse() + + monkeypatch.setattr(gate.urllib.request.OpenerDirector, "open", open_response) + monkeypatch.setattr(gate, "fetch_pr", lambda _repo, _number: {"headRefOid": head_sha}) + + started = time.monotonic() + with pytest.raises(gate.NoemaTransportError) as exc_info: + gate.call_llm( + "owner/repo", + 7, + {"title": "test", "headRefOid": head_sha}, + DIFF, + False, + head_sha, + changed_paths=("README.md",), + ) + elapsed = time.monotonic() - started + + message = str(exc_info.value) + assert "outcome must be falsified or confirmed" in message + assert "NoemaRepairDeadlineExceeded" in message + assert "wall-clock deadline" in message + assert elapsed < 1.0 + assert calls == 2 + assert signal.getitimer(signal.ITIMER_REAL)[0] == 0 +''' + TEST.write_text(text, encoding="utf-8") + + +def update_docs() -> None: + replacements = { + CHANGELOG: ( + "The one corrective HTTP request has a 15-minute client ceiling while the primary contextual-orchestrator review remains under its no-fixed-inference-timeout contract.", + "The one corrective attempt has a 15-minute absolute wall-clock deadline across open/read/decode/validation while the primary contextual-orchestrator review remains under its no-fixed-inference-timeout contract; unlike a urllib socket timeout, trickling response activity cannot renew that budget.", + ), + ARCHITECTURE: ( + "but is capped at 15 minutes because it repairs an\nalready-completed verdict rather than performing a second unbounded full\nreview.", + "but has one 15-minute process-level wall-clock deadline across open, read,\ndecode, and deterministic validation because it repairs an already-completed\nverdict rather than performing a second unbounded full review. This is not a\nsocket inactivity timeout, so response activity cannot renew the budget.", + ), + BASELINE: ( + "the one corrective request has a 900-second hard client ceiling;", + "the one corrective attempt has a 900-second absolute wall-clock deadline across open/read/decode/validation (not a renewable socket timeout);", + ), + DOCTORING: ( + "The *single corrective request* is different: it repairs an already-completed verdict and therefore has a hard 900-second `urllib` client timeout.", + "The *single corrective attempt* is different: it repairs an already-completed verdict and therefore has one 900-second process-level wall-clock deadline across open/read/decode/validation. It deliberately does not use `urllib`'s renewable socket-operation timeout.", + ), + } + for path, (old, new) in replacements.items(): + text = path.read_text(encoding="utf-8") + text = replace_once(text, old, new, str(path)) + path.write_text(text, encoding="utf-8") + + +def main() -> None: + update_source() + update_tests() + update_docs() + + +if __name__ == "__main__": + main() From d7a78c79f2df0c9510a61a95315757a5c5a553a2 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 00:27:53 +0900 Subject: [PATCH 09/24] fix(ci): make Noema repair workflow fail-closed and self-cleaning --- .../repair-noema-model-output-1617.yml | 172 +++--------------- 1 file changed, 21 insertions(+), 151 deletions(-) diff --git a/.github/workflows/repair-noema-model-output-1617.yml b/.github/workflows/repair-noema-model-output-1617.yml index 898d940379..8aeaa1f958 100644 --- a/.github/workflows/repair-noema-model-output-1617.yml +++ b/.github/workflows/repair-noema-model-output-1617.yml @@ -1,8 +1,9 @@ -name: Repair Noema model-output boundary 1617 +name: TEMP repair Noema model-output boundary 1617 on: push: - branches: [fix/noema-model-output-retry-20260901] + branches: + - fix/noema-model-output-retry-20260901 concurrency: group: repair-noema-model-output-1617 @@ -13,9 +14,8 @@ permissions: jobs: repair: - if: github.repository == 'ContextualWisdomLab/.github' && github.actor == 'seonghobae' runs-on: ubuntu-24.04 - timeout-minutes: 30 + timeout-minutes: 60 steps: - name: Checkout exact repair branch uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 @@ -32,164 +32,34 @@ jobs: - name: Install repository-pinned quality tools run: python -m pip install --require-hashes -r requirements-opencode-review-ci-hashes.txt - - name: Apply exact typed-evidence repair + - name: Apply typed-evidence repair and true wall-clock deadline run: | + set -euo pipefail PYTHONPATH=. python scripts/ci/repair_noema_model_output_1617.py - PYTHONPATH=. python scripts/ci/repair_noema_timeout_fixture_1617.py + PYTHONPATH=. python scripts/ci/repair_noema_wall_clock_1617.py - - name: Preserve no-fixed-inference-timeout policy and remove repair machinery + - name: Remove temporary repair machinery before verification run: | - PYTHONPATH=. python - <<'PY' - from pathlib import Path - - root = Path('.') - source_path = root / 'scripts/ci/noema_review_gate.py' - source = source_path.read_text(encoding='utf-8') - timeout_block = '''# A repair request corrects an already-completed model verdict; it is not a - # second unbounded full review. Fifteen minutes is the hard client-side - # ceiling for that one corrective HTTP request. The primary review remains - # governed by contextual-orchestrator rather than a fixed inference timeout. - NOEMA_REPAIR_TIMEOUT_SECONDS = 15 * 60 - - - ''' - if source.count(timeout_block) != 1: - raise RuntimeError('expected exactly one repair-timeout policy block') - source = source.replace(timeout_block, '', 1) - bounded_open = ''' if is_retry: - response_context = opener.open( # nosec B310 - request, timeout=NOEMA_REPAIR_TIMEOUT_SECONDS - ) - else: - response_context = opener.open(request) # nosec B310 - with response_context as response: - raw_bytes = response.read() - ''' - unbounded_open = ''' with opener.open(request) as response: # nosec B310 - raw_bytes = response.read() - ''' - if source.count(bounded_open) != 1: - raise RuntimeError('expected exactly one bounded repair opener block') - source = source.replace(bounded_open, unbounded_open, 1) - source = source.replace('Noema bounded repair transport was exhausted', 'Noema repair transport was exhausted') - source_path.write_text(source, encoding='utf-8') - - test_path = root / 'tests/test_noema_model_output_failure_classification.py' - tests = test_path.read_text(encoding='utf-8') - old_timeout_assert = ' assert requests[1][1]["timeout"] == gate.NOEMA_REPAIR_TIMEOUT_SECONDS\n' - if tests.count(old_timeout_assert) != 1: - raise RuntimeError('expected exactly one repair-timeout assertion') - tests = tests.replace(old_timeout_assert, ' assert requests[1][1] == {}\n', 1) - generic_marker = 'def test_retry_runtime_failure_preserves_initial_model_output_evidence' - if generic_marker not in tests: - tests += r''' - - -def test_retry_runtime_failure_preserves_initial_model_output_evidence(monkeypatch) -> None: - """An unexpected retry RuntimeError keeps the first validator diagnostic.""" - import json - - monkeypatch.setenv("NOEMA_LLM_API_URL", "https://llm.example/v1/chat/completions") - monkeypatch.setenv("NOEMA_LLM_API_KEY", "test-key") - head_sha = "c" * 40 - calls = 0 - - class Response: - def __enter__(self): - return self - - def __exit__(self, *_args): - return None + rm -f scripts/ci/repair_noema_model_output_1617.py + rm -f scripts/ci/repair_noema_timeout_fixture_1617.py + rm -f scripts/ci/repair_noema_wall_clock_1617.py + rm -f .github/workflows/repair-noema-model-output-1617.yml + test ! -e .github/workflows/repair-noema-model-output-1617.yml - def read(self): - return json.dumps( - {"choices": [{"message": {"content": json.dumps(_verdict())}}]} - ).encode() - - def open_response(_opener, _request, **_kwargs): - nonlocal calls - calls += 1 - if calls == 1: - return Response() - raise RuntimeError("synthetic retry runtime failure") - - monkeypatch.setattr(gate.urllib.request.OpenerDirector, "open", open_response) - monkeypatch.setattr(gate, "fetch_pr", lambda _repo, _number: {"headRefOid": head_sha}) - - with pytest.raises(RuntimeError) as exc_info: - gate.call_llm( - "owner/repo", - 7, - {"title": "test", "headRefOid": head_sha}, - DIFF, - False, - head_sha, - changed_paths=("README.md",), - ) - - message = str(exc_info.value) - assert "initial failure" in message - assert "outcome must be falsified or confirmed" in message - assert "synthetic retry runtime failure" in message - assert calls == 2 -''' - test_path.write_text(tests, encoding='utf-8') - - replacements = { - root / 'CHANGELOG.md': ( - 'The one corrective HTTP request has a 15-minute client ceiling while the primary contextual-orchestrator review remains under its no-fixed-inference-timeout contract.', - 'The single corrective request preserves the contextual-orchestrator no-fixed-inference-timeout contract while retaining the first validator diagnostic if later transport fails.', - ), - root / 'ARCHITECTURE.md': ( - 'but is capped at 15 minutes because it repairs an\nalready-completed verdict rather than performing a second unbounded full\nreview.', - 'and preserves the same no-fixed-inference-timeout policy as the primary\nreview; the retry is bounded by attempt count rather than an inference wall clock.', - ), - root / 'docs/product-technical-gap-baseline.md': ( - 'the one corrective request has a 900-second hard client ceiling;', - 'the one corrective request preserves the no-fixed-inference-timeout contract and is bounded to one repair attempt;', - ), - root / 'docs/doctoring/noema-model-output-repair-boundary.md': ( - 'The *single corrective request* is different: it repairs an already-completed verdict and therefore has a hard 900-second `urllib` client timeout.', - 'The single corrective request keeps the same no-fixed-inference-timeout contract and is bounded by allowing only one repair attempt.', - ), - } - for path, (old, new) in replacements.items(): - text = path.read_text(encoding='utf-8') - if old not in text: - raise RuntimeError(f'expected policy text missing from {path}') - path.write_text(text.replace(old, new, 1), encoding='utf-8') - - baseline = (root / 'docs/product-technical-gap-baseline.md') - text = baseline.read_text(encoding='utf-8') - text = text.replace( - 'The corrective HTTP request also had no client-side repair-specific ceiling, so a malformed first verdict could initiate another effectively full-duration request.', - 'A malformed first verdict could initiate one corrective request whose later transport failure replaced the first validator evidence in the top-level diagnostic.', - ) - text = text.replace('malformed-then-502 evidence preservation and the repair-only timeout', 'malformed-then-502 evidence preservation without a fixed inference timeout') - baseline.write_text(text, encoding='utf-8') - - doctor = root / 'docs/doctoring/noema-model-output-repair-boundary.md' - text = doctor.read_text(encoding='utf-8') - text = text.replace('with the 900-second repair-only timeout', 'while preserving the no-fixed-inference-timeout policy') - doctor.write_text(text, encoding='utf-8') - - for temporary in ( - root / 'scripts/ci/repair_noema_model_output_1617.py', - root / 'scripts/ci/repair_noema_timeout_fixture_1617.py', - root / '.github/workflows/repair-noema-model-output-1617.yml', - ): - temporary.unlink(missing_ok=True) - PY - - - name: Verify focused RED-to-GREEN regressions + - name: Verify focused Noema regressions run: PYTHONPATH=. python -m pytest -q tests/test_noema_model_output_failure_classification.py tests/test_noema_review_gate.py - - name: Verify complete suite and 100 percent source coverage + - name: Verify complete suite, source coverage, docs, and diff hygiene run: | + set -euo pipefail PYTHONPATH=. coverage run -m pytest tests -q coverage report --show-missing interrogate git diff --check + test ! -e .github/workflows/repair-noema-model-output-1617.yml + test ! -e scripts/ci/repair_noema_model_output_1617.py + test ! -e scripts/ci/repair_noema_timeout_fixture_1617.py + test ! -e scripts/ci/repair_noema_wall_clock_1617.py - name: Commit verified production repair run: | @@ -198,5 +68,5 @@ def test_retry_runtime_failure_preserves_initial_model_output_evidence(monkeypat git config user.email '41898282+github-actions[bot]@users.noreply.github.com' git add -A test -n "$(git status --porcelain)" - git commit -m 'fix(noema): classify malformed verdict repair evidence' + git commit -m 'fix(noema): bound malformed-verdict repair by wall clock' git push origin HEAD:fix/noema-model-output-retry-20260901 From 59e3151929cbf12cdfbc772355d433a562e624c0 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 00:38:30 +0900 Subject: [PATCH 10/24] test(noema): close repair deadline coverage gaps --- scripts/ci/repair_noema_coverage_1617.py | 131 +++++++++++++++++++++++ 1 file changed, 131 insertions(+) create mode 100644 scripts/ci/repair_noema_coverage_1617.py diff --git a/scripts/ci/repair_noema_coverage_1617.py b/scripts/ci/repair_noema_coverage_1617.py new file mode 100644 index 0000000000..c52a7349bb --- /dev/null +++ b/scripts/ci/repair_noema_coverage_1617.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python3 +"""Add fail-closed coverage for PR #1617's temporary production transform. + +This one-shot helper is removed by the repair workflow before the verified +production commit is created. +""" + +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] +TEST = ROOT / "tests/test_noema_model_output_failure_classification.py" + + +def main() -> None: + text = TEST.read_text(encoding="utf-8") + marker = "def test_repair_wall_clock_deadline_defensive_fail_closed_paths" + if marker in text: + raise RuntimeError("#1617 deadline coverage regressions already present") + text += r''' + + +def test_repair_wall_clock_deadline_defensive_fail_closed_paths(monkeypatch) -> None: + """Invalid budgets/platform state fail closed instead of weakening the bound.""" + import signal + + with pytest.raises(ValueError, match="must be positive"): + with gate._repair_wall_clock_deadline(0): + pass + + if not hasattr(signal, "setitimer"): + pytest.skip("remaining cases require POSIX setitimer") + + monkeypatch.delattr(gate.signal, "setitimer") + with pytest.raises(RuntimeError, match="requires POSIX setitimer support"): + with gate._repair_wall_clock_deadline(1): + pass + + +def test_repair_wall_clock_deadline_refuses_existing_process_alarm() -> None: + """Noema never overwrites another caller's active process alarm.""" + import signal + + if not hasattr(signal, "setitimer"): + pytest.skip("POSIX process timer is required by the Linux review runner") + signal.setitimer(signal.ITIMER_REAL, 30) + try: + with pytest.raises(RuntimeError, match="refused to overwrite"): + with gate._repair_wall_clock_deadline(1): + pass + finally: + signal.setitimer(signal.ITIMER_REAL, 0) + + +def test_repair_wall_clock_deadline_rejects_non_main_thread_signal_context(monkeypatch) -> None: + """A signal handler that cannot be installed fails closed before any timer starts.""" + import signal + + if not hasattr(signal, "setitimer"): + pytest.skip("POSIX process timer is required by the Linux review runner") + + def reject_signal(*_args, **_kwargs): + raise ValueError("signal only works in main thread") + + monkeypatch.setattr(gate.signal, "signal", reject_signal) + with pytest.raises(RuntimeError, match="process main thread"): + with gate._repair_wall_clock_deadline(1): + pass + assert signal.getitimer(signal.ITIMER_REAL)[0] == 0 + + +def test_repair_unexpected_runtime_failure_preserves_initial_model_evidence(monkeypatch) -> None: + """Unexpected corrective parser/runtime failures keep the first trusted diagnostic.""" + import json + + monkeypatch.setenv("NOEMA_LLM_API_URL", "https://llm.example/v1/chat/completions") + monkeypatch.setenv("NOEMA_LLM_API_KEY", "test-key") + head_sha = "e" * 40 + + class Response: + def __enter__(self): + return self + + def __exit__(self, *_args): + return None + + def read(self): + return json.dumps( + {"choices": [{"message": {"content": json.dumps(_verdict())}}]} + ).encode() + + monkeypatch.setattr( + gate.urllib.request.OpenerDirector, + "open", + lambda *_args, **_kwargs: Response(), + ) + monkeypatch.setattr(gate, "fetch_pr", lambda _repo, _number: {"headRefOid": head_sha}) + original_decode = gate.decode_llm_response_body + decode_calls = 0 + + def decode_once_then_fail(raw_bytes): + nonlocal decode_calls + decode_calls += 1 + if decode_calls == 2: + raise RuntimeError("repair parser invariant failed") + return original_decode(raw_bytes) + + monkeypatch.setattr(gate, "decode_llm_response_body", decode_once_then_fail) + + with pytest.raises(RuntimeError) as exc_info: + gate.call_llm( + "owner/repo", + 7, + {"title": "test", "headRefOid": head_sha}, + DIFF, + False, + head_sha, + changed_paths=("README.md",), + ) + + message = str(exc_info.value) + assert "Noema repair failed closed" in message + assert "outcome must be falsified or confirmed" in message + assert "repair parser invariant failed" in message + assert decode_calls == 2 +''' + TEST.write_text(text, encoding="utf-8") + + +if __name__ == "__main__": + main() From f380904a0eebb1b9cdf7236c5ecf577af6abcfab Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 00:38:46 +0900 Subject: [PATCH 11/24] fix(ci): verify all Noema repair deadline branches --- .github/workflows/repair-noema-model-output-1617.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/repair-noema-model-output-1617.yml b/.github/workflows/repair-noema-model-output-1617.yml index 8aeaa1f958..d036be0687 100644 --- a/.github/workflows/repair-noema-model-output-1617.yml +++ b/.github/workflows/repair-noema-model-output-1617.yml @@ -37,12 +37,14 @@ jobs: set -euo pipefail PYTHONPATH=. python scripts/ci/repair_noema_model_output_1617.py PYTHONPATH=. python scripts/ci/repair_noema_wall_clock_1617.py + PYTHONPATH=. python scripts/ci/repair_noema_coverage_1617.py - name: Remove temporary repair machinery before verification run: | rm -f scripts/ci/repair_noema_model_output_1617.py rm -f scripts/ci/repair_noema_timeout_fixture_1617.py rm -f scripts/ci/repair_noema_wall_clock_1617.py + rm -f scripts/ci/repair_noema_coverage_1617.py rm -f .github/workflows/repair-noema-model-output-1617.yml test ! -e .github/workflows/repair-noema-model-output-1617.yml @@ -60,6 +62,7 @@ jobs: test ! -e scripts/ci/repair_noema_model_output_1617.py test ! -e scripts/ci/repair_noema_timeout_fixture_1617.py test ! -e scripts/ci/repair_noema_wall_clock_1617.py + test ! -e scripts/ci/repair_noema_coverage_1617.py - name: Commit verified production repair run: | From fbe60245000ec6f6668f460b3268949436ac5167 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 00:39:56 +0900 Subject: [PATCH 12/24] fix(noema): close reviewed model-output repair gaps --- ...repair_noema_model_output_followup_1617.py | 194 ++++++++++++++++++ 1 file changed, 194 insertions(+) create mode 100644 scripts/ci/repair_noema_model_output_followup_1617.py diff --git a/scripts/ci/repair_noema_model_output_followup_1617.py b/scripts/ci/repair_noema_model_output_followup_1617.py new file mode 100644 index 0000000000..b5a0ee91d4 --- /dev/null +++ b/scripts/ci/repair_noema_model_output_followup_1617.py @@ -0,0 +1,194 @@ +#!/usr/bin/env python3 +"""Close the remaining reviewed #1617 model-output and coverage gaps. + +Temporary exact-head repair helper. The branch workflow removes this file before +verification and the production commit. +""" + +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +SOURCE = ROOT / "scripts/ci/noema_review_gate.py" +TEST = ROOT / "tests/test_noema_model_output_failure_classification.py" + + +def replace_once(text: str, old: str, new: str, label: str) -> str: + count = text.count(old) + if count != 1: + raise RuntimeError(f"{label}: expected exactly one match, found {count}") + return text.replace(old, new, 1) + + +def update_source() -> None: + text = SOURCE.read_text(encoding="utf-8") + + # A missing/invalid trusted diff is source evidence, not model output. + text = replace_once( + text, + ' raise NoemaModelOutputError("Noema formal verdict requires parseable changed-line evidence")\n', + ' raise RuntimeError("Noema formal verdict requires parseable changed-line evidence")\n', + "trusted diff classification", + ) + + deadline_class = '''class NoemaRepairDeadlineExceeded(TimeoutError): + """Raised when the corrective attempt exceeds its total wall-clock budget.""" +''' + diagnostic_helper = deadline_class + '''\n\ndef _stable_failure_diagnostic(exc: BaseException) -> str: + """Return bounded diagnostics without reflecting model-controlled text.""" + if isinstance(exc, NoemaModelOutputError): + return "model-output-contract-invalid" + return scrub_sensitive_data(str(exc)) or type(exc).__name__ +''' + text = replace_once( + text, + deadline_class, + diagnostic_helper, + "stable model-output diagnostic helper", + ) + + text = replace_once( + text, + ' current_failure = scrub_sensitive_data(str(exc)) or type(exc).__name__\n', + ' current_failure = _stable_failure_diagnostic(exc)\n', + "stable current failure diagnostic", + ) + + # Do not retain a model-controlled exception as an explicit cause: a raw + # unsupported decision/probe sentinel must not reappear in traceback output. + old_raise = ''' raise NoemaModelOutputError( + "Noema model-output repair remained invalid; " + f"initial failure: {initial_failure}; repair failure: {current_failure}" + ) from exc +''' + new_raise = ''' raise NoemaModelOutputError( + "Noema model-output repair remained invalid; " + f"initial failure: {initial_failure}; repair failure: {current_failure}" + ) from None +''' + text = replace_once(text, old_raise, new_raise, "model-output exception chaining") + SOURCE.write_text(text, encoding="utf-8") + + +def update_tests() -> None: + text = TEST.read_text(encoding="utf-8") + marker = "def test_unparseable_diff_remains_source_evidence" + if marker in text: + raise RuntimeError("follow-up #1617 regressions already present") + text += r''' + + +def test_unparseable_diff_remains_source_evidence() -> None: + """A location-free trusted diff is not retyped as model-output failure.""" + with pytest.raises(RuntimeError) as exc_info: + gate.validate_substantive_verdict(_verdict(), "not a unified diff", ["README.md"]) + assert not isinstance(exc_info.value, gate.NoemaModelOutputError) + assert "parseable changed-line evidence" in str(exc_info.value) + + +def test_model_sentinel_never_reaches_repair_prompt_or_final_diagnostic(monkeypatch) -> None: + """Model-controlled invalid values are replaced by a stable validator code.""" + import json + + sentinel = "MODEL_SENTINEL_DO_NOT_REFLECT" + monkeypatch.setenv("NOEMA_LLM_API_URL", "https://llm.example/v1/chat/completions") + monkeypatch.setenv("NOEMA_LLM_API_KEY", "test-key") + head_sha = "e" * 40 + requests = [] + + class Response: + def __enter__(self): + return self + + def __exit__(self, *_args): + return None + + def read(self): + return json.dumps( + {"choices": [{"message": {"content": json.dumps({"decision": sentinel})}}]} + ).encode() + + def open_response(_opener, request, **kwargs): + assert kwargs == {} + requests.append(request) + return Response() + + monkeypatch.setattr(gate.urllib.request.OpenerDirector, "open", open_response) + monkeypatch.setattr(gate, "fetch_pr", lambda _repo, _number: {"headRefOid": head_sha}) + + with pytest.raises(gate.NoemaModelOutputError) as exc_info: + gate.call_llm( + "owner/repo", + 7, + {"title": "test", "headRefOid": head_sha}, + DIFF, + False, + head_sha, + changed_paths=("README.md",), + ) + + assert len(requests) == 2 + repair_payload = requests[1].data.decode("utf-8") + assert sentinel not in repair_payload + assert "model-output-contract-invalid" in repair_payload + assert sentinel not in str(exc_info.value) + assert "model-output-contract-invalid" in str(exc_info.value) + assert exc_info.value.__cause__ is None + + +def test_stable_failure_diagnostic_keeps_transport_class_without_model_text() -> None: + """Trusted transport diagnostics remain useful while model text stays opaque.""" + assert gate._stable_failure_diagnostic(gate.NoemaModelOutputError("secret-ish model text")) == ( + "model-output-contract-invalid" + ) + assert gate._stable_failure_diagnostic(TimeoutError()) == "TimeoutError" + + +def test_repair_deadline_rejects_nonpositive_budget() -> None: + with pytest.raises(ValueError, match="must be positive"): + with gate._repair_wall_clock_deadline(0): + pass + + +def test_repair_deadline_requires_setitimer(monkeypatch) -> None: + monkeypatch.delattr(gate.signal, "setitimer") + with pytest.raises(RuntimeError, match="requires POSIX"): + with gate._repair_wall_clock_deadline(1): + pass + + +def test_repair_deadline_requires_itimer_real(monkeypatch) -> None: + monkeypatch.delattr(gate.signal, "ITIMER_REAL") + with pytest.raises(RuntimeError, match="requires POSIX"): + with gate._repair_wall_clock_deadline(1): + pass + + +@pytest.mark.parametrize("timer_state", [(1.0, 0.0), (0.0, 1.0)]) +def test_repair_deadline_refuses_existing_process_alarm(monkeypatch, timer_state) -> None: + monkeypatch.setattr(gate.signal, "getitimer", lambda _which: timer_state) + with pytest.raises(RuntimeError, match="active process alarm"): + with gate._repair_wall_clock_deadline(1): + pass + + +def test_repair_deadline_requires_main_thread_signal_registration(monkeypatch) -> None: + monkeypatch.setattr(gate.signal, "getitimer", lambda _which: (0.0, 0.0)) + + def reject_signal(*_args): + raise ValueError("signal only works in main thread") + + monkeypatch.setattr(gate.signal, "signal", reject_signal) + with pytest.raises(RuntimeError, match="process main thread"): + with gate._repair_wall_clock_deadline(1): + pass +''' + TEST.write_text(text, encoding="utf-8") + + +def main() -> None: + update_source() + update_tests() + + +if __name__ == "__main__": + main() From 8721ccf46b445ab7c8a0fde319898130e85df721 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 00:41:10 +0900 Subject: [PATCH 13/24] fix(ci): bind Noema repair to exact head and reviewed scope --- .../repair-noema-model-output-1617.yml | 86 +++++++++++++++++-- 1 file changed, 77 insertions(+), 9 deletions(-) diff --git a/.github/workflows/repair-noema-model-output-1617.yml b/.github/workflows/repair-noema-model-output-1617.yml index d036be0687..d314472fde 100644 --- a/.github/workflows/repair-noema-model-output-1617.yml +++ b/.github/workflows/repair-noema-model-output-1617.yml @@ -17,12 +17,19 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 60 steps: - - name: Checkout exact repair branch + - name: Checkout triggering repair head without credentials uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: - ref: fix/noema-model-output-retry-20260901 + ref: ${{ github.sha }} fetch-depth: 0 - persist-credentials: true + persist-credentials: false + + - name: Bind the single-writer branch to the triggering head + run: | + set -euo pipefail + writer_ref='refs/heads/fix/noema-model-output-retry-20260901' + remote_head="$(git ls-remote origin "$writer_ref" | awk '{print $1}')" + test "$remote_head" = "$GITHUB_SHA" - name: Set up Python uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 @@ -32,12 +39,13 @@ jobs: - name: Install repository-pinned quality tools run: python -m pip install --require-hashes -r requirements-opencode-review-ci-hashes.txt - - name: Apply typed-evidence repair and true wall-clock deadline + - name: Apply typed-evidence, deadline, coverage, and reviewed follow-up repairs run: | set -euo pipefail PYTHONPATH=. python scripts/ci/repair_noema_model_output_1617.py PYTHONPATH=. python scripts/ci/repair_noema_wall_clock_1617.py PYTHONPATH=. python scripts/ci/repair_noema_coverage_1617.py + PYTHONPATH=. python scripts/ci/repair_noema_model_output_followup_1617.py - name: Remove temporary repair machinery before verification run: | @@ -45,9 +53,43 @@ jobs: rm -f scripts/ci/repair_noema_timeout_fixture_1617.py rm -f scripts/ci/repair_noema_wall_clock_1617.py rm -f scripts/ci/repair_noema_coverage_1617.py + rm -f scripts/ci/repair_noema_model_output_followup_1617.py rm -f .github/workflows/repair-noema-model-output-1617.yml test ! -e .github/workflows/repair-noema-model-output-1617.yml + - name: Verify repair scope and required semantic targets + run: | + set -euo pipefail + python - <<'PY' + import subprocess + allowed = { + '.github/workflows/repair-noema-model-output-1617.yml', + 'ARCHITECTURE.md', + 'CHANGELOG.md', + 'docs/doctoring/noema-model-output-repair-boundary.md', + 'docs/product-technical-gap-baseline.md', + 'scripts/ci/noema_review_gate.py', + 'scripts/ci/repair_noema_coverage_1617.py', + 'scripts/ci/repair_noema_model_output_1617.py', + 'scripts/ci/repair_noema_model_output_followup_1617.py', + 'scripts/ci/repair_noema_timeout_fixture_1617.py', + 'scripts/ci/repair_noema_wall_clock_1617.py', + 'tests/test_noema_model_output_failure_classification.py', + } + changed = set(subprocess.check_output(['git', 'diff', '--name-only'], text=True).splitlines()) + unexpected = changed - allowed + if unexpected: + raise SystemExit(f'unexpected repair paths: {sorted(unexpected)}') + required = { + 'scripts/ci/noema_review_gate.py', + 'tests/test_noema_model_output_failure_classification.py', + } + missing = required - changed + if missing: + raise SystemExit(f'required repair targets unchanged: {sorted(missing)}') + print('verified repair scope:', *sorted(changed), sep='\n- ') + PY + - name: Verify focused Noema regressions run: PYTHONPATH=. python -m pytest -q tests/test_noema_model_output_failure_classification.py tests/test_noema_review_gate.py @@ -63,13 +105,39 @@ jobs: test ! -e scripts/ci/repair_noema_timeout_fixture_1617.py test ! -e scripts/ci/repair_noema_wall_clock_1617.py test ! -e scripts/ci/repair_noema_coverage_1617.py + test ! -e scripts/ci/repair_noema_model_output_followup_1617.py - - name: Commit verified production repair + - name: Commit verified production repair with an allowlisted scope run: | set -euo pipefail git config user.name 'github-actions[bot]' git config user.email '41898282+github-actions[bot]@users.noreply.github.com' - git add -A - test -n "$(git status --porcelain)" - git commit -m 'fix(noema): bound malformed-verdict repair by wall clock' - git push origin HEAD:fix/noema-model-output-retry-20260901 + git add -A -- \ + .github/workflows/repair-noema-model-output-1617.yml \ + ARCHITECTURE.md \ + CHANGELOG.md \ + docs/doctoring/noema-model-output-repair-boundary.md \ + docs/product-technical-gap-baseline.md \ + scripts/ci/noema_review_gate.py \ + scripts/ci/repair_noema_coverage_1617.py \ + scripts/ci/repair_noema_model_output_1617.py \ + scripts/ci/repair_noema_model_output_followup_1617.py \ + scripts/ci/repair_noema_timeout_fixture_1617.py \ + scripts/ci/repair_noema_wall_clock_1617.py \ + tests/test_noema_model_output_failure_classification.py + git diff --cached --check + test -z "$(git diff --name-only)" + git diff --cached --name-only | grep -Fx 'scripts/ci/noema_review_gate.py' + git diff --cached --name-only | grep -Fx 'tests/test_noema_model_output_failure_classification.py' + git commit -m 'fix(noema): bound and classify malformed-verdict repair' + + - name: Guard and push only the verified exact-head commit + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + writer_ref='refs/heads/fix/noema-model-output-retry-20260901' + remote_head="$(git ls-remote origin "$writer_ref" | awk '{print $1}')" + test "$remote_head" = "$GITHUB_SHA" + git push "https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" \ + HEAD:fix/noema-model-output-retry-20260901 From d8bb1ffe47f316afc3dcc01415be0b2412788c8d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 00:46:26 +0900 Subject: [PATCH 14/24] fix(ci): harden Noema 1617 repair writer boundary --- .../workflows/repair-noema-model-output-1617.yml | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/.github/workflows/repair-noema-model-output-1617.yml b/.github/workflows/repair-noema-model-output-1617.yml index d314472fde..bfb50f6141 100644 --- a/.github/workflows/repair-noema-model-output-1617.yml +++ b/.github/workflows/repair-noema-model-output-1617.yml @@ -10,10 +10,14 @@ concurrency: cancel-in-progress: true permissions: - contents: write + contents: read jobs: repair: + if: >- + github.repository == 'ContextualWisdomLab/.github' && + github.actor == 'seonghobae' && + github.triggering_actor == 'seonghobae' runs-on: ubuntu-24.04 timeout-minutes: 60 steps: @@ -133,11 +137,13 @@ jobs: - name: Guard and push only the verified exact-head commit env: - GH_TOKEN: ${{ github.token }} + GH_TOKEN: ${{ secrets.PR_REVIEW_MERGE_TOKEN || secrets.OPENCODE_APPROVE_TOKEN }} run: | set -euo pipefail - writer_ref='refs/heads/fix/noema-model-output-retry-20260901' + test -n "${GH_TOKEN:-}" + writer_branch='fix/noema-model-output-retry-20260901' + writer_ref="refs/heads/${writer_branch}" remote_head="$(git ls-remote origin "$writer_ref" | awk '{print $1}')" test "$remote_head" = "$GITHUB_SHA" - git push "https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" \ - HEAD:fix/noema-model-output-retry-20260901 + gh auth setup-git + git push origin HEAD:"$writer_branch" From ed2a5fe30dde6c10fa193381b34b4273003f735c Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 00:56:01 +0900 Subject: [PATCH 15/24] fix(noema): align stable repair diagnostics --- ...repair_noema_model_output_followup_1617.py | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/scripts/ci/repair_noema_model_output_followup_1617.py b/scripts/ci/repair_noema_model_output_followup_1617.py index b5a0ee91d4..de7ec18bed 100644 --- a/scripts/ci/repair_noema_model_output_followup_1617.py +++ b/scripts/ci/repair_noema_model_output_followup_1617.py @@ -10,6 +10,10 @@ ROOT = Path(__file__).resolve().parents[2] SOURCE = ROOT / "scripts/ci/noema_review_gate.py" TEST = ROOT / "tests/test_noema_model_output_failure_classification.py" +CHANGELOG = ROOT / "CHANGELOG.md" +ARCHITECTURE = ROOT / "ARCHITECTURE.md" +BASELINE = ROOT / "docs/product-technical-gap-baseline.md" +DOCTORING = ROOT / "docs/doctoring/noema-model-output-repair-boundary.md" def replace_once(text: str, old: str, new: str, label: str) -> str: @@ -19,6 +23,16 @@ def replace_once(text: str, old: str, new: str, label: str) -> str: return text.replace(old, new, 1) +def replace_exact_count( + text: str, old: str, new: str, expected_count: int, label: str +) -> str: + """Replace a reviewed generated fragment only when its multiplicity is exact.""" + count = text.count(old) + if count != expected_count: + raise RuntimeError(f"{label}: expected {expected_count} matches, found {count}") + return text.replace(old, new) + + def update_source() -> None: text = SOURCE.read_text(encoding="utf-8") @@ -71,6 +85,19 @@ def update_source() -> None: def update_tests() -> None: text = TEST.read_text(encoding="utf-8") + + # Three earlier one-shot transforms assert the model-controlled validator + # detail after call_llm(). The final contract intentionally exposes only a + # stable trusted code at that boundary; the direct validator regression at + # the top of the file keeps its detailed assertion. + text = replace_exact_count( + text, + ' assert "outcome must be falsified or confirmed" in message\n', + ' assert "model-output-contract-invalid" in message\n', + 3, + "generated call_llm stable-diagnostic assertions", + ) + marker = "def test_unparseable_diff_remains_source_evidence" if marker in text: raise RuntimeError("follow-up #1617 regressions already present") @@ -185,9 +212,36 @@ def reject_signal(*_args): TEST.write_text(text, encoding="utf-8") +def update_docs() -> None: + """Keep traceability aligned with the non-reflecting diagnostic contract.""" + replacements = { + CHANGELOG: ( + "NoemaTransportError preserves the first validator diagnostic plus the later transport class/status without logging raw model output or secrets.", + "NoemaTransportError preserves the stable model-output contract code plus the later transport class/status without logging raw model output or secrets.", + ), + ARCHITECTURE: ( + "transport error retains both the first trusted-validator diagnostic and the\nlater transport class/status while omitting raw model content and secrets.", + "transport error retains both the stable model-output contract code and the\nlater transport class/status while omitting raw model content and secrets.", + ), + BASELINE: ( + "the final fail-closed diagnostic preserves the sanitized first validator error plus the later typed transport evidence.", + "the final fail-closed diagnostic preserves a stable model-output contract code plus the later typed transport evidence without reflecting model-controlled values.", + ), + DOCTORING: ( + "A corrective transport failure is `NoemaTransportError` and carries the sanitized first validator diagnostic plus the later transport exception class/status.", + "A corrective transport failure is `NoemaTransportError` and carries a stable model-output contract code plus the later transport exception class/status; model-controlled validator values are not reflected into the retry prompt or public diagnostic.", + ), + } + for path, (old, new) in replacements.items(): + text = path.read_text(encoding="utf-8") + text = replace_once(text, old, new, f"stable diagnostic docs: {path}") + path.write_text(text, encoding="utf-8") + + def main() -> None: update_source() update_tests() + update_docs() if __name__ == "__main__": From 4a7005da4fb6d00357b2f092662cf41fd4920486 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 00:58:27 +0900 Subject: [PATCH 16/24] fix(noema): preserve actionable safe repair diagnostics --- ...repair_noema_model_output_followup_1617.py | 85 ++++++++++--------- 1 file changed, 45 insertions(+), 40 deletions(-) diff --git a/scripts/ci/repair_noema_model_output_followup_1617.py b/scripts/ci/repair_noema_model_output_followup_1617.py index de7ec18bed..cb5cf413d5 100644 --- a/scripts/ci/repair_noema_model_output_followup_1617.py +++ b/scripts/ci/repair_noema_model_output_followup_1617.py @@ -23,16 +23,6 @@ def replace_once(text: str, old: str, new: str, label: str) -> str: return text.replace(old, new, 1) -def replace_exact_count( - text: str, old: str, new: str, expected_count: int, label: str -) -> str: - """Replace a reviewed generated fragment only when its multiplicity is exact.""" - count = text.count(old) - if count != expected_count: - raise RuntimeError(f"{label}: expected {expected_count} matches, found {count}") - return text.replace(old, new) - - def update_source() -> None: text = SOURCE.read_text(encoding="utf-8") @@ -48,10 +38,31 @@ def update_source() -> None: """Raised when the corrective attempt exceeds its total wall-clock budget.""" ''' diagnostic_helper = deadline_class + '''\n\ndef _stable_failure_diagnostic(exc: BaseException) -> str: - """Return bounded diagnostics without reflecting model-controlled text.""" - if isinstance(exc, NoemaModelOutputError): - return "model-output-contract-invalid" - return scrub_sensitive_data(str(exc)) or type(exc).__name__ + """Return actionable trusted diagnostics without reflecting model values.""" + message = scrub_sensitive_data(str(exc)) or type(exc).__name__ + if not isinstance(exc, NoemaModelOutputError): + return message + + # Model-output exceptions are raised only by deterministic parsing and + # validation code. Preserve those static/structural diagnostics because + # they tell the corrective model and operators exactly which contract was + # violated. The one validator that embeds an untrusted model value is the + # unsupported-decision check; redact that value. Unknown model-output + # exception text fails closed to a stable code rather than being reflected. + if message.startswith("Noema LLM returned unsupported decision:"): + return "Noema LLM returned unsupported decision" + trusted_prefixes = ( + "Noema LLM response ", + "Noema formal verdict ", + "Noema reviewed line ", + "Noema adversarial validation ", + "Noema adversarial probe ", + "Noema approve ", + "Noema request_changes ", + ) + if message.startswith(trusted_prefixes): + return message + return "model-output-contract-invalid" ''' text = replace_once( text, @@ -85,19 +96,6 @@ def update_source() -> None: def update_tests() -> None: text = TEST.read_text(encoding="utf-8") - - # Three earlier one-shot transforms assert the model-controlled validator - # detail after call_llm(). The final contract intentionally exposes only a - # stable trusted code at that boundary; the direct validator regression at - # the top of the file keeps its detailed assertion. - text = replace_exact_count( - text, - ' assert "outcome must be falsified or confirmed" in message\n', - ' assert "model-output-contract-invalid" in message\n', - 3, - "generated call_llm stable-diagnostic assertions", - ) - marker = "def test_unparseable_diff_remains_source_evidence" if marker in text: raise RuntimeError("follow-up #1617 regressions already present") @@ -113,7 +111,7 @@ def test_unparseable_diff_remains_source_evidence() -> None: def test_model_sentinel_never_reaches_repair_prompt_or_final_diagnostic(monkeypatch) -> None: - """Model-controlled invalid values are replaced by a stable validator code.""" + """Model-controlled invalid values are redacted while the defect class stays actionable.""" import json sentinel = "MODEL_SENTINEL_DO_NOT_REFLECT" @@ -156,17 +154,24 @@ def open_response(_opener, request, **kwargs): assert len(requests) == 2 repair_payload = requests[1].data.decode("utf-8") assert sentinel not in repair_payload - assert "model-output-contract-invalid" in repair_payload + assert "Noema LLM returned unsupported decision" in repair_payload assert sentinel not in str(exc_info.value) - assert "model-output-contract-invalid" in str(exc_info.value) + assert "Noema LLM returned unsupported decision" in str(exc_info.value) assert exc_info.value.__cause__ is None -def test_stable_failure_diagnostic_keeps_transport_class_without_model_text() -> None: - """Trusted transport diagnostics remain useful while model text stays opaque.""" - assert gate._stable_failure_diagnostic(gate.NoemaModelOutputError("secret-ish model text")) == ( - "model-output-contract-invalid" +def test_stable_failure_diagnostic_preserves_trusted_structure_and_redacts_values() -> None: + """Trusted validator detail stays actionable; arbitrary model text stays opaque.""" + trusted = gate.NoemaModelOutputError( + "Noema adversarial probe 1 outcome must be falsified or confirmed" ) + assert gate._stable_failure_diagnostic(trusted) == str(trusted) + assert gate._stable_failure_diagnostic( + gate.NoemaModelOutputError("Noema LLM returned unsupported decision: 'SECRET_VALUE'") + ) == "Noema LLM returned unsupported decision" + assert gate._stable_failure_diagnostic( + gate.NoemaModelOutputError("secret-ish model text") + ) == "model-output-contract-invalid" assert gate._stable_failure_diagnostic(TimeoutError()) == "TimeoutError" @@ -213,28 +218,28 @@ def reject_signal(*_args): def update_docs() -> None: - """Keep traceability aligned with the non-reflecting diagnostic contract.""" + """Keep traceability aligned with the bounded actionable diagnostic contract.""" replacements = { CHANGELOG: ( "NoemaTransportError preserves the first validator diagnostic plus the later transport class/status without logging raw model output or secrets.", - "NoemaTransportError preserves the stable model-output contract code plus the later transport class/status without logging raw model output or secrets.", + "NoemaTransportError preserves the first trusted structural validator diagnostic plus the later transport class/status without reflecting model-controlled values or secrets.", ), ARCHITECTURE: ( "transport error retains both the first trusted-validator diagnostic and the\nlater transport class/status while omitting raw model content and secrets.", - "transport error retains both the stable model-output contract code and the\nlater transport class/status while omitting raw model content and secrets.", + "transport error retains both the first trusted structural validator diagnostic and the\nlater transport class/status while redacting model-controlled values and omitting raw model content and secrets.", ), BASELINE: ( "the final fail-closed diagnostic preserves the sanitized first validator error plus the later typed transport evidence.", - "the final fail-closed diagnostic preserves a stable model-output contract code plus the later typed transport evidence without reflecting model-controlled values.", + "the final fail-closed diagnostic preserves the first trusted structural validator error plus later typed transport evidence while redacting model-controlled values.", ), DOCTORING: ( "A corrective transport failure is `NoemaTransportError` and carries the sanitized first validator diagnostic plus the later transport exception class/status.", - "A corrective transport failure is `NoemaTransportError` and carries a stable model-output contract code plus the later transport exception class/status; model-controlled validator values are not reflected into the retry prompt or public diagnostic.", + "A corrective transport failure is `NoemaTransportError` and carries the first trusted structural validator diagnostic plus the later transport exception class/status; model-controlled values are redacted rather than reflected into the retry prompt or public diagnostic.", ), } for path, (old, new) in replacements.items(): text = path.read_text(encoding="utf-8") - text = replace_once(text, old, new, f"stable diagnostic docs: {path}") + text = replace_once(text, old, new, f"actionable diagnostic docs: {path}") path.write_text(text, encoding="utf-8") From 312df948b14682b43af6106ed116d5436d86e17f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 01:04:58 +0900 Subject: [PATCH 17/24] fix(noema): repair changelog transform drift --- scripts/ci/repair_noema_model_output_followup_1617.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/ci/repair_noema_model_output_followup_1617.py b/scripts/ci/repair_noema_model_output_followup_1617.py index cb5cf413d5..56f3336bc6 100644 --- a/scripts/ci/repair_noema_model_output_followup_1617.py +++ b/scripts/ci/repair_noema_model_output_followup_1617.py @@ -221,8 +221,8 @@ def update_docs() -> None: """Keep traceability aligned with the bounded actionable diagnostic contract.""" replacements = { CHANGELOG: ( - "NoemaTransportError preserves the first validator diagnostic plus the later transport class/status without logging raw model output or secrets.", - "NoemaTransportError preserves the first trusted structural validator diagnostic plus the later transport class/status without reflecting model-controlled values or secrets.", + "`NoemaTransportError` preserves the first validator diagnostic plus the later transport class/status without logging raw model output or secrets.", + "`NoemaTransportError` preserves the first trusted structural validator diagnostic plus the later transport class/status without reflecting model-controlled values or secrets.", ), ARCHITECTURE: ( "transport error retains both the first trusted-validator diagnostic and the\nlater transport class/status while omitting raw model content and secrets.", From 11b6b4e46eeb84c6c139b373c63c16fbd6abdad8 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 01:08:06 +0900 Subject: [PATCH 18/24] fix(noema): make repair trace updates drift-safe --- ...repair_noema_model_output_followup_1617.py | 91 ++++++++++++++----- 1 file changed, 68 insertions(+), 23 deletions(-) diff --git a/scripts/ci/repair_noema_model_output_followup_1617.py b/scripts/ci/repair_noema_model_output_followup_1617.py index 56f3336bc6..ffb200b359 100644 --- a/scripts/ci/repair_noema_model_output_followup_1617.py +++ b/scripts/ci/repair_noema_model_output_followup_1617.py @@ -218,29 +218,74 @@ def reject_signal(*_args): def update_docs() -> None: - """Keep traceability aligned with the bounded actionable diagnostic contract.""" - replacements = { - CHANGELOG: ( - "`NoemaTransportError` preserves the first validator diagnostic plus the later transport class/status without logging raw model output or secrets.", - "`NoemaTransportError` preserves the first trusted structural validator diagnostic plus the later transport class/status without reflecting model-controlled values or secrets.", - ), - ARCHITECTURE: ( - "transport error retains both the first trusted-validator diagnostic and the\nlater transport class/status while omitting raw model content and secrets.", - "transport error retains both the first trusted structural validator diagnostic and the\nlater transport class/status while redacting model-controlled values and omitting raw model content and secrets.", - ), - BASELINE: ( - "the final fail-closed diagnostic preserves the sanitized first validator error plus the later typed transport evidence.", - "the final fail-closed diagnostic preserves the first trusted structural validator error plus later typed transport evidence while redacting model-controlled values.", - ), - DOCTORING: ( - "A corrective transport failure is `NoemaTransportError` and carries the sanitized first validator diagnostic plus the later transport exception class/status.", - "A corrective transport failure is `NoemaTransportError` and carries the first trusted structural validator diagnostic plus the later transport exception class/status; model-controlled values are redacted rather than reflected into the retry prompt or public diagnostic.", - ), - } - for path, (old, new) in replacements.items(): - text = path.read_text(encoding="utf-8") - text = replace_once(text, old, new, f"actionable diagnostic docs: {path}") - path.write_text(text, encoding="utf-8") + """Add drift-safe traceability for the actionable diagnostic contract.""" + changelog = CHANGELOG.read_text(encoding="utf-8") + changelog_entry = ( + "- **Harden #1617 corrective diagnostics against model-value reflection.** " + "The repair prompt and final fail-closed error preserve deterministic structural validator evidence " + "needed to correct a malformed verdict, while model-controlled values (including an unsupported " + "decision value) are redacted and an unknown model-output diagnostic collapses to a stable code.\n" + ) + if changelog_entry not in changelog: + changelog = replace_once( + changelog, + "## [Unreleased]\n", + "## [Unreleased]\n" + changelog_entry, + "changelog unreleased heading", + ) + CHANGELOG.write_text(changelog, encoding="utf-8") + + architecture = ARCHITECTURE.read_text(encoding="utf-8") + architecture_marker = "#### Actionable Noema repair diagnostics" + if architecture_marker not in architecture: + architecture += """ + +#### Actionable Noema repair diagnostics + +The corrective prompt may retain only deterministic structural validator diagnostics +that are generated by trusted validation code. Model-controlled values are never +reflected into the corrective prompt or public exception chain: unsupported decision +values are reduced to their static defect class and unknown model-output diagnostics +collapse to a stable code. This keeps repair evidence actionable without turning the +reviewer itself into a data-reflection channel. +""" + ARCHITECTURE.write_text(architecture, encoding="utf-8") + + baseline = BASELINE.read_text(encoding="utf-8") + baseline_marker = "- **Diagnostic hardening:** #1617 corrective prompts" + if baseline_marker not in baseline: + baseline_heading = ( + "## 2026-09-01 Noema malformed-verdict retry classification and wall-clock bound (#1611/#1617)\n" + ) + baseline_note = ( + "\n- **Diagnostic hardening:** #1617 corrective prompts preserve only trusted structural validator " + "detail; model-controlled values are redacted, unknown model-output text becomes a stable defect " + "code, and repeated invalid-model exceptions do not retain the raw model exception as a cause.\n" + ) + baseline = replace_once( + baseline, + baseline_heading, + baseline_heading + baseline_note, + "baseline #1617 heading", + ) + BASELINE.write_text(baseline, encoding="utf-8") + + doctoring = DOCTORING.read_text(encoding="utf-8") + doctoring_marker = "## Actionable diagnostic boundary" + if doctoring_marker not in doctoring: + doctoring += """ + +## Actionable diagnostic boundary + +Corrective prompts need the deterministic *class* of a malformed verdict to repair it, +but do not need arbitrary model-produced values. Trusted structural validator messages +(such as a missing required field or an invalid adversarial-probe outcome class) remain +available after secret scrubbing. Unsupported decision values and unknown model-output +text are redacted to stable diagnostics, and a repeated invalid-model exception is raised +without retaining the raw model exception as an explicit cause. Tests use a sentinel value +to prove it reaches neither the retry prompt nor the final diagnostic. +""" + DOCTORING.write_text(doctoring, encoding="utf-8") def main() -> None: From b7fc695203d65745f0cce503d0b5ce6dd4b01496 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 01:12:39 +0900 Subject: [PATCH 19/24] fix(noema): preserve request-changes structural diagnostic --- scripts/ci/repair_noema_model_output_followup_1617.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/ci/repair_noema_model_output_followup_1617.py b/scripts/ci/repair_noema_model_output_followup_1617.py index ffb200b359..fd1c0ff60f 100644 --- a/scripts/ci/repair_noema_model_output_followup_1617.py +++ b/scripts/ci/repair_noema_model_output_followup_1617.py @@ -53,6 +53,7 @@ def update_source() -> None: return "Noema LLM returned unsupported decision" trusted_prefixes = ( "Noema LLM response ", + "Noema LLM request_changes ", "Noema formal verdict ", "Noema reviewed line ", "Noema adversarial validation ", @@ -166,6 +167,10 @@ def test_stable_failure_diagnostic_preserves_trusted_structure_and_redacts_value "Noema adversarial probe 1 outcome must be falsified or confirmed" ) assert gate._stable_failure_diagnostic(trusted) == str(trusted) + request_changes = gate.NoemaModelOutputError( + "Noema LLM request_changes response did not contain a substantive finding" + ) + assert gate._stable_failure_diagnostic(request_changes) == str(request_changes) assert gate._stable_failure_diagnostic( gate.NoemaModelOutputError("Noema LLM returned unsupported decision: 'SECRET_VALUE'") ) == "Noema LLM returned unsupported decision" From aa4ec462030dba523cb23416c81f0deb60f626f5 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 02:24:06 +0900 Subject: [PATCH 20/24] fix(noema): document repair deadline callback --- scripts/ci/repair_noema_wall_clock_1617.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/ci/repair_noema_wall_clock_1617.py b/scripts/ci/repair_noema_wall_clock_1617.py index ae8ba72b00..58c2e8c0d0 100644 --- a/scripts/ci/repair_noema_wall_clock_1617.py +++ b/scripts/ci/repair_noema_wall_clock_1617.py @@ -75,6 +75,7 @@ def _repair_wall_clock_deadline(seconds: float): previous_handler = signal.getsignal(signal.SIGALRM) def expire(_signum, _frame): + """Raise the typed deadline signal without reflecting response content.""" raise NoemaRepairDeadlineExceeded( f"Noema repair exceeded {seconds:g}-second absolute wall-clock deadline" ) From 0b33b0bc755f053437e384d274a0223c099dc74c Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 02:30:30 +0900 Subject: [PATCH 21/24] fix(ci): grant one-shot Noema repair writer token --- .github/workflows/repair-noema-model-output-1617.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/repair-noema-model-output-1617.yml b/.github/workflows/repair-noema-model-output-1617.yml index bfb50f6141..7f50b65c58 100644 --- a/.github/workflows/repair-noema-model-output-1617.yml +++ b/.github/workflows/repair-noema-model-output-1617.yml @@ -10,7 +10,7 @@ concurrency: cancel-in-progress: true permissions: - contents: read + contents: write jobs: repair: @@ -137,7 +137,7 @@ jobs: - name: Guard and push only the verified exact-head commit env: - GH_TOKEN: ${{ secrets.PR_REVIEW_MERGE_TOKEN || secrets.OPENCODE_APPROVE_TOKEN }} + GH_TOKEN: ${{ github.token }} run: | set -euo pipefail test -n "${GH_TOKEN:-}" From 17c28a8ca1ad29a36c2c48e6da1db63579c72723 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 02:30:59 +0900 Subject: [PATCH 22/24] chore(ci): retrigger verified Noema repair writer --- scripts/ci/repair_noema_timeout_fixture_1617.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/ci/repair_noema_timeout_fixture_1617.py b/scripts/ci/repair_noema_timeout_fixture_1617.py index 8a32c1879f..ea15ffae53 100644 --- a/scripts/ci/repair_noema_timeout_fixture_1617.py +++ b/scripts/ci/repair_noema_timeout_fixture_1617.py @@ -4,6 +4,7 @@ from pathlib import Path +# Temporary writer-token retry trigger; this helper is deleted by the repair. TEST = Path(__file__).resolve().parents[2] / "tests/test_noema_review_gate.py" From 430a007515e8510ded466636b65a34e06864ed0e Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 1 Sep 2026 17:33:47 +0000 Subject: [PATCH 23/24] fix(noema): bound and classify malformed-verdict repair --- .../repair-noema-model-output-1617.yml | 149 ------- ARCHITECTURE.md | 24 ++ CHANGELOG.md | 2 + .../noema-model-output-repair-boundary.md | 33 ++ docs/product-technical-gap-baseline.md | 11 + scripts/ci/noema_review_gate.py | 227 +++++++--- scripts/ci/repair_noema_coverage_1617.py | 131 ------ scripts/ci/repair_noema_model_output_1617.py | 291 ------------- ...repair_noema_model_output_followup_1617.py | 303 -------------- .../ci/repair_noema_timeout_fixture_1617.py | 34 -- scripts/ci/repair_noema_wall_clock_1617.py | 248 ----------- ...ema_model_output_failure_classification.py | 396 ++++++++++++++++++ 12 files changed, 635 insertions(+), 1214 deletions(-) delete mode 100644 .github/workflows/repair-noema-model-output-1617.yml create mode 100644 docs/doctoring/noema-model-output-repair-boundary.md delete mode 100644 scripts/ci/repair_noema_coverage_1617.py delete mode 100644 scripts/ci/repair_noema_model_output_1617.py delete mode 100644 scripts/ci/repair_noema_model_output_followup_1617.py delete mode 100644 scripts/ci/repair_noema_timeout_fixture_1617.py delete mode 100644 scripts/ci/repair_noema_wall_clock_1617.py diff --git a/.github/workflows/repair-noema-model-output-1617.yml b/.github/workflows/repair-noema-model-output-1617.yml deleted file mode 100644 index 7f50b65c58..0000000000 --- a/.github/workflows/repair-noema-model-output-1617.yml +++ /dev/null @@ -1,149 +0,0 @@ -name: TEMP repair Noema model-output boundary 1617 - -on: - push: - branches: - - fix/noema-model-output-retry-20260901 - -concurrency: - group: repair-noema-model-output-1617 - cancel-in-progress: true - -permissions: - contents: write - -jobs: - repair: - if: >- - github.repository == 'ContextualWisdomLab/.github' && - github.actor == 'seonghobae' && - github.triggering_actor == 'seonghobae' - runs-on: ubuntu-24.04 - timeout-minutes: 60 - steps: - - name: Checkout triggering repair head without credentials - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - with: - ref: ${{ github.sha }} - fetch-depth: 0 - persist-credentials: false - - - name: Bind the single-writer branch to the triggering head - run: | - set -euo pipefail - writer_ref='refs/heads/fix/noema-model-output-retry-20260901' - remote_head="$(git ls-remote origin "$writer_ref" | awk '{print $1}')" - test "$remote_head" = "$GITHUB_SHA" - - - name: Set up Python - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: '3.14' - - - name: Install repository-pinned quality tools - run: python -m pip install --require-hashes -r requirements-opencode-review-ci-hashes.txt - - - name: Apply typed-evidence, deadline, coverage, and reviewed follow-up repairs - run: | - set -euo pipefail - PYTHONPATH=. python scripts/ci/repair_noema_model_output_1617.py - PYTHONPATH=. python scripts/ci/repair_noema_wall_clock_1617.py - PYTHONPATH=. python scripts/ci/repair_noema_coverage_1617.py - PYTHONPATH=. python scripts/ci/repair_noema_model_output_followup_1617.py - - - name: Remove temporary repair machinery before verification - run: | - rm -f scripts/ci/repair_noema_model_output_1617.py - rm -f scripts/ci/repair_noema_timeout_fixture_1617.py - rm -f scripts/ci/repair_noema_wall_clock_1617.py - rm -f scripts/ci/repair_noema_coverage_1617.py - rm -f scripts/ci/repair_noema_model_output_followup_1617.py - rm -f .github/workflows/repair-noema-model-output-1617.yml - test ! -e .github/workflows/repair-noema-model-output-1617.yml - - - name: Verify repair scope and required semantic targets - run: | - set -euo pipefail - python - <<'PY' - import subprocess - allowed = { - '.github/workflows/repair-noema-model-output-1617.yml', - 'ARCHITECTURE.md', - 'CHANGELOG.md', - 'docs/doctoring/noema-model-output-repair-boundary.md', - 'docs/product-technical-gap-baseline.md', - 'scripts/ci/noema_review_gate.py', - 'scripts/ci/repair_noema_coverage_1617.py', - 'scripts/ci/repair_noema_model_output_1617.py', - 'scripts/ci/repair_noema_model_output_followup_1617.py', - 'scripts/ci/repair_noema_timeout_fixture_1617.py', - 'scripts/ci/repair_noema_wall_clock_1617.py', - 'tests/test_noema_model_output_failure_classification.py', - } - changed = set(subprocess.check_output(['git', 'diff', '--name-only'], text=True).splitlines()) - unexpected = changed - allowed - if unexpected: - raise SystemExit(f'unexpected repair paths: {sorted(unexpected)}') - required = { - 'scripts/ci/noema_review_gate.py', - 'tests/test_noema_model_output_failure_classification.py', - } - missing = required - changed - if missing: - raise SystemExit(f'required repair targets unchanged: {sorted(missing)}') - print('verified repair scope:', *sorted(changed), sep='\n- ') - PY - - - name: Verify focused Noema regressions - run: PYTHONPATH=. python -m pytest -q tests/test_noema_model_output_failure_classification.py tests/test_noema_review_gate.py - - - name: Verify complete suite, source coverage, docs, and diff hygiene - run: | - set -euo pipefail - PYTHONPATH=. coverage run -m pytest tests -q - coverage report --show-missing - interrogate - git diff --check - test ! -e .github/workflows/repair-noema-model-output-1617.yml - test ! -e scripts/ci/repair_noema_model_output_1617.py - test ! -e scripts/ci/repair_noema_timeout_fixture_1617.py - test ! -e scripts/ci/repair_noema_wall_clock_1617.py - test ! -e scripts/ci/repair_noema_coverage_1617.py - test ! -e scripts/ci/repair_noema_model_output_followup_1617.py - - - name: Commit verified production repair with an allowlisted scope - run: | - set -euo pipefail - git config user.name 'github-actions[bot]' - git config user.email '41898282+github-actions[bot]@users.noreply.github.com' - git add -A -- \ - .github/workflows/repair-noema-model-output-1617.yml \ - ARCHITECTURE.md \ - CHANGELOG.md \ - docs/doctoring/noema-model-output-repair-boundary.md \ - docs/product-technical-gap-baseline.md \ - scripts/ci/noema_review_gate.py \ - scripts/ci/repair_noema_coverage_1617.py \ - scripts/ci/repair_noema_model_output_1617.py \ - scripts/ci/repair_noema_model_output_followup_1617.py \ - scripts/ci/repair_noema_timeout_fixture_1617.py \ - scripts/ci/repair_noema_wall_clock_1617.py \ - tests/test_noema_model_output_failure_classification.py - git diff --cached --check - test -z "$(git diff --name-only)" - git diff --cached --name-only | grep -Fx 'scripts/ci/noema_review_gate.py' - git diff --cached --name-only | grep -Fx 'tests/test_noema_model_output_failure_classification.py' - git commit -m 'fix(noema): bound and classify malformed-verdict repair' - - - name: Guard and push only the verified exact-head commit - env: - GH_TOKEN: ${{ github.token }} - run: | - set -euo pipefail - test -n "${GH_TOKEN:-}" - writer_branch='fix/noema-model-output-retry-20260901' - writer_ref="refs/heads/${writer_branch}" - remote_head="$(git ls-remote origin "$writer_ref" | awk '{print $1}')" - test "$remote_head" = "$GITHUB_SHA" - gh auth setup-git - git push origin HEAD:"$writer_branch" diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 8038c3632e..8ff2b049ce 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -191,3 +191,27 @@ resolver conflict. — current increment's attestation decision and APA 7th citations. - [`docs/doctoring/sandboxed-web-readiness-loopback-boundary.md`](docs/doctoring/sandboxed-web-readiness-loopback-boundary.md) — loopback-only web E2E readiness polling and APA 7th citations. + + +### Noema model-output and repair boundary + +Noema separates deterministic model-output/schema failures from GitHub/source +findings and provider transport exhaustion. A malformed verdict remains +non-passing and is represented by `NoemaModelOutputError`. Its single corrective +request still routes only through the loopback contextual-orchestrator +`orchestrator/free` gateway, but has one 15-minute process-level wall-clock deadline across open, read, +decode, and deterministic validation because it repairs an already-completed +verdict rather than performing a second unbounded full review. This is not a +socket inactivity timeout, so response activity cannot renew the budget. If that corrective request encounters transport exhaustion, the typed +transport error retains both the first trusted-validator diagnostic and the +later transport class/status while omitting raw model content and secrets. + + +#### Actionable Noema repair diagnostics + +The corrective prompt may retain only deterministic structural validator diagnostics +that are generated by trusted validation code. Model-controlled values are never +reflected into the corrective prompt or public exception chain: unsupported decision +values are reduced to their static defect class and unknown model-output diagnostics +collapse to a stable code. This keeps repair evidence actionable without turning the +reviewer itself into a data-reflection channel. diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f0680a91d..11b5504b4a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ this file. The format follows Keep a Changelog, and versioned releases follow Semantic Versioning where the repository publishes a release. ## [Unreleased] +- **Harden #1617 corrective diagnostics against model-value reflection.** The repair prompt and final fail-closed error preserve deterministic structural validator evidence needed to correct a malformed verdict, while model-controlled values (including an unsupported decision value) are redacted and an unknown model-output diagnostic collapses to a stable code. +- **Classify and bound Noema malformed-verdict repair failures (#1611/#1617).** A schema-invalid model verdict now raises typed `NoemaModelOutputError` evidence instead of an undifferentiated runtime failure. The one corrective attempt has a 15-minute absolute wall-clock deadline across open/read/decode/validation while the primary contextual-orchestrator review remains under its no-fixed-inference-timeout contract; unlike a urllib socket timeout, trickling response activity cannot renew that budget. If the repair then fails at transport, `NoemaTransportError` preserves the first validator diagnostic plus the later transport class/status without logging raw model output or secrets. - Fix `existing_noema_review()` treating a "legacy" Noema review (one posted before `NOEMA_REVIEW_FOOTER_MARKER` existed) as proof the current head was already reviewed. `noema_review_handoff.py`'s `noema_review_state()` can never recognize such a review as a diff --git a/docs/doctoring/noema-model-output-repair-boundary.md b/docs/doctoring/noema-model-output-repair-boundary.md new file mode 100644 index 0000000000..d1602f92de --- /dev/null +++ b/docs/doctoring/noema-model-output-repair-boundary.md @@ -0,0 +1,33 @@ +# Noema model-output repair boundary + +## Incident + +On 2026-09-01 the required Noema review for `ContextualWisdomLab/naruon#1505` reached deterministic verdict validation, rejected an adversarial-probe `outcome` outside the closed `falsified|confirmed` domain, then spent the repair path on a long second model call that ultimately surfaced only `HTTP 502 Bad Gateway`. That final transport symptom erased the more informative first trusted-validator failure from the top-level diagnostic. + +## Decision + +1. Model-produced JSON/envelope/schema/semantic-contract failures are `NoemaModelOutputError`; they remain fail-closed and are not consumer-source findings. +2. The primary review keeps the accepted contextual-orchestrator no-fixed-inference-timeout contract. The *single corrective attempt* is different: it repairs an already-completed verdict and therefore has one 900-second process-level wall-clock deadline across open/read/decode/validation. It deliberately does not use `urllib`'s renewable socket-operation timeout. +3. A corrective transport failure is `NoemaTransportError` and carries the sanitized first validator diagnostic plus the later transport exception class/status. Raw model output is never copied into public Actions diagnostics. +4. Exact-head validation before retry and before publication remains mandatory. All model traffic remains on contextual-orchestrator `orchestrator/free`. + +## Verification + +The #1617 regression first proved RED because `NoemaModelOutputError` did not exist. The repair adds focused cases for malformed-verdict typing, malformed-then-502 evidence preservation with the 900-second repair-only timeout, and repeated malformed output remaining typed and non-passing. The repository full coverage/docstring gate is run before the one-shot repair workflow commits the result. + +## References + +Fielding, R., Nottingham, M., & Reschke, J. (2022). *HTTP semantics* (RFC 9110). Internet Engineering Task Force. + +Python Software Foundation. (2026). *urllib.request — Extensible library for opening URLs*. Python 3 documentation. + + +## Actionable diagnostic boundary + +Corrective prompts need the deterministic *class* of a malformed verdict to repair it, +but do not need arbitrary model-produced values. Trusted structural validator messages +(such as a missing required field or an invalid adversarial-probe outcome class) remain +available after secret scrubbing. Unsupported decision values and unknown model-output +text are redacted to stable diagnostics, and a repeated invalid-model exception is raised +without retaining the raw model exception as an explicit cause. Tests use a sentinel value +to prove it reaches neither the retry prompt nor the final diagnostic. diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 6a2bf678d4..a403a2241e 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -2562,3 +2562,14 @@ Zhang, S., Yu, Y., Li, Y., Zhao, W., Yang, Y., Zhang, Y., & Liu, T. (2025). *Con Xu, J., Sun, Q., Schwendeman, P., Nielsen, S., Cetin, E., & Tang, Y. (2026). *TRINITY: An evolved LLM coordinator* [Preprint]. arXiv. https://doi.org/10.48550/arXiv.2512.04695 Higgins, S. S., Crepalde, N., & Fernandes, L. (2021). Segmented multiplexity: A research agenda for multiplexity beyond the average. *PLOS ONE, 16*(9), e0257527. https://doi.org/10.1371/journal.pone.0257527 + + +## 2026-09-01 Noema malformed-verdict retry classification and wall-clock bound (#1611/#1617) + +- **Diagnostic hardening:** #1617 corrective prompts preserve only trusted structural validator detail; model-controlled values are redacted, unknown model-output text becomes a stable defect code, and repeated invalid-model exceptions do not retain the raw model exception as a cause. + +- **Observed consumer evidence:** `ContextualWisdomLab/naruon#1505@7da2a242e463f59d4580cb38e7591f1ba4b4049e`, Required Noema run `33460498090` / job `99742587317`. The first response reached the trusted semantic validator but used an out-of-domain adversarial-probe `outcome`; the generic repair attempt later ended as HTTP 502 after roughly 88 minutes. +- **Root cause:** model-output/schema rejection, repair transport exhaustion, and consumer-source findings shared an undifferentiated `RuntimeError` boundary. The corrective HTTP request also had no client-side repair-specific ceiling, so a malformed first verdict could initiate another effectively full-duration request. +- **Repair:** model-output/schema rejection is typed as `NoemaModelOutputError`; the one corrective attempt has a 900-second absolute wall-clock deadline across open/read/decode/validation (not a renewable socket timeout); repair transport exhaustion is typed as `NoemaTransportError`; and the final fail-closed diagnostic preserves the sanitized first validator error plus the later typed transport evidence. Primary review inference remains governed by contextual-orchestrator `orchestrator/free` and is not given a new fixed model-inference timeout. +- **Security/operability invariant:** raw model content, credentials, and provider secrets are never included in the combined diagnostic. Exact-head revalidation still occurs before retry and before publication. No direct-provider fallback or GitHub authority change is introduced. +- **Verification contract:** deterministic tests cover the original invalid `outcome`, malformed-then-502 evidence preservation and the repair-only timeout, and repeated malformed model output remaining typed and non-passing. The affected Naruon head must be re-run after protected integration; predecessor review/check evidence does not transfer. diff --git a/scripts/ci/noema_review_gate.py b/scripts/ci/noema_review_gate.py index 5dbeb65d79..4f82281fc3 100644 --- a/scripts/ci/noema_review_gate.py +++ b/scripts/ci/noema_review_gate.py @@ -6,12 +6,14 @@ import argparse import ast import base64 +import contextlib import hashlib import http.client import ipaddress import json import os import re +import signal import socket import subprocess import sys @@ -61,6 +63,53 @@ ORCHESTRATOR_LOOPBACK_HOSTS = frozenset({"127.0.0.1", "::1"}) ORCHESTRATOR_BASE_ENV = "CONTEXTUAL_ORCHESTRATOR_BASE_URL" +# A repair request corrects an already-completed model verdict; it is not a +# second unbounded full review. Fifteen minutes is an absolute wall-clock +# deadline for the complete corrective attempt (open/read/decode/validate), +# not a socket inactivity timeout. The primary review remains governed by +# contextual-orchestrator rather than a fixed inference timeout. +NOEMA_REPAIR_DEADLINE_SECONDS = 15 * 60 + + +class NoemaModelOutputError(RuntimeError): + """Raised when untrusted model output violates the trusted verdict contract.""" + + +class NoemaTransportError(RuntimeError): + """Raised when the bounded review transport cannot produce usable evidence.""" + + +class NoemaRepairDeadlineExceeded(TimeoutError): + """Raised when the corrective attempt exceeds its total wall-clock budget.""" + + +def _stable_failure_diagnostic(exc: BaseException) -> str: + """Return actionable trusted diagnostics without reflecting model values.""" + message = scrub_sensitive_data(str(exc)) or type(exc).__name__ + if not isinstance(exc, NoemaModelOutputError): + return message + + # Model-output exceptions are raised only by deterministic parsing and + # validation code. Preserve those static/structural diagnostics because + # they tell the corrective model and operators exactly which contract was + # violated. The one validator that embeds an untrusted model value is the + # unsupported-decision check; redact that value. Unknown model-output + # exception text fails closed to a stable code rather than being reflected. + if message.startswith("Noema LLM returned unsupported decision:"): + return "Noema LLM returned unsupported decision" + trusted_prefixes = ( + "Noema LLM response ", + "Noema LLM request_changes ", + "Noema formal verdict ", + "Noema reviewed line ", + "Noema adversarial validation ", + "Noema adversarial probe ", + "Noema approve ", + "Noema request_changes ", + ) + if message.startswith(trusted_prefixes): + return message + return "model-output-contract-invalid" # ⚡ Bolt: Pre-compiled regex patterns to avoid recompilation on every scrub_sensitive_data call. # Impact: Improves string processing performance in error reporting. @@ -387,57 +436,57 @@ def validate_substantive_verdict( reviewed_lines = verdict.get("reviewed_lines") if not isinstance(reviewed_lines, list) or not reviewed_lines: - raise RuntimeError("Noema formal verdict requires at least one reviewed changed line") + raise NoemaModelOutputError("Noema formal verdict requires at least one reviewed changed line") for index, reviewed in enumerate(reviewed_lines, start=1): if not isinstance(reviewed, dict): - raise RuntimeError(f"Noema reviewed line {index} must be an object") + raise NoemaModelOutputError(f"Noema reviewed line {index} must be an object") location = (reviewed.get("path"), reviewed.get("line"), reviewed.get("side")) if location not in locations: - raise RuntimeError(f"Noema reviewed line {index} is not an exact changed-side line") + raise NoemaModelOutputError(f"Noema reviewed line {index} is not an exact changed-side line") analysis = reviewed.get("analysis") if not isinstance(analysis, str) or not analysis.strip(): - raise RuntimeError(f"Noema reviewed line {index} requires concrete analysis") + raise NoemaModelOutputError(f"Noema reviewed line {index} requires concrete analysis") validation = verdict.get("adversarial_validation") if not isinstance(validation, dict): - raise RuntimeError("Noema formal verdict requires adversarial_validation") + raise NoemaModelOutputError("Noema formal verdict requires adversarial_validation") status = validation.get("status") expected_status = "passed" if decision == "approve" else "failed" if status != expected_status: - raise RuntimeError(f"Noema {decision} requires adversarial_validation.status={expected_status}") + raise NoemaModelOutputError(f"Noema {decision} requires adversarial_validation.status={expected_status}") residual_risk = validation.get("residual_risk") if not isinstance(residual_risk, str) or not residual_risk.strip(): - raise RuntimeError("Noema adversarial validation requires residual_risk") + raise NoemaModelOutputError("Noema adversarial validation requires residual_risk") probes = validation.get("probes") all_changed_paths = set(changed_paths) or {path for path, _line, _side in locations} required_probes = 2 if any(changed_file_is_material(path) for path in all_changed_paths) else 1 if not isinstance(probes, list) or len(probes) < required_probes: - raise RuntimeError(f"Noema adversarial validation requires at least {required_probes} concrete probe(s)") + raise NoemaModelOutputError(f"Noema adversarial validation requires at least {required_probes} concrete probe(s)") confirmed: set[tuple[str, int, str]] = set() identities: set[tuple[Any, ...]] = set() for index, probe in enumerate(probes, start=1): if not isinstance(probe, dict): - raise RuntimeError(f"Noema adversarial probe {index} must be an object") + raise NoemaModelOutputError(f"Noema adversarial probe {index} must be an object") location = (probe.get("path"), probe.get("line"), probe.get("side")) if location not in locations: - raise RuntimeError(f"Noema adversarial probe {index} is not an exact changed-side line") + raise NoemaModelOutputError(f"Noema adversarial probe {index} is not an exact changed-side line") for field in ("hypothesis", "attack_or_counterexample", "evidence"): value = probe.get(field) if not isinstance(value, str) or not value.strip(): - raise RuntimeError(f"Noema adversarial probe {index} requires {field}") + raise NoemaModelOutputError(f"Noema adversarial probe {index} requires {field}") outcome = probe.get("outcome") if outcome not in {"falsified", "confirmed"}: - raise RuntimeError(f"Noema adversarial probe {index} outcome must be falsified or confirmed") + raise NoemaModelOutputError(f"Noema adversarial probe {index} outcome must be falsified or confirmed") identity = (*location, probe["hypothesis"].strip().casefold(), probe["attack_or_counterexample"].strip().casefold()) if identity in identities: - raise RuntimeError(f"Noema adversarial probe {index} duplicates an earlier probe") + raise NoemaModelOutputError(f"Noema adversarial probe {index} duplicates an earlier probe") identities.add(identity) if outcome == "confirmed": confirmed.add((str(probe["path"]), int(probe["line"]), str(probe["side"]))) if decision == "approve" and confirmed: - raise RuntimeError("Noema approve cannot contain a confirmed adversarial probe") + raise NoemaModelOutputError("Noema approve cannot contain a confirmed adversarial probe") if decision == "request_changes": finding_locations = { (str(finding.get("file") or ""), finding.get("line"), str(finding.get("side") or "")) @@ -445,7 +494,7 @@ def validate_substantive_verdict( if isinstance(finding, dict) } if not confirmed or not confirmed.intersection(finding_locations): - raise RuntimeError("Noema request_changes requires a confirmed probe on a published finding") + raise NoemaModelOutputError("Noema request_changes requires a confirmed probe on a published finding") def truncate_text(text: str, limit: int) -> str: @@ -749,7 +798,7 @@ def _json_nesting_within_bound(text: str, start: int, max_depth: int) -> bool: def extract_json_object(text: str) -> dict[str, Any]: """Extract a JSON object from a strict or lightly wrapped LLM response. - Fails closed with ``RuntimeError`` — the same "no usable verdict" failure + Fails closed with ``NoemaModelOutputError`` — the same "no usable verdict" failure path ``call_llm`` already raises for an unsupported decision, a missing summary, or a malformed finding — instead of letting a malformed or truncated LLM response's ``json.JSONDecodeError`` propagate as an @@ -875,7 +924,7 @@ def extract_json_object(text: str) -> dict[str, Any]: return candidate if "{" not in stripped: - raise RuntimeError("Noema LLM response did not contain a JSON object") + raise NoemaModelOutputError("Noema LLM response did not contain a JSON object") exc = decode_error or json.JSONDecodeError( "No JSON object could be decoded", stripped, 0 @@ -886,7 +935,7 @@ def extract_json_object(text: str) -> dict[str, Any]: fingerprint = hashlib.sha256( stripped.encode("utf-8", errors="surrogatepass") ).hexdigest()[:16] - raise RuntimeError( + raise NoemaModelOutputError( f"Noema LLM response was not valid JSON ({exc}). Raw model output " "is not logged here (this pull_request_target workflow's logs " "are public and a finite secret-scrub pattern list cannot " @@ -918,21 +967,21 @@ def extract_llm_message_content(raw: str) -> str: try: data = json.loads(raw) except json.JSONDecodeError as exc: - raise RuntimeError(f"Noema LLM response body was not valid JSON: {exc}") from exc + raise NoemaModelOutputError(f"Noema LLM response body was not valid JSON: {exc}") from exc if not isinstance(data, dict): - raise RuntimeError( + raise NoemaModelOutputError( f"Noema LLM response body was not a JSON object (got {type(data).__name__})" ) choices = data.get("choices") if not choices: choices = [{}] elif not isinstance(choices, list): - raise RuntimeError( + raise NoemaModelOutputError( f"Noema LLM response 'choices' was not a list (got {type(choices).__name__})" ) first_choice = choices[0] if not isinstance(first_choice, dict): - raise RuntimeError( + raise NoemaModelOutputError( "Noema LLM response choices[0] was not a JSON object " f"(got {type(first_choice).__name__})" ) @@ -940,14 +989,14 @@ def extract_llm_message_content(raw: str) -> str: if not message: message = {} elif not isinstance(message, dict): - raise RuntimeError( + raise NoemaModelOutputError( f"Noema LLM response 'message' was not a JSON object (got {type(message).__name__})" ) content = message.get("content") if not content: content = "" elif not isinstance(content, str): - raise RuntimeError( + raise NoemaModelOutputError( f"Noema LLM response 'content' was not a string (got {type(content).__name__})" ) return content.strip() @@ -978,7 +1027,7 @@ def decode_llm_response_body(raw_bytes: bytes) -> str: return raw_bytes.decode("utf-8") except UnicodeDecodeError as exc: fingerprint = hashlib.sha256(raw_bytes).hexdigest()[:16] - raise RuntimeError( + raise NoemaModelOutputError( f"Noema LLM response body was not valid UTF-8 ({exc}). Raw " "response bytes are not logged here (this pull_request_target " "workflow's logs are public and a finite secret-scrub pattern " @@ -1075,6 +1124,43 @@ def reject_private_llm_url(api_url: str) -> None: raise ValueError("URL cannot target internal IP addresses") +@contextlib.contextmanager +def _repair_wall_clock_deadline(seconds: float): + """Interrupt the entire corrective attempt after ``seconds`` of wall time. + + ``urllib``'s timeout is a socket-operation timeout and can be extended by + trickling bytes. Required Noema Review runs on Linux, so ITIMER_REAL gives + the repair attempt one process-level wall-clock budget across open, read, + decode, and deterministic validation. An existing process alarm is not + overwritten; that condition fails closed instead. + """ + if seconds <= 0: + raise ValueError("repair wall-clock deadline must be positive") + if not hasattr(signal, "setitimer") or not hasattr(signal, "ITIMER_REAL"): + raise RuntimeError("repair wall-clock deadline requires POSIX setitimer support") + previous_remaining, previous_interval = signal.getitimer(signal.ITIMER_REAL) + if previous_remaining > 0 or previous_interval > 0: + raise RuntimeError("repair wall-clock deadline refused to overwrite an active process alarm") + previous_handler = signal.getsignal(signal.SIGALRM) + + def expire(_signum, _frame): + """Raise the typed deadline signal without reflecting response content.""" + raise NoemaRepairDeadlineExceeded( + f"Noema repair exceeded {seconds:g}-second absolute wall-clock deadline" + ) + + try: + signal.signal(signal.SIGALRM, expire) + except ValueError as exc: + raise RuntimeError("repair wall-clock deadline must run on the process main thread") from exc + signal.setitimer(signal.ITIMER_REAL, seconds) + try: + yield + finally: + signal.setitimer(signal.ITIMER_REAL, 0) + signal.signal(signal.SIGALRM, previous_handler) + + class StaleHeadDuringRepairRetryError(RuntimeError): """Raised when the PR head moves before ``call_llm``'s repair-retry request fires.""" @@ -1207,40 +1293,65 @@ def call_llm( ) opener = urllib.request.build_opener(NoRedirectHandler()) try: - with opener.open(request) as response: # nosec B310 - raw_bytes = response.read() - raw = decode_llm_response_body(raw_bytes) - content = extract_llm_message_content(raw) - verdict = extract_json_object(content) - decision = str(verdict.get("decision") or "").strip().lower() - if decision not in {"approve", "request_changes", "comment"}: - raise RuntimeError(f"Noema LLM returned unsupported decision: {decision!r}") - summary = verdict.get("summary") - if not isinstance(summary, str) or not summary.strip(): - raise RuntimeError("Noema LLM response did not contain a substantive summary") - findings = verdict.get("findings") - if not isinstance(findings, list) or any(not isinstance(finding, dict) for finding in findings): - raise RuntimeError("Noema LLM response findings must be a list of objects") - for finding in findings: - if ( - finding.get("severity") not in {"high", "medium", "low"} - or not isinstance(finding.get("file"), str) - or not finding["file"].strip() - or type(finding.get("line")) is not int - or finding["line"] <= 0 - or finding.get("side") not in {"RIGHT", "LEFT"} - or not isinstance(finding.get("message"), str) - or not finding["message"].strip() - ): - raise RuntimeError("Noema LLM response contained a malformed finding") - if decision == "request_changes" and not findings: - raise RuntimeError("Noema LLM request_changes response did not contain a substantive finding") - validate_substantive_verdict(verdict, diff, changed_paths) + deadline_context = ( + _repair_wall_clock_deadline(NOEMA_REPAIR_DEADLINE_SECONDS) + if is_retry + else contextlib.nullcontext() + ) + with deadline_context: + with opener.open(request) as response: # nosec B310 + raw_bytes = response.read() + raw = decode_llm_response_body(raw_bytes) + content = extract_llm_message_content(raw) + verdict = extract_json_object(content) + decision = str(verdict.get("decision") or "").strip().lower() + if decision not in {"approve", "request_changes", "comment"}: + raise NoemaModelOutputError(f"Noema LLM returned unsupported decision: {decision!r}") + summary = verdict.get("summary") + if not isinstance(summary, str) or not summary.strip(): + raise NoemaModelOutputError("Noema LLM response did not contain a substantive summary") + findings = verdict.get("findings") + if not isinstance(findings, list) or any(not isinstance(finding, dict) for finding in findings): + raise NoemaModelOutputError("Noema LLM response findings must be a list of objects") + for finding in findings: + if ( + finding.get("severity") not in {"high", "medium", "low"} + or not isinstance(finding.get("file"), str) + or not finding["file"].strip() + or type(finding.get("line")) is not int + or finding["line"] <= 0 + or finding.get("side") not in {"RIGHT", "LEFT"} + or not isinstance(finding.get("message"), str) + or not finding["message"].strip() + ): + raise NoemaModelOutputError("Noema LLM response contained a malformed finding") + if decision == "request_changes" and not findings: + raise NoemaModelOutputError("Noema LLM request_changes response did not contain a substantive finding") + validate_substantive_verdict(verdict, diff, changed_paths) except (RuntimeError, urllib.error.URLError, http.client.HTTPException, OSError) as exc: + current_failure = _stable_failure_diagnostic(exc) if is_retry: - if isinstance(exc, RuntimeError): - raise - raise RuntimeError(str(exc)) from exc + initial_failure = ( + scrub_sensitive_data(repair_error) + or "no diagnostic message was available" + ) + if isinstance(exc, NoemaModelOutputError): + raise NoemaModelOutputError( + "Noema model-output repair remained invalid; " + f"initial failure: {initial_failure}; repair failure: {current_failure}" + ) from None + if isinstance( + exc, (urllib.error.URLError, http.client.HTTPException, OSError) + ): + raise NoemaTransportError( + "Noema bounded repair transport was exhausted; " + f"initial failure: {initial_failure}; repair failure: " + f"{type(exc).__name__}: {current_failure}" + ) from exc + raise RuntimeError( + "Noema repair failed closed; " + f"initial failure: {initial_failure}; repair failure: {current_failure}" + ) from exc if str(fetch_pr(repo, number).get("headRefOid") or "").lower() != expected_head: raise StaleHeadDuringRepairRetryError( "Pull request head changed during review; stale before repair retry." @@ -1254,7 +1365,7 @@ def call_llm( expected_head, review_context, changed_paths, - str(exc), + current_failure, is_retry=True, ) return verdict diff --git a/scripts/ci/repair_noema_coverage_1617.py b/scripts/ci/repair_noema_coverage_1617.py deleted file mode 100644 index c52a7349bb..0000000000 --- a/scripts/ci/repair_noema_coverage_1617.py +++ /dev/null @@ -1,131 +0,0 @@ -#!/usr/bin/env python3 -"""Add fail-closed coverage for PR #1617's temporary production transform. - -This one-shot helper is removed by the repair workflow before the verified -production commit is created. -""" - -from pathlib import Path - - -ROOT = Path(__file__).resolve().parents[2] -TEST = ROOT / "tests/test_noema_model_output_failure_classification.py" - - -def main() -> None: - text = TEST.read_text(encoding="utf-8") - marker = "def test_repair_wall_clock_deadline_defensive_fail_closed_paths" - if marker in text: - raise RuntimeError("#1617 deadline coverage regressions already present") - text += r''' - - -def test_repair_wall_clock_deadline_defensive_fail_closed_paths(monkeypatch) -> None: - """Invalid budgets/platform state fail closed instead of weakening the bound.""" - import signal - - with pytest.raises(ValueError, match="must be positive"): - with gate._repair_wall_clock_deadline(0): - pass - - if not hasattr(signal, "setitimer"): - pytest.skip("remaining cases require POSIX setitimer") - - monkeypatch.delattr(gate.signal, "setitimer") - with pytest.raises(RuntimeError, match="requires POSIX setitimer support"): - with gate._repair_wall_clock_deadline(1): - pass - - -def test_repair_wall_clock_deadline_refuses_existing_process_alarm() -> None: - """Noema never overwrites another caller's active process alarm.""" - import signal - - if not hasattr(signal, "setitimer"): - pytest.skip("POSIX process timer is required by the Linux review runner") - signal.setitimer(signal.ITIMER_REAL, 30) - try: - with pytest.raises(RuntimeError, match="refused to overwrite"): - with gate._repair_wall_clock_deadline(1): - pass - finally: - signal.setitimer(signal.ITIMER_REAL, 0) - - -def test_repair_wall_clock_deadline_rejects_non_main_thread_signal_context(monkeypatch) -> None: - """A signal handler that cannot be installed fails closed before any timer starts.""" - import signal - - if not hasattr(signal, "setitimer"): - pytest.skip("POSIX process timer is required by the Linux review runner") - - def reject_signal(*_args, **_kwargs): - raise ValueError("signal only works in main thread") - - monkeypatch.setattr(gate.signal, "signal", reject_signal) - with pytest.raises(RuntimeError, match="process main thread"): - with gate._repair_wall_clock_deadline(1): - pass - assert signal.getitimer(signal.ITIMER_REAL)[0] == 0 - - -def test_repair_unexpected_runtime_failure_preserves_initial_model_evidence(monkeypatch) -> None: - """Unexpected corrective parser/runtime failures keep the first trusted diagnostic.""" - import json - - monkeypatch.setenv("NOEMA_LLM_API_URL", "https://llm.example/v1/chat/completions") - monkeypatch.setenv("NOEMA_LLM_API_KEY", "test-key") - head_sha = "e" * 40 - - class Response: - def __enter__(self): - return self - - def __exit__(self, *_args): - return None - - def read(self): - return json.dumps( - {"choices": [{"message": {"content": json.dumps(_verdict())}}]} - ).encode() - - monkeypatch.setattr( - gate.urllib.request.OpenerDirector, - "open", - lambda *_args, **_kwargs: Response(), - ) - monkeypatch.setattr(gate, "fetch_pr", lambda _repo, _number: {"headRefOid": head_sha}) - original_decode = gate.decode_llm_response_body - decode_calls = 0 - - def decode_once_then_fail(raw_bytes): - nonlocal decode_calls - decode_calls += 1 - if decode_calls == 2: - raise RuntimeError("repair parser invariant failed") - return original_decode(raw_bytes) - - monkeypatch.setattr(gate, "decode_llm_response_body", decode_once_then_fail) - - with pytest.raises(RuntimeError) as exc_info: - gate.call_llm( - "owner/repo", - 7, - {"title": "test", "headRefOid": head_sha}, - DIFF, - False, - head_sha, - changed_paths=("README.md",), - ) - - message = str(exc_info.value) - assert "Noema repair failed closed" in message - assert "outcome must be falsified or confirmed" in message - assert "repair parser invariant failed" in message - assert decode_calls == 2 -''' - TEST.write_text(text, encoding="utf-8") - - -if __name__ == "__main__": - main() diff --git a/scripts/ci/repair_noema_model_output_1617.py b/scripts/ci/repair_noema_model_output_1617.py deleted file mode 100644 index 609dada1ad..0000000000 --- a/scripts/ci/repair_noema_model_output_1617.py +++ /dev/null @@ -1,291 +0,0 @@ -#!/usr/bin/env python3 -"""Apply the one-shot, test-first Noema model-output repair for PR #1617. - -This helper exists only to make an exact, reviewable transformation on the -single-writer PR branch. The workflow that invokes it deletes this helper and -itself before committing the production repair. -""" - -from pathlib import Path - - -ROOT = Path(__file__).resolve().parents[2] -SOURCE = ROOT / "scripts/ci/noema_review_gate.py" -TEST = ROOT / "tests/test_noema_model_output_failure_classification.py" -CHANGELOG = ROOT / "CHANGELOG.md" -BASELINE = ROOT / "docs/product-technical-gap-baseline.md" -ARCHITECTURE = ROOT / "ARCHITECTURE.md" -DOCTORING = ROOT / "docs/doctoring/noema-model-output-repair-boundary.md" - - -def replace_once(text: str, old: str, new: str, label: str) -> str: - """Replace one exact source fragment and fail closed on drift.""" - count = text.count(old) - if count != 1: - raise RuntimeError(f"{label}: expected exactly one match, found {count}") - return text.replace(old, new, 1) - - -def replace_raises_between(text: str, start: str, end: str) -> str: - """Retype model-output validation errors within one bounded source span.""" - start_index = text.index(start) - end_index = text.index(end, start_index) - span = text[start_index:end_index] - if "raise RuntimeError(" not in span: - raise RuntimeError(f"{start.strip()}: no RuntimeError raises found") - span = span.replace("raise RuntimeError(", "raise NoemaModelOutputError(") - return text[:start_index] + span + text[end_index:] - - -def update_source() -> None: - """Implement typed model-output failures and a bounded one-time repair call.""" - text = SOURCE.read_text(encoding="utf-8") - text = replace_once( - text, - 'ORCHESTRATOR_BASE_ENV = "CONTEXTUAL_ORCHESTRATOR_BASE_URL"\n', - 'ORCHESTRATOR_BASE_ENV = "CONTEXTUAL_ORCHESTRATOR_BASE_URL"\n' - '# A repair request corrects an already-completed model verdict; it is not a\n' - '# second unbounded full review. Fifteen minutes is the hard client-side\n' - '# ceiling for that one corrective HTTP request. The primary review remains\n' - '# governed by contextual-orchestrator rather than a fixed inference timeout.\n' - 'NOEMA_REPAIR_TIMEOUT_SECONDS = 15 * 60\n\n\n' - 'class NoemaModelOutputError(RuntimeError):\n' - ' """Raised when untrusted model output violates the trusted verdict contract."""\n\n\n' - 'class NoemaTransportError(RuntimeError):\n' - ' """Raised when the bounded review transport cannot produce usable evidence."""\n', - "typed Noema error classes", - ) - - text = replace_raises_between( - text, - "def validate_substantive_verdict(\n", - "\ndef truncate_text(", - ) - text = replace_raises_between(text, "def extract_json_object(", "\ndef extract_llm_message_content(") - text = replace_raises_between( - text, - "def extract_llm_message_content(", - "\ndef decode_llm_response_body(", - ) - text = replace_raises_between( - text, - "def decode_llm_response_body(", - "\ndef _truthy_env(", - ) - - # Retype the immediate post-response verdict-shape checks. These are all - # model-output/schema failures, not GitHub/source or transport failures. - for old, new in ( - ( - 'raise RuntimeError(f"Noema LLM returned unsupported decision: {decision!r}")', - 'raise NoemaModelOutputError(f"Noema LLM returned unsupported decision: {decision!r}")', - ), - ( - 'raise RuntimeError("Noema LLM response did not contain a substantive summary")', - 'raise NoemaModelOutputError("Noema LLM response did not contain a substantive summary")', - ), - ( - 'raise RuntimeError("Noema LLM response findings must be a list of objects")', - 'raise NoemaModelOutputError("Noema LLM response findings must be a list of objects")', - ), - ( - 'raise RuntimeError("Noema LLM response contained a malformed finding")', - 'raise NoemaModelOutputError("Noema LLM response contained a malformed finding")', - ), - ( - 'raise RuntimeError("Noema LLM request_changes response did not contain a substantive finding")', - 'raise NoemaModelOutputError("Noema LLM request_changes response did not contain a substantive finding")', - ), - ): - text = replace_once(text, old, new, old) - - text = replace_once( - text, - """ with opener.open(request) as response: # nosec B310\n raw_bytes = response.read()\n""", - """ if is_retry:\n response_context = opener.open( # nosec B310\n request, timeout=NOEMA_REPAIR_TIMEOUT_SECONDS\n )\n else:\n response_context = opener.open(request) # nosec B310\n with response_context as response:\n raw_bytes = response.read()\n""", - "bounded repair HTTP timeout", - ) - - text = replace_once( - text, - """ except (RuntimeError, urllib.error.URLError, http.client.HTTPException, OSError) as exc:\n if is_retry:\n if isinstance(exc, RuntimeError):\n raise\n raise RuntimeError(str(exc)) from exc\n if str(fetch_pr(repo, number).get(\"headRefOid\") or \"\").lower() != expected_head:\n raise StaleHeadDuringRepairRetryError(\n \"Pull request head changed during review; stale before repair retry.\"\n ) from exc\n return call_llm(\n repo,\n number,\n pr,\n diff,\n truncated,\n expected_head,\n review_context,\n changed_paths,\n str(exc),\n is_retry=True,\n )\n""", - """ except (RuntimeError, urllib.error.URLError, http.client.HTTPException, OSError) as exc:\n current_failure = scrub_sensitive_data(str(exc)) or type(exc).__name__\n if is_retry:\n initial_failure = (\n scrub_sensitive_data(repair_error)\n or \"no diagnostic message was available\"\n )\n if isinstance(exc, NoemaModelOutputError):\n raise NoemaModelOutputError(\n \"Noema model-output repair remained invalid; \"\n f\"initial failure: {initial_failure}; repair failure: {current_failure}\"\n ) from exc\n if isinstance(\n exc, (urllib.error.URLError, http.client.HTTPException, OSError)\n ):\n raise NoemaTransportError(\n \"Noema bounded repair transport was exhausted; \"\n f\"initial failure: {initial_failure}; repair failure: \"\n f\"{type(exc).__name__}: {current_failure}\"\n ) from exc\n raise RuntimeError(\n \"Noema repair failed closed; \"\n f\"initial failure: {initial_failure}; repair failure: {current_failure}\"\n ) from exc\n if str(fetch_pr(repo, number).get(\"headRefOid\") or \"\").lower() != expected_head:\n raise StaleHeadDuringRepairRetryError(\n \"Pull request head changed during review; stale before repair retry.\"\n ) from exc\n return call_llm(\n repo,\n number,\n pr,\n diff,\n truncated,\n expected_head,\n review_context,\n changed_paths,\n current_failure,\n is_retry=True,\n )\n""", - "typed repair exhaustion", - ) - - text = text.replace( - "Fails closed with ``RuntimeError``", - "Fails closed with ``NoemaModelOutputError``", - ) - SOURCE.write_text(text, encoding="utf-8") - - -def update_tests() -> None: - """Extend the pre-existing RED with timeout and evidence-preservation coverage.""" - text = TEST.read_text(encoding="utf-8") - marker = "def test_bounded_repair_preserves_initial_schema_and_transport_evidence" - if marker in text: - raise RuntimeError("#1617 repair tests already present") - text += r''' - - -def test_bounded_repair_preserves_initial_schema_and_transport_evidence(monkeypatch) -> None: - """A malformed verdict followed by 502 keeps both typed evidence classes.""" - import json - import urllib.error - - monkeypatch.setenv("NOEMA_LLM_API_URL", "https://llm.example/v1/chat/completions") - monkeypatch.setenv("NOEMA_LLM_API_KEY", "test-key") - head_sha = "a" * 40 - requests: list[tuple[object, dict]] = [] - - class Response: - def __enter__(self): - return self - - def __exit__(self, *_args): - return None - - def read(self): - return json.dumps( - {"choices": [{"message": {"content": json.dumps(_verdict())}}]} - ).encode() - - def open_response(_opener, request, **kwargs): - requests.append((request, kwargs)) - if len(requests) == 1: - return Response() - raise urllib.error.HTTPError(request.full_url, 502, "Bad Gateway", {}, None) - - monkeypatch.setattr(gate.urllib.request.OpenerDirector, "open", open_response) - monkeypatch.setattr( - gate, - "fetch_pr", - lambda _repo, _number: {"headRefOid": head_sha}, - ) - - with pytest.raises(gate.NoemaTransportError) as exc_info: - gate.call_llm( - "owner/repo", - 7, - {"title": "test", "headRefOid": head_sha}, - DIFF, - False, - head_sha, - changed_paths=("README.md",), - ) - - message = str(exc_info.value) - assert "outcome must be falsified or confirmed" in message - assert "HTTPError" in message - assert "502" in message - assert len(requests) == 2 - assert requests[0][1] == {} - assert requests[1][1]["timeout"] == gate.NOEMA_REPAIR_TIMEOUT_SECONDS - - -def test_repeated_model_output_failure_remains_typed(monkeypatch) -> None: - """A second malformed verdict fails closed as model-output evidence.""" - import json - - monkeypatch.setenv("NOEMA_LLM_API_URL", "https://llm.example/v1/chat/completions") - monkeypatch.setenv("NOEMA_LLM_API_KEY", "test-key") - head_sha = "b" * 40 - - class Response: - def __enter__(self): - return self - - def __exit__(self, *_args): - return None - - def read(self): - return json.dumps( - {"choices": [{"message": {"content": json.dumps(_verdict())}}]} - ).encode() - - monkeypatch.setattr( - gate.urllib.request.OpenerDirector, - "open", - lambda *_args, **_kwargs: Response(), - ) - monkeypatch.setattr( - gate, - "fetch_pr", - lambda _repo, _number: {"headRefOid": head_sha}, - ) - - with pytest.raises(gate.NoemaModelOutputError) as exc_info: - gate.call_llm( - "owner/repo", - 7, - {"title": "test", "headRefOid": head_sha}, - DIFF, - False, - head_sha, - changed_paths=("README.md",), - ) - - assert "initial failure" in str(exc_info.value) - assert "repair failure" in str(exc_info.value) -''' - TEST.write_text(text, encoding="utf-8") - - -def update_docs() -> None: - """Record the RCA, bounded contract, and architecture consequence.""" - changelog = CHANGELOG.read_text(encoding="utf-8") - entry = """- **Classify and bound Noema malformed-verdict repair failures (#1611/#1617).** A schema-invalid model verdict now raises typed `NoemaModelOutputError` evidence instead of an undifferentiated runtime failure. The one corrective HTTP request has a 15-minute client ceiling while the primary contextual-orchestrator review remains under its no-fixed-inference-timeout contract. If the repair then fails at transport, `NoemaTransportError` preserves the first validator diagnostic plus the later transport class/status without logging raw model output or secrets.\n""" - changelog = replace_once(changelog, "## [Unreleased]\n", "## [Unreleased]\n" + entry, "changelog unreleased") - CHANGELOG.write_text(changelog, encoding="utf-8") - - architecture = ARCHITECTURE.read_text(encoding="utf-8") - architecture_note = """ - -### Noema model-output and repair boundary - -Noema separates deterministic model-output/schema failures from GitHub/source -findings and provider transport exhaustion. A malformed verdict remains -non-passing and is represented by `NoemaModelOutputError`. Its single corrective -request still routes only through the loopback contextual-orchestrator -`orchestrator/free` gateway, but is capped at 15 minutes because it repairs an -already-completed verdict rather than performing a second unbounded full -review. If that corrective request encounters transport exhaustion, the typed -transport error retains both the first trusted-validator diagnostic and the -later transport class/status while omitting raw model content and secrets. -""" - if "### Noema model-output and repair boundary" not in architecture: - architecture += architecture_note - ARCHITECTURE.write_text(architecture, encoding="utf-8") - - baseline = BASELINE.read_text(encoding="utf-8") - baseline_note = """ - -## 2026-09-01 Noema malformed-verdict retry classification and wall-clock bound (#1611/#1617) - -- **Observed consumer evidence:** `ContextualWisdomLab/naruon#1505@7da2a242e463f59d4580cb38e7591f1ba4b4049e`, Required Noema run `33460498090` / job `99742587317`. The first response reached the trusted semantic validator but used an out-of-domain adversarial-probe `outcome`; the generic repair attempt later ended as HTTP 502 after roughly 88 minutes. -- **Root cause:** model-output/schema rejection, repair transport exhaustion, and consumer-source findings shared an undifferentiated `RuntimeError` boundary. The corrective HTTP request also had no client-side repair-specific ceiling, so a malformed first verdict could initiate another effectively full-duration request. -- **Repair:** model-output/schema rejection is typed as `NoemaModelOutputError`; the one corrective request has a 900-second hard client ceiling; repair transport exhaustion is typed as `NoemaTransportError`; and the final fail-closed diagnostic preserves the sanitized first validator error plus the later typed transport evidence. Primary review inference remains governed by contextual-orchestrator `orchestrator/free` and is not given a new fixed model-inference timeout. -- **Security/operability invariant:** raw model content, credentials, and provider secrets are never included in the combined diagnostic. Exact-head revalidation still occurs before retry and before publication. No direct-provider fallback or GitHub authority change is introduced. -- **Verification contract:** deterministic tests cover the original invalid `outcome`, malformed-then-502 evidence preservation and the repair-only timeout, and repeated malformed model output remaining typed and non-passing. The affected Naruon head must be re-run after protected integration; predecessor review/check evidence does not transfer. -""" - if "## 2026-09-01 Noema malformed-verdict retry classification" not in baseline: - baseline += baseline_note - BASELINE.write_text(baseline, encoding="utf-8") - - DOCTORING.parent.mkdir(parents=True, exist_ok=True) - DOCTORING.write_text( - """# Noema model-output repair boundary\n\n## Incident\n\nOn 2026-09-01 the required Noema review for `ContextualWisdomLab/naruon#1505` reached deterministic verdict validation, rejected an adversarial-probe `outcome` outside the closed `falsified|confirmed` domain, then spent the repair path on a long second model call that ultimately surfaced only `HTTP 502 Bad Gateway`. That final transport symptom erased the more informative first trusted-validator failure from the top-level diagnostic.\n\n## Decision\n\n1. Model-produced JSON/envelope/schema/semantic-contract failures are `NoemaModelOutputError`; they remain fail-closed and are not consumer-source findings.\n2. The primary review keeps the accepted contextual-orchestrator no-fixed-inference-timeout contract. The *single corrective request* is different: it repairs an already-completed verdict and therefore has a hard 900-second `urllib` client timeout.\n3. A corrective transport failure is `NoemaTransportError` and carries the sanitized first validator diagnostic plus the later transport exception class/status. Raw model output is never copied into public Actions diagnostics.\n4. Exact-head validation before retry and before publication remains mandatory. All model traffic remains on contextual-orchestrator `orchestrator/free`.\n\n## Verification\n\nThe #1617 regression first proved RED because `NoemaModelOutputError` did not exist. The repair adds focused cases for malformed-verdict typing, malformed-then-502 evidence preservation with the 900-second repair-only timeout, and repeated malformed output remaining typed and non-passing. The repository full coverage/docstring gate is run before the one-shot repair workflow commits the result.\n\n## References\n\nFielding, R., Nottingham, M., & Reschke, J. (2022). *HTTP semantics* (RFC 9110). Internet Engineering Task Force.\n\nPython Software Foundation. (2026). *urllib.request — Extensible library for opening URLs*. Python 3 documentation.\n""", - encoding="utf-8", - ) - - -def main() -> None: - """Apply all production, regression, and traceability changes.""" - update_source() - update_tests() - update_docs() - - -if __name__ == "__main__": - main() diff --git a/scripts/ci/repair_noema_model_output_followup_1617.py b/scripts/ci/repair_noema_model_output_followup_1617.py deleted file mode 100644 index fd1c0ff60f..0000000000 --- a/scripts/ci/repair_noema_model_output_followup_1617.py +++ /dev/null @@ -1,303 +0,0 @@ -#!/usr/bin/env python3 -"""Close the remaining reviewed #1617 model-output and coverage gaps. - -Temporary exact-head repair helper. The branch workflow removes this file before -verification and the production commit. -""" - -from pathlib import Path - -ROOT = Path(__file__).resolve().parents[2] -SOURCE = ROOT / "scripts/ci/noema_review_gate.py" -TEST = ROOT / "tests/test_noema_model_output_failure_classification.py" -CHANGELOG = ROOT / "CHANGELOG.md" -ARCHITECTURE = ROOT / "ARCHITECTURE.md" -BASELINE = ROOT / "docs/product-technical-gap-baseline.md" -DOCTORING = ROOT / "docs/doctoring/noema-model-output-repair-boundary.md" - - -def replace_once(text: str, old: str, new: str, label: str) -> str: - count = text.count(old) - if count != 1: - raise RuntimeError(f"{label}: expected exactly one match, found {count}") - return text.replace(old, new, 1) - - -def update_source() -> None: - text = SOURCE.read_text(encoding="utf-8") - - # A missing/invalid trusted diff is source evidence, not model output. - text = replace_once( - text, - ' raise NoemaModelOutputError("Noema formal verdict requires parseable changed-line evidence")\n', - ' raise RuntimeError("Noema formal verdict requires parseable changed-line evidence")\n', - "trusted diff classification", - ) - - deadline_class = '''class NoemaRepairDeadlineExceeded(TimeoutError): - """Raised when the corrective attempt exceeds its total wall-clock budget.""" -''' - diagnostic_helper = deadline_class + '''\n\ndef _stable_failure_diagnostic(exc: BaseException) -> str: - """Return actionable trusted diagnostics without reflecting model values.""" - message = scrub_sensitive_data(str(exc)) or type(exc).__name__ - if not isinstance(exc, NoemaModelOutputError): - return message - - # Model-output exceptions are raised only by deterministic parsing and - # validation code. Preserve those static/structural diagnostics because - # they tell the corrective model and operators exactly which contract was - # violated. The one validator that embeds an untrusted model value is the - # unsupported-decision check; redact that value. Unknown model-output - # exception text fails closed to a stable code rather than being reflected. - if message.startswith("Noema LLM returned unsupported decision:"): - return "Noema LLM returned unsupported decision" - trusted_prefixes = ( - "Noema LLM response ", - "Noema LLM request_changes ", - "Noema formal verdict ", - "Noema reviewed line ", - "Noema adversarial validation ", - "Noema adversarial probe ", - "Noema approve ", - "Noema request_changes ", - ) - if message.startswith(trusted_prefixes): - return message - return "model-output-contract-invalid" -''' - text = replace_once( - text, - deadline_class, - diagnostic_helper, - "stable model-output diagnostic helper", - ) - - text = replace_once( - text, - ' current_failure = scrub_sensitive_data(str(exc)) or type(exc).__name__\n', - ' current_failure = _stable_failure_diagnostic(exc)\n', - "stable current failure diagnostic", - ) - - # Do not retain a model-controlled exception as an explicit cause: a raw - # unsupported decision/probe sentinel must not reappear in traceback output. - old_raise = ''' raise NoemaModelOutputError( - "Noema model-output repair remained invalid; " - f"initial failure: {initial_failure}; repair failure: {current_failure}" - ) from exc -''' - new_raise = ''' raise NoemaModelOutputError( - "Noema model-output repair remained invalid; " - f"initial failure: {initial_failure}; repair failure: {current_failure}" - ) from None -''' - text = replace_once(text, old_raise, new_raise, "model-output exception chaining") - SOURCE.write_text(text, encoding="utf-8") - - -def update_tests() -> None: - text = TEST.read_text(encoding="utf-8") - marker = "def test_unparseable_diff_remains_source_evidence" - if marker in text: - raise RuntimeError("follow-up #1617 regressions already present") - text += r''' - - -def test_unparseable_diff_remains_source_evidence() -> None: - """A location-free trusted diff is not retyped as model-output failure.""" - with pytest.raises(RuntimeError) as exc_info: - gate.validate_substantive_verdict(_verdict(), "not a unified diff", ["README.md"]) - assert not isinstance(exc_info.value, gate.NoemaModelOutputError) - assert "parseable changed-line evidence" in str(exc_info.value) - - -def test_model_sentinel_never_reaches_repair_prompt_or_final_diagnostic(monkeypatch) -> None: - """Model-controlled invalid values are redacted while the defect class stays actionable.""" - import json - - sentinel = "MODEL_SENTINEL_DO_NOT_REFLECT" - monkeypatch.setenv("NOEMA_LLM_API_URL", "https://llm.example/v1/chat/completions") - monkeypatch.setenv("NOEMA_LLM_API_KEY", "test-key") - head_sha = "e" * 40 - requests = [] - - class Response: - def __enter__(self): - return self - - def __exit__(self, *_args): - return None - - def read(self): - return json.dumps( - {"choices": [{"message": {"content": json.dumps({"decision": sentinel})}}]} - ).encode() - - def open_response(_opener, request, **kwargs): - assert kwargs == {} - requests.append(request) - return Response() - - monkeypatch.setattr(gate.urllib.request.OpenerDirector, "open", open_response) - monkeypatch.setattr(gate, "fetch_pr", lambda _repo, _number: {"headRefOid": head_sha}) - - with pytest.raises(gate.NoemaModelOutputError) as exc_info: - gate.call_llm( - "owner/repo", - 7, - {"title": "test", "headRefOid": head_sha}, - DIFF, - False, - head_sha, - changed_paths=("README.md",), - ) - - assert len(requests) == 2 - repair_payload = requests[1].data.decode("utf-8") - assert sentinel not in repair_payload - assert "Noema LLM returned unsupported decision" in repair_payload - assert sentinel not in str(exc_info.value) - assert "Noema LLM returned unsupported decision" in str(exc_info.value) - assert exc_info.value.__cause__ is None - - -def test_stable_failure_diagnostic_preserves_trusted_structure_and_redacts_values() -> None: - """Trusted validator detail stays actionable; arbitrary model text stays opaque.""" - trusted = gate.NoemaModelOutputError( - "Noema adversarial probe 1 outcome must be falsified or confirmed" - ) - assert gate._stable_failure_diagnostic(trusted) == str(trusted) - request_changes = gate.NoemaModelOutputError( - "Noema LLM request_changes response did not contain a substantive finding" - ) - assert gate._stable_failure_diagnostic(request_changes) == str(request_changes) - assert gate._stable_failure_diagnostic( - gate.NoemaModelOutputError("Noema LLM returned unsupported decision: 'SECRET_VALUE'") - ) == "Noema LLM returned unsupported decision" - assert gate._stable_failure_diagnostic( - gate.NoemaModelOutputError("secret-ish model text") - ) == "model-output-contract-invalid" - assert gate._stable_failure_diagnostic(TimeoutError()) == "TimeoutError" - - -def test_repair_deadline_rejects_nonpositive_budget() -> None: - with pytest.raises(ValueError, match="must be positive"): - with gate._repair_wall_clock_deadline(0): - pass - - -def test_repair_deadline_requires_setitimer(monkeypatch) -> None: - monkeypatch.delattr(gate.signal, "setitimer") - with pytest.raises(RuntimeError, match="requires POSIX"): - with gate._repair_wall_clock_deadline(1): - pass - - -def test_repair_deadline_requires_itimer_real(monkeypatch) -> None: - monkeypatch.delattr(gate.signal, "ITIMER_REAL") - with pytest.raises(RuntimeError, match="requires POSIX"): - with gate._repair_wall_clock_deadline(1): - pass - - -@pytest.mark.parametrize("timer_state", [(1.0, 0.0), (0.0, 1.0)]) -def test_repair_deadline_refuses_existing_process_alarm(monkeypatch, timer_state) -> None: - monkeypatch.setattr(gate.signal, "getitimer", lambda _which: timer_state) - with pytest.raises(RuntimeError, match="active process alarm"): - with gate._repair_wall_clock_deadline(1): - pass - - -def test_repair_deadline_requires_main_thread_signal_registration(monkeypatch) -> None: - monkeypatch.setattr(gate.signal, "getitimer", lambda _which: (0.0, 0.0)) - - def reject_signal(*_args): - raise ValueError("signal only works in main thread") - - monkeypatch.setattr(gate.signal, "signal", reject_signal) - with pytest.raises(RuntimeError, match="process main thread"): - with gate._repair_wall_clock_deadline(1): - pass -''' - TEST.write_text(text, encoding="utf-8") - - -def update_docs() -> None: - """Add drift-safe traceability for the actionable diagnostic contract.""" - changelog = CHANGELOG.read_text(encoding="utf-8") - changelog_entry = ( - "- **Harden #1617 corrective diagnostics against model-value reflection.** " - "The repair prompt and final fail-closed error preserve deterministic structural validator evidence " - "needed to correct a malformed verdict, while model-controlled values (including an unsupported " - "decision value) are redacted and an unknown model-output diagnostic collapses to a stable code.\n" - ) - if changelog_entry not in changelog: - changelog = replace_once( - changelog, - "## [Unreleased]\n", - "## [Unreleased]\n" + changelog_entry, - "changelog unreleased heading", - ) - CHANGELOG.write_text(changelog, encoding="utf-8") - - architecture = ARCHITECTURE.read_text(encoding="utf-8") - architecture_marker = "#### Actionable Noema repair diagnostics" - if architecture_marker not in architecture: - architecture += """ - -#### Actionable Noema repair diagnostics - -The corrective prompt may retain only deterministic structural validator diagnostics -that are generated by trusted validation code. Model-controlled values are never -reflected into the corrective prompt or public exception chain: unsupported decision -values are reduced to their static defect class and unknown model-output diagnostics -collapse to a stable code. This keeps repair evidence actionable without turning the -reviewer itself into a data-reflection channel. -""" - ARCHITECTURE.write_text(architecture, encoding="utf-8") - - baseline = BASELINE.read_text(encoding="utf-8") - baseline_marker = "- **Diagnostic hardening:** #1617 corrective prompts" - if baseline_marker not in baseline: - baseline_heading = ( - "## 2026-09-01 Noema malformed-verdict retry classification and wall-clock bound (#1611/#1617)\n" - ) - baseline_note = ( - "\n- **Diagnostic hardening:** #1617 corrective prompts preserve only trusted structural validator " - "detail; model-controlled values are redacted, unknown model-output text becomes a stable defect " - "code, and repeated invalid-model exceptions do not retain the raw model exception as a cause.\n" - ) - baseline = replace_once( - baseline, - baseline_heading, - baseline_heading + baseline_note, - "baseline #1617 heading", - ) - BASELINE.write_text(baseline, encoding="utf-8") - - doctoring = DOCTORING.read_text(encoding="utf-8") - doctoring_marker = "## Actionable diagnostic boundary" - if doctoring_marker not in doctoring: - doctoring += """ - -## Actionable diagnostic boundary - -Corrective prompts need the deterministic *class* of a malformed verdict to repair it, -but do not need arbitrary model-produced values. Trusted structural validator messages -(such as a missing required field or an invalid adversarial-probe outcome class) remain -available after secret scrubbing. Unsupported decision values and unknown model-output -text are redacted to stable diagnostics, and a repeated invalid-model exception is raised -without retaining the raw model exception as an explicit cause. Tests use a sentinel value -to prove it reaches neither the retry prompt nor the final diagnostic. -""" - DOCTORING.write_text(doctoring, encoding="utf-8") - - -def main() -> None: - update_source() - update_tests() - update_docs() - - -if __name__ == "__main__": - main() diff --git a/scripts/ci/repair_noema_timeout_fixture_1617.py b/scripts/ci/repair_noema_timeout_fixture_1617.py deleted file mode 100644 index ea15ffae53..0000000000 --- a/scripts/ci/repair_noema_timeout_fixture_1617.py +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env python3 -"""Update the existing Noema repair fixture for the repair-only timeout contract.""" - -from pathlib import Path - - -# Temporary writer-token retry trigger; this helper is deleted by the repair. -TEST = Path(__file__).resolve().parents[2] / "tests/test_noema_review_gate.py" - - -def main() -> None: - """Require no primary timeout and the bounded timeout on the one repair call.""" - text = TEST.read_text(encoding="utf-8") - old = ''' def open(self, request, timeout=None): - assert timeout is None - payloads.append(json.loads(request.data)) - return Response(invalid if len(payloads) == 1 else valid) -''' - new = ''' def open(self, request, timeout=None): - if payloads: - assert timeout == noema.NOEMA_REPAIR_TIMEOUT_SECONDS - else: - assert timeout is None - payloads.append(json.loads(request.data)) - return Response(invalid if len(payloads) == 1 else valid) -''' - count = text.count(old) - if count != 1: - raise RuntimeError(f"expected one repair-timeout fixture, found {count}") - TEST.write_text(text.replace(old, new, 1), encoding="utf-8") - - -if __name__ == "__main__": - main() diff --git a/scripts/ci/repair_noema_wall_clock_1617.py b/scripts/ci/repair_noema_wall_clock_1617.py deleted file mode 100644 index 58c2e8c0d0..0000000000 --- a/scripts/ci/repair_noema_wall_clock_1617.py +++ /dev/null @@ -1,248 +0,0 @@ -#!/usr/bin/env python3 -"""Finish PR #1617 with a true repair wall-clock deadline. - -Temporary one-shot branch repair helper. The repair workflow removes this file -before committing the production change. -""" - -from pathlib import Path -import textwrap - - -ROOT = Path(__file__).resolve().parents[2] -SOURCE = ROOT / "scripts/ci/noema_review_gate.py" -TEST = ROOT / "tests/test_noema_model_output_failure_classification.py" -CHANGELOG = ROOT / "CHANGELOG.md" -BASELINE = ROOT / "docs/product-technical-gap-baseline.md" -ARCHITECTURE = ROOT / "ARCHITECTURE.md" -DOCTORING = ROOT / "docs/doctoring/noema-model-output-repair-boundary.md" - - -def replace_once(text: str, old: str, new: str, label: str) -> str: - count = text.count(old) - if count != 1: - raise RuntimeError(f"{label}: expected exactly one match, found {count}") - return text.replace(old, new, 1) - - -def update_source() -> None: - text = SOURCE.read_text(encoding="utf-8") - text = replace_once(text, "import base64\n", "import base64\nimport contextlib\n", "contextlib import") - text = replace_once(text, "import re\n", "import re\nimport signal\n", "signal import") - text = replace_once( - text, - "# A repair request corrects an already-completed model verdict; it is not a\n" - "# second unbounded full review. Fifteen minutes is the hard client-side\n" - "# ceiling for that one corrective HTTP request. The primary review remains\n" - "# governed by contextual-orchestrator rather than a fixed inference timeout.\n" - "NOEMA_REPAIR_TIMEOUT_SECONDS = 15 * 60\n", - "# A repair request corrects an already-completed model verdict; it is not a\n" - "# second unbounded full review. Fifteen minutes is an absolute wall-clock\n" - "# deadline for the complete corrective attempt (open/read/decode/validate),\n" - "# not a socket inactivity timeout. The primary review remains governed by\n" - "# contextual-orchestrator rather than a fixed inference timeout.\n" - "NOEMA_REPAIR_DEADLINE_SECONDS = 15 * 60\n", - "repair deadline constant", - ) - marker = '''class NoemaTransportError(RuntimeError): - """Raised when the bounded review transport cannot produce usable evidence.""" -''' - addition = marker + '''\n\nclass NoemaRepairDeadlineExceeded(TimeoutError): - """Raised when the corrective attempt exceeds its total wall-clock budget.""" -''' - text = replace_once(text, marker, addition, "deadline error class") - - stale_marker = '''class StaleHeadDuringRepairRetryError(RuntimeError): - """Raised when the PR head moves before ``call_llm``'s repair-retry request fires.""" -''' - deadline_helper = '''@contextlib.contextmanager -def _repair_wall_clock_deadline(seconds: float): - """Interrupt the entire corrective attempt after ``seconds`` of wall time. - - ``urllib``'s timeout is a socket-operation timeout and can be extended by - trickling bytes. Required Noema Review runs on Linux, so ITIMER_REAL gives - the repair attempt one process-level wall-clock budget across open, read, - decode, and deterministic validation. An existing process alarm is not - overwritten; that condition fails closed instead. - """ - if seconds <= 0: - raise ValueError("repair wall-clock deadline must be positive") - if not hasattr(signal, "setitimer") or not hasattr(signal, "ITIMER_REAL"): - raise RuntimeError("repair wall-clock deadline requires POSIX setitimer support") - previous_remaining, previous_interval = signal.getitimer(signal.ITIMER_REAL) - if previous_remaining > 0 or previous_interval > 0: - raise RuntimeError("repair wall-clock deadline refused to overwrite an active process alarm") - previous_handler = signal.getsignal(signal.SIGALRM) - - def expire(_signum, _frame): - """Raise the typed deadline signal without reflecting response content.""" - raise NoemaRepairDeadlineExceeded( - f"Noema repair exceeded {seconds:g}-second absolute wall-clock deadline" - ) - - try: - signal.signal(signal.SIGALRM, expire) - except ValueError as exc: - raise RuntimeError("repair wall-clock deadline must run on the process main thread") from exc - signal.setitimer(signal.ITIMER_REAL, seconds) - try: - yield - finally: - signal.setitimer(signal.ITIMER_REAL, 0) - signal.signal(signal.SIGALRM, previous_handler) - - -''' + stale_marker - text = replace_once(text, stale_marker, deadline_helper, "deadline helper") - - old_open = ''' if is_retry: - response_context = opener.open( # nosec B310 - request, timeout=NOEMA_REPAIR_TIMEOUT_SECONDS - ) - else: - response_context = opener.open(request) # nosec B310 - with response_context as response: - raw_bytes = response.read() -''' - plain_open = ''' with opener.open(request) as response: # nosec B310 - raw_bytes = response.read() -''' - text = replace_once(text, old_open, plain_open, "remove socket timeout") - - try_marker = " try:\n with opener.open(request) as response: # nosec B310\n" - start = text.index(try_marker) - body_start = start + len(" try:\n") - except_marker = " except (RuntimeError, urllib.error.URLError, http.client.HTTPException, OSError) as exc:\n" - end = text.index(except_marker, body_start) - body = text[body_start:end] - wrapped = ( - " deadline_context = (\n" - " _repair_wall_clock_deadline(NOEMA_REPAIR_DEADLINE_SECONDS)\n" - " if is_retry\n" - " else contextlib.nullcontext()\n" - " )\n" - " with deadline_context:\n" - + textwrap.indent(body, " ") - ) - text = text[:body_start] + wrapped + text[end:] - SOURCE.write_text(text, encoding="utf-8") - - -def update_tests() -> None: - text = TEST.read_text(encoding="utf-8") - text = replace_once( - text, - ' assert requests[1][1]["timeout"] == gate.NOEMA_REPAIR_TIMEOUT_SECONDS\n', - ' assert requests[1][1] == {}\n', - "socket-timeout assertion", - ) - marker = "def test_total_repair_wall_clock_deadline_interrupts_slow_read" - if marker in text: - raise RuntimeError("wall-clock regression already present") - text += r''' - - -def test_total_repair_wall_clock_deadline_interrupts_slow_read(monkeypatch) -> None: - """Trickling/slow response activity cannot extend the one repair budget.""" - import json - import signal - import time - - if not hasattr(signal, "setitimer"): - pytest.skip("POSIX process timer is required by the Linux review runner") - - monkeypatch.setenv("NOEMA_LLM_API_URL", "https://llm.example/v1/chat/completions") - monkeypatch.setenv("NOEMA_LLM_API_KEY", "test-key") - monkeypatch.setattr(gate, "NOEMA_REPAIR_DEADLINE_SECONDS", 0.05) - head_sha = "d" * 40 - calls = 0 - - class FirstResponse: - def __enter__(self): - return self - - def __exit__(self, *_args): - return None - - def read(self): - return json.dumps( - {"choices": [{"message": {"content": json.dumps(_verdict())}}]} - ).encode() - - class SlowRepairResponse: - def __enter__(self): - return self - - def __exit__(self, *_args): - return None - - def read(self): - time.sleep(2) - return b"{}" - - def open_response(_opener, _request, **kwargs): - nonlocal calls - calls += 1 - assert kwargs == {} - return FirstResponse() if calls == 1 else SlowRepairResponse() - - monkeypatch.setattr(gate.urllib.request.OpenerDirector, "open", open_response) - monkeypatch.setattr(gate, "fetch_pr", lambda _repo, _number: {"headRefOid": head_sha}) - - started = time.monotonic() - with pytest.raises(gate.NoemaTransportError) as exc_info: - gate.call_llm( - "owner/repo", - 7, - {"title": "test", "headRefOid": head_sha}, - DIFF, - False, - head_sha, - changed_paths=("README.md",), - ) - elapsed = time.monotonic() - started - - message = str(exc_info.value) - assert "outcome must be falsified or confirmed" in message - assert "NoemaRepairDeadlineExceeded" in message - assert "wall-clock deadline" in message - assert elapsed < 1.0 - assert calls == 2 - assert signal.getitimer(signal.ITIMER_REAL)[0] == 0 -''' - TEST.write_text(text, encoding="utf-8") - - -def update_docs() -> None: - replacements = { - CHANGELOG: ( - "The one corrective HTTP request has a 15-minute client ceiling while the primary contextual-orchestrator review remains under its no-fixed-inference-timeout contract.", - "The one corrective attempt has a 15-minute absolute wall-clock deadline across open/read/decode/validation while the primary contextual-orchestrator review remains under its no-fixed-inference-timeout contract; unlike a urllib socket timeout, trickling response activity cannot renew that budget.", - ), - ARCHITECTURE: ( - "but is capped at 15 minutes because it repairs an\nalready-completed verdict rather than performing a second unbounded full\nreview.", - "but has one 15-minute process-level wall-clock deadline across open, read,\ndecode, and deterministic validation because it repairs an already-completed\nverdict rather than performing a second unbounded full review. This is not a\nsocket inactivity timeout, so response activity cannot renew the budget.", - ), - BASELINE: ( - "the one corrective request has a 900-second hard client ceiling;", - "the one corrective attempt has a 900-second absolute wall-clock deadline across open/read/decode/validation (not a renewable socket timeout);", - ), - DOCTORING: ( - "The *single corrective request* is different: it repairs an already-completed verdict and therefore has a hard 900-second `urllib` client timeout.", - "The *single corrective attempt* is different: it repairs an already-completed verdict and therefore has one 900-second process-level wall-clock deadline across open/read/decode/validation. It deliberately does not use `urllib`'s renewable socket-operation timeout.", - ), - } - for path, (old, new) in replacements.items(): - text = path.read_text(encoding="utf-8") - text = replace_once(text, old, new, str(path)) - path.write_text(text, encoding="utf-8") - - -def main() -> None: - update_source() - update_tests() - update_docs() - - -if __name__ == "__main__": - main() diff --git a/tests/test_noema_model_output_failure_classification.py b/tests/test_noema_model_output_failure_classification.py index cb29fc10ad..82305a6533 100644 --- a/tests/test_noema_model_output_failure_classification.py +++ b/tests/test_noema_model_output_failure_classification.py @@ -62,3 +62,399 @@ def test_invalid_probe_outcome_is_typed_model_output_failure() -> None: with pytest.raises(error_type, match="outcome must be falsified or confirmed"): gate.validate_substantive_verdict(_verdict(), DIFF, ["README.md"]) + + + +def test_bounded_repair_preserves_initial_schema_and_transport_evidence(monkeypatch) -> None: + """A malformed verdict followed by 502 keeps both typed evidence classes.""" + import json + import urllib.error + + monkeypatch.setenv("NOEMA_LLM_API_URL", "https://llm.example/v1/chat/completions") + monkeypatch.setenv("NOEMA_LLM_API_KEY", "test-key") + head_sha = "a" * 40 + requests: list[tuple[object, dict]] = [] + + class Response: + def __enter__(self): + return self + + def __exit__(self, *_args): + return None + + def read(self): + return json.dumps( + {"choices": [{"message": {"content": json.dumps(_verdict())}}]} + ).encode() + + def open_response(_opener, request, **kwargs): + requests.append((request, kwargs)) + if len(requests) == 1: + return Response() + raise urllib.error.HTTPError(request.full_url, 502, "Bad Gateway", {}, None) + + monkeypatch.setattr(gate.urllib.request.OpenerDirector, "open", open_response) + monkeypatch.setattr( + gate, + "fetch_pr", + lambda _repo, _number: {"headRefOid": head_sha}, + ) + + with pytest.raises(gate.NoemaTransportError) as exc_info: + gate.call_llm( + "owner/repo", + 7, + {"title": "test", "headRefOid": head_sha}, + DIFF, + False, + head_sha, + changed_paths=("README.md",), + ) + + message = str(exc_info.value) + assert "outcome must be falsified or confirmed" in message + assert "HTTPError" in message + assert "502" in message + assert len(requests) == 2 + assert requests[0][1] == {} + assert requests[1][1] == {} + + +def test_repeated_model_output_failure_remains_typed(monkeypatch) -> None: + """A second malformed verdict fails closed as model-output evidence.""" + import json + + monkeypatch.setenv("NOEMA_LLM_API_URL", "https://llm.example/v1/chat/completions") + monkeypatch.setenv("NOEMA_LLM_API_KEY", "test-key") + head_sha = "b" * 40 + + class Response: + def __enter__(self): + return self + + def __exit__(self, *_args): + return None + + def read(self): + return json.dumps( + {"choices": [{"message": {"content": json.dumps(_verdict())}}]} + ).encode() + + monkeypatch.setattr( + gate.urllib.request.OpenerDirector, + "open", + lambda *_args, **_kwargs: Response(), + ) + monkeypatch.setattr( + gate, + "fetch_pr", + lambda _repo, _number: {"headRefOid": head_sha}, + ) + + with pytest.raises(gate.NoemaModelOutputError) as exc_info: + gate.call_llm( + "owner/repo", + 7, + {"title": "test", "headRefOid": head_sha}, + DIFF, + False, + head_sha, + changed_paths=("README.md",), + ) + + assert "initial failure" in str(exc_info.value) + assert "repair failure" in str(exc_info.value) + + + +def test_total_repair_wall_clock_deadline_interrupts_slow_read(monkeypatch) -> None: + """Trickling/slow response activity cannot extend the one repair budget.""" + import json + import signal + import time + + if not hasattr(signal, "setitimer"): + pytest.skip("POSIX process timer is required by the Linux review runner") + + monkeypatch.setenv("NOEMA_LLM_API_URL", "https://llm.example/v1/chat/completions") + monkeypatch.setenv("NOEMA_LLM_API_KEY", "test-key") + monkeypatch.setattr(gate, "NOEMA_REPAIR_DEADLINE_SECONDS", 0.05) + head_sha = "d" * 40 + calls = 0 + + class FirstResponse: + def __enter__(self): + return self + + def __exit__(self, *_args): + return None + + def read(self): + return json.dumps( + {"choices": [{"message": {"content": json.dumps(_verdict())}}]} + ).encode() + + class SlowRepairResponse: + def __enter__(self): + return self + + def __exit__(self, *_args): + return None + + def read(self): + time.sleep(2) + return b"{}" + + def open_response(_opener, _request, **kwargs): + nonlocal calls + calls += 1 + assert kwargs == {} + return FirstResponse() if calls == 1 else SlowRepairResponse() + + monkeypatch.setattr(gate.urllib.request.OpenerDirector, "open", open_response) + monkeypatch.setattr(gate, "fetch_pr", lambda _repo, _number: {"headRefOid": head_sha}) + + started = time.monotonic() + with pytest.raises(gate.NoemaTransportError) as exc_info: + gate.call_llm( + "owner/repo", + 7, + {"title": "test", "headRefOid": head_sha}, + DIFF, + False, + head_sha, + changed_paths=("README.md",), + ) + elapsed = time.monotonic() - started + + message = str(exc_info.value) + assert "outcome must be falsified or confirmed" in message + assert "NoemaRepairDeadlineExceeded" in message + assert "wall-clock deadline" in message + assert elapsed < 1.0 + assert calls == 2 + assert signal.getitimer(signal.ITIMER_REAL)[0] == 0 + + + +def test_repair_wall_clock_deadline_defensive_fail_closed_paths(monkeypatch) -> None: + """Invalid budgets/platform state fail closed instead of weakening the bound.""" + import signal + + with pytest.raises(ValueError, match="must be positive"): + with gate._repair_wall_clock_deadline(0): + pass + + if not hasattr(signal, "setitimer"): + pytest.skip("remaining cases require POSIX setitimer") + + monkeypatch.delattr(gate.signal, "setitimer") + with pytest.raises(RuntimeError, match="requires POSIX setitimer support"): + with gate._repair_wall_clock_deadline(1): + pass + + +def test_repair_wall_clock_deadline_refuses_existing_process_alarm() -> None: + """Noema never overwrites another caller's active process alarm.""" + import signal + + if not hasattr(signal, "setitimer"): + pytest.skip("POSIX process timer is required by the Linux review runner") + signal.setitimer(signal.ITIMER_REAL, 30) + try: + with pytest.raises(RuntimeError, match="refused to overwrite"): + with gate._repair_wall_clock_deadline(1): + pass + finally: + signal.setitimer(signal.ITIMER_REAL, 0) + + +def test_repair_wall_clock_deadline_rejects_non_main_thread_signal_context(monkeypatch) -> None: + """A signal handler that cannot be installed fails closed before any timer starts.""" + import signal + + if not hasattr(signal, "setitimer"): + pytest.skip("POSIX process timer is required by the Linux review runner") + + def reject_signal(*_args, **_kwargs): + raise ValueError("signal only works in main thread") + + monkeypatch.setattr(gate.signal, "signal", reject_signal) + with pytest.raises(RuntimeError, match="process main thread"): + with gate._repair_wall_clock_deadline(1): + pass + assert signal.getitimer(signal.ITIMER_REAL)[0] == 0 + + +def test_repair_unexpected_runtime_failure_preserves_initial_model_evidence(monkeypatch) -> None: + """Unexpected corrective parser/runtime failures keep the first trusted diagnostic.""" + import json + + monkeypatch.setenv("NOEMA_LLM_API_URL", "https://llm.example/v1/chat/completions") + monkeypatch.setenv("NOEMA_LLM_API_KEY", "test-key") + head_sha = "e" * 40 + + class Response: + def __enter__(self): + return self + + def __exit__(self, *_args): + return None + + def read(self): + return json.dumps( + {"choices": [{"message": {"content": json.dumps(_verdict())}}]} + ).encode() + + monkeypatch.setattr( + gate.urllib.request.OpenerDirector, + "open", + lambda *_args, **_kwargs: Response(), + ) + monkeypatch.setattr(gate, "fetch_pr", lambda _repo, _number: {"headRefOid": head_sha}) + original_decode = gate.decode_llm_response_body + decode_calls = 0 + + def decode_once_then_fail(raw_bytes): + nonlocal decode_calls + decode_calls += 1 + if decode_calls == 2: + raise RuntimeError("repair parser invariant failed") + return original_decode(raw_bytes) + + monkeypatch.setattr(gate, "decode_llm_response_body", decode_once_then_fail) + + with pytest.raises(RuntimeError) as exc_info: + gate.call_llm( + "owner/repo", + 7, + {"title": "test", "headRefOid": head_sha}, + DIFF, + False, + head_sha, + changed_paths=("README.md",), + ) + + message = str(exc_info.value) + assert "Noema repair failed closed" in message + assert "outcome must be falsified or confirmed" in message + assert "repair parser invariant failed" in message + assert decode_calls == 2 + + + +def test_unparseable_diff_remains_source_evidence() -> None: + """A location-free trusted diff is not retyped as model-output failure.""" + with pytest.raises(RuntimeError) as exc_info: + gate.validate_substantive_verdict(_verdict(), "not a unified diff", ["README.md"]) + assert not isinstance(exc_info.value, gate.NoemaModelOutputError) + assert "parseable changed-line evidence" in str(exc_info.value) + + +def test_model_sentinel_never_reaches_repair_prompt_or_final_diagnostic(monkeypatch) -> None: + """Model-controlled invalid values are redacted while the defect class stays actionable.""" + import json + + sentinel = "MODEL_SENTINEL_DO_NOT_REFLECT" + monkeypatch.setenv("NOEMA_LLM_API_URL", "https://llm.example/v1/chat/completions") + monkeypatch.setenv("NOEMA_LLM_API_KEY", "test-key") + head_sha = "e" * 40 + requests = [] + + class Response: + def __enter__(self): + return self + + def __exit__(self, *_args): + return None + + def read(self): + return json.dumps( + {"choices": [{"message": {"content": json.dumps({"decision": sentinel})}}]} + ).encode() + + def open_response(_opener, request, **kwargs): + assert kwargs == {} + requests.append(request) + return Response() + + monkeypatch.setattr(gate.urllib.request.OpenerDirector, "open", open_response) + monkeypatch.setattr(gate, "fetch_pr", lambda _repo, _number: {"headRefOid": head_sha}) + + with pytest.raises(gate.NoemaModelOutputError) as exc_info: + gate.call_llm( + "owner/repo", + 7, + {"title": "test", "headRefOid": head_sha}, + DIFF, + False, + head_sha, + changed_paths=("README.md",), + ) + + assert len(requests) == 2 + repair_payload = requests[1].data.decode("utf-8") + assert sentinel not in repair_payload + assert "Noema LLM returned unsupported decision" in repair_payload + assert sentinel not in str(exc_info.value) + assert "Noema LLM returned unsupported decision" in str(exc_info.value) + assert exc_info.value.__cause__ is None + + +def test_stable_failure_diagnostic_preserves_trusted_structure_and_redacts_values() -> None: + """Trusted validator detail stays actionable; arbitrary model text stays opaque.""" + trusted = gate.NoemaModelOutputError( + "Noema adversarial probe 1 outcome must be falsified or confirmed" + ) + assert gate._stable_failure_diagnostic(trusted) == str(trusted) + request_changes = gate.NoemaModelOutputError( + "Noema LLM request_changes response did not contain a substantive finding" + ) + assert gate._stable_failure_diagnostic(request_changes) == str(request_changes) + assert gate._stable_failure_diagnostic( + gate.NoemaModelOutputError("Noema LLM returned unsupported decision: 'SECRET_VALUE'") + ) == "Noema LLM returned unsupported decision" + assert gate._stable_failure_diagnostic( + gate.NoemaModelOutputError("secret-ish model text") + ) == "model-output-contract-invalid" + assert gate._stable_failure_diagnostic(TimeoutError()) == "TimeoutError" + + +def test_repair_deadline_rejects_nonpositive_budget() -> None: + with pytest.raises(ValueError, match="must be positive"): + with gate._repair_wall_clock_deadline(0): + pass + + +def test_repair_deadline_requires_setitimer(monkeypatch) -> None: + monkeypatch.delattr(gate.signal, "setitimer") + with pytest.raises(RuntimeError, match="requires POSIX"): + with gate._repair_wall_clock_deadline(1): + pass + + +def test_repair_deadline_requires_itimer_real(monkeypatch) -> None: + monkeypatch.delattr(gate.signal, "ITIMER_REAL") + with pytest.raises(RuntimeError, match="requires POSIX"): + with gate._repair_wall_clock_deadline(1): + pass + + +@pytest.mark.parametrize("timer_state", [(1.0, 0.0), (0.0, 1.0)]) +def test_repair_deadline_refuses_existing_process_alarm(monkeypatch, timer_state) -> None: + monkeypatch.setattr(gate.signal, "getitimer", lambda _which: timer_state) + with pytest.raises(RuntimeError, match="active process alarm"): + with gate._repair_wall_clock_deadline(1): + pass + + +def test_repair_deadline_requires_main_thread_signal_registration(monkeypatch) -> None: + monkeypatch.setattr(gate.signal, "getitimer", lambda _which: (0.0, 0.0)) + + def reject_signal(*_args): + raise ValueError("signal only works in main thread") + + monkeypatch.setattr(gate.signal, "signal", reject_signal) + with pytest.raises(RuntimeError, match="process main thread"): + with gate._repair_wall_clock_deadline(1): + pass From ae75b1d0546cbd4f73806f70178e064fdadeca2e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 02:35:52 +0900 Subject: [PATCH 24/24] test(noema): preserve existing process alarm authority --- ...test_noema_repair_deadline_alarm_safety.py | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 tests/test_noema_repair_deadline_alarm_safety.py diff --git a/tests/test_noema_repair_deadline_alarm_safety.py b/tests/test_noema_repair_deadline_alarm_safety.py new file mode 100644 index 0000000000..11f5f9569f --- /dev/null +++ b/tests/test_noema_repair_deadline_alarm_safety.py @@ -0,0 +1,25 @@ +"""Regression coverage for Noema repair wall-clock alarm ownership.""" + +import pytest + +from scripts.ci import noema_review_gate as gate + + +def test_repair_deadline_refuses_to_clobber_an_existing_process_alarm(monkeypatch) -> None: + """A repair deadline must fail closed before replacing another alarm owner.""" + monkeypatch.setattr(gate.signal, "getitimer", lambda _kind: (5.0, 0.0)) + set_calls: list[tuple[object, ...]] = [] + monkeypatch.setattr( + gate.signal, + "setitimer", + lambda *args: set_calls.append(args), + ) + + with pytest.raises( + RuntimeError, + match="refused to overwrite an active process alarm", + ): + with gate._repair_wall_clock_deadline(0.05): + pytest.fail("deadline context must not run while another alarm is active") + + assert set_calls == []