diff --git a/docs/doctoring/noema-model-output-repair-boundary.md b/docs/doctoring/noema-model-output-repair-boundary.md new file mode 100644 index 0000000000..d1602f92de --- /dev/null +++ b/docs/doctoring/noema-model-output-repair-boundary.md @@ -0,0 +1,33 @@ +# Noema model-output repair boundary + +## Incident + +On 2026-09-01 the required Noema review for `ContextualWisdomLab/naruon#1505` reached deterministic verdict validation, rejected an adversarial-probe `outcome` outside the closed `falsified|confirmed` domain, then spent the repair path on a long second model call that ultimately surfaced only `HTTP 502 Bad Gateway`. That final transport symptom erased the more informative first trusted-validator failure from the top-level diagnostic. + +## Decision + +1. Model-produced JSON/envelope/schema/semantic-contract failures are `NoemaModelOutputError`; they remain fail-closed and are not consumer-source findings. +2. The primary review keeps the accepted contextual-orchestrator no-fixed-inference-timeout contract. The *single corrective attempt* is different: it repairs an already-completed verdict and therefore has one 900-second process-level wall-clock deadline across open/read/decode/validation. It deliberately does not use `urllib`'s renewable socket-operation timeout. +3. A corrective transport failure is `NoemaTransportError` and carries the sanitized first validator diagnostic plus the later transport exception class/status. Raw model output is never copied into public Actions diagnostics. +4. Exact-head validation before retry and before publication remains mandatory. All model traffic remains on contextual-orchestrator `orchestrator/free`. + +## Verification + +The #1617 regression first proved RED because `NoemaModelOutputError` did not exist. The repair adds focused cases for malformed-verdict typing, malformed-then-502 evidence preservation with the 900-second repair-only timeout, and repeated malformed output remaining typed and non-passing. The repository full coverage/docstring gate is run before the one-shot repair workflow commits the result. + +## References + +Fielding, R., Nottingham, M., & Reschke, J. (2022). *HTTP semantics* (RFC 9110). Internet Engineering Task Force. + +Python Software Foundation. (2026). *urllib.request — Extensible library for opening URLs*. Python 3 documentation. + + +## Actionable diagnostic boundary + +Corrective prompts need the deterministic *class* of a malformed verdict to repair it, +but do not need arbitrary model-produced values. Trusted structural validator messages +(such as a missing required field or an invalid adversarial-probe outcome class) remain +available after secret scrubbing. Unsupported decision values and unknown model-output +text are redacted to stable diagnostics, and a repeated invalid-model exception is raised +without retaining the raw model exception as an explicit cause. Tests use a sentinel value +to prove it reaches neither the retry prompt nor the final diagnostic. diff --git a/scripts/ci/noema_review_gate.py b/scripts/ci/noema_review_gate.py index 5dbeb65d79..4f82281fc3 100644 --- a/scripts/ci/noema_review_gate.py +++ b/scripts/ci/noema_review_gate.py @@ -6,12 +6,14 @@ import argparse import ast import base64 +import contextlib import hashlib import http.client import ipaddress import json import os import re +import signal import socket import subprocess import sys @@ -61,6 +63,53 @@ ORCHESTRATOR_LOOPBACK_HOSTS = frozenset({"127.0.0.1", "::1"}) ORCHESTRATOR_BASE_ENV = "CONTEXTUAL_ORCHESTRATOR_BASE_URL" +# A repair request corrects an already-completed model verdict; it is not a +# second unbounded full review. Fifteen minutes is an absolute wall-clock +# deadline for the complete corrective attempt (open/read/decode/validate), +# not a socket inactivity timeout. The primary review remains governed by +# contextual-orchestrator rather than a fixed inference timeout. +NOEMA_REPAIR_DEADLINE_SECONDS = 15 * 60 + + +class NoemaModelOutputError(RuntimeError): + """Raised when untrusted model output violates the trusted verdict contract.""" + + +class NoemaTransportError(RuntimeError): + """Raised when the bounded review transport cannot produce usable evidence.""" + + +class NoemaRepairDeadlineExceeded(TimeoutError): + """Raised when the corrective attempt exceeds its total wall-clock budget.""" + + +def _stable_failure_diagnostic(exc: BaseException) -> str: + """Return actionable trusted diagnostics without reflecting model values.""" + message = scrub_sensitive_data(str(exc)) or type(exc).__name__ + if not isinstance(exc, NoemaModelOutputError): + return message + + # Model-output exceptions are raised only by deterministic parsing and + # validation code. Preserve those static/structural diagnostics because + # they tell the corrective model and operators exactly which contract was + # violated. The one validator that embeds an untrusted model value is the + # unsupported-decision check; redact that value. Unknown model-output + # exception text fails closed to a stable code rather than being reflected. + if message.startswith("Noema LLM returned unsupported decision:"): + return "Noema LLM returned unsupported decision" + trusted_prefixes = ( + "Noema LLM response ", + "Noema LLM request_changes ", + "Noema formal verdict ", + "Noema reviewed line ", + "Noema adversarial validation ", + "Noema adversarial probe ", + "Noema approve ", + "Noema request_changes ", + ) + if message.startswith(trusted_prefixes): + return message + return "model-output-contract-invalid" # ⚡ Bolt: Pre-compiled regex patterns to avoid recompilation on every scrub_sensitive_data call. # Impact: Improves string processing performance in error reporting. @@ -387,57 +436,57 @@ def validate_substantive_verdict( reviewed_lines = verdict.get("reviewed_lines") if not isinstance(reviewed_lines, list) or not reviewed_lines: - raise RuntimeError("Noema formal verdict requires at least one reviewed changed line") + raise NoemaModelOutputError("Noema formal verdict requires at least one reviewed changed line") for index, reviewed in enumerate(reviewed_lines, start=1): if not isinstance(reviewed, dict): - raise RuntimeError(f"Noema reviewed line {index} must be an object") + raise NoemaModelOutputError(f"Noema reviewed line {index} must be an object") location = (reviewed.get("path"), reviewed.get("line"), reviewed.get("side")) if location not in locations: - raise RuntimeError(f"Noema reviewed line {index} is not an exact changed-side line") + raise NoemaModelOutputError(f"Noema reviewed line {index} is not an exact changed-side line") analysis = reviewed.get("analysis") if not isinstance(analysis, str) or not analysis.strip(): - raise RuntimeError(f"Noema reviewed line {index} requires concrete analysis") + raise NoemaModelOutputError(f"Noema reviewed line {index} requires concrete analysis") validation = verdict.get("adversarial_validation") if not isinstance(validation, dict): - raise RuntimeError("Noema formal verdict requires adversarial_validation") + raise NoemaModelOutputError("Noema formal verdict requires adversarial_validation") status = validation.get("status") expected_status = "passed" if decision == "approve" else "failed" if status != expected_status: - raise RuntimeError(f"Noema {decision} requires adversarial_validation.status={expected_status}") + raise NoemaModelOutputError(f"Noema {decision} requires adversarial_validation.status={expected_status}") residual_risk = validation.get("residual_risk") if not isinstance(residual_risk, str) or not residual_risk.strip(): - raise RuntimeError("Noema adversarial validation requires residual_risk") + raise NoemaModelOutputError("Noema adversarial validation requires residual_risk") probes = validation.get("probes") all_changed_paths = set(changed_paths) or {path for path, _line, _side in locations} required_probes = 2 if any(changed_file_is_material(path) for path in all_changed_paths) else 1 if not isinstance(probes, list) or len(probes) < required_probes: - raise RuntimeError(f"Noema adversarial validation requires at least {required_probes} concrete probe(s)") + raise NoemaModelOutputError(f"Noema adversarial validation requires at least {required_probes} concrete probe(s)") confirmed: set[tuple[str, int, str]] = set() identities: set[tuple[Any, ...]] = set() for index, probe in enumerate(probes, start=1): if not isinstance(probe, dict): - raise RuntimeError(f"Noema adversarial probe {index} must be an object") + raise NoemaModelOutputError(f"Noema adversarial probe {index} must be an object") location = (probe.get("path"), probe.get("line"), probe.get("side")) if location not in locations: - raise RuntimeError(f"Noema adversarial probe {index} is not an exact changed-side line") + raise NoemaModelOutputError(f"Noema adversarial probe {index} is not an exact changed-side line") for field in ("hypothesis", "attack_or_counterexample", "evidence"): value = probe.get(field) if not isinstance(value, str) or not value.strip(): - raise RuntimeError(f"Noema adversarial probe {index} requires {field}") + raise NoemaModelOutputError(f"Noema adversarial probe {index} requires {field}") outcome = probe.get("outcome") if outcome not in {"falsified", "confirmed"}: - raise RuntimeError(f"Noema adversarial probe {index} outcome must be falsified or confirmed") + raise NoemaModelOutputError(f"Noema adversarial probe {index} outcome must be falsified or confirmed") identity = (*location, probe["hypothesis"].strip().casefold(), probe["attack_or_counterexample"].strip().casefold()) if identity in identities: - raise RuntimeError(f"Noema adversarial probe {index} duplicates an earlier probe") + raise NoemaModelOutputError(f"Noema adversarial probe {index} duplicates an earlier probe") identities.add(identity) if outcome == "confirmed": confirmed.add((str(probe["path"]), int(probe["line"]), str(probe["side"]))) if decision == "approve" and confirmed: - raise RuntimeError("Noema approve cannot contain a confirmed adversarial probe") + raise NoemaModelOutputError("Noema approve cannot contain a confirmed adversarial probe") if decision == "request_changes": finding_locations = { (str(finding.get("file") or ""), finding.get("line"), str(finding.get("side") or "")) @@ -445,7 +494,7 @@ def validate_substantive_verdict( if isinstance(finding, dict) } if not confirmed or not confirmed.intersection(finding_locations): - raise RuntimeError("Noema request_changes requires a confirmed probe on a published finding") + raise NoemaModelOutputError("Noema request_changes requires a confirmed probe on a published finding") def truncate_text(text: str, limit: int) -> str: @@ -749,7 +798,7 @@ def _json_nesting_within_bound(text: str, start: int, max_depth: int) -> bool: def extract_json_object(text: str) -> dict[str, Any]: """Extract a JSON object from a strict or lightly wrapped LLM response. - Fails closed with ``RuntimeError`` — the same "no usable verdict" failure + Fails closed with ``NoemaModelOutputError`` — the same "no usable verdict" failure path ``call_llm`` already raises for an unsupported decision, a missing summary, or a malformed finding — instead of letting a malformed or truncated LLM response's ``json.JSONDecodeError`` propagate as an @@ -875,7 +924,7 @@ def extract_json_object(text: str) -> dict[str, Any]: return candidate if "{" not in stripped: - raise RuntimeError("Noema LLM response did not contain a JSON object") + raise NoemaModelOutputError("Noema LLM response did not contain a JSON object") exc = decode_error or json.JSONDecodeError( "No JSON object could be decoded", stripped, 0 @@ -886,7 +935,7 @@ def extract_json_object(text: str) -> dict[str, Any]: fingerprint = hashlib.sha256( stripped.encode("utf-8", errors="surrogatepass") ).hexdigest()[:16] - raise RuntimeError( + raise NoemaModelOutputError( f"Noema LLM response was not valid JSON ({exc}). Raw model output " "is not logged here (this pull_request_target workflow's logs " "are public and a finite secret-scrub pattern list cannot " @@ -918,21 +967,21 @@ def extract_llm_message_content(raw: str) -> str: try: data = json.loads(raw) except json.JSONDecodeError as exc: - raise RuntimeError(f"Noema LLM response body was not valid JSON: {exc}") from exc + raise NoemaModelOutputError(f"Noema LLM response body was not valid JSON: {exc}") from exc if not isinstance(data, dict): - raise RuntimeError( + raise NoemaModelOutputError( f"Noema LLM response body was not a JSON object (got {type(data).__name__})" ) choices = data.get("choices") if not choices: choices = [{}] elif not isinstance(choices, list): - raise RuntimeError( + raise NoemaModelOutputError( f"Noema LLM response 'choices' was not a list (got {type(choices).__name__})" ) first_choice = choices[0] if not isinstance(first_choice, dict): - raise RuntimeError( + raise NoemaModelOutputError( "Noema LLM response choices[0] was not a JSON object " f"(got {type(first_choice).__name__})" ) @@ -940,14 +989,14 @@ def extract_llm_message_content(raw: str) -> str: if not message: message = {} elif not isinstance(message, dict): - raise RuntimeError( + raise NoemaModelOutputError( f"Noema LLM response 'message' was not a JSON object (got {type(message).__name__})" ) content = message.get("content") if not content: content = "" elif not isinstance(content, str): - raise RuntimeError( + raise NoemaModelOutputError( f"Noema LLM response 'content' was not a string (got {type(content).__name__})" ) return content.strip() @@ -978,7 +1027,7 @@ def decode_llm_response_body(raw_bytes: bytes) -> str: return raw_bytes.decode("utf-8") except UnicodeDecodeError as exc: fingerprint = hashlib.sha256(raw_bytes).hexdigest()[:16] - raise RuntimeError( + raise NoemaModelOutputError( f"Noema LLM response body was not valid UTF-8 ({exc}). Raw " "response bytes are not logged here (this pull_request_target " "workflow's logs are public and a finite secret-scrub pattern " @@ -1075,6 +1124,43 @@ def reject_private_llm_url(api_url: str) -> None: raise ValueError("URL cannot target internal IP addresses") +@contextlib.contextmanager +def _repair_wall_clock_deadline(seconds: float): + """Interrupt the entire corrective attempt after ``seconds`` of wall time. + + ``urllib``'s timeout is a socket-operation timeout and can be extended by + trickling bytes. Required Noema Review runs on Linux, so ITIMER_REAL gives + the repair attempt one process-level wall-clock budget across open, read, + decode, and deterministic validation. An existing process alarm is not + overwritten; that condition fails closed instead. + """ + if seconds <= 0: + raise ValueError("repair wall-clock deadline must be positive") + if not hasattr(signal, "setitimer") or not hasattr(signal, "ITIMER_REAL"): + raise RuntimeError("repair wall-clock deadline requires POSIX setitimer support") + previous_remaining, previous_interval = signal.getitimer(signal.ITIMER_REAL) + if previous_remaining > 0 or previous_interval > 0: + raise RuntimeError("repair wall-clock deadline refused to overwrite an active process alarm") + previous_handler = signal.getsignal(signal.SIGALRM) + + def expire(_signum, _frame): + """Raise the typed deadline signal without reflecting response content.""" + raise NoemaRepairDeadlineExceeded( + f"Noema repair exceeded {seconds:g}-second absolute wall-clock deadline" + ) + + try: + signal.signal(signal.SIGALRM, expire) + except ValueError as exc: + raise RuntimeError("repair wall-clock deadline must run on the process main thread") from exc + signal.setitimer(signal.ITIMER_REAL, seconds) + try: + yield + finally: + signal.setitimer(signal.ITIMER_REAL, 0) + signal.signal(signal.SIGALRM, previous_handler) + + class StaleHeadDuringRepairRetryError(RuntimeError): """Raised when the PR head moves before ``call_llm``'s repair-retry request fires.""" @@ -1207,40 +1293,65 @@ def call_llm( ) opener = urllib.request.build_opener(NoRedirectHandler()) try: - with opener.open(request) as response: # nosec B310 - raw_bytes = response.read() - raw = decode_llm_response_body(raw_bytes) - content = extract_llm_message_content(raw) - verdict = extract_json_object(content) - decision = str(verdict.get("decision") or "").strip().lower() - if decision not in {"approve", "request_changes", "comment"}: - raise RuntimeError(f"Noema LLM returned unsupported decision: {decision!r}") - summary = verdict.get("summary") - if not isinstance(summary, str) or not summary.strip(): - raise RuntimeError("Noema LLM response did not contain a substantive summary") - findings = verdict.get("findings") - if not isinstance(findings, list) or any(not isinstance(finding, dict) for finding in findings): - raise RuntimeError("Noema LLM response findings must be a list of objects") - for finding in findings: - if ( - finding.get("severity") not in {"high", "medium", "low"} - or not isinstance(finding.get("file"), str) - or not finding["file"].strip() - or type(finding.get("line")) is not int - or finding["line"] <= 0 - or finding.get("side") not in {"RIGHT", "LEFT"} - or not isinstance(finding.get("message"), str) - or not finding["message"].strip() - ): - raise RuntimeError("Noema LLM response contained a malformed finding") - if decision == "request_changes" and not findings: - raise RuntimeError("Noema LLM request_changes response did not contain a substantive finding") - validate_substantive_verdict(verdict, diff, changed_paths) + deadline_context = ( + _repair_wall_clock_deadline(NOEMA_REPAIR_DEADLINE_SECONDS) + if is_retry + else contextlib.nullcontext() + ) + with deadline_context: + with opener.open(request) as response: # nosec B310 + raw_bytes = response.read() + raw = decode_llm_response_body(raw_bytes) + content = extract_llm_message_content(raw) + verdict = extract_json_object(content) + decision = str(verdict.get("decision") or "").strip().lower() + if decision not in {"approve", "request_changes", "comment"}: + raise NoemaModelOutputError(f"Noema LLM returned unsupported decision: {decision!r}") + summary = verdict.get("summary") + if not isinstance(summary, str) or not summary.strip(): + raise NoemaModelOutputError("Noema LLM response did not contain a substantive summary") + findings = verdict.get("findings") + if not isinstance(findings, list) or any(not isinstance(finding, dict) for finding in findings): + raise NoemaModelOutputError("Noema LLM response findings must be a list of objects") + for finding in findings: + if ( + finding.get("severity") not in {"high", "medium", "low"} + or not isinstance(finding.get("file"), str) + or not finding["file"].strip() + or type(finding.get("line")) is not int + or finding["line"] <= 0 + or finding.get("side") not in {"RIGHT", "LEFT"} + or not isinstance(finding.get("message"), str) + or not finding["message"].strip() + ): + raise NoemaModelOutputError("Noema LLM response contained a malformed finding") + if decision == "request_changes" and not findings: + raise NoemaModelOutputError("Noema LLM request_changes response did not contain a substantive finding") + validate_substantive_verdict(verdict, diff, changed_paths) except (RuntimeError, urllib.error.URLError, http.client.HTTPException, OSError) as exc: + current_failure = _stable_failure_diagnostic(exc) if is_retry: - if isinstance(exc, RuntimeError): - raise - raise RuntimeError(str(exc)) from exc + initial_failure = ( + scrub_sensitive_data(repair_error) + or "no diagnostic message was available" + ) + if isinstance(exc, NoemaModelOutputError): + raise NoemaModelOutputError( + "Noema model-output repair remained invalid; " + f"initial failure: {initial_failure}; repair failure: {current_failure}" + ) from None + if isinstance( + exc, (urllib.error.URLError, http.client.HTTPException, OSError) + ): + raise NoemaTransportError( + "Noema bounded repair transport was exhausted; " + f"initial failure: {initial_failure}; repair failure: " + f"{type(exc).__name__}: {current_failure}" + ) from exc + raise RuntimeError( + "Noema repair failed closed; " + f"initial failure: {initial_failure}; repair failure: {current_failure}" + ) from exc if str(fetch_pr(repo, number).get("headRefOid") or "").lower() != expected_head: raise StaleHeadDuringRepairRetryError( "Pull request head changed during review; stale before repair retry." @@ -1254,7 +1365,7 @@ def call_llm( expected_head, review_context, changed_paths, - str(exc), + current_failure, is_retry=True, ) return verdict diff --git a/tests/test_noema_model_output_failure_classification.py b/tests/test_noema_model_output_failure_classification.py new file mode 100644 index 0000000000..82305a6533 --- /dev/null +++ b/tests/test_noema_model_output_failure_classification.py @@ -0,0 +1,460 @@ +"""Regression for #1611: malformed model verdicts are infrastructure/model evidence. + +A schema-valid JSON envelope whose adversarial probe uses an out-of-domain +outcome is not a consumer repository defect. The deterministic validator must +still reject it, but with a typed model-output error so the retry/control plane +can preserve the distinction from source findings and provider exhaustion. +""" + +import pytest + +from scripts.ci import noema_review_gate as gate + + +DIFF = """diff --git a/README.md b/README.md +index 1111111..2222222 100644 +--- a/README.md ++++ b/README.md +@@ -1 +1 @@ +-old ++new +""" + + +def _verdict() -> dict: + return { + "decision": "approve", + "summary": "The changed line was reviewed.", + "reviewed_lines": [ + { + "path": "README.md", + "line": 1, + "side": "RIGHT", + "analysis": "The replacement is bounded and reviewable.", + } + ], + "adversarial_validation": { + "status": "passed", + "residual_risk": "No additional risk identified.", + "probes": [ + { + "path": "README.md", + "line": 1, + "side": "RIGHT", + "hypothesis": "The replacement could be wrong.", + "attack_or_counterexample": "Compare the exact changed line.", + "evidence": "Observed the exact replacement in the diff.", + "outcome": "passed", # real #1611 failure shape + } + ], + }, + "findings": [], + } + + +def test_invalid_probe_outcome_is_typed_model_output_failure() -> None: + """Reject malformed LLM evidence without reclassifying it as source failure.""" + error_type = getattr(gate, "NoemaModelOutputError", None) + assert error_type is not None, ( + "Noema must expose a typed model-output/schema failure so malformed " + "LLM evidence cannot collapse into an opaque generic RuntimeError" + ) + + with pytest.raises(error_type, match="outcome must be falsified or confirmed"): + gate.validate_substantive_verdict(_verdict(), DIFF, ["README.md"]) + + + +def test_bounded_repair_preserves_initial_schema_and_transport_evidence(monkeypatch) -> None: + """A malformed verdict followed by 502 keeps both typed evidence classes.""" + import json + import urllib.error + + monkeypatch.setenv("NOEMA_LLM_API_URL", "https://llm.example/v1/chat/completions") + monkeypatch.setenv("NOEMA_LLM_API_KEY", "test-key") + head_sha = "a" * 40 + requests: list[tuple[object, dict]] = [] + + class Response: + def __enter__(self): + return self + + def __exit__(self, *_args): + return None + + def read(self): + return json.dumps( + {"choices": [{"message": {"content": json.dumps(_verdict())}}]} + ).encode() + + def open_response(_opener, request, **kwargs): + requests.append((request, kwargs)) + if len(requests) == 1: + return Response() + raise urllib.error.HTTPError(request.full_url, 502, "Bad Gateway", {}, None) + + monkeypatch.setattr(gate.urllib.request.OpenerDirector, "open", open_response) + monkeypatch.setattr( + gate, + "fetch_pr", + lambda _repo, _number: {"headRefOid": head_sha}, + ) + + with pytest.raises(gate.NoemaTransportError) as exc_info: + gate.call_llm( + "owner/repo", + 7, + {"title": "test", "headRefOid": head_sha}, + DIFF, + False, + head_sha, + changed_paths=("README.md",), + ) + + message = str(exc_info.value) + assert "outcome must be falsified or confirmed" in message + assert "HTTPError" in message + assert "502" in message + assert len(requests) == 2 + assert requests[0][1] == {} + assert requests[1][1] == {} + + +def test_repeated_model_output_failure_remains_typed(monkeypatch) -> None: + """A second malformed verdict fails closed as model-output evidence.""" + import json + + monkeypatch.setenv("NOEMA_LLM_API_URL", "https://llm.example/v1/chat/completions") + monkeypatch.setenv("NOEMA_LLM_API_KEY", "test-key") + head_sha = "b" * 40 + + class Response: + def __enter__(self): + return self + + def __exit__(self, *_args): + return None + + def read(self): + return json.dumps( + {"choices": [{"message": {"content": json.dumps(_verdict())}}]} + ).encode() + + monkeypatch.setattr( + gate.urllib.request.OpenerDirector, + "open", + lambda *_args, **_kwargs: Response(), + ) + monkeypatch.setattr( + gate, + "fetch_pr", + lambda _repo, _number: {"headRefOid": head_sha}, + ) + + with pytest.raises(gate.NoemaModelOutputError) as exc_info: + gate.call_llm( + "owner/repo", + 7, + {"title": "test", "headRefOid": head_sha}, + DIFF, + False, + head_sha, + changed_paths=("README.md",), + ) + + assert "initial failure" in str(exc_info.value) + assert "repair failure" in str(exc_info.value) + + + +def test_total_repair_wall_clock_deadline_interrupts_slow_read(monkeypatch) -> None: + """Trickling/slow response activity cannot extend the one repair budget.""" + import json + import signal + import time + + if not hasattr(signal, "setitimer"): + pytest.skip("POSIX process timer is required by the Linux review runner") + + monkeypatch.setenv("NOEMA_LLM_API_URL", "https://llm.example/v1/chat/completions") + monkeypatch.setenv("NOEMA_LLM_API_KEY", "test-key") + monkeypatch.setattr(gate, "NOEMA_REPAIR_DEADLINE_SECONDS", 0.05) + head_sha = "d" * 40 + calls = 0 + + class FirstResponse: + def __enter__(self): + return self + + def __exit__(self, *_args): + return None + + def read(self): + return json.dumps( + {"choices": [{"message": {"content": json.dumps(_verdict())}}]} + ).encode() + + class SlowRepairResponse: + def __enter__(self): + return self + + def __exit__(self, *_args): + return None + + def read(self): + time.sleep(2) + return b"{}" + + def open_response(_opener, _request, **kwargs): + nonlocal calls + calls += 1 + assert kwargs == {} + return FirstResponse() if calls == 1 else SlowRepairResponse() + + monkeypatch.setattr(gate.urllib.request.OpenerDirector, "open", open_response) + monkeypatch.setattr(gate, "fetch_pr", lambda _repo, _number: {"headRefOid": head_sha}) + + started = time.monotonic() + with pytest.raises(gate.NoemaTransportError) as exc_info: + gate.call_llm( + "owner/repo", + 7, + {"title": "test", "headRefOid": head_sha}, + DIFF, + False, + head_sha, + changed_paths=("README.md",), + ) + elapsed = time.monotonic() - started + + message = str(exc_info.value) + assert "outcome must be falsified or confirmed" in message + assert "NoemaRepairDeadlineExceeded" in message + assert "wall-clock deadline" in message + assert elapsed < 1.0 + assert calls == 2 + assert signal.getitimer(signal.ITIMER_REAL)[0] == 0 + + + +def test_repair_wall_clock_deadline_defensive_fail_closed_paths(monkeypatch) -> None: + """Invalid budgets/platform state fail closed instead of weakening the bound.""" + import signal + + with pytest.raises(ValueError, match="must be positive"): + with gate._repair_wall_clock_deadline(0): + pass + + if not hasattr(signal, "setitimer"): + pytest.skip("remaining cases require POSIX setitimer") + + monkeypatch.delattr(gate.signal, "setitimer") + with pytest.raises(RuntimeError, match="requires POSIX setitimer support"): + with gate._repair_wall_clock_deadline(1): + pass + + +def test_repair_wall_clock_deadline_refuses_existing_process_alarm() -> None: + """Noema never overwrites another caller's active process alarm.""" + import signal + + if not hasattr(signal, "setitimer"): + pytest.skip("POSIX process timer is required by the Linux review runner") + signal.setitimer(signal.ITIMER_REAL, 30) + try: + with pytest.raises(RuntimeError, match="refused to overwrite"): + with gate._repair_wall_clock_deadline(1): + pass + finally: + signal.setitimer(signal.ITIMER_REAL, 0) + + +def test_repair_wall_clock_deadline_rejects_non_main_thread_signal_context(monkeypatch) -> None: + """A signal handler that cannot be installed fails closed before any timer starts.""" + import signal + + if not hasattr(signal, "setitimer"): + pytest.skip("POSIX process timer is required by the Linux review runner") + + def reject_signal(*_args, **_kwargs): + raise ValueError("signal only works in main thread") + + monkeypatch.setattr(gate.signal, "signal", reject_signal) + with pytest.raises(RuntimeError, match="process main thread"): + with gate._repair_wall_clock_deadline(1): + pass + assert signal.getitimer(signal.ITIMER_REAL)[0] == 0 + + +def test_repair_unexpected_runtime_failure_preserves_initial_model_evidence(monkeypatch) -> None: + """Unexpected corrective parser/runtime failures keep the first trusted diagnostic.""" + import json + + monkeypatch.setenv("NOEMA_LLM_API_URL", "https://llm.example/v1/chat/completions") + monkeypatch.setenv("NOEMA_LLM_API_KEY", "test-key") + head_sha = "e" * 40 + + class Response: + def __enter__(self): + return self + + def __exit__(self, *_args): + return None + + def read(self): + return json.dumps( + {"choices": [{"message": {"content": json.dumps(_verdict())}}]} + ).encode() + + monkeypatch.setattr( + gate.urllib.request.OpenerDirector, + "open", + lambda *_args, **_kwargs: Response(), + ) + monkeypatch.setattr(gate, "fetch_pr", lambda _repo, _number: {"headRefOid": head_sha}) + original_decode = gate.decode_llm_response_body + decode_calls = 0 + + def decode_once_then_fail(raw_bytes): + nonlocal decode_calls + decode_calls += 1 + if decode_calls == 2: + raise RuntimeError("repair parser invariant failed") + return original_decode(raw_bytes) + + monkeypatch.setattr(gate, "decode_llm_response_body", decode_once_then_fail) + + with pytest.raises(RuntimeError) as exc_info: + gate.call_llm( + "owner/repo", + 7, + {"title": "test", "headRefOid": head_sha}, + DIFF, + False, + head_sha, + changed_paths=("README.md",), + ) + + message = str(exc_info.value) + assert "Noema repair failed closed" in message + assert "outcome must be falsified or confirmed" in message + assert "repair parser invariant failed" in message + assert decode_calls == 2 + + + +def test_unparseable_diff_remains_source_evidence() -> None: + """A location-free trusted diff is not retyped as model-output failure.""" + with pytest.raises(RuntimeError) as exc_info: + gate.validate_substantive_verdict(_verdict(), "not a unified diff", ["README.md"]) + assert not isinstance(exc_info.value, gate.NoemaModelOutputError) + assert "parseable changed-line evidence" in str(exc_info.value) + + +def test_model_sentinel_never_reaches_repair_prompt_or_final_diagnostic(monkeypatch) -> None: + """Model-controlled invalid values are redacted while the defect class stays actionable.""" + import json + + sentinel = "MODEL_SENTINEL_DO_NOT_REFLECT" + monkeypatch.setenv("NOEMA_LLM_API_URL", "https://llm.example/v1/chat/completions") + monkeypatch.setenv("NOEMA_LLM_API_KEY", "test-key") + head_sha = "e" * 40 + requests = [] + + class Response: + def __enter__(self): + return self + + def __exit__(self, *_args): + return None + + def read(self): + return json.dumps( + {"choices": [{"message": {"content": json.dumps({"decision": sentinel})}}]} + ).encode() + + def open_response(_opener, request, **kwargs): + assert kwargs == {} + requests.append(request) + return Response() + + monkeypatch.setattr(gate.urllib.request.OpenerDirector, "open", open_response) + monkeypatch.setattr(gate, "fetch_pr", lambda _repo, _number: {"headRefOid": head_sha}) + + with pytest.raises(gate.NoemaModelOutputError) as exc_info: + gate.call_llm( + "owner/repo", + 7, + {"title": "test", "headRefOid": head_sha}, + DIFF, + False, + head_sha, + changed_paths=("README.md",), + ) + + assert len(requests) == 2 + repair_payload = requests[1].data.decode("utf-8") + assert sentinel not in repair_payload + assert "Noema LLM returned unsupported decision" in repair_payload + assert sentinel not in str(exc_info.value) + assert "Noema LLM returned unsupported decision" in str(exc_info.value) + assert exc_info.value.__cause__ is None + + +def test_stable_failure_diagnostic_preserves_trusted_structure_and_redacts_values() -> None: + """Trusted validator detail stays actionable; arbitrary model text stays opaque.""" + trusted = gate.NoemaModelOutputError( + "Noema adversarial probe 1 outcome must be falsified or confirmed" + ) + assert gate._stable_failure_diagnostic(trusted) == str(trusted) + request_changes = gate.NoemaModelOutputError( + "Noema LLM request_changes response did not contain a substantive finding" + ) + assert gate._stable_failure_diagnostic(request_changes) == str(request_changes) + assert gate._stable_failure_diagnostic( + gate.NoemaModelOutputError("Noema LLM returned unsupported decision: 'SECRET_VALUE'") + ) == "Noema LLM returned unsupported decision" + assert gate._stable_failure_diagnostic( + gate.NoemaModelOutputError("secret-ish model text") + ) == "model-output-contract-invalid" + assert gate._stable_failure_diagnostic(TimeoutError()) == "TimeoutError" + + +def test_repair_deadline_rejects_nonpositive_budget() -> None: + with pytest.raises(ValueError, match="must be positive"): + with gate._repair_wall_clock_deadline(0): + pass + + +def test_repair_deadline_requires_setitimer(monkeypatch) -> None: + monkeypatch.delattr(gate.signal, "setitimer") + with pytest.raises(RuntimeError, match="requires POSIX"): + with gate._repair_wall_clock_deadline(1): + pass + + +def test_repair_deadline_requires_itimer_real(monkeypatch) -> None: + monkeypatch.delattr(gate.signal, "ITIMER_REAL") + with pytest.raises(RuntimeError, match="requires POSIX"): + with gate._repair_wall_clock_deadline(1): + pass + + +@pytest.mark.parametrize("timer_state", [(1.0, 0.0), (0.0, 1.0)]) +def test_repair_deadline_refuses_existing_process_alarm(monkeypatch, timer_state) -> None: + monkeypatch.setattr(gate.signal, "getitimer", lambda _which: timer_state) + with pytest.raises(RuntimeError, match="active process alarm"): + with gate._repair_wall_clock_deadline(1): + pass + + +def test_repair_deadline_requires_main_thread_signal_registration(monkeypatch) -> None: + monkeypatch.setattr(gate.signal, "getitimer", lambda _which: (0.0, 0.0)) + + def reject_signal(*_args): + raise ValueError("signal only works in main thread") + + monkeypatch.setattr(gate.signal, "signal", reject_signal) + with pytest.raises(RuntimeError, match="process main thread"): + with gate._repair_wall_clock_deadline(1): + pass diff --git a/tests/test_noema_repair_deadline_alarm_safety.py b/tests/test_noema_repair_deadline_alarm_safety.py new file mode 100644 index 0000000000..11f5f9569f --- /dev/null +++ b/tests/test_noema_repair_deadline_alarm_safety.py @@ -0,0 +1,25 @@ +"""Regression coverage for Noema repair wall-clock alarm ownership.""" + +import pytest + +from scripts.ci import noema_review_gate as gate + + +def test_repair_deadline_refuses_to_clobber_an_existing_process_alarm(monkeypatch) -> None: + """A repair deadline must fail closed before replacing another alarm owner.""" + monkeypatch.setattr(gate.signal, "getitimer", lambda _kind: (5.0, 0.0)) + set_calls: list[tuple[object, ...]] = [] + monkeypatch.setattr( + gate.signal, + "setitimer", + lambda *args: set_calls.append(args), + ) + + with pytest.raises( + RuntimeError, + match="refused to overwrite an active process alarm", + ): + with gate._repair_wall_clock_deadline(0.05): + pytest.fail("deadline context must not run while another alarm is active") + + assert set_calls == []