diff --git a/CHANGELOG.md b/CHANGELOG.md index 1ead154999..683beee96f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +### Sidecar sanitizer keeps the exception type and innermost frame per traceback + +- `scripts/ci/sanitize_contextual_orchestrator_sidecar_stream.py` now reduces each Python traceback in the sidecar stream to one line, `unexpected_exception type= frame=contextual_orchestrator/.py::` (the type identifier and the innermost package frame only; the exception message, source echoes and non-package frames are never re-emitted; a traceback cut off by the sidecar dying or without a package frame reports `unknown`). The previous single, once-per-stream `sidecar emitted an unexpected exception` line kept neither the count nor the type: `.github#1812`'s strix run (33993155419) ended on 83 gateway `500 internal_error` responses -- the orchestrator's generic request handler prints one traceback per unhandled exception -- and no artifact could say which exception escaped or where. Chain sentences (`During handling of the above exception…`, `The above exception was the direct cause…`) are consumed, so a chained exception yields cause then effect. ### Contextual-orchestrator pin advance fixes orchestrator/free retry-stacking - Advanced the central sidecar's pinned immutable CO revision from `2e414d15` to protected `main@414f22973658c4ddc3d4320fcf7acd9b4e8ba991`, carrying contextual-orchestrator#1081's fix into Strix, OpenCode, and Noema. Root cause: `TaskOrchestrator._invoke`'s own retry-then-failover decision for a retryable 5xx (budgeted `1 + tool_retry_attempts` real tries per candidate) was getting multiplied by `ModelClient._send_with_retry`'s independent transient-retry-with-backoff underneath it (`max_retries + 1` further tries per call) -- up to 6 real network attempts against one already-flagged-flaky `orchestrator/free` agent before `_invoke` ever tried the next ranked candidate. Confirmed as the cause of independently observed incidents in #1912, #1231, #1503, and #1198, each spending 9-57+ minutes on one escalated route and surfacing that same route's model in its final error, never reaching a cleanly-ready sibling preflight had already found. The fix (`ModelClient.single_attempt_transport()`) changes only which agent gets tried next; no per-attempt timeout changed. Reproduced the bug directly against unmodified contextual-orchestrator `main` before the fix (6 real attempts) and confirmed the fix resolves it (<=2) before advancing this pin. `docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md`'s 2026-09-06 amendment and `tests/test_contextual_orchestrator_review_sidecar_contract.py`'s `ORCH_PIN_SHA` were updated alongside this pin. All callers still consume an exact SHA; no branch or tag is introduced. diff --git a/scripts/ci/sanitize_contextual_orchestrator_sidecar_stream.py b/scripts/ci/sanitize_contextual_orchestrator_sidecar_stream.py index a3227f2e32..e055513954 100644 --- a/scripts/ci/sanitize_contextual_orchestrator_sidecar_stream.py +++ b/scripts/ci/sanitize_contextual_orchestrator_sidecar_stream.py @@ -52,6 +52,27 @@ rf"^circuit_cleared agent_id={_AGENT_ID}$", ) ) +# Python traceback anatomy. The orchestrator's generic request handler +# (``server.py`` ``except Exception: traceback.print_exc(); _send_error(500, +# "internal_error", ...)``) prints one traceback per unhandled exception, so the +# exception *type* and the innermost ``contextual_orchestrator`` frame are the +# only evidence of what escaped. Frame lines are indented; the terminal line +# (``Type: message``) starts at column 0. Only the bounded type identifier and +# the package-relative frame are re-emitted -- never the message, which can +# carry provider bodies or credentials. +_TRACEBACK_FRAME = re.compile( + r'^\s+File ".*?[/\\]contextual_orchestrator[/\\](?P[A-Za-z0-9_][A-Za-z0-9_/\\]*\.py)", ' + r"line (?P\d+), in (?P[A-Za-z0-9_<>]{1,80})$" +) +_TRACEBACK_TERMINAL = re.compile( + r"^(?P[A-Za-z_][A-Za-z0-9_]{0,63}(?:\.[A-Za-z_][A-Za-z0-9_]{0,63}){0,8})(?::.*)?$" +) +_TRACEBACK_CHAIN_LINES = frozenset( + ( + "During handling of the above exception, another exception occurred:", + "The above exception was the direct cause of the following exception:", + ) +) _PREFIX_SUMMARIES = ( ("review sidecar preflight failed:", "review sidecar preflight failed"), ("review sidecar discovery failed:", "review sidecar discovery failed"), @@ -135,21 +156,65 @@ def sanitize_line(line: str) -> str | None: return None +def _traceback_summary(exception_type: str | None, frame: str | None) -> str: + """Return the one bounded line kept per traceback: exception type and innermost frame.""" + return ( + f"unexpected_exception type={exception_type or 'unknown'} " + f"frame={frame or 'unknown'}" + ) + + def main() -> int: - """Stream sanitized summaries to stdout without retaining raw provider text.""" + """Stream sanitized summaries to stdout without retaining raw provider text. + + A traceback opens at its ``Traceback`` header and closes at its column-0 + terminal ``Type: message`` line (emitting ``unexpected_exception type=... + frame=...``), at the next header or allowlisted line, or at end of stream + (``type=unknown``). Indented lines inside it are frames and source echoes: + consumed, not counted as omitted, and only a ``contextual_orchestrator`` + frame's package path, line and function are retained. Any other column-0 + line closes the traceback and is classified like every other line. + """ omitted = 0 - unexpected_exception_reported = False + in_traceback = False + frame: str | None = None for line in sys.stdin: - if line.lstrip().startswith("Traceback"): - if not unexpected_exception_reported: - print("sidecar emitted an unexpected exception", flush=True) - unexpected_exception_reported = True + stripped = line.strip() + if not stripped or stripped in _TRACEBACK_CHAIN_LINES: + # Blank lines carry nothing (Python pads chain sentences with them). + continue + if stripped.startswith("Traceback"): + if in_traceback: + print(_traceback_summary(None, frame), flush=True) + in_traceback, frame = True, None + continue + if in_traceback: + frame_match = _TRACEBACK_FRAME.match(line.rstrip("\n")) + if frame_match is not None: + module = frame_match.group("module").replace("\\", "/") + frame = f"contextual_orchestrator/{module}:{frame_match.group('line')}:{frame_match.group('function')}" + continue + if line[:1].isspace(): + continue + in_traceback = False + sanitized = sanitize_line(line) + terminal = _TRACEBACK_TERMINAL.match(stripped) if sanitized is None else None + if terminal is not None: + print(_traceback_summary(terminal.group("type"), frame), flush=True) + continue + print(_traceback_summary(None, frame), flush=True) + if sanitized is None: + omitted += 1 + continue + print(sanitized, flush=True) continue sanitized = sanitize_line(line) if sanitized is None: omitted += 1 continue print(sanitized, flush=True) + if in_traceback: + print(_traceback_summary(None, frame), flush=True) if omitted: print(f"omitted_unstructured_lines={omitted}", flush=True) return 0 diff --git a/tests/test_contextual_orchestrator_review_runtime_preflight.py b/tests/test_contextual_orchestrator_review_runtime_preflight.py index 902237bdad..69e62153ff 100644 --- a/tests/test_contextual_orchestrator_review_runtime_preflight.py +++ b/tests/test_contextual_orchestrator_review_runtime_preflight.py @@ -1857,16 +1857,148 @@ def test_sidecar_stream_sanitizer_summarizes_unstructured_and_traceback_lines( assert main() == 0 rendered = output.getvalue() + # The indented pseudo-frame is consumed as traceback body (not counted as + # omitted); each header closes at the next header or allowlisted line. assert rendered.splitlines() == [ "request_failed status=500 code=internal_error", - "sidecar emitted an unexpected exception", + "unexpected_exception type=unknown frame=unknown", + "unexpected_exception type=unknown frame=unknown", "review sidecar preflight failed", "client_disconnected", - "omitted_unstructured_lines=1", ] assert secret not in rendered +def _render_orchestrator_traceback(source: str, module: str, call: str) -> str: + """Run ``source`` as if it were a ``contextual_orchestrator`` module and return the real traceback.""" + import traceback + + namespace: dict[str, object] = {"__name__": f"contextual_orchestrator.{module}"} + exec( # noqa: S102 - test-only: the source is a literal in this file + compile(source, f"/opt/site-packages/contextual_orchestrator/{module}.py", "exec"), + namespace, + ) + try: + eval(call, namespace) # noqa: S307 - test-only literal + except Exception: # noqa: BLE001 - the traceback under test + return traceback.format_exc() + raise AssertionError("fixture did not raise") + + +def _sanitize_stream(monkeypatch: pytest.MonkeyPatch, text: str) -> list[str]: + """Run the sanitizer's ``main`` over ``text`` and return its output lines.""" + namespace = _load_sanitizer() + monkeypatch.setattr(sys, "stdin", io.StringIO(text)) + output = io.StringIO() + with redirect_stdout(output): + assert namespace["main"]() == 0 + return output.getvalue().splitlines() + + +def test_sidecar_stream_sanitizer_keeps_exception_type_and_innermost_frame( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A real traceback is reduced to its exception type and innermost package frame. + + `.github#1812`'s strix run (33993155419) died on 83 gateway ``500 + internal_error`` responses -- the orchestrator's generic handler prints one + traceback per unhandled exception -- and the sanitized stream kept a single + ``sidecar emitted an unexpected exception`` line, so neither the exception + type nor where it escaped survived into any artifact. + """ + secret = "sk-secret-must-not-enter-artifact" + rendered = _render_orchestrator_traceback( + "def _serve(payload):\n" + " return payload['model']\n" + "def do_POST(payload):\n" + " return _serve(payload)\n", + "server", + f"do_POST({{'token': '{secret}'}})", + ) + assert "KeyError: 'model'" in rendered + assert 'contextual_orchestrator/server.py", line 2, in _serve' in rendered + + lines = _sanitize_stream( + monkeypatch, + rendered + "request_failed status=500 code=internal_error\n", + ) + + assert lines == [ + "unexpected_exception type=KeyError frame=contextual_orchestrator/server.py:2:_serve", + "request_failed status=500 code=internal_error", + ] + assert secret not in "\n".join(lines) + assert "test_contextual_orchestrator" not in "\n".join(lines) + + +def test_sidecar_stream_sanitizer_keeps_dotted_exception_types_and_chains( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A package-defined exception keeps its dotted type; a chained traceback yields cause then effect.""" + rendered = _render_orchestrator_traceback( + "class ProviderResponseError(RuntimeError):\n" + " pass\n" + "def _parse(body):\n" + " return body['choices']\n" + "def proxy(body):\n" + " try:\n" + " return _parse(body)\n" + " except KeyError as exc:\n" + " raise ProviderResponseError('malformed body: sk-leak') from exc\n", + "transport", + "proxy({})", + ) + assert "The above exception was the direct cause of the following exception:" in rendered + + lines = _sanitize_stream(monkeypatch, rendered) + + assert lines == [ + "unexpected_exception type=KeyError frame=contextual_orchestrator/transport.py:4:_parse", + "unexpected_exception type=contextual_orchestrator.transport.ProviderResponseError " + "frame=contextual_orchestrator/transport.py:9:proxy", + ] + assert "sk-leak" not in "\n".join(lines) + + +def test_sidecar_stream_sanitizer_closes_a_truncated_traceback_at_end_of_stream( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A traceback cut off by the sidecar dying still reports its innermost frame.""" + lines = _sanitize_stream( + monkeypatch, + "Traceback (most recent call last):\n" + ' File "/x/site-packages/contextual_orchestrator/orchestrator.py", line 7824, in _invoke\n' + " result = await candidate.send(sk-secret)\n" + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + ) + assert lines == [ + "unexpected_exception type=unknown frame=contextual_orchestrator/orchestrator.py:7824:_invoke", + ] + + +def test_sidecar_stream_sanitizer_does_not_treat_free_text_as_an_exception( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A column-0 line that is neither a terminal nor allowlisted closes the traceback and is omitted. + + Unstructured lines outside any traceback keep counting as omitted, as before. + """ + lines = _sanitize_stream( + monkeypatch, + "Traceback (most recent call last):\n" + ' File "/x/site-packages/contextual_orchestrator/server.py", line 6288, in do_POST\n' + "provider said: sk-secret and more words\n" + "client_disconnected\n" + "provider body outside any traceback: sk-secret-two\n", + ) + assert lines == [ + "unexpected_exception type=unknown frame=contextual_orchestrator/server.py:6288:do_POST", + "client_disconnected", + "omitted_unstructured_lines=2", + ] + assert "sk-secret" not in "\n".join(lines) + + def test_sidecar_stream_sanitizer_omits_no_summary_for_fully_safe_input( monkeypatch: pytest.MonkeyPatch, ) -> None: