fix(sidecar): keep the exception type and innermost frame per traceback

CHANGELOG.md

@@ -1,3 +1,6 @@
+### Sidecar sanitizer keeps the exception type and innermost frame per traceback
+
+- `scripts/ci/sanitize_contextual_orchestrator_sidecar_stream.py` now reduces each Python traceback in the sidecar stream to one line, `unexpected_exception type=<ExceptionType> frame=contextual_orchestrator/<module>.py:<line>:<function>` (the type identifier and the innermost package frame only; the exception message, source echoes and non-package frames are never re-emitted; a traceback cut off by the sidecar dying or without a package frame reports `unknown`). The previous single, once-per-stream `sidecar emitted an unexpected exception` line kept neither the count nor the type: `.github#1812`'s strix run (33993155419) ended on 83 gateway `500 internal_error` responses -- the orchestrator's generic request handler prints one traceback per unhandled exception -- and no artifact could say which exception escaped or where. Chain sentences (`During handling of the above exception…`, `The above exception was the direct cause…`) are consumed, so a chained exception yields cause then effect.
 ### Contextual-orchestrator pin advance fixes orchestrator/free retry-stacking
 
 - Advanced the central sidecar's pinned immutable CO revision from `2e414d15` to protected `main@414f22973658c4ddc3d4320fcf7acd9b4e8ba991`, carrying contextual-orchestrator#1081's fix into Strix, OpenCode, and Noema. Root cause: `TaskOrchestrator._invoke`'s own retry-then-failover decision for a retryable 5xx (budgeted `1 + tool_retry_attempts` real tries per candidate) was getting multiplied by `ModelClient._send_with_retry`'s independent transient-retry-with-backoff underneath it (`max_retries + 1` further tries per call) -- up to 6 real network attempts against one already-flagged-flaky `orchestrator/free` agent before `_invoke` ever tried the next ranked candidate. Confirmed as the cause of independently observed incidents in #1912, #1231, #1503, and #1198, each spending 9-57+ minutes on one escalated route and surfacing that same route's model in its final error, never reaching a cleanly-ready sibling preflight had already found. The fix (`ModelClient.single_attempt_transport()`) changes only which agent gets tried next; no per-attempt timeout changed. Reproduced the bug directly against unmodified contextual-orchestrator `main` before the fix (6 real attempts) and confirmed the fix resolves it (<=2) before advancing this pin. `docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md`'s 2026-09-06 amendment and `tests/test_contextual_orchestrator_review_sidecar_contract.py`'s `ORCH_PIN_SHA` were updated alongside this pin. All callers still consume an exact SHA; no branch or tag is introduced.

scripts/ci/sanitize_contextual_orchestrator_sidecar_stream.py

@@ -52,6 +52,27 @@
         rf"^circuit_cleared agent_id={_AGENT_ID}$",
     )
 )
+# Python traceback anatomy. The orchestrator's generic request handler
+# (``server.py`` ``except Exception: traceback.print_exc(); _send_error(500,
+# "internal_error", ...)``) prints one traceback per unhandled exception, so the
+# exception *type* and the innermost ``contextual_orchestrator`` frame are the
+# only evidence of what escaped. Frame lines are indented; the terminal line
+# (``Type: message``) starts at column 0. Only the bounded type identifier and
+# the package-relative frame are re-emitted -- never the message, which can
+# carry provider bodies or credentials.
+_TRACEBACK_FRAME = re.compile(
+    r'^\s+File ".*?[/\\]contextual_orchestrator[/\\](?P<module>[A-Za-z0-9_][A-Za-z0-9_/\\]*\.py)", '
+    r"line (?P<line>\d+), in (?P<function>[A-Za-z0-9_<>]{1,80})$"
+)
+_TRACEBACK_TERMINAL = re.compile(
+    r"^(?P<type>[A-Za-z_][A-Za-z0-9_]{0,63}(?:\.[A-Za-z_][A-Za-z0-9_]{0,63}){0,8})(?::.*)?$"
+)
+_TRACEBACK_CHAIN_LINES = frozenset(
+    (
+        "During handling of the above exception, another exception occurred:",
+        "The above exception was the direct cause of the following exception:",
+    )
+)
 _PREFIX_SUMMARIES = (
     ("review sidecar preflight failed:", "review sidecar preflight failed"),
     ("review sidecar discovery failed:", "review sidecar discovery failed"),
@@ -135,21 +156,65 @@ def sanitize_line(line: str) -> str | None:
     return None
 
 
+def _traceback_summary(exception_type: str | None, frame: str | None) -> str:
+    """Return the one bounded line kept per traceback: exception type and innermost frame."""
+    return (
+        f"unexpected_exception type={exception_type or 'unknown'} "
+        f"frame={frame or 'unknown'}"
+    )
+
+
 def main() -> int:
-    """Stream sanitized summaries to stdout without retaining raw provider text."""
+    """Stream sanitized summaries to stdout without retaining raw provider text.
+
+    A traceback opens at its ``Traceback`` header and closes at its column-0
+    terminal ``Type: message`` line (emitting ``unexpected_exception type=...
+    frame=...``), at the next header or allowlisted line, or at end of stream
+    (``type=unknown``). Indented lines inside it are frames and source echoes:
+    consumed, not counted as omitted, and only a ``contextual_orchestrator``
+    frame's package path, line and function are retained. Any other column-0
+    line closes the traceback and is classified like every other line.
+    """
     omitted = 0
-    unexpected_exception_reported = False
+    in_traceback = False
+    frame: str | None = None
     for line in sys.stdin:
-        if line.lstrip().startswith("Traceback"):
-            if not unexpected_exception_reported:
-                print("sidecar emitted an unexpected exception", flush=True)
-                unexpected_exception_reported = True
+        stripped = line.strip()
+        if not stripped or stripped in _TRACEBACK_CHAIN_LINES:
+            # Blank lines carry nothing (Python pads chain sentences with them).
+            continue
+        if stripped.startswith("Traceback"):
+            if in_traceback:
+                print(_traceback_summary(None, frame), flush=True)
+            in_traceback, frame = True, None
+            continue
+        if in_traceback:
+            frame_match = _TRACEBACK_FRAME.match(line.rstrip("\n"))
+            if frame_match is not None:
+                module = frame_match.group("module").replace("\\", "/")
+                frame = f"contextual_orchestrator/{module}:{frame_match.group('line')}:{frame_match.group('function')}"
+                continue
+            if line[:1].isspace():
+                continue
+            in_traceback = False
+            sanitized = sanitize_line(line)
+            terminal = _TRACEBACK_TERMINAL.match(stripped) if sanitized is None else None
+            if terminal is not None:
+                print(_traceback_summary(terminal.group("type"), frame), flush=True)
+                continue
+            print(_traceback_summary(None, frame), flush=True)
+            if sanitized is None:
+                omitted += 1
+                continue
+            print(sanitized, flush=True)
             continue
         sanitized = sanitize_line(line)
         if sanitized is None:
             omitted += 1
             continue
         print(sanitized, flush=True)
+    if in_traceback:
+        print(_traceback_summary(None, frame), flush=True)
     if omitted:
         print(f"omitted_unstructured_lines={omitted}", flush=True)
     return 0

tests/test_contextual_orchestrator_review_runtime_preflight.py

@@ -1857,16 +1857,148 @@ def test_sidecar_stream_sanitizer_summarizes_unstructured_and_traceback_lines(
         assert main() == 0
 
     rendered = output.getvalue()
+    # The indented pseudo-frame is consumed as traceback body (not counted as
+    # omitted); each header closes at the next header or allowlisted line.
     assert rendered.splitlines() == [
         "request_failed status=500 code=internal_error",
-        "sidecar emitted an unexpected exception",
+        "unexpected_exception type=unknown frame=unknown",
+        "unexpected_exception type=unknown frame=unknown",
         "review sidecar preflight failed",
         "client_disconnected",
-        "omitted_unstructured_lines=1",
     ]
     assert secret not in rendered
 
 
+def _render_orchestrator_traceback(source: str, module: str, call: str) -> str:
+    """Run ``source`` as if it were a ``contextual_orchestrator`` module and return the real traceback."""
+    import traceback
+
+    namespace: dict[str, object] = {"__name__": f"contextual_orchestrator.{module}"}
+    exec(  # noqa: S102 - test-only: the source is a literal in this file
+        compile(source, f"/opt/site-packages/contextual_orchestrator/{module}.py", "exec"),
+        namespace,
+    )
+    try:
+        eval(call, namespace)  # noqa: S307 - test-only literal
+    except Exception:  # noqa: BLE001 - the traceback under test
+        return traceback.format_exc()
+    raise AssertionError("fixture did not raise")
+
+
+def _sanitize_stream(monkeypatch: pytest.MonkeyPatch, text: str) -> list[str]:
+    """Run the sanitizer's ``main`` over ``text`` and return its output lines."""
+    namespace = _load_sanitizer()
+    monkeypatch.setattr(sys, "stdin", io.StringIO(text))
+    output = io.StringIO()
+    with redirect_stdout(output):
+        assert namespace["main"]() == 0
+    return output.getvalue().splitlines()
+
+
+def test_sidecar_stream_sanitizer_keeps_exception_type_and_innermost_frame(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """A real traceback is reduced to its exception type and innermost package frame.
+
+    `.github#1812`'s strix run (33993155419) died on 83 gateway ``500
+    internal_error`` responses -- the orchestrator's generic handler prints one
+    traceback per unhandled exception -- and the sanitized stream kept a single
+    ``sidecar emitted an unexpected exception`` line, so neither the exception
+    type nor where it escaped survived into any artifact.
+    """
+    secret = "sk-secret-must-not-enter-artifact"
+    rendered = _render_orchestrator_traceback(
+        "def _serve(payload):\n"
+        "    return payload['model']\n"
+        "def do_POST(payload):\n"
+        "    return _serve(payload)\n",
+        "server",
+        f"do_POST({{'token': '{secret}'}})",
+    )
+    assert "KeyError: 'model'" in rendered
+    assert 'contextual_orchestrator/server.py", line 2, in _serve' in rendered
+
+    lines = _sanitize_stream(
+        monkeypatch,
+        rendered + "request_failed status=500 code=internal_error\n",
+    )
+
+    assert lines == [
+        "unexpected_exception type=KeyError frame=contextual_orchestrator/server.py:2:_serve",
+        "request_failed status=500 code=internal_error",
+    ]
+    assert secret not in "\n".join(lines)
+    assert "test_contextual_orchestrator" not in "\n".join(lines)
+
+
+def test_sidecar_stream_sanitizer_keeps_dotted_exception_types_and_chains(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """A package-defined exception keeps its dotted type; a chained traceback yields cause then effect."""
+    rendered = _render_orchestrator_traceback(
+        "class ProviderResponseError(RuntimeError):\n"
+        "    pass\n"
+        "def _parse(body):\n"
+        "    return body['choices']\n"
+        "def proxy(body):\n"
+        "    try:\n"
+        "        return _parse(body)\n"
+        "    except KeyError as exc:\n"
+        "        raise ProviderResponseError('malformed body: sk-leak') from exc\n",
+        "transport",
+        "proxy({})",
+    )
+    assert "The above exception was the direct cause of the following exception:" in rendered
+
+    lines = _sanitize_stream(monkeypatch, rendered)
+
+    assert lines == [
+        "unexpected_exception type=KeyError frame=contextual_orchestrator/transport.py:4:_parse",
+        "unexpected_exception type=contextual_orchestrator.transport.ProviderResponseError "
+        "frame=contextual_orchestrator/transport.py:9:proxy",
+    ]
+    assert "sk-leak" not in "\n".join(lines)
+
+
+def test_sidecar_stream_sanitizer_closes_a_truncated_traceback_at_end_of_stream(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """A traceback cut off by the sidecar dying still reports its innermost frame."""
+    lines = _sanitize_stream(
+        monkeypatch,
+        "Traceback (most recent call last):\n"
+        '  File "/x/site-packages/contextual_orchestrator/orchestrator.py", line 7824, in _invoke\n'
+        "    result = await candidate.send(sk-secret)\n"
+        "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
+    )
+    assert lines == [
+        "unexpected_exception type=unknown frame=contextual_orchestrator/orchestrator.py:7824:_invoke",
+    ]
+
+
+def test_sidecar_stream_sanitizer_does_not_treat_free_text_as_an_exception(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """A column-0 line that is neither a terminal nor allowlisted closes the traceback and is omitted.
+
+    Unstructured lines outside any traceback keep counting as omitted, as before.
+    """
+    lines = _sanitize_stream(
+        monkeypatch,
+        "Traceback (most recent call last):\n"
+        '  File "/x/site-packages/contextual_orchestrator/server.py", line 6288, in do_POST\n'
+        "provider said: sk-secret and more words\n"
+        "client_disconnected\n"
+        "provider body outside any traceback: sk-secret-two\n",
+    )
+    assert lines == [
+        "unexpected_exception type=unknown frame=contextual_orchestrator/server.py:6288:do_POST",
+        "client_disconnected",
+        "omitted_unstructured_lines=2",
+    ]
+    assert "sk-secret" not in "\n".join(lines)
+
+
 def test_sidecar_stream_sanitizer_omits_no_summary_for_fully_safe_input(
     monkeypatch: pytest.MonkeyPatch,
 ) -> None: