From a91257553c50b33a89af1f348818f896268a36c2 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 11:35:19 +0900 Subject: [PATCH] fix(sidecar): keep the exception type and innermost frame per traceback The sidecar stream sanitizer collapsed every Python traceback into a single "sidecar emitted an unexpected exception" line, printed once per stream. On .github#1812's strix run 33993155419 the gateway answered 83 requests with 500 internal_error -- contextual-orchestrator's generic `except Exception: traceback.print_exc(); _send_error(500, ...)` path -- and no artifact could say which exception escaped, where, or how many times. Each traceback now yields one bounded line, `unexpected_exception type= frame=contextual_orchestrator/.py::`: opened at the header, closed at the column-0 terminal line (or the next header, an allowlisted line, or end of stream -> type=unknown). Indented frame/source lines are consumed rather than counted as omitted; only a contextual_orchestrator frame's package path, line and function are kept, and the exception message is never re-emitted. Chain sentences are consumed so a chained exception yields cause then effect. Tests render real tracebacks with traceback.format_exc() from code compiled under a contextual_orchestrator/ filename (plain, dotted package type with a `from` chain, truncated at end of stream, free-text column-0 line), and the existing traceback expectation is updated. Only the sanitizer's own test consumed the old sentence. Co-Authored-By: Claude Fable 5.1 --- CHANGELOG.md | 4 + ..._contextual_orchestrator_sidecar_stream.py | 77 +++++++++- ...l_orchestrator_review_runtime_preflight.py | 136 +++++++++++++++++- 3 files changed, 209 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 55a2a2f211..30880d4844 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +### Sidecar sanitizer keeps the exception type and innermost frame per traceback + +- `scripts/ci/sanitize_contextual_orchestrator_sidecar_stream.py` now reduces each Python traceback in the sidecar stream to one line, `unexpected_exception type= frame=contextual_orchestrator/.py::` (the type identifier and the innermost package frame only; the exception message, source echoes and non-package frames are never re-emitted; a traceback cut off by the sidecar dying or without a package frame reports `unknown`). The previous single, once-per-stream `sidecar emitted an unexpected exception` line kept neither the count nor the type: `.github#1812`'s strix run (33993155419) ended on 83 gateway `500 internal_error` responses -- the orchestrator's generic request handler prints one traceback per unhandled exception -- and no artifact could say which exception escaped or where. Chain sentences (`During handling of the above exception…`, `The above exception was the direct cause…`) are consumed, so a chained exception yields cause then effect. + ### Noema review ships sidecar evidence on failure - `noema-review.yml` now uploads `strix_runs/contextual-orchestrator-sidecar.stderr.log` and `strix_runs/contextual-orchestrator-preflight.json` as the `noema-sidecar-evidence` artifact when the verdict phase fails (`if: failure()`, the same pinned `actions/upload-artifact` Strix uses, `if-no-files-found: ignore`, 5-day retention). Until now a failed Noema run left `artifacts=0` -- run `33981136873` spent 3122 s walking six ready routes twice each and ended in HTTP 502 with no per-route trace anywhere but the sidecar's stderr -- so the only diagnosis available was the caller's one-line summary. The stderr file is the sanitizer's bounded allowlist output (`sanitize_contextual_orchestrator_sidecar_stream.py`), the same file Strix already publishes in `strix-reports`; per-attempt route outcomes still need an allowlisted structured line from the orchestrator to appear in it. Refs #1935, #1939. diff --git a/scripts/ci/sanitize_contextual_orchestrator_sidecar_stream.py b/scripts/ci/sanitize_contextual_orchestrator_sidecar_stream.py index 51b9a5df27..0f6d22d33c 100644 --- a/scripts/ci/sanitize_contextual_orchestrator_sidecar_stream.py +++ b/scripts/ci/sanitize_contextual_orchestrator_sidecar_stream.py @@ -52,6 +52,27 @@ rf"^circuit_cleared agent_id={_AGENT_ID}$", ) ) +# Python traceback anatomy. The orchestrator's generic request handler +# (``server.py`` ``except Exception: traceback.print_exc(); _send_error(500, +# "internal_error", ...)``) prints one traceback per unhandled exception, so the +# exception *type* and the innermost ``contextual_orchestrator`` frame are the +# only evidence of what escaped. Frame lines are indented; the terminal line +# (``Type: message``) starts at column 0. Only the bounded type identifier and +# the package-relative frame are re-emitted -- never the message, which can +# carry provider bodies or credentials. +_TRACEBACK_FRAME = re.compile( + r'^\s+File ".*?[/\\]contextual_orchestrator[/\\](?P[A-Za-z0-9_][A-Za-z0-9_/\\]*\.py)", ' + r"line (?P\d+), in (?P[A-Za-z0-9_<>]{1,80})$" +) +_TRACEBACK_TERMINAL = re.compile( + r"^(?P[A-Za-z_][A-Za-z0-9_]{0,63}(?:\.[A-Za-z_][A-Za-z0-9_]{0,63}){0,8})(?::.*)?$" +) +_TRACEBACK_CHAIN_LINES = frozenset( + ( + "During handling of the above exception, another exception occurred:", + "The above exception was the direct cause of the following exception:", + ) +) _PREFIX_SUMMARIES = ( ("review sidecar preflight failed:", "review sidecar preflight failed"), ("review sidecar discovery failed:", "review sidecar discovery failed"), @@ -134,21 +155,65 @@ def sanitize_line(line: str) -> str | None: return None +def _traceback_summary(exception_type: str | None, frame: str | None) -> str: + """Return the one bounded line kept per traceback: exception type and innermost frame.""" + return ( + f"unexpected_exception type={exception_type or 'unknown'} " + f"frame={frame or 'unknown'}" + ) + + def main() -> int: - """Stream sanitized summaries to stdout without retaining raw provider text.""" + """Stream sanitized summaries to stdout without retaining raw provider text. + + A traceback opens at its ``Traceback`` header and closes at its column-0 + terminal ``Type: message`` line (emitting ``unexpected_exception type=... + frame=...``), at the next header or allowlisted line, or at end of stream + (``type=unknown``). Indented lines inside it are frames and source echoes: + consumed, not counted as omitted, and only a ``contextual_orchestrator`` + frame's package path, line and function are retained. Any other column-0 + line closes the traceback and is classified like every other line. + """ omitted = 0 - unexpected_exception_reported = False + in_traceback = False + frame: str | None = None for line in sys.stdin: - if line.lstrip().startswith("Traceback"): - if not unexpected_exception_reported: - print("sidecar emitted an unexpected exception", flush=True) - unexpected_exception_reported = True + stripped = line.strip() + if not stripped or stripped in _TRACEBACK_CHAIN_LINES: + # Blank lines carry nothing (Python pads chain sentences with them). + continue + if stripped.startswith("Traceback"): + if in_traceback: + print(_traceback_summary(None, frame), flush=True) + in_traceback, frame = True, None + continue + if in_traceback: + frame_match = _TRACEBACK_FRAME.match(line.rstrip("\n")) + if frame_match is not None: + module = frame_match.group("module").replace("\\", "/") + frame = f"contextual_orchestrator/{module}:{frame_match.group('line')}:{frame_match.group('function')}" + continue + if line[:1].isspace(): + continue + in_traceback = False + sanitized = sanitize_line(line) + terminal = _TRACEBACK_TERMINAL.match(stripped) if sanitized is None else None + if terminal is not None: + print(_traceback_summary(terminal.group("type"), frame), flush=True) + continue + print(_traceback_summary(None, frame), flush=True) + if sanitized is None: + omitted += 1 + continue + print(sanitized, flush=True) continue sanitized = sanitize_line(line) if sanitized is None: omitted += 1 continue print(sanitized, flush=True) + if in_traceback: + print(_traceback_summary(None, frame), flush=True) if omitted: print(f"omitted_unstructured_lines={omitted}", flush=True) return 0 diff --git a/tests/test_contextual_orchestrator_review_runtime_preflight.py b/tests/test_contextual_orchestrator_review_runtime_preflight.py index d0ace81e8b..092bb9f75b 100644 --- a/tests/test_contextual_orchestrator_review_runtime_preflight.py +++ b/tests/test_contextual_orchestrator_review_runtime_preflight.py @@ -1857,16 +1857,148 @@ def test_sidecar_stream_sanitizer_summarizes_unstructured_and_traceback_lines( assert main() == 0 rendered = output.getvalue() + # The indented pseudo-frame is consumed as traceback body (not counted as + # omitted); each header closes at the next header or allowlisted line. assert rendered.splitlines() == [ "request_failed status=500 code=internal_error", - "sidecar emitted an unexpected exception", + "unexpected_exception type=unknown frame=unknown", + "unexpected_exception type=unknown frame=unknown", "review sidecar preflight failed", "client_disconnected", - "omitted_unstructured_lines=1", ] assert secret not in rendered +def _render_orchestrator_traceback(source: str, module: str, call: str) -> str: + """Run ``source`` as if it were a ``contextual_orchestrator`` module and return the real traceback.""" + import traceback + + namespace: dict[str, object] = {"__name__": f"contextual_orchestrator.{module}"} + exec( # noqa: S102 - test-only: the source is a literal in this file + compile(source, f"/opt/site-packages/contextual_orchestrator/{module}.py", "exec"), + namespace, + ) + try: + eval(call, namespace) # noqa: S307 - test-only literal + except Exception: # noqa: BLE001 - the traceback under test + return traceback.format_exc() + raise AssertionError("fixture did not raise") + + +def _sanitize_stream(monkeypatch: pytest.MonkeyPatch, text: str) -> list[str]: + """Run the sanitizer's ``main`` over ``text`` and return its output lines.""" + namespace = _load_sanitizer() + monkeypatch.setattr(sys, "stdin", io.StringIO(text)) + output = io.StringIO() + with redirect_stdout(output): + assert namespace["main"]() == 0 + return output.getvalue().splitlines() + + +def test_sidecar_stream_sanitizer_keeps_exception_type_and_innermost_frame( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A real traceback is reduced to its exception type and innermost package frame. + + `.github#1812`'s strix run (33993155419) died on 83 gateway ``500 + internal_error`` responses -- the orchestrator's generic handler prints one + traceback per unhandled exception -- and the sanitized stream kept a single + ``sidecar emitted an unexpected exception`` line, so neither the exception + type nor where it escaped survived into any artifact. + """ + secret = "sk-secret-must-not-enter-artifact" + rendered = _render_orchestrator_traceback( + "def _serve(payload):\n" + " return payload['model']\n" + "def do_POST(payload):\n" + " return _serve(payload)\n", + "server", + f"do_POST({{'token': '{secret}'}})", + ) + assert "KeyError: 'model'" in rendered + assert 'contextual_orchestrator/server.py", line 2, in _serve' in rendered + + lines = _sanitize_stream( + monkeypatch, + rendered + "request_failed status=500 code=internal_error\n", + ) + + assert lines == [ + "unexpected_exception type=KeyError frame=contextual_orchestrator/server.py:2:_serve", + "request_failed status=500 code=internal_error", + ] + assert secret not in "\n".join(lines) + assert "test_contextual_orchestrator" not in "\n".join(lines) + + +def test_sidecar_stream_sanitizer_keeps_dotted_exception_types_and_chains( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A package-defined exception keeps its dotted type; a chained traceback yields cause then effect.""" + rendered = _render_orchestrator_traceback( + "class ProviderResponseError(RuntimeError):\n" + " pass\n" + "def _parse(body):\n" + " return body['choices']\n" + "def proxy(body):\n" + " try:\n" + " return _parse(body)\n" + " except KeyError as exc:\n" + " raise ProviderResponseError('malformed body: sk-leak') from exc\n", + "transport", + "proxy({})", + ) + assert "The above exception was the direct cause of the following exception:" in rendered + + lines = _sanitize_stream(monkeypatch, rendered) + + assert lines == [ + "unexpected_exception type=KeyError frame=contextual_orchestrator/transport.py:4:_parse", + "unexpected_exception type=contextual_orchestrator.transport.ProviderResponseError " + "frame=contextual_orchestrator/transport.py:9:proxy", + ] + assert "sk-leak" not in "\n".join(lines) + + +def test_sidecar_stream_sanitizer_closes_a_truncated_traceback_at_end_of_stream( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A traceback cut off by the sidecar dying still reports its innermost frame.""" + lines = _sanitize_stream( + monkeypatch, + "Traceback (most recent call last):\n" + ' File "/x/site-packages/contextual_orchestrator/orchestrator.py", line 7824, in _invoke\n' + " result = await candidate.send(sk-secret)\n" + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + ) + assert lines == [ + "unexpected_exception type=unknown frame=contextual_orchestrator/orchestrator.py:7824:_invoke", + ] + + +def test_sidecar_stream_sanitizer_does_not_treat_free_text_as_an_exception( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A column-0 line that is neither a terminal nor allowlisted closes the traceback and is omitted. + + Unstructured lines outside any traceback keep counting as omitted, as before. + """ + lines = _sanitize_stream( + monkeypatch, + "Traceback (most recent call last):\n" + ' File "/x/site-packages/contextual_orchestrator/server.py", line 6288, in do_POST\n' + "provider said: sk-secret and more words\n" + "client_disconnected\n" + "provider body outside any traceback: sk-secret-two\n", + ) + assert lines == [ + "unexpected_exception type=unknown frame=contextual_orchestrator/server.py:6288:do_POST", + "client_disconnected", + "omitted_unstructured_lines=2", + ] + assert "sk-secret" not in "\n".join(lines) + + def test_sidecar_stream_sanitizer_omits_no_summary_for_fully_safe_input( monkeypatch: pytest.MonkeyPatch, ) -> None: