feat(tracing): surface nested exec command exit codes as child tool spans

plugins/tracing/src/parse.ts

@@ -3,6 +3,7 @@ import type {
   EventMsgPayload,
   MessageContentPart,
   ModelStep,
+  NestedResult,
   ResponseItemFunctionCall,
   ResponseItemFunctionCallOutput,
   ResponseItemCustomToolCall,
@@ -76,6 +77,29 @@ export function parseArgs(raw: string): unknown {
   }
 }
 
+/**
+ * Code-mode `exec` outputs are arrays of text parts; each nested command result is a JSON
+ * part carrying `exit_code`. Anything else (plain text, other shapes) is ignored.
+ */
+export function extractNestedResults(output: unknown): NestedResult[] {
+  if (!Array.isArray(output)) return [];
+  const results: NestedResult[] = [];
+  for (const part of output) {
+    const text = part && typeof part === "object" ? (part as { text?: unknown }).text : undefined;
+    if (typeof text !== "string" || !text.trimStart().startsWith("{")) continue;
+    const parsed = parseArgs(text);
+    if (parsed === null || typeof parsed !== "object") continue;
+    const { exit_code, chunk_id, output: nestedOutput } = parsed as Record<string, unknown>;
+    if (typeof exit_code !== "number" || !Number.isFinite(exit_code)) continue;
+    results.push({
+      exitCode: exit_code,
+      ...(typeof chunk_id === "string" ? { chunkId: chunk_id } : {}),
+      ...(nestedOutput !== undefined ? { output: toText(nestedOutput) } : {}),
+    });
+  }
+  return results;
+}
+
 function extractToolError(payload: EventMsgPayload): string | undefined {
   const explicit = payload.error ?? payload.codex_error_info;
   if (explicit != null) {
@@ -318,6 +342,8 @@ export function parseSession(lines: RolloutLine[]): {
         const tc = toolCallsById.get(out.call_id);
         if (tc) {
           if (tc.output == null) tc.output = out.output;
+          const nested = extractNestedResults(out.output);
+          if (nested.length > 0) tc.nested = [...(tc.nested ?? []), ...nested];
           tc.endTime = Math.max(tc.endTime ?? ts, ts);
           if (tc.name === "spawn_agent") {
             const spawned = parseArgs(toText(out.output));

plugins/tracing/src/trace.ts

@@ -566,22 +566,50 @@ function emitToolCall(
   parent: LangfuseObservation,
   fallbackEnd: number,
 ): LangfuseObservation {
+  const nested = tc.nested ?? [];
+  // Exit 1 is not counted: rg/grep use it for "no match".
+  const nestedFailed = nested.filter((n) => n.exitCode >= 2).length;
   const tool = startObservation(
     toolObservationName(tc),
     {
       input: tc.args,
       output: tc.output != null ? toText(tc.output) : undefined,
       level: tc.error ? "ERROR" : undefined,
       statusMessage: tc.error,
-      metadata: { "codex.call_id": tc.callId, "codex.tool_name": tc.name || "tool" },
+      metadata: {
+        "codex.call_id": tc.callId,
+        "codex.tool_name": tc.name || "tool",
+        ...(nestedFailed > 0 ? { "codex.nested_failed_count": nestedFailed } : {}),
+      },
     },
     {
       asType: "tool",
       startTime: new Date(tc.startTime),
       parentSpanContext: parent.otelSpan.spanContext(),
     },
   );
-  tool.end(new Date(tc.endTime ?? fallbackEnd));
+  const end = new Date(tc.endTime ?? fallbackEnd);
+  for (const n of nested) {
+    const failed = n.exitCode >= 2;
+    startObservation(
+      "exec_command",
+      {
+        output: n.output,
+        level: failed ? "ERROR" : undefined,
+        statusMessage: failed ? `nested command exited with code ${n.exitCode}` : undefined,
+        metadata: {
+          "codex.exit_code": n.exitCode,
+          ...(n.chunkId !== undefined ? { "codex.chunk_id": n.chunkId } : {}),
+        },
+      },
+      {
+        asType: "tool",
+        startTime: new Date(tc.startTime),
+        parentSpanContext: tool.otelSpan.spanContext(),
+      },
+    ).end(end);
+  }
+  tool.end(end);
   return tool;
 }
 

plugins/tracing/src/types.ts

@@ -187,6 +187,13 @@ export type SystemPrompt = {
   changed: boolean;
 };
 
+/** A nested command result reported inside a tool output (e.g. each command run by code-mode `exec`). */
+export type NestedResult = {
+  exitCode: number;
+  chunkId?: string;
+  output?: string;
+};
+
 export type ToolCall = {
   callId: string;
   name: string;
@@ -195,6 +202,7 @@ export type ToolCall = {
   endTime?: number;
   output?: unknown;
   error?: string;
+  nested?: NestedResult[];
   mcp?: { server: string; tool: string };
 };
 

plugins/tracing/test/parse.test.ts

@@ -4,7 +4,7 @@ import { fileURLToPath } from "node:url";
 
 import { describe, expect, it } from "vitest";
 
-import { parseSession } from "../src/parse.js";
+import { extractNestedResults, parseSession } from "../src/parse.js";
 import type { RolloutLine } from "../src/types.js";
 
 const fixturesDir = path.join(
@@ -687,3 +687,36 @@ describe("user prompt extraction", () => {
     expect(turns[0]!.userInput).toBe("sag mal hallo");
   });
 });
+
+describe("extractNestedResults", () => {
+  const part = (text: string) => ({ type: "input_text", text });
+
+  it("reads exit_code, chunk_id and output from JSON parts", () => {
+    const nested = extractNestedResults([
+      part("Script completed\nWall time 1s\nOutput:\n"),
+      part(JSON.stringify({ chunk_id: "a4b7b0", exit_code: 2, output: "rg: No such file" })),
+      part(JSON.stringify({ exit_code: 0, output: "ok" })),
+    ]);
+    expect(nested).toEqual([
+      { exitCode: 2, chunkId: "a4b7b0", output: "rg: No such file" },
+      { exitCode: 0, output: "ok" },
+    ]);
+  });
+
+  it("ignores malformed parts and non-array outputs without throwing", () => {
+    expect(extractNestedResults("plain text")).toEqual([]);
+    expect(extractNestedResults(undefined)).toEqual([]);
+    expect(
+      extractNestedResults([
+        null,
+        "str",
+        { type: "input_text" },
+        part("{not json"),
+        part("[1,2]"),
+        part("null"),
+        part(JSON.stringify({ output: "no exit code" })),
+        part(JSON.stringify({ exit_code: "2" })),
+      ]),
+    ).toEqual([]);
+  });
+});

plugins/tracing/test/trace.test.ts

@@ -1157,3 +1157,107 @@ describe("attached mode (external parent span)", () => {
     expect(exporter.getFinishedSpans()).toHaveLength(0);
   });
 });
+
+describe("nested exec results", () => {
+  const nestedOutput = [
+    { type: "input_text", text: "Script completed\nOutput:\n" },
+    {
+      type: "input_text",
+      text: JSON.stringify({
+        chunk_id: "a4b7b0",
+        exit_code: 2,
+        output: "rg: path: No such file or directory",
+      }),
+    },
+    {
+      type: "input_text",
+      text: JSON.stringify({ chunk_id: "c1d2e3", exit_code: 1, output: "" }),
+    },
+    { type: "input_text", text: JSON.stringify({ exit_code: 0, output: "fine" }) },
+    { type: "input_text", text: "{not json" },
+  ];
+
+  let run = 0;
+
+  // Unique session/call ids per conversion so spans from another test cannot leak in.
+  async function convertExec(output: unknown): Promise<ReadableSpan[]> {
+    run += 1;
+    const callId = `call-n${run}`;
+    const dir = stageFixtures(); // A sessions/YYYY/MM/DD layout bounds the subagent scan.
+    const file = path.join(dir, "rollout-nested-main.jsonl");
+    const line = (timestamp: string, type: string, payload: unknown) =>
+      JSON.stringify({ timestamp, type, payload });
+    fs.writeFileSync(
+      file,
+      [
+        line("2026-06-03T10:00:00.000Z", "session_meta", { id: `sess-nested-${run}` }),
+        line("2026-06-03T10:00:01.000Z", "event_msg", {
+          type: "task_started",
+          turn_id: "turn-n",
+        }),
+        line("2026-06-03T10:00:01.300Z", "event_msg", {
+          type: "user_message",
+          message: "search",
+        }),
+        line("2026-06-03T10:00:02.000Z", "response_item", {
+          type: "custom_tool_call",
+          name: "exec",
+          call_id: callId,
+          input: "await tools.exec_command({})",
+        }),
+        line("2026-06-03T10:00:03.000Z", "response_item", {
+          type: "custom_tool_call_output",
+          call_id: callId,
+          output,
+        }),
+        line("2026-06-03T10:00:04.000Z", "event_msg", {
+          type: "task_complete",
+          turn_id: "turn-n",
+        }),
+      ].join("\n") + "\n",
+    );
+    await convertRollout(file, { config: baseConfig });
+    const tools = exporter.getFinishedSpans().filter((s) => obsType(s) === "tool");
+    const parent = tools.find(
+      (s) => attr(s, "langfuse.observation.metadata.codex.call_id") === callId,
+    );
+    if (!parent) return [];
+    return [parent, ...tools.filter((s) => parentId(s) === parent.spanContext().spanId)];
+  }
+
+  it("emits a child per nested result, ERROR only for exit >= 2, parent unchanged", async () => {
+    const [parent, ...children] = await convertExec(nestedOutput);
+    expect(children).toHaveLength(3);
+
+    const failed = children.find(
+      (s) => attr(s, "langfuse.observation.metadata.codex.exit_code") === "2",
+    )!;
+    expect(failed.name).toBe("exec_command");
+    expect(attr(failed, "langfuse.observation.level")).toBe("ERROR");
+    expect(attr(failed, "langfuse.observation.status_message")).toContain("code 2");
+    expect(attr(failed, "langfuse.observation.metadata.codex.chunk_id")).toBe("a4b7b0");
+    expect(attr(failed, "langfuse.observation.output")).toContain("No such file");
+
+    const noMatch = children.find(
+      (s) => attr(s, "langfuse.observation.metadata.codex.exit_code") === "1",
+    )!;
+    expect(attr(noMatch, "langfuse.observation.level")).not.toBe("ERROR");
+    const ok = children.find(
+      (s) => attr(s, "langfuse.observation.metadata.codex.exit_code") === "0",
+    )!;
+    expect(attr(ok, "langfuse.observation.level")).not.toBe("ERROR");
+
+    expect(attr(parent, "langfuse.observation.level")).not.toBe("ERROR");
+    expect(attr(parent, "langfuse.observation.status_message")).toBe("");
+    expect(attr(parent, "langfuse.observation.metadata.codex.nested_failed_count")).toBe("1");
+  });
+
+  it("emits no children and no failed count when nothing nested is parseable", async () => {
+    const tools = await convertExec([
+      { type: "input_text", text: "{not json" },
+      { type: "input_text", text: JSON.stringify({ output: "x" }) },
+    ]);
+    expect(tools).toHaveLength(1);
+    expect(attr(tools[0]!, "langfuse.observation.metadata.codex.nested_failed_count")).toBe("");
+  });
+});