fix(tracing): capture failed CommandExecution items

plugins/tracing/src/parse.ts

@@ -381,6 +381,20 @@ export function parseSession(lines: RolloutLine[]): {
         if (p.item.kind === "started" && typeof p.item.agent_thread_id === "string") {
           recordSubagentThread(p.item.agent_thread_id, p.item.id);
         }
+      } else if (et === "item_completed" && p.item?.type === "CommandExecution") {
+        const item = p.item;
+        if (
+          item.source !== "user_shell" &&
+          typeof item.id === "string" &&
+          (item.status === "failed" || item.status === "declined")
+        ) {
+          // Nested commands have their own IDs; do not attribute their failures to the wrapper.
+          const tc = toolCallsById.get(item.id);
+          if (tc)
+            tc.error ??=
+              item.aggregated_output ||
+              (typeof item.exit_code === "number" ? `Exit code: ${item.exit_code}` : item.status);
+        }
       } else if (et === "agent_message" && typeof p.message === "string") {
         turn!.lastAgentMessage = p.message;
       } else if (et === "token_count") {

plugins/tracing/src/types.ts

@@ -133,6 +133,10 @@ export type EventMsgPayload = {
     kind?: string;
     agent_thread_id?: string;
     content?: MessageContentPart[];
+    source?: string;
+    status?: string;
+    aggregated_output?: string | null;
+    exit_code?: number | null;
   } | null;
   new_thread_id?: string | null;
   event_id?: string;

plugins/tracing/test/fixtures/sessions/2026/06/03/rollout-command-failed.jsonl

@@ -0,0 +1,9 @@
+{"timestamp":"2026-06-03T14:00:00.000Z","type":"session_meta","payload":{"id":"sess-command-failed","cli_version":"0.149.0","model_provider":"openai"}}
+{"timestamp":"2026-06-03T14:00:01.000Z","type":"event_msg","payload":{"type":"task_started","turn_id":"turn-command-failed"}}
+{"timestamp":"2026-06-03T14:00:01.100Z","type":"event_msg","payload":{"type":"user_message","message":"Run the check"}}
+{"timestamp":"2026-06-03T14:00:02.000Z","type":"response_item","payload":{"type":"function_call","name":"exec_command","call_id":"call-exec","arguments":"{\"cmd\":\"check\"}"}}
+{"timestamp":"2026-06-03T14:00:02.100Z","type":"response_item","payload":{"type":"function_call_output","call_id":"call-exec","output":"Process running with session ID 123"}}
+{"timestamp":"2026-06-03T14:00:03.000Z","type":"response_item","payload":{"type":"function_call","name":"write_stdin","call_id":"call-stdin","arguments":"{\"session_id\":123,\"chars\":\"\"}"}}
+{"timestamp":"2026-06-03T14:00:03.100Z","type":"response_item","payload":{"type":"function_call_output","call_id":"call-stdin","output":"Process exited with code 1"}}
+{"timestamp":"2026-06-03T14:00:03.200Z","type":"event_msg","payload":{"type":"item_completed","turn_id":"turn-command-failed","item":{"type":"CommandExecution","id":"call-exec","command":["check"],"cwd":"/repo","source":"unified_exec_startup","status":"failed","aggregated_output":"check failed","exit_code":1}}}
+{"timestamp":"2026-06-03T14:00:04.000Z","type":"event_msg","payload":{"type":"task_complete","turn_id":"turn-command-failed"}}

plugins/tracing/test/parse.test.ts

@@ -5,7 +5,7 @@ import { fileURLToPath } from "node:url";
 import { describe, expect, it } from "vitest";
 
 import { parseSession } from "../src/parse.js";
-import type { RolloutLine } from "../src/types.js";
+import type { EventMsgPayload, RolloutLine } from "../src/types.js";
 
 const fixturesDir = path.join(
   path.dirname(fileURLToPath(import.meta.url)),
@@ -67,6 +67,81 @@ describe("parseSession", () => {
     expect(step2.toolCalls).toHaveLength(0);
   });
 
+  it.each([
+    { status: "failed", aggregated_output: "check failed", exit_code: 1, error: "check failed" },
+    { status: "failed", aggregated_output: "", exit_code: 2, error: "Exit code: 2" },
+    { status: "failed", aggregated_output: "", exit_code: null, error: "failed" },
+    { status: "declined", aggregated_output: "", exit_code: null, error: "declined" },
+  ])("records CommandExecution $status with error '$error'", ({ error, ...details }) => {
+    const lines = loadFixture("rollout-command-failed.jsonl");
+    const completion = lines.find((line) => line.payload.type === "item_completed")!;
+    Object.assign((completion.payload as EventMsgPayload).item!, details);
+    const { turns } = parseSession(lines);
+    const tools = turns[0].steps.flatMap((step) => step.toolCalls);
+    expect(tools).toHaveLength(2);
+    expect(tools[0]).toMatchObject({
+      callId: "call-exec",
+      error,
+      output: "Process running with session ID 123",
+      endTime: Date.parse("2026-06-03T14:00:02.100Z"),
+    });
+    expect(tools[1].error).toBeUndefined();
+  });
+
+  it.each([
+    { source: "user_shell" },
+    { id: "unknown-call" },
+    { status: "completed" },
+    { status: "in_progress" },
+  ])("ignores unrelated or non-failing command completions: %j", (details) => {
+    const lines = loadFixture("rollout-command-failed.jsonl");
+    const completion = lines.find((line) => line.payload.type === "item_completed")!;
+    Object.assign((completion.payload as EventMsgPayload).item!, details);
+    const tools = parseSession(lines).turns[0].steps.flatMap((step) => step.toolCalls);
+    expect(tools).toHaveLength(2);
+    expect(tools.every((tool) => tool.error === undefined)).toBe(true);
+  });
+
+  it("preserves an existing error when a command completion arrives", () => {
+    const lines = loadFixture("rollout-command-failed.jsonl");
+    lines.splice(7, 0, {
+      timestamp: "2026-06-03T14:00:03.150Z",
+      type: "event_msg",
+      payload: {
+        type: "exec_command_end",
+        call_id: "call-exec",
+        status: "failed",
+        error: "original error",
+      },
+    });
+    const tools = parseSession(lines).turns[0].steps.flatMap((step) => step.toolCalls);
+    expect(tools[0].error).toBe("original error");
+  });
+
+  it("does not attach a previous turn's command failure to the current turn", () => {
+    const lines = loadFixture("rollout-command-failed.jsonl");
+    lines.splice(
+      5,
+      0,
+      {
+        timestamp: "2026-06-03T14:00:02.200Z",
+        type: "event_msg",
+        payload: { type: "task_complete", turn_id: "turn-command-failed" },
+      },
+      {
+        timestamp: "2026-06-03T14:00:02.300Z",
+        type: "event_msg",
+        payload: { type: "task_started", turn_id: "turn-next" },
+      },
+    );
+    (lines[lines.length - 1].payload as EventMsgPayload).turn_id = "turn-next";
+    const { turns } = parseSession(lines);
+    expect(turns).toHaveLength(2);
+    const tools = turns.flatMap((turn) => turn.steps.flatMap((step) => step.toolCalls));
+    expect(tools.map((tool) => tool.callId)).toEqual(["call-exec", "call-stdin"]);
+    expect(tools.every((tool) => tool.error === undefined)).toBe(true);
+  });
+
   it("captures an explicitly invoked skill without letting it leak into the prompt", () => {
     const { turns } = parseSession(loadFixture("rollout-skills-main.jsonl"));
 

plugins/tracing/test/trace.test.ts

@@ -152,6 +152,20 @@ describe("convertRollout", () => {
     expect(attr(generations[0], "langfuse.observation.output")).toContain("call-1");
   });
 
+  it("marks the matching command as ERROR from item_completed without legacy end events", async () => {
+    const dir = stageFixtures();
+    await convertRollout(path.join(dir, "rollout-command-failed.jsonl"), { config: baseConfig });
+    const tools = exporter
+      .getFinishedSpans()
+      .filter((s) => obsType(s) === "tool")
+      .sort((a, b) => startMs(a) - startMs(b));
+    expect(tools).toHaveLength(2);
+    expect(attr(tools[0], "langfuse.observation.metadata.codex.call_id")).toBe("call-exec");
+    expect(attr(tools[0], "langfuse.observation.level")).toBe("ERROR");
+    expect(attr(tools[0], "langfuse.observation.status_message")).toBe("check failed");
+    expect(attr(tools[1], "langfuse.observation.level")).not.toBe("ERROR");
+  });
+
   it("ends a generation at the tool call it emitted, not at the end of the step", async () => {
     const dir = stageFixtures();
     await convertRollout(path.join(dir, "rollout-basic-main.jsonl"), { config: baseConfig });