diff --git a/plugins/tracing/src/parse.ts b/plugins/tracing/src/parse.ts index 5c70645..c0da662 100644 --- a/plugins/tracing/src/parse.ts +++ b/plugins/tracing/src/parse.ts @@ -3,6 +3,7 @@ import type { EventMsgPayload, MessageContentPart, ModelStep, + NestedResult, ResponseItemFunctionCall, ResponseItemFunctionCallOutput, ResponseItemCustomToolCall, @@ -76,6 +77,29 @@ export function parseArgs(raw: string): unknown { } } +/** + * Code-mode `exec` outputs are arrays of text parts; each nested command result is a JSON + * part carrying `exit_code`. Anything else (plain text, other shapes) is ignored. + */ +export function extractNestedResults(output: unknown): NestedResult[] { + if (!Array.isArray(output)) return []; + const results: NestedResult[] = []; + for (const part of output) { + const text = part && typeof part === "object" ? (part as { text?: unknown }).text : undefined; + if (typeof text !== "string" || !text.trimStart().startsWith("{")) continue; + const parsed = parseArgs(text); + if (parsed === null || typeof parsed !== "object") continue; + const { exit_code, chunk_id, output: nestedOutput } = parsed as Record; + if (typeof exit_code !== "number" || !Number.isFinite(exit_code)) continue; + results.push({ + exitCode: exit_code, + ...(typeof chunk_id === "string" ? { chunkId: chunk_id } : {}), + ...(nestedOutput !== undefined ? { output: toText(nestedOutput) } : {}), + }); + } + return results; +} + function extractToolError(payload: EventMsgPayload): string | undefined { const explicit = payload.error ?? payload.codex_error_info; if (explicit != null) { @@ -318,6 +342,8 @@ export function parseSession(lines: RolloutLine[]): { const tc = toolCallsById.get(out.call_id); if (tc) { if (tc.output == null) tc.output = out.output; + const nested = extractNestedResults(out.output); + if (nested.length > 0) tc.nested = [...(tc.nested ?? []), ...nested]; tc.endTime = Math.max(tc.endTime ?? ts, ts); if (tc.name === "spawn_agent") { const spawned = parseArgs(toText(out.output)); diff --git a/plugins/tracing/src/trace.ts b/plugins/tracing/src/trace.ts index b67396a..64aaa9b 100644 --- a/plugins/tracing/src/trace.ts +++ b/plugins/tracing/src/trace.ts @@ -566,6 +566,9 @@ function emitToolCall( parent: LangfuseObservation, fallbackEnd: number, ): LangfuseObservation { + const nested = tc.nested ?? []; + // Exit 1 is not counted: rg/grep use it for "no match". + const nestedFailed = nested.filter((n) => n.exitCode >= 2).length; const tool = startObservation( toolObservationName(tc), { @@ -573,7 +576,11 @@ function emitToolCall( output: tc.output != null ? toText(tc.output) : undefined, level: tc.error ? "ERROR" : undefined, statusMessage: tc.error, - metadata: { "codex.call_id": tc.callId, "codex.tool_name": tc.name || "tool" }, + metadata: { + "codex.call_id": tc.callId, + "codex.tool_name": tc.name || "tool", + ...(nestedFailed > 0 ? { "codex.nested_failed_count": nestedFailed } : {}), + }, }, { asType: "tool", @@ -581,7 +588,28 @@ function emitToolCall( parentSpanContext: parent.otelSpan.spanContext(), }, ); - tool.end(new Date(tc.endTime ?? fallbackEnd)); + const end = new Date(tc.endTime ?? fallbackEnd); + for (const n of nested) { + const failed = n.exitCode >= 2; + startObservation( + "exec_command", + { + output: n.output, + level: failed ? "ERROR" : undefined, + statusMessage: failed ? `nested command exited with code ${n.exitCode}` : undefined, + metadata: { + "codex.exit_code": n.exitCode, + ...(n.chunkId !== undefined ? { "codex.chunk_id": n.chunkId } : {}), + }, + }, + { + asType: "tool", + startTime: new Date(tc.startTime), + parentSpanContext: tool.otelSpan.spanContext(), + }, + ).end(end); + } + tool.end(end); return tool; } diff --git a/plugins/tracing/src/types.ts b/plugins/tracing/src/types.ts index 298bcc8..bd947ba 100644 --- a/plugins/tracing/src/types.ts +++ b/plugins/tracing/src/types.ts @@ -187,6 +187,13 @@ export type SystemPrompt = { changed: boolean; }; +/** A nested command result reported inside a tool output (e.g. each command run by code-mode `exec`). */ +export type NestedResult = { + exitCode: number; + chunkId?: string; + output?: string; +}; + export type ToolCall = { callId: string; name: string; @@ -195,6 +202,7 @@ export type ToolCall = { endTime?: number; output?: unknown; error?: string; + nested?: NestedResult[]; mcp?: { server: string; tool: string }; }; diff --git a/plugins/tracing/test/parse.test.ts b/plugins/tracing/test/parse.test.ts index 38b50ea..905546b 100644 --- a/plugins/tracing/test/parse.test.ts +++ b/plugins/tracing/test/parse.test.ts @@ -4,7 +4,7 @@ import { fileURLToPath } from "node:url"; import { describe, expect, it } from "vitest"; -import { parseSession } from "../src/parse.js"; +import { extractNestedResults, parseSession } from "../src/parse.js"; import type { RolloutLine } from "../src/types.js"; const fixturesDir = path.join( @@ -687,3 +687,36 @@ describe("user prompt extraction", () => { expect(turns[0]!.userInput).toBe("sag mal hallo"); }); }); + +describe("extractNestedResults", () => { + const part = (text: string) => ({ type: "input_text", text }); + + it("reads exit_code, chunk_id and output from JSON parts", () => { + const nested = extractNestedResults([ + part("Script completed\nWall time 1s\nOutput:\n"), + part(JSON.stringify({ chunk_id: "a4b7b0", exit_code: 2, output: "rg: No such file" })), + part(JSON.stringify({ exit_code: 0, output: "ok" })), + ]); + expect(nested).toEqual([ + { exitCode: 2, chunkId: "a4b7b0", output: "rg: No such file" }, + { exitCode: 0, output: "ok" }, + ]); + }); + + it("ignores malformed parts and non-array outputs without throwing", () => { + expect(extractNestedResults("plain text")).toEqual([]); + expect(extractNestedResults(undefined)).toEqual([]); + expect( + extractNestedResults([ + null, + "str", + { type: "input_text" }, + part("{not json"), + part("[1,2]"), + part("null"), + part(JSON.stringify({ output: "no exit code" })), + part(JSON.stringify({ exit_code: "2" })), + ]), + ).toEqual([]); + }); +}); diff --git a/plugins/tracing/test/trace.test.ts b/plugins/tracing/test/trace.test.ts index 36e015e..eb4d27d 100644 --- a/plugins/tracing/test/trace.test.ts +++ b/plugins/tracing/test/trace.test.ts @@ -1157,3 +1157,107 @@ describe("attached mode (external parent span)", () => { expect(exporter.getFinishedSpans()).toHaveLength(0); }); }); + +describe("nested exec results", () => { + const nestedOutput = [ + { type: "input_text", text: "Script completed\nOutput:\n" }, + { + type: "input_text", + text: JSON.stringify({ + chunk_id: "a4b7b0", + exit_code: 2, + output: "rg: path: No such file or directory", + }), + }, + { + type: "input_text", + text: JSON.stringify({ chunk_id: "c1d2e3", exit_code: 1, output: "" }), + }, + { type: "input_text", text: JSON.stringify({ exit_code: 0, output: "fine" }) }, + { type: "input_text", text: "{not json" }, + ]; + + let run = 0; + + // Unique session/call ids per conversion so spans from another test cannot leak in. + async function convertExec(output: unknown): Promise { + run += 1; + const callId = `call-n${run}`; + const dir = stageFixtures(); // A sessions/YYYY/MM/DD layout bounds the subagent scan. + const file = path.join(dir, "rollout-nested-main.jsonl"); + const line = (timestamp: string, type: string, payload: unknown) => + JSON.stringify({ timestamp, type, payload }); + fs.writeFileSync( + file, + [ + line("2026-06-03T10:00:00.000Z", "session_meta", { id: `sess-nested-${run}` }), + line("2026-06-03T10:00:01.000Z", "event_msg", { + type: "task_started", + turn_id: "turn-n", + }), + line("2026-06-03T10:00:01.300Z", "event_msg", { + type: "user_message", + message: "search", + }), + line("2026-06-03T10:00:02.000Z", "response_item", { + type: "custom_tool_call", + name: "exec", + call_id: callId, + input: "await tools.exec_command({})", + }), + line("2026-06-03T10:00:03.000Z", "response_item", { + type: "custom_tool_call_output", + call_id: callId, + output, + }), + line("2026-06-03T10:00:04.000Z", "event_msg", { + type: "task_complete", + turn_id: "turn-n", + }), + ].join("\n") + "\n", + ); + await convertRollout(file, { config: baseConfig }); + const tools = exporter.getFinishedSpans().filter((s) => obsType(s) === "tool"); + const parent = tools.find( + (s) => attr(s, "langfuse.observation.metadata.codex.call_id") === callId, + ); + if (!parent) return []; + return [parent, ...tools.filter((s) => parentId(s) === parent.spanContext().spanId)]; + } + + it("emits a child per nested result, ERROR only for exit >= 2, parent unchanged", async () => { + const [parent, ...children] = await convertExec(nestedOutput); + expect(children).toHaveLength(3); + + const failed = children.find( + (s) => attr(s, "langfuse.observation.metadata.codex.exit_code") === "2", + )!; + expect(failed.name).toBe("exec_command"); + expect(attr(failed, "langfuse.observation.level")).toBe("ERROR"); + expect(attr(failed, "langfuse.observation.status_message")).toContain("code 2"); + expect(attr(failed, "langfuse.observation.metadata.codex.chunk_id")).toBe("a4b7b0"); + expect(attr(failed, "langfuse.observation.output")).toContain("No such file"); + + const noMatch = children.find( + (s) => attr(s, "langfuse.observation.metadata.codex.exit_code") === "1", + )!; + expect(attr(noMatch, "langfuse.observation.level")).not.toBe("ERROR"); + const ok = children.find( + (s) => attr(s, "langfuse.observation.metadata.codex.exit_code") === "0", + )!; + expect(attr(ok, "langfuse.observation.level")).not.toBe("ERROR"); + + expect(attr(parent, "langfuse.observation.level")).not.toBe("ERROR"); + expect(attr(parent, "langfuse.observation.status_message")).toBe(""); + expect(attr(parent, "langfuse.observation.metadata.codex.nested_failed_count")).toBe("1"); + }); + + it("emits no children and no failed count when nothing nested is parseable", async () => { + const tools = await convertExec([ + { type: "input_text", text: "{not json" }, + { type: "input_text", text: JSON.stringify({ output: "x" }) }, + ]); + expect(tools).toHaveLength(1); + expect(attr(tools[0]!, "langfuse.observation.metadata.codex.nested_failed_count")).toBe(""); + }); +});