From 332ccf200f459b00162cbef0a1dc088f37ef6362 Mon Sep 17 00:00:00 2001 From: Muspi Merol Date: Mon, 2 Nov 2026 05:37:14 +0800 Subject: [PATCH 1/2] fix(tracing): preserve tool names in ChatML history --- plugins/tracing/src/trace.ts | 4 +- plugins/tracing/test/trace.test.ts | 74 +++++++++++++++++++++++++++++- 2 files changed, 75 insertions(+), 3 deletions(-) diff --git a/plugins/tracing/src/trace.ts b/plugins/tracing/src/trace.ts index b0de00d..51dcf71 100644 --- a/plugins/tracing/src/trace.ts +++ b/plugins/tracing/src/trace.ts @@ -337,7 +337,7 @@ function assistantMessage(step: ModelStep): ChatMlMessage { id: tc.callId, type: "function" as const, function: { - name: toolObservationName(tc), + name: tc.name, ...(tc.args !== undefined ? { arguments: toText(tc.args) } : {}), }, })), @@ -350,7 +350,7 @@ function toolMessages(step: ModelStep): ChatMlMessage[] { return step.toolCalls.map((tc) => ({ role: "tool" as const, tool_call_id: tc.callId, - name: toolObservationName(tc), + name: tc.name, content: tc.output != null ? toText(tc.output) : (tc.error ?? ""), ...(tc.error ? { is_error: true as const } : {}), })); diff --git a/plugins/tracing/test/trace.test.ts b/plugins/tracing/test/trace.test.ts index fd8c1be..f3ea606 100644 --- a/plugins/tracing/test/trace.test.ts +++ b/plugins/tracing/test/trace.test.ts @@ -783,7 +783,23 @@ describe("convertRollout", () => { it("captures web search, local shell, and MCP tool calls with specific names", async () => { const dir = stageFixtures(); - await convertRollout(path.join(dir, "rollout-tools-main.jsonl"), { config: baseConfig }); + const file = path.join(dir, "rollout-tools-main.jsonl"); + const lines = fs + .readFileSync(file, "utf-8") + .trim() + .split("\n") + .map((line) => JSON.parse(line)); + lines.splice( + lines.findIndex((line) => line.payload.role === "assistant"), + 0, + { + timestamp: "2026-06-03T12:00:04.700Z", + type: "event_msg", + payload: { type: "token_count" }, + }, + ); + fs.writeFileSync(file, lines.map((line) => JSON.stringify(line)).join("\n") + "\n"); + await convertRollout(file, { config: baseConfig }); const spans = exporter.getFinishedSpans(); const toolNames = spans @@ -798,6 +814,30 @@ describe("convertRollout", () => { const shell = spans.find((s) => s.name === "local_shell")!; expect(attr(shell, "langfuse.observation.output")).toContain("clean"); + + const generations = spans + .filter((s) => obsType(s) === "generation") + .sort((a, b) => startMs(a) - startMs(b)); + expect(generations).toHaveLength(2); + const history = JSON.parse(attr(generations[1], "langfuse.observation.input")); + expect(history).toContainEqual( + expect.objectContaining({ + role: "assistant", + tool_calls: expect.arrayContaining([ + expect.objectContaining({ + id: "call-mcp", + function: expect.objectContaining({ name: "linear__create_issue" }), + }), + ]), + }), + ); + expect(history).toContainEqual( + expect.objectContaining({ + role: "tool", + tool_call_id: "call-mcp", + name: "linear__create_issue", + }), + ); }); it("does not re-export a settings event between turns on every invocation", async () => { @@ -975,6 +1015,38 @@ describe("deterministic trace ids (trace_seed)", () => { }); describe("skill observations", () => { + it.each([true, false])( + "keeps real tool names in generation history with skill_tags=%s", + async (skill_tags) => { + const dir = stageFixtures(); + await convertRollout(path.join(dir, "rollout-skills-main.jsonl"), { + config: { ...baseConfig, skill_tags }, + }); + + const spans = exporter.getFinishedSpans(); + const generations = spans + .filter((s) => obsType(s) === "generation") + .sort((a, b) => startMs(a) - startMs(b)); + expect(generations).toHaveLength(2); + const history = JSON.parse(attr(generations[1], "langfuse.observation.input")); + expect(history).toContainEqual( + expect.objectContaining({ + role: "assistant", + tool_calls: expect.arrayContaining([ + expect.objectContaining({ + id: "call-doc", + function: expect.objectContaining({ name: "exec_command" }), + }), + ]), + }), + ); + expect(history).toContainEqual( + expect.objectContaining({ role: "tool", tool_call_id: "call-doc", name: "exec_command" }), + ); + expect(spans.find((s) => s.name === "skill:bug-mentor")).toBeDefined(); + }, + ); + it("names a skill-loading command after the skill, keeping its own timing", async () => { const dir = stageFixtures(); await convertRollout(path.join(dir, "rollout-skills-main.jsonl"), { config: baseConfig }); From 8e01bc0686e19fb86ee6f1a55a8dfb5b0375d025 Mon Sep 17 00:00:00 2001 From: Muspi Merol Date: Fri, 6 Nov 2026 11:18:00 +0800 Subject: [PATCH 2/2] fix(tracing): retain the tool name fallback in ChatML history --- plugins/tracing/src/trace.ts | 4 ++-- plugins/tracing/test/trace.test.ts | 34 ++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/plugins/tracing/src/trace.ts b/plugins/tracing/src/trace.ts index 51dcf71..0941cf5 100644 --- a/plugins/tracing/src/trace.ts +++ b/plugins/tracing/src/trace.ts @@ -337,7 +337,7 @@ function assistantMessage(step: ModelStep): ChatMlMessage { id: tc.callId, type: "function" as const, function: { - name: tc.name, + name: tc.name || "tool", ...(tc.args !== undefined ? { arguments: toText(tc.args) } : {}), }, })), @@ -350,7 +350,7 @@ function toolMessages(step: ModelStep): ChatMlMessage[] { return step.toolCalls.map((tc) => ({ role: "tool" as const, tool_call_id: tc.callId, - name: tc.name, + name: tc.name || "tool", content: tc.output != null ? toText(tc.output) : (tc.error ?? ""), ...(tc.error ? { is_error: true as const } : {}), })); diff --git a/plugins/tracing/test/trace.test.ts b/plugins/tracing/test/trace.test.ts index f3ea606..5f8c513 100644 --- a/plugins/tracing/test/trace.test.ts +++ b/plugins/tracing/test/trace.test.ts @@ -219,6 +219,40 @@ describe("convertRollout", () => { expect(meta("changed_this_turn")).toBe("true"); }); + it.each(["", undefined])("keeps the history fallback for tool name %j", async (name) => { + const dir = stageFixtures(); + const file = path.join(dir, "rollout-basic-main.jsonl"); + const lines = fs + .readFileSync(file, "utf8") + .trim() + .split("\n") + .map((line) => JSON.parse(line)); + // Malformed rollouts still need usable names in both halves of the tool exchange. + lines.find((line) => line.payload.type === "function_call").payload.name = name; + fs.writeFileSync(file, lines.map((line) => JSON.stringify(line)).join("\n") + "\n"); + await convertRollout(file, { config: baseConfig }); + + const generations = exporter + .getFinishedSpans() + .filter((s) => obsType(s) === "generation") + .sort((a, b) => startMs(a) - startMs(b)); + const history = JSON.parse(attr(generations[1], "langfuse.observation.input")); + expect(history).toContainEqual( + expect.objectContaining({ + role: "assistant", + tool_calls: [ + expect.objectContaining({ + id: "call-1", + function: expect.objectContaining({ name: "tool" }), + }), + ], + }), + ); + expect(history).toContainEqual( + expect.objectContaining({ role: "tool", tool_call_id: "call-1", name: "tool" }), + ); + }); + it("carries earlier turns of the thread into a later turn's generation input", async () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), "lf-codex-hist-")); const day = path.join(dir, "sessions", "2026", "06", "03");