diff --git a/src/providers/messages-to-responses.test.ts b/src/providers/messages-to-responses.test.ts index 8a25243..15326f6 100644 --- a/src/providers/messages-to-responses.test.ts +++ b/src/providers/messages-to-responses.test.ts @@ -146,4 +146,59 @@ describe("messages-to-responses", () => { }, ]); }); + + it("normalizes a plaintext reasoning item onto the assistant message", () => { + const output = responsesTurnToModelOutput({ + text: "Answer: B", + outputItems: [ + { + type: "reasoning", + id: "rs_1", + content: [{ type: "reasoning_text", text: "1. Analyze" }], + summary: [], + }, + { + type: "message", + content: [{ type: "output_text", text: "Answer: B" }], + }, + ], + functionCalls: [], + generationTimeMs: 7, + }); + expect(output.message.reasoning).toBe("1. Analyze"); + expect(output.message.reasoningDetails).toEqual([ + { type: "reasoning.text", text: "1. Analyze", id: "rs_1" }, + ]); + }); + + it("keeps an encrypted reasoning item as details without inventing readable text", () => { + const output = responsesTurnToModelOutput({ + text: "Answer: B", + outputItems: [ + { type: "reasoning", id: "rs_1", encrypted_content: "opaque" }, + ], + functionCalls: [], + generationTimeMs: 7, + }); + expect(output.message).not.toHaveProperty("reasoning"); + expect(output.message.reasoningDetails).toEqual([ + { type: "reasoning.encrypted", data: "opaque", id: "rs_1" }, + ]); + }); + + it("leaves both reasoning fields absent when the turn carries no reasoning", () => { + const output = responsesTurnToModelOutput({ + text: "Answer: B", + outputItems: [ + { + type: "message", + content: [{ type: "output_text", text: "Answer: B" }], + }, + ], + functionCalls: [], + generationTimeMs: 7, + }); + expect(output.message).not.toHaveProperty("reasoning"); + expect(output.message).not.toHaveProperty("reasoningDetails"); + }); }); diff --git a/src/providers/messages-to-responses.ts b/src/providers/messages-to-responses.ts index d99d2df..64ef412 100644 --- a/src/providers/messages-to-responses.ts +++ b/src/providers/messages-to-responses.ts @@ -6,6 +6,7 @@ import type { } from "../harness/core"; import { MessageRole } from "../harness/core"; import { definedValues } from "../internal/guards"; +import { extractReasoning } from "./responses-client"; import type { ResponsesFunctionTool, ResponsesInputItem, @@ -111,11 +112,13 @@ export function toolDefinitionToResponses( } export function responsesTurnToModelOutput(turn: ResponsesTurn): ModelOutput { + const reasoning = extractReasoning(turn.outputItems); return definedValues({ completion: turn.text, message: { role: MessageRole.Assistant, content: turn.text, + ...reasoning, ...definedValues({ toolCalls: turn.functionCalls.length > 0 diff --git a/src/providers/responses-client.test.ts b/src/providers/responses-client.test.ts index 10fba65..faa103b 100644 --- a/src/providers/responses-client.test.ts +++ b/src/providers/responses-client.test.ts @@ -28,6 +28,7 @@ import type { ModelErrorIdentifiers } from "./request-identifiers"; import { consumeStream, extractMessageText, + extractReasoning, findOutputItems, makeResponsesLayer, Responses, @@ -115,6 +116,176 @@ describe("extractMessageText", () => { expect(extractMessageText([])).toBe(""); }); }); +describe("extractReasoning", () => { + it("normalizes plaintext reasoning content into readable reasoning and text details", () => { + expect( + extractReasoning([ + { + type: "reasoning", + id: "rs_1", + content: [ + { type: "reasoning_text", text: "1. Analyze the request" }, + { type: "reasoning_text", text: "2. Answer" }, + ], + summary: [], + }, + { type: "message", content: [{ type: "output_text", text: "B" }] }, + ]) + ).toEqual({ + reasoning: "1. Analyze the request\n\n2. Answer", + reasoningDetails: [ + { + type: "reasoning.text", + text: "1. Analyze the request", + id: "rs_1", + }, + { type: "reasoning.text", text: "2. Answer", id: "rs_1" }, + ], + }); + }); + + it("carries the signature and format of a plaintext item", () => { + expect( + extractReasoning([ + { + type: "reasoning", + id: "rs_1", + format: "anthropic-claude-v1", + signature: "sig", + content: [{ type: "reasoning_text", text: "thought" }], + }, + ]) + ).toEqual({ + reasoning: "thought", + reasoningDetails: [ + { + type: "reasoning.text", + text: "thought", + id: "rs_1", + format: "anthropic-claude-v1", + signature: "sig", + }, + ], + }); + }); + + it("keeps an encrypted blob as a detail and emits no readable reasoning", () => { + expect( + extractReasoning([ + { + type: "reasoning", + id: "rs_2", + encrypted_content: "gAAAAAopaque", + summary: [], + }, + ]) + ).toEqual({ + reasoningDetails: [ + { type: "reasoning.encrypted", data: "gAAAAAopaque", id: "rs_2" }, + ], + }); + }); + + it("falls back to provider summaries when no plaintext is exposed", () => { + expect( + extractReasoning([ + { + type: "reasoning", + id: "rs_3", + summary: [ + { type: "summary_text", text: "Considered two options" }, + { type: "summary_text", text: "Picked the second" }, + ], + encrypted_content: "blob", + }, + ]) + ).toEqual({ + reasoning: "Considered two options\n\nPicked the second", + reasoningDetails: [ + { + type: "reasoning.summary", + summary: "Considered two options", + id: "rs_3", + }, + { + type: "reasoning.summary", + summary: "Picked the second", + id: "rs_3", + }, + { type: "reasoning.encrypted", data: "blob", id: "rs_3" }, + ], + }); + }); + + it("prefers plaintext over summaries for readable reasoning while keeping both details", () => { + const result = extractReasoning([ + { + type: "reasoning", + content: [{ type: "reasoning_text", text: "raw thought" }], + summary: [{ type: "summary_text", text: "short summary" }], + }, + ]); + expect(result.reasoning).toBe("raw thought"); + expect(result.reasoningDetails).toEqual([ + { type: "reasoning.text", text: "raw thought" }, + { type: "reasoning.summary", summary: "short summary" }, + ]); + }); + + it("joins plaintext across multiple reasoning items in wire order", () => { + expect( + extractReasoning([ + { + type: "reasoning", + content: [{ type: "reasoning_text", text: "first" }], + }, + { type: "function_call", call_id: "c1", name: "t", arguments: "{}" }, + { + type: "reasoning", + content: [{ type: "reasoning_text", text: "second" }], + }, + ]).reasoning + ).toBe("first\n\nsecond"); + }); + + it("returns nothing when no reasoning item is present", () => { + expect( + extractReasoning([ + { type: "message", content: [{ type: "output_text", text: "B" }] }, + ]) + ).toEqual({}); + }); + + it("returns nothing for a reasoning item carrying no reasoning at all", () => { + expect( + extractReasoning([ + { type: "reasoning", id: "rs_4", summary: [], content: [] }, + ]) + ).toEqual({}); + }); + + it("skips malformed parts rather than repairing them", () => { + expect( + extractReasoning([ + { + type: "reasoning", + content: [ + "not-an-object", + { type: "reasoning_text" }, + { type: "reasoning_text", text: "" }, + { type: "reasoning_text", text: 42 }, + { type: "reasoning_text", text: "kept" }, + ], + summary: "not-an-array", + encrypted_content: 7, + }, + ]) + ).toEqual({ + reasoning: "kept", + reasoningDetails: [{ type: "reasoning.text", text: "kept" }], + }); + }); +}); describe("findOutputItems", () => { it("returns all items matching the type", () => { const output = [ diff --git a/src/providers/responses-client.ts b/src/providers/responses-client.ts index 248a6e2..db83b5e 100644 --- a/src/providers/responses-client.ts +++ b/src/providers/responses-client.ts @@ -24,6 +24,7 @@ import { succeed as layerSucceed } from "effect/Layer"; import type { Citation, ModelUsage } from "../harness/core"; import { ModelError } from "../harness/core"; +import type { ReasoningDetails } from "../harness/reasoning-details"; import { Either } from "../internal/either"; import { definedValues, isRecord } from "../internal/guards"; import { parseSchema, z } from "../internal/zod"; @@ -70,6 +71,11 @@ export const ResponsesResultSchema = z.object({ export type ResponsesResult = z.infer; +export interface ResponsesReasoning { + readonly reasoning?: string; + readonly reasoningDetails?: ReasoningDetails; +} + const RawResponsesTerminalEventSchema = z.object({ type: z.union([ z.literal("response.completed"), @@ -576,6 +582,82 @@ export function extractMessageText( return text; } +export function extractReasoning( + output: readonly Record[] +): ResponsesReasoning { + const details: unknown[] = []; + const texts: string[] = []; + const summaries: string[] = []; + for (const item of output) { + if (item["type"] !== "reasoning") { + continue; + } + const id = stringField(item, "id"); + const format = stringField(item, "format"); + const signature = stringField(item, "signature"); + for (const text of partTexts(item["content"], "reasoning_text")) { + texts.push(text); + details.push( + definedValues({ + type: "reasoning.text", + text, + id, + format, + signature, + }) + ); + } + for (const summary of partTexts(item["summary"], "summary_text")) { + summaries.push(summary); + details.push( + definedValues({ + type: "reasoning.summary", + summary, + id, + format, + }) + ); + } + const encrypted = stringField(item, "encrypted_content"); + if (encrypted !== undefined) { + details.push( + definedValues({ + type: "reasoning.encrypted", + data: encrypted, + id, + format, + }) + ); + } + } + const readable = texts.length > 0 ? texts : summaries; + return definedValues({ + reasoning: readable.length > 0 ? readable.join("\n\n") : undefined, + reasoningDetails: details.length > 0 ? details : undefined, + }); +} + +function partTexts(parts: unknown, partType: string): string[] { + if (!Array.isArray(parts)) { + return []; + } + return parts.flatMap((part) => { + if (!isRecord(part) || part["type"] !== partType) { + return []; + } + const text = stringField(part, "text"); + return text !== undefined ? [text] : []; + }); +} + +function stringField( + record: Readonly>, + key: string +): string | undefined { + const value = record[key]; + return typeof value === "string" && value.length > 0 ? value : undefined; +} + export function findOutputItems( output: readonly Record[], itemType: string diff --git a/src/results/parquet.test.ts b/src/results/parquet.test.ts index e9fe883..7b97fc8 100644 --- a/src/results/parquet.test.ts +++ b/src/results/parquet.test.ts @@ -9,6 +9,7 @@ import { MessageRole, ScoreValue } from "../harness/core"; import type { SampleScore } from "../harness/metric"; import { assertRight, assertLeft } from "../internal/testing"; import { parseSchema } from "../internal/zod"; +import { responsesTurnToModelOutput } from "../providers/messages-to-responses"; import { readResultRows, runResultToParquet, @@ -454,6 +455,54 @@ describe("runResultToParquet", () => { ); expect(parsed[0]?.["reasoning"]).toBe("Step 1: ..."); }); + it("serializes reasoning from a provider Responses turn end to end", async () => { + const output = responsesTurnToModelOutput({ + text: "Answer: B", + outputItems: [ + { + type: "reasoning", + id: "rs_1", + content: [{ type: "reasoning_text", text: "1. Analyze the request" }], + summary: [], + }, + { + type: "reasoning", + id: "rs_2", + encrypted_content: "gAAAAAopaque", + summary: [], + }, + { + type: "message", + content: [{ type: "output_text", text: "Answer: B" }], + }, + ], + functionCalls: [], + generationTimeMs: 7, + }); + const bufferFromTurn = runResultToParquet({ + result: { + metrics: METRICS, + usage: USAGE, + sampleScores: [ + { + sampleId: "s0", + epoch: 0, + score: { value: ScoreValue.Correct, answer: "B", explanation: "" }, + messages: [output.message], + input: "q", + target: "B", + }, + ], + }, + meta: META, + }); + const turnRows = await readRows(bufferFromTurn); + const parsed: Record[] = JSON.parse( + turnRows[0]!.messages! + ); + expect(parsed[0]?.["reasoning"]).toBe("1. Analyze the request"); + expect(parsed[0]).not.toHaveProperty("reasoning_details"); + }); it("serializes multimodal content parts (image_url) in the messages JSON", async () => { const messages: readonly ModelMessage[] = [ {