From 3fbd974e9bcfbac90ab80aec2506ada80d13373c Mon Sep 17 00:00:00 2001 From: Alan Carroll Date: Fri, 4 Sep 2026 20:16:21 +0000 Subject: [PATCH 1/4] fix(providers): normalize Responses reasoning items onto assistant messages Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- src/providers/messages-to-responses.test.ts | 55 +++++++ src/providers/messages-to-responses.ts | 3 + src/providers/responses-reasoning.test.ts | 174 ++++++++++++++++++++ src/providers/responses-reasoning.ts | 105 ++++++++++++ src/results/parquet.test.ts | 52 ++++++ src/results/parquet.ts | 3 + 6 files changed, 392 insertions(+) create mode 100644 src/providers/responses-reasoning.test.ts create mode 100644 src/providers/responses-reasoning.ts diff --git a/src/providers/messages-to-responses.test.ts b/src/providers/messages-to-responses.test.ts index 8a25243..15326f6 100644 --- a/src/providers/messages-to-responses.test.ts +++ b/src/providers/messages-to-responses.test.ts @@ -146,4 +146,59 @@ describe("messages-to-responses", () => { }, ]); }); + + it("normalizes a plaintext reasoning item onto the assistant message", () => { + const output = responsesTurnToModelOutput({ + text: "Answer: B", + outputItems: [ + { + type: "reasoning", + id: "rs_1", + content: [{ type: "reasoning_text", text: "1. Analyze" }], + summary: [], + }, + { + type: "message", + content: [{ type: "output_text", text: "Answer: B" }], + }, + ], + functionCalls: [], + generationTimeMs: 7, + }); + expect(output.message.reasoning).toBe("1. Analyze"); + expect(output.message.reasoningDetails).toEqual([ + { type: "reasoning.text", text: "1. Analyze", id: "rs_1" }, + ]); + }); + + it("keeps an encrypted reasoning item as details without inventing readable text", () => { + const output = responsesTurnToModelOutput({ + text: "Answer: B", + outputItems: [ + { type: "reasoning", id: "rs_1", encrypted_content: "opaque" }, + ], + functionCalls: [], + generationTimeMs: 7, + }); + expect(output.message).not.toHaveProperty("reasoning"); + expect(output.message.reasoningDetails).toEqual([ + { type: "reasoning.encrypted", data: "opaque", id: "rs_1" }, + ]); + }); + + it("leaves both reasoning fields absent when the turn carries no reasoning", () => { + const output = responsesTurnToModelOutput({ + text: "Answer: B", + outputItems: [ + { + type: "message", + content: [{ type: "output_text", text: "Answer: B" }], + }, + ], + functionCalls: [], + generationTimeMs: 7, + }); + expect(output.message).not.toHaveProperty("reasoning"); + expect(output.message).not.toHaveProperty("reasoningDetails"); + }); }); diff --git a/src/providers/messages-to-responses.ts b/src/providers/messages-to-responses.ts index d99d2df..ca8d94f 100644 --- a/src/providers/messages-to-responses.ts +++ b/src/providers/messages-to-responses.ts @@ -11,6 +11,7 @@ import type { ResponsesInputItem, ResponsesTurn, } from "./responses-model"; +import { reasoningFromOutputItems } from "./responses-reasoning"; export function messagesToResponses( messages: readonly ModelMessage[] @@ -111,11 +112,13 @@ export function toolDefinitionToResponses( } export function responsesTurnToModelOutput(turn: ResponsesTurn): ModelOutput { + const reasoning = reasoningFromOutputItems(turn.outputItems); return definedValues({ completion: turn.text, message: { role: MessageRole.Assistant, content: turn.text, + ...reasoning, ...definedValues({ toolCalls: turn.functionCalls.length > 0 diff --git a/src/providers/responses-reasoning.test.ts b/src/providers/responses-reasoning.test.ts new file mode 100644 index 0000000..05db839 --- /dev/null +++ b/src/providers/responses-reasoning.test.ts @@ -0,0 +1,174 @@ +import { describe, expect, it } from "bun:test"; + +import { reasoningFromOutputItems } from "./responses-reasoning"; + +describe("reasoningFromOutputItems", () => { + it("normalizes plaintext reasoning content into readable reasoning and text details", () => { + expect( + reasoningFromOutputItems([ + { + type: "reasoning", + id: "rs_1", + content: [ + { type: "reasoning_text", text: "1. Analyze the request" }, + { type: "reasoning_text", text: "2. Answer" }, + ], + summary: [], + }, + { type: "message", content: [{ type: "output_text", text: "B" }] }, + ]) + ).toEqual({ + reasoning: "1. Analyze the request\n\n2. Answer", + reasoningDetails: [ + { + type: "reasoning.text", + text: "1. Analyze the request", + id: "rs_1", + }, + { type: "reasoning.text", text: "2. Answer", id: "rs_1" }, + ], + }); + }); + + it("carries the signature and format of a plaintext item", () => { + expect( + reasoningFromOutputItems([ + { + type: "reasoning", + id: "rs_1", + format: "anthropic-claude-v1", + signature: "sig", + content: [{ type: "reasoning_text", text: "thought" }], + }, + ]) + ).toEqual({ + reasoning: "thought", + reasoningDetails: [ + { + type: "reasoning.text", + text: "thought", + id: "rs_1", + format: "anthropic-claude-v1", + signature: "sig", + }, + ], + }); + }); + + it("keeps an encrypted blob as a detail and emits no readable reasoning", () => { + expect( + reasoningFromOutputItems([ + { + type: "reasoning", + id: "rs_2", + encrypted_content: "gAAAAAopaque", + summary: [], + }, + ]) + ).toEqual({ + reasoningDetails: [ + { type: "reasoning.encrypted", data: "gAAAAAopaque", id: "rs_2" }, + ], + }); + }); + + it("falls back to provider summaries when no plaintext is exposed", () => { + expect( + reasoningFromOutputItems([ + { + type: "reasoning", + id: "rs_3", + summary: [ + { type: "summary_text", text: "Considered two options" }, + { type: "summary_text", text: "Picked the second" }, + ], + encrypted_content: "blob", + }, + ]) + ).toEqual({ + reasoning: "Considered two options\n\nPicked the second", + reasoningDetails: [ + { + type: "reasoning.summary", + summary: "Considered two options", + id: "rs_3", + }, + { + type: "reasoning.summary", + summary: "Picked the second", + id: "rs_3", + }, + { type: "reasoning.encrypted", data: "blob", id: "rs_3" }, + ], + }); + }); + + it("prefers plaintext over summaries for readable reasoning while keeping both details", () => { + const result = reasoningFromOutputItems([ + { + type: "reasoning", + content: [{ type: "reasoning_text", text: "raw thought" }], + summary: [{ type: "summary_text", text: "short summary" }], + }, + ]); + expect(result.reasoning).toBe("raw thought"); + expect(result.reasoningDetails).toEqual([ + { type: "reasoning.text", text: "raw thought" }, + { type: "reasoning.summary", summary: "short summary" }, + ]); + }); + + it("joins plaintext across multiple reasoning items in wire order", () => { + expect( + reasoningFromOutputItems([ + { + type: "reasoning", + content: [{ type: "reasoning_text", text: "first" }], + }, + { type: "function_call", call_id: "c1", name: "t", arguments: "{}" }, + { + type: "reasoning", + content: [{ type: "reasoning_text", text: "second" }], + }, + ]).reasoning + ).toBe("first\n\nsecond"); + }); + + it("returns nothing when no reasoning item is present", () => { + expect( + reasoningFromOutputItems([ + { type: "message", content: [{ type: "output_text", text: "B" }] }, + ]) + ).toEqual({}); + }); + + it("returns nothing for a reasoning item carrying no reasoning at all", () => { + expect( + reasoningFromOutputItems([ + { type: "reasoning", id: "rs_4", summary: [], content: [] }, + ]) + ).toEqual({}); + }); + + it("skips malformed parts rather than repairing them", () => { + expect( + reasoningFromOutputItems([ + { + type: "reasoning", + content: [ + "not-an-object", + { type: "reasoning_text" }, + { type: "reasoning_text", text: "" }, + { type: "reasoning_text", text: 42 }, + { type: "reasoning_text", text: "kept" }, + ], + summary: "not-an-array", + encrypted_content: 7, + }, + ]) + ).toEqual({ + reasoning: "kept", + reasoningDetails: [{ type: "reasoning.text", text: "kept" }], + }); + }); +}); diff --git a/src/providers/responses-reasoning.ts b/src/providers/responses-reasoning.ts new file mode 100644 index 0000000..9de2db8 --- /dev/null +++ b/src/providers/responses-reasoning.ts @@ -0,0 +1,105 @@ +import type { ReasoningDetails } from "../harness/reasoning-details"; +import { definedValues, isRecord } from "../internal/guards"; + +export interface ResponsesReasoning { + readonly reasoning?: string; + readonly reasoningDetails?: ReasoningDetails; +} + +const ReasoningDetailType = { + Summary: "reasoning.summary", + Encrypted: "reasoning.encrypted", + Text: "reasoning.text", +} as const; + +const REASONING_JOINER = "\n\n"; + +export function reasoningFromOutputItems( + outputItems: readonly Record[] +): ResponsesReasoning { + const details: unknown[] = []; + const texts: string[] = []; + const summaries: string[] = []; + + for (const item of outputItems) { + if (item["type"] !== "reasoning") { + continue; + } + const id = stringField(item, "id"); + const format = stringField(item, "format"); + const signature = stringField(item, "signature"); + + for (const text of reasoningTexts(item)) { + texts.push(text); + details.push( + definedValues({ + type: ReasoningDetailType.Text, + text, + id, + format, + signature, + }) + ); + } + + for (const summary of summaryTexts(item)) { + summaries.push(summary); + details.push( + definedValues({ + type: ReasoningDetailType.Summary, + summary, + id, + format, + }) + ); + } + + const encrypted = stringField(item, "encrypted_content"); + if (encrypted !== undefined) { + details.push( + definedValues({ + type: ReasoningDetailType.Encrypted, + data: encrypted, + id, + format, + }) + ); + } + } + + const readable = texts.length > 0 ? texts : summaries; + return definedValues({ + reasoning: + readable.length > 0 ? readable.join(REASONING_JOINER) : undefined, + reasoningDetails: details.length > 0 ? details : undefined, + }); +} + +function reasoningTexts(item: Record): string[] { + return partTexts(item["content"], "text"); +} + +function summaryTexts(item: Record): string[] { + return partTexts(item["summary"], "text"); +} + +function partTexts(parts: unknown, key: string): string[] { + if (!Array.isArray(parts)) { + return []; + } + return parts.flatMap((part) => { + if (!isRecord(part)) { + return []; + } + const text = stringField(part, key); + return text !== undefined ? [text] : []; + }); +} + +function stringField( + record: Record, + key: string +): string | undefined { + const value = record[key]; + return typeof value === "string" && value.length > 0 ? value : undefined; +} diff --git a/src/results/parquet.test.ts b/src/results/parquet.test.ts index e9fe883..6e5a686 100644 --- a/src/results/parquet.test.ts +++ b/src/results/parquet.test.ts @@ -9,6 +9,7 @@ import { MessageRole, ScoreValue } from "../harness/core"; import type { SampleScore } from "../harness/metric"; import { assertRight, assertLeft } from "../internal/testing"; import { parseSchema } from "../internal/zod"; +import { responsesTurnToModelOutput } from "../providers/messages-to-responses"; import { readResultRows, runResultToParquet, @@ -454,6 +455,57 @@ describe("runResultToParquet", () => { ); expect(parsed[0]?.["reasoning"]).toBe("Step 1: ..."); }); + it("serializes reasoning from a provider Responses turn end to end", async () => { + const output = responsesTurnToModelOutput({ + text: "Answer: B", + outputItems: [ + { + type: "reasoning", + id: "rs_1", + content: [{ type: "reasoning_text", text: "1. Analyze the request" }], + summary: [], + }, + { + type: "reasoning", + id: "rs_2", + encrypted_content: "gAAAAAopaque", + summary: [], + }, + { + type: "message", + content: [{ type: "output_text", text: "Answer: B" }], + }, + ], + functionCalls: [], + generationTimeMs: 7, + }); + const bufferFromTurn = runResultToParquet({ + result: { + metrics: METRICS, + usage: USAGE, + sampleScores: [ + { + sampleId: "s0", + epoch: 0, + score: { value: ScoreValue.Correct, answer: "B", explanation: "" }, + messages: [output.message], + input: "q", + target: "B", + }, + ], + }, + meta: META, + }); + const turnRows = await readRows(bufferFromTurn); + const parsed: Record[] = JSON.parse( + turnRows[0]!.messages! + ); + expect(parsed[0]?.["reasoning"]).toBe("1. Analyze the request"); + expect(parsed[0]?.["reasoning_details"]).toEqual([ + { type: "reasoning.text", text: "1. Analyze the request", id: "rs_1" }, + { type: "reasoning.encrypted", data: "gAAAAAopaque", id: "rs_2" }, + ]); + }); it("serializes multimodal content parts (image_url) in the messages JSON", async () => { const messages: readonly ModelMessage[] = [ { diff --git a/src/results/parquet.ts b/src/results/parquet.ts index 661415d..5b74d18 100644 --- a/src/results/parquet.ts +++ b/src/results/parquet.ts @@ -289,6 +289,9 @@ function messageToPojo(msg: ModelMessage): Record { if (msg.reasoning !== undefined) { pojo["reasoning"] = msg.reasoning; } + if (msg.reasoningDetails !== undefined && msg.reasoningDetails.length > 0) { + pojo["reasoning_details"] = msg.reasoningDetails; + } if (msg.citations !== undefined && msg.citations.length > 0) { pojo["citations"] = msg.citations.map((c) => ({ url: c.url, From 01ee1884df1e71d250b46496dd00f822ac26cedc Mon Sep 17 00:00:00 2001 From: Alan Carroll Date: Fri, 4 Sep 2026 20:34:51 +0000 Subject: [PATCH 2/4] fix(results): keep parquet message payload at parity, no reasoning_details column Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- src/results/parquet.test.ts | 5 +---- src/results/parquet.ts | 3 --- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/src/results/parquet.test.ts b/src/results/parquet.test.ts index 6e5a686..7b97fc8 100644 --- a/src/results/parquet.test.ts +++ b/src/results/parquet.test.ts @@ -501,10 +501,7 @@ describe("runResultToParquet", () => { turnRows[0]!.messages! ); expect(parsed[0]?.["reasoning"]).toBe("1. Analyze the request"); - expect(parsed[0]?.["reasoning_details"]).toEqual([ - { type: "reasoning.text", text: "1. Analyze the request", id: "rs_1" }, - { type: "reasoning.encrypted", data: "gAAAAAopaque", id: "rs_2" }, - ]); + expect(parsed[0]).not.toHaveProperty("reasoning_details"); }); it("serializes multimodal content parts (image_url) in the messages JSON", async () => { const messages: readonly ModelMessage[] = [ diff --git a/src/results/parquet.ts b/src/results/parquet.ts index 5b74d18..661415d 100644 --- a/src/results/parquet.ts +++ b/src/results/parquet.ts @@ -289,9 +289,6 @@ function messageToPojo(msg: ModelMessage): Record { if (msg.reasoning !== undefined) { pojo["reasoning"] = msg.reasoning; } - if (msg.reasoningDetails !== undefined && msg.reasoningDetails.length > 0) { - pojo["reasoning_details"] = msg.reasoningDetails; - } if (msg.citations !== undefined && msg.citations.length > 0) { pojo["citations"] = msg.citations.map((c) => ({ url: c.url, From b13802278593407397eae502c73a7d14a1d0750a Mon Sep 17 00:00:00 2001 From: Alan Carroll Date: Fri, 4 Sep 2026 20:43:29 +0000 Subject: [PATCH 3/4] refactor(providers): move reasoning extraction beside the other output-item extractors Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- src/providers/messages-to-responses.ts | 4 +- src/providers/responses-client.test.ts | 171 +++++++++++++++++++++ src/providers/responses-client.ts | 91 +++++++++++ src/providers/responses-reasoning.test.ts | 174 ---------------------- src/providers/responses-reasoning.ts | 105 ------------- 5 files changed, 264 insertions(+), 281 deletions(-) delete mode 100644 src/providers/responses-reasoning.test.ts delete mode 100644 src/providers/responses-reasoning.ts diff --git a/src/providers/messages-to-responses.ts b/src/providers/messages-to-responses.ts index ca8d94f..64ef412 100644 --- a/src/providers/messages-to-responses.ts +++ b/src/providers/messages-to-responses.ts @@ -6,12 +6,12 @@ import type { } from "../harness/core"; import { MessageRole } from "../harness/core"; import { definedValues } from "../internal/guards"; +import { extractReasoning } from "./responses-client"; import type { ResponsesFunctionTool, ResponsesInputItem, ResponsesTurn, } from "./responses-model"; -import { reasoningFromOutputItems } from "./responses-reasoning"; export function messagesToResponses( messages: readonly ModelMessage[] @@ -112,7 +112,7 @@ export function toolDefinitionToResponses( } export function responsesTurnToModelOutput(turn: ResponsesTurn): ModelOutput { - const reasoning = reasoningFromOutputItems(turn.outputItems); + const reasoning = extractReasoning(turn.outputItems); return definedValues({ completion: turn.text, message: { diff --git a/src/providers/responses-client.test.ts b/src/providers/responses-client.test.ts index 10fba65..faa103b 100644 --- a/src/providers/responses-client.test.ts +++ b/src/providers/responses-client.test.ts @@ -28,6 +28,7 @@ import type { ModelErrorIdentifiers } from "./request-identifiers"; import { consumeStream, extractMessageText, + extractReasoning, findOutputItems, makeResponsesLayer, Responses, @@ -115,6 +116,176 @@ describe("extractMessageText", () => { expect(extractMessageText([])).toBe(""); }); }); +describe("extractReasoning", () => { + it("normalizes plaintext reasoning content into readable reasoning and text details", () => { + expect( + extractReasoning([ + { + type: "reasoning", + id: "rs_1", + content: [ + { type: "reasoning_text", text: "1. Analyze the request" }, + { type: "reasoning_text", text: "2. Answer" }, + ], + summary: [], + }, + { type: "message", content: [{ type: "output_text", text: "B" }] }, + ]) + ).toEqual({ + reasoning: "1. Analyze the request\n\n2. Answer", + reasoningDetails: [ + { + type: "reasoning.text", + text: "1. Analyze the request", + id: "rs_1", + }, + { type: "reasoning.text", text: "2. Answer", id: "rs_1" }, + ], + }); + }); + + it("carries the signature and format of a plaintext item", () => { + expect( + extractReasoning([ + { + type: "reasoning", + id: "rs_1", + format: "anthropic-claude-v1", + signature: "sig", + content: [{ type: "reasoning_text", text: "thought" }], + }, + ]) + ).toEqual({ + reasoning: "thought", + reasoningDetails: [ + { + type: "reasoning.text", + text: "thought", + id: "rs_1", + format: "anthropic-claude-v1", + signature: "sig", + }, + ], + }); + }); + + it("keeps an encrypted blob as a detail and emits no readable reasoning", () => { + expect( + extractReasoning([ + { + type: "reasoning", + id: "rs_2", + encrypted_content: "gAAAAAopaque", + summary: [], + }, + ]) + ).toEqual({ + reasoningDetails: [ + { type: "reasoning.encrypted", data: "gAAAAAopaque", id: "rs_2" }, + ], + }); + }); + + it("falls back to provider summaries when no plaintext is exposed", () => { + expect( + extractReasoning([ + { + type: "reasoning", + id: "rs_3", + summary: [ + { type: "summary_text", text: "Considered two options" }, + { type: "summary_text", text: "Picked the second" }, + ], + encrypted_content: "blob", + }, + ]) + ).toEqual({ + reasoning: "Considered two options\n\nPicked the second", + reasoningDetails: [ + { + type: "reasoning.summary", + summary: "Considered two options", + id: "rs_3", + }, + { + type: "reasoning.summary", + summary: "Picked the second", + id: "rs_3", + }, + { type: "reasoning.encrypted", data: "blob", id: "rs_3" }, + ], + }); + }); + + it("prefers plaintext over summaries for readable reasoning while keeping both details", () => { + const result = extractReasoning([ + { + type: "reasoning", + content: [{ type: "reasoning_text", text: "raw thought" }], + summary: [{ type: "summary_text", text: "short summary" }], + }, + ]); + expect(result.reasoning).toBe("raw thought"); + expect(result.reasoningDetails).toEqual([ + { type: "reasoning.text", text: "raw thought" }, + { type: "reasoning.summary", summary: "short summary" }, + ]); + }); + + it("joins plaintext across multiple reasoning items in wire order", () => { + expect( + extractReasoning([ + { + type: "reasoning", + content: [{ type: "reasoning_text", text: "first" }], + }, + { type: "function_call", call_id: "c1", name: "t", arguments: "{}" }, + { + type: "reasoning", + content: [{ type: "reasoning_text", text: "second" }], + }, + ]).reasoning + ).toBe("first\n\nsecond"); + }); + + it("returns nothing when no reasoning item is present", () => { + expect( + extractReasoning([ + { type: "message", content: [{ type: "output_text", text: "B" }] }, + ]) + ).toEqual({}); + }); + + it("returns nothing for a reasoning item carrying no reasoning at all", () => { + expect( + extractReasoning([ + { type: "reasoning", id: "rs_4", summary: [], content: [] }, + ]) + ).toEqual({}); + }); + + it("skips malformed parts rather than repairing them", () => { + expect( + extractReasoning([ + { + type: "reasoning", + content: [ + "not-an-object", + { type: "reasoning_text" }, + { type: "reasoning_text", text: "" }, + { type: "reasoning_text", text: 42 }, + { type: "reasoning_text", text: "kept" }, + ], + summary: "not-an-array", + encrypted_content: 7, + }, + ]) + ).toEqual({ + reasoning: "kept", + reasoningDetails: [{ type: "reasoning.text", text: "kept" }], + }); + }); +}); describe("findOutputItems", () => { it("returns all items matching the type", () => { const output = [ diff --git a/src/providers/responses-client.ts b/src/providers/responses-client.ts index 248a6e2..e08d89f 100644 --- a/src/providers/responses-client.ts +++ b/src/providers/responses-client.ts @@ -24,6 +24,7 @@ import { succeed as layerSucceed } from "effect/Layer"; import type { Citation, ModelUsage } from "../harness/core"; import { ModelError } from "../harness/core"; +import type { ReasoningDetails } from "../harness/reasoning-details"; import { Either } from "../internal/either"; import { definedValues, isRecord } from "../internal/guards"; import { parseSchema, z } from "../internal/zod"; @@ -70,6 +71,19 @@ export const ResponsesResultSchema = z.object({ export type ResponsesResult = z.infer; +export interface ResponsesReasoning { + readonly reasoning?: string; + readonly reasoningDetails?: ReasoningDetails; +} + +const ReasoningDetailType = { + Summary: "reasoning.summary", + Encrypted: "reasoning.encrypted", + Text: "reasoning.text", +} as const; + +const REASONING_JOINER = "\n\n"; + const RawResponsesTerminalEventSchema = z.object({ type: z.union([ z.literal("response.completed"), @@ -576,6 +590,83 @@ export function extractMessageText( return text; } +export function extractReasoning( + output: readonly Record[] +): ResponsesReasoning { + const details: unknown[] = []; + const texts: string[] = []; + const summaries: string[] = []; + for (const item of output) { + if (item["type"] !== "reasoning") { + continue; + } + const id = stringField(item, "id"); + const format = stringField(item, "format"); + const signature = stringField(item, "signature"); + for (const text of partTexts(item["content"], "reasoning_text")) { + texts.push(text); + details.push( + definedValues({ + type: ReasoningDetailType.Text, + text, + id, + format, + signature, + }) + ); + } + for (const summary of partTexts(item["summary"], "summary_text")) { + summaries.push(summary); + details.push( + definedValues({ + type: ReasoningDetailType.Summary, + summary, + id, + format, + }) + ); + } + const encrypted = stringField(item, "encrypted_content"); + if (encrypted !== undefined) { + details.push( + definedValues({ + type: ReasoningDetailType.Encrypted, + data: encrypted, + id, + format, + }) + ); + } + } + const readable = texts.length > 0 ? texts : summaries; + return definedValues({ + reasoning: + readable.length > 0 ? readable.join(REASONING_JOINER) : undefined, + reasoningDetails: details.length > 0 ? details : undefined, + }); +} + +function partTexts(parts: unknown, partType: string): string[] { + if (!Array.isArray(parts)) { + return []; + } + return parts.flatMap((part) => { + if (!isRecord(part) || part["type"] !== partType) { + return []; + } + const text = stringField(part, "text"); + return text !== undefined ? [text] : []; + }); +} + +function stringField( + record: Readonly>, + key: string +): string | undefined { + const value = record[key]; + return typeof value === "string" && value.length > 0 ? value : undefined; +} + export function findOutputItems( output: readonly Record[], itemType: string diff --git a/src/providers/responses-reasoning.test.ts b/src/providers/responses-reasoning.test.ts deleted file mode 100644 index 05db839..0000000 --- a/src/providers/responses-reasoning.test.ts +++ /dev/null @@ -1,174 +0,0 @@ -import { describe, expect, it } from "bun:test"; - -import { reasoningFromOutputItems } from "./responses-reasoning"; - -describe("reasoningFromOutputItems", () => { - it("normalizes plaintext reasoning content into readable reasoning and text details", () => { - expect( - reasoningFromOutputItems([ - { - type: "reasoning", - id: "rs_1", - content: [ - { type: "reasoning_text", text: "1. Analyze the request" }, - { type: "reasoning_text", text: "2. Answer" }, - ], - summary: [], - }, - { type: "message", content: [{ type: "output_text", text: "B" }] }, - ]) - ).toEqual({ - reasoning: "1. Analyze the request\n\n2. Answer", - reasoningDetails: [ - { - type: "reasoning.text", - text: "1. Analyze the request", - id: "rs_1", - }, - { type: "reasoning.text", text: "2. Answer", id: "rs_1" }, - ], - }); - }); - - it("carries the signature and format of a plaintext item", () => { - expect( - reasoningFromOutputItems([ - { - type: "reasoning", - id: "rs_1", - format: "anthropic-claude-v1", - signature: "sig", - content: [{ type: "reasoning_text", text: "thought" }], - }, - ]) - ).toEqual({ - reasoning: "thought", - reasoningDetails: [ - { - type: "reasoning.text", - text: "thought", - id: "rs_1", - format: "anthropic-claude-v1", - signature: "sig", - }, - ], - }); - }); - - it("keeps an encrypted blob as a detail and emits no readable reasoning", () => { - expect( - reasoningFromOutputItems([ - { - type: "reasoning", - id: "rs_2", - encrypted_content: "gAAAAAopaque", - summary: [], - }, - ]) - ).toEqual({ - reasoningDetails: [ - { type: "reasoning.encrypted", data: "gAAAAAopaque", id: "rs_2" }, - ], - }); - }); - - it("falls back to provider summaries when no plaintext is exposed", () => { - expect( - reasoningFromOutputItems([ - { - type: "reasoning", - id: "rs_3", - summary: [ - { type: "summary_text", text: "Considered two options" }, - { type: "summary_text", text: "Picked the second" }, - ], - encrypted_content: "blob", - }, - ]) - ).toEqual({ - reasoning: "Considered two options\n\nPicked the second", - reasoningDetails: [ - { - type: "reasoning.summary", - summary: "Considered two options", - id: "rs_3", - }, - { - type: "reasoning.summary", - summary: "Picked the second", - id: "rs_3", - }, - { type: "reasoning.encrypted", data: "blob", id: "rs_3" }, - ], - }); - }); - - it("prefers plaintext over summaries for readable reasoning while keeping both details", () => { - const result = reasoningFromOutputItems([ - { - type: "reasoning", - content: [{ type: "reasoning_text", text: "raw thought" }], - summary: [{ type: "summary_text", text: "short summary" }], - }, - ]); - expect(result.reasoning).toBe("raw thought"); - expect(result.reasoningDetails).toEqual([ - { type: "reasoning.text", text: "raw thought" }, - { type: "reasoning.summary", summary: "short summary" }, - ]); - }); - - it("joins plaintext across multiple reasoning items in wire order", () => { - expect( - reasoningFromOutputItems([ - { - type: "reasoning", - content: [{ type: "reasoning_text", text: "first" }], - }, - { type: "function_call", call_id: "c1", name: "t", arguments: "{}" }, - { - type: "reasoning", - content: [{ type: "reasoning_text", text: "second" }], - }, - ]).reasoning - ).toBe("first\n\nsecond"); - }); - - it("returns nothing when no reasoning item is present", () => { - expect( - reasoningFromOutputItems([ - { type: "message", content: [{ type: "output_text", text: "B" }] }, - ]) - ).toEqual({}); - }); - - it("returns nothing for a reasoning item carrying no reasoning at all", () => { - expect( - reasoningFromOutputItems([ - { type: "reasoning", id: "rs_4", summary: [], content: [] }, - ]) - ).toEqual({}); - }); - - it("skips malformed parts rather than repairing them", () => { - expect( - reasoningFromOutputItems([ - { - type: "reasoning", - content: [ - "not-an-object", - { type: "reasoning_text" }, - { type: "reasoning_text", text: "" }, - { type: "reasoning_text", text: 42 }, - { type: "reasoning_text", text: "kept" }, - ], - summary: "not-an-array", - encrypted_content: 7, - }, - ]) - ).toEqual({ - reasoning: "kept", - reasoningDetails: [{ type: "reasoning.text", text: "kept" }], - }); - }); -}); diff --git a/src/providers/responses-reasoning.ts b/src/providers/responses-reasoning.ts deleted file mode 100644 index 9de2db8..0000000 --- a/src/providers/responses-reasoning.ts +++ /dev/null @@ -1,105 +0,0 @@ -import type { ReasoningDetails } from "../harness/reasoning-details"; -import { definedValues, isRecord } from "../internal/guards"; - -export interface ResponsesReasoning { - readonly reasoning?: string; - readonly reasoningDetails?: ReasoningDetails; -} - -const ReasoningDetailType = { - Summary: "reasoning.summary", - Encrypted: "reasoning.encrypted", - Text: "reasoning.text", -} as const; - -const REASONING_JOINER = "\n\n"; - -export function reasoningFromOutputItems( - outputItems: readonly Record[] -): ResponsesReasoning { - const details: unknown[] = []; - const texts: string[] = []; - const summaries: string[] = []; - - for (const item of outputItems) { - if (item["type"] !== "reasoning") { - continue; - } - const id = stringField(item, "id"); - const format = stringField(item, "format"); - const signature = stringField(item, "signature"); - - for (const text of reasoningTexts(item)) { - texts.push(text); - details.push( - definedValues({ - type: ReasoningDetailType.Text, - text, - id, - format, - signature, - }) - ); - } - - for (const summary of summaryTexts(item)) { - summaries.push(summary); - details.push( - definedValues({ - type: ReasoningDetailType.Summary, - summary, - id, - format, - }) - ); - } - - const encrypted = stringField(item, "encrypted_content"); - if (encrypted !== undefined) { - details.push( - definedValues({ - type: ReasoningDetailType.Encrypted, - data: encrypted, - id, - format, - }) - ); - } - } - - const readable = texts.length > 0 ? texts : summaries; - return definedValues({ - reasoning: - readable.length > 0 ? readable.join(REASONING_JOINER) : undefined, - reasoningDetails: details.length > 0 ? details : undefined, - }); -} - -function reasoningTexts(item: Record): string[] { - return partTexts(item["content"], "text"); -} - -function summaryTexts(item: Record): string[] { - return partTexts(item["summary"], "text"); -} - -function partTexts(parts: unknown, key: string): string[] { - if (!Array.isArray(parts)) { - return []; - } - return parts.flatMap((part) => { - if (!isRecord(part)) { - return []; - } - const text = stringField(part, key); - return text !== undefined ? [text] : []; - }); -} - -function stringField( - record: Record, - key: string -): string | undefined { - const value = record[key]; - return typeof value === "string" && value.length > 0 ? value : undefined; -} From 8fb2c996c3c3489cca8f881b79160f04b6f83501 Mon Sep 17 00:00:00 2001 From: Alan Carroll Date: Fri, 4 Sep 2026 21:12:03 +0000 Subject: [PATCH 4/4] refactor(providers): inline reasoning detail literals in extractReasoning Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- src/providers/responses-client.ts | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/src/providers/responses-client.ts b/src/providers/responses-client.ts index e08d89f..db83b5e 100644 --- a/src/providers/responses-client.ts +++ b/src/providers/responses-client.ts @@ -76,14 +76,6 @@ export interface ResponsesReasoning { readonly reasoningDetails?: ReasoningDetails; } -const ReasoningDetailType = { - Summary: "reasoning.summary", - Encrypted: "reasoning.encrypted", - Text: "reasoning.text", -} as const; - -const REASONING_JOINER = "\n\n"; - const RawResponsesTerminalEventSchema = z.object({ type: z.union([ z.literal("response.completed"), @@ -607,7 +599,7 @@ export function extractReasoning( texts.push(text); details.push( definedValues({ - type: ReasoningDetailType.Text, + type: "reasoning.text", text, id, format, @@ -619,7 +611,7 @@ export function extractReasoning( summaries.push(summary); details.push( definedValues({ - type: ReasoningDetailType.Summary, + type: "reasoning.summary", summary, id, format, @@ -630,7 +622,7 @@ export function extractReasoning( if (encrypted !== undefined) { details.push( definedValues({ - type: ReasoningDetailType.Encrypted, + type: "reasoning.encrypted", data: encrypted, id, format, @@ -640,8 +632,7 @@ export function extractReasoning( } const readable = texts.length > 0 ? texts : summaries; return definedValues({ - reasoning: - readable.length > 0 ? readable.join(REASONING_JOINER) : undefined, + reasoning: readable.length > 0 ? readable.join("\n\n") : undefined, reasoningDetails: details.length > 0 ? details : undefined, }); }