Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 55 additions & 0 deletions src/providers/messages-to-responses.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -146,4 +146,59 @@ describe("messages-to-responses", () => {
},
]);
});

it("normalizes a plaintext reasoning item onto the assistant message", () => {
const output = responsesTurnToModelOutput({
text: "Answer: B",
outputItems: [
{
type: "reasoning",
id: "rs_1",
content: [{ type: "reasoning_text", text: "1. Analyze" }],
summary: [],
},
{
type: "message",
content: [{ type: "output_text", text: "Answer: B" }],
},
],
functionCalls: [],
generationTimeMs: 7,
});
expect(output.message.reasoning).toBe("1. Analyze");
expect(output.message.reasoningDetails).toEqual([
{ type: "reasoning.text", text: "1. Analyze", id: "rs_1" },
]);
});

it("keeps an encrypted reasoning item as details without inventing readable text", () => {
const output = responsesTurnToModelOutput({
text: "Answer: B",
outputItems: [
{ type: "reasoning", id: "rs_1", encrypted_content: "opaque" },
],
functionCalls: [],
generationTimeMs: 7,
});
expect(output.message).not.toHaveProperty("reasoning");
expect(output.message.reasoningDetails).toEqual([
{ type: "reasoning.encrypted", data: "opaque", id: "rs_1" },
]);
});

it("leaves both reasoning fields absent when the turn carries no reasoning", () => {
const output = responsesTurnToModelOutput({
text: "Answer: B",
outputItems: [
{
type: "message",
content: [{ type: "output_text", text: "Answer: B" }],
},
],
functionCalls: [],
generationTimeMs: 7,
});
expect(output.message).not.toHaveProperty("reasoning");
expect(output.message).not.toHaveProperty("reasoningDetails");
});
});
3 changes: 3 additions & 0 deletions src/providers/messages-to-responses.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import type {
} from "../harness/core";
import { MessageRole } from "../harness/core";
import { definedValues } from "../internal/guards";
import { extractReasoning } from "./responses-client";
import type {
ResponsesFunctionTool,
ResponsesInputItem,
Expand Down Expand Up @@ -111,11 +112,13 @@ export function toolDefinitionToResponses(
}

export function responsesTurnToModelOutput(turn: ResponsesTurn): ModelOutput {
const reasoning = extractReasoning(turn.outputItems);
return definedValues({
completion: turn.text,
message: {
role: MessageRole.Assistant,
content: turn.text,
...reasoning,
...definedValues({
toolCalls:
turn.functionCalls.length > 0
Expand Down
171 changes: 171 additions & 0 deletions src/providers/responses-client.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import type { ModelErrorIdentifiers } from "./request-identifiers";
import {
consumeStream,
extractMessageText,
extractReasoning,
findOutputItems,
makeResponsesLayer,
Responses,
Expand Down Expand Up @@ -115,6 +116,176 @@ describe("extractMessageText", () => {
expect(extractMessageText([])).toBe("");
});
});
describe("extractReasoning", () => {
it("normalizes plaintext reasoning content into readable reasoning and text details", () => {
expect(
extractReasoning([
{
type: "reasoning",
id: "rs_1",
content: [
{ type: "reasoning_text", text: "1. Analyze the request" },
{ type: "reasoning_text", text: "2. Answer" },
],
summary: [],
},
{ type: "message", content: [{ type: "output_text", text: "B" }] },
])
).toEqual({
reasoning: "1. Analyze the request\n\n2. Answer",
reasoningDetails: [
{
type: "reasoning.text",
text: "1. Analyze the request",
id: "rs_1",
},
{ type: "reasoning.text", text: "2. Answer", id: "rs_1" },
],
});
});

it("carries the signature and format of a plaintext item", () => {
expect(
extractReasoning([
{
type: "reasoning",
id: "rs_1",
format: "anthropic-claude-v1",
signature: "sig",
content: [{ type: "reasoning_text", text: "thought" }],
},
])
).toEqual({
reasoning: "thought",
reasoningDetails: [
{
type: "reasoning.text",
text: "thought",
id: "rs_1",
format: "anthropic-claude-v1",
signature: "sig",
},
],
});
});

it("keeps an encrypted blob as a detail and emits no readable reasoning", () => {
expect(
extractReasoning([
{
type: "reasoning",
id: "rs_2",
encrypted_content: "gAAAAAopaque",
summary: [],
},
])
).toEqual({
reasoningDetails: [
{ type: "reasoning.encrypted", data: "gAAAAAopaque", id: "rs_2" },
],
});
});

it("falls back to provider summaries when no plaintext is exposed", () => {
expect(
extractReasoning([
{
type: "reasoning",
id: "rs_3",
summary: [
{ type: "summary_text", text: "Considered two options" },
{ type: "summary_text", text: "Picked the second" },
],
encrypted_content: "blob",
},
])
).toEqual({
reasoning: "Considered two options\n\nPicked the second",
reasoningDetails: [
{
type: "reasoning.summary",
summary: "Considered two options",
id: "rs_3",
},
{
type: "reasoning.summary",
summary: "Picked the second",
id: "rs_3",
},
{ type: "reasoning.encrypted", data: "blob", id: "rs_3" },
],
});
});

it("prefers plaintext over summaries for readable reasoning while keeping both details", () => {
const result = extractReasoning([
{
type: "reasoning",
content: [{ type: "reasoning_text", text: "raw thought" }],
summary: [{ type: "summary_text", text: "short summary" }],
},
]);
expect(result.reasoning).toBe("raw thought");
expect(result.reasoningDetails).toEqual([
{ type: "reasoning.text", text: "raw thought" },
{ type: "reasoning.summary", summary: "short summary" },
]);
});

it("joins plaintext across multiple reasoning items in wire order", () => {
expect(
extractReasoning([
{
type: "reasoning",
content: [{ type: "reasoning_text", text: "first" }],
},
{ type: "function_call", call_id: "c1", name: "t", arguments: "{}" },
{
type: "reasoning",
content: [{ type: "reasoning_text", text: "second" }],
},
]).reasoning
).toBe("first\n\nsecond");
});

it("returns nothing when no reasoning item is present", () => {
expect(
extractReasoning([
{ type: "message", content: [{ type: "output_text", text: "B" }] },
])
).toEqual({});
});

it("returns nothing for a reasoning item carrying no reasoning at all", () => {
expect(
extractReasoning([
{ type: "reasoning", id: "rs_4", summary: [], content: [] },
])
).toEqual({});
});

it("skips malformed parts rather than repairing them", () => {
expect(
extractReasoning([
{
type: "reasoning",
content: [
"not-an-object",
{ type: "reasoning_text" },
{ type: "reasoning_text", text: "" },
{ type: "reasoning_text", text: 42 },
{ type: "reasoning_text", text: "kept" },
],
summary: "not-an-array",
encrypted_content: 7,
},
])
).toEqual({
reasoning: "kept",
reasoningDetails: [{ type: "reasoning.text", text: "kept" }],
});
});
});
describe("findOutputItems", () => {
it("returns all items matching the type", () => {
const output = [
Expand Down
82 changes: 82 additions & 0 deletions src/providers/responses-client.ts
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import { succeed as layerSucceed } from "effect/Layer";

import type { Citation, ModelUsage } from "../harness/core";
import { ModelError } from "../harness/core";
import type { ReasoningDetails } from "../harness/reasoning-details";
import { Either } from "../internal/either";
import { definedValues, isRecord } from "../internal/guards";
import { parseSchema, z } from "../internal/zod";
Expand Down Expand Up @@ -70,6 +71,11 @@ export const ResponsesResultSchema = z.object({

export type ResponsesResult = z.infer<typeof ResponsesResultSchema>;

export interface ResponsesReasoning {
readonly reasoning?: string;
readonly reasoningDetails?: ReasoningDetails;
}

const RawResponsesTerminalEventSchema = z.object({
type: z.union([
z.literal("response.completed"),
Expand Down Expand Up @@ -576,6 +582,82 @@ export function extractMessageText(
return text;
}

export function extractReasoning(
output: readonly Record<string, unknown>[]
): ResponsesReasoning {
const details: unknown[] = [];
const texts: string[] = [];
const summaries: string[] = [];
for (const item of output) {
if (item["type"] !== "reasoning") {
continue;
}
const id = stringField(item, "id");
const format = stringField(item, "format");
const signature = stringField(item, "signature");
for (const text of partTexts(item["content"], "reasoning_text")) {
texts.push(text);
details.push(
definedValues({
type: "reasoning.text",
text,
id,
format,
signature,
})
);
}
for (const summary of partTexts(item["summary"], "summary_text")) {
summaries.push(summary);
details.push(
definedValues({
type: "reasoning.summary",
summary,
id,
format,
})
);
}
const encrypted = stringField(item, "encrypted_content");
if (encrypted !== undefined) {
details.push(
definedValues({
type: "reasoning.encrypted",
data: encrypted,
id,
format,
})
);
}
}
const readable = texts.length > 0 ? texts : summaries;
return definedValues({
reasoning: readable.length > 0 ? readable.join("\n\n") : undefined,
reasoningDetails: details.length > 0 ? details : undefined,
});
}

function partTexts(parts: unknown, partType: string): string[] {
if (!Array.isArray(parts)) {
return [];
}
return parts.flatMap((part) => {
if (!isRecord(part) || part["type"] !== partType) {
return [];
}
const text = stringField(part, "text");
return text !== undefined ? [text] : [];
});
}

function stringField(
record: Readonly<Record<string, unknown>>,
key: string
): string | undefined {
const value = record[key];
return typeof value === "string" && value.length > 0 ? value : undefined;
}

export function findOutputItems(
output: readonly Record<string, unknown>[],
itemType: string
Expand Down
Loading
Loading