diff --git a/scripts/release-gate/summary-recovery.mjs b/scripts/release-gate/summary-recovery.mjs index fa7908f9b6..daa7fc059f 100644 --- a/scripts/release-gate/summary-recovery.mjs +++ b/scripts/release-gate/summary-recovery.mjs @@ -41,6 +41,7 @@ const sha = (file) => createHash("sha256").update(readFileSync(file)).digest("he let port, modelPort, modelServer, secret, installed, packageBefore, boot = 0; let heldCompression = null, failSummary = false, phase = "setup"; const markers = { BASE: "RECOVERED_BASE_COBALT", FINAL: "RECOVERED_FINAL_JUNIPER", LATE: "RECOVERED_LATE_QUARTZ" }; +const truncation = { session: "truncated-reasoning-session", accepted: "TRUNCATION_ACCEPTED_SAGE", marker: "TRUNCATED_REASONING_ONYX" }; function log(message) { const line = `[${((Date.now() - began) / 1000).toFixed(1)}s] ${message}`; console.log(line); appendFileSync(join(out, "gate.log"), `${line}\n`); @@ -155,7 +156,19 @@ async function startModel() { const item = { phase, boot, operation: compress ? "compression" : "summary", prompt }; modelRequests.push(item); let content; - if (compress) { + const truncatedCompression = compress && prompt.includes("GATE_INPUT_TRUNCATED"); + if (truncatedCompression || (summarize && phase === "truncated-reasoning-is-rejected")) { + const partial = compress + ? `Draft: decision${truncation.marker}${truncation.marker}The draft names ${truncation.marker} before the budget ran out.truncationfixture.txt8` + : `Draft: ${truncation.marker}The draft names ${truncation.marker} before the budget ran out.${truncation.marker}fixture.txttruncation`; + item.outcome = "truncated-reasoning"; item.response = partial; + response.writeHead(200, { "content-type": "application/json" }); + response.end(JSON.stringify({ choices: [{ finish_reason: "length", message: { role: "assistant", content: null, reasoning: partial } }] })); + return; + } + if (compress && prompt.includes("GATE_INPUT_ACCEPTED")) { + content = `decision${truncation.accepted}The verified result is ${truncation.accepted}.The local fixture completed processing and verified ${truncation.accepted}.truncationfixture.txt8`; + } else if (compress) { const token = prompt.match(/GATE_INPUT_(BASE|FINAL|LATE)\b/)?.[1]; assert(token, "Compression request lost the retained raw fixture"); item.token = token; @@ -232,14 +245,14 @@ async function crashDaemon() { await poll("crashed daemon ports closed", async () => (await Promise.all(daemonPorts().map(portFree))).every(Boolean), 15_000); modelServer.closeAllConnections(); ownedPids.clear(); log(`Native CLI boot ${boot} terminated`); } -function event(token) { - return { hookType: "post_tool_use", sessionId, project, cwd: root, timestamp: new Date().toISOString(), eventId: `summary-gate-${token.toLowerCase()}`, data: { tool_name: "Bash", tool_input: { command: `printf GATE_INPUT_${token}` }, tool_response: `GATE_INPUT_${token}: tool completed` } }; +function event(token, session = sessionId) { + return { hookType: "post_tool_use", sessionId: session, project, cwd: root, timestamp: new Date().toISOString(), eventId: `summary-gate-${token.toLowerCase()}`, data: { tool_name: "Bash", tool_input: { command: `printf GATE_INPUT_${token}` }, tool_response: `GATE_INPUT_${token}: tool completed` } }; } async function snapshot(name) { const data = await rest("/export", undefined, { record: false }); writeFileSync(join(out, `${name}.json`), JSON.stringify(data, null, 2) + "\n"); return data; } -const observations = (data) => data.observations?.[sessionId] ?? []; +const observations = (data, session = sessionId) => data.observations?.[session] ?? []; const summaryFor = (data) => data.summaries?.find((summary) => summary.sessionId === sessionId); async function waitSummary(expected) { return poll(`summary contains ${expected.join(", ")}`, async () => { @@ -305,6 +318,27 @@ async function main() { assert(!wire.some((entry) => /retry|repair/.test(entry.path)), "Recovery must use the autonomous runtime path"); assert(!modelRequests.some((entry) => entry.error), "The fixture rejected an unexpected model request"); }); + await scenario("truncated-reasoning-is-rejected", async () => { + const session = truncation.session; + const exported = () => rest("/export", undefined, { record: false }); + await rest("/session/start", { sessionId: session, project, cwd: root }); + await rest("/observe", event("ACCEPTED", session)); + await poll("accepted compression", async () => observations(await exported(), session).some((row) => row.title === truncation.accepted && row.compression?.status === "complete")); + const summary = await rest("/summarize", { sessionId: session }); + assert.equal(summary.success, false, "A summary reply cut at max_tokens must not succeed"); + assert.match(summary.error, /max_tokens/); + await rest("/observe", event("TRUNCATED", session)); + const failed = await poll("truncated compression rejected", async () => { + const row = observations(await exported(), session).find((item) => JSON.stringify(item.source ?? "").includes("GATE_INPUT_TRUNCATED")); + return row?.compression?.status === "failed" ? row : false; + }); + assert.equal(failed.compression.error, "compression_failed"); + const truncatedCalls = modelRequests.filter((request) => request.phase === phase && request.outcome === "truncated-reasoning"); + assert(truncatedCalls.some((request) => request.operation === "summary"), "The summary request must reach the truncating fixture"); + assert(truncatedCalls.some((request) => request.operation === "compression"), "The compression request must reach the truncating fixture"); + const after = await snapshot("after-truncated-reasoning"); + assert(!JSON.stringify(after).includes(truncation.marker), "Truncated reasoning must never be stored as model output"); + }); await scenario("installed-package-unchanged", async () => { assert.deepEqual(packageManifest(), packageBefore); assert.equal(sha(artifact.tarball), artifact.tarballSha256); }); diff --git a/src/providers/openai.ts b/src/providers/openai.ts index 6e2e9c2c6c..929dc4c9c8 100644 --- a/src/providers/openai.ts +++ b/src/providers/openai.ts @@ -108,14 +108,21 @@ export class OpenAIProvider implements MemoryProvider { const data = (await response.json()) as { choices?: Array<{ - message?: { content?: string; reasoning?: string; reasoning_content?: string }; + finish_reason?: string; + message?: { content?: string | null; reasoning?: string; reasoning_content?: string }; }>; }; - const message = data.choices?.[0]?.message; + const choice = data.choices?.[0]; + const message = choice?.message; const content = message?.content; - if (content) { + if (content?.trim()) { return content; } + if (choice?.finish_reason === "length") { + throw new Error( + `OpenAI response hit max_tokens (${this.maxTokens}) before any content. Raise MAX_TOKENS or lower OPENAI_REASONING_EFFORT.`, + ); + } const reasoning = message?.reasoning ?? message?.reasoning_content; if (reasoning) { return reasoning; diff --git a/test/fetch-timeout.test.ts b/test/fetch-timeout.test.ts index 0019a7708e..48f5a73eef 100644 --- a/test/fetch-timeout.test.ts +++ b/test/fetch-timeout.test.ts @@ -552,6 +552,39 @@ describe("OpenAIProvider thinking-model fallback", () => { expect(out).toBe("older shape"); }); + const emptyReasoningReplies = [null, "", " \n "].flatMap((content) => + (["reasoning", "reasoning_content"] as const).map((field) => ({ content, field })), + ); + + it.each( + emptyReasoningReplies.flatMap((reply) => (["compress", "summarize"] as const).map((method) => ({ ...reply, method }))), + )("$method rejects a reply cut at max_tokens with content $content and only $field", async ({ content, field, method }) => { + mockOpenAIResponse({ + choices: [{ finish_reason: "length", message: { content, [field]: "The user is asking me to reply with" } }], + }); + const provider = new OpenAIProvider("test-key", "gpt-5.6-luna", 16); + await expect(provider[method]("system", "user")).rejects.toThrow(/max_tokens \(16\)/); + }); + + it.each(emptyReasoningReplies)( + "keeps the $field fallback when the reply finished normally with content $content", + async ({ content, field }) => { + mockOpenAIResponse({ + choices: [{ finish_reason: "stop", message: { content, [field]: "complete answer" } }], + }); + const provider = new OpenAIProvider("test-key", "gpt-5.6-luna", 1024); + expect(await provider.summarize("system", "user")).toBe("complete answer"); + }, + ); + + it("returns nonempty content even when the reply hit max_tokens", async () => { + mockOpenAIResponse({ + choices: [{ finish_reason: "length", message: { content: "partial but real", reasoning: "ignore" } }], + }); + const provider = new OpenAIProvider("test-key", "gpt-5.6-luna", 16); + expect(await provider.compress("system", "user")).toBe("partial but real"); + }); + it("content wins over both reasoning fields when present", async () => { mockOpenAIResponse({ choices: [