From d08d63ddd4142a29c227e5a7f137e6a05552ad3e Mon Sep 17 00:00:00 2001 From: Rex57 <203883624+Rex57@users.noreply.github.com> Date: Thu, 8 Oct 2026 14:38:53 +0800 Subject: [PATCH 1/4] fix(openai): fail on a reply cut at max_tokens instead of returning partial reasoning When a reasoning model spends the whole max_tokens budget on reasoning, an OpenAI-compatible API answers 200 with finish_reason "length", no content and a partial reasoning field. The reasoning fallback returned that fragment as a successful result, so compression stored chain-of-thought as the summary. Throw a clear error for that case and keep the fallback for replies that finished normally. Signed-off-by: Rex57 <203883624+Rex57@users.noreply.github.com> --- src/providers/openai.ts | 11 ++++++++++- test/fetch-timeout.test.ts | 22 ++++++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/src/providers/openai.ts b/src/providers/openai.ts index 46500cf87b..847f103456 100644 --- a/src/providers/openai.ts +++ b/src/providers/openai.ts @@ -118,14 +118,23 @@ export class OpenAIProvider implements MemoryProvider { const data = (await response.json()) as { choices?: Array<{ + finish_reason?: string; message?: { content?: string; reasoning?: string; reasoning_content?: string }; }>; }; - const message = data.choices?.[0]?.message; + const choice = data.choices?.[0]; + const message = choice?.message; const content = message?.content; if (content) { return content; } + // A reasoning model that hits max_tokens returns only its partial chain of + // thought; returning that would store it as the compression result. + if (choice?.finish_reason === "length") { + throw new Error( + `OpenAI response hit max_tokens (${this.maxTokens}) before any content — raise MAX_TOKENS or lower OPENAI_REASONING_EFFORT.`, + ); + } const reasoning = message?.reasoning ?? message?.reasoning_content; if (reasoning) { return reasoning; diff --git a/test/fetch-timeout.test.ts b/test/fetch-timeout.test.ts index 1240e7603b..6adecba711 100644 --- a/test/fetch-timeout.test.ts +++ b/test/fetch-timeout.test.ts @@ -547,6 +547,28 @@ describe("OpenAIProvider thinking-model fallback", () => { expect(out).toBe("older shape"); }); + it("throws instead of returning partial reasoning when the reply hit max_tokens", async () => { + mockOpenAIResponse({ + choices: [ + { + finish_reason: "length", + message: { content: null, reasoning: "The user is asking me to reply with" }, + }, + ], + }); + const provider = new OpenAIProvider("test-key", "gpt-5.6-luna", 16); + await expect(provider.compress("system", "user")).rejects.toThrow(/max_tokens \(16\)/); + }); + + it("keeps the reasoning fallback when the reply finished normally", async () => { + mockOpenAIResponse({ + choices: [{ finish_reason: "stop", message: { content: "", reasoning_content: "complete answer" } }], + }); + const provider = new OpenAIProvider("test-key", "gpt-5.6-luna", 1024); + const out = await provider.compress("system", "user"); + expect(out).toBe("complete answer"); + }); + it("content wins over both reasoning fields when present", async () => { mockOpenAIResponse({ choices: [ From 0c269811e9015d45ba2d973d9829a77958472f3e Mon Sep 17 00:00:00 2001 From: Rex57 <203883624+Rex57@users.noreply.github.com> Date: Tue, 20 Oct 2026 15:25:40 +0800 Subject: [PATCH 2/4] test(openai): cover truncated reasoning in the provider and the packaged gate Provider tests now cover a reply cut at max_tokens with null or empty content and only reasoning or reasoning_content, through both compress and summarize; replies that finished normally with reasoning only (the fallback stays); and nonempty content, which is still returned even when finish_reason is "length". The packaged summary-recovery gate gains a truncated-reasoning-is-rejected scenario. Its loopback model returns finish_reason "length", null content and partial reasoning that parses as valid XML. Against the installed archive, summarize returns success: false with the max_tokens error, the compression is stored as failed (compression_failed), and the reasoning marker appears nowhere in the export. With the provider change reverted, the scenario fails because the partial reasoning is stored as the session summary. Signed-off-by: Rex57 <203883624+Rex57@users.noreply.github.com> --- scripts/release-gate/summary-recovery.mjs | 44 ++++++++++++++++++++--- test/fetch-timeout.test.ts | 37 ++++++++++++------- 2 files changed, 64 insertions(+), 17 deletions(-) diff --git a/scripts/release-gate/summary-recovery.mjs b/scripts/release-gate/summary-recovery.mjs index 42072a1814..9e74bca4da 100644 --- a/scripts/release-gate/summary-recovery.mjs +++ b/scripts/release-gate/summary-recovery.mjs @@ -41,6 +41,8 @@ const sha = (file) => createHash("sha256").update(readFileSync(file)).digest("he let port, modelPort, modelServer, secret, installed, packageBefore, boot = 0; let heldCompression = null, failSummary = false, phase = "setup"; const markers = { BASE: "RECOVERED_BASE_COBALT", FINAL: "RECOVERED_FINAL_JUNIPER", LATE: "RECOVERED_LATE_QUARTZ" }; +// A reasoning model cut at max_tokens returns only partial reasoning; this fixture makes that reasoning parse as valid XML. +const truncation = { session: "truncated-reasoning-session", accepted: "TRUNCATION_ACCEPTED_SAGE", marker: "TRUNCATED_REASONING_ONYX" }; function log(message) { const line = `[${((Date.now() - began) / 1000).toFixed(1)}s] ${message}`; console.log(line); appendFileSync(join(out, "gate.log"), `${line}\n`); @@ -155,7 +157,19 @@ async function startModel() { const item = { phase, boot, operation: compress ? "compression" : "summary", prompt }; modelRequests.push(item); let content; - if (compress) { + const truncatedCompression = compress && prompt.includes("GATE_INPUT_TRUNCATED"); + if (truncatedCompression || (summarize && phase === "truncated-reasoning-is-rejected")) { + const partial = compress + ? `Draft: decision${truncation.marker}${truncation.marker}The draft names ${truncation.marker} before the budget ran out.truncationfixture.txt8` + : `Draft: ${truncation.marker}The draft names ${truncation.marker} before the budget ran out.${truncation.marker}fixture.txttruncation`; + item.outcome = "truncated-reasoning"; item.response = partial; + response.writeHead(200, { "content-type": "application/json" }); + response.end(JSON.stringify({ choices: [{ finish_reason: "length", message: { role: "assistant", content: null, reasoning: partial } }] })); + return; + } + if (compress && prompt.includes("GATE_INPUT_ACCEPTED")) { + content = `decision${truncation.accepted}The verified result is ${truncation.accepted}.The local fixture completed processing and verified ${truncation.accepted}.truncationfixture.txt8`; + } else if (compress) { const token = prompt.match(/GATE_INPUT_(BASE|FINAL|LATE)\b/)?.[1]; assert(token, "Compression request lost the retained raw fixture"); item.token = token; @@ -221,14 +235,14 @@ async function crashDaemon() { await poll("crashed daemon ports closed", async () => (await Promise.all(daemonPorts().map(portFree))).every(Boolean), 15_000); modelServer.closeAllConnections(); ownedPids.clear(); log(`Native CLI boot ${boot} terminated`); } -function event(token) { - return { hookType: "post_tool_use", sessionId, project, cwd: root, timestamp: new Date().toISOString(), eventId: `summary-gate-${token.toLowerCase()}`, data: { tool_name: "Bash", tool_input: { command: `printf GATE_INPUT_${token}` }, tool_response: `GATE_INPUT_${token}: tool completed` } }; +function event(token, session = sessionId) { + return { hookType: "post_tool_use", sessionId: session, project, cwd: root, timestamp: new Date().toISOString(), eventId: `summary-gate-${token.toLowerCase()}`, data: { tool_name: "Bash", tool_input: { command: `printf GATE_INPUT_${token}` }, tool_response: `GATE_INPUT_${token}: tool completed` } }; } async function snapshot(name) { const data = await rest("/export", undefined, { record: false }); writeFileSync(join(out, `${name}.json`), JSON.stringify(data, null, 2) + "\n"); return data; } -const observations = (data) => data.observations?.[sessionId] ?? []; +const observations = (data, session = sessionId) => data.observations?.[session] ?? []; const summaryFor = (data) => data.summaries?.find((summary) => summary.sessionId === sessionId); async function waitSummary(expected) { return poll(`summary contains ${expected.join(", ")}`, async () => { @@ -294,6 +308,28 @@ async function main() { assert(!wire.some((entry) => /retry|repair/.test(entry.path)), "Recovery must use the autonomous runtime path"); assert(!modelRequests.some((entry) => entry.error), "The fixture rejected an unexpected model request"); }); + await scenario("truncated-reasoning-is-rejected", async () => { + const session = truncation.session; + const exported = () => rest("/export", undefined, { record: false }); + await rest("/session/start", { sessionId: session, project, cwd: root }); + await rest("/observe", event("ACCEPTED", session)); + await poll("accepted compression", async () => observations(await exported(), session).some((row) => row.title === truncation.accepted && row.compression?.status === "complete")); + // Summarize before the truncated compression so two provider errors stay below the circuit breaker threshold. + const summary = await rest("/summarize", { sessionId: session }); + assert.equal(summary.success, false, "A summary reply cut at max_tokens must not succeed"); + assert.match(summary.error, /max_tokens/); + await rest("/observe", event("TRUNCATED", session)); + const failed = await poll("truncated compression rejected", async () => { + const row = observations(await exported(), session).find((item) => JSON.stringify(item.source ?? "").includes("GATE_INPUT_TRUNCATED")); + return row?.compression?.status === "failed" ? row : false; + }); + assert.equal(failed.compression.error, "compression_failed"); + const truncatedCalls = modelRequests.filter((request) => request.phase === phase && request.outcome === "truncated-reasoning"); + assert(truncatedCalls.some((request) => request.operation === "summary"), "The summary request must reach the truncating fixture"); + assert(truncatedCalls.some((request) => request.operation === "compression"), "The compression request must reach the truncating fixture"); + const after = await snapshot("after-truncated-reasoning"); + assert(!JSON.stringify(after).includes(truncation.marker), "Truncated reasoning must never be stored as model output"); + }); await scenario("installed-package-unchanged", async () => { assert.deepEqual(packageManifest(), packageBefore); assert.equal(sha(artifact.tarball), artifact.tarballSha256); }); diff --git a/test/fetch-timeout.test.ts b/test/fetch-timeout.test.ts index f2be990c81..2f92013b23 100644 --- a/test/fetch-timeout.test.ts +++ b/test/fetch-timeout.test.ts @@ -552,26 +552,37 @@ describe("OpenAIProvider thinking-model fallback", () => { expect(out).toBe("older shape"); }); - it("throws instead of returning partial reasoning when the reply hit max_tokens", async () => { + const emptyReasoningReplies = [null, ""].flatMap((content) => + (["reasoning", "reasoning_content"] as const).map((field) => ({ content, field })), + ); + + it.each( + emptyReasoningReplies.flatMap((reply) => (["compress", "summarize"] as const).map((method) => ({ ...reply, method }))), + )("$method rejects a reply cut at max_tokens with content $content and only $field", async ({ content, field, method }) => { mockOpenAIResponse({ - choices: [ - { - finish_reason: "length", - message: { content: null, reasoning: "The user is asking me to reply with" }, - }, - ], + choices: [{ finish_reason: "length", message: { content, [field]: "The user is asking me to reply with" } }], }); const provider = new OpenAIProvider("test-key", "gpt-5.6-luna", 16); - await expect(provider.compress("system", "user")).rejects.toThrow(/max_tokens \(16\)/); + await expect(provider[method]("system", "user")).rejects.toThrow(/max_tokens \(16\)/); }); - it("keeps the reasoning fallback when the reply finished normally", async () => { + it.each(emptyReasoningReplies)( + "keeps the $field fallback when the reply finished normally with content $content", + async ({ content, field }) => { + mockOpenAIResponse({ + choices: [{ finish_reason: "stop", message: { content, [field]: "complete answer" } }], + }); + const provider = new OpenAIProvider("test-key", "gpt-5.6-luna", 1024); + expect(await provider.summarize("system", "user")).toBe("complete answer"); + }, + ); + + it("returns nonempty content even when the reply hit max_tokens", async () => { mockOpenAIResponse({ - choices: [{ finish_reason: "stop", message: { content: "", reasoning_content: "complete answer" } }], + choices: [{ finish_reason: "length", message: { content: "partial but real", reasoning: "ignore" } }], }); - const provider = new OpenAIProvider("test-key", "gpt-5.6-luna", 1024); - const out = await provider.compress("system", "user"); - expect(out).toBe("complete answer"); + const provider = new OpenAIProvider("test-key", "gpt-5.6-luna", 16); + expect(await provider.compress("system", "user")).toBe("partial but real"); }); it("content wins over both reasoning fields when present", async () => { From 146fcaf0f8e019ea6ef8f97f6e87cfc28d3b7445 Mon Sep 17 00:00:00 2001 From: Rohit Ghumare Date: Wed, 4 Nov 2026 11:44:02 +0530 Subject: [PATCH 3/4] fix(openai): drop explanatory comments and the dash in the max_tokens error The max_tokens error now reads as two sentences. The fixture and provider comments are removed; the tests already describe the behavior. --- scripts/release-gate/summary-recovery.mjs | 2 -- src/providers/openai.ts | 4 +--- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/scripts/release-gate/summary-recovery.mjs b/scripts/release-gate/summary-recovery.mjs index 781455845c..daa7fc059f 100644 --- a/scripts/release-gate/summary-recovery.mjs +++ b/scripts/release-gate/summary-recovery.mjs @@ -41,7 +41,6 @@ const sha = (file) => createHash("sha256").update(readFileSync(file)).digest("he let port, modelPort, modelServer, secret, installed, packageBefore, boot = 0; let heldCompression = null, failSummary = false, phase = "setup"; const markers = { BASE: "RECOVERED_BASE_COBALT", FINAL: "RECOVERED_FINAL_JUNIPER", LATE: "RECOVERED_LATE_QUARTZ" }; -// A reasoning model cut at max_tokens returns only partial reasoning; this fixture makes that reasoning parse as valid XML. const truncation = { session: "truncated-reasoning-session", accepted: "TRUNCATION_ACCEPTED_SAGE", marker: "TRUNCATED_REASONING_ONYX" }; function log(message) { const line = `[${((Date.now() - began) / 1000).toFixed(1)}s] ${message}`; @@ -325,7 +324,6 @@ async function main() { await rest("/session/start", { sessionId: session, project, cwd: root }); await rest("/observe", event("ACCEPTED", session)); await poll("accepted compression", async () => observations(await exported(), session).some((row) => row.title === truncation.accepted && row.compression?.status === "complete")); - // Summarize before the truncated compression so two provider errors stay below the circuit breaker threshold. const summary = await rest("/summarize", { sessionId: session }); assert.equal(summary.success, false, "A summary reply cut at max_tokens must not succeed"); assert.match(summary.error, /max_tokens/); diff --git a/src/providers/openai.ts b/src/providers/openai.ts index 8612954f10..c29234c8a8 100644 --- a/src/providers/openai.ts +++ b/src/providers/openai.ts @@ -118,11 +118,9 @@ export class OpenAIProvider implements MemoryProvider { if (content) { return content; } - // A reasoning model that hits max_tokens returns only its partial chain of - // thought; returning that would store it as the compression result. if (choice?.finish_reason === "length") { throw new Error( - `OpenAI response hit max_tokens (${this.maxTokens}) before any content — raise MAX_TOKENS or lower OPENAI_REASONING_EFFORT.`, + `OpenAI response hit max_tokens (${this.maxTokens}) before any content. Raise MAX_TOKENS or lower OPENAI_REASONING_EFFORT.`, ); } const reasoning = message?.reasoning ?? message?.reasoning_content; From 8a4dbc99452ac500e5840a69e33d1f782ba0e29e Mon Sep 17 00:00:00 2001 From: Rohit Ghumare Date: Wed, 4 Nov 2026 12:00:37 +0530 Subject: [PATCH 4/4] fix(openai): treat whitespace-only content as missing A reply cut at max_tokens with whitespace-only content was returned as a success, so the truncation error never fired, the fallback chain did not try the next provider and the circuit breaker recorded a success. Whitespace-only content now counts as missing, so it reaches the max_tokens error, or the reasoning fallback when the reply finished normally. --- src/providers/openai.ts | 2 +- test/fetch-timeout.test.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/providers/openai.ts b/src/providers/openai.ts index c29234c8a8..929dc4c9c8 100644 --- a/src/providers/openai.ts +++ b/src/providers/openai.ts @@ -115,7 +115,7 @@ export class OpenAIProvider implements MemoryProvider { const choice = data.choices?.[0]; const message = choice?.message; const content = message?.content; - if (content) { + if (content?.trim()) { return content; } if (choice?.finish_reason === "length") { diff --git a/test/fetch-timeout.test.ts b/test/fetch-timeout.test.ts index 2f92013b23..48f5a73eef 100644 --- a/test/fetch-timeout.test.ts +++ b/test/fetch-timeout.test.ts @@ -552,7 +552,7 @@ describe("OpenAIProvider thinking-model fallback", () => { expect(out).toBe("older shape"); }); - const emptyReasoningReplies = [null, ""].flatMap((content) => + const emptyReasoningReplies = [null, "", " \n "].flatMap((content) => (["reasoning", "reasoning_content"] as const).map((field) => ({ content, field })), );