Skip to content
42 changes: 38 additions & 4 deletions scripts/release-gate/summary-recovery.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ const sha = (file) => createHash("sha256").update(readFileSync(file)).digest("he
let port, modelPort, modelServer, secret, installed, packageBefore, boot = 0;
let heldCompression = null, failSummary = false, phase = "setup";
const markers = { BASE: "RECOVERED_BASE_COBALT", FINAL: "RECOVERED_FINAL_JUNIPER", LATE: "RECOVERED_LATE_QUARTZ" };
const truncation = { session: "truncated-reasoning-session", accepted: "TRUNCATION_ACCEPTED_SAGE", marker: "TRUNCATED_REASONING_ONYX" };
function log(message) {
const line = `[${((Date.now() - began) / 1000).toFixed(1)}s] ${message}`;
console.log(line); appendFileSync(join(out, "gate.log"), `${line}\n`);
Expand Down Expand Up @@ -155,7 +156,19 @@ async function startModel() {
const item = { phase, boot, operation: compress ? "compression" : "summary", prompt };
modelRequests.push(item);
let content;
if (compress) {
const truncatedCompression = compress && prompt.includes("GATE_INPUT_TRUNCATED");
if (truncatedCompression || (summarize && phase === "truncated-reasoning-is-rejected")) {
const partial = compress
? `Draft: <observation><type>decision</type><title>${truncation.marker}</title><facts><fact>${truncation.marker}</fact></facts><narrative>The draft names ${truncation.marker} before the budget ran out.</narrative><concepts><concept>truncation</concept></concepts><files><file>fixture.txt</file></files><importance>8</importance></observation>`
: `Draft: <summary><title>${truncation.marker}</title><narrative>The draft names ${truncation.marker} before the budget ran out.</narrative><decisions><decision>${truncation.marker}</decision></decisions><files><file>fixture.txt</file></files><concepts><concept>truncation</concept></concepts></summary>`;
item.outcome = "truncated-reasoning"; item.response = partial;
response.writeHead(200, { "content-type": "application/json" });
response.end(JSON.stringify({ choices: [{ finish_reason: "length", message: { role: "assistant", content: null, reasoning: partial } }] }));
return;
}
if (compress && prompt.includes("GATE_INPUT_ACCEPTED")) {
content = `<observation><type>decision</type><title>${truncation.accepted}</title><facts><fact>The verified result is ${truncation.accepted}.</fact></facts><narrative>The local fixture completed processing and verified ${truncation.accepted}.</narrative><concepts><concept>truncation</concept></concepts><files><file>fixture.txt</file></files><importance>8</importance></observation>`;
} else if (compress) {
const token = prompt.match(/GATE_INPUT_(BASE|FINAL|LATE)\b/)?.[1];
assert(token, "Compression request lost the retained raw fixture");
item.token = token;
Expand Down Expand Up @@ -232,14 +245,14 @@ async function crashDaemon() {
await poll("crashed daemon ports closed", async () => (await Promise.all(daemonPorts().map(portFree))).every(Boolean), 15_000);
modelServer.closeAllConnections(); ownedPids.clear(); log(`Native CLI boot ${boot} terminated`);
}
function event(token) {
return { hookType: "post_tool_use", sessionId, project, cwd: root, timestamp: new Date().toISOString(), eventId: `summary-gate-${token.toLowerCase()}`, data: { tool_name: "Bash", tool_input: { command: `printf GATE_INPUT_${token}` }, tool_response: `GATE_INPUT_${token}: tool completed` } };
function event(token, session = sessionId) {
return { hookType: "post_tool_use", sessionId: session, project, cwd: root, timestamp: new Date().toISOString(), eventId: `summary-gate-${token.toLowerCase()}`, data: { tool_name: "Bash", tool_input: { command: `printf GATE_INPUT_${token}` }, tool_response: `GATE_INPUT_${token}: tool completed` } };
}
async function snapshot(name) {
const data = await rest("/export", undefined, { record: false });
writeFileSync(join(out, `${name}.json`), JSON.stringify(data, null, 2) + "\n"); return data;
}
const observations = (data) => data.observations?.[sessionId] ?? [];
const observations = (data, session = sessionId) => data.observations?.[session] ?? [];
const summaryFor = (data) => data.summaries?.find((summary) => summary.sessionId === sessionId);
async function waitSummary(expected) {
return poll(`summary contains ${expected.join(", ")}`, async () => {
Expand Down Expand Up @@ -305,6 +318,27 @@ async function main() {
assert(!wire.some((entry) => /retry|repair/.test(entry.path)), "Recovery must use the autonomous runtime path");
assert(!modelRequests.some((entry) => entry.error), "The fixture rejected an unexpected model request");
});
await scenario("truncated-reasoning-is-rejected", async () => {
const session = truncation.session;
const exported = () => rest("/export", undefined, { record: false });
await rest("/session/start", { sessionId: session, project, cwd: root });
await rest("/observe", event("ACCEPTED", session));
await poll("accepted compression", async () => observations(await exported(), session).some((row) => row.title === truncation.accepted && row.compression?.status === "complete"));
const summary = await rest("/summarize", { sessionId: session });
assert.equal(summary.success, false, "A summary reply cut at max_tokens must not succeed");
assert.match(summary.error, /max_tokens/);
await rest("/observe", event("TRUNCATED", session));
const failed = await poll("truncated compression rejected", async () => {
const row = observations(await exported(), session).find((item) => JSON.stringify(item.source ?? "").includes("GATE_INPUT_TRUNCATED"));
return row?.compression?.status === "failed" ? row : false;
});
assert.equal(failed.compression.error, "compression_failed");
const truncatedCalls = modelRequests.filter((request) => request.phase === phase && request.outcome === "truncated-reasoning");
assert(truncatedCalls.some((request) => request.operation === "summary"), "The summary request must reach the truncating fixture");
assert(truncatedCalls.some((request) => request.operation === "compression"), "The compression request must reach the truncating fixture");
const after = await snapshot("after-truncated-reasoning");
assert(!JSON.stringify(after).includes(truncation.marker), "Truncated reasoning must never be stored as model output");
});
await scenario("installed-package-unchanged", async () => {
assert.deepEqual(packageManifest(), packageBefore); assert.equal(sha(artifact.tarball), artifact.tarballSha256);
});
Expand Down
13 changes: 10 additions & 3 deletions src/providers/openai.ts
Original file line number Diff line number Diff line change
Expand Up @@ -108,14 +108,21 @@ export class OpenAIProvider implements MemoryProvider {

const data = (await response.json()) as {
choices?: Array<{
message?: { content?: string; reasoning?: string; reasoning_content?: string };
finish_reason?: string;
message?: { content?: string | null; reasoning?: string; reasoning_content?: string };
}>;
};
const message = data.choices?.[0]?.message;
const choice = data.choices?.[0];
const message = choice?.message;
const content = message?.content;
if (content) {
if (content?.trim()) {
Comment thread
rohitg00 marked this conversation as resolved.
Resolved
return content;
}
if (choice?.finish_reason === "length") {
throw new Error(
`OpenAI response hit max_tokens (${this.maxTokens}) before any content. Raise MAX_TOKENS or lower OPENAI_REASONING_EFFORT.`,
);
}
Comment thread
rohitg00 marked this conversation as resolved.
Resolved
const reasoning = message?.reasoning ?? message?.reasoning_content;
if (reasoning) {
return reasoning;
Expand Down
33 changes: 33 additions & 0 deletions test/fetch-timeout.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -552,6 +552,39 @@ describe("OpenAIProvider thinking-model fallback", () => {
expect(out).toBe("older shape");
});

const emptyReasoningReplies = [null, "", " \n "].flatMap((content) =>
(["reasoning", "reasoning_content"] as const).map((field) => ({ content, field })),
);

it.each(
emptyReasoningReplies.flatMap((reply) => (["compress", "summarize"] as const).map((method) => ({ ...reply, method }))),
)("$method rejects a reply cut at max_tokens with content $content and only $field", async ({ content, field, method }) => {
mockOpenAIResponse({
choices: [{ finish_reason: "length", message: { content, [field]: "The user is asking me to reply with" } }],
});
const provider = new OpenAIProvider("test-key", "gpt-5.6-luna", 16);
await expect(provider[method]("system", "user")).rejects.toThrow(/max_tokens \(16\)/);
});

it.each(emptyReasoningReplies)(
"keeps the $field fallback when the reply finished normally with content $content",
async ({ content, field }) => {
mockOpenAIResponse({
choices: [{ finish_reason: "stop", message: { content, [field]: "complete answer" } }],
});
const provider = new OpenAIProvider("test-key", "gpt-5.6-luna", 1024);
expect(await provider.summarize("system", "user")).toBe("complete answer");
},
);

it("returns nonempty content even when the reply hit max_tokens", async () => {
mockOpenAIResponse({
choices: [{ finish_reason: "length", message: { content: "partial but real", reasoning: "ignore" } }],
});
const provider = new OpenAIProvider("test-key", "gpt-5.6-luna", 16);
expect(await provider.compress("system", "user")).toBe("partial but real");
});

it("content wins over both reasoning fields when present", async () => {
mockOpenAIResponse({
choices: [
Expand Down
Loading