From f2983206fba60d301310796952a16133bf26ff1a Mon Sep 17 00:00:00 2001 From: ework-agent Date: Tue, 15 Sep 2026 01:29:35 +0800 Subject: [PATCH] fix: preflight SSE extraction rejects truncated and failure streams (#780) extractSummaryFromSse conflated 'the stream delivered a complete summary' with 'the stream died mid-delivery': a gateway truncation (#764 shape: half-line data, no final event) left a partial accumulator that was persisted as a complete tier-1 summary - silently worse than an empty one, because #727's diagnosis chain only fires on empty results. - a data line that fails to parse is corruption (badFrame), not skipped noise - event: frame headers type frames whose payload lacks a type field, reset at every frame boundary (no cross-frame leakage) - failure terminals (response.incomplete/.failed/.error, generic error, bare error object) invalidate any text accumulated before them - responses requires the spec-mandatory response.completed terminal; its full response object stays authoritative over delta accumulation - delta-less terminals (response.output_text.done / output_item.done) now carry their text instead of extracting an empty string - anthropic/openai bodies must end on a frame boundary (CRLF-tolerant); finish_reason/[DONE]/message_stop are NOT required (#764: real gateways omit these occasionally) - diagnoseEmptySummary names truncation explicitly (incomplete data lines / missing final frame terminator) so logs say what actually happened Rejected streams return '' and flow into the existing diagnose + halving + cooldown chain (#726/#727), closing that recovery loop for truncated streams. Preflight: typecheck clean, 1535 pass / 0 fail / 2 skipped, build ok. New coverage: 23 unit tests (tests/preflight-sse-extraction.test.ts) + 1 full-pipeline integration test through the real proxy HTTP server. --- src/preflight.ts | 86 ++++++++++--- tests/preflight-empty-summary.test.ts | 57 ++++++++- tests/preflight-sse-extraction.test.ts | 160 +++++++++++++++++++++++++ 3 files changed, 287 insertions(+), 16 deletions(-) create mode 100644 tests/preflight-sse-extraction.test.ts diff --git a/src/preflight.ts b/src/preflight.ts index 8092f07c25..12b3b39a59 100644 --- a/src/preflight.ts +++ b/src/preflight.ts @@ -303,14 +303,41 @@ function summaryHeaders(deps: PreflightDeps): Record { return headers; } -// Extract the summary text from a buffered SSE body (the streaming twin of -// extractSummaryText). For Responses, prefer the response.completed event's -// full response object (reuses the JSON extractor); otherwise accumulate -// output_text deltas. Non-conforming upstreams that return plain JSON despite -// stream:true are handled by the caller's JSON fallback. -function extractSummaryFromSse(protocol: PreflightProtocol, text: string): string { +// #780: extraction carries a validity contract — it must separate "the stream +// delivered a complete summary" from "the stream died mid-delivery". The naive +// accumulator conflated the two: a gateway truncation (#764: half-line data, +// no [DONE]) left a partial `out` that was persisted as a complete tier-1 +// summary — silently worse than an empty one, because #727's diagnosis chain +// only fires on empty results. Rejection rules: +// - a data line that fails to parse is corruption (badFrame), not noise to skip +// - failure terminals (response.incomplete/.failed/.error, generic error / +// bare {error}) invalidate any text accumulated before them +// - responses requires the spec-mandatory response.completed terminal; its +// response object reuses the JSON extractor and is authoritative — once +// seen it is trusted as-is (no framing check on top, so gateways that close +// right after the final event without a trailing blank line are safe) +// - anthropic/openai do NOT require finish_reason/[DONE]/message_stop (#764: +// real gateways omit these occasionally); the body must at least end on a +// frame boundary (\n\n, CRLF-tolerant), else it may have been cut mid-frame +// A rejected stream returns "" so requestSummary routes it into +// diagnoseEmptySummary + the #726 halving/cooldown chain. +export function extractSummaryFromSse(protocol: PreflightProtocol, text: string): string { + const framed = /\r?\n\r?\n$/.test(text); let out = ""; + let terminalText = ""; + let completed = false; + let invalid = false; + let badFrame = false; + let eventType = ""; for (const line of text.split("\n")) { + if (line.trim() === "") { + eventType = ""; + continue; + } + if (line.startsWith("event:")) { + eventType = line.slice(6).trim(); + continue; + } if (!line.startsWith("data:")) continue; const payload = line.slice(5).trim(); if (!payload || payload === "[DONE]") continue; @@ -318,12 +345,21 @@ function extractSummaryFromSse(protocol: PreflightProtocol, text: string): strin try { obj = JSON.parse(payload); } catch { + badFrame = true; + continue; + } + if (!obj || typeof obj !== "object") { + badFrame = true; continue; } - if (!obj || typeof obj !== "object") continue; const o = obj as Record; + const type = typeof o.type === "string" ? o.type : eventType; + if (type === "error" || type === "response.incomplete" || type === "response.failed" || type === "response.error" || (!type && o.error && typeof o.error === "object")) { + invalid = true; + continue; + } if (protocol === "anthropic") { - if (o.type === "content_block_delta") { + if (type === "content_block_delta") { const d = o.delta as Record | undefined; if (d && d.type === "text_delta" && typeof d.text === "string") out += d.text; } @@ -334,15 +370,23 @@ function extractSummaryFromSse(protocol: PreflightProtocol, text: string): strin if (delta && typeof delta.content === "string") out += delta.content; } } else { - if (o.type === "response.output_text.delta" && typeof o.delta === "string") { + if (type === "response.output_text.delta" && typeof o.delta === "string") { out += o.delta; - } else if (o.type === "response.completed" && o.response && typeof o.response === "object") { - const full = extractSummaryText(protocol, o.response as Record); - if (full) return full; + } else if (type === "response.output_text.done" && typeof o.text === "string") { + terminalText += o.text; + } else if (type === "response.output_item.done" && o.item && typeof o.item === "object") { + terminalText += extractSummaryText("responses", { output: [o.item] }); + } else if (type === "response.completed" && o.response && typeof o.response === "object") { + completed = true; + const full = extractSummaryText("responses", o.response as Record); + if (full) terminalText = full; } } } - return out; + if (invalid) return ""; + if (protocol === "responses") return completed ? terminalText || out : ""; + if (badFrame || !framed) return ""; + return out || terminalText; } function extractSummaryText(protocol: PreflightProtocol, json: Record): string { @@ -424,6 +468,7 @@ export function diagnoseEmptySummary(text: string, json?: unknown): string { if (err) return err; } let sseEvents = 0; + let halfLines = 0; let firstPayload = ""; for (const line of text.split("\n")) { if (!line.startsWith("data:")) continue; @@ -434,14 +479,25 @@ export function diagnoseEmptySummary(text: string, json?: unknown): string { try { obj = JSON.parse(payload); } catch { + halfLines += 1; + continue; + } + if (!obj || typeof obj !== "object") { + halfLines += 1; continue; } - if (!obj || typeof obj !== "object") continue; sseEvents += 1; const err = extractStreamError(obj as Record); if (err) return err; } - if (sseEvents > 0) return `the upstream stream carried ${sseEvents} SSE event(s) but no summary text (first event: ${firstPayload})`; + // #780: the extractor rejects mid-frame-truncated streams (#764 shape) — say so + // explicitly instead of the generic no-text message, which reads like an + // upstream that simply never answered with a summary. + if (halfLines > 0 && sseEvents === 0) return `the upstream stream had ${halfLines} incomplete data line(s) and no parseable events (stream appears truncated)`; + if (sseEvents > 0) { + const truncated = halfLines > 0 || !/\r?\n\r?\n$/.test(text) ? ` (stream appears truncated: ${halfLines > 0 ? `${halfLines} incomplete data line(s)` : "no final frame terminator"})` : ""; + return `the upstream stream carried ${sseEvents} SSE event(s) but no summary text${truncated} (first event: ${firstPayload})`; + } const trimmed = text.trim(); if (!trimmed) return "the upstream returned an empty body"; return `the upstream returned a non-SSE body with no summary text (first 200 bytes: ${trimmed.slice(0, 200)})`; diff --git a/tests/preflight-empty-summary.test.ts b/tests/preflight-empty-summary.test.ts index 3a53fa6298..b60ed01d3d 100644 --- a/tests/preflight-empty-summary.test.ts +++ b/tests/preflight-empty-summary.test.ts @@ -51,6 +51,16 @@ function okSummarySse(res: http.ServerResponse): void { res.end(); } +// #780: two complete delta frames, then a half-line cut mid-JSON — no +// completed terminal, no final frame terminator (#764 gateway-corruption shape). +// The extractor must reject this as unusable rather than persist the partial text. +function truncatedSummarySse(res: http.ServerResponse): void { + res.write(sse("response.output_text.delta", { type: "response.output_text.delta", delta: SUMMARY_TEXT.slice(0, 40) })); + res.write(sse("response.output_text.delta", { type: "response.output_text.delta", delta: SUMMARY_TEXT.slice(40, 80) })); + res.write(`data: ${JSON.stringify({ type: "response.output_text.delta", delta: "tail" }).slice(0, 30)}\n`); + res.end(); +} + function forwardSse(res: http.ServerResponse, inputTokens = 800): void { res.write(sse("response.completed", { type: "response.completed", @@ -84,7 +94,7 @@ function inputContentChars(parsed: ParsedBody): number { return typeof c === "string" ? c.length : 0; } -function makeUpstream(calls: Call[], failAboveChars: number, forwardInputTokens = 800): http.Server { +function makeUpstream(calls: Call[], failAboveChars: number, forwardInputTokens = 800, truncateSummaries = false): http.Server { return http.createServer((req, res) => { const chunks: Buffer[] = []; req.on("data", (c: Buffer) => chunks.push(c)); @@ -98,6 +108,11 @@ function makeUpstream(calls: Call[], failAboveChars: number, forwardInputTokens res.end(JSON.stringify({ detail: "Stream must be set to true" })); return; } + if (truncateSummaries && isSummaryCall(parsed)) { + res.writeHead(200, { "content-type": "text/event-stream", "cache-control": "no-cache" }); + truncatedSummarySse(res); + return; + } if (failed) { res.writeHead(200, { "content-type": "text/event-stream", "cache-control": "no-cache" }); failedSummarySse(res); @@ -312,3 +327,43 @@ test("#726 diagnoseEmptySummary: extracts terminal error signals from 200 bodies /non-SSE body with no summary text/, ); }); + +test("#780 truncated summary stream is unusable: diagnosis names truncation, bounded calls, cooldown arms, nothing forwarded", async () => { + const calls: Call[] = []; + const upstream = makeUpstream(calls, 0, 800, true); + upstream.listen(0, "127.0.0.1"); + await once(upstream, "listening"); + const upstreamPort = (upstream.address() as { port: number }).port; + + const proxy = await startProxy(upstreamPort, { "gpt-7-sol": { context: 10_000 } }); + await once(proxy, "listening"); + const proxyPort = (proxy.address() as { port: number }).port; + + try { + const r = await driveResponses(proxyPort, upstreamPort, "s780-trunc", "gpt-7-sol", longResponsesInput(12)); + assert.equal(r.status, 502, `truncated summaries must fail-fast, got ${r.status}`); + const j = (await r.json()) as { error?: { code?: string; message?: string } }; + assert.equal(j.error?.code, "preflight_compress_failed"); + assert.ok( + j.error?.message?.includes("stream appears truncated"), + `fail-fast message must carry the truncation diagnosis, got: ${j.error?.message}`, + ); + const summaries = calls.filter((c) => c.summary); + assert.ok(summaries.length >= 2 && summaries.length <= 9, `summary calls must be bounded, got ${summaries.length}: ${JSON.stringify(summaries)}`); + assert.ok(!calls.some((c) => !c.summary), "nothing may be forwarded on failure"); + + const sess = listSessions().find((s) => s.id.includes("s780-trunc")); + assert.ok(sess?.metadata?.preflightDeadEnd, "zero-progress failure must arm the dead-end marker"); + + const callsBeforeRetry = calls.length; + const r2 = await driveResponses(proxyPort, upstreamPort, "s780-trunc", "gpt-7-sol", longResponsesInput(12)); + assert.equal(r2.status, 502, `cooldown must fail fast, got ${r2.status}`); + assert.equal(calls.length, callsBeforeRetry, "cooldown must spend ZERO upstream calls"); + } finally { + proxy.close(); + upstream.close(); + await new Promise((resolve, reject) => { + void Promise.allSettled([once(proxy, "close"), once(upstream, "close")]).then(() => resolve(), reject); + }); + } +}); diff --git a/tests/preflight-sse-extraction.test.ts b/tests/preflight-sse-extraction.test.ts new file mode 100644 index 0000000000..402104786a --- /dev/null +++ b/tests/preflight-sse-extraction.test.ts @@ -0,0 +1,160 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +process.env.NODE_ENV = "test"; + +// #780: validity contract of extractSummaryFromSse — a buffered SSE body either +// provably delivered a complete summary or it did not; anything less than proof +// yields "" so requestSummary routes it into diagnoseEmptySummary + the #726 +// halving/cooldown chain instead of persisting a silent partial summary. + +import { diagnoseEmptySummary, extractSummaryFromSse } from "../src/preflight.ts"; + +const R_DELTA = (delta: string): string => `data: ${JSON.stringify({ type: "response.output_text.delta", delta })}\n\n`; +const R_DONE = (text: string): string => `data: ${JSON.stringify({ type: "response.output_text.done", item_id: "msg_1", content_index: 0, text })}\n\n`; +const R_ITEM_DONE = (text: string): string => + `data: ${JSON.stringify({ type: "response.output_item.done", item_id: "msg_1", output_index: 0, item: { type: "message", id: "msg_1", role: "assistant", status: "completed", content: [{ type: "output_text", text }] } })}\n\n`; +const R_COMPLETED = (text?: string): string => + `data: ${JSON.stringify({ type: "response.completed", response: { id: "resp_1", status: "completed", output: text === undefined ? [] : [{ type: "message", id: "msg_1", role: "assistant", content: [{ type: "output_text", text }] }], usage: { input_tokens: 1, output_tokens: 1 } } })}\n\n`; +const A_DELTA = (text: string): string => `event: content_block_delta\ndata: ${JSON.stringify({ type: "content_block_delta", index: 0, delta: { type: "text_delta", text } })}\n\n`; +const O_DELTA = (content: string): string => `data: ${JSON.stringify({ choices: [{ index: 0, delta: { content } }] })}\n\n`; + +test("#780 responses: complete stream, completed full text is authoritative", () => { + assert.equal(extractSummaryFromSse("responses", R_DELTA("Hello ") + R_DELTA("world.") + R_COMPLETED("Hello world.")), "Hello world."); +}); + +test("#780 responses: completed with empty output falls back to accumulated deltas (#727 fixture shape)", () => { + assert.equal(extractSummaryFromSse("responses", R_DELTA("part one ") + R_DELTA("part two") + R_COMPLETED()), "part one part two"); +}); + +test("#780 responses: mid-frame truncation is rejected, not returned as a partial summary", () => { + assert.equal(extractSummaryFromSse("responses", R_DELTA("Hello ") + 'data: {"type":"response.output_text.delta","del'), ""); +}); + +test("#780 responses: clean framing without the mandatory completed terminal is rejected", () => { + assert.equal(extractSummaryFromSse("responses", R_DELTA("a summary that ended cleanly but never said completed")), ""); +}); + +test("#780 responses: delta-less terminals carry the text", () => { + assert.equal(extractSummaryFromSse("responses", R_DONE("Done-only summary.") + R_COMPLETED("Done-only summary.")), "Done-only summary."); + assert.equal(extractSummaryFromSse("responses", R_ITEM_DONE("Item-only summary.") + R_COMPLETED()), "Item-only summary."); +}); + +test("#780 responses: done + delta coexistence must not duplicate text", () => { + assert.equal(extractSummaryFromSse("responses", R_DELTA("Shared ") + R_DELTA("text") + R_DONE("Shared text") + R_COMPLETED("Shared text")), "Shared text"); +}); + +test("#780 responses: multiple output_text.done parts concatenate", () => { + assert.equal(extractSummaryFromSse("responses", R_DONE("alpha ") + R_DONE("beta") + R_COMPLETED("alpha beta")), "alpha beta"); +}); + +test("#780 responses: failure terminals invalidate previously accumulated text", () => { + assert.equal( + extractSummaryFromSse("responses", R_DELTA("partial ") + `data: ${JSON.stringify({ type: "response.failed", response: { status: "failed", error: { code: "boom", message: "nope" } } })}\n\n`), + "", + ); + assert.equal( + extractSummaryFromSse("responses", R_DELTA("partial ") + `data: ${JSON.stringify({ type: "response.incomplete", response: { status: "incomplete" } })}\n\n`), + "", + ); + assert.equal( + extractSummaryFromSse("responses", R_DELTA("partial ") + `data: ${JSON.stringify({ type: "error", error: { message: "mid-stream" } })}\n\n`), + "", + ); + assert.equal( + extractSummaryFromSse("responses", R_DELTA("partial ") + `data: ${JSON.stringify({ error: { message: "bare" } })}\n\n`), + "", + ); +}); + +test("#780 responses: event: header types frames whose payload lacks a type field", () => { + const body = + `event: response.output_text.delta\ndata: ${JSON.stringify({ delta: "Header-" })}\n\n` + + `event: response.completed\ndata: ${JSON.stringify({ response: { id: "resp_h", status: "completed", output: [{ type: "message", id: "m", role: "assistant", content: [{ type: "output_text", text: "Header-typed" }] }] } })}\n\n`; + assert.equal(extractSummaryFromSse("responses", body), "Header-typed"); +}); + +test("#780 responses: event: header must not leak across frame boundaries", () => { + const body = + `event: response.output_text.delta\ndata: ${JSON.stringify({ delta: "REAL" })}\n\n` + + `data: ${JSON.stringify({ delta: "LEAKED" })}\n\n` + + R_COMPLETED(); + assert.equal(extractSummaryFromSse("responses", body), "REAL"); +}); + +test("#780 responses: completed terminal is trusted despite corrupted bytes after it", () => { + assert.equal(extractSummaryFromSse("responses", R_DELTA("Trust ") + R_DELTA("me") + R_COMPLETED("Trust me") + 'data: {"type":"response.output_text.delta","de'), "Trust me"); +}); + +test("#780 anthropic: complete stream extracts text_deltas", () => { + const body = + `event: message_start\ndata: ${JSON.stringify({ type: "message_start", message: { id: "m1" } })}\n\n` + + A_DELTA("Anthropic ") + + A_DELTA("summary") + + `event: message_delta\ndata: ${JSON.stringify({ type: "message_delta", delta: { stop_reason: "end_turn" }, usage: { output_tokens: 2 } })}\n\n` + + `event: message_stop\ndata: ${JSON.stringify({ type: "message_stop" })}\n\n`; + assert.equal(extractSummaryFromSse("anthropic", body), "Anthropic summary"); +}); + +test("#780 anthropic: mid-frame truncation is rejected", () => { + assert.equal(extractSummaryFromSse("anthropic", A_DELTA("partial ") + 'data: {"type":"content_block_delta","delt'), ""); +}); + +test("#780 anthropic: cleanly framed stream without message_stop is accepted (#764: gateways omit terminals occasionally)", () => { + assert.equal(extractSummaryFromSse("anthropic", A_DELTA("lenient gateway text")), "lenient gateway text"); +}); + +test("#780 anthropic: event: header types frames whose payload lacks a type field", () => { + const body = `event: content_block_delta\ndata: ${JSON.stringify({ index: 0, delta: { type: "text_delta", text: "header-typed" } })}\n\n`; + assert.equal(extractSummaryFromSse("anthropic", body), "header-typed"); +}); + +test("#780 anthropic: event: header must not leak across frame boundaries", () => { + const body = + `event: content_block_delta\ndata: ${JSON.stringify({ delta: { type: "text_delta", text: "REAL" } })}\n\n` + + `data: ${JSON.stringify({ delta: { type: "text_delta", text: "LEAKED" } })}\n\n`; + assert.equal(extractSummaryFromSse("anthropic", body), "REAL"); +}); + +test("#780 anthropic: in-stream error invalidates accumulated text", () => { + const body = A_DELTA("partial ") + `event: error\ndata: ${JSON.stringify({ type: "error", error: { type: "overloaded_error", message: "try again" } })}\n\n`; + assert.equal(extractSummaryFromSse("anthropic", body), ""); +}); + +test("#780 openai: complete stream with finish_reason and [DONE]", () => { + const body = O_DELTA("OpenAI ") + O_DELTA("summary") + `data: ${JSON.stringify({ choices: [{ index: 0, delta: {}, finish_reason: "stop" }] })}\n\n` + "data: [DONE]\n\n"; + assert.equal(extractSummaryFromSse("openai", body), "OpenAI summary"); +}); + +test("#780 openai: missing finish_reason/[DONE] is tolerated when framing is clean (#764)", () => { + assert.equal(extractSummaryFromSse("openai", O_DELTA("tolerated ") + O_DELTA("gateway text")), "tolerated gateway text"); +}); + +test("#780 openai: mid-frame truncation is rejected", () => { + assert.equal(extractSummaryFromSse("openai", O_DELTA("partial ") + 'data: {"choices":[{"index":0,"delta":{"conten'), ""); +}); + +test("#780 openai: in-stream bare error object invalidates accumulated text", () => { + const body = O_DELTA("partial ") + `data: ${JSON.stringify({ error: { message: "rate limited" } })}\n\n`; + assert.equal(extractSummaryFromSse("openai", body), ""); +}); + +test("#780 CRLF-framed bodies are accepted", () => { + const lf = R_DELTA("crlf ") + R_DELTA("ok") + R_COMPLETED("crlf ok"); + assert.equal(extractSummaryFromSse("responses", lf.replace(/\n/g, "\r\n")), "crlf ok"); +}); + +test("#780 diagnoseEmptySummary names truncation explicitly", () => { + assert.match( + diagnoseEmptySummary('data: {"type":"response.output_text.delta","del'), + /incomplete data line\(s\) and no parseable events \(stream appears truncated\)/, + ); + assert.match( + diagnoseEmptySummary(R_DELTA("some text that looks like a summary but was cut") + 'data: {"broken'), + /stream appears truncated: 1 incomplete data line/, + ); + assert.match( + diagnoseEmptySummary(`data: ${JSON.stringify({ type: "response.output_text.delta", delta: "x" })}\n`), + /no final frame terminator/, + ); +});