diff --git a/src/plugin.ts b/src/plugin.ts index 00043ad931..76d47bf978 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -22,6 +22,37 @@ import { log as loggerLog } from "./logger.js"; import { ccrEnabled, ccrLoopConfig, contentStoreOf, retrieveToolName } from "./store.js"; import { imageUsageSuffix } from "./image-compress.js"; import { emitStreamError, emitUpstreamTruncation } from "./stream-error.js"; +import type { TruncationDiag } from "./stream-error.js"; + +/** #2328 Q2: what the pipes know about the upstream HTTP response whose body + * they are piping — threaded in by server.ts so the truncation diag can cite + * status/content-type without the pipes holding the Response object. */ +export interface UpstreamMeta { + status?: number; + contentType?: string; +} + +/** #2328 Q4: does the dangling partial event left at a cut carry the opening + * bytes of the wire's own terminal? Factual byte evidence only — this never + * guesses about frames that parsed but were classified otherwise. */ +function tailLooksTerminal(tail: string, protocol: WireProtocol | "responses"): boolean { + // [DONE] torn anywhere past `data: [DO` is still recognizable as the + // terminal byte arriving. + if (/^data: \[DO/.test(tail)) return true; + if (protocol === "anthropic") return /event: message_stop/.test(tail.slice(0, 64)) || /"type"\s*:\s*"message_stop"/.test(tail); + if (protocol === "openai") { + // `"finish_reason":null` rides EVERY chat chunk — only a quoted + // (non-null) value counts, which is the final chunk's signature. + return /"finish_reason"\s*:\s*"/.test(tail); + } + if (protocol === "google") { + // finishReason only ever appears on Gemini's terminal chunk. + return /"finishReason"\s*:\s*"/.test(tail); + } + // responses: the completion-family event is the terminal. + return /event: response\.(?:completed|failed|incomplete)/.test(tail.slice(0, 64)) || + /"type"\s*:\s*"response\.(?:completed|failed|incomplete)"/.test(tail); +} import { degenerateTurnWarning, endsWithDraftClose } from "./degenerate-turn.js"; import { PANEL_BOX_FOOTER } from "./acp-panel.js"; import { describeAdvisory, getAdvisoryState } from "./advisory.js"; @@ -1730,6 +1761,11 @@ export async function pipePluginChatWithStrip( // The shipped request text: an emission-shaped span the user asked to // output verbatim is echoed, not dropped (m00885). requestText?: string, + // #2328 Q2: upstream HTTP status/content-type of the response this stream + // came from, for the truncation diag. Optional so existing callers/tests + // stay source-compatible; the plugin lane's resolveFakeCompletion refetch + // swallows the original response object, hence optional. + upstreamMeta?: UpstreamMeta, ): Promise { let reader = stream.getReader(); let decoder = new TextDecoder("utf-8"); @@ -1826,6 +1862,16 @@ export async function pipePluginChatWithStrip( * the text that survives such a frame is the tag's own interior. */ let droppedTagInFrame = false; let finalFinishReason: string | undefined; + // #2328 Q2/Q4: termination diagnostics — every truncation emit below + // carries these on the in-band error frame (meta) and the log line. + let diagEvents = 0; + let diagBytes = 0; + let diagUnparseable = 0; + const diagLastTypes: string[] = []; + const diagPushType = (label: string) => { + diagLastTypes.push(label); + if (diagLastTypes.length > 5) diagLastTypes.shift(); + }; // #1501 option C: tool-call observations on this verbatim lane, keyed per // protocol (openai: choice:toolIndex, anthropic: block:N, google: // candidate/part). Bytes are forwarded untouched (#1039); the tracker only @@ -1968,6 +2014,27 @@ export async function pipePluginChatWithStrip( buf = ""; return true; }; + // #2328 Q2/Q4: the diag snapshot riding every truncation emit below. Built + // at emit time so the counters reflect the final (possibly retried) + // attempt; `buf` is whatever partial SSE event the cut left dangling. + const buildTruncationDiag = (cause: "eof" | "read-error"): TruncationDiag => ({ + cause, + classification: buf.length > 0 && tailLooksTerminal(buf, protocol) ? "terminal-bytes-unrecognized" : "no-terminal-seen", + protocol, + events: diagEvents, + unparseableForwarded: diagUnparseable, + bytes: diagBytes, + lastEventTypes: [...diagLastTypes], + eofBufferBytes: buf.length, + ...(buf.length > 0 ? { eofBufferHead: buf.slice(0, 80).replace(/[\r\n]+/g, "\\n") } : {}), + ...(finalFinishReason !== undefined ? { finishReason: finalFinishReason } : {}), + visibleChars: visibleTextChars, + sawStrippedEcho, + retryZeroByteCutSpent: truncationRetried, + retryDegenerateSpent: degenerateRetried, + ...(upstreamMeta?.status !== undefined ? { upstreamStatus: upstreamMeta.status } : {}), + ...(upstreamMeta?.contentType !== undefined ? { upstreamContentType: upstreamMeta.contentType } : {}), + }); /** While the retry stream feeds the client, the message the FIRST attempt * opened is still open: nothing may re-open it. Returns true when the event * was consumed. */ @@ -2486,6 +2553,7 @@ export async function pipePluginChatWithStrip( buf = ""; pendingFinal = resolved; } else { + if (value && value.length > 0) diagBytes += value.byteLength; pendingFinal = value && value.length > 0 ? decoder.decode(value, { stream: true }) : null; } if (pendingFinal !== null) { @@ -2499,6 +2567,8 @@ export async function pipePluginChatWithStrip( const jsonStr = dataLines.map((l) => l.slice(5).replace(/^ /, "")).join("\n").trim(); if (!jsonStr) continue; if (jsonStr === "[DONE]") { + diagEvents++; + diagPushType("[DONE]"); sawTerminal = true; await write(flushTails() + rawEvent + "\n\n"); continue; @@ -2507,11 +2577,28 @@ export async function pipePluginChatWithStrip( try { ev = JSON.parse(jsonStr) as Record; } catch { + diagEvents++; + diagUnparseable++; + diagPushType("unparseable"); // #2190: unparseable frames bypass every filter — audit them. auditRawForward(rawEvent); await write(rawEvent + "\n\n"); continue; } + diagEvents++; + // #2328: wire-native label where one exists; the chat wires + // without a `type` get a synthesized one so the last-events + // trail is readable in the diag. + if (protocol === "anthropic") diagPushType(typeof ev["type"] === "string" ? ev["type"] as string : "frame"); + else if (protocol === "openai") { + const c0 = Array.isArray(ev["choices"]) ? (ev["choices"] as unknown[])[0] : undefined; + const fr = c0 && typeof c0 === "object" ? (c0 as Record)["finish_reason"] : undefined; + diagPushType(typeof fr === "string" ? `chunk:finish=${fr}` : "chunk"); + } else if (protocol === "google") { + const c0 = Array.isArray(ev["candidates"]) ? (ev["candidates"] as unknown[])[0] : undefined; + const fr = c0 && typeof c0 === "object" ? (c0 as Record)["finishReason"] : undefined; + diagPushType(typeof fr === "string" ? `chunk:finish=${fr}` : "chunk"); + } else diagPushType("frame"); if (retryFraming(ev)) continue; if (protocol === "anthropic" && ev["type"] === "content_block_start") blocksForwarded++; if (ev["type"] === "message_stop") sawTerminal = true; @@ -2580,7 +2667,7 @@ export async function pipePluginChatWithStrip( maybeWarnNamelessToolCalls(); settleWitnesses(); if (truncated) { - emitUpstreamTruncation(res, protocol, finalFinishReason !== undefined, log); + emitUpstreamTruncation(res, protocol, finalFinishReason !== undefined, log, buildTruncationDiag("eof")); return; } } catch (e) { @@ -2604,7 +2691,7 @@ export async function pipePluginChatWithStrip( /* client half-gone; the emission below is best-effort too */ } loggerLog("warn", `[plugin] upstream stream read failed (${protocol}): ${String(e instanceof Error ? e.message : e)} — emitting in-band truncation signal`); - emitUpstreamTruncation(res, protocol, finalFinishReason !== undefined, log); + emitUpstreamTruncation(res, protocol, finalFinishReason !== undefined, log, buildTruncationDiag("read-error")); return; } finally { reader.releaseLock(); @@ -2725,11 +2812,22 @@ export async function pipePluginResponsesWithStrip( absorbInstructed?: boolean, // m00885: echoed (user-requested verbatim) emission spans survive. requestText?: string, + // #2328 Q2 — see pipePluginChatWithStrip. + upstreamMeta?: UpstreamMeta, ): Promise { let reader = stream.getReader(); let decoder = new TextDecoder("utf-8"); let buf = ""; const acc: UsageSample = {}; + // #2328 Q2/Q4 — see the chat-pipe twin. + let diagEvents = 0; + let diagBytes = 0; + let diagUnparseable = 0; + const diagLastTypes: string[] = []; + const diagPushType = (label: string) => { + diagLastTypes.push(label); + if (diagLastTypes.length > 5) diagLastTypes.shift(); + }; const onTagDrop = (snippet: string) => { loggerLog("warn", `[tag-echo] stripped model-emitted render tag (plugin passthrough): ${snippet.slice(0, 80).replace(/\n/g, " ")}`); log?.(`[tag-echo] stripped model-emitted render tag from plugin passthrough text`); @@ -2944,6 +3042,29 @@ export async function pipePluginResponsesWithStrip( if (typeof type === "string" && type.startsWith("response.reasoning_summary_")) return true; return type === "response.created" || type === "response.output_item.added" || type === "response.content_part.added"; }; + // #2328 Q2/Q4 — see the chat-pipe twin. Responses-specific extras: the + // last completion-family status observed (a failed/incomplete status on a + // "truncated" stream changes the diagnosis) and how many done-family + // frames were still held for the degenerate-turn retry when the cut hit. + const buildTruncationDiag = (cause: "eof" | "read-error"): TruncationDiag => ({ + cause, + classification: buf.length > 0 && tailLooksTerminal(buf, "responses") ? "terminal-bytes-unrecognized" : "no-terminal-seen", + protocol: "responses", + events: diagEvents, + unparseableForwarded: diagUnparseable, + bytes: diagBytes, + lastEventTypes: [...diagLastTypes], + eofBufferBytes: buf.length, + ...(buf.length > 0 ? { eofBufferHead: buf.slice(0, 80).replace(/[\r\n]+/g, "\\n") } : {}), + ...(responseStatus !== undefined ? { responseStatus } : {}), + visibleChars: visibleTextChars, + sawStrippedEcho, + retryZeroByteCutSpent: truncationRetried, + retryDegenerateSpent: degenerateRetried, + heldEvents: heldEvents.length, + ...(upstreamMeta?.status !== undefined ? { upstreamStatus: upstreamMeta.status } : {}), + ...(upstreamMeta?.contentType !== undefined ? { upstreamContentType: upstreamMeta.contentType } : {}), + }); /** Every id the retry carries is rewritten onto the first attempt's, so the * client's assembled item stays the one it already holds. */ const rewriteRetryIds = (ev: Record): void => { @@ -3072,6 +3193,7 @@ export async function pipePluginResponsesWithStrip( buf = ""; pendingFinal = resolved; } else { + if (value && value.length > 0) diagBytes += value.byteLength; pendingFinal = value && value.length > 0 ? decoder.decode(value, { stream: true }) : null; } if (pendingFinal !== null) { @@ -3084,7 +3206,11 @@ export async function pipePluginResponsesWithStrip( if (dataLines.length === 0) continue; const jsonStr = dataLines.map((l) => l.slice(5).replace(/^ /, "")).join("\n").trim(); if (!jsonStr || jsonStr === "[DONE]") { - if (jsonStr === "[DONE]") sawTerminal = true; + if (jsonStr === "[DONE]") { + diagEvents++; + diagPushType("[DONE]"); + sawTerminal = true; + } await write(rawEvent + "\n\n"); continue; } @@ -3092,11 +3218,16 @@ export async function pipePluginResponsesWithStrip( try { ev = JSON.parse(jsonStr) as Record; } catch { + diagEvents++; + diagUnparseable++; + diagPushType("unparseable"); // #2190: unparseable frames bypass every filter — audit them. auditRawForward(rawEvent); await write(rawEvent + "\n\n"); continue; } + diagEvents++; + if (typeof ev["type"] === "string") diagPushType(ev["type"] as string); const sample = usageFromSseEvent(ev); if (sample) mergeUsageSample(acc, sample); const type = ev["type"]; @@ -3305,7 +3436,7 @@ export async function pipePluginResponsesWithStrip( // done-family event. Responses has no separate finish-reason concept // (terminal events carry the status), so this is always the error shape. if (!sawTerminal && !res.destroyed && !res.writableEnded) { - emitUpstreamTruncation(res, "responses", false, log); + emitUpstreamTruncation(res, "responses", false, log, buildTruncationDiag("eof")); return; } } catch (e) { @@ -3326,7 +3457,7 @@ export async function pipePluginResponsesWithStrip( /* client half-gone; the emission below is best-effort too */ } loggerLog("warn", `[plugin] upstream stream read failed (responses): ${String(e instanceof Error ? e.message : e)} — emitting in-band truncation signal`); - emitUpstreamTruncation(res, "responses", false, log); + emitUpstreamTruncation(res, "responses", false, log, buildTruncationDiag("read-error")); return; } finally { reader.releaseLock(); diff --git a/src/server.ts b/src/server.ts index 60a29f790c..53df1224e6 100644 --- a/src/server.ts +++ b/src/server.ts @@ -118,6 +118,7 @@ import { personaNamespace } from "./persona-anchor.js"; import { prefixAffinity, type AnonymousAffinity } from "./prefix-affinity.js"; import { maybeAdoptForkBlocks, maybeAdoptResume } from "./fork-adoption.js"; import { publicForkInputMatches } from "./plugin.js"; +import type { UpstreamMeta } from "./plugin.js"; import { flushPrefixAffinity, hydratePrefixAffinity, scheduleAffinityPersist } from "./affinity-persist.js"; import { setSimhashAdoptionEnabled } from "./prefix-affinity.js"; import { consumePluginRegisterFor, flushConversations, handlePluginCompact, handlePluginManifest, handlePluginFork, handlePluginSnapshot, handlePluginRegister, handlePluginRuntimeInfo, handlePluginStatus, handlePluginTool, isPluginFoldCallId, loadConversations, pipePluginChatWithStrip, pipePluginJson, pipePluginResponsesWithStrip, pluginAgentHeader, pluginConversationHeader, pluginHeadersMatchModel, pluginReportedContextWindow, pluginReportedMaxOutput, pluginRequestAgentHeader, pluginRuntimeInfoFor, pluginRuntimeInfoForConversation, recordChainVerdict, recordPluginSession, rememberPluginMessages, resolveConversation, runtimeConversationId, takePendingPluginRegister } from "./plugin.js"; @@ -7047,6 +7048,11 @@ async function forward( dumpRaw = dumpStreamToFile(b, opts.dumpSse, `${Date.now()}-${safeSessionId(prepared.session.id)}-raw.sse`); } if (prepared.stream) { + // #2328: what the pipes know about the upstream HTTP response — + // first attempt's status/content-type (a resolveFakeCompletion + // re-send is not reflected; the pipes' own diag says which + // retry budget was spent instead). + const upstreamMeta: UpstreamMeta = { status: upstream.status, contentType: upstream.headers.get("content-type") ?? undefined }; if (prepared.protocol === "responses") { // #732/#821 applies to this pipe too (#871): the agent's own // body, held here with its URL and headers, is re-issued once @@ -7070,6 +7076,7 @@ async function forward( targetOrigin, absorbInstructed, wireBodyText, + upstreamMeta, ); } else { // #732/#821: the plugin pipe re-issues the agent's own body @@ -7096,6 +7103,7 @@ async function forward( targetOrigin, absorbInstructed, wireBodyText, + upstreamMeta, ); } } else { @@ -7186,6 +7194,8 @@ async function forward( prepared.compressInjected && prepared.processedMessages.length > 0; if (!useRewriter || prepared === null) { + // #2328: pipes cite the first upstream response's HTTP identity. + const upstreamMeta: UpstreamMeta = { status: upstream.status, contentType: upstream.headers.get("content-type") ?? undefined }; if (prepared && prepared.resetAfterSuccess) { const [toClient, toObserve] = responseBody.tee(); const observed = observeResponsesTerminalState(toObserve, prepared.stream); @@ -7212,6 +7222,10 @@ async function forward( log, label: prepared.session.id, }), + undefined, + undefined, + undefined, + upstreamMeta, ); } else { await pipeThrough(toClient, res); @@ -7252,6 +7266,10 @@ async function forward( log, label: p.session.id, }), + undefined, + undefined, + undefined, + upstreamMeta, ); } else { await pipePluginChatWithStrip( @@ -7271,6 +7289,10 @@ async function forward( log, label: p.session.id, }), + undefined, + undefined, + undefined, + upstreamMeta, ); } } else if ( diff --git a/src/stream-error.ts b/src/stream-error.ts index 87f1a0bbb7..1136c804d6 100644 --- a/src/stream-error.ts +++ b/src/stream-error.ts @@ -131,32 +131,83 @@ export function emitStreamError(res: http.ServerResponse, protocol: Protocol, me * only reachable case from a pipe that treats that chunk as terminal). * Never throws. */ -export function emitUpstreamTruncation(res: http.ServerResponse, protocol: Protocol, finished: boolean, log?: (msg: string) => void): void { +/** #2328 Q2: termination diagnostics carried on the in-band truncation + * error frame (`meta`) and in the log line, so "the upstream never sent a + * terminal event" (#721's no-terminal-seen) and "a terminal was arriving but + * its bytes never completed / were unrecognizable" can be told apart from + * the client side (#2347 forensics). Every field is plain-JSON safe. */ +export interface TruncationDiag { + /** Which pipe path fired: clean EOF without a terminal, or a read error. */ + cause: "eof" | "read-error"; + /** Q4 classification — see the helpers in plugin.ts that build this. */ + classification: "no-terminal-seen" | "terminal-bytes-unrecognized"; + /** Wire protocol of the truncated stream. */ + protocol: string; + /** Parsed SSE events (incl. [DONE] frames; excl. unparseable raw forwards). */ + events: number; + /** Frames forwarded verbatim because JSON.parse failed (#2190 audit twin). */ + unparseableForwarded: number; + /** Sum of decoded upstream bytes across the whole stream. */ + bytes: number; + /** Last ≤5 parsed event labels, oldest first (wire-native `type`, or a + * synthesized `chunk`/`chunk:finish=` for wires without one). */ + lastEventTypes: string[]; + /** Dangling partial event left in the normalize buffer at EOF/cut. */ + eofBufferBytes: number; + /** First ≤80 chars of that dangling buffer, newlines flattened. */ + eofBufferHead?: string; + /** Finish reason already delivered before the cut (chat wires), if any. */ + finishReason?: string; + /** Responses wire: status of the last completion-family frame, if any. */ + responseStatus?: string; + /** Post-filter visible chars the client actually assembled. */ + visibleChars: number; + /** The tag filter stripped echo markup somewhere in this stream. */ + sawStrippedEcho: boolean; + /** The one-shot #2171 re-issue was already spent earlier in this request. */ + retryZeroByteCutSpent: boolean; + /** The one-shot #732/#821 degenerate-turn re-issue was already spent. */ + retryDegenerateSpent: boolean; + /** Held done-family frames still buffered at the cut (responses wire). */ + heldEvents?: number; + /** Upstream HTTP status / content-type of the original response, when the + * caller still holds them (plugin lane re-fetches lose them by design). */ + upstreamStatus?: number; + upstreamContentType?: string; +} + +export function emitUpstreamTruncation(res: http.ServerResponse, protocol: Protocol, finished: boolean, log?: (msg: string) => void, diag?: TruncationDiag): void { const message = "upstream stream ended before a completion event; this turn may be incomplete"; const action = finished ? protocol === "google" ? "finish reason already delivered, stream complete" : "finish reason seen, synthesizing missing terminal byte" : "emitting in-band error"; - log?.(`[acp-proxy: upstream stream truncated (${protocol}) — ${action}]`); + // #2328: the diag blob rides the log line too — one grep for + // `upstream stream truncated` pulls the full classification + counters + // without needing the client-side error frame. + log?.(`[acp-proxy: upstream stream truncated (${protocol}) — ${action}${diag ? `; classification=${diag.classification} cause=${diag.cause}; diag=${JSON.stringify(diag)}` : ""}]`); + // #2328: attach the diagnostics as `meta` on the in-band error object. + // Wire-shape change (new field on an error-only frame) — human-merge item. + const metaField = diag === undefined ? {} : { meta: diag }; try { if (protocol === "openai") { if (finished) { safeWrite(res, "data: [DONE]\n\n"); } else { - safeWrite(res, `data: ${JSON.stringify({ error: { type: "server_error", code: "upstream_stream_truncated", message } })}\n\ndata: [DONE]\n\n`); + safeWrite(res, `data: ${JSON.stringify({ error: { type: "server_error", code: "upstream_stream_truncated", message, ...metaField } })}\n\ndata: [DONE]\n\n`); } } else if (protocol === "responses") { - safeWrite(res, `event: error\ndata: ${JSON.stringify({ type: "error", code: "upstream_stream_truncated", message })}\n\n`); + safeWrite(res, `event: error\ndata: ${JSON.stringify({ type: "error", code: "upstream_stream_truncated", message, ...metaField })}\n\n`); } else if (protocol === "google") { // The finishReason chunk IS Gemini's stream terminator: when it was // delivered, the client is already done and an extra terminal // object would read as a second answer. Only the mid-flight cut // needs the error frame (503/UNAVAILABLE — an upstream-cut stream). - if (!finished) safeWrite(res, `data: ${JSON.stringify({ error: { code: 503, message, status: "UNAVAILABLE" } })}\n\n`); + if (!finished) safeWrite(res, `data: ${JSON.stringify({ error: { code: 503, message, status: "UNAVAILABLE", ...metaField } })}\n\n`); } else { if (finished) { safeWrite(res, `event: message_stop\ndata: ${JSON.stringify({ type: "message_stop" })}\n\n`); } else { - safeWrite(res, `event: error\ndata: ${JSON.stringify({ type: "error", error: { type: "server_error", code: "upstream_stream_truncated", message } })}\n\n`); + safeWrite(res, `event: error\ndata: ${JSON.stringify({ type: "error", error: { type: "server_error", code: "upstream_stream_truncated", message, ...metaField } })}\n\n`); } } } catch { diff --git a/tests/issue2328-terminal-diag.test.ts b/tests/issue2328-terminal-diag.test.ts new file mode 100644 index 0000000000..458caec0f6 --- /dev/null +++ b/tests/issue2328-terminal-diag.test.ts @@ -0,0 +1,245 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { pipePluginChatWithStrip, pipePluginResponsesWithStrip } from "../src/plugin.ts"; + +// #2328 Q2/Q4: the plugin-lane strip pipes now attach termination +// diagnostics (meta) to the in-band upstream_stream_truncated error frame and +// to the log line, and classify the cut: +// - no-terminal-seen: EOF/read-error with no terminal bytes in flight. +// - terminal-bytes-unrecognized: the dangling partial event at the cut +// carries the opening bytes of the wire's own terminal ([DONE], +// message_stop, finishReason chunk, response.completed family). +// This is what makes "the upstream never finished the turn" distinguishable +// from "the terminal arrived but its bytes never completed" (#2347 forensics) +// without needing a raw-SSE dump. + +function makeRes(chunks: string[]) { + return { + writes: chunks, + write(b: Buffer | string) { + chunks.push(typeof b === "string" ? b : b.toString("utf8")); + return true; + }, + end(b?: Buffer | string) { + if (b !== undefined) chunks.push(typeof b === "string" ? b : b.toString("utf8")); + }, + once() {}, + destroyed: false, + writableEnded: false, + } as unknown as import("node:http").ServerResponse; +} + +function streamOf(events: string[], then?: "error"): ReadableStream { + const enc = new TextEncoder(); + let i = 0; + return new ReadableStream({ + pull(controller) { + if (i < events.length) { + controller.enqueue(enc.encode(events[i])); + i += 1; + } else if (then === "error") { + controller.error(new Error("upstream socket reset mid-stream")); + } else { + controller.close(); + } + }, + }); +} + +/** Pull the first data-frame JSON that carries the truncation code. */ +function truncationMeta(writes: string[], at: "top" | "nested"): Record | undefined { + for (const w of writes) { + for (const frame of w.split("\n\n")) { + const m = frame.match(/^(?:event: [^\n]+\n)?data: (.*)$/s); + if (!m) continue; + try { + const ev = JSON.parse(m[1]!) as Record; + if (at === "top" && ev["code"] === "upstream_stream_truncated") return ev["meta"] as Record; + if (at === "nested") { + const err = ev["error"] as Record | undefined; + // google's error object carries a numeric code (503) + + // gRPC status instead of the string code. + if (err && (err["code"] === "upstream_stream_truncated" || err["status"] === "UNAVAILABLE")) return err["meta"] as Record; + } + } catch { + /* not a JSON frame */ + } + } + } + return undefined; +} + +const openaiChunk = (fr: string | null) => `data: ${JSON.stringify({ object: "chat.completion.chunk", choices: [{ index: 0, delta: fr === null ? { content: "hi" } : {}, finish_reason: fr }] })}\n\n`; + +test("chat pipe (openai): dangling `data: [DO` at EOF classifies as terminal-bytes-unrecognized", async () => { + const out: string[] = []; + const logs: string[] = []; + await pipePluginChatWithStrip( + streamOf([openaiChunk(null), "data: [DO"]), + makeRes(out), + "openai", + undefined, + (m) => logs.push(m), + ); + const meta = truncationMeta(out, "nested"); + assert.ok(meta, "error frame carries meta"); + assert.equal(meta!["classification"], "terminal-bytes-unrecognized"); + assert.equal(meta!["cause"], "eof"); + assert.equal(meta!["protocol"], "openai"); + assert.equal(meta!["eofBufferBytes"], "data: [DO".length); + assert.equal(meta!["eofBufferHead"], "data: [DO"); + assert.deepEqual(meta!["lastEventTypes"], ["chunk"]); + assert.equal(meta!["events"], 1); + assert.ok((meta!["bytes"] as number) > 0); + assert.equal(meta!["retryZeroByteCutSpent"], false); + assert.equal(meta!["retryDegenerateSpent"], false); + // wire shape: openai meta nests inside `error`, and the [DONE] tail still + // follows the error frame (#721 contract unchanged). + assert.ok(out.join("").includes("data: [DONE]")); + // log line: one grep-able line with the classification + full diag. + const line = logs.find((l) => l.includes("upstream stream truncated")); + assert.ok(line, "log line emitted"); + assert.ok(line!.includes("classification=terminal-bytes-unrecognized")); + assert.ok(line!.includes("cause=eof")); + assert.ok(line!.includes("diag=")); +}); + +test("chat pipe (anthropic): clean EOF with no terminal classifies as no-terminal-seen", async () => { + const out: string[] = []; + await pipePluginChatWithStrip( + streamOf([ + `event: message_start\ndata: ${JSON.stringify({ type: "message_start" })}\n\n`, + `event: content_block_delta\ndata: ${JSON.stringify({ type: "content_block_delta", index: 0, delta: { type: "text_delta", text: "hi" } })}\n\n`, + ]), + makeRes(out), + "anthropic", + ); + const meta = truncationMeta(out, "nested"); + assert.ok(meta, "error frame carries meta"); + assert.equal(meta!["classification"], "no-terminal-seen"); + assert.equal(meta!["cause"], "eof"); + assert.equal(meta!["eofBufferBytes"], 0); + assert.equal(meta!["eofBufferHead"], undefined); + assert.deepEqual(meta!["lastEventTypes"], ["message_start", "content_block_delta"]); + assert.ok((meta!["visibleChars"] as number) >= 2); + assert.ok(out.join("").includes("event: error")); +}); + +test("chat pipe (google): meta rides the UNAVAILABLE error object and counts unparseable forwards", async () => { + const out: string[] = []; + await pipePluginChatWithStrip( + streamOf([ + // one parseable chunk, then one unparseable frame (forwarded raw), + // then EOF with nothing terminal in flight. + `data: ${JSON.stringify({ candidates: [{ index: 0, content: { role: "model", parts: [{ text: "hi" }] } }] })}\n\n`, + "data: {{{not-json\n\n", + ]), + makeRes(out), + "google", + ); + const meta = truncationMeta(out, "nested"); + assert.ok(meta, "error frame carries meta"); + assert.equal(meta!["classification"], "no-terminal-seen"); + assert.equal(meta!["unparseableForwarded"], 1); + assert.deepEqual(meta!["lastEventTypes"], ["chunk", "unparseable"]); + const frame = out.join(""); + // Gemini's error object carries a numeric code + gRPC status, no string + // code (see emitUpstreamTruncation) — UNAVAILABLE identifies the frame. + assert.ok(frame.includes('"status":"UNAVAILABLE"')); + assert.ok(frame.includes('"code":503')); +}); + +test("chat pipe: upstreamMeta threads HTTP status/content-type into the diag", async () => { + const out: string[] = []; + await pipePluginChatWithStrip( + streamOf([openaiChunk(null)]), + makeRes(out), + "openai", + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + { status: 200, contentType: "text/event-stream" }, + ); + const meta = truncationMeta(out, "nested"); + assert.ok(meta); + assert.equal(meta!["upstreamStatus"], 200); + assert.equal(meta!["upstreamContentType"], "text/event-stream"); +}); + +test("responses pipe: dangling response.completed head classifies as terminal-bytes-unrecognized", async () => { + const out: string[] = []; + const sse = (e: string, d: Record) => `event: ${e}\ndata: ${JSON.stringify({ type: e, ...d })}\n\n`; + await pipePluginResponsesWithStrip( + streamOf([ + sse("response.created", { response: { id: "resp_1" } }), + sse("response.output_item.added", { output_index: 0, item: { type: "message", id: "msg_1" } }), + sse("response.content_part.added", { item_id: "msg_1", output_index: 0 }), + sse("response.output_text.delta", { item_id: "msg_1", output_index: 0, delta: "he" }), + 'event: response.completed\ndata: {"type":"response.comp', + ]), + makeRes(out), + ); + const meta = truncationMeta(out, "top"); + assert.ok(meta, "error frame carries meta"); + assert.equal(meta!["classification"], "terminal-bytes-unrecognized"); + assert.equal(meta!["cause"], "eof"); + assert.equal(meta!["protocol"], "responses"); + assert.ok((meta!["eofBufferHead"] as string).includes("response.comp")); + assert.deepEqual(meta!["lastEventTypes"], [ + "response.created", + "response.output_item.added", + "response.content_part.added", + "response.output_text.delta", + ]); + assert.equal(meta!["heldEvents"], 0); + assert.ok((meta!["visibleChars"] as number) >= 2); +}); + +test("responses pipe: clean EOF after deltas classifies as no-terminal-seen", async () => { + const out: string[] = []; + const sse = (e: string, d: Record) => `event: ${e}\ndata: ${JSON.stringify({ type: e, ...d })}\n\n`; + await pipePluginResponsesWithStrip( + streamOf([ + sse("response.created", { response: { id: "resp_1" } }), + sse("response.output_item.added", { output_index: 0, item: { type: "message", id: "msg_1" } }), + sse("response.content_part.added", { item_id: "msg_1", output_index: 0 }), + sse("response.output_text.delta", { item_id: "msg_1", output_index: 0, delta: "hi" }), + ]), + makeRes(out), + ); + const meta = truncationMeta(out, "top"); + assert.ok(meta); + assert.equal(meta!["classification"], "no-terminal-seen"); + assert.equal(meta!["eofBufferBytes"], 0); +}); + +test("responses pipe: upstream read error records cause=read-error", async () => { + const out: string[] = []; + const sse = (e: string, d: Record) => `event: ${e}\ndata: ${JSON.stringify({ type: e, ...d })}\n\n`; + await pipePluginResponsesWithStrip( + streamOf([sse("response.output_text.delta", { item_id: "msg_1", output_index: 0, delta: "x" })], "error"), + makeRes(out), + ); + const meta = truncationMeta(out, "top"); + assert.ok(meta); + assert.equal(meta!["cause"], "read-error"); + assert.equal(meta!["classification"], "no-terminal-seen"); +}); + +test("chat pipe (anthropic): dangling message_stop head classifies as terminal-bytes-unrecognized", async () => { + const out: string[] = []; + await pipePluginChatWithStrip( + streamOf([ + `event: content_block_delta\ndata: ${JSON.stringify({ type: "content_block_delta", index: 0, delta: { type: "text_delta", text: "hi" } })}\n\n`, + "event: message_stop\ndata: {\"type\":\"mess", + ]), + makeRes(out), + "anthropic", + ); + const meta = truncationMeta(out, "nested"); + assert.ok(meta); + assert.equal(meta!["classification"], "terminal-bytes-unrecognized"); +});