diff --git a/src/plugin.ts b/src/plugin.ts index 649fda6aee..782eb3cca7 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -1205,6 +1205,13 @@ export async function pipePluginChatWithStrip( let reader = stream.getReader(); let decoder = new TextDecoder("utf-8"); let buf = ""; + // #1706: truncation forensics — when the stream dies without a terminal, + // "bytes in N ms, last byte M ms ago" names the cutting hop's idle budget + // (M≈BILI_UPSTREAM_TIMEOUT_MS ⇒ bili's own watchdog; M≈300s/600s ⇒ a + // middlebox; 0B ⇒ the connection never produced anything). + const tStart = Date.now(); + let lastByteAt = tStart; + let upstreamBytes = 0; const acc: UsageSample = {}; let sawStrippedEcho = false; const onTagDrop = (snippet: string) => { @@ -1811,6 +1818,8 @@ export async function pipePluginChatWithStrip( const { done, value } = await reader.read(); if (done) break; if (value && value.length > 0) { + lastByteAt = Date.now(); + upstreamBytes += value.length; buf = normalizeSseLineEndings(buf + decoder.decode(value, { stream: true })); let idx: number; while ((idx = buf.indexOf("\n\n")) !== -1) { @@ -1880,13 +1889,24 @@ export async function pipePluginChatWithStrip( // stream bare — the agent would persist the partial turn as complete // (the #719 chain). finished=true when a finish reason was delivered: // only the trailing terminal byte ([DONE]/message_stop) is missing. + // A final `data: [DONE]` that arrived without its terminating blank + // line is still the completion marker (some gateways close right after + // the line, #1706): recognize it at EOF and forward it properly + // terminated instead of reporting a completed stream as truncated. + let danglingDone = false; + if (!sawTerminal && protocol === "openai" && buf.length > 0) { + const dataLines = buf.split("\n").filter((l) => l.startsWith("data:")); + danglingDone = dataLines.length === 1 && dataLines[0].slice(5).replace(/^ /, "").trim() === "[DONE]"; + if (danglingDone) sawTerminal = true; + } const truncated = !sawTerminal && !res.destroyed && !res.writableEnded; // A dangling partial event left in buf by a mid-event cut would fuse // with the next complete frame — SSE joins every data line inside one // blank-line-delimited block — corrupting the truncation signal. Drop // it when the signal follows: an unterminated event is unparseable by - // the client anyway (same as the pre-#721 bare end). - if (!truncated && buf.length > 0 && !res.destroyed && !res.writableEnded) await write(offsetRetryIndices(buf)); + // the client anyway (same as the pre-#721 bare end). The recognized + // [DONE] sentinel gets its terminator restored (#1706). + if (!truncated && buf.length > 0 && !res.destroyed && !res.writableEnded) await write(offsetRetryIndices(danglingDone ? buf.replace(/\n+$/, "") + "\n\n" : buf)); const rest = flushTails(); if (rest.length > 0 && !res.destroyed && !res.writableEnded) await write(offsetRetryIndices(rest)); // Settle BEFORE res.end() in the finally below: the client can issue @@ -1898,6 +1918,7 @@ export async function pipePluginChatWithStrip( maybeWarnProtocolFragment(); maybeWarnNamelessToolCalls(); if (truncated) { + loggerLog("warn", `[plugin] upstream stream ended without terminal (${protocol}): ${upstreamBytes}B in ${Date.now() - tStart}ms, last byte ${Date.now() - lastByteAt}ms ago — emitting in-band truncation signal (#721)`); emitUpstreamTruncation(res, protocol, finalFinishReason !== undefined, log); return; } @@ -1920,7 +1941,7 @@ export async function pipePluginChatWithStrip( } catch { /* client half-gone; the emission below is best-effort too */ } - loggerLog("warn", `[plugin] upstream stream read failed (${protocol}): ${String(e instanceof Error ? e.message : e)} — emitting in-band truncation signal`); + loggerLog("warn", `[plugin] upstream stream read failed (${protocol}): ${String(e instanceof Error ? e.message : e)}; ${upstreamBytes}B in ${Date.now() - tStart}ms, last byte ${Date.now() - lastByteAt}ms ago — emitting in-band truncation signal (#721)`); emitUpstreamTruncation(res, protocol, finalFinishReason !== undefined, log); return; } finally { @@ -2003,6 +2024,10 @@ export async function pipePluginResponsesWithStrip( let reader = stream.getReader(); let decoder = new TextDecoder("utf-8"); let buf = ""; + // #1706: truncation forensics — same contract as the chat-pipe twin. + const tStart = Date.now(); + let lastByteAt = tStart; + let upstreamBytes = 0; const acc: UsageSample = {}; const onTagDrop = (snippet: string) => { loggerLog("warn", `[tag-echo] stripped model-emitted render tag (plugin passthrough): ${snippet.slice(0, 80).replace(/\n/g, " ")}`); @@ -2255,6 +2280,8 @@ export async function pipePluginResponsesWithStrip( const { done, value } = await reader.read(); if (done) break; if (value && value.length > 0) { + lastByteAt = Date.now(); + upstreamBytes += value.length; buf = normalizeSseLineEndings(buf + decoder.decode(value, { stream: true })); let idx: number; while ((idx = buf.indexOf("\n\n")) !== -1) { @@ -2457,6 +2484,7 @@ export async function pipePluginResponsesWithStrip( // done-family event. Responses has no separate finish-reason concept // (terminal events carry the status), so this is always the error shape. if (!sawTerminal && !res.destroyed && !res.writableEnded) { + loggerLog("warn", `[plugin] upstream stream ended without terminal (responses): ${upstreamBytes}B in ${Date.now() - tStart}ms, last byte ${Date.now() - lastByteAt}ms ago — emitting in-band truncation signal (#721)`); emitUpstreamTruncation(res, "responses", false, log); return; } @@ -2476,7 +2504,7 @@ export async function pipePluginResponsesWithStrip( } catch { /* client half-gone; the emission below is best-effort too */ } - loggerLog("warn", `[plugin] upstream stream read failed (responses): ${String(e instanceof Error ? e.message : e)} — emitting in-band truncation signal`); + loggerLog("warn", `[plugin] upstream stream read failed (responses): ${String(e instanceof Error ? e.message : e)}; ${upstreamBytes}B in ${Date.now() - tStart}ms, last byte ${Date.now() - lastByteAt}ms ago — emitting in-band truncation signal (#721)`); emitUpstreamTruncation(res, "responses", false, log); return; } finally { diff --git a/tests/issue1706-dangling-done.test.ts b/tests/issue1706-dangling-done.test.ts new file mode 100644 index 0000000000..90839592b2 --- /dev/null +++ b/tests/issue1706-dangling-done.test.ts @@ -0,0 +1,190 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { pipePluginChatWithStrip } from "../src/plugin.ts"; +import { setLogCapture } from "../src/logger.ts"; +import type { Session } from "../src/session.ts"; +import type http from "node:http"; + +// #1706: an upstream that emits its final `data: [DONE]` without the +// terminating blank line (and omits finish_reason) used to be reported as a +// truncated turn even though the completion marker arrived. Those streams are +// now recognized as complete at EOF; genuine cuts must STILL be reported. + +const TRUNC_MSG = "upstream stream ended before a completion event; this turn may be incomplete"; + +function makeSession(): Session { + return { + id: "testsess", + protocol: "openai", + upstreamOrigin: "http://127.0.0.1:9/v1", + label: "test", + createdAt: 0, + lastUsedAt: 0, + requests: 0, + lastInputTokens: 0, + stats: {}, + dirty: false, + } as unknown as Session; +} + +function makeRes(chunks: string[]) { + return { + writes: chunks, + write(b: Buffer | string) { + chunks.push(typeof b === "string" ? b : b.toString("utf8")); + return true; + }, + end(b?: Buffer | string) { + if (b !== undefined) chunks.push(typeof b === "string" ? b : b.toString("utf8")); + }, + once() {}, + destroyed: false, + writableEnded: false, + } as unknown as http.ServerResponse; +} + +function streamOf(events: string[]): ReadableStream { + const enc = new TextEncoder(); + let i = 0; + return new ReadableStream({ + pull(controller) { + if (i < events.length) { + controller.enqueue(enc.encode(events[i])); + i += 1; + } else { + controller.close(); + } + }, + }); +} + +function chatChunk(delta: Record, extra: Record = {}): string { + return `data: ${JSON.stringify({ id: "chatcmpl-1", object: "chat.completion.chunk", created: 1, model: "qwen", choices: [{ index: 0, delta, finish_reason: null }], ...extra })}\n\n`; +} + +async function run(events: string[]): Promise<{ out: string; logs: string[] }> { + const out: string[] = []; + const logs: string[] = []; + await pipePluginChatWithStrip(streamOf(events), makeRes(out), "openai", makeSession(), (m) => logs.push(m)); + return { out: out.join(""), logs }; +} + +test("#1706: final [DONE] missing only its terminating blank line is complete, not truncated", async () => { + const { out, logs } = await run([ + chatChunk({ role: "assistant" }), + chatChunk({ content: "hello" }), + "data: [DONE]\n", + ]); + assert.ok(!out.includes("upstream_stream_truncated"), `no error frame, got: ${out}`); + assert.ok(!logs.some((l) => l.includes("emitting in-band error")), `no truncation log, got: ${JSON.stringify(logs)}`); + assert.ok(out.endsWith("data: [DONE]\n\n"), `forwarded terminated sentinel, got tail: ${JSON.stringify(out.slice(-40))}`); +}); + +test("#1706: bare [DONE] with no newlines at all is still the completion marker", async () => { + const { out, logs } = await run([ + chatChunk({ role: "assistant" }), + chatChunk({ content: "hello" }), + "data: [DONE]", + ]); + assert.ok(!out.includes("upstream_stream_truncated"), `no error frame, got: ${out}`); + assert.ok(!logs.some((l) => l.includes("emitting in-band error"))); + assert.ok(out.endsWith("data: [DONE]\n\n"), `terminated sentinel restored, got tail: ${JSON.stringify(out.slice(-40))}`); +}); + +test("#1706: [DONE] without the space after the colon is normalized like the in-stream form", async () => { + const { out } = await run([ + chatChunk({ content: "hi" }), + "data:[DONE]\n", + ]); + assert.ok(!out.includes("upstream_stream_truncated"), `no error frame, got: ${out}`); + assert.ok(out.endsWith("data:[DONE]\n\n"), `sentinel forwarded verbatim + terminator, got tail: ${JSON.stringify(out.slice(-40))}`); +}); + +test("#1706 control: finish_reason delivered with no [DONE] at all stays on the silent-synthesis path", async () => { + const { out, logs } = await run([ + chatChunk({ content: "done" }), + chatChunk({}, { choices: [{ index: 0, delta: {}, finish_reason: "stop" }] }), + ]); + assert.ok(!out.includes("upstream_stream_truncated"), `no error frame, got: ${out}`); + assert.ok(logs.some((l) => l.includes("synthesizing missing terminal byte")), `synthesis log expected, got: ${JSON.stringify(logs)}`); + assert.ok(out.endsWith("data: [DONE]\n\n"), `got tail: ${JSON.stringify(out.slice(-40))}`); +}); + +test("#1706 control: genuine mid-stream cut still emits the in-band truncation signal + forensics log", async () => { + const sink: string[] = []; + setLogCapture((_level, msg) => sink.push(msg)); + let result: { out: string; logs: string[] }; + try { + result = await run([ + chatChunk({ role: "assistant" }), + chatChunk({ content: "hel" }), + ]); + } finally { + setLogCapture(null); + } + assert.ok(result.out.includes("upstream_stream_truncated"), `error frame expected, got: ${result.out}`); + assert.ok(result.out.includes(TRUNC_MSG), `message expected, got: ${result.out}`); + assert.ok(result.logs.some((l) => l.includes("emitting in-band error")), `got: ${JSON.stringify(result.logs)}`); + assert.ok(result.out.endsWith("data: [DONE]\n\n"), `client still gets a terminal, got tail: ${JSON.stringify(result.out.slice(-40))}`); + // #1706 forensics: bytes/elapsed/silence since last byte name the cutting hop's idle budget. + const forensic = sink.filter((l) => l.includes("upstream stream ended without terminal (openai)")); + assert.equal(forensic.length, 1, `one forensics line, got: ${JSON.stringify(forensic)}`); + assert.match(forensic[0], /\d+B in \d+ms, last byte \d+ms ago — emitting in-band truncation signal \(#721\)/, forensic[0]); +}); + +test("#1706 control: upstream read failure still emits the signal and logs the error with forensics", async () => { + const enc = new TextEncoder(); + const events = [chatChunk({ role: "assistant" })]; + let i = 0; + const cutStream = new ReadableStream({ + pull(controller) { + if (i < events.length) { + controller.enqueue(enc.encode(events[i])); + i += 1; + } else { + throw new Error("terminated"); + } + }, + }); + const out: string[] = []; + const logs: string[] = []; + const sink: string[] = []; + setLogCapture((_level, msg) => sink.push(msg)); + try { + await pipePluginChatWithStrip(cutStream, makeRes(out), "openai", makeSession(), (m) => logs.push(m)); + } finally { + setLogCapture(null); + } + const joined = out.join(""); + assert.ok(joined.includes("upstream_stream_truncated"), `error frame expected, got: ${joined}`); + assert.ok(logs.some((l) => l.includes("emitting in-band error")), `got: ${JSON.stringify(logs)}`); + const forensic = sink.filter((l) => l.includes("upstream stream read failed (openai): terminated;")); + assert.equal(forensic.length, 1, `one forensics line, got: ${JSON.stringify(sink)}`); + assert.match(forensic[0], /\d+B in \d+ms, last byte \d+ms ago — emitting in-band truncation signal \(#721\)/, forensic[0]); +}); + +test("#1706 control: a dangling block carrying more than one data line is NOT recognized as [DONE]", async () => { + const { out } = await run([ + chatChunk({ content: "x" }), + `data: ${JSON.stringify({ id: "chatcmpl-1", object: "chat.completion.chunk", created: 1, model: "qwen", choices: [{ index: 0, delta: {}, finish_reason: null }] })}`, + "\ndata: [DONE]", + ]); + assert.ok(out.includes("upstream_stream_truncated"), `conservative error frame expected, got: ${out}`); +}); + +test("#1706 control: a sentinel cut mid-token is NOT recognized as [DONE]", async () => { + const { out } = await run([ + chatChunk({ content: "x" }), + "data: [DO\n", + ]); + assert.ok(out.includes("upstream_stream_truncated"), `partial sentinel must stay on the truncation path, got: ${out}`); +}); + +test("#1706 control: properly terminated [DONE] remains clean", async () => { + const { out, logs } = await run([ + chatChunk({ content: "ok" }), + "data: [DONE]\n\n", + ]); + assert.ok(!out.includes("upstream_stream_truncated"), `got: ${out}`); + assert.ok(!logs.some((l) => l.includes("truncated")), `got: ${JSON.stringify(logs)}`); +});