Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 32 additions & 4 deletions src/plugin.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1205,6 +1205,13 @@ export async function pipePluginChatWithStrip(
let reader = stream.getReader();
let decoder = new TextDecoder("utf-8");
let buf = "";
// #1706: truncation forensics — when the stream dies without a terminal,
// "bytes in N ms, last byte M ms ago" names the cutting hop's idle budget
// (M≈BILI_UPSTREAM_TIMEOUT_MS ⇒ bili's own watchdog; M≈300s/600s ⇒ a
// middlebox; 0B ⇒ the connection never produced anything).
const tStart = Date.now();
let lastByteAt = tStart;
let upstreamBytes = 0;
const acc: UsageSample = {};
let sawStrippedEcho = false;
const onTagDrop = (snippet: string) => {
Expand Down Expand Up @@ -1811,6 +1818,8 @@ export async function pipePluginChatWithStrip(
const { done, value } = await reader.read();
if (done) break;
if (value && value.length > 0) {
lastByteAt = Date.now();
upstreamBytes += value.length;
buf = normalizeSseLineEndings(buf + decoder.decode(value, { stream: true }));
let idx: number;
while ((idx = buf.indexOf("\n\n")) !== -1) {
Expand Down Expand Up @@ -1880,13 +1889,24 @@ export async function pipePluginChatWithStrip(
// stream bare — the agent would persist the partial turn as complete
// (the #719 chain). finished=true when a finish reason was delivered:
// only the trailing terminal byte ([DONE]/message_stop) is missing.
// A final `data: [DONE]` that arrived without its terminating blank
// line is still the completion marker (some gateways close right after
// the line, #1706): recognize it at EOF and forward it properly
// terminated instead of reporting a completed stream as truncated.
let danglingDone = false;
if (!sawTerminal && protocol === "openai" && buf.length > 0) {
const dataLines = buf.split("\n").filter((l) => l.startsWith("data:"));
danglingDone = dataLines.length === 1 && dataLines[0].slice(5).replace(/^ /, "").trim() === "[DONE]";
if (danglingDone) sawTerminal = true;
}
const truncated = !sawTerminal && !res.destroyed && !res.writableEnded;
// A dangling partial event left in buf by a mid-event cut would fuse
// with the next complete frame — SSE joins every data line inside one
// blank-line-delimited block — corrupting the truncation signal. Drop
// it when the signal follows: an unterminated event is unparseable by
// the client anyway (same as the pre-#721 bare end).
if (!truncated && buf.length > 0 && !res.destroyed && !res.writableEnded) await write(offsetRetryIndices(buf));
// the client anyway (same as the pre-#721 bare end). The recognized
// [DONE] sentinel gets its terminator restored (#1706).
if (!truncated && buf.length > 0 && !res.destroyed && !res.writableEnded) await write(offsetRetryIndices(danglingDone ? buf.replace(/\n+$/, "") + "\n\n" : buf));
const rest = flushTails();
if (rest.length > 0 && !res.destroyed && !res.writableEnded) await write(offsetRetryIndices(rest));
// Settle BEFORE res.end() in the finally below: the client can issue
Expand All @@ -1898,6 +1918,7 @@ export async function pipePluginChatWithStrip(
maybeWarnProtocolFragment();
maybeWarnNamelessToolCalls();
if (truncated) {
loggerLog("warn", `[plugin] upstream stream ended without terminal (${protocol}): ${upstreamBytes}B in ${Date.now() - tStart}ms, last byte ${Date.now() - lastByteAt}ms ago — emitting in-band truncation signal (#721)`);
emitUpstreamTruncation(res, protocol, finalFinishReason !== undefined, log);
return;
}
Expand All @@ -1920,7 +1941,7 @@ export async function pipePluginChatWithStrip(
} catch {
/* client half-gone; the emission below is best-effort too */
}
loggerLog("warn", `[plugin] upstream stream read failed (${protocol}): ${String(e instanceof Error ? e.message : e)} — emitting in-band truncation signal`);
loggerLog("warn", `[plugin] upstream stream read failed (${protocol}): ${String(e instanceof Error ? e.message : e)}; ${upstreamBytes}B in ${Date.now() - tStart}ms, last byte ${Date.now() - lastByteAt}ms ago — emitting in-band truncation signal (#721)`);
emitUpstreamTruncation(res, protocol, finalFinishReason !== undefined, log);
return;
} finally {
Expand Down Expand Up @@ -2003,6 +2024,10 @@ export async function pipePluginResponsesWithStrip(
let reader = stream.getReader();
let decoder = new TextDecoder("utf-8");
let buf = "";
// #1706: truncation forensics — same contract as the chat-pipe twin.
const tStart = Date.now();
let lastByteAt = tStart;
let upstreamBytes = 0;
const acc: UsageSample = {};
const onTagDrop = (snippet: string) => {
loggerLog("warn", `[tag-echo] stripped model-emitted render tag (plugin passthrough): ${snippet.slice(0, 80).replace(/\n/g, " ")}`);
Expand Down Expand Up @@ -2255,6 +2280,8 @@ export async function pipePluginResponsesWithStrip(
const { done, value } = await reader.read();
if (done) break;
if (value && value.length > 0) {
lastByteAt = Date.now();
upstreamBytes += value.length;
buf = normalizeSseLineEndings(buf + decoder.decode(value, { stream: true }));
let idx: number;
while ((idx = buf.indexOf("\n\n")) !== -1) {
Expand Down Expand Up @@ -2457,6 +2484,7 @@ export async function pipePluginResponsesWithStrip(
// done-family event. Responses has no separate finish-reason concept
// (terminal events carry the status), so this is always the error shape.
if (!sawTerminal && !res.destroyed && !res.writableEnded) {
loggerLog("warn", `[plugin] upstream stream ended without terminal (responses): ${upstreamBytes}B in ${Date.now() - tStart}ms, last byte ${Date.now() - lastByteAt}ms ago — emitting in-band truncation signal (#721)`);
emitUpstreamTruncation(res, "responses", false, log);
return;
}
Expand All @@ -2476,7 +2504,7 @@ export async function pipePluginResponsesWithStrip(
} catch {
/* client half-gone; the emission below is best-effort too */
}
loggerLog("warn", `[plugin] upstream stream read failed (responses): ${String(e instanceof Error ? e.message : e)} — emitting in-band truncation signal`);
loggerLog("warn", `[plugin] upstream stream read failed (responses): ${String(e instanceof Error ? e.message : e)}; ${upstreamBytes}B in ${Date.now() - tStart}ms, last byte ${Date.now() - lastByteAt}ms ago — emitting in-band truncation signal (#721)`);
emitUpstreamTruncation(res, "responses", false, log);
return;
} finally {
Expand Down
190 changes: 190 additions & 0 deletions tests/issue1706-dangling-done.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,190 @@
import { test } from "node:test";
import assert from "node:assert/strict";
import { pipePluginChatWithStrip } from "../src/plugin.ts";
import { setLogCapture } from "../src/logger.ts";
import type { Session } from "../src/session.ts";
import type http from "node:http";

// #1706: an upstream that emits its final `data: [DONE]` without the
// terminating blank line (and omits finish_reason) used to be reported as a
// truncated turn even though the completion marker arrived. Those streams are
// now recognized as complete at EOF; genuine cuts must STILL be reported.

const TRUNC_MSG = "upstream stream ended before a completion event; this turn may be incomplete";

function makeSession(): Session {
return {
id: "testsess",
protocol: "openai",
upstreamOrigin: "http://127.0.0.1:9/v1",
label: "test",
createdAt: 0,
lastUsedAt: 0,
requests: 0,
lastInputTokens: 0,
stats: {},
dirty: false,
} as unknown as Session;
}

function makeRes(chunks: string[]) {
return {
writes: chunks,
write(b: Buffer | string) {
chunks.push(typeof b === "string" ? b : b.toString("utf8"));
return true;
},
end(b?: Buffer | string) {
if (b !== undefined) chunks.push(typeof b === "string" ? b : b.toString("utf8"));
},
once() {},
destroyed: false,
writableEnded: false,
} as unknown as http.ServerResponse;
}

function streamOf(events: string[]): ReadableStream<Uint8Array> {
const enc = new TextEncoder();
let i = 0;
return new ReadableStream<Uint8Array>({
pull(controller) {
if (i < events.length) {
controller.enqueue(enc.encode(events[i]));
i += 1;
} else {
controller.close();
}
},
});
}

function chatChunk(delta: Record<string, unknown>, extra: Record<string, unknown> = {}): string {
return `data: ${JSON.stringify({ id: "chatcmpl-1", object: "chat.completion.chunk", created: 1, model: "qwen", choices: [{ index: 0, delta, finish_reason: null }], ...extra })}\n\n`;
}

async function run(events: string[]): Promise<{ out: string; logs: string[] }> {
const out: string[] = [];
const logs: string[] = [];
await pipePluginChatWithStrip(streamOf(events), makeRes(out), "openai", makeSession(), (m) => logs.push(m));
return { out: out.join(""), logs };
}

test("#1706: final [DONE] missing only its terminating blank line is complete, not truncated", async () => {
const { out, logs } = await run([
chatChunk({ role: "assistant" }),
chatChunk({ content: "hello" }),
"data: [DONE]\n",
]);
assert.ok(!out.includes("upstream_stream_truncated"), `no error frame, got: ${out}`);
assert.ok(!logs.some((l) => l.includes("emitting in-band error")), `no truncation log, got: ${JSON.stringify(logs)}`);
assert.ok(out.endsWith("data: [DONE]\n\n"), `forwarded terminated sentinel, got tail: ${JSON.stringify(out.slice(-40))}`);
});

test("#1706: bare [DONE] with no newlines at all is still the completion marker", async () => {
const { out, logs } = await run([
chatChunk({ role: "assistant" }),
chatChunk({ content: "hello" }),
"data: [DONE]",
]);
assert.ok(!out.includes("upstream_stream_truncated"), `no error frame, got: ${out}`);
assert.ok(!logs.some((l) => l.includes("emitting in-band error")));
assert.ok(out.endsWith("data: [DONE]\n\n"), `terminated sentinel restored, got tail: ${JSON.stringify(out.slice(-40))}`);
});

test("#1706: [DONE] without the space after the colon is normalized like the in-stream form", async () => {
const { out } = await run([
chatChunk({ content: "hi" }),
"data:[DONE]\n",
]);
assert.ok(!out.includes("upstream_stream_truncated"), `no error frame, got: ${out}`);
assert.ok(out.endsWith("data:[DONE]\n\n"), `sentinel forwarded verbatim + terminator, got tail: ${JSON.stringify(out.slice(-40))}`);
});

test("#1706 control: finish_reason delivered with no [DONE] at all stays on the silent-synthesis path", async () => {
const { out, logs } = await run([
chatChunk({ content: "done" }),
chatChunk({}, { choices: [{ index: 0, delta: {}, finish_reason: "stop" }] }),
]);
assert.ok(!out.includes("upstream_stream_truncated"), `no error frame, got: ${out}`);
assert.ok(logs.some((l) => l.includes("synthesizing missing terminal byte")), `synthesis log expected, got: ${JSON.stringify(logs)}`);
assert.ok(out.endsWith("data: [DONE]\n\n"), `got tail: ${JSON.stringify(out.slice(-40))}`);
});

test("#1706 control: genuine mid-stream cut still emits the in-band truncation signal + forensics log", async () => {
const sink: string[] = [];
setLogCapture((_level, msg) => sink.push(msg));
let result: { out: string; logs: string[] };
try {
result = await run([
chatChunk({ role: "assistant" }),
chatChunk({ content: "hel" }),
]);
} finally {
setLogCapture(null);
}
assert.ok(result.out.includes("upstream_stream_truncated"), `error frame expected, got: ${result.out}`);
assert.ok(result.out.includes(TRUNC_MSG), `message expected, got: ${result.out}`);
assert.ok(result.logs.some((l) => l.includes("emitting in-band error")), `got: ${JSON.stringify(result.logs)}`);
assert.ok(result.out.endsWith("data: [DONE]\n\n"), `client still gets a terminal, got tail: ${JSON.stringify(result.out.slice(-40))}`);
// #1706 forensics: bytes/elapsed/silence since last byte name the cutting hop's idle budget.
const forensic = sink.filter((l) => l.includes("upstream stream ended without terminal (openai)"));
assert.equal(forensic.length, 1, `one forensics line, got: ${JSON.stringify(forensic)}`);
assert.match(forensic[0], /\d+B in \d+ms, last byte \d+ms ago — emitting in-band truncation signal \(#721\)/, forensic[0]);
});

test("#1706 control: upstream read failure still emits the signal and logs the error with forensics", async () => {
const enc = new TextEncoder();
const events = [chatChunk({ role: "assistant" })];
let i = 0;
const cutStream = new ReadableStream<Uint8Array>({
pull(controller) {
if (i < events.length) {
controller.enqueue(enc.encode(events[i]));
i += 1;
} else {
throw new Error("terminated");
}
},
});
const out: string[] = [];
const logs: string[] = [];
const sink: string[] = [];
setLogCapture((_level, msg) => sink.push(msg));
try {
await pipePluginChatWithStrip(cutStream, makeRes(out), "openai", makeSession(), (m) => logs.push(m));
} finally {
setLogCapture(null);
}
const joined = out.join("");
assert.ok(joined.includes("upstream_stream_truncated"), `error frame expected, got: ${joined}`);
assert.ok(logs.some((l) => l.includes("emitting in-band error")), `got: ${JSON.stringify(logs)}`);
const forensic = sink.filter((l) => l.includes("upstream stream read failed (openai): terminated;"));
assert.equal(forensic.length, 1, `one forensics line, got: ${JSON.stringify(sink)}`);
assert.match(forensic[0], /\d+B in \d+ms, last byte \d+ms ago — emitting in-band truncation signal \(#721\)/, forensic[0]);
});

test("#1706 control: a dangling block carrying more than one data line is NOT recognized as [DONE]", async () => {
const { out } = await run([
chatChunk({ content: "x" }),
`data: ${JSON.stringify({ id: "chatcmpl-1", object: "chat.completion.chunk", created: 1, model: "qwen", choices: [{ index: 0, delta: {}, finish_reason: null }] })}`,
"\ndata: [DONE]",
]);
assert.ok(out.includes("upstream_stream_truncated"), `conservative error frame expected, got: ${out}`);
});

test("#1706 control: a sentinel cut mid-token is NOT recognized as [DONE]", async () => {
const { out } = await run([
chatChunk({ content: "x" }),
"data: [DO\n",
]);
assert.ok(out.includes("upstream_stream_truncated"), `partial sentinel must stay on the truncation path, got: ${out}`);
});

test("#1706 control: properly terminated [DONE] remains clean", async () => {
const { out, logs } = await run([
chatChunk({ content: "ok" }),
"data: [DONE]\n\n",
]);
assert.ok(!out.includes("upstream_stream_truncated"), `got: ${out}`);
assert.ok(!logs.some((l) => l.includes("truncated")), `got: ${JSON.stringify(logs)}`);
});
Loading