From c4bdb05ed9f3cb209c4230b218e25dceca7d8457 Mon Sep 17 00:00:00 2001 From: Afloat Date: Fri, 20 Nov 2026 18:52:02 +0800 Subject: [PATCH 1/4] fix(responses): preserve content indices in synthesized streams Set content_index=0 on the single-part text and marker lifecycles and legacy stream-error completions. Retain the source content index when proxy and plugin filters flush pending text, including plugin retry reframing. This prevents official SDK consumers from rejecting valid answers such as text ending in a buffered "m" as a missing content part. Record WC-026, enforce it in the matching fake and exercise actual OpenAI SDK consumption. The unchanged production fails ten focused regressions with ten controls passing; the fixed relevant suites pass all 193 tests. Typecheck, build, unused-export and test-import-graph gates also pass. (cherry picked from commit 3eec3863d3c80a5551aa579b2053d72e0aba9f53) --- src/loop/adapter-responses.ts | 14 +-- src/plugin.ts | 6 +- src/stream-error.ts | 8 +- tests/proxy-stream-error.test.ts | 1 + tests/responses-content-index.test.ts | 134 ++++++++++++++++++++++++++ tests/wire-contract-fakes.ts | 16 +++ tests/wire-contract.test.ts | 22 +++++ 7 files changed, 187 insertions(+), 14 deletions(-) create mode 100644 tests/responses-content-index.test.ts diff --git a/src/loop/adapter-responses.ts b/src/loop/adapter-responses.ts index 24f9d14605..eda5d21d50 100644 --- a/src/loop/adapter-responses.ts +++ b/src/loop/adapter-responses.ts @@ -237,10 +237,10 @@ function buildMessageItemSequence(itemId: string, outputIndex: number, text: str return Buffer.from( [ `event: response.output_item.added\ndata: ${JSON.stringify({ type: "response.output_item.added", output_index: outputIndex, item })}\n\n`, - `event: response.content_part.added\ndata: ${JSON.stringify({ type: "response.content_part.added", item_id: itemId, output_index: outputIndex, part })}\n\n`, - `event: response.output_text.delta\ndata: ${JSON.stringify({ type: "response.output_text.delta", item_id: itemId, output_index: outputIndex, delta: text })}\n\n`, - `event: response.output_text.done\ndata: ${JSON.stringify({ type: "response.output_text.done", item_id: itemId, output_index: outputIndex, text })}\n\n`, - `event: response.content_part.done\ndata: ${JSON.stringify({ type: "response.content_part.done", item_id: itemId, output_index: outputIndex, part: { type: "output_text", text } })}\n\n`, + `event: response.content_part.added\ndata: ${JSON.stringify({ type: "response.content_part.added", item_id: itemId, output_index: outputIndex, content_index: 0, part })}\n\n`, + `event: response.output_text.delta\ndata: ${JSON.stringify({ type: "response.output_text.delta", item_id: itemId, output_index: outputIndex, content_index: 0, delta: text })}\n\n`, + `event: response.output_text.done\ndata: ${JSON.stringify({ type: "response.output_text.done", item_id: itemId, output_index: outputIndex, content_index: 0, text })}\n\n`, + `event: response.content_part.done\ndata: ${JSON.stringify({ type: "response.content_part.done", item_id: itemId, output_index: outputIndex, content_index: 0, part: { type: "output_text", text } })}\n\n`, `event: response.output_item.done\ndata: ${JSON.stringify({ type: "response.output_item.done", output_index: outputIndex, item: doneItem })}\n\n`, ].join(""), "utf8", @@ -413,12 +413,12 @@ export function createResponsesAdapter(textProtocol?: boolean, projection?: Resp // identity filters, byte-verbatim (never rewritten; matches the // openai adapter's reasoning_filter and #1960/KDD#10). const reasoningFilters = new Map(); - let lastTextRef: { itemId: string; outputIndex: number } | null = null; + let lastTextRef: { itemId: string; outputIndex: number; contentIndex: unknown } | null = null; const flushFilter = function* (): Generator { const tail = tagFilter.flush(); if (tail.length > 0) { const raw = lastTextRef - ? rebuildResponsesEvent("response.output_text.delta", { type: "response.output_text.delta", item_id: lastTextRef.itemId, output_index: lastTextRef.outputIndex, delta: tail }) + ? rebuildResponsesEvent("response.output_text.delta", { type: "response.output_text.delta", item_id: lastTextRef.itemId, output_index: lastTextRef.outputIndex, content_index: lastTextRef.contentIndex, delta: tail }) : undefined; yield { kind: "text", delta: tail, ...(raw ? { raw } : {}) } as ParsedStreamEvent; } @@ -545,7 +545,7 @@ export function createResponsesAdapter(textProtocol?: boolean, projection?: Resp const itemId = typeof obj.item_id === "string" ? obj.item_id : ""; const outputIndex = typeof obj.output_index === "number" ? obj.output_index : 0; const mapped = remapped.get(itemId); - lastTextRef = { itemId: mapped ? mapped.id : itemId, outputIndex: mapped ? mapped.index : outputIndex }; + lastTextRef = { itemId: mapped ? mapped.id : itemId, outputIndex: mapped ? mapped.index : outputIndex, contentIndex: obj.content_index }; const clean = tagFilter.push(delta); if (clean.length > 0) { if (mapped) { diff --git a/src/plugin.ts b/src/plugin.ts index 867038aef8..52488ebbab 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -4612,13 +4612,13 @@ export async function pipePluginResponsesWithStrip( if (proseAcc.length === 0 || !session) return; scanModelOutputForPhantomAttachments(session.id, proseAcc); }; - let lastDeltaMeta: { item_id?: unknown; output_index?: unknown } | null = null; + let lastDeltaMeta: { item_id?: unknown; output_index?: unknown; content_index?: unknown } | null = null; const flushTail = (after: string) => { const tail = tagFilter.flush(); if (tail.length > 0) { visibleTextChars += tail.length; answerProse += tail; - const meta = inRetry && heldItemId !== undefined ? { item_id: heldItemId, output_index: heldOutputIndex } : (lastDeltaMeta ?? {}); + const meta = inRetry && heldItemId !== undefined ? { ...lastDeltaMeta, item_id: heldItemId, output_index: heldOutputIndex } : (lastDeltaMeta ?? {}); appendReplyDelta(meta, tail); return `data: ${JSON.stringify({ type: "response.output_text.delta", ...meta, delta: tail })}\n\n` + after; } @@ -5070,7 +5070,7 @@ export async function pipePluginResponsesWithStrip( continue; } const clean = tagFilter.push(delta); - lastDeltaMeta = { item_id: ev["item_id"], output_index: ev["output_index"] }; + lastDeltaMeta = { item_id: ev["item_id"], output_index: ev["output_index"], content_index: ev["content_index"] }; if (!inRetry && heldItemId === undefined && ev["item_id"] !== undefined) { heldItemId = ev["item_id"]; heldOutputIndex = ev["output_index"]; diff --git a/src/stream-error.ts b/src/stream-error.ts index c1495a747e..2054993913 100644 --- a/src/stream-error.ts +++ b/src/stream-error.ts @@ -90,10 +90,10 @@ function legacyCompletionChunk(protocol: Protocol, visible: string, state?: Lega const oi = 0; const errorItem = { type: "message", id: itemId, role: "assistant", content: [{ type: "output_text", text: visible }] }; return `event: response.output_item.added\ndata: ${JSON.stringify({ type: "response.output_item.added", output_index: oi, item: { type: "message", id: itemId, role: "assistant", content: [] } })}\n\n` + - `event: response.content_part.added\ndata: ${JSON.stringify({ type: "response.content_part.added", item_id: itemId, output_index: oi, part: { type: "output_text", text: "" } })}\n\n` + - `event: response.output_text.delta\ndata: ${JSON.stringify({ type: "response.output_text.delta", item_id: itemId, output_index: oi, delta: visible })}\n\n` + - `event: response.output_text.done\ndata: ${JSON.stringify({ type: "response.output_text.done", item_id: itemId, output_index: oi, text: visible })}\n\n` + - `event: response.content_part.done\ndata: ${JSON.stringify({ type: "response.content_part.done", item_id: itemId, output_index: oi, part: { type: "output_text", text: visible } })}\n\n` + + `event: response.content_part.added\ndata: ${JSON.stringify({ type: "response.content_part.added", item_id: itemId, output_index: oi, content_index: 0, part: { type: "output_text", text: "" } })}\n\n` + + `event: response.output_text.delta\ndata: ${JSON.stringify({ type: "response.output_text.delta", item_id: itemId, output_index: oi, content_index: 0, delta: visible })}\n\n` + + `event: response.output_text.done\ndata: ${JSON.stringify({ type: "response.output_text.done", item_id: itemId, output_index: oi, content_index: 0, text: visible })}\n\n` + + `event: response.content_part.done\ndata: ${JSON.stringify({ type: "response.content_part.done", item_id: itemId, output_index: oi, content_index: 0, part: { type: "output_text", text: visible } })}\n\n` + `event: response.output_item.done\ndata: ${JSON.stringify({ type: "response.output_item.done", output_index: oi, item: errorItem })}\n\n` + `event: response.completed\ndata: ${JSON.stringify({ type: "response.completed", response: { status: "completed", output: [errorItem] } })}\n\n`; } diff --git a/tests/proxy-stream-error.test.ts b/tests/proxy-stream-error.test.ts index 61c9b211fa..f5d1eade21 100644 --- a/tests/proxy-stream-error.test.ts +++ b/tests/proxy-stream-error.test.ts @@ -163,6 +163,7 @@ test("emitStreamError: responses errorShape=\"completion\" emits a full item lif for (const block of itemEvents) { assert.match(block, new RegExp(`"item_id":"${itemId}"`)); assert.match(block, /"output_index":0/); + assert.match(block, /"content_index":0/); } // The completed response contains the error item in its output array. const completed = out.split("\n\n").find((b) => b.includes("response.completed")); diff --git a/tests/responses-content-index.test.ts b/tests/responses-content-index.test.ts new file mode 100644 index 0000000000..0c44b5cf83 --- /dev/null +++ b/tests/responses-content-index.test.ts @@ -0,0 +1,134 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import type { ServerResponse } from "node:http"; +import { createRequire } from "node:module"; +import { createResponsesAdapter } from "../src/loop/adapter-responses.ts"; +import { pipePluginResponsesWithStrip } from "../src/plugin.ts"; +import { emitStreamError } from "../src/stream-error.ts"; +import { validateResponsesContentIndex } from "./wire-contract-fakes.ts"; + +function frame(event: Record): string { + return `event: ${String(event.type)}\ndata: ${JSON.stringify(event)}\n\n`; +} + +function created(): string { + return frame({ type: "response.created", response: { id: "resp_fixture", object: "response", status: "in_progress", output: [] } }); +} + +function payloads(raw: string): Record[] { + return raw.split("\n").filter((line) => line.startsWith("data: ")).map((line) => JSON.parse(line.slice(6)) as Record); +} + +function collector(): { res: ServerResponse; chunks: string[] } { + const chunks: string[] = []; + const res = { + write(chunk: string | Buffer) { chunks.push(chunk.toString()); return true; }, + end() { return this; }, + once() { return this; }, + destroyed: false, + writableEnded: false, + } as unknown as ServerResponse; + return { res, chunks }; +} + +async function consume(raw: string) { + const requireSdk = createRequire(import.meta.resolve("@earendil-works/pi-coding-agent")); + const { default: OpenAI } = await import(requireSdk.resolve("openai")); + const client = new OpenAI({ + apiKey: "synthetic-test-key", + fetch: async () => new Response(raw, { headers: { "content-type": "text/event-stream" } }), + }); + const stream = client.responses.stream({ model: "synthetic", input: "synthetic" }); + const deltas: Array<{ delta: string; content_index: number; output_index: number }> = []; + stream.on("response.output_text.delta", (event: { delta: string; content_index: number; output_index: number }) => deltas.push(event)); + const response = await stream.finalResponse(); + return { deltas, response }; +} + +function upstreamText(text: string, contentIndex: number, id = "msg_fixture"): string { + const earlier = Array.from({ length: contentIndex }, () => ({ type: "output_text", text: "earlier part" })); + const content = [...earlier, { type: "output_text", text }]; + const item = { type: "message", id, role: "assistant", content }; + const ref = { item_id: id, output_index: 0, content_index: contentIndex }; + return created() + [ + { type: "response.output_item.added", output_index: 0, item: { ...item, content: [] } }, + ...earlier.map((part, index) => ({ type: "response.content_part.added", item_id: id, output_index: 0, content_index: index, part })), + { type: "response.content_part.added", ...ref, part: { type: "output_text", text: "" } }, + { type: "response.output_text.delta", ...ref, delta: text }, + { type: "response.output_text.done", ...ref, text }, + { type: "response.content_part.done", ...ref, part: { type: "output_text", text } }, + { type: "response.output_item.done", output_index: 0, item }, + { type: "response.completed", response: { id: "resp_fixture", status: "completed", output: [item] } }, + ].map(frame).join(""); +} + +test("official Responses SDK consumes a legacy stream-error completion", async () => { + const { res, chunks } = collector(); + emitStreamError(res, "responses", "kaboom", undefined, "completion"); + const { deltas, response } = await consume(created() + chunks.join("")); + assert.deepEqual(deltas.map((event) => event.delta), ["\n❌ [ACP] stream error: kaboom"]); + assert.equal(response.status, "completed"); + assert.equal(response.output[0].content[0].text, deltas[0].delta); +}); + +test("official Responses SDK still rejects the default native error", async () => { + const { res, chunks } = collector(); + emitStreamError(res, "responses", "kaboom"); + await assert.rejects(consume(created() + chunks.join("")), { type: "error", code: "stream_error", message: "[acp-proxy: kaboom]" }); +}); + +for (const kind of ["text", "marker"] as const) { + test(`official Responses SDK consumes synthesized ${kind} content`, async () => { + const adapter = createResponsesAdapter(); + const chunk = kind === "text" ? adapter.emitText("generated text") : adapter.emitMarker("acp_status", "ready"); + const events = payloads(chunk.toString()); + const item = events.find((event) => event.type === "response.output_item.done")?.item; + assert.ok(item); + const { deltas } = await consume(created() + chunk + frame({ type: "response.completed", response: { id: "resp_fixture", status: "completed", output: [item] } })); + assert.equal(deltas.length, 1); + assert.equal(deltas[0].content_index, 0); + if (kind === "text") assert.equal(deltas[0].delta, "generated text"); + else assert.match(deltas[0].delta, /\[ACP\]/); + assert.deepEqual(events.flatMap(validateResponsesContentIndex), []); + }); +} + +for (const mode of ["proxy", "plugin"] as const) { + for (const contentIndex of [0, 1]) { + test(`${mode} Responses text flush preserves content index ${contentIndex}`, async () => { + const input = upstreamText("Length: 10 m", contentIndex); + const { res, chunks } = collector(); + if (mode === "plugin") { + await pipePluginResponsesWithStrip(new Response(input).body!, res); + } else { + const adapter = createResponsesAdapter(); + for await (const event of adapter.parseStream(new Response(input).body!, 1)) { + if (event.kind === "meta") chunks.push(event.chunk.toString()); + else if (event.kind === "text" && event.raw) chunks.push(event.raw.toString()); + } + chunks.push(adapter.emitCompletion().toString()); + } + const { deltas, response } = await consume(chunks.join("")); + assert.deepEqual(deltas.map((event) => event.delta), ["Length: 10 ", "m"]); + for (const delta of deltas) assert.equal(delta.content_index, contentIndex); + assert.equal(response.output[0].content[contentIndex].text, "Length: 10 m"); + if (contentIndex > 0) assert.equal(response.output[0].content[0].text, "earlier part"); + assert.deepEqual(payloads(chunks.join("")).flatMap(validateResponsesContentIndex), []); + }); + } +} + +test("plugin Responses retry flush preserves its content index", async () => { + const { res, chunks } = collector(); + let retries = 0; + const echo = "\x3cacp tokens=\"1\" type=\"text\"\x3em00001\x3c/acp\x3e"; + await pipePluginResponsesWithStrip(new Response(upstreamText(echo, 0)).body!, res, undefined, undefined, async () => { + retries++; + return new Response(upstreamText("Length: 10 m", 0, "msg_retry")).body; + }); + assert.equal(retries, 1); + const { deltas } = await consume(chunks.join("")); + assert.equal(deltas.map((event) => event.delta).join(""), "Length: 10 m"); + for (const delta of deltas) assert.equal(delta.content_index, 0); + assert.deepEqual(payloads(chunks.join("")).flatMap(validateResponsesContentIndex), []); +}); diff --git a/tests/wire-contract-fakes.ts b/tests/wire-contract-fakes.ts index 9b6797ee1b..ecb00b1426 100644 --- a/tests/wire-contract-fakes.ts +++ b/tests/wire-contract-fakes.ts @@ -31,6 +31,13 @@ interface WireRule { } export const WIRE_RULES: readonly WireRule[] = [ + { + id: "WC-026", + wire: "responses", + lane: "responses-sse", + summary: "Responses text/content-part SSE events carry a non-negative integer content_index identifying the part inside the indexed output item", + provenance: "OpenAI openai-node v6.26.0 requires content_index in ResponseTextDeltaEvent/ResponseTextDoneEvent/ResponseContentPartAddedEvent/ResponseContentPartDoneEvent (https://github.com/openai/openai-node/blob/v6.26.0/src/resources/responses/responses.ts); ResponseStream indexes output.content[event.content_index] (https://github.com/openai/openai-node/blob/v6.26.0/src/lib/responses/ResponseStream.ts). Reproduced against that official SDK: bili's synthesized messages, legacy stream-error completions and filtered text tails omitted indices, so finalResponse() rejected with 'missing content at index undefined'. Gates exercise the actual SDK in proxy/plugin paths and the emitted error frames; the fake validates its outgoing SSE frames with the same constraint.", + }, { id: "WC-017", wire: "responses", @@ -772,6 +779,8 @@ interface FakeUpstream { } function sse(res: http.ServerResponse, events: Array<[string, unknown]>, opts?: { done?: boolean }): void { + const violations = events.flatMap(([, data]) => validateResponsesContentIndex(data)); + if (violations.length) throw new Error(violations.join("; ")); res.writeHead(200, { "content-type": "text/event-stream", "cache-control": "no-cache", connection: "keep-alive" }); // OpenAI streams carry no `event:` line at all — only Anthropic/Gemini name their events. for (const [ev, data] of events) { @@ -811,6 +820,13 @@ function openAiChatReply(res: http.ServerResponse, parsed: Record= 0 + ? [] + : ["WC-026 Responses content-part events require a non-negative integer content_index"]; +} + function responsesReply(res: http.ServerResponse, parsed: Record, text: string): void { const msgId = "msg_fake"; const msg = { type: "message", id: msgId, role: "assistant", content: [{ type: "output_text", text }] }; diff --git a/tests/wire-contract.test.ts b/tests/wire-contract.test.ts index bda8c0071e..6ccfe5e93f 100644 --- a/tests/wire-contract.test.ts +++ b/tests/wire-contract.test.ts @@ -21,6 +21,8 @@ import assert from "node:assert/strict"; import http from "node:http"; import { once } from "node:events"; import { readFileSync } from "node:fs"; +import { emitStreamError } from "../src/stream-error.ts"; +import { validateResponsesContentIndex } from "./wire-contract-fakes.ts"; process.env.NODE_ENV = "test"; @@ -57,6 +59,25 @@ import { verifySdkHmac, WIRE_RULES, WIRE_HEADER_RULES, VALIDATORS, startFakeUpst type ToolShape = Record; +test("WC-026 Responses content-part indices are explicit, including legacy error completions", () => { + for (const type of ["response.content_part.added", "response.output_text.delta", "response.output_text.done", "response.content_part.done"]) { + for (const content_index of [0, 2]) assert.deepEqual(validateResponsesContentIndex({ type, content_index }), []); + for (const content_index of [undefined, null, "0", -1, 0.5]) assert.match(validateResponsesContentIndex({ type, content_index })[0], /WC-026/); + } + assert.deepEqual(validateResponsesContentIndex({ type: "response.output_item.done" }), []); + const frames: Record[] = []; + const res = { + write(chunk: string) { + for (const line of chunk.split("\n")) if (line.startsWith("data: ")) frames.push(JSON.parse(line.slice(6))); + return true; + }, + end() { return this; }, + } as unknown as http.ServerResponse; + emitStreamError(res, "responses", "fixture failure", undefined, "completion"); + assert.equal(frames.length, 7); + assert.deepEqual(frames.flatMap(validateResponsesContentIndex), []); +}); + test("WC-013: WebSocket transport envelopes omit HTTP-only fields", () => { assert.deepEqual(validateResponsesWsCreate({ type: "response.create", input: [] }), []); for (const key of ["stream", "background", "stream_options"]) assert.match(validateResponsesWsCreate({ type: "response.create", [key]: true })[0], /WC-013/); @@ -369,6 +390,7 @@ test("wire-contract ledger: every lane-scoped rule has a live enforcement clause // Lane-scoped constraints hold on ONE client's lane only — the shared wire validator // cannot enforce them (WC-013 forbids exactly what WC-017 requires, same frame shape). const laneProbes: Array<{ ruleId: string; validate: (body: unknown) => string[]; body: unknown }> = [ + { ruleId: "WC-026", validate: validateResponsesContentIndex, body: { type: "response.output_text.delta", output_index: 0, delta: "x" } }, { ruleId: "WC-017", validate: validateCodexResponsesWsCreate, body: { type: "response.create", input: [] } }, // WC-019/020/021 are the Vertex Anthropic lane (lane: "vertex-anthropic") — the wire rules that // hold only on that lane, so the shared anthropic validator must not carry them. WC-019 is From 17350dea177d82d2a6145fb08d21ae23baabb8b2 Mon Sep 17 00:00:00 2001 From: Afloat Date: Fri, 20 Nov 2026 19:27:48 +0800 Subject: [PATCH 2/4] fix: preserve responses beyond the fake-completion buffer cap Stop bounded inspection at the first over-budget chunk and replay its complete prefix and remaining stream through the existing response pipes. Do not classify a partial response or alter its fake-completion streak. Keep retry idle timers and client-abort listeners until the live response and optional raw dump finish. Error-body truncation remains unchanged. Validation: 5 baseline behavior failures and 2 passing exact-cap controls; 74 targeted tests pass. Typecheck, build and static gates pass. Codex preflight finds the CLI/artifact but the live upstream is unavailable; the real four-phase E2E and full remote CI remain pending. (cherry picked from commit 2703cbe80f5fef6071dbff7d7eb4c603ac46bde4) --- docs/configuration.md | 1 + docs/configuration.zh-CN.md | 1 + src/server/relay.ts | 41 ++-- src/server/stream-io.ts | 57 +++++ tests/fake-completion-buffer-limit.test.ts | 242 +++++++++++++++++++++ tests/stream-buffer-budget.test.ts | 107 +++++++++ 6 files changed, 436 insertions(+), 13 deletions(-) create mode 100644 tests/fake-completion-buffer-limit.test.ts create mode 100644 tests/stream-buffer-budget.test.ts diff --git a/docs/configuration.md b/docs/configuration.md index a2ba64df70..1ebf16324b 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1113,6 +1113,7 @@ Security boundary: the file controls ONLY *when* to upgrade and *which published - **Default:** `{ retries: 0, bufCapBytes: 16777216 }` - **Status:** ACTIVE - **Description:** Fake-completion safety net for hosts that abort mid-stream (#371) — **opt-in**, `retries: 0` (default) is the pre-#371 passthrough. `bufCapBytes` is the OOM guard on the buffered response. Twins `BILI_FAKE_COMPLETION_RETRIES`, `BILI_FAKE_BUF_CAP`. +- A response exceeding `bufCapBytes` continues through the normal streaming path, including its already-read prefix. The incomplete inspection neither retries nor changes the session's fake-completion streak; the cap does not truncate the answer. This applies to the first response and hinted retries. ### `codexCompact` / `ccrRetrievalTtlMs` / `decompressTmpCap` diff --git a/docs/configuration.zh-CN.md b/docs/configuration.zh-CN.md index 94d38a80f2..704430f02c 100644 --- a/docs/configuration.zh-CN.md +++ b/docs/configuration.zh-CN.md @@ -1107,6 +1107,7 @@ - **默认:** `{ retries: 0, bufCapBytes: 16777216 }` - **状态:** ACTIVE - **说明:** 针对流中途被宿主掐断场景的 fake-completion 安全网(#371)—— **显式开启**,`retries: 0`(默认)即 #371 之前的透传行为。`bufCapBytes` 是缓冲响应的 OOM 护栏。对应 `BILI_FAKE_COMPLETION_RETRIES`、`BILI_FAKE_BUF_CAP`。 +- 响应超过 `bufCapBytes` 时,已读前缀和后续内容会继续进入正常流式处理。未完成的检查不会触发重试或改变会话的 fake-completion 连续计数;该上限不会截断回答。首次响应与附带纠正提示的重试响应都遵循此规则。 ### `codexCompact` / `ccrRetrievalTtlMs` / `decompressTmpCap` diff --git a/src/server/relay.ts b/src/server/relay.ts index 7ce093b91b..63fdf4cfd4 100644 --- a/src/server/relay.ts +++ b/src/server/relay.ts @@ -76,7 +76,7 @@ import { buildForwardHeaders, connectionNamedHeaders, RESPONSE_ONLY_STRIP_HEADER import { bodyDumpEnabled, logDumpFailure } from "./observability.js"; import { rejectionDiagnosticSuffix } from "./rejection-diagnostics.js"; import { effectiveAbsorbBlock, effectiveBlockOpsEnabled } from "./prepare-responses.js"; -import { awaitDrain, bufferToStream, dumpStreamToFile, pipeThrough, readStreamToBuffer } from "./stream-io.js"; +import { awaitDrain, bufferStreamIfFits, bufferToStream, dumpStreamToFile, pipeThrough, readStreamToBuffer } from "./stream-io.js"; import { buildForwardTarget, measuredBreakdownClause, measuredOutbound, outboundPayloadBreakdown, registerRequestAbort, beginStreamKeepalive, effectiveTokenCount, googlePathKind, imageBillingFor, imageTokenCapFor, inboundResignCredential, outboundContextEstimates, resolveUpstream, stripKernelSummaries, usageGradeInputBaseline, DOOMED_PROBE_ABSOLUTE_BYTES, DOOMED_PROBE_BYTES_PER_TOKEN, type Prepared } from "../server.js"; /** Retry the same input when the provider proved only its output reservation overflows. */ @@ -2292,6 +2292,7 @@ const reply = prepared && !prepared.sidePassthrough ? createHandoffReply(prepare // leaked a live idle timer. (#721/#2563: the SSE pipes themselves no // longer rethrow an upstream cut — they emit an in-band truncation // signal, or end raw for permitlist hosts that own the handling.) + let clearFakeRetryTimer: (() => void) | undefined; try { let pluginBody = upstream.body as ReadableStream; // #1295: the JSONL wire has no fake-completion shape to detect — @@ -2304,7 +2305,7 @@ const reply = prepared && !prepared.sidePassthrough ? createHandoffReply(prepare // fake completion. The agent owning compression does not change this: // the retry is a proxy→upstream HTTP call whose base must be what // upstream just accepted (aligned with the proxy lane below). - const resolvedBuf = await resolveFakeCompletion(pluginBody, { + const resolved = await resolveFakeCompletion(pluginBody, { protocol: prepared.protocol, wireBody, upstreamUrl, @@ -2315,7 +2316,8 @@ const reply = prepared && !prepared.sidePassthrough ? createHandoffReply(prepare log, model: prepared.model ?? prepared.google?.model, }); - pluginBody = bufferToStream(resolvedBuf); + pluginBody = resolved.body; + clearFakeRetryTimer = resolved.clearTimer; } // #2347: raw-dump twin of the tee below — this branch returns // before reaching it, so plugin-mode turns need their own. @@ -2421,7 +2423,8 @@ const reply = prepared && !prepared.sidePassthrough ? createHandoffReply(prepare clearUpstreamTimer(); // #2347: finish the dump before the turn closes — its value is the // full upstream tail, which the client may have stopped reading early. - if (dumpRaw) await dumpRaw; + try { if (dumpRaw) await dumpRaw; } + finally { clearFakeRetryTimer?.(); } } return; } @@ -2457,6 +2460,7 @@ const reply = prepared && !prepared.sidePassthrough ? createHandoffReply(prepare // response (any turn, not just compress-injected). Buffering is required: // the retry re-requests before the client sees the fake completion. let responseBody: ReadableStream = upstream.body; + let clearFakeRetryTimer: (() => void) | undefined; // #411: every body-consuming path below must clear the upstream idle timer // even when it throws (client abort / upstream cut) — previously an abort // skipped the trailing clearUpstreamTimer and leaked a live idle @@ -2470,7 +2474,7 @@ const reply = prepared && !prepared.sidePassthrough ? createHandoffReply(prepare // never the raw client body: upstream may have rejected those raw // bytes earlier in this session, and a 400'd hint would present the // fake completion instead of a corrected turn. - const resolvedBuf = await resolveFakeCompletion(upstream.body, { + const resolved = await resolveFakeCompletion(upstream.body, { protocol: prepared.protocol, wireBody, wrapBody: v1Active ? v1Wrap : undefined, @@ -2482,7 +2486,8 @@ const reply = prepared && !prepared.sidePassthrough ? createHandoffReply(prepare log, model: prepared.model ?? prepared.google?.model, }); - responseBody = bufferToStream(resolvedBuf); + responseBody = resolved.body; + clearFakeRetryTimer = resolved.clearTimer; } // #2347: dump the FINAL upstream body for every prepared streaming turn, // regardless of rewriter branch — the previous tee sat INSIDE the useRewriter @@ -2979,13 +2984,14 @@ const reply = prepared && !prepared.sidePassthrough ? createHandoffReply(prepare // the full upstream tail, which the client may have stopped reading // early. Idempotent where the rewriter lane already awaited it in its // own finally; this covers the non-rewriter branch's throw paths too. - if (dumpRaw) await dumpRaw; + try { if (dumpRaw) await dumpRaw; } + finally { clearFakeRetryTimer?.(); } } } // #371: buffer the raw upstream response, detect a fake completion (tool-call // XML, no real tool block), and — bounded per turn and per session — re-request -// upstream with a corrective hint. Returns the bytes to stream to the client +// upstream with a corrective hint. Returns the body to stream to the client // (the retry's response when a retry recovered, else the original). The session // streak (metadata.fakeCompletionStreak) counts consecutive fake-completion // turns: it gates retries (skip once >= cap) and resets to 0 on a clean turn. @@ -3018,8 +3024,11 @@ async function resolveFakeCompletion( * must go back out inside the CloudCode envelope. */ wrapBody?: (body: string | Buffer) => string | Buffer; }, -): Promise { - let buffer = await readStreamToBuffer(stream, fakeBufCap()); +): Promise<{ body: ReadableStream; clearTimer?: () => void }> { + const buffered = await bufferStreamIfFits(stream, fakeBufCap()); + // An incomplete inspection cannot classify this turn or change its streak. + if (!Buffer.isBuffer(buffered)) return { body: buffered }; + let buffer = buffered; const max = maxFakeCompletionRetries(); const sid = opts.session.id; const priorStreak = (opts.session.metadata.fakeCompletionStreak as number | undefined) ?? 0; @@ -3047,14 +3056,20 @@ async function resolveFakeCompletion( opts.log("warn", `[${sid}] fake-completion retry failed: ${String(e)}; presenting original`); break; } + let handedOff = false; try { if (!r.response.ok || !r.response.body) { opts.log("warn", `[${sid}] fake-completion retry rejected (HTTP ${r.response.status}); presenting original`); break; } - buffer = await readStreamToBuffer(r.response.body, fakeBufCap()); + const retried = await bufferStreamIfFits(r.response.body, fakeBufCap()); + if (!Buffer.isBuffer(retried)) { + handedOff = true; + return { body: retried, clearTimer: r.clearTimer }; + } + buffer = retried; } finally { - r.clearTimer(); + if (!handedOff) r.clearTimer(); } if (!isFakeCompletion(opts.protocol, buffer.toString("utf8"))) break; } @@ -3062,5 +3077,5 @@ async function resolveFakeCompletion( const stillFake = isFakeCompletion(opts.protocol, buffer.toString("utf8")); opts.session.metadata.fakeCompletionStreak = stillFake ? priorStreak + 1 : 0; markDirty(opts.session); - return buffer; + return { body: bufferToStream(buffer) }; } diff --git a/src/server/stream-io.ts b/src/server/stream-io.ts index 849c66cb58..c75ca6155f 100644 --- a/src/server/stream-io.ts +++ b/src/server/stream-io.ts @@ -31,6 +31,63 @@ export async function readStreamToBuffer(stream: ReadableStream, max return Buffer.concat(chunks); } +/** Buffer a response only while it fits the inspection budget. On overflow, + * replay the retained prefix and continue streaming without dropping bytes. */ +export async function bufferStreamIfFits(stream: ReadableStream, maxBytes: number): Promise> { + const reader = stream.getReader(); + const chunks: Buffer[] = []; + let kept = 0; + let handedOff = false; + try { + for (;;) { + const { done, value } = await reader.read(); + if (done) return Buffer.concat(chunks, kept); + if (value.byteLength === 0) continue; + if (kept + value.byteLength <= maxBytes) { + chunks.push(Buffer.from(value)); + kept += value.byteLength; + continue; + } + let ended = false; + const replay = new ReadableStream({ + start(controller) { + for (const chunk of chunks) controller.enqueue(chunk); + chunks.length = 0; + controller.enqueue(value); + }, + async pull(controller) { + try { + const next = await reader.read(); + if (ended) return; + if (next.done) { + ended = true; + reader.releaseLock(); + controller.close(); + } else { + controller.enqueue(next.value); + } + } catch (error) { + if (!ended) { + ended = true; + reader.releaseLock(); + controller.error(error); + } + } + }, + async cancel(reason) { + ended = true; + try { await reader.cancel(reason); } + finally { reader.releaseLock(); } + }, + }); + handedOff = true; + return replay; + } + } finally { + if (!handedOff) reader.releaseLock(); + } +} + export function bufferToStream(buf: Buffer): ReadableStream { return new ReadableStream({ start(controller) { diff --git a/tests/fake-completion-buffer-limit.test.ts b/tests/fake-completion-buffer-limit.test.ts new file mode 100644 index 0000000000..077ad922e4 --- /dev/null +++ b/tests/fake-completion-buffer-limit.test.ts @@ -0,0 +1,242 @@ +import assert from "node:assert/strict"; +import { randomUUID } from "node:crypto"; +import http from "node:http"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; +import { defaultConfig } from "acp-kernel"; +import type { ProxyOptions } from "../src/config.ts"; +import { _liveUpstreamTimersForTest } from "../src/fetch-util.ts"; +import { SessionStore, _setStoreForTest } from "../src/persist.ts"; +import { _resetPluginStateForTest } from "../src/plugin.ts"; +import { _setForTest as setRegistryForTest } from "../src/registry.ts"; +import { startServer } from "../src/server.ts"; +import { _resetSessionsForTest, getSession } from "../src/session.ts"; + +process.env.NODE_ENV = "test"; + +const FAKE = "FAKE-MARKER \x3cinvoke name=\"example\"\x3e\x3c/invoke\x3e"; +const LARGE = "begin-😀-" + "complete response ".repeat(600) + "-TAIL-MARKER"; + +function sse(text: string, toolArguments?: string): string { + const base = { id: "chatcmpl-buffer", object: "chat.completion.chunk", model: "buffer-test", created: 1 }; + const tool = toolArguments === undefined ? "" : `data: ${JSON.stringify({ ...base, choices: [{ index: 0, delta: { tool_calls: [{ index: 0, id: "call-preserved", type: "function", function: { name: "write_file", arguments: toolArguments } }] } }] })}\n\n`; + return `data: ${JSON.stringify({ ...base, choices: [{ index: 0, delta: { role: "assistant", content: text } }] })}\n\n` + + tool + + `data: ${JSON.stringify({ ...base, choices: [{ index: 0, delta: {}, finish_reason: toolArguments === undefined ? "stop" : "tool_calls" }], usage: { prompt_tokens: 10, completion_tokens: 20, total_tokens: 30 } })}\n\n` + + "data: [DONE]\n\n"; +} + +function json(text: string): string { + return JSON.stringify({ id: "chatcmpl-buffer", object: "chat.completion", model: "buffer-test", choices: [{ index: 0, message: { role: "assistant", content: text }, finish_reason: "stop" }] }); +} + +function textOf(raw: string): string { + let text = ""; + for (const line of raw.split("\n")) { + if (!line.startsWith("data:") || line.slice(5).trim() === "[DONE]") continue; + const event = JSON.parse(line.slice(5)); + text += event.choices?.[0]?.delta?.content ?? ""; + } + return text; +} + +type Reply = string | ((res: http.ServerResponse) => void); + +async function fixture(mode: "proxy" | "plugin", replies: Reply[], cap: number, opts: { json?: boolean; idleMs?: number; dumpSse?: string; priorStreak?: number } = {}) { + const before = { retries: process.env.BILI_FAKE_COMPLETION_RETRIES, cap: process.env.BILI_FAKE_BUF_CAP, idle: process.env.BILI_UPSTREAM_TIMEOUT_MS, firstByte: process.env.BILI_UPSTREAM_FIRST_BYTE_MS }; + process.env.BILI_FAKE_COMPLETION_RETRIES = "1"; + process.env.BILI_FAKE_BUF_CAP = String(cap); + if (opts.idleMs) { + process.env.BILI_UPSTREAM_TIMEOUT_MS = String(opts.idleMs); + process.env.BILI_UPSTREAM_FIRST_BYTE_MS = "10000"; + } + let hits = 0; + const upstream = http.createServer((req, res) => { + req.resume(); + req.on("end", () => { + res.writeHead(200, { "content-type": opts.json ? "application/json" : "text/event-stream" }); + const reply = replies[Math.min(hits++, replies.length - 1)]; + if (typeof reply === "string") res.end(reply); + else reply(res); + }); + }); + await new Promise((resolve) => upstream.listen(0, "127.0.0.1", resolve)); + const address = upstream.address(); + assert.ok(address && typeof address !== "string"); + _setStoreForTest(new SessionStore({ enabled: false })); + _resetSessionsForTest(); + setRegistryForTest({}); + _resetPluginStateForTest(); + const proxy = await startServer({ + port: 0, host: "127.0.0.1", upstream: "http://127.0.0.1", + routes: { [`http://127.0.0.1:${address.port}`]: { models: { "buffer-test": { context: 100_000 } } } }, + modelContextLimit: 100_000, kernelConfig: defaultConfig(100_000), + compress: { injectTool: false, injectNudge: false }, promptCache: { routing: "auto" }, + sessionHeader: "x-acp-session", log: false, debug: false, passthrough: false, + autoUpdate: false, compat: { roles: {} }, streamErrorShape: "protocol", passthroughSource: null, + autoRestartOnUpdate: false, updateTag: "latest", advisoryCheck: false, releaseNotesCheck: false, + mitm: { enabled: false, domains: [] }, + ...(opts.dumpSse ? { dumpSse: opts.dumpSse } : {}), + } as ProxyOptions); + if (!proxy.listening) await new Promise((resolve) => proxy.once("listening", resolve)); + const proxyAddress = proxy.address(); + assert.ok(proxyAddress && typeof proxyAddress !== "string"); + const session = "buffer-limit-" + randomUUID(); + if (opts.priorStreak !== undefined) getSession(session).metadata.fakeCompletionStreak = opts.priorStreak; + return { + async request(signal?: AbortSignal) { + return fetch(`http://127.0.0.1:${proxyAddress.port}/bili/http://127.0.0.1:${address.port}/v1/chat/completions`, { + method: "POST", + headers: { "content-type": "application/json", "x-acp-session": session, ...(mode === "plugin" ? { "x-bili-plugin": "pi" } : {}) }, + body: JSON.stringify({ model: "buffer-test", stream: true, max_tokens: 4096, messages: [{ role: "user", content: "Return the complete result" }] }), + signal, + }); + }, + hits: () => hits, + session: () => getSession(session), + async close() { + proxy.closeAllConnections(); + upstream.closeAllConnections(); + await Promise.all([new Promise((resolve) => proxy.close(() => resolve())), new Promise((resolve) => upstream.close(() => resolve()))]); + if (before.retries === undefined) delete process.env.BILI_FAKE_COMPLETION_RETRIES; + else process.env.BILI_FAKE_COMPLETION_RETRIES = before.retries; + if (before.cap === undefined) delete process.env.BILI_FAKE_BUF_CAP; + else process.env.BILI_FAKE_BUF_CAP = before.cap; + if (before.idle === undefined) delete process.env.BILI_UPSTREAM_TIMEOUT_MS; + else process.env.BILI_UPSTREAM_TIMEOUT_MS = before.idle; + if (before.firstByte === undefined) delete process.env.BILI_UPSTREAM_FIRST_BYTE_MS; + else process.env.BILI_UPSTREAM_FIRST_BYTE_MS = before.firstByte; + }, + }; +} + +for (const mode of ["proxy", "plugin"] as const) { + for (const retry of [false, true]) { + test(`${mode}: buffer cap preserves a ${retry ? "retried" : "first"} successful SSE response`, async () => { + const rig = await fixture(mode, retry ? [sse(FAKE), sse(LARGE)] : [sse(LARGE)], 1024); + try { + const response = await rig.request(); + assert.equal(response.status, 200); + const raw = await response.text(); + assert.equal(textOf(raw), LARGE); + assert.ok(raw.includes("[DONE]")); + assert.ok(raw.includes('"completion_tokens":20')); + assert.equal(rig.hits(), retry ? 2 : 1); + assert.equal(rig.session().metadata.fakeCompletionStreak, undefined); + assert.equal(_liveUpstreamTimersForTest(), 0); + } finally { await rig.close(); } + }); + } + + test(`${mode}: a fake completion exactly at the buffer cap is still checked`, async () => { + const fake = sse(FAKE); + const rig = await fixture(mode, [fake, sse("recovered")], Buffer.byteLength(fake)); + try { + assert.equal(textOf(await (await rig.request()).text()), "recovered"); + assert.equal(rig.hits(), 2); + } finally { await rig.close(); } + }); +} + +test("proxy: buffer cap preserves JSON returned to a streaming request", async () => { + const rig = await fixture("proxy", [json(LARGE)], 1024, { json: true }); + try { + const response = await rig.request(); + assert.equal(response.status, 200); + const body = await response.json() as { choices: Array<{ message: { content: string } }> }; + assert.equal(body.choices[0].message.content, LARGE); + assert.equal(rig.hits(), 1); + } finally { await rig.close(); } +}); + +async function within(promise: Promise, label: string): Promise { + let timer: NodeJS.Timeout | undefined; + try { + return await Promise.race([promise, new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error(label)), 5_000); + })]); + } finally { clearTimeout(timer); } +} + +async function timersCleared(): Promise { + const deadline = Date.now() + 5_000; + while (_liveUpstreamTimersForTest() !== 0 && Date.now() < deadline) await new Promise((resolve) => setTimeout(resolve, 10)); + assert.equal(_liveUpstreamTimersForTest(), 0, "upstream timers were not released"); +} + +for (const mode of ["proxy", "plugin"] as const) { + test(`${mode}: a tool call beyond the inspection cap remains byte-exact without a retry`, async () => { + const args = JSON.stringify({ contents: LARGE }); + const rig = await fixture(mode, [sse(FAKE + LARGE, args), sse("unexpected retry")], 1024); + try { + const raw = await (await rig.request()).text(); + const events = raw.split("\n").filter((line) => line.startsWith("data:") && !line.includes("[DONE]")).map((line) => JSON.parse(line.slice(5))); + const calls = events.flatMap((event) => event.choices?.[0]?.delta?.tool_calls ?? []); + assert.equal(calls.length, 1); + assert.equal(calls[0].id, "call-preserved"); + assert.equal(calls[0].function.arguments, args); + assert.equal(rig.hits(), 1); + } finally { await rig.close(); } + }); + + test(`${mode}: incomplete inspection leaves the prior fake-completion streak intact`, async () => { + const rig = await fixture(mode, [sse(LARGE)], 1024, { priorStreak: 1 }); + try { + assert.equal(textOf(await (await rig.request()).text()), LARGE); + assert.equal(rig.session().metadata.fakeCompletionStreak, 1); + assert.equal(rig.hits(), 1); + } finally { await rig.close(); } + }); + + test(`${mode}: an over-budget retry still aborts on upstream silence`, { timeout: 15_000 }, async () => { + const closed = Promise.withResolvers(); + const prefix = sse(LARGE).split("\n\n")[0] + "\n\n"; + const rig = await fixture(mode, [sse(FAKE), (res) => { + res.once("close", closed.resolve); + res.write(prefix); + }], 1024, { idleMs: 1500 }); + try { + const response = await rig.request(); + const raw = await within(response.text(), "retry idle watchdog was detached before the body finished"); + assert.equal(textOf(raw), LARGE); + assert.equal(rig.hits(), 2); + await within(closed.promise, "stalled retry connection stayed open"); + await timersCleared(); + } finally { await rig.close(); } + }); + + test(`${mode}: disconnect cancels an over-budget retry and its raw-dump branch`, { timeout: 15_000 }, async () => { + const closed = Promise.withResolvers(); + const prefix = sse(LARGE).split("\n\n")[0] + "\n\n"; + const dir = await mkdtemp(join(tmpdir(), "bili-fake-buffer-dump-")); + const rig = await fixture(mode, [sse(FAKE), (res) => { + res.once("close", closed.resolve); + res.write(prefix); + }], 1024, { dumpSse: dir }); + const abort = new AbortController(); + try { + const response = await rig.request(abort.signal); + const reader = response.body!.getReader(); + let raw = ""; + const decoder = new TextDecoder(); + await within((async () => { + while (!raw.includes("-TAIL-MARKER")) { + const next = await reader.read(); + assert.equal(next.done, false); + raw += decoder.decode(next.value, { stream: true }); + } + })(), "over-budget prefix was withheld until upstream EOF"); + assert.equal(textOf(raw), LARGE); + abort.abort(); + await within(closed.promise, "client abort did not close the live retry"); + await timersCleared(); + reader.releaseLock(); + } finally { + abort.abort(); + await rig.close(); + await rm(dir, { recursive: true, force: true }); + } + }); +} diff --git a/tests/stream-buffer-budget.test.ts b/tests/stream-buffer-budget.test.ts new file mode 100644 index 0000000000..19bd7a090b --- /dev/null +++ b/tests/stream-buffer-budget.test.ts @@ -0,0 +1,107 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { bufferStreamIfFits, readStreamToBuffer } from "../src/server/stream-io.ts"; + +function source(chunks: Uint8Array[]): ReadableStream { + return new ReadableStream({ + start(controller) { + for (const chunk of chunks) controller.enqueue(chunk); + controller.close(); + }, + }); +} + +test("inspection buffers a complete body at or below its byte budget", async () => { + for (const cap of [0, 1, 4, 16]) { + const bytes = Buffer.alloc(cap, 65); + const stream = source([bytes.subarray(0, 1), Buffer.alloc(0), bytes.subarray(1)]); + const result = await bufferStreamIfFits(stream, cap); + assert.ok(Buffer.isBuffer(result)); + assert.deepEqual(result, bytes); + assert.equal(stream.locked, false); + } +}); + +test("overflow replays split UTF-8 bytes once and does not drain the unread tail", async () => { + const bytes = Buffer.from("a😀b-remaining-tail"); + const chunks = [bytes.subarray(0, 2), bytes.subarray(2, 4), bytes.subarray(4)]; + let reads = 0; + const stream = new ReadableStream({ + pull(controller) { + if (reads < chunks.length) controller.enqueue(chunks[reads++]); + else controller.close(); + }, + }, { highWaterMark: 0 }); + const result = await bufferStreamIfFits(stream, 3); + assert.ok(!Buffer.isBuffer(result)); + assert.equal(reads, 2, "inspection stops at the first over-budget chunk"); + assert.deepEqual(Buffer.from(await new Response(result).arrayBuffer()), bytes); + assert.equal(stream.locked, false); +}); + +test("cancelling an overflow before replay cancels and unlocks its source", async () => { + const reason = new Error("client disconnected before replay"); + let cancelled: unknown; + const stream = new ReadableStream({ + start(controller) { controller.enqueue(Buffer.from("over budget")); }, + cancel(value) { cancelled = value; }, + }); + const result = await bufferStreamIfFits(stream, 2); + assert.ok(!Buffer.isBuffer(result)); + await result.cancel(reason); + assert.equal(cancelled, reason); + assert.equal(stream.locked, false); +}); + +test("cancelling a replay with a pending tail read releases both readers", async () => { + const waiting = Promise.withResolvers(); + let cancelled = false; + let first = true; + const stream = new ReadableStream({ + pull(controller) { + if (first) { + first = false; + controller.enqueue(Buffer.from("over budget")); + } else waiting.resolve(); + }, + cancel() { cancelled = true; }, + }, { highWaterMark: 0 }); + const result = await bufferStreamIfFits(stream, 2); + assert.ok(!Buffer.isBuffer(result)); + const reader = result.getReader(); + assert.equal(Buffer.from((await reader.read()).value!).toString(), "over budget"); + const pending = reader.read(); + await waiting.promise; + await reader.cancel(); + assert.equal((await pending).done, true); + reader.releaseLock(); + assert.equal(cancelled, true); + assert.equal(stream.locked, false); + assert.equal(result.locked, false); +}); + +test("source errors propagate before and after the buffer limit", async () => { + for (const cap of [1, 100]) { + const failure = new Error("source read failed"); + let first = true; + const stream = new ReadableStream({ + pull(controller) { + if (first) { + first = false; + controller.enqueue(Buffer.from("prefix")); + } else controller.error(failure); + }, + }, { highWaterMark: 0 }); + await assert.rejects(async () => { + const result = await bufferStreamIfFits(stream, cap); + if (!Buffer.isBuffer(result)) await new Response(result).arrayBuffer(); + }, (error: unknown) => error === failure); + assert.equal(stream.locked, false); + } +}); + +test("the existing error-body reader still retains only its capped prefix", async () => { + const stream = source([Buffer.from("prefix"), Buffer.from("tail")]); + assert.equal((await readStreamToBuffer(stream, 3)).toString(), "pre"); + assert.equal(stream.locked, false); +}); From d60f690e22007c18626693c07daf7f5604d30e09 Mon Sep 17 00:00:00 2001 From: Markus Eisele Date: Wed, 18 Nov 2026 09:18:05 +0100 Subject: [PATCH 3/4] fix(openai): keep provider usage fields in synthesized completion Preserve the last upstream usage object when emitting the synthesized completion chunk. OpenAI clients that price a turn from provider fields (for example cost/credits) otherwise lose them. Forward the trailing usage-only frame to clients that set stream_options.include_usage, including text-only rounds, and reset the remembered usage per stream round so a later round cannot inherit a previous cost. Adds five regression tests in tests/openai-usage-frame-forward.test.ts. (cherry picked from commit 6228c3954218edbcdd550c6e990d91a2d57034e4) --- src/loop/adapter-openai.ts | 31 ++++++- tests/openai-usage-frame-forward.test.ts | 104 +++++++++++++++++++++++ 2 files changed, 133 insertions(+), 2 deletions(-) create mode 100644 tests/openai-usage-frame-forward.test.ts diff --git a/src/loop/adapter-openai.ts b/src/loop/adapter-openai.ts index 94efbafa5a..50afc60eef 100644 --- a/src/loop/adapter-openai.ts +++ b/src/loop/adapter-openai.ts @@ -195,8 +195,24 @@ function openaiCachedTokens(u: Record): number | null { export function createOpenaiAdapter(requestBody: Record, clientSystem?: string, absorbName?: string, notes?: string[], errorShape: "protocol" | "completion" = "protocol"): CompressLoopAdapter { const model = (requestBody.model as string) ?? "unknown"; + // OpenAI clients opt into the trailing usage frame with + // stream_options.include_usage. bili forces the upstream option on for its + // own ledger, but the frame must also reach the CLIENT when it asked for + // it: cost/context-aware clients read provider fields from that frame, and + // swallowing it on plain-text rounds silently loses them. Tool-call rounds + // keep the existing verbatim forward. + const clientWantsUsage = (() => { + const so = (requestBody as { stream_options?: unknown }).stream_options; + return typeof so === "object" && so !== null + && (so as Record).include_usage === true; + })(); let responseId = `chatcmpl-proxy-${Date.now()}`; let toolIndex = 0; + // Last upstream usage object seen on the wire. The synthesized completion + // frame re-states tokens; provider-specific fields (cost/credits, cache + // details, validity flags) must survive that re-statement — dropping them + // breaks any client that prices or accounts the turn from them. + let lastUsageRaw: Record | null = null; const makeBase = () => ({ id: responseId, @@ -280,6 +296,9 @@ export function createOpenaiAdapter(requestBody: Record, client }, async *parseStream(upstream, round) { + // Per-round scope: a later round that emits no usage frame must not + // inherit the previous round's provider fields (e.g. `credits`). + lastUsageRaw = null; const pending = new Map(); // #206: strip model-imitated render tags from PROSE deltas; the // filter may hold back a short tail, flushed at finish/[DONE]. @@ -535,6 +554,7 @@ export function createOpenaiAdapter(requestBody: Record, client if (!choice) { if (parsed.usage) { const u = parsed.usage as Record; + lastUsageRaw = u; yield { kind: "usage", inputTokens: typeof u.prompt_tokens === "number" ? u.prompt_tokens : undefined, @@ -545,7 +565,7 @@ export function createOpenaiAdapter(requestBody: Record, client // from this trailing empty-choices frame; raw tool-call rounds // must forward it verbatim, not swallow it into the internal // ledger. - if (sawRealToolCall) { + if (sawRealToolCall || clientWantsUsage) { yield { kind: "meta", chunk: rawBuf } as ParsedStreamEvent; } } @@ -578,6 +598,7 @@ export function createOpenaiAdapter(requestBody: Record, client // only the ledger gains the sample. if (parsed.usage && !finishReason && delta && Object.keys(delta).length === 0) { const u = parsed.usage as Record; + lastUsageRaw = u; yield { kind: "usage", inputTokens: typeof u.prompt_tokens === "number" ? u.prompt_tokens : undefined, @@ -591,6 +612,7 @@ export function createOpenaiAdapter(requestBody: Record, client const hadToolCalls = [...pending.values()].some((tc) => tc.name.length > 0 || tc.id.length > 0); yield* settleToolCalls(); const u = parsed.usage as Record | undefined; + if (u) lastUsageRaw = u; yield { kind: "usage", inputTokens: typeof u?.prompt_tokens === "number" ? u.prompt_tokens : undefined, @@ -727,13 +749,18 @@ export function createOpenaiAdapter(requestBody: Record, client // Numeric fields must never serialize to absent/undefined: strict // clients (dsh's mapUsage) compute over prompt_tokens/completion_tokens // and a usage object missing them yields NaN (non-JSON-serializable). + const raw = lastUsageRaw ?? {}; + const rawDetails = raw.prompt_tokens_details && typeof raw.prompt_tokens_details === "object" + ? raw.prompt_tokens_details as Record + : undefined; const usage = opts?.usage ? { + ...raw, prompt_tokens: opts.usage.inputTokens ?? 0, completion_tokens: opts.usage.outputTokens ?? 0, total_tokens: (opts.usage.inputTokens ?? 0) + (opts.usage.outputTokens ?? 0), ...(typeof opts.usage.cachedTokens === "number" - ? { prompt_tokens_details: { cached_tokens: opts.usage.cachedTokens } } + ? { prompt_tokens_details: { ...rawDetails, cached_tokens: opts.usage.cachedTokens } } : {}), } : null; diff --git a/tests/openai-usage-frame-forward.test.ts b/tests/openai-usage-frame-forward.test.ts new file mode 100644 index 0000000000..6052dd73d4 --- /dev/null +++ b/tests/openai-usage-frame-forward.test.ts @@ -0,0 +1,104 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { createOpenaiAdapter } from "../src/loop/index.ts"; +import type { ParsedStreamEvent } from "../src/loop/core.ts"; + +// Regression: a plain-text OpenAI turn with stream_options.include_usage must +// forward the trailing usage-only frame to the client. bili used to swallow it +// unless a tool call was seen, so clients that price the turn from that frame +// (IBM Bob) reported cost 0 while the account was still charged. + +const enc = new TextEncoder(); +const dec = new TextDecoder(); + +const frame = (obj: Record) => enc.encode(`data: ${JSON.stringify(obj)}\n\n`); + +const textTurn = (...extra: Uint8Array[]): ReadableStream => + new ReadableStream({ + start(controller) { + controller.enqueue(frame({ id: "c1", object: "chat.completion.chunk", created: 1, model: "gpt", choices: [{ index: 0, delta: { role: "assistant" }, finish_reason: null }] })); + controller.enqueue(frame({ id: "c1", object: "chat.completion.chunk", created: 1, model: "gpt", choices: [{ index: 0, delta: { content: "ok" }, finish_reason: null }] })); + controller.enqueue(frame({ id: "c1", object: "chat.completion.chunk", created: 1, model: "gpt", choices: [{ index: 0, delta: {}, finish_reason: "stop" }] })); + for (const e of extra) controller.enqueue(e); + controller.enqueue(enc.encode("data: [DONE]\n\n")); + controller.close(); + }, + }); + +const usageFrame = (prompt: number, completion: number) => + frame({ + id: "c1", + object: "chat.completion.chunk", + created: 1, + model: "gpt", + choices: [], + usage: { prompt_tokens: prompt, completion_tokens: completion, total_tokens: prompt + completion }, + }); + +const collect = async (body: Record, stream: ReadableStream) => { + const adapter = createOpenaiAdapter(body); + const events: ParsedStreamEvent[] = []; + for await (const ev of adapter.parseStream(stream, 1)) events.push(ev); + return events; +}; + +const wire = (events: ParsedStreamEvent[]): string => + events + .filter((ev) => ev.kind === "meta") + .map((ev) => dec.decode((ev as { chunk: Uint8Array }).chunk)) + .join(""); + +test("include_usage client receives the trailing usage-only frame on a text turn", async () => { + const events = await collect( + { model: "gpt", stream: true, stream_options: { include_usage: true } }, + textTurn(usageFrame(123, 4)), + ); + const out = wire(events); + assert.match(out, /"usage":\{"prompt_tokens":123/); + assert.match(out, /"completion_tokens":4/); +}); + +test("client that did not opt in does not receive the usage-only frame", async () => { + const events = await collect({ model: "gpt", stream: true }, textTurn(usageFrame(123, 4))); + assert.doesNotMatch(wire(events), /"prompt_tokens"/); +}); + +test("tool-call round still forwards the usage frame verbatim", async () => { + const toolCall = frame({ + id: "c1", + object: "chat.completion.chunk", + created: 1, + model: "gpt", + choices: [{ index: 0, delta: { tool_calls: [{ index: 0, id: "call_1", type: "function", function: { name: "bash", arguments: "{}" } }] }, finish_reason: "tool_calls" }], + }); + const events = await collect({ model: "gpt", stream: true }, textTurn(toolCall, usageFrame(50, 2))); + assert.match(wire(events), /"prompt_tokens":50/); +}); + +test("synthesized completion preserves provider-specific usage fields (credits)", async () => { + const adapter = createOpenaiAdapter({ model: "gpt", stream: true, stream_options: { include_usage: true } }); + const usageWithCredits = frame({ + choices: [{ delta: {}, index: 0 }], + created: 1, + id: "c1", + model: "gpt", + object: "chat.completion.chunk", + usage: { prompt_tokens: 123, completion_tokens: 4, total_tokens: 127, credits: 0.42, cache_validity: false }, + }); + for await (const _ of adapter.parseStream(textTurn(usageWithCredits), 1)) { /* drain */ } + const completion = adapter.emitCompletion({ usage: { inputTokens: 123, outputTokens: 4, cachedTokens: 0 } }).toString("utf8"); + assert.match(completion, /"credits":0.42/); + assert.match(completion, /"prompt_tokens":123/); +}); + +test("a round without usage does not inherit the previous round's credits", async () => { + const adapter = createOpenaiAdapter({ model: "gpt", stream: true, stream_options: { include_usage: true } }); + const withCredits = frame({ + choices: [{ delta: {}, index: 0 }], created: 1, id: "c1", model: "gpt", object: "chat.completion.chunk", + usage: { prompt_tokens: 10, completion_tokens: 2, total_tokens: 12, credits: 0.42 }, + }); + for await (const _ of adapter.parseStream(textTurn(withCredits), 1)) { /* drain */ } + for await (const _ of adapter.parseStream(textTurn(), 2)) { /* drain, no usage frame */ } + const completion = adapter.emitCompletion({ usage: { inputTokens: 10, outputTokens: 2, cachedTokens: 0 } }).toString("utf8"); + assert.doesNotMatch(completion, /"credits"/); +}); From 0e6b83aab4cac8fd80b8667dabe64a7aa33b30d4 Mon Sep 17 00:00:00 2001 From: ranxianglei Date: Sat, 21 Nov 2026 15:45:09 +0800 Subject: [PATCH 4/4] fix(tests): resolve the openai SDK via file:// URL so the import works on Windows (ERR_UNSUPPORTED_ESM_URL_SCHEME) --- tests/responses-content-index.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/responses-content-index.test.ts b/tests/responses-content-index.test.ts index 0c44b5cf83..3b2aa687a7 100644 --- a/tests/responses-content-index.test.ts +++ b/tests/responses-content-index.test.ts @@ -2,6 +2,7 @@ import { test } from "node:test"; import assert from "node:assert/strict"; import type { ServerResponse } from "node:http"; import { createRequire } from "node:module"; +import { pathToFileURL } from "node:url"; import { createResponsesAdapter } from "../src/loop/adapter-responses.ts"; import { pipePluginResponsesWithStrip } from "../src/plugin.ts"; import { emitStreamError } from "../src/stream-error.ts"; @@ -33,7 +34,7 @@ function collector(): { res: ServerResponse; chunks: string[] } { async function consume(raw: string) { const requireSdk = createRequire(import.meta.resolve("@earendil-works/pi-coding-agent")); - const { default: OpenAI } = await import(requireSdk.resolve("openai")); + const { default: OpenAI } = await import(pathToFileURL(requireSdk.resolve("openai")).href); const client = new OpenAI({ apiKey: "synthetic-test-key", fetch: async () => new Response(raw, { headers: { "content-type": "text/event-stream" } }),