diff --git a/docs/configuration.md b/docs/configuration.md index a2ba64df70..1ebf16324b 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1113,6 +1113,7 @@ Security boundary: the file controls ONLY *when* to upgrade and *which published - **Default:** `{ retries: 0, bufCapBytes: 16777216 }` - **Status:** ACTIVE - **Description:** Fake-completion safety net for hosts that abort mid-stream (#371) — **opt-in**, `retries: 0` (default) is the pre-#371 passthrough. `bufCapBytes` is the OOM guard on the buffered response. Twins `BILI_FAKE_COMPLETION_RETRIES`, `BILI_FAKE_BUF_CAP`. +- A response exceeding `bufCapBytes` continues through the normal streaming path, including its already-read prefix. The incomplete inspection neither retries nor changes the session's fake-completion streak; the cap does not truncate the answer. This applies to the first response and hinted retries. ### `codexCompact` / `ccrRetrievalTtlMs` / `decompressTmpCap` diff --git a/docs/configuration.zh-CN.md b/docs/configuration.zh-CN.md index 94d38a80f2..704430f02c 100644 --- a/docs/configuration.zh-CN.md +++ b/docs/configuration.zh-CN.md @@ -1107,6 +1107,7 @@ - **默认:** `{ retries: 0, bufCapBytes: 16777216 }` - **状态:** ACTIVE - **说明:** 针对流中途被宿主掐断场景的 fake-completion 安全网(#371)—— **显式开启**,`retries: 0`(默认)即 #371 之前的透传行为。`bufCapBytes` 是缓冲响应的 OOM 护栏。对应 `BILI_FAKE_COMPLETION_RETRIES`、`BILI_FAKE_BUF_CAP`。 +- 响应超过 `bufCapBytes` 时,已读前缀和后续内容会继续进入正常流式处理。未完成的检查不会触发重试或改变会话的 fake-completion 连续计数;该上限不会截断回答。首次响应与附带纠正提示的重试响应都遵循此规则。 ### `codexCompact` / `ccrRetrievalTtlMs` / `decompressTmpCap` diff --git a/src/loop/adapter-openai.ts b/src/loop/adapter-openai.ts index 94efbafa5a..50afc60eef 100644 --- a/src/loop/adapter-openai.ts +++ b/src/loop/adapter-openai.ts @@ -195,8 +195,24 @@ function openaiCachedTokens(u: Record): number | null { export function createOpenaiAdapter(requestBody: Record, clientSystem?: string, absorbName?: string, notes?: string[], errorShape: "protocol" | "completion" = "protocol"): CompressLoopAdapter { const model = (requestBody.model as string) ?? "unknown"; + // OpenAI clients opt into the trailing usage frame with + // stream_options.include_usage. bili forces the upstream option on for its + // own ledger, but the frame must also reach the CLIENT when it asked for + // it: cost/context-aware clients read provider fields from that frame, and + // swallowing it on plain-text rounds silently loses them. Tool-call rounds + // keep the existing verbatim forward. + const clientWantsUsage = (() => { + const so = (requestBody as { stream_options?: unknown }).stream_options; + return typeof so === "object" && so !== null + && (so as Record).include_usage === true; + })(); let responseId = `chatcmpl-proxy-${Date.now()}`; let toolIndex = 0; + // Last upstream usage object seen on the wire. The synthesized completion + // frame re-states tokens; provider-specific fields (cost/credits, cache + // details, validity flags) must survive that re-statement — dropping them + // breaks any client that prices or accounts the turn from them. + let lastUsageRaw: Record | null = null; const makeBase = () => ({ id: responseId, @@ -280,6 +296,9 @@ export function createOpenaiAdapter(requestBody: Record, client }, async *parseStream(upstream, round) { + // Per-round scope: a later round that emits no usage frame must not + // inherit the previous round's provider fields (e.g. `credits`). + lastUsageRaw = null; const pending = new Map(); // #206: strip model-imitated render tags from PROSE deltas; the // filter may hold back a short tail, flushed at finish/[DONE]. @@ -535,6 +554,7 @@ export function createOpenaiAdapter(requestBody: Record, client if (!choice) { if (parsed.usage) { const u = parsed.usage as Record; + lastUsageRaw = u; yield { kind: "usage", inputTokens: typeof u.prompt_tokens === "number" ? u.prompt_tokens : undefined, @@ -545,7 +565,7 @@ export function createOpenaiAdapter(requestBody: Record, client // from this trailing empty-choices frame; raw tool-call rounds // must forward it verbatim, not swallow it into the internal // ledger. - if (sawRealToolCall) { + if (sawRealToolCall || clientWantsUsage) { yield { kind: "meta", chunk: rawBuf } as ParsedStreamEvent; } } @@ -578,6 +598,7 @@ export function createOpenaiAdapter(requestBody: Record, client // only the ledger gains the sample. if (parsed.usage && !finishReason && delta && Object.keys(delta).length === 0) { const u = parsed.usage as Record; + lastUsageRaw = u; yield { kind: "usage", inputTokens: typeof u.prompt_tokens === "number" ? u.prompt_tokens : undefined, @@ -591,6 +612,7 @@ export function createOpenaiAdapter(requestBody: Record, client const hadToolCalls = [...pending.values()].some((tc) => tc.name.length > 0 || tc.id.length > 0); yield* settleToolCalls(); const u = parsed.usage as Record | undefined; + if (u) lastUsageRaw = u; yield { kind: "usage", inputTokens: typeof u?.prompt_tokens === "number" ? u.prompt_tokens : undefined, @@ -727,13 +749,18 @@ export function createOpenaiAdapter(requestBody: Record, client // Numeric fields must never serialize to absent/undefined: strict // clients (dsh's mapUsage) compute over prompt_tokens/completion_tokens // and a usage object missing them yields NaN (non-JSON-serializable). + const raw = lastUsageRaw ?? {}; + const rawDetails = raw.prompt_tokens_details && typeof raw.prompt_tokens_details === "object" + ? raw.prompt_tokens_details as Record + : undefined; const usage = opts?.usage ? { + ...raw, prompt_tokens: opts.usage.inputTokens ?? 0, completion_tokens: opts.usage.outputTokens ?? 0, total_tokens: (opts.usage.inputTokens ?? 0) + (opts.usage.outputTokens ?? 0), ...(typeof opts.usage.cachedTokens === "number" - ? { prompt_tokens_details: { cached_tokens: opts.usage.cachedTokens } } + ? { prompt_tokens_details: { ...rawDetails, cached_tokens: opts.usage.cachedTokens } } : {}), } : null; diff --git a/src/loop/adapter-responses.ts b/src/loop/adapter-responses.ts index 24f9d14605..eda5d21d50 100644 --- a/src/loop/adapter-responses.ts +++ b/src/loop/adapter-responses.ts @@ -237,10 +237,10 @@ function buildMessageItemSequence(itemId: string, outputIndex: number, text: str return Buffer.from( [ `event: response.output_item.added\ndata: ${JSON.stringify({ type: "response.output_item.added", output_index: outputIndex, item })}\n\n`, - `event: response.content_part.added\ndata: ${JSON.stringify({ type: "response.content_part.added", item_id: itemId, output_index: outputIndex, part })}\n\n`, - `event: response.output_text.delta\ndata: ${JSON.stringify({ type: "response.output_text.delta", item_id: itemId, output_index: outputIndex, delta: text })}\n\n`, - `event: response.output_text.done\ndata: ${JSON.stringify({ type: "response.output_text.done", item_id: itemId, output_index: outputIndex, text })}\n\n`, - `event: response.content_part.done\ndata: ${JSON.stringify({ type: "response.content_part.done", item_id: itemId, output_index: outputIndex, part: { type: "output_text", text } })}\n\n`, + `event: response.content_part.added\ndata: ${JSON.stringify({ type: "response.content_part.added", item_id: itemId, output_index: outputIndex, content_index: 0, part })}\n\n`, + `event: response.output_text.delta\ndata: ${JSON.stringify({ type: "response.output_text.delta", item_id: itemId, output_index: outputIndex, content_index: 0, delta: text })}\n\n`, + `event: response.output_text.done\ndata: ${JSON.stringify({ type: "response.output_text.done", item_id: itemId, output_index: outputIndex, content_index: 0, text })}\n\n`, + `event: response.content_part.done\ndata: ${JSON.stringify({ type: "response.content_part.done", item_id: itemId, output_index: outputIndex, content_index: 0, part: { type: "output_text", text } })}\n\n`, `event: response.output_item.done\ndata: ${JSON.stringify({ type: "response.output_item.done", output_index: outputIndex, item: doneItem })}\n\n`, ].join(""), "utf8", @@ -413,12 +413,12 @@ export function createResponsesAdapter(textProtocol?: boolean, projection?: Resp // identity filters, byte-verbatim (never rewritten; matches the // openai adapter's reasoning_filter and #1960/KDD#10). const reasoningFilters = new Map(); - let lastTextRef: { itemId: string; outputIndex: number } | null = null; + let lastTextRef: { itemId: string; outputIndex: number; contentIndex: unknown } | null = null; const flushFilter = function* (): Generator { const tail = tagFilter.flush(); if (tail.length > 0) { const raw = lastTextRef - ? rebuildResponsesEvent("response.output_text.delta", { type: "response.output_text.delta", item_id: lastTextRef.itemId, output_index: lastTextRef.outputIndex, delta: tail }) + ? rebuildResponsesEvent("response.output_text.delta", { type: "response.output_text.delta", item_id: lastTextRef.itemId, output_index: lastTextRef.outputIndex, content_index: lastTextRef.contentIndex, delta: tail }) : undefined; yield { kind: "text", delta: tail, ...(raw ? { raw } : {}) } as ParsedStreamEvent; } @@ -545,7 +545,7 @@ export function createResponsesAdapter(textProtocol?: boolean, projection?: Resp const itemId = typeof obj.item_id === "string" ? obj.item_id : ""; const outputIndex = typeof obj.output_index === "number" ? obj.output_index : 0; const mapped = remapped.get(itemId); - lastTextRef = { itemId: mapped ? mapped.id : itemId, outputIndex: mapped ? mapped.index : outputIndex }; + lastTextRef = { itemId: mapped ? mapped.id : itemId, outputIndex: mapped ? mapped.index : outputIndex, contentIndex: obj.content_index }; const clean = tagFilter.push(delta); if (clean.length > 0) { if (mapped) { diff --git a/src/plugin.ts b/src/plugin.ts index 867038aef8..52488ebbab 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -4612,13 +4612,13 @@ export async function pipePluginResponsesWithStrip( if (proseAcc.length === 0 || !session) return; scanModelOutputForPhantomAttachments(session.id, proseAcc); }; - let lastDeltaMeta: { item_id?: unknown; output_index?: unknown } | null = null; + let lastDeltaMeta: { item_id?: unknown; output_index?: unknown; content_index?: unknown } | null = null; const flushTail = (after: string) => { const tail = tagFilter.flush(); if (tail.length > 0) { visibleTextChars += tail.length; answerProse += tail; - const meta = inRetry && heldItemId !== undefined ? { item_id: heldItemId, output_index: heldOutputIndex } : (lastDeltaMeta ?? {}); + const meta = inRetry && heldItemId !== undefined ? { ...lastDeltaMeta, item_id: heldItemId, output_index: heldOutputIndex } : (lastDeltaMeta ?? {}); appendReplyDelta(meta, tail); return `data: ${JSON.stringify({ type: "response.output_text.delta", ...meta, delta: tail })}\n\n` + after; } @@ -5070,7 +5070,7 @@ export async function pipePluginResponsesWithStrip( continue; } const clean = tagFilter.push(delta); - lastDeltaMeta = { item_id: ev["item_id"], output_index: ev["output_index"] }; + lastDeltaMeta = { item_id: ev["item_id"], output_index: ev["output_index"], content_index: ev["content_index"] }; if (!inRetry && heldItemId === undefined && ev["item_id"] !== undefined) { heldItemId = ev["item_id"]; heldOutputIndex = ev["output_index"]; diff --git a/src/server/relay.ts b/src/server/relay.ts index 7ce093b91b..63fdf4cfd4 100644 --- a/src/server/relay.ts +++ b/src/server/relay.ts @@ -76,7 +76,7 @@ import { buildForwardHeaders, connectionNamedHeaders, RESPONSE_ONLY_STRIP_HEADER import { bodyDumpEnabled, logDumpFailure } from "./observability.js"; import { rejectionDiagnosticSuffix } from "./rejection-diagnostics.js"; import { effectiveAbsorbBlock, effectiveBlockOpsEnabled } from "./prepare-responses.js"; -import { awaitDrain, bufferToStream, dumpStreamToFile, pipeThrough, readStreamToBuffer } from "./stream-io.js"; +import { awaitDrain, bufferStreamIfFits, bufferToStream, dumpStreamToFile, pipeThrough, readStreamToBuffer } from "./stream-io.js"; import { buildForwardTarget, measuredBreakdownClause, measuredOutbound, outboundPayloadBreakdown, registerRequestAbort, beginStreamKeepalive, effectiveTokenCount, googlePathKind, imageBillingFor, imageTokenCapFor, inboundResignCredential, outboundContextEstimates, resolveUpstream, stripKernelSummaries, usageGradeInputBaseline, DOOMED_PROBE_ABSOLUTE_BYTES, DOOMED_PROBE_BYTES_PER_TOKEN, type Prepared } from "../server.js"; /** Retry the same input when the provider proved only its output reservation overflows. */ @@ -2292,6 +2292,7 @@ const reply = prepared && !prepared.sidePassthrough ? createHandoffReply(prepare // leaked a live idle timer. (#721/#2563: the SSE pipes themselves no // longer rethrow an upstream cut — they emit an in-band truncation // signal, or end raw for permitlist hosts that own the handling.) + let clearFakeRetryTimer: (() => void) | undefined; try { let pluginBody = upstream.body as ReadableStream; // #1295: the JSONL wire has no fake-completion shape to detect — @@ -2304,7 +2305,7 @@ const reply = prepared && !prepared.sidePassthrough ? createHandoffReply(prepare // fake completion. The agent owning compression does not change this: // the retry is a proxy→upstream HTTP call whose base must be what // upstream just accepted (aligned with the proxy lane below). - const resolvedBuf = await resolveFakeCompletion(pluginBody, { + const resolved = await resolveFakeCompletion(pluginBody, { protocol: prepared.protocol, wireBody, upstreamUrl, @@ -2315,7 +2316,8 @@ const reply = prepared && !prepared.sidePassthrough ? createHandoffReply(prepare log, model: prepared.model ?? prepared.google?.model, }); - pluginBody = bufferToStream(resolvedBuf); + pluginBody = resolved.body; + clearFakeRetryTimer = resolved.clearTimer; } // #2347: raw-dump twin of the tee below — this branch returns // before reaching it, so plugin-mode turns need their own. @@ -2421,7 +2423,8 @@ const reply = prepared && !prepared.sidePassthrough ? createHandoffReply(prepare clearUpstreamTimer(); // #2347: finish the dump before the turn closes — its value is the // full upstream tail, which the client may have stopped reading early. - if (dumpRaw) await dumpRaw; + try { if (dumpRaw) await dumpRaw; } + finally { clearFakeRetryTimer?.(); } } return; } @@ -2457,6 +2460,7 @@ const reply = prepared && !prepared.sidePassthrough ? createHandoffReply(prepare // response (any turn, not just compress-injected). Buffering is required: // the retry re-requests before the client sees the fake completion. let responseBody: ReadableStream = upstream.body; + let clearFakeRetryTimer: (() => void) | undefined; // #411: every body-consuming path below must clear the upstream idle timer // even when it throws (client abort / upstream cut) — previously an abort // skipped the trailing clearUpstreamTimer and leaked a live idle @@ -2470,7 +2474,7 @@ const reply = prepared && !prepared.sidePassthrough ? createHandoffReply(prepare // never the raw client body: upstream may have rejected those raw // bytes earlier in this session, and a 400'd hint would present the // fake completion instead of a corrected turn. - const resolvedBuf = await resolveFakeCompletion(upstream.body, { + const resolved = await resolveFakeCompletion(upstream.body, { protocol: prepared.protocol, wireBody, wrapBody: v1Active ? v1Wrap : undefined, @@ -2482,7 +2486,8 @@ const reply = prepared && !prepared.sidePassthrough ? createHandoffReply(prepare log, model: prepared.model ?? prepared.google?.model, }); - responseBody = bufferToStream(resolvedBuf); + responseBody = resolved.body; + clearFakeRetryTimer = resolved.clearTimer; } // #2347: dump the FINAL upstream body for every prepared streaming turn, // regardless of rewriter branch — the previous tee sat INSIDE the useRewriter @@ -2979,13 +2984,14 @@ const reply = prepared && !prepared.sidePassthrough ? createHandoffReply(prepare // the full upstream tail, which the client may have stopped reading // early. Idempotent where the rewriter lane already awaited it in its // own finally; this covers the non-rewriter branch's throw paths too. - if (dumpRaw) await dumpRaw; + try { if (dumpRaw) await dumpRaw; } + finally { clearFakeRetryTimer?.(); } } } // #371: buffer the raw upstream response, detect a fake completion (tool-call // XML, no real tool block), and — bounded per turn and per session — re-request -// upstream with a corrective hint. Returns the bytes to stream to the client +// upstream with a corrective hint. Returns the body to stream to the client // (the retry's response when a retry recovered, else the original). The session // streak (metadata.fakeCompletionStreak) counts consecutive fake-completion // turns: it gates retries (skip once >= cap) and resets to 0 on a clean turn. @@ -3018,8 +3024,11 @@ async function resolveFakeCompletion( * must go back out inside the CloudCode envelope. */ wrapBody?: (body: string | Buffer) => string | Buffer; }, -): Promise { - let buffer = await readStreamToBuffer(stream, fakeBufCap()); +): Promise<{ body: ReadableStream; clearTimer?: () => void }> { + const buffered = await bufferStreamIfFits(stream, fakeBufCap()); + // An incomplete inspection cannot classify this turn or change its streak. + if (!Buffer.isBuffer(buffered)) return { body: buffered }; + let buffer = buffered; const max = maxFakeCompletionRetries(); const sid = opts.session.id; const priorStreak = (opts.session.metadata.fakeCompletionStreak as number | undefined) ?? 0; @@ -3047,14 +3056,20 @@ async function resolveFakeCompletion( opts.log("warn", `[${sid}] fake-completion retry failed: ${String(e)}; presenting original`); break; } + let handedOff = false; try { if (!r.response.ok || !r.response.body) { opts.log("warn", `[${sid}] fake-completion retry rejected (HTTP ${r.response.status}); presenting original`); break; } - buffer = await readStreamToBuffer(r.response.body, fakeBufCap()); + const retried = await bufferStreamIfFits(r.response.body, fakeBufCap()); + if (!Buffer.isBuffer(retried)) { + handedOff = true; + return { body: retried, clearTimer: r.clearTimer }; + } + buffer = retried; } finally { - r.clearTimer(); + if (!handedOff) r.clearTimer(); } if (!isFakeCompletion(opts.protocol, buffer.toString("utf8"))) break; } @@ -3062,5 +3077,5 @@ async function resolveFakeCompletion( const stillFake = isFakeCompletion(opts.protocol, buffer.toString("utf8")); opts.session.metadata.fakeCompletionStreak = stillFake ? priorStreak + 1 : 0; markDirty(opts.session); - return buffer; + return { body: bufferToStream(buffer) }; } diff --git a/src/server/stream-io.ts b/src/server/stream-io.ts index 849c66cb58..c75ca6155f 100644 --- a/src/server/stream-io.ts +++ b/src/server/stream-io.ts @@ -31,6 +31,63 @@ export async function readStreamToBuffer(stream: ReadableStream, max return Buffer.concat(chunks); } +/** Buffer a response only while it fits the inspection budget. On overflow, + * replay the retained prefix and continue streaming without dropping bytes. */ +export async function bufferStreamIfFits(stream: ReadableStream, maxBytes: number): Promise> { + const reader = stream.getReader(); + const chunks: Buffer[] = []; + let kept = 0; + let handedOff = false; + try { + for (;;) { + const { done, value } = await reader.read(); + if (done) return Buffer.concat(chunks, kept); + if (value.byteLength === 0) continue; + if (kept + value.byteLength <= maxBytes) { + chunks.push(Buffer.from(value)); + kept += value.byteLength; + continue; + } + let ended = false; + const replay = new ReadableStream({ + start(controller) { + for (const chunk of chunks) controller.enqueue(chunk); + chunks.length = 0; + controller.enqueue(value); + }, + async pull(controller) { + try { + const next = await reader.read(); + if (ended) return; + if (next.done) { + ended = true; + reader.releaseLock(); + controller.close(); + } else { + controller.enqueue(next.value); + } + } catch (error) { + if (!ended) { + ended = true; + reader.releaseLock(); + controller.error(error); + } + } + }, + async cancel(reason) { + ended = true; + try { await reader.cancel(reason); } + finally { reader.releaseLock(); } + }, + }); + handedOff = true; + return replay; + } + } finally { + if (!handedOff) reader.releaseLock(); + } +} + export function bufferToStream(buf: Buffer): ReadableStream { return new ReadableStream({ start(controller) { diff --git a/src/stream-error.ts b/src/stream-error.ts index c1495a747e..2054993913 100644 --- a/src/stream-error.ts +++ b/src/stream-error.ts @@ -90,10 +90,10 @@ function legacyCompletionChunk(protocol: Protocol, visible: string, state?: Lega const oi = 0; const errorItem = { type: "message", id: itemId, role: "assistant", content: [{ type: "output_text", text: visible }] }; return `event: response.output_item.added\ndata: ${JSON.stringify({ type: "response.output_item.added", output_index: oi, item: { type: "message", id: itemId, role: "assistant", content: [] } })}\n\n` + - `event: response.content_part.added\ndata: ${JSON.stringify({ type: "response.content_part.added", item_id: itemId, output_index: oi, part: { type: "output_text", text: "" } })}\n\n` + - `event: response.output_text.delta\ndata: ${JSON.stringify({ type: "response.output_text.delta", item_id: itemId, output_index: oi, delta: visible })}\n\n` + - `event: response.output_text.done\ndata: ${JSON.stringify({ type: "response.output_text.done", item_id: itemId, output_index: oi, text: visible })}\n\n` + - `event: response.content_part.done\ndata: ${JSON.stringify({ type: "response.content_part.done", item_id: itemId, output_index: oi, part: { type: "output_text", text: visible } })}\n\n` + + `event: response.content_part.added\ndata: ${JSON.stringify({ type: "response.content_part.added", item_id: itemId, output_index: oi, content_index: 0, part: { type: "output_text", text: "" } })}\n\n` + + `event: response.output_text.delta\ndata: ${JSON.stringify({ type: "response.output_text.delta", item_id: itemId, output_index: oi, content_index: 0, delta: visible })}\n\n` + + `event: response.output_text.done\ndata: ${JSON.stringify({ type: "response.output_text.done", item_id: itemId, output_index: oi, content_index: 0, text: visible })}\n\n` + + `event: response.content_part.done\ndata: ${JSON.stringify({ type: "response.content_part.done", item_id: itemId, output_index: oi, content_index: 0, part: { type: "output_text", text: visible } })}\n\n` + `event: response.output_item.done\ndata: ${JSON.stringify({ type: "response.output_item.done", output_index: oi, item: errorItem })}\n\n` + `event: response.completed\ndata: ${JSON.stringify({ type: "response.completed", response: { status: "completed", output: [errorItem] } })}\n\n`; } diff --git a/tests/fake-completion-buffer-limit.test.ts b/tests/fake-completion-buffer-limit.test.ts new file mode 100644 index 0000000000..077ad922e4 --- /dev/null +++ b/tests/fake-completion-buffer-limit.test.ts @@ -0,0 +1,242 @@ +import assert from "node:assert/strict"; +import { randomUUID } from "node:crypto"; +import http from "node:http"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; +import { defaultConfig } from "acp-kernel"; +import type { ProxyOptions } from "../src/config.ts"; +import { _liveUpstreamTimersForTest } from "../src/fetch-util.ts"; +import { SessionStore, _setStoreForTest } from "../src/persist.ts"; +import { _resetPluginStateForTest } from "../src/plugin.ts"; +import { _setForTest as setRegistryForTest } from "../src/registry.ts"; +import { startServer } from "../src/server.ts"; +import { _resetSessionsForTest, getSession } from "../src/session.ts"; + +process.env.NODE_ENV = "test"; + +const FAKE = "FAKE-MARKER \x3cinvoke name=\"example\"\x3e\x3c/invoke\x3e"; +const LARGE = "begin-😀-" + "complete response ".repeat(600) + "-TAIL-MARKER"; + +function sse(text: string, toolArguments?: string): string { + const base = { id: "chatcmpl-buffer", object: "chat.completion.chunk", model: "buffer-test", created: 1 }; + const tool = toolArguments === undefined ? "" : `data: ${JSON.stringify({ ...base, choices: [{ index: 0, delta: { tool_calls: [{ index: 0, id: "call-preserved", type: "function", function: { name: "write_file", arguments: toolArguments } }] } }] })}\n\n`; + return `data: ${JSON.stringify({ ...base, choices: [{ index: 0, delta: { role: "assistant", content: text } }] })}\n\n` + + tool + + `data: ${JSON.stringify({ ...base, choices: [{ index: 0, delta: {}, finish_reason: toolArguments === undefined ? "stop" : "tool_calls" }], usage: { prompt_tokens: 10, completion_tokens: 20, total_tokens: 30 } })}\n\n` + + "data: [DONE]\n\n"; +} + +function json(text: string): string { + return JSON.stringify({ id: "chatcmpl-buffer", object: "chat.completion", model: "buffer-test", choices: [{ index: 0, message: { role: "assistant", content: text }, finish_reason: "stop" }] }); +} + +function textOf(raw: string): string { + let text = ""; + for (const line of raw.split("\n")) { + if (!line.startsWith("data:") || line.slice(5).trim() === "[DONE]") continue; + const event = JSON.parse(line.slice(5)); + text += event.choices?.[0]?.delta?.content ?? ""; + } + return text; +} + +type Reply = string | ((res: http.ServerResponse) => void); + +async function fixture(mode: "proxy" | "plugin", replies: Reply[], cap: number, opts: { json?: boolean; idleMs?: number; dumpSse?: string; priorStreak?: number } = {}) { + const before = { retries: process.env.BILI_FAKE_COMPLETION_RETRIES, cap: process.env.BILI_FAKE_BUF_CAP, idle: process.env.BILI_UPSTREAM_TIMEOUT_MS, firstByte: process.env.BILI_UPSTREAM_FIRST_BYTE_MS }; + process.env.BILI_FAKE_COMPLETION_RETRIES = "1"; + process.env.BILI_FAKE_BUF_CAP = String(cap); + if (opts.idleMs) { + process.env.BILI_UPSTREAM_TIMEOUT_MS = String(opts.idleMs); + process.env.BILI_UPSTREAM_FIRST_BYTE_MS = "10000"; + } + let hits = 0; + const upstream = http.createServer((req, res) => { + req.resume(); + req.on("end", () => { + res.writeHead(200, { "content-type": opts.json ? "application/json" : "text/event-stream" }); + const reply = replies[Math.min(hits++, replies.length - 1)]; + if (typeof reply === "string") res.end(reply); + else reply(res); + }); + }); + await new Promise((resolve) => upstream.listen(0, "127.0.0.1", resolve)); + const address = upstream.address(); + assert.ok(address && typeof address !== "string"); + _setStoreForTest(new SessionStore({ enabled: false })); + _resetSessionsForTest(); + setRegistryForTest({}); + _resetPluginStateForTest(); + const proxy = await startServer({ + port: 0, host: "127.0.0.1", upstream: "http://127.0.0.1", + routes: { [`http://127.0.0.1:${address.port}`]: { models: { "buffer-test": { context: 100_000 } } } }, + modelContextLimit: 100_000, kernelConfig: defaultConfig(100_000), + compress: { injectTool: false, injectNudge: false }, promptCache: { routing: "auto" }, + sessionHeader: "x-acp-session", log: false, debug: false, passthrough: false, + autoUpdate: false, compat: { roles: {} }, streamErrorShape: "protocol", passthroughSource: null, + autoRestartOnUpdate: false, updateTag: "latest", advisoryCheck: false, releaseNotesCheck: false, + mitm: { enabled: false, domains: [] }, + ...(opts.dumpSse ? { dumpSse: opts.dumpSse } : {}), + } as ProxyOptions); + if (!proxy.listening) await new Promise((resolve) => proxy.once("listening", resolve)); + const proxyAddress = proxy.address(); + assert.ok(proxyAddress && typeof proxyAddress !== "string"); + const session = "buffer-limit-" + randomUUID(); + if (opts.priorStreak !== undefined) getSession(session).metadata.fakeCompletionStreak = opts.priorStreak; + return { + async request(signal?: AbortSignal) { + return fetch(`http://127.0.0.1:${proxyAddress.port}/bili/http://127.0.0.1:${address.port}/v1/chat/completions`, { + method: "POST", + headers: { "content-type": "application/json", "x-acp-session": session, ...(mode === "plugin" ? { "x-bili-plugin": "pi" } : {}) }, + body: JSON.stringify({ model: "buffer-test", stream: true, max_tokens: 4096, messages: [{ role: "user", content: "Return the complete result" }] }), + signal, + }); + }, + hits: () => hits, + session: () => getSession(session), + async close() { + proxy.closeAllConnections(); + upstream.closeAllConnections(); + await Promise.all([new Promise((resolve) => proxy.close(() => resolve())), new Promise((resolve) => upstream.close(() => resolve()))]); + if (before.retries === undefined) delete process.env.BILI_FAKE_COMPLETION_RETRIES; + else process.env.BILI_FAKE_COMPLETION_RETRIES = before.retries; + if (before.cap === undefined) delete process.env.BILI_FAKE_BUF_CAP; + else process.env.BILI_FAKE_BUF_CAP = before.cap; + if (before.idle === undefined) delete process.env.BILI_UPSTREAM_TIMEOUT_MS; + else process.env.BILI_UPSTREAM_TIMEOUT_MS = before.idle; + if (before.firstByte === undefined) delete process.env.BILI_UPSTREAM_FIRST_BYTE_MS; + else process.env.BILI_UPSTREAM_FIRST_BYTE_MS = before.firstByte; + }, + }; +} + +for (const mode of ["proxy", "plugin"] as const) { + for (const retry of [false, true]) { + test(`${mode}: buffer cap preserves a ${retry ? "retried" : "first"} successful SSE response`, async () => { + const rig = await fixture(mode, retry ? [sse(FAKE), sse(LARGE)] : [sse(LARGE)], 1024); + try { + const response = await rig.request(); + assert.equal(response.status, 200); + const raw = await response.text(); + assert.equal(textOf(raw), LARGE); + assert.ok(raw.includes("[DONE]")); + assert.ok(raw.includes('"completion_tokens":20')); + assert.equal(rig.hits(), retry ? 2 : 1); + assert.equal(rig.session().metadata.fakeCompletionStreak, undefined); + assert.equal(_liveUpstreamTimersForTest(), 0); + } finally { await rig.close(); } + }); + } + + test(`${mode}: a fake completion exactly at the buffer cap is still checked`, async () => { + const fake = sse(FAKE); + const rig = await fixture(mode, [fake, sse("recovered")], Buffer.byteLength(fake)); + try { + assert.equal(textOf(await (await rig.request()).text()), "recovered"); + assert.equal(rig.hits(), 2); + } finally { await rig.close(); } + }); +} + +test("proxy: buffer cap preserves JSON returned to a streaming request", async () => { + const rig = await fixture("proxy", [json(LARGE)], 1024, { json: true }); + try { + const response = await rig.request(); + assert.equal(response.status, 200); + const body = await response.json() as { choices: Array<{ message: { content: string } }> }; + assert.equal(body.choices[0].message.content, LARGE); + assert.equal(rig.hits(), 1); + } finally { await rig.close(); } +}); + +async function within(promise: Promise, label: string): Promise { + let timer: NodeJS.Timeout | undefined; + try { + return await Promise.race([promise, new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error(label)), 5_000); + })]); + } finally { clearTimeout(timer); } +} + +async function timersCleared(): Promise { + const deadline = Date.now() + 5_000; + while (_liveUpstreamTimersForTest() !== 0 && Date.now() < deadline) await new Promise((resolve) => setTimeout(resolve, 10)); + assert.equal(_liveUpstreamTimersForTest(), 0, "upstream timers were not released"); +} + +for (const mode of ["proxy", "plugin"] as const) { + test(`${mode}: a tool call beyond the inspection cap remains byte-exact without a retry`, async () => { + const args = JSON.stringify({ contents: LARGE }); + const rig = await fixture(mode, [sse(FAKE + LARGE, args), sse("unexpected retry")], 1024); + try { + const raw = await (await rig.request()).text(); + const events = raw.split("\n").filter((line) => line.startsWith("data:") && !line.includes("[DONE]")).map((line) => JSON.parse(line.slice(5))); + const calls = events.flatMap((event) => event.choices?.[0]?.delta?.tool_calls ?? []); + assert.equal(calls.length, 1); + assert.equal(calls[0].id, "call-preserved"); + assert.equal(calls[0].function.arguments, args); + assert.equal(rig.hits(), 1); + } finally { await rig.close(); } + }); + + test(`${mode}: incomplete inspection leaves the prior fake-completion streak intact`, async () => { + const rig = await fixture(mode, [sse(LARGE)], 1024, { priorStreak: 1 }); + try { + assert.equal(textOf(await (await rig.request()).text()), LARGE); + assert.equal(rig.session().metadata.fakeCompletionStreak, 1); + assert.equal(rig.hits(), 1); + } finally { await rig.close(); } + }); + + test(`${mode}: an over-budget retry still aborts on upstream silence`, { timeout: 15_000 }, async () => { + const closed = Promise.withResolvers(); + const prefix = sse(LARGE).split("\n\n")[0] + "\n\n"; + const rig = await fixture(mode, [sse(FAKE), (res) => { + res.once("close", closed.resolve); + res.write(prefix); + }], 1024, { idleMs: 1500 }); + try { + const response = await rig.request(); + const raw = await within(response.text(), "retry idle watchdog was detached before the body finished"); + assert.equal(textOf(raw), LARGE); + assert.equal(rig.hits(), 2); + await within(closed.promise, "stalled retry connection stayed open"); + await timersCleared(); + } finally { await rig.close(); } + }); + + test(`${mode}: disconnect cancels an over-budget retry and its raw-dump branch`, { timeout: 15_000 }, async () => { + const closed = Promise.withResolvers(); + const prefix = sse(LARGE).split("\n\n")[0] + "\n\n"; + const dir = await mkdtemp(join(tmpdir(), "bili-fake-buffer-dump-")); + const rig = await fixture(mode, [sse(FAKE), (res) => { + res.once("close", closed.resolve); + res.write(prefix); + }], 1024, { dumpSse: dir }); + const abort = new AbortController(); + try { + const response = await rig.request(abort.signal); + const reader = response.body!.getReader(); + let raw = ""; + const decoder = new TextDecoder(); + await within((async () => { + while (!raw.includes("-TAIL-MARKER")) { + const next = await reader.read(); + assert.equal(next.done, false); + raw += decoder.decode(next.value, { stream: true }); + } + })(), "over-budget prefix was withheld until upstream EOF"); + assert.equal(textOf(raw), LARGE); + abort.abort(); + await within(closed.promise, "client abort did not close the live retry"); + await timersCleared(); + reader.releaseLock(); + } finally { + abort.abort(); + await rig.close(); + await rm(dir, { recursive: true, force: true }); + } + }); +} diff --git a/tests/openai-usage-frame-forward.test.ts b/tests/openai-usage-frame-forward.test.ts new file mode 100644 index 0000000000..6052dd73d4 --- /dev/null +++ b/tests/openai-usage-frame-forward.test.ts @@ -0,0 +1,104 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { createOpenaiAdapter } from "../src/loop/index.ts"; +import type { ParsedStreamEvent } from "../src/loop/core.ts"; + +// Regression: a plain-text OpenAI turn with stream_options.include_usage must +// forward the trailing usage-only frame to the client. bili used to swallow it +// unless a tool call was seen, so clients that price the turn from that frame +// (IBM Bob) reported cost 0 while the account was still charged. + +const enc = new TextEncoder(); +const dec = new TextDecoder(); + +const frame = (obj: Record) => enc.encode(`data: ${JSON.stringify(obj)}\n\n`); + +const textTurn = (...extra: Uint8Array[]): ReadableStream => + new ReadableStream({ + start(controller) { + controller.enqueue(frame({ id: "c1", object: "chat.completion.chunk", created: 1, model: "gpt", choices: [{ index: 0, delta: { role: "assistant" }, finish_reason: null }] })); + controller.enqueue(frame({ id: "c1", object: "chat.completion.chunk", created: 1, model: "gpt", choices: [{ index: 0, delta: { content: "ok" }, finish_reason: null }] })); + controller.enqueue(frame({ id: "c1", object: "chat.completion.chunk", created: 1, model: "gpt", choices: [{ index: 0, delta: {}, finish_reason: "stop" }] })); + for (const e of extra) controller.enqueue(e); + controller.enqueue(enc.encode("data: [DONE]\n\n")); + controller.close(); + }, + }); + +const usageFrame = (prompt: number, completion: number) => + frame({ + id: "c1", + object: "chat.completion.chunk", + created: 1, + model: "gpt", + choices: [], + usage: { prompt_tokens: prompt, completion_tokens: completion, total_tokens: prompt + completion }, + }); + +const collect = async (body: Record, stream: ReadableStream) => { + const adapter = createOpenaiAdapter(body); + const events: ParsedStreamEvent[] = []; + for await (const ev of adapter.parseStream(stream, 1)) events.push(ev); + return events; +}; + +const wire = (events: ParsedStreamEvent[]): string => + events + .filter((ev) => ev.kind === "meta") + .map((ev) => dec.decode((ev as { chunk: Uint8Array }).chunk)) + .join(""); + +test("include_usage client receives the trailing usage-only frame on a text turn", async () => { + const events = await collect( + { model: "gpt", stream: true, stream_options: { include_usage: true } }, + textTurn(usageFrame(123, 4)), + ); + const out = wire(events); + assert.match(out, /"usage":\{"prompt_tokens":123/); + assert.match(out, /"completion_tokens":4/); +}); + +test("client that did not opt in does not receive the usage-only frame", async () => { + const events = await collect({ model: "gpt", stream: true }, textTurn(usageFrame(123, 4))); + assert.doesNotMatch(wire(events), /"prompt_tokens"/); +}); + +test("tool-call round still forwards the usage frame verbatim", async () => { + const toolCall = frame({ + id: "c1", + object: "chat.completion.chunk", + created: 1, + model: "gpt", + choices: [{ index: 0, delta: { tool_calls: [{ index: 0, id: "call_1", type: "function", function: { name: "bash", arguments: "{}" } }] }, finish_reason: "tool_calls" }], + }); + const events = await collect({ model: "gpt", stream: true }, textTurn(toolCall, usageFrame(50, 2))); + assert.match(wire(events), /"prompt_tokens":50/); +}); + +test("synthesized completion preserves provider-specific usage fields (credits)", async () => { + const adapter = createOpenaiAdapter({ model: "gpt", stream: true, stream_options: { include_usage: true } }); + const usageWithCredits = frame({ + choices: [{ delta: {}, index: 0 }], + created: 1, + id: "c1", + model: "gpt", + object: "chat.completion.chunk", + usage: { prompt_tokens: 123, completion_tokens: 4, total_tokens: 127, credits: 0.42, cache_validity: false }, + }); + for await (const _ of adapter.parseStream(textTurn(usageWithCredits), 1)) { /* drain */ } + const completion = adapter.emitCompletion({ usage: { inputTokens: 123, outputTokens: 4, cachedTokens: 0 } }).toString("utf8"); + assert.match(completion, /"credits":0.42/); + assert.match(completion, /"prompt_tokens":123/); +}); + +test("a round without usage does not inherit the previous round's credits", async () => { + const adapter = createOpenaiAdapter({ model: "gpt", stream: true, stream_options: { include_usage: true } }); + const withCredits = frame({ + choices: [{ delta: {}, index: 0 }], created: 1, id: "c1", model: "gpt", object: "chat.completion.chunk", + usage: { prompt_tokens: 10, completion_tokens: 2, total_tokens: 12, credits: 0.42 }, + }); + for await (const _ of adapter.parseStream(textTurn(withCredits), 1)) { /* drain */ } + for await (const _ of adapter.parseStream(textTurn(), 2)) { /* drain, no usage frame */ } + const completion = adapter.emitCompletion({ usage: { inputTokens: 10, outputTokens: 2, cachedTokens: 0 } }).toString("utf8"); + assert.doesNotMatch(completion, /"credits"/); +}); diff --git a/tests/proxy-stream-error.test.ts b/tests/proxy-stream-error.test.ts index 61c9b211fa..f5d1eade21 100644 --- a/tests/proxy-stream-error.test.ts +++ b/tests/proxy-stream-error.test.ts @@ -163,6 +163,7 @@ test("emitStreamError: responses errorShape=\"completion\" emits a full item lif for (const block of itemEvents) { assert.match(block, new RegExp(`"item_id":"${itemId}"`)); assert.match(block, /"output_index":0/); + assert.match(block, /"content_index":0/); } // The completed response contains the error item in its output array. const completed = out.split("\n\n").find((b) => b.includes("response.completed")); diff --git a/tests/responses-content-index.test.ts b/tests/responses-content-index.test.ts new file mode 100644 index 0000000000..3b2aa687a7 --- /dev/null +++ b/tests/responses-content-index.test.ts @@ -0,0 +1,135 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import type { ServerResponse } from "node:http"; +import { createRequire } from "node:module"; +import { pathToFileURL } from "node:url"; +import { createResponsesAdapter } from "../src/loop/adapter-responses.ts"; +import { pipePluginResponsesWithStrip } from "../src/plugin.ts"; +import { emitStreamError } from "../src/stream-error.ts"; +import { validateResponsesContentIndex } from "./wire-contract-fakes.ts"; + +function frame(event: Record): string { + return `event: ${String(event.type)}\ndata: ${JSON.stringify(event)}\n\n`; +} + +function created(): string { + return frame({ type: "response.created", response: { id: "resp_fixture", object: "response", status: "in_progress", output: [] } }); +} + +function payloads(raw: string): Record[] { + return raw.split("\n").filter((line) => line.startsWith("data: ")).map((line) => JSON.parse(line.slice(6)) as Record); +} + +function collector(): { res: ServerResponse; chunks: string[] } { + const chunks: string[] = []; + const res = { + write(chunk: string | Buffer) { chunks.push(chunk.toString()); return true; }, + end() { return this; }, + once() { return this; }, + destroyed: false, + writableEnded: false, + } as unknown as ServerResponse; + return { res, chunks }; +} + +async function consume(raw: string) { + const requireSdk = createRequire(import.meta.resolve("@earendil-works/pi-coding-agent")); + const { default: OpenAI } = await import(pathToFileURL(requireSdk.resolve("openai")).href); + const client = new OpenAI({ + apiKey: "synthetic-test-key", + fetch: async () => new Response(raw, { headers: { "content-type": "text/event-stream" } }), + }); + const stream = client.responses.stream({ model: "synthetic", input: "synthetic" }); + const deltas: Array<{ delta: string; content_index: number; output_index: number }> = []; + stream.on("response.output_text.delta", (event: { delta: string; content_index: number; output_index: number }) => deltas.push(event)); + const response = await stream.finalResponse(); + return { deltas, response }; +} + +function upstreamText(text: string, contentIndex: number, id = "msg_fixture"): string { + const earlier = Array.from({ length: contentIndex }, () => ({ type: "output_text", text: "earlier part" })); + const content = [...earlier, { type: "output_text", text }]; + const item = { type: "message", id, role: "assistant", content }; + const ref = { item_id: id, output_index: 0, content_index: contentIndex }; + return created() + [ + { type: "response.output_item.added", output_index: 0, item: { ...item, content: [] } }, + ...earlier.map((part, index) => ({ type: "response.content_part.added", item_id: id, output_index: 0, content_index: index, part })), + { type: "response.content_part.added", ...ref, part: { type: "output_text", text: "" } }, + { type: "response.output_text.delta", ...ref, delta: text }, + { type: "response.output_text.done", ...ref, text }, + { type: "response.content_part.done", ...ref, part: { type: "output_text", text } }, + { type: "response.output_item.done", output_index: 0, item }, + { type: "response.completed", response: { id: "resp_fixture", status: "completed", output: [item] } }, + ].map(frame).join(""); +} + +test("official Responses SDK consumes a legacy stream-error completion", async () => { + const { res, chunks } = collector(); + emitStreamError(res, "responses", "kaboom", undefined, "completion"); + const { deltas, response } = await consume(created() + chunks.join("")); + assert.deepEqual(deltas.map((event) => event.delta), ["\n❌ [ACP] stream error: kaboom"]); + assert.equal(response.status, "completed"); + assert.equal(response.output[0].content[0].text, deltas[0].delta); +}); + +test("official Responses SDK still rejects the default native error", async () => { + const { res, chunks } = collector(); + emitStreamError(res, "responses", "kaboom"); + await assert.rejects(consume(created() + chunks.join("")), { type: "error", code: "stream_error", message: "[acp-proxy: kaboom]" }); +}); + +for (const kind of ["text", "marker"] as const) { + test(`official Responses SDK consumes synthesized ${kind} content`, async () => { + const adapter = createResponsesAdapter(); + const chunk = kind === "text" ? adapter.emitText("generated text") : adapter.emitMarker("acp_status", "ready"); + const events = payloads(chunk.toString()); + const item = events.find((event) => event.type === "response.output_item.done")?.item; + assert.ok(item); + const { deltas } = await consume(created() + chunk + frame({ type: "response.completed", response: { id: "resp_fixture", status: "completed", output: [item] } })); + assert.equal(deltas.length, 1); + assert.equal(deltas[0].content_index, 0); + if (kind === "text") assert.equal(deltas[0].delta, "generated text"); + else assert.match(deltas[0].delta, /\[ACP\]/); + assert.deepEqual(events.flatMap(validateResponsesContentIndex), []); + }); +} + +for (const mode of ["proxy", "plugin"] as const) { + for (const contentIndex of [0, 1]) { + test(`${mode} Responses text flush preserves content index ${contentIndex}`, async () => { + const input = upstreamText("Length: 10 m", contentIndex); + const { res, chunks } = collector(); + if (mode === "plugin") { + await pipePluginResponsesWithStrip(new Response(input).body!, res); + } else { + const adapter = createResponsesAdapter(); + for await (const event of adapter.parseStream(new Response(input).body!, 1)) { + if (event.kind === "meta") chunks.push(event.chunk.toString()); + else if (event.kind === "text" && event.raw) chunks.push(event.raw.toString()); + } + chunks.push(adapter.emitCompletion().toString()); + } + const { deltas, response } = await consume(chunks.join("")); + assert.deepEqual(deltas.map((event) => event.delta), ["Length: 10 ", "m"]); + for (const delta of deltas) assert.equal(delta.content_index, contentIndex); + assert.equal(response.output[0].content[contentIndex].text, "Length: 10 m"); + if (contentIndex > 0) assert.equal(response.output[0].content[0].text, "earlier part"); + assert.deepEqual(payloads(chunks.join("")).flatMap(validateResponsesContentIndex), []); + }); + } +} + +test("plugin Responses retry flush preserves its content index", async () => { + const { res, chunks } = collector(); + let retries = 0; + const echo = "\x3cacp tokens=\"1\" type=\"text\"\x3em00001\x3c/acp\x3e"; + await pipePluginResponsesWithStrip(new Response(upstreamText(echo, 0)).body!, res, undefined, undefined, async () => { + retries++; + return new Response(upstreamText("Length: 10 m", 0, "msg_retry")).body; + }); + assert.equal(retries, 1); + const { deltas } = await consume(chunks.join("")); + assert.equal(deltas.map((event) => event.delta).join(""), "Length: 10 m"); + for (const delta of deltas) assert.equal(delta.content_index, 0); + assert.deepEqual(payloads(chunks.join("")).flatMap(validateResponsesContentIndex), []); +}); diff --git a/tests/stream-buffer-budget.test.ts b/tests/stream-buffer-budget.test.ts new file mode 100644 index 0000000000..19bd7a090b --- /dev/null +++ b/tests/stream-buffer-budget.test.ts @@ -0,0 +1,107 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { bufferStreamIfFits, readStreamToBuffer } from "../src/server/stream-io.ts"; + +function source(chunks: Uint8Array[]): ReadableStream { + return new ReadableStream({ + start(controller) { + for (const chunk of chunks) controller.enqueue(chunk); + controller.close(); + }, + }); +} + +test("inspection buffers a complete body at or below its byte budget", async () => { + for (const cap of [0, 1, 4, 16]) { + const bytes = Buffer.alloc(cap, 65); + const stream = source([bytes.subarray(0, 1), Buffer.alloc(0), bytes.subarray(1)]); + const result = await bufferStreamIfFits(stream, cap); + assert.ok(Buffer.isBuffer(result)); + assert.deepEqual(result, bytes); + assert.equal(stream.locked, false); + } +}); + +test("overflow replays split UTF-8 bytes once and does not drain the unread tail", async () => { + const bytes = Buffer.from("a😀b-remaining-tail"); + const chunks = [bytes.subarray(0, 2), bytes.subarray(2, 4), bytes.subarray(4)]; + let reads = 0; + const stream = new ReadableStream({ + pull(controller) { + if (reads < chunks.length) controller.enqueue(chunks[reads++]); + else controller.close(); + }, + }, { highWaterMark: 0 }); + const result = await bufferStreamIfFits(stream, 3); + assert.ok(!Buffer.isBuffer(result)); + assert.equal(reads, 2, "inspection stops at the first over-budget chunk"); + assert.deepEqual(Buffer.from(await new Response(result).arrayBuffer()), bytes); + assert.equal(stream.locked, false); +}); + +test("cancelling an overflow before replay cancels and unlocks its source", async () => { + const reason = new Error("client disconnected before replay"); + let cancelled: unknown; + const stream = new ReadableStream({ + start(controller) { controller.enqueue(Buffer.from("over budget")); }, + cancel(value) { cancelled = value; }, + }); + const result = await bufferStreamIfFits(stream, 2); + assert.ok(!Buffer.isBuffer(result)); + await result.cancel(reason); + assert.equal(cancelled, reason); + assert.equal(stream.locked, false); +}); + +test("cancelling a replay with a pending tail read releases both readers", async () => { + const waiting = Promise.withResolvers(); + let cancelled = false; + let first = true; + const stream = new ReadableStream({ + pull(controller) { + if (first) { + first = false; + controller.enqueue(Buffer.from("over budget")); + } else waiting.resolve(); + }, + cancel() { cancelled = true; }, + }, { highWaterMark: 0 }); + const result = await bufferStreamIfFits(stream, 2); + assert.ok(!Buffer.isBuffer(result)); + const reader = result.getReader(); + assert.equal(Buffer.from((await reader.read()).value!).toString(), "over budget"); + const pending = reader.read(); + await waiting.promise; + await reader.cancel(); + assert.equal((await pending).done, true); + reader.releaseLock(); + assert.equal(cancelled, true); + assert.equal(stream.locked, false); + assert.equal(result.locked, false); +}); + +test("source errors propagate before and after the buffer limit", async () => { + for (const cap of [1, 100]) { + const failure = new Error("source read failed"); + let first = true; + const stream = new ReadableStream({ + pull(controller) { + if (first) { + first = false; + controller.enqueue(Buffer.from("prefix")); + } else controller.error(failure); + }, + }, { highWaterMark: 0 }); + await assert.rejects(async () => { + const result = await bufferStreamIfFits(stream, cap); + if (!Buffer.isBuffer(result)) await new Response(result).arrayBuffer(); + }, (error: unknown) => error === failure); + assert.equal(stream.locked, false); + } +}); + +test("the existing error-body reader still retains only its capped prefix", async () => { + const stream = source([Buffer.from("prefix"), Buffer.from("tail")]); + assert.equal((await readStreamToBuffer(stream, 3)).toString(), "pre"); + assert.equal(stream.locked, false); +}); diff --git a/tests/wire-contract-fakes.ts b/tests/wire-contract-fakes.ts index 9b6797ee1b..ecb00b1426 100644 --- a/tests/wire-contract-fakes.ts +++ b/tests/wire-contract-fakes.ts @@ -31,6 +31,13 @@ interface WireRule { } export const WIRE_RULES: readonly WireRule[] = [ + { + id: "WC-026", + wire: "responses", + lane: "responses-sse", + summary: "Responses text/content-part SSE events carry a non-negative integer content_index identifying the part inside the indexed output item", + provenance: "OpenAI openai-node v6.26.0 requires content_index in ResponseTextDeltaEvent/ResponseTextDoneEvent/ResponseContentPartAddedEvent/ResponseContentPartDoneEvent (https://github.com/openai/openai-node/blob/v6.26.0/src/resources/responses/responses.ts); ResponseStream indexes output.content[event.content_index] (https://github.com/openai/openai-node/blob/v6.26.0/src/lib/responses/ResponseStream.ts). Reproduced against that official SDK: bili's synthesized messages, legacy stream-error completions and filtered text tails omitted indices, so finalResponse() rejected with 'missing content at index undefined'. Gates exercise the actual SDK in proxy/plugin paths and the emitted error frames; the fake validates its outgoing SSE frames with the same constraint.", + }, { id: "WC-017", wire: "responses", @@ -772,6 +779,8 @@ interface FakeUpstream { } function sse(res: http.ServerResponse, events: Array<[string, unknown]>, opts?: { done?: boolean }): void { + const violations = events.flatMap(([, data]) => validateResponsesContentIndex(data)); + if (violations.length) throw new Error(violations.join("; ")); res.writeHead(200, { "content-type": "text/event-stream", "cache-control": "no-cache", connection: "keep-alive" }); // OpenAI streams carry no `event:` line at all — only Anthropic/Gemini name their events. for (const [ev, data] of events) { @@ -811,6 +820,13 @@ function openAiChatReply(res: http.ServerResponse, parsed: Record= 0 + ? [] + : ["WC-026 Responses content-part events require a non-negative integer content_index"]; +} + function responsesReply(res: http.ServerResponse, parsed: Record, text: string): void { const msgId = "msg_fake"; const msg = { type: "message", id: msgId, role: "assistant", content: [{ type: "output_text", text }] }; diff --git a/tests/wire-contract.test.ts b/tests/wire-contract.test.ts index bda8c0071e..6ccfe5e93f 100644 --- a/tests/wire-contract.test.ts +++ b/tests/wire-contract.test.ts @@ -21,6 +21,8 @@ import assert from "node:assert/strict"; import http from "node:http"; import { once } from "node:events"; import { readFileSync } from "node:fs"; +import { emitStreamError } from "../src/stream-error.ts"; +import { validateResponsesContentIndex } from "./wire-contract-fakes.ts"; process.env.NODE_ENV = "test"; @@ -57,6 +59,25 @@ import { verifySdkHmac, WIRE_RULES, WIRE_HEADER_RULES, VALIDATORS, startFakeUpst type ToolShape = Record; +test("WC-026 Responses content-part indices are explicit, including legacy error completions", () => { + for (const type of ["response.content_part.added", "response.output_text.delta", "response.output_text.done", "response.content_part.done"]) { + for (const content_index of [0, 2]) assert.deepEqual(validateResponsesContentIndex({ type, content_index }), []); + for (const content_index of [undefined, null, "0", -1, 0.5]) assert.match(validateResponsesContentIndex({ type, content_index })[0], /WC-026/); + } + assert.deepEqual(validateResponsesContentIndex({ type: "response.output_item.done" }), []); + const frames: Record[] = []; + const res = { + write(chunk: string) { + for (const line of chunk.split("\n")) if (line.startsWith("data: ")) frames.push(JSON.parse(line.slice(6))); + return true; + }, + end() { return this; }, + } as unknown as http.ServerResponse; + emitStreamError(res, "responses", "fixture failure", undefined, "completion"); + assert.equal(frames.length, 7); + assert.deepEqual(frames.flatMap(validateResponsesContentIndex), []); +}); + test("WC-013: WebSocket transport envelopes omit HTTP-only fields", () => { assert.deepEqual(validateResponsesWsCreate({ type: "response.create", input: [] }), []); for (const key of ["stream", "background", "stream_options"]) assert.match(validateResponsesWsCreate({ type: "response.create", [key]: true })[0], /WC-013/); @@ -369,6 +390,7 @@ test("wire-contract ledger: every lane-scoped rule has a live enforcement clause // Lane-scoped constraints hold on ONE client's lane only — the shared wire validator // cannot enforce them (WC-013 forbids exactly what WC-017 requires, same frame shape). const laneProbes: Array<{ ruleId: string; validate: (body: unknown) => string[]; body: unknown }> = [ + { ruleId: "WC-026", validate: validateResponsesContentIndex, body: { type: "response.output_text.delta", output_index: 0, delta: "x" } }, { ruleId: "WC-017", validate: validateCodexResponsesWsCreate, body: { type: "response.create", input: [] } }, // WC-019/020/021 are the Vertex Anthropic lane (lane: "vertex-anthropic") — the wire rules that // hold only on that lane, so the shared anthropic validator must not carry them. WC-019 is