diff --git a/CONFIGURATION.md b/CONFIGURATION.md index 8ac177db79..eb7b30929e 100644 --- a/CONFIGURATION.md +++ b/CONFIGURATION.md @@ -359,6 +359,7 @@ Environment variables take precedence over the config file. They are useful for | `ACP_COMPRESS_NUDGE` | Set to `0` to disable nudge injection (same as `"compress.injectNudge": false`). | | `ACP_MODEL_CONTEXT_LIMIT` | Override the context limit globally (absolute token count). | | `BILI_IMAGE_TOKEN_CAP` | Cap the per-image token estimate used by the preflight size gate and output clamp (#488/#496). By default an inline `data:` image counts as `base64 length / 4` tokens with **no cap** — correct for byte-billing relays, but a large over-estimate for pixel-tile upstreams (official Anthropic/OpenAI), which bill each image at roughly 1.1K–1.6K tokens regardless of byte size. Set this to your upstream's per-image tile cost so the gate reflects real billing; unset = no cap (current default). | +| `BILI_PREFLIGHT_HOLD_MS` | Grace period (ms) before a long preflight compression starts holding the client with keep-alive bytes (default `30000`; see #568 / README "Preflight hold"). | | `BILI_CONFIG_FILE` | Override the config file path (point at any JSON file). | | `ACP_PORT` / `PORT` | Override the listen port. | | `ACP_HOST` | Override the listen host. | diff --git a/CONFIGURATION.zh-CN.md b/CONFIGURATION.zh-CN.md index e047c3e9e0..574e475f66 100644 --- a/CONFIGURATION.zh-CN.md +++ b/CONFIGURATION.zh-CN.md @@ -357,6 +357,7 @@ | `ACP_COMPRESS_NUDGE` | 设为 `0` 禁用 nudge 注入(等同 `"compress.injectNudge": false`)。 | | `ACP_MODEL_CONTEXT_LIMIT` | 全局覆盖上下文上限(绝对 token 数)。 | | `BILI_IMAGE_TOKEN_CAP` | 预检尺寸门与输出钳制用的单图 token 估算上限(#488/#496)。默认内联 `data:` 图片按 `base64 长度 / 4` 计 token、**无上限** —— 对字节计费 relay 正确,但对像素 tile 计费的官方上游(Anthropic/OpenAI)会严重高估(后者无论字节多少,每图约计 1.1K–1.6K token)。设为你上游的单图 tile 成本,可让尺寸门反映真实计费;不设置 = 无上限(当前默认)。 | +| `BILI_PREFLIGHT_HOLD_MS` | 预压缩超过该宽限期(毫秒)后,代理提前提交响应并用保活字节挂住客户端(默认 `30000`;见 #568 / README「预压缩挂起」)。 | | `BILI_CONFIG_FILE` | 覆盖配置文件路径(指向任意 JSON 文件)。 | | `ACP_PORT` / `PORT` | 覆盖监听端口。 | | `ACP_HOST` | 覆盖监听主机。 | diff --git a/README.md b/README.md index b997a01f15..478a46bb38 100644 --- a/README.md +++ b/README.md @@ -112,6 +112,7 @@ turn is a semantic mismatch the model tolerates (it is clearly marked summaries (their tool call isn't in the agent's history and the agent's view skips `acp_summary`) — but that needs the id match above, which doesn't occur. + ## Which do I need? Pick by your client: diff --git a/README.zh-CN.md b/README.zh-CN.md index 580fd50bf7..be46b74062 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -164,6 +164,7 @@ bili --host 0.0.0.0 # 所有网卡(或直接用局域网 IP) 端点仍仅限本机访问。 - 启动时的 `[security]` 警告会提醒上述事项。 + ### 调试 三种方式打开详细日志(优先级:参数 > 环境变量 > 配置): diff --git a/src/server.ts b/src/server.ts index af08630f83..02e5022c25 100644 --- a/src/server.ts +++ b/src/server.ts @@ -45,7 +45,7 @@ import { getSession, listSessions, type Session, initSessions, markDirty, flushA import { COMPRESS_TOOL, ACP_TOOLS_ANTHROPIC, ACP_TOOLS_OPENAI, ACP_TOOLS_RESPONSES, ACP_READONLY_TOOLS_RESPONSES, COMPRESS_TOOL_NAME, buildCompressSystemPrompt, buildCompressHybridSystemPrompt, withStagedCompressGuidance } from "./compress-tool.js"; import { rewriteJsonResponse, type RewriteCtx } from "./stream.js"; import { applyRanges } from "./stream.js"; -import { preflightCompress, estimateCoreMessages, estimateRawBodyTokens, estimateCoreMessagesUpper } from "./preflight.js"; +import { preflightCompress, estimateCoreMessages, estimateRawBodyTokens, estimateCoreMessagesUpper, type PreflightResult } from "./preflight.js"; import { imageTokensInRawBody, imageTokensInParsedBody } from "./image-tokens.js"; import { renderUI, handleConfigGet, handleConfigPut } from "./web/index.js"; import { reapOrphanBlocks } from "./orphan-gc.js"; @@ -63,7 +63,7 @@ import { stripAcpPanelMessages, stripAcpPanelResponsesInput } from "./acp-panel. import { rewriteOpenaiJsonResponse } from "./stream-openai.js"; import { rewriteResponsesJsonResponse } from "./stream-responses.js"; import { observeResponsesTerminalState } from "./stream-terminal.js"; -import { emitStreamError } from "./stream-error.js"; +import { emitPreflightError, emitStreamError } from "./stream-error.js"; import { affinityToken, clientConversationHeader, codexTurnIdentity, preferPromptCacheKeyIdentity, type ConversationIdentity } from "./session-id.js"; import { prefixAffinity, type AnonymousAffinity } from "./prefix-affinity.js"; import { flushPrefixAffinity, hydratePrefixAffinity, scheduleAffinityPersist } from "./affinity-persist.js"; @@ -1581,22 +1581,43 @@ async function handle( // #301: the payload still overflows the window and // preflight could not fix it — answer with a // structured error instead of forwarding. - if (outcome.respond && !res.headersSent && !res.destroyed) { - // Retry-After on the 503 (rate-limited) path: gives - // well-behaved clients a backoff signal instead of - // hammering the rate-limited upstream (#301). - res.writeHead(outcome.status, { - "content-type": "application/json", - ...(outcome.status === 503 ? { "retry-after": "30" } : {}), - }); - res.end(JSON.stringify({ - error: { - type: "server_error", - code: "preflight_compress_failed", - message: outcome.message, - retryable: outcome.retryable, - }, - })); + if (outcome.respond && !res.destroyed) { + if (res.headersSent) { + // #568: the hold already committed 200 early — the status + // can no longer change, so deliver the same error in-band + // (protocol error event for SSE, identical JSON body for + // non-stream) instead of a status code we lost. + if (prepared!.stream) { + emitPreflightError(res, prepared!.protocol, { message: outcome.message, retryable: outcome.retryable }, (m) => log("warn", m)); + } else { + try { + res.end(JSON.stringify({ + error: { + type: "server_error", + code: "preflight_compress_failed", + message: outcome.message, + retryable: outcome.retryable, + }, + })); + } catch { /* client gone */ } + } + } else { + // Retry-After on the 503 (rate-limited) path: gives + // well-behaved clients a backoff signal instead of + // hammering the rate-limited upstream (#301). + res.writeHead(outcome.status, { + "content-type": "application/json", + ...(outcome.status === 503 ? { "retry-after": "30" } : {}), + }); + res.end(JSON.stringify({ + error: { + type: "server_error", + code: "preflight_compress_failed", + message: outcome.message, + retryable: outcome.retryable, + }, + })); + } } return; } @@ -2753,6 +2774,65 @@ function isPreflightFailFast(outcome: Prepared | PreflightFailFast): outcome is return "failFast" in outcome; } +// #568: preflight compression sends ZERO bytes to the client while it runs, so +// undici's default headersTimeout (300s) kills any multi-round compression that +// crosses it — the proxy then aborts on the detected disconnect and the client +// retries into the same wall (5-minute death loop). Once the work outlives this +// grace period the response is committed early (200 + protocol framing) and the +// client is held with periodic keep-alive bytes until the real response exists. +// 30s protects every client whose header deadline exceeds 30s (undici's 300s +// default included); shorter preflights keep full status-code fidelity. +const PREFLIGHT_HOLD_GRACE_DEFAULT_MS = 30_000; +const PREFLIGHT_KEEPALIVE_MS = 15_000; + +function preflightHoldGraceMs(): number { + const raw = process.env.BILI_PREFLIGHT_HOLD_MS; + if (!raw) return PREFLIGHT_HOLD_GRACE_DEFAULT_MS; + const v = Number(raw); + return Number.isFinite(v) && v >= 0 ? Math.floor(v) : PREFLIGHT_HOLD_GRACE_DEFAULT_MS; +} + +/** #568: commit the response early so a long preflight cannot lose the client + * to its header timeout. Streaming clients get an SSE stream with keep-alive + * comment lines (`: bili-preflight` — a spec-mandated no-op for every SSE + * consumer, same pattern as OpenAI's SSE pings); non-streaming clients get + * chunked JSON padded with whitespace (valid JSON padding). Each byte resets + * undici's bodyTimeout (inactivity-based), holding the client for the whole + * compression. Returns a stop() ending the keep-alive, or undefined when + * nothing could be committed (headers already sent / socket gone — the + * existing res "close" abort then handles cancellation). */ +function beginPreflightHold(res: http.ServerResponse, prepared: Prepared, log: (level: string, msg: string) => void): (() => void) | undefined { + if (res.headersSent || res.destroyed || res.writableEnded) return undefined; + const sid = prepared.session.id; + const keepAlive = prepared.stream ? ": bili-preflight\n\n" : " "; + try { + if (prepared.stream) { + res.writeHead(200, { + "content-type": "text/event-stream", + "cache-control": "no-cache", + "x-accel-buffering": "no", + "x-bili-preflight": "compressing", + }); + } else { + res.writeHead(200, { "content-type": "application/json", "x-bili-preflight": "compressing" }); + } + } catch { + return undefined; + } + log("info", `[${sid}] preflight still running after ${preflightHoldGraceMs()}ms grace — committed early ${prepared.stream ? "SSE" : "JSON"} headers + keep-alive to hold the client (#568)`); + try { + res.write(keepAlive); + } catch { /* client gone */ } + const iv = setInterval(() => { + try { + res.write(keepAlive); + } catch { + clearInterval(iv); + } + }, PREFLIGHT_KEEPALIVE_MS); + return () => clearInterval(iv); +} + async function preflightCompressIfNeeded( prepared: Prepared, runPrepare: () => Prepared, @@ -2868,25 +2948,36 @@ async function preflightCompressIfNeeded( if (!res.writableEnded) clientAbort.abort(); }); const started = Date.now(); - const result = await preflightCompress( - { - core, - session, - config, - prompts: prepared.prompts ?? defaultPrompts, - protocol: prepared.protocol, - url: upstreamUrl, - headers, - model, - proxyUrl, - signal: clientAbort.signal, - log, - imageFloor: imageTokens, - wireOverhead: overheadEstimate, - unknownBaseline, - }, - prepared.originalMessages, - ); + let stopHold: (() => void) | undefined; + const holdTimer = setTimeout(() => { + stopHold = beginPreflightHold(res, prepared, log); + }, preflightHoldGraceMs()); + holdTimer.unref(); + let result: PreflightResult; + try { + result = await preflightCompress( + { + core, + session, + config, + prompts: prepared.prompts ?? defaultPrompts, + protocol: prepared.protocol, + url: upstreamUrl, + headers, + model, + proxyUrl, + signal: clientAbort.signal, + log, + imageFloor: imageTokens, + wireOverhead: overheadEstimate, + unknownBaseline, + }, + prepared.originalMessages, + ); + } finally { + clearTimeout(holdTimer); + stopHold?.(); + } // #330: decide forward/fail on the payload actually forwarded, not // result.payloadEstimate — the preflight's relaxed-zone processTurn trims // that estimate more than the normal-config prepare does, which can turn a @@ -2939,7 +3030,7 @@ async function forward( // forged success response — serve it without contacting upstream. if (prepared?.codexForge) { log("info", `[${prepared.session.id}] codex compact served locally (${prepared.codexForge.kind}); upstream not contacted`); - res.writeHead(200, { "content-type": prepared.codexForge.contentType }); + if (!res.headersSent) res.writeHead(200, { "content-type": prepared.codexForge.contentType }); res.end(prepared.codexForge.body); return; } @@ -3302,6 +3393,18 @@ async function forward( if (bodyText.length > 600) snippet += " …"; if (!snippet) snippet = "(no body)"; loggerLog("warn", `[${errSid}] ← upstream ${upstream.status}${reqIdText}: ${snippet}`); + if (res.headersSent) { + // #568: the preflight hold already committed 200 early — the status can no + // longer change, so deliver the upstream failure in-band (protocol error + // event for streams; verbatim error body under 200 otherwise). + if (prepared?.stream) { + emitStreamError(res, prepared.protocol, `upstream HTTP ${upstream.status}: ${snippet}`, (m) => loggerLog("info", m)); + } else { + try { res.end(errBody ?? undefined); } catch { /* client gone */ } + } + clearUpstreamTimer(); + return; + } const errHeaders: Record = { ...respHeaders }; // Drop the upstream framing headers unconditionally: when errBody is // present a fixed-length write replaces them, and when errBody is @@ -3316,7 +3419,9 @@ async function forward( return; } // 2xx path: now safe to commit the status + headers, then stream the body. - res.writeHead(upstream.status, respHeaders); + // When the #568 hold already committed an early 200, the upstream's own + // headers (x-request-id etc.) are dropped — informational only. + if (!res.headersSent) res.writeHead(upstream.status, respHeaders); if (!upstream.body) { res.end(); clearUpstreamTimer(); diff --git a/src/stream-error.ts b/src/stream-error.ts index 9e76093e7f..41a9ec521a 100644 --- a/src/stream-error.ts +++ b/src/stream-error.ts @@ -70,3 +70,35 @@ export function emitStreamError(res: http.ServerResponse, protocol: Protocol, me } } } + +/** + * #568: deliver a preflight fail-fast error IN-BAND, after the proxy already + * committed `200` early to hold the client through a long compression (see + * beginPreflightHold in server.ts). The status line can no longer change, so + * the failure rides on a protocol-native error event: + * - openai: top-level `error` in a data frame, then `[DONE]`. + * - anthropic: `event: error` (the SDK's standard API-error channel). + * - responses: `event: error` (Responses SSE spec). + * Never throws. + */ +export function emitPreflightError(res: http.ServerResponse, protocol: Protocol, error: { message: string; retryable: boolean }, log?: (msg: string) => void): void { + const err = { type: "server_error", code: "preflight_compress_failed", message: error.message, retryable: error.retryable }; + log?.(`[acp-proxy: preflight failed after early response commit — delivering in-band: ${error.message}]`); + try { + if (protocol === "openai") { + safeWrite(res, `data: ${JSON.stringify({ error: err })}\n\ndata: [DONE]\n\n`); + } else if (protocol === "responses") { + safeWrite(res, `event: error\ndata: ${JSON.stringify({ type: "error", code: err.code, message: err.message })}\n\n`); + } else { + safeWrite(res, `event: error\ndata: ${JSON.stringify({ type: "error", error: { type: "server_error", code: err.code, message: err.message } })}\n\n`); + } + } catch { + /* best-effort */ + } finally { + try { + res.end(); + } catch { + /* already closed */ + } + } +} diff --git a/tests/preflight-hold.test.ts b/tests/preflight-hold.test.ts new file mode 100644 index 0000000000..6af76f0df9 --- /dev/null +++ b/tests/preflight-hold.test.ts @@ -0,0 +1,342 @@ +import assert from "node:assert/strict"; +import http from "node:http"; +import { once } from "node:events"; +import test from "node:test"; + +process.env.NODE_ENV = "test"; +// Fail fast on the very first 429 instead of the default 3 attempts with +// exponential backoff — the fail-fast tests below want the error immediately. +process.env.BILI_REPLAY_RETRY_MAX = "1"; +// Shrink the preflight hold grace so a 1.5s-slow summarization call reliably +// outlives it (default is 30s — too slow for a test). +process.env.BILI_PREFLIGHT_HOLD_MS = "300"; + +import { defaultConfig } from "acp-kernel"; +import { startServer, type ProxyOptions } from "../src/server.ts"; +import { SessionStore, _setStoreForTest } from "../src/persist.ts"; +import { _setForTest as setRegistryForTest } from "../src/registry.ts"; + +// #568: preflight compression previously sent ZERO bytes until it finished — +// clients with undici's default 300s headersTimeout aborted mid-compression, +// the proxy logged "summarization aborted: client disconnected" and never +// forwarded: a repeating 5-minute death loop. When preflight outlives the +// hold grace (BILI_PREFLIGHT_HOLD_MS, default 30s) the proxy must commit the +// response early (stream: 200 + SSE comment keep-alives; non-stream: 200 + +// whitespace) so the client's header timeout can never fire. Late failures +// then arrive in-band (protocol error event / JSON body) instead of as a +// status code that can no longer change. + +const SLOW_MS = 1500; + +const SUMMARY_TEXT = + "PREFLIGHT SUMMARY: the segment covered a multi-step debugging session. " + + "Key decisions: chose the preflight approach over lossy truncation because the payload must stay coherent. " + + "Files touched: src/a.ts:10, src/b.ts:20. Outcome: fixed and verified by tests."; + +function okSse(inputTokens: number): string { + return ( + `event: message_start\ndata: ${JSON.stringify({ type: "message_start", message: { id: "m1", role: "assistant", usage: { input_tokens: inputTokens } } })}\n\n` + + `event: content_block_start\ndata: ${JSON.stringify({ type: "content_block_start", index: 0, content_block: { type: "text", text: "" } })}\n\n` + + `event: content_block_delta\ndata: ${JSON.stringify({ type: "content_block_delta", index: 0, delta: { type: "text_delta", text: "ok" } })}\n\n` + + `event: content_block_stop\ndata: ${JSON.stringify({ type: "content_block_stop", index: 0 })}\n\n` + + `event: message_delta\ndata: ${JSON.stringify({ type: "message_delta", delta: { stop_reason: "end_turn", stop_sequence: null }, usage: { output_tokens: 3 } })}\n\n` + + `event: message_stop\ndata: ${JSON.stringify({ type: "message_stop" })}\n\n` + ); +} + +function bigConversation(): Array<{ role: string; content: string }> { + const msgs: Array<{ role: string; content: string }> = []; + for (let i = 0; i < 12; i++) { + const role = i % 2 === 0 ? "user" : "assistant"; + const filler = `MARKER_${i}_content_`.repeat(250); + msgs.push({ role, content: `Message ${i} of the long conversation. ${filler}` }); + } + return msgs; +} + +type Call = { raw: string }; + +type TimedResponse = { + status: number; + headers: http.IncomingHttpHeaders; + /** ms from request start to response headers */ + tHeaderMs: number; + /** ms from request start to response end */ + tEndMs: number; + body: string; +}; + +function timedPost(url: string, headers: Record, body: string): Promise { + return new Promise((resolve, reject) => { + const t0 = Date.now(); + const req = http.request(url, { method: "POST", headers }, (res) => { + const tHeaderMs = Date.now() - t0; + const chunks: Buffer[] = []; + res.on("data", (c: Buffer) => chunks.push(c)); + res.on("end", () => { + resolve({ + status: res.statusCode ?? 0, + headers: res.headers, + tHeaderMs, + tEndMs: Date.now() - t0, + body: Buffer.concat(chunks).toString("utf8"), + }); + }); + }); + req.on("error", reject); + req.end(body); + }); +} + +function startProxy(upstreamPort: number, models: Record, limit: number): Promise { + _setStoreForTest(new SessionStore({ enabled: false })); + setRegistryForTest({}); + return startServer({ + port: 0, + host: "127.0.0.1", + upstream: "http://127.0.0.1", + routes: { [`http://127.0.0.1:${upstreamPort}`]: { models } }, + modelContextLimit: limit, + kernelConfig: defaultConfig(limit), + compress: { injectTool: true, injectNudge: true }, + promptCache: { routing: "auto" }, + sessionHeader: "x-acp-session", + log: false, + debug: false, + passthrough: false, + autoUpdate: false, + mitm: { enabled: false, domains: [] }, + } as ProxyOptions); +} + +test("#568: stream + slow preflight → early 200 + SSE keep-alive, then the real stream", async () => { + const calls: Call[] = []; + const upstream = http.createServer((req, res) => { + const chunks: Buffer[] = []; + req.on("data", (c: Buffer) => chunks.push(c)); + req.on("end", () => { + const raw = Buffer.concat(chunks).toString("utf8"); + let parsed: { stream?: boolean } = {}; + try { parsed = JSON.parse(raw); } catch { /* keep {} */ } + calls.push({ raw }); + if (parsed.stream) { + res.writeHead(200, { "content-type": "text/event-stream" }); + res.end(okSse(1000)); + } else { + // Slow summarization call — outlives the 300ms hold grace. + setTimeout(() => { + res.writeHead(200, { "content-type": "application/json" }); + res.end(JSON.stringify({ + id: "msg_summary", + type: "message", + role: "assistant", + model: "claude-small", + content: [{ type: "text", text: SUMMARY_TEXT }], + stop_reason: "end_turn", + usage: { input_tokens: 500, output_tokens: 50 }, + })); + }, SLOW_MS); + } + }); + }); + upstream.listen(0, "127.0.0.1"); + await once(upstream, "listening"); + const upstreamPort = upstream.address().port; + + const proxy = await startProxy(upstreamPort, { "claude-small": { context: 10_000 } }, 400_000); + await once(proxy, "listening"); + const proxyPort = proxy.address().port; + + try { + // Fresh session, ~13k-token history vs 10k window → preflight fires + // and its summarization call takes 1.5s ≫ the 300ms grace. + const r = await timedPost( + `http://127.0.0.1:${proxyPort}/bili/http://127.0.0.1:${upstreamPort}/v1/messages`, + { "content-type": "application/json", "x-acp-session": "hold-stream-ok-sess" }, + JSON.stringify({ model: "claude-small", max_tokens: 1024, stream: true, messages: bigConversation() }), + ); + assert.equal(r.status, 200); + assert.equal(String(r.headers["content-type"]), "text/event-stream"); + assert.equal(r.headers["x-bili-preflight"], "compressing", "early commit is marked for observability"); + assert.ok(r.tHeaderMs < SLOW_MS, `headers arrived before summarization finished (${r.tHeaderMs}ms)`); + assert.ok(r.tEndMs >= SLOW_MS, `the response waited for the slow preflight (${r.tEndMs}ms) — the hold did real work`); + assert.ok(r.body.includes(": bili-preflight"), "an SSE comment keep-alive was sent while compressing"); + assert.ok(r.body.includes('"text_delta"'), "the real stream content follows the keep-alive"); + assert.ok(r.body.includes("message_stop"), "the stream completed"); + + const forwards = calls.filter((c) => c.raw.includes('"stream":true')); + assert.equal(forwards.length, 1, "exactly one forward upstream"); + const fwd = forwards[0]!.raw; + assert.ok(fwd.includes(SUMMARY_TEXT), "the rebuilt payload carries the preflight summary"); + assert.ok(!fwd.includes("MARKER_1_"), "compressed messages are out of the payload"); + assert.ok(fwd.includes("MARKER_11_"), "recent protected messages remain in the payload"); + } finally { + proxy.close(); + await once(proxy, "close"); + upstream.close(); + await once(upstream, "close"); + } +}); + +test("#568: stream + slow preflight 429 → early-committed 200 carries the error in-band, nothing forwarded", async () => { + const calls: Call[] = []; + const upstream = http.createServer((req, res) => { + const chunks: Buffer[] = []; + req.on("data", (c: Buffer) => chunks.push(c)); + req.on("end", () => { + const raw = Buffer.concat(chunks).toString("utf8"); + let parsed: { stream?: boolean } = {}; + try { parsed = JSON.parse(raw); } catch { /* keep {} */ } + calls.push({ raw }); + if (parsed.stream) { + res.writeHead(200, { "content-type": "text/event-stream" }); + res.end(okSse(1000)); + } else { + // Slow rate-limited summarization call (#292 scenario, slow edition). + setTimeout(() => { + res.writeHead(429, { "content-type": "application/json" }); + res.end(JSON.stringify({ error: { message: "rate limited", type: "rate_limit_error" } })); + }, SLOW_MS); + } + }); + }); + upstream.listen(0, "127.0.0.1"); + await once(upstream, "listening"); + const upstreamPort = upstream.address().port; + + const proxy = await startProxy(upstreamPort, { "claude-small": { context: 10_000 } }, 400_000); + await once(proxy, "listening"); + const proxyPort = proxy.address().port; + + try { + const r = await timedPost( + `http://127.0.0.1:${proxyPort}/bili/http://127.0.0.1:${upstreamPort}/v1/messages`, + { "content-type": "application/json", "x-acp-session": "hold-stream-429-sess" }, + JSON.stringify({ model: "claude-small", max_tokens: 1024, stream: true, messages: bigConversation() }), + ); + // The hold already committed 200 before the 429 landed — the client + // gets the structured error IN-BAND instead of a hang/abort. + assert.equal(r.status, 200, "status was committed early and cannot become 503 anymore"); + assert.equal(r.headers["x-bili-preflight"], "compressing"); + assert.ok(r.body.includes("event: error"), "SSE error event delivered in-band"); + assert.ok(r.body.includes("preflight_compress_failed"), "structured error code preserved in-band"); + assert.ok(r.body.includes("rate limited") || r.body.includes("429"), "the cause is named"); + assert.equal(calls.filter((c) => c.raw.includes('"stream":true')).length, 0, "the over-window payload was NOT forwarded"); + assert.ok(calls.filter((c) => !c.raw.includes('"stream":true')).length >= 1, "the (slow, 429'd) summarization call happened"); + } finally { + proxy.close(); + await once(proxy, "close"); + upstream.close(); + await once(upstream, "close"); + } +}); + +test("#568: Responses protocol + slow preflight 429 → early-committed 200, in-band error event", async () => { + const calls: Call[] = []; + const upstream = http.createServer((req, res) => { + const chunks: Buffer[] = []; + req.on("data", (c: Buffer) => chunks.push(c)); + req.on("end", () => { + const raw = Buffer.concat(chunks).toString("utf8"); + let parsed: { stream?: boolean } = {}; + try { parsed = JSON.parse(raw); } catch { /* keep {} */ } + calls.push({ raw }); + if (parsed.stream === false) { + setTimeout(() => { + res.writeHead(429, { "content-type": "application/json" }); + res.end(JSON.stringify({ error: { message: "rate limited", type: "rate_limit_error" } })); + }, SLOW_MS); + } else { + res.writeHead(200, { "content-type": "text/event-stream", "cache-control": "no-cache" }); + res.end(`event: response.completed\ndata: ${JSON.stringify({ response: { id: "resp_done", status: "completed", output: [], usage: { input_tokens: 1000, output_tokens: 5, total_tokens: 1005 } } })}\n\n`); + } + }); + }); + upstream.listen(0, "127.0.0.1"); + await once(upstream, "listening"); + const upstreamPort = upstream.address().port; + + const proxy = await startProxy(upstreamPort, { "gpt-resp": { context: 10_000 } }, 10_000); + await once(proxy, "listening"); + const proxyPort = proxy.address().port; + + try { + const input = bigConversation().map((m) => ({ type: "message", role: m.role, content: m.content })); + const r = await timedPost( + `http://127.0.0.1:${proxyPort}/bili/http://127.0.0.1:${upstreamPort}/v1/responses`, + { "content-type": "application/json", "x-acp-session": "hold-resp-429-sess" }, + JSON.stringify({ model: "gpt-resp", stream: true, session_id: "hold-resp-429-sess", instructions: "You are the test coding agent.", input }), + ); + assert.equal(r.status, 200, "status was committed early and cannot become 503 anymore"); + assert.ok(r.body.includes("event: error"), "Responses-API error event delivered in-band"); + assert.ok(r.body.includes("preflight_compress_failed"), "structured error code preserved in-band"); + assert.equal(calls.filter((c) => !c.raw.includes('"stream":false')).length, 0, "nothing was forwarded"); + } finally { + proxy.close(); + await once(proxy, "close"); + upstream.close(); + await once(upstream, "close"); + } +}); + +test("#568: non-stream + slow preflight → early 200 + whitespace keep-alive, then valid JSON", async () => { + const calls: Call[] = []; + const upstream = http.createServer((req, res) => { + const chunks: Buffer[] = []; + req.on("data", (c: Buffer) => chunks.push(c)); + req.on("end", () => { + const raw = Buffer.concat(chunks).toString("utf8"); + let parsed: { stream?: boolean; max_tokens?: number } = {}; + try { parsed = JSON.parse(raw); } catch { /* keep {} */ } + calls.push({ raw }); + // Summarization calls are sized to MAX_SUMMARY_OUTPUT_TOKENS (8192); + // the main request keeps its own max_tokens (1024). + if (parsed.max_tokens === 8192) { + setTimeout(() => { + res.writeHead(200, { "content-type": "application/json" }); + res.end(JSON.stringify({ choices: [{ message: { role: "assistant", content: SUMMARY_TEXT } }] })); + }, SLOW_MS); + } else { + res.writeHead(200, { "content-type": "application/json" }); + res.end(JSON.stringify({ + id: "chatcmpl-hold", + object: "chat.completion", + choices: [{ index: 0, message: { role: "assistant", content: "ok" }, finish_reason: "stop" }], + usage: { prompt_tokens: 1000, completion_tokens: 3, total_tokens: 1003 }, + })); + } + }); + }); + upstream.listen(0, "127.0.0.1"); + await once(upstream, "listening"); + const upstreamPort = upstream.address().port; + + const proxy = await startProxy(upstreamPort, { "gpt-small": { context: 10_000 } }, 400_000); + await once(proxy, "listening"); + const proxyPort = proxy.address().port; + + try { + const r = await timedPost( + `http://127.0.0.1:${proxyPort}/bili/http://127.0.0.1:${upstreamPort}/v1/chat/completions`, + { "content-type": "application/json", "x-acp-session": "hold-json-ok-sess" }, + JSON.stringify({ model: "gpt-small", max_tokens: 1024, messages: bigConversation() }), + ); + assert.equal(r.status, 200); + assert.equal(String(r.headers["content-type"]), "application/json"); + assert.equal(r.headers["x-bili-preflight"], "compressing"); + assert.ok(r.tHeaderMs < SLOW_MS, `headers arrived before summarization finished (${r.tHeaderMs}ms)`); + assert.ok(r.tEndMs >= SLOW_MS, `the response waited for the slow preflight (${r.tEndMs}ms)`); + assert.ok(r.body.startsWith(" "), "whitespace keep-alive precedes the JSON body"); + const json = JSON.parse(r.body) as { choices?: Array<{ message?: { content?: string } }> }; + assert.equal(json.choices?.[0]?.message?.content, "ok", "the JSON body stays parseable despite the padding byte"); + + const forwards = calls.filter((c) => !c.raw.includes('"max_tokens":8192')); + assert.equal(forwards.length, 1, "exactly one forward upstream"); + assert.ok(forwards[0]!.raw.includes(SUMMARY_TEXT), "the rebuilt payload carries the preflight summary"); + } finally { + proxy.close(); + await once(proxy, "close"); + upstream.close(); + await once(upstream, "close"); + } +});