From 06666f0a5dd6ad7335e8a1e2a752eacd28ad6a73 Mon Sep 17 00:00:00 2001 From: ranxianglei Date: Tue, 25 Aug 2026 01:24:07 +0800 Subject: [PATCH] fix: arm emergency shrink on upstream failure to break relay-5xx deadlock (#11) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A failed request (relay 5xx, network error) never reports usage, so lastInputTokens stays frozen and every retry re-sends the same oversized payload — the relay fails it again. On 5xx and network-level failures, arm the emergency shrink with a local estimate of the body just sent: the next turn's processTurn fires the kernel's emergency nudge and tool-result truncate server-side, shrinking the payload without model cooperation. 4xx responses are excluded (auth/quota errors must not distort the usage signal). --- src/server.ts | 37 +++++ tests/e2e-relay-5xx-selfheal.test.ts | 197 +++++++++++++++++++++++++++ 2 files changed, 234 insertions(+) create mode 100644 tests/e2e-relay-5xx-selfheal.test.ts diff --git a/src/server.ts b/src/server.ts index 5a8d8af7cf..3f580d5e50 100644 --- a/src/server.ts +++ b/src/server.ts @@ -1332,6 +1332,33 @@ function injectResponsesTool(tools: unknown[] | undefined, toolsToAdd: readonly return [...tools, ...additions]; } +// #11: a failed request (relay 5xx, socket reset, timeout) never reports +// usage, so lastInputTokens stays frozen and each retry re-sends the same +// oversized payload — the relay fails it again (deadlock). Arm the emergency +// shrink with a local estimate of the body just sent. Deliberate exception +// to "tokenCount must be real usage" above: it only RAISES the value (a +// lower bound can only compress earlier, never later), the kernel no-ops +// below truncate.threshold, and the next successful usage report overwrites +// it. markDirty is needed because error paths return before forward()'s +// trailing save. +function armFailureShrink( + prepared: Prepared | null, + body: Buffer | string | undefined, + log: (level: string, msg: string) => void, + reason: string, +): void { + const s = prepared?.session; + if (!s || body === undefined) return; + try { + const est = estimateTokensFast(typeof body === "string" ? body : body.toString("utf8")); + if (est > s.stats.lastInputTokens) { + s.stats.lastInputTokens = est; + markDirty(s); + log("warn", `[${s.id}] ${reason} with no usage report — armed emergency shrink with local estimate ${est} tokens`); + } + } catch { /* non-text body — nothing to estimate */ } +} + async function forward( req: http.IncomingMessage, res: http.ServerResponse, @@ -1487,6 +1514,9 @@ async function forward( recordUpstreamConnection(upstreamUrl, proxyUrl); } catch (error) { recordUpstreamConnection(upstreamUrl, proxyUrl, error); + // #11: a network-level failure also never reports usage — arm the + // emergency shrink the same way as the 5xx branch below. + if (req.method !== "GET" && req.method !== "HEAD") armFailureShrink(prepared, body, log, "network failure"); throw new Error(`upstream request failed: ${formatUpstreamError(error, upstreamUrl, proxyUrl)}`, { cause: error }); } const { response: upstream, clearTimer: clearUpstreamTimer } = upstreamResult; @@ -1583,6 +1613,13 @@ async function forward( // markDirty, so schedule the save HERE or the self-heal is // lost on restart and the next overflow must be re-learned. markDirty(s); + } else if (upstream.status >= 500) { + // #11: relay/gateway 5xx — no usage report will arrive, so + // arm the emergency shrink with a local estimate of what we + // just sent (see armFailureShrink for the deadlock this + // breaks). Generic relay errors (new_api_error etc.) are not + // overflow signatures, so this is the only self-heal path. + armFailureShrink(prepared, prepared.body, log, `upstream ${upstream.status}`); } } // #174: always log a non-2xx upstream response (status + request-id + diff --git a/tests/e2e-relay-5xx-selfheal.test.ts b/tests/e2e-relay-5xx-selfheal.test.ts new file mode 100644 index 0000000000..8353a76f84 --- /dev/null +++ b/tests/e2e-relay-5xx-selfheal.test.ts @@ -0,0 +1,197 @@ +import assert from "node:assert/strict"; +import http from "node:http"; +import { once } from "node:events"; +import test from "node:test"; + +process.env.NODE_ENV = "test"; + +import { defaultConfig } from "acp-kernel"; +import { startServer, type ProxyOptions } from "../src/server.ts"; +import { SessionStore, _setStoreForTest } from "../src/persist.ts"; +import { _setForTest as setRegistryForTest } from "../src/registry.ts"; +import { listSessions } from "../src/session.ts"; + +// #11: a relay that 500s an oversized request also never reports usage, so +// lastInputTokens stays frozen and every retry re-sends the same payload — +// a deadlock. On a 5xx the proxy must arm the emergency shrink with a local +// estimate of the body it just sent, so the NEXT turn's processTurn fires +// the kernel's emergency nudge + tool-result truncate and the payload +// shrinks. The recovery turn's real usage report must then overwrite the +// armed value. + +const RELAY_500_BODY = JSON.stringify({ + error: { type: "new_api_error", message: "upstream error: do request failed (request id: abc)" }, +}); + +function okSse(): string { + return ( + `event: message_start\ndata: ${JSON.stringify({ type: "message_start", message: { id: "m1", role: "assistant", usage: { input_tokens: 5000 } } })}\n\n` + + `event: content_block_start\ndata: ${JSON.stringify({ type: "content_block_start", index: 0, content_block: { type: "text", text: "" } })}\n\n` + + `event: content_block_delta\ndata: ${JSON.stringify({ type: "content_block_delta", index: 0, delta: { type: "text_delta", text: "ok" } })}\n\n` + + `event: content_block_stop\ndata: ${JSON.stringify({ type: "content_block_stop", index: 0 })}\n\n` + + `event: message_delta\ndata: ${JSON.stringify({ type: "message_delta", delta: { stop_reason: "end_turn", stop_sequence: null }, usage: { output_tokens: 3 } })}\n\n` + + `event: message_stop\ndata: ${JSON.stringify({ type: "message_stop" })}\n\n` + ); +} + +function bigSession(): unknown[] { + // Roughly the #11 shape: a long history whose bulk is chunky tool + // results (what emergency-truncate targets), far past the 16k window + // configured below. Each tool_result is 12k chars (~3k tokens) — above + // the 1000-token floor and the 2k+2k keep-prefix/suffix, so each is a + // truncation candidate. + const msgs: unknown[] = []; + for (let i = 0; i < 40; i++) { + msgs.push({ role: "user", content: `turn ${i} ` + "x".repeat(500) }); + msgs.push({ role: "assistant", content: [{ type: "tool_use", id: `t${i}`, name: "run", input: {} }] }); + msgs.push({ role: "user", content: [{ type: "tool_result", tool_use_id: `t${i}`, content: "z".repeat(12_000) }] }); + msgs.push({ role: "assistant", content: [{ type: "text", text: `reply ${i}` }] }); + } + return msgs; +} + +test("e2e: relay 5xx on oversized request → arm emergency shrink → next turn truncates", async () => { + let call = 0; + let sawShrunkPayload = false; + let firstPayloadTokens = 0; + let secondPayloadTokens = 0; + const upstream = http.createServer((req, res) => { + const chunks: Buffer[] = []; + req.on("data", (c: Buffer) => chunks.push(c)); + req.on("end", () => { + const bodyText = Buffer.concat(chunks).toString("utf8"); + if (call === 0) { + firstPayloadTokens = Math.ceil(bodyText.length / 4); + // The relay fails the oversized request with its own error. + res.writeHead(500, { "content-type": "application/json" }); + res.end(RELAY_500_BODY); + } else { + secondPayloadTokens = Math.ceil(bodyText.length / 4); + sawShrunkPayload = bodyText.includes("[truncated for context space]"); + res.writeHead(200, { "content-type": "text/event-stream" }); + res.end(okSse()); + } + call += 1; + }); + }); + upstream.listen(0, "127.0.0.1"); + await once(upstream, "listening"); + const upstreamPort = upstream.address().port; + + const store = new SessionStore({ enabled: false }); + _setStoreForTest(store); + setRegistryForTest({}); + const proxy = await startServer({ + port: 0, + host: "127.0.0.1", + upstream: "http://127.0.0.1", + routes: { [`http://127.0.0.1:${upstreamPort}`]: { models: { "claude-relay": { context: 16_000 } } } }, + modelContextLimit: 16_000, + kernelConfig: defaultConfig(16_000), + compress: { injectTool: true, injectNudge: true }, + promptCache: { routing: "auto" }, + sessionHeader: "x-acp-session", + log: false, + debug: false, + passthrough: false, + autoUpdate: false, + mitm: { enabled: false, domains: [] }, + } as ProxyOptions); + await once(proxy, "listening"); + const proxyPort = proxy.address().port; + + try { + const url = `http://127.0.0.1:${proxyPort}/bili/http://127.0.0.1:${upstreamPort}/v1/messages`; + const headers = { "content-type": "application/json", "x-acp-session": "relay500-sess" }; + + // --- Request 1: oversized payload → relay 500, passed through verbatim. + const body1 = JSON.stringify({ model: "claude-relay", max_tokens: 1024, stream: true, messages: bigSession() }); + const r1 = await fetch(url, { method: "POST", headers, body: body1 }); + assert.equal(r1.status, 500); + const r1text = await r1.text(); + assert.ok(r1text.includes("new_api_error"), "relay error body must pass through"); + + // The 5xx armed the emergency shrink with a local estimate of the + // body actually sent (>= a rough char/8 lower bound of the payload). + const s = listSessions()[0]; + assert.ok(s, "session exists"); + assert.ok( + s.stats.lastInputTokens >= Math.ceil(body1.length / 8), + `armed with local estimate (lastInputTokens=${s.stats.lastInputTokens}, body ~${Math.ceil(body1.length / 4)} tokens)`, + ); + + // --- Request 2: same history resent by the client. The armed value + // drives emergency nudge + tool-result truncate server-side, so the + // forwarded payload shrinks and the (now healthy) relay accepts it. + const r2 = await fetch(url, { method: "POST", headers, body: body1 }); + assert.equal(r2.status, 200, "second request recovers after emergency shrink"); + await r2.text(); + + assert.ok(sawShrunkPayload, "recovery payload contains the truncation marker"); + assert.ok(secondPayloadTokens < firstPayloadTokens, `payload shrank (${firstPayloadTokens} → ${secondPayloadTokens} tokens)`); + + // The real usage report from the successful turn overwrote the armed value. + const s2 = listSessions()[0]; + assert.equal(s2?.stats.lastInputTokens, 5000); + } finally { + proxy.close(); + await once(proxy, "close"); + upstream.close(); + await once(upstream, "close"); + } +}); + +test("e2e: 4xx (auth) must NOT arm the emergency shrink", async () => { + const upstream = http.createServer((req, res) => { + req.resume(); + req.on("end", () => { + res.writeHead(401, { "content-type": "application/json" }); + res.end(JSON.stringify({ error: { type: "authentication_error", message: "invalid api key" } })); + }); + }); + upstream.listen(0, "127.0.0.1"); + await once(upstream, "listening"); + const upstreamPort = upstream.address().port; + + const store = new SessionStore({ enabled: false }); + _setStoreForTest(store); + setRegistryForTest({}); + const proxy = await startServer({ + port: 0, + host: "127.0.0.1", + upstream: "http://127.0.0.1", + routes: { [`http://127.0.0.1:${upstreamPort}`]: { models: { "claude-relay": { context: 16_000 } } } }, + modelContextLimit: 16_000, + kernelConfig: defaultConfig(16_000), + compress: { injectTool: true, injectNudge: true }, + promptCache: { routing: "auto" }, + sessionHeader: "x-acp-session", + log: false, + debug: false, + passthrough: false, + autoUpdate: false, + mitm: { enabled: false, domains: [] }, + } as ProxyOptions); + await once(proxy, "listening"); + const proxyPort = proxy.address().port; + + try { + const url = `http://127.0.0.1:${proxyPort}/bili/http://127.0.0.1:${upstreamPort}/v1/messages`; + const body = JSON.stringify({ model: "claude-relay", max_tokens: 1024, stream: true, messages: [{ role: "user", content: "x".repeat(200_000) }] }); + const r1 = await fetch(url, { + method: "POST", + headers: { "content-type": "application/json", "x-acp-session": "auth4xx-sess" }, + body, + }); + assert.equal(r1.status, 401); + await r1.text(); + const s = listSessions()[0]; + assert.ok(s, "session exists"); + assert.equal(s.stats.lastInputTokens, 0, "4xx must not arm the emergency shrink"); + } finally { + proxy.close(); + await once(proxy, "close"); + upstream.close(); + await once(upstream, "close"); + } +});