Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 37 additions & 0 deletions src/server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1332,6 +1332,33 @@ function injectResponsesTool(tools: unknown[] | undefined, toolsToAdd: readonly
return [...tools, ...additions];
}

// #11: a failed request (relay 5xx, socket reset, timeout) never reports
// usage, so lastInputTokens stays frozen and each retry re-sends the same
// oversized payload — the relay fails it again (deadlock). Arm the emergency
// shrink with a local estimate of the body just sent. Deliberate exception
// to "tokenCount must be real usage" above: it only RAISES the value (a
// lower bound can only compress earlier, never later), the kernel no-ops
// below truncate.threshold, and the next successful usage report overwrites
// it. markDirty is needed because error paths return before forward()'s
// trailing save.
function armFailureShrink(
prepared: Prepared | null,
body: Buffer | string | undefined,
log: (level: string, msg: string) => void,
reason: string,
): void {
const s = prepared?.session;
if (!s || body === undefined) return;
try {
const est = estimateTokensFast(typeof body === "string" ? body : body.toString("utf8"));
if (est > s.stats.lastInputTokens) {
s.stats.lastInputTokens = est;
markDirty(s);
log("warn", `[${s.id}] ${reason} with no usage report — armed emergency shrink with local estimate ${est} tokens`);
}
} catch { /* non-text body — nothing to estimate */ }
}

async function forward(
req: http.IncomingMessage,
res: http.ServerResponse,
Expand Down Expand Up @@ -1487,6 +1514,9 @@ async function forward(
recordUpstreamConnection(upstreamUrl, proxyUrl);
} catch (error) {
recordUpstreamConnection(upstreamUrl, proxyUrl, error);
// #11: a network-level failure also never reports usage — arm the
// emergency shrink the same way as the 5xx branch below.
if (req.method !== "GET" && req.method !== "HEAD") armFailureShrink(prepared, body, log, "network failure");
throw new Error(`upstream request failed: ${formatUpstreamError(error, upstreamUrl, proxyUrl)}`, { cause: error });
}
const { response: upstream, clearTimer: clearUpstreamTimer } = upstreamResult;
Expand Down Expand Up @@ -1583,6 +1613,13 @@ async function forward(
// markDirty, so schedule the save HERE or the self-heal is
// lost on restart and the next overflow must be re-learned.
markDirty(s);
} else if (upstream.status >= 500) {
// #11: relay/gateway 5xx — no usage report will arrive, so
// arm the emergency shrink with a local estimate of what we
// just sent (see armFailureShrink for the deadlock this
// breaks). Generic relay errors (new_api_error etc.) are not
// overflow signatures, so this is the only self-heal path.
armFailureShrink(prepared, prepared.body, log, `upstream ${upstream.status}`);
}
}
// #174: always log a non-2xx upstream response (status + request-id +
Expand Down
197 changes: 197 additions & 0 deletions tests/e2e-relay-5xx-selfheal.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,197 @@
import assert from "node:assert/strict";
import http from "node:http";
import { once } from "node:events";
import test from "node:test";

process.env.NODE_ENV = "test";

import { defaultConfig } from "acp-kernel";
import { startServer, type ProxyOptions } from "../src/server.ts";
import { SessionStore, _setStoreForTest } from "../src/persist.ts";
import { _setForTest as setRegistryForTest } from "../src/registry.ts";
import { listSessions } from "../src/session.ts";

// #11: a relay that 500s an oversized request also never reports usage, so
// lastInputTokens stays frozen and every retry re-sends the same payload —
// a deadlock. On a 5xx the proxy must arm the emergency shrink with a local
// estimate of the body it just sent, so the NEXT turn's processTurn fires
// the kernel's emergency nudge + tool-result truncate and the payload
// shrinks. The recovery turn's real usage report must then overwrite the
// armed value.

const RELAY_500_BODY = JSON.stringify({
error: { type: "new_api_error", message: "upstream error: do request failed (request id: abc)" },
});

function okSse(): string {
return (
`event: message_start\ndata: ${JSON.stringify({ type: "message_start", message: { id: "m1", role: "assistant", usage: { input_tokens: 5000 } } })}\n\n` +
`event: content_block_start\ndata: ${JSON.stringify({ type: "content_block_start", index: 0, content_block: { type: "text", text: "" } })}\n\n` +
`event: content_block_delta\ndata: ${JSON.stringify({ type: "content_block_delta", index: 0, delta: { type: "text_delta", text: "ok" } })}\n\n` +
`event: content_block_stop\ndata: ${JSON.stringify({ type: "content_block_stop", index: 0 })}\n\n` +
`event: message_delta\ndata: ${JSON.stringify({ type: "message_delta", delta: { stop_reason: "end_turn", stop_sequence: null }, usage: { output_tokens: 3 } })}\n\n` +
`event: message_stop\ndata: ${JSON.stringify({ type: "message_stop" })}\n\n`
);
}

function bigSession(): unknown[] {
// Roughly the #11 shape: a long history whose bulk is chunky tool
// results (what emergency-truncate targets), far past the 16k window
// configured below. Each tool_result is 12k chars (~3k tokens) — above
// the 1000-token floor and the 2k+2k keep-prefix/suffix, so each is a
// truncation candidate.
const msgs: unknown[] = [];
for (let i = 0; i < 40; i++) {
msgs.push({ role: "user", content: `turn ${i} ` + "x".repeat(500) });
msgs.push({ role: "assistant", content: [{ type: "tool_use", id: `t${i}`, name: "run", input: {} }] });
msgs.push({ role: "user", content: [{ type: "tool_result", tool_use_id: `t${i}`, content: "z".repeat(12_000) }] });
msgs.push({ role: "assistant", content: [{ type: "text", text: `reply ${i}` }] });
}
return msgs;
}

test("e2e: relay 5xx on oversized request → arm emergency shrink → next turn truncates", async () => {
let call = 0;
let sawShrunkPayload = false;
let firstPayloadTokens = 0;
let secondPayloadTokens = 0;
const upstream = http.createServer((req, res) => {
const chunks: Buffer[] = [];
req.on("data", (c: Buffer) => chunks.push(c));
req.on("end", () => {
const bodyText = Buffer.concat(chunks).toString("utf8");
if (call === 0) {
firstPayloadTokens = Math.ceil(bodyText.length / 4);
// The relay fails the oversized request with its own error.
res.writeHead(500, { "content-type": "application/json" });
res.end(RELAY_500_BODY);
} else {
secondPayloadTokens = Math.ceil(bodyText.length / 4);
sawShrunkPayload = bodyText.includes("[truncated for context space]");
res.writeHead(200, { "content-type": "text/event-stream" });
res.end(okSse());
}
call += 1;
});
});
upstream.listen(0, "127.0.0.1");
await once(upstream, "listening");
const upstreamPort = upstream.address().port;

const store = new SessionStore({ enabled: false });
_setStoreForTest(store);
setRegistryForTest({});
const proxy = await startServer({
port: 0,
host: "127.0.0.1",
upstream: "http://127.0.0.1",
routes: { [`http://127.0.0.1:${upstreamPort}`]: { models: { "claude-relay": { context: 16_000 } } } },
modelContextLimit: 16_000,
kernelConfig: defaultConfig(16_000),
compress: { injectTool: true, injectNudge: true },
promptCache: { routing: "auto" },
sessionHeader: "x-acp-session",
log: false,
debug: false,
passthrough: false,
autoUpdate: false,
mitm: { enabled: false, domains: [] },
} as ProxyOptions);
await once(proxy, "listening");
const proxyPort = proxy.address().port;

try {
const url = `http://127.0.0.1:${proxyPort}/bili/http://127.0.0.1:${upstreamPort}/v1/messages`;
const headers = { "content-type": "application/json", "x-acp-session": "relay500-sess" };

// --- Request 1: oversized payload → relay 500, passed through verbatim.
const body1 = JSON.stringify({ model: "claude-relay", max_tokens: 1024, stream: true, messages: bigSession() });
const r1 = await fetch(url, { method: "POST", headers, body: body1 });
assert.equal(r1.status, 500);
const r1text = await r1.text();
assert.ok(r1text.includes("new_api_error"), "relay error body must pass through");

// The 5xx armed the emergency shrink with a local estimate of the
// body actually sent (>= a rough char/8 lower bound of the payload).
const s = listSessions()[0];
assert.ok(s, "session exists");
assert.ok(
s.stats.lastInputTokens >= Math.ceil(body1.length / 8),
`armed with local estimate (lastInputTokens=${s.stats.lastInputTokens}, body ~${Math.ceil(body1.length / 4)} tokens)`,
);

// --- Request 2: same history resent by the client. The armed value
// drives emergency nudge + tool-result truncate server-side, so the
// forwarded payload shrinks and the (now healthy) relay accepts it.
const r2 = await fetch(url, { method: "POST", headers, body: body1 });
assert.equal(r2.status, 200, "second request recovers after emergency shrink");
await r2.text();

assert.ok(sawShrunkPayload, "recovery payload contains the truncation marker");
assert.ok(secondPayloadTokens < firstPayloadTokens, `payload shrank (${firstPayloadTokens} → ${secondPayloadTokens} tokens)`);

// The real usage report from the successful turn overwrote the armed value.
const s2 = listSessions()[0];
assert.equal(s2?.stats.lastInputTokens, 5000);
} finally {
proxy.close();
await once(proxy, "close");
upstream.close();
await once(upstream, "close");
}
});

test("e2e: 4xx (auth) must NOT arm the emergency shrink", async () => {
const upstream = http.createServer((req, res) => {
req.resume();
req.on("end", () => {
res.writeHead(401, { "content-type": "application/json" });
res.end(JSON.stringify({ error: { type: "authentication_error", message: "invalid api key" } }));
});
});
upstream.listen(0, "127.0.0.1");
await once(upstream, "listening");
const upstreamPort = upstream.address().port;

const store = new SessionStore({ enabled: false });
_setStoreForTest(store);
setRegistryForTest({});
const proxy = await startServer({
port: 0,
host: "127.0.0.1",
upstream: "http://127.0.0.1",
routes: { [`http://127.0.0.1:${upstreamPort}`]: { models: { "claude-relay": { context: 16_000 } } } },
modelContextLimit: 16_000,
kernelConfig: defaultConfig(16_000),
compress: { injectTool: true, injectNudge: true },
promptCache: { routing: "auto" },
sessionHeader: "x-acp-session",
log: false,
debug: false,
passthrough: false,
autoUpdate: false,
mitm: { enabled: false, domains: [] },
} as ProxyOptions);
await once(proxy, "listening");
const proxyPort = proxy.address().port;

try {
const url = `http://127.0.0.1:${proxyPort}/bili/http://127.0.0.1:${upstreamPort}/v1/messages`;
const body = JSON.stringify({ model: "claude-relay", max_tokens: 1024, stream: true, messages: [{ role: "user", content: "x".repeat(200_000) }] });
const r1 = await fetch(url, {
method: "POST",
headers: { "content-type": "application/json", "x-acp-session": "auth4xx-sess" },
body,
});
assert.equal(r1.status, 401);
await r1.text();
const s = listSessions()[0];
assert.ok(s, "session exists");
assert.equal(s.stats.lastInputTokens, 0, "4xx must not arm the emergency shrink");
} finally {
proxy.close();
await once(proxy, "close");
upstream.close();
await once(upstream, "close");
}
});
Loading