From 2b911f950ab0fbc7cb458b6e198328a6ba5df5ca Mon Sep 17 00:00:00 2001 From: ci-check Date: Tue, 3 Nov 2026 16:17:12 +0800 Subject: [PATCH 1/2] fix: attribute client-disconnect aborts to the client, not the upstream (#5598) When the downstream client cancels mid-forward (user hits stop, SDK timeout), the relay catch classified the resulting AbortError without the external-abort context, so describeUpstreamFailure billed it to the upstream's idle budget: kind=upstream-timeout / 'check upstream health' on the last-connection status surface, the egress line and the 502 diag body. The retry ladder and alert/circuit gates already classified client-abort correctly (classifyUpstreamFailure with externalAborted); only these three display surfaces lagged. Plumb clientAbort.signal.aborted through recordUpstreamConnection/describeUpstreamFailure/formatUpstreamError at the relay call sites; behavior of retry, alerts and circuit is unchanged. --- src/server/relay.ts | 9 +- src/upstream-proxy.ts | 15 +- ...ue5598-client-abort-classification.test.ts | 150 ++++++++++++++++++ 3 files changed, 165 insertions(+), 9 deletions(-) create mode 100644 tests/issue5598-client-abort-classification.test.ts diff --git a/src/server/relay.ts b/src/server/relay.ts index e534f29b2b..54752c02e2 100644 --- a/src/server/relay.ts +++ b/src/server/relay.ts @@ -879,9 +879,12 @@ async function forwardCaptured( } catch (error) { const oversize = extractWsOversizeError(error); if (oversize === undefined) { - recordUpstreamConnection(upstreamUrl, proxyUrl, error, skippedProxy); + // #5598: plumb the client-disconnect state into the classification — + // a user hitting stop must not surface as "check upstream health" + // on the status surface or in the egress line. + recordUpstreamConnection(upstreamUrl, proxyUrl, error, skippedProxy, clientAbort.signal.aborted); recordUpstreamAlert(upstreamUrl, error, proxyUrl !== undefined, skippedProxy); - const failure = describeUpstreamFailure(error, upstreamUrl, proxyUrl, skippedProxy); + const failure = describeUpstreamFailure(error, upstreamUrl, proxyUrl, skippedProxy, clientAbort.signal.aborted); recordUpstreamCircuitFailure(upstreamUrl, error, proxyUrl !== undefined, failure.message); // #604: a network-level failure (socket reset, timeout abort) also never // reports usage — arm the emergency shrink like the 5xx branch below. @@ -945,7 +948,7 @@ async function forwardCaptured( log("warn", `[${prepared.session.id}] [ws-byte-shed] shrunken resend still rejected (HTTP ${retried.response.status}); passing the retry response through`); } } catch (retryErr) { - log("warn", `[${prepared?.session.id ?? "unknown"}] [ws-byte-shed] shrunken resend failed: ${formatUpstreamError(retryErr, upstreamUrl, proxyUrl, skippedProxy)}`); + log("warn", `[${prepared?.session.id ?? "unknown"}] [ws-byte-shed] shrunken resend failed: ${formatUpstreamError(retryErr, upstreamUrl, proxyUrl, skippedProxy, clientAbort.signal.aborted)}`); throw new Error(wsOversizeDiagnosis(sentBytes, stats, budget), { cause: retryErr }); } } diff --git a/src/upstream-proxy.ts b/src/upstream-proxy.ts index e4031dae8a..33a347d4f1 100644 --- a/src/upstream-proxy.ts +++ b/src/upstream-proxy.ts @@ -562,9 +562,12 @@ interface UpstreamFailureParts { detail: UpstreamFailureDetail; } -export function describeUpstreamFailure(error: unknown, url: string, proxyUrl?: string, skipped?: SkippedProxyInfo): UpstreamFailureParts { +export function describeUpstreamFailure(error: unknown, url: string, proxyUrl?: string, skipped?: SkippedProxyInfo, externalAborted?: boolean): UpstreamFailureParts { const chain = errorChain(error); - const kind = classifyUpstreamFailure(error, { viaProxy: proxyUrl !== undefined }); + // #5598: the caller's external (client-disconnect) signal state decides the + // attribution — without it an AbortError is billed to the upstream's idle + // budget ("check upstream health") when the user simply hit stop. + const kind = classifyUpstreamFailure(error, { viaProxy: proxyUrl !== undefined, externalAborted }); const fields = ["code", "errno", "syscall", "address", "port"]; const parts: string[] = [`kind=${kind}`]; // Error text from undici/OS layers embeds the endpoint identity @@ -605,16 +608,16 @@ export function describeUpstreamFailure(error: unknown, url: string, proxyUrl?: return { message: parts.join(" "), detail: { kind, ...(codeValue !== undefined ? { code: String(codeValue) } : {}), ...(targetHostPort !== "" ? { host: maskHostPortForDisplay(targetHostPort) } : {}), hint } }; } -export function formatUpstreamError(error: unknown, url: string, proxyUrl?: string, skipped?: SkippedProxyInfo): string { - return describeUpstreamFailure(error, url, proxyUrl, skipped).message; +export function formatUpstreamError(error: unknown, url: string, proxyUrl?: string, skipped?: SkippedProxyInfo, externalAborted?: boolean): string { + return describeUpstreamFailure(error, url, proxyUrl, skipped, externalAborted).message; } -export function recordUpstreamConnection(url: string, proxyUrl: string | undefined, error?: unknown, skipped?: SkippedProxyInfo): void { +export function recordUpstreamConnection(url: string, proxyUrl: string | undefined, error?: unknown, skipped?: SkippedProxyInfo, externalAborted?: boolean): void { lastConnection = { url, proxy: redactProxyUrl(proxyUrl), connected: error === undefined, - ...(error === undefined ? {} : { error: formatUpstreamError(error, url, proxyUrl, skipped) }), + ...(error === undefined ? {} : { error: formatUpstreamError(error, url, proxyUrl, skipped, externalAborted) }), checkedAt: new Date().toISOString(), }; } diff --git a/tests/issue5598-client-abort-classification.test.ts b/tests/issue5598-client-abort-classification.test.ts new file mode 100644 index 0000000000..cdc274cf9a --- /dev/null +++ b/tests/issue5598-client-abort-classification.test.ts @@ -0,0 +1,150 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import http from "node:http"; +import { once } from "node:events"; +import { defaultConfig } from "acp-kernel"; +import { startServer } from "../src/server.ts"; +import type { ProxyOptions } from "../src/config.ts"; +import { SessionStore, _setStoreForTest } from "../src/persist.ts"; +import { _setForTest as setRegistryForTest } from "../src/registry.ts"; +import { setLogCapture } from "../src/logger.ts"; +import { describeUpstreamFailure, formatUpstreamError, getUpstreamConnectionStatus, _resetUpstreamProxyForTest, recordUpstreamConnection } from "../src/upstream-proxy.ts"; + +/** #5598 field report: a client (user stop / SDK timeout) closes the connection + * before any upstream byte returns — nginx records 499, and bili's relay catch + * classified the resulting AbortError as upstream-timeout ("check upstream + * health") because it never passed the clientAbort signal state into the + * classification. Retry/alert/circuit were already correctly gated on + * externalAborted; the misattribution lived in the last-connection status + * surface, the egress line detail, and the 502 diagnostic body. */ + +function abortError(): Error { + return new DOMException("This operation was aborted", "AbortError"); +} + +function close(server: http.Server): Promise { + return new Promise((resolve, reject) => server.close((error) => (error ? reject(error) : resolve()))); +} + +interface Captured { + level: string; + msg: string; +} + +async function waitFor(predicate: () => boolean, what: string, timeoutMs = 8000, dump?: () => string): Promise { + const start = Date.now(); + while (!predicate()) { + if (Date.now() - start > timeoutMs) throw new Error(`timed out waiting for ${what}${dump ? `\ncaptured so far:\n${dump()}` : ""}`); + await new Promise((r) => setTimeout(r, 25)); + } +} + +test("describeUpstreamFailure: external-abort context attributes the AbortError to the client, not the upstream (#5598)", () => { + const withCtx = describeUpstreamFailure(abortError(), "http://upstream.example/v1/messages", undefined, undefined, true); + assert.equal(withCtx.detail.kind, "client-abort"); + assert.match(withCtx.message, /kind=client-abort/); + assert.doesNotMatch(withCtx.message, /check upstream health/); + assert.match(withCtx.message, /downstream client disconnected/); + + // Negative control: without the context (internal idle-watchdog abort) the + // legacy attribution must stay untouched. + const withoutCtx = describeUpstreamFailure(abortError(), "http://upstream.example/v1/messages"); + assert.equal(withoutCtx.detail.kind, "upstream-timeout"); + assert.match(withoutCtx.message, /check upstream health/); +}); + +test("formatUpstreamError + recordUpstreamConnection: status surface says kind=client-abort after a user cancel (#5598)", () => { + _resetUpstreamProxyForTest(); + try { + assert.match(formatUpstreamError(abortError(), "http://upstream.example", undefined, undefined, true), /kind=client-abort/); + recordUpstreamConnection("http://upstream.example", undefined, abortError(), undefined, true); + const status = getUpstreamConnectionStatus(); + assert.equal(status.connected, false); + assert.match(status.error ?? "", /kind=client-abort/); + assert.doesNotMatch(status.error ?? "", /check upstream health/); + + // Legacy shape (no context) unchanged. + recordUpstreamConnection("http://upstream.example", undefined, abortError()); + assert.match(getUpstreamConnectionStatus().error ?? "", /kind=upstream-timeout/); + } finally { + _resetUpstreamProxyForTest(); + } +}); + +test("client cancels mid-forward: logs and status attribute the failure to the client, no error-level upstream blame (#5598)", async () => { + let upstreamSawRequest = false; + const upstream = http.createServer((req, res) => { + upstreamSawRequest = true; + req.resume(); + // Never answer — hold the socket open until the client goes away. + void res; + }); + const captured: Captured[] = []; + setLogCapture((level, msg) => captured.push({ level, msg })); + _setStoreForTest(new SessionStore({ enabled: false })); + setRegistryForTest({}); + upstream.listen(0, "127.0.0.1"); + await once(upstream, "listening"); + const upstreamPort = (upstream.address() as { port: number }).port; + const opts: ProxyOptions = { + port: 0, + host: "127.0.0.1", + upstream: "http://127.0.0.1", + routes: { + [`http://127.0.0.1:${upstreamPort}`]: { models: { "gpt-test": { context: 400_000 } } }, + }, + modelContextLimit: 400_000, + kernelConfig: defaultConfig(400_000), + compress: { injectTool: false, injectNudge: false }, + promptCache: { routing: "auto" }, + sessionHeader: "x-acp-session", + // #5598: log must be ON — the request pipeline routes every line through + // logMsg(), which drops ALL levels (incl. warn) when opts.log is false, + // before the capture hook ever sees them. + log: true, + debug: false, + passthrough: false, + autoUpdate: false, + compat: { roles: {} }, + streamErrorShape: "protocol", + passthroughSource: null, + autoRestartOnUpdate: false, + updateTag: "latest", + advisoryCheck: false, + releaseNotesCheck: false, + mitm: { enabled: false, domains: [] }, + }; + const proxy = await startServer(opts); + await once(proxy, "listening"); + const proxyPort = (proxy.address() as { port: number }).port; + const controller = new AbortController(); + try { + const fetchPromise = fetch(`http://127.0.0.1:${proxyPort}/bili/http://127.0.0.1:${upstreamPort}/v1/chat/completions`, { + method: "POST", + headers: { "content-type": "application/json", "x-acp-session": "issue5598-1" }, + body: JSON.stringify({ model: "gpt-test", messages: [{ role: "user", content: "hi" }] }), + signal: controller.signal, + }).catch((e) => e); + // Wait until the proxy has actually started the upstream fetch so the + // abort lands INSIDE fetchWithTimeout, not during body reading. + await waitFor(() => upstreamSawRequest, "upstream request to arrive"); + await new Promise((r) => setTimeout(r, 150)); + controller.abort(); + const clientErr = await fetchPromise; + assert.ok(clientErr instanceof Error && /abort/i.test(String(clientErr)), `expected the client fetch to reject with its own abort, got: ${String(clientErr)}`); + await waitFor(() => captured.some((c) => c.msg.includes("client aborted mid-stream")), "egress client-abort info line", 8000, () => captured.map((c) => `${c.level}: ${c.msg}`).join("\n")); + + const status = getUpstreamConnectionStatus(); + assert.equal(status.connected, false); + assert.match(status.error ?? "", /kind=client-abort/); + assert.doesNotMatch(status.error ?? "", /check upstream health/); + assert.ok(captured.some((c) => c.msg.includes("client disconnected mid-stream")), "relay disconnect info line missing"); + const blaming = captured.filter((c) => c.level === "error" && c.msg.includes("kind=")); + assert.deepEqual(blaming, [], `no error-level line may blame the upstream for a client cancel: ${JSON.stringify(blaming)}`); + } finally { + controller.abort(); + setLogCapture(null); + await close(proxy); + await close(upstream); + } +}); From fd29be36a5e89d243b8bdff9c5949928a22ade84 Mon Sep 17 00:00:00 2001 From: ework-agent Date: Tue, 3 Nov 2026 17:23:12 +0800 Subject: [PATCH 2/2] =?UTF-8?q?fix:=20correct=20the=20#5598=20test=20comme?= =?UTF-8?q?nt=20=E2=80=94=20alert/circuit=20gate=20on=20ALERT=5FKINDS,=20n?= =?UTF-8?q?ot=20externalAborted?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The header claimed retry/alert/circuit were 'already correctly gated on externalAborted'. Only the retry ladder (src/fetch-util.ts) receives externalAborted; recordUpstreamAlert/recordUpstreamCircuitFailure classify without it and stay safe because upstream-timeout/client-abort are absent from ALERT_KINDS (src/upstream-alerts.ts). Comment now matches the code facts. --- tests/issue5598-client-abort-classification.test.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/issue5598-client-abort-classification.test.ts b/tests/issue5598-client-abort-classification.test.ts index cdc274cf9a..c130b7b9d0 100644 --- a/tests/issue5598-client-abort-classification.test.ts +++ b/tests/issue5598-client-abort-classification.test.ts @@ -14,9 +14,10 @@ import { describeUpstreamFailure, formatUpstreamError, getUpstreamConnectionStat * before any upstream byte returns — nginx records 499, and bili's relay catch * classified the resulting AbortError as upstream-timeout ("check upstream * health") because it never passed the clientAbort signal state into the - * classification. Retry/alert/circuit were already correctly gated on - * externalAborted; the misattribution lived in the last-connection status - * surface, the egress line detail, and the 502 diagnostic body. */ + * classification. The retry ladder already classified with externalAborted, + * and alert/circuit never count these kinds (ALERT_KINDS whitelist); the + * misattribution lived in the last-connection status surface, the egress line + * detail, and the 502 diagnostic body. */ function abortError(): Error { return new DOMException("This operation was aborted", "AbortError");