diff --git a/src/upstream-alerts.ts b/src/upstream-alerts.ts index a09a4a7fdf..27e3fb2ec8 100644 --- a/src/upstream-alerts.ts +++ b/src/upstream-alerts.ts @@ -92,11 +92,15 @@ function pruneStaleAlerts(now: number): void { } } -/** #5146: pull the errno off a Node error when present (ECONNREFUSED, ENOTFOUND, …) so the +/** #5146: pull the errno off a Node error or its cause (ECONNREFUSED, ENOTFOUND, …) so the * /__bili/upstreams report can show WHY beyond the classified kind. undefined for non-errors. */ function errnoCode(error: unknown): string | undefined { - const c = typeof error === "object" && error !== null ? (error as { code?: unknown }).code : undefined; - return typeof c === "string" && c.length > 0 ? c : undefined; + for (let depth = 0; typeof error === "object" && error !== null && depth < 8; depth++) { + const current = error as { code?: unknown; cause?: unknown }; + if (typeof current.code === "string" && current.code.length > 0) return current.code; + error = current.cause; + } + return undefined; } /** Record one failed model request. Dedup key (kind, host): repeats refresh diff --git a/tests/upstream-alert-error-codes.test.ts b/tests/upstream-alert-error-codes.test.ts new file mode 100644 index 0000000000..e4f4dccf36 --- /dev/null +++ b/tests/upstream-alert-error-codes.test.ts @@ -0,0 +1,116 @@ +import assert from "node:assert/strict"; +import http from "node:http"; +import { beforeEach, test } from "node:test"; +import { + getOpenUpstreamCircuits, + getUpstreamAlerts, + getUpstreamHealthReport, + recordUpstreamAlert, + recordUpstreamCircuitFailure, + _resetUpstreamAlertsForTest, + _resetUpstreamCircuitsForTest, +} from "../src/upstream-alerts.ts"; + +const upstream = "https://example.test/v1/messages"; + +function wrapped(code: string, depth: number): Error { + let error: Error = Object.assign(new Error("transport failed"), { code }); + for (let i = 0; i < depth; i++) error = new TypeError("fetch failed", { cause: error }); + return error; +} + +beforeEach(() => { + _resetUpstreamAlertsForTest(); + _resetUpstreamCircuitsForTest(); +}); + +for (const [code, depth, viaProxy, kind] of [ + ["ECONNREFUSED", 0, false, "connect-refused"], + ["ECONNREFUSED", 1, false, "connect-refused"], + ["ENOTFOUND", 3, false, "dns"], + ["ECONNRESET", 1, true, "proxy-reset"], + ["UND_ERR_CONNECT_TIMEOUT", 2, false, "connect-timeout"], + ["CERT_HAS_EXPIRED", 1, false, "tls"], + ["ENETUNREACH", 7, false, "upstream-unreachable"], +] as const) { + test(`${code} at cause depth ${depth} reaches alerts and open-circuit health`, () => { + const error = wrapped(code, depth); + recordUpstreamAlert(upstream, error, viaProxy); + assert.equal(getUpstreamAlerts()[0]?.kind, kind); + assert.equal(getUpstreamAlerts()[0]?.code, code); + assert.equal(getUpstreamHealthReport().upstreams[0]?.code, code); + + for (let i = 0; i < 3; i++) recordUpstreamCircuitFailure(upstream, error, viaProxy); + assert.equal(getOpenUpstreamCircuits()[0]?.lastFailure?.code, code); + assert.equal(getUpstreamHealthReport().upstreams[0]?.code, code); + assert.equal(getUpstreamHealthReport().upstreams[0]?.kind, kind); + assert.equal(getUpstreamHealthReport().upstreams[0]?.count, 3); + }); +} + +test("an outer nonempty code keeps precedence over its cause", () => { + const error = Object.assign(new Error("outer refusal", { cause: wrapped("ECONNRESET", 0) }), { code: "ECONNREFUSED" }); + recordUpstreamAlert(upstream, error, false); + assert.equal(getUpstreamAlerts()[0]?.kind, "connect-refused"); + assert.equal(getUpstreamAlerts()[0]?.code, "ECONNREFUSED"); +}); + +for (const code of ["", 0]) { + test(`a nonusable outer code ${JSON.stringify(code)} does not hide the cause errno`, () => { + const error = Object.assign(new Error("wrapper", { cause: wrapped("ECONNRESET", 0) }), { code }); + recordUpstreamAlert(upstream, error, false); + assert.equal(getUpstreamAlerts()[0]?.kind, "upstream-reset"); + assert.equal(getUpstreamAlerts()[0]?.code, "ECONNRESET"); + }); +} + +test("a repeated failure updates the same alert with the new nested errno", () => { + recordUpstreamAlert(upstream, wrapped("ECONNRESET", 0), false); + recordUpstreamAlert(upstream, wrapped("EPIPE", 1), false); + assert.equal(getUpstreamAlerts().length, 1); + assert.equal(getUpstreamAlerts()[0].count, 2); + assert.equal(getUpstreamAlerts()[0].code, "EPIPE"); +}); + +test("an error beyond the taxonomy's eight-entry cause bound stays unclassified", () => { + recordUpstreamAlert(upstream, wrapped("ECONNREFUSED", 8), false); + assert.deepEqual(getUpstreamAlerts(), []); + for (let i = 0; i < 3; i++) recordUpstreamCircuitFailure(upstream, wrapped("ECONNREFUSED", 0), false); + recordUpstreamCircuitFailure(upstream, wrapped("ECONNREFUSED", 8), false); + assert.equal(getOpenUpstreamCircuits()[0]?.lastFailure?.kind, "unknown"); + assert.equal(getOpenUpstreamCircuits()[0]?.lastFailure?.code, undefined); +}); + +test("a cyclic cause without a code terminates without inventing an errno", () => { + const error = new Error("cyclic cause"); + error.cause = error; + recordUpstreamAlert(upstream, error, false); + assert.deepEqual(getUpstreamAlerts(), []); + for (let i = 0; i < 3; i++) recordUpstreamCircuitFailure(upstream, wrapped("ECONNREFUSED", 0), false); + recordUpstreamCircuitFailure(upstream, error, false); + assert.equal(getOpenUpstreamCircuits()[0]?.lastFailure?.code, undefined); +}); + +test("native Fetch socket errors retain their actual cause code in both health windows", async () => { + const server = http.createServer((req) => req.socket.destroy()); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const address = server.address(); + assert.ok(address && typeof address === "object"); + const url = `http://127.0.0.1:${address.port}/v1/messages`; + try { + let failure: unknown; + try { await fetch(url); } catch (error) { failure = error; } + assert.ok(failure instanceof TypeError); + assert.equal((failure as Error & { code?: unknown }).code, undefined); + assert.ok(failure.cause instanceof Error); + assert.equal((failure.cause as Error & { code?: unknown }).code, "UND_ERR_SOCKET"); + recordUpstreamAlert(url, failure, false); + assert.equal(getUpstreamHealthReport().upstreams[0]?.code, "UND_ERR_SOCKET"); + for (let i = 0; i < 3; i++) recordUpstreamCircuitFailure(url, failure, false); + assert.equal(getOpenUpstreamCircuits()[0]?.lastFailure?.code, "UND_ERR_SOCKET"); + assert.equal(getUpstreamHealthReport().upstreams[0]?.code, "UND_ERR_SOCKET"); + } finally { + server.closeAllConnections(); + await new Promise((resolve) => server.close(() => resolve())); + } +}); diff --git a/tests/upstream-alerts.test.ts b/tests/upstream-alerts.test.ts index c72a512390..fd261a051a 100644 --- a/tests/upstream-alerts.test.ts +++ b/tests/upstream-alerts.test.ts @@ -383,10 +383,15 @@ test("#1682: transport failure appears in /__bili/overview and clears on recover const a1 = await overviewAlerts(proxyPort); assert.equal(a1.length, 1); assert.equal(a1[0].kind, "connect-refused"); + assert.equal(a1[0].code, "ECONNREFUSED"); assert.equal(a1[0].host, `127.0.0.1:${deadPort}`); assert.equal(a1[0].count, 1); assert.ok(typeof a1[0].hint === "string" && a1[0].hint.length > 0); assert.equal(a1[0].firstSeen, a1[0].lastSeen); + const healthResponse = await fetch(`http://127.0.0.1:${proxyPort}/__bili/upstreams`); + assert.equal(healthResponse.status, 200); + const health = await healthResponse.json() as ReturnType; + assert.equal(health.upstreams[0]?.code, "ECONNREFUSED"); // 2) second failure → same entry, count bumped, no stacking const r2 = await chatPost(proxyPort, deadPort, "ep-alerts-1");