diff --git a/reference/architecture.md b/reference/architecture.md index 33fe32a5ce..f929892475 100644 --- a/reference/architecture.md +++ b/reference/architecture.md @@ -31,6 +31,7 @@ billion-context/ │ ├── knobs.ts # Single knob resolver: env > config file > default (#2030) │ ├── config.ts # Config file (billion-context.json) + providers table + loadOptions │ ├── instance.ts # Live-instance registry record in the proxy-origin file (#394/#403/#417) +│ ├── port-owner.ts # Post-bind port-ownership verification + pointer-clobber guard (#2941) │ ├── paths.ts # XDG paths (config/cache/state) + relocation env vars │ ├── logger.ts # Tee logger: state-dir file + stderr │ ├── log-mask.ts # Log safety: masks credentials/endpoints (#255) diff --git a/src/port-owner.ts b/src/port-owner.ts new file mode 100644 index 0000000000..a0e2409def --- /dev/null +++ b/src/port-owner.ts @@ -0,0 +1,179 @@ +import { isProxyInstanceFile, type ProxyInstanceFile } from "./instance.js"; + +/** #2941: post-bind port-ownership verification + pointer-clobber guard for + * same-lane double-spawn residuals (#707 follow-ups). Conservative by + * design: positive evidence only — never yield/refuse on ambiguity. */ + +interface OwnerProbeInfo { + pid?: number; + instanceId?: string; + /** Negative attribution: responder is up but pre-registration (#2941 + * readiness gate) — see findForeignOwner for the handling. */ + reason?: string; +} + +/** Structural fetch shape — global `fetch` satisfies it; tests inject bare objects. */ +export type HealthFetcher = (url: string, init?: RequestInit) => Promise<{ ok: boolean; json(): Promise }>; + +interface ProbeDeps { + fetchImpl?: HealthFetcher; + sleepImpl?: (ms: number) => Promise; +} + +const DEFAULT_ATTEMPTS = 3; +const DEFAULT_INTERVAL_MS = 100; +const DEFAULT_TIMEOUT_MS = 1_500; + +// #2941/#1529: a default fetch socket lingers ~4s (pooled keep-alive) AFTER +// the post-bind self-probe completes, sitting in OUR OWN conn ledger and +// inflating [exposure] liveConns in exactly the window the #1529 +// hold-until-peer-FIN contract samples it (liveConns=2 instead of 1). +// `connection: close` makes every probe a dedicated socket that dies with +// its response — verified: immediate server-side close vs ~3.8s linger. +function probeFetch(url: string, init?: RequestInit): Promise { + const headers = new Headers(init?.headers); + headers.set("connection", "close"); + return fetch(url, { ...init, headers }); +} + +function healthUrl(origin: string): string { + return origin.replace(/\/+$/, "") + "/__bili/health"; +} + +/** One /__bili/health read. undefined when dead/non-JSON/not-ok — absence of + * evidence is not evidence of a foreign owner. */ +export async function probeOwnerInfo( + origin: string, + fetchImpl: HealthFetcher = probeFetch, + timeoutMs: number = DEFAULT_TIMEOUT_MS, +): Promise { + try { + const res = await fetchImpl(healthUrl(origin), { signal: AbortSignal.timeout(timeoutMs) }); + if (!res.ok) { + // #2941: our own pre-registration 503 carries reason:"starting" — + // surface it so findForeignOwner self-attributes instead of burning + // its probe budget on itself. + const neg = (await res.json()) as { ok?: unknown; reason?: unknown }; + if (neg.ok === false && typeof neg.reason === "string") return { reason: neg.reason }; + return undefined; + } + const data = (await res.json()) as { ok?: unknown; pid?: unknown; instanceId?: unknown }; + if (data.ok !== true) return undefined; + const info: OwnerProbeInfo = {}; + if (typeof data.pid === "number" && Number.isInteger(data.pid) && data.pid > 0) info.pid = data.pid; + if (typeof data.instanceId === "string" && data.instanceId !== "") info.instanceId = data.instanceId; + return info; + } catch { + return undefined; + } +} + +/** #2941: foreign live listener co-listening on our just-bound port? Multiple + * probes: dual-bind-tolerant platforms may distribute connections among the + * listeners, so one round-trip can hit our own socket. A responder WITHOUT a + * pid field (pre-#1753 builds) is unattributable — yielding on it would + * ratchet sticky ports during upgrade overlaps (#1723). */ +export async function findForeignOwner( + origin: string, + ownPid: number, + opts: { attempts?: number; intervalMs?: number; timeoutMs?: number } & ProbeDeps = {}, +): Promise { + const attempts = Math.max(1, opts.attempts ?? DEFAULT_ATTEMPTS); + const intervalMs = opts.intervalMs ?? DEFAULT_INTERVAL_MS; + const timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS; + const fetchImpl = opts.fetchImpl ?? probeFetch; + const sleepImpl = opts.sleepImpl ?? ((ms: number) => new Promise((r) => setTimeout(r, ms))); + for (let i = 0; i < attempts; i++) { + if (i > 0) await sleepImpl(intervalMs); + const info = await probeOwnerInfo(origin, fetchImpl, timeoutMs); + if (info?.pid !== undefined && info.pid !== ownPid) return info; + // #2941: a "starting" responder on our just-bound port is ourselves + // mid-announce — /__bili/health gates until registration completes, so + // without this every boot burned the full probe budget (~200ms+) on its + // own pre-registration 503s and delayed pointer/registry publication + // past callers that expect registration right after bind (#394 warn + // timing in multi-instance-live). Bind is exclusive on every platform + // bili ships on (Linux EADDRINUSE; Windows per libuv issue #1360), so no + // foreign listener can be in that state here; a hypothetical dual-bind + // co-starter would be unregistered anyway — the registry/pointer layer + // surfaces the collision at registration time. Never foreign. + if (info?.reason === "starting") return undefined; + } + return undefined; +} + +type PortClaimOutcome = + | { kind: "claimed" } + | { kind: "ladder"; nextPort: number; foreign: OwnerProbeInfo } + | { kind: "die"; message: string }; + +function portOf(origin: string): number { + try { + return Number(new URL(origin).port || 80); + } catch { + return 0; + } +} + +/** #2941: what to do with a bound port whose ownership check found a foreign + * listener — mirrors the EADDRINUSE ladder contract (#407/#964): late binder + * moves +1, ephemeral after the attempt budget, strict-port/manual fail fast. */ +export async function resolvePortClaim(params: { + origin: string; + ownPid: number; + /** launchToken && !strictPort — the retry contract exists only for + * launcher-spawned children (#407); manual `bili start` stays fail-fast. */ + canLadder: boolean; + listenAttempts: number; + maxListenAttempts: number; + probe?: { attempts?: number; intervalMs?: number; timeoutMs?: number } & ProbeDeps; +}): Promise { + const foreign = await findForeignOwner(params.origin, params.ownPid, params.probe); + if (foreign === undefined) return { kind: "claimed" }; + const who = `another live proxy (pid ${foreign.pid}${foreign.instanceId !== undefined ? `, id ${foreign.instanceId.slice(0, 8)}` : ""})`; + if (!params.canLadder) { + return { kind: "die", message: `port ${portOf(params.origin)} is shared with ${who} — refusing to serve a contested port (#2941)` }; + } + const attempts = params.listenAttempts + 1; + const nextPort = attempts >= params.maxListenAttempts ? 0 : portOf(params.origin) + 1; + return { kind: "ladder", nextPort, foreign }; +} + +interface PointerGuardResult { + allow: boolean; + holder?: { pid: number; instanceId: string; origin: string }; +} + +/** #2941: proxy-origin pointer overwrite identity check. Refuses ONLY a + * different live instance whose recorded pid health-checks as serving the + * SAME address. Everything else allows the write: missing/legacy records, + * dead pids, draining predecessors (alive process, closed listener — must + * keep the #1723 takeover working), responder-pid mismatch (stale record — + * overwriting repairs it), cross-address holders (last-writer-wins by + * design, #1232). */ +export async function mayClobberPointer(params: { + candidateOrigin: string; + candidateInstanceId: string; + read: () => ProxyInstanceFile | { origin: string } | undefined; + isAlive: (pid: number) => boolean; + healthOf: (origin: string) => Promise; +}): Promise { + const current = params.read(); + if (!isProxyInstanceFile(current)) return { allow: true }; + if (current.instanceId === params.candidateInstanceId) return { allow: true }; + if (!sameAddress(current.origin, params.candidateOrigin)) return { allow: true }; + if (current.pid <= 0 || !params.isAlive(current.pid)) return { allow: true }; + const health = await params.healthOf(current.origin); + if (health !== undefined && health.pid === current.pid) { + return { allow: false, holder: { pid: current.pid, instanceId: current.instanceId, origin: current.origin } }; + } + return { allow: true }; +} + +function sameAddress(a: string, b: string): boolean { + try { + return new URL(a).host === new URL(b).host; + } catch { + return false; + } +} diff --git a/src/server.ts b/src/server.ts index bdad26c4b8..b77edd702b 100644 --- a/src/server.ts +++ b/src/server.ts @@ -97,7 +97,8 @@ import { getStore } from "./persist.js"; import { log as loggerLog, configureLogger, getLogPath, closeLogger, isFileLoggingActive, isStreamWriteError, isBenignSocketRaceError, enterSessionContext, muteStderrLeg } from "./logger.js"; import { queryLogLines } from "./web/logs-query.js"; import { configFile, defaultLogFile, stateDir } from "./paths.js"; -import { atomicWriteInstanceFile, clearProxyInstanceFile, entryScriptFingerprint, findSameLanePredecessor, isPidAlive, listInstances, registerInstanceAndWarn, unregisterInstance, warnOnNewPeers, type ProxyInstanceFile } from "./instance.js"; +import { atomicWriteInstanceFile, clearProxyInstanceFile, entryScriptFingerprint, findSameLanePredecessor, isPidAlive, listInstances, readProxyInstanceFile, registerInstanceAndWarn, unregisterInstance, warnOnNewPeers, type ProxyInstanceFile } from "./instance.js"; +import { mayClobberPointer, probeOwnerInfo, resolvePortClaim } from "./port-owner.js"; import { hoistTrappedToolItems } from "./tool-pair-order.js"; import { computeAnthropicMessageMarks, stampAnthropicSystemCacheControl, anthropicToolsCarryCacheControl } from "./loop/cache-control.js"; import { reconcileSystemAnchor } from "./system-anchor.js"; @@ -422,6 +423,13 @@ export async function startServer(opts: ProxyOptions): Promise { // (the chain check only compares against the other running instance). const instanceId = randomUUID(); const instanceStartedAt = Date.now(); + // #2941: the bind happens BEFORE announceListening writes this instance's + // identity files (pointer + registry entry), and the health route is + // stateless — so between bind and registration a probe sees "healthy" for + // an instance no discovery pass can find, and a concurrent same-lane + // launcher spawns a duplicate instead of attaching or waiting. The health + // endpoint reports not-ready until registration completes. + let bootReady = false; // #7 (shared stable-port proxy): the parent-gone watchdog watches a SET of // pids, not one. The spawning hook seeds it via BILI_PARENT_PID; every // ATTACHING session (POST /__bili/watcher) adds its claude host, so a @@ -462,7 +470,7 @@ export async function startServer(opts: ProxyOptions): Promise { // as the context-window source for zero-config `/p/` routes that have no // per-model config. A miss falls back to the prefix table + default. void loadRegistry(); - const adminCtx = { opts, core, config, log, instanceId, instanceStartedAt, proxyWatchers, initialWatcherPid }; + const adminCtx = { opts, core, config, log, instanceId, instanceStartedAt, proxyWatchers, initialWatcherPid, bootReady: () => bootReady }; const dispatch = async (req: http.IncomingMessage, res: http.ServerResponse): Promise => { armRequestWatchdog(req, res, log); @@ -885,19 +893,26 @@ export async function startServer(opts: ProxyOptions): Promise { } return pred !== undefined; }; - const announceListening = (): void => { - const actualPort = server.address() === null ? opts.port : (server.address() as { port: number }).port; + // Discovery origin local MCP shells dial: collapse wildcard + // binds to loopback (localhost may resolve to ::1, where an + // IPv4-only listener is absent) and bracket bare IPv6 literals + // so the file always holds a valid URL. + const boundOrigin = (actualPort: number): string => { + const originHost = opts.host === "0.0.0.0" || opts.host === "::" || opts.host === "localhost" ? "127.0.0.1" : opts.host.includes(":") && !opts.host.startsWith("[") ? `[${opts.host}]` : opts.host; + return `http://${originHost}:${actualPort}`; + }; + const announceListening = async (): Promise => { + // #2941: closed between bind and announce (fast exit, test teardown) — + // announcing would register a dead listener under the configured port + // (port 0 → bogus http://host:0 banners/registry records); skip entirely. + if (server.address() === null) return; + const actualPort = (server.address() as { port: number }).port; const nonLoopbackBind = opts.host === "0.0.0.0" || opts.host === "::" || !isLoopbackAddress(opts.host); // Honest bind display: a wildcard bind shows as 0.0.0.0 (the user // chose to expose the proxy — hiding it behind "localhost" made // remote setups look broken in the log, see #240). const displayHost = nonLoopbackBind ? opts.host : opts.host === "0.0.0.0" ? "localhost" : opts.host; - // Discovery origin local MCP shells dial: collapse wildcard - // binds to loopback (localhost may resolve to ::1, where an - // IPv4-only listener is absent) and bracket bare IPv6 literals - // so the file always holds a valid URL. - const originHost = opts.host === "0.0.0.0" || opts.host === "::" || opts.host === "localhost" ? "127.0.0.1" : opts.host.includes(":") && !opts.host.startsWith("[") ? `[${opts.host}]` : opts.host; - const origin = `http://${originHost}:${actualPort}`; + const origin = boundOrigin(actualPort); const instanceRecord: ProxyInstanceFile = { origin, instanceId, @@ -920,7 +935,21 @@ export async function startServer(opts: ProxyOptions): Promise { // restores the pre-#2265 exact-hash-only resolution (hot kill-switch). setSimhashAdoptionEnabled(opts.affinitySimhash ?? true); hydratePrefixAffinity(); - atomicWriteInstanceFile(instanceRecord); + // #2941: never clobber a live pointer on this port — a foreign + // instance health-verified as serving here keeps it; our registry + // entry below still records us for lane-aware discovery. + const guard = await mayClobberPointer({ + candidateOrigin: origin, + candidateInstanceId: instanceId, + read: readProxyInstanceFile, + isAlive: isPidAlive, + healthOf: probeOwnerInfo, + }); + if (guard.allow) { + atomicWriteInstanceFile(instanceRecord); + } else if (guard.holder !== undefined) { + log("warn", `[pointer] keeping proxy-origin held by live instance ${guard.holder.instanceId.slice(0, 8)} (pid ${guard.holder.pid}, ${guard.holder.origin}) on this port — refusing to clobber a live pointer (#2941)`); + } } catch { // best-effort discovery hint for host-spawned MCP shells } @@ -1011,10 +1040,60 @@ export async function startServer(opts: ProxyOptions): Promise { if (opts.debug) { log("info", `[debug] build features: raw-HTTP-capture(${bodyDumpEnabled() ? "on" : "off"}) | remote_compaction_v2-strip(on) | cert-MITM-launcher(on) | strip-acp-summary(on) — seeing this line confirms the launcher build (not registry 0.1.34)`); } + // #2941: LAST statement — registration above is done; degraded boots + // (best-effort write failed) still flip ready, serving as before. + bootReady = true; + }; + // #2941: after a successful bind, verify we are the SOLE owner of the + // port before registering. Where socket semantics tolerate a concurrent + // bind onto an actively-listening port, a same-lane double spawn would + // otherwise serve ONE port from two state machines silently; stock libuv + // enforces exclusivity (EADDRINUSE ladder below), so this is a defensive + // second layer — see src/port-owner.ts. + const claimOrAnnounce = (): void => { + void (async () => { + const addr = server.address(); + if (addr !== null && typeof addr === "object") { + const outcome = await resolvePortClaim({ + origin: boundOrigin(addr.port), + ownPid: process.pid, + canLadder: Boolean(launchToken) && !strictPort, + listenAttempts, + maxListenAttempts: MAX_LISTEN_ATTEMPTS, + }); + if (outcome.kind === "ladder") { + listenAttempts += 1; + log("warn", `port ${lastTriedPort} shared with foreign listener (pid ${outcome.foreign.pid}) — ${outcome.nextPort === 0 ? "retrying on an ephemeral port" : `retrying on port ${outcome.nextPort}`} (#2941)`); + // closeAllConnections: idle keep-alive sockets from early + // probes must not stall the re-listen callback. + server.close(() => attemptListen(outcome.nextPort)); + server.closeAllConnections?.(); + return; + } + if (outcome.kind === "die") { + log("error", `listen failed: ${outcome.message}`); + shuttingDown = true; + server.close(); + flushConversations(); + void flushAllSessions().finally(() => { + closeLogger(); + process.exit(1); + }); + return; + } + } + await announceListening(); + })(); }; const attemptListen = (port: number): void => { lastTriedPort = port; - server.listen(port, opts.host, announceListening); + // #2941: a failed listen never fires 'listening', so each attempt's + // callback stacks on the server and a later successful bind fires ALL + // accumulated claimOrAnnounce closures at once (double announce, + // double registration, MaxListenersExceededWarning on long waits). + // Only our own callbacks ever use this event. + server.removeAllListeners("listening"); + server.listen(port, opts.host, claimOrAnnounce); }; attemptListen(opts.port); // Listen errors (EADDRINUSE port taken, EACCES privileged port, EAFNOSUPPORT diff --git a/src/server/admin.ts b/src/server/admin.ts index 9f51be005c..154698296b 100644 --- a/src/server/admin.ts +++ b/src/server/admin.ts @@ -40,6 +40,10 @@ interface AdminCtx { instanceStartedAt: number; proxyWatchers: Set; initialWatcherPid: number | null; + /** #2941: false until startup registration (pointer + registry entry) has + * completed — the bind precedes it, so health must not claim readiness + * before discovery can find this instance. */ + bootReady: () => boolean; } // #1537: reduce a Host-header value or a URL hostname to its bare lowercase @@ -171,6 +175,15 @@ export async function handleAdminRoute(req: http.IncomingMessage, res: http.Serv return; } if (req.method === "GET" && req.url === "/__bili/health") { + // #2941: 503 while registration is still pending — every consumer + // (launcher spawn-wait, attach discovery, host-native respawn) keys on + // HTTP ok, so one gate covers all of them; pre-boot the instance is + // bound but not yet findable by name. + if (!ctx.bootReady()) { + res.writeHead(503, { "content-type": "application/json" }); + res.end(JSON.stringify({ ok: false, reason: "starting" })); + return; + } res.writeHead(200, { "content-type": "application/json" }); // #1322: watchdog state is part of the health contract — attachers and // operators can see whether this proxy dies with its sessions (armed) diff --git a/tests/admin-probe.test.ts b/tests/admin-probe.test.ts index e21090036a..c1944e6fd9 100644 --- a/tests/admin-probe.test.ts +++ b/tests/admin-probe.test.ts @@ -13,6 +13,18 @@ import { SessionStore, _setStoreForTest } from "../src/persist.ts"; import { _setForTest as setRegistryForTest } from "../src/registry.ts"; import { rmrf } from "./tmp-rm.ts"; +// #2941: registration now completes AFTER the bind (post-bind ownership +// check), so "listening" no longer implies health-readiness — poll for it. +async function waitReady(port: number): Promise { + for (let i = 0; i < 200; i++) { + try { + if ((await fetch(`http://127.0.0.1:${port}/__bili/health`, { signal: AbortSignal.timeout(500) })).status === 200) return; + } catch {} + await new Promise((r) => setTimeout(r, 50)); + } + throw new Error(`proxy on port ${port} did not become ready`); +} + /** #1073: forward-proxy-style (absolute-form) management probes addressed to * bili instances must return real health state instead of being tunneled and * 403'd by the target's own admin gate — while every #409 security property @@ -144,6 +156,8 @@ test("integration: absolute-form management probes answer with real health state const port1 = (p1.address() as { port: number }).port; const port2 = (p2.address() as { port: number }).port; try { + await waitReady(port1); + await waitReady(port2); const health1 = await get(port1, "/__bili/health"); const health2 = await get(port2, "/__bili/health"); assert.equal(health1.status, 200); diff --git a/tests/banner-identity.test.ts b/tests/banner-identity.test.ts index f26c668576..3e54841820 100644 --- a/tests/banner-identity.test.ts +++ b/tests/banner-identity.test.ts @@ -59,16 +59,29 @@ async function startBare(opts: Partial, captured: Captured[]): Pro test("banner carries pid + instance id and fires exactly once per listen", async () => { const captured: Captured[] = []; let dir: string | undefined; + let proxy: http.Server | undefined; try { - const { proxy, dir: d } = await startBare({}, captured); - dir = d; + const started = await startBare({}, captured); + proxy = started.proxy; + dir = started.dir; const port = (proxy.address() as { port: number }).port; + // #2941: registration now completes AFTER the bind (post-bind ownership + // check), so "listening" no longer implies the banner is logged yet — poll for it. + const t0 = Date.now(); + while (!captured.some((c) => c.msg.includes("listening on")) && Date.now() - t0 < 10_000) { + await new Promise((r) => setTimeout(r, 25)); + } const banners = captured.filter((c) => c.msg.includes("listening on")); assert.equal(banners.length, 1, "one banner per successful listen"); assert.match(banners[0]!.msg, new RegExp(`^acp-proxy v[\\d.]+ \\(pid ${process.pid}, id [0-9a-f]{8}\\) listening on http://127\\.0\\.0\\.1:${port}`)); assert.ok(banners[0]!.msg.includes("web UI:"), "rest of the banner line is unchanged"); - await new Promise((resolve, reject) => proxy.close((e) => (e ? reject(e) : resolve()))); } finally { + // best-effort close on EVERY path — an assertion failure above must not + // leak the server handle (it would pin this process's event loop forever). + if (proxy !== undefined) { + const p = proxy; + await new Promise((resolve) => p.close(() => resolve())); + } setLogCapture(null); closeLogger(); if (dir) rmrf(dir); diff --git a/tests/blind-tunnel-stats.test.ts b/tests/blind-tunnel-stats.test.ts index 6178b37405..77fd31702e 100644 --- a/tests/blind-tunnel-stats.test.ts +++ b/tests/blind-tunnel-stats.test.ts @@ -13,6 +13,18 @@ import { _setForTest as setRegistryForTest } from "../src/registry.ts"; import { recordBlindTunnel, getBlindTunnelStats, _resetBlindTunnelStatsForTest } from "../src/mitm.ts"; import { rmrf } from "./tmp-rm.ts"; +// #2941: registration now completes AFTER the bind (post-bind ownership +// check), so "listening" no longer implies health-readiness — poll for it. +async function waitReady(port: number): Promise { + for (let i = 0; i < 200; i++) { + try { + if ((await fetch(`http://127.0.0.1:${port}/__bili/health`, { signal: AbortSignal.timeout(500) })).status === 200) return; + } catch {} + await new Promise((r) => setTimeout(r, 50)); + } + throw new Error(`proxy on port ${port} did not become ready`); +} + /** #897: /__bili/health and /__bili/stats must expose the blind-tunnel counter * (loopback-only admin endpoints), so an operator can see which CONNECT * targets were relayed opaquely and never entered the compression pipeline. */ @@ -62,6 +74,7 @@ test("health + stats expose blindTunnels counts with real hosts (#897)", async ( proxy = await startServer(opts); await once(proxy, "listening"); const port = (proxy.address() as { port: number }).port; + await waitReady(port); const emptyHealth = (await (await fetch(`http://127.0.0.1:${port}/__bili/health`)).json()) as { blindTunnels: BlindTunnels }; assert.deepEqual(emptyHealth.blindTunnels, { total: 0, hosts: {} }, "fresh instance reports zero blind tunnels"); diff --git a/tests/port-owner.test.ts b/tests/port-owner.test.ts new file mode 100644 index 0000000000..97e6fd644a --- /dev/null +++ b/tests/port-owner.test.ts @@ -0,0 +1,259 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import http from "node:http"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { findForeignOwner, mayClobberPointer, probeOwnerInfo, resolvePortClaim, type HealthFetcher } from "../src/port-owner.ts"; +import { atomicWriteInstanceFile, isPidAlive, readProxyInstanceFile, type ProxyInstanceFile } from "../src/instance.ts"; +import { rmrf } from "./tmp-rm.ts"; + +const OWN_PID = process.pid; +const FOREIGN_PID = 4_242_424; + +function healthFetch(body: unknown): HealthFetcher { + return async () => ({ ok: true, json: async () => body }); +} + +function seqFetch(...bodies: unknown[]): { fetchImpl: HealthFetcher; calls: () => number } { + let n = 0; + const fetchImpl: HealthFetcher = async () => { + const body = bodies[Math.min(n, bodies.length - 1)]; + n += 1; + return { ok: true, json: async () => body }; + }; + return { fetchImpl, calls: () => n }; +} + +const NO_SLEEP = async (): Promise => {}; + +test("probeOwnerInfo: extracts pid + instanceId from a health body", async () => { + const info = await probeOwnerInfo("http://127.0.0.1:9", healthFetch({ ok: true, pid: FOREIGN_PID, instanceId: "abc-123" })); + assert.deepEqual(info, { pid: FOREIGN_PID, instanceId: "abc-123" }); +}); + +test("probeOwnerInfo: unreachable / not-ok / non-ok bodies yield undefined or unattributable info", async () => { + const throwing: HealthFetcher = async () => { throw new Error("ECONNREFUSED"); }; + assert.equal(await probeOwnerInfo("http://127.0.0.1:1", throwing), undefined); + const notOk: HealthFetcher = async () => ({ ok: false, json: async () => ({}) }); + assert.equal(await probeOwnerInfo("http://127.0.0.1:1", notOk), undefined); + const notHealthy: HealthFetcher = async () => ({ ok: true, json: async () => ({ ok: false, pid: FOREIGN_PID }) }); + assert.equal(await probeOwnerInfo("http://127.0.0.1:1", notHealthy), undefined); + // pre-#1753 shape: healthy but no pid field — unattributable, NOT foreign + const noPid: HealthFetcher = async () => ({ ok: true, json: async () => ({ ok: true }) }); + assert.deepEqual(await probeOwnerInfo("http://127.0.0.1:1", noPid), {}); +}); + +test("probeOwnerInfo: pre-registration 503 surfaces reason:'starting' (negative attribution, #2941)", async () => { + const starting: HealthFetcher = async () => ({ ok: false, json: async () => ({ ok: false, reason: "starting" }) }); + assert.deepEqual(await probeOwnerInfo("http://127.0.0.1:1", starting), { reason: "starting" }); + const noBody: HealthFetcher = async () => ({ ok: false, json: async () => { throw new Error("no body"); } }); + assert.equal(await probeOwnerInfo("http://127.0.0.1:1", noBody), undefined); +}); + +test("findForeignOwner: 'starting' responder stops the probe loop after one round-trip (#2941 gate window)", async () => { + let n = 0; + const fetchImpl: HealthFetcher = async () => { + n += 1; + return { ok: false, json: async () => ({ ok: false, reason: "starting" }) }; + }; + const sleeps: number[] = []; + assert.equal( + await findForeignOwner("http://127.0.0.1:9", OWN_PID, { attempts: 3, fetchImpl, sleepImpl: async (ms) => { sleeps.push(ms); } }), + undefined, + ); + assert.equal(n, 1); + assert.deepEqual(sleeps, []); +}); + +test("findForeignOwner: own-pid probe keeps looping; a later 'starting' probe stops it", async () => { + let n = 0; + const fetchImpl: HealthFetcher = async () => { + n += 1; + return n === 1 + ? { ok: true, json: async () => ({ ok: true, pid: OWN_PID }) } + : { ok: false, json: async () => ({ ok: false, reason: "starting" }) }; + }; + assert.equal( + await findForeignOwner("http://127.0.0.1:9", OWN_PID, { attempts: 3, fetchImpl, sleepImpl: NO_SLEEP }), + undefined, + ); + assert.equal(n, 2); +}); + +test("findForeignOwner: all probes ours → undefined", async () => { + const { fetchImpl } = seqFetch({ ok: true, pid: OWN_PID }, { ok: true, pid: OWN_PID }, { ok: true, pid: OWN_PID }); + assert.equal(await findForeignOwner("http://127.0.0.1:9", OWN_PID, { fetchImpl, sleepImpl: NO_SLEEP }), undefined); +}); + +test("findForeignOwner: connection-distributed co-listener caught on a later probe", async () => { + const { fetchImpl, calls } = seqFetch({ ok: true, pid: OWN_PID }, { ok: true, pid: FOREIGN_PID, instanceId: "foreign-inst" }, { ok: true, pid: OWN_PID }); + const foreign = await findForeignOwner("http://127.0.0.1:9", OWN_PID, { fetchImpl, sleepImpl: NO_SLEEP }); + assert.deepEqual(foreign, { pid: FOREIGN_PID, instanceId: "foreign-inst" }); + assert.equal(calls(), 2); +}); + +test("findForeignOwner: unattributable responders are never treated as foreign", async () => { + const { fetchImpl } = seqFetch({ ok: true }, { ok: true }, { ok: true }); + assert.equal(await findForeignOwner("http://127.0.0.1:9", OWN_PID, { fetchImpl, sleepImpl: NO_SLEEP }), undefined); +}); + +test("findForeignOwner: unreachable probes → undefined", async () => { + const throwing: HealthFetcher = async () => { throw new Error("ECONNREFUSED"); }; + assert.equal(await findForeignOwner("http://127.0.0.1:1", OWN_PID, { fetchImpl: throwing, sleepImpl: NO_SLEEP }), undefined); +}); + +test("findForeignOwner: honors the attempt budget exactly", async () => { + const { fetchImpl, calls } = seqFetch({ ok: true, pid: OWN_PID }); + const sleeps: number[] = []; + await findForeignOwner("http://127.0.0.1:9", OWN_PID, { attempts: 5, fetchImpl, sleepImpl: async (ms) => { sleeps.push(ms); } }); + assert.equal(calls(), 5); + assert.deepEqual(sleeps, [100, 100, 100, 100]); +}); + +test("findForeignOwner: real socket answering a foreign pid is detected", async () => { + const server = http.createServer((req, res) => { + if (req.url === "/__bili/health") { + res.setHeader("content-type", "application/json"); + res.end(JSON.stringify({ ok: true, pid: FOREIGN_PID, instanceId: "foreign-inst" })); + return; + } + res.statusCode = 404; + res.end(); + }); + await new Promise((r) => server.listen(0, "127.0.0.1", () => r())); + const port = (server.address() as { port: number }).port; + try { + const foreign = await findForeignOwner(`http://127.0.0.1:${port}`, OWN_PID, { intervalMs: 10 }); + assert.deepEqual(foreign, { pid: FOREIGN_PID, instanceId: "foreign-inst" }); + } finally { + server.closeAllConnections?.(); + await new Promise((r) => server.close(() => r())); + } +}); + +test("findForeignOwner: real socket answering OUR pid → no foreign owner", async () => { + const server = http.createServer((req, res) => { + if (req.url === "/__bili/health") { + res.setHeader("content-type", "application/json"); + res.end(JSON.stringify({ ok: true, pid: OWN_PID, instanceId: "own-inst" })); + return; + } + res.statusCode = 404; + res.end(); + }); + await new Promise((r) => server.listen(0, "127.0.0.1", () => r())); + const port = (server.address() as { port: number }).port; + try { + assert.equal(await findForeignOwner(`http://127.0.0.1:${port}`, OWN_PID, { intervalMs: 10 }), undefined); + } finally { + server.closeAllConnections?.(); + await new Promise((r) => server.close(() => r())); + } +}); + +test("resolvePortClaim: sole owner → claimed", async () => { + const { fetchImpl } = seqFetch({ ok: true, pid: OWN_PID }); + const outcome = await resolvePortClaim({ origin: "http://127.0.0.1:18787", ownPid: OWN_PID, canLadder: true, listenAttempts: 0, maxListenAttempts: 17, probe: { fetchImpl, sleepImpl: NO_SLEEP } }); + assert.deepEqual(outcome, { kind: "claimed" }); +}); + +test("resolvePortClaim: foreign owner + ladder rights → move +1", async () => { + const { fetchImpl } = seqFetch({ ok: true, pid: FOREIGN_PID }); + const outcome = await resolvePortClaim({ origin: "http://127.0.0.1:18787", ownPid: OWN_PID, canLadder: true, listenAttempts: 0, maxListenAttempts: 17, probe: { fetchImpl, sleepImpl: NO_SLEEP } }); + assert.deepEqual(outcome, { kind: "ladder", nextPort: 18788, foreign: { pid: FOREIGN_PID } }); +}); + +test("resolvePortClaim: exhausted attempt budget → ephemeral port", async () => { + const { fetchImpl } = seqFetch({ ok: true, pid: FOREIGN_PID }); + const outcome = await resolvePortClaim({ origin: "http://127.0.0.1:18787", ownPid: OWN_PID, canLadder: true, listenAttempts: 16, maxListenAttempts: 17, probe: { fetchImpl, sleepImpl: NO_SLEEP } }); + assert.equal(outcome.kind === "ladder" ? outcome.nextPort : null, 0); +}); + +test("resolvePortClaim: foreign owner without ladder rights → die naming the foreign pid", async () => { + const { fetchImpl } = seqFetch({ ok: true, pid: FOREIGN_PID, instanceId: "ffffffff-0000-0000-0000-000000000000" }); + const outcome = await resolvePortClaim({ origin: "http://127.0.0.1:18787", ownPid: OWN_PID, canLadder: false, listenAttempts: 0, maxListenAttempts: 17, probe: { fetchImpl, sleepImpl: NO_SLEEP } }); + assert.equal(outcome.kind, "die"); + if (outcome.kind === "die") { + assert.match(outcome.message, /18787/); + assert.match(outcome.message, /pid 4242424/); + assert.match(outcome.message, /ffffffff/); + } +}); + +function holderRecord(over: Partial = {}): ProxyInstanceFile { + return { + origin: "http://127.0.0.1:18787", + instanceId: "inst-holder", + pid: process.pid, + startedAt: 1_000, + host: "127.0.0.1", + port: 18787, + passthrough: false, + mitmDomains: [], + modelWindows: {}, + ...over, + }; +} + +const CANDIDATE = { candidateOrigin: "http://127.0.0.1:18787", candidateInstanceId: "inst-candidate" }; +const aliveSelf = (pid: number): boolean => pid === process.pid; +const healthServing = async (): Promise<{ pid?: number; instanceId?: string } | undefined> => ({ pid: process.pid }); + +test("mayClobberPointer: missing or legacy record → allowed", async () => { + assert.equal((await mayClobberPointer({ ...CANDIDATE, read: () => undefined, isAlive: aliveSelf, healthOf: healthServing })).allow, true); + assert.equal((await mayClobberPointer({ ...CANDIDATE, read: () => ({ origin: "http://127.0.0.1:18787" }), isAlive: aliveSelf, healthOf: healthServing })).allow, true); +}); + +test("mayClobberPointer: same instance re-registering → allowed without health probe", async () => { + let probed = 0; + const result = await mayClobberPointer({ ...CANDIDATE, candidateInstanceId: "inst-holder", read: () => holderRecord(), isAlive: aliveSelf, healthOf: async () => { probed += 1; return { pid: process.pid }; } }); + assert.equal(result.allow, true); + assert.equal(probed, 0); +}); + +test("mayClobberPointer: cross-address holder → allowed (last-writer-wins by design, #1232)", async () => { + const result = await mayClobberPointer({ ...CANDIDATE, candidateOrigin: "http://127.0.0.1:18788", read: () => holderRecord(), isAlive: aliveSelf, healthOf: healthServing }); + assert.equal(result.allow, true); +}); + +test("mayClobberPointer: dead holder pid → allowed, no health probe", async () => { + let probed = 0; + const result = await mayClobberPointer({ ...CANDIDATE, read: () => holderRecord({ pid: 4_000_000 }), isAlive: aliveSelf, healthOf: async () => { probed += 1; return { pid: 4_000_000 }; } }); + assert.equal(result.allow, true); + assert.equal(probed, 0); +}); + +test("mayClobberPointer: draining predecessor (alive pid, closed listener) → allowed (#1723 takeover intact)", async () => { + const result = await mayClobberPointer({ ...CANDIDATE, read: () => holderRecord(), isAlive: aliveSelf, healthOf: async () => undefined }); + assert.equal(result.allow, true); +}); + +test("mayClobberPointer: live holder serving the same address → REFUSED with holder identity", async () => { + const result = await mayClobberPointer({ ...CANDIDATE, read: () => holderRecord(), isAlive: aliveSelf, healthOf: healthServing }); + assert.equal(result.allow, false); + assert.deepEqual(result.holder, { pid: process.pid, instanceId: "inst-holder", origin: "http://127.0.0.1:18787" }); +}); + +test("mayClobberPointer: stale record (responder pid ≠ recorded pid) → allowed, overwrite repairs it", async () => { + const result = await mayClobberPointer({ ...CANDIDATE, read: () => holderRecord(), isAlive: aliveSelf, healthOf: async () => ({ pid: FOREIGN_PID }) }); + assert.equal(result.allow, true); +}); + +test("mayClobberPointer: end-to-end through the real proxy-origin file", async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "bili-port-owner-")); + const prev = process.env.XDG_STATE_HOME; + process.env.XDG_STATE_HOME = dir; + try { + atomicWriteInstanceFile(holderRecord()); + const refused = await mayClobberPointer({ ...CANDIDATE, read: readProxyInstanceFile, isAlive: isPidAlive, healthOf: healthServing }); + assert.equal(refused.allow, false); + assert.equal(refused.holder?.instanceId, "inst-holder"); + const takenOver = await mayClobberPointer({ ...CANDIDATE, candidateInstanceId: "inst-holder", read: readProxyInstanceFile, isAlive: isPidAlive, healthOf: healthServing }); + assert.equal(takenOver.allow, true); + } finally { + if (prev === undefined) delete process.env.XDG_STATE_HOME; + else process.env.XDG_STATE_HOME = prev; + await rmrf(dir); + } +}); diff --git a/tests/watchdog-health-daemon.test.ts b/tests/watchdog-health-daemon.test.ts index b1ae2e2195..38806ffb20 100644 --- a/tests/watchdog-health-daemon.test.ts +++ b/tests/watchdog-health-daemon.test.ts @@ -10,6 +10,18 @@ import type { AddressInfo } from "node:net"; import type { ProxyOptions } from "../src/config.ts"; import { rmrf } from "./tmp-rm.ts"; +// #2941: registration now completes AFTER the bind (post-bind ownership +// check), so "listening" no longer implies health-readiness — poll for it. +async function waitReady(port: number): Promise { + for (let i = 0; i < 200; i++) { + try { + if ((await fetch(`http://127.0.0.1:${port}/__bili/health`, { signal: AbortSignal.timeout(500) })).status === 200) return; + } catch {} + await new Promise((r) => setTimeout(r, 50)); + } + throw new Error(`proxy on port ${port} did not become ready`); +} + /** #1322 counterpart of watchdog-health.test.ts: a proxy started WITHOUT * BILI_PARENT_PID is a daemon — its health must say `armed:false` and it must * keep refusing watcher registrations (409) while staying up. Separate file @@ -88,6 +100,7 @@ test("health exposes an unarmed watchdog on a daemon proxy; registration stays r const proxy = await startServer(opts); if (!proxy.listening) await once(proxy, "listening"); const port = (proxy.address() as AddressInfo).port; + await waitReady(port); try { const h = await getJson(port, "/__bili/health"); assert.equal(h.status, 200); diff --git a/tests/watchdog-health.test.ts b/tests/watchdog-health.test.ts index b5df39e444..37d639bb6a 100644 --- a/tests/watchdog-health.test.ts +++ b/tests/watchdog-health.test.ts @@ -10,6 +10,18 @@ import type { AddressInfo } from "node:net"; import type { ProxyOptions } from "../src/config.ts"; import { rmrf } from "./tmp-rm.ts"; +// #2941: registration now completes AFTER the bind (post-bind ownership +// check), so "listening" no longer implies health-readiness — poll for it. +async function waitReady(port: number): Promise { + for (let i = 0; i < 200; i++) { + try { + if ((await fetch(`http://127.0.0.1:${port}/__bili/health`, { signal: AbortSignal.timeout(500) })).status === 200) return; + } catch {} + await new Promise((r) => setTimeout(r, 50)); + } + throw new Error(`proxy on port ${port} did not become ready`); +} + /** #1322: /__bili/health must expose the session-lifecycle watchdog state so * attachers and operators can tell a session-owned proxy (armed — dies with * its sessions) from a daemon squatting a stable port (unarmed — outlives all @@ -91,6 +103,7 @@ mitm: { enabled: false, domains: [] }, const proxy = await startServer(opts); if (!proxy.listening) await once(proxy, "listening"); const port = (proxy.address() as AddressInfo).port; + await waitReady(port); try { const h = await getJson(port, "/__bili/health"); assert.equal(h.status, 200);