diff --git a/src/instance.ts b/src/instance.ts index 3fdeeb25d8..70bdc882dd 100644 --- a/src/instance.ts +++ b/src/instance.ts @@ -248,6 +248,12 @@ export function classifySameLanePeer( for (const e of entries) { if (e.port !== port || e.lane !== lane) continue; if (!isPidAlive(e.pid)) continue; + // #4982: a draining instance stamped portReleasedAt when its listener + // closed — its pid still lives but the socket no longer does, so it + // cannot be the EADDRINUSE holder even though every other identity cell + // matches. This is exactly the stale-twin misclassification that killed + // #4982's CI child while a cross-lane instance owned the freed port. + if (e.portReleasedAt !== undefined) continue; if (e.codeFingerprint !== ownFingerprint) continue; return { kind: sameShape(e) ? "twin" : "drifted", entry: e }; } @@ -482,6 +488,14 @@ export interface RegistryEntry { modelMaxOutputs?: Record; lane?: string; codeFingerprint?: string; + /** #4982: set when this instance released its listening socket (graceful + * shutdown start). The process can stay alive for seconds-to-minutes while + * it drains keep-alive connections and flushes state — during that window + * it CANNOT be the EADDRINUSE holder of `port` anymore, so consumers that + * judge PORT OWNERSHIP (classifySameLanePeer) must skip marked entries, + * while consumers that wait for the process to EXIT (findSameLanePredecessor + * → wait-predecessor) keep counting them. */ + portReleasedAt?: number; } /** Cross-instance liveness registry (#394/#527): one marker file per instance @@ -549,6 +563,7 @@ function coerceEntry(value: unknown): RegistryEntry | undefined { ...(Object.keys(maxOutputs).length > 0 ? { modelMaxOutputs: maxOutputs } : {}), ...(typeof o.lane === "string" && o.lane !== "" ? { lane: o.lane } : {}), ...(typeof o.codeFingerprint === "string" && o.codeFingerprint !== "" ? { codeFingerprint: o.codeFingerprint } : {}), + ...(typeof o.portReleasedAt === "number" && Number.isFinite(o.portReleasedAt) ? { portReleasedAt: o.portReleasedAt } : {}), }; } @@ -704,3 +719,21 @@ export function unregisterInstance(instanceId: string): void { fs.unlinkSync(registryEntryFile(instanceId)); } catch {} } + +/** #4982: withdraw the port-ownership claim the moment the listening socket + * closes (graceful shutdown start). The drain can outlive the release by + * minutes (keep-alive pins, slow flushes), and during that window a stale + * "I hold port N" marker let a same-lane newcomer misclassify us as its twin + * and die with the adopt signal while a cross-lane instance owned the freed + * socket (#4982). Rewrites ONLY our own marker file with the stamp — + * per-instance files mean no cross-process read-modify-write. Best-effort: + * a failed mark degrades to the pre-#4982 behavior, where the health + * verification in the EADDRINUSE path still catches the stale claim. */ +export function markPortReleased(instanceId: string): void { + try { + const file = registryEntryFile(instanceId); + const entry = coerceEntry(safeReadJson(file)); + if (!entry || entry.portReleasedAt !== undefined) return; + atomicWriteJson({ ...entry, portReleasedAt: Date.now() }, file); + } catch {} +} diff --git a/src/port-owner.ts b/src/port-owner.ts index a0e2409def..aa2c4fc34c 100644 --- a/src/port-owner.ts +++ b/src/port-owner.ts @@ -102,6 +102,36 @@ export async function findForeignOwner( return undefined; } +/** #4982: does the listener on `origin` answer health AS the registry entry's + * claimed owner (exact pid + instance identity)? A registry marker is a CLAIM + * written at announce time, not proof: between the claim and our bind the + * socket can move on — the claimed owner enters graceful shutdown (listener + * closed, process still draining) and a different instance binds the freed + * port. Declaring a twin collision on an unverified claim kills the newcomer + * with the adopt signal over a port it could have laddered off (#4982). + * An ANSWERED probe decides immediately (match or not); only a silent or + * non-bili responder is inconclusive and retried within the standard probe + * budget — a saturated-but-alive twin must not be mistaken for a moved + * socket either. */ +export async function verifyEntryOwnsPort( + origin: string, + expected: { pid: number; instanceId: string }, + opts: { attempts?: number; intervalMs?: number; timeoutMs?: number } & ProbeDeps = {}, +): Promise { + const attempts = Math.max(1, opts.attempts ?? DEFAULT_ATTEMPTS); + const intervalMs = opts.intervalMs ?? DEFAULT_INTERVAL_MS; + const timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS; + const fetchImpl = opts.fetchImpl ?? probeFetch; + const sleepImpl = opts.sleepImpl ?? ((ms: number) => new Promise((r) => setTimeout(r, ms))); + for (let i = 0; i < attempts; i++) { + if (i > 0) await sleepImpl(intervalMs); + const info = await probeOwnerInfo(origin, fetchImpl, timeoutMs); + if (info === undefined) continue; // inconclusive — retry + return info.pid === expected.pid && info.instanceId === expected.instanceId; + } + return false; +} + type PortClaimOutcome = | { kind: "claimed" } | { kind: "ladder"; nextPort: number; foreign: OwnerProbeInfo } diff --git a/src/server.ts b/src/server.ts index f29398d857..ccea8cf428 100644 --- a/src/server.ts +++ b/src/server.ts @@ -97,10 +97,10 @@ import { getStore, setGlobalSessionCommitListener } from "./persist.js"; import { log as loggerLog, configureLogger, getLogPath, closeLogger, isStreamWriteError, isBenignSocketRaceError, enterSessionContext, muteStderrLeg, onFileLegLive, logTimestamp } from "./logger.js"; import { queryLogLines } from "./web/logs-query.js"; import { configFile, defaultLogFile, stateDir } from "./paths.js"; -import { atomicWriteInstanceFile, clearProxyInstanceFile, classifySameLanePeer, entryScriptFingerprint, entryShape, findSameLanePredecessor, findVerifiedSameLaneTwin, isPidAlive, listInstances, probePid, readProxyInstanceFile, registerInstanceAndWarn, sameInstanceShape, unregisterInstance, warnOnNewPeers, writeSpawnPortLog, type InstanceShape, type PidProbe, type ProxyInstanceFile, type RegistryEntry, type SameLanePeer } from "./instance.js"; +import { atomicWriteInstanceFile, clearProxyInstanceFile, classifySameLanePeer, entryScriptFingerprint, entryShape, findSameLanePredecessor, findVerifiedSameLaneTwin, isPidAlive, listInstances, markPortReleased, probePid, readProxyInstanceFile, registerInstanceAndWarn, sameInstanceShape, unregisterInstance, warnOnNewPeers, writeSpawnPortLog, type InstanceShape, type PidProbe, type ProxyInstanceFile, type RegistryEntry, type SameLanePeer } from "./instance.js"; import { createServedMarkerWriter, reapStaleWriterMarkers, writerMarkerDir, type ServedMarkerWriter } from "./served-markers.js"; import { recordSplitBrainContest } from "./storage/runtime-status.js"; -import { mayClobberPointer, probeOwnerInfo, resolvePortClaim } from "./port-owner.js"; +import { mayClobberPointer, probeOwnerInfo, resolvePortClaim, verifyEntryOwnsPort } from "./port-owner.js"; import { hoistTrappedToolItems } from "./tool-pair-order.js"; import { computeAnthropicMessageMarks, stampAnthropicSystemCacheControl, anthropicToolsCarryCacheControl } from "./loop/cache-control.js"; import { reconcileSystemAnchor } from "./system-anchor.js"; @@ -1392,72 +1392,113 @@ export async function startServer(opts: ProxyOptions): Promise { // Node treats it as an unhandled 'error' and throws, aborting before the // graceful-shutdown flush can run. Catch, log a human-readable message, // flush sessions, and exit cleanly (exit code 1 so callers/scripts notice). + // #4982: the twin verdict needs an async health probe (verifyEntryOwnsPort), + // so the handler body runs through a promise. At most one listen attempt is + // ever in flight (each retry is scheduled FROM this handler), so two error + // handlers can never interleave. server.on("error", (err: NodeJS.ErrnoException) => { - let busyReason: FailFastReason | undefined; - let peer: SameLanePeer | undefined; - if (isPortBusyError(err.code, lastTriedPort)) { - // #3493: win32 reports a cross-family bind collision (IPv6 wildcard - // holder squats the IPv4 bind) as EACCES on an unprivileged port — - // classify it like a true EADDRINUSE so it ladders instead of dying - // with the misleading privileged-port hint. - peer = classifySameLanePeer(listInstances(), lastTriedPort, launcherLane, ownFingerprint, (e) => sameInstanceShape(entryShape(e), ownShape)); - const action = decideEaddrinuse({ - upgradeReplacement, - twinHolds: peer?.kind === "twin", - launchToken: launchToken !== undefined, - strictPort, - attempts: listenAttempts, - maxAttempts: MAX_LISTEN_ATTEMPTS, - predecessorTicksLeft, - predecessorHolds: sameLanePredecessorHolds(lastTriedPort), - lastTriedPort, - }); - if (action.kind === "wait-predecessor") { - predecessorTicksLeft -= 1; - setTimeout(() => attemptListen(action.port), PREDECESSOR_WAIT_MS); - return; - } - if (action.kind === "ladder") { - listenAttempts += 1; - // #3801: laddering OFF a same-build same-lane holder means it - // drifted (a true twin would have failed fast above) — name - // the coexistence so the dual-instance state in the logs is - // explainable: both run, each serves its own sessions. - const driftNote = peer?.kind === "drifted" - ? ` — held by a same-lane bili instance (pid ${peer.entry.pid}) whose config drifted since it spawned; it keeps serving its own sessions while this one takes the next free port` - : ""; - log("warn", `port ${lastTriedPort} busy${err.code === "EADDRINUSE" ? "" : ` (${err.code})`} — ${action.nextPort === 0 ? "retrying on an ephemeral port" : `retrying on port ${action.nextPort}`}${driftNote} (#3801)`); - attemptListen(action.nextPort); - return; + void handleListenError(err); + }); + const handleListenError = async (err: NodeJS.ErrnoException): Promise => { + try { + let busyReason: FailFastReason | undefined; + let peer: SameLanePeer | undefined; + if (isPortBusyError(err.code, lastTriedPort)) { + // #3493: win32 reports a cross-family bind collision (IPv6 wildcard + // holder squats the IPv4 bind) as EACCES on an unprivileged port — + // classify it like a true EADDRINUSE so it ladders instead of dying + // with the misleading privileged-port hint. + peer = classifySameLanePeer(listInstances(), lastTriedPort, launcherLane, ownFingerprint, (e) => sameInstanceShape(entryShape(e), ownShape)); + // #4982: a registry twin is only a twin while it still OWNS the + // contended socket. The marker is a claim written at announce time; + // by now the claimed owner may have entered graceful shutdown + // (listener closed, process still draining) and a DIFFERENT + // instance may hold the freed port — dying with the adopt signal + // there strands the launch behind a cross-lane stranger we are + // forbidden to attach to (#1225). Verify the claim against the live + // listener first; an unverified claim falls through to the plain + // foreign-holder policy below (ladder or fail-fast). + let twinHolds = false; + if (peer?.kind === "twin") { + const claimed = peer.entry; + twinHolds = await verifyEntryOwnsPort(claimed.origin, { pid: claimed.pid, instanceId: claimed.instanceId }); + if (!twinHolds) { + peer = undefined; + log("warn", `port ${lastTriedPort}: registry twin ${claimed.instanceId.slice(0, 8)} (pid ${claimed.pid}) no longer answers as itself — the socket moved on while it drained; treating the port as foreign-held instead of dying twin-collision (#4982)`); + } + } + // shutdown() may have raced the probe — its own exit path owns + // the outcome from here on. + if (shuttingDown) return; + const action = decideEaddrinuse({ + upgradeReplacement, + twinHolds, + launchToken: launchToken !== undefined, + strictPort, + attempts: listenAttempts, + maxAttempts: MAX_LISTEN_ATTEMPTS, + predecessorTicksLeft, + predecessorHolds: sameLanePredecessorHolds(lastTriedPort), + lastTriedPort, + }); + if (action.kind === "wait-predecessor") { + predecessorTicksLeft -= 1; + setTimeout(() => attemptListen(action.port), PREDECESSOR_WAIT_MS); + return; + } + if (action.kind === "ladder") { + listenAttempts += 1; + // #3801: laddering OFF a same-build same-lane holder means it + // drifted (a true twin would have failed fast above) — name + // the coexistence so the dual-instance state in the logs is + // explainable: both run, each serves its own sessions. + const driftNote = peer?.kind === "drifted" + ? ` — held by a same-lane bili instance (pid ${peer.entry.pid}) whose config drifted since it spawned; it keeps serving its own sessions while this one takes the next free port` + : ""; + log("warn", `port ${lastTriedPort} busy${err.code === "EADDRINUSE" ? "" : ` (${err.code})`} — ${action.nextPort === 0 ? "retrying on an ephemeral port" : `retrying on port ${action.nextPort}`}${driftNote} (#3801)`); + attemptListen(action.nextPort); + return; + } + busyReason = action.reason; } - busyReason = action.reason; + const hint = + isPortBusyError(err.code, lastTriedPort) + ? busyReason === "twin-collision" + ? ` — port ${lastTriedPort} is already served by a same-build bili instance (pid ${peer?.entry.pid}, lane ${launcherLane}) — exiting instead of hopping: the duplicate loses and the serving instance keeps serving; the next launch will attach to it (#3044). A spawner reads this exit as an adopt signal and attaches to the serving instance itself (#3654).` + : busyReason === "upgrade-replacement" + ? ` — this process is an upgrade-restart replacement (#3044): it must own port ${lastTriedPort} exactly, but the prior instance still holds it. Exiting instead of hopping to ${lastTriedPort + 1} — a drifted replacement serves no client and leaks while the prior instance keeps serving; the upgrade retries on the next cooldown cycle.` + : busyReason === "strict-port" + ? ` — port ${lastTriedPort} is pinned (strict-port mode) but already in use. Free it or point the client at another port (e.g. BILI_CLAUDE_NATIVE_PORT for the claude native posture).` + : ` — port ${lastTriedPort} is already in use. Stop the other process or use --port .` + : err.code === "EACCES" + ? ` — port ${lastTriedPort} requires privileges. Use a port >= 1024.` + : ""; + const listenLine = `listen failed: ${err.code ?? ""} ${err.message}${hint}`; + log("error", listenLine); + // #3654/#3116: a spawner's failure message points users at the per-port + // log, but post-#2864 this child mutes its stderr leg into bili.log — + // without this line that file stays 0 bytes and the diagnosis dies there. + if (launchToken !== undefined) writeSpawnPortLog(opts.port, `${logTimestamp()} [error] [v=${VERSION}] ${listenLine}`); + shuttingDown = true; + server.close(); + flushConversations(); + void flushAllSessions().finally(() => { + closeLogger(); + process.exit(eaddrinuseExitCode(busyReason)); + }); + } catch (e) { + // The listen already failed; a crash here must still end the process + // cleanly instead of leaving a dead listener running. + log("error", `listen-error handling crashed: ${String(e)}`); + shuttingDown = true; + server.close(); + flushConversations(); + void flushAllSessions().finally(() => { + closeLogger(); + process.exit(1); + }); } - const hint = - isPortBusyError(err.code, lastTriedPort) - ? busyReason === "twin-collision" - ? ` — port ${lastTriedPort} is already served by a same-build bili instance (pid ${peer?.entry.pid}, lane ${launcherLane}) — exiting instead of hopping: the duplicate loses and the serving instance keeps serving; the next launch will attach to it (#3044). A spawner reads this exit as an adopt signal and attaches to the serving instance itself (#3654).` - : busyReason === "upgrade-replacement" - ? ` — this process is an upgrade-restart replacement (#3044): it must own port ${lastTriedPort} exactly, but the prior instance still holds it. Exiting instead of hopping to ${lastTriedPort + 1} — a drifted replacement serves no client and leaks while the prior instance keeps serving; the upgrade retries on the next cooldown cycle.` - : busyReason === "strict-port" - ? ` — port ${lastTriedPort} is pinned (strict-port mode) but already in use. Free it or point the client at another port (e.g. BILI_CLAUDE_NATIVE_PORT for the claude native posture).` - : ` — port ${lastTriedPort} is already in use. Stop the other process or use --port .` - : err.code === "EACCES" - ? ` — port ${lastTriedPort} requires privileges. Use a port >= 1024.` - : ""; - const listenLine = `listen failed: ${err.code ?? ""} ${err.message}${hint}`; - log("error", listenLine); - // #3654/#3116: a spawner's failure message points users at the per-port - // log, but post-#2864 this child mutes its stderr leg into bili.log — - // without this line that file stays 0 bytes and the diagnosis dies there. - if (launchToken !== undefined) writeSpawnPortLog(opts.port, `${logTimestamp()} [error] [v=${VERSION}] ${listenLine}`); - shuttingDown = true; - server.close(); - flushConversations(); - void flushAllSessions().finally(() => { - closeLogger(); - process.exit(eaddrinuseExitCode(busyReason)); - }); - }); + }; // Catch stray rejections/throws from background work (compress loops, // auto-update, initSessions) that escape the per-request try/catch — // Node 20+ aborts the process on these by default. Log loudly and flush. @@ -1521,6 +1562,11 @@ export async function startServer(opts: ProxyOptions): Promise { server.close(() => { finishDrain(); }); + // #4982: the listener just stopped accepting — withdraw the port + // ownership claim NOW instead of at process exit. The drain can outlive + // the release by minutes (keep-alive pins), and a stale claim let a + // same-lane newcomer misclassify us as its twin and die (#4982). + markPortReleased(instanceId); // Hard fallback: if connections hang (client never closes), don't // block shutdown forever — force-exit after a grace window. #3890: // while connections are still ALIVE the window re-arms (bounded by diff --git a/tests/issue4982-draining-twin-ladder.test.ts b/tests/issue4982-draining-twin-ladder.test.ts new file mode 100644 index 0000000000..b1bc2db71b --- /dev/null +++ b/tests/issue4982-draining-twin-ladder.test.ts @@ -0,0 +1,440 @@ +// #4982: a same-lane proxy entering graceful shutdown releases its listening +// socket IMMEDIATELY (server.close) but keeps its registry marker until the +// drain/flush finishes (seconds-to-minutes later, pinned by keep-alive +// connections). During that window the marker still claims "I hold port N" +// while a DIFFERENT instance can bind the freed port. A same-lane newcomer +// hitting EADDRINUSE there misclassified the draining predecessor as its +// twin and died with the adopt signal (exit 3) — stranding its launcher +// behind a cross-lane stranger it is forbidden to attach to (#1225). CI hit +// this in the #3203 concurrent-sessions e2e (release v0.1.209, PR #4982). +// +// Two complementary fixes, each pinned here: +// B' (root-cause hygiene): shutdown() stamps portReleasedAt on its own +// marker the moment the listener closes; classifySameLanePeer skips +// stamped entries, so a draining instance can never be mistaken for the +// port holder again (integration test A exercises this end to end). +// A (defense in depth): the late EADDRINUSE path verifies the claimed +// twin's socket ownership via /__bili/health (pid + instance identity, +// the #4341 criterion) before declaring twin-collision; an unverified +// claim falls through to the plain foreign-holder policy and ladders +// (integration test B exercises this with an UNSTAMPED marker, i.e. a +// pre-fix build or a failed stamp). + +import test from "node:test"; +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { spawn, type ChildProcess } from "node:child_process"; +import fs from "node:fs"; +import net from "node:net"; +import os from "node:os"; +import path from "node:path"; +import type { AddressInfo } from "node:net"; +import { classifySameLanePeer, markPortReleased, type InstanceShape, type RegistryEntry } from "../src/instance.ts"; +import { verifyEntryOwnsPort } from "../src/port-owner.ts"; +import { assertPortDead } from "./port-race.ts"; + +const ROOT = path.join(import.meta.dirname, ".."); +const ENTRY = path.join(ROOT, "src/index.ts"); +const BASE = 18861; +const OWN_FP = "fp-our-build"; + +function ownShape(over: Partial = {}): InstanceShape { + return { + host: "127.0.0.1", + passthrough: false, + mitmDomains: [], + modelWindows: {}, + modelMaxOutputs: {}, + ...over, + }; +} + +function holder(over: Partial = {}): RegistryEntry { + return { + instanceId: "holder-4982", + pid: process.pid, + port: BASE, + origin: `http://127.0.0.1:${BASE}`, + startedAt: Date.now(), + host: "127.0.0.1", + passthrough: false, + mitmDomains: [], + modelWindows: {}, + modelMaxOutputs: {}, + lane: "pi", + codeFingerprint: OWN_FP, + ...over, + }; +} + +function sameShape(own: InstanceShape): (entry: RegistryEntry) => boolean { + return (entry) => + entry.host === own.host && + Boolean(entry.passthrough) === own.passthrough && + JSON.stringify(entry.mitmDomains ?? []) === JSON.stringify(own.mitmDomains) && + JSON.stringify(entry.modelWindows ?? {}) === JSON.stringify(own.modelWindows) && + JSON.stringify(entry.modelMaxOutputs ?? {}) === JSON.stringify(own.modelMaxOutputs); +} + +test("classifySameLanePeer: a draining marker (portReleasedAt) is never a twin or drifted peer", () => { + const live = holder(); + const drained = holder({ instanceId: "drained-4982", portReleasedAt: Date.now() }); + const shape = sameShape(ownShape()); + assert.equal(classifySameLanePeer([live], BASE, "pi", OWN_FP, shape)?.kind, "twin", "control: an unstamped live marker still classifies"); + assert.equal(classifySameLanePeer([drained], BASE, "pi", OWN_FP, shape), undefined, "a stamped marker must be skipped even with a live pid and full identity match"); + assert.equal(classifySameLanePeer([drained, live], BASE, "pi", OWN_FP, shape)?.entry.instanceId, live.instanceId, "an unstamped twin behind a drained one still classifies"); +}); + +test("markPortReleased: stamps only its own marker, idempotent, missing-file no-op", () => { + const sandbox = fs.mkdtempSync(path.join(os.tmpdir(), "bili-4982-mark-")); + const prevState = process.env.XDG_STATE_HOME; + try { + process.env.XDG_STATE_HOME = path.join(sandbox, "state"); + const regDir = path.join(process.env.XDG_STATE_HOME as string, "billion-context", "instances"); + fs.mkdirSync(regDir, { recursive: true }); + const base = { + startedAt: Date.now(), + host: "127.0.0.1", + passthrough: false, + mitmDomains: [], + modelWindows: {}, + modelMaxOutputs: {}, + lane: "pi", + codeFingerprint: OWN_FP, + }; + fs.writeFileSync(path.join(regDir, "aaa.json"), JSON.stringify({ instanceId: "aaa", pid: 1, port: 1, origin: "http://127.0.0.1:1", ...base }) + "\n"); + fs.writeFileSync(path.join(regDir, "bbb.json"), JSON.stringify({ instanceId: "bbb", pid: 2, port: 2, origin: "http://127.0.0.1:2", ...base }) + "\n"); + + markPortReleased("aaa"); + const a = JSON.parse(fs.readFileSync(path.join(regDir, "aaa.json"), "utf8")); + const b = JSON.parse(fs.readFileSync(path.join(regDir, "bbb.json"), "utf8")); + assert.equal(typeof a.portReleasedAt, "number", "own marker gets the stamp"); + assert.equal(b.portReleasedAt, undefined, "other instances' markers must stay untouched"); + + markPortReleased("aaa"); + const a2 = JSON.parse(fs.readFileSync(path.join(regDir, "aaa.json"), "utf8")); + assert.equal(a2.portReleasedAt, a.portReleasedAt, "idempotent — re-stamping must not move the timestamp"); + + markPortReleased("missing"); // no-op, must not throw + } finally { + if (prevState === undefined) delete process.env.XDG_STATE_HOME; + else process.env.XDG_STATE_HOME = prevState; + fs.rmSync(sandbox, { recursive: true, force: true }); + } +}); + +test("verifyEntryOwnsPort: an answered probe decides immediately (match or mismatch)", async () => { + let calls = 0; + const fetchImpl = async () => { + calls += 1; + return { ok: true, json: async () => ({ ok: true, pid: 4242, instanceId: "inst-a" }) }; + }; + const sleepImpl = async () => { throw new Error("an answered probe must not sleep/retry"); }; + assert.equal(await verifyEntryOwnsPort("http://127.0.0.1:1", { pid: 4242, instanceId: "inst-a" }, { fetchImpl, sleepImpl }), true); + assert.equal(calls, 1); + assert.equal(await verifyEntryOwnsPort("http://127.0.0.1:1", { pid: 9999, instanceId: "inst-a" }, { fetchImpl, sleepImpl }), false, "a pid mismatch decides immediately"); + assert.equal(calls, 2); + assert.equal(await verifyEntryOwnsPort("http://127.0.0.1:1", { pid: 4242, instanceId: "other" }, { fetchImpl, sleepImpl }), false, "an instance mismatch decides immediately"); + assert.equal(calls, 3); +}); + +test("verifyEntryOwnsPort: only a silent responder is retried within the probe budget", async () => { + let calls = 0; + let sleeps = 0; + const sleepImpl = async () => { sleeps += 1; }; + const silent = async () => { + calls += 1; + throw new Error("probe timeout"); + }; + assert.equal(await verifyEntryOwnsPort("http://127.0.0.1:1", { pid: 1, instanceId: "i" }, { fetchImpl: silent, sleepImpl, attempts: 3 }), false); + assert.equal(calls, 3, "silent responder exhausts the attempt budget"); + assert.equal(sleeps, 2, "sleeps between attempts only"); + + calls = 0; + sleeps = 0; + const thenMatch = async () => { + calls += 1; + if (calls === 1) throw new Error("probe timeout"); + return { ok: true, json: async () => ({ ok: true, pid: 1, instanceId: "i" }) }; + }; + assert.equal(await verifyEntryOwnsPort("http://127.0.0.1:1", { pid: 1, instanceId: "i" }, { fetchImpl: thenMatch, sleepImpl }), true, "a late answer within budget verifies"); + assert.equal(calls, 2); + assert.equal(sleeps, 1); +}); + +// --------------------------------------------------------------------------- +// Integration helpers (conventions from issue4341-early-twin-adoption.test.ts) +// --------------------------------------------------------------------------- + +function sandboxDirs(sandbox: string): { stateHome: string; dataHome: string; configHome: string; cacheHome: string; sessions: string } { + return { + stateHome: path.join(sandbox, "state"), + dataHome: path.join(sandbox, "data"), + configHome: path.join(sandbox, "config"), + cacheHome: path.join(sandbox, "cache"), + sessions: path.join(sandbox, "data", "sessions"), + }; +} + +function makeEnv(sandbox: string): NodeJS.ProcessEnv { + const dirs = sandboxDirs(sandbox); + const env: NodeJS.ProcessEnv = { + ...process.env, + NODE_ENV: "test", + HOME: sandbox, + XDG_STATE_HOME: dirs.stateHome, + XDG_DATA_HOME: dirs.dataHome, + XDG_CONFIG_HOME: dirs.configHome, + XDG_CACHE_HOME: dirs.cacheHome, + BILI_SESSIONS_DIR: dirs.sessions, + BILI_LAUNCH_TOKEN: "issue-4982-test", + BILI_LAUNCHER_LANE: "pi", + BILI_PARENT_PID: String(process.pid), + BILI_STORAGE_UNIFIED: "full", + BILI_MITM: "0", + ACP_PASSTHROUGH: "0", + BILI_LAUNCHER_MODEL_WINDOWS: "{}", + BILI_LAUNCHER_MODEL_MAX_OUTPUTS: "{}", + }; + delete env.NODE_TEST_CONTEXT; + delete env.BILI_ENCRYPTION_KEY; + return env; +} + +async function freePort(): Promise { + const server = net.createServer(); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", resolve); + }); + const port = (server.address() as AddressInfo).port; + await new Promise((resolve) => server.close(() => resolve())); + return port; +} + +async function waitForStatus(port: number, timeoutMs: number): Promise { + const deadline = Date.now() + timeoutMs; + for (;;) { + try { + const res = await fetch(`http://127.0.0.1:${port}/__bili/status`); + if (res.status === 200) return true; + } catch { + // connection refused while the child is still starting up — expected + } + if (Date.now() >= deadline) return false; + await new Promise((resolve) => setTimeout(resolve, 100)); + } +} + +async function killChild(child: ChildProcess): Promise { + if (child.exitCode !== null || child.signalCode !== null) return; + child.kill("SIGTERM"); + await Promise.race([ + new Promise((resolve) => child.once("exit", () => resolve())), + new Promise((resolve) => setTimeout(resolve, 10_000)), + ]); + if (child.exitCode === null && child.signalCode === null) child.kill("SIGKILL"); +} + +/** A TCP listener that accepts connections and then says nothing — invisible + * to /__bili/health probes (they time out as "no evidence") but still owns + * the socket, so binders get EADDRINUSE. Deterministically forces the LATE + * EADDRINUSE decision path instead of the pre-bind foreign-owner ladder. */ +function silentTcpListener(port: number): Promise { + return new Promise((resolve, reject) => { + const server = net.createServer(() => { + /* accept and hang — never answer HTTP */ + }); + server.once("error", reject); + server.listen(port, "127.0.0.1", () => resolve(server)); + }); +} + +function tcpRefused(port: number): Promise { + return new Promise((resolve) => { + const s = net.connect({ port, host: "127.0.0.1" }); + let done = false; + const finish = (v: boolean): void => { + if (done) return; + done = true; + s.destroy(); + resolve(v); + }; + s.once("connect", () => finish(false)); + s.once("error", () => finish(true)); + setTimeout(() => finish(false), 2_000).unref(); + }); +} + +async function waitUntil(cond: () => boolean | Promise, timeoutMs: number, label: string): Promise { + const deadline = Date.now() + timeoutMs; + for (;;) { + if (await cond()) return; + if (Date.now() >= deadline) throw new Error(`timed out waiting for ${label}`); + await new Promise((resolve) => setTimeout(resolve, 100)); + } +} + +function readMarkers(stateHome: string): Array> { + const dir = path.join(stateHome, "billion-context", "instances"); + let names: string[]; + try { + names = fs.readdirSync(dir); + } catch { + return []; + } + const out: Array> = []; + for (const name of names) { + if (!name.endsWith(".json")) continue; + try { + out.push(JSON.parse(fs.readFileSync(path.join(dir, name), "utf8"))); + } catch { + // torn write in flight — skip + } + } + return out; +} + +function spawnBili(sandbox: string, port: number): { child: ChildProcess; out: { value: string } } { + const child = spawn(process.execPath, ["--import", "tsx", ENTRY, "start", "--host", "127.0.0.1", "--port", String(port)], { + cwd: ROOT, + env: makeEnv(sandbox), + stdio: ["ignore", "pipe", "pipe"], + }); + const out = { value: "" }; + child.stdout.setEncoding("utf8"); + child.stderr.setEncoding("utf8"); + child.stdout.on("data", (chunk: string) => { out.value += chunk; }); + child.stderr.on("data", (chunk: string) => { out.value += chunk; }); + return { child, out }; +} + +/** Wait for the child's own registry marker, then for it to serve health on + * the port IT registered (laddering may have moved it off the requested one). */ +async function waitUntilLadderedHealthy(stateHome: string, child: ChildProcess, label: string): Promise { + let port = 0; + await waitUntil(async () => { + const m = readMarkers(stateHome).find((e) => e.pid === child.pid); + if (!m || typeof m.port !== "number") return false; + port = m.port as number; + return waitForStatus(port, 5_000); + }, 90_000, `${label} to register and become healthy`); + return port; +} + +test("#4982 e2e: draining same-lane predecessor + squatter → newcomer ladders instead of dying", { timeout: 180_000 }, async () => { + const sandbox = fs.mkdtempSync(path.join(os.tmpdir(), "bili-4982a-")); + const { stateHome } = sandboxDirs(sandbox); + const port = await freePort(); + await assertPortDead(port, { label: "4982 drain-test port" }); + try { + // P1: the serving proxy that will enter graceful shutdown. + const p1 = spawnBili(sandbox, port); + if (!(await waitForStatus(port, 90_000))) throw new Error(`P1 never became ready:\n${p1.out.value}`); + const p1Marker = readMarkers(stateHome).find((m) => m.pid === p1.child.pid); + assert.ok(p1Marker, "P1 must have registered itself"); + + // Pin P1's drain with a held connection so it stays alive (listener + // closed, process + marker present) long enough to run the race. + const held = await new Promise((resolve, reject) => { + const s = net.connect(port, "127.0.0.1"); + s.once("connect", () => resolve(s)); + s.once("error", reject); + }); + + p1.child.kill("SIGTERM"); + const p1MarkerFile = path.join(stateHome, "billion-context", "instances", `${p1Marker.instanceId}.json`); + await waitUntil(async () => { + let stamped = false; + try { + stamped = (JSON.parse(fs.readFileSync(p1MarkerFile, "utf8")).portReleasedAt as unknown) !== undefined; + } catch { + return false; + } + return stamped && (await tcpRefused(port)); + }, 20_000, "P1 to stamp portReleasedAt and release the socket"); + + // The squatter takes the freed port BEFORE the newcomer starts — but + // silently, so the newcomer's pre-bind claim probe sees no evidence. + const squatter = await silentTcpListener(port); + + // P2: same lane, same requested port — the CI victim. + const p2 = spawnBili(sandbox, port); + const result = await Promise.race([ + new Promise<"exited">((resolve) => p2.child.once("exit", () => resolve("exited"))), + (async () => { + const servedPort = await waitUntilLadderedHealthy(stateHome, p2.child, "P2"); + return { kind: "healthy" as const, servedPort }; + })(), + ]); + if (typeof result === "string") { + throw new Error(`P2 exited before becoming healthy (the #4982 failure mode):\n${p2.out.value}`); + } + assert.notEqual(result.servedPort, port, "P2 must ladder off the contended port"); + const p2Marker = readMarkers(stateHome).find((m) => m.pid === p2.child.pid); + assert.equal(p2Marker?.lane, "pi"); + assert.equal(p2Marker?.port, result.servedPort); + assert.doesNotMatch(p2.out.value, /twin-collision/, "the stale claim must not produce a twin-collision verdict"); + + await killChild(p2.child); + held.destroy(); + await killChild(p1.child); + await new Promise((resolve) => squatter.close(() => resolve())); + } finally { + fs.rmSync(sandbox, { recursive: true, force: true }); + } +}); + +test("#4982 e2e: unverified registry twin claim (unstamped marker) → ladder, not adopt-death", { timeout: 180_000 }, async () => { + const sandbox = fs.mkdtempSync(path.join(os.tmpdir(), "bili-4982b-")); + const { stateHome } = sandboxDirs(sandbox); + const port = await freePort(); + await assertPortDead(port, { label: "4982 unverified-twin port" }); + try { + // A registry marker claiming `port` with a LIVE pid and full identity + // match — but NO portReleasedAt stamp (pre-fix build, or a failed + // stamp), and the socket is actually owned by a silent squatter. + const registryDir = path.join(stateHome, "billion-context", "instances"); + fs.mkdirSync(registryDir, { recursive: true }); + const fingerprint = createHash("sha256").update(fs.readFileSync(ENTRY)).digest("hex"); + fs.writeFileSync( + path.join(registryDir, "stale-4982.json"), + JSON.stringify({ + instanceId: "stale-4982", + pid: process.pid, + port, + origin: `http://127.0.0.1:${port}`, + startedAt: Date.now(), + host: "127.0.0.1", + passthrough: false, + mitmDomains: [], + modelWindows: {}, + modelMaxOutputs: {}, + lane: "pi", + codeFingerprint: fingerprint, + }) + "\n", + ); + const squatter = await silentTcpListener(port); + + const p2 = spawnBili(sandbox, port); + const result = await Promise.race([ + new Promise<"exited">((resolve) => p2.child.once("exit", () => resolve("exited"))), + (async () => { + const servedPort = await waitUntilLadderedHealthy(stateHome, p2.child, "P2"); + return { kind: "healthy" as const, servedPort }; + })(), + ]); + if (typeof result === "string") { + throw new Error(`P2 exited before becoming healthy (the #4982 failure mode):\n${p2.out.value}`); + } + assert.notEqual(result.servedPort, port, "P2 must ladder off the contended port"); + const logText = fs.readFileSync(path.join(stateHome, "billion-context", "bili.log"), "utf8"); + assert.match(logText, /no longer answers as itself/, "the demotion from twin to foreign-held must be logged"); + assert.doesNotMatch(logText, /twin-collision/, "an unverified claim must never reach the twin-collision verdict"); + + await killChild(p2.child); + await new Promise((resolve) => squatter.close(() => resolve())); + } finally { + fs.rmSync(sandbox, { recursive: true, force: true }); + } +});