From 33bec442d95e93808a87a07584820a520c69109e Mon Sep 17 00:00:00 2001 From: ci-check Date: Mon, 9 Nov 2026 00:46:58 +0800 Subject: [PATCH 1/3] fix(launcher): force-takeover the wedged same-lane proxy holding the canonical port (#5955) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The attach/rotation decision table had one uncovered cell: a same-lane resident on the canonical port running DIFFERENT bytes whose health probe never answers (BUSY: TCP connect succeeds while the event loop stays inside long sync slices, e.g. a multi-minute fold on an oversized session). Attach vetoes foreign bytes (#1225), #4125's upward-adoption veto-lift needs an ANSWERED version a BUSY holder cannot give, #3544 rotation excludes version-unverifiable residents by design (#4811), and the spawned child waits 5s behind a same-lane predecessor then ladders away (+1 sticky drift) or dies. No actor ever resolved the holder, so the port stayed wedged until the process died on its own — which the event-loop watchdog only guarantees for >=5min CONTINUOUS blocks; the saturation class never trips it. Oversized sessions then 422'd indefinitely on the stuck lane. Resolution: bounded patience across bring-up cycles, then a supervised forced takeover. A persisted observation ledger (state dir, atomic write, keyed origin#pid#birth-stamp so a REUSED pid starts its own window instead of inheriting the dead occupant's history) accumulates sightings across client restarts. After WEDGED_TAKEOVER_WINDOW_MS (10min = 2x the watchdog hard timeout) of continuous observations across >=3 distinct cycles, the launcher requests a graceful shutdown, terminates if the listener survives, hard-kills as last resort, verifies the listener closed, gives kernel socket state a settle after an abrupt kill, and continues to the normal spawn. Any doubt falls through to the untouched spawn ladder — never worse than today. Adjacent surfaces deliberately unchanged: #3544 rotation (healthy older holders), #4125 upward adoption (healthy newer holders), #4811 same-lane attach (matching bytes), user-zone and foreign-lane holders all keep their existing paths. killAbandonedChild keeps its exact behavior (group kill + handle-level SIGKILL) via the new shared hard-kill helper. --- src/instance.ts | 5 +- src/launcher.ts | 456 +++++++++++++-- ...ssue5955-wedged-same-lane-takeover.test.ts | 540 ++++++++++++++++++ 3 files changed, 967 insertions(+), 34 deletions(-) create mode 100644 tests/issue5955-wedged-same-lane-takeover.test.ts diff --git a/src/instance.ts b/src/instance.ts index 10b874953a..7fa0dd5749 100644 --- a/src/instance.ts +++ b/src/instance.ts @@ -362,8 +362,9 @@ export function resolveOwnFingerprint(spawnedValue: string | undefined, scriptPa /** tmp+fsync+rename (same shape as web/api.ts atomicWriteConfig) — a torn * write must never leave a half-file behind (#406 family). Shared by the - * proxy-origin record and every registry marker. */ -function atomicWriteJson(obj: unknown, filePath: string): void { + * proxy-origin record, every registry marker, and the #5955 wedged-holder + * ledger. */ +export function atomicWriteJson(obj: unknown, filePath: string): void { fs.mkdirSync(path.dirname(filePath), { recursive: true }); const tempPath = `${filePath}.${process.pid}.${randomUUID()}.tmp`; let descriptor: number | undefined; diff --git a/src/launcher.ts b/src/launcher.ts index c287017b5b..333012a3f3 100644 --- a/src/launcher.ts +++ b/src/launcher.ts @@ -36,11 +36,12 @@ import path from "node:path"; import { createRequire } from "node:module"; import { Worker } from "node:worker_threads"; import { pathToFileURL } from "node:url"; -import { execFileSync, spawn, spawnSync, type StdioOptions } from "node:child_process"; +import { execFile, execFileSync, spawn, spawnSync, type StdioOptions } from "node:child_process"; import { DEFAULT_MITM_DOMAINS } from "./mitm.js"; import { isPrivateUpstreamHost } from "./private-host.js"; import { redactProxyUrl } from "./upstream-fail.js"; import { + atomicWriteJson, claimStartingMarker, clearStartingMarker, discoverLiveInstances, @@ -56,6 +57,7 @@ import { type ProxyInstanceFile, type ProxyStartingMarker, } from "./instance.js"; +import { stateDir } from "./paths.js"; import { TWIN_ADOPT_EXIT_CODE } from "./listen-policy.js"; import { tcpConnectable } from "./liveness.js"; import { selfPackageRoot, isBiliPiEntry, ompPluginLoadedFrom, dshNativeInstalled, claudeNativeInstalled, piPackageStrings } from "./plugin-install.js"; @@ -351,6 +353,21 @@ export interface LauncherDeps { /** #3654: registry view behind serving-twin adoption after exit code 3 * (default: discoverLiveInstances, instance.ts). */ listAdoptCandidates?: () => ProxyInstanceFile[]; + /** #5955: graceful termination REQUEST for an escalated wedged holder — + * POSIX SIGTERM (a saturated-but-yielding loop can still drain); win32 + * has no graceful node signal, so the default no-ops and the caller goes + * straight to the hard kill. Tests inject a recorder. */ + signalWedgedHolder?: (pid: number) => void; + /** #5955: unconditional hard-kill of an escalated wedged holder (POSIX + * SIGKILL / win32 taskkill /F /T tree-kill, mirroring killAbandonedChild). + * Tests inject a recorder so no real pid is ever touched. */ + killWedgedHolder?: (pid: number) => void; + /** #5955: where the wedged-holder observation ledger persists (state dir by + * default via wedgedLedgerFilePath(); tests point it at a scratch file). */ + wedgedLedgerPath?: () => string; + /** #5955: process birth-stamp lookup for PID-reuse detection in the ledger + * key (default: processBirthStamp — /proc on linux, ps/powershell elsewhere). */ + processBirthStamp?: (pid: number) => Promise; } export function isLaunchClient(value: string): value is ClientName { @@ -4958,6 +4975,314 @@ async function rotateStalePredecessor( } } +// #5955: the decision table had one uncovered cell — a SAME-LANE resident on +// our canonical port running DIFFERENT bytes whose health probe never answers +// (BUSY: TCP connect succeeds while the event loop stays inside long sync +// slices, e.g. a multi-minute fold on an oversized session). Every existing +// surface skips such a holder: attach vetoes foreign bytes (#1225), #4125's +// upward-adoption veto-lift needs an ANSWERED version a BUSY holder cannot +// give, #3544 rotation excludes version-unverifiable residents by design +// (#4811), and the spawned child waits 5s behind a same-lane predecessor then +// ladders away (+1 sticky drift, #4568 family) or dies. No actor ever +// resolved the holder, so the port stayed wedged until the process died on its +// own — which the event-loop watchdog only guarantees for ≥5min CONTINUOUS +// blocks; the saturation class never trips it, so the lane lost its proxy +// indefinitely. Resolution = bounded patience across bring-up cycles, then a +// supervised forced takeover: ask for a graceful shutdown (insurance for a +// loop that briefly unblocks — the listener closes on accept), terminate if +// the listener survives, verify the port released, continue to the normal +// spawn. The victim loses at most its in-flight turn (sessions persist +// per-turn); the window bounds worst-case unavailability from ∞ to +// WEDGED_TAKEOVER_WINDOW_MS instead of risking an immediate kill of a +// slow-but-alive fold. +export const WEDGED_TAKEOVER_WINDOW_MS = 600_000; // 10 min = 2× WATCHDOG_HARD_TIMEOUT_MS (event-loop-watchdog.ts) +export const WEDGED_MIN_OBSERVATIONS = 3; // distinct bring-up cycles, not milliseconds — retry cadence varies per host +export const WEDGED_OBSERVATION_STALE_MS = 300_000; // gap > this ⇒ the consecutive run was interrupted ⇒ the window restarts +const WEDGED_TERM_GRACE_MS = 2_000; // SIGTERM drain budget before the hard kill (POSIX only) +const WEDGED_RELEASE_WAIT_MS = 5_000; // per-stage budget to observe the listener close +const WEDGED_POST_KILL_SETTLE_MS = 2_000; // grace for kernel socket release after an ABRUPT kill before we bind (#5955) + +interface WedgedLedgerEntry { + pid: number; + firstBusyAt: number; + lastObservedAt: number; + observations: number; +} + +/** #5955: the observation ledger is PERSISTED in the state dir — during a + * wedge the client process restarts on every bring-up attempt, so an + * in-process counter would reset every cycle and the patience window could + * never fill (reporter evidence: `bootstrap failed` repeating each round). + * Keyed by origin#pid#birth-stamp, so a new occupant — including a REUSED pid + * on Windows, where dense kill/restart recycles pids fast — starts its own + * window instead of inheriting the dead occupant's history. Concurrent hosts + * sharing one file can race on read-modify-write; the worst case loses one + * observation (the window lengthens by one cycle), which is safe. */ +export function wedgedLedgerFilePath(): string { + return path.join(stateDir(), "wedged-takeover.json"); +} + +function isWedgedLedgerEntry(value: unknown): value is WedgedLedgerEntry { + if (typeof value !== "object" || value === null) return false; + const e = value as Record; + return typeof e.pid === "number" && Number.isInteger(e.pid) && e.pid > 0 + && typeof e.firstBusyAt === "number" && Number.isFinite(e.firstBusyAt) + && typeof e.lastObservedAt === "number" && Number.isFinite(e.lastObservedAt) + && typeof e.observations === "number" && Number.isInteger(e.observations) && e.observations >= 1; +} + +/** Load the persisted ledger. Missing file = empty; a corrupt file or + * malformed entry is dropped LOUDLY — a broken state file must never block + * bring-up (the patience window simply restarts from zero). */ +function readWedgedLedger(file: string): Record { + let raw: string; + try { + raw = fs.readFileSync(file, "utf8"); + } catch (error) { + if ((error as NodeJS.ErrnoException)?.code !== "ENOENT") { + teeLog("warn", `bili: cannot read wedged-takeover ledger at ${file}: ${String(error)} — starting fresh, patience window restarts (#5955)`); + } + return {}; + } + const entries: Record = {}; + let unparseable = false; + let dropped = 0; + try { + const parsed: unknown = JSON.parse(raw); + if (typeof parsed !== "object" || parsed === null || !("entries" in parsed)) throw new Error("missing entries"); + for (const [key, value] of Object.entries((parsed as { entries: Record }).entries)) { + if (isWedgedLedgerEntry(value)) entries[key] = value; + else dropped += 1; + } + } catch { + unparseable = true; + } + if (unparseable || dropped > 0) { + teeLog("warn", `bili: wedged-takeover ledger at ${file} is ${unparseable ? "unparseable" : `malformed (${dropped} bad entr${dropped === 1 ? "y" : "ies"})`} — starting fresh, patience window restarts (#5955)`); + } + return entries; +} + +/** Persist the ledger. A failure here must NOT block bring-up: this cycle's + * observation is lost and the window may take one more cycle. */ +function writeWedgedLedger(file: string, entries: Record): void { + try { + atomicWriteJson({ entries }, file); + } catch (error) { + teeLog("warn", `bili: failed to persist wedged-takeover ledger at ${file}: ${String(error)} — observation not saved, patience window may extend by one cycle (#5955)`); + } +} + +function runCommandCapture(command: string, args: readonly string[], timeoutMs: number): Promise { + return new Promise((resolve, reject) => { + execFile(command, [...args], { timeout: timeoutMs, windowsHide: true }, (error, stdout) => { + if (error !== null) reject(error instanceof Error ? error : new Error(String(error))); + else resolve(String(stdout)); + }); + }); +} + +/** #5955: process birth stamp for the ledger key — PID REUSE guard. Windows + * recycles pids quickly after dense kill/restart; without a birth stamp a + * reused pid would inherit the dead occupant's observation history and get + * force-taken-over on first sight. Sources (all best-effort, zero new deps): + * linux: /proc//stat field 22 (starttime, ticks since boot) + * win32: powershell Get-Process StartTime (~1-3s cold start, rare path only) + * other POSIX: `ps -o lstart=` + * Failure → undefined → the key degrades to origin#pid (previous behavior); a + * stamp that appears or disappears across observations restarts the window + * once — conservative (longer patience, never a premature kill). */ +export async function processBirthStamp(pid: number): Promise { + if (!Number.isInteger(pid) || pid <= 0) return undefined; + try { + if (process.platform === "linux") { + const raw = fs.readFileSync(`/proc/${pid}/stat`, "utf8"); + const closeParen = raw.lastIndexOf(")"); + if (closeParen < 0) return undefined; + // Fields after comm start at field 3 (state); starttime is field 22 → index 19. + const fields = raw.slice(closeParen + 1).trim().split(/\s+/); + const starttime = fields[19]; + if (starttime !== undefined && /^\d+$/.test(starttime)) return `boot-${starttime}`; + return undefined; + } + if (process.platform === "win32") { + const out = await runCommandCapture("powershell", ["-NoProfile", "-NonInteractive", "-Command", `(Get-Process -Id ${pid} -ErrorAction Stop).StartTime.ToString('o')`], 10_000); + const trimmed = out.trim(); + return trimmed === "" ? undefined : `ps-${trimmed}`; + } + const out = await runCommandCapture("ps", ["-o", "lstart=", "-p", String(pid)], 2_000); + const trimmed = out.trim(); + return trimmed === "" ? undefined : `ps-${trimmed}`; + } catch { + return undefined; + } +} + +/** #5955: pure cell check — is this probed candidate exactly the unresolved + * deadlock cell? Same declared lane, our target port, foreign bytes + * (fingerprint mismatch BEFORE any shape verdict), and BUSY (health silent, + * TCP alive — probeLiveInstances kept it only because the connect succeeded). + * A holder that ANSWERS health is someone else's cell (#4125 adoption / + * #3544 rotation); matching bytes are attachable per #4811 same-lane + * discipline. */ +export function isWedgedSameLaneCandidate( + c: { inst: ProxyInstanceFile; health: HealthInfo | undefined }, + opts: LaunchOptions, + targetPort: number, + codeFingerprint: string | undefined, + foreignLanes: readonly string[], +): boolean { + if (c.inst.port !== targetPort) return false; + if (opts.lane === undefined || c.inst.lane !== opts.lane) return false; + if (isUserZoneInstance(c.inst)) return false; + if (c.health !== undefined) return false; + return instanceCompatible(c.inst, opts, codeFingerprint, foreignLanes) === "code-fingerprint-mismatch"; +} + +/** #5955: pure ledger step — record one observation of a wedged candidate and + * decide whether the patience window elapsed. A gap longer than the stale + * threshold means the consecutive run was interrupted (the holder answered or + * left between bring-ups) and the window restarts; short gaps bridge, so the + * window measures cumulative wedging within one continuous episode. */ +export function stepWedgedLedger( + existing: WedgedLedgerEntry | undefined, + pid: number, + nowMs: number, +): { entry: WedgedLedgerEntry; escalate: boolean; fresh: boolean } { + const fresh = existing === undefined || nowMs - existing.lastObservedAt > WEDGED_OBSERVATION_STALE_MS; + const entry = fresh + ? { pid, firstBusyAt: nowMs, lastObservedAt: nowMs, observations: 1 } + : { pid, firstBusyAt: existing.firstBusyAt, lastObservedAt: nowMs, observations: existing.observations + 1 }; + const escalate = nowMs - entry.firstBusyAt >= WEDGED_TAKEOVER_WINDOW_MS && entry.observations >= WEDGED_MIN_OBSERVATIONS; + return { entry, escalate, fresh }; +} + +async function waitForListenerClose( + origin: string, + isConnectable: (origin: string) => Promise, + now: () => number, + sleep: (ms: number) => Promise, + budgetMs: number, +): Promise { + const deadline = now() + budgetMs; + for (;;) { + if (!(await isConnectable(origin))) return true; + if (now() >= deadline) return false; + await sleep(HEALTH_POLL_INTERVAL_MS); + } +} + +/** #5955: bounded-patience forced takeover of the one decision-table cell + * every other surface skips (see the block comment above). No-op unless a + * candidate exists AND its ledger says the patience window elapsed; on any + * doubt the caller's normal spawn ladder runs untouched. */ +async function resolveWedgedSameLaneHolder( + probed: Array<{ inst: ProxyInstanceFile; health: HealthInfo | undefined }>, + resolveTargetPort: () => number | undefined, + opts: LaunchOptions, + attachUrl: string | undefined, + foreignLanes: readonly string[], + codeFingerprint: string | undefined, + ledgerPath: string, + birthStamp: (pid: number) => Promise, + now: () => number, + sleep: (ms: number) => Promise, + requestShutdown: (origin: string) => Promise, + isConnectable: (origin: string) => Promise, + signalHolder: (pid: number) => void, + hardKillHolder: (pid: number) => void, + diag: (msg: string) => void, +): Promise { + if (attachUrl !== undefined) return; + // #1660: touching the sticky zone resolver has side effects; only do it when a + // wedged-cell holder actually exists. + const maybeWedged = probed.some((c) => + c.health === undefined && opts.lane !== undefined && c.inst.lane === opts.lane + && !isUserZoneInstance(c.inst) + && instanceCompatible(c.inst, opts, codeFingerprint, foreignLanes) === "code-fingerprint-mismatch"); + if (!maybeWedged) return; + const targetPort = resolveTargetPort(); + if (targetPort === undefined) return; + const entries = readWedgedLedger(ledgerPath); + for (const [key, entry] of Object.entries(entries)) { + if (!isPidAlive(entry.pid)) delete entries[key]; + } + let dirty = false; + const persist = (): void => { + if (!dirty) return; + dirty = false; + if (Object.keys(entries).length === 0) { + try { fs.unlinkSync(ledgerPath); } catch { /* already gone */ } + return; + } + writeWedgedLedger(ledgerPath, entries); + }; + for (const c of probed) { + if (!isWedgedSameLaneCandidate(c, opts, targetPort, codeFingerprint, foreignLanes)) continue; + // PID-REUSE guard (#5955): Windows recycles pids quickly after dense + // kill/restart. The key carries the occupant's birth stamp so a REUSED + // pid starts a fresh window instead of inheriting the dead occupant's + // observation history and getting force-taken-over on first sight. + const stamp = await birthStamp(c.inst.pid); + const key = stamp !== undefined ? `${c.inst.origin}#${c.inst.pid}#${stamp}` : `${c.inst.origin}#${c.inst.pid}`; + const { entry, escalate, fresh } = stepWedgedLedger(entries[key], c.inst.pid, now()); + entries[key] = entry; + dirty = true; + if (fresh) { + // User-visible announcement at patience-window start: without it the + // user sees only 422 refusals during the whole window (#5955). + const startMsg = `native-attach: ${c.inst.origin} (pid ${c.inst.pid}) unresponsive — TCP alive but health silent on port ${targetPort}; patience window started, forced takeover after ${WEDGED_TAKEOVER_WINDOW_MS / 1000}s of continuous observations (#5955)`; + console.error(startMsg); + teeLog("warn", startMsg); + diag(startMsg); + } else if (!escalate) { + diag(`native-attach: watching ${c.inst.origin} (pid ${c.inst.pid}): BUSY same-lane holder with foreign bytes holds port ${targetPort} — observation ${entry.observations}, ${Math.round((now() - entry.firstBusyAt) / 1000)}s of ${WEDGED_TAKEOVER_WINDOW_MS / 1000}s patience before forced takeover (#5955)`); + } + if (!escalate) { + persist(); + continue; + } + const msg = `native-attach: forcing takeover of wedged same-lane proxy ${c.inst.origin} (pid ${c.inst.pid}): BUSY (TCP alive, health silent) for ${Math.round((now() - entry.firstBusyAt) / 1000)}s across ${entry.observations} bring-up cycles while holding port ${targetPort} with foreign bytes — requesting graceful shutdown, then terminating (#5955)`; + console.error(msg); + teeLog("warn", msg); + const t0 = now(); + await requestShutdown(c.inst.origin); + let released = await waitForListenerClose(c.inst.origin, isConnectable, now, sleep, WEDGED_RELEASE_WAIT_MS); + let killedHard = false; + if (!released) { + signalHolder(c.inst.pid); + released = await waitForListenerClose(c.inst.origin, isConnectable, now, sleep, WEDGED_TERM_GRACE_MS); + } + if (!released) { + hardKillHolder(c.inst.pid); + killedHard = true; + released = await waitForListenerClose(c.inst.origin, isConnectable, now, sleep, WEDGED_RELEASE_WAIT_MS); + } + if (released) { + if (killedHard) { + // An abrupt kill (SIGKILL / taskkill /F) leaves kernel socket state + // behind; give it a moment before the child tries to bind so the + // takeover does not ladder to +1 over residual TIME_WAIT (#5955). + await sleep(WEDGED_POST_KILL_SETTLE_MS); + } + delete entries[key]; + dirty = true; + const done = `native-attach: wedged holder pid ${c.inst.pid} released port ${targetPort} after ${now() - t0}ms — continuing to spawn`; + console.error(done); + teeLog("info", done); + } else { + // Keep the ledger entry: the occupant survived every stage; the next + // bring-up cycle retries the escalation instead of starting over. + const failed = `native-attach: FAILED to release port ${targetPort} from wedged holder pid ${c.inst.pid} — falling back to the normal spawn ladder (may ladder to the next free port); takeover retries on the next bring-up (#5955)`; + console.error(failed); + teeLog("warn", failed); + } + persist(); + return; + } + persist(); +} + /** #3544: owner heartbeat — re-asserts our liveness every beat so the proxy's * watchdog can age out a RECYCLED pid: on Windows pid reuse makes the * kill(pid,0) existence check identity-blind (a dead owner's pid can read @@ -5322,6 +5647,18 @@ export async function ensureProxyRunning( console.error(msg); attachDiag(msg); } + // #3544/#5955: the canonical port a replacement would bind — resolved + // LAZILY (only once a takeover candidate exists) because the lane + // preference resolver is stateful/sticky (#1660) and must not be touched + // on bring-ups where nothing rotates or takes over. + const resolveTargetPort = (): number | undefined => { + const rotLane = opts.lane !== undefined && opts.port <= 0 ? opts.lane : undefined; + return opts.port > 0 + ? opts.port + : rotLane !== undefined + ? (deps?.zonePreferredPort ?? lanePreferredPort)(rotLane) + : undefined; + }; // #3544: nothing attachable above — but a strictly-older generation may // still hold our canonical port (its fingerprint differs by definition, so // it was never attachable). Rotate it instead of letting the update sit @@ -5329,14 +5666,7 @@ export async function ensureProxyRunning( { await rotateStalePredecessor( probed, - () => { - const rotLane = opts.lane !== undefined && opts.port <= 0 ? opts.lane : undefined; - return opts.port > 0 - ? opts.port - : rotLane !== undefined - ? (deps?.zonePreferredPort ?? lanePreferredPort)(rotLane) - : undefined; - }, + resolveTargetPort, opts, attachUrl, foreignLanes, @@ -5346,6 +5676,29 @@ export async function ensureProxyRunning( deps?.requestInstanceShutdown ?? requestInstanceShutdown, ); } + // #5955: the cell neither attach nor rotation covers — a BUSY same-lane + // holder with foreign bytes on our canonical port. Bounded patience across + // bring-up cycles, then a supervised forced takeover; on any doubt the + // spawn ladder below runs untouched. + { + await resolveWedgedSameLaneHolder( + probed, + resolveTargetPort, + opts, + attachUrl, + foreignLanes, + codeFingerprint, + deps?.wedgedLedgerPath?.() ?? wedgedLedgerFilePath(), + deps?.processBirthStamp ?? ((pid: number) => processBirthStamp(pid)), + now, + sleepImpl, + deps?.requestInstanceShutdown ?? requestInstanceShutdown, + deps?.isConnectable ?? tcpConnectable, + deps?.signalWedgedHolder ?? signalWedgedHolderDefault, + deps?.killWedgedHolder ?? killWedgedHolderDefault, + attachDiag, + ); + } if (!attachExternal && opts.strictPort) { // #1335: the only compatible listener is an unarmed one squatting OUR // pinned port — self-managed fallback is impossible here (we cannot @@ -5940,6 +6293,63 @@ export async function ensureProxyRunning( } } +/** #2260(A)/#5955: unconditional platform-native hard-kill of a proxy process + * BY PID, all platforms. Win32 uses taskkill /F /T because TerminateProcess + * on the direct pid alone leaks descendants (a detached child that spawned + * its own helpers, e.g. self-update npm runs, survives); the direct SIGKILL + * stays as the fallback if taskkill itself fails. Failures are surfaced, + * never swallowed (#4125): a "killed" holder that survives keeps holding its + * listener, and the caller's release verification is what catches that. */ +function hardKillProxyTree(pid: number, label: string): void { + if (process.platform !== "win32") { + try { + process.kill(pid, "SIGKILL"); + } catch (err) { + // ESRCH = the pid is already gone — the goal is met, not a failure. + if ((err as NodeJS.ErrnoException)?.code !== "ESRCH") { + teeLog("warn", `bili: SIGKILL of ${label} ${pid} failed (${err instanceof Error ? err.message : String(err)}) — its listener may survive the cleanup (#4125)`); + } + } + return; + } + let tkError: string | undefined; + try { + const tk = spawnSync("taskkill", ["/F", "/T", "/PID", String(pid)], { windowsHide: true, stdio: "ignore" }); + if (tk.error !== undefined) tkError = tk.error.message; + else if (tk.status !== 0) tkError = `exit ${tk.status}`; + } catch (err) { + tkError = err instanceof Error ? err.message : String(err); + } + // #4125: a failed taskkill must not vanish silently — surface it; a non- + // zero exit can also mean the pid already left (harmless — the direct + // kill below still runs). + if (tkError !== undefined) { + teeLog("warn", `bili: taskkill of ${label} ${pid} reported failure (${tkError}) — its listener may survive the cleanup (#4125)`); + } + try { + process.kill(pid, "SIGKILL"); + } catch (err) { + teeLog("warn", `bili: direct SIGKILL of ${label} ${pid} failed (${err instanceof Error ? err.message : String(err)}) — its listener may survive the cleanup (#4125)`); + } +} + +/** #5955 defaults for the LauncherDeps takeover seams. POSIX sends SIGTERM + * first (a saturated-but-yielding loop can still drain); win32 has no + * graceful node signal, so it no-ops and the caller goes straight to the + * hard kill. */ +function signalWedgedHolderDefault(pid: number): void { + if (process.platform === "win32") return; + try { + process.kill(pid, "SIGTERM"); + } catch { + /* already gone */ + } +} + +function killWedgedHolderDefault(pid: number): void { + hardKillProxyTree(pid, "wedged holder"); +} + /** #2260(A)/#2187: unconditional hard-kill of a spawned-but-abandoned proxy * child, all platforms. Deliberately NOT stopProxy(): its win32 path returns * without killing because the #414 design assumes THIS process exits ≤2s @@ -5958,32 +6368,14 @@ function killAbandonedChild(child: SpawnChild | undefined): void { } catch { /* group already gone */ } - } else { - // #3116: win32 has no process-group kill — TerminateProcess on the direct - // child alone leaks its descendants (a detached:true child that spawned - // its own helpers, e.g. self-update npm runs, survives the "abandoned" - // kill). taskkill /T walks the whole tree; the direct kill below stays - // as the fallback if taskkill itself fails. - let tkError: string | undefined; - try { - const tk = spawnSync("taskkill", ["/F", "/T", "/PID", String(child.pid)], { windowsHide: true, stdio: "ignore" }); - if (tk.error !== undefined) tkError = tk.error.message; - else if (tk.status !== 0) tkError = `exit ${tk.status}`; - } catch (err) { - tkError = err instanceof Error ? err.message : String(err); - } - // #4125: a failed taskkill used to vanish silently here — exactly how a - // "killed" abandoned child could keep holding its listener. Surface it; - // a non-zero exit can also mean the pid already left (harmless — the - // direct kill below still runs). - if (tkError !== undefined) { - teeLog("warn", `bili: taskkill of abandoned proxy ${child.pid} reported failure (${tkError}) — its listener may survive the cleanup (#4125)`); - } } + hardKillProxyTree(child.pid, "abandoned proxy"); + // The ChildProcess handle needs its own SIGKILL to close stdio and mark the + // handle dead — the pid-level kills above do not do that (#2260/#2187). try { child.kill?.("SIGKILL"); - } catch (err) { - teeLog("warn", `bili: direct SIGKILL of abandoned proxy ${child.pid} failed (${err instanceof Error ? err.message : String(err)}) — its listener may survive the cleanup (#4125)`); + } catch { + teeLog("warn", `bili: SIGKILL of abandoned proxy ${child.pid} via its own handle failed — it may survive the cleanup (#4125)`); } } diff --git a/tests/issue5955-wedged-same-lane-takeover.test.ts b/tests/issue5955-wedged-same-lane-takeover.test.ts new file mode 100644 index 0000000000..6c99abfbb5 --- /dev/null +++ b/tests/issue5955-wedged-same-lane-takeover.test.ts @@ -0,0 +1,540 @@ +// #5955 — zombie proxy holds the canonical port + the fingerprint branch only +// skips without terminating → oversized sessions stuck at 422. +// +// The bring-up decision table had no row for "same-lane resident on the +// canonical port, DIFFERENT bytes, UNANSWERED health (BUSY)": attach vetoes +// it (code-fingerprint-mismatch), #4125 upward adoption demands an answered +// version, #3544 rotation excludes unverifiable residents (deliberate, #4811), +// and the spawned child ladders away or exits code 1. Nobody ever resolved +// the holder; the port stayed wedged until the process died on its own +// (guaranteed only for >=5min CONTINUOUS event-loop blocks). +// +// This suite pins the bounded-patience + forced-takeover step: +// - pure rows for isWedgedSameLaneCandidate / stepWedgedLedger (both +// directions, exact boundaries); +// - full bring-up path: escalation only after WEDGED_TAKEOVER_WINDOW_MS AND +// WEDGED_MIN_OBSERVATIONS, graceful shutdown first, then SIGTERM, then +// SIGKILL/taskkill, and the listener-close verification BEFORE spawn; +// - the observation ledger PERSISTS in the state dir — during a wedge the +// client restarts on every failed bootstrap round, so a fresh process must +// continue the previous process's window (and a REUSED pid must NOT +// inherit the dead occupant's history — birth-stamp key); +// - kill failure → fall through to the normal spawn ladder (never worse +// than today), ledger entry kept, takeover retried on the next cycle; +// - regression guards: healthy OLDER holder still routes through #3544 +// rotation ONLY, healthy NEWER holder still adopted (#4125), SAME-build +// BUSY holder attached and NEVER killed (#4811), user-zone / foreign-lane +// / foreign-port / attachUrl holders all untouched. +// +// Hermeticity: XDG_STATE_HOME is redirected to a scratch dir BEFORE any test +// runs, so probeLiveInstances' REAL registry view (discoverLiveInstances), +// the starting marker, and the default ledger file all live in scratch — no +// machine-local proxy state can leak into the candidate set. + +import test, { beforeEach, afterEach } from "node:test"; +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import type { ProxyInstanceFile } from "../src/instance.ts"; +import { removeStartingMarker } from "../src/instance.ts"; +import { + WEDGED_MIN_OBSERVATIONS, + WEDGED_OBSERVATION_STALE_MS, + WEDGED_TAKEOVER_WINDOW_MS, + ensureProxyRunning, + isWedgedSameLaneCandidate, + processBirthStamp, + stepWedgedLedger, + type SpawnChild, + type SpawnFn, +} from "../src/launcher.ts"; + +const SCRATCH_STATE = path.join(os.tmpdir(), `bili-state-5955-${process.pid}`); +fs.mkdirSync(path.join(SCRATCH_STATE, "billion-context"), { recursive: true }); +process.env.XDG_STATE_HOME = SCRATCH_STATE; + +const HOST = "127.0.0.1"; +const PORT = 18787; +const ORIGIN = `http://${HOST}:${PORT}`; +const CHILD_PID = 42_471; +// Inter-cycle advance: > starting-marker TTL (90s) so each cycle re-claims its +// marker, and + one post-spawn poll (200ms) stays <= WEDGED_OBSERVATION_STALE_MS +// so observations BRIDGE across cycles instead of restarting the window. +const GAP_MS = 280_000; + +const FP_SCRIPT = path.join(os.tmpdir(), `bili-fp-5955-${process.pid}.js`); +fs.writeFileSync(FP_SCRIPT, "// fingerprint fixture\n"); +const FP_HASH = createHash("sha256").update(fs.readFileSync(FP_SCRIPT)).digest("hex"); + +let ledgerFile = ""; + +function makeClock() { + let t = 1_000_000_000; + return { + now: () => t, + sleep: (ms: number) => new Promise((res) => { t += Math.max(1, ms); res(); }), + advance: (ms: number) => { t += ms; }, + }; +} + +interface World { + clock: ReturnType; + connectable: boolean; + gracefulShutdown: boolean; + killReleases: boolean; + shutdownCalls: string[]; + signalCalls: number[]; + killCalls: number[]; + spawnCalls: number; + order: string[]; + diagMsgs: string[]; +} + +function makeWorld(opts: { gracefulShutdown: boolean; killReleases: boolean }): World { + return { + clock: makeClock(), + connectable: true, + gracefulShutdown: opts.gracefulShutdown, + killReleases: opts.killReleases, + shutdownCalls: [], + signalCalls: [], + killCalls: [], + spawnCalls: 0, + order: [], + diagMsgs: [], + }; +} + +function wedgedHolder(): ProxyInstanceFile { + return { + origin: ORIGIN, + instanceId: "zombie-instance", + pid: process.pid, + startedAt: Date.now(), + host: HOST, + port: PORT, + passthrough: false, + mitmDomains: [], + modelWindows: {}, + codeFingerprint: "foreign-build", + lane: "dsh", + launchToken: "zombie-token", + }; +} + +// Own-child record found by the post-spawn pid fallback (the resident must come +// through the instance FILE view; the registry view feeds the pid fallback +// only — same discipline as the #3544 suite). +function mineMarker(): ProxyInstanceFile { + return { + origin: ORIGIN, + instanceId: "mine-1", + pid: CHILD_PID, + startedAt: Date.now(), + host: HOST, + port: PORT, + passthrough: false, + mitmDomains: [], + modelWindows: {}, + codeFingerprint: FP_HASH, + lane: "dsh", + launchToken: "mine-token", + }; +} + +function makeFakeChild(pid: number): SpawnChild { + return { pid, unref() {}, kill() { return true; }, on() {} }; +} + +type Deps = Parameters[1]; + +function makeDeps(w: World, holder: ProxyInstanceFile): Deps { + return { + fetchImpl: async (url: string) => ({ ok: url.startsWith(ORIGIN) }), + fetchHealthInfo: async (origin: string) => (origin === holder.origin ? undefined : { ok: false }), + readInstanceFile: () => holder, + listRegistryInstances: () => [mineMarker()], + spawnImpl: (_command: string, _args: readonly string[], _options: Parameters[2]): SpawnChild => { + w.spawnCalls += 1; + w.order.push("spawn"); + return makeFakeChild(CHILD_PID); + }, + now: () => w.clock.now(), + sleep: (ms: number) => w.clock.sleep(ms), + scriptPath: FP_SCRIPT, + nodeRuntime: process.execPath, + registerWatcher: async () => "ok", + resolveAttachUrl: () => undefined, + resolveAttachExternal: () => false, + resolveAttachPolicy: () => "own-lane", + attachLanePriority: () => [], + requestInstanceShutdown: async (origin: string) => { + w.shutdownCalls.push(origin); + w.order.push("shutdown"); + if (w.gracefulShutdown) w.connectable = false; + return true; + }, + isConnectable: async (origin: string) => origin === holder.origin && w.connectable, + signalWedgedHolder: (pid: number) => { w.signalCalls.push(pid); w.order.push("signal"); }, + killWedgedHolder: (pid: number) => { + w.killCalls.push(pid); + w.order.push("kill"); + if (w.killReleases) w.connectable = false; + }, + wedgedLedgerPath: () => ledgerFile, + // Deterministic birth stamp: stable per pid within a world unless a row + // overrides the seam to simulate PID REUSE. + processBirthStamp: async (pid: number) => `stamp-${pid}`, + attachDiag: (msg: string) => { w.diagMsgs.push(msg); }, + }; +} + +async function runCycle( + w: World, + holder: ProxyInstanceFile, + overrides?: { opts?: Record; deps?: Partial }, +) { + const opts = { + host: HOST, + port: PORT, + passthrough: false, + debug: false, + lane: "dsh", + parentPid: 999_999, + ...(overrides?.opts ?? {}), + }; + const deps = { ...makeDeps(w, holder), ...(overrides?.deps ?? {}) }; + return ensureProxyRunning(opts, deps); +} + +/** Runs bring-up cycles GAP_MS apart until pred() holds; returns the last handle. */ +async function cycleUntil( + w: World, + holder: ProxyInstanceFile, + pred: () => boolean, + maxCycles: number, + overrides?: { opts?: Record; deps?: Partial }, +) { + let handle: Awaited> | undefined; + for (let i = 1; i <= maxCycles; i++) { + if (i > 1) w.clock.advance(GAP_MS); + handle = await runCycle(w, holder, overrides); + if (pred()) break; + } + assert.ok(handle, `expected a handle within ${maxCycles} cycles`); + return handle; +} + +const watchingLines = (w: World) => w.diagMsgs.filter((m) => m.includes("patience before forced takeover")); +const startLines = (w: World) => w.diagMsgs.filter((m) => m.includes("patience window started")); + +function readLedger(): Record { + const raw = JSON.parse(fs.readFileSync(ledgerFile, "utf8")) as { entries: Record }; + return raw.entries; +} + +beforeEach(() => { + ledgerFile = path.join(SCRATCH_STATE, "billion-context", `wedged-ledger-${Math.random().toString(36).slice(2)}.json`); + try { fs.unlinkSync(ledgerFile); } catch { /* fresh */ } + removeStartingMarker(); +}); +afterEach(() => { + try { fs.unlinkSync(ledgerFile); } catch { /* already gone */ } + removeStartingMarker(); +}); + +test("#5955 isWedgedSameLaneCandidate: positive + single-flip negatives", () => { + const baseOpts = { host: HOST, port: PORT, passthrough: false, debug: false, lane: "dsh" }; + const pos = { inst: wedgedHolder(), health: undefined }; + + assert.equal(isWedgedSameLaneCandidate(pos, baseOpts, PORT, FP_HASH, []), true, "wedged cell: foreign bytes, same lane, BUSY, canonical port"); + + const wrongPort = { ...pos, inst: { ...pos.inst, port: 18799, origin: `http://${HOST}:18799` } }; + assert.equal(isWedgedSameLaneCandidate(wrongPort, baseOpts, PORT, FP_HASH, []), false, "different port"); + + const foreignLane = { ...pos, inst: { ...pos.inst, lane: "pi" } }; + assert.equal(isWedgedSameLaneCandidate(foreignLane, baseOpts, PORT, FP_HASH, []), false, "foreign lane"); + + const noLaneOpts = { ...baseOpts, lane: undefined }; + assert.equal(isWedgedSameLaneCandidate(pos, noLaneOpts, PORT, FP_HASH, []), false, "laneless bring-up"); + + const userZone = { ...pos, inst: { ...pos.inst, lane: undefined, launchToken: undefined } }; + assert.equal(isWedgedSameLaneCandidate(userZone, baseOpts, PORT, FP_HASH, []), false, "user zone"); + + const answered = { ...pos, health: { ok: true, instanceId: "zombie-instance", pid: process.pid, watchdog: { armed: true }, version: "0.0.1" } }; + assert.equal(isWedgedSameLaneCandidate(answered, baseOpts, PORT, FP_HASH, []), false, "answered health is not BUSY"); + + const sameFp = { ...pos, inst: { ...pos.inst, codeFingerprint: "foreign-build" } }; + assert.equal(isWedgedSameLaneCandidate(sameFp, baseOpts, PORT, "foreign-build", []), false, "matching bytes are attachable, never a takeover target"); + + const shapeOnly = { ...pos, inst: { ...pos.inst, codeFingerprint: "foreign-build", modelWindows: { gpt: 200_000 } } }; + assert.equal(isWedgedSameLaneCandidate(shapeOnly, baseOpts, PORT, "foreign-build", []), false, "shape-only mismatch is not the fp-mismatch cell"); +}); + +test("#5955 stepWedgedLedger: boundaries (window, min observations, stale gap, fresh flag)", () => { + const PID = 1234; + const r0 = stepWedgedLedger(undefined, PID, 0); + assert.equal(r0.entry.observations, 1); + assert.equal(r0.entry.firstBusyAt, 0); + assert.equal(r0.fresh, true, "first observation is a fresh window"); + assert.equal(r0.escalate, false, "fresh observation cannot escalate"); + + let e = stepWedgedLedger(r0.entry, PID, 200_000); + assert.equal(e.entry.observations, 2); + assert.equal(e.fresh, false, "short gap bridges the window"); + assert.equal(e.escalate, false); + e = stepWedgedLedger(e.entry, PID, 400_000); + assert.equal(e.entry.observations, 3); + assert.equal(e.escalate, false, "min observations reached but window not elapsed"); + e = stepWedgedLedger(e.entry, PID, WEDGED_TAKEOVER_WINDOW_MS - 1); + assert.equal(e.escalate, false, "one ms short of the window"); + assert.equal(e.entry.firstBusyAt, 0, "window keeps counting from first observation"); + + const atWindow = stepWedgedLedger(e.entry, PID, WEDGED_TAKEOVER_WINDOW_MS); + assert.equal(atWindow.escalate, true, "window elapsed + enough observations → escalate"); + + // Window elapsed but still below WEDGED_MIN_OBSERVATIONS even after this + // step: patience continues (a long-running host that rarely re-brings-up + // must not be killed off a couple of old sightings). Gap == STALE bridges + // exactly. + const sparse = { pid: PID, firstBusyAt: 0, lastObservedAt: WEDGED_OBSERVATION_STALE_MS, observations: WEDGED_MIN_OBSERVATIONS - 2 }; + const sparseStep = stepWedgedLedger(sparse, PID, WEDGED_TAKEOVER_WINDOW_MS); + assert.equal(sparseStep.entry.observations, WEDGED_MIN_OBSERVATIONS - 1); + assert.equal(sparseStep.entry.firstBusyAt, 0, "gap == STALE bridges the window"); + assert.equal(sparseStep.escalate, false, "min observations gate holds even past the window"); + + // One ms over STALE: the run was interrupted (host restarted) → the window + // restarts from now with a single fresh observation. + const stale = { pid: PID, firstBusyAt: 0, lastObservedAt: WEDGED_OBSERVATION_STALE_MS - 1, observations: 5 }; + const staleStep = stepWedgedLedger(stale, PID, WEDGED_TAKEOVER_WINDOW_MS); + assert.equal(staleStep.entry.firstBusyAt, WEDGED_TAKEOVER_WINDOW_MS, "stale gap restarts the window"); + assert.equal(staleStep.entry.observations, 1); + assert.equal(staleStep.fresh, true, "stale gap is reported as a fresh window"); + assert.equal(staleStep.escalate, false); +}); + +test("#5955 processBirthStamp: own pid resolves to a stable non-empty stamp", async () => { + const a = await processBirthStamp(process.pid); + const b = await processBirthStamp(process.pid); + assert.ok(a !== undefined && a.length > 0, "birth stamp available on this platform"); + assert.equal(b, a, "stable across reads"); + assert.equal(await processBirthStamp(-1), undefined, "invalid pid → undefined (key degrades to origin#pid)"); +}); + +test("#5955 full path: graceful shutdown frees the port, no signal/kill needed", async () => { + const w = makeWorld({ gracefulShutdown: true, killReleases: false }); + const holder = wedgedHolder(); + const handle = await cycleUntil(w, holder, () => w.shutdownCalls.length === 1, 6); + + assert.equal(w.shutdownCalls.length, 1, "exactly one graceful shutdown attempt"); + assert.deepEqual(w.shutdownCalls, [ORIGIN]); + assert.equal(w.signalCalls.length, 0, "graceful release needs no signal"); + assert.equal(w.killCalls.length, 0, "graceful release needs no kill"); + assert.deepEqual(w.order.filter((s) => s !== "spawn"), ["shutdown"]); + assert.equal(w.spawnCalls, 4, "each pre-escalation cycle spawns; the escalating cycle binds the freed port"); + assert.equal(handle.port, PORT); + assert.ok(handle.child, "the escalated cycle must spawn a live child"); + assert.equal(startLines(w).length, 1, "patience-window start announced once, user-visibly"); + assert.equal(watchingLines(w).length, 2, "non-escalating cycles logged the observability line"); + assert.equal(fs.existsSync(ledgerFile), false, "successful takeover clears the persisted ledger"); +}); + +test("#5955 full path: unresponsive holder → shutdown, then SIGTERM, then hard kill, in order", async () => { + const w = makeWorld({ gracefulShutdown: false, killReleases: true }); + const holder = wedgedHolder(); + const handle = await cycleUntil(w, holder, () => w.killCalls.length === 1, 6); + + assert.deepEqual(w.order.filter((s) => s !== "spawn"), ["shutdown", "signal", "kill"], "escalation ladder order"); + assert.deepEqual(w.signalCalls, [holder.pid]); + assert.deepEqual(w.killCalls, [holder.pid]); + assert.equal(w.spawnCalls, 4); + assert.equal(handle.port, PORT); + assert.ok(handle.child); + assert.equal(fs.existsSync(ledgerFile), false, "successful forced takeover clears the persisted ledger"); +}); + +test("#5955 full path: kill fails → fall through to normal spawn, ledger kept, retry next cycle", async () => { + const w = makeWorld({ gracefulShutdown: false, killReleases: false }); + const holder = wedgedHolder(); + const handle = await cycleUntil(w, holder, () => w.killCalls.length === 2, 8); + + assert.equal(w.shutdownCalls.length, 2, "one escalation attempt per qualifying cycle"); + assert.equal(w.signalCalls.length, 2); + assert.equal(w.killCalls.length, 2); + assert.equal(w.spawnCalls, 5, "every cycle still spawns — never worse than today while the holder survives"); + assert.equal(handle.port, PORT); + assert.ok(handle.child); + assert.equal(startLines(w).length, 1); + assert.equal(watchingLines(w).length, 2, "only the pre-escalation cycles logged the watching line"); + const entries = readLedger(); + assert.equal(Object.keys(entries).length, 1, "surviving occupant keeps its ledger entry for the retry"); + assert.equal(Object.values(entries)[0].observations, 5, "observations accumulate across failed escalations"); +}); + +test("#5955 full path: port:0 bring-up resolves the sticky lane port lazily", async () => { + const w = makeWorld({ gracefulShutdown: true, killReleases: false }); + const holder = wedgedHolder(); + let zonePortCalls = 0; + const handle = await cycleUntil( + w, + holder, + () => w.shutdownCalls.length === 1, + 6, + { + opts: { port: 0 }, + deps: { zonePreferredPort: (lane: string) => { zonePortCalls += 1; assert.equal(lane, "dsh"); return PORT; } }, + }, + ); + assert.ok(zonePortCalls >= 1, "sticky resolver touched only because a real wedged-cell holder existed"); + assert.equal(handle.port, PORT, "after takeover the spawn binds the lane's preferred port"); + assert.ok(handle.child); +}); + +test("#5955 full path: observations persist across client restarts (file ledger)", async () => { + const holder = wedgedHolder(); + const clock = makeClock(); + // "Process 1": one bring-up cycle in its own memory. + const w1 = makeWorld({ gracefulShutdown: false, killReleases: false }); + w1.clock = clock; + await runCycle(w1, holder); + const key = `${ORIGIN}#${holder.pid}#stamp-${holder.pid}`; + const after1 = readLedger(); + assert.equal(Object.keys(after1).length, 1); + assert.equal(after1[key].observations, 1); + + // "Process 2": a FRESH world (fresh in-memory state) GAP later, same disk. + clock.advance(GAP_MS); + const w2 = makeWorld({ gracefulShutdown: false, killReleases: false }); + w2.clock = clock; + await runCycle(w2, holder); + const after2 = readLedger(); + assert.equal(after2[key].observations, 2, "the restarted client continues the previous process's window"); + assert.equal(after2[key].firstBusyAt, after1[key].firstBusyAt, "the window keeps counting from the first sighting"); + assert.equal(w2.shutdownCalls.length, 0, "no premature takeover after a mere two sightings"); +}); + +test("#5955 full path: reused pid gets a fresh window (birth-stamp key)", async () => { + const holder = wedgedHolder(); + const w = makeWorld({ gracefulShutdown: false, killReleases: false }); + let stamp = "boot-old"; + await runCycle(w, holder, { deps: { processBirthStamp: async () => stamp } }); + w.clock.advance(GAP_MS); + await runCycle(w, holder, { deps: { processBirthStamp: async () => stamp } }); + // The OS hands the SAME pid to a brand-new process (different birth time). + stamp = "boot-new"; + w.clock.advance(GAP_MS); + const handle = await runCycle(w, holder, { deps: { processBirthStamp: async () => stamp } }); + + const entries = readLedger(); + assert.equal(Object.keys(entries).length, 2, "old-stamp and new-stamp keys coexist (dead-pid prune cleans up later)"); + assert.equal(entries[`${ORIGIN}#${holder.pid}#boot-old`].observations, 2); + assert.equal(entries[`${ORIGIN}#${holder.pid}#boot-new`].observations, 1, "reused pid starts at observation 1"); + assert.equal(w.shutdownCalls.length, 0, "never force-take-over a newborn instance on inherited history"); + assert.equal(startLines(w).length, 2, "the new occupant's patience window is announced again"); + assert.equal(handle.port, PORT); +}); + +test("#5955 negative: healthy OLDER holder still routes through #3544 rotation ONLY", async () => { + const w = makeWorld({ gracefulShutdown: true, killReleases: false }); + const holder = wedgedHolder(); + const handle = await runCycle(w, holder, { + deps: { + fetchHealthInfo: async (origin: string) => origin === holder.origin + ? { ok: true, instanceId: "zombie-instance", pid: process.pid, watchdog: { armed: true }, version: "0.0.1" } + : { ok: false }, + fetchHealthInfoNoReuse: async () => ({ ok: false }), + }, + }); + + assert.deepEqual(w.shutdownCalls, [ORIGIN], "#3544 rotation performs the graceful shutdown"); + assert.equal(w.signalCalls.length, 0, "rotation never signals"); + assert.equal(w.killCalls.length, 0, "rotation never kills"); + assert.equal(w.spawnCalls, 1); + assert.equal(handle.port, PORT); + assert.equal(watchingLines(w).length, 0, "no wedged-takeover activity for a healthy holder"); + assert.equal(startLines(w).length, 0); +}); + +test("#5955 negative: healthy NEWER holder still adopted (#4125), never touched", async () => { + const w = makeWorld({ gracefulShutdown: true, killReleases: false }); + const holder = wedgedHolder(); + const handle = await runCycle(w, holder, { + deps: { + fetchHealthInfo: async (origin: string) => origin === holder.origin + ? { ok: true, instanceId: "zombie-instance", pid: process.pid, watchdog: { armed: true }, version: "999.0.0" } + : { ok: false }, + }, + }); + + assert.equal(w.spawnCalls, 0, "newer same-lane holder is adopted, not bypassed"); + assert.equal(w.shutdownCalls.length, 0); + assert.equal(w.signalCalls.length, 0); + assert.equal(w.killCalls.length, 0); + assert.equal(handle.attached, true); + assert.equal(handle.child, undefined); +}); + +test("#5955 negative: SAME-build BUSY holder is attached and NEVER killed (#4811)", async () => { + const w = makeWorld({ gracefulShutdown: true, killReleases: false }); + const holder = { ...wedgedHolder(), codeFingerprint: FP_HASH }; + const handle = await runCycle(w, holder); + + assert.equal(w.spawnCalls, 0, "our own build attaches even while BUSY"); + assert.equal(w.shutdownCalls.length, 0, "never shut down our own build"); + assert.equal(w.signalCalls.length, 0, "never signal our own build"); + assert.equal(w.killCalls.length, 0, "never kill our own build"); + assert.equal(handle.attached, true); + assert.equal(watchingLines(w).length, 0); +}); + +test("#5955 negative: user-zone BUSY holder untouched", async () => { + const w = makeWorld({ gracefulShutdown: true, killReleases: false }); + const holder = { ...wedgedHolder(), lane: undefined, launchToken: undefined }; + const handle = await runCycle(w, holder); + + assert.equal(w.spawnCalls, 1); + assert.equal(w.shutdownCalls.length, 0); + assert.equal(w.signalCalls.length, 0); + assert.equal(w.killCalls.length, 0); + assert.equal(handle.port, PORT); + assert.equal(watchingLines(w).length, 0); +}); + +test("#5955 negative: foreign-lane BUSY holder untouched", async () => { + const w = makeWorld({ gracefulShutdown: true, killReleases: false }); + const holder = { ...wedgedHolder(), lane: "pi" }; + const handle = await runCycle(w, holder); + + assert.equal(w.spawnCalls, 1); + assert.equal(w.shutdownCalls.length, 0); + assert.equal(w.killCalls.length, 0); + assert.equal(handle.port, PORT); + assert.equal(watchingLines(w).length, 0); +}); + +test("#5955 negative: BUSY holder on a different port untouched", async () => { + const w = makeWorld({ gracefulShutdown: true, killReleases: false }); + const holder = { ...wedgedHolder(), port: 18799, origin: `http://${HOST}:18799` }; + const handle = await runCycle(w, holder); + + assert.equal(w.spawnCalls, 1); + assert.equal(w.shutdownCalls.length, 0); + assert.equal(w.killCalls.length, 0); + assert.equal(handle.port, PORT, "the spawn still binds the canonical port"); + assert.equal(watchingLines(w).length, 0); +}); + +test("#5955 negative: explicit attach target skips the takeover step entirely", async () => { + const w = makeWorld({ gracefulShutdown: true, killReleases: false }); + const holder = wedgedHolder(); + const handle = await runCycle(w, holder, { deps: { resolveAttachUrl: () => ORIGIN } }); + + assert.equal(w.spawnCalls, 1); + assert.equal(w.shutdownCalls.length, 0); + assert.equal(w.signalCalls.length, 0); + assert.equal(w.killCalls.length, 0); + assert.equal(handle.port, PORT); + assert.equal(watchingLines(w).length, 0); +}); From e7d717937c809ed4799f3361e8b5276ea3bc3f4c Mon Sep 17 00:00:00 2001 From: ci-check Date: Mon, 9 Nov 2026 01:16:23 +0800 Subject: [PATCH 2/3] fix(launcher): stop exporting wedgedLedgerFilePath (unused-exports gate) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit wedgedLedgerFilePath() is referenced only inside launcher.ts — the deps-injection fallback at line 5691 — while tests supply their own path via deps.wedgedLedgerPath. The leading `export` made it INTERNAL_ONLY under scripts/check-unused-exports.mjs. Drop it to match the sibling module-local ledger helpers (isWedgedLedgerEntry, readWedgedLedger). No behavior change. --- src/launcher.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/launcher.ts b/src/launcher.ts index 333012a3f3..f8493b25c9 100644 --- a/src/launcher.ts +++ b/src/launcher.ts @@ -5018,7 +5018,7 @@ interface WedgedLedgerEntry { * window instead of inheriting the dead occupant's history. Concurrent hosts * sharing one file can race on read-modify-write; the worst case loses one * observation (the window lengthens by one cycle), which is safe. */ -export function wedgedLedgerFilePath(): string { +function wedgedLedgerFilePath(): string { return path.join(stateDir(), "wedged-takeover.json"); } From 2826c518f842464a7ebb71c1d201c785c0761d25 Mon Sep 17 00:00:00 2001 From: ci-check Date: Mon, 9 Nov 2026 02:05:18 +0800 Subject: [PATCH 3/3] fix(launcher): field-tune wedged-takeover timings + quiet win32 ESRCH alarm (#5955) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Acceptance run on Windows (issue reporter, branch @ 33bec44) passed all three points but measured ~12.4s of pure budget spin in the escalation ladder against a structurally unresponsive holder (the port released in ~250ms after the hard kill), plus a false-alarm warn on every successful win32 takeover: - Escalation stages tightened from 3s+5s+2s+5s to 1s+2s+1s+2s. The shutdown-request timeout is now parameterized (requestInstanceShutdown gains an optional budget; #3544 rotation keeps its 3s default). Worst-case release verdict drops from ~15s to ~6s (+2s post-kill settle unchanged). The graceful-first discipline is kept: after a 10-minute health silence it is insurance, bounded to at most ~3s of cost. - hardKillProxyTree win32 fallback no longer warns when the direct SIGKILL throws ESRCH — taskkill already reaped the tree, which IS the goal; mirrors the POSIX branch suppression. --- src/launcher.ts | 47 +++++++++++++++++++++++++++++++++-------------- 1 file changed, 33 insertions(+), 14 deletions(-) diff --git a/src/launcher.ts b/src/launcher.ts index f8493b25c9..1531e61124 100644 --- a/src/launcher.ts +++ b/src/launcher.ts @@ -329,8 +329,10 @@ export interface LauncherDeps { * Default hits the real endpoint; tests stub it. */ fetchHealthInfoNoReuse?: (origin: string) => Promise; /** #3544: ask a stale resident to drain+exit (POST /__bili/shutdown). + * The optional budget lets a caller shorten the request timeout for a + * holder that is already known-unresponsive (#5955 wedged takeover). * Default hits the real endpoint; tests stub it. */ - requestInstanceShutdown?: (origin: string) => Promise; + requestInstanceShutdown?: (origin: string, timeoutMs?: number) => Promise; /** #1292: simulated OS for spawn planning and platform-gated arg * construction — tests drive win32 paths from a POSIX host. Defaults * to process.platform. */ @@ -4890,7 +4892,7 @@ const OWNER_HEARTBEAT_POST_TIMEOUT_MS = 2_000; const WATCHER_REG_ATTEMPTS = 3; const WATCHER_REG_RETRY_BACKOFF_MS = 500; -export async function requestInstanceShutdown(origin: string): Promise { +export async function requestInstanceShutdown(origin: string, timeoutMs: number = SHUTDOWN_REQUEST_TIMEOUT_MS): Promise { try { const res = await fetch(`${origin}/__bili/shutdown`, { method: "POST", @@ -4899,7 +4901,7 @@ export async function requestInstanceShutdown(origin: string): Promise // would only extend the drain it is about to run. headers: { "content-type": "application/json", connection: "close" }, body: "{}", - signal: AbortSignal.timeout(SHUTDOWN_REQUEST_TIMEOUT_MS), + signal: AbortSignal.timeout(timeoutMs), }); return res.ok; } catch { @@ -4988,18 +4990,30 @@ async function rotateStalePredecessor( // own — which the event-loop watchdog only guarantees for ≥5min CONTINUOUS // blocks; the saturation class never trips it, so the lane lost its proxy // indefinitely. Resolution = bounded patience across bring-up cycles, then a -// supervised forced takeover: ask for a graceful shutdown (insurance for a -// loop that briefly unblocks — the listener closes on accept), terminate if -// the listener survives, verify the port released, continue to the normal -// spawn. The victim loses at most its in-flight turn (sessions persist +// supervised forced takeover: ask for a graceful shutdown (a loop that +// briefly unblocks can still answer and close its listener cleanly), terminate +// if the listener survives, verify the port released, continue to the normal +// spawn. The stage budgets are deliberately SHORT (field-tuned #5955): after +// 10 minutes of health silence the graceful path is insurance, not +// expectation — the whole ladder must not add more than a few seconds on top +// of the window. The victim loses at most its in-flight turn (sessions persist // per-turn); the window bounds worst-case unavailability from ∞ to // WEDGED_TAKEOVER_WINDOW_MS instead of risking an immediate kill of a // slow-but-alive fold. export const WEDGED_TAKEOVER_WINDOW_MS = 600_000; // 10 min = 2× WATCHDOG_HARD_TIMEOUT_MS (event-loop-watchdog.ts) export const WEDGED_MIN_OBSERVATIONS = 3; // distinct bring-up cycles, not milliseconds — retry cadence varies per host export const WEDGED_OBSERVATION_STALE_MS = 300_000; // gap > this ⇒ the consecutive run was interrupted ⇒ the window restarts -const WEDGED_TERM_GRACE_MS = 2_000; // SIGTERM drain budget before the hard kill (POSIX only) -const WEDGED_RELEASE_WAIT_MS = 5_000; // per-stage budget to observe the listener close +// Escalation-stage budgets, field-tuned (#5955 acceptance run on Windows: the +// original 3+5+2+5s ladder spent ~12.4s of PURE SPIN against a structurally +// unresponsive holder before the hard kill released the port in ~250ms). A +// holder silent on health for 10 minutes will not complete an HTTP round-trip +// in 3s — if it can answer at all it answers promptly, so every stage is a +// tight multiple of the 200ms poll interval. Worst case is now ~6s to the +// release verdict (+settle), not ~15s (+settle). +const WEDGED_SHUTDOWN_BUDGET_MS = 1_000; // request timeout for the graceful-shutdown POST +const WEDGED_POST_SHUTDOWN_WAIT_MS = 2_000; // budget to observe the listener close after the graceful request +const WEDGED_TERM_GRACE_MS = 1_000; // SIGTERM drain budget before the hard kill (POSIX only) +const WEDGED_POST_KILL_WAIT_MS = 2_000; // budget to observe the listener close after a hard kill (measured release ≤252ms) const WEDGED_POST_KILL_SETTLE_MS = 2_000; // grace for kernel socket release after an ABRUPT kill before we bind (#5955) interface WedgedLedgerEntry { @@ -5187,7 +5201,7 @@ async function resolveWedgedSameLaneHolder( birthStamp: (pid: number) => Promise, now: () => number, sleep: (ms: number) => Promise, - requestShutdown: (origin: string) => Promise, + requestShutdown: (origin: string, timeoutMs?: number) => Promise, isConnectable: (origin: string) => Promise, signalHolder: (pid: number) => void, hardKillHolder: (pid: number) => void, @@ -5246,8 +5260,8 @@ async function resolveWedgedSameLaneHolder( console.error(msg); teeLog("warn", msg); const t0 = now(); - await requestShutdown(c.inst.origin); - let released = await waitForListenerClose(c.inst.origin, isConnectable, now, sleep, WEDGED_RELEASE_WAIT_MS); + await requestShutdown(c.inst.origin, WEDGED_SHUTDOWN_BUDGET_MS); + let released = await waitForListenerClose(c.inst.origin, isConnectable, now, sleep, WEDGED_POST_SHUTDOWN_WAIT_MS); let killedHard = false; if (!released) { signalHolder(c.inst.pid); @@ -5256,7 +5270,7 @@ async function resolveWedgedSameLaneHolder( if (!released) { hardKillHolder(c.inst.pid); killedHard = true; - released = await waitForListenerClose(c.inst.origin, isConnectable, now, sleep, WEDGED_RELEASE_WAIT_MS); + released = await waitForListenerClose(c.inst.origin, isConnectable, now, sleep, WEDGED_POST_KILL_WAIT_MS); } if (released) { if (killedHard) { @@ -6329,7 +6343,12 @@ function hardKillProxyTree(pid: number, label: string): void { try { process.kill(pid, "SIGKILL"); } catch (err) { - teeLog("warn", `bili: direct SIGKILL of ${label} ${pid} failed (${err instanceof Error ? err.message : String(err)}) — its listener may survive the cleanup (#4125)`); + // ESRCH = the tree is already gone (taskkill reaped it) — the goal is + // met; warning there would log a false alarm on EVERY successful win32 + // takeover (#5955 field report). + if ((err as NodeJS.ErrnoException)?.code !== "ESRCH") { + teeLog("warn", `bili: direct SIGKILL of ${label} ${pid} failed (${err instanceof Error ? err.message : String(err)}) — its listener may survive the cleanup (#4125)`); + } } }