Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 34 additions & 11 deletions src/launcher.ts
Original file line number Diff line number Diff line change
Expand Up @@ -330,7 +330,7 @@ export interface LauncherDeps {
fetchHealthInfoNoReuse?: (origin: string) => Promise<HealthInfo | undefined>;
/** #3544: ask a stale resident to drain+exit (POST /__bili/shutdown).
* Default hits the real endpoint; tests stub it. */
requestInstanceShutdown?: (origin: string) => Promise<boolean>;
requestInstanceShutdown?: (origin: string, timeoutMs?: number) => Promise<boolean>;
/** #1292: simulated OS for spawn planning and platform-gated arg
* construction — tests drive win32 paths from a POSIX host. Defaults
* to process.platform. */
Expand Down Expand Up @@ -4890,7 +4890,7 @@ const OWNER_HEARTBEAT_POST_TIMEOUT_MS = 2_000;
const WATCHER_REG_ATTEMPTS = 3;
const WATCHER_REG_RETRY_BACKOFF_MS = 500;

export async function requestInstanceShutdown(origin: string): Promise<boolean> {
export async function requestInstanceShutdown(origin: string, timeoutMs = SHUTDOWN_REQUEST_TIMEOUT_MS): Promise<boolean> {
try {
const res = await fetch(`${origin}/__bili/shutdown`, {
method: "POST",
Expand All @@ -4899,7 +4899,10 @@ export async function requestInstanceShutdown(origin: string): Promise<boolean>
// would only extend the drain it is about to run.
headers: { "content-type": "application/json", connection: "close" },
body: "{}",
signal: AbortSignal.timeout(SHUTDOWN_REQUEST_TIMEOUT_MS),
// #5955: the wedged-takeover path passes a SHORT budget — a holder
// whose health is silent never reads this POST, so the full 3s is dead
// weight on the takeover's critical path.
signal: AbortSignal.timeout(timeoutMs),
});
return res.ok;
} catch {
Expand Down Expand Up @@ -4998,9 +5001,21 @@ async function rotateStalePredecessor(
export const WEDGED_TAKEOVER_WINDOW_MS = 600_000; // 10 min = 2× WATCHDOG_HARD_TIMEOUT_MS (event-loop-watchdog.ts)
export const WEDGED_MIN_OBSERVATIONS = 3; // distinct bring-up cycles, not milliseconds — retry cadence varies per host
export const WEDGED_OBSERVATION_STALE_MS = 300_000; // gap > this ⇒ the consecutive run was interrupted ⇒ the window restarts
const WEDGED_TERM_GRACE_MS = 2_000; // SIGTERM drain budget before the hard kill (POSIX only)
const WEDGED_RELEASE_WAIT_MS = 5_000; // per-stage budget to observe the listener close
const WEDGED_RELEASE_WAIT_MS = 5_000; // budget to observe the listener close AFTER the hard kill (the only stage that can actually need it)
// #5955 (measured, win32 acceptance run): taskkill /F /T → first successful
// bind is 162–252 ms across 4 runs, so 2s is an ~8× safety margin, not a tuned
// value — kept deliberately conservative because a lost race costs a whole
// bring-up cycle.
const WEDGED_POST_KILL_SETTLE_MS = 2_000; // grace for kernel socket release after an ABRUPT kill before we bind (#5955)
// #5955: a takeover candidate is BUSY *by construction* (TCP alive, health
// silent across ≥ WEDGED_MIN_OBSERVATIONS bring-up cycles) — its event loop is
// saturated, so the graceful stages below are insurance, not a realistic path.
// Paying the full graceful budget there is pure latency: the measured win32
// takeover spent 12.4s of 13.8s inside them (3s POST timeout + 5s wait + 2s
// SIGTERM drain) before the hard kill it always ends in. Short stage budgets
// cut the worst case to ~4s without removing any stage.
export const WEDGED_BUSY_REQUEST_TIMEOUT_MS = 750; // shutdown POST budget for a health-silent holder
const WEDGED_BUSY_GRACE_MS = 500; // per-stage budget to observe the listener close

interface WedgedLedgerEntry {
pid: number;
Expand Down Expand Up @@ -5187,7 +5202,7 @@ async function resolveWedgedSameLaneHolder(
birthStamp: (pid: number) => Promise<string | undefined>,
now: () => number,
sleep: (ms: number) => Promise<void>,
requestShutdown: (origin: string) => Promise<boolean>,
requestShutdown: (origin: string, timeoutMs?: number) => Promise<boolean>,
isConnectable: (origin: string) => Promise<boolean>,
signalHolder: (pid: number) => void,
hardKillHolder: (pid: number) => void,
Expand Down Expand Up @@ -5246,12 +5261,14 @@ async function resolveWedgedSameLaneHolder(
console.error(msg);
teeLog("warn", msg);
const t0 = now();
await requestShutdown(c.inst.origin);
let released = await waitForListenerClose(c.inst.origin, isConnectable, now, sleep, WEDGED_RELEASE_WAIT_MS);
// #5955: BUSY by construction — see WEDGED_BUSY_*_MS. Every stage stays,
// each gets a short budget instead of the full graceful ladder.
await requestShutdown(c.inst.origin, WEDGED_BUSY_REQUEST_TIMEOUT_MS);
let released = await waitForListenerClose(c.inst.origin, isConnectable, now, sleep, WEDGED_BUSY_GRACE_MS);
let killedHard = false;
if (!released) {
signalHolder(c.inst.pid);
released = await waitForListenerClose(c.inst.origin, isConnectable, now, sleep, WEDGED_TERM_GRACE_MS);
released = await waitForListenerClose(c.inst.origin, isConnectable, now, sleep, WEDGED_BUSY_GRACE_MS);
}
if (!released) {
hardKillHolder(c.inst.pid);
Expand Down Expand Up @@ -6300,7 +6317,7 @@ export async function ensureProxyRunning(
* stays as the fallback if taskkill itself fails. Failures are surfaced,
* never swallowed (#4125): a "killed" holder that survives keeps holding its
* listener, and the caller's release verification is what catches that. */
function hardKillProxyTree(pid: number, label: string): void {
export function hardKillProxyTree(pid: number, label: string): void {
if (process.platform !== "win32") {
try {
process.kill(pid, "SIGKILL");
Expand Down Expand Up @@ -6329,7 +6346,13 @@ function hardKillProxyTree(pid: number, label: string): void {
try {
process.kill(pid, "SIGKILL");
} catch (err) {
teeLog("warn", `bili: direct SIGKILL of ${label} ${pid} failed (${err instanceof Error ? err.message : String(err)}) — its listener may survive the cleanup (#4125)`);
// #5955: ESRCH here means taskkill ALREADY reaped the pid — the goal is
// met, and reporting a possible survivor on every single takeover is the
// noise that trains operators to ignore this log. Mirrors the POSIX
// branch above; any other errno is still surfaced (#4125).
if ((err as NodeJS.ErrnoException)?.code !== "ESRCH") {
teeLog("warn", `bili: direct SIGKILL of ${label} ${pid} failed (${err instanceof Error ? err.message : String(err)}) — its listener may survive the cleanup (#4125)`);
}
}
}

Expand Down
57 changes: 56 additions & 1 deletion tests/issue5955-wedged-same-lane-takeover.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -33,17 +33,21 @@

import test, { beforeEach, afterEach } from "node:test";
import assert from "node:assert/strict";
import { spawnSync } from "node:child_process";
import { createHash } from "node:crypto";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { setLogCapture } from "../src/logger.ts";
import type { ProxyInstanceFile } from "../src/instance.ts";
import { removeStartingMarker } from "../src/instance.ts";
import {
WEDGED_BUSY_REQUEST_TIMEOUT_MS,
WEDGED_MIN_OBSERVATIONS,
WEDGED_OBSERVATION_STALE_MS,
WEDGED_TAKEOVER_WINDOW_MS,
ensureProxyRunning,
hardKillProxyTree,
isWedgedSameLaneCandidate,
processBirthStamp,
stepWedgedLedger,
Expand Down Expand Up @@ -85,6 +89,7 @@ interface World {
gracefulShutdown: boolean;
killReleases: boolean;
shutdownCalls: string[];
shutdownTimeouts: Array<number | undefined>;
signalCalls: number[];
killCalls: number[];
spawnCalls: number;
Expand All @@ -99,6 +104,7 @@ function makeWorld(opts: { gracefulShutdown: boolean; killReleases: boolean }):
gracefulShutdown: opts.gracefulShutdown,
killReleases: opts.killReleases,
shutdownCalls: [],
shutdownTimeouts: [],
signalCalls: [],
killCalls: [],
spawnCalls: 0,
Expand Down Expand Up @@ -170,8 +176,9 @@ function makeDeps(w: World, holder: ProxyInstanceFile): Deps {
resolveAttachExternal: () => false,
resolveAttachPolicy: () => "own-lane",
attachLanePriority: () => [],
requestInstanceShutdown: async (origin: string) => {
requestInstanceShutdown: async (origin: string, timeoutMs?: number) => {
w.shutdownCalls.push(origin);
w.shutdownTimeouts.push(timeoutMs);
w.order.push("shutdown");
if (w.gracefulShutdown) w.connectable = false;
return true;
Expand Down Expand Up @@ -355,6 +362,54 @@ test("#5955 full path: unresponsive holder → shutdown, then SIGTERM, then hard
assert.equal(fs.existsSync(ledgerFile), false, "successful forced takeover clears the persisted ledger");
});

test("#5955 escalation pays the SHORT BUSY budget, not the graceful ladder", async () => {
// Measured on the acceptance box: 12.4s of a 13.8s takeover sat inside the
// graceful stages a BUSY holder can never answer (3s POST timeout + 5s
// listener wait + 2s SIGTERM drain) before the hard kill it always needs.
const w = makeWorld({ gracefulShutdown: false, killReleases: true });
const holder = wedgedHolder();
let handle: Awaited<ReturnType<typeof runCycle>> | undefined;
for (let i = 1; i <= 3; i++) {
if (i > 1) w.clock.advance(GAP_MS);
handle = await runCycle(w, holder);
}
assert.equal(w.killCalls.length, 0, "still patient before the window elapses");

w.clock.advance(GAP_MS);
const t0 = w.clock.now();
handle = await runCycle(w, holder);
const escalationMs = w.clock.now() - t0;

assert.equal(w.killCalls.length, 1, "the escalating cycle still ends in the hard kill");
assert.equal(handle?.port, PORT);
assert.equal(w.shutdownTimeouts.length, 1);
assert.equal(w.shutdownTimeouts[0], WEDGED_BUSY_REQUEST_TIMEOUT_MS, "BUSY shutdown POST gets the short budget, not 3s");
assert.ok(escalationMs >= 2_000, `the post-kill settle stays conservative (got ${escalationMs}ms)`);
assert.ok(escalationMs <= 4_500, `worst-case takeover budget bounded to ~4s (got ${escalationMs}ms)`);
});

test("#5955 hard kill: a pid already reaped by taskkill is silent, real failures still surface", () => {
// win32: taskkill /F /T reaps the tree, then the defensive direct SIGKILL
// finds the pid gone and used to emit "its listener may survive the
// cleanup (#4125)" on EVERY takeover — noise that hides real survivors.
const dead = spawnSync(process.execPath, ["-e", ""], { stdio: "ignore" });
assert.equal(dead.status, 0);
assert.equal(typeof dead.pid, "number");
const lines: string[] = [];
setLogCapture((level, msg) => { lines.push(`${level}:${msg}`); });
try {
hardKillProxyTree(dead.pid as number, "wedged holder");
} finally {
setLogCapture(null);
}
assert.equal(lines.filter((l) => l.includes("direct SIGKILL")).length, 0, "a kill that found the pid gone is not a survivor report");
assert.equal(
lines.filter((l) => l.includes("taskkill")).length,
process.platform === "win32" ? 1 : 0,
"a real hard-kill failure is still surfaced (#4125)",
);
});

test("#5955 full path: kill fails → fall through to normal spawn, ledger kept, retry next cycle", async () => {
const w = makeWorld({ gracefulShutdown: false, killReleases: false });
const holder = wedgedHolder();
Expand Down
Loading