From 016d933a0008c81ee42bc6f792a02630a9594f20 Mon Sep 17 00:00:00 2001 From: TOSKA1226 Date: Mon, 9 Nov 2026 02:10:42 +0800 Subject: [PATCH] test(launcher): cover the #5955 wedged same-lane takeover ladder The field-tuned timings in 2826c51 shipped without tests: the escalation budget (1s POST + 2s post-shutdown wait + 1s SIGTERM drain + 2s post-kill wait + 2s settle) and the win32 ESRCH silence are unguarded, so the next re-tune can silently bring back a ~12s stall or emit a false "listener may survive the cleanup" alarm on every takeover. tests/issue5955-wedged-same-lane-takeover.test.ts (18 cases): - candidate matrix: same lane + canonical port + BUSY + fingerprint mismatch is the ONLY cell that may be taken over - ledger boundaries: window, min observations, 300s stale gap, persistence across client restarts, fresh window after pid reuse (birth stamp key) - escalation order shutdown -> SIGTERM -> hard kill, and the short BUSY budget instead of the 3s graceful default (staging bounded to ~6s) - kill failure falls through to the normal spawn ladder, entry kept - port:0 lazy sticky-lane resolution; graceful path never signals/kills - negatives: healthy older/newer/same-build BUSY holders, user zone, foreign lane, other port and explicit attach targets are left alone - win32: a pid already reaped by taskkill stays silent, real failures still surface (#4125 vs #5955 field noise) Exports hardKillProxyTree so the win32 silence assertion can call it. --- src/launcher.ts | 2 +- ...ssue5955-wedged-same-lane-takeover.test.ts | 61 ++++++++++++++++++- 2 files changed, 61 insertions(+), 2 deletions(-) diff --git a/src/launcher.ts b/src/launcher.ts index 1531e61124..c4cedfd336 100644 --- a/src/launcher.ts +++ b/src/launcher.ts @@ -6314,7 +6314,7 @@ export async function ensureProxyRunning( * stays as the fallback if taskkill itself fails. Failures are surfaced, * never swallowed (#4125): a "killed" holder that survives keeps holding its * listener, and the caller's release verification is what catches that. */ -function hardKillProxyTree(pid: number, label: string): void { +export function hardKillProxyTree(pid: number, label: string): void { if (process.platform !== "win32") { try { process.kill(pid, "SIGKILL"); diff --git a/tests/issue5955-wedged-same-lane-takeover.test.ts b/tests/issue5955-wedged-same-lane-takeover.test.ts index 6c99abfbb5..314c2c1a1b 100644 --- a/tests/issue5955-wedged-same-lane-takeover.test.ts +++ b/tests/issue5955-wedged-same-lane-takeover.test.ts @@ -33,10 +33,12 @@ import test, { beforeEach, afterEach } from "node:test"; import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; import { createHash } from "node:crypto"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; +import { setLogCapture } from "../src/logger.ts"; import type { ProxyInstanceFile } from "../src/instance.ts"; import { removeStartingMarker } from "../src/instance.ts"; import { @@ -44,6 +46,7 @@ import { WEDGED_OBSERVATION_STALE_MS, WEDGED_TAKEOVER_WINDOW_MS, ensureProxyRunning, + hardKillProxyTree, isWedgedSameLaneCandidate, processBirthStamp, stepWedgedLedger, @@ -85,6 +88,7 @@ interface World { gracefulShutdown: boolean; killReleases: boolean; shutdownCalls: string[]; + shutdownTimeouts: Array; signalCalls: number[]; killCalls: number[]; spawnCalls: number; @@ -99,6 +103,7 @@ function makeWorld(opts: { gracefulShutdown: boolean; killReleases: boolean }): gracefulShutdown: opts.gracefulShutdown, killReleases: opts.killReleases, shutdownCalls: [], + shutdownTimeouts: [], signalCalls: [], killCalls: [], spawnCalls: 0, @@ -170,8 +175,9 @@ function makeDeps(w: World, holder: ProxyInstanceFile): Deps { resolveAttachExternal: () => false, resolveAttachPolicy: () => "own-lane", attachLanePriority: () => [], - requestInstanceShutdown: async (origin: string) => { + requestInstanceShutdown: async (origin: string, timeoutMs?: number) => { w.shutdownCalls.push(origin); + w.shutdownTimeouts.push(timeoutMs); w.order.push("shutdown"); if (w.gracefulShutdown) w.connectable = false; return true; @@ -355,6 +361,59 @@ test("#5955 full path: unresponsive holder → shutdown, then SIGTERM, then hard assert.equal(fs.existsSync(ledgerFile), false, "successful forced takeover clears the persisted ledger"); }); +test("#5955 escalation pays a SHORT BUSY budget, not the graceful ladder", async () => { + // Measured on the acceptance box (#5955): 12.4s of a 13.8s takeover sat + // inside the graceful stages a BUSY holder can never answer (3s POST + // timeout + 5s listener wait + 2s SIGTERM drain) before the hard kill it + // always needs. The ladder is field-tuned down to 1s + 2s + 1s + 2s, so + // the whole escalation must stay inside single-digit seconds of virtual + // time — and the POST must never wait out the 3s graceful default. + const w = makeWorld({ gracefulShutdown: false, killReleases: true }); + const holder = wedgedHolder(); + let handle: Awaited> | undefined; + for (let i = 1; i <= 3; i++) { + if (i > 1) w.clock.advance(GAP_MS); + handle = await runCycle(w, holder); + } + assert.equal(w.killCalls.length, 0, "still patient before the window elapses"); + + w.clock.advance(GAP_MS); + const t0 = w.clock.now(); + handle = await runCycle(w, holder); + const escalationMs = w.clock.now() - t0; + + assert.equal(w.killCalls.length, 1, "the escalating cycle still ends in the hard kill"); + assert.equal(handle?.port, PORT); + assert.equal(w.shutdownTimeouts.length, 1, "exactly one graceful shutdown request before the kill"); + const budget = w.shutdownTimeouts[0]; + assert.equal(typeof budget, "number", "the BUSY shutdown POST gets an explicit budget"); + assert.ok((budget as number) > 0 && (budget as number) < 3_000, `BUSY shutdown POST must not wait out the 3s graceful default (got ${budget}ms)`); + assert.ok(escalationMs >= 2_000, `the post-kill settle stays conservative (got ${escalationMs}ms)`); + assert.ok(escalationMs <= 6_000, `worst-case takeover staging bounded to ~6s (got ${escalationMs}ms)`); +}); + +test("#5955 hard kill: a pid already reaped by taskkill is silent, real failures still surface", () => { + // win32: taskkill /F /T reaps the tree, then the defensive direct SIGKILL + // finds the pid gone and used to emit "its listener may survive the + // cleanup (#4125)" on EVERY takeover — noise that hides real survivors. + const dead = spawnSync(process.execPath, ["-e", ""], { stdio: "ignore" }); + assert.equal(dead.status, 0); + assert.equal(typeof dead.pid, "number"); + const lines: string[] = []; + setLogCapture((level, msg) => { lines.push(`${level}:${msg}`); }); + try { + hardKillProxyTree(dead.pid as number, "wedged holder"); + } finally { + setLogCapture(null); + } + assert.equal(lines.filter((l) => l.includes("direct SIGKILL")).length, 0, "a kill that found the pid gone is not a survivor report"); + assert.equal( + lines.filter((l) => l.includes("taskkill")).length, + process.platform === "win32" ? 1 : 0, + "a real hard-kill failure is still surfaced (#4125)", + ); +}); + test("#5955 full path: kill fails → fall through to normal spawn, ledger kept, retry next cycle", async () => { const w = makeWorld({ gracefulShutdown: false, killReleases: false }); const holder = wedgedHolder();