diff --git a/src/launcher.ts b/src/launcher.ts index c5473ce834..381eef45d1 100644 --- a/src/launcher.ts +++ b/src/launcher.ts @@ -5234,6 +5234,23 @@ export async function ensureProxyRunning( // "child exited (code 0)" without the runtime path forced users to // guess WHICH program was actually spawned. const runtime = deps.nodeRuntime ?? resolveNodeRuntime(); + // #4125: the port log is SHARED across spawns (append fd, size-based + // rotation only) — a zero-output failed child used to leave the + // PREVIOUS spawn's content behind, and every quoted sample misattributed + // it (observed: a week-old EADDRINUSE line quoted as the current + // failure's evidence). The header marks where THIS spawn's segment + // starts; advance spawnLogOffset past it so "N bytes since this spawn" + // counts only the child's own output (the header is launcher-written, + // not child output). Published BEFORE the spawn — like #707's starting + // marker — because a spawned child can write before a post-spawn + // append lands, which would push its earliest bytes (crash tails, + // listen lines) below the offset into "previous spawn" territory. + // childPid stays off the header (unknown pre-spawn); the child names + // itself in its own pid-tagged listen line (#4994). + try { + fs.appendFileSync(logPath, `\n=== bili spawn ${new Date(now()).toISOString()} launcherPid=${process.pid} port=${port} ===\n`); + spawnLogOffset = fs.statSync(logPath).size; + } catch { /* diagnostics only */ } try { child = spawnImpl( runtime, @@ -5291,6 +5308,9 @@ export async function ensureProxyRunning( // #1753: announce a healthy-but-foreign responder on the preferred port // at most once — silence would hide exactly the misroute this fix closes. let squatterAnnounced = false; + // #4125: same discipline for the PRIMARY settle branches (launchToken / + // registry-by-pid) — announce a contested claimed origin at most once. + let contestedAnnounced = false; child.on?.("exit", (...rest: unknown[]) => { childExit = { code: typeof rest[0] === "number" ? rest[0] : null, @@ -5404,21 +5424,17 @@ export async function ensureProxyRunning( const logTail = (): string => { try { const size = fs.statSync(logPath).size; - // "0 bytes" alone cannot tell "child wrote nothing" from "output + // #4125/#4811: report bytes SINCE THIS SPAWN, not the whole + // shared file — a zero-output failed child must not be + // described by (or quoted from) a previous spawn's content. + // "0" still cannot tell "child wrote nothing" from "output // rotated away" (rotatePortLogIfNeeded renames to .old before // the fd is opened) — say which when the rotated half exists. - // Same honesty when the child appended NOTHING to a non-empty - // file: the bytes on disk then belong to an EARLIER spawn, and - // quoting them as this child's head is exactly the #4811 - // misdirection (a shrunken file means rotation under us — the - // bytes left ARE ours, so the tail sample stands). The .old - // guard keeps that claim honest: if a rotation happened, the - // captured offset is untrustworthy and the sample path applies. - const rotatedFile = fs.existsSync(`${logPath}.old`); - const rotated = rotatedFile ? `; earlier output rotated to ${logPath}.old` : ""; - if (size === 0) return `, ${size} bytes — EMPTY, the child produced no output of its own${rotated}`; - if (size === spawnLogOffset && !rotatedFile) return `, ${size} bytes — this child appended nothing of its own (earlier content stays on disk until rotation)`; - return `, ${size} bytes${rotated}${logHeadSample()}${logTailSample()}`; + const own = Math.max(0, size - spawnLogOffset); + const rotated = fs.existsSync(`${logPath}.old`) ? `; earlier output rotated to ${logPath}.old` : ""; + return own <= 0 + ? `, ${own} bytes since this spawn — EMPTY, the child produced no output of its own (any earlier content in this shared file belongs to previous spawns)${rotated}` + : `, ${own} bytes since this spawn${rotated}${logHeadSample()}${logTailSample()}`; } catch { return ""; } @@ -5470,6 +5486,30 @@ export async function ensureProxyRunning( return { text: `last health probe failed with ${d}`, selfHealthy: false }; }; + // #4125: capture WHAT the planned port was doing at SPAWN time. The + // failure-time diagnosis (diagnosePort) only sees the END state; the + // start state separates "predecessor still draining/exiting on the + // port" from "child wedged on something unrelated" (e.g. a storage lock + // left by a forced exit) — the two read identically at the end, and in + // the field case the end state was plain ECONNREFUSED while the real + // story was the start state. + const initialPortNote = await (async (): Promise => { + const origin = proxyOrigin(opts.host, port); + let info: HealthInfo | undefined; + let det: ProbeOutcome = { ok: false, detail: "probe error" }; + try { + const [i, d] = await Promise.all([fetchHealthInfo(origin), fetchDetail(healthUrl(origin))]); + info = i; + det = d; + } catch { /* observability only — keep the generic clause */ } + if (info?.ok) return `healthy bili instance${info.pid !== undefined ? ` (pid ${info.pid})` : ""}`; + if (det.ok) return "foreign HTTP answer"; + const d = det.detail ?? "unknown error"; + if (/^(ECONNREFUSED|ECONNRESET|EHOSTUNREACH|ENETUNREACH|EPIPE)$/i.test(d)) return "nothing listening"; + if (/^HTTP \d{3}$/.test(d)) return `${d} answer`; + if (/^timeout after \d+ms$/i.test(d) && (await (deps.isConnectable ?? tcpConnectable)(origin))) return "busy (TCP up, no health answer)"; + return `probe failed (${d})`; + })(); const deadline = now() + SPAWN_BUDGET_MS; // #2187: announce the phase boundary once — a live child past the // initial window is still being waited for, and the log is the only @@ -5480,13 +5520,21 @@ export async function ensureProxyRunning( if (!slowStartNoticed && now() >= deadline - SLOW_START_GRACE_MS) { slowStartNoticed = true; console.error( - `bili: spawned proxy not healthy after ${SPAWN_WAIT_MS}ms — child pid ${child.pid ?? "?"} is still starting up; extending wait to ${SPAWN_BUDGET_MS}ms total (${(await diagnosePort(true)).text}) (log: ${logPath}${logTail()})`, + `bili: spawned proxy not healthy after ${SPAWN_WAIT_MS}ms — child pid ${child.pid ?? "?"} is still starting up; extending wait to ${SPAWN_BUDGET_MS}ms total (${(await diagnosePort(true)).text}; at spawn: ${initialPortNote}) (log: ${logPath}${logTail()})`, ); } await sleepImpl(HEALTH_POLL_INTERVAL_MS); const inst = readInstance(); if (isProxyInstanceFile(inst) && inst.launchToken === launchToken) { - if (await probeHealth(inst.origin, fetchImpl)) { + // #4125: the same identity discipline as the preferred-port + // fallback below — a healthy answer at the origin the child + // CLAIMED is proof of our child only while it does not positively + // name a different pid. A foreign responder that took over that + // address must not be exported to the client as ours (one writer + // per copy, #991/#1225); absent pid info keeps legacy semantics. + const claimed = await fetchHealthInfo(inst.origin); + const healthy = claimed?.ok ?? (await probeHealth(inst.origin, fetchImpl)); + if (healthy && (claimed?.pid === undefined || child.pid === undefined || claimed.pid === child.pid)) { // #1660: settle the lane's sticky record on the port the // child actually bound (preferred or laddered) so every // later launch of this lane tries it first. @@ -5494,6 +5542,12 @@ export async function ensureProxyRunning( startOwnerHeartbeat(inst.origin, watchPid); return { origin: inst.origin, port: inst.port, child, logPath }; } + if (healthy && claimed?.pid !== undefined && child.pid !== undefined && !contestedAnnounced) { + contestedAnnounced = true; + attachDiag( + `bili: ${inst.origin} answers health (pid ${claimed.pid}) but this launcher's child is pid ${child.pid} — refusing to claim a contested origin (#4125)`, + ); + } continue; } // #2954: the single proxy-origin file is last-writer-wins machine-wide — @@ -5506,10 +5560,24 @@ export async function ensureProxyRunning( // the preferred-port fallback below, minus the port assumption. if (child.pid !== undefined) { const mine = listRegistry().find((e) => e.pid === child.pid); - if (mine && mine.origin !== "" && (await probeHealth(mine.origin, fetchImpl))) { - settleZonePort(mine.port); - startOwnerHeartbeat(mine.origin, watchPid); - return { origin: mine.origin, port: mine.port, child, logPath }; + // #4125: same contested-origin discipline as the launchToken + // branch above — the registry record vouches for the pid, but a + // healthy answer at that origin naming a DIFFERENT pid is a + // foreign responder, not our child. + if (mine && mine.origin !== "") { + const claimed = await fetchHealthInfo(mine.origin); + const healthy = claimed?.ok ?? (await probeHealth(mine.origin, fetchImpl)); + if (healthy && (claimed?.pid === undefined || claimed.pid === child.pid)) { + settleZonePort(mine.port); + startOwnerHeartbeat(mine.origin, watchPid); + return { origin: mine.origin, port: mine.port, child, logPath }; + } + if (healthy && claimed?.pid !== undefined && !contestedAnnounced) { + contestedAnnounced = true; + attachDiag( + `bili: ${mine.origin} answers health (pid ${claimed.pid}) but this launcher's child is pid ${child.pid} — refusing to claim a contested origin (#4125)`, + ); + } } } // Fallback for a child that cannot write the instance file (broken @@ -5631,7 +5699,7 @@ export async function ensureProxyRunning( const detail = childExit.code !== null ? `code ${childExit.code}${childExit.code === TWIN_ADOPT_EXIT_CODE ? " (twin-collision signal)" : ""}` : childExit.signal ? `signal ${childExit.signal}` : "unknown reason"; - throw new Error(`bili: proxy child exited before becoming healthy (${detail}) — ${(await diagnosePort(false)).text} (runtime: ${runtime}) (log: ${logPath}${logTail()})${failureHint()}`); + throw new Error(`bili: proxy child exited before becoming healthy (${detail}) — ${(await diagnosePort(false)).text} (at spawn: ${initialPortNote}) (runtime: ${runtime}) (log: ${logPath}${logTail()})${failureHint()}`); } // #4994: where THIS child says it bound, parsed from its own log // segment ([spawnLogOffset..end] — no other spawn's lines can answer; @@ -5704,9 +5772,9 @@ export async function ensureProxyRunning( } killAbandonedChild(child); console.error( - `bili: killed spawned proxy (pid ${child.pid ?? "?"}) after ${SPAWN_BUDGET_MS}ms without health — ${diagnosis.text} (log: ${logPath}${logTail()})`, + `bili: killed spawned proxy (pid ${child.pid ?? "?"}) after ${SPAWN_BUDGET_MS}ms without health — ${diagnosis.text} (at spawn: ${initialPortNote}) (log: ${logPath}${logTail()})`, ); - throw new Error(`bili: proxy did not become healthy within ${SPAWN_BUDGET_MS}ms — ${diagnosis.text} (runtime: ${runtime}) (log: ${logPath}${logTail()})${failureHint()}`); + throw new Error(`bili: proxy did not become healthy within ${SPAWN_BUDGET_MS}ms — ${diagnosis.text} (at spawn: ${initialPortNote}) (runtime: ${runtime}) (log: ${logPath}${logTail()})${failureHint()}`); } finally { if (claimed) clearStartingMarker(launchToken); } diff --git a/src/port-owner.ts b/src/port-owner.ts index 36432bc8fa..843bad1aa4 100644 --- a/src/port-owner.ts +++ b/src/port-owner.ts @@ -98,10 +98,13 @@ export async function findForeignOwner( // own pre-registration 503s and delayed pointer/registry publication // past callers that expect registration right after bind (#394 warn // timing in multi-instance-live). Bind is exclusive on every platform - // bili ships on (Linux EADDRINUSE; Windows per libuv issue #1360), so no - // foreign listener can be in that state here; a hypothetical dual-bind - // co-starter would be unregistered anyway — the registry/pointer layer - // surfaces the collision at registration time. Never foreign. + // bili ships on (Linux EADDRINUSE; Windows — libuv deliberately omits + // SO_REUSEADDR there because setting it would allow stealing a port + // held by another application, libuv src/win/tcp.c uv__tcp_try_bind), + // so no foreign listener can be in that state here; a hypothetical + // dual-bind co-starter would be unregistered anyway — the registry/ + // pointer layer surfaces the collision at registration time. Never + // foreign. if (info?.reason === "starting") return undefined; } return undefined; diff --git a/src/server.ts b/src/server.ts index b716992248..7c063b5164 100644 --- a/src/server.ts +++ b/src/server.ts @@ -1856,6 +1856,15 @@ export async function startServer(opts: ProxyOptions): Promise { hbStreaks.delete(pid); log("info", `[watchdog] pid ${pid} heartbeating again — clearing stale-heartbeat streak (#4994)`); } + } else if (decision.heartbeatVeto) { + // #4125: fresh beat vs failed existence probe — announce the + // disagreement once per episode so the field can attribute + // what the probe actually failed with (same errno clue as + // the #3890 line below). + if ((deadStreak.get(pid) ?? 0) === 0) { + log("info", `[watchdog] kill-0 probe failed for pid ${pid} (code=${probe.errCode ?? "?"}) but its heartbeat beat recently — trusting the identity-anchored heartbeat, failure not counted (#4125/#3890)`); + } + deadStreak.delete(pid); } else { if (decision.streak === 1) { // Field instrumentation (#3890): Windows collapses every @@ -2262,16 +2271,26 @@ interface WatcherTickDecision { streak: number; /** Stale-while-alive streak after this tick (0 when the beat is fresh or removed). #4994. */ hbStreak: number; + /** #4125: the probe failed but a FRESH heartbeat vouched for the pid — the failure was neither counted nor removing. */ + heartbeatVeto?: boolean; } -/** #3890/#4994: pure per-pid death-streak step. Each axis confirms - * independently over CONSECUTIVE agreeing ticks: a failed kill-0 probe +/** #3890/#4994/#4125: pure per-pid death-streak step. Two axes confirm + * independently over CONSECUTIVE agreeing ticks (#4994): a failed kill-0 probe * increments its streak and only removes at confirmTicks; a stale-while-alive * reading increments the heartbeat streak and only removes at * heartbeatConfirmTicks. A live probe resets the kill-0 axis, a fresh beat * resets the heartbeat axis, and a tick that disagrees with an axis breaks * THAT axis's run (episodes never accumulate across interruptions). Legacy - * pids (no heartbeat record) keep kill-0-only semantics (#3544). */ + * pids (no heartbeat record) keep kill-0-only semantics (#3544). On top of + * that, #4125: a FAILED probe on a pid whose last beat is still FRESH is + * vetoed entirely — not counted, not removed — because the beat is + * identity-anchored (only the true owner posts its own pid over loopback) + * while kill-0 is an identity-blind existence check; disagreement resolves in + * favor of the stronger proof. Field case: three consecutive transient + * OpenProcess failures against a LIVE host burned the whole confirm window + * and force-exited a healthy proxy mid-session; the same host pid then + * re-armed fine on the replacement proxy, proving it never died. */ export function tickWatchedPid(probe: PidProbe, lastSeen: number | undefined, nowMs: number, prevStreak: number | undefined, prevHbStreak: number | undefined, opts: { heartbeatGraceMs: number; confirmTicks: number; heartbeatConfirmTicks: number }): WatcherTickDecision { if (probe.alive) { const staleHeartbeat = watcherIsDead(true, lastSeen, nowMs, opts.heartbeatGraceMs); @@ -2281,6 +2300,9 @@ export function tickWatchedPid(probe: PidProbe, lastSeen: number | undefined, no ? { removed: true, staleHeartbeat: true, streak: 0, hbStreak: 0 } : { removed: false, staleHeartbeat: true, streak: 0, hbStreak }; } + if (lastSeen !== undefined && nowMs - lastSeen <= opts.heartbeatGraceMs) { + return { removed: false, staleHeartbeat: false, streak: 0, hbStreak: 0, heartbeatVeto: true }; + } const streak = (prevStreak ?? 0) + 1; return streak >= opts.confirmTicks ? { removed: true, staleHeartbeat: false, streak: 0, hbStreak: 0 } diff --git a/tests/issue3116-spawn-failure-diagnostics.test.ts b/tests/issue3116-spawn-failure-diagnostics.test.ts index 0f96a2cbe2..e82e9e456b 100644 --- a/tests/issue3116-spawn-failure-diagnostics.test.ts +++ b/tests/issue3116-spawn-failure-diagnostics.test.ts @@ -94,7 +94,8 @@ test("#3116 timeout with nothing listening: error says so, names the live child, const msg = (err as Error).message; assert.match(msg, new RegExp(`did not become healthy within ${SPAWN_BUDGET_MS}ms`)); assert.ok(msg.includes(`nothing is listening on port ${PORT} — the spawned child (still alive, pid 42471) never bound it`), msg); - assert.ok(msg.includes("0 bytes — EMPTY, the child produced no output of its own"), msg); + // #4125: the size is now scoped to THIS spawn's segment of the shared file. + assert.ok(msg.includes("0 bytes since this spawn — EMPTY"), msg); assert.deepEqual(kills, ["SIGKILL"], "the abandoned child must be hard-killed exactly once"); const slowStart = stderrLines.find((l) => l.includes("extending wait")); assert.ok(slowStart, "the slow-start notice must still fire"); @@ -211,7 +212,8 @@ test("#3116 early exit: code stays in the message, diagnosis marks the child alr const msg = (err as Error).message; assert.match(msg, /exited before becoming healthy \(code 1\)/); assert.ok(msg.includes(`nothing is listening on port ${PORT} — the spawned child (already exited, pid 42474) never bound it`), msg); - assert.ok(msg.includes("0 bytes — EMPTY, the child produced no output of its own"), msg); + // #4125: same scoped-wording change as the timeout case above. + assert.ok(msg.includes("0 bytes since this spawn — EMPTY"), msg); }); test("#3116 describeProbeFailure extracts the socket error class", () => { diff --git a/tests/issue3544-stale-proxy-rotation.test.ts b/tests/issue3544-stale-proxy-rotation.test.ts index b4ce5d9527..b14aa39515 100644 --- a/tests/issue3544-stale-proxy-rotation.test.ts +++ b/tests/issue3544-stale-proxy-rotation.test.ts @@ -111,11 +111,13 @@ async function runBringUp(t: BringUpCase) { const clock = fakeClock(); const shutdownCalls: string[] = []; let spawnCalls = 0; + let spawned = false; let staleDead = false; const stalePort = t.stalePort ?? 18787; const staleOrigin = `http://127.0.0.1:${stalePort}`; const spawnImpl: SpawnFn = () => { spawnCalls += 1; + spawned = true; return makeFakeChild(CHILD_PID); }; const handle = await ensureProxyRunning( @@ -124,6 +126,10 @@ async function runBringUp(t: BringUpCase) { { fetchImpl: async (url) => ({ ok: url.startsWith(ORIGIN) }), fetchHealthInfo: async (origin) => { + // #4125: post-spawn, the settle branches verify responder identity + // through health INFO (not the bare probe) — model the real child, + // whose /__bili/health names its own pid. + if (spawned && origin === ORIGIN) return { ok: true, pid: CHILD_PID }; if (origin !== staleOrigin || staleDead) return { ok: false }; const info: { ok: boolean; instanceId: string; pid: number; watchdog: { armed: boolean }; version?: string } = { ok: true, diff --git a/tests/issue4125-settle-identity.test.ts b/tests/issue4125-settle-identity.test.ts new file mode 100644 index 0000000000..c7fdabbe43 --- /dev/null +++ b/tests/issue4125-settle-identity.test.ts @@ -0,0 +1,227 @@ +// #4125: the spawn-wait settle loop's PRIMARY branches (launchToken handshake, +// registry-by-pid) used to trust ANY healthy answer at the origin our child +// CLAIMED — while the preferred-port fallback already verified the responder's +// pid (#1753). A foreign responder that took over that address (port-stealing +// non-libuv process, cross-build confusion) was exported to the client as ours: +// one writer per copy violated (#991/#1225). These tests pin BOTH directions of +// the gate (#3617): own/absent pid settles, a positively-foreign pid never does. +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import type { ProxyInstanceFile as InstanceFile } from "../src/instance.ts"; +import { ensureProxyRunning, type SpawnChild, type SpawnFn } from "../src/launcher.ts"; + +const FP_SCRIPT = path.join(os.tmpdir(), `bili-4125-fp-${process.pid}.js`); +fs.writeFileSync(FP_SCRIPT, "// fingerprint fixture\n"); +const FP_HASH = createHash("sha256").update(fs.readFileSync(FP_SCRIPT)).digest("hex"); + +function makeFakeChild(pid: number): SpawnChild { + const handlers = new Map void)[]>(); + return { + pid, + unref() {}, + kill() { + return true; + }, + on(event, listener) { + const list = handlers.get(event) ?? []; + list.push(listener); + handlers.set(event, list); + return this; + }, + }; +} + +function recordedInstance(over: Partial = {}): InstanceFile { + return { + origin: "http://127.0.0.1:8787", + instanceId: "inst-1", + pid: process.pid, + startedAt: Date.now(), + host: "127.0.0.1", + port: 8787, + passthrough: false, + mitmDomains: [], + modelWindows: {}, + codeFingerprint: FP_HASH, + ...over, + }; +} + +test("ensureProxyRunning: launchToken record settles when the claimed origin's health names the spawned child (#4125)", async () => { + let childToken = ""; + const handle = await ensureProxyRunning( + { host: "127.0.0.1", port: 8787, passthrough: false, debug: false }, + { + fetchImpl: async () => ({ ok: true }), + fetchHealthInfo: async (origin) => + origin === "http://127.0.0.1:8795" ? { ok: true, pid: 42470 } : { ok: false }, + spawnImpl: (_cmd, _args, options) => { + childToken = options.env?.BILI_LAUNCH_TOKEN ?? ""; + return makeFakeChild(42470); + }, + sleep: () => Promise.resolve(), + scriptPath: FP_SCRIPT, + listRegistryInstances: () => [], + readInstanceFile: () => + childToken + ? recordedInstance({ origin: "http://127.0.0.1:8795", port: 8795, launchToken: childToken }) + : undefined, + }, + ); + assert.equal(handle.origin, "http://127.0.0.1:8795"); + assert.equal(handle.port, 8795); +}); + +test("ensureProxyRunning: a foreign responder on the child's CLAIMED origin is not exported as ours (#4125)", async () => { + let childToken = ""; + const diag: string[] = []; + let ticks = 0; + await assert.rejects( + ensureProxyRunning( + { host: "127.0.0.1", port: 8787, passthrough: false, debug: false }, + { + fetchImpl: async () => ({ ok: true }), + // The child's own record points at 8796 — but the answer there + // comes from ANOTHER process's pid. + fetchHealthInfo: async (origin) => + origin === "http://127.0.0.1:8796" ? { ok: true, pid: 875250 } : { ok: false }, + spawnImpl: (_cmd, _args, options) => { + childToken = options.env?.BILI_LAUNCH_TOKEN ?? ""; + return makeFakeChild(42471); + }, + attachDiag: (m) => diag.push(m), + now: () => ticks * 1000, + sleep: () => { + ticks += 10; + return Promise.resolve(); + }, + scriptPath: FP_SCRIPT, + listRegistryInstances: () => [], + readInstanceFile: () => + childToken + ? recordedInstance({ origin: "http://127.0.0.1:8796", port: 8796, launchToken: childToken }) + : undefined, + }, + ), + /did not become healthy/, + ); + const contested = diag.filter((m) => m.includes("refusing to claim a contested origin")); + assert.equal(contested.length, 1, "the contested-origin diagnostic fires exactly once"); + assert.match(contested[0], /pid 875250/); + assert.match(contested[0], /child is pid 42471/); +}); + +test("ensureProxyRunning: a claimed-origin responder WITHOUT a pid still settles (legacy fail-open, #4125)", async () => { + let childToken = ""; + const handle = await ensureProxyRunning( + { host: "127.0.0.1", port: 8787, passthrough: false, debug: false }, + { + fetchImpl: async () => ({ ok: true }), + fetchHealthInfo: async (origin) => + origin === "http://127.0.0.1:8797" ? { ok: true } : { ok: false }, + spawnImpl: (_cmd, _args, options) => { + childToken = options.env?.BILI_LAUNCH_TOKEN ?? ""; + return makeFakeChild(42472); + }, + sleep: () => Promise.resolve(), + scriptPath: FP_SCRIPT, + listRegistryInstances: () => [], + readInstanceFile: () => + childToken + ? recordedInstance({ origin: "http://127.0.0.1:8797", port: 8797, launchToken: childToken }) + : undefined, + }, + ); + assert.equal(handle.origin, "http://127.0.0.1:8797"); + assert.equal(handle.port, 8797); +}); + +test("ensureProxyRunning: registry-by-pid settles when that entry's origin names the spawned child (#4125)", async () => { + let spawned = false; + const handle = await ensureProxyRunning( + { host: "127.0.0.1", port: 8787, passthrough: false, debug: false }, + { + fetchImpl: async () => ({ ok: true }), + fetchHealthInfo: async (origin) => + origin === "http://127.0.0.1:8798" ? { ok: true, pid: 42473 } : { ok: false }, + spawnImpl: () => { + spawned = true; + return makeFakeChild(42473); + }, + sleep: () => Promise.resolve(), + scriptPath: FP_SCRIPT, + listRegistryInstances: () => + spawned + ? [recordedInstance({ instanceId: "reg-1", pid: 42473, origin: "http://127.0.0.1:8798", port: 8798 })] + : [], + readInstanceFile: () => undefined, + }, + ); + assert.equal(handle.origin, "http://127.0.0.1:8798"); + assert.equal(handle.port, 8798); +}); + +test("ensureProxyRunning: a foreign responder on the registry entry's origin ends in a loud failure, not a misroute (#4125)", async () => { + let spawned = false; + const diag: string[] = []; + let ticks = 0; + await assert.rejects( + ensureProxyRunning( + { host: "127.0.0.1", port: 8787, passthrough: false, debug: false }, + { + fetchImpl: async () => ({ ok: true }), + fetchHealthInfo: async (origin) => + origin === "http://127.0.0.1:8799" ? { ok: true, pid: 875250 } : { ok: false }, + spawnImpl: () => { + spawned = true; + return makeFakeChild(42474); + }, + attachDiag: (m) => diag.push(m), + now: () => ticks * 1000, + sleep: () => { + ticks += 10; + return Promise.resolve(); + }, + scriptPath: FP_SCRIPT, + listRegistryInstances: () => + spawned + ? [recordedInstance({ instanceId: "reg-1", pid: 42474, origin: "http://127.0.0.1:8799", port: 8799 })] + : [], + readInstanceFile: () => undefined, + }, + ), + /did not become healthy/, + ); + const contested = diag.filter((m) => m.includes("refusing to claim a contested origin")); + assert.equal(contested.length, 1, "the contested-origin diagnostic fires exactly once"); + assert.match(contested[0], /pid 875250/); + assert.match(contested[0], /child is pid 42474/); +}); + +test("ensureProxyRunning: health-info probe failure falls back to the legacy boolean probe (#4125)", async () => { + let childToken = ""; + const handle = await ensureProxyRunning( + { host: "127.0.0.1", port: 8787, passthrough: false, debug: false }, + { + fetchImpl: async () => ({ ok: true }), + fetchHealthInfo: async () => undefined, + spawnImpl: (_cmd, _args, options) => { + childToken = options.env?.BILI_LAUNCH_TOKEN ?? ""; + return makeFakeChild(42475); + }, + sleep: () => Promise.resolve(), + scriptPath: FP_SCRIPT, + listRegistryInstances: () => [], + readInstanceFile: () => + childToken + ? recordedInstance({ origin: "http://127.0.0.1:8800", port: 8800, launchToken: childToken }) + : undefined, + }, + ); + assert.equal(handle.origin, "http://127.0.0.1:8800"); + assert.equal(handle.port, 8800); +}); diff --git a/tests/issue4125-spawn-log-segment.test.ts b/tests/issue4125-spawn-log-segment.test.ts new file mode 100644 index 0000000000..d598fc1165 --- /dev/null +++ b/tests/issue4125-spawn-log-segment.test.ts @@ -0,0 +1,165 @@ +// #4125: the port log (%TEMP%/bili-proxy-.log) is SHARED across spawns +// (append fd, size-based rotation only). Field case (2026-10-25): a failed +// child wrote 0 bytes, so the failure message quoted the PREVIOUS spawn's +// content — a week-old EADDRINUSE line presented as the current failure's +// evidence — and nothing recorded what the port was doing at SPAWN time. +// Fix: each spawn appends a `=== bili spawn ... ===` header, samples/sizes +// are scoped to this spawn's segment ("N bytes since this spawn"), and every +// terminal failure message carries "(at spawn: )". All probes go +// through injected seams — no real network I/O. + +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { ensureProxyRunning, SPAWN_BUDGET_MS, type ProbeOutcome, type SpawnChild, type SpawnFn } from "../src/launcher.ts"; + +const PORT = 47891; +const LOG_FILE = path.join(os.tmpdir(), `bili-proxy-${PORT}.log`); +const LOG_OLD = `${LOG_FILE}.old`; +const STALE_MARKER = "EADDRINUSE stale evidence from a previous week (pid 38768)"; + +function cleanupLogs(): void { + for (const f of [LOG_FILE, LOG_OLD]) { + try { fs.unlinkSync(f); } catch {} + } +} + +function fakeClock() { + let ticks = 0; + return { + now: () => ticks * 1000, + sleep: () => { + ticks += 1; + return Promise.resolve(); + }, + }; +} + +function makeAliveChild(pid: number): SpawnChild { + return { pid, unref() {}, kill() { return true; }, on() {} }; +} + +test("#4125 zero-output child with stale shared content: message says 0-bytes-since-spawn, never quotes the stale half", async () => { + cleanupLogs(); + fs.writeFileSync(LOG_FILE, `old line\n${STALE_MARKER}\n`); + const clock = fakeClock(); + let err: unknown; + try { + await ensureProxyRunning( + { host: "127.0.0.1", port: PORT, passthrough: false, debug: false }, + { + fetchImpl: async () => ({ ok: false }), + fetchHealthInfo: async () => undefined, + fetchDetailImpl: async (): Promise => ({ ok: false, detail: "ECONNREFUSED" }), + spawnImpl: (() => makeAliveChild(42481)) as SpawnFn, + now: clock.now, + sleep: clock.sleep, + readInstanceFile: () => undefined, + }, + ); + assert.fail("expected rejection"); + } catch (e) { + err = e; + } finally { + const content = (() => { try { return fs.readFileSync(LOG_FILE, "utf8"); } catch { return ""; } })(); + cleanupLogs(); + const msg = (err as Error).message; + assert.match(msg, new RegExp(`did not become healthy within ${SPAWN_BUDGET_MS}ms`)); + assert.ok(msg.includes("0 bytes since this spawn — EMPTY"), msg); + assert.ok(!msg.includes(STALE_MARKER), "stale previous-spawn content must not be quoted as this failure's evidence"); + assert.ok(content.includes("=== bili spawn"), "the spawn header must mark this segment in the shared file"); + assert.ok(content.includes(`port=${PORT}`), "the header names the port for attribution (childPid is unknown pre-spawn)"); + assert.ok(content.includes(STALE_MARKER), "previous content stays on disk (rotation policy unchanged) — only the QUOTING changed"); + } +}); + +test("#4125 child that writes early: the quoted head comes from THIS spawn's segment", async () => { + cleanupLogs(); + fs.writeFileSync(LOG_FILE, `${STALE_MARKER}\n`); + const clock = fakeClock(); + let err: unknown; + try { + await ensureProxyRunning( + { host: "127.0.0.1", port: PORT, passthrough: false, debug: false }, + { + fetchImpl: async () => ({ ok: false }), + fetchHealthInfo: async () => undefined, + fetchDetailImpl: async (): Promise => ({ ok: false, detail: "ECONNREFUSED" }), + spawnImpl: (() => { + // A real child cannot write before the parent returns from + // spawn() and appends the header — defer like one would. + queueMicrotask(() => fs.appendFileSync(LOG_FILE, "child stdout line\n")); + return makeAliveChild(42482); + }) as SpawnFn, + now: clock.now, + sleep: clock.sleep, + readInstanceFile: () => undefined, + }, + ); + assert.fail("expected rejection"); + } catch (e) { + err = e; + } finally { + cleanupLogs(); + const msg = (err as Error).message; + assert.ok(msg.includes("bytes since this spawn"), msg); + assert.ok(msg.includes('child output head: "child stdout line"'), msg); + assert.ok(!msg.includes(STALE_MARKER), msg); + } +}); + +test("#4125 at-spawn port state: a healthy foreign instance holding the planned port is named", async () => { + cleanupLogs(); + const clock = fakeClock(); + let err: unknown; + try { + await ensureProxyRunning( + { host: "127.0.0.1", port: PORT, passthrough: false, debug: false }, + { + fetchImpl: async () => ({ ok: true }), + fetchHealthInfo: async () => ({ ok: true, pid: 99999 }), + fetchDetailImpl: async (): Promise => ({ ok: true }), + spawnImpl: (() => makeAliveChild(42483)) as SpawnFn, + now: clock.now, + sleep: clock.sleep, + readInstanceFile: () => undefined, + }, + ); + assert.fail("expected rejection"); + } catch (e) { + err = e; + } finally { + cleanupLogs(); + const msg = (err as Error).message; + assert.ok(msg.includes("(at spawn: healthy bili instance (pid 99999))"), msg); + } +}); + +test("#4125 at-spawn port state: an empty port reads 'nothing listening'", async () => { + cleanupLogs(); + const clock = fakeClock(); + let err: unknown; + try { + await ensureProxyRunning( + { host: "127.0.0.1", port: PORT, passthrough: false, debug: false }, + { + fetchImpl: async () => ({ ok: false }), + fetchHealthInfo: async () => undefined, + fetchDetailImpl: async (): Promise => ({ ok: false, detail: "ECONNREFUSED" }), + spawnImpl: (() => makeAliveChild(42484)) as SpawnFn, + now: clock.now, + sleep: clock.sleep, + readInstanceFile: () => undefined, + }, + ); + assert.fail("expected rejection"); + } catch (e) { + err = e; + } finally { + cleanupLogs(); + const msg = (err as Error).message; + assert.ok(msg.includes("(at spawn: nothing listening)"), msg); + } +}); diff --git a/tests/issue4125-watchdog-heartbeat-veto.test.ts b/tests/issue4125-watchdog-heartbeat-veto.test.ts new file mode 100644 index 0000000000..ba4c39f828 --- /dev/null +++ b/tests/issue4125-watchdog-heartbeat-veto.test.ts @@ -0,0 +1,78 @@ +// #4125: the parent-gone watchdog had TWO disagreeing liveness signals — the +// identity-blind kill-0 existence probe and the identity-anchored owner +// heartbeat (#3544) — and the #3890 kill-0 death streak ran regardless of +// heartbeat freshness. Field case (2026-10-25, DSH web host): three +// consecutive transient OpenProcess failures against a LIVE host burned the +// whole confirm window (~6s) and force-exited a healthy, compressing proxy +// mid-session; the same host pid then re-armed fine on the replacement proxy, +// proving it never died. The shutdown reason string carried no +// [stale-heartbeat] suffix, pinning this to the kill-0 streak path. +// Fix: a FRESH heartbeat vetoes failed probes entirely — the beat is the +// stronger proof (only the true owner posts its own pid over loopback). +// Stale-heartbeat removal is tick-confirmed per #4994 (immediate removal let +// a recycled pid exit ~8s early in one field case); legacy kill-0-only +// semantics are unchanged. Both axes pinned below. + +import test from "node:test"; +import assert from "node:assert/strict"; +import { tickWatchedPid } from "../src/server.ts"; + +const NOW = 1_000_000; +const OPTS = { heartbeatGraceMs: 15_000, confirmTicks: 3, heartbeatConfirmTicks: 5 }; +const dead = (errCode?: string) => ({ alive: false, errCode }); +const aliveProbe = { alive: true }; + +test("#4125 veto: fresh heartbeats keep a pid out of the death streak across consecutive failed probes", () => { + let d = tickWatchedPid(dead("ESRCH"), NOW - 10_000, NOW, undefined, undefined, OPTS); + assert.deepEqual(d, { removed: false, staleHeartbeat: false, streak: 0, hbStreak: 0, heartbeatVeto: true }); + d = tickWatchedPid(dead("ESRCH"), NOW - 12_000, NOW + 2_000, d.streak, undefined, OPTS); + assert.equal(d.removed, false, "second consecutive failure still vetoed while the beat ages but stays fresh"); + assert.equal(d.heartbeatVeto, true); + d = tickWatchedPid(dead("ESRCH"), NOW - 14_000, NOW + 4_000, d.streak, undefined, OPTS); + assert.equal(d.removed, false, "third failure too — the confirm window cannot burn under a live heartbeat"); +}); + +test("#4125 veto boundary: a beat exactly at the grace edge is still fresh; one past it counts again", () => { + const at = tickWatchedPid(dead(), NOW - 15_000, NOW, undefined, undefined, OPTS); + assert.equal(at.heartbeatVeto, true, "age == grace counts as fresh (<=)"); + const past = tickWatchedPid(dead(), NOW - 15_001, NOW, undefined, undefined, OPTS); + assert.equal(past.heartbeatVeto, undefined, "stale beat falls back to the #3890 streak"); + assert.equal(past.streak, 1); +}); + +test("#4125 veto reset: counting resumes from 1 after vetoes, but three unvetoed failures still remove", () => { + let d = tickWatchedPid(dead(), NOW - 10_000, NOW, undefined, undefined, OPTS); + assert.equal(d.heartbeatVeto, true); + d = tickWatchedPid(dead(), NOW - 20_000, NOW + 2_000, d.streak, undefined, OPTS); + assert.equal(d.streak, 1, "beat went stale → counting resumes, not accumulating pre-veto history"); + d = tickWatchedPid(dead(), NOW - 21_000, NOW + 4_000, d.streak, undefined, OPTS); + assert.equal(d.streak, 2); + d = tickWatchedPid(dead(), NOW - 22_000, NOW + 6_000, d.streak, undefined, OPTS); + assert.equal(d.removed, true, "a genuinely silent owner still exits promptly"); + assert.equal(d.staleHeartbeat, false, "removal here is the kill-0 verdict, not the heartbeat one"); +}); + +test("#4125 regression pin: legacy pids (no heartbeat record) keep #3890 streak semantics exactly", () => { + let d = tickWatchedPid(dead("ESRCH"), undefined, NOW, undefined, undefined, OPTS); + assert.equal(d.heartbeatVeto, undefined); + assert.equal(d.streak, 1); + d = tickWatchedPid(dead("ESRCH"), undefined, NOW + 2_000, d.streak, undefined, OPTS); + assert.equal(d.streak, 2); + d = tickWatchedPid(dead("ESRCH"), undefined, NOW + 4_000, d.streak, undefined, OPTS); + assert.deepEqual(d, { removed: true, staleHeartbeat: false, streak: 0, hbStreak: 0 }); +}); + +test("#4994 dual-axis pin: an ALIVE process with a stale heartbeat removes only after confirmed ticks", () => { + assert.deepEqual(tickWatchedPid(aliveProbe, NOW - 1_000, NOW, undefined, undefined, OPTS), { removed: false, staleHeartbeat: false, streak: 0, hbStreak: 0 }); + // a live probe with a fresh beat clears any prior counting on BOTH axes + assert.deepEqual(tickWatchedPid(aliveProbe, NOW - 1_000, NOW, 2, 4, OPTS), { removed: false, staleHeartbeat: false, streak: 0, hbStreak: 0 }); + let d = tickWatchedPid(aliveProbe, NOW - 16_000, NOW, undefined, undefined, OPTS); + assert.deepEqual(d, { removed: false, staleHeartbeat: true, streak: 0, hbStreak: 1 }, "first stale-beat tick flags but does not remove"); + for (let i = 2; i <= 4; i++) { + d = tickWatchedPid(aliveProbe, NOW - 16_000 - i * 1_000, NOW + i * 1_000, d.streak, d.hbStreak, OPTS); + assert.equal(d.removed, false, `stale-beat tick ${i} must not remove yet`); + assert.equal(d.hbStreak, i); + } + d = tickWatchedPid(aliveProbe, NOW - 21_000, NOW + 5_000, d.streak, d.hbStreak, OPTS); + assert.deepEqual(d, { removed: true, staleHeartbeat: true, streak: 0, hbStreak: 0 }, "fifth confirmed stale beat removes"); +}); diff --git a/tests/issue4811-dsh-busy-drop-selfheal.test.ts b/tests/issue4811-dsh-busy-drop-selfheal.test.ts index 4e526086d9..7d4a4dba30 100644 --- a/tests/issue4811-dsh-busy-drop-selfheal.test.ts +++ b/tests/issue4811-dsh-busy-drop-selfheal.test.ts @@ -362,7 +362,7 @@ test("B: the failure report samples this spawn's output, not offset 0 of a rotat const err2 = await pending2.then(() => undefined, (e: unknown) => e); assert.ok(err2 instanceof Error, "the silent child must still fail the bring-up"); assert.equal(secondSpawnCalls, 1); - assert.match(err2.message, /this child appended nothing of its own/, "a zero-output child must be said so, not quoted from history"); + assert.match(err2.message, /0 bytes since this spawn — EMPTY/, "a zero-output child must be said so, not quoted from history"); assert.doesNotMatch(err2.message, /OLD NOISE|NEW LINE/, "an earlier spawn's bytes must never be quoted as this child's output"); } finally { for (const [k, v] of Object.entries(prev)) if (v === undefined) delete process.env[k]; else process.env[k] = v;