From 3ededa8c89af54c8efca1449bde1fbaa66cf7d7c Mon Sep 17 00:00:00 2001 From: ci-check Date: Sun, 25 Oct 2026 05:06:29 +0800 Subject: [PATCH 1/4] fix: verify responder identity before claiming a settled proxy origin (#4125) The ensureProxyRunning settle loop trusted any healthy /__bili/health answer at the child's claimed origin in its two primary branches (launchToken match, registry-by-pid), while only the stale-record fallback verified the responder pid (#1753). A foreign listener positively answering with a different pid therefore counted as success: the launcher would hand that origin out as our own (single-writer violation) or burn the full spawn budget probing a contested port. Both primary branches now use fetchHealthInfo and accept only when the reported pid is absent (legacy fail-open preserved) or equals the spawned child's pid; a positive foreign-pid answer is rejected with a once-diagnostic naming both pids. Terminal paths unchanged. Also corrects the stale libuv-issue citation in port-owner.ts (libuv omits SO_REUSEADDR on Windows precisely because setting it allows stealing another application's port). Tests: tests/issue4125-settle-identity.test.ts pins accept/reject/fail-open per branch (both gate directions, #3617 discipline); launcher suite green. --- src/launcher.ts | 41 ++++- src/port-owner.ts | 11 +- tests/issue4125-settle-identity.test.ts | 227 ++++++++++++++++++++++++ 3 files changed, 270 insertions(+), 9 deletions(-) create mode 100644 tests/issue4125-settle-identity.test.ts diff --git a/src/launcher.ts b/src/launcher.ts index 3b520a9714..ded537624b 100644 --- a/src/launcher.ts +++ b/src/launcher.ts @@ -4999,6 +4999,9 @@ export async function ensureProxyRunning( // #1753: announce a healthy-but-foreign responder on the preferred port // at most once — silence would hide exactly the misroute this fix closes. let squatterAnnounced = false; + // #4125: same discipline for the PRIMARY settle branches (launchToken / + // registry-by-pid) — announce a contested claimed origin at most once. + let contestedAnnounced = false; child.on?.("exit", (...rest: unknown[]) => { childExit = { code: typeof rest[0] === "number" ? rest[0] : null, @@ -5103,7 +5106,15 @@ export async function ensureProxyRunning( await sleepImpl(HEALTH_POLL_INTERVAL_MS); const inst = readInstance(); if (isProxyInstanceFile(inst) && inst.launchToken === launchToken) { - if (await probeHealth(inst.origin, fetchImpl)) { + // #4125: the same identity discipline as the preferred-port + // fallback below — a healthy answer at the origin the child + // CLAIMED is proof of our child only while it does not positively + // name a different pid. A foreign responder that took over that + // address must not be exported to the client as ours (one writer + // per copy, #991/#1225); absent pid info keeps legacy semantics. + const claimed = await fetchHealthInfo(inst.origin); + const healthy = claimed?.ok ?? (await probeHealth(inst.origin, fetchImpl)); + if (healthy && (claimed?.pid === undefined || child.pid === undefined || claimed.pid === child.pid)) { // #1660: settle the lane's sticky record on the port the // child actually bound (preferred or laddered) so every // later launch of this lane tries it first. @@ -5111,6 +5122,12 @@ export async function ensureProxyRunning( startOwnerHeartbeat(inst.origin, watchPid); return { origin: inst.origin, port: inst.port, child, logPath }; } + if (healthy && claimed?.pid !== undefined && child.pid !== undefined && !contestedAnnounced) { + contestedAnnounced = true; + attachDiag( + `bili: ${inst.origin} answers health (pid ${claimed.pid}) but this launcher's child is pid ${child.pid} — refusing to claim a contested origin (#4125)`, + ); + } continue; } // #2954: the single proxy-origin file is last-writer-wins machine-wide — @@ -5123,10 +5140,24 @@ export async function ensureProxyRunning( // the preferred-port fallback below, minus the port assumption. if (child.pid !== undefined) { const mine = listRegistry().find((e) => e.pid === child.pid); - if (mine && mine.origin !== "" && (await probeHealth(mine.origin, fetchImpl))) { - settleZonePort(mine.port); - startOwnerHeartbeat(mine.origin, watchPid); - return { origin: mine.origin, port: mine.port, child, logPath }; + // #4125: same contested-origin discipline as the launchToken + // branch above — the registry record vouches for the pid, but a + // healthy answer at that origin naming a DIFFERENT pid is a + // foreign responder, not our child. + if (mine && mine.origin !== "") { + const claimed = await fetchHealthInfo(mine.origin); + const healthy = claimed?.ok ?? (await probeHealth(mine.origin, fetchImpl)); + if (healthy && (claimed?.pid === undefined || claimed.pid === child.pid)) { + settleZonePort(mine.port); + startOwnerHeartbeat(mine.origin, watchPid); + return { origin: mine.origin, port: mine.port, child, logPath }; + } + if (healthy && claimed?.pid !== undefined && !contestedAnnounced) { + contestedAnnounced = true; + attachDiag( + `bili: ${mine.origin} answers health (pid ${claimed.pid}) but this launcher's child is pid ${child.pid} — refusing to claim a contested origin (#4125)`, + ); + } } } // Fallback for a child that cannot write the instance file (broken diff --git a/src/port-owner.ts b/src/port-owner.ts index a0e2409def..8aed75e050 100644 --- a/src/port-owner.ts +++ b/src/port-owner.ts @@ -93,10 +93,13 @@ export async function findForeignOwner( // own pre-registration 503s and delayed pointer/registry publication // past callers that expect registration right after bind (#394 warn // timing in multi-instance-live). Bind is exclusive on every platform - // bili ships on (Linux EADDRINUSE; Windows per libuv issue #1360), so no - // foreign listener can be in that state here; a hypothetical dual-bind - // co-starter would be unregistered anyway — the registry/pointer layer - // surfaces the collision at registration time. Never foreign. + // bili ships on (Linux EADDRINUSE; Windows — libuv deliberately omits + // SO_REUSEADDR there because setting it would allow stealing a port + // held by another application, libuv src/win/tcp.c uv__tcp_try_bind), + // so no foreign listener can be in that state here; a hypothetical + // dual-bind co-starter would be unregistered anyway — the registry/ + // pointer layer surfaces the collision at registration time. Never + // foreign. if (info?.reason === "starting") return undefined; } return undefined; diff --git a/tests/issue4125-settle-identity.test.ts b/tests/issue4125-settle-identity.test.ts new file mode 100644 index 0000000000..c7fdabbe43 --- /dev/null +++ b/tests/issue4125-settle-identity.test.ts @@ -0,0 +1,227 @@ +// #4125: the spawn-wait settle loop's PRIMARY branches (launchToken handshake, +// registry-by-pid) used to trust ANY healthy answer at the origin our child +// CLAIMED — while the preferred-port fallback already verified the responder's +// pid (#1753). A foreign responder that took over that address (port-stealing +// non-libuv process, cross-build confusion) was exported to the client as ours: +// one writer per copy violated (#991/#1225). These tests pin BOTH directions of +// the gate (#3617): own/absent pid settles, a positively-foreign pid never does. +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import type { ProxyInstanceFile as InstanceFile } from "../src/instance.ts"; +import { ensureProxyRunning, type SpawnChild, type SpawnFn } from "../src/launcher.ts"; + +const FP_SCRIPT = path.join(os.tmpdir(), `bili-4125-fp-${process.pid}.js`); +fs.writeFileSync(FP_SCRIPT, "// fingerprint fixture\n"); +const FP_HASH = createHash("sha256").update(fs.readFileSync(FP_SCRIPT)).digest("hex"); + +function makeFakeChild(pid: number): SpawnChild { + const handlers = new Map void)[]>(); + return { + pid, + unref() {}, + kill() { + return true; + }, + on(event, listener) { + const list = handlers.get(event) ?? []; + list.push(listener); + handlers.set(event, list); + return this; + }, + }; +} + +function recordedInstance(over: Partial = {}): InstanceFile { + return { + origin: "http://127.0.0.1:8787", + instanceId: "inst-1", + pid: process.pid, + startedAt: Date.now(), + host: "127.0.0.1", + port: 8787, + passthrough: false, + mitmDomains: [], + modelWindows: {}, + codeFingerprint: FP_HASH, + ...over, + }; +} + +test("ensureProxyRunning: launchToken record settles when the claimed origin's health names the spawned child (#4125)", async () => { + let childToken = ""; + const handle = await ensureProxyRunning( + { host: "127.0.0.1", port: 8787, passthrough: false, debug: false }, + { + fetchImpl: async () => ({ ok: true }), + fetchHealthInfo: async (origin) => + origin === "http://127.0.0.1:8795" ? { ok: true, pid: 42470 } : { ok: false }, + spawnImpl: (_cmd, _args, options) => { + childToken = options.env?.BILI_LAUNCH_TOKEN ?? ""; + return makeFakeChild(42470); + }, + sleep: () => Promise.resolve(), + scriptPath: FP_SCRIPT, + listRegistryInstances: () => [], + readInstanceFile: () => + childToken + ? recordedInstance({ origin: "http://127.0.0.1:8795", port: 8795, launchToken: childToken }) + : undefined, + }, + ); + assert.equal(handle.origin, "http://127.0.0.1:8795"); + assert.equal(handle.port, 8795); +}); + +test("ensureProxyRunning: a foreign responder on the child's CLAIMED origin is not exported as ours (#4125)", async () => { + let childToken = ""; + const diag: string[] = []; + let ticks = 0; + await assert.rejects( + ensureProxyRunning( + { host: "127.0.0.1", port: 8787, passthrough: false, debug: false }, + { + fetchImpl: async () => ({ ok: true }), + // The child's own record points at 8796 — but the answer there + // comes from ANOTHER process's pid. + fetchHealthInfo: async (origin) => + origin === "http://127.0.0.1:8796" ? { ok: true, pid: 875250 } : { ok: false }, + spawnImpl: (_cmd, _args, options) => { + childToken = options.env?.BILI_LAUNCH_TOKEN ?? ""; + return makeFakeChild(42471); + }, + attachDiag: (m) => diag.push(m), + now: () => ticks * 1000, + sleep: () => { + ticks += 10; + return Promise.resolve(); + }, + scriptPath: FP_SCRIPT, + listRegistryInstances: () => [], + readInstanceFile: () => + childToken + ? recordedInstance({ origin: "http://127.0.0.1:8796", port: 8796, launchToken: childToken }) + : undefined, + }, + ), + /did not become healthy/, + ); + const contested = diag.filter((m) => m.includes("refusing to claim a contested origin")); + assert.equal(contested.length, 1, "the contested-origin diagnostic fires exactly once"); + assert.match(contested[0], /pid 875250/); + assert.match(contested[0], /child is pid 42471/); +}); + +test("ensureProxyRunning: a claimed-origin responder WITHOUT a pid still settles (legacy fail-open, #4125)", async () => { + let childToken = ""; + const handle = await ensureProxyRunning( + { host: "127.0.0.1", port: 8787, passthrough: false, debug: false }, + { + fetchImpl: async () => ({ ok: true }), + fetchHealthInfo: async (origin) => + origin === "http://127.0.0.1:8797" ? { ok: true } : { ok: false }, + spawnImpl: (_cmd, _args, options) => { + childToken = options.env?.BILI_LAUNCH_TOKEN ?? ""; + return makeFakeChild(42472); + }, + sleep: () => Promise.resolve(), + scriptPath: FP_SCRIPT, + listRegistryInstances: () => [], + readInstanceFile: () => + childToken + ? recordedInstance({ origin: "http://127.0.0.1:8797", port: 8797, launchToken: childToken }) + : undefined, + }, + ); + assert.equal(handle.origin, "http://127.0.0.1:8797"); + assert.equal(handle.port, 8797); +}); + +test("ensureProxyRunning: registry-by-pid settles when that entry's origin names the spawned child (#4125)", async () => { + let spawned = false; + const handle = await ensureProxyRunning( + { host: "127.0.0.1", port: 8787, passthrough: false, debug: false }, + { + fetchImpl: async () => ({ ok: true }), + fetchHealthInfo: async (origin) => + origin === "http://127.0.0.1:8798" ? { ok: true, pid: 42473 } : { ok: false }, + spawnImpl: () => { + spawned = true; + return makeFakeChild(42473); + }, + sleep: () => Promise.resolve(), + scriptPath: FP_SCRIPT, + listRegistryInstances: () => + spawned + ? [recordedInstance({ instanceId: "reg-1", pid: 42473, origin: "http://127.0.0.1:8798", port: 8798 })] + : [], + readInstanceFile: () => undefined, + }, + ); + assert.equal(handle.origin, "http://127.0.0.1:8798"); + assert.equal(handle.port, 8798); +}); + +test("ensureProxyRunning: a foreign responder on the registry entry's origin ends in a loud failure, not a misroute (#4125)", async () => { + let spawned = false; + const diag: string[] = []; + let ticks = 0; + await assert.rejects( + ensureProxyRunning( + { host: "127.0.0.1", port: 8787, passthrough: false, debug: false }, + { + fetchImpl: async () => ({ ok: true }), + fetchHealthInfo: async (origin) => + origin === "http://127.0.0.1:8799" ? { ok: true, pid: 875250 } : { ok: false }, + spawnImpl: () => { + spawned = true; + return makeFakeChild(42474); + }, + attachDiag: (m) => diag.push(m), + now: () => ticks * 1000, + sleep: () => { + ticks += 10; + return Promise.resolve(); + }, + scriptPath: FP_SCRIPT, + listRegistryInstances: () => + spawned + ? [recordedInstance({ instanceId: "reg-1", pid: 42474, origin: "http://127.0.0.1:8799", port: 8799 })] + : [], + readInstanceFile: () => undefined, + }, + ), + /did not become healthy/, + ); + const contested = diag.filter((m) => m.includes("refusing to claim a contested origin")); + assert.equal(contested.length, 1, "the contested-origin diagnostic fires exactly once"); + assert.match(contested[0], /pid 875250/); + assert.match(contested[0], /child is pid 42474/); +}); + +test("ensureProxyRunning: health-info probe failure falls back to the legacy boolean probe (#4125)", async () => { + let childToken = ""; + const handle = await ensureProxyRunning( + { host: "127.0.0.1", port: 8787, passthrough: false, debug: false }, + { + fetchImpl: async () => ({ ok: true }), + fetchHealthInfo: async () => undefined, + spawnImpl: (_cmd, _args, options) => { + childToken = options.env?.BILI_LAUNCH_TOKEN ?? ""; + return makeFakeChild(42475); + }, + sleep: () => Promise.resolve(), + scriptPath: FP_SCRIPT, + listRegistryInstances: () => [], + readInstanceFile: () => + childToken + ? recordedInstance({ origin: "http://127.0.0.1:8800", port: 8800, launchToken: childToken }) + : undefined, + }, + ); + assert.equal(handle.origin, "http://127.0.0.1:8800"); + assert.equal(handle.port, 8800); +}); From 4939841666534df7f45c7b03a71caa3f0bb4503f Mon Sep 17 00:00:00 2001 From: ci-check Date: Sun, 25 Oct 2026 05:36:25 +0800 Subject: [PATCH 2/4] =?UTF-8?q?fix(ci):=20drop=20orphaned=20ClientBuildChe?= =?UTF-8?q?ck=20export=20=E2=80=94=20master-born-red=20unused-exports?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #4545's dsh-lane-only trim removed every external use of the type; it is now referenced only inside client-build-note.ts, which trips the INTERNAL_ONLY gate on clean master (reproduced locally). Drop the leading export per the gate's prescribed fix — a future lane re-adds it when it actually opts in. --- src/agent/client-build-note.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/agent/client-build-note.ts b/src/agent/client-build-note.ts index 8ca7e9ddaf..6c08149309 100644 --- a/src/agent/client-build-note.ts +++ b/src/agent/client-build-note.ts @@ -29,7 +29,7 @@ import path from "node:path"; import { isVersionNewer } from "../update.js"; import { logHostEvent } from "./host-log.js"; -export type ClientBuildCheck = { +type ClientBuildCheck = { /** Short human label used in the note text ("dsh", "pi", ...). */ label: string; /** The client's npm id, as shown in the note (@deepseek-ai/dsh, ...). */ From aebbafb8d3fec3b57edd3c773b3a3396f2918172 Mon Sep 17 00:00:00 2001 From: ci-check Date: Sun, 25 Oct 2026 15:26:03 +0800 Subject: [PATCH 3/4] fix: fresh owner heartbeat vetoes transient kill-0 failures in the parent-gone watchdog (#4125) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The watchdog had two disagreeing liveness signals: the identity-blind kill-0 existence probe (death streak of confirmTicks, #3890) and the identity-anchored owner heartbeat (#3544). The streak ran regardless of heartbeat freshness, so three consecutive transient OpenProcess failures against a LIVE host burned the whole confirm window and force-exited a healthy proxy mid-session. Field case 2026-10-25 (DSH web host, reported in #4125): parent-gone fired for a pid that was alive throughout; the shutdown reason carried no [stale-heartbeat] suffix, pinning it to the kill-0 path. A FAILED probe on a heartbeat-opted owner no longer counts while its last beat is within the grace window: the beat is the stronger proof (only the true owner posts its own pid over loopback). The disagreement is logged once per episode with the probe errno. Stale-heartbeat removal (#3544) and legacy kill-0-only semantics for non-heartbeat pids are unchanged (pinned in tests). Behavior change: a truly dead parent whose last heartbeat was delivered moments before death now keeps the proxy alive up to the heartbeat grace (15s) plus idle grace (5s) instead of ~11s — bounded orphan-lifetime extension in exchange for not killing live sessions on transient probe flaps. --- src/server.ts | 23 ++++++- .../issue4125-watchdog-heartbeat-veto.test.ts | 67 +++++++++++++++++++ 2 files changed, 89 insertions(+), 1 deletion(-) create mode 100644 tests/issue4125-watchdog-heartbeat-veto.test.ts diff --git a/src/server.ts b/src/server.ts index 2dd74daee8..65175bf9f7 100644 --- a/src/server.ts +++ b/src/server.ts @@ -1589,6 +1589,15 @@ export async function startServer(opts: ProxyOptions): Promise { deadStreak.delete(pid); log("info", `[watchdog] pid ${pid} alive again after failed kill-0 probe(s) — clearing death streak (#3890)`); } + } else if (decision.heartbeatVeto) { + // #4125: fresh beat vs failed existence probe — announce the + // disagreement once per episode so the field can attribute + // what the probe actually failed with (same errno clue as + // the #3890 line below). + if ((deadStreak.get(pid) ?? 0) === 0) { + log("info", `[watchdog] kill-0 probe failed for pid ${pid} (code=${probe.errCode ?? "?"}) but its heartbeat beat recently — trusting the identity-anchored heartbeat, failure not counted (#4125/#3890)`); + } + deadStreak.delete(pid); } else { if (decision.streak === 1) { // Field instrumentation (#3890): Windows collapses every @@ -1933,17 +1942,29 @@ interface WatcherTickDecision { staleHeartbeat: boolean; /** Failed-probe streak after this tick (0 when alive or removed). */ streak: number; + /** #4125: the probe failed but a FRESH heartbeat vouched for the pid — the failure was neither counted nor removing. */ + heartbeatVeto?: boolean; } /** #3890: pure per-pid death-streak step. A live probe resets the streak; a * failed probe increments it and only removes the pid once the streak reaches * confirmTicks. Heartbeat expiry stays immediate (it is a time-based liveness - * proof, not an existence probe — #3544 semantics). */ + * proof, not an existence probe — #3544 semantics). #4125: a FAILED probe on + * a heartbeat-opted owner does NOT count while its last beat is still fresh — + * the beat is identity-anchored (only the true owner posts its own pid over + * loopback), kill-0 is an identity-blind existence check, and disagreement + * resolves in favor of the stronger proof. Field case: three consecutive + * transient OpenProcess failures against a LIVE host burned the whole confirm + * window and force-exited a healthy proxy mid-session; the same host pid then + * re-armed fine on the replacement proxy, proving it never died. */ export function tickWatchedPid(probe: PidProbe, lastSeen: number | undefined, nowMs: number, prevStreak: number | undefined, opts: { heartbeatGraceMs: number; confirmTicks: number }): WatcherTickDecision { if (probe.alive) { const staleHeartbeat = watcherIsDead(true, lastSeen, nowMs, opts.heartbeatGraceMs); return { removed: staleHeartbeat, staleHeartbeat, streak: 0 }; } + if (lastSeen !== undefined && nowMs - lastSeen <= opts.heartbeatGraceMs) { + return { removed: false, staleHeartbeat: false, streak: 0, heartbeatVeto: true }; + } const streak = (prevStreak ?? 0) + 1; return streak >= opts.confirmTicks ? { removed: true, staleHeartbeat: false, streak: 0 } : { removed: false, staleHeartbeat: false, streak }; } diff --git a/tests/issue4125-watchdog-heartbeat-veto.test.ts b/tests/issue4125-watchdog-heartbeat-veto.test.ts new file mode 100644 index 0000000000..8a05a890b3 --- /dev/null +++ b/tests/issue4125-watchdog-heartbeat-veto.test.ts @@ -0,0 +1,67 @@ +// #4125: the parent-gone watchdog had TWO disagreeing liveness signals — the +// identity-blind kill-0 existence probe and the identity-anchored owner +// heartbeat (#3544) — and the #3890 kill-0 death streak ran regardless of +// heartbeat freshness. Field case (2026-10-25, DSH web host): three +// consecutive transient OpenProcess failures against a LIVE host burned the +// whole confirm window (~6s) and force-exited a healthy, compressing proxy +// mid-session; the same host pid then re-armed fine on the replacement proxy, +// proving it never died. The shutdown reason string carried no +// [stale-heartbeat] suffix, pinning this to the kill-0 streak path. +// Fix: a FRESH heartbeat vetoes failed probes entirely — the beat is the +// stronger proof (only the true owner posts its own pid over loopback). +// Stale-heartbeat removal (#3544) and legacy kill-0-only semantics are +// unchanged and pinned below. + +import test from "node:test"; +import assert from "node:assert/strict"; +import { tickWatchedPid } from "../src/server.ts"; + +const NOW = 1_000_000; +const OPTS = { heartbeatGraceMs: 15_000, confirmTicks: 3 }; +const dead = (errCode?: string) => ({ alive: false, errCode }); +const aliveProbe = { alive: true }; + +test("#4125 veto: fresh heartbeats keep a pid out of the death streak across consecutive failed probes", () => { + let d = tickWatchedPid(dead("ESRCH"), NOW - 10_000, NOW, undefined, OPTS); + assert.deepEqual(d, { removed: false, staleHeartbeat: false, streak: 0, heartbeatVeto: true }); + d = tickWatchedPid(dead("ESRCH"), NOW - 12_000, NOW + 2_000, d.streak, OPTS); + assert.equal(d.removed, false, "second consecutive failure still vetoed while the beat ages but stays fresh"); + assert.equal(d.heartbeatVeto, true); + d = tickWatchedPid(dead("ESRCH"), NOW - 14_000, NOW + 4_000, d.streak, OPTS); + assert.equal(d.removed, false, "third failure too — the confirm window cannot burn under a live heartbeat"); +}); + +test("#4125 veto boundary: a beat exactly at the grace edge is still fresh; one past it counts again", () => { + const at = tickWatchedPid(dead(), NOW - 15_000, NOW, undefined, OPTS); + assert.equal(at.heartbeatVeto, true, "age == grace counts as fresh (<=)"); + const past = tickWatchedPid(dead(), NOW - 15_001, NOW, undefined, OPTS); + assert.equal(past.heartbeatVeto, undefined, "stale beat falls back to the #3890 streak"); + assert.equal(past.streak, 1); +}); + +test("#4125 veto reset: counting resumes from 1 after vetoes, but three unvetoed failures still remove", () => { + let d = tickWatchedPid(dead(), NOW - 10_000, NOW, undefined, OPTS); + assert.equal(d.heartbeatVeto, true); + d = tickWatchedPid(dead(), NOW - 20_000, NOW + 2_000, d.streak, OPTS); + assert.equal(d.streak, 1, "beat went stale → counting resumes, not accumulating pre-veto history"); + d = tickWatchedPid(dead(), NOW - 21_000, NOW + 4_000, d.streak, OPTS); + assert.equal(d.streak, 2); + d = tickWatchedPid(dead(), NOW - 22_000, NOW + 6_000, d.streak, OPTS); + assert.equal(d.removed, true, "a genuinely silent owner still exits promptly"); + assert.equal(d.staleHeartbeat, false, "removal here is the kill-0 verdict, not the heartbeat one"); +}); + +test("#4125 regression pin: legacy pids (no heartbeat record) keep #3890 streak semantics exactly", () => { + let d = tickWatchedPid(dead("ESRCH"), undefined, NOW, undefined, OPTS); + assert.equal(d.heartbeatVeto, undefined); + assert.equal(d.streak, 1); + d = tickWatchedPid(dead("ESRCH"), undefined, NOW + 2_000, d.streak, OPTS); + assert.equal(d.streak, 2); + d = tickWatchedPid(dead("ESRCH"), undefined, NOW + 4_000, d.streak, OPTS); + assert.deepEqual(d, { removed: true, staleHeartbeat: false, streak: 0 }); +}); + +test("#4125 regression pin: the ALIVE-probe branch is untouched (#3544 stale-heartbeat removal stays immediate)", () => { + assert.deepEqual(tickWatchedPid(aliveProbe, NOW - 1_000, NOW, undefined, OPTS), { removed: false, staleHeartbeat: false, streak: 0 }); + assert.deepEqual(tickWatchedPid(aliveProbe, NOW - 16_000, NOW, undefined, OPTS), { removed: true, staleHeartbeat: true, streak: 0 }); +}); From 4537b63f4e51219ef725de366ba83ac57936f648 Mon Sep 17 00:00:00 2001 From: ci-check Date: Sun, 25 Oct 2026 15:26:14 +0800 Subject: [PATCH 4/4] fix: scope spawn-log evidence to this spawn and record the at-spawn port state (#4125) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The port log (%TEMP%/bili-proxy-.log) is shared across spawns (append fd, size-based rotation only). A zero-output failed child left the PREVIOUS spawn's content behind and every quoted sample misattributed it — field case 2026-10-25: a week-old EADDRINUSE line quoted as the current failure's evidence. Nothing recorded what the port was doing at SPAWN time, while the failure-time diagnosis only sees the END state; 'predecessor still draining' and 'child wedged elsewhere' read identically at the end. Each spawn now appends a '=== bili spawn launcherPid childPid port ===' header; size reports and head samples are scoped to this spawn's segment ('N bytes since this spawn'); every terminal failure message (slow-start notice, early exit, timeout) carries '(at spawn: )'. Rotation policy and on-disk content are unchanged — only the quoting and the new fields. Also fixes a fake gap this PR exposed: the #3544 bring-up harness modeled the spawned child through the bare probe seam but not the health-info seam, so the settle-identity commit in this PR starved its settle loop. The fake now models a real child whose /__bili/health names its own pid post-spawn. Behavior change: failure/diagnostic messages gain fields (additive); the empty-log wording changes from '0 bytes — EMPTY' to '0 bytes since this spawn — EMPTY'. --- src/launcher.ts | 62 ++++++- ...ssue3116-spawn-failure-diagnostics.test.ts | 6 +- tests/issue3544-stale-proxy-rotation.test.ts | 6 + tests/issue4125-spawn-log-segment.test.ts | 165 ++++++++++++++++++ 4 files changed, 228 insertions(+), 11 deletions(-) create mode 100644 tests/issue4125-spawn-log-segment.test.ts diff --git a/src/launcher.ts b/src/launcher.ts index ded537624b..6877af5c5e 100644 --- a/src/launcher.ts +++ b/src/launcher.ts @@ -4984,6 +4984,18 @@ export async function ensureProxyRunning( fs.closeSync(logFd); } catch {} } + // #4125: the port log is SHARED across spawns (append fd, size-based + // rotation only) — a zero-output failed child used to leave the + // PREVIOUS spawn's content behind, and every quoted sample misattributed + // it (observed: a week-old EADDRINUSE line quoted as the current + // failure's evidence). Mark where THIS spawn's segment starts; the + // samples below read from here on, and "N bytes" means N bytes since + // this spawn. + try { + fs.appendFileSync(logPath, `\n=== bili spawn ${new Date(now()).toISOString()} launcherPid=${process.pid} childPid=${child.pid ?? "?"} port=${port} ===\n`); + } catch { /* diagnostics only */ } + let spawnLogStartBytes = 0; + try { spawnLogStartBytes = fs.statSync(logPath).size; } catch { /* unreadable — treat as fresh */ } try { child.unref?.(); } catch {} @@ -5028,7 +5040,9 @@ export async function ensureProxyRunning( const fd = fs.openSync(logPath, "r"); try { const buf = Buffer.alloc(256); - const n = fs.readSync(fd, buf, 0, buf.length, 0); + // #4125: from THIS spawn's segment start — the shared file + // may hold previous spawns' output before that offset. + const n = fs.readSync(fd, buf, 0, buf.length, spawnLogStartBytes); if (n === 0) return ""; let s = buf.subarray(0, n).toString("utf8").replace(/\s+/g, " ").trim(); if (s.length > 200) s = `${s.slice(0, 200)}…`; @@ -5043,11 +5057,17 @@ export async function ensureProxyRunning( const logTail = (): string => { try { const size = fs.statSync(logPath).size; - // "0 bytes" alone cannot tell "child wrote nothing" from "output - // rotated away" (rotatePortLogIfNeeded renames to .old before - // the fd is opened) — say which when the rotated half exists. + // #4125: report bytes SINCE THIS SPAWN, not the whole shared + // file — a zero-output failed child must not be described by + // (or quoted from) a previous spawn's content. "0" still cannot + // tell "child wrote nothing" from "output rotated away" + // (rotatePortLogIfNeeded renames to .old before the fd is + // opened) — say which when the rotated half exists. + const own = Math.max(0, size - spawnLogStartBytes); const rotated = fs.existsSync(`${logPath}.old`) ? `; earlier output rotated to ${logPath}.old` : ""; - return size === 0 ? `, ${size} bytes — EMPTY, the child produced no output of its own${rotated}` : `, ${size} bytes${rotated}${logHeadSample()}`; + return own <= 0 + ? `, ${own} bytes since this spawn — EMPTY, the child produced no output of its own (any earlier content in this shared file belongs to previous spawns)${rotated}` + : `, ${own} bytes since this spawn${rotated}${logHeadSample()}`; } catch { return ""; } @@ -5090,6 +5110,30 @@ export async function ensureProxyRunning( return `last health probe failed with ${d}`; }; + // #4125: capture WHAT the planned port was doing at SPAWN time. The + // failure-time diagnosis (diagnosePort) only sees the END state; the + // start state separates "predecessor still draining/exiting on the + // port" from "child wedged on something unrelated" (e.g. a storage lock + // left by a forced exit) — the two read identically at the end, and in + // the field case the end state was plain ECONNREFUSED while the real + // story was the start state. + const initialPortNote = await (async (): Promise => { + const origin = proxyOrigin(opts.host, port); + let info: HealthInfo | undefined; + let det: ProbeOutcome = { ok: false, detail: "probe error" }; + try { + const [i, d] = await Promise.all([fetchHealthInfo(origin), fetchDetail(healthUrl(origin))]); + info = i; + det = d; + } catch { /* observability only — keep the generic clause */ } + if (info?.ok) return `healthy bili instance${info.pid !== undefined ? ` (pid ${info.pid})` : ""}`; + if (det.ok) return "foreign HTTP answer"; + const d = det.detail ?? "unknown error"; + if (/^(ECONNREFUSED|ECONNRESET|EHOSTUNREACH|ENETUNREACH|EPIPE)$/i.test(d)) return "nothing listening"; + if (/^HTTP \d{3}$/.test(d)) return `${d} answer`; + if (/^timeout after \d+ms$/i.test(d) && (await (deps.isConnectable ?? tcpConnectable)(origin))) return "busy (TCP up, no health answer)"; + return `probe failed (${d})`; + })(); const deadline = now() + SPAWN_BUDGET_MS; // #2187: announce the phase boundary once — a live child past the // initial window is still being waited for, and the log is the only @@ -5100,7 +5144,7 @@ export async function ensureProxyRunning( if (!slowStartNoticed && now() >= deadline - SLOW_START_GRACE_MS) { slowStartNoticed = true; console.error( - `bili: spawned proxy not healthy after ${SPAWN_WAIT_MS}ms — child pid ${child.pid ?? "?"} is still starting up; extending wait to ${SPAWN_BUDGET_MS}ms total (${await diagnosePort(true)}) (log: ${logPath}${logTail()})`, + `bili: spawned proxy not healthy after ${SPAWN_WAIT_MS}ms — child pid ${child.pid ?? "?"} is still starting up; extending wait to ${SPAWN_BUDGET_MS}ms total (${await diagnosePort(true)}; at spawn: ${initialPortNote}) (log: ${logPath}${logTail()})`, ); } await sleepImpl(HEALTH_POLL_INTERVAL_MS); @@ -5258,7 +5302,7 @@ export async function ensureProxyRunning( const detail = childExit.code !== null ? `code ${childExit.code}${childExit.code === TWIN_ADOPT_EXIT_CODE ? " (twin-collision signal)" : ""}` : childExit.signal ? `signal ${childExit.signal}` : "unknown reason"; - throw new Error(`bili: proxy child exited before becoming healthy (${detail}) — ${await diagnosePort(false)} (runtime: ${runtime}) (log: ${logPath}${logTail()})`); + throw new Error(`bili: proxy child exited before becoming healthy (${detail}) — ${await diagnosePort(false)} (at spawn: ${initialPortNote}) (runtime: ${runtime}) (log: ${logPath}${logTail()})`); } // #2260(A)/#2187: the timeout is the ONE terminal path where the // spawned child is STILL ALIVE. Abandoning it leaked a detached proxy @@ -5271,9 +5315,9 @@ export async function ensureProxyRunning( const diagnosis = await diagnosePort(true); killAbandonedChild(child); console.error( - `bili: killed spawned proxy (pid ${child.pid ?? "?"}) after ${SPAWN_BUDGET_MS}ms without health — ${diagnosis} (log: ${logPath}${logTail()})`, + `bili: killed spawned proxy (pid ${child.pid ?? "?"}) after ${SPAWN_BUDGET_MS}ms without health — ${diagnosis} (at spawn: ${initialPortNote}) (log: ${logPath}${logTail()})`, ); - throw new Error(`bili: proxy did not become healthy within ${SPAWN_BUDGET_MS}ms — ${diagnosis} (runtime: ${runtime}) (log: ${logPath}${logTail()})`); + throw new Error(`bili: proxy did not become healthy within ${SPAWN_BUDGET_MS}ms — ${diagnosis} (at spawn: ${initialPortNote}) (runtime: ${runtime}) (log: ${logPath}${logTail()})`); } finally { if (claimed) clearStartingMarker(launchToken); } diff --git a/tests/issue3116-spawn-failure-diagnostics.test.ts b/tests/issue3116-spawn-failure-diagnostics.test.ts index 0f96a2cbe2..e82e9e456b 100644 --- a/tests/issue3116-spawn-failure-diagnostics.test.ts +++ b/tests/issue3116-spawn-failure-diagnostics.test.ts @@ -94,7 +94,8 @@ test("#3116 timeout with nothing listening: error says so, names the live child, const msg = (err as Error).message; assert.match(msg, new RegExp(`did not become healthy within ${SPAWN_BUDGET_MS}ms`)); assert.ok(msg.includes(`nothing is listening on port ${PORT} — the spawned child (still alive, pid 42471) never bound it`), msg); - assert.ok(msg.includes("0 bytes — EMPTY, the child produced no output of its own"), msg); + // #4125: the size is now scoped to THIS spawn's segment of the shared file. + assert.ok(msg.includes("0 bytes since this spawn — EMPTY"), msg); assert.deepEqual(kills, ["SIGKILL"], "the abandoned child must be hard-killed exactly once"); const slowStart = stderrLines.find((l) => l.includes("extending wait")); assert.ok(slowStart, "the slow-start notice must still fire"); @@ -211,7 +212,8 @@ test("#3116 early exit: code stays in the message, diagnosis marks the child alr const msg = (err as Error).message; assert.match(msg, /exited before becoming healthy \(code 1\)/); assert.ok(msg.includes(`nothing is listening on port ${PORT} — the spawned child (already exited, pid 42474) never bound it`), msg); - assert.ok(msg.includes("0 bytes — EMPTY, the child produced no output of its own"), msg); + // #4125: same scoped-wording change as the timeout case above. + assert.ok(msg.includes("0 bytes since this spawn — EMPTY"), msg); }); test("#3116 describeProbeFailure extracts the socket error class", () => { diff --git a/tests/issue3544-stale-proxy-rotation.test.ts b/tests/issue3544-stale-proxy-rotation.test.ts index b4ce5d9527..b14aa39515 100644 --- a/tests/issue3544-stale-proxy-rotation.test.ts +++ b/tests/issue3544-stale-proxy-rotation.test.ts @@ -111,11 +111,13 @@ async function runBringUp(t: BringUpCase) { const clock = fakeClock(); const shutdownCalls: string[] = []; let spawnCalls = 0; + let spawned = false; let staleDead = false; const stalePort = t.stalePort ?? 18787; const staleOrigin = `http://127.0.0.1:${stalePort}`; const spawnImpl: SpawnFn = () => { spawnCalls += 1; + spawned = true; return makeFakeChild(CHILD_PID); }; const handle = await ensureProxyRunning( @@ -124,6 +126,10 @@ async function runBringUp(t: BringUpCase) { { fetchImpl: async (url) => ({ ok: url.startsWith(ORIGIN) }), fetchHealthInfo: async (origin) => { + // #4125: post-spawn, the settle branches verify responder identity + // through health INFO (not the bare probe) — model the real child, + // whose /__bili/health names its own pid. + if (spawned && origin === ORIGIN) return { ok: true, pid: CHILD_PID }; if (origin !== staleOrigin || staleDead) return { ok: false }; const info: { ok: boolean; instanceId: string; pid: number; watchdog: { armed: boolean }; version?: string } = { ok: true, diff --git a/tests/issue4125-spawn-log-segment.test.ts b/tests/issue4125-spawn-log-segment.test.ts new file mode 100644 index 0000000000..6c36cff70e --- /dev/null +++ b/tests/issue4125-spawn-log-segment.test.ts @@ -0,0 +1,165 @@ +// #4125: the port log (%TEMP%/bili-proxy-.log) is SHARED across spawns +// (append fd, size-based rotation only). Field case (2026-10-25): a failed +// child wrote 0 bytes, so the failure message quoted the PREVIOUS spawn's +// content — a week-old EADDRINUSE line presented as the current failure's +// evidence — and nothing recorded what the port was doing at SPAWN time. +// Fix: each spawn appends a `=== bili spawn ... ===` header, samples/sizes +// are scoped to this spawn's segment ("N bytes since this spawn"), and every +// terminal failure message carries "(at spawn: )". All probes go +// through injected seams — no real network I/O. + +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { ensureProxyRunning, SPAWN_BUDGET_MS, type ProbeOutcome, type SpawnChild, type SpawnFn } from "../src/launcher.ts"; + +const PORT = 47891; +const LOG_FILE = path.join(os.tmpdir(), `bili-proxy-${PORT}.log`); +const LOG_OLD = `${LOG_FILE}.old`; +const STALE_MARKER = "EADDRINUSE stale evidence from a previous week (pid 38768)"; + +function cleanupLogs(): void { + for (const f of [LOG_FILE, LOG_OLD]) { + try { fs.unlinkSync(f); } catch {} + } +} + +function fakeClock() { + let ticks = 0; + return { + now: () => ticks * 1000, + sleep: () => { + ticks += 1; + return Promise.resolve(); + }, + }; +} + +function makeAliveChild(pid: number): SpawnChild { + return { pid, unref() {}, kill() { return true; }, on() {} }; +} + +test("#4125 zero-output child with stale shared content: message says 0-bytes-since-spawn, never quotes the stale half", async () => { + cleanupLogs(); + fs.writeFileSync(LOG_FILE, `old line\n${STALE_MARKER}\n`); + const clock = fakeClock(); + let err: unknown; + try { + await ensureProxyRunning( + { host: "127.0.0.1", port: PORT, passthrough: false, debug: false }, + { + fetchImpl: async () => ({ ok: false }), + fetchHealthInfo: async () => undefined, + fetchDetailImpl: async (): Promise => ({ ok: false, detail: "ECONNREFUSED" }), + spawnImpl: (() => makeAliveChild(42481)) as SpawnFn, + now: clock.now, + sleep: clock.sleep, + readInstanceFile: () => undefined, + }, + ); + assert.fail("expected rejection"); + } catch (e) { + err = e; + } finally { + const content = (() => { try { return fs.readFileSync(LOG_FILE, "utf8"); } catch { return ""; } })(); + cleanupLogs(); + const msg = (err as Error).message; + assert.match(msg, new RegExp(`did not become healthy within ${SPAWN_BUDGET_MS}ms`)); + assert.ok(msg.includes("0 bytes since this spawn — EMPTY"), msg); + assert.ok(!msg.includes(STALE_MARKER), "stale previous-spawn content must not be quoted as this failure's evidence"); + assert.ok(content.includes("=== bili spawn"), "the spawn header must mark this segment in the shared file"); + assert.ok(content.includes(`childPid=${42481}`), "the header names the child for attribution"); + assert.ok(content.includes(STALE_MARKER), "previous content stays on disk (rotation policy unchanged) — only the QUOTING changed"); + } +}); + +test("#4125 child that writes early: the quoted head comes from THIS spawn's segment", async () => { + cleanupLogs(); + fs.writeFileSync(LOG_FILE, `${STALE_MARKER}\n`); + const clock = fakeClock(); + let err: unknown; + try { + await ensureProxyRunning( + { host: "127.0.0.1", port: PORT, passthrough: false, debug: false }, + { + fetchImpl: async () => ({ ok: false }), + fetchHealthInfo: async () => undefined, + fetchDetailImpl: async (): Promise => ({ ok: false, detail: "ECONNREFUSED" }), + spawnImpl: (() => { + // A real child cannot write before the parent returns from + // spawn() and appends the header — defer like one would. + queueMicrotask(() => fs.appendFileSync(LOG_FILE, "child stdout line\n")); + return makeAliveChild(42482); + }) as SpawnFn, + now: clock.now, + sleep: clock.sleep, + readInstanceFile: () => undefined, + }, + ); + assert.fail("expected rejection"); + } catch (e) { + err = e; + } finally { + cleanupLogs(); + const msg = (err as Error).message; + assert.ok(msg.includes("bytes since this spawn"), msg); + assert.ok(msg.includes('child output head: "child stdout line"'), msg); + assert.ok(!msg.includes(STALE_MARKER), msg); + } +}); + +test("#4125 at-spawn port state: a healthy foreign instance holding the planned port is named", async () => { + cleanupLogs(); + const clock = fakeClock(); + let err: unknown; + try { + await ensureProxyRunning( + { host: "127.0.0.1", port: PORT, passthrough: false, debug: false }, + { + fetchImpl: async () => ({ ok: true }), + fetchHealthInfo: async () => ({ ok: true, pid: 99999 }), + fetchDetailImpl: async (): Promise => ({ ok: true }), + spawnImpl: (() => makeAliveChild(42483)) as SpawnFn, + now: clock.now, + sleep: clock.sleep, + readInstanceFile: () => undefined, + }, + ); + assert.fail("expected rejection"); + } catch (e) { + err = e; + } finally { + cleanupLogs(); + const msg = (err as Error).message; + assert.ok(msg.includes("(at spawn: healthy bili instance (pid 99999))"), msg); + } +}); + +test("#4125 at-spawn port state: an empty port reads 'nothing listening'", async () => { + cleanupLogs(); + const clock = fakeClock(); + let err: unknown; + try { + await ensureProxyRunning( + { host: "127.0.0.1", port: PORT, passthrough: false, debug: false }, + { + fetchImpl: async () => ({ ok: false }), + fetchHealthInfo: async () => undefined, + fetchDetailImpl: async (): Promise => ({ ok: false, detail: "ECONNREFUSED" }), + spawnImpl: (() => makeAliveChild(42484)) as SpawnFn, + now: clock.now, + sleep: clock.sleep, + readInstanceFile: () => undefined, + }, + ); + assert.fail("expected rejection"); + } catch (e) { + err = e; + } finally { + cleanupLogs(); + const msg = (err as Error).message; + assert.ok(msg.includes("(at spawn: nothing listening)"), msg); + } +});