Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
112 changes: 90 additions & 22 deletions src/launcher.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5234,6 +5234,23 @@ export async function ensureProxyRunning(
// "child exited (code 0)" without the runtime path forced users to
// guess WHICH program was actually spawned.
const runtime = deps.nodeRuntime ?? resolveNodeRuntime();
// #4125: the port log is SHARED across spawns (append fd, size-based
// rotation only) — a zero-output failed child used to leave the
// PREVIOUS spawn's content behind, and every quoted sample misattributed
// it (observed: a week-old EADDRINUSE line quoted as the current
// failure's evidence). The header marks where THIS spawn's segment
// starts; advance spawnLogOffset past it so "N bytes since this spawn"
// counts only the child's own output (the header is launcher-written,
// not child output). Published BEFORE the spawn — like #707's starting
// marker — because a spawned child can write before a post-spawn
// append lands, which would push its earliest bytes (crash tails,
// listen lines) below the offset into "previous spawn" territory.
// childPid stays off the header (unknown pre-spawn); the child names
// itself in its own pid-tagged listen line (#4994).
try {
fs.appendFileSync(logPath, `\n=== bili spawn ${new Date(now()).toISOString()} launcherPid=${process.pid} port=${port} ===\n`);
spawnLogOffset = fs.statSync(logPath).size;
} catch { /* diagnostics only */ }
try {
child = spawnImpl(
runtime,
Expand Down Expand Up @@ -5291,6 +5308,9 @@ export async function ensureProxyRunning(
// #1753: announce a healthy-but-foreign responder on the preferred port
// at most once — silence would hide exactly the misroute this fix closes.
let squatterAnnounced = false;
// #4125: same discipline for the PRIMARY settle branches (launchToken /
// registry-by-pid) — announce a contested claimed origin at most once.
let contestedAnnounced = false;
child.on?.("exit", (...rest: unknown[]) => {
childExit = {
code: typeof rest[0] === "number" ? rest[0] : null,
Expand Down Expand Up @@ -5404,21 +5424,17 @@ export async function ensureProxyRunning(
const logTail = (): string => {
try {
const size = fs.statSync(logPath).size;
// "0 bytes" alone cannot tell "child wrote nothing" from "output
// #4125/#4811: report bytes SINCE THIS SPAWN, not the whole
// shared file — a zero-output failed child must not be
// described by (or quoted from) a previous spawn's content.
// "0" still cannot tell "child wrote nothing" from "output
// rotated away" (rotatePortLogIfNeeded renames to .old before
// the fd is opened) — say which when the rotated half exists.
// Same honesty when the child appended NOTHING to a non-empty
// file: the bytes on disk then belong to an EARLIER spawn, and
// quoting them as this child's head is exactly the #4811
// misdirection (a shrunken file means rotation under us — the
// bytes left ARE ours, so the tail sample stands). The .old
// guard keeps that claim honest: if a rotation happened, the
// captured offset is untrustworthy and the sample path applies.
const rotatedFile = fs.existsSync(`${logPath}.old`);
const rotated = rotatedFile ? `; earlier output rotated to ${logPath}.old` : "";
if (size === 0) return `, ${size} bytes — EMPTY, the child produced no output of its own${rotated}`;
if (size === spawnLogOffset && !rotatedFile) return `, ${size} bytes — this child appended nothing of its own (earlier content stays on disk until rotation)`;
return `, ${size} bytes${rotated}${logHeadSample()}${logTailSample()}`;
const own = Math.max(0, size - spawnLogOffset);
const rotated = fs.existsSync(`${logPath}.old`) ? `; earlier output rotated to ${logPath}.old` : "";
return own <= 0
? `, ${own} bytes since this spawn — EMPTY, the child produced no output of its own (any earlier content in this shared file belongs to previous spawns)${rotated}`
: `, ${own} bytes since this spawn${rotated}${logHeadSample()}${logTailSample()}`;
} catch {
return "";
}
Expand Down Expand Up @@ -5470,6 +5486,30 @@ export async function ensureProxyRunning(
return { text: `last health probe failed with ${d}`, selfHealthy: false };
};

// #4125: capture WHAT the planned port was doing at SPAWN time. The
// failure-time diagnosis (diagnosePort) only sees the END state; the
// start state separates "predecessor still draining/exiting on the
// port" from "child wedged on something unrelated" (e.g. a storage lock
// left by a forced exit) — the two read identically at the end, and in
// the field case the end state was plain ECONNREFUSED while the real
// story was the start state.
const initialPortNote = await (async (): Promise<string> => {
const origin = proxyOrigin(opts.host, port);
let info: HealthInfo | undefined;
let det: ProbeOutcome = { ok: false, detail: "probe error" };
try {
const [i, d] = await Promise.all([fetchHealthInfo(origin), fetchDetail(healthUrl(origin))]);
info = i;
det = d;
} catch { /* observability only — keep the generic clause */ }
if (info?.ok) return `healthy bili instance${info.pid !== undefined ? ` (pid ${info.pid})` : ""}`;
if (det.ok) return "foreign HTTP answer";
const d = det.detail ?? "unknown error";
if (/^(ECONNREFUSED|ECONNRESET|EHOSTUNREACH|ENETUNREACH|EPIPE)$/i.test(d)) return "nothing listening";
if (/^HTTP \d{3}$/.test(d)) return `${d} answer`;
if (/^timeout after \d+ms$/i.test(d) && (await (deps.isConnectable ?? tcpConnectable)(origin))) return "busy (TCP up, no health answer)";
return `probe failed (${d})`;
})();
const deadline = now() + SPAWN_BUDGET_MS;
// #2187: announce the phase boundary once — a live child past the
// initial window is still being waited for, and the log is the only
Expand All @@ -5480,20 +5520,34 @@ export async function ensureProxyRunning(
if (!slowStartNoticed && now() >= deadline - SLOW_START_GRACE_MS) {
slowStartNoticed = true;
console.error(
`bili: spawned proxy not healthy after ${SPAWN_WAIT_MS}ms — child pid ${child.pid ?? "?"} is still starting up; extending wait to ${SPAWN_BUDGET_MS}ms total (${(await diagnosePort(true)).text}) (log: ${logPath}${logTail()})`,
`bili: spawned proxy not healthy after ${SPAWN_WAIT_MS}ms — child pid ${child.pid ?? "?"} is still starting up; extending wait to ${SPAWN_BUDGET_MS}ms total (${(await diagnosePort(true)).text}; at spawn: ${initialPortNote}) (log: ${logPath}${logTail()})`,
);
}
await sleepImpl(HEALTH_POLL_INTERVAL_MS);
const inst = readInstance();
if (isProxyInstanceFile(inst) && inst.launchToken === launchToken) {
if (await probeHealth(inst.origin, fetchImpl)) {
// #4125: the same identity discipline as the preferred-port
// fallback below — a healthy answer at the origin the child
// CLAIMED is proof of our child only while it does not positively
// name a different pid. A foreign responder that took over that
// address must not be exported to the client as ours (one writer
// per copy, #991/#1225); absent pid info keeps legacy semantics.
const claimed = await fetchHealthInfo(inst.origin);
const healthy = claimed?.ok ?? (await probeHealth(inst.origin, fetchImpl));
if (healthy && (claimed?.pid === undefined || child.pid === undefined || claimed.pid === child.pid)) {
// #1660: settle the lane's sticky record on the port the
// child actually bound (preferred or laddered) so every
// later launch of this lane tries it first.
settleZonePort(inst.port);
startOwnerHeartbeat(inst.origin, watchPid);
return { origin: inst.origin, port: inst.port, child, logPath };
}
if (healthy && claimed?.pid !== undefined && child.pid !== undefined && !contestedAnnounced) {
contestedAnnounced = true;
attachDiag(
`bili: ${inst.origin} answers health (pid ${claimed.pid}) but this launcher's child is pid ${child.pid} — refusing to claim a contested origin (#4125)`,
);
}
continue;
}
// #2954: the single proxy-origin file is last-writer-wins machine-wide —
Expand All @@ -5506,10 +5560,24 @@ export async function ensureProxyRunning(
// the preferred-port fallback below, minus the port assumption.
if (child.pid !== undefined) {
const mine = listRegistry().find((e) => e.pid === child.pid);
if (mine && mine.origin !== "" && (await probeHealth(mine.origin, fetchImpl))) {
settleZonePort(mine.port);
startOwnerHeartbeat(mine.origin, watchPid);
return { origin: mine.origin, port: mine.port, child, logPath };
// #4125: same contested-origin discipline as the launchToken
// branch above — the registry record vouches for the pid, but a
// healthy answer at that origin naming a DIFFERENT pid is a
// foreign responder, not our child.
if (mine && mine.origin !== "") {
const claimed = await fetchHealthInfo(mine.origin);
const healthy = claimed?.ok ?? (await probeHealth(mine.origin, fetchImpl));
if (healthy && (claimed?.pid === undefined || claimed.pid === child.pid)) {
settleZonePort(mine.port);
startOwnerHeartbeat(mine.origin, watchPid);
return { origin: mine.origin, port: mine.port, child, logPath };
}
if (healthy && claimed?.pid !== undefined && !contestedAnnounced) {
contestedAnnounced = true;
attachDiag(
`bili: ${mine.origin} answers health (pid ${claimed.pid}) but this launcher's child is pid ${child.pid} — refusing to claim a contested origin (#4125)`,
);
}
}
}
// Fallback for a child that cannot write the instance file (broken
Expand Down Expand Up @@ -5631,7 +5699,7 @@ export async function ensureProxyRunning(
const detail = childExit.code !== null
? `code ${childExit.code}${childExit.code === TWIN_ADOPT_EXIT_CODE ? " (twin-collision signal)" : ""}`
: childExit.signal ? `signal ${childExit.signal}` : "unknown reason";
throw new Error(`bili: proxy child exited before becoming healthy (${detail}) — ${(await diagnosePort(false)).text} (runtime: ${runtime}) (log: ${logPath}${logTail()})${failureHint()}`);
throw new Error(`bili: proxy child exited before becoming healthy (${detail}) — ${(await diagnosePort(false)).text} (at spawn: ${initialPortNote}) (runtime: ${runtime}) (log: ${logPath}${logTail()})${failureHint()}`);
}
// #4994: where THIS child says it bound, parsed from its own log
// segment ([spawnLogOffset..end] — no other spawn's lines can answer;
Expand Down Expand Up @@ -5704,9 +5772,9 @@ export async function ensureProxyRunning(
}
killAbandonedChild(child);
console.error(
`bili: killed spawned proxy (pid ${child.pid ?? "?"}) after ${SPAWN_BUDGET_MS}ms without health — ${diagnosis.text} (log: ${logPath}${logTail()})`,
`bili: killed spawned proxy (pid ${child.pid ?? "?"}) after ${SPAWN_BUDGET_MS}ms without health — ${diagnosis.text} (at spawn: ${initialPortNote}) (log: ${logPath}${logTail()})`,
);
throw new Error(`bili: proxy did not become healthy within ${SPAWN_BUDGET_MS}ms — ${diagnosis.text} (runtime: ${runtime}) (log: ${logPath}${logTail()})${failureHint()}`);
throw new Error(`bili: proxy did not become healthy within ${SPAWN_BUDGET_MS}ms — ${diagnosis.text} (at spawn: ${initialPortNote}) (runtime: ${runtime}) (log: ${logPath}${logTail()})${failureHint()}`);
} finally {
if (claimed) clearStartingMarker(launchToken);
}
Expand Down
11 changes: 7 additions & 4 deletions src/port-owner.ts
Original file line number Diff line number Diff line change
Expand Up @@ -98,10 +98,13 @@ export async function findForeignOwner(
// own pre-registration 503s and delayed pointer/registry publication
// past callers that expect registration right after bind (#394 warn
// timing in multi-instance-live). Bind is exclusive on every platform
// bili ships on (Linux EADDRINUSE; Windows per libuv issue #1360), so no
// foreign listener can be in that state here; a hypothetical dual-bind
// co-starter would be unregistered anyway — the registry/pointer layer
// surfaces the collision at registration time. Never foreign.
// bili ships on (Linux EADDRINUSE; Windows — libuv deliberately omits
// SO_REUSEADDR there because setting it would allow stealing a port
// held by another application, libuv src/win/tcp.c uv__tcp_try_bind),
// so no foreign listener can be in that state here; a hypothetical
// dual-bind co-starter would be unregistered anyway — the registry/
// pointer layer surfaces the collision at registration time. Never
// foreign.
if (info?.reason === "starting") return undefined;
}
return undefined;
Expand Down
28 changes: 25 additions & 3 deletions src/server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1856,6 +1856,15 @@ export async function startServer(opts: ProxyOptions): Promise<http.Server> {
hbStreaks.delete(pid);
log("info", `[watchdog] pid ${pid} heartbeating again — clearing stale-heartbeat streak (#4994)`);
}
} else if (decision.heartbeatVeto) {
// #4125: fresh beat vs failed existence probe — announce the
// disagreement once per episode so the field can attribute
// what the probe actually failed with (same errno clue as
// the #3890 line below).
if ((deadStreak.get(pid) ?? 0) === 0) {
log("info", `[watchdog] kill-0 probe failed for pid ${pid} (code=${probe.errCode ?? "?"}) but its heartbeat beat recently — trusting the identity-anchored heartbeat, failure not counted (#4125/#3890)`);
}
deadStreak.delete(pid);
} else {
if (decision.streak === 1) {
// Field instrumentation (#3890): Windows collapses every
Expand Down Expand Up @@ -2262,16 +2271,26 @@ interface WatcherTickDecision {
streak: number;
/** Stale-while-alive streak after this tick (0 when the beat is fresh or removed). #4994. */
hbStreak: number;
/** #4125: the probe failed but a FRESH heartbeat vouched for the pid — the failure was neither counted nor removing. */
heartbeatVeto?: boolean;
}

/** #3890/#4994: pure per-pid death-streak step. Each axis confirms
* independently over CONSECUTIVE agreeing ticks: a failed kill-0 probe
/** #3890/#4994/#4125: pure per-pid death-streak step. Two axes confirm
* independently over CONSECUTIVE agreeing ticks (#4994): a failed kill-0 probe
* increments its streak and only removes at confirmTicks; a stale-while-alive
* reading increments the heartbeat streak and only removes at
* heartbeatConfirmTicks. A live probe resets the kill-0 axis, a fresh beat
* resets the heartbeat axis, and a tick that disagrees with an axis breaks
* THAT axis's run (episodes never accumulate across interruptions). Legacy
* pids (no heartbeat record) keep kill-0-only semantics (#3544). */
* pids (no heartbeat record) keep kill-0-only semantics (#3544). On top of
* that, #4125: a FAILED probe on a pid whose last beat is still FRESH is
* vetoed entirely — not counted, not removed — because the beat is
* identity-anchored (only the true owner posts its own pid over loopback)
* while kill-0 is an identity-blind existence check; disagreement resolves in
* favor of the stronger proof. Field case: three consecutive transient
* OpenProcess failures against a LIVE host burned the whole confirm window
* and force-exited a healthy proxy mid-session; the same host pid then
* re-armed fine on the replacement proxy, proving it never died. */
export function tickWatchedPid(probe: PidProbe, lastSeen: number | undefined, nowMs: number, prevStreak: number | undefined, prevHbStreak: number | undefined, opts: { heartbeatGraceMs: number; confirmTicks: number; heartbeatConfirmTicks: number }): WatcherTickDecision {
if (probe.alive) {
const staleHeartbeat = watcherIsDead(true, lastSeen, nowMs, opts.heartbeatGraceMs);
Expand All @@ -2281,6 +2300,9 @@ export function tickWatchedPid(probe: PidProbe, lastSeen: number | undefined, no
? { removed: true, staleHeartbeat: true, streak: 0, hbStreak: 0 }
: { removed: false, staleHeartbeat: true, streak: 0, hbStreak };
}
if (lastSeen !== undefined && nowMs - lastSeen <= opts.heartbeatGraceMs) {
return { removed: false, staleHeartbeat: false, streak: 0, hbStreak: 0, heartbeatVeto: true };
}
const streak = (prevStreak ?? 0) + 1;
return streak >= opts.confirmTicks
? { removed: true, staleHeartbeat: false, streak: 0, hbStreak: 0 }
Expand Down
6 changes: 4 additions & 2 deletions tests/issue3116-spawn-failure-diagnostics.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,8 @@ test("#3116 timeout with nothing listening: error says so, names the live child,
const msg = (err as Error).message;
assert.match(msg, new RegExp(`did not become healthy within ${SPAWN_BUDGET_MS}ms`));
assert.ok(msg.includes(`nothing is listening on port ${PORT} — the spawned child (still alive, pid 42471) never bound it`), msg);
assert.ok(msg.includes("0 bytes — EMPTY, the child produced no output of its own"), msg);
// #4125: the size is now scoped to THIS spawn's segment of the shared file.
assert.ok(msg.includes("0 bytes since this spawn — EMPTY"), msg);
assert.deepEqual(kills, ["SIGKILL"], "the abandoned child must be hard-killed exactly once");
const slowStart = stderrLines.find((l) => l.includes("extending wait"));
assert.ok(slowStart, "the slow-start notice must still fire");
Expand Down Expand Up @@ -211,7 +212,8 @@ test("#3116 early exit: code stays in the message, diagnosis marks the child alr
const msg = (err as Error).message;
assert.match(msg, /exited before becoming healthy \(code 1\)/);
assert.ok(msg.includes(`nothing is listening on port ${PORT} — the spawned child (already exited, pid 42474) never bound it`), msg);
assert.ok(msg.includes("0 bytes — EMPTY, the child produced no output of its own"), msg);
// #4125: same scoped-wording change as the timeout case above.
assert.ok(msg.includes("0 bytes since this spawn — EMPTY"), msg);
});

test("#3116 describeProbeFailure extracts the socket error class", () => {
Expand Down
Loading
Loading