diff --git a/src/agent/opencode-native.ts b/src/agent/opencode-native.ts index b98da7c455..0af4987946 100644 --- a/src/agent/opencode-native.ts +++ b/src/agent/opencode-native.ts @@ -141,7 +141,7 @@ async function oversizeDirectRefusal(e: V2HttpRequestEvent, s: V2State, reason?: ); } return new Error( - `bili: proxy unavailable and this request cannot be sent uncompressed — body ~${floorTokens} tok (floor estimate ${bytes}B/4) exceeds the ${model.providerID}/${model.id} context window (${window}); upstream would reject it. Recovery is automatic (#3308); retry shortly, or start a fresh session if the local history has outgrown the window.`, + `bili: proxy unavailable and this request cannot be sent uncompressed — body ~${floorTokens} tok (floor estimate ${bytes}B/4) exceeds the ${model.providerID}/${model.id} context window (${window}); upstream would reject it. The proxy respawns automatically (#3308), but this request is NOT retried on its own — resubmit once it is back; if your local history has outgrown the window, start a fresh session instead (a resubmit cannot fit either).`, ); } diff --git a/src/instance.ts b/src/instance.ts index 2faa7b8b13..1bc7398ffb 100644 --- a/src/instance.ts +++ b/src/instance.ts @@ -420,6 +420,29 @@ export function isPidAlive(pid: number): boolean { } } +export interface PidProbe { + alive: boolean; + /** Errno code of the failed kill-0 (e.g. "ESRCH"); undefined when the + * probe succeeded or no probe ran at all (invalid pid). */ + errCode?: string; +} + +/** #3890: kill-0 with errno visibility. On Windows Node maps every + * OpenProcess failure other than ERROR_ACCESS_DENIED to ESRCH, so a LIVE + * process can read dead with no way to tell which Win32 error caused it. + * Callers that act destructively on a dead verdict log errCode on first + * sight so field episodes can be attributed. */ +export function probePid(pid: number): PidProbe { + if (!Number.isInteger(pid) || pid <= 0) return { alive: false }; + try { + process.kill(pid, 0); + return { alive: true }; + } catch (err) { + const code = (err as NodeJS.ErrnoException).code; + return { alive: code === "EPERM", errCode: code }; + } +} + export interface RegistryEntry { instanceId: string; pid: number; diff --git a/src/server.ts b/src/server.ts index fddc3bf4cd..8961e91e75 100644 --- a/src/server.ts +++ b/src/server.ts @@ -98,7 +98,7 @@ import { getStore } from "./persist.js"; import { log as loggerLog, configureLogger, getLogPath, closeLogger, isStreamWriteError, isBenignSocketRaceError, enterSessionContext, muteStderrLeg, onFileLegLive } from "./logger.js"; import { queryLogLines } from "./web/logs-query.js"; import { configFile, defaultLogFile, stateDir } from "./paths.js"; -import { atomicWriteInstanceFile, clearProxyInstanceFile, classifySameLanePeer, entryScriptFingerprint, entryShape, findSameLanePredecessor, isPidAlive, listInstances, readProxyInstanceFile, registerInstanceAndWarn, sameInstanceShape, unregisterInstance, warnOnNewPeers, writeSpawnPortLog, type InstanceShape, type ProxyInstanceFile, type SameLanePeer } from "./instance.js"; +import { atomicWriteInstanceFile, clearProxyInstanceFile, classifySameLanePeer, entryScriptFingerprint, entryShape, findSameLanePredecessor, isPidAlive, listInstances, probePid, readProxyInstanceFile, registerInstanceAndWarn, sameInstanceShape, unregisterInstance, warnOnNewPeers, writeSpawnPortLog, type InstanceShape, type PidProbe, type ProxyInstanceFile, type RegistryEntry, type SameLanePeer } from "./instance.js"; import { mayClobberPointer, probeOwnerInfo, resolvePortClaim } from "./port-owner.js"; import { hoistTrappedToolItems } from "./tool-pair-order.js"; import { computeAnthropicMessageMarks, stampAnthropicSystemCacheControl, anthropicToolsCarryCacheControl } from "./loop/cache-control.js"; @@ -1407,6 +1407,10 @@ export async function startServer(opts: ProxyOptions): Promise { closeLogger(); process.exit(0); }; + const finishDrain = (): void => { + flushConversations(); + void flushAllSessions().finally(finishShutdown); + }; const shutdown = (sig: string) => { if (shuttingDown) return; shuttingDown = true; @@ -1417,16 +1421,29 @@ export async function startServer(opts: ProxyOptions): Promise { // before invoking cb, so in-flight SSE streams get a chance to finish // rather than being yanked mid-chunk. server.close(() => { - flushConversations(); - void flushAllSessions().finally(finishShutdown); + finishDrain(); }); // Hard fallback: if connections hang (client never closes), don't - // block shutdown forever — force-exit after a grace window. - setTimeout(() => { - log("warn", `shutdown grace window elapsed; forcing exit (liveConns=${connRecords.size})`); - flushConversations(); - void flushAllSessions().finally(finishShutdown); - }, 10_000).unref?.(); + // block shutdown forever — force-exit after a grace window. #3890: + // while connections are still ALIVE the window re-arms (bounded by + // SHUTDOWN_DRAIN_MAX_PASSES) instead of cutting in-flight upstream + // streams mid-response — idle keep-alives don't hold the proxy, the + // keep-alive reaper clears them within one window, so a healthy drain + // still exits promptly and a stuck socket can pin at most the cap. + let drainPasses = 0; + const armDrainBackstop = (): void => { + setTimeout(() => { + if (connRecords.size > 0 && drainPasses < SHUTDOWN_DRAIN_MAX_PASSES) { + drainPasses += 1; + log("warn", `shutdown grace window elapsed but liveConns=${connRecords.size} — extending drain ${drainPasses}/${SHUTDOWN_DRAIN_MAX_PASSES} (#3890)`); + armDrainBackstop(); + return; + } + log("warn", `shutdown grace window elapsed; forcing exit (liveConns=${connRecords.size})`); + finishDrain(); + }, SHUTDOWN_DRAIN_GRACE_MS).unref?.(); + }; + armDrainBackstop(); }; // #3544: POST /__bili/shutdown (stale-proxy rotation from a newer // generation's bring-up) takes this exact graceful path. @@ -1454,21 +1471,44 @@ export async function startServer(opts: ProxyOptions): Promise { let idleSince: number | null = null; let lastDead: number[] = []; let lastStaleHb: number[] = []; + // #3890: consecutive failed kill-0 probes per pid (see WATCHER_DEAD_CONFIRM_TICKS). + const deadStreak = new Map(); const watcher = setInterval(() => { const dead: number[] = []; const staleByHeartbeat: number[] = []; const tickNow = Date.now(); for (const pid of proxyWatchers) { - const alive = isPidAlive(pid); + const probe = probePid(pid); // #3544: heartbeat-opted owners ALSO die by liveness-proof // expiry — kill(pid,0) is an existence check, identity-blind // under Windows pid reuse. Legacy pids (no heartbeat on record) - // keep kill-0-only semantics. - if (watcherIsDead(alive, watcherLastSeen.get(pid), tickNow, WATCHER_HEARTBEAT_GRACE_MS)) { + // keep kill-0-only semantics. #3890: a failed kill-0 must be + // CONFIRMED over consecutive ticks before it removes the pid — + // one transient OpenProcess failure used to be enough to kill + // the whole proxy while its parent lived on. + const decision = tickWatchedPid(probe, watcherLastSeen.get(pid), tickNow, deadStreak.get(pid), { heartbeatGraceMs: WATCHER_HEARTBEAT_GRACE_MS, confirmTicks: WATCHER_DEAD_CONFIRM_TICKS }); + if (decision.removed) { proxyWatchers.delete(pid); watcherLastSeen.delete(pid); + deadStreak.delete(pid); dead.push(pid); - if (alive) staleByHeartbeat.push(pid); + if (decision.staleHeartbeat) staleByHeartbeat.push(pid); + continue; + } + if (probe.alive) { + if ((deadStreak.get(pid) ?? 0) > 0) { + deadStreak.delete(pid); + log("info", `[watchdog] pid ${pid} alive again after failed kill-0 probe(s) — clearing death streak (#3890)`); + } + } else { + if (decision.streak === 1) { + // Field instrumentation (#3890): Windows collapses every + // non-ACCESS_DENIED OpenProcess failure into ESRCH, so + // the errno is the only clue left to attribute false-dead + // episodes — log it once per episode. + log("info", `[watchdog] kill-0 probe failed for pid ${pid} (code=${probe.errCode ?? "?"}) — requiring ${WATCHER_DEAD_CONFIRM_TICKS} consecutive failures before believing it (#3890)`); + } + deadStreak.set(pid, decision.streak); } } if (dead.length > 0) lastDead = dead; @@ -1763,6 +1803,46 @@ export function watcherIsDead(alive: boolean, lastSeen: number | undefined, nowM return lastSeen !== undefined && nowMs - lastSeen > heartbeatGraceMs; } +// #3890: the shutdown backstop window, and how many times it may RE-ARM while +// connections are still alive. One window is the historical hard fallback; +// extra passes exist so an in-flight upstream stream finishes instead of being +// cut mid-response when its owner's death triggers the exit. Worst-case total +// drain ≈ (1 + SHUTDOWN_DRAIN_MAX_PASSES) windows. +const SHUTDOWN_DRAIN_GRACE_MS = 10_000; +const SHUTDOWN_DRAIN_MAX_PASSES = 10; + +// #3890: how many CONSECUTIVE failed kill-0 probes must agree before the +// watchdog believes a watched pid has died. Pre-fix, ONE failed probe removed +// the pid from the watcher set and the empty-set grace window that followed +// never re-probed it — a single transient OpenProcess failure on Windows was +// enough to force-exit the proxy ~6s later while its parent lived on (12 such +// events in 2.5h against one live pid). At the 2s tick, three confirmations +// span ~4–6s: long enough to ride out transient failures, short enough that a +// genuinely dead parent still exits promptly. +const WATCHER_DEAD_CONFIRM_TICKS = 3; + +interface WatcherTickDecision { + /** Remove this pid from the watcher set on this tick. */ + removed: boolean; + /** Removed because the heartbeat expired while the pid still reads alive (#3544). */ + staleHeartbeat: boolean; + /** Failed-probe streak after this tick (0 when alive or removed). */ + streak: number; +} + +/** #3890: pure per-pid death-streak step. A live probe resets the streak; a + * failed probe increments it and only removes the pid once the streak reaches + * confirmTicks. Heartbeat expiry stays immediate (it is a time-based liveness + * proof, not an existence probe — #3544 semantics). */ +export function tickWatchedPid(probe: PidProbe, lastSeen: number | undefined, nowMs: number, prevStreak: number | undefined, opts: { heartbeatGraceMs: number; confirmTicks: number }): WatcherTickDecision { + if (probe.alive) { + const staleHeartbeat = watcherIsDead(true, lastSeen, nowMs, opts.heartbeatGraceMs); + return { removed: staleHeartbeat, staleHeartbeat, streak: 0 }; + } + const streak = (prevStreak ?? 0) + 1; + return streak >= opts.confirmTicks ? { removed: true, staleHeartbeat: false, streak: 0 } : { removed: false, staleHeartbeat: false, streak }; +} + const ACP_TAG_MARK = "\x3cacp "; diff --git a/tests/claude-native.test.ts b/tests/claude-native.test.ts index 194a7d1d42..44c8c8bb5e 100644 --- a/tests/claude-native.test.ts +++ b/tests/claude-native.test.ts @@ -1615,8 +1615,10 @@ test("hook e2e: watchdog tracks the claude host, not the transient sh wrapper", assert.ok(await canConnect(port), "proxy survives the transient sh wrapper's death"); // Session end: claude dies, the proxy must follow within a few ticks. + // #3890 widened owner-death→exit by up to ~6s (three consecutive failed + // probes now confirm death instead of one) — deadline re-sized to match. if (claudePid > 1) killPid(claudePid); - const deadline = Date.now() + 15_000; + const deadline = Date.now() + 30_000; while (Date.now() < deadline && (await canConnect(port))) await new Promise((r) => setTimeout(r, 250)); assert.equal(await canConnect(port), false, "proxy exits with the claude session"); } finally { @@ -1697,17 +1699,22 @@ test("watcher route: shared proxies take watcher registrations, daemons refuse ( // Grace: keeper1 dies, and BEFORE the idle grace expires a new owner // registers (the live race: spawner exits right after a second session // starts). The pending shutdown must be cancelled — the proxy stays up - // well past the grace window. + // well past the grace window. #3890: keeper1's removal now takes ~6s + // (three consecutive failed probes confirm death), so the late + // registration lands AFTER removal but INSIDE the 5s idle grace that + // follows it — still exercising the cancellation of an armed pending + // shutdown (which would otherwise fire ~11s after the death). if (keeperPid > 1) killPid(keeperPid); - await new Promise((r) => setTimeout(r, 3000)); + await new Promise((r) => setTimeout(r, 8000)); const late = await post({ pid: keeper2Pid }); assert.equal(late.status, 200, "late registration within the grace window accepted"); - await new Promise((r) => setTimeout(r, 7000)); + await new Promise((r) => setTimeout(r, 8000)); assert.ok(await canConnect(port), "grace registration cancelled the pending shutdown"); // LAST owner dies → armed proxy must follow (≤ a few ticks + grace). + // #3890 widened owner-death→exit by up to ~6s (confirmation ticks). if (keeper2Pid > 1) killPid(keeper2Pid); - const deadline = Date.now() + 15_000; + const deadline = Date.now() + 30_000; while (Date.now() < deadline && (await canConnect(port))) await new Promise((r) => setTimeout(r, 250)); assert.equal(await canConnect(port), false, "armed proxy exits when its watcher dies"); @@ -1915,8 +1922,9 @@ test("hook e2e: shared proxy survives the first session's exit, dies after the l assert.equal(instMid.pid, proxyPid, "same proxy instance throughout — no respawn"); // Last session exits → the proxy must follow within a few ticks. + // #3890 widened owner-death→exit by up to ~6s (confirmation ticks). if (claudeB > 1) killPid(claudeB); - const deadline = Date.now() + 15_000; + const deadline = Date.now() + 30_000; while (Date.now() < deadline && (await canConnect(port))) await new Promise((r) => setTimeout(r, 250)); assert.equal(await canConnect(port), false, "proxy exits after the LAST session"); } finally { diff --git a/tests/fixtures/issue3890-drain-bili.mts b/tests/fixtures/issue3890-drain-bili.mts new file mode 100644 index 0000000000..cb51b79a81 --- /dev/null +++ b/tests/fixtures/issue3890-drain-bili.mts @@ -0,0 +1,38 @@ +// Child-process fixture for tests/issue3890-parent-gone-confirm-drain.test.ts: +// a minimal bili server in its OWN process because finishShutdown ends with +// process.exit(0) — an in-process test would kill the runner. Prints +// "READY " on stdout once listening; all bili logs go to stderr +// (logFile: "off"), which the parent test asserts against. +import { defaultConfig } from "acp-kernel"; +import { startServer } from "../../src/server.js"; +import type { ProxyOptions } from "../../src/config.js"; + +const opts: ProxyOptions = { + port: 0, + host: "127.0.0.1", + upstream: "http://127.0.0.1:1", + routes: {}, + modelContextLimit: 400_000, + kernelConfig: defaultConfig(400_000), + compress: { injectTool: true, injectNudge: true }, + promptCache: { routing: "auto" }, + sessionHeader: "x-acp-session", + log: true, + logFile: "off", + debug: false, + passthrough: false, + autoUpdate: false, + autoRestartOnUpdate: false, + advisoryCheck: false, + releaseNotesCheck: false, + compat: { roles: {} }, + passthroughSource: null, + updateTag: "latest", + mitm: { enabled: false, domains: [] }, +}; +startServer(opts).then((server) => { + server.on("listening", () => { + const port = (server.address() as { port: number }).port; + console.log(`READY ${port}`); + }); +}); diff --git a/tests/issue3890-parent-gone-confirm-drain.test.ts b/tests/issue3890-parent-gone-confirm-drain.test.ts new file mode 100644 index 0000000000..04d8da9dee --- /dev/null +++ b/tests/issue3890-parent-gone-confirm-drain.test.ts @@ -0,0 +1,146 @@ +// #3890: Windows parent-gone false positive — a LIVE parent pid (opencode-cli.exe) +// was judged dead by the watchdog and the proxy force-exited 12× in 2.5h, each +// time cutting in-flight upstream streams. Two mechanisms are pinned here: +// (1) a kill-0 death verdict now requires WATCHER_DEAD_CONFIRM_TICKS +// CONSECUTIVE failed probes (tickWatchedPid — pure, unit-tested below); +// one transient OpenProcess failure can no longer kill the proxy, while +// heartbeat expiry for a live pid stays immediate (#3544 unchanged); +// (2) the shutdown backstop RE-ARMS its grace window while connections are +// still alive (bounded passes) instead of force-exiting at 10s and +// yanking in-flight streams (live child-process test — pre-fix behavior +// exits mid-drain). + +import test from "node:test"; +import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import { mkdirSync } from "node:fs"; +import { tmpdir } from "node:os"; +import net from "node:net"; +import path from "node:path"; +import { once } from "node:events"; +import { fileURLToPath } from "node:url"; +import { tickWatchedPid } from "../src/server.ts"; +import type { PidProbe } from "../src/instance.ts"; +import { rmrf } from "./tmp-rm.ts"; + +const OPTS = { heartbeatGraceMs: 15_000, confirmTicks: 3 }; +const NOW = 1_000_000; +const dead = (code = "ESRCH"): PidProbe => ({ alive: false, errCode: code }); +const aliveProbe: PidProbe = { alive: true }; + +test("#3890 tickWatchedPid: one failed probe does NOT remove the pid", () => { + const d = tickWatchedPid(dead(), undefined, NOW, undefined, OPTS); + assert.deepEqual(d, { removed: false, staleHeartbeat: false, streak: 1 }, "a single flaky probe is not conclusive"); +}); + +test("#3890 tickWatchedPid: blip shorter than confirmTicks survives; a live probe resets the streak", () => { + let d = tickWatchedPid(dead(), undefined, NOW, undefined, OPTS); + assert.equal(d.removed, false); + d = tickWatchedPid(dead(), undefined, NOW + 2_000, d.streak, OPTS); + assert.equal(d.removed, false); + assert.equal(d.streak, 2); + d = tickWatchedPid(aliveProbe, undefined, NOW + 4_000, d.streak, OPTS); + assert.equal(d.removed, false); + assert.equal(d.streak, 0, "the live probe clears the pending death"); + d = tickWatchedPid(dead(), undefined, NOW + 6_000, d.streak, OPTS); + assert.equal(d.streak, 1, "a later episode starts from zero again"); +}); + +test("#3890 tickWatchedPid: confirmTicks consecutive failures remove the pid", () => { + let d = tickWatchedPid(dead(), undefined, NOW, undefined, OPTS); + d = tickWatchedPid(dead(), undefined, NOW + 2_000, d.streak, OPTS); + assert.equal(d.removed, false, "two failures are not yet conclusive"); + d = tickWatchedPid(dead(), undefined, NOW + 4_000, d.streak, OPTS); + assert.equal(d.removed, true, "the third consecutive failure confirms death"); + assert.equal(d.staleHeartbeat, false); +}); + +test("#3890 tickWatchedPid: live pids keep legacy semantics (no heartbeat record ⇒ never removed)", () => { + assert.deepEqual(tickWatchedPid(aliveProbe, undefined, NOW, undefined, OPTS), { removed: false, staleHeartbeat: false, streak: 0 }); +}); + +test("#3890 tickWatchedPid: heartbeat expiry stays immediate for a LIVE pid (#3544 unchanged)", () => { + assert.deepEqual(tickWatchedPid(aliveProbe, NOW - 1_000, NOW, undefined, OPTS), { removed: false, staleHeartbeat: false, streak: 0 }, "fresh beat keeps it"); + const stale = tickWatchedPid(aliveProbe, NOW - 16_000, NOW, undefined, OPTS); + assert.equal(stale.removed, true, "missed beats beyond grace mark the slot dead immediately — no confirmation needed for a time-based liveness proof"); + assert.equal(stale.staleHeartbeat, true); +}); + +test("#3890 tickWatchedPid: a DEAD pid never takes the heartbeat fast path", () => { + let d = tickWatchedPid(dead("ESRCH"), NOW - 60_000, NOW, undefined, OPTS); + assert.equal(d.removed, false, "stale lastSeen does not shortcut kill-0 confirmation"); + d = tickWatchedPid(dead("ESRCH"), NOW - 60_000, NOW, d.streak, OPTS); + assert.equal(d.removed, false); + d = tickWatchedPid(dead("ESRCH"), NOW - 60_000, NOW, d.streak, OPTS); + assert.equal(d.removed, true); + assert.equal(d.staleHeartbeat, false, "removal via kill-0 confirmation is never tagged stale-heartbeat"); +}); + +// ── live drain backstop (child process — finishShutdown ends in process.exit) ── + +const FIXTURE = fileURLToPath(new URL("./fixtures/issue3890-drain-bili.mts", import.meta.url)); +const READY_TIMEOUT_MS = 30_000; +// The pre-fix hard backstop force-exited 10s after shutdown() regardless of +// live connections. The fixed proxy must still be ALIVE well past that point +// while a connection is held, and exit promptly once it drains. +const HOLD_CHECK_MS = 13_000; +const EXIT_GRACE_MS = 20_000; + +test("#3890 shutdown backstop extends while a connection is alive, then exits when it drains", { timeout: 120_000 }, async () => { + const root = path.join(tmpdir(), `bili-3890drain-${process.pid}-${Date.now()}`); + const env = { ...process.env }; + env.XDG_CONFIG_HOME = path.join(root, "cfg"); + env.XDG_STATE_HOME = path.join(root, "state"); + env.XDG_CACHE_HOME = path.join(root, "cache"); + for (const d of [env.XDG_CONFIG_HOME, env.XDG_STATE_HOME, env.XDG_CACHE_HOME]) mkdirSync(d, { recursive: true }); + delete env.BILI_PARENT_PID; + delete env.BILLION_CONTEXT_PROXY; + const child = spawn(process.execPath, ["--import", "tsx", FIXTURE], { env, stdio: ["ignore", "pipe", "pipe"] }); + let stdout = ""; + let stderr = ""; + child.stdout!.on("data", (c) => { stdout += String(c); }); + child.stderr!.on("data", (c) => { stderr += String(c); }); + let exitInfo: { code: number | null } | null = null; + child.on("exit", (code) => { exitInfo = { code }; }); + let held: net.Socket | null = null; + try { + const deadline = Date.now() + READY_TIMEOUT_MS; + let port = 0; + while (!port && Date.now() < deadline) { + const m = /^READY (\d+)$/m.exec(stdout); + if (m) port = Number(m[1]); + else await new Promise((r) => setTimeout(r, 100)); + } + assert.ok(port > 0, `fixture reported READY (stderr tail: ${stderr.slice(-500)})`); + // A bare held socket: registered in connRecords, never requests, never + // FINs until destroyed — deterministic liveConns≥1 without depending on + // keep-alive reaper timing. + held = new net.Socket(); + held.connect(port, "127.0.0.1"); + await once(held, "connect"); + const res = await fetch(`http://127.0.0.1:${port}/__bili/shutdown`, { method: "POST" }); + assert.equal(res.status, 202, "admin shutdown accepted"); + // Pre-fix: force-exit at ~10s with liveConns=1. Post-fix: the window + // re-arms and the proxy stays up serving/draining. + const checkDeadline = Date.now() + HOLD_CHECK_MS; + while (Date.now() < checkDeadline) { + if (exitInfo !== null) break; + await new Promise((r) => setTimeout(r, 250)); + } + assert.equal(exitInfo, null, `proxy must stay alive ${HOLD_CHECK_MS}ms into the shutdown drain while a connection is open — pre-#3890 behavior force-exited at 10s (stderr tail: ${stderr.slice(-500)})`); + assert.match(stderr, /extending drain \d+\/\d+/, "the extension is logged, not an accidental slowdown"); + held.destroy(); + held = null; + const exitDeadline = Date.now() + EXIT_GRACE_MS; + while (exitInfo === null && Date.now() < exitDeadline) await new Promise((r) => setTimeout(r, 250)); + if (exitInfo === null) throw new Error(`proxy exits once the last connection drains — still up after ${EXIT_GRACE_MS}ms (stderr tail: ${stderr.slice(-500)})`); + assert.equal(exitInfo.code, 0, "clean exit code after drain"); + } finally { + held?.destroy(); + if (exitInfo === null && child.pid) { + child.kill("SIGKILL"); + await once(child, "exit").catch(() => {}); + } + try { rmrf(root); } catch {} + } +});