Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/agent/opencode-native.ts
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,7 @@ async function oversizeDirectRefusal(e: V2HttpRequestEvent, s: V2State, reason?:
);
}
return new Error(
`bili: proxy unavailable and this request cannot be sent uncompressed — body ~${floorTokens} tok (floor estimate ${bytes}B/4) exceeds the ${model.providerID}/${model.id} context window (${window}); upstream would reject it. Recovery is automatic (#3308); retry shortly, or start a fresh session if the local history has outgrown the window.`,
`bili: proxy unavailable and this request cannot be sent uncompressed — body ~${floorTokens} tok (floor estimate ${bytes}B/4) exceeds the ${model.providerID}/${model.id} context window (${window}); upstream would reject it. The proxy respawns automatically (#3308), but this request is NOT retried on its own — resubmit once it is back; if your local history has outgrown the window, start a fresh session instead (a resubmit cannot fit either).`,
);
}

Expand Down
23 changes: 23 additions & 0 deletions src/instance.ts
Original file line number Diff line number Diff line change
Expand Up @@ -420,6 +420,29 @@ export function isPidAlive(pid: number): boolean {
}
}

export interface PidProbe {
alive: boolean;
/** Errno code of the failed kill-0 (e.g. "ESRCH"); undefined when the
* probe succeeded or no probe ran at all (invalid pid). */
errCode?: string;
}

/** #3890: kill-0 with errno visibility. On Windows Node maps every
* OpenProcess failure other than ERROR_ACCESS_DENIED to ESRCH, so a LIVE
* process can read dead with no way to tell which Win32 error caused it.
* Callers that act destructively on a dead verdict log errCode on first
* sight so field episodes can be attributed. */
export function probePid(pid: number): PidProbe {
if (!Number.isInteger(pid) || pid <= 0) return { alive: false };
try {
process.kill(pid, 0);
return { alive: true };
} catch (err) {
const code = (err as NodeJS.ErrnoException).code;
return { alive: code === "EPERM", errCode: code };
}
}

export interface RegistryEntry {
instanceId: string;
pid: number;
Expand Down
106 changes: 93 additions & 13 deletions src/server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ import { getStore } from "./persist.js";
import { log as loggerLog, configureLogger, getLogPath, closeLogger, isStreamWriteError, isBenignSocketRaceError, enterSessionContext, muteStderrLeg, onFileLegLive } from "./logger.js";
import { queryLogLines } from "./web/logs-query.js";
import { configFile, defaultLogFile, stateDir } from "./paths.js";
import { atomicWriteInstanceFile, clearProxyInstanceFile, classifySameLanePeer, entryScriptFingerprint, entryShape, findSameLanePredecessor, isPidAlive, listInstances, readProxyInstanceFile, registerInstanceAndWarn, sameInstanceShape, unregisterInstance, warnOnNewPeers, writeSpawnPortLog, type InstanceShape, type ProxyInstanceFile, type SameLanePeer } from "./instance.js";
import { atomicWriteInstanceFile, clearProxyInstanceFile, classifySameLanePeer, entryScriptFingerprint, entryShape, findSameLanePredecessor, isPidAlive, listInstances, probePid, readProxyInstanceFile, registerInstanceAndWarn, sameInstanceShape, unregisterInstance, warnOnNewPeers, writeSpawnPortLog, type InstanceShape, type PidProbe, type ProxyInstanceFile, type RegistryEntry, type SameLanePeer } from "./instance.js";
import { mayClobberPointer, probeOwnerInfo, resolvePortClaim } from "./port-owner.js";
import { hoistTrappedToolItems } from "./tool-pair-order.js";
import { computeAnthropicMessageMarks, stampAnthropicSystemCacheControl, anthropicToolsCarryCacheControl } from "./loop/cache-control.js";
Expand Down Expand Up @@ -1407,6 +1407,10 @@ export async function startServer(opts: ProxyOptions): Promise<http.Server> {
closeLogger();
process.exit(0);
};
const finishDrain = (): void => {
flushConversations();
void flushAllSessions().finally(finishShutdown);
};
const shutdown = (sig: string) => {
if (shuttingDown) return;
shuttingDown = true;
Expand All @@ -1417,16 +1421,29 @@ export async function startServer(opts: ProxyOptions): Promise<http.Server> {
// before invoking cb, so in-flight SSE streams get a chance to finish
// rather than being yanked mid-chunk.
server.close(() => {
flushConversations();
void flushAllSessions().finally(finishShutdown);
finishDrain();
});
// Hard fallback: if connections hang (client never closes), don't
// block shutdown forever — force-exit after a grace window.
setTimeout(() => {
log("warn", `shutdown grace window elapsed; forcing exit (liveConns=${connRecords.size})`);
flushConversations();
void flushAllSessions().finally(finishShutdown);
}, 10_000).unref?.();
// block shutdown forever — force-exit after a grace window. #3890:
// while connections are still ALIVE the window re-arms (bounded by
// SHUTDOWN_DRAIN_MAX_PASSES) instead of cutting in-flight upstream
// streams mid-response — idle keep-alives don't hold the proxy, the
// keep-alive reaper clears them within one window, so a healthy drain
// still exits promptly and a stuck socket can pin at most the cap.
let drainPasses = 0;
const armDrainBackstop = (): void => {
setTimeout(() => {
if (connRecords.size > 0 && drainPasses < SHUTDOWN_DRAIN_MAX_PASSES) {
drainPasses += 1;
log("warn", `shutdown grace window elapsed but liveConns=${connRecords.size} — extending drain ${drainPasses}/${SHUTDOWN_DRAIN_MAX_PASSES} (#3890)`);
armDrainBackstop();
return;
}
log("warn", `shutdown grace window elapsed; forcing exit (liveConns=${connRecords.size})`);
finishDrain();
}, SHUTDOWN_DRAIN_GRACE_MS).unref?.();
};
armDrainBackstop();
};
// #3544: POST /__bili/shutdown (stale-proxy rotation from a newer
// generation's bring-up) takes this exact graceful path.
Expand Down Expand Up @@ -1454,21 +1471,44 @@ export async function startServer(opts: ProxyOptions): Promise<http.Server> {
let idleSince: number | null = null;
let lastDead: number[] = [];
let lastStaleHb: number[] = [];
// #3890: consecutive failed kill-0 probes per pid (see WATCHER_DEAD_CONFIRM_TICKS).
const deadStreak = new Map<number, number>();
const watcher = setInterval(() => {
const dead: number[] = [];
const staleByHeartbeat: number[] = [];
const tickNow = Date.now();
for (const pid of proxyWatchers) {
const alive = isPidAlive(pid);
const probe = probePid(pid);
// #3544: heartbeat-opted owners ALSO die by liveness-proof
// expiry — kill(pid,0) is an existence check, identity-blind
// under Windows pid reuse. Legacy pids (no heartbeat on record)
// keep kill-0-only semantics.
if (watcherIsDead(alive, watcherLastSeen.get(pid), tickNow, WATCHER_HEARTBEAT_GRACE_MS)) {
// keep kill-0-only semantics. #3890: a failed kill-0 must be
// CONFIRMED over consecutive ticks before it removes the pid —
// one transient OpenProcess failure used to be enough to kill
// the whole proxy while its parent lived on.
const decision = tickWatchedPid(probe, watcherLastSeen.get(pid), tickNow, deadStreak.get(pid), { heartbeatGraceMs: WATCHER_HEARTBEAT_GRACE_MS, confirmTicks: WATCHER_DEAD_CONFIRM_TICKS });
if (decision.removed) {
proxyWatchers.delete(pid);
watcherLastSeen.delete(pid);
deadStreak.delete(pid);
dead.push(pid);
if (alive) staleByHeartbeat.push(pid);
if (decision.staleHeartbeat) staleByHeartbeat.push(pid);
continue;
}
if (probe.alive) {
if ((deadStreak.get(pid) ?? 0) > 0) {
deadStreak.delete(pid);
log("info", `[watchdog] pid ${pid} alive again after failed kill-0 probe(s) — clearing death streak (#3890)`);
}
} else {
if (decision.streak === 1) {
// Field instrumentation (#3890): Windows collapses every
// non-ACCESS_DENIED OpenProcess failure into ESRCH, so
// the errno is the only clue left to attribute false-dead
// episodes — log it once per episode.
log("info", `[watchdog] kill-0 probe failed for pid ${pid} (code=${probe.errCode ?? "?"}) — requiring ${WATCHER_DEAD_CONFIRM_TICKS} consecutive failures before believing it (#3890)`);
}
deadStreak.set(pid, decision.streak);
}
}
if (dead.length > 0) lastDead = dead;
Expand Down Expand Up @@ -1763,6 +1803,46 @@ export function watcherIsDead(alive: boolean, lastSeen: number | undefined, nowM
return lastSeen !== undefined && nowMs - lastSeen > heartbeatGraceMs;
}

// #3890: the shutdown backstop window, and how many times it may RE-ARM while
// connections are still alive. One window is the historical hard fallback;
// extra passes exist so an in-flight upstream stream finishes instead of being
// cut mid-response when its owner's death triggers the exit. Worst-case total
// drain ≈ (1 + SHUTDOWN_DRAIN_MAX_PASSES) windows.
const SHUTDOWN_DRAIN_GRACE_MS = 10_000;
const SHUTDOWN_DRAIN_MAX_PASSES = 10;

// #3890: how many CONSECUTIVE failed kill-0 probes must agree before the
// watchdog believes a watched pid has died. Pre-fix, ONE failed probe removed
// the pid from the watcher set and the empty-set grace window that followed
// never re-probed it — a single transient OpenProcess failure on Windows was
// enough to force-exit the proxy ~6s later while its parent lived on (12 such
// events in 2.5h against one live pid). At the 2s tick, three confirmations
// span ~4–6s: long enough to ride out transient failures, short enough that a
// genuinely dead parent still exits promptly.
const WATCHER_DEAD_CONFIRM_TICKS = 3;

interface WatcherTickDecision {
/** Remove this pid from the watcher set on this tick. */
removed: boolean;
/** Removed because the heartbeat expired while the pid still reads alive (#3544). */
staleHeartbeat: boolean;
/** Failed-probe streak after this tick (0 when alive or removed). */
streak: number;
}

/** #3890: pure per-pid death-streak step. A live probe resets the streak; a
* failed probe increments it and only removes the pid once the streak reaches
* confirmTicks. Heartbeat expiry stays immediate (it is a time-based liveness
* proof, not an existence probe — #3544 semantics). */
export function tickWatchedPid(probe: PidProbe, lastSeen: number | undefined, nowMs: number, prevStreak: number | undefined, opts: { heartbeatGraceMs: number; confirmTicks: number }): WatcherTickDecision {
if (probe.alive) {
const staleHeartbeat = watcherIsDead(true, lastSeen, nowMs, opts.heartbeatGraceMs);
return { removed: staleHeartbeat, staleHeartbeat, streak: 0 };
}
const streak = (prevStreak ?? 0) + 1;
return streak >= opts.confirmTicks ? { removed: true, staleHeartbeat: false, streak: 0 } : { removed: false, staleHeartbeat: false, streak };
}


const ACP_TAG_MARK = "\x3cacp ";

Expand Down
20 changes: 14 additions & 6 deletions tests/claude-native.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1615,8 +1615,10 @@ test("hook e2e: watchdog tracks the claude host, not the transient sh wrapper",
assert.ok(await canConnect(port), "proxy survives the transient sh wrapper's death");

// Session end: claude dies, the proxy must follow within a few ticks.
// #3890 widened owner-death→exit by up to ~6s (three consecutive failed
// probes now confirm death instead of one) — deadline re-sized to match.
if (claudePid > 1) killPid(claudePid);
const deadline = Date.now() + 15_000;
const deadline = Date.now() + 30_000;
while (Date.now() < deadline && (await canConnect(port))) await new Promise((r) => setTimeout(r, 250));
assert.equal(await canConnect(port), false, "proxy exits with the claude session");
} finally {
Expand Down Expand Up @@ -1697,17 +1699,22 @@ test("watcher route: shared proxies take watcher registrations, daemons refuse (
// Grace: keeper1 dies, and BEFORE the idle grace expires a new owner
// registers (the live race: spawner exits right after a second session
// starts). The pending shutdown must be cancelled — the proxy stays up
// well past the grace window.
// well past the grace window. #3890: keeper1's removal now takes ~6s
// (three consecutive failed probes confirm death), so the late
// registration lands AFTER removal but INSIDE the 5s idle grace that
// follows it — still exercising the cancellation of an armed pending
// shutdown (which would otherwise fire ~11s after the death).
if (keeperPid > 1) killPid(keeperPid);
await new Promise((r) => setTimeout(r, 3000));
await new Promise((r) => setTimeout(r, 8000));
const late = await post({ pid: keeper2Pid });
assert.equal(late.status, 200, "late registration within the grace window accepted");
await new Promise((r) => setTimeout(r, 7000));
await new Promise((r) => setTimeout(r, 8000));
assert.ok(await canConnect(port), "grace registration cancelled the pending shutdown");

// LAST owner dies → armed proxy must follow (≤ a few ticks + grace).
// #3890 widened owner-death→exit by up to ~6s (confirmation ticks).
if (keeper2Pid > 1) killPid(keeper2Pid);
const deadline = Date.now() + 15_000;
const deadline = Date.now() + 30_000;
while (Date.now() < deadline && (await canConnect(port))) await new Promise((r) => setTimeout(r, 250));
assert.equal(await canConnect(port), false, "armed proxy exits when its watcher dies");

Expand Down Expand Up @@ -1915,8 +1922,9 @@ test("hook e2e: shared proxy survives the first session's exit, dies after the l
assert.equal(instMid.pid, proxyPid, "same proxy instance throughout — no respawn");

// Last session exits → the proxy must follow within a few ticks.
// #3890 widened owner-death→exit by up to ~6s (confirmation ticks).
if (claudeB > 1) killPid(claudeB);
const deadline = Date.now() + 15_000;
const deadline = Date.now() + 30_000;
while (Date.now() < deadline && (await canConnect(port))) await new Promise((r) => setTimeout(r, 250));
assert.equal(await canConnect(port), false, "proxy exits after the LAST session");
} finally {
Expand Down
38 changes: 38 additions & 0 deletions tests/fixtures/issue3890-drain-bili.mts
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
// Child-process fixture for tests/issue3890-parent-gone-confirm-drain.test.ts:
// a minimal bili server in its OWN process because finishShutdown ends with
// process.exit(0) — an in-process test would kill the runner. Prints
// "READY <port>" on stdout once listening; all bili logs go to stderr
// (logFile: "off"), which the parent test asserts against.
import { defaultConfig } from "acp-kernel";
import { startServer } from "../../src/server.js";
import type { ProxyOptions } from "../../src/config.js";

const opts: ProxyOptions = {
port: 0,
host: "127.0.0.1",
upstream: "http://127.0.0.1:1",
routes: {},
modelContextLimit: 400_000,
kernelConfig: defaultConfig(400_000),
compress: { injectTool: true, injectNudge: true },
promptCache: { routing: "auto" },
sessionHeader: "x-acp-session",
log: true,
logFile: "off",
debug: false,
passthrough: false,
autoUpdate: false,
autoRestartOnUpdate: false,
advisoryCheck: false,
releaseNotesCheck: false,
compat: { roles: {} },
passthroughSource: null,
updateTag: "latest",
mitm: { enabled: false, domains: [] },
};
startServer(opts).then((server) => {
server.on("listening", () => {
const port = (server.address() as { port: number }).port;
console.log(`READY ${port}`);
});
});
Loading
Loading