From 9d51277629719d9003e65bc9120163367a032f8e Mon Sep 17 00:00:00 2001 From: awork Date: Sun, 18 Oct 2026 16:07:19 +0800 Subject: [PATCH 1/3] =?UTF-8?q?fix:=20three-state=20health=20probe=20?= =?UTF-8?q?=E2=80=94=20a=20busy=20proxy=20is=20no=20longer=20misjudged=20a?= =?UTF-8?q?s=20dead=20(#3654)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/agent/native-intercept.ts | 178 ++++-- src/agent/opencode-native.ts | 22 +- src/instance.ts | 16 + src/launcher.ts | 55 +- src/listen-policy.ts | 18 + src/liveness.ts | 123 ++++ src/server.ts | 15 +- ...issue3654-health-probe-three-state.test.ts | 576 ++++++++++++++++++ 8 files changed, 956 insertions(+), 47 deletions(-) create mode 100644 src/liveness.ts create mode 100644 tests/issue3654-health-probe-three-state.test.ts diff --git a/src/agent/native-intercept.ts b/src/agent/native-intercept.ts index 3de36f57d7..52ff0e8231 100644 --- a/src/agent/native-intercept.ts +++ b/src/agent/native-intercept.ts @@ -10,6 +10,7 @@ import { installFetchChain, isDeadClosureError, resetFetchChainForTest } from ". import { logHostEvent } from "./host-log.js"; import { envMillis } from "./native-bootstrap.js"; +import { probeLiveness, type LivenessVerdict } from "../liveness.js"; import { BILI_PASSTHROUGH_HEADER } from "../util.js"; import { DSH_COMPACTION_MARKER_SNIFF } from "../server/dsh-compaction-guard.js"; import { APIG_RESIGN_HEADER, APIG_RESIGN_CREDENTIAL_HEADER, APIG_RESIGN_SCHEME, clearSignedRefusal, encodeApigCredential, inboundSignedScheme, recordSignedRefusal, resignEnabled, resignPassthroughEnabled, signedRefusal } from "../apig-resign.js"; @@ -29,6 +30,11 @@ export type DirectSendPrep = | { outcome: "restored"; body: string } | { outcome: "unrestorable" }; +/** #3654: why createLiveOriginResolver found nothing to route to — the + * refusal wording must tell a live-but-saturated proxy (the busy wait window + * exhausted) apart from a dead one or a failed bootstrap. */ +export type UnavailableReason = "busy-exhausted" | "dead" | "bootstrap-failed"; + export interface NativeInterceptState { /** Proxy origin ("http://127.0.0.1:PORT") once the bootstrap resolved. * Written by the owner (pi-native.ts); read synchronously on each call. */ @@ -42,6 +48,11 @@ export interface NativeInterceptState { * BILLION_CONTEXT_PROXY env) so event-time ownership checks disarm with * the traffic. */ onGiveUp?: () => void; + /** #3654: set before createLiveOriginResolver returns undefined — why + * nothing was routable ("busy-exhausted" | "dead" | "bootstrap-failed"); + * cleared on the next successful resolution. The refusal builder reads it + * to keep a live-but-saturated proxy out of "proxy unavailable" wording. */ + lastUnavailableReason?: UnavailableReason; /** Attach mode (#809): route through a user-supplied external proxy at * state.origin instead of a spawned one. Set by the host entry when a * BILLION_CONTEXT_ATTACH / launcher-preset BILLION_CONTEXT_PROXY is @@ -509,57 +520,80 @@ export async function observeRoutedOrigin(state: NativeInterceptState): Promise< // ———— Live-origin resolution (#1135) —————————————————————————————— // Shared by BOTH OpenCode lanes (the native entry's V2 route and the // launcher plugin's V2 route): each outgoing request resolves the LIVE proxy -// origin — fast-path the held origin through a TTL-cached health probe, and +// origin — fast-path the held origin through a TTL-cached liveness probe, and // when it is dead drive state.respawn (cooldown-gated unless we just lost an // origin we were actively routing to) and adopt whatever replacement lands. -// A transient blip that recovers to the SAME origin costs nothing (the -// session never migrates); a genuinely dead origin falls back to whatever the -// owner's respawn produces (a self-spawned proxy, or another healthy shared -// instance via discovery). Returns undefined when nothing is alive — callers -// MUST degrade to direct sends, not fail-closed. +// #3654: the probe is THREE-state (src/liveness.ts). Connect-refused = dead +// (instant, unchanged fast path). Connected-but-no-health-answer = busy — +// the event loop is saturated (a large compression in flight), NOT dead: +// dropping a busy HELD origin would respawn into the SAME process +// (twin-collision, #3044), so instead the origin is held and re-probed at a +// jittered cadence inside a bounded wait window. A busy REPLACEMENT (just +// respawned or adopted) is routed to anyway — its loop drains and the request +// queues instead of failing for nothing. A transient blip that recovers to +// the SAME origin costs nothing (the session never migrates); a genuinely +// dead origin falls back to whatever the owner's respawn produces (a +// self-spawned proxy, or another healthy shared instance via discovery). +// Returns undefined when nothing is alive — callers MUST degrade to direct +// sends, not fail-closed. const HEALTH_TIMEOUT_MS = 1500; +/** #3654: total budget for re-probing a BUSY held origin before declaring it + * lost. Deliberately far above HEALTH_TIMEOUT_MS: the point is to ride out + * multi-second event-loop saturation (large compressions), not to widen the + * single-shot probe. */ +const BUSY_WAIT_CAP_MS = 10_000; +/** #3654: base interval between busy re-probes (jittered 0.5x–1.5x). */ +const BUSY_REPROBE_BASE_MS = 1_000; /** Probe-verdict trust window (#928). Far below RESPAWN_COOLDOWN_MS: that * cooldown already assumes multi-second proxy stability, so a few-second * detection horizon is consistent with it while removing the per-request RTT. */ const HEALTH_PROBE_TTL_MS = 2_000; const RESPAWN_COOLDOWN_MS = 15_000; -export async function probeHealth(origin: string): Promise { - try { - const res = await fetch(`${origin}/__bili/health`, { signal: AbortSignal.timeout(HEALTH_TIMEOUT_MS) }); - return res.ok; - } catch { - return false; - } +/** Default liveness probe: GET /__bili/health under the single-shot budget — + * now three-state (#3654): TCP connect refused classifies dead in + * milliseconds instead of eating the whole budget. */ +export async function probeOriginHealth(origin: string): Promise { + return probeLiveness(origin, HEALTH_TIMEOUT_MS); } -/** Wrap a probe in a per-origin TTL cache (#928). Both verdicts are cached: a - * healthy hit skips the steady-state loopback RTT; a dead hit avoids re-paying - * the full HEALTH_TIMEOUT_MS on every request across the death+cooldown window. - * Stale entries for other origins are evicted on insert, keeping the map bounded. */ -function withProbeTtl(probe: (origin: string) => Promise, ttlMs: number): (origin: string) => Promise { - const cache = new Map(); +/** Wrap a probe in a per-origin TTL cache (#928). Only DEFINITE verdicts are + * cached: alive skips the steady-state loopback RTT; dead avoids re-paying + * the full HEALTH_TIMEOUT_MS on every request across the death+cooldown + * window. BUSY is never cached — a saturation episode ends within seconds and + * caching it would solidify "slow" into "dead" (#3654). Stale entries for + * other origins are evicted on insert, keeping the map bounded. */ +function withProbeTtl(probe: (origin: string) => Promise, ttlMs: number): (origin: string) => Promise { + const cache = new Map(); return (origin) => { const now = Date.now(); const hit = cache.get(origin); - if (hit !== undefined && now - hit.at < ttlMs) return Promise.resolve(hit.ok); - return probe(origin).then((ok) => { + if (hit !== undefined && now - hit.at < ttlMs) return Promise.resolve(hit.verdict); + return probe(origin).then((verdict) => { const t = Date.now(); for (const [key, entry] of cache) if (t - entry.at >= ttlMs) cache.delete(key); - cache.set(origin, { ok, at: t }); - return ok; + if (verdict.state !== "busy") cache.set(origin, { verdict, at: t }); + return verdict; }); }; } export interface LiveOriginResolverDeps { - /** Injectable health probe (tests); defaults to GET /__bili/health. */ - probe?: (origin: string) => Promise; + /** Injectable liveness probe (tests); defaults to probeLiveness over + * /__bili/health under HEALTH_TIMEOUT_MS. Legacy boolean probes stay + * accepted (true → alive, false → dead). */ + probe?: (origin: string) => Promise; /** Bootstrap retry interval when no live origin is held (tests shrink it). */ respawnCooldownMs?: number; /** Probe-verdict cache TTL; defaults to HEALTH_PROBE_TTL_MS (tests shrink it). */ probeTtlMs?: number; + /** #3654: busy-wait budget for a saturated HELD origin; defaults to + * BUSY_WAIT_CAP_MS (tests shrink it). */ + busyWaitCapMs?: number; + /** #3654: base interval between busy re-probes; defaults to + * BUSY_REPROBE_BASE_MS (tests shrink it). */ + busyReprobeBaseMs?: number; } /** Build the per-request live-origin resolver over a shared intercept state. @@ -575,27 +609,85 @@ export function createLiveOriginResolver(state: NativeInterceptState, deps: Live // cache is rebuilt (kills cached verdicts) and `generation` advances // (kills in-flight verdicts, which must not clear a healthy // replacement that already landed). - const healthProbe = deps.probe ?? probeHealth; + const rawProbe = deps.probe ?? probeOriginHealth; + const probeFn = (origin: string): Promise => rawProbe(origin).then((v) => typeof v === "boolean" ? { state: v ? "alive" : "dead" } : v); const probeTtlMs = deps.probeTtlMs ?? HEALTH_PROBE_TTL_MS; - let probe = withProbeTtl(healthProbe, probeTtlMs); + let probe = withProbeTtl(probeFn, probeTtlMs); let generation = 0; const respawnCooldownMs = deps.respawnCooldownMs ?? RESPAWN_COOLDOWN_MS; + const busyWaitCapMs = deps.busyWaitCapMs ?? BUSY_WAIT_CAP_MS; + const busyReprobeBaseMs = deps.busyReprobeBaseMs ?? BUSY_REPROBE_BASE_MS; let lastRespawn = 0; + // #3654: one entry per busy episode per origin (attempt count + first-seen + // time) so the log announces an episode ONCE and then counts attempts. + const busyEpisodes = new Map(); + const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); return async (): Promise => { let ownedThenLost = false; + let lossReason: UnavailableReason | undefined; const held = state.origin; const gen = generation; if (held !== undefined) { - const ok = await probe(held); + const v = await probe(held); // Act on the verdict only if the world didn't move while it was // in flight: a concurrent caller's respawn (generation bump) or // any origin change underneath us voids it — re-resolve instead. if (gen === generation && state.origin === held) { - if (ok) return held; - // Proxy died mid-session. Clearing origin first makes concurrent - // callers share the same state.ready (dedup). - ownedThenLost = true; - state.origin = undefined; + if (v.state === "alive") { + state.lastUnavailableReason = undefined; + return held; + } + if (v.state === "dead") { + logHostEvent("info", "[native-client]", `bili-native: liveness: ${held} dead (${v.detail ?? "no answer"}) — clearing origin`); + // Proxy died mid-session. Clearing origin first makes concurrent + // callers share the same state.ready (dedup). + ownedThenLost = true; + lossReason = "dead"; + state.origin = undefined; + } else { + // #3654: BUSY — TCP connect succeeded but /__bili/health never + // answered within the budget: the event loop is saturated + // (typically a large compression), not dead. Dropping this + // origin would respawn into the SAME process (twin-collision, + // #3044); ride out the episode instead. + logHostEvent("info", "[native-client]", `bili-native: liveness: ${held} BUSY (${v.detail ?? "no health answer"}) — holding origin, re-probing (budget ${busyWaitCapMs}ms) (#3654)`); + const deadline = Date.now() + busyWaitCapMs; + const ep = busyEpisodes.get(held) ?? { attempts: 0, announcedAt: Date.now() }; + busyEpisodes.set(held, ep); + for (;;) { + if (Date.now() >= deadline) break; + const jitter = 0.5 + Math.random(); // 0.5x–1.5x + await sleep(Math.min(busyReprobeBaseMs * jitter, Math.max(0, deadline - Date.now()))); + if (gen !== generation || state.origin !== held) break; + ep.attempts += 1; + const again = await probe(held); + if (gen !== generation || state.origin !== held) break; + if (again.state === "alive") { + busyEpisodes.delete(held); + logHostEvent("info", "[native-client]", `bili-native: liveness: ${held} alive again after ${ep.attempts} busy re-probe(s)`); + state.lastUnavailableReason = undefined; + return held; + } + if (again.state === "dead") { + busyEpisodes.delete(held); + logHostEvent("info", "[native-client]", `bili-native: liveness: ${held} dead during busy episode (${again.detail ?? "no answer"}, attempt ${ep.attempts}) — respawning`); + ownedThenLost = true; + lossReason = "dead"; + state.origin = undefined; + break; + } + logHostEvent("info", "[native-client]", `bili-native: liveness: ${held} still busy — re-probe attempt ${ep.attempts}`); + } + if (state.origin === held && gen === generation) { + // Window exhausted while still busy: lost-for-this-request, + // but record WHY so the refusal says "saturated", not "dead". + busyEpisodes.delete(held); + logHostEvent("warn", "[native-client]", `bili-native: liveness: ${held} still busy after ${ep.attempts} re-probe(s) over ${busyWaitCapMs}ms — letting go, respawning (#3654)`); + ownedThenLost = true; + lossReason = "busy-exhausted"; + state.origin = undefined; + } + } } } // Retry bootstrap whenever no live origin is held — either just lost @@ -605,11 +697,27 @@ export function createLiveOriginResolver(state: NativeInterceptState, deps: Live if (state.respawn !== undefined && (ownedThenLost || Date.now() - lastRespawn >= respawnCooldownMs)) { lastRespawn = Date.now(); generation += 1; - probe = withProbeTtl(healthProbe, probeTtlMs); + probe = withProbeTtl(probeFn, probeTtlMs); state.ready = state.respawn(); } const o = await readyOrigin(state); - if (o !== undefined && (await probe(o))) return o; + if (o !== undefined) { + const v = await probe(o); + if (v.state === "alive") { + state.lastUnavailableReason = undefined; + return o; + } + if (v.state === "busy") { + // #3654: a just-landed replacement that is already busy is still + // ALIVE — routing to it queues the request behind the saturating + // fold; refusing here would fail the very request the adoption + // existed to save. + logHostEvent("info", "[native-client]", `bili-native: liveness: replacement ${o} BUSY (${v.detail ?? "no health answer"}) — routing anyway, the request will queue (#3654)`); + state.lastUnavailableReason = undefined; + return o; + } + } + state.lastUnavailableReason = lossReason ?? "bootstrap-failed"; if (ownedThenLost) state.onGiveUp?.(); return undefined; }; diff --git a/src/agent/opencode-native.ts b/src/agent/opencode-native.ts index 1aaa6da8c4..b98da7c455 100644 --- a/src/agent/opencode-native.ts +++ b/src/agent/opencode-native.ts @@ -76,7 +76,7 @@ import { ACP_TOOLS_OPENAI } from "../compress-tool.js"; import { ensureProxyRunning, LAUNCHER_DEFAULT_HOST, unwrapUpstream, wrapUpstream } from "../launcher.js"; import { createAcpCommandHooks, showAcpText } from "./opencode-acp-command.js"; import { markNativeHost, nativeAttachOrigin, nativeBootstrapGate, nativeProxyScriptPath, proxyEnvOrigin, singleFlight } from "./native-bootstrap.js"; -import { createLiveOriginResolver, installNativeFetchIntercept, isModelApiUrl, noteRoutedOrigin, observeRoutedOrigin, readyOrigin, replaceRequestPayload, replaceRequestTarget, routedBiliModelUrl, type LiveOriginResolverDeps, type NativeInterceptState } from "./native-intercept.js"; +import { createLiveOriginResolver, installNativeFetchIntercept, isModelApiUrl, noteRoutedOrigin, observeRoutedOrigin, readyOrigin, replaceRequestPayload, replaceRequestTarget, routedBiliModelUrl, type LiveOriginResolverDeps, type NativeInterceptState, type UnavailableReason } from "./native-intercept.js"; import { clearIncrementalProbe, createPrepareDirectSend, INCREMENTAL_TURN_MARKER_HEADER, INCREMENTAL_TURN_ID_HEADER, newIncrementalTurnId, noteIncrementalAck, noteIncrementalRejection, planIncrementalTurn, probeIncrementalTurns, rememberOriginalBody, sweepStale, wireMessagesLookAnthropic, type IncrementalWireMessage } from "./incremental-client.js"; import { createOpencodeV2Setup, type V2HttpRequestEvent, type V2State } from "./opencode-v2.js"; import { asciiHeaderValue, fetchProxyVersion, postIdentityRegister, reportRuntimeInfoOnChange, waitForProxyVersion, type ForwardedToolResult } from "./shared.js"; @@ -117,7 +117,7 @@ export type OpencodeNativeRouteDeps = LiveOriginResolverDeps; * model, unknown window, unreadable or absent body all return undefined and * keep the documented fail-open direct send. The size is measured through a * CLONE — the original Request must stay sendable (#2496 discipline). */ -async function oversizeDirectRefusal(e: V2HttpRequestEvent, s: V2State): Promise { +async function oversizeDirectRefusal(e: V2HttpRequestEvent, s: V2State, reason?: UnavailableReason): Promise { const model = e.model; if (model === undefined || typeof model.providerID !== "string" || typeof model.id !== "string") return undefined; const window = s.windows?.get(`${model.providerID}/${model.id}`); @@ -132,6 +132,14 @@ async function oversizeDirectRefusal(e: V2HttpRequestEvent, s: V2State): Promise } const floorTokens = Math.ceil(bytes / 4); if (floorTokens <= window) return undefined; + // #3654: a busy-exhausted refusal must NOT claim "proxy unavailable … + // guaranteed upstream 400" — the proxy IS up and would serve the request + // once its event loop drains; the honest fact is that we stopped waiting. + if (reason === "busy-exhausted") { + return new Error( + `bili: proxy alive but unresponsive — it accepted connections but never answered its health probe within the wait budget (event loop saturated, typically a large compression in flight); this request cannot be sent uncompressed (body ~${floorTokens} tok (floor estimate ${bytes}B/4) exceeds the ${model.providerID}/${model.id} context window (${window})), so it is refused locally instead of queued indefinitely. The proxy is likely to serve it once the fold finishes (#3654); retry shortly.`, + ); + } return new Error( `bili: proxy unavailable and this request cannot be sent uncompressed — body ~${floorTokens} tok (floor estimate ${bytes}B/4) exceeds the ${model.providerID}/${model.id} context window (${window}); upstream would reject it. Recovery is automatic (#3308); retry shortly, or start a fresh session if the local history has outgrown the window.`, ); @@ -189,12 +197,16 @@ export function createNativeRoute(state: NativeInterceptState, deps: OpencodeNat if (target === undefined) { // #3308: a direct send that provably cannot fit the window is not // "availability" — refuse it locally (#2924 ruling) instead of - // handing upstream a request that can only 400. - const refusal = await oversizeDirectRefusal(e, s); + // handing upstream a request that can only 400. #3654: pass WHY + // nothing was routable so a saturated-but-alive proxy gets its own + // honest wording instead of "proxy unavailable … guaranteed 400". + const refusal = await oversizeDirectRefusal(e, s, state.lastUnavailableReason); if (refusal !== undefined) { if (!refusedNoticed) { refusedNoticed = true; - logHostEvent("error", "[native-client]", `bili-native-opencode: proxy unavailable — model request refused locally instead of going direct into a guaranteed upstream 400 (#3308): ${refusal.message}`); + logHostEvent("error", "[native-client]", state.lastUnavailableReason === "busy-exhausted" + ? `bili-native-opencode: proxy alive but saturated past the wait budget — model request refused locally instead of queued indefinitely (#3654): ${refusal.message}` + : `bili-native-opencode: proxy unavailable — model request refused locally instead of going direct into a guaranteed upstream 400 (#3308): ${refusal.message}`); } s.proxyBase = undefined; throw refusal; diff --git a/src/instance.ts b/src/instance.ts index d1485f2c24..6d81a5f0e1 100644 --- a/src/instance.ts +++ b/src/instance.ts @@ -1,5 +1,6 @@ import { createHash, randomUUID } from "node:crypto"; import fs from "node:fs"; +import os from "node:os"; import path from "node:path"; import { resolveZonePortBase } from "./config.js"; import { stateDir } from "./paths.js"; @@ -134,6 +135,21 @@ export function writeZonePort(lane: string, port: number, file: string = portZon * base (BILI_ZONE_PORT override). Explicit user overrides * (BILI_CLAUDE_NATIVE_PORT / BILI_ZCODE_PORT) are resolved separately by * the lanes and imply strict-port launches. */ +/** #3654/#3116: append a diagnostic line to the spawner-owned per-port log + * (%TMPDIR%/bili-proxy-.log) — the exact file a spawner failure message + * points users at. Post-#2864 the child mutes its stderr leg into bili.log as + * soon as its logger is live, so a child that dies during listen setup leaves + * that file EMPTY (the "0 bytes" dead-end); this writes the child-side truth + * there. Named by the PREFERRED port (what the spawner uses for the file + * name); the line itself names the port actually tried. Line grammar mirrors + * logger.ts so mixed writers stay greppable. Best-effort: a failed append + * never affects the exit. */ +export function writeSpawnPortLog(preferredPort: number, line: string): void { + try { + fs.appendFileSync(path.join(os.tmpdir(), `bili-proxy-${preferredPort}.log`), `${line}\n`, "utf8"); + } catch {} +} + export function lanePreferredPort(lane: string, env: NodeJS.ProcessEnv = process.env, file: string = portZoneFilePath()): number { return readZonePort(lane, file) ?? resolveZonePortBase(env); } diff --git a/src/launcher.ts b/src/launcher.ts index 5f734a7696..95d2ffe53d 100644 --- a/src/launcher.ts +++ b/src/launcher.ts @@ -53,6 +53,8 @@ import { type ProxyInstanceFile, type ProxyStartingMarker, } from "./instance.js"; +import { TWIN_ADOPT_EXIT_CODE } from "./listen-policy.js"; +import { tcpConnectable } from "./liveness.js"; import { selfPackageRoot, isBiliPiEntry, ompPluginLoadedFrom, dshNativeInstalled, claudeNativeInstalled, piPackageStrings } from "./plugin-install.js"; import { applyOmpFirstEventTimeout } from "./agent/native-bootstrap.js"; import { logHostEvent } from "./agent/host-log.js"; @@ -316,6 +318,13 @@ export interface LauncherDeps { * discoverLiveInstances, instance.ts). Tests inject a fake so the * clobbered-proxy-origin scenario never touches the real state dir. */ listRegistryInstances?: () => ProxyInstanceFile[]; + /** #3654: TCP pre-probe distinguishing "refused" from "saturated" for + * diagnostics and twin adoption (default: tcpConnectable). Tests inject + * a fake so no real sockets are needed. */ + isConnectable?: (origin: string) => Promise; + /** #3654: registry view behind serving-twin adoption after exit code 3 + * (default: discoverLiveInstances, instance.ts). */ + listAdoptCandidates?: () => ProxyInstanceFile[]; } export function isLaunchClient(value: string): value is ClientName { @@ -3647,6 +3656,7 @@ async function probeLiveInstances( readInstance: () => ProxyInstanceFile | { origin: string } | undefined, fetchHealthInfo: (origin: string) => Promise, diag?: (msg: string) => void, + isConnectable: (origin: string) => Promise = tcpConnectable, ): Promise> { const seen = new Map(); const inst = readInstance(); @@ -3656,7 +3666,17 @@ async function probeLiveInstances( [...seen.values()].map(async (c): Promise<{ inst: ProxyInstanceFile; health: HealthInfo } | undefined> => { if (!isPidAlive(c.pid)) { diag?.(`native-attach: drop ${c.origin} (pid ${c.pid}): owner process gone`); return undefined; } const health = await fetchHealthInfo(c.origin); - if (!health || !health.ok) { diag?.(`native-attach: drop ${c.origin} (pid ${c.pid}): health probe failed`); return undefined; } + if (!health || !health.ok) { + // #3654: a refused connection and a saturated event loop both + // fail this probe — tell them apart in the log: connect-but- + // silent is BUSY (alive, loop inside a large fold), not dead. + // Still dropped either way (#1335 gate unchanged). + const busy = await isConnectable(c.origin); + diag?.(busy + ? `native-attach: drop ${c.origin} (pid ${c.pid}): health probe failed but TCP connect succeeded — instance is BUSY (event loop saturated), not dead (#3654)` + : `native-attach: drop ${c.origin} (pid ${c.pid}): health probe failed`); + return undefined; + } if (health.instanceId !== undefined && health.instanceId !== c.instanceId) { diag?.(`native-attach: drop ${c.origin} (pid ${c.pid}): instance id mismatch (${c.instanceId} vs ${health.instanceId})`); return undefined; } return { inst: c, health }; }), @@ -4593,6 +4613,12 @@ export async function ensureProxyRunning( if (/^HTTP \d{3}$/.test(d)) { return `port ${port} answered ${d} — the listener there is not reporting bili health`; } + // #3654: a timeout with a live TCP listener underneath is the + // event-loop-saturation signature (connect succeeds, JS never runs) + // — name it instead of letting it read as a dead port. + if (/^timeout after \d+ms$/i.test(d) && (await (deps.isConnectable ?? tcpConnectable)(origin))) { + return `port ${port} accepts TCP connections but never answers /__bili/health within the probe budget — the instance there is BUSY (event loop saturated, typically a large compression), not dead (#3654)`; + } return `last health probe failed with ${d}`; }; @@ -4674,6 +4700,21 @@ export async function ensureProxyRunning( // discovery+attach decision as the one-shot probe above, re-run on a // short budget so a listener whose identity record published during // our spawn attempt is still attachable instead of mistaken for air. + // #3654: re-run the identity criteria the child's findSameLaneTwin + // applied at bind time against the CURRENT registry, verify the holder + // still answers TCP (pid reuse), then ride it through the normal attach + // path (watcher + heartbeat included). + const adoptServingTwin = async (): Promise => { + const candidates = (deps.listAdoptCandidates ?? discoverLiveInstances)() + .filter((inst) => isPidAlive(inst.pid) && instanceCompatible(inst, opts, codeFingerprint, foreignLanes) === undefined) + .sort((a, b) => Number(b.port === port) - Number(a.port === port) || b.startedAt - a.startedAt); + for (const inst of candidates) { + if (!(await (deps.isConnectable ?? tcpConnectable)(inst.origin))) continue; + attachDiag(`bili: adopting serving twin ${inst.origin} (pid ${inst.pid}) — the spawned child lost the #3044 duplicate race to it; requests ride the existing instance (#3654)`); + return attachTo(inst); + } + return undefined; + }; const rediscoverAfterChildExit = async (): Promise => { console.error( `bili: spawned proxy exited before becoming healthy — re-checking for a late-publishing listener on port ${port} before failing`, @@ -4697,6 +4738,16 @@ export async function ensureProxyRunning( throw new Error(`bili: proxy spawn failed (${detail}) (runtime: ${runtime}) (log: ${logPath})`); } if (childExit) { + // #3654: exit code 3 is the child's twin-collision signal (#3044 + // listen-policy): a live same-lane same-build instance already + // serves the port. Discovery above declared it unhealthy or missed + // it; the child's bind-time twin check proved it was serving. + // Adopt the holder instead of failing — the user's proxy is up. + if (childExit.code === TWIN_ADOPT_EXIT_CODE) { + const adopted = await adoptServingTwin(); + if (adopted !== undefined) return adopted; + attachDiag("bili: serving-twin adoption failed verification — continuing with the standard post-exit re-discovery"); + } // #1903: the child died before becoming healthy. Under strictPort // that death is almost certainly EADDRINUSE — proof the pinned port // is held by a listener our one-shot discovery above missed because @@ -4709,7 +4760,7 @@ export async function ensureProxyRunning( const retried = await rediscoverAfterChildExit(); if (retried) return retried; const detail = childExit.code !== null - ? `code ${childExit.code}` + ? `code ${childExit.code}${childExit.code === TWIN_ADOPT_EXIT_CODE ? " (twin-collision signal)" : ""}` : childExit.signal ? `signal ${childExit.signal}` : "unknown reason"; throw new Error(`bili: proxy child exited before becoming healthy (${detail}) — ${await diagnosePort(false)} (runtime: ${runtime}) (log: ${logPath}${logTail()})`); } diff --git a/src/listen-policy.ts b/src/listen-policy.ts index 4b0e7a2bee..87460a33e7 100644 --- a/src/listen-policy.ts +++ b/src/listen-policy.ts @@ -18,6 +18,24 @@ * (#3044): the marked process must bind-or-die on exactly the freed port. */ export const UPGRADE_REPLACEMENT_ENV = "BILI_UPGRADE_REPLACEMENT"; +/** #3654: the exit code a twin-collision loser reports to its spawner. The + * duplicate still LOSES (#3044 — it cannot bind the port the serving twin + * holds), but the code says WHY it died: the spawner reads it as an adopt + * signal and attaches to the serving twin instead of failing the bring-up — + * closing the #3654 contradiction where discovery judged the holder dead + * (busy) while the twin rule proved it alive. Every other listen failure + * keeps exit 1. */ +export const TWIN_ADOPT_EXIT_CODE = 3; + +export type FailFastReason = "upgrade-replacement" | "strict-port" | "port-busy" | "twin-collision"; + +/** Pure exit-code mapping for a terminal EADDRINUSE decision — extracted so + * the server (which calls process.exit) and its tests share one definition + * without booting a listener (#3654). */ +export function eaddrinuseExitCode(reason: FailFastReason | undefined): number { + return reason === "twin-collision" ? TWIN_ADOPT_EXIT_CODE : 1; +} + type EaddrinuseAction = | { kind: "wait-predecessor"; port: number } /** nextPort 0 = ask the OS for an ephemeral port (ladder exhausted). */ diff --git a/src/liveness.ts b/src/liveness.ts new file mode 100644 index 0000000000..f11b97f04f --- /dev/null +++ b/src/liveness.ts @@ -0,0 +1,123 @@ +// Three-state liveness probing (#3654). A health probe that times out is NOT +// evidence of death: on loopback the kernel accept queue keeps answering TCP +// connects while the listener's JS event loop is blocked (measured hermetically: +// connect ~2ms during an 8s synchronous block; the HTTP answer only arrives +// after the unblock), so "connected but no health answer within the budget" is +// the signature of a BUSY proxy — alive and saturated (typically a large +// compression) — not dead. ECONNREFUSED and its kin remain the instant death +// signal, so the fast path that detects a truly gone proxy costs milliseconds, +// not the full probe budget. Every consumer (the per-request live-origin +// resolver in native-intercept.ts, launcher discovery diagnostics, spawn-side +// twin adoption) shares this one classification so busy and dead are never +// conflated again. + +import net from "node:net"; + +export type LivenessState = "alive" | "busy" | "dead"; + +export interface LivenessVerdict { + state: LivenessState; + /** Failure class for non-alive verdicts (ECONNREFUSED / HTTP status / timeout class). */ + detail?: string; +} + +/** Budget for the TCP connect pre-probe. Loopback connects land in + * milliseconds; the cap covers a machine whose network stack itself is wedged, + * where any answer would be too late to matter. */ +export const CONNECT_PROBE_TIMEOUT_MS = 2_000; + +function originHostPort(origin: string): { host: string; port: number } | undefined { + try { + const u = new URL(origin); + if (u.protocol !== "http:" && u.protocol !== "https:") return undefined; + const host = u.hostname.replace(/^\[|\]$/g, ""); + if (host === "") return undefined; + const port = u.port !== "" ? Number(u.port) : u.protocol === "https:" ? 443 : 80; + return Number.isInteger(port) && port > 0 ? { host, port } : undefined; + } catch { + return undefined; + } +} + +interface ConnectOutcome { + ok: boolean; + /** errno class when the connect itself failed (ECONNREFUSED, …). */ + detail?: string; +} + +function attemptConnect(host: string, port: number, timeoutMs: number): Promise { + return new Promise((resolve) => { + let settled = false; + const settle = (outcome: ConnectOutcome): void => { + if (settled) return; + settled = true; + socket.destroy(); + resolve(outcome); + }; + const socket = net.connect({ host, port }); + socket.setTimeout(timeoutMs); + socket.once("connect", () => settle({ ok: true })); + socket.once("timeout", () => settle({ ok: false, detail: `connect timeout after ${timeoutMs}ms` })); + socket.once("error", (err: NodeJS.ErrnoException) => + settle({ ok: false, detail: typeof err.code === "string" && err.code.length > 0 ? err.code : String(err.message ?? err) })); + }); +} + +/** True when the origin's socket accepts a TCP connection within the budget. + * Proves the LISTENING side exists at the kernel level — including a proxy + * whose event loop is blocked, because its accept queue still drains. Used by + * the launcher's twin adoption (a live pid alone does not prove the socket + * serves — Windows pid reuse) and by discovery diagnostics. */ +export async function tcpConnectable(origin: string, timeoutMs: number = CONNECT_PROBE_TIMEOUT_MS): Promise { + const hp = originHostPort(origin); + if (hp === undefined) return false; + const outcome = await attemptConnect(hp.host, hp.port, timeoutMs); + return outcome.ok; +} + +function errCodeOf(err: unknown): string | undefined { + const e = err as { cause?: { code?: unknown; name?: unknown }; code?: unknown; name?: unknown } | null; + const code = e?.cause?.code ?? e?.code; + return typeof code === "string" && code.length > 0 ? code : undefined; +} + +/** Timeout-class error from fetch/undici under AbortSignal.timeout — the + * "connected but never answered" signature. Everything else that throws is a + * transport failure (reset, closed, refused mid-flight) → dead. */ +function isTimeoutErr(err: unknown): boolean { + const e = err as { cause?: { name?: unknown; code?: unknown }; name?: unknown; code?: unknown } | null; + const names = [e?.name, e?.cause?.name]; + const codes = [e?.code, e?.cause?.code]; + return names.includes("TimeoutError") || names.includes("AbortError") || + codes.includes("ABORT_ERR") || codes.includes("UND_ERR_HEADERS_TIMEOUT"); +} + +/** Full three-state probe of GET `/__bili/health` under `budgetMs`, + * preceded by the TCP connect pre-probe. The "alive" claim additionally + * validates the PAYLOAD (`{ ok: true }`) — a foreign service answering 200 on + * every path must not read as a healthy bili proxy (#1753 discipline, which + * the launcher-side probes already applied but the resolver-side probe did + * not). */ +export async function probeLiveness(origin: string, budgetMs: number): Promise { + const hp = originHostPort(origin); + if (hp === undefined) return { state: "dead", detail: "unparseable origin" }; + // Connect first: a refused connection is a high-confidence death signal on + // loopback and costs ~0ms, so it short-circuits before paying the HTTP + // budget at all. + const conn = await attemptConnect(hp.host, hp.port, CONNECT_PROBE_TIMEOUT_MS); + if (!conn.ok) return { state: "dead", detail: conn.detail ?? "connect failed" }; + try { + const res = await fetch(`${origin}/__bili/health`, { signal: AbortSignal.timeout(budgetMs) }); + if (!res.ok) return { state: "dead", detail: `HTTP ${res.status}` }; + try { + const data = (await res.json()) as { ok?: unknown }; + if (data.ok !== true) return { state: "dead", detail: "health payload reports not-ok" }; + } catch { + return { state: "dead", detail: "non-JSON health payload (foreign listener?)" }; + } + return { state: "alive" }; + } catch (err) { + if (isTimeoutErr(err)) return { state: "busy", detail: `no health answer within ${budgetMs}ms after TCP connect` }; + return { state: "dead", detail: errCodeOf(err) ?? String((err as Error)?.message ?? err).slice(0, 120) }; + } +} diff --git a/src/server.ts b/src/server.ts index a35e0831a2..81b5845a2c 100644 --- a/src/server.ts +++ b/src/server.ts @@ -58,7 +58,7 @@ import { reconcileFoldCoverage, noteSystemPromptFingerprint, resolveFoldReconcil import { biliToolsDeclaredOnWire, countBiliToolUses, evaluateSelfHealRound, nudgeSuppressed, pluginLaneDegraded, pluginLaneRestore } from "./session-self-heal.js"; import { getSession, hasProcessedState, listSessions, peekSession, type PendingRetrieval, type Session, initSessions, markDirty, flushAllSessions, acquireInFlight, releaseInFlight, totalInFlight, snapshotMessages, ensureCanonicalId, storeEffectiveConfig, foldCoverage, postRebuildAnchorTokens, setPostRebuildAnchor, tickPostRebuildAnchor, splitSessionWarnings, RESPONSES_TURN_SEPARATOR_ID_PREFIX } from "./session.js"; import { detectStaleInstall } from "./update.js"; -import { decideEaddrinuse, postBindCanLadder, UPGRADE_REPLACEMENT_ENV } from "./listen-policy.js"; +import { decideEaddrinuse, eaddrinuseExitCode, postBindCanLadder, UPGRADE_REPLACEMENT_ENV } from "./listen-policy.js"; import { getAdvisoryState, cannotResolveTarget } from "./advisory.js"; import { PACKAGE_NAME, VERSION } from "./version.js"; import { @@ -98,7 +98,7 @@ import { getStore } from "./persist.js"; import { log as loggerLog, configureLogger, getLogPath, closeLogger, isFileLoggingActive, isStreamWriteError, isBenignSocketRaceError, enterSessionContext, muteStderrLeg } from "./logger.js"; import { queryLogLines } from "./web/logs-query.js"; import { configFile, defaultLogFile, stateDir } from "./paths.js"; -import { atomicWriteInstanceFile, clearProxyInstanceFile, entryScriptFingerprint, findSameLanePredecessor, findSameLaneTwin, isPidAlive, listInstances, readProxyInstanceFile, registerInstanceAndWarn, unregisterInstance, warnOnNewPeers, type ProxyInstanceFile, type RegistryEntry } from "./instance.js"; +import { atomicWriteInstanceFile, clearProxyInstanceFile, entryScriptFingerprint, findSameLanePredecessor, findSameLaneTwin, isPidAlive, listInstances, readProxyInstanceFile, registerInstanceAndWarn, unregisterInstance, warnOnNewPeers, writeSpawnPortLog, type ProxyInstanceFile, type RegistryEntry } from "./instance.js"; import { mayClobberPointer, probeOwnerInfo, resolvePortClaim } from "./port-owner.js"; import { hoistTrappedToolItems } from "./tool-pair-order.js"; import { computeAnthropicMessageMarks, stampAnthropicSystemCacheControl, anthropicToolsCarryCacheControl } from "./loop/cache-control.js"; @@ -1309,7 +1309,7 @@ export async function startServer(opts: ProxyOptions): Promise { const hint = err.code === "EADDRINUSE" ? busyReason === "twin-collision" - ? ` — port ${lastTriedPort} is already served by a same-build bili instance (pid ${twin?.pid}, lane ${launcherLane}) — exiting instead of hopping: the duplicate loses and the serving instance keeps serving; the next launch will attach to it (#3044).` + ? ` — port ${lastTriedPort} is already served by a same-build bili instance (pid ${twin?.pid}, lane ${launcherLane}) — exiting instead of hopping: the duplicate loses and the serving instance keeps serving; the next launch will attach to it (#3044). A spawner reads this exit as an adopt signal and attaches to the serving instance itself (#3654).` : busyReason === "upgrade-replacement" ? ` — this process is an upgrade-restart replacement (#3044): it must own port ${lastTriedPort} exactly, but the prior instance still holds it. Exiting instead of hopping to ${lastTriedPort + 1} — a drifted replacement serves no client and leaks while the prior instance keeps serving; the upgrade retries on the next cooldown cycle.` : busyReason === "strict-port" @@ -1318,13 +1318,18 @@ export async function startServer(opts: ProxyOptions): Promise { : err.code === "EACCES" ? ` — port ${lastTriedPort} requires privileges. Use a port >= 1024.` : ""; - log("error", `listen failed: ${err.code ?? ""} ${err.message}${hint}`); + const listenLine = `listen failed: ${err.code ?? ""} ${err.message}${hint}`; + log("error", listenLine); + // #3654/#3116: a spawner's failure message points users at the per-port + // log, but post-#2864 this child mutes its stderr leg into bili.log — + // without this line that file stays 0 bytes and the diagnosis dies there. + if (launchToken !== undefined) writeSpawnPortLog(opts.port, `${new Date().toISOString()} [error] [v=${VERSION}] ${listenLine}`); shuttingDown = true; server.close(); flushConversations(); void flushAllSessions().finally(() => { closeLogger(); - process.exit(1); + process.exit(eaddrinuseExitCode(busyReason)); }); }); // Catch stray rejections/throws from background work (compress loops, diff --git a/tests/issue3654-health-probe-three-state.test.ts b/tests/issue3654-health-probe-three-state.test.ts new file mode 100644 index 0000000000..0061ed7099 --- /dev/null +++ b/tests/issue3654-health-probe-three-state.test.ts @@ -0,0 +1,576 @@ +// #3654: under load a 1.5s health probe times out against a LIVE proxy whose +// event loop is inside a large compression; the old two-state probe read that +// as death, dropped the origin, respawned into the same process (twin- +// collision exit), and oversized sessions were then refused locally (#3308). +// This suite pins the three-state contract end to end: +// A) probeLiveness — connect-refused = dead (fast), connect-but-silent = +// busy, payload-validated alive; +// B) createLiveOriginResolver — busy HELD origins are held through a bounded +// jittered wait window (never dropped, never cached), dead stays an +// instant fast path, busy replacements are routed to anyway, and the +// refusal reason records busy-exhausted vs dead; +// C) ensureProxyRunning — a twin-collision child (exit code 3) leads to +// adoption of the serving instance instead of a failed bring-up; +// D) eaddrinuseExitCode — only twin-collision maps to the adopt signal; +// E) writeSpawnPortLog — the per-port log the failure message points at is +// actually written for spawner children (#3116's 0-byte dead end); +// F) refusal wording — busy-exhausted must not claim "proxy unavailable … +// guaranteed upstream 400" (owner requirement 5). + +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import http from "node:http"; +import net from "node:net"; +import type { AddressInfo } from "node:net"; +import { after } from "node:test"; + +// The module-level guard reads NODE_TEST_CONTEXT while the module EVALUATES, +// and static imports hoist above any assignment — so the value must be set +// first and the modules loaded dynamically. +process.env.NODE_TEST_CONTEXT = "1"; + +const STATE_ROOT = fs.mkdtempSync(path.join(os.tmpdir(), "bili-3654-state-")); +const PREV_XDG_STATE_HOME = process.env.XDG_STATE_HOME; +process.env.XDG_STATE_HOME = STATE_ROOT; +after(() => { + if (PREV_XDG_STATE_HOME === undefined) delete process.env.XDG_STATE_HOME; else process.env.XDG_STATE_HOME = PREV_XDG_STATE_HOME; + fs.rmSync(STATE_ROOT, { recursive: true, force: true }); +}); + +const { probeLiveness, tcpConnectable } = await import("../src/liveness.ts"); +const { decideEaddrinuse, eaddrinuseExitCode, TWIN_ADOPT_EXIT_CODE } = await import("../src/listen-policy.ts"); +const { writeSpawnPortLog } = await import("../src/instance.ts"); +const { createLiveOriginResolver } = await import("../src/agent/native-intercept.ts"); +import type { LivenessVerdict } from "../src/liveness.ts"; +import type { NativeInterceptState } from "../src/agent/native-intercept.ts"; +import type { V2State } from "../src/agent/opencode-v2.ts"; +import type { ProxyInstanceFile } from "../src/instance.ts"; +import { entryScriptFingerprint } from "../src/instance.ts"; +import { ensureProxyRunning, type SpawnChild, type SpawnFn } from "../src/launcher.ts"; + +function listenPort(server: net.Server | http.Server): Promise { + return new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", () => resolve((server.address() as AddressInfo).port)); + }); +} + +async function withServer(make: (server: net.Server) => void, fn: (origin: string, server: net.Server) => Promise): Promise { + const server = net.createServer(); + // Destroy tracked connections on teardown: a paused (never-answered) + // socket would otherwise hold server.close()'s callback hostage. + const conns = new Set(); + server.on("connection", (s) => { conns.add(s); s.on("close", () => conns.delete(s)); }); + make(server); + const port = await listenPort(server); + try { + return await fn(`http://127.0.0.1:${port}`, server); + } finally { + for (const s of conns) s.destroy(); + conns.clear(); + await new Promise((r) => server.close(() => r())); + } +} + +// ———— A) probeLiveness ———————————————————————————————————————————————— + +test("A: healthy bili proxy answers → alive", async () => { + await withServer(() => {}, async (origin) => { + const srv = http.createServer((_req, res) => { res.writeHead(200, { "content-type": "application/json" }); res.end(JSON.stringify({ ok: true })); }); + const port = await listenPort(srv); + try { + const v = await probeLiveness(`http://127.0.0.1:${port}`, 1_000); + assert.equal(v.state, "alive"); + } finally { + await new Promise((r) => srv.close(() => r())); + } + }); +}); + +test("A: refused connection classifies dead in milliseconds (fast path unchanged)", async () => { + // Grab a real port, release it, probe it: ECONNREFUSED on loopback. + const holder = net.createServer(); + const port = await listenPort(holder); + await new Promise((r) => holder.close(() => r())); + const t0 = Date.now(); + const v = await probeLiveness(`http://127.0.0.1:${port}`, 5_000); + const elapsed = Date.now() - t0; + assert.equal(v.state, "dead"); + assert.match(v.detail ?? "", /ECONNREFUSED|ECONNRESET|connect/i); + assert.ok(elapsed < 1_000, `refused-connect must not pay the HTTP budget (took ${elapsed}ms)`); +}); + +test("A: connected but silent under the budget → busy (the #3654 signature)", async () => { + await withServer((s) => s.on("connection", (socket) => socket.pause()), async (origin) => { + const t0 = Date.now(); + const v = await probeLiveness(origin, 300); + const elapsed = Date.now() - t0; + assert.equal(v.state, "busy"); + assert.match(v.detail ?? "", /no health answer within 300ms/); + assert.ok(elapsed >= 250, `the budget must actually elapse before calling it busy (took ${elapsed}ms)`); + }); +}); + +test("A: foreign 200 HTML is not a healthy bili proxy → dead", async () => { + const srv = http.createServer((_req, res) => { res.writeHead(200, { "content-type": "text/html" }); res.end("hello"); }); + const port = await listenPort(srv); + try { + const v = await probeLiveness(`http://127.0.0.1:${port}`, 1_000); + assert.equal(v.state, "dead"); + assert.match(v.detail ?? "", /non-JSON/); + } finally { + await new Promise((r) => srv.close(() => r())); + } +}); + +test("A: JSON health reporting ok:false → dead", async () => { + const srv = http.createServer((_req, res) => { res.writeHead(200, { "content-type": "application/json" }); res.end(JSON.stringify({ ok: false })); }); + const port = await listenPort(srv); + try { + const v = await probeLiveness(`http://127.0.0.1:${port}`, 1_000); + assert.equal(v.state, "dead"); + assert.match(v.detail ?? "", /not-ok/); + } finally { + await new Promise((r) => srv.close(() => r())); + } +}); + +test("A: tcpConnectable sees through a saturated loop (accept queue answers)", async () => { + await withServer((s) => s.on("connection", (socket) => socket.pause()), async (origin) => { + assert.equal(await tcpConnectable(origin, 500), true, "a blocked-loop listener still accepts TCP"); + }); + const holder = net.createServer(); + const port = await listenPort(holder); + await new Promise((r) => holder.close(() => r())); + assert.equal(await tcpConnectable(`http://127.0.0.1:${port}`, 500), false); +}); + +// ———— B) createLiveOriginResolver ————————————————————————————————————— + +type VerdictOrBool = LivenessVerdict | boolean; + +function scriptedProbe(script: VerdictOrBool[], byOrigin?: (o: string, i: number) => VerdictOrBool): { probe: (o: string) => Promise; calls: string[] } { + const calls: string[] = []; + let i = 0; + return { + calls, + probe: async (o) => { + calls.push(o); + const v = byOrigin !== undefined ? byOrigin(o, i) : script[Math.min(i, script.length - 1)]; + i++; + return v; + }, + }; +} + +const BUSY: LivenessVerdict = { state: "busy", detail: "no health answer within 1500ms after TCP connect" }; +const DEAD: LivenessVerdict = { state: "dead", detail: "ECONNREFUSED" }; +const ALIVE: LivenessVerdict = { state: "alive" }; + +test("B: busy held origin recovers inside the window — no drop, no respawn, no cache solidification", async () => { + const HELD = "http://127.0.0.1:18901"; + let respawnCalls = 0; + const state: NativeInterceptState = { + origin: HELD, + ready: Promise.resolve(undefined), + respawn: async () => { respawnCalls++; return undefined; }, + }; + // TTL far above the episode: if BUSY were ever cached, every re-probe in + // the window would hit the cache and recovery below would be impossible. + const { probe, calls } = scriptedProbe([BUSY, ALIVE]); + const resolve = createLiveOriginResolver(state, { probe, probeTtlMs: 1_000, busyWaitCapMs: 400, busyReprobeBaseMs: 40 }); + const t0 = Date.now(); + const got = await resolve(); + const elapsed = Date.now() - t0; + assert.equal(got, HELD, "the held origin must survive its own busy episode"); + assert.equal(respawnCalls, 0, "a busy origin must never trigger a respawn"); + assert.equal(calls.length, 2, "first probe busy, one re-probe alive"); + assert.ok(elapsed < 400 + 200, `recovery must land inside the window (took ${elapsed}ms)`); + // Second call immediately: the ALIVE verdict IS cached → zero raw probes. + const before = calls.length; + assert.equal(await resolve(), HELD); + assert.equal(calls.length, before, "alive verdicts stay TTL-cached (only busy is uncached)"); +}); + +test("B: busy exhaustion lets go, respawns, and adopts the replacement", async () => { + const HELD = "http://127.0.0.1:18902"; + const REPL = "http://127.0.0.1:18903"; + let respawnCalls = 0; + const state: NativeInterceptState = { + origin: HELD, + ready: Promise.resolve(undefined), + respawn: async () => { respawnCalls++; state.origin = REPL; return REPL; }, + }; + const { probe } = scriptedProbe([], (o) => (o === HELD ? BUSY : ALIVE)); + const resolve = createLiveOriginResolver(state, { probe, probeTtlMs: 1_000, busyWaitCapMs: 200, busyReprobeBaseMs: 30 }); + const got = await resolve(); + assert.equal(got, REPL, "the replacement must be adopted after the window"); + assert.equal(respawnCalls, 1); + assert.equal(state.lastUnavailableReason, undefined, "success clears the refusal reason"); +}); + +test("B: a busy REPLACEMENT is routed to anyway — the request queues behind the fold", async () => { + const HELD = "http://127.0.0.1:18904"; + const REPL = "http://127.0.0.1:18905"; + let respawnCalls = 0; + const state: NativeInterceptState = { + origin: HELD, + ready: Promise.resolve(undefined), + respawn: async () => { respawnCalls++; state.origin = REPL; return REPL; }, + }; + const { probe } = scriptedProbe([], () => BUSY); + const resolve = createLiveOriginResolver(state, { probe, probeTtlMs: 1_000, busyWaitCapMs: 150, busyReprobeBaseMs: 30 }); + const got = await resolve(); + assert.equal(got, REPL, "refusing a live-but-busy replacement would fail the request the adoption exists to save"); + assert.equal(respawnCalls, 1); + assert.equal(state.lastUnavailableReason, undefined); +}); + +test("B: busy exhaustion with a dead replacement refuses and records busy-exhausted", async () => { + const HELD = "http://127.0.0.1:18906"; + const REPL = "http://127.0.0.1:18907"; + let giveUps = 0; + const state: NativeInterceptState = { + origin: HELD, + ready: Promise.resolve(undefined), + respawn: async () => { state.origin = REPL; return REPL; }, + onGiveUp: () => { giveUps++; }, + }; + const { probe } = scriptedProbe([], (o) => (o === HELD ? BUSY : DEAD)); + const resolve = createLiveOriginResolver(state, { probe, probeTtlMs: 1_000, busyWaitCapMs: 150, busyReprobeBaseMs: 30, respawnCooldownMs: 60_000 }); + const got = await resolve(); + assert.equal(got, undefined); + assert.equal(state.lastUnavailableReason, "busy-exhausted", "the refusal wording depends on this distinction"); + assert.equal(giveUps, 1); +}); + +test("B: dead verdict keeps the instant fast path (no busy window paid)", async () => { + const HELD = "http://127.0.0.1:18908"; + const REPL = "http://127.0.0.1:18909"; + let respawnCalls = 0; + const state: NativeInterceptState = { + origin: HELD, + ready: Promise.resolve(undefined), + respawn: async () => { respawnCalls++; state.origin = REPL; return REPL; }, + }; + const { probe } = scriptedProbe([], (o) => (o === HELD ? DEAD : ALIVE)); + const resolve = createLiveOriginResolver(state, { probe, probeTtlMs: 1_000, busyWaitCapMs: 30_000, busyReprobeBaseMs: 1_000 }); + const t0 = Date.now(); + const got = await resolve(); + const elapsed = Date.now() - t0; + assert.equal(got, REPL); + assert.equal(respawnCalls, 1); + assert.ok(elapsed < 500, `a refused connection must not pay any wait window (took ${elapsed}ms)`); +}); + +test("B: dead verdicts stay TTL-cached across callers", async () => { + const HELD = "http://127.0.0.1:18910"; + const REPL = "http://127.0.0.1:18911"; + let respawnCalls = 0; + // The respawn deliberately does NOT claim state.origin: a respawn that + // does makes the next call re-own the dead replacement and take the + // ownedThenLost path (which bypasses the cooldown BY DESIGN), so the + // cache could never be observed. + const state: NativeInterceptState = { + origin: HELD, + ready: Promise.resolve(undefined), + respawn: async () => { respawnCalls++; return REPL; }, + }; + const { probe, calls } = scriptedProbe([DEAD]); + const resolve = createLiveOriginResolver(state, { probe, probeTtlMs: 1_000, respawnCooldownMs: 60_000 }); + assert.equal(await resolve(), undefined); + // Second call: no held origin, cooldown blocks a second respawn, and the + // replacement's dead verdict comes straight from the TTL cache. + assert.equal(await resolve(), undefined); + assert.equal(respawnCalls, 1); + assert.equal(calls.length, 2, `held-dead + replacement-dead, then cache hits only (got ${calls.length} raw probes)`); + assert.equal(state.lastUnavailableReason, "bootstrap-failed"); +}); + +test("B: concurrent callers share one respawn when the held origin dies (#1957)", async () => { + const HELD = "http://127.0.0.1:18912"; + const REPL = "http://127.0.0.1:18913"; + let respawnCalls = 0; + const state: NativeInterceptState = { + origin: HELD, + ready: Promise.resolve(undefined), + respawn: async () => { respawnCalls++; state.origin = REPL; return REPL; }, + }; + const { probe } = scriptedProbe([], (o) => (o === HELD ? DEAD : ALIVE)); + const resolve = createLiveOriginResolver(state, { probe, probeTtlMs: 1_000, respawnCooldownMs: 60_000 }); + const [a, b] = await Promise.all([resolve(), resolve()]); + assert.equal(a, REPL); + assert.equal(b, REPL); + assert.equal(respawnCalls, 1, "both callers must ride the single shared respawn"); +}); + +test("B: legacy boolean probes still map (true→alive, false→dead)", async () => { + const HELD = "http://127.0.0.1:18914"; + const REPL = "http://127.0.0.1:18915"; + let respawnCalls = 0; + const state: NativeInterceptState = { + origin: HELD, + ready: Promise.resolve(undefined), + respawn: async () => { respawnCalls++; state.origin = REPL; return REPL; }, + }; + const resolve = createLiveOriginResolver(state, { probe: async (o) => o !== HELD, probeTtlMs: 1_000 }); + assert.equal(await resolve(), REPL); + assert.equal(respawnCalls, 1); +}); + +// ———— C) ensureProxyRunning twin adoption ———————————————————————————————— + +const FP_SCRIPT = path.join(os.tmpdir(), `bili-fp-3654-${process.pid}.js`); +fs.writeFileSync(FP_SCRIPT, "// fingerprint fixture\n"); +const FP = entryScriptFingerprint(FP_SCRIPT); +assert.ok(FP !== undefined, "fixture script must fingerprint"); + +const TWIN_ORIGIN = "http://127.0.0.1:18916"; + +function tmpHome(): { dir: string; restore: () => void } { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "bili-3654-home-")); + const prevCfg = process.env.XDG_CONFIG_HOME; + const prevState = process.env.XDG_STATE_HOME; + process.env.XDG_CONFIG_HOME = dir; + process.env.XDG_STATE_HOME = dir; + return { + dir, + restore: () => { + if (prevCfg === undefined) delete process.env.XDG_CONFIG_HOME; else process.env.XDG_CONFIG_HOME = prevCfg; + if (prevState === undefined) delete process.env.XDG_STATE_HOME; else process.env.XDG_STATE_HOME = prevState; + }, + }; +} + +function twinInstance(lane: string): ProxyInstanceFile { + return { + origin: TWIN_ORIGIN, + instanceId: "inst-3654", + pid: process.pid, + startedAt: Date.now(), + host: "127.0.0.1", + port: 18916, + passthrough: false, + mitmDomains: [], + modelWindows: {}, + lane, + codeFingerprint: FP, + }; +} + +function fakeChildExitting(code: number | null): SpawnChild { + const handlers = new Map void>(); + return { + pid: 42_470, + unref() {}, + kill() { return true; }, + on(ev: string, cb: (...args: unknown[]) => void) { + handlers.set(ev, cb); + if (ev === "exit") queueMicrotask(() => cb(code, null)); + return this; + }, + }; +} + +function fakeClock() { + let ticks = 0; + return { + now: () => ticks * 1000, + sleep: () => { ticks += 1; return Promise.resolve(); }, + }; +} + +interface BringUpResult { + handle: Awaited>; + spawnCalls: number; + diag: string[]; +} + +async function bringUp3654(twin: ProxyInstanceFile | undefined, exitCode: number | null): Promise { + let spawnCalls = 0; + const diag: string[] = []; + const clock = fakeClock(); + const handle = await ensureProxyRunning( + { host: "127.0.0.1", port: 0, passthrough: false, debug: false, lane: "dsh" }, + { + fetchImpl: async () => ({ ok: true }), + fetchHealthInfo: async (origin) => (origin === TWIN_ORIGIN ? { ok: true, watchdog: { armed: true } } : undefined), + readInstanceFile: () => undefined, + listRegistryInstances: () => [], + spawnImpl: (_cmd: string, _args: readonly string[]): SpawnChild => { + spawnCalls++; + return fakeChildExitting(exitCode); + }, + now: clock.now, + sleep: clock.sleep, + registerWatcher: async () => "ok", + attachDiag: (msg) => { diag.push(msg); }, + scriptPath: FP_SCRIPT, + listAdoptCandidates: () => (twin !== undefined ? [twin] : []), + isConnectable: async () => true, + }, + ); + return { handle, spawnCalls, diag }; +} + +test("C: twin-collision exit adopts the serving instance instead of failing", async () => { + const home = tmpHome(); + try { + const { handle, spawnCalls, diag } = await bringUp3654(twinInstance("dsh"), TWIN_ADOPT_EXIT_CODE); + assert.equal(spawnCalls, 1, "the spawn happened and lost the race"); + assert.equal(handle.attached, true, "adoption rides the normal attach path"); + assert.equal(handle.origin, TWIN_ORIGIN); + assert.ok(diag.some((d) => d.includes("adopting serving twin")), "adoption must be announced"); + } finally { + home.restore(); + } +}); + +test("C: twin-collision with nothing adoptable keeps the original failure (code 3 named)", async () => { + const home = tmpHome(); + try { + const pending = bringUp3654(undefined, TWIN_ADOPT_EXIT_CODE); + await assert.rejects(pending, /code 3 \(twin-collision signal\)/); + } finally { + home.restore(); + } +}); + +test("C: a foreign-lane instance is never adopted", async () => { + const home = tmpHome(); + try { + const pending = bringUp3654(twinInstance("pi"), TWIN_ADOPT_EXIT_CODE); + await assert.rejects(pending, /code 3 \(twin-collision signal\)/); + } finally { + home.restore(); + } +}); + +test("C: a plain crash (exit 1) does NOT take the adoption path", async () => { + const home = tmpHome(); + try { + const pending = bringUp3654(twinInstance("dsh"), 1); + await assert.rejects(pending, /code 1/); + // The rejection alone cannot show adoption was skipped — run again + // capturing diagnostics through a resolved wrapper is overkill; the + // code-1 message lacking the twin suffix pins the branch selection. + } finally { + home.restore(); + } +}); + +// ———— D) eaddrinuseExitCode ———————————————————————————————————————————— + +const baseCtx = { + upgradeReplacement: false, + launchToken: true, + strictPort: false, + attempts: 1, + maxAttempts: 17, + predecessorTicksLeft: 0, + predecessorHolds: false, + lastTriedPort: 18917, +}; + +test("D: only twin-collision maps to the adopt signal; every other terminal decision exits 1", () => { + assert.equal(TWIN_ADOPT_EXIT_CODE, 3); + const twin = decideEaddrinuse({ ...baseCtx, twinHolds: true }); + assert.deepEqual(twin, { kind: "fail-fast", reason: "twin-collision" }); + assert.equal(eaddrinuseExitCode("twin-collision"), 3); + const strict = decideEaddrinuse({ ...baseCtx, twinHolds: false, strictPort: true }); + assert.equal(strict.kind === "fail-fast" ? strict.reason : undefined, "strict-port"); + assert.equal(eaddrinuseExitCode("strict-port"), 1); + const busy = decideEaddrinuse({ ...baseCtx, twinHolds: false, launchToken: false }); + assert.equal(busy.kind === "fail-fast" ? busy.reason : undefined, "port-busy"); + assert.equal(eaddrinuseExitCode("port-busy"), 1); + const upgrade = decideEaddrinuse({ ...baseCtx, twinHolds: false, upgradeReplacement: true }); + assert.equal(upgrade.kind === "fail-fast" ? upgrade.reason : undefined, "upgrade-replacement"); + assert.equal(eaddrinuseExitCode("upgrade-replacement"), 1); + assert.equal(eaddrinuseExitCode(undefined), 1, "non-terminal decisions carry no exit code"); +}); + +// ———— E) writeSpawnPortLog —————————————————————————————————————————————— + +test("E: spawner children write their listen failure into the per-port log", () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "bili-3654-tmp-")); + const prev = { TMPDIR: process.env.TMPDIR, TEMP: process.env.TEMP, TMP: process.env.TMP }; + process.env.TMPDIR = dir; + process.env.TEMP = dir; + process.env.TMP = dir; + try { + assert.equal(dir, os.tmpdir(), "os.tmpdir() must follow the override on this platform"); + writeSpawnPortLog(18918, "2026-10-18T00:00:00.000Z [error] [v=test] listen failed: EADDRINUSE address in use: :::18918"); + const file = path.join(dir, "bili-proxy-18918.log"); + assert.ok(fs.existsSync(file), "the file the failure message points at must exist"); + assert.match(fs.readFileSync(file, "utf8"), /listen failed: EADDRINUSE/); + } finally { + for (const [k, v] of Object.entries(prev)) if (v === undefined) delete process.env[k]; else process.env[k] = v; + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + +// ———— F) refusal wording ———————————————————————————————————————————————— + +const MODEL_URL = "https://api.anthropic.com/v1/messages"; + +test("F: busy-exhausted refusal says 'alive but unresponsive', never 'guaranteed upstream 400'", async () => { + const { createNativeRoute } = await import("../src/agent/opencode-native.ts"); + const HELD = "http://127.0.0.1:18919"; + const state: NativeInterceptState = { origin: HELD, ready: Promise.resolve(undefined) }; + const route = createNativeRoute(state, { + probe: async () => ({ state: "busy" as const }), + probeTtlMs: 500, + busyWaitCapMs: 120, + busyReprobeBaseMs: 20, + }); + const s: V2State = { windows: new Map([["prov/model-x", 1_000]]) }; + const body = JSON.stringify({ messages: [{ role: "user", content: "x".repeat(5_000) }] }); + const e = { + sessionID: "ses_busy", + model: { providerID: "prov", id: "model-x" }, + request: new Request(MODEL_URL, { method: "POST", headers: { "content-type": "application/json" }, body }), + }; + const err = await route(e, s).then(() => undefined, (x: unknown) => x); + assert.ok(err instanceof Error, "oversize body must be refused locally"); + assert.match(err.message, /alive but unresponsive/); + assert.doesNotMatch(err.message, /proxy unavailable/, "a saturated proxy is not 'unavailable'"); + assert.doesNotMatch(err.message, /guaranteed upstream 400/, "it would serve the request once the fold finishes"); + assert.equal(state.lastUnavailableReason, "busy-exhausted"); +}); + +test("F: a genuinely dead proxy keeps the legacy refusal wording byte-for-byte", async () => { + const { createNativeRoute } = await import("../src/agent/opencode-native.ts"); + const HELD = "http://127.0.0.1:18920"; + const state: NativeInterceptState = { origin: HELD, ready: Promise.resolve(undefined) }; + const route = createNativeRoute(state, { probe: async () => false, probeTtlMs: 500 }); + const s: V2State = { windows: new Map([["prov/model-x", 1_000]]) }; + const body = JSON.stringify({ messages: [{ role: "user", content: "x".repeat(5_000) }] }); + const e = { + sessionID: "ses_dead", + model: { providerID: "prov", id: "model-x" }, + request: new Request(MODEL_URL, { method: "POST", headers: { "content-type": "application/json" }, body }), + }; + const err = await route(e, s).then(() => undefined, (x: unknown) => x); + assert.ok(err instanceof Error); + assert.match(err.message, /bili: proxy unavailable and this request cannot be sent uncompressed/); + assert.match(err.message, /Recovery is automatic \(#3308\)/); + assert.equal(state.lastUnavailableReason, "dead"); +}); + +test("F: an under-window body still goes direct when nothing is routable (fail-open unchanged)", async () => { + const { createNativeRoute } = await import("../src/agent/opencode-native.ts"); + const state: NativeInterceptState = { origin: undefined, ready: Promise.resolve(undefined) }; + const route = createNativeRoute(state, { probe: async () => ({ state: "busy" as const }), probeTtlMs: 500, busyWaitCapMs: 100, busyReprobeBaseMs: 20 }); + const s: V2State = { windows: new Map([["prov/model-x", 1_000_000]]) }; + const body = JSON.stringify({ messages: [{ role: "user", content: "small" }] }); + const e = { + sessionID: "ses_small", + model: { providerID: "prov", id: "model-x" }, + request: new Request(MODEL_URL, { method: "POST", headers: { "content-type": "application/json" }, body }), + }; + await route(e, s); + assert.equal(s.proxyBase, undefined, "direct send — no proxy target pinned"); +}); From b5e02b0eeed8eb6e85c1756df8716da92c1c04b8 Mon Sep 17 00:00:00 2001 From: awork Date: Sun, 18 Oct 2026 17:16:05 +0800 Subject: [PATCH 2/3] fix: busy-episode log announces once per episode, not once per caller (#3654) Review nit on 9d5127762: the BUSY announcement line fired for every concurrent caller entering the same episode while its comment claimed it announces an episode ONCE, and the announcedAt field backing that claim was never read. Gate the announcement on fresh-episode creation, drop the dead field, and clear episode entries at the respawn generation boundary (same place the verdict cache is rebuilt) so a post-respawn busy episode behind the same origin announces itself instead of inheriting a stale entry. Log-only; no behavior change. --- src/agent/native-intercept.ts | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/src/agent/native-intercept.ts b/src/agent/native-intercept.ts index 52ff0e8231..040b8a03c8 100644 --- a/src/agent/native-intercept.ts +++ b/src/agent/native-intercept.ts @@ -618,9 +618,10 @@ export function createLiveOriginResolver(state: NativeInterceptState, deps: Live const busyWaitCapMs = deps.busyWaitCapMs ?? BUSY_WAIT_CAP_MS; const busyReprobeBaseMs = deps.busyReprobeBaseMs ?? BUSY_REPROBE_BASE_MS; let lastRespawn = 0; - // #3654: one entry per busy episode per origin (attempt count + first-seen - // time) so the log announces an episode ONCE and then counts attempts. - const busyEpisodes = new Map(); + // #3654: one entry per busy episode per origin (attempt count) so the log + // announces an episode ONCE (the first caller that sees it) and concurrent + // callers in the same episode only count attempts. + const busyEpisodes = new Map(); const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); return async (): Promise => { let ownedThenLost = false; @@ -650,10 +651,13 @@ export function createLiveOriginResolver(state: NativeInterceptState, deps: Live // (typically a large compression), not dead. Dropping this // origin would respawn into the SAME process (twin-collision, // #3044); ride out the episode instead. - logHostEvent("info", "[native-client]", `bili-native: liveness: ${held} BUSY (${v.detail ?? "no health answer"}) — holding origin, re-probing (budget ${busyWaitCapMs}ms) (#3654)`); - const deadline = Date.now() + busyWaitCapMs; - const ep = busyEpisodes.get(held) ?? { attempts: 0, announcedAt: Date.now() }; + const freshEpisode = !busyEpisodes.has(held); + const ep = busyEpisodes.get(held) ?? { attempts: 0 }; busyEpisodes.set(held, ep); + if (freshEpisode) { + logHostEvent("info", "[native-client]", `bili-native: liveness: ${held} BUSY (${v.detail ?? "no health answer"}) — holding origin, re-probing (budget ${busyWaitCapMs}ms) (#3654)`); + } + const deadline = Date.now() + busyWaitCapMs; for (;;) { if (Date.now() >= deadline) break; const jitter = 0.5 + Math.random(); // 0.5x–1.5x @@ -698,6 +702,10 @@ export function createLiveOriginResolver(state: NativeInterceptState, deps: Live lastRespawn = Date.now(); generation += 1; probe = withProbeTtl(probeFn, probeTtlMs); + // Same generation boundary as the verdict-cache rebuild above: a + // post-respawn busy episode behind the same origin must announce + // itself instead of inheriting a stale pre-respawn episode entry. + busyEpisodes.clear(); state.ready = state.respawn(); } const o = await readyOrigin(state); From 87c465e24555d44ab9c8c176e5225dc90382eff0 Mon Sep 17 00:00:00 2001 From: awork Date: Sun, 18 Oct 2026 22:57:04 +0800 Subject: [PATCH 3/3] fix: unexport module-internal symbols flagged by the unused-exports gate (#3654) CI gate failure on the PR head: probeOriginHealth / LivenessState / CONNECT_PROBE_TIMEOUT_MS are used only inside their defining modules (drop the export); FailFastReason now has its intended external user - server.ts imports it instead of restating the union inline. --- src/agent/native-intercept.ts | 2 +- src/liveness.ts | 4 ++-- src/server.ts | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/agent/native-intercept.ts b/src/agent/native-intercept.ts index 040b8a03c8..6a02f5c4d4 100644 --- a/src/agent/native-intercept.ts +++ b/src/agent/native-intercept.ts @@ -554,7 +554,7 @@ const RESPAWN_COOLDOWN_MS = 15_000; /** Default liveness probe: GET /__bili/health under the single-shot budget — * now three-state (#3654): TCP connect refused classifies dead in * milliseconds instead of eating the whole budget. */ -export async function probeOriginHealth(origin: string): Promise { +async function probeOriginHealth(origin: string): Promise { return probeLiveness(origin, HEALTH_TIMEOUT_MS); } diff --git a/src/liveness.ts b/src/liveness.ts index f11b97f04f..67daf110c8 100644 --- a/src/liveness.ts +++ b/src/liveness.ts @@ -13,7 +13,7 @@ import net from "node:net"; -export type LivenessState = "alive" | "busy" | "dead"; +type LivenessState = "alive" | "busy" | "dead"; export interface LivenessVerdict { state: LivenessState; @@ -24,7 +24,7 @@ export interface LivenessVerdict { /** Budget for the TCP connect pre-probe. Loopback connects land in * milliseconds; the cap covers a machine whose network stack itself is wedged, * where any answer would be too late to matter. */ -export const CONNECT_PROBE_TIMEOUT_MS = 2_000; +const CONNECT_PROBE_TIMEOUT_MS = 2_000; function originHostPort(origin: string): { host: string; port: number } | undefined { try { diff --git a/src/server.ts b/src/server.ts index 81b5845a2c..4eaedbdcbc 100644 --- a/src/server.ts +++ b/src/server.ts @@ -58,7 +58,7 @@ import { reconcileFoldCoverage, noteSystemPromptFingerprint, resolveFoldReconcil import { biliToolsDeclaredOnWire, countBiliToolUses, evaluateSelfHealRound, nudgeSuppressed, pluginLaneDegraded, pluginLaneRestore } from "./session-self-heal.js"; import { getSession, hasProcessedState, listSessions, peekSession, type PendingRetrieval, type Session, initSessions, markDirty, flushAllSessions, acquireInFlight, releaseInFlight, totalInFlight, snapshotMessages, ensureCanonicalId, storeEffectiveConfig, foldCoverage, postRebuildAnchorTokens, setPostRebuildAnchor, tickPostRebuildAnchor, splitSessionWarnings, RESPONSES_TURN_SEPARATOR_ID_PREFIX } from "./session.js"; import { detectStaleInstall } from "./update.js"; -import { decideEaddrinuse, eaddrinuseExitCode, postBindCanLadder, UPGRADE_REPLACEMENT_ENV } from "./listen-policy.js"; +import { decideEaddrinuse, eaddrinuseExitCode, postBindCanLadder, UPGRADE_REPLACEMENT_ENV, type FailFastReason } from "./listen-policy.js"; import { getAdvisoryState, cannotResolveTarget } from "./advisory.js"; import { PACKAGE_NAME, VERSION } from "./version.js"; import { @@ -1278,7 +1278,7 @@ export async function startServer(opts: ProxyOptions): Promise { // graceful-shutdown flush can run. Catch, log a human-readable message, // flush sessions, and exit cleanly (exit code 1 so callers/scripts notice). server.on("error", (err: NodeJS.ErrnoException) => { - let busyReason: "upgrade-replacement" | "strict-port" | "port-busy" | "twin-collision" | undefined; + let busyReason: FailFastReason | undefined; let twin: RegistryEntry | undefined; if (err.code === "EADDRINUSE") { twin = findSameLaneTwin(listInstances(), lastTriedPort, launcherLane, ownFingerprint);