diff --git a/src/agent/native-intercept.ts b/src/agent/native-intercept.ts index 3de36f57d7..6a02f5c4d4 100644 --- a/src/agent/native-intercept.ts +++ b/src/agent/native-intercept.ts @@ -10,6 +10,7 @@ import { installFetchChain, isDeadClosureError, resetFetchChainForTest } from ". import { logHostEvent } from "./host-log.js"; import { envMillis } from "./native-bootstrap.js"; +import { probeLiveness, type LivenessVerdict } from "../liveness.js"; import { BILI_PASSTHROUGH_HEADER } from "../util.js"; import { DSH_COMPACTION_MARKER_SNIFF } from "../server/dsh-compaction-guard.js"; import { APIG_RESIGN_HEADER, APIG_RESIGN_CREDENTIAL_HEADER, APIG_RESIGN_SCHEME, clearSignedRefusal, encodeApigCredential, inboundSignedScheme, recordSignedRefusal, resignEnabled, resignPassthroughEnabled, signedRefusal } from "../apig-resign.js"; @@ -29,6 +30,11 @@ export type DirectSendPrep = | { outcome: "restored"; body: string } | { outcome: "unrestorable" }; +/** #3654: why createLiveOriginResolver found nothing to route to — the + * refusal wording must tell a live-but-saturated proxy (the busy wait window + * exhausted) apart from a dead one or a failed bootstrap. */ +export type UnavailableReason = "busy-exhausted" | "dead" | "bootstrap-failed"; + export interface NativeInterceptState { /** Proxy origin ("http://127.0.0.1:PORT") once the bootstrap resolved. * Written by the owner (pi-native.ts); read synchronously on each call. */ @@ -42,6 +48,11 @@ export interface NativeInterceptState { * BILLION_CONTEXT_PROXY env) so event-time ownership checks disarm with * the traffic. */ onGiveUp?: () => void; + /** #3654: set before createLiveOriginResolver returns undefined — why + * nothing was routable ("busy-exhausted" | "dead" | "bootstrap-failed"); + * cleared on the next successful resolution. The refusal builder reads it + * to keep a live-but-saturated proxy out of "proxy unavailable" wording. */ + lastUnavailableReason?: UnavailableReason; /** Attach mode (#809): route through a user-supplied external proxy at * state.origin instead of a spawned one. Set by the host entry when a * BILLION_CONTEXT_ATTACH / launcher-preset BILLION_CONTEXT_PROXY is @@ -509,57 +520,80 @@ export async function observeRoutedOrigin(state: NativeInterceptState): Promise< // ———— Live-origin resolution (#1135) —————————————————————————————— // Shared by BOTH OpenCode lanes (the native entry's V2 route and the // launcher plugin's V2 route): each outgoing request resolves the LIVE proxy -// origin — fast-path the held origin through a TTL-cached health probe, and +// origin — fast-path the held origin through a TTL-cached liveness probe, and // when it is dead drive state.respawn (cooldown-gated unless we just lost an // origin we were actively routing to) and adopt whatever replacement lands. -// A transient blip that recovers to the SAME origin costs nothing (the -// session never migrates); a genuinely dead origin falls back to whatever the -// owner's respawn produces (a self-spawned proxy, or another healthy shared -// instance via discovery). Returns undefined when nothing is alive — callers -// MUST degrade to direct sends, not fail-closed. +// #3654: the probe is THREE-state (src/liveness.ts). Connect-refused = dead +// (instant, unchanged fast path). Connected-but-no-health-answer = busy — +// the event loop is saturated (a large compression in flight), NOT dead: +// dropping a busy HELD origin would respawn into the SAME process +// (twin-collision, #3044), so instead the origin is held and re-probed at a +// jittered cadence inside a bounded wait window. A busy REPLACEMENT (just +// respawned or adopted) is routed to anyway — its loop drains and the request +// queues instead of failing for nothing. A transient blip that recovers to +// the SAME origin costs nothing (the session never migrates); a genuinely +// dead origin falls back to whatever the owner's respawn produces (a +// self-spawned proxy, or another healthy shared instance via discovery). +// Returns undefined when nothing is alive — callers MUST degrade to direct +// sends, not fail-closed. const HEALTH_TIMEOUT_MS = 1500; +/** #3654: total budget for re-probing a BUSY held origin before declaring it + * lost. Deliberately far above HEALTH_TIMEOUT_MS: the point is to ride out + * multi-second event-loop saturation (large compressions), not to widen the + * single-shot probe. */ +const BUSY_WAIT_CAP_MS = 10_000; +/** #3654: base interval between busy re-probes (jittered 0.5x–1.5x). */ +const BUSY_REPROBE_BASE_MS = 1_000; /** Probe-verdict trust window (#928). Far below RESPAWN_COOLDOWN_MS: that * cooldown already assumes multi-second proxy stability, so a few-second * detection horizon is consistent with it while removing the per-request RTT. */ const HEALTH_PROBE_TTL_MS = 2_000; const RESPAWN_COOLDOWN_MS = 15_000; -export async function probeHealth(origin: string): Promise { - try { - const res = await fetch(`${origin}/__bili/health`, { signal: AbortSignal.timeout(HEALTH_TIMEOUT_MS) }); - return res.ok; - } catch { - return false; - } +/** Default liveness probe: GET /__bili/health under the single-shot budget — + * now three-state (#3654): TCP connect refused classifies dead in + * milliseconds instead of eating the whole budget. */ +async function probeOriginHealth(origin: string): Promise { + return probeLiveness(origin, HEALTH_TIMEOUT_MS); } -/** Wrap a probe in a per-origin TTL cache (#928). Both verdicts are cached: a - * healthy hit skips the steady-state loopback RTT; a dead hit avoids re-paying - * the full HEALTH_TIMEOUT_MS on every request across the death+cooldown window. - * Stale entries for other origins are evicted on insert, keeping the map bounded. */ -function withProbeTtl(probe: (origin: string) => Promise, ttlMs: number): (origin: string) => Promise { - const cache = new Map(); +/** Wrap a probe in a per-origin TTL cache (#928). Only DEFINITE verdicts are + * cached: alive skips the steady-state loopback RTT; dead avoids re-paying + * the full HEALTH_TIMEOUT_MS on every request across the death+cooldown + * window. BUSY is never cached — a saturation episode ends within seconds and + * caching it would solidify "slow" into "dead" (#3654). Stale entries for + * other origins are evicted on insert, keeping the map bounded. */ +function withProbeTtl(probe: (origin: string) => Promise, ttlMs: number): (origin: string) => Promise { + const cache = new Map(); return (origin) => { const now = Date.now(); const hit = cache.get(origin); - if (hit !== undefined && now - hit.at < ttlMs) return Promise.resolve(hit.ok); - return probe(origin).then((ok) => { + if (hit !== undefined && now - hit.at < ttlMs) return Promise.resolve(hit.verdict); + return probe(origin).then((verdict) => { const t = Date.now(); for (const [key, entry] of cache) if (t - entry.at >= ttlMs) cache.delete(key); - cache.set(origin, { ok, at: t }); - return ok; + if (verdict.state !== "busy") cache.set(origin, { verdict, at: t }); + return verdict; }); }; } export interface LiveOriginResolverDeps { - /** Injectable health probe (tests); defaults to GET /__bili/health. */ - probe?: (origin: string) => Promise; + /** Injectable liveness probe (tests); defaults to probeLiveness over + * /__bili/health under HEALTH_TIMEOUT_MS. Legacy boolean probes stay + * accepted (true → alive, false → dead). */ + probe?: (origin: string) => Promise; /** Bootstrap retry interval when no live origin is held (tests shrink it). */ respawnCooldownMs?: number; /** Probe-verdict cache TTL; defaults to HEALTH_PROBE_TTL_MS (tests shrink it). */ probeTtlMs?: number; + /** #3654: busy-wait budget for a saturated HELD origin; defaults to + * BUSY_WAIT_CAP_MS (tests shrink it). */ + busyWaitCapMs?: number; + /** #3654: base interval between busy re-probes; defaults to + * BUSY_REPROBE_BASE_MS (tests shrink it). */ + busyReprobeBaseMs?: number; } /** Build the per-request live-origin resolver over a shared intercept state. @@ -575,27 +609,89 @@ export function createLiveOriginResolver(state: NativeInterceptState, deps: Live // cache is rebuilt (kills cached verdicts) and `generation` advances // (kills in-flight verdicts, which must not clear a healthy // replacement that already landed). - const healthProbe = deps.probe ?? probeHealth; + const rawProbe = deps.probe ?? probeOriginHealth; + const probeFn = (origin: string): Promise => rawProbe(origin).then((v) => typeof v === "boolean" ? { state: v ? "alive" : "dead" } : v); const probeTtlMs = deps.probeTtlMs ?? HEALTH_PROBE_TTL_MS; - let probe = withProbeTtl(healthProbe, probeTtlMs); + let probe = withProbeTtl(probeFn, probeTtlMs); let generation = 0; const respawnCooldownMs = deps.respawnCooldownMs ?? RESPAWN_COOLDOWN_MS; + const busyWaitCapMs = deps.busyWaitCapMs ?? BUSY_WAIT_CAP_MS; + const busyReprobeBaseMs = deps.busyReprobeBaseMs ?? BUSY_REPROBE_BASE_MS; let lastRespawn = 0; + // #3654: one entry per busy episode per origin (attempt count) so the log + // announces an episode ONCE (the first caller that sees it) and concurrent + // callers in the same episode only count attempts. + const busyEpisodes = new Map(); + const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); return async (): Promise => { let ownedThenLost = false; + let lossReason: UnavailableReason | undefined; const held = state.origin; const gen = generation; if (held !== undefined) { - const ok = await probe(held); + const v = await probe(held); // Act on the verdict only if the world didn't move while it was // in flight: a concurrent caller's respawn (generation bump) or // any origin change underneath us voids it — re-resolve instead. if (gen === generation && state.origin === held) { - if (ok) return held; - // Proxy died mid-session. Clearing origin first makes concurrent - // callers share the same state.ready (dedup). - ownedThenLost = true; - state.origin = undefined; + if (v.state === "alive") { + state.lastUnavailableReason = undefined; + return held; + } + if (v.state === "dead") { + logHostEvent("info", "[native-client]", `bili-native: liveness: ${held} dead (${v.detail ?? "no answer"}) — clearing origin`); + // Proxy died mid-session. Clearing origin first makes concurrent + // callers share the same state.ready (dedup). + ownedThenLost = true; + lossReason = "dead"; + state.origin = undefined; + } else { + // #3654: BUSY — TCP connect succeeded but /__bili/health never + // answered within the budget: the event loop is saturated + // (typically a large compression), not dead. Dropping this + // origin would respawn into the SAME process (twin-collision, + // #3044); ride out the episode instead. + const freshEpisode = !busyEpisodes.has(held); + const ep = busyEpisodes.get(held) ?? { attempts: 0 }; + busyEpisodes.set(held, ep); + if (freshEpisode) { + logHostEvent("info", "[native-client]", `bili-native: liveness: ${held} BUSY (${v.detail ?? "no health answer"}) — holding origin, re-probing (budget ${busyWaitCapMs}ms) (#3654)`); + } + const deadline = Date.now() + busyWaitCapMs; + for (;;) { + if (Date.now() >= deadline) break; + const jitter = 0.5 + Math.random(); // 0.5x–1.5x + await sleep(Math.min(busyReprobeBaseMs * jitter, Math.max(0, deadline - Date.now()))); + if (gen !== generation || state.origin !== held) break; + ep.attempts += 1; + const again = await probe(held); + if (gen !== generation || state.origin !== held) break; + if (again.state === "alive") { + busyEpisodes.delete(held); + logHostEvent("info", "[native-client]", `bili-native: liveness: ${held} alive again after ${ep.attempts} busy re-probe(s)`); + state.lastUnavailableReason = undefined; + return held; + } + if (again.state === "dead") { + busyEpisodes.delete(held); + logHostEvent("info", "[native-client]", `bili-native: liveness: ${held} dead during busy episode (${again.detail ?? "no answer"}, attempt ${ep.attempts}) — respawning`); + ownedThenLost = true; + lossReason = "dead"; + state.origin = undefined; + break; + } + logHostEvent("info", "[native-client]", `bili-native: liveness: ${held} still busy — re-probe attempt ${ep.attempts}`); + } + if (state.origin === held && gen === generation) { + // Window exhausted while still busy: lost-for-this-request, + // but record WHY so the refusal says "saturated", not "dead". + busyEpisodes.delete(held); + logHostEvent("warn", "[native-client]", `bili-native: liveness: ${held} still busy after ${ep.attempts} re-probe(s) over ${busyWaitCapMs}ms — letting go, respawning (#3654)`); + ownedThenLost = true; + lossReason = "busy-exhausted"; + state.origin = undefined; + } + } } } // Retry bootstrap whenever no live origin is held — either just lost @@ -605,11 +701,31 @@ export function createLiveOriginResolver(state: NativeInterceptState, deps: Live if (state.respawn !== undefined && (ownedThenLost || Date.now() - lastRespawn >= respawnCooldownMs)) { lastRespawn = Date.now(); generation += 1; - probe = withProbeTtl(healthProbe, probeTtlMs); + probe = withProbeTtl(probeFn, probeTtlMs); + // Same generation boundary as the verdict-cache rebuild above: a + // post-respawn busy episode behind the same origin must announce + // itself instead of inheriting a stale pre-respawn episode entry. + busyEpisodes.clear(); state.ready = state.respawn(); } const o = await readyOrigin(state); - if (o !== undefined && (await probe(o))) return o; + if (o !== undefined) { + const v = await probe(o); + if (v.state === "alive") { + state.lastUnavailableReason = undefined; + return o; + } + if (v.state === "busy") { + // #3654: a just-landed replacement that is already busy is still + // ALIVE — routing to it queues the request behind the saturating + // fold; refusing here would fail the very request the adoption + // existed to save. + logHostEvent("info", "[native-client]", `bili-native: liveness: replacement ${o} BUSY (${v.detail ?? "no health answer"}) — routing anyway, the request will queue (#3654)`); + state.lastUnavailableReason = undefined; + return o; + } + } + state.lastUnavailableReason = lossReason ?? "bootstrap-failed"; if (ownedThenLost) state.onGiveUp?.(); return undefined; }; diff --git a/src/agent/opencode-native.ts b/src/agent/opencode-native.ts index 1aaa6da8c4..b98da7c455 100644 --- a/src/agent/opencode-native.ts +++ b/src/agent/opencode-native.ts @@ -76,7 +76,7 @@ import { ACP_TOOLS_OPENAI } from "../compress-tool.js"; import { ensureProxyRunning, LAUNCHER_DEFAULT_HOST, unwrapUpstream, wrapUpstream } from "../launcher.js"; import { createAcpCommandHooks, showAcpText } from "./opencode-acp-command.js"; import { markNativeHost, nativeAttachOrigin, nativeBootstrapGate, nativeProxyScriptPath, proxyEnvOrigin, singleFlight } from "./native-bootstrap.js"; -import { createLiveOriginResolver, installNativeFetchIntercept, isModelApiUrl, noteRoutedOrigin, observeRoutedOrigin, readyOrigin, replaceRequestPayload, replaceRequestTarget, routedBiliModelUrl, type LiveOriginResolverDeps, type NativeInterceptState } from "./native-intercept.js"; +import { createLiveOriginResolver, installNativeFetchIntercept, isModelApiUrl, noteRoutedOrigin, observeRoutedOrigin, readyOrigin, replaceRequestPayload, replaceRequestTarget, routedBiliModelUrl, type LiveOriginResolverDeps, type NativeInterceptState, type UnavailableReason } from "./native-intercept.js"; import { clearIncrementalProbe, createPrepareDirectSend, INCREMENTAL_TURN_MARKER_HEADER, INCREMENTAL_TURN_ID_HEADER, newIncrementalTurnId, noteIncrementalAck, noteIncrementalRejection, planIncrementalTurn, probeIncrementalTurns, rememberOriginalBody, sweepStale, wireMessagesLookAnthropic, type IncrementalWireMessage } from "./incremental-client.js"; import { createOpencodeV2Setup, type V2HttpRequestEvent, type V2State } from "./opencode-v2.js"; import { asciiHeaderValue, fetchProxyVersion, postIdentityRegister, reportRuntimeInfoOnChange, waitForProxyVersion, type ForwardedToolResult } from "./shared.js"; @@ -117,7 +117,7 @@ export type OpencodeNativeRouteDeps = LiveOriginResolverDeps; * model, unknown window, unreadable or absent body all return undefined and * keep the documented fail-open direct send. The size is measured through a * CLONE — the original Request must stay sendable (#2496 discipline). */ -async function oversizeDirectRefusal(e: V2HttpRequestEvent, s: V2State): Promise { +async function oversizeDirectRefusal(e: V2HttpRequestEvent, s: V2State, reason?: UnavailableReason): Promise { const model = e.model; if (model === undefined || typeof model.providerID !== "string" || typeof model.id !== "string") return undefined; const window = s.windows?.get(`${model.providerID}/${model.id}`); @@ -132,6 +132,14 @@ async function oversizeDirectRefusal(e: V2HttpRequestEvent, s: V2State): Promise } const floorTokens = Math.ceil(bytes / 4); if (floorTokens <= window) return undefined; + // #3654: a busy-exhausted refusal must NOT claim "proxy unavailable … + // guaranteed upstream 400" — the proxy IS up and would serve the request + // once its event loop drains; the honest fact is that we stopped waiting. + if (reason === "busy-exhausted") { + return new Error( + `bili: proxy alive but unresponsive — it accepted connections but never answered its health probe within the wait budget (event loop saturated, typically a large compression in flight); this request cannot be sent uncompressed (body ~${floorTokens} tok (floor estimate ${bytes}B/4) exceeds the ${model.providerID}/${model.id} context window (${window})), so it is refused locally instead of queued indefinitely. The proxy is likely to serve it once the fold finishes (#3654); retry shortly.`, + ); + } return new Error( `bili: proxy unavailable and this request cannot be sent uncompressed — body ~${floorTokens} tok (floor estimate ${bytes}B/4) exceeds the ${model.providerID}/${model.id} context window (${window}); upstream would reject it. Recovery is automatic (#3308); retry shortly, or start a fresh session if the local history has outgrown the window.`, ); @@ -189,12 +197,16 @@ export function createNativeRoute(state: NativeInterceptState, deps: OpencodeNat if (target === undefined) { // #3308: a direct send that provably cannot fit the window is not // "availability" — refuse it locally (#2924 ruling) instead of - // handing upstream a request that can only 400. - const refusal = await oversizeDirectRefusal(e, s); + // handing upstream a request that can only 400. #3654: pass WHY + // nothing was routable so a saturated-but-alive proxy gets its own + // honest wording instead of "proxy unavailable … guaranteed 400". + const refusal = await oversizeDirectRefusal(e, s, state.lastUnavailableReason); if (refusal !== undefined) { if (!refusedNoticed) { refusedNoticed = true; - logHostEvent("error", "[native-client]", `bili-native-opencode: proxy unavailable — model request refused locally instead of going direct into a guaranteed upstream 400 (#3308): ${refusal.message}`); + logHostEvent("error", "[native-client]", state.lastUnavailableReason === "busy-exhausted" + ? `bili-native-opencode: proxy alive but saturated past the wait budget — model request refused locally instead of queued indefinitely (#3654): ${refusal.message}` + : `bili-native-opencode: proxy unavailable — model request refused locally instead of going direct into a guaranteed upstream 400 (#3308): ${refusal.message}`); } s.proxyBase = undefined; throw refusal; diff --git a/src/instance.ts b/src/instance.ts index d1485f2c24..6d81a5f0e1 100644 --- a/src/instance.ts +++ b/src/instance.ts @@ -1,5 +1,6 @@ import { createHash, randomUUID } from "node:crypto"; import fs from "node:fs"; +import os from "node:os"; import path from "node:path"; import { resolveZonePortBase } from "./config.js"; import { stateDir } from "./paths.js"; @@ -134,6 +135,21 @@ export function writeZonePort(lane: string, port: number, file: string = portZon * base (BILI_ZONE_PORT override). Explicit user overrides * (BILI_CLAUDE_NATIVE_PORT / BILI_ZCODE_PORT) are resolved separately by * the lanes and imply strict-port launches. */ +/** #3654/#3116: append a diagnostic line to the spawner-owned per-port log + * (%TMPDIR%/bili-proxy-.log) — the exact file a spawner failure message + * points users at. Post-#2864 the child mutes its stderr leg into bili.log as + * soon as its logger is live, so a child that dies during listen setup leaves + * that file EMPTY (the "0 bytes" dead-end); this writes the child-side truth + * there. Named by the PREFERRED port (what the spawner uses for the file + * name); the line itself names the port actually tried. Line grammar mirrors + * logger.ts so mixed writers stay greppable. Best-effort: a failed append + * never affects the exit. */ +export function writeSpawnPortLog(preferredPort: number, line: string): void { + try { + fs.appendFileSync(path.join(os.tmpdir(), `bili-proxy-${preferredPort}.log`), `${line}\n`, "utf8"); + } catch {} +} + export function lanePreferredPort(lane: string, env: NodeJS.ProcessEnv = process.env, file: string = portZoneFilePath()): number { return readZonePort(lane, file) ?? resolveZonePortBase(env); } diff --git a/src/launcher.ts b/src/launcher.ts index 5f734a7696..95d2ffe53d 100644 --- a/src/launcher.ts +++ b/src/launcher.ts @@ -53,6 +53,8 @@ import { type ProxyInstanceFile, type ProxyStartingMarker, } from "./instance.js"; +import { TWIN_ADOPT_EXIT_CODE } from "./listen-policy.js"; +import { tcpConnectable } from "./liveness.js"; import { selfPackageRoot, isBiliPiEntry, ompPluginLoadedFrom, dshNativeInstalled, claudeNativeInstalled, piPackageStrings } from "./plugin-install.js"; import { applyOmpFirstEventTimeout } from "./agent/native-bootstrap.js"; import { logHostEvent } from "./agent/host-log.js"; @@ -316,6 +318,13 @@ export interface LauncherDeps { * discoverLiveInstances, instance.ts). Tests inject a fake so the * clobbered-proxy-origin scenario never touches the real state dir. */ listRegistryInstances?: () => ProxyInstanceFile[]; + /** #3654: TCP pre-probe distinguishing "refused" from "saturated" for + * diagnostics and twin adoption (default: tcpConnectable). Tests inject + * a fake so no real sockets are needed. */ + isConnectable?: (origin: string) => Promise; + /** #3654: registry view behind serving-twin adoption after exit code 3 + * (default: discoverLiveInstances, instance.ts). */ + listAdoptCandidates?: () => ProxyInstanceFile[]; } export function isLaunchClient(value: string): value is ClientName { @@ -3647,6 +3656,7 @@ async function probeLiveInstances( readInstance: () => ProxyInstanceFile | { origin: string } | undefined, fetchHealthInfo: (origin: string) => Promise, diag?: (msg: string) => void, + isConnectable: (origin: string) => Promise = tcpConnectable, ): Promise> { const seen = new Map(); const inst = readInstance(); @@ -3656,7 +3666,17 @@ async function probeLiveInstances( [...seen.values()].map(async (c): Promise<{ inst: ProxyInstanceFile; health: HealthInfo } | undefined> => { if (!isPidAlive(c.pid)) { diag?.(`native-attach: drop ${c.origin} (pid ${c.pid}): owner process gone`); return undefined; } const health = await fetchHealthInfo(c.origin); - if (!health || !health.ok) { diag?.(`native-attach: drop ${c.origin} (pid ${c.pid}): health probe failed`); return undefined; } + if (!health || !health.ok) { + // #3654: a refused connection and a saturated event loop both + // fail this probe — tell them apart in the log: connect-but- + // silent is BUSY (alive, loop inside a large fold), not dead. + // Still dropped either way (#1335 gate unchanged). + const busy = await isConnectable(c.origin); + diag?.(busy + ? `native-attach: drop ${c.origin} (pid ${c.pid}): health probe failed but TCP connect succeeded — instance is BUSY (event loop saturated), not dead (#3654)` + : `native-attach: drop ${c.origin} (pid ${c.pid}): health probe failed`); + return undefined; + } if (health.instanceId !== undefined && health.instanceId !== c.instanceId) { diag?.(`native-attach: drop ${c.origin} (pid ${c.pid}): instance id mismatch (${c.instanceId} vs ${health.instanceId})`); return undefined; } return { inst: c, health }; }), @@ -4593,6 +4613,12 @@ export async function ensureProxyRunning( if (/^HTTP \d{3}$/.test(d)) { return `port ${port} answered ${d} — the listener there is not reporting bili health`; } + // #3654: a timeout with a live TCP listener underneath is the + // event-loop-saturation signature (connect succeeds, JS never runs) + // — name it instead of letting it read as a dead port. + if (/^timeout after \d+ms$/i.test(d) && (await (deps.isConnectable ?? tcpConnectable)(origin))) { + return `port ${port} accepts TCP connections but never answers /__bili/health within the probe budget — the instance there is BUSY (event loop saturated, typically a large compression), not dead (#3654)`; + } return `last health probe failed with ${d}`; }; @@ -4674,6 +4700,21 @@ export async function ensureProxyRunning( // discovery+attach decision as the one-shot probe above, re-run on a // short budget so a listener whose identity record published during // our spawn attempt is still attachable instead of mistaken for air. + // #3654: re-run the identity criteria the child's findSameLaneTwin + // applied at bind time against the CURRENT registry, verify the holder + // still answers TCP (pid reuse), then ride it through the normal attach + // path (watcher + heartbeat included). + const adoptServingTwin = async (): Promise => { + const candidates = (deps.listAdoptCandidates ?? discoverLiveInstances)() + .filter((inst) => isPidAlive(inst.pid) && instanceCompatible(inst, opts, codeFingerprint, foreignLanes) === undefined) + .sort((a, b) => Number(b.port === port) - Number(a.port === port) || b.startedAt - a.startedAt); + for (const inst of candidates) { + if (!(await (deps.isConnectable ?? tcpConnectable)(inst.origin))) continue; + attachDiag(`bili: adopting serving twin ${inst.origin} (pid ${inst.pid}) — the spawned child lost the #3044 duplicate race to it; requests ride the existing instance (#3654)`); + return attachTo(inst); + } + return undefined; + }; const rediscoverAfterChildExit = async (): Promise => { console.error( `bili: spawned proxy exited before becoming healthy — re-checking for a late-publishing listener on port ${port} before failing`, @@ -4697,6 +4738,16 @@ export async function ensureProxyRunning( throw new Error(`bili: proxy spawn failed (${detail}) (runtime: ${runtime}) (log: ${logPath})`); } if (childExit) { + // #3654: exit code 3 is the child's twin-collision signal (#3044 + // listen-policy): a live same-lane same-build instance already + // serves the port. Discovery above declared it unhealthy or missed + // it; the child's bind-time twin check proved it was serving. + // Adopt the holder instead of failing — the user's proxy is up. + if (childExit.code === TWIN_ADOPT_EXIT_CODE) { + const adopted = await adoptServingTwin(); + if (adopted !== undefined) return adopted; + attachDiag("bili: serving-twin adoption failed verification — continuing with the standard post-exit re-discovery"); + } // #1903: the child died before becoming healthy. Under strictPort // that death is almost certainly EADDRINUSE — proof the pinned port // is held by a listener our one-shot discovery above missed because @@ -4709,7 +4760,7 @@ export async function ensureProxyRunning( const retried = await rediscoverAfterChildExit(); if (retried) return retried; const detail = childExit.code !== null - ? `code ${childExit.code}` + ? `code ${childExit.code}${childExit.code === TWIN_ADOPT_EXIT_CODE ? " (twin-collision signal)" : ""}` : childExit.signal ? `signal ${childExit.signal}` : "unknown reason"; throw new Error(`bili: proxy child exited before becoming healthy (${detail}) — ${await diagnosePort(false)} (runtime: ${runtime}) (log: ${logPath}${logTail()})`); } diff --git a/src/listen-policy.ts b/src/listen-policy.ts index 4b0e7a2bee..87460a33e7 100644 --- a/src/listen-policy.ts +++ b/src/listen-policy.ts @@ -18,6 +18,24 @@ * (#3044): the marked process must bind-or-die on exactly the freed port. */ export const UPGRADE_REPLACEMENT_ENV = "BILI_UPGRADE_REPLACEMENT"; +/** #3654: the exit code a twin-collision loser reports to its spawner. The + * duplicate still LOSES (#3044 — it cannot bind the port the serving twin + * holds), but the code says WHY it died: the spawner reads it as an adopt + * signal and attaches to the serving twin instead of failing the bring-up — + * closing the #3654 contradiction where discovery judged the holder dead + * (busy) while the twin rule proved it alive. Every other listen failure + * keeps exit 1. */ +export const TWIN_ADOPT_EXIT_CODE = 3; + +export type FailFastReason = "upgrade-replacement" | "strict-port" | "port-busy" | "twin-collision"; + +/** Pure exit-code mapping for a terminal EADDRINUSE decision — extracted so + * the server (which calls process.exit) and its tests share one definition + * without booting a listener (#3654). */ +export function eaddrinuseExitCode(reason: FailFastReason | undefined): number { + return reason === "twin-collision" ? TWIN_ADOPT_EXIT_CODE : 1; +} + type EaddrinuseAction = | { kind: "wait-predecessor"; port: number } /** nextPort 0 = ask the OS for an ephemeral port (ladder exhausted). */ diff --git a/src/liveness.ts b/src/liveness.ts new file mode 100644 index 0000000000..67daf110c8 --- /dev/null +++ b/src/liveness.ts @@ -0,0 +1,123 @@ +// Three-state liveness probing (#3654). A health probe that times out is NOT +// evidence of death: on loopback the kernel accept queue keeps answering TCP +// connects while the listener's JS event loop is blocked (measured hermetically: +// connect ~2ms during an 8s synchronous block; the HTTP answer only arrives +// after the unblock), so "connected but no health answer within the budget" is +// the signature of a BUSY proxy — alive and saturated (typically a large +// compression) — not dead. ECONNREFUSED and its kin remain the instant death +// signal, so the fast path that detects a truly gone proxy costs milliseconds, +// not the full probe budget. Every consumer (the per-request live-origin +// resolver in native-intercept.ts, launcher discovery diagnostics, spawn-side +// twin adoption) shares this one classification so busy and dead are never +// conflated again. + +import net from "node:net"; + +type LivenessState = "alive" | "busy" | "dead"; + +export interface LivenessVerdict { + state: LivenessState; + /** Failure class for non-alive verdicts (ECONNREFUSED / HTTP status / timeout class). */ + detail?: string; +} + +/** Budget for the TCP connect pre-probe. Loopback connects land in + * milliseconds; the cap covers a machine whose network stack itself is wedged, + * where any answer would be too late to matter. */ +const CONNECT_PROBE_TIMEOUT_MS = 2_000; + +function originHostPort(origin: string): { host: string; port: number } | undefined { + try { + const u = new URL(origin); + if (u.protocol !== "http:" && u.protocol !== "https:") return undefined; + const host = u.hostname.replace(/^\[|\]$/g, ""); + if (host === "") return undefined; + const port = u.port !== "" ? Number(u.port) : u.protocol === "https:" ? 443 : 80; + return Number.isInteger(port) && port > 0 ? { host, port } : undefined; + } catch { + return undefined; + } +} + +interface ConnectOutcome { + ok: boolean; + /** errno class when the connect itself failed (ECONNREFUSED, …). */ + detail?: string; +} + +function attemptConnect(host: string, port: number, timeoutMs: number): Promise { + return new Promise((resolve) => { + let settled = false; + const settle = (outcome: ConnectOutcome): void => { + if (settled) return; + settled = true; + socket.destroy(); + resolve(outcome); + }; + const socket = net.connect({ host, port }); + socket.setTimeout(timeoutMs); + socket.once("connect", () => settle({ ok: true })); + socket.once("timeout", () => settle({ ok: false, detail: `connect timeout after ${timeoutMs}ms` })); + socket.once("error", (err: NodeJS.ErrnoException) => + settle({ ok: false, detail: typeof err.code === "string" && err.code.length > 0 ? err.code : String(err.message ?? err) })); + }); +} + +/** True when the origin's socket accepts a TCP connection within the budget. + * Proves the LISTENING side exists at the kernel level — including a proxy + * whose event loop is blocked, because its accept queue still drains. Used by + * the launcher's twin adoption (a live pid alone does not prove the socket + * serves — Windows pid reuse) and by discovery diagnostics. */ +export async function tcpConnectable(origin: string, timeoutMs: number = CONNECT_PROBE_TIMEOUT_MS): Promise { + const hp = originHostPort(origin); + if (hp === undefined) return false; + const outcome = await attemptConnect(hp.host, hp.port, timeoutMs); + return outcome.ok; +} + +function errCodeOf(err: unknown): string | undefined { + const e = err as { cause?: { code?: unknown; name?: unknown }; code?: unknown; name?: unknown } | null; + const code = e?.cause?.code ?? e?.code; + return typeof code === "string" && code.length > 0 ? code : undefined; +} + +/** Timeout-class error from fetch/undici under AbortSignal.timeout — the + * "connected but never answered" signature. Everything else that throws is a + * transport failure (reset, closed, refused mid-flight) → dead. */ +function isTimeoutErr(err: unknown): boolean { + const e = err as { cause?: { name?: unknown; code?: unknown }; name?: unknown; code?: unknown } | null; + const names = [e?.name, e?.cause?.name]; + const codes = [e?.code, e?.cause?.code]; + return names.includes("TimeoutError") || names.includes("AbortError") || + codes.includes("ABORT_ERR") || codes.includes("UND_ERR_HEADERS_TIMEOUT"); +} + +/** Full three-state probe of GET `/__bili/health` under `budgetMs`, + * preceded by the TCP connect pre-probe. The "alive" claim additionally + * validates the PAYLOAD (`{ ok: true }`) — a foreign service answering 200 on + * every path must not read as a healthy bili proxy (#1753 discipline, which + * the launcher-side probes already applied but the resolver-side probe did + * not). */ +export async function probeLiveness(origin: string, budgetMs: number): Promise { + const hp = originHostPort(origin); + if (hp === undefined) return { state: "dead", detail: "unparseable origin" }; + // Connect first: a refused connection is a high-confidence death signal on + // loopback and costs ~0ms, so it short-circuits before paying the HTTP + // budget at all. + const conn = await attemptConnect(hp.host, hp.port, CONNECT_PROBE_TIMEOUT_MS); + if (!conn.ok) return { state: "dead", detail: conn.detail ?? "connect failed" }; + try { + const res = await fetch(`${origin}/__bili/health`, { signal: AbortSignal.timeout(budgetMs) }); + if (!res.ok) return { state: "dead", detail: `HTTP ${res.status}` }; + try { + const data = (await res.json()) as { ok?: unknown }; + if (data.ok !== true) return { state: "dead", detail: "health payload reports not-ok" }; + } catch { + return { state: "dead", detail: "non-JSON health payload (foreign listener?)" }; + } + return { state: "alive" }; + } catch (err) { + if (isTimeoutErr(err)) return { state: "busy", detail: `no health answer within ${budgetMs}ms after TCP connect` }; + return { state: "dead", detail: errCodeOf(err) ?? String((err as Error)?.message ?? err).slice(0, 120) }; + } +} diff --git a/src/server.ts b/src/server.ts index a35e0831a2..4eaedbdcbc 100644 --- a/src/server.ts +++ b/src/server.ts @@ -58,7 +58,7 @@ import { reconcileFoldCoverage, noteSystemPromptFingerprint, resolveFoldReconcil import { biliToolsDeclaredOnWire, countBiliToolUses, evaluateSelfHealRound, nudgeSuppressed, pluginLaneDegraded, pluginLaneRestore } from "./session-self-heal.js"; import { getSession, hasProcessedState, listSessions, peekSession, type PendingRetrieval, type Session, initSessions, markDirty, flushAllSessions, acquireInFlight, releaseInFlight, totalInFlight, snapshotMessages, ensureCanonicalId, storeEffectiveConfig, foldCoverage, postRebuildAnchorTokens, setPostRebuildAnchor, tickPostRebuildAnchor, splitSessionWarnings, RESPONSES_TURN_SEPARATOR_ID_PREFIX } from "./session.js"; import { detectStaleInstall } from "./update.js"; -import { decideEaddrinuse, postBindCanLadder, UPGRADE_REPLACEMENT_ENV } from "./listen-policy.js"; +import { decideEaddrinuse, eaddrinuseExitCode, postBindCanLadder, UPGRADE_REPLACEMENT_ENV, type FailFastReason } from "./listen-policy.js"; import { getAdvisoryState, cannotResolveTarget } from "./advisory.js"; import { PACKAGE_NAME, VERSION } from "./version.js"; import { @@ -98,7 +98,7 @@ import { getStore } from "./persist.js"; import { log as loggerLog, configureLogger, getLogPath, closeLogger, isFileLoggingActive, isStreamWriteError, isBenignSocketRaceError, enterSessionContext, muteStderrLeg } from "./logger.js"; import { queryLogLines } from "./web/logs-query.js"; import { configFile, defaultLogFile, stateDir } from "./paths.js"; -import { atomicWriteInstanceFile, clearProxyInstanceFile, entryScriptFingerprint, findSameLanePredecessor, findSameLaneTwin, isPidAlive, listInstances, readProxyInstanceFile, registerInstanceAndWarn, unregisterInstance, warnOnNewPeers, type ProxyInstanceFile, type RegistryEntry } from "./instance.js"; +import { atomicWriteInstanceFile, clearProxyInstanceFile, entryScriptFingerprint, findSameLanePredecessor, findSameLaneTwin, isPidAlive, listInstances, readProxyInstanceFile, registerInstanceAndWarn, unregisterInstance, warnOnNewPeers, writeSpawnPortLog, type ProxyInstanceFile, type RegistryEntry } from "./instance.js"; import { mayClobberPointer, probeOwnerInfo, resolvePortClaim } from "./port-owner.js"; import { hoistTrappedToolItems } from "./tool-pair-order.js"; import { computeAnthropicMessageMarks, stampAnthropicSystemCacheControl, anthropicToolsCarryCacheControl } from "./loop/cache-control.js"; @@ -1278,7 +1278,7 @@ export async function startServer(opts: ProxyOptions): Promise { // graceful-shutdown flush can run. Catch, log a human-readable message, // flush sessions, and exit cleanly (exit code 1 so callers/scripts notice). server.on("error", (err: NodeJS.ErrnoException) => { - let busyReason: "upgrade-replacement" | "strict-port" | "port-busy" | "twin-collision" | undefined; + let busyReason: FailFastReason | undefined; let twin: RegistryEntry | undefined; if (err.code === "EADDRINUSE") { twin = findSameLaneTwin(listInstances(), lastTriedPort, launcherLane, ownFingerprint); @@ -1309,7 +1309,7 @@ export async function startServer(opts: ProxyOptions): Promise { const hint = err.code === "EADDRINUSE" ? busyReason === "twin-collision" - ? ` — port ${lastTriedPort} is already served by a same-build bili instance (pid ${twin?.pid}, lane ${launcherLane}) — exiting instead of hopping: the duplicate loses and the serving instance keeps serving; the next launch will attach to it (#3044).` + ? ` — port ${lastTriedPort} is already served by a same-build bili instance (pid ${twin?.pid}, lane ${launcherLane}) — exiting instead of hopping: the duplicate loses and the serving instance keeps serving; the next launch will attach to it (#3044). A spawner reads this exit as an adopt signal and attaches to the serving instance itself (#3654).` : busyReason === "upgrade-replacement" ? ` — this process is an upgrade-restart replacement (#3044): it must own port ${lastTriedPort} exactly, but the prior instance still holds it. Exiting instead of hopping to ${lastTriedPort + 1} — a drifted replacement serves no client and leaks while the prior instance keeps serving; the upgrade retries on the next cooldown cycle.` : busyReason === "strict-port" @@ -1318,13 +1318,18 @@ export async function startServer(opts: ProxyOptions): Promise { : err.code === "EACCES" ? ` — port ${lastTriedPort} requires privileges. Use a port >= 1024.` : ""; - log("error", `listen failed: ${err.code ?? ""} ${err.message}${hint}`); + const listenLine = `listen failed: ${err.code ?? ""} ${err.message}${hint}`; + log("error", listenLine); + // #3654/#3116: a spawner's failure message points users at the per-port + // log, but post-#2864 this child mutes its stderr leg into bili.log — + // without this line that file stays 0 bytes and the diagnosis dies there. + if (launchToken !== undefined) writeSpawnPortLog(opts.port, `${new Date().toISOString()} [error] [v=${VERSION}] ${listenLine}`); shuttingDown = true; server.close(); flushConversations(); void flushAllSessions().finally(() => { closeLogger(); - process.exit(1); + process.exit(eaddrinuseExitCode(busyReason)); }); }); // Catch stray rejections/throws from background work (compress loops, diff --git a/tests/issue3654-health-probe-three-state.test.ts b/tests/issue3654-health-probe-three-state.test.ts new file mode 100644 index 0000000000..0061ed7099 --- /dev/null +++ b/tests/issue3654-health-probe-three-state.test.ts @@ -0,0 +1,576 @@ +// #3654: under load a 1.5s health probe times out against a LIVE proxy whose +// event loop is inside a large compression; the old two-state probe read that +// as death, dropped the origin, respawned into the same process (twin- +// collision exit), and oversized sessions were then refused locally (#3308). +// This suite pins the three-state contract end to end: +// A) probeLiveness — connect-refused = dead (fast), connect-but-silent = +// busy, payload-validated alive; +// B) createLiveOriginResolver — busy HELD origins are held through a bounded +// jittered wait window (never dropped, never cached), dead stays an +// instant fast path, busy replacements are routed to anyway, and the +// refusal reason records busy-exhausted vs dead; +// C) ensureProxyRunning — a twin-collision child (exit code 3) leads to +// adoption of the serving instance instead of a failed bring-up; +// D) eaddrinuseExitCode — only twin-collision maps to the adopt signal; +// E) writeSpawnPortLog — the per-port log the failure message points at is +// actually written for spawner children (#3116's 0-byte dead end); +// F) refusal wording — busy-exhausted must not claim "proxy unavailable … +// guaranteed upstream 400" (owner requirement 5). + +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import http from "node:http"; +import net from "node:net"; +import type { AddressInfo } from "node:net"; +import { after } from "node:test"; + +// The module-level guard reads NODE_TEST_CONTEXT while the module EVALUATES, +// and static imports hoist above any assignment — so the value must be set +// first and the modules loaded dynamically. +process.env.NODE_TEST_CONTEXT = "1"; + +const STATE_ROOT = fs.mkdtempSync(path.join(os.tmpdir(), "bili-3654-state-")); +const PREV_XDG_STATE_HOME = process.env.XDG_STATE_HOME; +process.env.XDG_STATE_HOME = STATE_ROOT; +after(() => { + if (PREV_XDG_STATE_HOME === undefined) delete process.env.XDG_STATE_HOME; else process.env.XDG_STATE_HOME = PREV_XDG_STATE_HOME; + fs.rmSync(STATE_ROOT, { recursive: true, force: true }); +}); + +const { probeLiveness, tcpConnectable } = await import("../src/liveness.ts"); +const { decideEaddrinuse, eaddrinuseExitCode, TWIN_ADOPT_EXIT_CODE } = await import("../src/listen-policy.ts"); +const { writeSpawnPortLog } = await import("../src/instance.ts"); +const { createLiveOriginResolver } = await import("../src/agent/native-intercept.ts"); +import type { LivenessVerdict } from "../src/liveness.ts"; +import type { NativeInterceptState } from "../src/agent/native-intercept.ts"; +import type { V2State } from "../src/agent/opencode-v2.ts"; +import type { ProxyInstanceFile } from "../src/instance.ts"; +import { entryScriptFingerprint } from "../src/instance.ts"; +import { ensureProxyRunning, type SpawnChild, type SpawnFn } from "../src/launcher.ts"; + +function listenPort(server: net.Server | http.Server): Promise { + return new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", () => resolve((server.address() as AddressInfo).port)); + }); +} + +async function withServer(make: (server: net.Server) => void, fn: (origin: string, server: net.Server) => Promise): Promise { + const server = net.createServer(); + // Destroy tracked connections on teardown: a paused (never-answered) + // socket would otherwise hold server.close()'s callback hostage. + const conns = new Set(); + server.on("connection", (s) => { conns.add(s); s.on("close", () => conns.delete(s)); }); + make(server); + const port = await listenPort(server); + try { + return await fn(`http://127.0.0.1:${port}`, server); + } finally { + for (const s of conns) s.destroy(); + conns.clear(); + await new Promise((r) => server.close(() => r())); + } +} + +// ———— A) probeLiveness ———————————————————————————————————————————————— + +test("A: healthy bili proxy answers → alive", async () => { + await withServer(() => {}, async (origin) => { + const srv = http.createServer((_req, res) => { res.writeHead(200, { "content-type": "application/json" }); res.end(JSON.stringify({ ok: true })); }); + const port = await listenPort(srv); + try { + const v = await probeLiveness(`http://127.0.0.1:${port}`, 1_000); + assert.equal(v.state, "alive"); + } finally { + await new Promise((r) => srv.close(() => r())); + } + }); +}); + +test("A: refused connection classifies dead in milliseconds (fast path unchanged)", async () => { + // Grab a real port, release it, probe it: ECONNREFUSED on loopback. + const holder = net.createServer(); + const port = await listenPort(holder); + await new Promise((r) => holder.close(() => r())); + const t0 = Date.now(); + const v = await probeLiveness(`http://127.0.0.1:${port}`, 5_000); + const elapsed = Date.now() - t0; + assert.equal(v.state, "dead"); + assert.match(v.detail ?? "", /ECONNREFUSED|ECONNRESET|connect/i); + assert.ok(elapsed < 1_000, `refused-connect must not pay the HTTP budget (took ${elapsed}ms)`); +}); + +test("A: connected but silent under the budget → busy (the #3654 signature)", async () => { + await withServer((s) => s.on("connection", (socket) => socket.pause()), async (origin) => { + const t0 = Date.now(); + const v = await probeLiveness(origin, 300); + const elapsed = Date.now() - t0; + assert.equal(v.state, "busy"); + assert.match(v.detail ?? "", /no health answer within 300ms/); + assert.ok(elapsed >= 250, `the budget must actually elapse before calling it busy (took ${elapsed}ms)`); + }); +}); + +test("A: foreign 200 HTML is not a healthy bili proxy → dead", async () => { + const srv = http.createServer((_req, res) => { res.writeHead(200, { "content-type": "text/html" }); res.end("hello"); }); + const port = await listenPort(srv); + try { + const v = await probeLiveness(`http://127.0.0.1:${port}`, 1_000); + assert.equal(v.state, "dead"); + assert.match(v.detail ?? "", /non-JSON/); + } finally { + await new Promise((r) => srv.close(() => r())); + } +}); + +test("A: JSON health reporting ok:false → dead", async () => { + const srv = http.createServer((_req, res) => { res.writeHead(200, { "content-type": "application/json" }); res.end(JSON.stringify({ ok: false })); }); + const port = await listenPort(srv); + try { + const v = await probeLiveness(`http://127.0.0.1:${port}`, 1_000); + assert.equal(v.state, "dead"); + assert.match(v.detail ?? "", /not-ok/); + } finally { + await new Promise((r) => srv.close(() => r())); + } +}); + +test("A: tcpConnectable sees through a saturated loop (accept queue answers)", async () => { + await withServer((s) => s.on("connection", (socket) => socket.pause()), async (origin) => { + assert.equal(await tcpConnectable(origin, 500), true, "a blocked-loop listener still accepts TCP"); + }); + const holder = net.createServer(); + const port = await listenPort(holder); + await new Promise((r) => holder.close(() => r())); + assert.equal(await tcpConnectable(`http://127.0.0.1:${port}`, 500), false); +}); + +// ———— B) createLiveOriginResolver ————————————————————————————————————— + +type VerdictOrBool = LivenessVerdict | boolean; + +function scriptedProbe(script: VerdictOrBool[], byOrigin?: (o: string, i: number) => VerdictOrBool): { probe: (o: string) => Promise; calls: string[] } { + const calls: string[] = []; + let i = 0; + return { + calls, + probe: async (o) => { + calls.push(o); + const v = byOrigin !== undefined ? byOrigin(o, i) : script[Math.min(i, script.length - 1)]; + i++; + return v; + }, + }; +} + +const BUSY: LivenessVerdict = { state: "busy", detail: "no health answer within 1500ms after TCP connect" }; +const DEAD: LivenessVerdict = { state: "dead", detail: "ECONNREFUSED" }; +const ALIVE: LivenessVerdict = { state: "alive" }; + +test("B: busy held origin recovers inside the window — no drop, no respawn, no cache solidification", async () => { + const HELD = "http://127.0.0.1:18901"; + let respawnCalls = 0; + const state: NativeInterceptState = { + origin: HELD, + ready: Promise.resolve(undefined), + respawn: async () => { respawnCalls++; return undefined; }, + }; + // TTL far above the episode: if BUSY were ever cached, every re-probe in + // the window would hit the cache and recovery below would be impossible. + const { probe, calls } = scriptedProbe([BUSY, ALIVE]); + const resolve = createLiveOriginResolver(state, { probe, probeTtlMs: 1_000, busyWaitCapMs: 400, busyReprobeBaseMs: 40 }); + const t0 = Date.now(); + const got = await resolve(); + const elapsed = Date.now() - t0; + assert.equal(got, HELD, "the held origin must survive its own busy episode"); + assert.equal(respawnCalls, 0, "a busy origin must never trigger a respawn"); + assert.equal(calls.length, 2, "first probe busy, one re-probe alive"); + assert.ok(elapsed < 400 + 200, `recovery must land inside the window (took ${elapsed}ms)`); + // Second call immediately: the ALIVE verdict IS cached → zero raw probes. + const before = calls.length; + assert.equal(await resolve(), HELD); + assert.equal(calls.length, before, "alive verdicts stay TTL-cached (only busy is uncached)"); +}); + +test("B: busy exhaustion lets go, respawns, and adopts the replacement", async () => { + const HELD = "http://127.0.0.1:18902"; + const REPL = "http://127.0.0.1:18903"; + let respawnCalls = 0; + const state: NativeInterceptState = { + origin: HELD, + ready: Promise.resolve(undefined), + respawn: async () => { respawnCalls++; state.origin = REPL; return REPL; }, + }; + const { probe } = scriptedProbe([], (o) => (o === HELD ? BUSY : ALIVE)); + const resolve = createLiveOriginResolver(state, { probe, probeTtlMs: 1_000, busyWaitCapMs: 200, busyReprobeBaseMs: 30 }); + const got = await resolve(); + assert.equal(got, REPL, "the replacement must be adopted after the window"); + assert.equal(respawnCalls, 1); + assert.equal(state.lastUnavailableReason, undefined, "success clears the refusal reason"); +}); + +test("B: a busy REPLACEMENT is routed to anyway — the request queues behind the fold", async () => { + const HELD = "http://127.0.0.1:18904"; + const REPL = "http://127.0.0.1:18905"; + let respawnCalls = 0; + const state: NativeInterceptState = { + origin: HELD, + ready: Promise.resolve(undefined), + respawn: async () => { respawnCalls++; state.origin = REPL; return REPL; }, + }; + const { probe } = scriptedProbe([], () => BUSY); + const resolve = createLiveOriginResolver(state, { probe, probeTtlMs: 1_000, busyWaitCapMs: 150, busyReprobeBaseMs: 30 }); + const got = await resolve(); + assert.equal(got, REPL, "refusing a live-but-busy replacement would fail the request the adoption exists to save"); + assert.equal(respawnCalls, 1); + assert.equal(state.lastUnavailableReason, undefined); +}); + +test("B: busy exhaustion with a dead replacement refuses and records busy-exhausted", async () => { + const HELD = "http://127.0.0.1:18906"; + const REPL = "http://127.0.0.1:18907"; + let giveUps = 0; + const state: NativeInterceptState = { + origin: HELD, + ready: Promise.resolve(undefined), + respawn: async () => { state.origin = REPL; return REPL; }, + onGiveUp: () => { giveUps++; }, + }; + const { probe } = scriptedProbe([], (o) => (o === HELD ? BUSY : DEAD)); + const resolve = createLiveOriginResolver(state, { probe, probeTtlMs: 1_000, busyWaitCapMs: 150, busyReprobeBaseMs: 30, respawnCooldownMs: 60_000 }); + const got = await resolve(); + assert.equal(got, undefined); + assert.equal(state.lastUnavailableReason, "busy-exhausted", "the refusal wording depends on this distinction"); + assert.equal(giveUps, 1); +}); + +test("B: dead verdict keeps the instant fast path (no busy window paid)", async () => { + const HELD = "http://127.0.0.1:18908"; + const REPL = "http://127.0.0.1:18909"; + let respawnCalls = 0; + const state: NativeInterceptState = { + origin: HELD, + ready: Promise.resolve(undefined), + respawn: async () => { respawnCalls++; state.origin = REPL; return REPL; }, + }; + const { probe } = scriptedProbe([], (o) => (o === HELD ? DEAD : ALIVE)); + const resolve = createLiveOriginResolver(state, { probe, probeTtlMs: 1_000, busyWaitCapMs: 30_000, busyReprobeBaseMs: 1_000 }); + const t0 = Date.now(); + const got = await resolve(); + const elapsed = Date.now() - t0; + assert.equal(got, REPL); + assert.equal(respawnCalls, 1); + assert.ok(elapsed < 500, `a refused connection must not pay any wait window (took ${elapsed}ms)`); +}); + +test("B: dead verdicts stay TTL-cached across callers", async () => { + const HELD = "http://127.0.0.1:18910"; + const REPL = "http://127.0.0.1:18911"; + let respawnCalls = 0; + // The respawn deliberately does NOT claim state.origin: a respawn that + // does makes the next call re-own the dead replacement and take the + // ownedThenLost path (which bypasses the cooldown BY DESIGN), so the + // cache could never be observed. + const state: NativeInterceptState = { + origin: HELD, + ready: Promise.resolve(undefined), + respawn: async () => { respawnCalls++; return REPL; }, + }; + const { probe, calls } = scriptedProbe([DEAD]); + const resolve = createLiveOriginResolver(state, { probe, probeTtlMs: 1_000, respawnCooldownMs: 60_000 }); + assert.equal(await resolve(), undefined); + // Second call: no held origin, cooldown blocks a second respawn, and the + // replacement's dead verdict comes straight from the TTL cache. + assert.equal(await resolve(), undefined); + assert.equal(respawnCalls, 1); + assert.equal(calls.length, 2, `held-dead + replacement-dead, then cache hits only (got ${calls.length} raw probes)`); + assert.equal(state.lastUnavailableReason, "bootstrap-failed"); +}); + +test("B: concurrent callers share one respawn when the held origin dies (#1957)", async () => { + const HELD = "http://127.0.0.1:18912"; + const REPL = "http://127.0.0.1:18913"; + let respawnCalls = 0; + const state: NativeInterceptState = { + origin: HELD, + ready: Promise.resolve(undefined), + respawn: async () => { respawnCalls++; state.origin = REPL; return REPL; }, + }; + const { probe } = scriptedProbe([], (o) => (o === HELD ? DEAD : ALIVE)); + const resolve = createLiveOriginResolver(state, { probe, probeTtlMs: 1_000, respawnCooldownMs: 60_000 }); + const [a, b] = await Promise.all([resolve(), resolve()]); + assert.equal(a, REPL); + assert.equal(b, REPL); + assert.equal(respawnCalls, 1, "both callers must ride the single shared respawn"); +}); + +test("B: legacy boolean probes still map (true→alive, false→dead)", async () => { + const HELD = "http://127.0.0.1:18914"; + const REPL = "http://127.0.0.1:18915"; + let respawnCalls = 0; + const state: NativeInterceptState = { + origin: HELD, + ready: Promise.resolve(undefined), + respawn: async () => { respawnCalls++; state.origin = REPL; return REPL; }, + }; + const resolve = createLiveOriginResolver(state, { probe: async (o) => o !== HELD, probeTtlMs: 1_000 }); + assert.equal(await resolve(), REPL); + assert.equal(respawnCalls, 1); +}); + +// ———— C) ensureProxyRunning twin adoption ———————————————————————————————— + +const FP_SCRIPT = path.join(os.tmpdir(), `bili-fp-3654-${process.pid}.js`); +fs.writeFileSync(FP_SCRIPT, "// fingerprint fixture\n"); +const FP = entryScriptFingerprint(FP_SCRIPT); +assert.ok(FP !== undefined, "fixture script must fingerprint"); + +const TWIN_ORIGIN = "http://127.0.0.1:18916"; + +function tmpHome(): { dir: string; restore: () => void } { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "bili-3654-home-")); + const prevCfg = process.env.XDG_CONFIG_HOME; + const prevState = process.env.XDG_STATE_HOME; + process.env.XDG_CONFIG_HOME = dir; + process.env.XDG_STATE_HOME = dir; + return { + dir, + restore: () => { + if (prevCfg === undefined) delete process.env.XDG_CONFIG_HOME; else process.env.XDG_CONFIG_HOME = prevCfg; + if (prevState === undefined) delete process.env.XDG_STATE_HOME; else process.env.XDG_STATE_HOME = prevState; + }, + }; +} + +function twinInstance(lane: string): ProxyInstanceFile { + return { + origin: TWIN_ORIGIN, + instanceId: "inst-3654", + pid: process.pid, + startedAt: Date.now(), + host: "127.0.0.1", + port: 18916, + passthrough: false, + mitmDomains: [], + modelWindows: {}, + lane, + codeFingerprint: FP, + }; +} + +function fakeChildExitting(code: number | null): SpawnChild { + const handlers = new Map void>(); + return { + pid: 42_470, + unref() {}, + kill() { return true; }, + on(ev: string, cb: (...args: unknown[]) => void) { + handlers.set(ev, cb); + if (ev === "exit") queueMicrotask(() => cb(code, null)); + return this; + }, + }; +} + +function fakeClock() { + let ticks = 0; + return { + now: () => ticks * 1000, + sleep: () => { ticks += 1; return Promise.resolve(); }, + }; +} + +interface BringUpResult { + handle: Awaited>; + spawnCalls: number; + diag: string[]; +} + +async function bringUp3654(twin: ProxyInstanceFile | undefined, exitCode: number | null): Promise { + let spawnCalls = 0; + const diag: string[] = []; + const clock = fakeClock(); + const handle = await ensureProxyRunning( + { host: "127.0.0.1", port: 0, passthrough: false, debug: false, lane: "dsh" }, + { + fetchImpl: async () => ({ ok: true }), + fetchHealthInfo: async (origin) => (origin === TWIN_ORIGIN ? { ok: true, watchdog: { armed: true } } : undefined), + readInstanceFile: () => undefined, + listRegistryInstances: () => [], + spawnImpl: (_cmd: string, _args: readonly string[]): SpawnChild => { + spawnCalls++; + return fakeChildExitting(exitCode); + }, + now: clock.now, + sleep: clock.sleep, + registerWatcher: async () => "ok", + attachDiag: (msg) => { diag.push(msg); }, + scriptPath: FP_SCRIPT, + listAdoptCandidates: () => (twin !== undefined ? [twin] : []), + isConnectable: async () => true, + }, + ); + return { handle, spawnCalls, diag }; +} + +test("C: twin-collision exit adopts the serving instance instead of failing", async () => { + const home = tmpHome(); + try { + const { handle, spawnCalls, diag } = await bringUp3654(twinInstance("dsh"), TWIN_ADOPT_EXIT_CODE); + assert.equal(spawnCalls, 1, "the spawn happened and lost the race"); + assert.equal(handle.attached, true, "adoption rides the normal attach path"); + assert.equal(handle.origin, TWIN_ORIGIN); + assert.ok(diag.some((d) => d.includes("adopting serving twin")), "adoption must be announced"); + } finally { + home.restore(); + } +}); + +test("C: twin-collision with nothing adoptable keeps the original failure (code 3 named)", async () => { + const home = tmpHome(); + try { + const pending = bringUp3654(undefined, TWIN_ADOPT_EXIT_CODE); + await assert.rejects(pending, /code 3 \(twin-collision signal\)/); + } finally { + home.restore(); + } +}); + +test("C: a foreign-lane instance is never adopted", async () => { + const home = tmpHome(); + try { + const pending = bringUp3654(twinInstance("pi"), TWIN_ADOPT_EXIT_CODE); + await assert.rejects(pending, /code 3 \(twin-collision signal\)/); + } finally { + home.restore(); + } +}); + +test("C: a plain crash (exit 1) does NOT take the adoption path", async () => { + const home = tmpHome(); + try { + const pending = bringUp3654(twinInstance("dsh"), 1); + await assert.rejects(pending, /code 1/); + // The rejection alone cannot show adoption was skipped — run again + // capturing diagnostics through a resolved wrapper is overkill; the + // code-1 message lacking the twin suffix pins the branch selection. + } finally { + home.restore(); + } +}); + +// ———— D) eaddrinuseExitCode ———————————————————————————————————————————— + +const baseCtx = { + upgradeReplacement: false, + launchToken: true, + strictPort: false, + attempts: 1, + maxAttempts: 17, + predecessorTicksLeft: 0, + predecessorHolds: false, + lastTriedPort: 18917, +}; + +test("D: only twin-collision maps to the adopt signal; every other terminal decision exits 1", () => { + assert.equal(TWIN_ADOPT_EXIT_CODE, 3); + const twin = decideEaddrinuse({ ...baseCtx, twinHolds: true }); + assert.deepEqual(twin, { kind: "fail-fast", reason: "twin-collision" }); + assert.equal(eaddrinuseExitCode("twin-collision"), 3); + const strict = decideEaddrinuse({ ...baseCtx, twinHolds: false, strictPort: true }); + assert.equal(strict.kind === "fail-fast" ? strict.reason : undefined, "strict-port"); + assert.equal(eaddrinuseExitCode("strict-port"), 1); + const busy = decideEaddrinuse({ ...baseCtx, twinHolds: false, launchToken: false }); + assert.equal(busy.kind === "fail-fast" ? busy.reason : undefined, "port-busy"); + assert.equal(eaddrinuseExitCode("port-busy"), 1); + const upgrade = decideEaddrinuse({ ...baseCtx, twinHolds: false, upgradeReplacement: true }); + assert.equal(upgrade.kind === "fail-fast" ? upgrade.reason : undefined, "upgrade-replacement"); + assert.equal(eaddrinuseExitCode("upgrade-replacement"), 1); + assert.equal(eaddrinuseExitCode(undefined), 1, "non-terminal decisions carry no exit code"); +}); + +// ———— E) writeSpawnPortLog —————————————————————————————————————————————— + +test("E: spawner children write their listen failure into the per-port log", () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "bili-3654-tmp-")); + const prev = { TMPDIR: process.env.TMPDIR, TEMP: process.env.TEMP, TMP: process.env.TMP }; + process.env.TMPDIR = dir; + process.env.TEMP = dir; + process.env.TMP = dir; + try { + assert.equal(dir, os.tmpdir(), "os.tmpdir() must follow the override on this platform"); + writeSpawnPortLog(18918, "2026-10-18T00:00:00.000Z [error] [v=test] listen failed: EADDRINUSE address in use: :::18918"); + const file = path.join(dir, "bili-proxy-18918.log"); + assert.ok(fs.existsSync(file), "the file the failure message points at must exist"); + assert.match(fs.readFileSync(file, "utf8"), /listen failed: EADDRINUSE/); + } finally { + for (const [k, v] of Object.entries(prev)) if (v === undefined) delete process.env[k]; else process.env[k] = v; + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + +// ———— F) refusal wording ———————————————————————————————————————————————— + +const MODEL_URL = "https://api.anthropic.com/v1/messages"; + +test("F: busy-exhausted refusal says 'alive but unresponsive', never 'guaranteed upstream 400'", async () => { + const { createNativeRoute } = await import("../src/agent/opencode-native.ts"); + const HELD = "http://127.0.0.1:18919"; + const state: NativeInterceptState = { origin: HELD, ready: Promise.resolve(undefined) }; + const route = createNativeRoute(state, { + probe: async () => ({ state: "busy" as const }), + probeTtlMs: 500, + busyWaitCapMs: 120, + busyReprobeBaseMs: 20, + }); + const s: V2State = { windows: new Map([["prov/model-x", 1_000]]) }; + const body = JSON.stringify({ messages: [{ role: "user", content: "x".repeat(5_000) }] }); + const e = { + sessionID: "ses_busy", + model: { providerID: "prov", id: "model-x" }, + request: new Request(MODEL_URL, { method: "POST", headers: { "content-type": "application/json" }, body }), + }; + const err = await route(e, s).then(() => undefined, (x: unknown) => x); + assert.ok(err instanceof Error, "oversize body must be refused locally"); + assert.match(err.message, /alive but unresponsive/); + assert.doesNotMatch(err.message, /proxy unavailable/, "a saturated proxy is not 'unavailable'"); + assert.doesNotMatch(err.message, /guaranteed upstream 400/, "it would serve the request once the fold finishes"); + assert.equal(state.lastUnavailableReason, "busy-exhausted"); +}); + +test("F: a genuinely dead proxy keeps the legacy refusal wording byte-for-byte", async () => { + const { createNativeRoute } = await import("../src/agent/opencode-native.ts"); + const HELD = "http://127.0.0.1:18920"; + const state: NativeInterceptState = { origin: HELD, ready: Promise.resolve(undefined) }; + const route = createNativeRoute(state, { probe: async () => false, probeTtlMs: 500 }); + const s: V2State = { windows: new Map([["prov/model-x", 1_000]]) }; + const body = JSON.stringify({ messages: [{ role: "user", content: "x".repeat(5_000) }] }); + const e = { + sessionID: "ses_dead", + model: { providerID: "prov", id: "model-x" }, + request: new Request(MODEL_URL, { method: "POST", headers: { "content-type": "application/json" }, body }), + }; + const err = await route(e, s).then(() => undefined, (x: unknown) => x); + assert.ok(err instanceof Error); + assert.match(err.message, /bili: proxy unavailable and this request cannot be sent uncompressed/); + assert.match(err.message, /Recovery is automatic \(#3308\)/); + assert.equal(state.lastUnavailableReason, "dead"); +}); + +test("F: an under-window body still goes direct when nothing is routable (fail-open unchanged)", async () => { + const { createNativeRoute } = await import("../src/agent/opencode-native.ts"); + const state: NativeInterceptState = { origin: undefined, ready: Promise.resolve(undefined) }; + const route = createNativeRoute(state, { probe: async () => ({ state: "busy" as const }), probeTtlMs: 500, busyWaitCapMs: 100, busyReprobeBaseMs: 20 }); + const s: V2State = { windows: new Map([["prov/model-x", 1_000_000]]) }; + const body = JSON.stringify({ messages: [{ role: "user", content: "small" }] }); + const e = { + sessionID: "ses_small", + model: { providerID: "prov", id: "model-x" }, + request: new Request(MODEL_URL, { method: "POST", headers: { "content-type": "application/json" }, body }), + }; + await route(e, s); + assert.equal(s.proxyBase, undefined, "direct send — no proxy target pinned"); +});