From 4cd1849ab4f27b3fe9d3be090dafd71baa19ebbb Mon Sep 17 00:00:00 2001 From: ework-agent Date: Mon, 5 Oct 2026 23:55:44 +0800 Subject: [PATCH] fix: wait out slow live-child proxy starts and self-heal dsh spawn-mode boot failures in-process (#2187) --- src/agent/dsh-native-client.ts | 21 ++- src/agent/dsh-native.ts | 15 ++ src/launcher.ts | 43 +++-- tests/dsh-settings-ui.test.ts | 86 +++++++++- tests/issue2187-dsh-spawn-retry.test.ts | 168 ++++++++++++++++++++ tests/issue2187-launcher-slow-start.test.ts | 147 +++++++++++++++++ tests/launcher.test.ts | 2 +- 7 files changed, 462 insertions(+), 20 deletions(-) create mode 100644 tests/issue2187-dsh-spawn-retry.test.ts create mode 100644 tests/issue2187-launcher-slow-start.test.ts diff --git a/src/agent/dsh-native-client.ts b/src/agent/dsh-native-client.ts index 6499e704c9..5beb155b0e 100644 --- a/src/agent/dsh-native-client.ts +++ b/src/agent/dsh-native-client.ts @@ -41,12 +41,16 @@ export const inject = ["slots", "locale"]; const NS = "bili"; -// #1809: live-origin probe cadence — first attempt immediate, then a retry -// every POLL_INTERVAL_MS up to POLL_MAX_ATTEMPTS total (~30s of coverage for -// a slow spawn-mode bootstrap, bounded so an absent host costs no more). +// #1809/#2187: live-origin probe cadence — first attempt immediate, then a +// retry every POLL_INTERVAL_MS for POLL_MAX_ATTEMPTS total (~30s), after +// which probing CONTINUES at SLOW_POLL_INTERVAL_MS until the origin arrives +// or the page unmounts. #2187: the server side now heals a failed spawn-mode +// bootstrap in the background and a slow Windows boot can take minutes, so +// stopping at ~30s stranded the entry on "not bound" for the whole session. const ORIGIN_PATH = "/bili/origin"; const POLL_INTERVAL_MS = 3000; const POLL_MAX_ATTEMPTS = 10; +const SLOW_POLL_INTERVAL_MS = 10000; const zh: Dict = { "nav": "bili设置", @@ -75,9 +79,9 @@ function openExternal(url: string): void { if (typeof w.open === "function") w.open(url, "_blank", "noopener,noreferrer"); } -/** #1809: poll the host's live origin route until one arrives; returns the - * cancel used as the effect cleanup (no fetch ⇒ no-op, older hosts without - * the route simply stay degraded after the attempts are exhausted). */ +/** #1809/#2187: poll the host's live origin route until one arrives; returns + * the cancel used as the effect cleanup (no fetch ⇒ no-op, older hosts + * without the route simply stay degraded). */ function probeOrigin(onOrigin: (origin: string) => void): () => void { if (typeof fetch !== "function") return () => {}; let cancelled = false; @@ -99,7 +103,10 @@ function probeOrigin(onOrigin: (origin: string) => void): () => void { } if (resolved) return; attempts += 1; - if (!cancelled && attempts < POLL_MAX_ATTEMPTS) timer = setTimeout(() => void poll(), POLL_INTERVAL_MS); + // #2187: fast phase for the first POLL_MAX_ATTEMPTS, then an endless + // slow phase — the origin can land at any time (background heal). + const delay = attempts < POLL_MAX_ATTEMPTS ? POLL_INTERVAL_MS : SLOW_POLL_INTERVAL_MS; + if (!cancelled) timer = setTimeout(() => void poll(), delay); }; void poll(); return () => { diff --git a/src/agent/dsh-native.ts b/src/agent/dsh-native.ts index b754fc264d..c0fc3989bc 100644 --- a/src/agent/dsh-native.ts +++ b/src/agent/dsh-native.ts @@ -838,6 +838,16 @@ export function apply(ctx: PluginContext): void { state.ready = trackChain(start()); } + // #2187: arm the recovery loop at PLAN time, not on first traffic. After + // a boot-time spawn failure NO caller can reach maybeRetry — headersFor + // is only consulted once an origin has resolved, /acp early-returns while + // base-less, and the recovery timer below arms itself only from inside + // maybeRetry — so the lane stayed degraded for the whole session. Happy + // path: single-flight joins the bootstrap started above (no second + // spawn); failure path: the timer re-fires state.respawn() every + // RETRY_INTERVAL_MS until a proxy lands. + maybeRetry(ctx); + // #1158 L2: a refusal sends model traffic DIRECT. First refusal per // endpoint logs once; same-state refusals accumulate silently and re-print // only on an attribution STATE change (none↔threw), always carrying the @@ -1096,6 +1106,11 @@ export function _resetRegisterForTest(base: string | undefined): void { recoveryTimer = undefined; } activeCtx = undefined; + // #2187: apply() now reaches the respawn arming on every plan, so a stale + // single-flight wrapper (capturing a previous test's spawn stub) must not + // survive a reset. + state.respawn = undefined; + state.onGiveUp = undefined; register.base = base; register.toolsReady = false; register.dead = false; diff --git a/src/launcher.ts b/src/launcher.ts index 080b286703..bcd00ef574 100644 --- a/src/launcher.ts +++ b/src/launcher.ts @@ -153,11 +153,20 @@ export type BaseClientName = "claude" | "codex" | "pi" | "omp" | "opencode" | "h const HEALTH_PATH = "/__bili/health"; const HEALTH_POLL_INTERVAL_MS = 200; const SPAWN_WAIT_MS = 20000; +// #2187: grace beyond the initial wait for a LIVE child still starting. +// Spawn-window startups have been observed taking 26-45s on slow Windows +// machines (AV / first-run disk activity); the one-shot 20s budget misread +// that as failure and left the lane degraded for the whole session. +const SLOW_START_GRACE_MS = 40_000; +// Total wait for a live child: initial window + grace. A child that EXITS or +// fails to spawn still fails fast (the poll loop breaks on exit/error). +export const SPAWN_BUDGET_MS = SPAWN_WAIT_MS + SLOW_START_GRACE_MS; const PROBE_TIMEOUT_MS = 1500; -// #707: max age of a starting marker still treated as an in-progress bring-up. -// A well-behaved starter resolves within SPAWN_WAIT_MS; the slack covers slow -// disks and client teardown before it clears the marker. -const STARTING_MARKER_TTL_MS = SPAWN_WAIT_MS + 30_000; +// #707/#2187: max age of a starting marker still treated as an in-progress +// bring-up. A well-behaved starter resolves within SPAWN_BUDGET_MS (a live +// child may take the whole budget); the slack covers slow disks and client +// teardown before it clears the marker. +const STARTING_MARKER_TTL_MS = SPAWN_BUDGET_MS + 30_000; // #1903: budget for the post-exit re-discovery below. A spawned child dying // before becoming healthy is evidence the port it wanted is HELD — but our // one-shot discovery snapshot may predate the holder publishing its identity @@ -3439,8 +3448,10 @@ function pickAttachable( } /** #707: wait for another launcher's in-flight bring-up to produce a live - * instance. Bounded by SPAWN_WAIT_MS; breaks early when the starting marker - * disappears (starter gave up / crashed). The final probe closes the + * instance. Bounded by SPAWN_BUDGET_MS — the starter's own live-child wait + * has the same envelope (#2187); bailing earlier would double-spawn against + * a still-starting child; breaks early when the starting marker disappears + * (starter gave up / crashed). The final probe closes the * deadline-boundary sliver: the starter's own poll window ends ~now, and its * success path clears the marker — indistinguishable from a failure bail * without one last look. */ @@ -3455,7 +3466,7 @@ async function waitForStarterInstance( refusedLog: Set, diag?: (msg: string) => void, ): Promise { - const deadline = now() + SPAWN_WAIT_MS; + const deadline = now() + SPAWN_BUDGET_MS; const probe = async (): Promise => pickAttachable(await probeLiveInstances(readInstance, fetchHealthInfo, diag), opts, codeFingerprint, attachExternal, refusedLog, diag); let inst: ProxyInstanceFile | undefined; @@ -3764,7 +3775,7 @@ export async function ensureProxyRunning( }; // #1225: an in-flight starter of a DIFFERENT declared lane can never // produce an instance we may attach to — waiting would only stall this - // launch behind its SPAWN_WAIT_MS window. Undeclared lanes wildcard. + // launch behind its SPAWN_BUDGET_MS window. Undeclared lanes wildcard. const starterLaneMatches = (m: ProxyStartingMarker): boolean => opts.lane === undefined || m.lane === undefined || m.lane === opts.lane; const marker = readStartingMarker(); @@ -3876,7 +3887,7 @@ export async function ensureProxyRunning( // #401/#480: fail fast when OUR spawned child dies before becoming // healthy — otherwise a startup crash (bad config, missing upstream, …) - // burns the whole SPAWN_WAIT_MS poll window before erroring. + // burns the whole spawn wait budget before erroring. let childExit: { code: number | null; signal: string | null } | undefined; // #809/D: an async spawn failure (EACCES/ENOENT on the resolved runtime) // emits 'error', not 'exit'. Unhandled, it becomes an uncaughtException @@ -3895,9 +3906,19 @@ export async function ensureProxyRunning( childError = rest[0]; }); - const deadline = now() + SPAWN_WAIT_MS; + const deadline = now() + SPAWN_BUDGET_MS; + // #2187: announce the phase boundary once — a live child past the + // initial window is still being waited for, and the log is the only + // place to see that the budget was extended instead of failing. + let slowStartNoticed = false; while (now() < deadline) { if (childExit || childError !== undefined) break; + if (!slowStartNoticed && now() >= deadline - SLOW_START_GRACE_MS) { + slowStartNoticed = true; + console.error( + `bili: spawned proxy not healthy after ${SPAWN_WAIT_MS}ms — child pid ${child.pid ?? "?"} is still starting up; extending wait to ${SPAWN_BUDGET_MS}ms total (log: ${logPath})`, + ); + } await sleepImpl(HEALTH_POLL_INTERVAL_MS); const inst = readInstance(); if (isProxyInstanceFile(inst) && inst.launchToken === launchToken) { @@ -3979,7 +4000,7 @@ export async function ensureProxyRunning( : childExit.signal ? `signal ${childExit.signal}` : "unknown reason"; throw new Error(`bili: proxy child exited before becoming healthy (${detail}) (log: ${logPath})`); } - throw new Error(`bili: proxy did not become healthy within ${SPAWN_WAIT_MS}ms (log: ${logPath})`); + throw new Error(`bili: proxy did not become healthy within ${SPAWN_BUDGET_MS}ms (log: ${logPath})`); } finally { if (claimed) clearStartingMarker(launchToken); } diff --git a/tests/dsh-settings-ui.test.ts b/tests/dsh-settings-ui.test.ts index 1cf537132e..815015e2bd 100644 --- a/tests/dsh-settings-ui.test.ts +++ b/tests/dsh-settings-ui.test.ts @@ -427,7 +427,7 @@ test("#1590/#2125: client bundle registers the settings.section entry bili AND t assert.ok(degBundleTexts.some((t) => t.includes("/acp")), "the bundle degraded hint points at /acp"); }); -test("#1809/#2125: client polls /bili/origin while unresolved — upgrades on success, stays degraded and cancels when absent (both slots)", async () => { +test("#1809/#2125: client polls /bili/origin while unresolved — upgrades on success, stays degraded and cancels when absent (both slots); #2187 keeps probing in a slow phase past the fast attempts", async () => { const { build } = await import("esbuild"); const configs = ((await import("../tsup.config.ts")).default) as unknown as Array<{ entry: Record; @@ -595,6 +595,90 @@ test("#1809/#2125: client polls /bili/origin while unresolved — upgrades on su assert.deepEqual(opened, ["http://127.0.0.1:9999/__bili/"]); m.runCleanups(); } + + { + // #2187: past the fast phase, probing CONTINUES at the slow cadence + // instead of stopping — a late-arriving origin (background heal) must + // still upgrade the entry. Manual clock: capture each scheduled retry + // and fire it by hand so the test never waits in wall time. + const pending: Array<{ id: number; delay: number; fn: () => void }> = []; + let nextId = 1; + let polls = 0; + let landed = false; + const fakeSetTimeout = ((fn: () => void, delay?: number): unknown => { + const id = nextId++; + pending.push({ id, delay: delay ?? 0, fn }); + return id; + }) as unknown as typeof setTimeout; + const fakeClearTimeout = ((id: unknown): void => { + const i = pending.findIndex((p) => p.id === id); + if (i >= 0) pending.splice(i, 1); + }) as unknown as typeof clearTimeout; + const m = mount({ + fetch: async () => { + polls += 1; + if (polls >= 13) landed = true; + return { ok: true, json: async () => ({ origin: polls >= 13 ? "http://127.0.0.1:9997" : null }) }; + }, + setTimeout: fakeSetTimeout, + clearTimeout: fakeClearTimeout, + }); + m.resetHooks(); + const first = m.component() as ElementNode; + assert.equal(findButton(first), undefined, "first paint before the probe resolves is still degraded"); + const delays: number[] = []; + for (;;) { + await tick(); + if (landed) break; + if (pending.length === 0) throw new Error("polling stopped before the origin arrived"); + const t = pending.shift()!; + delays.push(t.delay); + t.fn(); + } + const second = m.component() as ElementNode; + const button = findButton(second); + assert.ok(button !== undefined, "a late-arriving origin upgrades the entry without a reload"); + assert.deepEqual(delays.slice(0, 9), Array(9).fill(3000), "fast phase keeps the original cadence"); + assert.ok(delays.length > 9, `probing must continue past the fast phase, got ${delays.length} retries`); + assert.deepEqual(delays.slice(9), Array(delays.length - 9).fill(10000), "slow phase uses the reduced cadence"); + m.runCleanups(); + } + + { + // #2187: unmount still cancels the pending retry even in the slow phase + const pending: Array<{ id: number; delay: number; fn: () => void }> = []; + let nextId = 1; + let cleared = 0; + const fakeSetTimeout = ((fn: () => void, delay?: number): unknown => { + const id = nextId++; + pending.push({ id, delay: delay ?? 0, fn }); + return id; + }) as unknown as typeof setTimeout; + const fakeClearTimeout = ((id: unknown): void => { + cleared += 1; + const i = pending.findIndex((p) => p.id === id); + if (i >= 0) pending.splice(i, 1); + }) as unknown as typeof clearTimeout; + const m = mount({ + fetch: async () => ({ ok: true, json: async () => ({ origin: null }) }), + setTimeout: fakeSetTimeout, + clearTimeout: fakeClearTimeout, + }); + m.resetHooks(); + m.component(); + const delays: number[] = []; + for (let i = 0; i < 12; i++) { + await tick(); + if (pending.length === 0) break; + const t = pending.shift()!; + delays.push(t.delay); + t.fn(); + } + assert.ok(delays.includes(10000), `reached the slow phase, got delays ${JSON.stringify(delays)}`); + m.runCleanups(); + assert.ok(cleared >= 1, "the pending retry is cancelled on unmount"); + assert.equal(pending.length, 0, "no timers survive unmount"); + } }); test("#2125: the manifest declares no dsh.client.inject (pre-0.2 dsh compat)", () => { diff --git a/tests/issue2187-dsh-spawn-retry.test.ts b/tests/issue2187-dsh-spawn-retry.test.ts new file mode 100644 index 0000000000..5808287129 --- /dev/null +++ b/tests/issue2187-dsh-spawn-retry.test.ts @@ -0,0 +1,168 @@ +// #2187: a boot-time spawn failure must self-heal within the process. Before +// this fix the retry machinery was unreachable after a bootstrap failure — +// every production caller of maybeRetry sat behind a bound origin, so one +// failed spawn meant permanent direct-send and /bili/origin stuck at null. +// apply() now arms the retry at plan time; these tests pin that the recovery +// timer drives retries with ZERO model traffic until a proxy lands. + +import test, { afterEach } from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import http from "node:http"; +import { apply, _resetRegisterForTest, _setSpawnForTest, _settleNativeForTest, _stateHeadersForTest } from "../src/agent/dsh-native.ts"; +import { rmrf } from "./tmp-rm.ts"; + +afterEach(() => { void _settleNativeForTest(); }); + +type MockTool = { name: string; description?: string; inputSchema: Record }; +type RegisteredTool = { + name: string; + description?: string; + parameters: unknown; + output: { schema: unknown; render: (args: unknown, value: unknown) => Array<{ type: string; text: string }> }; + execute: (args: Record, exec: { agent?: { session?: { id?: unknown } }; signal?: AbortSignal }) => Promise; +}; + +function mockBiliHandler(manifestDelayMs = 0): (req: http.IncomingMessage, res: http.ServerResponse) => void { + const manifestTools: MockTool[] = [ + { + name: "compress", + description: "Compress a range of messages", + inputSchema: { type: "object", properties: { summary: { type: "string" } }, required: ["summary"] }, + }, + ]; + return (_req, res) => { + if ((_req.url ?? "") === "/__bili/plugin/manifest") { + const respond = () => { + res.writeHead(200, { "content-type": "application/json" }); + res.end(JSON.stringify({ version: "0.1.119", tools: { anthropic: manifestTools.map((t) => ({ name: t.name, description: t.description, input_schema: t.inputSchema })) } })); + }; + if (manifestDelayMs > 0) setTimeout(respond, manifestDelayMs); + else respond(); + return; + } + res.writeHead(404); + res.end("{}"); + }; +} + +async function startMockProxy(): Promise<{ origin: string; close: () => void }> { + const server = http.createServer(mockBiliHandler()); + return new Promise((resolve) => { + server.listen(0, "127.0.0.1", () => { + const addr = server.address() as { port: number }; + resolve({ origin: `http://127.0.0.1:${addr.port}`, close: () => server.close() }); + }); + }); +} + +type HostCtx = Parameters[0]; + +function mockCtx() { + const tools: RegisteredTool[] = []; + const commands: Array<{ name: string; handler: () => Promise<{ kind: string; text: string }> }> = []; + let initiator: { session?: { id?: unknown } } | undefined = undefined; + return { + tools: { register: (t: RegisteredTool) => { tools.push(t); } }, + commands: { register: (c: { name: string; handler: () => Promise<{ kind: string; text: string }> }) => commands.push(c) }, + agents: { currentInitiator: () => initiator }, + setInitiator: (i: { session?: { id?: unknown } } | undefined) => (initiator = i), + registeredTools: tools, + registeredCommands: commands, + inject: (_deps: readonly string[], _cb: (sub: HostCtx) => void) => {}, + }; +} + +async function withEnv(env: Record, fn: () => Promise | T): Promise { + const saved: Record = {}; + for (const k of Object.keys(env)) { + saved[k] = process.env[k]; + const v = env[k]; + if (v === undefined) delete process.env[k]; + else process.env[k] = v; + } + try { + return await fn(); + } finally { + for (const [k, v] of Object.entries(saved)) { + if (v === undefined) delete process.env[k]; + else process.env[k] = v; + } + } +} + +async function waitFor(cond: () => boolean, what: string): Promise { + const deadline = Date.now() + 15000; + while (!cond()) { + if (Date.now() > deadline) throw new Error(`timeout waiting for ${what}`); + await new Promise((r) => setTimeout(r, 10)); + } +} + +test("#2187 boot-time spawn failure self-heals with zero model traffic — timer-driven retries land the proxy and register the tools", async () => { + const proxy = await startMockProxy(); + const home = fs.mkdtempSync(path.join(os.tmpdir(), "bili-dsh-2187-spawn-")); + try { + await withEnv( + { DSH_HOME: home, BILLION_CONTEXT_PROXY: undefined, BILI_DSH_RECOVERY_INTERVAL_MS: "20", BILI_DSH_RETRY_INTERVAL_MS: "20" }, + async () => { + _resetRegisterForTest(undefined); // base-less spawn plan + let attempts = 0; + _setSpawnForTest(() => { + attempts += 1; + return Promise.resolve(attempts <= 2 ? undefined : proxy.origin); + }); + try { + const ctx = mockCtx(); + ctx.setInitiator({ session: { id: "dsh-2187-session" } }); + apply(ctx); + // two consecutive boot failures — before the fix nothing + // would ever retry after this point + await waitFor(() => attempts >= 3, "a third attempt fires after two failed boots"); + await waitFor(() => ctx.registeredTools.length === 1, "timer-driven registration with zero model traffic"); + assert.equal(ctx.registeredTools[0].name, "compress"); + await waitFor( + () => _stateHeadersForTest()?.("http://example.test/v1/messages") !== undefined, + "headers stamped after the heal", + ); + } finally { + _setSpawnForTest(undefined); + } + }, + ); + } finally { + proxy.close(); + rmrf(home); + _resetRegisterForTest(undefined); + } +}); + +test("#2187 persistent spawn failure keeps retrying instead of degrading permanently", async () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "bili-dsh-2187-persist-")); + try { + await withEnv( + { DSH_HOME: home, BILLION_CONTEXT_PROXY: undefined, BILI_DSH_RECOVERY_INTERVAL_MS: "20", BILI_DSH_RETRY_INTERVAL_MS: "20" }, + async () => { + _resetRegisterForTest(undefined); + let attempts = 0; + _setSpawnForTest(() => { + attempts += 1; + return Promise.resolve(undefined); + }); + try { + apply(mockCtx()); + // the old behavior was exactly ONE attempt for the whole + // process lifetime; the recovery timer must keep firing + await waitFor(() => attempts >= 4, "retries keep firing while every attempt fails"); + } finally { + _setSpawnForTest(undefined); + } + }, + ); + } finally { + rmrf(home); + _resetRegisterForTest(undefined); + } +}); diff --git a/tests/issue2187-launcher-slow-start.test.ts b/tests/issue2187-launcher-slow-start.test.ts new file mode 100644 index 0000000000..e39f34ee16 --- /dev/null +++ b/tests/issue2187-launcher-slow-start.test.ts @@ -0,0 +1,147 @@ +// #2187: a LIVE spawned child past the initial 20s window must keep being +// waited for (extended budget), not abandoned — observed Windows spawn-window +// startups take 26-45s, and the one-shot budget misread that as failure. The +// cross-process waiter shares the same envelope, or it bails at 20s and +// double-spawns against a still-starting child. + +import test from "node:test"; +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { claimStartingMarker, removeStartingMarker, type ProxyInstanceFile as InstanceFile } from "../src/instance.ts"; +import { ensureProxyRunning, SPAWN_BUDGET_MS, type SpawnChild, type SpawnFn } from "../src/launcher.ts"; + +const FP_SCRIPT = path.join(os.tmpdir(), `bili-fp-2187-${process.pid}.js`); +fs.writeFileSync(FP_SCRIPT, "// fingerprint fixture\n"); +const FP_HASH = createHash("sha256").update(fs.readFileSync(FP_SCRIPT)).digest("hex"); + +function recordedInstance(over: Partial = {}): InstanceFile { + return { + origin: "http://127.0.0.1:8787", + instanceId: "inst-1", + pid: process.pid, + startedAt: Date.now(), + host: "127.0.0.1", + port: 8787, + passthrough: false, + mitmDomains: [], + modelWindows: {}, + codeFingerprint: FP_HASH, + ...over, + }; +} + +function makeFakeChild(pid: number): SpawnChild { + const handlers = new Map void)[]>(); + return { + pid, + unref() {}, + kill() { + return true; + }, + on(event, listener) { + const list = handlers.get(event) ?? []; + list.push(listener); + handlers.set(event, list); + }, + }; +} + +// Fake clock: 1 tick = 1 second — spans the observed slow-start envelope +// (26-45s) without burning wall-clock time. +function fakeClock() { + let ticks = 0; + return { + now: () => ticks * 1000, + sleep: () => { + ticks += 1; + return Promise.resolve(); + }, + ticks: () => ticks, + }; +} + +for (const bindAt of [26, 45]) { + test(`#2187 live child binding at ~${bindAt}s resolves within the extended budget`, async () => { + let spawnCalls = 0; + let childToken = ""; + const clock = fakeClock(); + const spawnImpl: SpawnFn = (_cmd, _args, options) => { + spawnCalls++; + childToken = (options.env?.BILI_LAUNCH_TOKEN as string) ?? ""; + return makeFakeChild(42460); + }; + const handle = await ensureProxyRunning( + { host: "127.0.0.1", port: 8787, passthrough: false, debug: false }, + { + fetchImpl: async () => ({ ok: true }), + fetchHealthInfo: async () => ({ ok: true, pid: 42460 }), + spawnImpl, + now: clock.now, + sleep: clock.sleep, + readInstanceFile: () => + clock.ticks() >= bindAt + ? recordedInstance({ launchToken: childToken, origin: "http://127.0.0.1:8787", port: 8787, pid: 42460 }) + : undefined, + }, + ); + assert.equal(spawnCalls, 1); + assert.ok(handle.child, "the still-starting child was waited for, not abandoned"); + assert.equal(handle.origin, "http://127.0.0.1:8787"); + assert.ok(clock.ticks() <= Math.ceil(SPAWN_BUDGET_MS / 1000), `bound at ${bindAt}s must not burn the whole budget`); + }); +} + +test("#2187 never-healthy child fails after the FULL extended budget, not the initial window", async () => { + const clock = fakeClock(); + let spawnCalls = 0; + const spawnImpl: SpawnFn = () => { + spawnCalls++; + return makeFakeChild(42461); + }; + await assert.rejects( + ensureProxyRunning( + { host: "127.0.0.1", port: 8787, passthrough: false, debug: false }, + { + fetchImpl: async () => ({ ok: false }), + fetchHealthInfo: async () => ({ ok: false }), + spawnImpl, + now: clock.now, + sleep: clock.sleep, + readInstanceFile: () => undefined, + }, + ), + new RegExp(`did not become healthy within ${SPAWN_BUDGET_MS}ms`), + ); + assert.equal(spawnCalls, 1); + assert.ok(clock.ticks() >= 59, `the full budget (~${Math.ceil(SPAWN_BUDGET_MS / 1000)}s) must be consumed, got ${clock.ticks()}s`); +}); + +test("#2187 cross-process waiter: instance appearing at ~45s attaches instead of double-spawning", async () => { + try { + claimStartingMarker({ token: "starter-2187", pid: process.pid, host: "127.0.0.1", port: 8788, startedAt: Date.now() }); + const clock = fakeClock(); + const handle = await ensureProxyRunning( + { host: "127.0.0.1", port: 8787, passthrough: false, debug: false }, + { + spawnImpl: () => { + throw new Error("double-spawn: another launch was still bringing its proxy up"); + }, + fetchImpl: async () => ({ ok: true }), + fetchHealthInfo: async () => ({ ok: true, instanceId: "inst-9", watchdog: { armed: true } }), + readInstanceFile: () => + clock.ticks() >= 45 ? recordedInstance({ instanceId: "inst-9", origin: "http://127.0.0.1:8788", port: 8788 }) : undefined, + now: clock.now, + sleep: clock.sleep, + registerWatcher: async () => "ok" as const, + scriptPath: FP_SCRIPT, + }, + ); + assert.equal(handle.attached, true); + assert.equal(handle.origin, "http://127.0.0.1:8788"); + } finally { + removeStartingMarker(); + } +}); diff --git a/tests/launcher.test.ts b/tests/launcher.test.ts index 0315d88c46..ec5966f54c 100644 --- a/tests/launcher.test.ts +++ b/tests/launcher.test.ts @@ -1933,7 +1933,7 @@ test("ensureProxyRunning: stale starting marker (dead owner) → removed, then s test("ensureProxyRunning: expired starting marker (hung owner) → spawns (#707)", async () => { try { - claimStartingMarker({ token: "starter-c", pid: process.pid, host: "127.0.0.1", port: 8789, startedAt: Date.now() - 55_000 }); + claimStartingMarker({ token: "starter-c", pid: process.pid, host: "127.0.0.1", port: 8789, startedAt: Date.now() - 95_000 }); // #2187: past the extended 90s marker TTL let spawnCalls = 0; const handle = await ensureProxyRunning( { host: "127.0.0.1", port: 8787, passthrough: false, debug: false },