From 2a7e5d14ae6aae4a4d28f79ca15175c6dfef9c58 Mon Sep 17 00:00:00 2001 From: ework-agent Date: Fri, 11 Sep 2026 21:52:02 +0800 Subject: [PATCH 1/2] =?UTF-8?q?fix(launcher):=20cross-process=20startup-wi?= =?UTF-8?q?ndow=20race=20=E2=80=94=20starting=20marker=20prevents=20double?= =?UTF-8?q?-spawn=20(#707)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 1 + src/instance.ts | 85 ++++++++++++ src/launcher.ts | 178 ++++++++++++++++--------- tests/instance-governance.test.ts | 44 +++++++ tests/launcher.test.ts | 209 +++++++++++++++++++++++++++++- 5 files changed, 455 insertions(+), 62 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 19bf0905e6..a8f79581bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ Versions follow the merge of a `*_release-v*` branch; CI publishes to npm on tag ### Fixes +- **Cross-process startup-window race: concurrent launches no longer double-spawn proxies (#707)**: `ensureProxyRunning` only attached when it found a *healthy* instance at probe time — between the launcher's spawn and the child's first successful listen there was zero on-disk evidence that a bring-up was in flight, so two launches inside that window (~1s+) each spawned their own child: two live proxies over one sessions dir (double writers), last-writer-wins instance file, and the extra proxy living until ITS OWN spawner exited. The starter now claims `/proxy-starting` with O_EXCL before spawning (carrying `{token, pid, host, port, startedAt}`); a concurrent caller that sees an active marker (owner pid alive AND younger than SPAWN_WAIT_MS + 30s) waits for the starter's instance with the same probe as attach (pid alive + `/__bili/health` + instanceId match), bounded by SPAWN_WAIT_MS, and attaches on success; marker disappearance (starter gave up / crashed) breaks the wait early so the caller spawns immediately. The starter clears the marker on every terminal path via a token-checked clear; a hard crash leaves a stale marker that the dead-owner/TTL checks render inert. Incompatible-config fall-through is preserved (a waited-for instance that isn't compatible with our launch options still spawns, matching today's incompatible-healthy-instance behavior); explicit `bili start` bypasses the marker entirely (it's a user action, not auto-bootstrap). In-process dedup (`singleFlight`, #706) is untouched — this is the cross-process half. - **DeepSeek thinking mode 400 fixed: turns are atomic, reasoning-drop self-disables on strict-echo upstreams (#684)**: two paths could ship a rebuilt request whose assistant `tool_calls` turn lost its `reasoning_content` — (1) the proxy's recent-zone carve kept the last tool-call of a multi-call turn visible while its reasoning run folded into the block (kernel boundary logic only pulled the FIRST companion message after reasoning), and (2) #651's reasoning-drop stripped reasoning from a closed compress turn even on upstreams that document thinking-mode `reasoning_content` as mandatory round-trip. DeepSeek rejects both with 400 (`The \`reasoning_content\` in the thinking mode must be passed back to the API`) and the turn dies. Fixes: the kernel now computes atomic turn groups (reasoning run + its full assistant burst + every result paired by `toolCallId` — all fold together or none do; a range that would split a turn is withdrawn with a warning, or rejected with an explanatory error when nothing else remains), the forward boundary pull covers the whole burst, and the reasoning-drop never fires when the session is known strict-echo — statically for `deepseek` upstreams, or learned from the first 400 whose body mentions `reasoning_content` (persisted on the session, self-healing like #552). Three wire-level exit sentinels (`[reasoning-pair-violated]` on chat/completions and Responses, `[thinking-pair-violated]` on Anthropic) log loudly if any future path reintroduces a split turn. - **Windows: spaced client/extension paths no longer break launched clients or silently drop extensions (#679, PR #682)**: `runClient` spawned clients with `shell: true` and no per-token quoting, so on Windows cmd.exe re-split the joined line at the first space — a client binary under a spaced path (`C:\Users\John Doe\...`) died with `'C:\Users\John' is not recognized`, and a surviving client whose `-e ` path was split launched WITHOUT the bili extension (compression silently never activates). Spawning is now routed by executable class: real `.exe`/node binaries spawn directly (the OS quotes argv, spaces intact), npm `.cmd`/`.bat`/extensionless shims go through an explicit `comspec /d /s /c "…"` line with correct per-token quoting (`windowsVerbatimArguments`); `shell: true` is gone (also silences Node's DEP0190 warning). Known edge: `BILI_CLIENT_BIN` pointing at a `.js` file now fails loudly instead of relying on file association via the old shell path. - **Windows: codex MCP server delivered via CODEX_HOME overlay, not inline `-c` (#681, PR #683)**: cmd.exe cannot transmit a literal `"` to a child, but every valid TOML string needs one — so the inline `-c mcp_servers.bili.command="C:\\Program Files\\nodejs\\node.exe"` arg arrived at codex as an unquoted, space-split, invalid TOML value for ANY absolute Windows path (spaced stock install just the common case), the bili MCP server never registered, and plugin-mode compression silently never activated. On `win32` the launcher now writes `[mcp_servers.bili]` (command/args/env.BILI_MCP_PROXY/env.BILI_CONVERSATION_ID) into a persistent `-bili` overlay's merged `config.toml` — sharing every other real-home entry (auth.json, sessions, model settings) — and points the spawned codex at it via `CODEX_HOME`. POSIX keeps the working inline `-c` path byte-identical; if the overlay can't be built the launch degrades to wire mode (server-side compression still active) with a warning. Real-Windows end-to-end verification tracked in #686. diff --git a/src/instance.ts b/src/instance.ts index 8dea89b68b..8c933cc5d2 100644 --- a/src/instance.ts +++ b/src/instance.ts @@ -113,6 +113,91 @@ export function clearProxyInstanceFile(instanceId: string, file?: string): void } } +/** Cross-process starting marker (#707): while a launcher sits between spawn + * and healthy, no instance record exists yet, so concurrent launches see + * nothing to attach to and double-spawn — two writers over one sessions dir. + * The starter claims this file with O_EXCL before spawning; concurrent + * callers find it and wait for the bring-up instead of spawning their own. */ +export interface ProxyStartingMarker { + token: string; + pid: number; + host: string; + port: number; + startedAt: number; +} + +export function startingMarkerPath(): string { + return path.join(stateDir(), "proxy-starting"); +} + +export function readStartingMarker(file?: string): ProxyStartingMarker | undefined { + let raw: string; + try { + raw = fs.readFileSync(file ?? startingMarkerPath(), "utf8"); + } catch { + return undefined; + } + try { + const parsed = JSON.parse(raw) as Partial; + if (typeof parsed.token !== "string" || parsed.token === "") return undefined; + if (typeof parsed.pid !== "number" || !Number.isInteger(parsed.pid) || parsed.pid <= 0) return undefined; + if (typeof parsed.startedAt !== "number") return undefined; + return { + token: parsed.token, + pid: parsed.pid, + host: typeof parsed.host === "string" ? parsed.host : "", + port: typeof parsed.port === "number" ? parsed.port : 0, + startedAt: parsed.startedAt, + }; + } catch { + return undefined; + } +} + +/** Claim the marker atomically across processes (O_EXCL create — a rename-based + * write would leave the same race at the marker level). Returns false when a + * claimant already holds it or the state dir is unwritable; coordination then + * degrades to the pre-#707 behavior instead of failing the launch. */ +export function claimStartingMarker(marker: ProxyStartingMarker, file?: string): boolean { + const filePath = file ?? startingMarkerPath(); + let descriptor: number | undefined; + try { + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + descriptor = fs.openSync(filePath, "wx", 0o644); + fs.writeSync(descriptor, JSON.stringify(marker) + "\n", null, "utf8"); + fs.fsyncSync(descriptor); + descriptor = undefined; + return true; + } catch { + if (descriptor !== undefined) { + try { + fs.closeSync(descriptor); + } catch {} + } + return false; + } +} + +/** Remove the marker only if it still carries OUR token (a newer claimant may + * have taken it over). Mirrors clearProxyInstanceFile. */ +export function clearStartingMarker(token: string, file?: string): void { + const filePath = file ?? startingMarkerPath(); + const current = readStartingMarker(filePath); + if (current && current.token === token) { + try { + fs.unlinkSync(filePath); + } catch {} + } +} + +/** Unconditional best-effort removal — used only on markers the caller has + * already verified stale (dead owner / expired). */ +export function removeStartingMarker(file?: string): void { + try { + fs.unlinkSync(file ?? startingMarkerPath()); + } catch {} +} + /** pid liveness (kill-0). EPERM means the process exists but is owned by * another user — still alive. */ export function isPidAlive(pid: number): boolean { diff --git a/src/launcher.ts b/src/launcher.ts index 8eb31df617..f56c072265 100644 --- a/src/launcher.ts +++ b/src/launcher.ts @@ -33,7 +33,17 @@ import path from "node:path"; import { pathToFileURL } from "node:url"; import { spawn, type StdioOptions } from "node:child_process"; import { DEFAULT_MITM_DOMAINS } from "./mitm.js"; -import { isProxyInstanceFile, isPidAlive, readProxyInstanceFile, type ProxyInstanceFile } from "./instance.js"; +import { + claimStartingMarker, + clearStartingMarker, + isPidAlive, + isProxyInstanceFile, + readProxyInstanceFile, + readStartingMarker, + removeStartingMarker, + type ProxyInstanceFile, + type ProxyStartingMarker, +} from "./instance.js"; import { selfPackageRoot, isBiliPiEntry, ompPluginLoadedFrom } from "./plugin-install.js"; /** Absolute path of a file inside our dist/, resolved via the package root @@ -104,6 +114,10 @@ const HEALTH_PATH = "/__bili/health"; const HEALTH_POLL_INTERVAL_MS = 200; const SPAWN_WAIT_MS = 20000; const PROBE_TIMEOUT_MS = 1500; +// #707: max age of a starting marker still treated as an in-progress bring-up. +// A well-behaved starter resolves within SPAWN_WAIT_MS; the slack covers slow +// disks and client teardown before it clears the marker. +const STARTING_MARKER_TTL_MS = SPAWN_WAIT_MS + 30_000; const DEFAULT_MITM_DOMAIN_SET = new Set(DEFAULT_MITM_DOMAINS.map((d) => d.toLowerCase())); @@ -1630,6 +1644,12 @@ async function fetchHealthInfoDefault(origin: string): Promise 0 - ? { BILI_LAUNCHER_MODEL_WINDOWS: JSON.stringify(opts.modelWindows) } - : {}), + let child: SpawnChild; + try { + child = spawnImpl( + process.execPath, + [script, ...proxyStartArgs({ ...opts, port })], + { + detached: true, + stdio: ["ignore", logFd, logFd], + env: { + ...stripInheritedProxy(process.env), + BILI_LAUNCH_TOKEN: launchToken, + BILI_PARENT_PID: String(process.pid), + ...(opts.mitmDomains && opts.mitmDomains.length + ? { BILI_MITM_DOMAINS: opts.mitmDomains.join(",") } + : {}), + ...(opts.modelWindows && Object.keys(opts.modelWindows).length > 0 + ? { BILI_LAUNCHER_MODEL_WINDOWS: JSON.stringify(opts.modelWindows) } + : {}), + }, }, - }, - ); - } finally { + ); + } finally { + try { + fs.closeSync(logFd); + } catch {} + } try { - fs.closeSync(logFd); + child.unref?.(); } catch {} - } - try { - child.unref?.(); - } catch {} - // #401/#480: fail fast when OUR spawned child dies before becoming - // healthy — otherwise a startup crash (bad config, missing upstream, …) - // burns the whole SPAWN_WAIT_MS poll window before erroring. - let childExit: { code: number | null; signal: string | null } | undefined; - child.on?.("exit", (...rest: unknown[]) => { - childExit = { - code: typeof rest[0] === "number" ? rest[0] : null, - signal: typeof rest[1] === "string" ? rest[1] : null, - }; - }); + // #401/#480: fail fast when OUR spawned child dies before becoming + // healthy — otherwise a startup crash (bad config, missing upstream, …) + // burns the whole SPAWN_WAIT_MS poll window before erroring. + let childExit: { code: number | null; signal: string | null } | undefined; + child.on?.("exit", (...rest: unknown[]) => { + childExit = { + code: typeof rest[0] === "number" ? rest[0] : null, + signal: typeof rest[1] === "string" ? rest[1] : null, + }; + }); - const deadline = now() + SPAWN_WAIT_MS; - while (now() < deadline) { - if (childExit) break; - await sleepImpl(HEALTH_POLL_INTERVAL_MS); - const inst = readInstance(); - if (isProxyInstanceFile(inst) && inst.launchToken === launchToken) { - if (await probeHealth(inst.origin, fetchImpl)) { - return { origin: inst.origin, port: inst.port, child, logPath }; + const deadline = now() + SPAWN_WAIT_MS; + while (now() < deadline) { + if (childExit) break; + await sleepImpl(HEALTH_POLL_INTERVAL_MS); + const inst = readInstance(); + if (isProxyInstanceFile(inst) && inst.launchToken === launchToken) { + if (await probeHealth(inst.origin, fetchImpl)) { + return { origin: inst.origin, port: inst.port, child, logPath }; + } + continue; + } + // Fallback for a child that cannot write the instance file (broken + // state dir) or an old pre-handshake binary: only trust the preferred + // origin when NO record vouches for it — a LIVE record's owner owns + // the discovery surface and our child is retry-binding elsewhere. + // A stale record (dead pid / legacy plain) cannot vouch for anything. + const stale = !isProxyInstanceFile(inst) || !isPidAlive(inst.pid); + if (stale && (await probeHealth(proxyOrigin(opts.host, port), fetchImpl))) { + return { origin: proxyOrigin(opts.host, port), port, child, logPath }; } - continue; } - // Fallback for a child that cannot write the instance file (broken - // state dir) or an old pre-handshake binary: only trust the preferred - // origin when NO record vouches for it — a LIVE record's owner owns - // the discovery surface and our child is retry-binding elsewhere. - // A stale record (dead pid / legacy plain) cannot vouch for anything. - const stale = !isProxyInstanceFile(inst) || !isPidAlive(inst.pid); - if (stale && (await probeHealth(proxyOrigin(opts.host, port), fetchImpl))) { - return { origin: proxyOrigin(opts.host, port), port, child, logPath }; + if (childExit) { + const detail = childExit.code !== null + ? `code ${childExit.code}` + : childExit.signal ? `signal ${childExit.signal}` : "unknown reason"; + throw new Error(`bili: proxy child exited before becoming healthy (${detail}) (log: ${logPath})`); } + throw new Error(`bili: proxy did not become healthy within ${SPAWN_WAIT_MS}ms (log: ${logPath})`); + } finally { + clearStartingMarker(launchToken); } - if (childExit) { - const detail = childExit.code !== null - ? `code ${childExit.code}` - : childExit.signal ? `signal ${childExit.signal}` : "unknown reason"; - throw new Error(`bili: proxy child exited before becoming healthy (${detail}) (log: ${logPath})`); - } - throw new Error(`bili: proxy did not become healthy within ${SPAWN_WAIT_MS}ms (log: ${logPath})`); } export function stopProxy(handle: ProxyHandle): void { diff --git a/tests/instance-governance.test.ts b/tests/instance-governance.test.ts index fb6a8afa91..4ec1bad17c 100644 --- a/tests/instance-governance.test.ts +++ b/tests/instance-governance.test.ts @@ -5,12 +5,17 @@ import os from "node:os"; import path from "node:path"; import { atomicWriteInstanceFile, + claimStartingMarker, clearProxyInstanceFile, + clearStartingMarker, instanceFilePath, isPidAlive, isProxyInstanceFile, readProxyInstanceFile, + readStartingMarker, registerInstanceAndWarn, + removeStartingMarker, + startingMarkerPath, unregisterInstance, type ProxyInstanceFile, } from "../src/instance.ts"; @@ -90,6 +95,45 @@ test("clearProxyInstanceFile: only removes its own record", () => { } }); +test("starting marker: claim is exclusive, read validates, clear is token-checked (#707)", () => { + const st = tmpStateDir(); + try { + const m1 = { token: "t1", pid: process.pid, host: "127.0.0.1", port: 8787, startedAt: 1_000_000 }; + assert.equal(claimStartingMarker(m1), true, "first claim wins"); + assert.equal(claimStartingMarker({ ...m1, token: "t2" }), false, "O_EXCL: second claimant rejected"); + assert.deepEqual(readStartingMarker(), m1); + + clearStartingMarker("t2"); + assert.ok(readStartingMarker(), "token mismatch → not removed"); + clearStartingMarker("t1"); + assert.equal(readStartingMarker(), undefined); + assert.equal(claimStartingMarker(m1), true, "claim succeeds again after clear"); + + removeStartingMarker(); + assert.equal(readStartingMarker(), undefined); + } finally { + st.restore(); + } +}); + +test("starting marker: garbage file reads as absent but still blocks a claim (#707)", () => { + const st = tmpStateDir(); + try { + fs.mkdirSync(path.join(st.dir, "billion-context"), { recursive: true }); + fs.writeFileSync(startingMarkerPath(), "{{{garbage"); + assert.equal(readStartingMarker(), undefined); + fs.writeFileSync(startingMarkerPath(), JSON.stringify({ token: "", pid: 1, startedAt: 1 })); + assert.equal(readStartingMarker(), undefined, "empty token invalid"); + assert.equal( + claimStartingMarker({ token: "t", pid: process.pid, host: "h", port: 0, startedAt: 2 }), + false, + "existing file blocks the claim; caller degrades to pre-#707 behavior", + ); + } finally { + st.restore(); + } +}); + test("resolveProxyOrigin: env wins, JSON file parsed, default fallback", () => { const st = tmpStateDir(); const prevEnv = process.env.BILI_MCP_PROXY; diff --git a/tests/launcher.test.ts b/tests/launcher.test.ts index 66686e88a3..686546e482 100644 --- a/tests/launcher.test.ts +++ b/tests/launcher.test.ts @@ -1,4 +1,4 @@ -import { test } from "node:test"; +import { test, after } from "node:test"; import assert from "node:assert/strict"; import fs from "node:fs"; import type { PathLike } from "node:fs"; @@ -6,7 +6,12 @@ type SymlinkKind = "dir" | "file" | "junction"; import net from "node:net"; import os from "node:os"; import path from "node:path"; -import type { ProxyInstanceFile as InstanceFile } from "../src/instance.ts"; +import { + claimStartingMarker, + readStartingMarker, + removeStartingMarker, + type ProxyInstanceFile as InstanceFile, +} from "../src/instance.ts"; import { LAUNCHER_DEFAULT_HOST, isLaunchClient, @@ -80,6 +85,16 @@ import { } from "../src/launcher.ts"; import { _setForTest as registrySetForTest, _resetForTest as registryResetForTest } from "../src/registry.ts"; +// ensureProxyRunning coordinates across processes via /proxy-starting (#707) +// — point the state dir at a throwaway so these tests never touch the real one. +const prevXdgState = process.env.XDG_STATE_HOME; +process.env.XDG_STATE_HOME = fs.mkdtempSync(path.join(os.tmpdir(), "bili-launcher-state-")); +after(() => { + removeStartingMarker(); + if (prevXdgState === undefined) delete process.env.XDG_STATE_HOME; + else process.env.XDG_STATE_HOME = prevXdgState; +}); + test("isLaunchClient: pi/claude/codex/omp/opencode/pi-test true, others false", () => { assert.equal(isLaunchClient("pi"), true); assert.equal(isLaunchClient("claude"), true); @@ -956,6 +971,196 @@ test("ensureProxyRunning: dead recorded pid is ignored (no attach)", async () => assert.equal(spawnCalls, 1); }); +test("ensureProxyRunning: active starting marker → waits, then attaches instead of spawning (#707)", async () => { + try { + claimStartingMarker({ token: "starter-a", pid: process.pid, host: "127.0.0.1", port: 8788, startedAt: Date.now() }); + let reads = 0; + const handle = await ensureProxyRunning( + { host: "127.0.0.1", port: 8787, passthrough: false, debug: false }, + { + spawnImpl: () => { + throw new Error("double-spawn: another launch was still bringing its proxy up"); + }, + fetchImpl: async () => ({ ok: true }), + fetchHealthInfo: async () => ({ ok: true, instanceId: "inst-9" }), + readInstanceFile: () => (reads++ < 2 ? undefined : recordedInstance({ instanceId: "inst-9", origin: "http://127.0.0.1:8788", port: 8788 })), + sleep: () => Promise.resolve(), + }, + ); + assert.equal(handle.attached, true); + assert.equal(handle.origin, "http://127.0.0.1:8788"); + } finally { + removeStartingMarker(); + } +}); + +test("ensureProxyRunning: stale starting marker (dead owner) → removed, then spawns (#707)", async () => { + try { + claimStartingMarker({ token: "starter-b", pid: 99999999, host: "127.0.0.1", port: 8789, startedAt: Date.now() }); + let spawnCalls = 0; + const handle = await ensureProxyRunning( + { host: "127.0.0.1", port: 8787, passthrough: false, debug: false }, + { + spawnImpl: () => { + spawnCalls++; + return makeFakeChild(42451); + }, + fetchImpl: async () => ({ ok: true }), + readInstanceFile: () => undefined, + sleep: () => Promise.resolve(), + }, + ); + assert.equal(spawnCalls, 1); + assert.ok(handle.child); + assert.equal(readStartingMarker(), undefined); + } finally { + removeStartingMarker(); + } +}); + +test("ensureProxyRunning: expired starting marker (hung owner) → spawns (#707)", async () => { + try { + claimStartingMarker({ token: "starter-c", pid: process.pid, host: "127.0.0.1", port: 8789, startedAt: Date.now() - 55_000 }); + let spawnCalls = 0; + const handle = await ensureProxyRunning( + { host: "127.0.0.1", port: 8787, passthrough: false, debug: false }, + { + spawnImpl: () => { + spawnCalls++; + return makeFakeChild(42452); + }, + fetchImpl: async () => ({ ok: true }), + readInstanceFile: () => undefined, + sleep: () => Promise.resolve(), + }, + ); + assert.equal(spawnCalls, 1); + assert.ok(handle.child); + assert.equal(readStartingMarker(), undefined); + } finally { + removeStartingMarker(); + } +}); + +test("ensureProxyRunning: waiter bails early when the starter clears its marker (#707)", async () => { + try { + claimStartingMarker({ token: "starter-d", pid: process.pid, host: "127.0.0.1", port: 8790, startedAt: Date.now() }); + let sleeps = 0; + let spawnCalls = 0; + const t0 = Date.now(); + const handle = await ensureProxyRunning( + { host: "127.0.0.1", port: 8787, passthrough: false, debug: false }, + { + spawnImpl: () => { + spawnCalls++; + return makeFakeChild(42453); + }, + fetchImpl: async () => ({ ok: true }), + readInstanceFile: () => undefined, + sleep: () => { + if (++sleeps === 1) removeStartingMarker(); + return Promise.resolve(); + }, + }, + ); + assert.equal(spawnCalls, 1); + assert.ok(handle.child); + assert.ok(Date.now() - t0 < 2000, `early bail took ${Date.now() - t0}ms; must not burn the full wait window`); + } finally { + removeStartingMarker(); + } +}); + +test("ensureProxyRunning: starter claims marker before spawning and clears it when done (#707)", async () => { + try { + let childToken = ""; + let markerTokenAtSpawn: string | undefined; + const spawnImpl: SpawnFn = (_cmd, _args, options) => { + childToken = (options.env?.BILI_LAUNCH_TOKEN as string) ?? ""; + markerTokenAtSpawn = readStartingMarker()?.token; + return makeFakeChild(42454); + }; + const handle = await ensureProxyRunning( + { host: "127.0.0.1", port: 8787, passthrough: false, debug: false }, + { + spawnImpl, + fetchImpl: async () => ({ ok: true }), + readInstanceFile: () => (childToken ? recordedInstance({ launchToken: childToken }) : undefined), + sleep: () => new Promise((r) => setTimeout(r, 0)), + }, + ); + assert.ok(childToken.length > 0); + assert.equal(markerTokenAtSpawn, childToken); + assert.ok(handle.child); + assert.equal(readStartingMarker(), undefined); + } finally { + removeStartingMarker(); + } +}); + +test("ensureProxyRunning: starter clears marker when the child exits pre-bind (#707)", async () => { + try { + const child: SpawnChild = { + pid: 42455, + unref() {}, + kill() { + return true; + }, + on(event, listener) { + if (event === "exit") setImmediate(() => listener(1, null)); + return undefined; + }, + }; + await assert.rejects( + ensureProxyRunning( + { host: "127.0.0.1", port: 8787, passthrough: false, debug: false }, + { + spawnImpl: () => child, + fetchImpl: async () => ({ ok: false }), + fetchHealthInfo: async () => undefined, + readInstanceFile: () => undefined, + sleep: () => new Promise((r) => setTimeout(r, 1)), + }, + ), + /exited before becoming healthy/, + ); + assert.equal(readStartingMarker(), undefined); + } finally { + removeStartingMarker(); + } +}); + +test("ensureProxyRunning: incompatible bring-up (modelWindows) is not attached — spawns anyway (#707)", async () => { + try { + claimStartingMarker({ token: "starter-g", pid: process.pid, host: "127.0.0.1", port: 8791, startedAt: Date.now() }); + let spawnCalls = 0; + let ticks = 0; + await assert.rejects( + ensureProxyRunning( + { host: "127.0.0.1", port: 8787, passthrough: false, debug: false, modelWindows: { m1: 100000 } }, + { + spawnImpl: () => { + spawnCalls++; + return makeFakeChild(42456); + }, + fetchImpl: async () => ({ ok: false }), + fetchHealthInfo: async () => ({ ok: true, instanceId: "inst-1" }), + readInstanceFile: () => recordedInstance(), + now: () => ticks * 1000, + sleep: () => { + ticks += 10; + return Promise.resolve(); + }, + }, + ), + /did not become healthy/, + ); + assert.equal(spawnCalls, 1); + } finally { + removeStartingMarker(); + } +}); + test("ensureProxyRunning: port 0 (no explicit --port) spawns on an OS-assigned ephemeral port (#446)", async () => { let spawnedArgs: string[] | null = null; const spawnImpl: SpawnFn = (_cmd, args) => { From 61ae4fd319fa0e224691c197c9f304c0a8dca233 Mon Sep 17 00:00:00 2001 From: ework-agent Date: Fri, 11 Sep 2026 22:19:50 +0800 Subject: [PATCH 2/2] =?UTF-8?q?fix(launcher):=20make=20the=20O=5FEXCL=20cl?= =?UTF-8?q?aim=20the=20arbiter=20=E2=80=94=20close=20read=E2=86=92claim=20?= =?UTF-8?q?TOCTOU=20and=20deadline-boundary=20sliver=20(#707=20review)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two residual double-spawn windows in the starting-marker protocol, found in review: 1. The pre-spawn readStartingMarker() was treated as the coordination decision while the O_EXCL claim result was ignored: two processes that both read 'no marker' before either claimed would each spawn (the loser's EEXIST was dropped on the floor). The default port-0 path widens the window with await pickEphemeralPort() between read and claim. Now a claim loser re-reads the marker: active holder → wait for it (same probe as attach); stale/garbage file → remove and re-claim once; still losing → degrade to pre-#707 spawn. This also self-heals an unreadable marker file (crash mid-write), which previously blocked every future claim forever. 2. The wait loop's deadline coincides with the starter's own poll deadline: a starter succeeding in its last ~200ms could be missed by the waiter's final poll, which then fell through to spawning a duplicate. The success path clears the marker, so the miss is indistinguishable from a failure bail without one more look — waitForStarterInstance now ends with a final probe after the loop exits. Tests: garbage-marker self-heal (fails on the pre-review code), deadline-boundary attach instead of double-spawn (fails on the pre-review code), hung-starter bounded-wait pin. Preflight: typecheck clean, 1379 pass / 0 fail / 2 skipped, build ok. --- CHANGELOG.md | 2 +- src/launcher.ts | 84 ++++++++++++++++++++++++++++---------- tests/launcher.test.ts | 92 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 156 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a8f79581bb..688dca312a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ Versions follow the merge of a `*_release-v*` branch; CI publishes to npm on tag ### Fixes -- **Cross-process startup-window race: concurrent launches no longer double-spawn proxies (#707)**: `ensureProxyRunning` only attached when it found a *healthy* instance at probe time — between the launcher's spawn and the child's first successful listen there was zero on-disk evidence that a bring-up was in flight, so two launches inside that window (~1s+) each spawned their own child: two live proxies over one sessions dir (double writers), last-writer-wins instance file, and the extra proxy living until ITS OWN spawner exited. The starter now claims `/proxy-starting` with O_EXCL before spawning (carrying `{token, pid, host, port, startedAt}`); a concurrent caller that sees an active marker (owner pid alive AND younger than SPAWN_WAIT_MS + 30s) waits for the starter's instance with the same probe as attach (pid alive + `/__bili/health` + instanceId match), bounded by SPAWN_WAIT_MS, and attaches on success; marker disappearance (starter gave up / crashed) breaks the wait early so the caller spawns immediately. The starter clears the marker on every terminal path via a token-checked clear; a hard crash leaves a stale marker that the dead-owner/TTL checks render inert. Incompatible-config fall-through is preserved (a waited-for instance that isn't compatible with our launch options still spawns, matching today's incompatible-healthy-instance behavior); explicit `bili start` bypasses the marker entirely (it's a user action, not auto-bootstrap). In-process dedup (`singleFlight`, #706) is untouched — this is the cross-process half. +- **Cross-process startup-window race: concurrent launches no longer double-spawn proxies (#707)**: `ensureProxyRunning` only attached when it found a *healthy* instance at probe time — between the launcher's spawn and the child's first successful listen there was zero on-disk evidence that a bring-up was in flight, so two launches inside that window (~1s+) each spawned their own child: two live proxies over one sessions dir (double writers), last-writer-wins instance file, and the extra proxy living until ITS OWN spawner exited. The starter now claims `/proxy-starting` with O_EXCL before spawning (carrying `{token, pid, host, port, startedAt}`); a concurrent caller that sees an active marker (owner pid alive AND younger than SPAWN_WAIT_MS + 30s) waits for the starter's instance with the same probe as attach (pid alive + `/__bili/health` + instanceId match), bounded by SPAWN_WAIT_MS, and attaches on success; marker disappearance (starter gave up / crashed) breaks the wait early so the caller spawns immediately. The starter clears the marker on every terminal path via a token-checked clear; a hard crash leaves a stale marker that the dead-owner/TTL checks render inert. The O_EXCL claim itself is the cross-process arbiter: the pre-spawn read is only a fast path, so a loser of the claim (read→claim race, or a stale/garbage marker file left by a crash mid-write) re-checks and either waits for the live holder or removes-and-reclaims the slot — it never spawns blindly, and an unreadable marker file self-heals on the next launch instead of permanently degrading coordination. A final probe after the wait deadline catches a starter that succeeded in its last moments — its success path clears the marker, which otherwise looks identical to a failure bail. Incompatible-config fall-through is preserved (a waited-for instance that isn't compatible with our launch options still spawns, matching today's incompatible-healthy-instance behavior); explicit `bili start` bypasses the marker entirely (it's a user action, not auto-bootstrap). In-process dedup (`singleFlight`, #706) is untouched — this is the cross-process half. - **DeepSeek thinking mode 400 fixed: turns are atomic, reasoning-drop self-disables on strict-echo upstreams (#684)**: two paths could ship a rebuilt request whose assistant `tool_calls` turn lost its `reasoning_content` — (1) the proxy's recent-zone carve kept the last tool-call of a multi-call turn visible while its reasoning run folded into the block (kernel boundary logic only pulled the FIRST companion message after reasoning), and (2) #651's reasoning-drop stripped reasoning from a closed compress turn even on upstreams that document thinking-mode `reasoning_content` as mandatory round-trip. DeepSeek rejects both with 400 (`The \`reasoning_content\` in the thinking mode must be passed back to the API`) and the turn dies. Fixes: the kernel now computes atomic turn groups (reasoning run + its full assistant burst + every result paired by `toolCallId` — all fold together or none do; a range that would split a turn is withdrawn with a warning, or rejected with an explanatory error when nothing else remains), the forward boundary pull covers the whole burst, and the reasoning-drop never fires when the session is known strict-echo — statically for `deepseek` upstreams, or learned from the first 400 whose body mentions `reasoning_content` (persisted on the session, self-healing like #552). Three wire-level exit sentinels (`[reasoning-pair-violated]` on chat/completions and Responses, `[thinking-pair-violated]` on Anthropic) log loudly if any future path reintroduces a split turn. - **Windows: spaced client/extension paths no longer break launched clients or silently drop extensions (#679, PR #682)**: `runClient` spawned clients with `shell: true` and no per-token quoting, so on Windows cmd.exe re-split the joined line at the first space — a client binary under a spaced path (`C:\Users\John Doe\...`) died with `'C:\Users\John' is not recognized`, and a surviving client whose `-e ` path was split launched WITHOUT the bili extension (compression silently never activates). Spawning is now routed by executable class: real `.exe`/node binaries spawn directly (the OS quotes argv, spaces intact), npm `.cmd`/`.bat`/extensionless shims go through an explicit `comspec /d /s /c "…"` line with correct per-token quoting (`windowsVerbatimArguments`); `shell: true` is gone (also silences Node's DEP0190 warning). Known edge: `BILI_CLIENT_BIN` pointing at a `.js` file now fails loudly instead of relying on file association via the old shell path. - **Windows: codex MCP server delivered via CODEX_HOME overlay, not inline `-c` (#681, PR #683)**: cmd.exe cannot transmit a literal `"` to a child, but every valid TOML string needs one — so the inline `-c mcp_servers.bili.command="C:\\Program Files\\nodejs\\node.exe"` arg arrived at codex as an unquoted, space-split, invalid TOML value for ANY absolute Windows path (spaced stock install just the common case), the bili MCP server never registered, and plugin-mode compression silently never activated. On `win32` the launcher now writes `[mcp_servers.bili]` (command/args/env.BILI_MCP_PROXY/env.BILI_CONVERSATION_ID) into a persistent `-bili` overlay's merged `config.toml` — sharing every other real-home entry (auth.json, sessions, model settings) — and points the spawned codex at it via `CODEX_HOME`. POSIX keeps the working inline `-c` path byte-identical; if the overlay can't be built the launch degrades to wire mode (server-side compression still active) with a warning. Real-Windows end-to-end verification tracked in #686. diff --git a/src/launcher.ts b/src/launcher.ts index f56c072265..6133015a71 100644 --- a/src/launcher.ts +++ b/src/launcher.ts @@ -1673,6 +1673,30 @@ async function probeExistingInstance( return inst; } +/** #707: wait for another launcher's in-flight bring-up to produce a live + * instance. Bounded by SPAWN_WAIT_MS; breaks early when the starting marker + * disappears (starter gave up / crashed). The final probe closes the + * deadline-boundary sliver: the starter's own poll window ends ~now, and its + * success path clears the marker — indistinguishable from a failure bail + * without one last look. */ +async function waitForStarterInstance( + readInstance: () => ProxyInstanceFile | { origin: string } | undefined, + fetchHealthInfo: (origin: string) => Promise, + now: () => number, + sleepImpl: (ms: number) => Promise, +): Promise { + const deadline = now() + SPAWN_WAIT_MS; + let inst: ProxyInstanceFile | undefined; + while (now() < deadline) { + await sleepImpl(HEALTH_POLL_INTERVAL_MS); + inst = await probeExistingInstance(readInstance, fetchHealthInfo); + if (inst) break; + const still = readStartingMarker(); + if (!still || !isStartingMarkerActive(still, now())) break; + } + return inst ?? (await probeExistingInstance(readInstance, fetchHealthInfo)); +} + export function findFreePort(preferred: number, host = LAUNCHER_DEFAULT_HOST): Promise { const tryBind = (port: number): Promise => new Promise((resolve) => { @@ -1760,27 +1784,24 @@ export async function ensureProxyRunning( // the attach above saw nothing). Wait for ITS instance instead of spawning // a second writer over the same sessions dir. In-process dedup is separate // (singleFlight, #706); this is the cross-process half. + const waitForOtherStarter = async (): Promise => { + console.error("bili: another bili launch is bringing up a proxy — waiting for it instead of spawning a second"); + const waited = await waitForStarterInstance(readInstance, fetchHealthInfo, now, sleepImpl); + if (waited && instanceCompatible(waited, opts)) { + console.error(`bili: attaching to running proxy at ${waited.origin} (pid ${waited.pid})`); + return { origin: waited.origin, port: waited.port, attached: true }; + } + // starter failed/timed out (or incompatible config) — caller falls + // through and spawns itself, as before + return undefined; + }; const marker = readStartingMarker(); if (marker) { if (!isStartingMarkerActive(marker, now())) { removeStartingMarker(); } else { - console.error("bili: another bili launch is bringing up a proxy — waiting for it instead of spawning a second"); - const waitedDeadline = now() + SPAWN_WAIT_MS; - let waited: ProxyInstanceFile | undefined; - while (now() < waitedDeadline) { - await sleepImpl(HEALTH_POLL_INTERVAL_MS); - waited = await probeExistingInstance(readInstance, fetchHealthInfo); - if (waited) break; - const still = readStartingMarker(); - if (!still || !isStartingMarkerActive(still, now())) break; - } - if (waited && instanceCompatible(waited, opts)) { - console.error(`bili: attaching to running proxy at ${waited.origin} (pid ${waited.pid})`); - return { origin: waited.origin, port: waited.port, attached: true }; - } - // starter failed/timed out (or incompatible config) — fall through - // and spawn ourselves, as before + const attached = await waitForOtherStarter(); + if (attached) return attached; } } @@ -1798,10 +1819,31 @@ export async function ensureProxyRunning( const logPath = path.join(os.tmpdir(), `bili-proxy-${port}.log`); const logFd = fs.openSync(logPath, "a"); // #707: publish the starting marker BEFORE spawning so concurrent launches - // wait for this bring-up instead of double-spawning. Cleared on every - // terminal path below; a hard crash leaves a stale marker that the - // dead-owner/TTL check treats as inert. - claimStartingMarker({ token: launchToken, pid: process.pid, host: opts.host, port, startedAt: now() }); + // wait for this bring-up instead of double-spawning. The O_EXCL claim is + // the cross-process arbiter: the read above is only a fast path, so a + // loser of the claim must re-check and wait instead of spawning blindly. + // Cleared on every terminal path below; a hard crash leaves a stale marker + // that the dead-owner/TTL check treats as inert. + const claimMarker = (): boolean => + claimStartingMarker({ token: launchToken, pid: process.pid, host: opts.host, port, startedAt: now() }); + let claimed = claimMarker(); + if (!claimed) { + const holder = readStartingMarker(); + if (holder && isStartingMarkerActive(holder, now())) { + // Lost the read→claim race to a live starter — honor its bring-up. + const attached = await waitForOtherStarter(); + if (attached) return attached; + claimed = claimMarker(); + } else { + // Stale or unreadable (crash mid-write): safe to remove — while any + // marker file exists, O_EXCL bars a newer claimant, so we cannot + // clobber a live coordinator. Retry once to take the slot. + removeStartingMarker(); + claimed = claimMarker(); + } + // Still unclaimed (unwritable state dir, or lost the retry race): + // degrade to pre-#707 behavior — spawn without coordinating. + } try { let child: SpawnChild; try { @@ -1873,7 +1915,7 @@ export async function ensureProxyRunning( } throw new Error(`bili: proxy did not become healthy within ${SPAWN_WAIT_MS}ms (log: ${logPath})`); } finally { - clearStartingMarker(launchToken); + if (claimed) clearStartingMarker(launchToken); } } diff --git a/tests/launcher.test.ts b/tests/launcher.test.ts index 686546e482..5753b5bb46 100644 --- a/tests/launcher.test.ts +++ b/tests/launcher.test.ts @@ -10,6 +10,7 @@ import { claimStartingMarker, readStartingMarker, removeStartingMarker, + startingMarkerPath, type ProxyInstanceFile as InstanceFile, } from "../src/instance.ts"; import { @@ -1161,6 +1162,97 @@ test("ensureProxyRunning: incompatible bring-up (modelWindows) is not attached } }); +test("ensureProxyRunning: unreadable starting marker is self-healed — removed, slot re-claimed (#707)", async () => { + try { + fs.writeFileSync(startingMarkerPath(), "{{{garbage"); + let spawnCalls = 0; + let childToken = ""; + let markerTokenAtSpawn: string | undefined; + const spawnImpl: SpawnFn = (_cmd, _args, options) => { + spawnCalls++; + childToken = (options.env?.BILI_LAUNCH_TOKEN as string) ?? ""; + markerTokenAtSpawn = readStartingMarker()?.token; + return makeFakeChild(42460); + }; + const handle = await ensureProxyRunning( + { host: "127.0.0.1", port: 8787, passthrough: false, debug: false }, + { + spawnImpl, + fetchImpl: async () => ({ ok: true }), + readInstanceFile: () => (childToken ? recordedInstance({ launchToken: childToken }) : undefined), + sleep: () => new Promise((r) => setTimeout(r, 0)), + }, + ); + assert.equal(spawnCalls, 1); + assert.ok(handle.child); + assert.equal(markerTokenAtSpawn, childToken, "re-claim after garbage removal carries our token"); + assert.equal(readStartingMarker(), undefined, "marker cleared after success"); + } finally { + removeStartingMarker(); + } +}); + +test("ensureProxyRunning: instance appearing at the wait deadline is attached, not double-spawned (#707)", async () => { + try { + claimStartingMarker({ token: "starter-h", pid: process.pid, host: "127.0.0.1", port: 8792, startedAt: Date.now() }); + let ticks = 0; + let reads = 0; + const handle = await ensureProxyRunning( + { host: "127.0.0.1", port: 8787, passthrough: false, debug: false }, + { + spawnImpl: () => { + throw new Error("double-spawn: instance appeared at the deadline"); + }, + fetchImpl: async () => ({ ok: true }), + fetchHealthInfo: async () => ({ ok: true, instanceId: "inst-b" }), + readInstanceFile: () => + reads++ < 3 ? undefined : recordedInstance({ instanceId: "inst-b", origin: "http://127.0.0.1:8792", port: 8792 }), + now: () => ticks * 1000, + sleep: () => { + ticks += 10; + return Promise.resolve(); + }, + }, + ); + assert.equal(handle.attached, true); + assert.equal(handle.origin, "http://127.0.0.1:8792"); + } finally { + removeStartingMarker(); + } +}); + +test("ensureProxyRunning: hung starter (marker never clears) is bounded — waits max twice, then spawns (#707)", async () => { + try { + claimStartingMarker({ token: "starter-j", pid: process.pid, host: "127.0.0.1", port: 8793, startedAt: Date.now() }); + let spawnCalls = 0; + let ticks = 0; + await assert.rejects( + ensureProxyRunning( + { host: "127.0.0.1", port: 8787, passthrough: false, debug: false }, + { + spawnImpl: () => { + spawnCalls++; + return makeFakeChild(42461); + }, + fetchImpl: async () => ({ ok: false }), + fetchHealthInfo: async () => undefined, + readInstanceFile: () => undefined, + now: () => ticks * 1000, + sleep: () => { + ticks += 10; + return Promise.resolve(); + }, + }, + ), + /did not become healthy/, + ); + assert.equal(spawnCalls, 1, "bounded waits end in a spawn attempt, never an infinite loop"); + assert.equal(readStartingMarker()?.token, "starter-j", "foreign marker left untouched"); + } finally { + removeStartingMarker(); + } +}); + test("ensureProxyRunning: port 0 (no explicit --port) spawns on an OS-assigned ephemeral port (#446)", async () => { let spawnedArgs: string[] | null = null; const spawnImpl: SpawnFn = (_cmd, args) => {