Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions reference/architecture.md
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ billion-context/
│ ├── knobs.ts # Single knob resolver: env > config file > default (#2030)
│ ├── config.ts # Config file (billion-context.json) + providers table + loadOptions
│ ├── instance.ts # Live-instance registry record in the proxy-origin file (#394/#403/#417)
│ ├── port-owner.ts # Post-bind port-ownership verification + pointer-clobber guard (#2941)
│ ├── paths.ts # XDG paths (config/cache/state) + relocation env vars
│ ├── logger.ts # Tee logger: state-dir file + stderr
│ ├── log-mask.ts # Log safety: masks credentials/endpoints (#255)
Expand Down
179 changes: 179 additions & 0 deletions src/port-owner.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,179 @@
import { isProxyInstanceFile, type ProxyInstanceFile } from "./instance.js";

/** #2941: post-bind port-ownership verification + pointer-clobber guard for
* same-lane double-spawn residuals (#707 follow-ups). Conservative by
* design: positive evidence only — never yield/refuse on ambiguity. */

interface OwnerProbeInfo {
pid?: number;
instanceId?: string;
/** Negative attribution: responder is up but pre-registration (#2941
* readiness gate) — see findForeignOwner for the handling. */
reason?: string;
}

/** Structural fetch shape — global `fetch` satisfies it; tests inject bare objects. */
export type HealthFetcher = (url: string, init?: RequestInit) => Promise<{ ok: boolean; json(): Promise<unknown> }>;

interface ProbeDeps {
fetchImpl?: HealthFetcher;
sleepImpl?: (ms: number) => Promise<void>;
}

const DEFAULT_ATTEMPTS = 3;
const DEFAULT_INTERVAL_MS = 100;
const DEFAULT_TIMEOUT_MS = 1_500;

// #2941/#1529: a default fetch socket lingers ~4s (pooled keep-alive) AFTER
// the post-bind self-probe completes, sitting in OUR OWN conn ledger and
// inflating [exposure] liveConns in exactly the window the #1529
// hold-until-peer-FIN contract samples it (liveConns=2 instead of 1).
// `connection: close` makes every probe a dedicated socket that dies with
// its response — verified: immediate server-side close vs ~3.8s linger.
function probeFetch(url: string, init?: RequestInit): Promise<Response> {
const headers = new Headers(init?.headers);
headers.set("connection", "close");
return fetch(url, { ...init, headers });
}

function healthUrl(origin: string): string {
return origin.replace(/\/+$/, "") + "/__bili/health";
}

/** One /__bili/health read. undefined when dead/non-JSON/not-ok — absence of
* evidence is not evidence of a foreign owner. */
export async function probeOwnerInfo(
origin: string,
fetchImpl: HealthFetcher = probeFetch,
timeoutMs: number = DEFAULT_TIMEOUT_MS,
): Promise<OwnerProbeInfo | undefined> {
try {
const res = await fetchImpl(healthUrl(origin), { signal: AbortSignal.timeout(timeoutMs) });
if (!res.ok) {
// #2941: our own pre-registration 503 carries reason:"starting" —
// surface it so findForeignOwner self-attributes instead of burning
// its probe budget on itself.
const neg = (await res.json()) as { ok?: unknown; reason?: unknown };
if (neg.ok === false && typeof neg.reason === "string") return { reason: neg.reason };
return undefined;
}
const data = (await res.json()) as { ok?: unknown; pid?: unknown; instanceId?: unknown };
if (data.ok !== true) return undefined;
const info: OwnerProbeInfo = {};
if (typeof data.pid === "number" && Number.isInteger(data.pid) && data.pid > 0) info.pid = data.pid;
if (typeof data.instanceId === "string" && data.instanceId !== "") info.instanceId = data.instanceId;
return info;
} catch {
return undefined;
}
}

/** #2941: foreign live listener co-listening on our just-bound port? Multiple
* probes: dual-bind-tolerant platforms may distribute connections among the
* listeners, so one round-trip can hit our own socket. A responder WITHOUT a
* pid field (pre-#1753 builds) is unattributable — yielding on it would
* ratchet sticky ports during upgrade overlaps (#1723). */
export async function findForeignOwner(
origin: string,
ownPid: number,
opts: { attempts?: number; intervalMs?: number; timeoutMs?: number } & ProbeDeps = {},
): Promise<OwnerProbeInfo | undefined> {
const attempts = Math.max(1, opts.attempts ?? DEFAULT_ATTEMPTS);
const intervalMs = opts.intervalMs ?? DEFAULT_INTERVAL_MS;
const timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS;
const fetchImpl = opts.fetchImpl ?? probeFetch;
const sleepImpl = opts.sleepImpl ?? ((ms: number) => new Promise((r) => setTimeout(r, ms)));
for (let i = 0; i < attempts; i++) {
if (i > 0) await sleepImpl(intervalMs);
const info = await probeOwnerInfo(origin, fetchImpl, timeoutMs);
if (info?.pid !== undefined && info.pid !== ownPid) return info;
// #2941: a "starting" responder on our just-bound port is ourselves
// mid-announce — /__bili/health gates until registration completes, so
// without this every boot burned the full probe budget (~200ms+) on its
// own pre-registration 503s and delayed pointer/registry publication
// past callers that expect registration right after bind (#394 warn
// timing in multi-instance-live). Bind is exclusive on every platform
// bili ships on (Linux EADDRINUSE; Windows per libuv issue #1360), so no
// foreign listener can be in that state here; a hypothetical dual-bind
// co-starter would be unregistered anyway — the registry/pointer layer
// surfaces the collision at registration time. Never foreign.
if (info?.reason === "starting") return undefined;
}
return undefined;
}

type PortClaimOutcome =
| { kind: "claimed" }
| { kind: "ladder"; nextPort: number; foreign: OwnerProbeInfo }
| { kind: "die"; message: string };

function portOf(origin: string): number {
try {
return Number(new URL(origin).port || 80);
} catch {
return 0;
}
}

/** #2941: what to do with a bound port whose ownership check found a foreign
* listener — mirrors the EADDRINUSE ladder contract (#407/#964): late binder
* moves +1, ephemeral after the attempt budget, strict-port/manual fail fast. */
export async function resolvePortClaim(params: {
origin: string;
ownPid: number;
/** launchToken && !strictPort — the retry contract exists only for
* launcher-spawned children (#407); manual `bili start` stays fail-fast. */
canLadder: boolean;
listenAttempts: number;
maxListenAttempts: number;
probe?: { attempts?: number; intervalMs?: number; timeoutMs?: number } & ProbeDeps;
}): Promise<PortClaimOutcome> {
const foreign = await findForeignOwner(params.origin, params.ownPid, params.probe);
if (foreign === undefined) return { kind: "claimed" };
const who = `another live proxy (pid ${foreign.pid}${foreign.instanceId !== undefined ? `, id ${foreign.instanceId.slice(0, 8)}` : ""})`;
if (!params.canLadder) {
return { kind: "die", message: `port ${portOf(params.origin)} is shared with ${who} — refusing to serve a contested port (#2941)` };
}
const attempts = params.listenAttempts + 1;
const nextPort = attempts >= params.maxListenAttempts ? 0 : portOf(params.origin) + 1;
return { kind: "ladder", nextPort, foreign };
}

interface PointerGuardResult {
allow: boolean;
holder?: { pid: number; instanceId: string; origin: string };
}

/** #2941: proxy-origin pointer overwrite identity check. Refuses ONLY a
* different live instance whose recorded pid health-checks as serving the
* SAME address. Everything else allows the write: missing/legacy records,
* dead pids, draining predecessors (alive process, closed listener — must
* keep the #1723 takeover working), responder-pid mismatch (stale record —
* overwriting repairs it), cross-address holders (last-writer-wins by
* design, #1232). */
export async function mayClobberPointer(params: {
candidateOrigin: string;
candidateInstanceId: string;
read: () => ProxyInstanceFile | { origin: string } | undefined;
isAlive: (pid: number) => boolean;
healthOf: (origin: string) => Promise<OwnerProbeInfo | undefined>;
}): Promise<PointerGuardResult> {
const current = params.read();
if (!isProxyInstanceFile(current)) return { allow: true };
if (current.instanceId === params.candidateInstanceId) return { allow: true };
if (!sameAddress(current.origin, params.candidateOrigin)) return { allow: true };
if (current.pid <= 0 || !params.isAlive(current.pid)) return { allow: true };
const health = await params.healthOf(current.origin);
if (health !== undefined && health.pid === current.pid) {
return { allow: false, holder: { pid: current.pid, instanceId: current.instanceId, origin: current.origin } };
}
return { allow: true };
}

function sameAddress(a: string, b: string): boolean {
try {
return new URL(a).host === new URL(b).host;
} catch {
return false;
}
}
103 changes: 91 additions & 12 deletions src/server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,8 @@ import { getStore } from "./persist.js";
import { log as loggerLog, configureLogger, getLogPath, closeLogger, isFileLoggingActive, isStreamWriteError, isBenignSocketRaceError, enterSessionContext, muteStderrLeg } from "./logger.js";
import { queryLogLines } from "./web/logs-query.js";
import { configFile, defaultLogFile, stateDir } from "./paths.js";
import { atomicWriteInstanceFile, clearProxyInstanceFile, entryScriptFingerprint, findSameLanePredecessor, isPidAlive, listInstances, registerInstanceAndWarn, unregisterInstance, warnOnNewPeers, type ProxyInstanceFile } from "./instance.js";
import { atomicWriteInstanceFile, clearProxyInstanceFile, entryScriptFingerprint, findSameLanePredecessor, isPidAlive, listInstances, readProxyInstanceFile, registerInstanceAndWarn, unregisterInstance, warnOnNewPeers, type ProxyInstanceFile } from "./instance.js";
import { mayClobberPointer, probeOwnerInfo, resolvePortClaim } from "./port-owner.js";
import { hoistTrappedToolItems } from "./tool-pair-order.js";
import { computeAnthropicMessageMarks, stampAnthropicSystemCacheControl, anthropicToolsCarryCacheControl } from "./loop/cache-control.js";
import { reconcileSystemAnchor } from "./system-anchor.js";
Expand Down Expand Up @@ -422,6 +423,13 @@ export async function startServer(opts: ProxyOptions): Promise<http.Server> {
// (the chain check only compares against the other running instance).
const instanceId = randomUUID();
const instanceStartedAt = Date.now();
// #2941: the bind happens BEFORE announceListening writes this instance's
// identity files (pointer + registry entry), and the health route is
// stateless — so between bind and registration a probe sees "healthy" for
// an instance no discovery pass can find, and a concurrent same-lane
// launcher spawns a duplicate instead of attaching or waiting. The health
// endpoint reports not-ready until registration completes.
let bootReady = false;
// #7 (shared stable-port proxy): the parent-gone watchdog watches a SET of
// pids, not one. The spawning hook seeds it via BILI_PARENT_PID; every
// ATTACHING session (POST /__bili/watcher) adds its claude host, so a
Expand Down Expand Up @@ -462,7 +470,7 @@ export async function startServer(opts: ProxyOptions): Promise<http.Server> {
// as the context-window source for zero-config `/p/` routes that have no
// per-model config. A miss falls back to the prefix table + default.
void loadRegistry();
const adminCtx = { opts, core, config, log, instanceId, instanceStartedAt, proxyWatchers, initialWatcherPid };
const adminCtx = { opts, core, config, log, instanceId, instanceStartedAt, proxyWatchers, initialWatcherPid, bootReady: () => bootReady };

const dispatch = async (req: http.IncomingMessage, res: http.ServerResponse): Promise<void> => {
armRequestWatchdog(req, res, log);
Expand Down Expand Up @@ -885,19 +893,26 @@ export async function startServer(opts: ProxyOptions): Promise<http.Server> {
}
return pred !== undefined;
};
const announceListening = (): void => {
const actualPort = server.address() === null ? opts.port : (server.address() as { port: number }).port;
// Discovery origin local MCP shells dial: collapse wildcard
// binds to loopback (localhost may resolve to ::1, where an
// IPv4-only listener is absent) and bracket bare IPv6 literals
// so the file always holds a valid URL.
const boundOrigin = (actualPort: number): string => {
const originHost = opts.host === "0.0.0.0" || opts.host === "::" || opts.host === "localhost" ? "127.0.0.1" : opts.host.includes(":") && !opts.host.startsWith("[") ? `[${opts.host}]` : opts.host;
return `http://${originHost}:${actualPort}`;
};
const announceListening = async (): Promise<void> => {
// #2941: closed between bind and announce (fast exit, test teardown) —
// announcing would register a dead listener under the configured port
// (port 0 → bogus http://host:0 banners/registry records); skip entirely.
if (server.address() === null) return;
const actualPort = (server.address() as { port: number }).port;
const nonLoopbackBind = opts.host === "0.0.0.0" || opts.host === "::" || !isLoopbackAddress(opts.host);
// Honest bind display: a wildcard bind shows as 0.0.0.0 (the user
// chose to expose the proxy — hiding it behind "localhost" made
// remote setups look broken in the log, see #240).
const displayHost = nonLoopbackBind ? opts.host : opts.host === "0.0.0.0" ? "localhost" : opts.host;
// Discovery origin local MCP shells dial: collapse wildcard
// binds to loopback (localhost may resolve to ::1, where an
// IPv4-only listener is absent) and bracket bare IPv6 literals
// so the file always holds a valid URL.
const originHost = opts.host === "0.0.0.0" || opts.host === "::" || opts.host === "localhost" ? "127.0.0.1" : opts.host.includes(":") && !opts.host.startsWith("[") ? `[${opts.host}]` : opts.host;
const origin = `http://${originHost}:${actualPort}`;
const origin = boundOrigin(actualPort);
const instanceRecord: ProxyInstanceFile = {
origin,
instanceId,
Expand All @@ -920,7 +935,21 @@ export async function startServer(opts: ProxyOptions): Promise<http.Server> {
// restores the pre-#2265 exact-hash-only resolution (hot kill-switch).
setSimhashAdoptionEnabled(opts.affinitySimhash ?? true);
hydratePrefixAffinity();
atomicWriteInstanceFile(instanceRecord);
// #2941: never clobber a live pointer on this port — a foreign
// instance health-verified as serving here keeps it; our registry
// entry below still records us for lane-aware discovery.
const guard = await mayClobberPointer({
candidateOrigin: origin,
candidateInstanceId: instanceId,
read: readProxyInstanceFile,
isAlive: isPidAlive,
healthOf: probeOwnerInfo,
});
if (guard.allow) {
atomicWriteInstanceFile(instanceRecord);
} else if (guard.holder !== undefined) {
log("warn", `[pointer] keeping proxy-origin held by live instance ${guard.holder.instanceId.slice(0, 8)} (pid ${guard.holder.pid}, ${guard.holder.origin}) on this port — refusing to clobber a live pointer (#2941)`);
}
} catch {
// best-effort discovery hint for host-spawned MCP shells
}
Expand Down Expand Up @@ -1011,10 +1040,60 @@ export async function startServer(opts: ProxyOptions): Promise<http.Server> {
if (opts.debug) {
log("info", `[debug] build features: raw-HTTP-capture(${bodyDumpEnabled() ? "on" : "off"}) | remote_compaction_v2-strip(on) | cert-MITM-launcher(on) | strip-acp-summary(on) — seeing this line confirms the launcher build (not registry 0.1.34)`);
}
// #2941: LAST statement — registration above is done; degraded boots
// (best-effort write failed) still flip ready, serving as before.
bootReady = true;
};
// #2941: after a successful bind, verify we are the SOLE owner of the
// port before registering. Where socket semantics tolerate a concurrent
// bind onto an actively-listening port, a same-lane double spawn would
// otherwise serve ONE port from two state machines silently; stock libuv
// enforces exclusivity (EADDRINUSE ladder below), so this is a defensive
// second layer — see src/port-owner.ts.
const claimOrAnnounce = (): void => {
void (async () => {
const addr = server.address();
if (addr !== null && typeof addr === "object") {
const outcome = await resolvePortClaim({
origin: boundOrigin(addr.port),
ownPid: process.pid,
canLadder: Boolean(launchToken) && !strictPort,
listenAttempts,
maxListenAttempts: MAX_LISTEN_ATTEMPTS,
});
if (outcome.kind === "ladder") {
listenAttempts += 1;
log("warn", `port ${lastTriedPort} shared with foreign listener (pid ${outcome.foreign.pid}) — ${outcome.nextPort === 0 ? "retrying on an ephemeral port" : `retrying on port ${outcome.nextPort}`} (#2941)`);
// closeAllConnections: idle keep-alive sockets from early
// probes must not stall the re-listen callback.
server.close(() => attemptListen(outcome.nextPort));
server.closeAllConnections?.();
return;
}
if (outcome.kind === "die") {
log("error", `listen failed: ${outcome.message}`);
shuttingDown = true;
server.close();
flushConversations();
void flushAllSessions().finally(() => {
closeLogger();
process.exit(1);
});
return;
}
}
await announceListening();
})();
};
const attemptListen = (port: number): void => {
lastTriedPort = port;
server.listen(port, opts.host, announceListening);
// #2941: a failed listen never fires 'listening', so each attempt's
// callback stacks on the server and a later successful bind fires ALL
// accumulated claimOrAnnounce closures at once (double announce,
// double registration, MaxListenersExceededWarning on long waits).
// Only our own callbacks ever use this event.
server.removeAllListeners("listening");
server.listen(port, opts.host, claimOrAnnounce);
};
attemptListen(opts.port);
// Listen errors (EADDRINUSE port taken, EACCES privileged port, EAFNOSUPPORT
Expand Down
Loading
Loading