Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions scripts/live-files.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,10 @@ export const LIVE_FILES = [
// servers; every socket is listen(0), spawns are mocked, so the zone band
// is belt-and-braces like #3890's — the entry is for the tripwire
"tests/issue4811-dsh-busy-drop-selfheal.test.ts",
// #4977: spawns real CLI children against listen(0) ports (zombie holds
// the port; the ghost row binds it); zone band is belt-and-braces like
// #4341's — the entry is for the SPAWN_RE tripwire
"tests/issue4977-present-twin-persist-gate.test.ts",
// #4982: integration rows spawn bili children against freePort() picks and
// follow each child's REGISTERED port (ladder-tolerant); BASE is an
// in-memory-only value, so no zone band is needed — the entry is for the
Expand Down
35 changes: 35 additions & 0 deletions src/instance.ts
Original file line number Diff line number Diff line change
Expand Up @@ -281,6 +281,41 @@ export async function findVerifiedSameLaneTwin(
return undefined;
}

/** #4977: the recovery-window complement to findVerifiedSameLaneTwin. A registry
* twin whose health probe is SILENT while its socket still accepts connections
* is present but unserviceable — dying or event-loop-wedged (#3654 three-state
* liveness), NOT a healthy peer. The verified path correctly refuses to adopt
* such an occupant; pre-#4977 the child then proceeded into persistence
* initialization, opening index.db under the zombie before its own listen
* failed EADDRINUSE — a real two-writer window on one store (#3923/#3959) plus
* a misleading "another process is already writing" report, all on a round
* that ends in the twin-collision exit anyway (the late EADDRINUSE policy
* classifies the same registry evidence and fail-fasts). Positive-evidence-only
* discipline (#4341): the socket check runs FIRST and is decisive in the
* negative — no live listener means nothing to adjudicate (stale marker over a
* free or foreign port), so we fall through and let the bind proceed; a health
* probe that ANSWERS also falls through — matches belong to the verified path,
* mismatches must not be refused here. */
export async function findPresentUnverifiedSameLaneTwin(
entries: RegistryEntry[],
port: number,
lane: string | undefined,
ownFingerprint: string | undefined,
sameShape: (e: RegistryEntry) => boolean,
healthOf: (origin: string) => Promise<{ pid?: number; instanceId?: string } | undefined>,
socketAlive: (origin: string) => Promise<boolean>,
): Promise<RegistryEntry | undefined> {
for (const entry of entries) {
const peer = classifySameLanePeer([entry], port, lane, ownFingerprint, sameShape);
if (peer?.kind !== "twin") continue;
if (!(await socketAlive(entry.origin))) continue;
const health = await healthOf(entry.origin);
if (health !== undefined) continue;
return entry;
}
return undefined;
}

/** #1225: content identity of a bili entry script (sha256 of its bytes).
* The spawned child records this for ITS script; an attaching launcher
* compares it against the hash of the script it would spawn — so "same
Expand Down
56 changes: 42 additions & 14 deletions src/server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -98,11 +98,12 @@ import { getStore, setGlobalSessionCommitListener } from "./persist.js";
import { log as loggerLog, configureLogger, getLogPath, closeLogger, isStreamWriteError, isBenignSocketRaceError, enterSessionContext, muteStderrLeg, onFileLegLive, shouldMuteStderrLegAfterFileLive, logTimestamp } from "./logger.js";
import { queryLogLines } from "./web/logs-query.js";
import { configFile, defaultLogFile, sessionsDir, stateDir } from "./paths.js";
import { atomicWriteInstanceFile, clearProxyInstanceFile, classifySameLanePeer, entryShape, findSameLanePredecessor, findVerifiedSameLaneTwin, isPidAlive, listInstances, markPortReleased, probePid, readProxyInstanceFile, registerInstanceAndWarn, resolveOwnFingerprint, sameInstanceShape, SPAWN_FINGERPRINT_ENV, unregisterInstance, warnOnNewPeers, writeSpawnPortLog, type InstanceShape, type PidProbe, type ProxyInstanceFile, type RegistryEntry, type SameLanePeer } from "./instance.js";
import { atomicWriteInstanceFile, clearProxyInstanceFile, classifySameLanePeer, entryShape, findPresentUnverifiedSameLaneTwin, findSameLanePredecessor, findVerifiedSameLaneTwin, isPidAlive, listInstances, markPortReleased, probePid, readProxyInstanceFile, registerInstanceAndWarn, resolveOwnFingerprint, sameInstanceShape, SPAWN_FINGERPRINT_ENV, unregisterInstance, warnOnNewPeers, writeSpawnPortLog, type InstanceShape, type PidProbe, type ProxyInstanceFile, type RegistryEntry, type SameLanePeer } from "./instance.js";
import { createServedMarkerWriter, peerStillContests, reapStaleWriterMarkers, writerMarkerDir, type ServedMarkerWriter } from "./served-markers.js";
import { recordSplitBrainContest, registerSplitBrainEvidenceCheck } from "./storage/runtime-status.js";
import { holderProbeOrigin, mayClobberPointer, probeHolderForHint, probeOwnerInfo, resolvePortClaim, verifyEntryOwnsPort } from "./port-owner.js";
import { armEventLoopWatchdog } from "./event-loop-watchdog.js";
import { tcpConnectable } from "./liveness.js";
import { hoistTrappedToolItems } from "./tool-pair-order.js";
import { computeAnthropicMessageMarks, stampAnthropicSystemCacheControl, anthropicToolsCarryCacheControl } from "./loop/cache-control.js";
import { reconcileSystemAnchor } from "./system-anchor.js";
Expand Down Expand Up @@ -678,31 +679,58 @@ export async function startServer(opts: ProxyOptions): Promise<http.Server> {
onSpawnFailure: (err) => logMsg(opts, "warn", `[event-loop-watchdog] could not arm (${String(err)}) — continuing unwatched (#4762)`),
});

const instances = listInstances();
/** Both pre-persistence twin exits below share this tail: createEngineContext
* has not run yet, so its configureLogger call has not happened either —
* open the file leg here (mirroring it) so the twin line lands in bili.log
* like the late-path twin line, gate on opts.log like every other
* startServer line, and drain the stream before exiting or the buffered
* line can be lost with the process. */
const exitBeforePersistence = async (hint: string): Promise<never> => {
const listenLine = `listen failed: EADDRINUSE listen EADDRINUSE: address already in use ${opts.host}:${opts.port}${hint}`;
configureLogger(opts.logFile ?? defaultLogFile());
logMsg(opts, "error", listenLine);
if (launchToken !== undefined) writeSpawnPortLog(opts.port, `${logTimestamp()} [error] [v=${VERSION}] ${listenLine}`);
await closeLogger();
process.exit(eaddrinuseExitCode("twin-collision"));
};
const earlyTwin = await findVerifiedSameLaneTwin(
listInstances(),
instances,
opts.port,
launcherLane,
ownFingerprint,
(entry) => sameInstanceShape(entryShape(entry), ownShape),
(origin) => probeOwnerInfo(origin),
);
if (earlyTwin !== undefined) {
// createEngineContext has not run yet, so its configureLogger call
// has not happened either — open the file leg here (mirroring it) so
// the adopt line lands in bili.log like the late-path twin line, gate
// on opts.log like every other startServer line, and drain the stream
// before exiting or the buffered line can be lost with the process.
configureLogger(opts.logFile ?? defaultLogFile());
// #4762: the twin verdict above rested on ONE health probe during
// classification — a wedged zombie drains probes intermittently, so
// re-probe at the moment of death and say which case we actually hit.
const holderFragment = await probeHolderForHint({ origin: holderProbeOrigin(opts.host, opts.port), knownPid: earlyTwin.pid });
const hint = ` — port ${opts.port} is already served by a same-build bili instance (pid ${earlyTwin.pid}, lane ${launcherLane}) — exiting before persistence initialization instead of hopping: the duplicate loses and the serving instance keeps serving; the next launch will attach to it (#4341/#3044). A spawner reads this exit as an adopt signal and attaches to the serving instance itself (#3654). ${holderFragment}.`;
const listenLine = `listen failed: EADDRINUSE listen EADDRINUSE: address already in use ${opts.host}:${opts.port}${hint}`;
logMsg(opts, "error", listenLine);
if (launchToken !== undefined) writeSpawnPortLog(opts.port, `${logTimestamp()} [error] [v=${VERSION}] ${listenLine}`);
await closeLogger();
process.exit(eaddrinuseExitCode("twin-collision"));
await exitBeforePersistence(` — port ${opts.port} is already served by a same-build bili instance (pid ${earlyTwin.pid}, lane ${launcherLane}) — exiting before persistence initialization instead of hopping: the duplicate loses and the serving instance keeps serving; the next launch will attach to it (#4341/#3044). A spawner reads this exit as an adopt signal and attaches to the serving instance itself (#3654). ${holderFragment}.`);
}
// #4977: the verified path above only adopts an occupant that PROVES itself
// healthy. The complement case — same-build twin marker whose socket still
// accepts connections but whose health probe is silent (dying or
// event-loop-wedged, #3654) — used to fall through into persistence init
// and open index.db under the zombie before the listen failed EADDRINUSE
// (two-writer window #3923/#3959). Same outcome, decided earlier: once the
// bind proved the port busy, the late policy classified this exact registry
// evidence and fail-fast-exited with the identical code.
// #5018 decision (owner, 2026-10-31): option A — the early adopt-exit wins
// over #5016's late ladder for this unstamped same-lane corner; the
// cross-lane squatter form stays ladder-off (lane matching excludes it).
const presentTwin = await findPresentUnverifiedSameLaneTwin(
instances,
opts.port,
launcherLane,
ownFingerprint,
(entry) => sameInstanceShape(entryShape(entry), ownShape),
(origin) => probeOwnerInfo(origin),
(origin) => tcpConnectable(origin),
);
if (presentTwin !== undefined) {
await exitBeforePersistence(` — port ${opts.port} is still held by a same-build bili instance (pid ${presentTwin.pid}, lane ${launcherLane}) whose health probe is silent while its socket accepts connections: it is dying or event-loop-wedged (#3654), not serving — exiting before persistence initialization instead of opening index.db under it (#3923/#3959 two-writer window); the spawner reads this exit as an adopt signal and retries once the port frees (#4811/#4977).`);
}

const engine = await createEngineContext(opts);
Expand Down
5 changes: 4 additions & 1 deletion tests/auto-restart.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -174,8 +174,11 @@ function makeReadyStub(name: string): string {
"if (!Number.isFinite(port)) process.exit(64);",
// #3182: a stolen port must be distinguishable from a product
// failure — exit 91 lets the retry wrapper below re-run the scenario.
// EACCES/EPERM = the OS refused the bind outright (Windows dynamic
// exclusion ranges on shared runners) — same "port unusable" class,
// not a product failure.
"const srv = http.createServer((_req, res) => res.end('ok'));",
"srv.on('error', (e) => process.exit(e && e.code === 'EADDRINUSE' ? 91 : 1));",
"srv.on('error', (e) => process.exit(e && ['EADDRINUSE', 'EACCES', 'EPERM'].includes(e.code) ? 91 : 1));",
"srv.listen(port, '127.0.0.1');",
].join("\n"));
return p;
Expand Down
Loading
Loading