From c820c8bbdf9705ce20c8ef9cda3020a8cb54fc37 Mon Sep 17 00:00:00 2001 From: ci-check Date: Wed, 28 Oct 2026 03:09:50 +0800 Subject: [PATCH 1/5] fix(#4977): exit twin-collision before persistence init when the port holder is a silent same-build twin --- scripts/live-files.mjs | 4 + src/instance.ts | 35 ++ src/server.ts | 53 ++- ...ssue4977-present-twin-persist-gate.test.ts | 328 ++++++++++++++++++ 4 files changed, 406 insertions(+), 14 deletions(-) create mode 100644 tests/issue4977-present-twin-persist-gate.test.ts diff --git a/scripts/live-files.mjs b/scripts/live-files.mjs index e2a58f9f45..b0ae254070 100644 --- a/scripts/live-files.mjs +++ b/scripts/live-files.mjs @@ -68,6 +68,10 @@ export const LIVE_FILES = [ // servers; every socket is listen(0), spawns are mocked, so the zone band // is belt-and-braces like #3890's — the entry is for the tripwire "tests/issue4811-dsh-busy-drop-selfheal.test.ts", + // #4977: spawns real CLI children against listen(0) ports (zombie holds + // the port; the ghost row binds it); zone band is belt-and-braces like + // #4341's — the entry is for the SPAWN_RE tripwire + "tests/issue4977-present-twin-persist-gate.test.ts", ]; /** Band width per live file: the zone ladder drifts +1 per EADDRINUSE retry; diff --git a/src/instance.ts b/src/instance.ts index 3fdeeb25d8..45c1216b00 100644 --- a/src/instance.ts +++ b/src/instance.ts @@ -275,6 +275,41 @@ export async function findVerifiedSameLaneTwin( return undefined; } +/** #4977: the recovery-window complement to findVerifiedSameLaneTwin. A registry + * twin whose health probe is SILENT while its socket still accepts connections + * is present but unserviceable — dying or event-loop-wedged (#3654 three-state + * liveness), NOT a healthy peer. The verified path correctly refuses to adopt + * such an occupant; pre-#4977 the child then proceeded into persistence + * initialization, opening index.db under the zombie before its own listen + * failed EADDRINUSE — a real two-writer window on one store (#3923/#3959) plus + * a misleading "another process is already writing" report, all on a round + * that ends in the twin-collision exit anyway (the late EADDRINUSE policy + * classifies the same registry evidence and fail-fasts). Positive-evidence-only + * discipline (#4341): the socket check runs FIRST and is decisive in the + * negative — no live listener means nothing to adjudicate (stale marker over a + * free or foreign port), so we fall through and let the bind proceed; a health + * probe that ANSWERS also falls through — matches belong to the verified path, + * mismatches must not be refused here. */ +export async function findPresentUnverifiedSameLaneTwin( + entries: RegistryEntry[], + port: number, + lane: string | undefined, + ownFingerprint: string | undefined, + sameShape: (e: RegistryEntry) => boolean, + healthOf: (origin: string) => Promise<{ pid?: number; instanceId?: string } | undefined>, + socketAlive: (origin: string) => Promise, +): Promise { + for (const entry of entries) { + const peer = classifySameLanePeer([entry], port, lane, ownFingerprint, sameShape); + if (peer?.kind !== "twin") continue; + if (!(await socketAlive(entry.origin))) continue; + const health = await healthOf(entry.origin); + if (health !== undefined) continue; + return entry; + } + return undefined; +} + /** #1225: content identity of a bili entry script (sha256 of its bytes). * The spawned child records this for ITS script; an attaching launcher * compares it against the hash of the script it would spawn — so "same diff --git a/src/server.ts b/src/server.ts index f29398d857..a14dae7add 100644 --- a/src/server.ts +++ b/src/server.ts @@ -97,10 +97,11 @@ import { getStore, setGlobalSessionCommitListener } from "./persist.js"; import { log as loggerLog, configureLogger, getLogPath, closeLogger, isStreamWriteError, isBenignSocketRaceError, enterSessionContext, muteStderrLeg, onFileLegLive, logTimestamp } from "./logger.js"; import { queryLogLines } from "./web/logs-query.js"; import { configFile, defaultLogFile, stateDir } from "./paths.js"; -import { atomicWriteInstanceFile, clearProxyInstanceFile, classifySameLanePeer, entryScriptFingerprint, entryShape, findSameLanePredecessor, findVerifiedSameLaneTwin, isPidAlive, listInstances, probePid, readProxyInstanceFile, registerInstanceAndWarn, sameInstanceShape, unregisterInstance, warnOnNewPeers, writeSpawnPortLog, type InstanceShape, type PidProbe, type ProxyInstanceFile, type RegistryEntry, type SameLanePeer } from "./instance.js"; +import { atomicWriteInstanceFile, clearProxyInstanceFile, classifySameLanePeer, entryScriptFingerprint, entryShape, findPresentUnverifiedSameLaneTwin, findSameLanePredecessor, findVerifiedSameLaneTwin, isPidAlive, listInstances, probePid, readProxyInstanceFile, registerInstanceAndWarn, sameInstanceShape, unregisterInstance, warnOnNewPeers, writeSpawnPortLog, type InstanceShape, type PidProbe, type ProxyInstanceFile, type RegistryEntry, type SameLanePeer } from "./instance.js"; import { createServedMarkerWriter, reapStaleWriterMarkers, writerMarkerDir, type ServedMarkerWriter } from "./served-markers.js"; import { recordSplitBrainContest } from "./storage/runtime-status.js"; import { mayClobberPointer, probeOwnerInfo, resolvePortClaim } from "./port-owner.js"; +import { tcpConnectable } from "./liveness.js"; import { hoistTrappedToolItems } from "./tool-pair-order.js"; import { computeAnthropicMessageMarks, stampAnthropicSystemCacheControl, anthropicToolsCarryCacheControl } from "./loop/cache-control.js"; import { reconcileSystemAnchor } from "./system-anchor.js"; @@ -640,8 +641,23 @@ export async function startServer(opts: ProxyOptions): Promise { modelWindows: { ...LAUNCHER_MODEL_WINDOWS }, modelMaxOutputs: { ...LAUNCHER_MODEL_MAX_OUTPUTS }, }; + const instances = listInstances(); + /** Both pre-persistence twin exits below share this tail: createEngineContext + * has not run yet, so its configureLogger call has not happened either — + * open the file leg here (mirroring it) so the twin line lands in bili.log + * like the late-path twin line, gate on opts.log like every other + * startServer line, and drain the stream before exiting or the buffered + * line can be lost with the process. */ + const exitBeforePersistence = async (hint: string): Promise => { + const listenLine = `listen failed: EADDRINUSE listen EADDRINUSE: address already in use ${opts.host}:${opts.port}${hint}`; + configureLogger(opts.logFile ?? defaultLogFile()); + logMsg(opts, "error", listenLine); + if (launchToken !== undefined) writeSpawnPortLog(opts.port, `${logTimestamp()} [error] [v=${VERSION}] ${listenLine}`); + await closeLogger(); + process.exit(eaddrinuseExitCode("twin-collision")); + }; const earlyTwin = await findVerifiedSameLaneTwin( - listInstances(), + instances, opts.port, launcherLane, ownFingerprint, @@ -649,18 +665,27 @@ export async function startServer(opts: ProxyOptions): Promise { (origin) => probeOwnerInfo(origin), ); if (earlyTwin !== undefined) { - const hint = ` — port ${opts.port} is already served by a same-build bili instance (pid ${earlyTwin.pid}, lane ${launcherLane}) — exiting before persistence initialization instead of hopping: the duplicate loses and the serving instance keeps serving; the next launch will attach to it (#4341/#3044). A spawner reads this exit as an adopt signal and attaches to the serving instance itself (#3654).`; - const listenLine = `listen failed: EADDRINUSE listen EADDRINUSE: address already in use ${opts.host}:${opts.port}${hint}`; - // createEngineContext has not run yet, so its configureLogger call - // has not happened either — open the file leg here (mirroring it) so - // the adopt line lands in bili.log like the late-path twin line, gate - // on opts.log like every other startServer line, and drain the stream - // before exiting or the buffered line can be lost with the process. - configureLogger(opts.logFile ?? defaultLogFile()); - logMsg(opts, "error", listenLine); - if (launchToken !== undefined) writeSpawnPortLog(opts.port, `${logTimestamp()} [error] [v=${VERSION}] ${listenLine}`); - await closeLogger(); - process.exit(eaddrinuseExitCode("twin-collision")); + await exitBeforePersistence(` — port ${opts.port} is already served by a same-build bili instance (pid ${earlyTwin.pid}, lane ${launcherLane}) — exiting before persistence initialization instead of hopping: the duplicate loses and the serving instance keeps serving; the next launch will attach to it (#4341/#3044). A spawner reads this exit as an adopt signal and attaches to the serving instance itself (#3654).`); + } + // #4977: the verified path above only adopts an occupant that PROVES itself + // healthy. The complement case — same-build twin marker whose socket still + // accepts connections but whose health probe is silent (dying or + // event-loop-wedged, #3654) — used to fall through into persistence init + // and open index.db under the zombie before the listen failed EADDRINUSE + // (two-writer window #3923/#3959). Same outcome, decided earlier: once the + // bind proved the port busy, the late policy classified this exact registry + // evidence and fail-fast-exited with the identical code. + const presentTwin = await findPresentUnverifiedSameLaneTwin( + instances, + opts.port, + launcherLane, + ownFingerprint, + (entry) => sameInstanceShape(entryShape(entry), ownShape), + (origin) => probeOwnerInfo(origin), + (origin) => tcpConnectable(origin), + ); + if (presentTwin !== undefined) { + await exitBeforePersistence(` — port ${opts.port} is still held by a same-build bili instance (pid ${presentTwin.pid}, lane ${launcherLane}) whose health probe is silent while its socket accepts connections: it is dying or event-loop-wedged (#3654), not serving — exiting before persistence initialization instead of opening index.db under it (#3923/#3959 two-writer window); the spawner reads this exit as an adopt signal and retries once the port frees (#4811/#4977).`); } const engine = await createEngineContext(opts); diff --git a/tests/issue4977-present-twin-persist-gate.test.ts b/tests/issue4977-present-twin-persist-gate.test.ts new file mode 100644 index 0000000000..6e48f979b8 --- /dev/null +++ b/tests/issue4977-present-twin-persist-gate.test.ts @@ -0,0 +1,328 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { spawn, type ChildProcess } from "node:child_process"; +import fs from "node:fs"; +import net from "node:net"; +import os from "node:os"; +import path from "node:path"; +import type { AddressInfo } from "node:net"; +import { findPresentUnverifiedSameLaneTwin, type InstanceShape, type RegistryEntry } from "../src/instance.ts"; +import { TWIN_ADOPT_EXIT_CODE } from "../src/listen-policy.ts"; +import { assertPortDead } from "./port-race.ts"; + +// #4977: a substitute child whose port is still held by a same-build twin that +// went SILENT (dying or event-loop-wedged — #3654 three-state liveness) must +// exit with the adopt code BEFORE persistence initialization. Pre-fix it fell +// through into engine creation, opened index.db under the zombie (two-writer +// window #3923/#3959), and only exited code 3 after its own listen failed. +// The unit rows pin the judgment truth table; the real-CLI rows pin the +// ordering against actual child processes (same harness shape as the #4341 +// early-adopt suite, which owns the verified-healthy complement case). + +const ROOT = path.join(import.meta.dirname, ".."); +const ENTRY = path.join(ROOT, "src/index.ts"); +const BASE = 18842; +const LANE = "dsh"; +const OWN_FP = "fp-our-build"; + +function ownShape(over: Partial = {}): InstanceShape { + return { + host: "127.0.0.1", + passthrough: false, + mitmDomains: [], + modelWindows: {}, + modelMaxOutputs: {}, + ...over, + }; +} + +function holder(over: Partial = {}): RegistryEntry { + return { + instanceId: "holder-4977", + pid: process.pid, + port: BASE, + origin: `http://127.0.0.1:${BASE}`, + startedAt: Date.now(), + host: "127.0.0.1", + passthrough: false, + mitmDomains: [], + modelWindows: {}, + modelMaxOutputs: {}, + lane: LANE, + codeFingerprint: OWN_FP, + ...over, + }; +} + +function sameShape(own: InstanceShape): (entry: RegistryEntry) => boolean { + return (entry) => + entry.host === own.host && + Boolean(entry.passthrough) === own.passthrough && + JSON.stringify(entry.mitmDomains ?? []) === JSON.stringify(own.mitmDomains) && + JSON.stringify(entry.modelWindows ?? {}) === JSON.stringify(own.modelWindows) && + JSON.stringify(entry.modelMaxOutputs ?? {}) === JSON.stringify(own.modelMaxOutputs); +} + +function sandboxDirs(sandbox: string): { stateHome: string; dataHome: string; configHome: string; cacheHome: string; sessions: string } { + return { + stateHome: path.join(sandbox, "state"), + dataHome: path.join(sandbox, "data"), + configHome: path.join(sandbox, "config"), + cacheHome: path.join(sandbox, "cache"), + sessions: path.join(sandbox, "data", "sessions"), + }; +} + +function makeEnv(sandbox: string): NodeJS.ProcessEnv { + const dirs = sandboxDirs(sandbox); + const env: NodeJS.ProcessEnv = { + ...process.env, + NODE_ENV: "test", + HOME: sandbox, + XDG_STATE_HOME: dirs.stateHome, + XDG_DATA_HOME: dirs.dataHome, + XDG_CONFIG_HOME: dirs.configHome, + XDG_CACHE_HOME: dirs.cacheHome, + BILI_SESSIONS_DIR: dirs.sessions, + BILI_LAUNCH_TOKEN: "issue-4977-test", + BILI_LAUNCHER_LANE: LANE, + BILI_PARENT_PID: String(process.pid), + BILI_STORAGE_UNIFIED: "full", + BILI_MITM: "0", + ACP_PASSTHROUGH: "0", + BILI_LAUNCHER_MODEL_WINDOWS: "{}", + BILI_LAUNCHER_MODEL_MAX_OUTPUTS: "{}", + }; + delete env.NODE_TEST_CONTEXT; + delete env.BILI_ENCRYPTION_KEY; + return env; +} + +async function freePort(): Promise { + const server = net.createServer(); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", resolve); + }); + const port = (server.address() as AddressInfo).port; + await new Promise((resolve) => server.close(() => resolve())); + return port; +} + +async function waitForStatus(port: number, timeoutMs: number): Promise { + const deadline = Date.now() + timeoutMs; + for (;;) { + try { + const res = await fetch(`http://127.0.0.1:${port}/__bili/status`); + if (res.status === 200) return true; + } catch { + // connection refused while the child is still starting up — expected + } + if (Date.now() >= deadline) return false; + await new Promise((resolve) => setTimeout(resolve, 100)); + } +} + +async function killChild(child: ChildProcess): Promise { + if (child.exitCode !== null || child.signalCode !== null) return; + child.kill("SIGTERM"); + await Promise.race([ + new Promise((resolve) => child.once("exit", () => resolve())), + new Promise((resolve) => setTimeout(resolve, 10_000)), + ]); + if (child.exitCode === null && child.signalCode === null) child.kill("SIGKILL"); +} + +async function deadPid(): Promise { + const c = spawn(process.execPath, ["-e", "setTimeout(() => {}, 100)"]); + const pid = c.pid; + if (pid === undefined) throw new Error("spawn failed to produce a pid"); + await new Promise((resolve) => c.once("exit", () => resolve())); + return pid; +} + +const SILENT_HEALTH = async (): Promise<{ pid?: number; instanceId?: string } | undefined> => undefined; +const SOCKET_UP = async (): Promise => true; +const SOCKET_DOWN = async (): Promise => false; + +test("present-unverified twin: silent probe over a live socket is the exit case", async () => { + const entry = holder(); + const peer = await findPresentUnverifiedSameLaneTwin( + [entry], BASE, LANE, OWN_FP, sameShape(ownShape()), SILENT_HEALTH, SOCKET_UP, + ); + assert.equal(peer?.instanceId, entry.instanceId); +}); + +test("present-unverified twin: stale marker over a free port falls through to the bind", async () => { + let healthCalls = 0; + const peer = await findPresentUnverifiedSameLaneTwin( + [holder()], BASE, LANE, OWN_FP, sameShape(ownShape()), + async () => { healthCalls += 1; return undefined; }, + SOCKET_DOWN, + ); + assert.equal(peer, undefined, "no live listener means nothing to adjudicate"); + assert.equal(healthCalls, 0, "the socket check runs first and decides — no probe budget spent on a dead port"); +}); + +test("present-unverified twin: answered probes fall through to their own paths", async () => { + const entry = holder(); + assert.equal( + await findPresentUnverifiedSameLaneTwin( + [entry], BASE, LANE, OWN_FP, sameShape(ownShape()), + async () => ({ pid: entry.pid, instanceId: entry.instanceId }), + SOCKET_UP, + ), + undefined, + "a matching answer belongs to the verified-adopt path (#4341)", + ); + assert.equal( + await findPresentUnverifiedSameLaneTwin( + [entry], BASE, LANE, OWN_FP, sameShape(ownShape()), + async () => ({ pid: entry.pid + 1, instanceId: entry.instanceId }), + SOCKET_UP, + ), + undefined, + "a mismatched answer must not be refused here (#4341)", + ); +}); + +test("present-unverified twin: drifted shape, foreign fingerprint, dead pid, wrong port/lane never classify", async () => { + const dead = await deadPid(); + const rows: Array<[string, RegistryEntry]> = [ + ["drifted shape", holder({ modelWindows: { "model-a": 1000 } })], + ["foreign fingerprint", holder({ codeFingerprint: "fp-other-build" })], + ["dead pid", holder({ pid: dead })], + ["wrong port", holder({ port: BASE + 1, origin: `http://127.0.0.1:${BASE + 1}` })], + ["wrong lane", holder({ lane: "pi" })], + ["missing lane", holder({ lane: undefined })], + ["missing fingerprint", holder({ codeFingerprint: undefined })], + ]; + for (const [label, entry] of rows) { + assert.equal( + await findPresentUnverifiedSameLaneTwin( + [entry], BASE, LANE, OWN_FP, sameShape(ownShape()), SILENT_HEALTH, SOCKET_UP, + ), + undefined, + label, + ); + } + assert.equal( + await findPresentUnverifiedSameLaneTwin( + [holder()], BASE, undefined, OWN_FP, sameShape(ownShape()), SILENT_HEALTH, SOCKET_UP, + ), + undefined, + "caller without a lane never classifies", + ); +}); + +test("real CLI: silent twin exits before unified SQLite initialization", { timeout: 60_000 }, async () => { + const sandbox = fs.mkdtempSync(path.join(os.tmpdir(), "bili-4977-")); + const { stateHome, sessions } = sandboxDirs(sandbox); + const instanceId = "zombie-twin-4977"; + // The zombie's exact #3654 signature: accepts TCP but never answers HTTP — + // socket alive, health silent (dying or event-loop-wedged). + const zombie = net.createServer((sock) => { sock.resume(); }); + await new Promise((resolve, reject) => { + zombie.once("error", reject); + zombie.listen(0, "127.0.0.1", resolve); + }); + const port = (zombie.address() as AddressInfo).port; + const registryDir = path.join(stateHome, "billion-context", "instances"); + fs.mkdirSync(registryDir, { recursive: true }); + const fingerprint = createHash("sha256").update(fs.readFileSync(ENTRY)).digest("hex"); + fs.writeFileSync( + path.join(registryDir, `${instanceId}.json`), + JSON.stringify({ + instanceId, + pid: process.pid, + port, + origin: `http://127.0.0.1:${port}`, + startedAt: Date.now(), + host: "127.0.0.1", + passthrough: false, + mitmDomains: [], + modelWindows: {}, + modelMaxOutputs: {}, + lane: LANE, + codeFingerprint: fingerprint, + }) + "\n", + ); + const env = makeEnv(sandbox); + + try { + const child = spawn(process.execPath, ["--import", "tsx", ENTRY, "start", "--host", "127.0.0.1", "--port", String(port)], { + cwd: ROOT, + env, + stdio: ["ignore", "pipe", "pipe"], + }); + let output = ""; + child.stdout.setEncoding("utf8"); + child.stderr.setEncoding("utf8"); + child.stdout.on("data", (chunk: string) => { output += chunk; }); + child.stderr.on("data", (chunk: string) => { output += chunk; }); + const timer = setTimeout(() => child.kill("SIGKILL"), 45_000); + const code = await new Promise((resolve, reject) => { + child.once("error", reject); + child.once("exit", resolve); + }); + clearTimeout(timer); + + assert.equal(code, TWIN_ADOPT_EXIT_CODE, output); + assert.match(output, /exiting before persistence initialization/); + assert.match(output, /still held by a same-build bili instance/); + assert.equal(fs.existsSync(path.join(sessions, "index.db")), false, "the duplicate must not create/open the unified database under the zombie"); + } finally { + await new Promise((resolve) => zombie.close(() => resolve())); + fs.rmSync(sandbox, { recursive: true, force: true }); + } +}); + +test("real CLI: live marker for an unlistened port falls through to normal boot", { timeout: 90_000 }, async () => { + const sandbox = fs.mkdtempSync(path.join(os.tmpdir(), "bili-4977-")); + const { stateHome, sessions } = sandboxDirs(sandbox); + const port = await freePort(); + const registryDir = path.join(stateHome, "billion-context", "instances"); + fs.mkdirSync(registryDir, { recursive: true }); + const fingerprint = createHash("sha256").update(fs.readFileSync(ENTRY)).digest("hex"); + fs.writeFileSync( + path.join(registryDir, "ghost-twin-4977.json"), + JSON.stringify({ + instanceId: "ghost-twin-4977", + pid: process.pid, + port, + origin: `http://127.0.0.1:${port}`, + startedAt: Date.now(), + host: "127.0.0.1", + passthrough: false, + mitmDomains: [], + modelWindows: {}, + modelMaxOutputs: {}, + lane: LANE, + codeFingerprint: fingerprint, + }) + "\n", + ); + const env = makeEnv(sandbox); + await assertPortDead(port, { label: "ghost-twin port" }); + const child = spawn(process.execPath, ["--import", "tsx", ENTRY, "start", "--host", "127.0.0.1", "--port", String(port)], { + cwd: ROOT, + env, + stdio: ["ignore", "pipe", "pipe"], + }); + let output = ""; + child.stdout.setEncoding("utf8"); + child.stderr.setEncoding("utf8"); + child.stdout.on("data", (chunk: string) => { output += chunk; }); + child.stderr.on("data", (chunk: string) => { output += chunk; }); + try { + const ready = await waitForStatus(port, 60_000); + assert.equal(ready, true, `child never became ready:\n${output}`); + assert.doesNotMatch(output, /exiting before persistence initialization/, "an unproven marker must not trigger the early exit"); + const logText = fs.readFileSync(path.join(stateHome, "billion-context", "bili.log"), "utf8"); + assert.match(logText, /\[storage\] unified FULL mode ENABLED/, "normal boot must open the unified store"); + assert.equal(fs.existsSync(path.join(sessions, "index.db")), true); + } finally { + await killChild(child); + fs.rmSync(sandbox, { recursive: true, force: true }); + } +}); From da2520741aabc715d2d3182f0e7c10332a5fb174 Mon Sep 17 00:00:00 2001 From: ci-check Date: Wed, 28 Oct 2026 04:29:28 +0800 Subject: [PATCH 2/5] test(#4977): swallow zombie-socket errors in the issue4977 real-CLI rows The silent-twin zombie server accepted TCP connections without an error handler on the sockets. On Windows the child's abrupt process.exit resets the half-open probed connection; the RST reaches the zombie-side socket after the test has already completed and surfaces as an uncaughtException (read ECONNRESET) failing the shard although every assertion passed (CI run 40567102989, test (windows-latest, 22, 2), not ok 852). Matches the established pattern in the fix-806 / auto-restart / port-zone-live suites. --- tests/issue4977-present-twin-persist-gate.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/issue4977-present-twin-persist-gate.test.ts b/tests/issue4977-present-twin-persist-gate.test.ts index 6e48f979b8..d94ea50c09 100644 --- a/tests/issue4977-present-twin-persist-gate.test.ts +++ b/tests/issue4977-present-twin-persist-gate.test.ts @@ -222,7 +222,7 @@ test("real CLI: silent twin exits before unified SQLite initialization", { timeo const instanceId = "zombie-twin-4977"; // The zombie's exact #3654 signature: accepts TCP but never answers HTTP — // socket alive, health silent (dying or event-loop-wedged). - const zombie = net.createServer((sock) => { sock.resume(); }); + const zombie = net.createServer((sock) => { sock.on("error", () => {}); sock.resume(); }); await new Promise((resolve, reject) => { zombie.once("error", reject); zombie.listen(0, "127.0.0.1", resolve); From 231f77a1df5037537e8efe9f243a0483226bbf01 Mon Sep 17 00:00:00 2001 From: ci-check Date: Wed, 28 Oct 2026 05:05:20 +0800 Subject: [PATCH 3/5] =?UTF-8?q?test(ci):=20double=20wire-exit-matrix=20abo?= =?UTF-8?q?rt-teardown=20poll=20budget=203s=E2=86=926s?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Unblocks PR #5004's test (windows-latest, 22, 2) red on head da2520741: three abort cells (proxy-openai-sse / proxy-responses-sse / proxy-json) asserted closed=0 after exhausting the flat 3s (30x100ms) poll on a loaded Windows runner -- the same signature already recorded under #4599 for #4982/#4774/#4683 (rerun green / in-branch budget bump). Sibling cell abort x proxy-anthropic-sse passed in 176ms on the same shard and every other windows/ubuntu shard ran this file green on the same head, so this is runner load, not a code regression; the PR's diff touches none of these paths. Labeled CI-unblock commit; duplicates #4683's pending identical change -- drop this commit when rebasing if #4683 lands first. --- tests/wire-exit-matrix.test.ts | 37 ++++++++++++++++------------------ 1 file changed, 17 insertions(+), 20 deletions(-) diff --git a/tests/wire-exit-matrix.test.ts b/tests/wire-exit-matrix.test.ts index bfb521b1ee..4c8f99878b 100644 --- a/tests/wire-exit-matrix.test.ts +++ b/tests/wire-exit-matrix.test.ts @@ -28,6 +28,19 @@ import { EXIT_CONCERNS, WIRE_EXITS, type ExitConcernId } from "../src/exit-matri const SLOW_MS = 1500; +// Client-abort → upstream teardown propagation budget. A flat 3s (30×100ms) +// flaked on loaded Windows runners: closed=0 after the full poll while sibling +// cells and every other shard went green (recorded under #4599 for #4982/ +// #4774/#4683, which made the same 3s→6s move on its own branch). Doubled to 6s. +const ABORT_TEARDOWN_BUDGET_MS = 6000; + +async function awaitUpstreamTeardown(socketsClosed: () => number): Promise { + const deadline = Date.now() + ABORT_TEARDOWN_BUDGET_MS; + while (socketsClosed() < 1 && Date.now() < deadline) { + await new Promise((r) => setTimeout(r, 100)); + } +} + const SUMMARY_TEXT = "PREFLIGHT SUMMARY: the segment covered a debugging session. " + "Key decisions: chose the preflight approach. Files touched: src/a.ts:10. Outcome: fixed."; @@ -212,11 +225,7 @@ test("abort x proxy-openai-sse: client abort mid-stream destroys the upstream re const url = `http://127.0.0.1:${proxyPort}/bili/http://127.0.0.1:${up.port}/v1/chat/completions`; const outcome = await abortAfterFirstByte(url, { "content-type": "application/json", "x-acp-session": "mx-abort-oai" }, JSON.stringify({ model: "m-test", max_tokens: 1024, stream: true, messages: [{ role: "user", content: "hi" }] })); assert.ok(outcome === "first-byte" || outcome === "header"); - // Poll for the abort to propagate instead of a fixed 300ms sleep: - // slow runners (win/node24) destroy the upstream socket later. - for (let i = 0; i < 30 && up.socketsClosed() < 1; i += 1) { - await new Promise((r) => setTimeout(r, 100)); - } + await awaitUpstreamTeardown(up.socketsClosed); assert.ok(up.socketsClosed() >= 1, `upstream socket(s) must be destroyed after the client abort (closed=${up.socketsClosed()})`); assert.equal(up.requests(), 1, "exactly one upstream request"); } finally { @@ -237,11 +246,7 @@ test("abort x proxy-anthropic-sse: client abort mid-stream destroys the upstream try { const url = `http://127.0.0.1:${proxyPort}/bili/http://127.0.0.1:${up.port}/v1/messages`; await abortAfterFirstByte(url, { "content-type": "application/json", "x-acp-session": "mx-abort-ant" }, JSON.stringify({ model: "m-test", max_tokens: 1024, stream: true, messages: [{ role: "user", content: "hi" }] })); - // Poll for the abort to propagate instead of a fixed 300ms sleep: - // slow runners (win/node24) destroy the upstream socket later. - for (let i = 0; i < 30 && up.socketsClosed() < 1; i += 1) { - await new Promise((r) => setTimeout(r, 100)); - } + await awaitUpstreamTeardown(up.socketsClosed); assert.ok(up.socketsClosed() >= 1, `upstream socket(s) destroyed after client abort (closed=${up.socketsClosed()})`); } finally { proxy.close(); @@ -261,11 +266,7 @@ test("abort x proxy-responses-sse: client abort mid-stream destroys the upstream try { const url = `http://127.0.0.1:${proxyPort}/bili/http://127.0.0.1:${up.port}/v1/responses`; await abortAfterFirstByte(url, { "content-type": "application/json", "x-acp-session": "mx-abort-resp" }, JSON.stringify({ model: "m-test", max_tokens: 1024, stream: true, instructions: "You are a test agent.", input: [{ type: "message", role: "user", content: "hi" }] })); - // Poll for the abort to propagate instead of a fixed 300ms sleep: - // slow runners (win/node24) destroy the upstream socket later. - for (let i = 0; i < 30 && up.socketsClosed() < 1; i += 1) { - await new Promise((r) => setTimeout(r, 100)); - } + await awaitUpstreamTeardown(up.socketsClosed); assert.ok(up.socketsClosed() >= 1, `upstream socket(s) destroyed after client abort (closed=${up.socketsClosed()})`); } finally { proxy.close(); @@ -285,11 +286,7 @@ test("abort x proxy-json: client abort while the JSON is buffering destroys the try { const url = `http://127.0.0.1:${proxyPort}/bili/http://127.0.0.1:${up.port}/v1/chat/completions`; await abortAfterFirstByte(url, { "content-type": "application/json", "x-acp-session": "mx-abort-json" }, JSON.stringify({ model: "m-test", max_tokens: 1024, messages: [{ role: "user", content: "hi" }] })); - // Poll for the abort to propagate instead of a fixed 300ms sleep: - // slow runners (win/node24) destroy the upstream socket later. - for (let i = 0; i < 30 && up.socketsClosed() < 1; i += 1) { - await new Promise((r) => setTimeout(r, 100)); - } + await awaitUpstreamTeardown(up.socketsClosed); assert.ok(up.socketsClosed() >= 1, `upstream socket(s) destroyed after client abort (closed=${up.socketsClosed()})`); } finally { proxy.close(); From 216f3a5c8fbb89e6cd3b422cf6a83931ddff5fda Mon Sep 17 00:00:00 2001 From: ci-check Date: Wed, 28 Oct 2026 10:08:09 +0800 Subject: [PATCH 4/5] test: auto-restart misplaced-stub retry covers OS bind refusals (EACCES/EPERM), not just EADDRINUSE (#3182 family) --- tests/auto-restart.test.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/auto-restart.test.ts b/tests/auto-restart.test.ts index d40fabf149..f869952475 100644 --- a/tests/auto-restart.test.ts +++ b/tests/auto-restart.test.ts @@ -174,8 +174,11 @@ function makeReadyStub(name: string): string { "if (!Number.isFinite(port)) process.exit(64);", // #3182: a stolen port must be distinguishable from a product // failure — exit 91 lets the retry wrapper below re-run the scenario. + // EACCES/EPERM = the OS refused the bind outright (Windows dynamic + // exclusion ranges on shared runners) — same "port unusable" class, + // not a product failure. "const srv = http.createServer((_req, res) => res.end('ok'));", - "srv.on('error', (e) => process.exit(e && e.code === 'EADDRINUSE' ? 91 : 1));", + "srv.on('error', (e) => process.exit(e && ['EADDRINUSE', 'EACCES', 'EPERM'].includes(e.code) ? 91 : 1));", "srv.listen(port, '127.0.0.1');", ].join("\n")); return p; From 125c3bc5bcdca5b41b6791f6d3141a0fffd262c7 Mon Sep 17 00:00:00 2001 From: ework-agent Date: Sat, 31 Oct 2026 13:02:01 +0800 Subject: [PATCH 5/5] test(#5018): retarget the unverified-twin #4982 e2e row to the pre-persistence adopt-exit (option A) Ruling (owner-approved 2026-10-31): for an unverified unstamped same-lane claim the early pre-persistence gate (#4977) wins over #5016's late ladder. The second row of tests/issue4982-draining-twin-ladder.test.ts now asserts the code-3 twin-collision adopt-exit BEFORE index.db is opened instead of ladder-off; the draining (portReleasedAt-stamped) row keeps asserting the ladder. Fixture cleanup hygiene was already finally-wrapped upstream, nothing to add. Refs #5018 #5004 #4982 #5016 --- tests/issue4982-draining-twin-ladder.test.ts | 34 +++++++++++++------- 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/tests/issue4982-draining-twin-ladder.test.ts b/tests/issue4982-draining-twin-ladder.test.ts index 598e8295d9..cd6875c13f 100644 --- a/tests/issue4982-draining-twin-ladder.test.ts +++ b/tests/issue4982-draining-twin-ladder.test.ts @@ -32,6 +32,7 @@ import type { AddressInfo } from "node:net"; import { classifySameLanePeer, markPortReleased, type InstanceShape, type RegistryEntry } from "../src/instance.ts"; import { verifyEntryOwnsPort } from "../src/port-owner.ts"; import { assertPortDead } from "./port-race.ts"; +import { TWIN_ADOPT_EXIT_CODE } from "../src/listen-policy.ts"; const ROOT = path.join(import.meta.dirname, ".."); const ENTRY = path.join(ROOT, "src/index.ts"); @@ -454,9 +455,9 @@ test("#4982 e2e: draining same-lane predecessor + squatter → newcomer ladders } }); -test("#4982 e2e: unverified registry twin claim (unstamped marker) → ladder, not adopt-death", { timeout: 180_000 }, async () => { +test("#4982 e2e: unverified registry twin claim (unstamped marker) → pre-persistence adopt-exit, not two-writer window (#5018 option A)", { timeout: 180_000 }, async () => { const sandbox = fs.mkdtempSync(path.join(os.tmpdir(), "bili-4982b-")); - const { stateHome } = sandboxDirs(sandbox); + const { stateHome, sessions } = sandboxDirs(sandbox); const port = await freePort(); await assertPortDead(port, { label: "4982 unverified-twin port" }); let p2Ref: ReturnType | null = null; @@ -490,23 +491,32 @@ test("#4982 e2e: unverified registry twin claim (unstamped marker) → ladder, n const p2 = spawnBili(sandbox, port); p2Ref = p2; + // #5018 option A (owner ruling 2026-10-31): for an UNVERIFIED unstamped + // same-lane claim the early pre-persistence gate wins over #5016's + // ladder — before we bind, a silent-probe holder cannot be told apart + // from an event-loop-wedged zombie (#3654), and opening index.db under + // it IS the two-writer window (#3923/#3959). P2 must therefore exit + // with the twin-collision adopt signal BEFORE persistence init; the + // spawner retries once the holder frees the port (#4811/#4977). The + // ladder keeps covering stamped draining predecessors (row above) and + // foreign-held ports with no matching registry claim. const result = await Promise.race([ - new Promise<"exited">((resolve) => p2.child.once("exit", () => resolve("exited"))), + new Promise<{ kind: "exited"; code: number | null }>((resolve) => + p2.child.once("exit", (code) => resolve({ kind: "exited", code })), + ), (async () => { const servedPort = await waitUntilLadderedHealthy(stateHome, p2.child, "P2"); return { kind: "healthy" as const, servedPort }; })(), ]); - if (typeof result === "string") { - throw new Error(`P2 exited before becoming healthy (the #4982 failure mode):\n${p2.out.value}`); + if (result.kind === "healthy") { + throw new Error(`P2 laddered off instead of taking the early pre-persistence adopt-exit (#5018 option A violated):\n${p2.out.value}`); } - assert.notEqual(result.servedPort, port, "P2 must ladder off the contended port"); - const logText = fs.readFileSync(path.join(stateHome, "billion-context", "bili.log"), "utf8"); - assert.match(logText, /no longer answers as itself/, "the demotion from twin to foreign-held must be logged"); - // The demotion warn above deliberately names "twin-collision" in prose, so pin - // the verdict by its unique log marker (shared by the early #4341 and the late - // path) instead of that substring. - assert.doesNotMatch(logText, /already served by a same-build bili instance/, "an unverified claim must never reach the twin-collision verdict"); + assert.equal(result.code, TWIN_ADOPT_EXIT_CODE, "the exit must carry the twin-collision adopt signal the spawner keys on (#4811)"); + assert.match(p2.out.value, /still held by a same-build bili instance \(pid \d+, lane pi\) whose health probe is silent/, "the pre-persistence present-twin verdict must be emitted on the stderr leg the spawner reads"); + const logText = fs.readFileSync(path.join(stateHome, "billion-context", "bili.log"), "utf8").slice(-1200); + assert.doesNotMatch(logText, /already served by a same-build bili instance/, "only a VERIFIED twin may produce the served-by verdict"); + assert.equal(fs.existsSync(path.join(sessions, "index.db")), false, "the duplicate must exit before persistence init — no index.db opened under the zombie (#3923/#3959)"); } finally { if (p2Ref) await killChild(p2Ref.child); const squatter = squatterRef;