From ef2e7ed71eecda4501f5c38c3e8428ba5b31ebd3 Mon Sep 17 00:00:00 2001 From: ci-check Date: Sun, 25 Oct 2026 08:04:05 +0800 Subject: [PATCH 1/4] fix: self-clear the stale split-brain verdict once the peer stops contesting (#4626) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The surfaced contest was persisted into the storage runtime state at detection time and replayed verbatim on every /__bili/status read until a manual ack — but detection fires while the restarting peer's tombstone marker is still seconds fresh (its graceful-shutdown dispose() rewrites the marker), so a restart handover left an alarming card behind for hours after the peer pid was gone. The stored verdict now re-evaluates against CURRENT evidence at the status read, with the same never-freeze discipline as the sibling second-process warning: the server registers the very scan that produced the verdict (findContestedWriters over its own writers dir), and when that scan no longer reports the accused peer (pid gone + marker past its freshness grace, or the contest window lapsed) the entry clears itself with an info log line. Without a registered check the legacy replay behavior stands (standalone record/ack use). The fresh-tombstone case keeps persisting: a dead pid alone is not proof of absence on every platform, so the notice stays honest while marker freshness says the peer may be alive. --- src/server.ts | 10 ++- src/storage/runtime-status.ts | 20 +++++ tests/issue-split-brain-surface.test.ts | 114 +++++++++++++++++++++++- 3 files changed, 138 insertions(+), 6 deletions(-) diff --git a/src/server.ts b/src/server.ts index 3030b92888..bbe9df668b 100644 --- a/src/server.ts +++ b/src/server.ts @@ -99,8 +99,8 @@ import { log as loggerLog, configureLogger, getLogPath, closeLogger, isStreamWri import { queryLogLines } from "./web/logs-query.js"; import { configFile, defaultLogFile, stateDir } from "./paths.js"; import { atomicWriteInstanceFile, clearProxyInstanceFile, classifySameLanePeer, entryScriptFingerprint, entryShape, findSameLanePredecessor, findVerifiedSameLaneTwin, isPidAlive, listInstances, probePid, readProxyInstanceFile, registerInstanceAndWarn, sameInstanceShape, unregisterInstance, warnOnNewPeers, writeSpawnPortLog, type InstanceShape, type PidProbe, type ProxyInstanceFile, type RegistryEntry, type SameLanePeer } from "./instance.js"; -import { createServedMarkerWriter, reapStaleWriterMarkers, writerMarkerDir, type ServedMarkerWriter } from "./served-markers.js"; -import { recordSplitBrainContest } from "./storage/runtime-status.js"; +import { createServedMarkerWriter, findContestedWriters, reapStaleWriterMarkers, writerMarkerDir, type ServedMarkerWriter } from "./served-markers.js"; +import { recordSplitBrainContest, registerSplitBrainEvidenceCheck } from "./storage/runtime-status.js"; import { mayClobberPointer, probeOwnerInfo, resolvePortClaim } from "./port-owner.js"; import { hoistTrappedToolItems } from "./tool-pair-order.js"; import { computeAnthropicMessageMarks, stampAnthropicSystemCacheControl, anthropicToolsCarryCacheControl } from "./loop/cache-control.js"; @@ -1231,6 +1231,12 @@ export async function startServer(opts: ProxyOptions): Promise { }, ); reapStaleWriterMarkers(markerDir, Date.now()); + // #4626: a surfaced verdict re-evaluates against CURRENT marker + // evidence at every status read — reuse of findContestedWriters + // keeps ONE definition of "still contested", so the restart- + // handover notice clears itself once the peer provably stops. + registerSplitBrainEvidenceCheck((contest) => + findContestedWriters(markerDir, instanceId, contest.sessionId, Date.now()).some((p) => p.instanceId === contest.peer.instanceId)); setGlobalSessionCommitListener((id) => servedWriter?.noteServed(id)); } const nOverrides = Object.keys(opts.routes).length; diff --git a/src/storage/runtime-status.ts b/src/storage/runtime-status.ts index f2c0aee365..1b683f3997 100644 --- a/src/storage/runtime-status.ts +++ b/src/storage/runtime-status.ts @@ -137,6 +137,18 @@ export function ackSplitBrainContest(sessionId: string): boolean { const MAX_SPLIT_BRAIN_ACKS = 64; +// #4626: the stored verdict must be a function of CURRENT evidence, not a +// replay of one moment's snapshot — at detection time the restart-handover +// tombstone is seconds fresh, but it must not pin the notice forever. The +// server registers the same writer-marker scan that produced the verdict +// (findContestedWriters); without a registered check the stored entry is +// replayed as-is (legacy behavior, standalone record/ack use). +let splitBrainEvidenceCheck: ((contest: SplitBrainContestInput) => boolean) | null = null; + +export function registerSplitBrainEvidenceCheck(check: ((contest: SplitBrainContestInput) => boolean) | null): void { + splitBrainEvidenceCheck = check; +} + /** #4175: the last successful VACUUM INTO backup. */ export function recordBackupOutcome(lastPath: string, lastBytes: number): void { state.backup = { lastPath, lastAt: Date.now(), lastBytes }; @@ -145,6 +157,7 @@ export function recordBackupOutcome(lastPath: string, lastBytes: number): void { export function _resetStorageRuntimeStatusForTest(): void { state = freshState(); cooldownCache = null; + splitBrainEvidenceCheck = null; } /** #4175: true when a pid belongs to a live process (EPERM = alive, foreign user). */ @@ -271,6 +284,13 @@ function getStorageRuntimeStatus(): StorageRuntimeStatus { state.secondProcess = null; loggerLog("info", `[storage] second-process pid ${sp.pid} has exited — stale warning cleared`); } + // #4626: same never-freeze discipline as above — once the peer no longer + // contests (its marker evidence is gone), the stored verdict clears itself. + const sb = state.splitBrain; + if (sb !== null && splitBrainEvidenceCheck !== null && !splitBrainEvidenceCheck(sb)) { + state.splitBrain = null; + loggerLog("info", `[split-brain] session ${sb.sessionId} notice cleared — peer ${sb.peer.instanceId} (pid ${sb.peer.pid}) no longer contests it`); + } return { requested, effective, diff --git a/tests/issue-split-brain-surface.test.ts b/tests/issue-split-brain-surface.test.ts index e6b368cdeb..07bd6051ed 100644 --- a/tests/issue-split-brain-surface.test.ts +++ b/tests/issue-split-brain-surface.test.ts @@ -12,6 +12,9 @@ // G2 the surfaced entry is dismissable (POST /__bili/storage/split-brain/ // ack): ack clears it, the SAME contest stays quiet, a NEW contest // re-surfaces, an unknown session answers 400. +// G3 the stored verdict never outlives its evidence (#4626): it persists +// while the peer's marker evidence holds and self-clears at the status +// read once the peer provably stops contesting (both directions). import test from "node:test"; import assert from "node:assert/strict"; import http from "node:http"; @@ -23,8 +26,8 @@ import { spawn, type ChildProcess } from "node:child_process"; import { startServer } from "../src/server.ts"; import { UnifiedStore } from "../src/storage/store.ts"; import { probeBuiltinSqlite, probeHostBetterSqlite3, setPreferredSqliteEngineForTests, setSqliteEnginesUnavailableForTests } from "../src/storage/driver.ts"; -import { _resetStorageRuntimeStatusForTest, ackSplitBrainContest, recordSecondProcessOwner, recordSplitBrainContest, storageStatusWire, type SplitBrainContestInput } from "../src/storage/runtime-status.ts"; -import { createServedMarkerWriter, writerMarkerDir, type WriterIdentity } from "../src/served-markers.ts"; +import { _resetStorageRuntimeStatusForTest, ackSplitBrainContest, recordSecondProcessOwner, recordSplitBrainContest, registerSplitBrainEvidenceCheck, storageStatusWire, type SplitBrainContestInput } from "../src/storage/runtime-status.ts"; +import { createServedMarkerWriter, findContestedWriters, writerMarkerDir, type WriterIdentity } from "../src/served-markers.ts"; import { isPidAlive } from "../src/instance.ts"; import { setLogCapture } from "../src/logger.ts"; import { renderPage } from "../src/web/page.ts"; @@ -114,6 +117,21 @@ function writeInstanceEntry(stateRoot: string, id: string, pid: number, lane?: s fs.writeFileSync(path.join(dir, `${id}.json`), JSON.stringify(entry)); } +// Beyond any sane pid_max (Linux default 4194304) — provably dead. +const DEAD_PID = 999_999_999; + +function writePeerMarker(dir: string, id: string, pid: number, servedAt: Record): string { + fs.mkdirSync(dir, { recursive: true }); + const file = path.join(dir, `${id}.json`); + fs.writeFileSync(file, JSON.stringify({ v: 1, instanceId: id, pid, origin: "http://127.0.0.1:8787", version: "0.1.205", lane: "dsh", servedAt })); + return file; +} + +function backdate(file: string, ageMs: number): void { + const t = (Date.now() - ageMs) / 1000; + fs.utimesSync(file, t, t); +} + interface SplitBrainWire { sessionId: string; title?: string; @@ -304,6 +322,81 @@ test("#3959 G2: ack clears the surfaced entry; same contest stays quiet, a new o _resetStorageRuntimeStatusForTest(); }); +// ---- #4626: the stored verdict must not outlive its evidence ------------------ + +test("#4626 G3: stored verdict persists while evidence holds, self-clears once the peer stops contesting", () => { + _resetStorageRuntimeStatusForTest(); + const logs: string[] = []; + setLogCapture((level, msg) => logs.push(`${level} ${msg}`)); + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "bili-4626-g3-")); + try { + const dir = writerMarkerDir(tmp); + const base: SplitBrainContestInput = { + sessionId: "sess-stale", + title: "Fix auth bug", + self: { instanceId: "self-1", pid: process.pid, origin: "http://127.0.0.1:18802", lane: "pi", version: "0.1.205" }, + peer: { instanceId: "peer-gone", pid: DEAD_PID, origin: "http://127.0.0.1:8787", lane: "dsh", version: "0.1.204", servedAt: Date.now() - 1000 }, + at: Date.now(), + }; + // The server's registration shape (src/server.ts): the verdict holds iff + // the SAME scan that raised it still reports the accused peer. + registerSplitBrainEvidenceCheck((c) => + findContestedWriters(dir, "self-1", c.sessionId, Date.now()).some((p) => p.instanceId === c.peer.instanceId)); + + // Phase A — fresh tombstone (dead pid, marker just rewritten by the + // peer's graceful-shutdown flush): the notice MUST persist; a dead pid + // alone is not proof of absence on every platform. + const file = writePeerMarker(dir, "peer-gone", DEAD_PID, { [base.sessionId]: base.peer.servedAt }); + recordSplitBrainContest(base); + assert.equal(storageStatusWire().splitBrain?.sessionId, "sess-stale", "fresh tombstone keeps the notice honest"); + + // Phase B — dead pid + marker older than the freshness grace: the next + // status read clears the stored verdict by itself (#4626). + backdate(file, 40_000); + assert.equal(storageStatusWire().splitBrain, null, "dead peer + stale marker clears the stored verdict at the status read"); + assert.ok(logs.some((l) => l.startsWith("info ") && l.includes("[split-brain] session sess-stale") && l.includes("peer-gone") && l.includes(String(DEAD_PID))), "the clear is logged with full ids"); + assert.equal(storageStatusWire().splitBrain, null, "stays cleared on subsequent reads"); + + // Phase C — an evidence-based clear must NOT poison the ack guard: if + // the same peer contests again, the notice surfaces again. + backdate(file, 0); + recordSplitBrainContest(base); + assert.equal(storageStatusWire().splitBrain?.sessionId, "sess-stale", "a renewed contest re-surfaces after an evidence-based clear"); + } finally { + setLogCapture(null); + _resetStorageRuntimeStatusForTest(); + rmrf(tmp); + } +}); + +test("#4626 G3-negative: without a registered check the stored verdict stands (legacy replay)", () => { + _resetStorageRuntimeStatusForTest(); + const base: SplitBrainContestInput = { + sessionId: "sess-legacy", + self: { instanceId: "self-1", pid: process.pid, origin: "http://127.0.0.1:18802", lane: "pi", version: "0.1.205" }, + peer: { instanceId: "peer-x", pid: DEAD_PID, origin: "http://127.0.0.1:8787", lane: "dsh", version: "0.1.204", servedAt: Date.now() - 1000 }, + at: Date.now(), + }; + try { + recordSplitBrainContest(base); + assert.equal(storageStatusWire().splitBrain?.sessionId, "sess-legacy", "no registered check ⇒ stored entry replays as-is"); + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "bili-4626-g3n-")); + try { + const dir = writerMarkerDir(tmp); + writePeerMarker(dir, "peer-live", process.pid, { [base.sessionId]: Date.now() - 1000 }); + registerSplitBrainEvidenceCheck((c) => + findContestedWriters(dir, "self-1", c.sessionId, Date.now()).some((p) => p.instanceId === c.peer.instanceId)); + const live: SplitBrainContestInput = { ...base, peer: { ...base.peer, instanceId: "peer-live", pid: process.pid } }; + recordSplitBrainContest(live); + assert.equal(storageStatusWire().splitBrain?.sessionId, "sess-legacy", "live peer + fresh marker keeps the notice surfaced"); + } finally { + rmrf(tmp); + } + } finally { + _resetStorageRuntimeStatusForTest(); + } +}); + // ---- web surface: wire + endpoint + page/client/i18n --------------------------- test("#3959 web: status carries secondProcess/splitBrain; ack endpoint clears; page/client/i18n wired", async () => { @@ -322,11 +415,17 @@ test("#3959 web: status carries secondProcess/splitBrain; ack endpoint clears; p assert.equal(s.storage.secondProcess, null, "no second process recorded → null"); assert.equal(s.storage.splitBrain, null, "no contest recorded → null"); + // #4626: surfaced entries re-evaluate against CURRENT writer-marker + // evidence at each status read — give the recorded contest real + // evidence in the server's writers dir (fresh mtime keeps it contested + // even though DEAD_PID is provably not a live process). + const peerMarker = writePeerMarker(path.join(base, "state", "billion-context", "writers"), "peer-1", DEAD_PID, { "sess-web": Date.now() - 1000 }); + const contest: SplitBrainContestInput = { sessionId: "sess-web", title: "Fix auth bug", self: { instanceId: "self-1", pid: process.pid, origin: "http://127.0.0.1:18802", lane: "pi", version: "0.1.202" }, - peer: { instanceId: "peer-1", pid: 4242, origin: "http://127.0.0.1:8787", lane: "dsh", version: "0.1.201", servedAt: 1000 }, + peer: { instanceId: "peer-1", pid: DEAD_PID, origin: "http://127.0.0.1:8787", lane: "dsh", version: "0.1.201", servedAt: 1000 }, at: Date.now(), }; recordSplitBrainContest(contest); @@ -334,7 +433,7 @@ test("#3959 web: status carries secondProcess/splitBrain; ack endpoint clears; p assert.equal(s.storage.splitBrain?.sessionId, "sess-web", "status wire carries the surfaced contest"); assert.equal(s.storage.splitBrain?.title, "Fix auth bug"); assert.equal(s.storage.splitBrain?.peer.lane, "dsh"); - assert.equal(s.storage.splitBrain?.peer.pid, 4242); + assert.equal(s.storage.splitBrain?.peer.pid, DEAD_PID); const acked = await postJson(port, "/__bili/storage/split-brain/ack", { sessionId: "sess-web" }); assert.equal(acked.status, 200, "ack of the surfaced session answers 200"); @@ -351,6 +450,13 @@ test("#3959 web: status carries secondProcess/splitBrain; ack endpoint clears; p s = await fetchJson<{ storage: StorageWireShape }>(port, "/__bili/status"); assert.equal(s.storage.splitBrain?.sessionId, "sess-web", "a NEW contest re-surfaces on the wire after ack"); + // #4626 end-to-end: once the peer's marker is past its freshness grace + // and its pid is provably dead, the next status read clears the notice + // WITHOUT a manual ack. + backdate(peerMarker, 40_000); + s = await fetchJson<{ storage: StorageWireShape }>(port, "/__bili/status"); + assert.equal(s.storage.splitBrain, null, "stale verdict self-clears at the status read without a manual ack"); + const html = renderPage("http://127.0.0.1:1", "0.0.0"); assert.ok(html.includes('id="split-brain-banner"'), "page carries the split-brain banner div"); assert.ok(WEB_CLIENT.includes('"split-brain-banner"'), "client targets the banner element"); From cf6994af3394060ff7a18ebbe8030dbc0b0c3e80 Mon Sep 17 00:00:00 2001 From: ci-check Date: Sun, 25 Oct 2026 08:04:05 +0800 Subject: [PATCH 2/4] fix: show the full session id in the split-brain ack log and web card (#4626) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit slice(0, 8) of a "session-…" id renders literally "session-" — the ack log line and the web banner both showed an unusable prefix. Log the full id (the [split-brain] error line already does) and give the card a 24-char prefix so distinct sessions are tellable apart. --- src/server/admin.ts | 3 ++- src/web/client.ts | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/server/admin.ts b/src/server/admin.ts index bd0fb963e9..67f702b56d 100644 --- a/src/server/admin.ts +++ b/src/server/admin.ts @@ -981,7 +981,8 @@ async function handleSplitBrainAck(req: http.IncomingMessage, res: http.ServerRe res.end(JSON.stringify({ error: `no surfaced split-brain entry for session: ${sessionId}` }, null, 2)); return; } - log("info", `[storage] split-brain notice for session ${sessionId.slice(0, 8)} acknowledged via web UI`); + // #4626: full id — .slice(0, 8) of "session-…" renders literally "session-" + log("info", `[storage] split-brain notice for session ${sessionId} acknowledged via web UI`); res.writeHead(200, { "content-type": "application/json" }); res.end(JSON.stringify({ ok: true }, null, 2)); } diff --git a/src/web/client.ts b/src/web/client.ts index 7556a9efb2..e3d0b4e5ac 100644 --- a/src/web/client.ts +++ b/src/web/client.ts @@ -1223,10 +1223,12 @@ function changelogCatOf(rawLine) { sbb.hidden = false; sbb.classList.add("show"); const laneOf = (x) => (x && typeof x.lane === "string" && x.lane ? x.lane : t("sys.split_brain_lane_undeclared")); + // #4626: .slice(0, 8) of "session-…" is literally "session-" — show enough to tell sessions apart + const shortSid = sb.sessionId.length > 24 ? sb.sessionId.slice(0, 24) + "…" : sb.sessionId; sbb.innerHTML = "" + escapeHtml(t("sys.split_brain_title")) + " " + escapeHtml(t("sys.split_brain", { - title: typeof sb.title === "string" && sb.title ? sb.title : sb.sessionId.slice(0, 8), - sid: sb.sessionId.slice(0, 8), + title: typeof sb.title === "string" && sb.title ? sb.title : shortSid, + sid: shortSid, selfPid: String(sb.self.pid), selfLane: laneOf(sb.self), peerPid: String(sb.peer.pid), From 8f51ea084f3a6518d0c47eb1b30b22cfe19586ac Mon Sep 17 00:00:00 2001 From: ci-check Date: Mon, 26 Oct 2026 17:47:07 +0800 Subject: [PATCH 3/4] fix: stop flagging gracefully-shutdown peers as split-brain writers (#4626) --- src/served-markers.ts | 36 ++++++++++++++++++++++++++++-------- tests/served-markers.test.ts | 29 +++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 8 deletions(-) diff --git a/src/served-markers.ts b/src/served-markers.ts index 6ae0aa8e25..a2f0d6bafa 100644 --- a/src/served-markers.ts +++ b/src/served-markers.ts @@ -11,6 +11,11 @@ // names both sides in an error line when it commits a session another live // instance committed within the window. Pure observability: no routing, // locking, or serving behavior changes. +// #4626: an instance that shut down GRACEFULLY stamps its final marker with +// `disposedAt`; such a peer is attribution-only and can never be flagged as +// a concurrent writer — its pid is provably gone by construction (it ran our +// shutdown path), so the freshness grace below stays reserved for peers that +// died without it (crash/kill) where the pid probe may be blind. import fs from "node:fs"; import path from "node:path"; @@ -35,6 +40,11 @@ interface WriterMarkerFile { lane?: string; /** sessionId → last commit time (ms epoch). */ servedAt: Record; + /** Set only on the final marker write of a graceful shutdown (#4626): + * the instance is provably gone — peers keep the file for attribution + * but never treat it as a concurrent writer. Additive; markers from + * older builds carry none. */ + disposedAt?: number; } /** A surfaced split-brain contest: which session (id + resolved title) and @@ -68,7 +78,9 @@ const MAX_TRACKED_SESSIONS = 128; const VERDICT_TTL_MS = 60_000; /** A peer counts as live when its pid is alive OR its marker was written * within this window (covers platforms where the pid check cannot see the - * peer process). */ + * peer process). Gracefully-shutdown instances are excluded BEFORE this + * test via their disposedAt stamp (#4626) — their death is certain, so no + * grace applies. */ const PEER_FILE_FRESH_MS = 30_000; /** reapStaleWriterMarkers removes files whose pid is dead AND whose mtime is * older than this. */ @@ -127,7 +139,7 @@ export function createServedMarkerWriter(identity: WriterIdentity, dir: string, } }; - const writeMarker = (): void => { + const writeMarker = (final: boolean): void => { const now = Date.now(); prune(now); const body: WriterMarkerFile = { @@ -138,6 +150,7 @@ export function createServedMarkerWriter(identity: WriterIdentity, dir: string, version: identity.version, ...(identity.lane === undefined || identity.lane === "" ? {} : { lane: identity.lane }), servedAt: Object.fromEntries(servedAt), + ...(final ? { disposedAt: now } : {}), }; try { fs.mkdirSync(dir, { recursive: true }); @@ -152,7 +165,7 @@ export function createServedMarkerWriter(identity: WriterIdentity, dir: string, flushTimer = undefined; if (disposed) return; lastFlushAt = Date.now(); - writeMarker(); + writeMarker(false); }; const scheduleFlush = (): void => { @@ -172,7 +185,7 @@ export function createServedMarkerWriter(identity: WriterIdentity, dir: string, const title = opts?.resolveTitle?.(sessionId); const laneOf = (x: WriterIdentity): string => (x.lane === undefined || x.lane === "" ? "" : `, lane ${x.lane}`); const titleOf = title === undefined || title === "" ? "" : ` "${title}"`; - log("error", `[split-brain] session ${sessionId}${titleOf} committed by BOTH this instance (pid ${identity.pid}, ${identity.origin}${laneOf(identity)}, v${identity.version}) and another live instance (pid ${peer.pid}, ${peer.origin}${laneOf(peer)}, v${peer.version}) within ${CONTEST_WINDOW_MS / 60000}m — fold decisions are invisible across instances and the last writer wins (#394/#405); stop one of them or align your client config so one conversation rides one proxy (#3923)`); + log("error", `[split-brain] session ${sessionId}${titleOf} committed by BOTH this instance (pid ${identity.pid}, ${identity.origin}${laneOf(identity)}, v${identity.version}) and another instance (pid ${peer.pid}, ${peer.origin}${laneOf(peer)}, v${peer.version}) within ${CONTEST_WINDOW_MS / 60000}m — fold decisions are invisible across instances and the last writer wins (#394/#405); stop one of them or align your client config so one conversation rides one proxy (#3923)`); opts?.onContest?.({ sessionId, ...(title === undefined || title === "" ? {} : { title }), self: identity, peer, at: now }); } }; @@ -189,8 +202,10 @@ export function createServedMarkerWriter(identity: WriterIdentity, dir: string, disposed = true; if (flushTimer !== undefined) clearTimeout(flushTimer); // final write (bypasses the disposed guard): peers should still be - // able to attribute this instance's last commits after shutdown - writeMarker(); + // able to attribute this instance's last commits after shutdown; + // the disposedAt stamp (#4626) keeps that attribution while + // excluding us from every peer's contest scan + writeMarker(true); }, __flushForTest(): void { flush(); @@ -202,8 +217,10 @@ export function createServedMarkerWriter(identity: WriterIdentity, dir: string, } /** Peer marker files that record a commit of `sessionId` inside the contest - * window by a LIVE foreign instance (pid alive or file fresh). Malformed or - * tearing files are skipped silently — a marker is observability, not data. */ + * window by a LIVE foreign instance (pid alive or file fresh). Markers + * stamped `disposedAt` by a gracefully-shutdown peer are attribution-only + * and never contested (#4626). Malformed or tearing files are skipped + * silently — a marker is observability, not data. */ export function findContestedWriters(dir: string, selfInstanceId: string, sessionId: string, now: number): Array { let entries: fs.Dirent[]; try { @@ -218,6 +235,9 @@ export function findContestedWriters(dir: string, selfInstanceId: string, sessio try { const raw = JSON.parse(fs.readFileSync(file, "utf8")) as Partial; if (raw.v !== 1 || typeof raw.instanceId !== "string" || raw.instanceId === selfInstanceId) continue; + // #4626: gracefully-shutdown peer — attribution only, never a + // concurrent writer (its death is certain; no freshness grace). + if (typeof raw.disposedAt === "number") continue; const at = raw.servedAt !== null && typeof raw.servedAt === "object" ? (raw.servedAt as Record)[sessionId] : undefined; if (typeof at !== "number" || now - at > CONTEST_WINDOW_MS) continue; const mtimeMs = fs.statSync(file).mtimeMs; diff --git a/tests/served-markers.test.ts b/tests/served-markers.test.ts index 93f75d662b..0183ace9e3 100644 --- a/tests/served-markers.test.ts +++ b/tests/served-markers.test.ts @@ -111,6 +111,35 @@ test("dead-pid peer counts while its marker is fresh, stops counting once stale" assert.equal(found.length, 0, "dead pid + stale mtime = gone"); }); +test("#4626 gracefully-shutdown peer (disposedAt stamp) never contests, even with fresh mtime", () => { + const dir = writerMarkerDir(freshDir("bili-4626-disposed-")); + const deadPeer: WriterIdentity = { ...PEER, pid: DEAD_PID }; + fs.mkdirSync(dir, { recursive: true }); + // The exact shape dispose() leaves behind: dead pid, mtime ≈ death time, + // a commit inside the window seconds earlier. + const file = path.join(dir, `${deadPeer.instanceId}.json`); + fs.writeFileSync(file, JSON.stringify({ v: 1, ...deadPeer, servedAt: { "sess-h": Date.now() - 1000 }, disposedAt: Date.now() - 9000 })); + backdate(file, 8000); + assert.deepEqual(findContestedWriters(dir, SELF.instanceId, "sess-h", Date.now()), [], "handover tombstone is attribution-only — no false split-brain verdict"); +}); + +test("#4626 e2e: graceful rotation — successor commits the same session without a verdict", () => { + const dir = writerMarkerDir(freshDir("bili-4626-handover-")); + const logs: Array<[string, string]> = []; + const outgoing = createServedMarkerWriter(PEER, dir, (level, msg) => logs.push([level, msg])); + outgoing.noteServed("sess-rot"); + outgoing.__flushForTest(); + const mid = JSON.parse(fs.readFileSync(path.join(dir, "peer-1.json"), "utf8")) as Record; + assert.equal(mid.disposedAt, undefined, "periodic flushes carry no disposedAt stamp"); + outgoing.dispose(); + const final = JSON.parse(fs.readFileSync(path.join(dir, "peer-1.json"), "utf8")) as Record; + assert.equal(typeof final.disposedAt, "number", "the final shutdown write is stamped disposedAt"); + const incoming = createServedMarkerWriter(SELF, dir, (level, msg) => logs.push([level, msg])); + incoming.noteServed("sess-rot"); + assert.equal(logs.length, 0, "sequential handover inside the 10m window raises no [split-brain] line"); + incoming.dispose(); +}); + test("malformed or foreign-shape marker files are skipped, never fatal", () => { const dir = writerMarkerDir(freshDir("bili-3923-malformed-")); fs.mkdirSync(dir, { recursive: true }); From 4368daf64310d7600b7ccad6e18b883fc3b094b0 Mon Sep 17 00:00:00 2001 From: ci-check Date: Mon, 26 Oct 2026 09:20:57 +0800 Subject: [PATCH 4/4] fix: fail the split-brain evidence check closed when marker evidence is unreadable (#4626) --- src/served-markers.ts | 62 +++++++++++++++++++++---- src/server.ts | 11 +++-- src/storage/runtime-status.ts | 20 ++++---- tests/issue-split-brain-surface.test.ts | 51 ++++++++++++++++++-- 4 files changed, 119 insertions(+), 25 deletions(-) diff --git a/src/served-markers.ts b/src/served-markers.ts index a2f0d6bafa..c6e30bde74 100644 --- a/src/served-markers.ts +++ b/src/served-markers.ts @@ -216,6 +216,21 @@ export function createServedMarkerWriter(identity: WriterIdentity, dir: string, }; } +/** One definition of "still contests" (#4626): the contested `servedAt` + * timestamp, or null when the marker does not contest — shared by the dir + * scan below and the direct-path clear check, so detection and clearing can + * never drift apart. */ +function contestedServedAt(raw: Partial, selfInstanceId: string, sessionId: string, mtimeMs: number, now: number): number | null { + if (raw.v !== 1 || typeof raw.instanceId !== "string" || raw.instanceId === selfInstanceId) return null; + // #4626: gracefully-shutdown peer — attribution only, never a concurrent + // writer (its death is certain; no freshness grace). + if (typeof raw.disposedAt === "number") return null; + const at = raw.servedAt !== null && typeof raw.servedAt === "object" ? (raw.servedAt as Record)[sessionId] : undefined; + if (typeof at !== "number" || now - at > CONTEST_WINDOW_MS) return null; + const alive = typeof raw.pid === "number" ? isPidAlive(raw.pid) : false; + return alive || now - mtimeMs <= PEER_FILE_FRESH_MS ? at : null; +} + /** Peer marker files that record a commit of `sessionId` inside the contest * window by a LIVE foreign instance (pid alive or file fresh). Markers * stamped `disposedAt` by a gracefully-shutdown peer are attribution-only @@ -234,15 +249,9 @@ export function findContestedWriters(dir: string, selfInstanceId: string, sessio const file = path.join(dir, entry.name); try { const raw = JSON.parse(fs.readFileSync(file, "utf8")) as Partial; - if (raw.v !== 1 || typeof raw.instanceId !== "string" || raw.instanceId === selfInstanceId) continue; - // #4626: gracefully-shutdown peer — attribution only, never a - // concurrent writer (its death is certain; no freshness grace). - if (typeof raw.disposedAt === "number") continue; - const at = raw.servedAt !== null && typeof raw.servedAt === "object" ? (raw.servedAt as Record)[sessionId] : undefined; - if (typeof at !== "number" || now - at > CONTEST_WINDOW_MS) continue; const mtimeMs = fs.statSync(file).mtimeMs; - const alive = typeof raw.pid === "number" ? isPidAlive(raw.pid) : false; - if (!alive && now - mtimeMs > PEER_FILE_FRESH_MS) continue; + const at = contestedServedAt(raw, selfInstanceId, sessionId, mtimeMs, now); + if (at === null || typeof raw.instanceId !== "string") continue; out.push({ instanceId: raw.instanceId, pid: typeof raw.pid === "number" ? raw.pid : -1, @@ -258,6 +267,43 @@ export function findContestedWriters(dir: string, selfInstanceId: string, sessio return out; } +/** #4626 fail-closed companion to findContestedWriters: does the ACCUSED + * peer's own marker still contest `sessionId`? true = still contested, + * false = provably stopped (marker reaped/absent, dead pid past the + * freshness grace, or the contest window lapsed), null = UNKNOWN (dir or + * file unreadable, torn mid-write) — a transient FS error must never drop a + * real notice, so callers keep it on null. Reads one file, not the dir. */ +export function peerStillContests(dir: string, selfInstanceId: string, peerInstanceId: string, sessionId: string, now: number): boolean | null { + let dirStat: fs.Stats; + try { + dirStat = fs.statSync(dir); + } catch { + return null; + } + if (!dirStat.isDirectory()) return null; + const file = path.join(dir, `${peerInstanceId}.json`); + let content: string; + try { + content = fs.readFileSync(file, "utf8"); + } catch (err) { + return (err as NodeJS.ErrnoException).code === "ENOENT" ? false : null; + } + let raw: Partial; + try { + raw = JSON.parse(content) as Partial; + } catch { + return null; + } + if (raw.instanceId !== peerInstanceId) return null; + let mtimeMs: number; + try { + mtimeMs = fs.statSync(file).mtimeMs; + } catch { + return null; + } + return contestedServedAt(raw, selfInstanceId, sessionId, mtimeMs, now) !== null; +} + /** Startup hygiene: drop marker files whose instance is provably gone (pid * dead) and stale beyond the reap horizon. Fresh files from a just-crashed * peer stay — findContestedWriters' freshness test still honors them for diff --git a/src/server.ts b/src/server.ts index bbe9df668b..758b048000 100644 --- a/src/server.ts +++ b/src/server.ts @@ -99,7 +99,7 @@ import { log as loggerLog, configureLogger, getLogPath, closeLogger, isStreamWri import { queryLogLines } from "./web/logs-query.js"; import { configFile, defaultLogFile, stateDir } from "./paths.js"; import { atomicWriteInstanceFile, clearProxyInstanceFile, classifySameLanePeer, entryScriptFingerprint, entryShape, findSameLanePredecessor, findVerifiedSameLaneTwin, isPidAlive, listInstances, probePid, readProxyInstanceFile, registerInstanceAndWarn, sameInstanceShape, unregisterInstance, warnOnNewPeers, writeSpawnPortLog, type InstanceShape, type PidProbe, type ProxyInstanceFile, type RegistryEntry, type SameLanePeer } from "./instance.js"; -import { createServedMarkerWriter, findContestedWriters, reapStaleWriterMarkers, writerMarkerDir, type ServedMarkerWriter } from "./served-markers.js"; +import { createServedMarkerWriter, peerStillContests, reapStaleWriterMarkers, writerMarkerDir, type ServedMarkerWriter } from "./served-markers.js"; import { recordSplitBrainContest, registerSplitBrainEvidenceCheck } from "./storage/runtime-status.js"; import { mayClobberPointer, probeOwnerInfo, resolvePortClaim } from "./port-owner.js"; import { hoistTrappedToolItems } from "./tool-pair-order.js"; @@ -1232,11 +1232,12 @@ export async function startServer(opts: ProxyOptions): Promise { ); reapStaleWriterMarkers(markerDir, Date.now()); // #4626: a surfaced verdict re-evaluates against CURRENT marker - // evidence at every status read — reuse of findContestedWriters - // keeps ONE definition of "still contested", so the restart- - // handover notice clears itself once the peer provably stops. + // evidence at every status read — the accused peer's own marker, + // judged by the same "still contests" definition as detection, so + // the restart-handover notice clears itself once the peer + // PROVABLY stops; an unreadable marker fails closed (keeps it). registerSplitBrainEvidenceCheck((contest) => - findContestedWriters(markerDir, instanceId, contest.sessionId, Date.now()).some((p) => p.instanceId === contest.peer.instanceId)); + peerStillContests(markerDir, instanceId, contest.peer.instanceId, contest.sessionId, Date.now())); setGlobalSessionCommitListener((id) => servedWriter?.noteServed(id)); } const nOverrides = Object.keys(opts.routes).length; diff --git a/src/storage/runtime-status.ts b/src/storage/runtime-status.ts index 1b683f3997..fff3bd91c9 100644 --- a/src/storage/runtime-status.ts +++ b/src/storage/runtime-status.ts @@ -140,12 +140,15 @@ const MAX_SPLIT_BRAIN_ACKS = 64; // #4626: the stored verdict must be a function of CURRENT evidence, not a // replay of one moment's snapshot — at detection time the restart-handover // tombstone is seconds fresh, but it must not pin the notice forever. The -// server registers the same writer-marker scan that produced the verdict -// (findContestedWriters); without a registered check the stored entry is -// replayed as-is (legacy behavior, standalone record/ack use). -let splitBrainEvidenceCheck: ((contest: SplitBrainContestInput) => boolean) | null = null; +// server registers the evidence check (the accused peer's own marker, judged +// by the same "still contests" definition as detection); without a registered +// check the stored entry is replayed as-is (legacy behavior, standalone +// record/ack use). The check is TRI-STATE and fails closed: an unknown result +// (the marker could not be read at status-read time) keeps the notice — +// dropping a real notice on a transient FS error is the wrong direction. +let splitBrainEvidenceCheck: ((contest: SplitBrainContestInput) => boolean | null) | null = null; -export function registerSplitBrainEvidenceCheck(check: ((contest: SplitBrainContestInput) => boolean) | null): void { +export function registerSplitBrainEvidenceCheck(check: ((contest: SplitBrainContestInput) => boolean | null) | null): void { splitBrainEvidenceCheck = check; } @@ -284,10 +287,11 @@ function getStorageRuntimeStatus(): StorageRuntimeStatus { state.secondProcess = null; loggerLog("info", `[storage] second-process pid ${sp.pid} has exited — stale warning cleared`); } - // #4626: same never-freeze discipline as above — once the peer no longer - // contests (its marker evidence is gone), the stored verdict clears itself. + // #4626: same never-freeze discipline as above — once the peer PROVABLY + // stops contesting, the stored verdict clears itself; an unknown verdict + // (marker unreadable at read time) keeps it. const sb = state.splitBrain; - if (sb !== null && splitBrainEvidenceCheck !== null && !splitBrainEvidenceCheck(sb)) { + if (sb !== null && splitBrainEvidenceCheck !== null && splitBrainEvidenceCheck(sb) === false) { state.splitBrain = null; loggerLog("info", `[split-brain] session ${sb.sessionId} notice cleared — peer ${sb.peer.instanceId} (pid ${sb.peer.pid}) no longer contests it`); } diff --git a/tests/issue-split-brain-surface.test.ts b/tests/issue-split-brain-surface.test.ts index 07bd6051ed..e57708e7c4 100644 --- a/tests/issue-split-brain-surface.test.ts +++ b/tests/issue-split-brain-surface.test.ts @@ -15,6 +15,9 @@ // G3 the stored verdict never outlives its evidence (#4626): it persists // while the peer's marker evidence holds and self-clears at the status // read once the peer provably stops contesting (both directions). +// G4 the evidence check fails closed (#4626): an unreadable writers dir or +// torn peer marker is UNKNOWN and keeps the notice; only provable +// absence/staleness clears. import test from "node:test"; import assert from "node:assert/strict"; import http from "node:http"; @@ -27,7 +30,7 @@ import { startServer } from "../src/server.ts"; import { UnifiedStore } from "../src/storage/store.ts"; import { probeBuiltinSqlite, probeHostBetterSqlite3, setPreferredSqliteEngineForTests, setSqliteEnginesUnavailableForTests } from "../src/storage/driver.ts"; import { _resetStorageRuntimeStatusForTest, ackSplitBrainContest, recordSecondProcessOwner, recordSplitBrainContest, registerSplitBrainEvidenceCheck, storageStatusWire, type SplitBrainContestInput } from "../src/storage/runtime-status.ts"; -import { createServedMarkerWriter, findContestedWriters, writerMarkerDir, type WriterIdentity } from "../src/served-markers.ts"; +import { createServedMarkerWriter, peerStillContests, writerMarkerDir, type WriterIdentity } from "../src/served-markers.ts"; import { isPidAlive } from "../src/instance.ts"; import { setLogCapture } from "../src/logger.ts"; import { renderPage } from "../src/web/page.ts"; @@ -339,9 +342,9 @@ test("#4626 G3: stored verdict persists while evidence holds, self-clears once t at: Date.now(), }; // The server's registration shape (src/server.ts): the verdict holds iff - // the SAME scan that raised it still reports the accused peer. + // the accused peer's own marker still contests — tri-state, fails closed. registerSplitBrainEvidenceCheck((c) => - findContestedWriters(dir, "self-1", c.sessionId, Date.now()).some((p) => p.instanceId === c.peer.instanceId)); + peerStillContests(dir, "self-1", c.peer.instanceId, c.sessionId, Date.now())); // Phase A — fresh tombstone (dead pid, marker just rewritten by the // peer's graceful-shutdown flush): the notice MUST persist; a dead pid @@ -385,7 +388,7 @@ test("#4626 G3-negative: without a registered check the stored verdict stands (l const dir = writerMarkerDir(tmp); writePeerMarker(dir, "peer-live", process.pid, { [base.sessionId]: Date.now() - 1000 }); registerSplitBrainEvidenceCheck((c) => - findContestedWriters(dir, "self-1", c.sessionId, Date.now()).some((p) => p.instanceId === c.peer.instanceId)); + peerStillContests(dir, "self-1", c.peer.instanceId, c.sessionId, Date.now())); const live: SplitBrainContestInput = { ...base, peer: { ...base.peer, instanceId: "peer-live", pid: process.pid } }; recordSplitBrainContest(live); assert.equal(storageStatusWire().splitBrain?.sessionId, "sess-legacy", "live peer + fresh marker keeps the notice surfaced"); @@ -397,6 +400,46 @@ test("#4626 G3-negative: without a registered check the stored verdict stands (l } }); +test("#4626 G4: evidence check fails closed — unknown evidence keeps the notice", () => { + _resetStorageRuntimeStatusForTest(); + const logs: string[] = []; + setLogCapture((level, msg) => logs.push(`${level} ${msg}`)); + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "bili-4626-g4-")); + try { + const dir = writerMarkerDir(tmp); + const base: SplitBrainContestInput = { + sessionId: "sess-fc", + self: { instanceId: "self-1", pid: process.pid, origin: "http://127.0.0.1:18802", lane: "pi", version: "0.1.205" }, + peer: { instanceId: "peer-gone", pid: DEAD_PID, origin: "http://127.0.0.1:8787", lane: "dsh", version: "0.1.204", servedAt: Date.now() - 1000 }, + at: Date.now(), + }; + registerSplitBrainEvidenceCheck((c) => + peerStillContests(dir, "self-1", c.peer.instanceId, c.sessionId, Date.now())); + const file = writePeerMarker(dir, "peer-gone", DEAD_PID, { [base.sessionId]: base.peer.servedAt }); + recordSplitBrainContest(base); + assert.equal(storageStatusWire().splitBrain?.sessionId, "sess-fc"); + + // Writers dir vanishes (state-dir move / transient FS error): UNKNOWN — + // the notice stays and no false "no longer contests" line appears. + fs.rmSync(dir, { recursive: true }); + assert.equal(storageStatusWire().splitBrain?.sessionId, "sess-fc", "missing writers dir ⇒ unknown ⇒ notice kept"); + assert.ok(!logs.some((l) => l.includes("no longer contests")), "an unknown verdict never logs a false clear"); + + // Dir restored but the accused marker reaped: PROVABLY STOPPED — clears. + fs.mkdirSync(dir, { recursive: true }); + assert.equal(storageStatusWire().splitBrain, null, "absent peer marker ⇒ cleared at the status read"); + + // Torn/malformed marker: UNKNOWN again — a renewed verdict stays up. + fs.writeFileSync(file, "{ torn mid-write"); + recordSplitBrainContest(base); + assert.equal(storageStatusWire().splitBrain?.sessionId, "sess-fc", "unreadable peer marker ⇒ unknown ⇒ notice kept"); + } finally { + setLogCapture(null); + _resetStorageRuntimeStatusForTest(); + rmrf(tmp); + } +}); + // ---- web surface: wire + endpoint + page/client/i18n --------------------------- test("#3959 web: status carries secondProcess/splitBrain; ack endpoint clears; page/client/i18n wired", async () => {