diff --git a/src/served-markers.ts b/src/served-markers.ts index 6ae0aa8e25..c6e30bde74 100644 --- a/src/served-markers.ts +++ b/src/served-markers.ts @@ -11,6 +11,11 @@ // names both sides in an error line when it commits a session another live // instance committed within the window. Pure observability: no routing, // locking, or serving behavior changes. +// #4626: an instance that shut down GRACEFULLY stamps its final marker with +// `disposedAt`; such a peer is attribution-only and can never be flagged as +// a concurrent writer — its pid is provably gone by construction (it ran our +// shutdown path), so the freshness grace below stays reserved for peers that +// died without it (crash/kill) where the pid probe may be blind. import fs from "node:fs"; import path from "node:path"; @@ -35,6 +40,11 @@ interface WriterMarkerFile { lane?: string; /** sessionId → last commit time (ms epoch). */ servedAt: Record; + /** Set only on the final marker write of a graceful shutdown (#4626): + * the instance is provably gone — peers keep the file for attribution + * but never treat it as a concurrent writer. Additive; markers from + * older builds carry none. */ + disposedAt?: number; } /** A surfaced split-brain contest: which session (id + resolved title) and @@ -68,7 +78,9 @@ const MAX_TRACKED_SESSIONS = 128; const VERDICT_TTL_MS = 60_000; /** A peer counts as live when its pid is alive OR its marker was written * within this window (covers platforms where the pid check cannot see the - * peer process). */ + * peer process). Gracefully-shutdown instances are excluded BEFORE this + * test via their disposedAt stamp (#4626) — their death is certain, so no + * grace applies. */ const PEER_FILE_FRESH_MS = 30_000; /** reapStaleWriterMarkers removes files whose pid is dead AND whose mtime is * older than this. */ @@ -127,7 +139,7 @@ export function createServedMarkerWriter(identity: WriterIdentity, dir: string, } }; - const writeMarker = (): void => { + const writeMarker = (final: boolean): void => { const now = Date.now(); prune(now); const body: WriterMarkerFile = { @@ -138,6 +150,7 @@ export function createServedMarkerWriter(identity: WriterIdentity, dir: string, version: identity.version, ...(identity.lane === undefined || identity.lane === "" ? {} : { lane: identity.lane }), servedAt: Object.fromEntries(servedAt), + ...(final ? { disposedAt: now } : {}), }; try { fs.mkdirSync(dir, { recursive: true }); @@ -152,7 +165,7 @@ export function createServedMarkerWriter(identity: WriterIdentity, dir: string, flushTimer = undefined; if (disposed) return; lastFlushAt = Date.now(); - writeMarker(); + writeMarker(false); }; const scheduleFlush = (): void => { @@ -172,7 +185,7 @@ export function createServedMarkerWriter(identity: WriterIdentity, dir: string, const title = opts?.resolveTitle?.(sessionId); const laneOf = (x: WriterIdentity): string => (x.lane === undefined || x.lane === "" ? "" : `, lane ${x.lane}`); const titleOf = title === undefined || title === "" ? "" : ` "${title}"`; - log("error", `[split-brain] session ${sessionId}${titleOf} committed by BOTH this instance (pid ${identity.pid}, ${identity.origin}${laneOf(identity)}, v${identity.version}) and another live instance (pid ${peer.pid}, ${peer.origin}${laneOf(peer)}, v${peer.version}) within ${CONTEST_WINDOW_MS / 60000}m — fold decisions are invisible across instances and the last writer wins (#394/#405); stop one of them or align your client config so one conversation rides one proxy (#3923)`); + log("error", `[split-brain] session ${sessionId}${titleOf} committed by BOTH this instance (pid ${identity.pid}, ${identity.origin}${laneOf(identity)}, v${identity.version}) and another instance (pid ${peer.pid}, ${peer.origin}${laneOf(peer)}, v${peer.version}) within ${CONTEST_WINDOW_MS / 60000}m — fold decisions are invisible across instances and the last writer wins (#394/#405); stop one of them or align your client config so one conversation rides one proxy (#3923)`); opts?.onContest?.({ sessionId, ...(title === undefined || title === "" ? {} : { title }), self: identity, peer, at: now }); } }; @@ -189,8 +202,10 @@ export function createServedMarkerWriter(identity: WriterIdentity, dir: string, disposed = true; if (flushTimer !== undefined) clearTimeout(flushTimer); // final write (bypasses the disposed guard): peers should still be - // able to attribute this instance's last commits after shutdown - writeMarker(); + // able to attribute this instance's last commits after shutdown; + // the disposedAt stamp (#4626) keeps that attribution while + // excluding us from every peer's contest scan + writeMarker(true); }, __flushForTest(): void { flush(); @@ -201,9 +216,26 @@ export function createServedMarkerWriter(identity: WriterIdentity, dir: string, }; } +/** One definition of "still contests" (#4626): the contested `servedAt` + * timestamp, or null when the marker does not contest — shared by the dir + * scan below and the direct-path clear check, so detection and clearing can + * never drift apart. */ +function contestedServedAt(raw: Partial, selfInstanceId: string, sessionId: string, mtimeMs: number, now: number): number | null { + if (raw.v !== 1 || typeof raw.instanceId !== "string" || raw.instanceId === selfInstanceId) return null; + // #4626: gracefully-shutdown peer — attribution only, never a concurrent + // writer (its death is certain; no freshness grace). + if (typeof raw.disposedAt === "number") return null; + const at = raw.servedAt !== null && typeof raw.servedAt === "object" ? (raw.servedAt as Record)[sessionId] : undefined; + if (typeof at !== "number" || now - at > CONTEST_WINDOW_MS) return null; + const alive = typeof raw.pid === "number" ? isPidAlive(raw.pid) : false; + return alive || now - mtimeMs <= PEER_FILE_FRESH_MS ? at : null; +} + /** Peer marker files that record a commit of `sessionId` inside the contest - * window by a LIVE foreign instance (pid alive or file fresh). Malformed or - * tearing files are skipped silently — a marker is observability, not data. */ + * window by a LIVE foreign instance (pid alive or file fresh). Markers + * stamped `disposedAt` by a gracefully-shutdown peer are attribution-only + * and never contested (#4626). Malformed or tearing files are skipped + * silently — a marker is observability, not data. */ export function findContestedWriters(dir: string, selfInstanceId: string, sessionId: string, now: number): Array { let entries: fs.Dirent[]; try { @@ -217,12 +249,9 @@ export function findContestedWriters(dir: string, selfInstanceId: string, sessio const file = path.join(dir, entry.name); try { const raw = JSON.parse(fs.readFileSync(file, "utf8")) as Partial; - if (raw.v !== 1 || typeof raw.instanceId !== "string" || raw.instanceId === selfInstanceId) continue; - const at = raw.servedAt !== null && typeof raw.servedAt === "object" ? (raw.servedAt as Record)[sessionId] : undefined; - if (typeof at !== "number" || now - at > CONTEST_WINDOW_MS) continue; const mtimeMs = fs.statSync(file).mtimeMs; - const alive = typeof raw.pid === "number" ? isPidAlive(raw.pid) : false; - if (!alive && now - mtimeMs > PEER_FILE_FRESH_MS) continue; + const at = contestedServedAt(raw, selfInstanceId, sessionId, mtimeMs, now); + if (at === null || typeof raw.instanceId !== "string") continue; out.push({ instanceId: raw.instanceId, pid: typeof raw.pid === "number" ? raw.pid : -1, @@ -238,6 +267,43 @@ export function findContestedWriters(dir: string, selfInstanceId: string, sessio return out; } +/** #4626 fail-closed companion to findContestedWriters: does the ACCUSED + * peer's own marker still contest `sessionId`? true = still contested, + * false = provably stopped (marker reaped/absent, dead pid past the + * freshness grace, or the contest window lapsed), null = UNKNOWN (dir or + * file unreadable, torn mid-write) — a transient FS error must never drop a + * real notice, so callers keep it on null. Reads one file, not the dir. */ +export function peerStillContests(dir: string, selfInstanceId: string, peerInstanceId: string, sessionId: string, now: number): boolean | null { + let dirStat: fs.Stats; + try { + dirStat = fs.statSync(dir); + } catch { + return null; + } + if (!dirStat.isDirectory()) return null; + const file = path.join(dir, `${peerInstanceId}.json`); + let content: string; + try { + content = fs.readFileSync(file, "utf8"); + } catch (err) { + return (err as NodeJS.ErrnoException).code === "ENOENT" ? false : null; + } + let raw: Partial; + try { + raw = JSON.parse(content) as Partial; + } catch { + return null; + } + if (raw.instanceId !== peerInstanceId) return null; + let mtimeMs: number; + try { + mtimeMs = fs.statSync(file).mtimeMs; + } catch { + return null; + } + return contestedServedAt(raw, selfInstanceId, sessionId, mtimeMs, now) !== null; +} + /** Startup hygiene: drop marker files whose instance is provably gone (pid * dead) and stale beyond the reap horizon. Fresh files from a just-crashed * peer stay — findContestedWriters' freshness test still honors them for diff --git a/src/server.ts b/src/server.ts index 3030b92888..758b048000 100644 --- a/src/server.ts +++ b/src/server.ts @@ -99,8 +99,8 @@ import { log as loggerLog, configureLogger, getLogPath, closeLogger, isStreamWri import { queryLogLines } from "./web/logs-query.js"; import { configFile, defaultLogFile, stateDir } from "./paths.js"; import { atomicWriteInstanceFile, clearProxyInstanceFile, classifySameLanePeer, entryScriptFingerprint, entryShape, findSameLanePredecessor, findVerifiedSameLaneTwin, isPidAlive, listInstances, probePid, readProxyInstanceFile, registerInstanceAndWarn, sameInstanceShape, unregisterInstance, warnOnNewPeers, writeSpawnPortLog, type InstanceShape, type PidProbe, type ProxyInstanceFile, type RegistryEntry, type SameLanePeer } from "./instance.js"; -import { createServedMarkerWriter, reapStaleWriterMarkers, writerMarkerDir, type ServedMarkerWriter } from "./served-markers.js"; -import { recordSplitBrainContest } from "./storage/runtime-status.js"; +import { createServedMarkerWriter, peerStillContests, reapStaleWriterMarkers, writerMarkerDir, type ServedMarkerWriter } from "./served-markers.js"; +import { recordSplitBrainContest, registerSplitBrainEvidenceCheck } from "./storage/runtime-status.js"; import { mayClobberPointer, probeOwnerInfo, resolvePortClaim } from "./port-owner.js"; import { hoistTrappedToolItems } from "./tool-pair-order.js"; import { computeAnthropicMessageMarks, stampAnthropicSystemCacheControl, anthropicToolsCarryCacheControl } from "./loop/cache-control.js"; @@ -1231,6 +1231,13 @@ export async function startServer(opts: ProxyOptions): Promise { }, ); reapStaleWriterMarkers(markerDir, Date.now()); + // #4626: a surfaced verdict re-evaluates against CURRENT marker + // evidence at every status read — the accused peer's own marker, + // judged by the same "still contests" definition as detection, so + // the restart-handover notice clears itself once the peer + // PROVABLY stops; an unreadable marker fails closed (keeps it). + registerSplitBrainEvidenceCheck((contest) => + peerStillContests(markerDir, instanceId, contest.peer.instanceId, contest.sessionId, Date.now())); setGlobalSessionCommitListener((id) => servedWriter?.noteServed(id)); } const nOverrides = Object.keys(opts.routes).length; diff --git a/src/server/admin.ts b/src/server/admin.ts index bd0fb963e9..67f702b56d 100644 --- a/src/server/admin.ts +++ b/src/server/admin.ts @@ -981,7 +981,8 @@ async function handleSplitBrainAck(req: http.IncomingMessage, res: http.ServerRe res.end(JSON.stringify({ error: `no surfaced split-brain entry for session: ${sessionId}` }, null, 2)); return; } - log("info", `[storage] split-brain notice for session ${sessionId.slice(0, 8)} acknowledged via web UI`); + // #4626: full id — .slice(0, 8) of "session-…" renders literally "session-" + log("info", `[storage] split-brain notice for session ${sessionId} acknowledged via web UI`); res.writeHead(200, { "content-type": "application/json" }); res.end(JSON.stringify({ ok: true }, null, 2)); } diff --git a/src/storage/runtime-status.ts b/src/storage/runtime-status.ts index f2c0aee365..fff3bd91c9 100644 --- a/src/storage/runtime-status.ts +++ b/src/storage/runtime-status.ts @@ -137,6 +137,21 @@ export function ackSplitBrainContest(sessionId: string): boolean { const MAX_SPLIT_BRAIN_ACKS = 64; +// #4626: the stored verdict must be a function of CURRENT evidence, not a +// replay of one moment's snapshot — at detection time the restart-handover +// tombstone is seconds fresh, but it must not pin the notice forever. The +// server registers the evidence check (the accused peer's own marker, judged +// by the same "still contests" definition as detection); without a registered +// check the stored entry is replayed as-is (legacy behavior, standalone +// record/ack use). The check is TRI-STATE and fails closed: an unknown result +// (the marker could not be read at status-read time) keeps the notice — +// dropping a real notice on a transient FS error is the wrong direction. +let splitBrainEvidenceCheck: ((contest: SplitBrainContestInput) => boolean | null) | null = null; + +export function registerSplitBrainEvidenceCheck(check: ((contest: SplitBrainContestInput) => boolean | null) | null): void { + splitBrainEvidenceCheck = check; +} + /** #4175: the last successful VACUUM INTO backup. */ export function recordBackupOutcome(lastPath: string, lastBytes: number): void { state.backup = { lastPath, lastAt: Date.now(), lastBytes }; @@ -145,6 +160,7 @@ export function recordBackupOutcome(lastPath: string, lastBytes: number): void { export function _resetStorageRuntimeStatusForTest(): void { state = freshState(); cooldownCache = null; + splitBrainEvidenceCheck = null; } /** #4175: true when a pid belongs to a live process (EPERM = alive, foreign user). */ @@ -271,6 +287,14 @@ function getStorageRuntimeStatus(): StorageRuntimeStatus { state.secondProcess = null; loggerLog("info", `[storage] second-process pid ${sp.pid} has exited — stale warning cleared`); } + // #4626: same never-freeze discipline as above — once the peer PROVABLY + // stops contesting, the stored verdict clears itself; an unknown verdict + // (marker unreadable at read time) keeps it. + const sb = state.splitBrain; + if (sb !== null && splitBrainEvidenceCheck !== null && splitBrainEvidenceCheck(sb) === false) { + state.splitBrain = null; + loggerLog("info", `[split-brain] session ${sb.sessionId} notice cleared — peer ${sb.peer.instanceId} (pid ${sb.peer.pid}) no longer contests it`); + } return { requested, effective, diff --git a/src/web/client.ts b/src/web/client.ts index 7556a9efb2..e3d0b4e5ac 100644 --- a/src/web/client.ts +++ b/src/web/client.ts @@ -1223,10 +1223,12 @@ function changelogCatOf(rawLine) { sbb.hidden = false; sbb.classList.add("show"); const laneOf = (x) => (x && typeof x.lane === "string" && x.lane ? x.lane : t("sys.split_brain_lane_undeclared")); + // #4626: .slice(0, 8) of "session-…" is literally "session-" — show enough to tell sessions apart + const shortSid = sb.sessionId.length > 24 ? sb.sessionId.slice(0, 24) + "…" : sb.sessionId; sbb.innerHTML = "" + escapeHtml(t("sys.split_brain_title")) + " " + escapeHtml(t("sys.split_brain", { - title: typeof sb.title === "string" && sb.title ? sb.title : sb.sessionId.slice(0, 8), - sid: sb.sessionId.slice(0, 8), + title: typeof sb.title === "string" && sb.title ? sb.title : shortSid, + sid: shortSid, selfPid: String(sb.self.pid), selfLane: laneOf(sb.self), peerPid: String(sb.peer.pid), diff --git a/tests/issue-split-brain-surface.test.ts b/tests/issue-split-brain-surface.test.ts index e6b368cdeb..e57708e7c4 100644 --- a/tests/issue-split-brain-surface.test.ts +++ b/tests/issue-split-brain-surface.test.ts @@ -12,6 +12,12 @@ // G2 the surfaced entry is dismissable (POST /__bili/storage/split-brain/ // ack): ack clears it, the SAME contest stays quiet, a NEW contest // re-surfaces, an unknown session answers 400. +// G3 the stored verdict never outlives its evidence (#4626): it persists +// while the peer's marker evidence holds and self-clears at the status +// read once the peer provably stops contesting (both directions). +// G4 the evidence check fails closed (#4626): an unreadable writers dir or +// torn peer marker is UNKNOWN and keeps the notice; only provable +// absence/staleness clears. import test from "node:test"; import assert from "node:assert/strict"; import http from "node:http"; @@ -23,8 +29,8 @@ import { spawn, type ChildProcess } from "node:child_process"; import { startServer } from "../src/server.ts"; import { UnifiedStore } from "../src/storage/store.ts"; import { probeBuiltinSqlite, probeHostBetterSqlite3, setPreferredSqliteEngineForTests, setSqliteEnginesUnavailableForTests } from "../src/storage/driver.ts"; -import { _resetStorageRuntimeStatusForTest, ackSplitBrainContest, recordSecondProcessOwner, recordSplitBrainContest, storageStatusWire, type SplitBrainContestInput } from "../src/storage/runtime-status.ts"; -import { createServedMarkerWriter, writerMarkerDir, type WriterIdentity } from "../src/served-markers.ts"; +import { _resetStorageRuntimeStatusForTest, ackSplitBrainContest, recordSecondProcessOwner, recordSplitBrainContest, registerSplitBrainEvidenceCheck, storageStatusWire, type SplitBrainContestInput } from "../src/storage/runtime-status.ts"; +import { createServedMarkerWriter, peerStillContests, writerMarkerDir, type WriterIdentity } from "../src/served-markers.ts"; import { isPidAlive } from "../src/instance.ts"; import { setLogCapture } from "../src/logger.ts"; import { renderPage } from "../src/web/page.ts"; @@ -114,6 +120,21 @@ function writeInstanceEntry(stateRoot: string, id: string, pid: number, lane?: s fs.writeFileSync(path.join(dir, `${id}.json`), JSON.stringify(entry)); } +// Beyond any sane pid_max (Linux default 4194304) — provably dead. +const DEAD_PID = 999_999_999; + +function writePeerMarker(dir: string, id: string, pid: number, servedAt: Record): string { + fs.mkdirSync(dir, { recursive: true }); + const file = path.join(dir, `${id}.json`); + fs.writeFileSync(file, JSON.stringify({ v: 1, instanceId: id, pid, origin: "http://127.0.0.1:8787", version: "0.1.205", lane: "dsh", servedAt })); + return file; +} + +function backdate(file: string, ageMs: number): void { + const t = (Date.now() - ageMs) / 1000; + fs.utimesSync(file, t, t); +} + interface SplitBrainWire { sessionId: string; title?: string; @@ -304,6 +325,121 @@ test("#3959 G2: ack clears the surfaced entry; same contest stays quiet, a new o _resetStorageRuntimeStatusForTest(); }); +// ---- #4626: the stored verdict must not outlive its evidence ------------------ + +test("#4626 G3: stored verdict persists while evidence holds, self-clears once the peer stops contesting", () => { + _resetStorageRuntimeStatusForTest(); + const logs: string[] = []; + setLogCapture((level, msg) => logs.push(`${level} ${msg}`)); + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "bili-4626-g3-")); + try { + const dir = writerMarkerDir(tmp); + const base: SplitBrainContestInput = { + sessionId: "sess-stale", + title: "Fix auth bug", + self: { instanceId: "self-1", pid: process.pid, origin: "http://127.0.0.1:18802", lane: "pi", version: "0.1.205" }, + peer: { instanceId: "peer-gone", pid: DEAD_PID, origin: "http://127.0.0.1:8787", lane: "dsh", version: "0.1.204", servedAt: Date.now() - 1000 }, + at: Date.now(), + }; + // The server's registration shape (src/server.ts): the verdict holds iff + // the accused peer's own marker still contests — tri-state, fails closed. + registerSplitBrainEvidenceCheck((c) => + peerStillContests(dir, "self-1", c.peer.instanceId, c.sessionId, Date.now())); + + // Phase A — fresh tombstone (dead pid, marker just rewritten by the + // peer's graceful-shutdown flush): the notice MUST persist; a dead pid + // alone is not proof of absence on every platform. + const file = writePeerMarker(dir, "peer-gone", DEAD_PID, { [base.sessionId]: base.peer.servedAt }); + recordSplitBrainContest(base); + assert.equal(storageStatusWire().splitBrain?.sessionId, "sess-stale", "fresh tombstone keeps the notice honest"); + + // Phase B — dead pid + marker older than the freshness grace: the next + // status read clears the stored verdict by itself (#4626). + backdate(file, 40_000); + assert.equal(storageStatusWire().splitBrain, null, "dead peer + stale marker clears the stored verdict at the status read"); + assert.ok(logs.some((l) => l.startsWith("info ") && l.includes("[split-brain] session sess-stale") && l.includes("peer-gone") && l.includes(String(DEAD_PID))), "the clear is logged with full ids"); + assert.equal(storageStatusWire().splitBrain, null, "stays cleared on subsequent reads"); + + // Phase C — an evidence-based clear must NOT poison the ack guard: if + // the same peer contests again, the notice surfaces again. + backdate(file, 0); + recordSplitBrainContest(base); + assert.equal(storageStatusWire().splitBrain?.sessionId, "sess-stale", "a renewed contest re-surfaces after an evidence-based clear"); + } finally { + setLogCapture(null); + _resetStorageRuntimeStatusForTest(); + rmrf(tmp); + } +}); + +test("#4626 G3-negative: without a registered check the stored verdict stands (legacy replay)", () => { + _resetStorageRuntimeStatusForTest(); + const base: SplitBrainContestInput = { + sessionId: "sess-legacy", + self: { instanceId: "self-1", pid: process.pid, origin: "http://127.0.0.1:18802", lane: "pi", version: "0.1.205" }, + peer: { instanceId: "peer-x", pid: DEAD_PID, origin: "http://127.0.0.1:8787", lane: "dsh", version: "0.1.204", servedAt: Date.now() - 1000 }, + at: Date.now(), + }; + try { + recordSplitBrainContest(base); + assert.equal(storageStatusWire().splitBrain?.sessionId, "sess-legacy", "no registered check ⇒ stored entry replays as-is"); + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "bili-4626-g3n-")); + try { + const dir = writerMarkerDir(tmp); + writePeerMarker(dir, "peer-live", process.pid, { [base.sessionId]: Date.now() - 1000 }); + registerSplitBrainEvidenceCheck((c) => + peerStillContests(dir, "self-1", c.peer.instanceId, c.sessionId, Date.now())); + const live: SplitBrainContestInput = { ...base, peer: { ...base.peer, instanceId: "peer-live", pid: process.pid } }; + recordSplitBrainContest(live); + assert.equal(storageStatusWire().splitBrain?.sessionId, "sess-legacy", "live peer + fresh marker keeps the notice surfaced"); + } finally { + rmrf(tmp); + } + } finally { + _resetStorageRuntimeStatusForTest(); + } +}); + +test("#4626 G4: evidence check fails closed — unknown evidence keeps the notice", () => { + _resetStorageRuntimeStatusForTest(); + const logs: string[] = []; + setLogCapture((level, msg) => logs.push(`${level} ${msg}`)); + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "bili-4626-g4-")); + try { + const dir = writerMarkerDir(tmp); + const base: SplitBrainContestInput = { + sessionId: "sess-fc", + self: { instanceId: "self-1", pid: process.pid, origin: "http://127.0.0.1:18802", lane: "pi", version: "0.1.205" }, + peer: { instanceId: "peer-gone", pid: DEAD_PID, origin: "http://127.0.0.1:8787", lane: "dsh", version: "0.1.204", servedAt: Date.now() - 1000 }, + at: Date.now(), + }; + registerSplitBrainEvidenceCheck((c) => + peerStillContests(dir, "self-1", c.peer.instanceId, c.sessionId, Date.now())); + const file = writePeerMarker(dir, "peer-gone", DEAD_PID, { [base.sessionId]: base.peer.servedAt }); + recordSplitBrainContest(base); + assert.equal(storageStatusWire().splitBrain?.sessionId, "sess-fc"); + + // Writers dir vanishes (state-dir move / transient FS error): UNKNOWN — + // the notice stays and no false "no longer contests" line appears. + fs.rmSync(dir, { recursive: true }); + assert.equal(storageStatusWire().splitBrain?.sessionId, "sess-fc", "missing writers dir ⇒ unknown ⇒ notice kept"); + assert.ok(!logs.some((l) => l.includes("no longer contests")), "an unknown verdict never logs a false clear"); + + // Dir restored but the accused marker reaped: PROVABLY STOPPED — clears. + fs.mkdirSync(dir, { recursive: true }); + assert.equal(storageStatusWire().splitBrain, null, "absent peer marker ⇒ cleared at the status read"); + + // Torn/malformed marker: UNKNOWN again — a renewed verdict stays up. + fs.writeFileSync(file, "{ torn mid-write"); + recordSplitBrainContest(base); + assert.equal(storageStatusWire().splitBrain?.sessionId, "sess-fc", "unreadable peer marker ⇒ unknown ⇒ notice kept"); + } finally { + setLogCapture(null); + _resetStorageRuntimeStatusForTest(); + rmrf(tmp); + } +}); + // ---- web surface: wire + endpoint + page/client/i18n --------------------------- test("#3959 web: status carries secondProcess/splitBrain; ack endpoint clears; page/client/i18n wired", async () => { @@ -322,11 +458,17 @@ test("#3959 web: status carries secondProcess/splitBrain; ack endpoint clears; p assert.equal(s.storage.secondProcess, null, "no second process recorded → null"); assert.equal(s.storage.splitBrain, null, "no contest recorded → null"); + // #4626: surfaced entries re-evaluate against CURRENT writer-marker + // evidence at each status read — give the recorded contest real + // evidence in the server's writers dir (fresh mtime keeps it contested + // even though DEAD_PID is provably not a live process). + const peerMarker = writePeerMarker(path.join(base, "state", "billion-context", "writers"), "peer-1", DEAD_PID, { "sess-web": Date.now() - 1000 }); + const contest: SplitBrainContestInput = { sessionId: "sess-web", title: "Fix auth bug", self: { instanceId: "self-1", pid: process.pid, origin: "http://127.0.0.1:18802", lane: "pi", version: "0.1.202" }, - peer: { instanceId: "peer-1", pid: 4242, origin: "http://127.0.0.1:8787", lane: "dsh", version: "0.1.201", servedAt: 1000 }, + peer: { instanceId: "peer-1", pid: DEAD_PID, origin: "http://127.0.0.1:8787", lane: "dsh", version: "0.1.201", servedAt: 1000 }, at: Date.now(), }; recordSplitBrainContest(contest); @@ -334,7 +476,7 @@ test("#3959 web: status carries secondProcess/splitBrain; ack endpoint clears; p assert.equal(s.storage.splitBrain?.sessionId, "sess-web", "status wire carries the surfaced contest"); assert.equal(s.storage.splitBrain?.title, "Fix auth bug"); assert.equal(s.storage.splitBrain?.peer.lane, "dsh"); - assert.equal(s.storage.splitBrain?.peer.pid, 4242); + assert.equal(s.storage.splitBrain?.peer.pid, DEAD_PID); const acked = await postJson(port, "/__bili/storage/split-brain/ack", { sessionId: "sess-web" }); assert.equal(acked.status, 200, "ack of the surfaced session answers 200"); @@ -351,6 +493,13 @@ test("#3959 web: status carries secondProcess/splitBrain; ack endpoint clears; p s = await fetchJson<{ storage: StorageWireShape }>(port, "/__bili/status"); assert.equal(s.storage.splitBrain?.sessionId, "sess-web", "a NEW contest re-surfaces on the wire after ack"); + // #4626 end-to-end: once the peer's marker is past its freshness grace + // and its pid is provably dead, the next status read clears the notice + // WITHOUT a manual ack. + backdate(peerMarker, 40_000); + s = await fetchJson<{ storage: StorageWireShape }>(port, "/__bili/status"); + assert.equal(s.storage.splitBrain, null, "stale verdict self-clears at the status read without a manual ack"); + const html = renderPage("http://127.0.0.1:1", "0.0.0"); assert.ok(html.includes('id="split-brain-banner"'), "page carries the split-brain banner div"); assert.ok(WEB_CLIENT.includes('"split-brain-banner"'), "client targets the banner element"); diff --git a/tests/served-markers.test.ts b/tests/served-markers.test.ts index 93f75d662b..0183ace9e3 100644 --- a/tests/served-markers.test.ts +++ b/tests/served-markers.test.ts @@ -111,6 +111,35 @@ test("dead-pid peer counts while its marker is fresh, stops counting once stale" assert.equal(found.length, 0, "dead pid + stale mtime = gone"); }); +test("#4626 gracefully-shutdown peer (disposedAt stamp) never contests, even with fresh mtime", () => { + const dir = writerMarkerDir(freshDir("bili-4626-disposed-")); + const deadPeer: WriterIdentity = { ...PEER, pid: DEAD_PID }; + fs.mkdirSync(dir, { recursive: true }); + // The exact shape dispose() leaves behind: dead pid, mtime ≈ death time, + // a commit inside the window seconds earlier. + const file = path.join(dir, `${deadPeer.instanceId}.json`); + fs.writeFileSync(file, JSON.stringify({ v: 1, ...deadPeer, servedAt: { "sess-h": Date.now() - 1000 }, disposedAt: Date.now() - 9000 })); + backdate(file, 8000); + assert.deepEqual(findContestedWriters(dir, SELF.instanceId, "sess-h", Date.now()), [], "handover tombstone is attribution-only — no false split-brain verdict"); +}); + +test("#4626 e2e: graceful rotation — successor commits the same session without a verdict", () => { + const dir = writerMarkerDir(freshDir("bili-4626-handover-")); + const logs: Array<[string, string]> = []; + const outgoing = createServedMarkerWriter(PEER, dir, (level, msg) => logs.push([level, msg])); + outgoing.noteServed("sess-rot"); + outgoing.__flushForTest(); + const mid = JSON.parse(fs.readFileSync(path.join(dir, "peer-1.json"), "utf8")) as Record; + assert.equal(mid.disposedAt, undefined, "periodic flushes carry no disposedAt stamp"); + outgoing.dispose(); + const final = JSON.parse(fs.readFileSync(path.join(dir, "peer-1.json"), "utf8")) as Record; + assert.equal(typeof final.disposedAt, "number", "the final shutdown write is stamped disposedAt"); + const incoming = createServedMarkerWriter(SELF, dir, (level, msg) => logs.push([level, msg])); + incoming.noteServed("sess-rot"); + assert.equal(logs.length, 0, "sequential handover inside the 10m window raises no [split-brain] line"); + incoming.dispose(); +}); + test("malformed or foreign-shape marker files are skipped, never fatal", () => { const dir = writerMarkerDir(freshDir("bili-3923-malformed-")); fs.mkdirSync(dir, { recursive: true });