From b5ec3bf3bcb22b0d774eeeaff5a3f11a16c88877 Mon Sep 17 00:00:00 2001 From: ranxianglei Date: Tue, 27 Oct 2026 23:55:28 +0800 Subject: [PATCH] =?UTF-8?q?served-markers:=20graceful=20dispose=20records?= =?UTF-8?q?=20stoppedAt=20=E2=80=94=20restart=20flows=20no=20longer=20rais?= =?UTF-8?q?e=20split-brain=20alarms=20(#3923)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A SIGTERM'd proxy's final marker write (dispose flush) lands with a fresh mtime seconds before its successor rebinds; findContestedWriters' 30s freshness fallback then counts the dead predecessor as live and logs a false [split-brain] error (observed in release-gate lane A proxy restart). Fix: dispose() stamps stoppedAt on the final marker. Peers skip markers carrying stoppedAt unconditionally — a recorded graceful stop is positively-known death and must not contest, even while the file is fresh or the pid has been reused. The freshness fallback is unchanged for crashed peers (no stoppedAt), preserving cross-platform pid invisibility coverage. Two regression tests pin both paths. --- CHANGELOG.md | 1 + src/served-markers.ts | 25 +++++++++++++++++++++---- tests/served-markers.test.ts | 28 ++++++++++++++++++++++++++++ 3 files changed, 50 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 220781101a..42fa43f2c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ Versions follow the merge of a `*_release-v*` branch; CI publishes to npm on tag ### Fixed - 块区间边界在重存时不再回退:折叠后客户端重发的视图不含已覆盖原文,重入库时该批次无法重新解析区间(-1/-1),旧逻辑会把已解析的真实区间抹掉。现在区间证据一旦解析即持久(入库 `CASE` 保留;sqlite/mysql 双后端),未知块保持诚实缺口(#4608)。 +- 优雅停机(SIGTERM → dispose)的 writer 标记现在记录 `stoppedAt`:重启流程(杀旧进程→立即重绑)不再对已确认死亡的前任实例误报 [split-brain]——30s 新鲜度回退从此只覆盖无法证明死亡的崩溃场景(#3923;release-gate lane A 代理重启时观察到误报)。 ### Added diff --git a/src/served-markers.ts b/src/served-markers.ts index 6ae0aa8e25..146dd18ec6 100644 --- a/src/served-markers.ts +++ b/src/served-markers.ts @@ -33,6 +33,12 @@ interface WriterMarkerFile { origin: string; version: string; lane?: string; + /** Set only by a graceful dispose(): this instance recorded its FINAL + * marker and will never commit again. Peers skip such markers outright — + * a positively-known dead predecessor is not a concurrent writer, even + * while its file is fresh (restart flows kill + immediately rebind). + * Markers from older builds carry none and stay valid. */ + stoppedAt?: number; /** sessionId → last commit time (ms epoch). */ servedAt: Record; } @@ -68,7 +74,8 @@ const MAX_TRACKED_SESSIONS = 128; const VERDICT_TTL_MS = 60_000; /** A peer counts as live when its pid is alive OR its marker was written * within this window (covers platforms where the pid check cannot see the - * peer process). */ + * peer process). Gracefully-stopped peers are excluded upstream via + * stoppedAt — this fallback only ever rescues CRASHED peers. */ const PEER_FILE_FRESH_MS = 30_000; /** reapStaleWriterMarkers removes files whose pid is dead AND whose mtime is * older than this. */ @@ -113,6 +120,7 @@ export function createServedMarkerWriter(identity: WriterIdentity, dir: string, let flushTimer: ReturnType | undefined; let lastFlushAt = 0; let disposed = false; + let stoppedAt: number | undefined; const prune = (now: number): void => { for (const [id, at] of [...servedAt]) { @@ -137,6 +145,7 @@ export function createServedMarkerWriter(identity: WriterIdentity, dir: string, origin: identity.origin, version: identity.version, ...(identity.lane === undefined || identity.lane === "" ? {} : { lane: identity.lane }), + ...(stoppedAt === undefined ? {} : { stoppedAt }), servedAt: Object.fromEntries(servedAt), }; try { @@ -189,7 +198,11 @@ export function createServedMarkerWriter(identity: WriterIdentity, dir: string, disposed = true; if (flushTimer !== undefined) clearTimeout(flushTimer); // final write (bypasses the disposed guard): peers should still be - // able to attribute this instance's last commits after shutdown + // able to attribute this instance's last commits after shutdown. + // stoppedAt marks the stop as GRACEFUL so peers never flag this + // predecessor in a split-brain contest — the freshness fallback + // must only rescue crashed peers whose death we cannot prove. + stoppedAt = Date.now(); writeMarker(); }, __flushForTest(): void { @@ -202,8 +215,11 @@ export function createServedMarkerWriter(identity: WriterIdentity, dir: string, } /** Peer marker files that record a commit of `sessionId` inside the contest - * window by a LIVE foreign instance (pid alive or file fresh). Malformed or - * tearing files are skipped silently — a marker is observability, not data. */ + * window by a LIVE foreign instance (pid alive or file fresh). Markers + * carrying stoppedAt (graceful shutdown) are skipped unconditionally — a + * restart flow's predecessor died seconds ago and must not raise an alarm. + * Malformed or tearing files are skipped silently — a marker is + * observability, not data. */ export function findContestedWriters(dir: string, selfInstanceId: string, sessionId: string, now: number): Array { let entries: fs.Dirent[]; try { @@ -218,6 +234,7 @@ export function findContestedWriters(dir: string, selfInstanceId: string, sessio try { const raw = JSON.parse(fs.readFileSync(file, "utf8")) as Partial; if (raw.v !== 1 || typeof raw.instanceId !== "string" || raw.instanceId === selfInstanceId) continue; + if (typeof raw.stoppedAt === "number") continue; const at = raw.servedAt !== null && typeof raw.servedAt === "object" ? (raw.servedAt as Record)[sessionId] : undefined; if (typeof at !== "number" || now - at > CONTEST_WINDOW_MS) continue; const mtimeMs = fs.statSync(file).mtimeMs; diff --git a/tests/served-markers.test.ts b/tests/served-markers.test.ts index 93f75d662b..296113cd2a 100644 --- a/tests/served-markers.test.ts +++ b/tests/served-markers.test.ts @@ -111,6 +111,34 @@ test("dead-pid peer counts while its marker is fresh, stops counting once stale" assert.equal(found.length, 0, "dead pid + stale mtime = gone"); }); +test("graceful dispose records stoppedAt and never contests a fresh predecessor (#4982 gate restart)", () => { + const dir = writerMarkerDir(freshDir("bili-3923-stop-")); + const logs: Array<[string, string]> = []; + // predecessor committed sess-a, then shut down gracefully (SIGTERM → dispose) + const pred = createServedMarkerWriter({ ...PEER }, dir, () => {}); + pred.noteServed("sess-a"); + pred.__flushForTest(); + pred.dispose(); + const body = JSON.parse(fs.readFileSync(path.join(dir, "peer-1.json"), "utf8")) as Record; + assert.equal(typeof body.stoppedAt, "number", "dispose records a graceful stop"); + // successor rebinds seconds later and commits the same session — the + // predecessor's marker is still FRESH, but a known-dead instance must not + // raise an alarm + const w = createServedMarkerWriter(SELF, dir, (level, msg) => logs.push([level, msg])); + w.noteServed("sess-a"); + assert.equal(logs.length, 0, "a gracefully-stopped predecessor is not a concurrent writer"); + w.dispose(); +}); + +test("stoppedAt skips even when the peer pid is alive", () => { + const dir = writerMarkerDir(freshDir("bili-3923-stopped-pid-")); + fs.mkdirSync(dir, { recursive: true }); + fs.writeFileSync(path.join(dir, "peer-1.json"), JSON.stringify({ + v: 1, ...PEER, servedAt: { "sess-x": Date.now() - 1_000 }, stoppedAt: Date.now() - 2_000, + })); + assert.deepEqual(findContestedWriters(dir, SELF.instanceId, "sess-x", Date.now()), [], "stoppedAt outranks liveness — a recorded stop is positively-known death"); +}); + test("malformed or foreign-shape marker files are skipped, never fatal", () => { const dir = writerMarkerDir(freshDir("bili-3923-malformed-")); fs.mkdirSync(dir, { recursive: true });