Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
92 changes: 79 additions & 13 deletions src/served-markers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,11 @@
// names both sides in an error line when it commits a session another live
// instance committed within the window. Pure observability: no routing,
// locking, or serving behavior changes.
// #4626: an instance that shut down GRACEFULLY stamps its final marker with
// `disposedAt`; such a peer is attribution-only and can never be flagged as
// a concurrent writer — its pid is provably gone by construction (it ran our
// shutdown path), so the freshness grace below stays reserved for peers that
// died without it (crash/kill) where the pid probe may be blind.

import fs from "node:fs";
import path from "node:path";
Expand All @@ -35,6 +40,11 @@ interface WriterMarkerFile {
lane?: string;
/** sessionId → last commit time (ms epoch). */
servedAt: Record<string, number>;
/** Set only on the final marker write of a graceful shutdown (#4626):
* the instance is provably gone — peers keep the file for attribution
* but never treat it as a concurrent writer. Additive; markers from
* older builds carry none. */
disposedAt?: number;
}

/** A surfaced split-brain contest: which session (id + resolved title) and
Expand Down Expand Up @@ -68,7 +78,9 @@ const MAX_TRACKED_SESSIONS = 128;
const VERDICT_TTL_MS = 60_000;
/** A peer counts as live when its pid is alive OR its marker was written
* within this window (covers platforms where the pid check cannot see the
* peer process). */
* peer process). Gracefully-shutdown instances are excluded BEFORE this
* test via their disposedAt stamp (#4626) — their death is certain, so no
* grace applies. */
const PEER_FILE_FRESH_MS = 30_000;
/** reapStaleWriterMarkers removes files whose pid is dead AND whose mtime is
* older than this. */
Expand Down Expand Up @@ -127,7 +139,7 @@ export function createServedMarkerWriter(identity: WriterIdentity, dir: string,
}
};

const writeMarker = (): void => {
const writeMarker = (final: boolean): void => {
const now = Date.now();
prune(now);
const body: WriterMarkerFile = {
Expand All @@ -138,6 +150,7 @@ export function createServedMarkerWriter(identity: WriterIdentity, dir: string,
version: identity.version,
...(identity.lane === undefined || identity.lane === "" ? {} : { lane: identity.lane }),
servedAt: Object.fromEntries(servedAt),
...(final ? { disposedAt: now } : {}),
};
try {
fs.mkdirSync(dir, { recursive: true });
Expand All @@ -152,7 +165,7 @@ export function createServedMarkerWriter(identity: WriterIdentity, dir: string,
flushTimer = undefined;
if (disposed) return;
lastFlushAt = Date.now();
writeMarker();
writeMarker(false);
};

const scheduleFlush = (): void => {
Expand All @@ -172,7 +185,7 @@ export function createServedMarkerWriter(identity: WriterIdentity, dir: string,
const title = opts?.resolveTitle?.(sessionId);
const laneOf = (x: WriterIdentity): string => (x.lane === undefined || x.lane === "" ? "" : `, lane ${x.lane}`);
const titleOf = title === undefined || title === "" ? "" : ` "${title}"`;
log("error", `[split-brain] session ${sessionId}${titleOf} committed by BOTH this instance (pid ${identity.pid}, ${identity.origin}${laneOf(identity)}, v${identity.version}) and another live instance (pid ${peer.pid}, ${peer.origin}${laneOf(peer)}, v${peer.version}) within ${CONTEST_WINDOW_MS / 60000}m — fold decisions are invisible across instances and the last writer wins (#394/#405); stop one of them or align your client config so one conversation rides one proxy (#3923)`);
log("error", `[split-brain] session ${sessionId}${titleOf} committed by BOTH this instance (pid ${identity.pid}, ${identity.origin}${laneOf(identity)}, v${identity.version}) and another instance (pid ${peer.pid}, ${peer.origin}${laneOf(peer)}, v${peer.version}) within ${CONTEST_WINDOW_MS / 60000}m — fold decisions are invisible across instances and the last writer wins (#394/#405); stop one of them or align your client config so one conversation rides one proxy (#3923)`);
opts?.onContest?.({ sessionId, ...(title === undefined || title === "" ? {} : { title }), self: identity, peer, at: now });
}
};
Expand All @@ -189,8 +202,10 @@ export function createServedMarkerWriter(identity: WriterIdentity, dir: string,
disposed = true;
if (flushTimer !== undefined) clearTimeout(flushTimer);
// final write (bypasses the disposed guard): peers should still be
// able to attribute this instance's last commits after shutdown
writeMarker();
// able to attribute this instance's last commits after shutdown;
// the disposedAt stamp (#4626) keeps that attribution while
// excluding us from every peer's contest scan
writeMarker(true);
},
__flushForTest(): void {
flush();
Expand All @@ -201,9 +216,26 @@ export function createServedMarkerWriter(identity: WriterIdentity, dir: string,
};
}

/** One definition of "still contests" (#4626): the contested `servedAt`
* timestamp, or null when the marker does not contest — shared by the dir
* scan below and the direct-path clear check, so detection and clearing can
* never drift apart. */
function contestedServedAt(raw: Partial<WriterMarkerFile>, selfInstanceId: string, sessionId: string, mtimeMs: number, now: number): number | null {
if (raw.v !== 1 || typeof raw.instanceId !== "string" || raw.instanceId === selfInstanceId) return null;
// #4626: gracefully-shutdown peer — attribution only, never a concurrent
// writer (its death is certain; no freshness grace).
if (typeof raw.disposedAt === "number") return null;
const at = raw.servedAt !== null && typeof raw.servedAt === "object" ? (raw.servedAt as Record<string, unknown>)[sessionId] : undefined;
if (typeof at !== "number" || now - at > CONTEST_WINDOW_MS) return null;
const alive = typeof raw.pid === "number" ? isPidAlive(raw.pid) : false;
return alive || now - mtimeMs <= PEER_FILE_FRESH_MS ? at : null;
}

/** Peer marker files that record a commit of `sessionId` inside the contest
* window by a LIVE foreign instance (pid alive or file fresh). Malformed or
* tearing files are skipped silently — a marker is observability, not data. */
* window by a LIVE foreign instance (pid alive or file fresh). Markers
* stamped `disposedAt` by a gracefully-shutdown peer are attribution-only
* and never contested (#4626). Malformed or tearing files are skipped
* silently — a marker is observability, not data. */
export function findContestedWriters(dir: string, selfInstanceId: string, sessionId: string, now: number): Array<WriterIdentity & { servedAt: number }> {
let entries: fs.Dirent[];
try {
Expand All @@ -217,12 +249,9 @@ export function findContestedWriters(dir: string, selfInstanceId: string, sessio
const file = path.join(dir, entry.name);
try {
const raw = JSON.parse(fs.readFileSync(file, "utf8")) as Partial<WriterMarkerFile>;
if (raw.v !== 1 || typeof raw.instanceId !== "string" || raw.instanceId === selfInstanceId) continue;
const at = raw.servedAt !== null && typeof raw.servedAt === "object" ? (raw.servedAt as Record<string, unknown>)[sessionId] : undefined;
if (typeof at !== "number" || now - at > CONTEST_WINDOW_MS) continue;
const mtimeMs = fs.statSync(file).mtimeMs;
const alive = typeof raw.pid === "number" ? isPidAlive(raw.pid) : false;
if (!alive && now - mtimeMs > PEER_FILE_FRESH_MS) continue;
const at = contestedServedAt(raw, selfInstanceId, sessionId, mtimeMs, now);
if (at === null || typeof raw.instanceId !== "string") continue;
out.push({
instanceId: raw.instanceId,
pid: typeof raw.pid === "number" ? raw.pid : -1,
Expand All @@ -238,6 +267,43 @@ export function findContestedWriters(dir: string, selfInstanceId: string, sessio
return out;
}

/** #4626 fail-closed companion to findContestedWriters: does the ACCUSED
* peer's own marker still contest `sessionId`? true = still contested,
* false = provably stopped (marker reaped/absent, dead pid past the
* freshness grace, or the contest window lapsed), null = UNKNOWN (dir or
* file unreadable, torn mid-write) — a transient FS error must never drop a
* real notice, so callers keep it on null. Reads one file, not the dir. */
export function peerStillContests(dir: string, selfInstanceId: string, peerInstanceId: string, sessionId: string, now: number): boolean | null {
let dirStat: fs.Stats;
try {
dirStat = fs.statSync(dir);
} catch {
return null;
}
if (!dirStat.isDirectory()) return null;
const file = path.join(dir, `${peerInstanceId}.json`);
let content: string;
try {
content = fs.readFileSync(file, "utf8");
} catch (err) {
return (err as NodeJS.ErrnoException).code === "ENOENT" ? false : null;
}
let raw: Partial<WriterMarkerFile>;
try {
raw = JSON.parse(content) as Partial<WriterMarkerFile>;
} catch {
return null;
}
if (raw.instanceId !== peerInstanceId) return null;
let mtimeMs: number;
try {
mtimeMs = fs.statSync(file).mtimeMs;
} catch {
return null;
}
return contestedServedAt(raw, selfInstanceId, sessionId, mtimeMs, now) !== null;
}

/** Startup hygiene: drop marker files whose instance is provably gone (pid
* dead) and stale beyond the reap horizon. Fresh files from a just-crashed
* peer stay — findContestedWriters' freshness test still honors them for
Expand Down
11 changes: 9 additions & 2 deletions src/server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -99,8 +99,8 @@ import { log as loggerLog, configureLogger, getLogPath, closeLogger, isStreamWri
import { queryLogLines } from "./web/logs-query.js";
import { configFile, defaultLogFile, stateDir } from "./paths.js";
import { atomicWriteInstanceFile, clearProxyInstanceFile, classifySameLanePeer, entryScriptFingerprint, entryShape, findSameLanePredecessor, findVerifiedSameLaneTwin, isPidAlive, listInstances, probePid, readProxyInstanceFile, registerInstanceAndWarn, sameInstanceShape, unregisterInstance, warnOnNewPeers, writeSpawnPortLog, type InstanceShape, type PidProbe, type ProxyInstanceFile, type RegistryEntry, type SameLanePeer } from "./instance.js";
import { createServedMarkerWriter, reapStaleWriterMarkers, writerMarkerDir, type ServedMarkerWriter } from "./served-markers.js";
import { recordSplitBrainContest } from "./storage/runtime-status.js";
import { createServedMarkerWriter, peerStillContests, reapStaleWriterMarkers, writerMarkerDir, type ServedMarkerWriter } from "./served-markers.js";
import { recordSplitBrainContest, registerSplitBrainEvidenceCheck } from "./storage/runtime-status.js";
import { mayClobberPointer, probeOwnerInfo, resolvePortClaim } from "./port-owner.js";
import { hoistTrappedToolItems } from "./tool-pair-order.js";
import { computeAnthropicMessageMarks, stampAnthropicSystemCacheControl, anthropicToolsCarryCacheControl } from "./loop/cache-control.js";
Expand Down Expand Up @@ -1231,6 +1231,13 @@ export async function startServer(opts: ProxyOptions): Promise<http.Server> {
},
);
reapStaleWriterMarkers(markerDir, Date.now());
// #4626: a surfaced verdict re-evaluates against CURRENT marker
// evidence at every status read — the accused peer's own marker,
// judged by the same "still contests" definition as detection, so
// the restart-handover notice clears itself once the peer
// PROVABLY stops; an unreadable marker fails closed (keeps it).
registerSplitBrainEvidenceCheck((contest) =>
peerStillContests(markerDir, instanceId, contest.peer.instanceId, contest.sessionId, Date.now()));
setGlobalSessionCommitListener((id) => servedWriter?.noteServed(id));
}
const nOverrides = Object.keys(opts.routes).length;
Expand Down
3 changes: 2 additions & 1 deletion src/server/admin.ts
Original file line number Diff line number Diff line change
Expand Up @@ -981,7 +981,8 @@ async function handleSplitBrainAck(req: http.IncomingMessage, res: http.ServerRe
res.end(JSON.stringify({ error: `no surfaced split-brain entry for session: ${sessionId}` }, null, 2));
return;
}
log("info", `[storage] split-brain notice for session ${sessionId.slice(0, 8)} acknowledged via web UI`);
// #4626: full id — .slice(0, 8) of "session-…" renders literally "session-"
log("info", `[storage] split-brain notice for session ${sessionId} acknowledged via web UI`);
res.writeHead(200, { "content-type": "application/json" });
res.end(JSON.stringify({ ok: true }, null, 2));
}
Expand Down
24 changes: 24 additions & 0 deletions src/storage/runtime-status.ts
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,21 @@ export function ackSplitBrainContest(sessionId: string): boolean {

const MAX_SPLIT_BRAIN_ACKS = 64;

// #4626: the stored verdict must be a function of CURRENT evidence, not a
// replay of one moment's snapshot — at detection time the restart-handover
// tombstone is seconds fresh, but it must not pin the notice forever. The
// server registers the evidence check (the accused peer's own marker, judged
// by the same "still contests" definition as detection); without a registered
// check the stored entry is replayed as-is (legacy behavior, standalone
// record/ack use). The check is TRI-STATE and fails closed: an unknown result
// (the marker could not be read at status-read time) keeps the notice —
// dropping a real notice on a transient FS error is the wrong direction.
let splitBrainEvidenceCheck: ((contest: SplitBrainContestInput) => boolean | null) | null = null;

export function registerSplitBrainEvidenceCheck(check: ((contest: SplitBrainContestInput) => boolean | null) | null): void {
splitBrainEvidenceCheck = check;
}

/** #4175: the last successful VACUUM INTO backup. */
export function recordBackupOutcome(lastPath: string, lastBytes: number): void {
state.backup = { lastPath, lastAt: Date.now(), lastBytes };
Expand All @@ -145,6 +160,7 @@ export function recordBackupOutcome(lastPath: string, lastBytes: number): void {
export function _resetStorageRuntimeStatusForTest(): void {
state = freshState();
cooldownCache = null;
splitBrainEvidenceCheck = null;
}

/** #4175: true when a pid belongs to a live process (EPERM = alive, foreign user). */
Expand Down Expand Up @@ -271,6 +287,14 @@ function getStorageRuntimeStatus(): StorageRuntimeStatus {
state.secondProcess = null;
loggerLog("info", `[storage] second-process pid ${sp.pid} has exited — stale warning cleared`);
}
// #4626: same never-freeze discipline as above — once the peer PROVABLY
// stops contesting, the stored verdict clears itself; an unknown verdict
// (marker unreadable at read time) keeps it.
const sb = state.splitBrain;
if (sb !== null && splitBrainEvidenceCheck !== null && splitBrainEvidenceCheck(sb) === false) {
state.splitBrain = null;
loggerLog("info", `[split-brain] session ${sb.sessionId} notice cleared — peer ${sb.peer.instanceId} (pid ${sb.peer.pid}) no longer contests it`);
}
return {
requested,
effective,
Expand Down
6 changes: 4 additions & 2 deletions src/web/client.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1223,10 +1223,12 @@ function changelogCatOf(rawLine) {
sbb.hidden = false;
sbb.classList.add("show");
const laneOf = (x) => (x && typeof x.lane === "string" && x.lane ? x.lane : t("sys.split_brain_lane_undeclared"));
// #4626: .slice(0, 8) of "session-…" is literally "session-" — show enough to tell sessions apart
const shortSid = sb.sessionId.length > 24 ? sb.sessionId.slice(0, 24) + "…" : sb.sessionId;
sbb.innerHTML = "<strong>" + escapeHtml(t("sys.split_brain_title")) + "</strong> "
+ escapeHtml(t("sys.split_brain", {
title: typeof sb.title === "string" && sb.title ? sb.title : sb.sessionId.slice(0, 8),
sid: sb.sessionId.slice(0, 8),
title: typeof sb.title === "string" && sb.title ? sb.title : shortSid,
sid: shortSid,
selfPid: String(sb.self.pid),
selfLane: laneOf(sb.self),
peerPid: String(sb.peer.pid),
Expand Down
Loading
Loading