Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 39 additions & 3 deletions src/persist.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1438,6 +1438,25 @@ export class SessionStore {
if (this.shadow.driftedSessions.has(id)) {
const driftedEnvelope = this.loadEnvelope(id, meta);
if (driftedEnvelope) {
// #4323: the latch historically served the envelope
// unconditionally on the assumption it was the fresher copy
// — true for one-off external churn, FALSE under a
// multi-writer ping-pong (#394): the last file write may be
// an OLDER generation than the index (measured incident:
// envelope 12 active blocks vs index 16 — serving the file
// put ~240K folded tokens back into context). Fold progress
// is the freshness proxy owner-ruled for this class:
// strictly more active blocks wins; a tie or an unreadable
// other side keeps the historical envelope preference.
const envActive = countActiveBlocks(driftedEnvelope.payload);
const idxActive = this.shadow.activeBlockCount(id);
if (idxActive !== null && idxActive > envActive) {
const unified = this.loadUnified(id);
if (unified) {
this.log("warn", `[persist] serving ${id} from the UNIFIED store despite the drift latch: legacy envelope holds ${envActive} active block(s), the index holds ${idxActive} — taking the fuller fold coverage instead of the older generation (#4323/#394)`);
return this.afterLoad(id, unified);
}
}
const drifted = buildSession(driftedEnvelope.payload);
if (hasNegativePersistedTokens(driftedEnvelope.payload)) {
this.scheduleSave(drifted);
Expand Down Expand Up @@ -2263,7 +2282,7 @@ export class SessionStore {
* bytes on disk and the ownership fast path stayed live. */
scheduleSave(session: Session): void {
if (!this.saveContentStore(session)) return;
if (this.writerOffActive()) {
if (this.writerOffActive() || this.envelopeFrozen(session.id)) {
// Phase 3 (#2960): index-only persistence — the builder closure
// runs the sidecar ingest (raw tail + live CCR) and the stale-disk
// guard; no envelope write is armed, so no debounce is needed.
Expand Down Expand Up @@ -2348,7 +2367,7 @@ export class SessionStore {
* either. Do NOT regress this by eager-snapshotting the record here. */
async writeNow(session: Session): Promise<void> {
if (!this.saveContentStore(session)) throw new Error(`Originals were not saved for ${session.id}`);
if (this.writerOffActive()) {
if (this.writerOffActive() || this.envelopeFrozen(session.id)) {
// #3133: drop the armed microtask (token mismatch below) — this
// sync build subsumes it; later mutations re-arm on their own.
this.writerOffPending.delete(session.id);
Expand Down Expand Up @@ -2428,7 +2447,7 @@ export class SessionStore {
* failure for a never-persisted session or it is lost permanently). */
flushSync(session: Session): boolean {
if (!this.saveContentStore(session)) return false;
if (this.writerOffActive()) {
if (this.writerOffActive() || this.envelopeFrozen(session.id)) {
// #3133: same subsume rule as writeNow.
this.writerOffPending.delete(session.id);
try {
Expand Down Expand Up @@ -2549,6 +2568,14 @@ export class SessionStore {
return this.jsonWriterOff && this.shadow !== undefined && !this.shadow.stats.disabled;
}

/** #4323: per-session envelope freeze (escalated structural drift, see
* UnifiedShadow.envelopeFrozen) — the save routes index-only exactly
* like writerOff, so a multi-writer ping-pong (#394) can no longer
* churn the file copy from this process. */
private envelopeFrozen(sessionId: string): boolean {
return this.shadow?.envelopeFrozen(sessionId) ?? false;
}

/** Cancel all pending writes without flushing (e.g. for tests). */
cancelAll(): void {
for (const timer of this.saveTimers.values()) clearTimeout(timer);
Expand Down Expand Up @@ -2792,6 +2819,15 @@ function hasNegativePersistedTokens(parsed: PersistedSession): boolean {
return (typeof last === "number" && last < 0) || (typeof ctx === "number" && ctx < 0);
}

/** #4323: active fold blocks a persisted record carries — the freshness
* proxy for drift-latched source selection (loadSync). Defensive: malformed
* files must count as zero, never throw into the load path. */
function countActiveBlocks(payload: PersistedSession): number {
const blocks = payload.state?.blocks;
if (!Array.isArray(blocks)) return 0;
return blocks.filter((b) => !!b && b.active === true).length;
}

/** #2671 Phase 1.5: adapter that lets buildSession (with its full forward-
* compat pass, incl. mergeState) rebuild a Session from a unified-store
* view. `messagesFolded` is only ever set when `messages` is present — an
Expand Down
6 changes: 6 additions & 0 deletions src/storage/runtime-status.ts
Original file line number Diff line number Diff line change
Expand Up @@ -175,6 +175,8 @@ interface StorageRuntimeStatus {
integrity: { ok: boolean; checkedAt: number; durationMs: number; detail?: string } | null;
secondProcess: { pid: number; at: number } | null;
backup: { lastPath: string; lastAt: number; lastBytes: number } | null;
/** #4323: sessions whose JSON-envelope writes this process froze after repeated non-converging drift samples. */
frozenEnvelopeSessions: number;
}

function getStorageRuntimeStatus(): StorageRuntimeStatus {
Expand Down Expand Up @@ -215,6 +217,7 @@ function getStorageRuntimeStatus(): StorageRuntimeStatus {
integrity: state.integrity,
secondProcess: state.secondProcess,
backup: state.backup,
frozenEnvelopeSessions: sidecar?.frozenEnvelopeIds.size ?? 0,
};
}

Expand All @@ -238,6 +241,8 @@ interface StorageStatusWire {
integrity: { ok: boolean; checkedAt: number; durationMs: number; detail?: string } | null;
secondProcess: { pid: number; at: number } | null;
backup: { lastPath: string; lastAt: number; lastBytes: number } | null;
/** #4323: envelope-write freeze count on the wire. */
frozenEnvelopeSessions: number;
}

export function storageStatusWire(): StorageStatusWire {
Expand All @@ -258,6 +263,7 @@ export function storageStatusWire(): StorageStatusWire {
integrity: st.integrity,
secondProcess: st.secondProcess,
backup: st.backup,
frozenEnvelopeSessions: st.frozenEnvelopeSessions,
};
}

Expand Down
69 changes: 69 additions & 0 deletions src/storage/shadow.ts
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,16 @@ export const MAX_CONSECUTIVE_FAILURES = 3;
* session always verifies (cold-path full check), then every N-th. */
export const VERIFY_EVERY_INGESTS = 20;

/** #4323: consecutive sampled drifts required before escalating. A one-off
* drift is transient external churn (the watermark clear below heals it);
* N in a row proves the views are STRUCTURALLY divergent — the multi-writer
* signature (#394), where nothing this process does can converge them.
* Escalation = loud WARN with both sides' fold counts + freeze of THIS
* process's JSON envelope writes for the session (persist.ts consults
* envelopeFrozen on every save route); the unified store keeps receiving
* every save, and the first clean sample re-admits the session. */
export const DRIFT_ESCALATE_STREAK = 3;

/** #3823: on win32 "unable to open database file" (SQLITE_CANTOPEN) is almost
* always an OS-level refusal — antivirus/Defender holding the dir, OneDrive
* placeholder files under the profile dir, a path past MAX_PATH, or a
Expand Down Expand Up @@ -106,6 +116,14 @@ export class UnifiedShadow {
* until a save verifies clean again; surfaced on /__bili/status wire as
* driftedSessions so drift is observable, not just logged. */
readonly driftedSessions = new Set<string>();
/** #4323: per-session consecutive sampled-drift count (resets on a clean
* sample). Per-process only, same as ingestCounts. */
readonly driftStreaks = new Map<string, number>();
/** #4323: sessions whose drift escalated to DRIFT_ESCALATE_STREAK — this
* process stops writing their JSON envelopes (persist.ts routes their
* saves index-only) so a multi-writer ping-pong (#394) can no longer
* churn the file copy; the first clean sample re-admits. */
readonly frozenEnvelopeIds = new Set<string>();
/** #3931 step 3: the sqlite fast lane. Null when the selected backend
* is a network backend (mysql) — then every sync facade method below
* returns its miss value (null / [] / no-op) and callers fall back to
Expand Down Expand Up @@ -364,6 +382,29 @@ export class UnifiedShadow {
// to distinguish from one-off external churn.
const repeat = this.driftedSessions.has(legacy.id);
this.driftedSessions.add(legacy.id);
// #4323: count CONSECUTIVE sampled drifts — membership alone
// caps at "repeat yes/no" and cannot distinguish 3+ samples.
// N in a row proves structural divergence (multi-writer
// #394): escalate ONCE at the threshold with a loud WARN
// naming both sides' fold counts, then freeze THIS
// process's envelope writes for the session so the ping-
// pong stops churning the file copy (persist.ts consults
// envelopeFrozen on every save route; the unified store
// keeps receiving every save either way).
const streak = (this.driftStreaks.get(legacy.id) ?? 0) + 1;
this.driftStreaks.set(legacy.id, streak);
if (streak === DRIFT_ESCALATE_STREAK) {
this.frozenEnvelopeIds.add(legacy.id);
let sides = "";
try {
const envBlocks = expectView.state.blocks.filter((b) => b.active).length;
const idxBlocks = (gotView.state.blocks as { active?: boolean }[] | null | undefined)?.filter((b) => b.active === true).length ?? 0;
sides = ` [envelope ${envBlocks} active block(s) vs index ${idxBlocks}]`;
} catch {
// counts must never break the heal path
}
loggerLog("warn", `[storage] DIGEST DRIFT ESCALATED for ${legacy.id}: ${streak} consecutive sampled drifts — the views have stopped converging${sides}. This usually means MULTIPLE bili instances share one sessions dir (#394 anti-pattern; second-instance detection: #4177). Freezing THIS process's JSON envelope writes for the session until a clean sample re-admits it — the unified store keeps receiving every save (#4323)`);
}
// Panels consume {disabled, lastError} — make the sampled
// degradation visible there, not just in the counter/log.
this.stats.lastError = `digest drift for ${legacy.id}: unified replay != legacy load view`;
Expand Down Expand Up @@ -394,6 +435,12 @@ export class UnifiedShadow {
// index-first reads (the watermark re-verify path above
// proved the views converged again).
this.driftedSessions.delete(legacy.id);
// #4323: convergence also ends the streak and unfreezes
// this process's envelope writes for the session.
this.driftStreaks.delete(legacy.id);
if (this.frozenEnvelopeIds.delete(legacy.id)) {
loggerLog("info", `[storage] drift resolved for ${legacy.id}: clean sample re-admits the session — envelope writes resume (#4323)`);
}
}
}
this.consecutiveFailures = 0;
Expand Down Expand Up @@ -474,6 +521,26 @@ export class UnifiedShadow {
}
}

/** #4323: true once this session's drift escalated and no clean sample
* has since re-admitted it. Sqlite lanes only — network backends keep
* the envelope as their sync read path, where freezing would strand
* the session. persist.ts consults this on every save route. */
envelopeFrozen(id: string): boolean {
return this.sync !== null && this.frozenEnvelopeIds.has(id);
}

/** #4323: active block rows the unified store holds for `id` — null when
* the sqlite fast lane is unavailable; callers must not guess from null. */
activeBlockCount(id: string): number | null {
if (this.sync === null) return null;
if (this.stats.disabled) return null;
try {
return this.sync.getBlocks(id).filter((b) => b.active).length;
} catch {
return null;
}
}

/** Organic import (full mode): ingest a legacy record the index has not
* seen so future loads serve from the unified store. Never throws. */
importRecord(record: unknown): LegacySessionLike | null {
Expand Down Expand Up @@ -501,6 +568,8 @@ export class UnifiedShadow {
remove(id: string): void {
if (this.sync === null) return;
this.ingestCounts.delete(id);
this.driftStreaks.delete(id);
this.frozenEnvelopeIds.delete(id);
if (this.stats.disabled) return;
try {
this.sync.deleteSession(id);
Expand Down
Loading
Loading