Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 34 additions & 1 deletion src/persist.ts
Original file line number Diff line number Diff line change
Expand Up @@ -349,6 +349,9 @@ export class SessionStore {
/** #3199: post-boot background alias scan (unified modes only) — null in
* legacy mode and until bootBounded defers work. */
private remainderScan: Promise<void> | null = null;
/** #3406 item 3: boot start instant — sessions born AFTER this are the
* signature of a label resume that raced the background scan. */
private bootStartedAtMs = 0;
/** #2674: monotonic per-process counter for unique content-store temp
* names (pid + seq makes collisions across processes/rapid saves
* unreachable). */
Expand Down Expand Up @@ -651,6 +654,7 @@ export class SessionStore {
if (!Number.isFinite(limits.maxSessions) || limits.maxSessions < 0 || !Number.isFinite(limits.maxBytes) || limits.maxBytes < 0) {
throw new Error("Invalid session boot limits");
}
this.bootStartedAtMs = Date.now();
this.boundedReads = true;
mkdirSync(this.sessionsDir, { recursive: true });
this.diskFiles = [];
Expand Down Expand Up @@ -826,6 +830,7 @@ export class SessionStore {
private async scanRemainderInBackground(parsedFiles: Set<string>, canonical: Set<string>): Promise<void> {
let scanned = 0;
let aliased = 0;
const seenLabels: Array<{ label: string; id: string; file: string }> = [];
for (const file of this.diskFiles) {
if (parsedFiles.has(file)) continue;
scanned++;
Expand All @@ -835,16 +840,44 @@ export class SessionStore {
if (rec && fileNameMatchesId(path.basename(file), rec.payload.id, rec.meta.protocol, rec.meta.upstreamOrigin)) {
const savedAt = rec.payload.savedAt ?? 0;
if (rec.payload.id === rec.meta.label || rec.payload.id.startsWith("pfa-")) this.noteCanonical(rec.payload.id, canonical);
else if (rec.meta.label) aliased += this.upsertLegacyAlias(rec.meta.label, { file, id: rec.payload.id, savedAt }, canonical) ? 1 : 0;
else if (rec.meta.label) {
seenLabels.push({ label: rec.meta.label, id: rec.payload.id, file });
aliased += this.upsertLegacyAlias(rec.meta.label, { file, id: rec.payload.id, savedAt }, canonical) ? 1 : 0;
}
}
}
await new Promise<void>((resolve) => setImmediate(resolve));
}
this.detectLabelResumeRaces(seenLabels);
if (scanned > 0) {
this.log("info", `[persist] background index scan finished after boot: ${scanned} file(s) scanned, ${aliased} alias(es) recorded`);
}
}

/** #3406 item 3 (#3199 race, detection half): a label-based resume that
* lands while this scan is still running finds no alias yet and mints an
* EMPTY session under the label; once the scan reaches the older file the
* direct-id file wins every later lookup, so the prior conversation stays
* unreachable under the label. Pure detection — zero state mutation. At
* completion, every scanned label that ALSO resolves to its own direct-id
* file whose payload was born AFTER boot started (the racy session's
* signature; a healthy alias-resume re-key keeps its original createdAt
* and stays quiet) gets one loud warn naming both states. */
private detectLabelResumeRaces(seenLabels: Array<{ label: string; id: string; file: string }>): void {
const flagged = new Set<string>();
for (const entry of seenLabels) {
if (flagged.has(entry.label)) continue;
const expected = path.basename(relPathFor(entry.label));
const direct = this.diskFiles.find((f) => f !== entry.file
&& (fileNameMatchesId(path.basename(f), entry.label) || path.basename(f).endsWith(`_${expected}`)));
if (!direct) continue;
const env = this.readEnvelopeFile(direct, entry.label);
if (!env || typeof env.payload.createdAt !== "number" || env.payload.createdAt <= this.bootStartedAtMs) continue;
flagged.add(entry.label);
this.log("warn", `[persist] label resume raced the post-boot background scan: "${entry.label}" was resumed before the scan recorded its alias, so a fresh empty session was minted under the label; the prior conversation survives at ${entry.file} (id ${entry.id}). Resume by id ${entry.id}, or delete the empty ${entry.label} state file if the new conversation supersedes it (#3406/#3199).`);
}
}

/** #3199: resolves when the post-boot background alias scan has finished
* (immediate in legacy mode or when nothing was deferred). */
backgroundScanComplete(): Promise<void> {
Expand Down
300 changes: 300 additions & 0 deletions tests/issue3406-label-race-detection.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,300 @@
import test from "node:test";
import assert from "node:assert/strict";
import { mkdirSync, mkdtempSync, readdirSync, readFileSync, statSync, utimesSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import path from "node:path";
import { createInitialState } from "acp-kernel";
import type { CompressionBlock } from "acp-kernel";
import { flatFileNameFor } from "acp-kernel/persist";
import { SessionStore, relPathFor } from "../src/persist.ts";
import type { Session } from "../src/session.ts";
import { _resetSessionsForTest } from "../src/session.ts";
import { rmrf } from "./tmp-rm.ts";

// #3406 item 3 (detection half of the #3199 race): when a label-based resume
// lands while the post-boot background alias scan is still running, the alias
// does not exist yet, so getSession mints a fresh EMPTY session under the
// label; once the scan reaches the older file the direct-id file wins every
// later lookup and the prior conversation stays unreachable under the label.
// A full fix needs an index/format decision (owner-gated); these tests pin
// the stopgap: at scan completion the store logs ONE loud warn naming both
// states — for BOTH on-disk shapes of the racy direct-id file (flat and
// protocol-namespaced, #1937) — and stays silent for the two look-alike
// shapes that are NOT the race (a healthy alias-resume re-key keeps its
// original createdAt; a plain unrecorded alias has no conflicting direct-id
// file at all).

// Plain JSON session files so seeds and round-trips stay deterministic (#1080)
process.env.BILI_PERSIST_ZSTD = "0";

// Hermetic config: a developer machine's real billion-context.json must not
// leak storage settings into this suite's tmp-dir stores (same discipline as
// tests/issue3199-overview-cold-scan.test.ts).
process.env.BILI_CONFIG_FILE = path.join(
mkdtempSync(path.join(tmpdir(), "bili-3406-hermetic-")),
"no-config.json",
);
// Absent everywhere resolves to the shadow DEFAULT (#2815 rollout flip); pin
// the explicit legacy baseline — legs flip per test.
process.env.BILI_STORAGE_UNIFIED = "legacy";

interface Stats {
requests: number;
tokensSaved: number;
inputTokens: number;
cachedTokens: number;
outputTokens: number;
cacheSamples: number;
lastInputTokens: number;
contextTokens: number;
retrieveCalls: number;
retrieveHits: number;
retrieveMisses: number;
retrieveDropped: number;
retrieveDelivered: number;
storedBytes: number;
storeBytesSaved: number;
compressCreditTokens: number;
rangeRestores: number;
}

function zeroStats(): Stats {
return { requests: 0, tokensSaved: 0, inputTokens: 0, cachedTokens: 0, outputTokens: 0, cacheSamples: 0, lastInputTokens: 0, contextTokens: 0, retrieveCalls: 0, retrieveHits: 0, retrieveMisses: 0, retrieveDropped: 0, retrieveDelivered: 0, storedBytes: 0, storeBytesSaved: 0, compressCreditTokens: 0, rangeRestores: 0 };
}

function makeSession(id: string, meta: Session["meta"] = {}, patch: Partial<Stats> = {}, lastSeen = Date.now()): Session {
return {
id,
meta,
stats: { ...zeroStats(), ...patch },
metadata: {},
state: createInitialState(),
createdAt: lastSeen,
lastSeen,
blockContents: new Map(),
inFlight: 0,
persisted: false,
pendingRetrievals: [],
};
}

const mkBlock = (blockId: string, compressedTokens: number, active: boolean, topic = "T"): CompressionBlock => ({
blockId,
runId: "r1",
tier: 1,
topic,
summary: "S",
directMessageIds: [],
effectiveMessageIds: [],
directBlockIds: [],
compressedTokens,
createdAt: Date.now(),
survivedCount: 1,
generation: "young",
active,
});

/** Full persisted-state shape (kernel initial state + our blocks) so hand-
* written records survive both summary extraction AND buildSession's compat
* pass in the loadSync legs. */
function stateWith(blocks: CompressionBlock[]): unknown {
const s = JSON.parse(JSON.stringify(createInitialState())) as { blocks: unknown[] };
s.blocks = blocks;
return s;
}

/** id → absolute envelope path, by scanning the corpus (layout-independent —
* protocol-namespaced files live in subdirectories, #1937). */
function fileById(root: string, id: string): string | null {
const walk = (r: string): string[] => {
const out: string[] = [];
for (const e of readdirSync(r, { withFileTypes: true })) {
const p = path.join(r, e.name);
if (e.isDirectory()) out.push(...walk(p));
else if (e.name.endsWith(".json")) out.push(p);
}
return out;
};
for (const p of walk(root)) {
try {
const env = JSON.parse(readFileSync(p, "utf8")) as { id?: unknown; payload?: { id?: unknown } };
if (env.id === id || env.payload?.id === id) return p;
} catch {
// non-JSON / unreadable: not our file
}
}
return null;
}

interface LogLine { level: "info" | "warn" | "error"; msg: string }

interface UnderLabelShape { createdAt: number; savedAt: number; mtimeSec: number; namespaced?: boolean }

/** Corpus shared by all legs: 4 filler sessions (newest mtimes — they take
* the whole warm window) plus the OLD labeled session outside the window.
* `underLabel` adds the conflicting direct-id file when given. Returns every
* seeded id with its on-disk path (mtime already stamped). */
async function seedCorpus(dir: string, underLabel?: UnderLabelShape): Promise<Array<{ id: string; file: string }>> {
const seeder = new SessionStore({ dir, debounceMs: 0, enabled: true });
assert.ok(seeder.replicasAvailable(), "sidecar must be live for these legs (engine available)");
const base = Math.floor(Date.now() / 1000) - 5 * 60;
const entries: Array<{ id: string; file: string }> = [];
for (let i = 0; i < 4; i++) {
const id = `s3406-fill-${i}`;
const s = makeSession(id, { protocol: "openai", title: `Fill ${i}`, label: id }, { requests: i + 1, contextTokens: 100 * (i + 1) });
s.state.blocks.push(mkBlock(`sf${i}`, 100 * (i + 1), true, `Fill ${i}`));
await seeder.writeNow(s);
entries.push({ id, file: "" });
}
seeder.closeQuietly();
for (const e of entries) {
e.file = fileById(dir, e.id)!;
assert.ok(e.file, `session file for ${e.id}`);
utimesSync(e.file, base + Number(e.id.slice(-1)) * 60, base + Number(e.id.slice(-1)) * 60);
}
const oldFile = path.join(dir, flatFileNameFor("s3406-old"));
writeFileSync(oldFile, JSON.stringify({
version: 2,
savedAt: (base - 120) * 1000,
createdAt: (base - 120) * 1000,
id: "s3406-old",
meta: { protocol: "openai", title: "Old Conversation", label: "s3406-label" },
stats: { ...zeroStats(), requests: 7, contextTokens: 700 },
state: stateWith([]),
}), "utf8");
utimesSync(oldFile, base - 120, base - 120);
entries.push({ id: "s3406-old", file: oldFile });
if (underLabel) {
// Production racy sessions carry protocol meta and persist NAMESPACED
// (proto/<host>_<hash>.json, #1937) — matched by the suffix branch;
// namespaced:false pins the flat fallback shape.
const rel = underLabel.namespaced ? relPathFor("s3406-label", "openai") : flatFileNameFor("s3406-label");
const nFile = path.join(dir, rel);
mkdirSync(path.dirname(nFile), { recursive: true });
writeFileSync(nFile, JSON.stringify({
version: 2,
savedAt: underLabel.savedAt,
createdAt: underLabel.createdAt,
id: "s3406-label",
meta: { protocol: "openai", title: "Minted Under Label", label: "s3406-label" },
stats: { ...zeroStats() },
state: stateWith([]),
}), "utf8");
utimesSync(nFile, underLabel.mtimeSec, underLabel.mtimeSec);
entries.push({ id: "s3406-label", file: nFile });
}
return entries;
}

function withShadowMode<T>(name: string, fn: (dir: string) => Promise<T>): void {
test(name, async () => {
const dir = mkdtempSync(path.join(tmpdir(), "bili-3406-"));
const prevUnified = process.env.BILI_STORAGE_UNIFIED;
process.env.BILI_STORAGE_UNIFIED = "shadow";
try {
await fn(dir);
} finally {
if (prevUnified === undefined) delete process.env.BILI_STORAGE_UNIFIED; else process.env.BILI_STORAGE_UNIFIED = prevUnified;
_resetSessionsForTest();
rmrf(dir);
}
});
}

async function bootAndScan(dir: string, logs: LogLine[]): Promise<SessionStore> {
let maxFileSize = 0;
for (const f of readdirSync(dir, { recursive: true }) as string[]) {
if (!f.endsWith(".json") || f.endsWith(".content-store.json")) continue;
try { maxFileSize = Math.max(maxFileSize, statSync(path.join(dir, f)).size); } catch { /* raced deletion */ }
}
const store = new SessionStore({ dir, debounceMs: 0, enabled: true, log: (level, msg) => logs.push({ level, msg }) });
await store.boot({ maxSessions: 4, maxBytes: maxFileSize * 5 });
await store.backgroundScanComplete();
return store;
}

withShadowMode("#3406 item 3: racy empty-under-label session gets exactly one loud warn at scan completion", async (dir) => {
// createdAt/savedAt stamped AFTER the boot below starts (future dates make
// the "born after bootStartedAtMs" discriminator deterministic — no sleeps).
const entries = await seedCorpus(dir, { createdAt: Date.now() + 60_000, savedAt: Date.now() + 60_000, mtimeSec: Math.floor(Date.now() / 1000) - 6 * 60 });
const logs: LogLine[] = [];
// Byte budget = EXACTLY the four filler sizes: the parse window stops the
// moment the fillers fill it, so the two late files (racy label + old
// labeled) both fall through to the background scan — the race shape.
const fillsTotal = entries.filter((e) => e.id.startsWith("s3406-fill-")).reduce((sum, e) => sum + statSync(e.file).size, 0);
const store = new SessionStore({ dir, debounceMs: 0, enabled: true, log: (level, msg) => logs.push({ level, msg }) });
const booted = await store.boot({ maxSessions: 4, maxBytes: fillsTotal });
assert.deepEqual([...booted.keys()].sort(), ["s3406-fill-0", "s3406-fill-1", "s3406-fill-2", "s3406-fill-3"], "warm set = the 4 fillers only");

// Symptom pin: while the scan has not recorded the alias, resuming by
// label hands back the EMPTY session minted under the label.
const resumed = store.loadSync("s3406-label");
assert.ok(resumed, "direct-id file resolves under the label");
assert.equal(resumed!.stats.requests, 0, "the resumed state is the empty racy one");

await store.backgroundScanComplete();
const warns = logs.filter((l) => l.level === "warn" && l.msg.includes("label resume raced"));
assert.equal(warns.length, 1, "exactly one race warning");
assert.ok(warns[0].msg.includes("s3406-label"), "warn names the label");
assert.ok(warns[0].msg.includes("s3406-old"), "warn names the surviving conversation id");

// The prior conversation stays reachable by its real id.
const old = store.loadSync("s3406-old");
assert.ok(old, "old state loads by id");
assert.equal(old!.stats.contextTokens, 700);
// …and the label still serves the empty state (the stranded shape the warn describes).
const again = store.loadSync("s3406-label");
assert.equal(again!.stats.requests, 0, "direct-id file keeps winning lookups under the label");
store.closeQuietly();
});

withShadowMode("#3406 item 3: namespaced (production-shape) racy session gets exactly one loud warn at scan completion", async (dir) => {
// Same race as leg 1, but the racy direct-id file sits where real traffic
// puts it: openai/<host>_<hash>.json — detection must find it through the
// suffix branch, not just the flat-name match.
const entries = await seedCorpus(dir, { createdAt: Date.now() + 60_000, savedAt: Date.now() + 60_000, mtimeSec: Math.floor(Date.now() / 1000) - 6 * 60, namespaced: true });
const logs: LogLine[] = [];
const fillsTotal = entries.filter((e) => e.id.startsWith("s3406-fill-")).reduce((sum, e) => sum + statSync(e.file).size, 0);
const store = new SessionStore({ dir, debounceMs: 0, enabled: true, log: (level, msg) => logs.push({ level, msg }) });
const booted = await store.boot({ maxSessions: 4, maxBytes: fillsTotal });
assert.deepEqual([...booted.keys()].sort(), ["s3406-fill-0", "s3406-fill-1", "s3406-fill-2", "s3406-fill-3"], "warm set = the 4 fillers only");
const racyFile = entries.find((e) => e.id === "s3406-label")!.file;
assert.ok(path.relative(dir, racyFile).includes(path.sep), "racy file is protocol-namespaced, not flat");

const resumed = store.loadSync("s3406-label");
assert.ok(resumed, "namespaced direct-id file resolves under the label");
assert.equal(resumed!.stats.requests, 0, "the resumed state is the empty racy one");

await store.backgroundScanComplete();
const warns = logs.filter((l) => l.level === "warn" && l.msg.includes("label resume raced"));
assert.equal(warns.length, 1, "exactly one race warning");
assert.ok(warns[0].msg.includes("s3406-label"), "warn names the label");
assert.ok(warns[0].msg.includes("s3406-old"), "warn names the surviving conversation id");

const old = store.loadSync("s3406-old");
assert.ok(old, "old state loads by id");
assert.equal(old!.stats.contextTokens, 700);
const again = store.loadSync("s3406-label");
assert.equal(again!.stats.requests, 0, "namespaced direct-id file keeps winning lookups under the label");
store.closeQuietly();
});

withShadowMode("#3406 item 3 control: a healthy alias-resume re-key (original createdAt) stays silent", async (dir) => {
const base = Math.floor(Date.now() / 1000) - 5 * 60;
await seedCorpus(dir, { createdAt: (base - 60) * 1000, savedAt: (base - 60) * 1000, mtimeSec: base - 60 });
const logs: LogLine[] = [];
const store = await bootAndScan(dir, logs);
assert.equal(logs.filter((l) => l.msg.includes("label resume raced")).length, 0, "re-keyed healthy resume must not warn");
store.closeQuietly();
});

withShadowMode("#3406 item 3 control: a plain unrecorded alias with no conflicting file stays silent", async (dir) => {
await seedCorpus(dir);
const logs: LogLine[] = [];
const store = await bootAndScan(dir, logs);
assert.equal(logs.filter((l) => l.msg.includes("label resume raced")).length, 0, "plain alias resolution must not warn");
const aliased = store.loadSync("s3406-label");
assert.ok(aliased, "alias still resolves through the scan");
assert.equal(aliased!.stats.contextTokens, 700, "alias serves the old conversation");
store.closeQuietly();
});
Loading