Skip to content
Open
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
86 changes: 67 additions & 19 deletions tests/issue5412-async-drift-latch-parity.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,14 +6,30 @@
* `completeLog = hasRawTail || driftedSessions.has(id)` so a latched save
* claims authoritative-set authority and its stale sweep retires residue
* rows even without a raw tail. The async port
* (`UnifiedShadow.enqueueAsyncRecord`) was not updated: under the same
* latch it kept ingesting `{ completeLog: hasRawTail }`, i.e. false — every
* latched save re-took the incomplete view, the next sample drifted again,
* and a session frozen via the #4323 escalation could never converge back
* to clean on an async backend (mysql etc.). Positive gate pinned here:
* latched folded save → foreign row retired. Negative gate: an UNLATCHED
* folded save leaves foreign rows untouched (no raw tail, no latch) — the
* historical behavior must not leak into async either way.
* (`UnifiedShadow.enqueueAsyncRecord`) originally kept
* `{ completeLog: hasRawTail }`: under the latch that is always false, so
* every latched save re-took the incomplete view, each sampled verify kept
* drifting, the watermark was cleared every time, and a session frozen via
* the #4323 escalation could never release its latch on an async backend
* (mysql etc.). Fixed in #5420 by mirroring the sync condition verbatim.
*
* Gates (async-lane shadow over a REAL sqlite backend via openBackend —
* sync === null forces record() onto enqueueAsyncRecord's serial queue, the
* same queue an opt-in network backend runs on):
* A. POSITIVE GATE. A drift-latched folded save runs the authoritative-set
* sweep and retires the foreign row out of the index.
* B. NEGATIVE GATE. An UNLATCHED folded save leaves foreign rows untouched
* (no raw tail, no latch -> no sweep authority over the table) — the
* historical behavior must not leak into async either way.
*
* Hardened 2026-11-11 during review of PR #5421: the original gates waited
* a fixed settle(300ms) for the serial queue to drain, which could false-red
* gate A on loaded runners and pass gate B vacuously if the folded save had
* not been processed yet. Both gates now poll bounded conditions (row
* retired / ingests counter) instead of sleeping; cleanup uses closeAsync()
* (close() is a no-op in async mode and leaked the sqlite handle); the
* BILI_STORAGE_UNIFIED restore captures the previous value before deleting
* it.
*/

import { test } from "node:test";
Expand All @@ -31,6 +47,7 @@ process.env.BILI_PERSIST_ZSTD = "0";
// Hermeticity pin, same family as the #4323/#5408 suites: a developer
// machine with storage.unified in its real config must not leak a tier.
const HERMETIC = fs.mkdtempSync(path.join(os.tmpdir(), "bili-5412-hermetic-"));
const PREV_UNIFIED = process.env.BILI_STORAGE_UNIFIED;
process.env.BILI_CONFIG_FILE = path.join(HERMETIC, "no-config.json");
delete process.env.BILI_STORAGE_UNIFIED;

Expand All @@ -46,11 +63,6 @@ function tmpDir(tag: string): string {
return fs.mkdtempSync(path.join(HERMETIC, `${tag}-`));
}

/** Give the serial backend queue a turn so enqueueAsyncRecord drains. */
function settle(ms = 300): Promise<void> {
return new Promise((r) => setTimeout(r, ms));
}

function msg(i: number, text = `hello ${i}`): CoreMessage {
return { id: `h_msg${i}`, role: i % 2 === 0 ? "user" : "assistant", contentType: "text", text } as CoreMessage;
}
Expand All @@ -75,6 +87,16 @@ function mkLegacy(id: string, lastMessages: LegacySessionLike["lastMessages"]):
};
}

interface StatsView {
ingests: number;
verifyOk: number;
verifyDrift: number;
}

function stats(shadow: UnifiedShadow): StatsView {
return (shadow as unknown as { stats: StatsView }).stats;
}

/** Async-lane shadow over a REAL sqlite backend: sync === null forces
* record() onto enqueueAsyncRecord — the same queue an opt-in network
* backend runs on, without needing a live mysql server. */
Expand All @@ -85,6 +107,17 @@ function openAsyncShadow(dir: string): Promise<{ shadow: UnifiedShadow }> {
}));
}

/** Count the session's live (non-retired) message rows. */
function countMessages(dir: string, sessionId: string): number | null {
const conn = openSqliteDatabase(path.join(dir, "index.db")).db;
try {
const row = conn.prepare("SELECT COUNT(*) AS n FROM messages WHERE session_id = ? AND retired_at IS NULL").get(sessionId) as { n: number } | undefined;
return row ? row.n : null;
} finally {
conn.close();
}
}

/** Insert a foreign message row (identity absent from any plan this test
* produces) — the residue shape a second writer or a lost rewind leaves.
* (Replicated from the #5408 suite.) */
Expand Down Expand Up @@ -117,24 +150,34 @@ function latchDrift(shadow: UnifiedShadow, id: string): void {
sh.driftedSessions.add(id);
}

/** Poll an async-queue effect to completion (bounded) instead of a fixed sleep. */
async function waitFor(fn: () => boolean, what: string, timeoutMs = 5000): Promise<void> {
const deadline = Date.now() + timeoutMs;
for (;;) {
if (fn()) return;
if (Date.now() > deadline) throw new Error(`timed out waiting for ${what}`);
await new Promise((r) => setTimeout(r, 10));
}
}

test("#5412 A: an async drift-latched folded save sweeps the foreign row out of the index", async () => {
const dir = tmpDir("latched");
try {
const { shadow } = await openAsyncShadow(dir);
const ids = "5412-latched";
try {
shadow.record(mkLegacy(ids, Array.from({ length: 5 }, (_, i) => msg(i))));
await settle();
await waitFor(() => (countMessages(dir, ids) ?? 0) >= 5, "first ingest to land 5 rows");
insertForeignRow(dir, ids);
assert.equal(foreignRowRetired(dir, ids), false, "foreign row present before the latched save");

latchDrift(shadow, ids);
shadow.record({ ...mkLegacy(ids, []), lastMessagesFolded: true });
await settle();

await waitFor(() => foreignRowRetired(dir, ids) === true, "latched folded save to run the authoritative-set sweep");
assert.equal(foreignRowRetired(dir, ids), true, "latched folded save on the async queue must run the authoritative-set sweep and retire the foreign row");
} finally {
shadow.close();
await shadow.closeAsync();
}
} finally {
rmBestEffort(dir);
Expand All @@ -148,19 +191,24 @@ test("#5412 B negative gate: an async UNLATCHED folded save leaves foreign rows
const ids = "5412-control";
try {
shadow.record(mkLegacy(ids, Array.from({ length: 5 }, (_, i) => msg(i))));
await settle();
await waitFor(() => (countMessages(dir, ids) ?? 0) >= 5, "first ingest to land 5 rows");
insertForeignRow(dir, ids);

// No latch: the historical behavior — a folded save (no raw tail)
// does not claim authoritative-set authority over the table.
shadow.record({ ...mkLegacy(ids, []), lastMessagesFolded: true });
await settle();
await waitFor(() => stats(shadow).ingests >= 2, "folded save to process");

assert.equal(foreignRowRetired(dir, ids), false, "unlatched folded save must not sweep (no raw tail, no latch)");
} finally {
shadow.close();
await shadow.closeAsync();
}
} finally {
rmBestEffort(dir);
}
});

test.after(() => {
if (PREV_UNIFIED === undefined) delete process.env.BILI_STORAGE_UNIFIED;
else process.env.BILI_STORAGE_UNIFIED = PREV_UNIFIED;
});
Loading