From bbb49d3e4e7e74ff253355626bf33e34cd9ce1c9 Mon Sep 17 00:00:00 2001 From: Omar Gerardo Date: Wed, 26 Aug 2026 12:52:47 +0800 Subject: [PATCH 1/4] fix(state): reclaim index generations the manifest can no longer name saveShardedIndex wrote each new generation's shards, published a manifest naming it, then deleted the previous generation only when its manifest read had succeeded. That read is the first thing the function does, and it fails under exactly the load that makes saves slow, so `previous` became null and the whole preceding generation was stranded. Nothing revisits it: a later save only inspects its own predecessor, and loadShardedData reads only the shards the current manifest lists. Two further paths reached the same state. The manifest catch block ends in a throw, so a write that threw after committing skipped the cleanup below it. And a kill between the first shard write and the cleanup left a generation nothing had recorded. Verified on a deployed store: 8 dead BM25 generations, 339 MB, 33.5% of a 1013 MB store, the oldest 13 days old, against 35 MB of actual observations. Every orphan's generation id decodes to a restart or an incident. Record each generation in a ledger beside its manifest, written before the first shard write, and reclaim from it after publish and again on load. StateKV lists keys within a scope and never scopes by prefix, so a generation that is not recorded cannot be found again by anything. Serialise saves. scheduleSave fires save() unawaited from a timer while flushIndexSave awaits save() on every delete path, and stop() clears the timer without awaiting a save already running, so saves overlap by construction. Two of them read-modify-writing one ledger drop each other's entries, and the one that publishes second reclaims the shards the first is still writing, leaving it to publish a manifest naming data that is already gone. The queue never coalesces: flushIndexSave awaits save() to make a delete durable, so returning an in-flight promise that started before the delete would report success for a snapshot without it. The load-time reclaim joins the same queue. An unreadable ledger skips that cycle's tracking and reclaim rather than failing the save. state::get timing out is the condition this bug appears under, and aborting there would stop persisting the index entirely, which costs a full-corpus rebuild. A ledger whose shape is not recognised is left untouched, never rewritten. Reclaim keeps malformed entries instead of iterating them, so a GC step can never fail a load, and emits one audit row per sweep rather than one per shard, per the policy in src/functions/audit.ts. Each index saves independently, since one try around both would let a BM25 failure stop the vector index persisting, and a lost vector index is never rebuilt: both rebuild triggers key on the BM25 size. A failed delete keeps its shards listed and retries, rather than stranding them. The ledger cannot discover generations orphaned before it existed; those need a one-time reclaim. Closes #1115 --- src/state/index-persistence.ts | 402 +++++++++++++++++++++++++++++++-- test/index-persistence.test.ts | 377 +++++++++++++++++++++++++++++++ 2 files changed, 754 insertions(+), 25 deletions(-) diff --git a/src/state/index-persistence.ts b/src/state/index-persistence.ts index 6df0e2fdaf..8b0d125423 100644 --- a/src/state/index-persistence.ts +++ b/src/state/index-persistence.ts @@ -24,6 +24,27 @@ type IndexShardManifest = { chars: number; }; +// Suffix for the reclaim ledger that sits beside each manifest. +const GC_LEDGER_SUFFIX = ":gc"; + +// Stands in for a manifest written before generations were recorded, whose +// `generation` field is absent. Cannot collide with createIndexGeneration(). +const PRE_LEDGER_GENERATION = "pre-ledger"; + +// Every generation whose shards may still be on disk, live one included. The +// manifest alone cannot answer that: it names only the generation that is +// current, so a generation stranded by a failed read, a throw after commit, or +// a kill mid-cleanup becomes unreachable the moment the next manifest replaces +// it. Nothing else enumerates shard scopes — StateKV lists keys within a scope, +// not scopes by prefix — so what is not recorded here can never be found again. +type IndexGcLedger = { + v: 1; + generations: Array<{ + generation: string; + shards: Array<{ scope: string; key: string }>; + }>; +}; + type IndexPersistenceOptions = { shardChars?: number; createGeneration?: () => string; @@ -67,7 +88,8 @@ function isValidShardDescriptor( export class IndexPersistence { private timer: ReturnType | null = null; - private lastFailureLogAt = 0; + private lastFailureLogAt = new Map(); + private queue: Promise = Promise.resolve(); constructor( private kv: StateKV, @@ -83,7 +105,7 @@ export class IndexPersistence { // under sustained iii-engine write timeouts (issue #204). Funnel // rejections through logFailure() instead. this.timer = setTimeout(() => { - this.save().catch((err) => this.logFailure(err)); + this.save().catch((err) => this.logFailure("index", err)); }, DEBOUNCE_MS); } @@ -92,13 +114,52 @@ export class IndexPersistence { clearTimeout(this.timer); this.timer = null; } + return this.enqueue(() => this.runSave()); + } + + /** + * Serialise everything that read-modify-writes the gc ledger. + * + * Two saves genuinely overlap here: scheduleSave fires save() unawaited from + * a timer, flushIndexSave awaits save() on every delete path + * (src/functions/search.ts), and stop() clears the timer without awaiting a + * save already running. Overlapping saves drop each other's ledger entries, + * and worse, a save that publishes second reclaims the shards the first is + * still writing — leaving the first to publish a manifest naming data that + * is already gone, which fails the next load closed. + * + * Queue, never coalesce. flushIndexSave awaits this to make a delete + * durable, so handing back an in-flight promise that started before the + * delete would report success for a snapshot that does not contain it. The + * cost is real: a delete-path flush waits out the save ahead of it. + */ + private enqueue(work: () => Promise): Promise { + // runSave never rejects and the tail below always fulfils, so the queue + // cannot be left rejected and needs no rejection handler here. + const run = this.queue.then(work); + this.queue = run.then( + () => undefined, + () => undefined, + ); + return run; + } + + private async runSave(): Promise { + // Each index fails on its own. One try around both would let a BM25 + // failure stop the vector index persisting at all, and a lost vector index + // is never rebuilt: both rebuild triggers key on the BM25 size + // (src/index.ts, src/functions/search.ts). try { await this.saveBm25Index(this.bm25.serialize()); - if (this.vector) { + } catch (err) { + this.logFailure("BM25", err); + } + if (this.vector) { + try { await this.saveVectorIndex(this.vector.serialize()); + } catch (err) { + this.logFailure("vector", err); } - } catch (err) { - this.logFailure(err); } } @@ -129,16 +190,19 @@ export class IndexPersistence { } } - private logFailure(err: unknown): void { + private logFailure(index: string, err: unknown): void { const now = Date.now(); // Throttle: persistence failures under load arrive in bursts // (iii-engine queue pressure). Logging every debounce flush adds - // noise without information. - if (now - this.lastFailureLogAt < FAILURE_LOG_THROTTLE_MS) return; - this.lastFailureLogAt = now; + // noise without information. Throttled PER INDEX, so a vector failure + // right after a BM25 one is not swallowed — the two fail independently + // now, and at 3am you need to know which one stopped persisting. + const lastAt = this.lastFailureLogAt.get(index) ?? 0; + if (now - lastAt < FAILURE_LOG_THROTTLE_MS) return; + this.lastFailureLogAt.set(index, now); const code = (err as { code?: string })?.code; const message = err instanceof Error ? err.message : String(err); - logger.warn("index persistence: failed to save BM25/vector index", { + logger.warn(`index persistence: failed to save ${index} index`, { code, message, hint: @@ -192,6 +256,16 @@ export class IndexPersistence { chunks.push(chunk); } + // Record the generation BEFORE the first shard write. A kill anywhere from + // here to the manifest publish would otherwise leave shards on disk that + // nothing references and nothing can enumerate. + const tracked = await this.trackGeneration( + manifestKey, + generation, + shards, + previous, + ); + const writeResults = await Promise.allSettled( shards.map(async (shard, index) => { const chunk = chunks[index] ?? ""; @@ -211,7 +285,14 @@ export class IndexPersistence { (result): result is PromiseRejectedResult => result.status === "rejected", ); if (failedWrite) { - await this.deleteShards(shards, "shard_write_rollback"); + const allGone = await this.deleteShards(shards, "shard_write_rollback"); + // Drop the entry only if every shard actually went. A shard that survived + // its delete is unnameable once its entry is gone, which is the contract + // this file states for reclaim: a delete failure costs a retry, never a + // stranded generation. + if (tracked && allGone) { + await this.untrackGeneration(manifestKey, generation); + } throw failedWrite.reason; } @@ -249,23 +330,275 @@ export class IndexPersistence { error: errorMessage(err), }); } else { - await this.deleteShards(shards, "manifest_publish_rollback"); + const allGone = await this.deleteShards( + shards, + "manifest_publish_rollback", + ); + if (tracked && allGone) { + await this.untrackGeneration(manifestKey, generation); + } } throw err; } await this.deleteKey(KV.bm25Index, legacyKey, "legacy_cleanup"); - if (previous?.v === 1 && Array.isArray(previous.shards)) { - const currentShardIds = new Set( - shards.map((shard) => `${shard.scope}\0${shard.key}`), + // Only reclaim when this generation is actually in the ledger. Reclaiming + // against a ledger that does not list us would treat live shards as dead. + if (tracked) { + await this.reclaimGenerations(manifestKey, generation); + } else if (previous?.v === 1 && Array.isArray(previous.shards)) { + // The ledger was unusable this cycle, so nothing above will ever revisit + // `previous`. Fall back to the pre-ledger cleanup: the manifest just + // published supersedes it, and saves are serialised, so its shards are + // dead. Without this, a run of unusable cycles orphans one generation + // each — strictly worse than the code this replaced, which always had + // this path. + const liveIds = new Set( + shards.map((shard) => `${shard.scope}\u0000${shard.key}`), + ); + await this.deleteShards( + previous.shards.filter( + (shard) => + isValidShardDescriptor(shard) && + !liveIds.has(`${shard.scope}\u0000${shard.key}`), + ), + "previous_generation_cleanup", + ); + } + } + + /** Drop a generation's entry after its shards have been rolled back. */ + private async untrackGeneration( + manifestKey: string, + generation: string, + ): Promise { + try { + const ledger = await this.readLedger(manifestKey); + const generations = ledger.generations.filter( + (entry) => entry?.generation !== generation, ); - for (const shard of previous.shards) { - if (currentShardIds.has(`${shard.scope}\0${shard.key}`)) continue; - await this.deleteShards([shard], "previous_generation_cleanup"); + if (generations.length === ledger.generations.length) return; + await this.kv.set(KV.bm25Index, this.gcKey(manifestKey), { + v: 1, + generations, + }); + } catch { + // Best effort. A surviving entry costs a retry on the next reclaim, and + // this runs while a save is already failing — never make that worse. + } + } + + private gcKey(manifestKey: string): string { + return `${manifestKey}${GC_LEDGER_SUFFIX}`; + } + + private async readLedger(manifestKey: string): Promise { + // Throws rather than returning a blank ledger, because a blank one would + // be written straight back over whatever is stored. Callers catch it and + // skip tracking for that cycle; they must never treat it as "no ledger". + const stored = await this.kv.get( + KV.bm25Index, + this.gcKey(manifestKey), + ); + if (stored == null) return { v: 1, generations: [] }; + // Present but unrecognised: a newer version, or a rollback to this build + // after one that wrote a different shape. Overwriting drops every + // generation it tracked, so leave it exactly where it is. + if (stored.v !== 1 || !Array.isArray(stored.generations)) { + throw new Error( + `index gc ledger ${this.gcKey(manifestKey)} has an unrecognised shape ` + + `(v=${String((stored as { v?: unknown }).v)}); refusing to overwrite it`, + ); + } + return stored; + } + + private async trackGeneration( + manifestKey: string, + generation: string, + shards: IndexShardManifest["shards"], + previous: IndexShardManifest | null, + ): Promise { + try { + // The WHOLE body runs under this guard, not just the ledger read. A + // malformed ledger entry or manifest shard would otherwise throw a + // TypeError out of saveShardedIndex before any shard write, leaving one + // throttled log line per 60s as the only trace while BM25 stopped + // persisting for good. isValidShardDescriptor exists in this file + // because a per-shard-malformed manifest is already its threat model. + return await this.recordGeneration( + manifestKey, + generation, + shards, + previous, + ); + } catch (err) { + // A state::get brownout is the exact condition this bug appears under, + // so this path is not rare. Aborting the save here would stop persisting + // the index at all, which is worse than the leak being fixed. Writing a + // fresh ledger would drop every generation the stored one lists. Do + // neither: let the shards and manifest land, skip this cycle's tracking + // and its reclaim, and leak at most one generation instead of one per + // brownout. + // + // That leak can OUTLIVE the unreadable window. This generation publishes + // untracked; the next save re-seeds it from `previous`, but that read is + // itself `.catch(() => null)`, so if it also fails the generation is in + // no ledger and reclaim only ever looks at what precedes the live entry. + // With no scope enumeration in StateKV, nothing can find it again. + // Throttled for the same reason every other failure log here is: an + // unusable ledger stays unusable, so this fires on every debounce. + const throttleKey = `gc:${manifestKey}`; + const now = Date.now(); + if (now - (this.lastFailureLogAt.get(throttleKey) ?? 0) >= FAILURE_LOG_THROTTLE_MS) { + this.lastFailureLogAt.set(throttleKey, now); + logger.warn( + "index persistence: gc ledger unavailable, skipping reclaim", + { manifestKey, message: errorMessage(err) }, + ); } + return false; } } + private async recordGeneration( + manifestKey: string, + generation: string, + shards: IndexShardManifest["shards"], + previous: IndexShardManifest | null, + ): Promise { + const ledger = await this.readLedger(manifestKey); + // Optional-chained so one malformed entry costs that entry, not the whole + // cycle's tracking. reclaimGenerations tolerates them the same way. + const known = new Set(ledger.generations.map((entry) => entry?.generation)); + + // Seed the generation the pre-ledger code left live, so upgrading does not + // strand it. Only reachable while `previous` is readable, which is exactly + // the case the old cleanup already handled. + // + // `generation` is optional on the manifest and older stores really do omit + // it, so fall back to a sentinel rather than skipping the seed. The + // sentinel cannot collide with createIndexGeneration()'s `idx_` ids, and + // since every manifest written from here on carries a generation, the + // seeded entry always ends up preceding a live one and gets reclaimed. + if (previous?.v === 1 && Array.isArray(previous.shards)) { + const previousGeneration = previous.generation ?? PRE_LEDGER_GENERATION; + if (!known.has(previousGeneration)) { + ledger.generations.push({ + generation: previousGeneration, + shards: previous.shards + .filter(isValidShardDescriptor) + .map(({ scope, key }) => ({ scope, key })), + }); + known.add(previousGeneration); + } + } + + if (!known.has(generation)) { + ledger.generations.push({ + generation, + shards: shards.map(({ scope, key }) => ({ scope, key })), + }); + } + + // ponytail: unbounded while EVERY delete keeps failing — each debounce + // appends a generation carrying all its shard descriptors, and + // reclaimGenerations only rewrites when something was actually reclaimed. + // Add a length cap if that ever shows up in the wild; a cap is not one + // line, and this needs a total delete outage to reach. + await this.kv.set( + KV.bm25Index, + this.gcKey(manifestKey), + ledger, + ); + return true; + } + + /** + * Delete every tracked generation except the live one, then rewrite the + * ledger with whatever survived. A shard whose delete failed stays listed and + * is retried on the next save or load, so a delete failure costs a retry + * instead of stranding the generation for good. + */ + private async reclaimGenerations( + manifestKey: string, + liveGeneration: string | undefined, + ): Promise { + if (!liveGeneration) return; + const ledger = await this.readLedger(manifestKey).catch(() => null); + if (!ledger) return; + + // Reclaim strictly what precedes the live generation in the ledger, which + // trackGeneration appends to in creation order. Anything at or after the + // live entry is either live or a save still in flight: setIndexPersistence + // runs before load() in src/index.ts, so a request arriving during boot can + // have a save writing shards while this reclaim runs, and deleting those + // would publish a manifest whose shards are already half gone. A generation + // stranded after the live one is not lost, only deferred — it becomes + // reclaimable as soon as a newer generation is published. + const liveIndex = ledger.generations.findIndex( + (entry) => entry?.generation === liveGeneration, + ); + // Live generation untracked (a pre-ledger store, or a manifest written + // before this shipped). Nothing can be classified as superseded, so leave + // every entry alone rather than guess. + if (liveIndex < 0) return; + + const reclaimedPaths: string[] = []; + let failed = 0; + const survivors: IndexGcLedger["generations"] = []; + for (const [index, entry] of ledger.generations.entries()) { + // A malformed entry is kept, never iterated. This also runs on the load + // path, where a throw would null the loaded index and trigger the + // full-corpus rebuild. A GC step must not be able to fail a load. + if (index >= liveIndex || !entry || !Array.isArray(entry.shards)) { + survivors.push(entry); + continue; + } + const stranded: IndexGcLedger["generations"][number]["shards"] = []; + for (const shard of entry.shards) { + try { + await this.kv.delete(shard.scope, shard.key); + reclaimedPaths.push(statePath(shard.scope, shard.key)); + } catch { + failed += 1; + stranded.push(shard); + } + } + if (stranded.length > 0) { + survivors.push({ generation: entry.generation, shards: stranded }); + } + } + + // One audit row for the sweep, not one per shard. src/functions/audit.ts + // sets the policy: automatic bulk sweeps emit a single row listing every + // removed id, because per-item rows flood the log. A reclaim on a badly + // leaked store is well over a thousand shards, and this runs during boot. + if (reclaimedPaths.length > 0) { + await this.auditIndexPersistence("delete", reclaimedPaths, { + manifestKey, + reason: "generation_reclaim", + liveGeneration, + // `evicted` is the field name src/functions/audit.ts specifies for a + // sweep; retention.ts is the reference shape. `failed` surfaces deletes + // that will be retried, which would otherwise vanish silently. + evicted: reclaimedPaths.length, + failed, + }); + } + + // A successful delete is the only thing that shrinks the ledger, so with + // none there is nothing to rewrite. Without this, every boot writes the + // ledger back unchanged. + if (reclaimedPaths.length === 0) return; + await this.kv + .set(KV.bm25Index, this.gcKey(manifestKey), { + v: 1, + generations: survivors, + }) + .catch(() => undefined); + } + private async auditIndexPersistence( action: string, targetIds: string[], @@ -280,35 +613,41 @@ export class IndexPersistence { ); } + /** Reports whether the delete landed, so the reclaim path can retry the rest. */ private async deleteKey( scope: string, key: string, reason: string, - ): Promise { - let result = "deleted"; + ): Promise { + let ok = true; let error: string | undefined; try { await this.kv.delete(scope, key); } catch (err) { - result = "failed"; + ok = false; error = errorMessage(err); } await this.auditIndexPersistence("delete", [statePath(scope, key)], { scope, key, reason, - result, + result: ok ? "deleted" : "failed", error, }); + return ok; } private async deleteShards( shards: IndexShardManifest["shards"], reason: string, - ): Promise { + ): Promise { + let allGone = true; for (const shard of shards) { - await this.deleteKey(shard.scope, shard.key, reason); + if (!(await this.deleteKey(shard.scope, shard.key, reason))) { + allGone = false; + } } + return allGone; } private async isManifestPublished( @@ -368,7 +707,20 @@ export class IndexPersistence { manifest.value != null && typeof manifest.value === "object" ) { - return this.loadManifestData(manifest.value, label); + const data = await this.loadManifestData(manifest.value, label); + // Boot is the only point that sees a generation stranded by a kill: the + // save path only ever inspects its own predecessor. Reclaim once the live + // generation has actually loaded — a failed load must not authorise + // deleting anything. + if (data !== null) { + // Through the same queue as save(), or this sweep's ledger rewrite + // clobbers a concurrent save's entry. Never allowed to fail the load. + const live = manifest.value.generation; + await this + .enqueue(() => this.reclaimGenerations(manifestKey, live)) + .catch(() => undefined); + } + return data; } const legacy = await this.readIndexValue( diff --git a/test/index-persistence.test.ts b/test/index-persistence.test.ts index 929791657f..3dff054091 100644 --- a/test/index-persistence.test.ts +++ b/test/index-persistence.test.ts @@ -563,6 +563,383 @@ describe("IndexPersistence", () => { expect(loaded.bm25!.search("alpha").length).toBe(0); }); + it("reclaims the previous generation when the previous manifest read fails (#1115)", async () => { + const previous = makeBm25("obs_old", "alpha previous snapshot"); + await new IndexPersistence(kv as never, previous, null, { + shardChars: 80, + createGeneration: () => "gen_old", + }).save(); + const oldShardScope = "mem:index:bm25:bm25:gen_old:00000"; + await expect(kv.get(oldShardScope, "data")).resolves.not.toBeNull(); + + // The manifest read that opens saveShardedIndex times out. It used to be + // swallowed into `previous = null`, which skipped the cleanup guard and + // stranded gen_old's shards with nothing left to ever revisit them. + const readFailsKv = { + ...kv, + get: vi.fn(async (scope: string, key: string): Promise => { + if (scope === BM25_SCOPE && key === BM25_MANIFEST_KEY) { + throw new Error("Invocation timeout after 180000ms: state::get"); + } + return kv.get(scope, key); + }), + }; + + const next = makeBm25("obs_new", "bravo new snapshot"); + await new IndexPersistence(readFailsKv as never, next, null, { + shardChars: 80, + createGeneration: () => "gen_new", + }).save(); + + const manifest = await getBm25Manifest(kv); + expect(manifest.generation).toBe("gen_new"); + await expect(kv.get(oldShardScope, "data")).resolves.toBeNull(); + }); + + it("reclaims a generation stranded by a failed cleanup on the next load (#1115)", async () => { + const previous = makeBm25("obs_old", "alpha previous snapshot"); + await new IndexPersistence(kv as never, previous, null, { + shardChars: 80, + createGeneration: () => "gen_old", + }).save(); + const oldShardScope = "mem:index:bm25:bm25:gen_old:00000"; + + const cleanupKv = { + ...kv, + delete: vi.fn(async () => { + throw new Error("cleanup failed"); + }), + }; + const next = makeBm25("obs_new", "bravo new snapshot"); + await new IndexPersistence(cleanupKv as never, next, null, { + shardChars: 80, + createGeneration: () => "gen_new", + }).save(); + + // Cleanup failed, so gen_old is still on disk. Nothing in the save path + // will revisit it — a later save only ever inspects its own predecessor. + await expect(kv.get(oldShardScope, "data")).resolves.not.toBeNull(); + + const loaded = await new IndexPersistence( + kv as never, + new SearchIndex(), + null, + ).load(); + expect(loaded.bm25!.search("bravo").length).toBe(1); + await expect(kv.get(oldShardScope, "data")).resolves.toBeNull(); + }); + + it("reclaims the previous vector generation when the vector manifest read fails (#1115)", async () => { + await new IndexPersistence( + kv as never, + makeBm25("obs_old", "alpha previous snapshot"), + makeVector("obs_old"), + { shardChars: 80, createGeneration: () => "gen_old" }, + ).save(); + const oldVectorScope = "mem:index:bm25:vectors:gen_old:00000"; + await expect(kv.get(oldVectorScope, "data")).resolves.not.toBeNull(); + + const readFailsKv = { + ...kv, + get: vi.fn(async (scope: string, key: string): Promise => { + if (scope === BM25_SCOPE && key === VECTOR_MANIFEST_KEY) { + throw new Error("Invocation timeout after 180000ms: state::get"); + } + return kv.get(scope, key); + }), + }; + + await new IndexPersistence( + readFailsKv as never, + makeBm25("obs_new", "bravo new snapshot"), + makeVector("obs_new"), + { shardChars: 80, createGeneration: () => "gen_new" }, + ).save(); + + await expect(kv.get(oldVectorScope, "data")).resolves.toBeNull(); + }); + + it("leaves a generation recorded after the live one untouched on load (#1115)", async () => { + await new IndexPersistence( + kv as never, + makeBm25("obs_old", "alpha previous snapshot"), + null, + { shardChars: 80, createGeneration: () => "gen_live" }, + ).save(); + + // A concurrent save has recorded its generation and is mid-write, but has + // not published its manifest yet. setIndexPersistence runs before load() + // in src/index.ts, so a request arriving during boot produces exactly this. + const inflightScope = "mem:index:bm25:bm25:gen_inflight:00000"; + await kv.set(inflightScope, "data", "partial shard"); + const gcKey = `${BM25_MANIFEST_KEY}:gc`; + const ledger = await kv.get<{ + v: 1; + generations: Array<{ + generation: string; + shards: Array<{ scope: string; key: string }>; + }>; + }>(BM25_SCOPE, gcKey); + ledger!.generations.push({ + generation: "gen_inflight", + shards: [{ scope: inflightScope, key: "data" }], + }); + await kv.set(BM25_SCOPE, gcKey, ledger); + + await new IndexPersistence(kv as never, new SearchIndex(), null).load(); + + // Deleting it would leave the in-flight save publishing a manifest whose + // shards are already gone, which fails closed on the next load. + await expect(kv.get(inflightScope, "data")).resolves.toBe("partial shard"); + }); + + it("reclaims a pre-ledger manifest that carries no generation (#1115)", async () => { + const legacyScope = "mem:index:bm25:bm25:gen_legacy:00000"; + await kv.set(legacyScope, "data", "legacy shard"); + await kv.set(BM25_SCOPE, BM25_MANIFEST_KEY, { + v: 1, + shards: [{ scope: legacyScope, key: "data", chars: 12 }], + chars: 12, + }); + + await new IndexPersistence( + kv as never, + makeBm25("obs_new", "bravo new snapshot"), + null, + { shardChars: 80, createGeneration: () => "gen_new" }, + ).save(); + + await expect(kv.get(legacyScope, "data")).resolves.toBeNull(); + }); + + it("keeps a published manifest whole when two saves overlap (#1115)", async () => { + // scheduleSave fires save() unawaited from a timer while flushIndexSave + // awaits save() on every delete path, so two saves on ONE instance is the + // normal shape, not a contrivance. Without the queue, whichever published + // second reclaimed the other's shards and left it naming data already gone. + vi.useRealTimers(); + const store = new Map>(); + const slowKv = { + get: async (scope: string, key: string): Promise => + (store.get(scope)?.get(key) as T) ?? null, + set: async (scope: string, key: string, data: T): Promise => { + // Stall one of the first generation's shard writes so the second save + // overtakes it. + if (scope.includes(":gen_0:") && scope.endsWith("00005")) { + await new Promise((resolve) => setTimeout(resolve, 200)); + } + if (!store.has(scope)) store.set(scope, new Map()); + store.get(scope)!.set(key, data); + return data; + }, + delete: async (scope: string, key: string): Promise => { + store.get(scope)?.delete(key); + }, + list: async (scope: string): Promise => + Array.from((store.get(scope)?.values() ?? []) as Iterable), + }; + + const bm25 = new SearchIndex(); + for (let i = 0; i < 30; i++) { + bm25.add( + makeObs({ id: `obs_${i}`, title: `lorem ipsum dolor sit amet ${i}` }), + ); + } + let generation = 0; + const persistence = new IndexPersistence(slowKv as never, bm25, null, { + shardChars: 400, + createGeneration: () => `gen_${generation++}`, + }); + + // Let the first save get into its shard writes before the second starts, + // so the second is the one that publishes. + const first = persistence.save(); + await new Promise((resolve) => setTimeout(resolve, 20)); + const second = persistence.save(); + await Promise.all([first, second]); + + const manifest = await slowKv.get( + BM25_SCOPE, + BM25_MANIFEST_KEY, + ); + const missing: string[] = []; + for (const shard of manifest!.shards) { + if ((await slowKv.get(shard.scope, shard.key)) === null) { + missing.push(shard.scope); + } + } + expect(missing).toEqual([]); + + const loaded = await new IndexPersistence( + slowKv as never, + new SearchIndex(), + null, + ).load(); + expect(loaded.bm25).not.toBeNull(); + }); + + it("still persists the index when the gc ledger is unreadable (#1115)", async () => { + // state::get timing out is the condition this bug appears under. Aborting + // the save there would stop persisting entirely, which is worse than the + // leak: a lost BM25 index costs a full-corpus rebuild. + const failingKv = { + ...kv, + get: vi.fn(async (scope: string, key: string): Promise => { + if (scope === BM25_SCOPE && key === `${BM25_MANIFEST_KEY}:gc`) { + throw new Error("Invocation timeout after 180000ms: state::get"); + } + return kv.get(scope, key); + }), + }; + + await new IndexPersistence( + failingKv as never, + makeBm25("obs_new", "bravo new snapshot"), + null, + { shardChars: 80, createGeneration: () => "gen_new" }, + ).save(); + + const manifest = await getBm25Manifest(kv); + expect(manifest.generation).toBe("gen_new"); + await expect( + kv.get(manifest.shards[0].scope, manifest.shards[0].key), + ).resolves.not.toBeNull(); + }); + + it("still persists when the gc ledger holds a malformed entry (#1115)", async () => { + // A null entry used to throw a TypeError out of saveShardedIndex before any + // shard write, leaving one throttled log line per 60s while BM25 silently + // stopped persisting for good. + await kv.set(BM25_SCOPE, `${BM25_MANIFEST_KEY}:gc`, { + v: 1, + generations: [null], + }); + + await new IndexPersistence( + kv as never, + makeBm25("obs_new", "bravo new snapshot"), + null, + { shardChars: 80, createGeneration: () => "gen_new" }, + ).save(); + + const manifest = await getBm25Manifest(kv); + expect(manifest.generation).toBe("gen_new"); + await expect( + kv.get(manifest.shards[0].scope, manifest.shards[0].key), + ).resolves.not.toBeNull(); + }); + + it("still reclaims when the gc ledger holds a malformed entry (#1115)", async () => { + // Publishing is not enough. A malformed entry that silently disabled + // reclaim would leak a generation every cycle while this test stayed green. + await new IndexPersistence(kv as never, makeBm25("obs_old", "alpha"), null, { + shardChars: 80, + createGeneration: () => "gen_old", + }).save(); + const oldShardScope = "mem:index:bm25:bm25:gen_old:00000"; + + const gcKey = `${BM25_MANIFEST_KEY}:gc`; + const ledger = await kv.get<{ v: 1; generations: unknown[] }>( + BM25_SCOPE, + gcKey, + ); + ledger!.generations.unshift(null); + await kv.set(BM25_SCOPE, gcKey, ledger); + + await new IndexPersistence(kv as never, makeBm25("obs_new", "bravo"), null, { + shardChars: 80, + createGeneration: () => "gen_new", + }).save(); + + await expect(kv.get(oldShardScope, "data")).resolves.toBeNull(); + }); + + it("falls back to previous-generation cleanup when the ledger is unusable (#1115)", async () => { + // Measured regression guard: with no fallback, an unusable ledger leaked + // one generation per cycle — strictly worse than the code this replaced, + // which always had this path. Without this test the `else if` can be + // deleted with the whole suite green. + await new IndexPersistence(kv as never, makeBm25("obs_old", "alpha"), null, { + shardChars: 80, + createGeneration: () => "gen_old", + }).save(); + const oldShardScope = "mem:index:bm25:bm25:gen_old:00000"; + await expect(kv.get(oldShardScope, "data")).resolves.not.toBeNull(); + + // Ledger read fails, manifest read succeeds. Both are state::get with + // independent timeouts, so this split is ordinary, not contrived. + const ledgerFailsKv = { + ...kv, + get: vi.fn(async (scope: string, key: string): Promise => { + if (scope === BM25_SCOPE && key === `${BM25_MANIFEST_KEY}:gc`) { + throw new Error("Invocation timeout after 180000ms: state::get"); + } + return kv.get(scope, key); + }), + }; + + await new IndexPersistence( + ledgerFailsKv as never, + makeBm25("obs_new", "bravo"), + null, + { shardChars: 80, createGeneration: () => "gen_new" }, + ).save(); + + const manifest = await getBm25Manifest(kv); + expect(manifest.generation).toBe("gen_new"); + await expect(kv.get(oldShardScope, "data")).resolves.toBeNull(); + }); + + it("keeps the ledger entry when a rollback delete fails (#1115)", async () => { + // Shards that survived their rollback delete must keep the entry that + // names them, or nothing can ever reclaim them. + let shardWrites = 0; + const failingKv = { + ...kv, + set: vi.fn(async (scope: string, key: string, data: T): Promise => { + if (scope.includes(":gen_new:")) { + shardWrites += 1; + if (shardWrites === 2) throw new Error("shard write failed"); + } + return kv.set(scope, key, data); + }), + delete: vi.fn(async () => { + throw new Error("rollback delete failed"); + }), + }; + + await new IndexPersistence( + failingKv as never, + makeBm25("obs_new", "bravo new snapshot"), + null, + { shardChars: 80, createGeneration: () => "gen_new" }, + ).save(); + + const ledger = await kv.get<{ + v: 1; + generations: Array<{ generation: string }>; + }>(BM25_SCOPE, `${BM25_MANIFEST_KEY}:gc`); + expect(ledger!.generations.map((entry) => entry.generation)).toContain( + "gen_new", + ); + }); + + it("refuses to overwrite a gc ledger it does not recognise (#1115)", async () => { + const gcKey = `${BM25_MANIFEST_KEY}:gc`; + const future = { v: 2, generations: [], writtenByANewerBuild: true }; + await kv.set(BM25_SCOPE, gcKey, future); + + await new IndexPersistence( + kv as never, + makeBm25("obs_new", "bravo new snapshot"), + null, + { shardChars: 80, createGeneration: () => "gen_new" }, + ).save(); + + // Rewriting it as a v1 ledger would drop every generation it tracked. + await expect(kv.get(BM25_SCOPE, gcKey)).resolves.toEqual(future); + }); + it("keeps the previous vector generation when vector save fails after BM25 publish", async () => { const previousBm25 = makeBm25("obs_old", "alpha previous snapshot"); const previousVector = makeVector("obs_old"); From c6851e70c1dc732b3aed0a20a17ca19a7759f8da Mon Sep 17 00:00:00 2001 From: Omar Gerardo Date: Wed, 26 Aug 2026 23:22:15 +0800 Subject: [PATCH 2/4] fix(state): coalesce onto a queued save so a delete does not wait out the line Serialising every save fixed the ledger races, and introduced a worse problem than the one it solved. flushIndexSave awaits save() on five delete paths (governance.ts:33,139, remember.ts:321, retention.ts:392, auto-forget.ts:194), so once saves are queued rather than overlapping, a delete waits for every save ahead of it. That is fine when a save takes a second. When saves are timing out at 180s it turns a delete into an unbounded wait and the request dies. Measured on the deployed instance: no index save completed in a 29 minute window, and 5xx rates rose sharply from the commit that added the queue. The original reasoning against coalescing was right about one case and too broad about the other. A save that is ALREADY RUNNING must never be handed back: its snapshot predates the caller's mutation, so reporting success would tell a delete it was durable when it was not. But a save that is queued and has not started yet has not taken its snapshot, so it will serialise whatever the caller just changed. Those callers can share it. Track the queued-but-not-started save and hand it to arriving callers, clearing it the moment it begins to run so the next caller queues its own. Six concurrent delete flushes now cost at most two saves instead of six, with the durability guarantee unchanged. --- src/state/index-persistence.ts | 25 ++++++++++++++++- test/index-persistence.test.ts | 51 ++++++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+), 1 deletion(-) diff --git a/src/state/index-persistence.ts b/src/state/index-persistence.ts index 8b0d125423..4666f41cd5 100644 --- a/src/state/index-persistence.ts +++ b/src/state/index-persistence.ts @@ -90,6 +90,15 @@ export class IndexPersistence { private timer: ReturnType | null = null; private lastFailureLogAt = new Map(); private queue: Promise = Promise.resolve(); + /** + * A save that is queued but has not started running yet. + * + * It will serialise the index as it stands at the moment it runs, so it + * already covers every caller that arrives before then. Handing them this + * promise instead of queueing another identical save is what keeps a delete + * from waiting out an unbounded line of saves ahead of it. + */ + private pendingSave: Promise | null = null; constructor( private kv: StateKV, @@ -114,7 +123,21 @@ export class IndexPersistence { clearTimeout(this.timer); this.timer = null; } - return this.enqueue(() => this.runSave()); + // Coalesce onto a save that is queued but has not started. It has not taken + // its snapshot yet, so it will include whatever this caller just changed. + // Only a save that is ALREADY RUNNING must not be handed back: its snapshot + // predates this caller's mutation, and reporting success for it would tell + // a delete it was durable when it was not. + if (this.pendingSave) return this.pendingSave; + + const pending = this.enqueue(() => { + // Running now. Its snapshot is fixed from here, so the next caller needs + // a save of its own. + if (this.pendingSave === pending) this.pendingSave = null; + return this.runSave(); + }); + this.pendingSave = pending; + return pending; } /** diff --git a/test/index-persistence.test.ts b/test/index-persistence.test.ts index 3dff054091..4597594a1b 100644 --- a/test/index-persistence.test.ts +++ b/test/index-persistence.test.ts @@ -1168,3 +1168,54 @@ describe("IndexPersistence", () => { await expect(persistence.load()).resolves.toBeDefined(); }); }); + +describe("IndexPersistence save coalescing", () => { + let kv: ReturnType; + + beforeEach(() => { + vi.useFakeTimers(); + kv = mockKV(); + }); + afterEach(() => vi.useRealTimers()); + + it("does not queue a second identical save behind one that has not started", async () => { + // flushIndexSave awaits save() on every delete path. Serialising every one + // of them means a delete waits out every save ahead of it, which is fine + // when a save takes a second and is an outage when saves are timing out at + // 180s. A save that is queued but not yet running will serialise the index + // as it stands when it runs, so it already covers these callers. + let manifestWrites = 0; + let releaseFirst!: () => void; + const firstStarted = new Promise((r) => { + releaseFirst = r; + }); + let started = 0; + const slowKv = { + ...kv, + set: async (scope: string, key: string, data: T): Promise => { + if (scope === BM25_SCOPE && key === BM25_MANIFEST_KEY) { + manifestWrites++; + if (++started === 1) await firstStarted; + } + return kv.set(scope, key, data); + }, + }; + + const persistence = new IndexPersistence( + slowKv as never, + makeBm25("obs_1", "alpha"), + null, + { shardChars: 400 }, + ); + + const first = persistence.save(); + // Five more delete-path flushes arrive while the first is still running. + const rest = Array.from({ length: 5 }, () => persistence.save()); + releaseFirst(); + await Promise.all([first, ...rest]); + + // One running save plus at most one queued behind it covers all six + // callers. Six serialised saves would be the regression. + expect(manifestWrites).toBeLessThanOrEqual(2); + }); +}); From 81127a618ef8776fdb9525af1679ffea01b8f228 Mon Sep 17 00:00:00 2001 From: Omar Gerardo Date: Thu, 27 Aug 2026 00:56:04 +0800 Subject: [PATCH 3/4] refactor(state): name the coalescing field for the invariant it holds Review flagged the comments added with the coalescing fix as explaining what the code does, which AGENTS.md rules out under Coding Standards for src. `pendingSave` needed three comment blocks because the name did not say which saves qualify. `unstartedSave` does: only a save that has not begun running may be handed to a new caller, since a save already in flight took its snapshot before that caller's mutation and reporting success for it would tell a delete it was durable when it was not. Rename plus comment removal, no behaviour change. Also removes an authoring marker that should not have shipped upstream. What it recorded is a genuine limitation and moves to the PR body: the gc ledger's `generations` array grows unbounded while every shard delete keeps failing, because each debounce appends a generation carrying its full shard list and `reclaimGenerations` only rewrites the ledger when something was actually reclaimed. Reaching that state takes a total delete outage, and capping the length is more than a one-line change, so it is disclosed rather than fixed here. --- src/state/index-persistence.ts | 28 ++++------------------------ 1 file changed, 4 insertions(+), 24 deletions(-) diff --git a/src/state/index-persistence.ts b/src/state/index-persistence.ts index 4666f41cd5..64d19f70d5 100644 --- a/src/state/index-persistence.ts +++ b/src/state/index-persistence.ts @@ -90,15 +90,7 @@ export class IndexPersistence { private timer: ReturnType | null = null; private lastFailureLogAt = new Map(); private queue: Promise = Promise.resolve(); - /** - * A save that is queued but has not started running yet. - * - * It will serialise the index as it stands at the moment it runs, so it - * already covers every caller that arrives before then. Handing them this - * promise instead of queueing another identical save is what keeps a delete - * from waiting out an unbounded line of saves ahead of it. - */ - private pendingSave: Promise | null = null; + private unstartedSave: Promise | null = null; constructor( private kv: StateKV, @@ -123,20 +115,13 @@ export class IndexPersistence { clearTimeout(this.timer); this.timer = null; } - // Coalesce onto a save that is queued but has not started. It has not taken - // its snapshot yet, so it will include whatever this caller just changed. - // Only a save that is ALREADY RUNNING must not be handed back: its snapshot - // predates this caller's mutation, and reporting success for it would tell - // a delete it was durable when it was not. - if (this.pendingSave) return this.pendingSave; + if (this.unstartedSave) return this.unstartedSave; const pending = this.enqueue(() => { - // Running now. Its snapshot is fixed from here, so the next caller needs - // a save of its own. - if (this.pendingSave === pending) this.pendingSave = null; + if (this.unstartedSave === pending) this.unstartedSave = null; return this.runSave(); }); - this.pendingSave = pending; + this.unstartedSave = pending; return pending; } @@ -524,11 +509,6 @@ export class IndexPersistence { }); } - // ponytail: unbounded while EVERY delete keeps failing — each debounce - // appends a generation carrying all its shard descriptors, and - // reclaimGenerations only rewrites when something was actually reclaimed. - // Add a length cap if that ever shows up in the wild; a cap is not one - // line, and this needs a total delete outage to reach. await this.kv.set( KV.bm25Index, this.gcKey(manifestKey), From 001581974c1b3731bee645f02422e4cf834e1ba2 Mon Sep 17 00:00:00 2001 From: Omar Gerardo Date: Mon, 31 Aug 2026 12:27:15 +0800 Subject: [PATCH 4/4] fix(state): reclaim vector generations when no vector index is loaded runSave gates the vector save on `this.vector`, and src/index.ts builds a vector index only when an embedding provider exists. On a boot without one, saveVectorIndex never runs, so the vector ledger is never read and its superseded generations are never reclaimed. The bm25 save is unconditional and keeps cycling beside it, which is why the leak is one-sided. Nothing else revisits those shards. Reclaim lives inside the save this branch skips, and StateKV.list takes an exact scope rather than a prefix, so a generation that is in no ledger entry cannot be enumerated back. A production store carried 9 orphaned vector shards, 17 MB, created minutes after a redeploy that came up without a usable provider. Its vectors ledger listed only the live generation. Reclaim now runs on its own when there is no vector index to save. It reads the manifest solely to learn which generation is live; a missing or nameless manifest reclaims nothing, which is the fail-safe direction. The test drives a failed-delete brownout, then a boot with no vector index, and asserts the superseded generation is gone. It fails against the unmodified file, and returning early from the new path fails it again. --- src/state/index-persistence.ts | 27 ++++++++++++++++++++++ test/index-persistence.test.ts | 41 ++++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+) diff --git a/src/state/index-persistence.ts b/src/state/index-persistence.ts index 64d19f70d5..cbf4d7ac4d 100644 --- a/src/state/index-persistence.ts +++ b/src/state/index-persistence.ts @@ -168,6 +168,18 @@ export class IndexPersistence { } catch (err) { this.logFailure("vector", err); } + } else { + // No embedding provider this boot, so there is no vector index to save + // (src/index.ts builds one only when a provider exists). The vector + // manifest and its ledger still sit on disk from a boot that had one, + // and nothing else will ever revisit them: reclaim runs inside the save + // this branch skips, and StateKV cannot enumerate scopes to find the + // shards later. Reclaim on its own so that generation is not stranded. + try { + await this.reclaimVectorGenerations(); + } catch (err) { + this.logFailure("vector", err); + } } } @@ -229,6 +241,21 @@ export class IndexPersistence { ); } + /** + * Reclaim superseded vector generations without saving a vector index. + * + * Reads the manifest only to learn which generation is live; the ledger + * holds everything else. A manifest that is missing or nameless reclaims + * nothing, which is the fail-safe direction. + */ + private async reclaimVectorGenerations(): Promise { + const manifest = await this.kv + .get(KV.bm25Index, VECTOR_MANIFEST_KEY) + .catch(() => null); + if (manifest?.v !== 1 || !manifest.generation) return; + await this.reclaimGenerations(VECTOR_MANIFEST_KEY, manifest.generation); + } + private async saveVectorIndex(serialized: string): Promise { await this.saveShardedIndex( serialized, diff --git a/test/index-persistence.test.ts b/test/index-persistence.test.ts index 4597594a1b..ca3c1e0ae2 100644 --- a/test/index-persistence.test.ts +++ b/test/index-persistence.test.ts @@ -629,6 +629,47 @@ describe("IndexPersistence", () => { await expect(kv.get(oldShardScope, "data")).resolves.toBeNull(); }); + // A boot with no embedding provider constructs IndexPersistence with a null + // vector index (src/index.ts sets it from the provider), and runSave gates + // the vector save on it. The vector manifest and ledger still exist on disk, + // but nothing revisits them, so the generation that was live at that moment + // strands forever: StateKV cannot enumerate scopes to find it later. + // Production carried exactly this — 9 vector shards, 17 MB, orphaned by a + // redeploy that came up without a provider. + it("reclaims the previous vector generation when no vector index is loaded (#1115)", async () => { + await new IndexPersistence(kv as never, makeBm25("obs_old", "alpha"), makeVector("obs_old"), { + shardChars: 80, + createGeneration: () => "gen_old", + }).save(); + const oldVectorScope = "mem:index:bm25:vectors:gen_old:00000"; + await expect(kv.get(oldVectorScope, "data")).resolves.not.toBeNull(); + + // A boot WITH a provider publishes gen_mid over it, but its deletes fail, + // so gen_old survives on disk and stays in the ledger as superseded. That + // is the state a brownout leaves behind, and reclaim is built to finish it + // on a later pass. + const deleteFailsKv = { + ...kv, + delete: vi.fn(async () => { + throw new Error("delete failed"); + }), + }; + await new IndexPersistence(deleteFailsKv as never, makeBm25("obs_mid", "charlie"), makeVector("obs_mid"), { + shardChars: 80, + createGeneration: () => "gen_mid", + }).save(); + await expect(kv.get(oldVectorScope, "data")).resolves.not.toBeNull(); + + // Next boot has no embedding provider. Without the fix nothing revisits + // the vector manifest, so gen_old stays on disk forever. + await new IndexPersistence(kv as never, makeBm25("obs_new", "bravo"), null, { + shardChars: 80, + createGeneration: () => "gen_new", + }).save(); + + await expect(kv.get(oldVectorScope, "data")).resolves.toBeNull(); + }); + it("reclaims the previous vector generation when the vector manifest read fails (#1115)", async () => { await new IndexPersistence( kv as never,