From a0b0a0a90992b5f8bbf26faf60e157b9cfed11b1 Mon Sep 17 00:00:00 2001 From: Omar Gerardo Date: Wed, 26 Aug 2026 20:57:41 +0800 Subject: [PATCH 01/10] perf(state): write only the vector buckets that changed Vector shards were chunked by character offset into one serialised blob, so a single new observation shifted every downstream byte and rewrote every shard. That is O(entire corpus) per change. Measured on a deployed store: the vector index had not persisted since 06:21Z while sessions were still being written at 11:12Z, and BM25 had saved at 11:03Z. runSave writes BM25 (~164 MB) then vectors (~266 MB), so the failure lands on the larger write every time. Because flushIndexSave is awaited on every delete path to make a delete durable, a vector save that never completes means no vector delete is durable either. Partition the vector index into a fixed number of buckets by hash(obsId). Each save serialises a bucket, hashes it, and writes it only when the hash differs from the one the manifest recorded. Bucket keys are deterministic, so there is no generation to strand. Per-bucket hashes live in the manifest rather than process memory. The service restarts often, and an in-memory cache would force a full rewrite after every restart, which is the cost this change removes. Buckets are still chunked to shardChars. Bucket size grows with the corpus and the engine rejects an oversized state::set payload, so chunking bounds the payload while bucketing bounds how much is rewritten; both are needed. Nothing is deleted until the manifest naming its replacement is live. Deleting first destroys data the still-current manifest points at when a publish fails. v1 manifests still load, and the first save migrates and reclaims the shards they name. That first save is not cheap: with no previous v2 manifest every bucket counts as changed, so the change only takes effect after one full vector save lands. BM25 is unchanged; its inverted index is term to docIds, so one document dirties many term buckets and that needs a different design. --- src/state/index-persistence.ts | 267 ++++++++++++++++++++++++++-- src/state/vector-index.ts | 91 ++++++++++ test/index-persistence.test.ts | 316 ++++++++++++++++++++++++++++++++- 3 files changed, 657 insertions(+), 17 deletions(-) diff --git a/src/state/index-persistence.ts b/src/state/index-persistence.ts index 6df0e2fdaf..22d634f543 100644 --- a/src/state/index-persistence.ts +++ b/src/state/index-persistence.ts @@ -1,3 +1,4 @@ +import { createHash } from "node:crypto"; import { SearchIndex } from "./search-index.js"; import { VectorIndex } from "./vector-index.js"; import type { StateKV } from "./kv.js"; @@ -13,10 +14,14 @@ const BM25_MANIFEST_KEY = "data:manifest"; const BM25_SHARD_SCOPE_PREFIX = `${KV.bm25Index}:bm25:`; const VECTOR_KEY = "vectors"; const VECTOR_MANIFEST_KEY = "vectors:manifest"; -const VECTOR_SHARD_SCOPE_PREFIX = `${KV.bm25Index}:vectors:`; const INDEX_SHARD_KEY = "data"; const DEFAULT_INDEX_SHARD_CHARS = 2_000_000; +// Fixed scope for bucketed vector shards. No generation component: bucket keys +// are deterministic and overwritten in place, so nothing can be stranded. +const VECTOR_BUCKET_SCOPE = `${KV.bm25Index}:vectors:v2`; +const DEFAULT_VECTOR_BUCKETS = 256; + type IndexShardManifest = { v: 1; generation?: string; @@ -24,11 +29,86 @@ type IndexShardManifest = { chars: number; }; +/** + * Bucketed vector manifest. + * + * `shards` maps bucket key -> that bucket's content hash, chunk count, and + * length, and holds only non-empty buckets. + * + * Keeping the hashes *in the manifest* rather than in process memory is what + * makes this work in production: the service restarts often, and an in-memory + * cache would force a full rewrite of every bucket after each restart, which + * is the cost this change exists to remove. + * + * A bucket is still chunked to `shardChars`, because bucket size is unbounded + * — it grows with the corpus — and the engine rejects an oversized + * `state::set` payload. Chunking is what bounds the payload; bucketing is what + * bounds how much gets rewritten. They solve different problems and both are + * needed. + */ +type VectorBucketEntry = { hash: string; chunks: number; chars: number }; + +type VectorBucketManifest = { + v: 2; + buckets: number; + shards: Record; +}; + +function isVectorBucketManifest(value: unknown): value is VectorBucketManifest { + if (!value || typeof value !== "object") return false; + const candidate = value as Partial; + return ( + candidate.v === 2 && + Number.isInteger(candidate.buckets) && + (candidate.buckets as number) >= 1 && + !!candidate.shards && + typeof candidate.shards === "object" + ); +} + +function isValidBucketEntry(value: unknown): value is VectorBucketEntry { + if (!value || typeof value !== "object") return false; + const candidate = value as Partial; + return ( + typeof candidate.hash === "string" && + candidate.hash.length > 0 && + Number.isInteger(candidate.chunks) && + (candidate.chunks as number) >= 1 && + Number.isInteger(candidate.chars) && + (candidate.chars as number) >= 0 + ); +} + +function vectorBucketKey(bucket: number): string { + return `b${String(bucket).padStart(4, "0")}`; +} + +function vectorChunkKey(bucketKey: string, chunk: number): string { + return `${bucketKey}:${String(chunk).padStart(5, "0")}`; +} + +// sha1 over the bucket body. This one IS a content check — a collision means a +// changed bucket is silently never persisted — so it must not be the 32-bit +// hash used for bucket assignment. +function contentHash(value: string): string { + return createHash("sha1").update(value).digest("hex"); +} + type IndexPersistenceOptions = { shardChars?: number; createGeneration?: () => string; + vectorBuckets?: number; }; +function vectorBucketCount(options: IndexPersistenceOptions): number { + const configured = options.vectorBuckets; + if (typeof configured !== "number" || !Number.isFinite(configured)) { + return DEFAULT_VECTOR_BUCKETS; + } + const whole = Math.floor(configured); + return whole >= 1 ? whole : DEFAULT_VECTOR_BUCKETS; +} + function shardChars(options: IndexPersistenceOptions): number { const configured = options.shardChars; if (typeof configured !== "number" || !Number.isFinite(configured)) { @@ -95,7 +175,7 @@ export class IndexPersistence { try { await this.saveBm25Index(this.bm25.serialize()); if (this.vector) { - await this.saveVectorIndex(this.vector.serialize()); + await this.saveVectorBuckets(this.vector); } } catch (err) { this.logFailure(err); @@ -114,9 +194,15 @@ export class IndexPersistence { bm25 = SearchIndex.deserialize(bm25Data); } - const vecData = await this.loadVectorData(); - if (vecData && typeof vecData === "string") { - vector = VectorIndex.deserialize(vecData); + // v2 (bucketed) is read directly into an index; the v1 offset-chunked + // path still exists so an upgrade loses nothing and a downgrade is only a + // rebuild, not data loss. + vector = await this.loadVectorBuckets(); + if (!vector) { + const vecData = await this.loadVectorData(); + if (vecData && typeof vecData === "string") { + vector = VectorIndex.deserialize(vecData); + } } return { bm25, vector }; @@ -157,13 +243,109 @@ export class IndexPersistence { ); } - private async saveVectorIndex(serialized: string): Promise { - await this.saveShardedIndex( - serialized, + /** + * Persist the vector index as fixed, independently-addressed buckets, + * writing only the buckets whose contents actually changed. + * + * The offset-chunked format this replaces was O(entire corpus) per change: + * shard boundaries are character offsets into one serialised blob, so a + * single inserted vector shifts every downstream byte and rewrites every + * shard. In production that was 431 MB per save against a corpus of 35 MB of + * observations, and no save had completed in over four hours. + * + * Cross-bucket atomicity is deliberately given up here. It buys nothing + * today, because the atomic save never completes. Recovery is convergent + * instead: a bucket written before a crash disagrees with the manifest that + * was never published, so the next save sees the hash mismatch and rewrites + * it. Vectors are immutable per obsId, so a surviving older bucket is still + * correct rather than stale. + */ + private async saveVectorBuckets(vector: VectorIndex): Promise { + const previous = await this.kv + .get(KV.bm25Index, VECTOR_MANIFEST_KEY) + .catch(() => null); + const buckets = vectorBucketCount(this.options); + const chunkChars = shardChars(this.options); + + // Only trust previous hashes when the bucket count matches. A different + // count remaps every obsId, so every stored bucket is meaningless. + const previousV2 = isVectorBucketManifest(previous) ? previous : null; + const previousShards = + previousV2 && previousV2.buckets === buckets ? previousV2.shards : {}; + + const nextShards: Record = {}; + // Nothing is deleted until the manifest naming its replacement is live. + // Deleting first is how a failed publish destroys data the still-current + // manifest points at — the old manifest survives the failure, so anything + // it names has to survive with it. + const pendingDeletes: string[] = []; + let written = 0; + + for (const [bucket, body] of vector.serializeBuckets(buckets)) { + const bucketKey = vectorBucketKey(bucket); + const hash = contentHash(body); + const chunks = Math.max(1, Math.ceil(body.length / chunkChars)); + const prior = isValidBucketEntry(previousShards[bucketKey]) + ? previousShards[bucketKey] + : undefined; + nextShards[bucketKey] = { hash, chunks, chars: body.length }; + if (prior?.hash === hash) continue; + + for (let i = 0; i < chunks; i++) { + await this.kv.set( + VECTOR_BUCKET_SCOPE, + vectorChunkKey(bucketKey, i), + body.slice(i * chunkChars, (i + 1) * chunkChars), + ); + } + written++; + // This bucket shrank: its old tail chunks are no longer addressed. + for (let i = chunks; i < (prior?.chunks ?? 0); i++) { + pendingDeletes.push(vectorChunkKey(bucketKey, i)); + } + } + + // Buckets that emptied entirely since the last save. + for (const [bucketKey, prior] of Object.entries(previousShards)) { + if (bucketKey in nextShards) continue; + if (!isValidBucketEntry(prior)) continue; + for (let i = 0; i < prior.chunks; i++) { + pendingDeletes.push(vectorChunkKey(bucketKey, i)); + } + } + + const nextManifest: VectorBucketManifest = { v: 2, buckets, shards: nextShards }; + await this.kv.set( + KV.bm25Index, VECTOR_MANIFEST_KEY, - VECTOR_KEY, - VECTOR_SHARD_SCOPE_PREFIX, + nextManifest, + ); + + // Safe from here: the new manifest is live and names none of these. + for (const key of pendingDeletes) { + await this.deleteKey(VECTOR_BUCKET_SCOPE, key, "vector_bucket_reclaim"); + } + + await this.auditIndexPersistence( + "vector_bucket_publish", + [statePath(KV.bm25Index, VECTOR_MANIFEST_KEY)], + { + manifestKey: VECTOR_MANIFEST_KEY, + buckets, + written, + unchanged: Object.keys(nextShards).length - written, + reclaimed: pendingDeletes.length, + }, ); + + // One-time migration off the offset-chunked format. A v1 manifest names + // its own shards, so this needs no enumeration — which matters, because + // StateKV.list returns values, not keys, and cannot be used here. + const legacy = previous as IndexShardManifest | null; + if (legacy?.v === 1 && Array.isArray(legacy.shards)) { + await this.deleteShards(legacy.shards, "vector_v1_migration"); + } + await this.deleteKey(KV.bm25Index, VECTOR_KEY, "legacy_cleanup"); } private async saveShardedIndex( @@ -346,6 +528,71 @@ export class IndexPersistence { return this.loadShardedData(VECTOR_KEY, VECTOR_MANIFEST_KEY, "vector"); } + /** + * Read the bucketed vector index, or null when the store is still on the v1 + * format so the caller can fall back. + * + * A missing or unreadable bucket is skipped rather than failing the whole + * load. Unlike an offset-chunked shard — where a hole corrupts the single + * JSON document and the index must be rejected wholesale — each bucket is an + * independent document, so losing one costs exactly the vectors it held and + * the next save rewrites it. + */ + private async loadVectorBuckets(): Promise { + const manifest = await this.kv + .get(KV.bm25Index, VECTOR_MANIFEST_KEY) + .catch(() => null); + if (!isVectorBucketManifest(manifest)) return null; + + const entries = Object.entries(manifest.shards); + if (entries.length === 0) return new VectorIndex(); + + const index = new VectorIndex(); + let missing = 0; + let corrupt = 0; + for (const [bucketKey, entry] of entries) { + if (!isValidBucketEntry(entry)) { + corrupt++; + continue; + } + const parts: string[] = []; + let complete = true; + for (let i = 0; i < entry.chunks; i++) { + const chunk = await this.kv + .get(VECTOR_BUCKET_SCOPE, vectorChunkKey(bucketKey, i)) + .catch(() => null); + if (typeof chunk !== "string") { + complete = false; + break; + } + parts.push(chunk); + } + if (!complete) { + missing++; + continue; + } + const body = parts.join(""); + // Verifying the hash is what makes a torn write safe. A bucket half + // rewritten before a crash reassembles into invalid JSON; without this + // check it would be silently parsed as empty and its vectors lost with + // no signal. Skipping it instead costs only that bucket, and the next + // save rewrites it because the hash still disagrees. + if (contentHash(body) !== entry.hash) { + corrupt++; + continue; + } + index.mergeSerialized(body); + } + if (missing > 0 || corrupt > 0) { + logger.warn("index persistence: vector buckets skipped", { + missing, + corrupt, + total: entries.length, + }); + } + return index; + } + private async loadShardedData( legacyKey: string, manifestKey: string, diff --git a/src/state/vector-index.ts b/src/state/vector-index.ts index d4b8bda760..35e53e2e29 100644 --- a/src/state/vector-index.ts +++ b/src/state/vector-index.ts @@ -20,6 +20,30 @@ function base64ToFloat32(b64: string): Float32Array { ); } +// FNV-1a. Only needs to spread obsIds evenly across buckets and be stable +// across processes — it is never a content check, so a non-cryptographic +// 32-bit hash is the right size. Math.imul keeps the multiply in int32. +function fnv1a32(str: string): number { + let hash = 0x811c9dc5; + for (let i = 0; i < str.length; i++) { + hash ^= str.charCodeAt(i); + hash = Math.imul(hash, 0x01000193); + } + return hash >>> 0; +} + +/** + * Which bucket an observation's vector belongs to. + * + * Deterministic and stable, which is the whole point: bucket keys are reused + * in place on every save, so there is no generation to strand and orphaned + * shards are structurally impossible (unlike the offset-chunked format, where + * one insert shifts every downstream chunk boundary and rewrites everything). + */ +export function vectorBucketOf(obsId: string, bucketCount: number): number { + return fnv1a32(obsId) % bucketCount; +} + function cosineSimilarity(a: Float32Array, b: Float32Array): number { if (a.length !== b.length) return 0; let dot = 0; @@ -135,6 +159,73 @@ export class VectorIndex { return JSON.stringify(data); } + /** + * Serialise one bucket at a time, in the same row shape `serialize()` emits + * so both formats deserialise through one code path. + * + * A generator rather than a `Map` on purpose: materialising + * every bucket would hold the whole index as strings at once (~266 MB in + * production) inside an already memory-constrained process. Yielding lets the + * caller write and drop each bucket, so the peak is a single bucket. + * + * Grouping first costs one pass and holds ids only, not embeddings. + * Empty buckets are not yielded — the caller reconciles those against the + * previous manifest and deletes them. + */ + *serializeBuckets(bucketCount: number): Generator<[number, string]> { + const groups = new Map(); + for (const obsId of this.vectors.keys()) { + const bucket = vectorBucketOf(obsId, bucketCount); + const ids = groups.get(bucket); + if (ids) ids.push(obsId); + else groups.set(bucket, [obsId]); + } + for (const [bucket, ids] of groups) { + const rows: Array<[string, { embedding: string; sessionId: string }]> = []; + for (const obsId of ids) { + const entry = this.vectors.get(obsId); + if (!entry) continue; + rows.push([ + obsId, + { + embedding: float32ToBase64(entry.embedding), + sessionId: entry.sessionId, + }, + ]); + } + yield [bucket, JSON.stringify(rows)]; + } + } + + /** Merge serialised rows into this index. Malformed rows are skipped. */ + mergeSerialized(json: string): void { + let data: unknown; + try { + data = JSON.parse(json); + } catch { + return; + } + if (!Array.isArray(data)) return; + for (const row of data) { + try { + if (!Array.isArray(row) || row.length < 2) continue; + const [obsId, entry] = row; + if ( + typeof obsId !== "string" || + typeof entry?.embedding !== "string" || + typeof entry?.sessionId !== "string" + ) + continue; + this.vectors.set(obsId, { + embedding: base64ToFloat32(entry.embedding), + sessionId: entry.sessionId, + }); + } catch { + continue; + } + } + } + static deserialize(json: string): VectorIndex { const idx = new VectorIndex(); let data: unknown; diff --git a/test/index-persistence.test.ts b/test/index-persistence.test.ts index 929791657f..a95b0c71f3 100644 --- a/test/index-persistence.test.ts +++ b/test/index-persistence.test.ts @@ -9,6 +9,13 @@ const BM25_LEGACY_KEY = "data"; const BM25_MANIFEST_KEY = "data:manifest"; const VECTOR_LEGACY_KEY = "vectors"; const VECTOR_MANIFEST_KEY = "vectors:manifest"; +const VECTOR_BUCKET_SCOPE = "mem:index:bm25:vectors:v2"; + +type TestVectorBucketManifest = { + v: 2; + buckets: number; + shards: Record; +}; type TestIndexShardManifest = { v: 1; @@ -256,17 +263,21 @@ describe("IndexPersistence", () => { }); await persistence.save(); - const manifest = await kv.get( + const manifest = await kv.get( BM25_SCOPE, VECTOR_MANIFEST_KEY, ); expect(manifest).not.toBeNull(); - expect(manifest!.generation).toBe("gen_vector"); - expect(manifest!.shards.length).toBeGreaterThan(1); - expect(manifest!.shards[0].scope).toContain(":gen_vector:"); + expect(manifest!.v).toBe(2); + const bucketKeys = Object.keys(manifest!.shards); + expect(bucketKeys.length).toBe(1); + // shardChars: 40 against a 32-float embedding, so the bucket must still be + // chunked. Bucketing bounds the rewrite; chunking bounds the payload. + expect(manifest!.shards[bucketKeys[0]].chunks).toBeGreaterThan(1); await expect(kv.get(BM25_SCOPE, VECTOR_LEGACY_KEY)).resolves.toBeNull(); + // Vector payloads live outside the BM25 scope. await expect( - kv.get(manifest!.shards[0].scope, manifest!.shards[0].key), + kv.get(VECTOR_BUCKET_SCOPE, `${bucketKeys[0]}:00000`), ).resolves.toEqual(expect.any(String)); const loaded = await persistence.load(); @@ -289,12 +300,18 @@ describe("IndexPersistence", () => { createGeneration: () => "gen_empty", }).save(); - const vectorManifest = await kv.get( + const vectorManifest = await kv.get( BM25_SCOPE, VECTOR_MANIFEST_KEY, ); expect(vectorManifest).not.toBeNull(); - expect(vectorManifest!.generation).toBe("gen_empty"); + expect(vectorManifest!.v).toBe(2); + // The cleared index publishes a manifest naming no buckets, and the + // previous bucket is reclaimed rather than left to reload. + expect(Object.keys(vectorManifest!.shards)).toHaveLength(0); + await expect( + kv.list(VECTOR_BUCKET_SCOPE), + ).resolves.toHaveLength(0); const loaded = await new IndexPersistence( kv as never, new SearchIndex(), @@ -791,3 +808,288 @@ describe("IndexPersistence", () => { await expect(persistence.load()).resolves.toBeDefined(); }); }); + +// Counts writes landing on vector bucket payload keys, so assertions are about +// payload rewrites rather than manifest bookkeeping. +function countingKV() { + const inner = mockKV(); + let bucketWrites = 0; + const writtenKeys: string[] = []; + return { + ...inner, + resetCount: () => { + bucketWrites = 0; + writtenKeys.length = 0; + }, + get bucketWrites() { + return bucketWrites; + }, + get writtenKeys() { + return writtenKeys; + }, + set: async (scope: string, key: string, data: T): Promise => { + if (scope === VECTOR_BUCKET_SCOPE) { + bucketWrites++; + writtenKeys.push(key); + } + return inner.set(scope, key, data); + }, + }; +} + +describe("IndexPersistence vector bucketing", () => { + let kv: ReturnType; + + beforeEach(() => { + vi.useFakeTimers(); + kv = countingKV(); + }); + afterEach(() => vi.useRealTimers()); + + function seeded(count: number): VectorIndex { + const vector = new VectorIndex(); + for (let i = 0; i < count; i++) { + vector.add(`obs_${i}`, "ses_1", new Float32Array([i, i + 1, i + 2])); + } + return vector; + } + + it("writes O(1) buckets when one vector is added, not O(corpus)", async () => { + const vector = seeded(60); + const persistence = new IndexPersistence(kv as never, new SearchIndex(), vector, { + shardChars: 400, + }); + + await persistence.save(); + expect(kv.bucketWrites).toBeGreaterThan(5); + + vector.add("obs_new", "ses_1", new Float32Array([9, 9, 9])); + kv.resetCount(); + await persistence.save(); + + expect(kv.bucketWrites).toBeLessThanOrEqual(2); + }); + + it("rewrites nothing when the index has not changed", async () => { + const vector = seeded(40); + const persistence = new IndexPersistence(kv as never, new SearchIndex(), vector, { + shardChars: 400, + }); + await persistence.save(); + + kv.resetCount(); + await persistence.save(); + + expect(kv.bucketWrites).toBe(0); + }); + + it("rewrites nothing after a restart when nothing changed", async () => { + const vector = seeded(40); + await new IndexPersistence(kv as never, new SearchIndex(), vector, { + shardChars: 400, + }).save(); + + // Fresh instance: hashes come from the manifest, not process memory. This + // is what stops a restart-heavy service rewriting the whole index. + kv.resetCount(); + const restarted = new IndexPersistence(kv as never, new SearchIndex(), seeded(40), { + shardChars: 400, + }); + await restarted.save(); + + expect(kv.bucketWrites).toBe(0); + }); + + it("writes only the removed observation's bucket on delete", async () => { + const vector = seeded(60); + const persistence = new IndexPersistence(kv as never, new SearchIndex(), vector, { + shardChars: 400, + }); + await persistence.save(); + + vector.remove("obs_7"); + kv.resetCount(); + await persistence.save(); + + expect(kv.bucketWrites).toBeLessThanOrEqual(1); + const loaded = await persistence.load(); + expect(loaded.vector!.size).toBe(59); + }); + + it("round-trips every vector through the bucketed format", async () => { + const vector = seeded(50); + const persistence = new IndexPersistence(kv as never, new SearchIndex(), vector, { + shardChars: 400, + }); + await persistence.save(); + + const loaded = await persistence.load(); + expect(loaded.vector!.size).toBe(50); + expect( + loaded.vector!.search(new Float32Array([7, 8, 9]))[0]?.obsId, + ).toBe("obs_7"); + }); + + it("bounds each payload by shardChars even when a bucket is large", async () => { + // One bucket, many vectors in it: only chunking can bound the payload. + const vector = new VectorIndex(); + for (let i = 0; i < 40; i++) { + vector.add( + `obs_${i}`, + "ses_1", + new Float32Array(Array.from({ length: 32 }, (_, n) => n + i)), + ); + } + const maxChars = 120; + const guarded = { + ...kv, + set: async (scope: string, key: string, data: T): Promise => { + if (typeof data === "string" && data.length > maxChars) { + throw new Error(`oversized state::set payload: ${scope}/${key}`); + } + return kv.set(scope, key, data); + }, + }; + await new IndexPersistence(guarded as never, new SearchIndex(), vector, { + shardChars: maxChars, + vectorBuckets: 1, + }).save(); + + const loaded = await new IndexPersistence( + kv as never, + new SearchIndex(), + null, + ).load(); + expect(loaded.vector!.size).toBe(40); + }); + + it("loads a v1 manifest and migrates it to buckets on the next save", async () => { + // Write the legacy offset-chunked format by hand, exactly as the v1 code + // shaped it, then prove no vectors are lost across the upgrade. + const legacy = seeded(20); + const serialized = legacy.serialize(); + const shards: Array<{ scope: string; key: string; chars: number }> = []; + const chunkChars = 200; + for (let offset = 0; offset < serialized.length; offset += chunkChars) { + const scope = `mem:index:bm25:vectors:gen_v1:${String( + shards.length, + ).padStart(5, "0")}`; + const chunk = serialized.slice(offset, offset + chunkChars); + shards.push({ scope, key: "data", chars: chunk.length }); + await kv.set(scope, "data", chunk); + } + await kv.set(BM25_SCOPE, VECTOR_MANIFEST_KEY, { + v: 1, + generation: "gen_v1", + shards, + chars: serialized.length, + }); + + // Load path still understands v1. + const before = await new IndexPersistence( + kv as never, + new SearchIndex(), + null, + ).load(); + expect(before.vector!.size).toBe(20); + + // First save migrates and reclaims the v1 shards. + await new IndexPersistence(kv as never, new SearchIndex(), seeded(20), { + shardChars: 400, + }).save(); + + const manifest = await kv.get( + BM25_SCOPE, + VECTOR_MANIFEST_KEY, + ); + expect(manifest!.v).toBe(2); + for (const shard of shards) { + await expect(kv.get(shard.scope, shard.key)).resolves.toBeNull(); + } + const after = await new IndexPersistence( + kv as never, + new SearchIndex(), + null, + ).load(); + expect(after.vector!.size).toBe(20); + }); + + it("keeps previous buckets loadable when the manifest publish fails", async () => { + const vector = seeded(30); + await new IndexPersistence(kv as never, new SearchIndex(), vector, { + shardChars: 400, + }).save(); + + const failing = { + ...kv, + set: async (scope: string, key: string, data: T): Promise => { + if (scope === BM25_SCOPE && key === VECTOR_MANIFEST_KEY) { + throw new Error("vector manifest write failed"); + } + return kv.set(scope, key, data); + }, + }; + // Empty index: without deferring deletes until after publish, this would + // reclaim every bucket the still-live manifest names and lose the lot. + await new IndexPersistence( + failing as never, + new SearchIndex(), + new VectorIndex(), + { shardChars: 400 }, + ).save(); + + const loaded = await new IndexPersistence( + kv as never, + new SearchIndex(), + null, + ).load(); + expect(loaded.vector!.size).toBe(30); + }); + + it("skips a bucket whose body disagrees with its manifest hash", async () => { + const vector = seeded(30); + // Large shardChars keeps every bucket to a single chunk, so the stand-in + // body below is the bucket in full rather than a fragment of one. + const persistence = new IndexPersistence(kv as never, new SearchIndex(), vector, { + shardChars: 100_000, + }); + await persistence.save(); + + const manifest = await kv.get( + BM25_SCOPE, + VECTOR_MANIFEST_KEY, + ); + const [tornKey] = Object.keys(manifest!.shards); + + // Critically, this stand-in is VALID JSON in the right row shape. An + // invalid body would be dropped by mergeSerialized regardless, so the test + // would pass with the hash check removed and prove nothing. Only content + // that would otherwise load cleanly can show the hash check doing work. + const ghost = JSON.stringify([ + [ + "obs_ghost", + { + embedding: Buffer.from( + new Float32Array([1, 2, 3]).buffer, + ).toString("base64"), + sessionId: "ses_1", + }, + ], + ]); + await kv.set(VECTOR_BUCKET_SCOPE, `${tornKey}:00000`, ghost); + + const loaded = await new IndexPersistence( + kv as never, + new SearchIndex(), + null, + ).load(); + // The disagreeing bucket is dropped whole rather than trusted. + expect( + loaded.vector!.search(new Float32Array([1, 2, 3]), 50) + .some((r) => r.obsId === "obs_ghost"), + ).toBe(false); + // Every other bucket still loads. + expect(loaded.vector!.size).toBeGreaterThan(0); + expect(loaded.vector!.size).toBeLessThan(30); + }); +}); From faf223abe78477c3a353c0ccda52e37c4820e90b Mon Sep 17 00:00:00 2001 From: Omar Gerardo Date: Wed, 26 Aug 2026 20:57:56 +0800 Subject: [PATCH 02/10] fix(state): compare the addressing scheme, not just bucket contents A content hash cannot see an addressing change: the bytes are identical, they just belong somewhere else now. The manifest recorded only content, so two changes silently corrupted the index. Changing the bucket count remapped every obsId while the reclaim walk consulted a map that had been reset to empty, stranding every old bucket permanently with nothing able to name them again. Changing shardChars left the chunk count recomputed but the write skipped, because the hash still matched. The manifest then claimed a chunk count the stored bucket did not have, load read the wrong number of keys, and the reassembled body failed its own hash check. Silent vector loss that looked like corruption rather than a config change. Record layout, buckets, and chunkChars in the manifest and compare all three before skipping any write. A mismatch forces a full rewrite plus reclaim of every key the old manifest named. layout is versioned separately from v so an older manifest stays parseable for reclaim; bumping v would make it unreadable and strand the very keys that need deleting. Also fixes tail-chunk reclaim, which referenced the bucket map where it meant the bucket entry and so never ran. A v2 manifest reaching the v1 loader no longer reports itself as invalid. That only happens when a transient manifest read sends the vector load down the fallback path, and it is the wrong signal to hand an operator mid-incident. Review cuts: deserialize now delegates to mergeSerialized instead of repeating its parse loop verbatim, the unread chars manifest field is gone, and one test subsumed by the restart variant is removed. --- src/state/index-persistence.ts | 111 ++++++++++++++++++--------- src/state/vector-index.ts | 39 ++-------- test/index-persistence.test.ts | 134 +++++++++++++++++++++++++++------ 3 files changed, 191 insertions(+), 93 deletions(-) diff --git a/src/state/index-persistence.ts b/src/state/index-persistence.ts index 22d634f543..2f58360e43 100644 --- a/src/state/index-persistence.ts +++ b/src/state/index-persistence.ts @@ -32,28 +32,47 @@ type IndexShardManifest = { /** * Bucketed vector manifest. * - * `shards` maps bucket key -> that bucket's content hash, chunk count, and - * length, and holds only non-empty buckets. + * `shards` maps bucket key -> content hash and chunk count, non-empty buckets + * only. * - * Keeping the hashes *in the manifest* rather than in process memory is what - * makes this work in production: the service restarts often, and an in-memory - * cache would force a full rewrite of every bucket after each restart, which - * is the cost this change exists to remove. + * Hashes live in the manifest, not process memory: restarts must not force a + * full rewrite. Buckets bound how much is rewritten; chunks bound the payload + * size, since a bucket grows with the corpus and the engine rejects an + * oversized `state::set`. + */ +// No `chars`: the sha1 already proves both content and length. v1 needed a +// length check only because it had no hash to check against. +type VectorBucketEntry = { hash: string; chunks: number }; + +/** + * Version of the *addressing scheme* — which bucket an obsId lands in and which + * keys that bucket occupies. Bump it whenever that mapping changes, including a + * change to the bucket hash function or the key format. + * + * This exists because a content hash cannot see an addressing change: the bytes + * are identical, they just belong somewhere else now. Without this, such a + * change silently skips every write while the manifest claims new locations, + * and the vectors are lost on the next load. * - * A bucket is still chunked to `shardChars`, because bucket size is unbounded - * — it grows with the corpus — and the engine rejects an oversized - * `state::set` payload. Chunking is what bounds the payload; bucketing is what - * bounds how much gets rewritten. They solve different problems and both are - * needed. + * It is deliberately separate from `v`. `v` describes the manifest's shape, and + * keeping it stable is what lets a newer build still parse an older manifest + * well enough to reclaim the keys it named. Bumping `v` instead would make the + * old manifest unreadable and strand every key in it. */ -type VectorBucketEntry = { hash: string; chunks: number; chars: number }; +const VECTOR_LAYOUT = 1; type VectorBucketManifest = { v: 2; + layout: number; buckets: number; + chunkChars: number; shards: Record; }; +// `layout` and `chunkChars` are not required here on purpose. A manifest +// missing them simply fails the layout comparison, which triggers a full +// rewrite and a full reclaim — the correct, self-healing response — whereas +// rejecting the manifest outright would strand the keys it names. function isVectorBucketManifest(value: unknown): value is VectorBucketManifest { if (!value || typeof value !== "object") return false; const candidate = value as Partial; @@ -69,13 +88,13 @@ function isVectorBucketManifest(value: unknown): value is VectorBucketManifest { function isValidBucketEntry(value: unknown): value is VectorBucketEntry { if (!value || typeof value !== "object") return false; const candidate = value as Partial; + // The chunks bound matters beyond shape: it is the loop bound for both the + // reclaim walks below, so a poisoned value would spin a garbage-length loop. return ( typeof candidate.hash === "string" && candidate.hash.length > 0 && Number.isInteger(candidate.chunks) && - (candidate.chunks as number) >= 1 && - Number.isInteger(candidate.chars) && - (candidate.chars as number) >= 0 + (candidate.chunks as number) >= 1 ); } @@ -247,11 +266,8 @@ export class IndexPersistence { * Persist the vector index as fixed, independently-addressed buckets, * writing only the buckets whose contents actually changed. * - * The offset-chunked format this replaces was O(entire corpus) per change: - * shard boundaries are character offsets into one serialised blob, so a - * single inserted vector shifts every downstream byte and rewrites every - * shard. In production that was 431 MB per save against a corpus of 35 MB of - * observations, and no save had completed in over four hours. + * Replaces an offset-chunked format that was O(entire corpus) per change; + * see the commit message for the production measurements. * * Cross-bucket atomicity is deliberately given up here. It buys nothing * today, because the atomic save never completes. Recovery is convergent @@ -267,11 +283,22 @@ export class IndexPersistence { const buckets = vectorBucketCount(this.options); const chunkChars = shardChars(this.options); - // Only trust previous hashes when the bucket count matches. A different - // count remaps every obsId, so every stored bucket is meaningless. const previousV2 = isVectorBucketManifest(previous) ? previous : null; - const previousShards = - previousV2 && previousV2.buckets === buckets ? previousV2.shards : {}; + + // Every input that decides *where* a vector's bytes live. If any of them + // moved, a matching content hash proves nothing — the bytes are the same + // but their home is not — so nothing on disk may be skipped. + const layoutMatches = + !!previousV2 && + previousV2.layout === VECTOR_LAYOUT && + previousV2.buckets === buckets && + previousV2.chunkChars === chunkChars; + + // One map, used for two things: comparison (only when the layout matches) + // and reclaim (always). Reclaim must see the previous manifest even on a + // layout change, because those keys are exactly the ones about to become + // unreachable, and nothing can enumerate them afterwards. + const prior = previousV2?.shards ?? {}; const nextShards: Record = {}; // Nothing is deleted until the manifest naming its replacement is live. @@ -285,11 +312,11 @@ export class IndexPersistence { const bucketKey = vectorBucketKey(bucket); const hash = contentHash(body); const chunks = Math.max(1, Math.ceil(body.length / chunkChars)); - const prior = isValidBucketEntry(previousShards[bucketKey]) - ? previousShards[bucketKey] + const priorEntry = isValidBucketEntry(prior[bucketKey]) + ? prior[bucketKey] : undefined; - nextShards[bucketKey] = { hash, chunks, chars: body.length }; - if (prior?.hash === hash) continue; + nextShards[bucketKey] = { hash, chunks }; + if (layoutMatches && priorEntry?.hash === hash) continue; for (let i = 0; i < chunks; i++) { await this.kv.set( @@ -300,21 +327,29 @@ export class IndexPersistence { } written++; // This bucket shrank: its old tail chunks are no longer addressed. - for (let i = chunks; i < (prior?.chunks ?? 0); i++) { + for (let i = chunks; i < (priorEntry?.chunks ?? 0); i++) { pendingDeletes.push(vectorChunkKey(bucketKey, i)); } } - // Buckets that emptied entirely since the last save. - for (const [bucketKey, prior] of Object.entries(previousShards)) { + // Buckets the new manifest does not name: emptied since the last save, or + // left behind by a layout change that remapped them elsewhere. Either way + // this is the last moment anything can still name them. + for (const [bucketKey, priorEntry] of Object.entries(prior)) { if (bucketKey in nextShards) continue; - if (!isValidBucketEntry(prior)) continue; - for (let i = 0; i < prior.chunks; i++) { + if (!isValidBucketEntry(priorEntry)) continue; + for (let i = 0; i < priorEntry.chunks; i++) { pendingDeletes.push(vectorChunkKey(bucketKey, i)); } } - const nextManifest: VectorBucketManifest = { v: 2, buckets, shards: nextShards }; + const nextManifest: VectorBucketManifest = { + v: 2, + layout: VECTOR_LAYOUT, + buckets, + chunkChars, + shards: nextShards, + }; await this.kv.set( KV.bm25Index, VECTOR_MANIFEST_KEY, @@ -545,8 +580,6 @@ export class IndexPersistence { if (!isVectorBucketManifest(manifest)) return null; const entries = Object.entries(manifest.shards); - if (entries.length === 0) return new VectorIndex(); - const index = new VectorIndex(); let missing = 0; let corrupt = 0; @@ -611,6 +644,12 @@ export class IndexPersistence { // `manifest.v` with TypeError. Treat both null and undefined as // "no manifest" and fall through to the legacy path. The shape // check stays so a malformed-but-present row still fails closed. + // A bucketed manifest is handled by loadVectorBuckets, not here. Without + // this, a transient manifest-read failure that sends the vector load down + // the v1 path would report a perfectly valid manifest as invalid — the + // wrong signal to hand an operator during exactly the incident this format + // exists to fix. + if (isVectorBucketManifest(manifest.value)) return null; if ( manifest.value != null && typeof manifest.value === "object" diff --git a/src/state/vector-index.ts b/src/state/vector-index.ts index 35e53e2e29..d130e60d1b 100644 --- a/src/state/vector-index.ts +++ b/src/state/vector-index.ts @@ -32,15 +32,10 @@ function fnv1a32(str: string): number { return hash >>> 0; } -/** - * Which bucket an observation's vector belongs to. - * - * Deterministic and stable, which is the whole point: bucket keys are reused - * in place on every save, so there is no generation to strand and orphaned - * shards are structurally impossible (unlike the offset-chunked format, where - * one insert shifts every downstream chunk boundary and rewrites everything). - */ -export function vectorBucketOf(obsId: string, bucketCount: number): number { +// Deterministic and stable across processes: bucket keys are reused in place, +// so nothing can be stranded. Changing this is an addressing change — bump +// VECTOR_LAYOUT in index-persistence.ts if you do. +function vectorBucketOf(obsId: string, bucketCount: number): number { return fnv1a32(obsId) % bucketCount; } @@ -228,31 +223,7 @@ export class VectorIndex { static deserialize(json: string): VectorIndex { const idx = new VectorIndex(); - let data: unknown; - try { - data = JSON.parse(json); - } catch { - return idx; - } - if (!Array.isArray(data)) return idx; - for (const row of data) { - try { - if (!Array.isArray(row) || row.length < 2) continue; - const [obsId, entry] = row; - if ( - typeof obsId !== "string" || - typeof entry?.embedding !== "string" || - typeof entry?.sessionId !== "string" - ) - continue; - idx.vectors.set(obsId, { - embedding: base64ToFloat32(entry.embedding), - sessionId: entry.sessionId, - }); - } catch { - continue; - } - } + idx.mergeSerialized(json); return idx; } } diff --git a/test/index-persistence.test.ts b/test/index-persistence.test.ts index a95b0c71f3..d865e83d79 100644 --- a/test/index-persistence.test.ts +++ b/test/index-persistence.test.ts @@ -13,8 +13,10 @@ const VECTOR_BUCKET_SCOPE = "mem:index:bm25:vectors:v2"; type TestVectorBucketManifest = { v: 2; + layout: number; buckets: number; - shards: Record; + chunkChars: number; + shards: Record; }; type TestIndexShardManifest = { @@ -814,24 +816,16 @@ describe("IndexPersistence", () => { function countingKV() { const inner = mockKV(); let bucketWrites = 0; - const writtenKeys: string[] = []; return { ...inner, resetCount: () => { bucketWrites = 0; - writtenKeys.length = 0; }, get bucketWrites() { return bucketWrites; }, - get writtenKeys() { - return writtenKeys; - }, set: async (scope: string, key: string, data: T): Promise => { - if (scope === VECTOR_BUCKET_SCOPE) { - bucketWrites++; - writtenKeys.push(key); - } + if (scope === VECTOR_BUCKET_SCOPE) bucketWrites++; return inner.set(scope, key, data); }, }; @@ -870,19 +864,8 @@ describe("IndexPersistence vector bucketing", () => { expect(kv.bucketWrites).toBeLessThanOrEqual(2); }); - it("rewrites nothing when the index has not changed", async () => { - const vector = seeded(40); - const persistence = new IndexPersistence(kv as never, new SearchIndex(), vector, { - shardChars: 400, - }); - await persistence.save(); - - kv.resetCount(); - await persistence.save(); - - expect(kv.bucketWrites).toBe(0); - }); - + // Covers the same-instance case too: saveVectorBuckets holds no state between + // saves, so a fresh instance and a reused one run the identical path. it("rewrites nothing after a restart when nothing changed", async () => { const vector = seeded(40); await new IndexPersistence(kv as never, new SearchIndex(), vector, { @@ -1093,3 +1076,108 @@ describe("IndexPersistence vector bucketing", () => { expect(loaded.vector!.size).toBeLessThan(30); }); }); + +// A content hash cannot see an addressing change: same bytes, different home. +// These two cover that whole family — the bucket count and the chunk size are +// the two inputs that decide where a vector's bytes live. +describe("IndexPersistence vector layout changes", () => { + let kv: ReturnType; + + beforeEach(() => { + vi.useFakeTimers(); + kv = countingKV(); + }); + afterEach(() => vi.useRealTimers()); + + function seeded(count: number): VectorIndex { + const vector = new VectorIndex(); + for (let i = 0; i < count; i++) { + vector.add(`obs_${i}`, "ses_1", new Float32Array([i, i + 1, i + 2])); + } + return vector; + } + + it("reclaims buckets stranded by a smaller bucket count", async () => { + await new IndexPersistence(kv as never, new SearchIndex(), seeded(40), { + shardChars: 400, + vectorBuckets: 8, + }).save(); + + const wide = await kv.get( + BM25_SCOPE, + VECTOR_MANIFEST_KEY, + ); + const widened = Object.keys(wide!.shards); + // Needs keys that only the 8-bucket layout can produce. + const beyondNarrow = widened.filter((key) => Number(key.slice(1)) >= 4); + expect(beyondNarrow.length).toBeGreaterThan(0); + + await new IndexPersistence(kv as never, new SearchIndex(), seeded(40), { + shardChars: 400, + vectorBuckets: 4, + }).save(); + + // Every key the old layout owned and the new one cannot name must be gone. + for (const bucketKey of beyondNarrow) { + await expect( + kv.get(VECTOR_BUCKET_SCOPE, `${bucketKey}:00000`), + ).resolves.toBeNull(); + } + const narrow = await kv.get( + BM25_SCOPE, + VECTOR_MANIFEST_KEY, + ); + expect(narrow!.buckets).toBe(4); + const loaded = await new IndexPersistence( + kv as never, + new SearchIndex(), + null, + ).load(); + expect(loaded.vector!.size).toBe(40); + }); + + it("survives a chunk-size change that leaves content identical", async () => { + // Small chunks first, so buckets span several keys. + await new IndexPersistence(kv as never, new SearchIndex(), seeded(30), { + shardChars: 60, + vectorBuckets: 4, + }).save(); + + const wide = await kv.get( + BM25_SCOPE, + VECTOR_MANIFEST_KEY, + ); + const multiChunk = Object.entries(wide!.shards).find( + ([, entry]) => entry.chunks > 1, + ); + expect(multiChunk).toBeDefined(); + const [shrinkingKey, wideEntry] = multiChunk!; + + // Same vectors, larger chunks. The bucket bodies are byte-identical, so the + // content hash matches and a hash-only skip would write nothing at all — + // while the manifest records the new, smaller chunk count. Load would then + // read too few keys and drop every bucket. + await new IndexPersistence(kv as never, new SearchIndex(), seeded(30), { + shardChars: 100_000, + vectorBuckets: 4, + }).save(); + + // The bucket now needs fewer keys, so its old tail must be reclaimed + // rather than left addressable-by-nobody. + for (let i = 1; i < wideEntry.chunks; i++) { + await expect( + kv.get(VECTOR_BUCKET_SCOPE, `${shrinkingKey}:${String(i).padStart(5, "0")}`), + ).resolves.toBeNull(); + } + + const loaded = await new IndexPersistence( + kv as never, + new SearchIndex(), + null, + ).load(); + expect(loaded.vector!.size).toBe(30); + expect( + loaded.vector!.search(new Float32Array([7, 8, 9]))[0]?.obsId, + ).toBe("obs_7"); + }); +}); From 03d2dbe89b2f4a0a9f8b745364a7739e029f89ad Mon Sep 17 00:00:00 2001 From: Omar Gerardo Date: Wed, 26 Aug 2026 20:58:10 +0800 Subject: [PATCH 03/10] fix(state): address vector buckets by content so a torn save loses nothing Corrects a claim made in the previous commit. It said recovery from a torn write was convergent. It was not, and the failure was silent data loss. Buckets were overwritten in place. A save that died between the chunk writes and the manifest publish left the bucket no longer matching the hash the still current manifest recorded, so load dropped it. Nothing restored it: rebuild is gated on the BM25 index being empty (src/index.ts, src/functions/search.ts) and never on the vector index, so with BM25 populated no re-embed ever runs. The next save then serialised the index without those vectors and overwrote the bucket, making the loss permanent. The generation swap this format replaced lost nothing in the same situation, so that was a durability regression. Chunk keys now carry the bucket's content hash. The old bucket stays readable until the new manifest names the new one, so a save that dies partway loses nothing and reclaim stays exact because the old manifest still names the old keys. Never delete a key the new manifest names. Old and new keys can legitimately collide: identical content re-chunked keeps the same bucket and hash, so chunk zero's key is unchanged even when the old entry listed more chunks. Reclaiming that entry blind deleted a bucket the manifest was actively pointing at. Carry unfinished reclaim work in the manifest. Deletes run after the publish, so dying partway stranded the remainder with nothing able to name them. The list is published with the manifest and drained by the next save, and a delete that fails stays on the list rather than being reported as done. Stop swallowing the previous-manifest read error. Treating a transient engine timeout as "no previous manifest" is indistinguishable from a genuinely absent one, and that mistake strands the entire previous generation without needing a crash. Let it throw; the debounce retries. Save vectors before BM25. The vector save is the larger of the two and the one that has not completed in production; running it second lets a BM25 save consume the engine's budget first. Each index now gets its own try, because one try around both would let a vector failure silently block BM25 once the order is reversed. Skip the publish entirely when nothing changed and nothing needs reclaiming, rather than rewriting an identical manifest every debounce. The restart test now goes through load() instead of seeding a fresh identical index, which left the path that actually keeps hashes stable across a restart unpinned. --- src/state/index-persistence.ts | 174 ++++++++++++++++++++++++++++----- test/index-persistence.test.ts | 160 ++++++++++++++++++++++++++++-- 2 files changed, 298 insertions(+), 36 deletions(-) diff --git a/src/state/index-persistence.ts b/src/state/index-persistence.ts index 2f58360e43..50d6321231 100644 --- a/src/state/index-persistence.ts +++ b/src/state/index-persistence.ts @@ -67,6 +67,11 @@ type VectorBucketManifest = { buckets: number; chunkChars: number; shards: Record; + // Keys superseded by this manifest but not yet deleted. Published *with* the + // manifest so the work survives a crash: deletes run after the publish, and + // dying partway would otherwise strand the remainder with nothing able to + // name them. The next save drains whatever is left. + reclaim?: Array<{ scope: string; key: string }>; }; // `layout` and `chunkChars` are not required here on purpose. A manifest @@ -102,8 +107,47 @@ function vectorBucketKey(bucket: number): string { return `b${String(bucket).padStart(4, "0")}`; } -function vectorChunkKey(bucketKey: string, chunk: number): string { - return `${bucketKey}:${String(chunk).padStart(5, "0")}`; +/** + * Chunk keys carry the bucket's content hash, so a bucket's bytes are never + * overwritten in place. + * + * This is what makes a torn write survivable. Writing in place looks safe + * because the manifest is published last, but it is not: the half-written + * bucket no longer matches the hash the *old* manifest recorded, load drops it, + * and nothing restores it — rebuild is gated on the BM25 index being empty + * (src/index.ts, src/functions/search.ts), never on the vector index. The next + * save then serialises the index without those vectors and persists the loss. + * + * Addressing by content instead means the old bucket stays readable until the + * new manifest names the new one, so a save that dies partway loses nothing. + */ +function vectorChunkKey( + bucketKey: string, + hash: string, + chunk: number, +): string { + return `${bucketKey}:${hash.slice(0, 12)}:${String(chunk).padStart(5, "0")}`; +} + +function isReclaimTarget( + value: unknown, +): value is { scope: string; key: string } { + if (!value || typeof value !== "object") return false; + const candidate = value as { scope?: unknown; key?: unknown }; + return ( + typeof candidate.scope === "string" && + candidate.scope.length > 0 && + typeof candidate.key === "string" && + candidate.key.length > 0 + ); +} + +function bucketChunkKeys(bucketKey: string, entry: VectorBucketEntry): string[] { + const keys: string[] = []; + for (let i = 0; i < entry.chunks; i++) { + keys.push(vectorChunkKey(bucketKey, entry.hash, i)); + } + return keys; } // sha1 over the bucket body. This one IS a content check — a collision means a @@ -191,11 +235,25 @@ export class IndexPersistence { clearTimeout(this.timer); this.timer = null; } - try { - await this.saveBm25Index(this.bm25.serialize()); - if (this.vector) { + // Vectors first. This is the save that has been failing in production — it + // is the larger of the two and it runs second, so a BM25 save that consumes + // the engine's budget starves it. Going first also makes a vector delete + // durable before anything else can get in the way, which is what + // flushIndexSave is awaited for on the delete paths. + // + // Each index gets its own try. One try around both would let a failure in + // whichever runs first stop the other from persisting at all — harmless + // when BM25 led and vectors trailed, but reversing the order without this + // would make a vector failure silently block BM25 too. + if (this.vector) { + try { await this.saveVectorBuckets(this.vector); + } catch (err) { + this.logFailure(err); } + } + try { + await this.saveBm25Index(this.bm25.serialize()); } catch (err) { this.logFailure(err); } @@ -277,9 +335,16 @@ export class IndexPersistence { * correct rather than stale. */ private async saveVectorBuckets(vector: VectorIndex): Promise { - const previous = await this.kv - .get(KV.bm25Index, VECTOR_MANIFEST_KEY) - .catch(() => null); + // Deliberately unguarded. Swallowing a read failure here would report "no + // previous manifest", which is indistinguishable from a genuinely absent + // one — and that mistake strands the entire previous generation, because + // the manifest is the only thing that can name it. One transient engine + // timeout would be enough. Let it throw; runSave logs and the debounce + // retries in seconds. + const previous = await this.kv.get( + KV.bm25Index, + VECTOR_MANIFEST_KEY, + ); const buckets = vectorBucketCount(this.options); const chunkChars = shardChars(this.options); @@ -305,7 +370,13 @@ export class IndexPersistence { // Deleting first is how a failed publish destroys data the still-current // manifest points at — the old manifest survives the failure, so anything // it names has to survive with it. - const pendingDeletes: string[] = []; + const pendingDeletes: Array<{ scope: string; key: string }> = []; + + // Deletes a previous save published but died before finishing. Carrying + // them forward is the whole reason the list is in the manifest. + for (const target of previousV2?.reclaim ?? []) { + if (isReclaimTarget(target)) pendingDeletes.push(target); + } let written = 0; for (const [bucket, body] of vector.serializeBuckets(buckets)) { @@ -321,14 +392,17 @@ export class IndexPersistence { for (let i = 0; i < chunks; i++) { await this.kv.set( VECTOR_BUCKET_SCOPE, - vectorChunkKey(bucketKey, i), + vectorChunkKey(bucketKey, hash, i), body.slice(i * chunkChars, (i + 1) * chunkChars), ); } written++; - // This bucket shrank: its old tail chunks are no longer addressed. - for (let i = chunks; i < (priorEntry?.chunks ?? 0); i++) { - pendingDeletes.push(vectorChunkKey(bucketKey, i)); + // The bucket's old content lives under different keys entirely, so it is + // still intact right now and is reclaimed only after the publish. + if (priorEntry) { + for (const key of bucketChunkKeys(bucketKey, priorEntry)) { + pendingDeletes.push({ scope: VECTOR_BUCKET_SCOPE, key }); + } } } @@ -338,17 +412,48 @@ export class IndexPersistence { for (const [bucketKey, priorEntry] of Object.entries(prior)) { if (bucketKey in nextShards) continue; if (!isValidBucketEntry(priorEntry)) continue; - for (let i = 0; i < priorEntry.chunks; i++) { - pendingDeletes.push(vectorChunkKey(bucketKey, i)); + for (const key of bucketChunkKeys(bucketKey, priorEntry)) { + pendingDeletes.push({ scope: VECTOR_BUCKET_SCOPE, key }); + } + } + + // One-time migration off the offset-chunked format. A v1 manifest names its + // own shards, so this needs no enumeration — which matters, because + // StateKV.list returns values, not keys, and cannot be used here. + const legacy = previous as IndexShardManifest | null; + if (legacy?.v === 1 && Array.isArray(legacy.shards)) { + for (const shard of legacy.shards) { + if (isValidShardDescriptor(shard)) { + pendingDeletes.push({ scope: shard.scope, key: shard.key }); + } } } + // Never delete a key the new manifest still names. Old and new keys can + // collide legitimately: identical content re-chunked keeps the same bucket + // and hash, so chunk 0's key is unchanged even though the old entry listed + // more chunks. Reclaiming that entry blind would delete a bucket the + // manifest is actively pointing at. + const liveKeys = new Set(); + for (const [bucketKey, entry] of Object.entries(nextShards)) { + for (const key of bucketChunkKeys(bucketKey, entry)) liveKeys.add(key); + } + const reclaimTargets = pendingDeletes.filter( + (target) => + !(target.scope === VECTOR_BUCKET_SCOPE && liveKeys.has(target.key)), + ); + + // Genuinely nothing to do. Republishing an identical manifest every + // debounce costs a write and two audit rows for no change. + if (written === 0 && reclaimTargets.length === 0 && previousV2) return; + const nextManifest: VectorBucketManifest = { v: 2, layout: VECTOR_LAYOUT, buckets, chunkChars, shards: nextShards, + ...(reclaimTargets.length > 0 ? { reclaim: reclaimTargets } : {}), }; await this.kv.set( KV.bm25Index, @@ -357,8 +462,24 @@ export class IndexPersistence { ); // Safe from here: the new manifest is live and names none of these. - for (const key of pendingDeletes) { - await this.deleteKey(VECTOR_BUCKET_SCOPE, key, "vector_bucket_reclaim"); + const undeleted: Array<{ scope: string; key: string }> = []; + for (const target of reclaimTargets) { + const gone = await this.deleteKey( + target.scope, + target.key, + "vector_bucket_reclaim", + ); + if (!gone) undeleted.push(target); + } + if (reclaimTargets.length > 0) { + // Record the drain, keeping anything that did not actually go. Clearing + // the list on a failed delete would strand those keys permanently, and + // not clearing it at all would let the list grow without bound. + const { reclaim: _drained, ...rest } = nextManifest; + await this.kv.set(KV.bm25Index, VECTOR_MANIFEST_KEY, { + ...rest, + ...(undeleted.length > 0 ? { reclaim: undeleted } : {}), + }); } await this.auditIndexPersistence( @@ -369,17 +490,10 @@ export class IndexPersistence { buckets, written, unchanged: Object.keys(nextShards).length - written, - reclaimed: pendingDeletes.length, + reclaimed: reclaimTargets.length, }, ); - // One-time migration off the offset-chunked format. A v1 manifest names - // its own shards, so this needs no enumeration — which matters, because - // StateKV.list returns values, not keys, and cannot be used here. - const legacy = previous as IndexShardManifest | null; - if (legacy?.v === 1 && Array.isArray(legacy.shards)) { - await this.deleteShards(legacy.shards, "vector_v1_migration"); - } await this.deleteKey(KV.bm25Index, VECTOR_KEY, "legacy_cleanup"); } @@ -497,11 +611,13 @@ export class IndexPersistence { ); } + /** Returns whether the key is actually gone. Callers that track reclaim work + * must keep a failed delete queued rather than reporting it done. */ private async deleteKey( scope: string, key: string, reason: string, - ): Promise { + ): Promise { let result = "deleted"; let error: string | undefined; try { @@ -517,6 +633,7 @@ export class IndexPersistence { result, error, }); + return result === "deleted"; } private async deleteShards( @@ -592,7 +709,10 @@ export class IndexPersistence { let complete = true; for (let i = 0; i < entry.chunks; i++) { const chunk = await this.kv - .get(VECTOR_BUCKET_SCOPE, vectorChunkKey(bucketKey, i)) + .get( + VECTOR_BUCKET_SCOPE, + vectorChunkKey(bucketKey, entry.hash, i), + ) .catch(() => null); if (typeof chunk !== "string") { complete = false; diff --git a/test/index-persistence.test.ts b/test/index-persistence.test.ts index d865e83d79..5c9325efd4 100644 --- a/test/index-persistence.test.ts +++ b/test/index-persistence.test.ts @@ -11,6 +11,12 @@ const VECTOR_LEGACY_KEY = "vectors"; const VECTOR_MANIFEST_KEY = "vectors:manifest"; const VECTOR_BUCKET_SCOPE = "mem:index:bm25:vectors:v2"; +// Mirrors the production key builder. Chunk keys carry the bucket's content +// hash so a bucket is never overwritten in place. +function chunkKey(bucketKey: string, hash: string, chunk = 0): string { + return `${bucketKey}:${hash.slice(0, 12)}:${String(chunk).padStart(5, "0")}`; +} + type TestVectorBucketManifest = { v: 2; layout: number; @@ -279,7 +285,10 @@ describe("IndexPersistence", () => { await expect(kv.get(BM25_SCOPE, VECTOR_LEGACY_KEY)).resolves.toBeNull(); // Vector payloads live outside the BM25 scope. await expect( - kv.get(VECTOR_BUCKET_SCOPE, `${bucketKeys[0]}:00000`), + kv.get( + VECTOR_BUCKET_SCOPE, + chunkKey(bucketKeys[0], manifest!.shards[bucketKeys[0]].hash), + ), ).resolves.toEqual(expect.any(String)); const loaded = await persistence.load(); @@ -872,12 +881,24 @@ describe("IndexPersistence vector bucketing", () => { shardChars: 400, }).save(); - // Fresh instance: hashes come from the manifest, not process memory. This - // is what stops a restart-heavy service rewriting the whole index. + // Go through the real restart path. Seeding a fresh identical index would + // skip load() entirely, leaving the thing that actually keeps hashes stable + // across a restart — deserialize -> mergeSerialized row order -> identical + // serialised bytes — completely unpinned. + const reloaded = await new IndexPersistence( + kv as never, + new SearchIndex(), + null, + ).load(); + expect(reloaded.vector!.size).toBe(40); + kv.resetCount(); - const restarted = new IndexPersistence(kv as never, new SearchIndex(), seeded(40), { - shardChars: 400, - }); + const restarted = new IndexPersistence( + kv as never, + new SearchIndex(), + reloaded.vector, + { shardChars: 400 }, + ); await restarted.save(); expect(kv.bucketWrites).toBe(0); @@ -1043,6 +1064,7 @@ describe("IndexPersistence vector bucketing", () => { VECTOR_MANIFEST_KEY, ); const [tornKey] = Object.keys(manifest!.shards); + const tornHash = manifest!.shards[tornKey].hash; // Critically, this stand-in is VALID JSON in the right row shape. An // invalid body would be dropped by mergeSerialized regardless, so the test @@ -1059,7 +1081,7 @@ describe("IndexPersistence vector bucketing", () => { }, ], ]); - await kv.set(VECTOR_BUCKET_SCOPE, `${tornKey}:00000`, ghost); + await kv.set(VECTOR_BUCKET_SCOPE, chunkKey(tornKey, tornHash), ghost); const loaded = await new IndexPersistence( kv as never, @@ -1120,7 +1142,10 @@ describe("IndexPersistence vector layout changes", () => { // Every key the old layout owned and the new one cannot name must be gone. for (const bucketKey of beyondNarrow) { await expect( - kv.get(VECTOR_BUCKET_SCOPE, `${bucketKey}:00000`), + kv.get( + VECTOR_BUCKET_SCOPE, + chunkKey(bucketKey, wide!.shards[bucketKey].hash), + ), ).resolves.toBeNull(); } const narrow = await kv.get( @@ -1164,9 +1189,15 @@ describe("IndexPersistence vector layout changes", () => { // The bucket now needs fewer keys, so its old tail must be reclaimed // rather than left addressable-by-nobody. + // Chunk 0 keeps its key: same bucket, same content hash, same index. Only + // the tail is no longer addressed, and reclaiming it must not take chunk 0 + // with it. + await expect( + kv.get(VECTOR_BUCKET_SCOPE, chunkKey(shrinkingKey, wideEntry.hash, 0)), + ).resolves.toEqual(expect.any(String)); for (let i = 1; i < wideEntry.chunks; i++) { await expect( - kv.get(VECTOR_BUCKET_SCOPE, `${shrinkingKey}:${String(i).padStart(5, "0")}`), + kv.get(VECTOR_BUCKET_SCOPE, chunkKey(shrinkingKey, wideEntry.hash, i)), ).resolves.toBeNull(); } @@ -1181,3 +1212,114 @@ describe("IndexPersistence vector layout changes", () => { ).toBe("obs_7"); }); }); + +describe("IndexPersistence torn vector save", () => { + let kv: ReturnType; + + beforeEach(() => { + vi.useFakeTimers(); + kv = countingKV(); + }); + afterEach(() => vi.useRealTimers()); + + function seeded(count: number): VectorIndex { + const vector = new VectorIndex(); + for (let i = 0; i < count; i++) { + vector.add(`obs_${i}`, "ses_1", new Float32Array([i, i + 1, i + 2])); + } + return vector; + } + + it("loses nothing when a save dies partway through writing buckets", async () => { + await new IndexPersistence(kv as never, new SearchIndex(), seeded(40), { + shardChars: 400, + vectorBuckets: 8, + }).save(); + + const before = await new IndexPersistence( + kv as never, + new SearchIndex(), + null, + ).load(); + expect(before.vector!.size).toBe(40); + + // A second save with changed content that dies after some bucket writes and + // before the manifest publish. With in-place keys this would corrupt the + // buckets the live manifest still names, load would drop them, and — since + // rebuild is gated on BM25 being empty, never on the vector index — the + // next save would serialise the absence and lose them for good. + const changed = seeded(40); + for (let i = 0; i < 40; i++) { + changed.add(`obs_${i}`, "ses_1", new Float32Array([i + 99, i, i])); + } + let writes = 0; + const dying = { + ...kv, + set: async (scope: string, key: string, data: T): Promise => { + if (scope === VECTOR_BUCKET_SCOPE && ++writes > 3) { + throw new Error("engine died mid-save"); + } + return kv.set(scope, key, data); + }, + }; + await new IndexPersistence(dying as never, new SearchIndex(), changed, { + shardChars: 400, + vectorBuckets: 8, + }).save(); + + // The old generation is still fully intact and loadable. + const after = await new IndexPersistence( + kv as never, + new SearchIndex(), + null, + ).load(); + expect(after.vector!.size).toBe(40); + expect( + after.vector!.search(new Float32Array([7, 8, 9]), 50)[0]?.obsId, + ).toBe("obs_7"); + }); + + it("resumes a reclaim that was published but never finished", async () => { + await new IndexPersistence(kv as never, new SearchIndex(), seeded(40), { + shardChars: 400, + vectorBuckets: 8, + }).save(); + + // Save again with different content, failing every delete so the reclaim + // list is published but never drained. + const changed = seeded(40); + changed.add("obs_extra", "ses_1", new Float32Array([5, 5, 5])); + const undeletable = { + ...kv, + delete: async (): Promise => { + throw new Error("delete unavailable"); + }, + }; + await new IndexPersistence(undeletable as never, new SearchIndex(), changed, { + shardChars: 400, + vectorBuckets: 8, + }).save(); + + const stalled = await kv.get; + }>(BM25_SCOPE, VECTOR_MANIFEST_KEY); + expect(stalled!.reclaim!.length).toBeGreaterThan(0); + const stranded = stalled!.reclaim!.map((t) => t.key); + + // A later healthy save must finish the job rather than orphan those keys. + await new IndexPersistence(kv as never, new SearchIndex(), changed, { + shardChars: 400, + vectorBuckets: 8, + }).save(); + + for (const key of stranded) { + await expect(kv.get(VECTOR_BUCKET_SCOPE, key)).resolves.toBeNull(); + } + const loaded = await new IndexPersistence( + kv as never, + new SearchIndex(), + null, + ).load(); + expect(loaded.vector!.size).toBe(41); + }); +}); From 905be675d8a3dd6ea64801a767c8ff824d196051 Mon Sep 17 00:00:00 2001 From: Omar Gerardo Date: Wed, 26 Aug 2026 20:58:25 +0800 Subject: [PATCH 04/10] fix(state): a failed read is not an empty store, and a clear is not a delete Three ways the bucketed vector index destroyed data, all measured rather than inferred, all with the same shape: something the code could not observe was treated as authority to delete. serializeBuckets is a generator, and the caller awaits a KV write between yields. It read each row from the live Map at yield time, so a concurrent search that triggers rebuildIndex, whose first statements clear both indexes synchronously before any await, emptied the Map underneath the loop. Every remaining bucket then serialised to "[]", got hashed, written, and published as that bucket's true content, and the real content behind it was reclaimed. Manifest and disk agreed perfectly, so the load-time hash check could not see it. Measured: 60 durable vectors to 4, with 54 buckets stored as "[]". Snapshot the entry references when grouping instead. The Float32Arrays are not copied, and only one bucket's base64 is materialised at a time, so the memory property that matters is unchanged. The load path still swallowed read errors with catch(() => null) after the save path had been hardened against exactly that. A single timed-out manifest read returned no manifest, fell through to a v1 path that no longer has anything, left the live index empty, and the next save reclaimed every bucket the old manifest named. Measured: 60 to 0 from one transient read. A timed-out chunk read did the same to one bucket at a time: skipped at load, absent from memory, deleted on the next save. Measured: 58/60 loaded, then 58/60 permanently. Both come from the same confusion between "I could not read this" and "this does not exist". Track whether a vector load read everything the manifest named, and while it did not, preserve every unread bucket in the next manifest instead of reclaiming it. The data stays referenced and a later clean load recovers it. Rebuild cannot cover any of this: it is gated on the BM25 index being empty (src/index.ts, src/functions/search.ts) and never on the vector index, so a vector index that silently loses entries is never repopulated. --- src/state/index-persistence.ts | 39 ++++++++++- src/state/vector-index.ts | 53 +++++++++------ test/index-persistence.test.ts | 116 +++++++++++++++++++++++++++++++++ 3 files changed, 186 insertions(+), 22 deletions(-) diff --git a/src/state/index-persistence.ts b/src/state/index-persistence.ts index 50d6321231..b4f9952ea3 100644 --- a/src/state/index-persistence.ts +++ b/src/state/index-persistence.ts @@ -211,6 +211,17 @@ function isValidShardDescriptor( export class IndexPersistence { private timer: ReturnType | null = null; private lastFailureLogAt = 0; + /** + * Set when a vector load could not read everything the manifest named. + * + * `.catch(() => null)` on a read makes "I could not read this" look identical + * to "this does not exist", and reclaim treats absence as authority to + * delete. One timed-out read at boot would otherwise leave the in-memory + * index short, and the next save would serialise that gap and permanently + * delete the buckets behind it. While this is set, save preserves every + * unread bucket instead of reclaiming it. + */ + private vectorLoadIncomplete = false; constructor( private kv: StateKV, @@ -412,6 +423,13 @@ export class IndexPersistence { for (const [bucketKey, priorEntry] of Object.entries(prior)) { if (bucketKey in nextShards) continue; if (!isValidBucketEntry(priorEntry)) continue; + if (this.vectorLoadIncomplete) { + // This bucket is missing from memory because we could not read it, not + // because anything was deleted. Keep naming it so it stays reachable + // and a later clean load can recover it. + nextShards[bucketKey] = priorEntry; + continue; + } for (const key of bucketChunkKeys(bucketKey, priorEntry)) { pendingDeletes.push({ scope: VECTOR_BUCKET_SCOPE, key }); } @@ -691,9 +709,20 @@ export class IndexPersistence { * the next save rewrites it. */ private async loadVectorBuckets(): Promise { - const manifest = await this.kv - .get(KV.bm25Index, VECTOR_MANIFEST_KEY) - .catch(() => null); + let manifest: unknown; + try { + manifest = await this.kv.get(KV.bm25Index, VECTOR_MANIFEST_KEY); + } catch (err) { + // A read that failed is not a store that is empty. Falling through to the + // v1 path here finds nothing (v1 was migrated away), load returns a null + // vector index, and the next save reclaims every bucket the manifest + // still names. One transient timeout, whole index gone. + logger.warn("index persistence: vector manifest read failed", { + message: errorMessage(err), + }); + this.vectorLoadIncomplete = true; + return new VectorIndex(); + } if (!isVectorBucketManifest(manifest)) return null; const entries = Object.entries(manifest.shards); @@ -737,6 +766,10 @@ export class IndexPersistence { index.mergeSerialized(body); } if (missing > 0 || corrupt > 0) { + // Anything not read is still on disk and still referenced. Mark the load + // incomplete so the next save preserves those buckets rather than + // treating their absence from memory as a deletion. + this.vectorLoadIncomplete = true; logger.warn("index persistence: vector buckets skipped", { missing, corrupt, diff --git a/src/state/vector-index.ts b/src/state/vector-index.ts index d130e60d1b..7e5b6543d6 100644 --- a/src/state/vector-index.ts +++ b/src/state/vector-index.ts @@ -168,27 +168,42 @@ export class VectorIndex { * previous manifest and deletes them. */ *serializeBuckets(bucketCount: number): Generator<[number, string]> { - const groups = new Map(); - for (const obsId of this.vectors.keys()) { + // Snapshot entry references up front rather than reading the live Map as + // each bucket is yielded. + // + // The caller awaits a KV write between yields, and rebuildIndex() calls + // vectorIndex.clear() synchronously before its first await + // (src/functions/search.ts). A search that triggers a rebuild mid-save + // would therefore empty the Map underneath this loop, every remaining + // bucket would serialise to "[]", and those empty buckets would be hashed, + // written, and published as the bucket's true content — with the manifest + // and disk in perfect agreement, so the load-time hash check cannot see it. + // + // Holding references costs nothing: the Float32Arrays already exist and are + // not copied. Only one bucket's base64 is materialised at a time, which is + // the memory property that matters. + const groups = new Map< + number, + Array<[string, { embedding: Float32Array; sessionId: string }]> + >(); + for (const [obsId, entry] of this.vectors) { const bucket = vectorBucketOf(obsId, bucketCount); - const ids = groups.get(bucket); - if (ids) ids.push(obsId); - else groups.set(bucket, [obsId]); + const rows = groups.get(bucket); + if (rows) rows.push([obsId, entry]); + else groups.set(bucket, [[obsId, entry]]); } - for (const [bucket, ids] of groups) { - const rows: Array<[string, { embedding: string; sessionId: string }]> = []; - for (const obsId of ids) { - const entry = this.vectors.get(obsId); - if (!entry) continue; - rows.push([ - obsId, - { - embedding: float32ToBase64(entry.embedding), - sessionId: entry.sessionId, - }, - ]); - } - yield [bucket, JSON.stringify(rows)]; + for (const [bucket, rows] of groups) { + const serialised = rows.map( + ([obsId, entry]) => + [ + obsId, + { + embedding: float32ToBase64(entry.embedding), + sessionId: entry.sessionId, + }, + ] as [string, { embedding: string; sessionId: string }], + ); + yield [bucket, JSON.stringify(serialised)]; } } diff --git a/test/index-persistence.test.ts b/test/index-persistence.test.ts index 5c9325efd4..32cc26ea25 100644 --- a/test/index-persistence.test.ts +++ b/test/index-persistence.test.ts @@ -1323,3 +1323,119 @@ describe("IndexPersistence torn vector save", () => { expect(loaded.vector!.size).toBe(41); }); }); + +// A failed read is not an empty store, and a mid-save mutation is not a +// deletion. Both were measured to destroy data before these guards existed. +describe("IndexPersistence vector save/load hazards", () => { + let kv: ReturnType; + + beforeEach(() => { + vi.useFakeTimers(); + kv = countingKV(); + }); + afterEach(() => vi.useRealTimers()); + + function seeded(count: number): VectorIndex { + const vector = new VectorIndex(); + for (let i = 0; i < count; i++) { + vector.add(`obs_${i}`, "ses_1", new Float32Array([i, i + 1, i + 2])); + } + return vector; + } + + async function loadedSize(): Promise { + const loaded = await new IndexPersistence( + kv as never, + new SearchIndex(), + null, + ).load(); + return loaded.vector?.size ?? 0; + } + + it("does not persist empty buckets when the index is cleared mid-save", async () => { + const vector = seeded(60); + // rebuildIndex() calls vectorIndex.clear() synchronously before its first + // await, so a search-triggered rebuild lands between two bucket writes. + let clearedAt = 0; + const clearing = { + ...kv, + set: async (scope: string, key: string, data: T): Promise => { + if (scope === VECTOR_BUCKET_SCOPE && ++clearedAt === 2) vector.clear(); + return kv.set(scope, key, data); + }, + }; + + await new IndexPersistence(clearing as never, new SearchIndex(), vector, { + shardChars: 400, + vectorBuckets: 8, + }).save(); + + // Reading the live Map lazily would serialise every later bucket as "[]", + // hash it, publish it as truth, and reclaim the real content behind it. + expect(await loadedSize()).toBe(60); + }); + + it("does not wipe the index when the manifest read fails at load", async () => { + await new IndexPersistence(kv as never, new SearchIndex(), seeded(60), { + shardChars: 400, + vectorBuckets: 8, + }).save(); + + const blindKv = { + ...kv, + get: async (scope: string, key: string): Promise => { + if (scope === BM25_SCOPE && key === VECTOR_MANIFEST_KEY) { + throw new Error("manifest read timed out"); + } + return kv.get(scope, key); + }, + }; + // Same instance loads then saves, exactly as src/index.ts does. + const persistence = new IndexPersistence( + blindKv as never, + new SearchIndex(), + new VectorIndex(), + { shardChars: 400, vectorBuckets: 8 }, + ); + await persistence.load(); + await persistence.save(); + + expect(await loadedSize()).toBe(60); + }); + + it("does not delete a bucket whose chunk read failed at load", async () => { + await new IndexPersistence(kv as never, new SearchIndex(), seeded(60), { + shardChars: 400, + vectorBuckets: 8, + }).save(); + + const manifest = await kv.get( + BM25_SCOPE, + VECTOR_MANIFEST_KEY, + ); + const [victim] = Object.keys(manifest!.shards); + const victimChunk = chunkKey(victim, manifest!.shards[victim].hash, 0); + + const flakyKv = { + ...kv, + get: async (scope: string, key: string): Promise => { + if (scope === VECTOR_BUCKET_SCOPE && key === victimChunk) { + throw new Error("chunk read timed out"); + } + return kv.get(scope, key); + }, + }; + const persistence = new IndexPersistence( + flakyKv as never, + new SearchIndex(), + new VectorIndex(), + { shardChars: 400, vectorBuckets: 8 }, + ); + const partial = await persistence.load(); + expect(partial.vector!.size).toBeLessThan(60); + + // The blip must not be converted into a permanent delete. + await persistence.save(); + expect(await loadedSize()).toBe(60); + }); +}); From 0c93c333fd49e25b526e6eccb06532650cee5ed3 Mon Sep 17 00:00:00 2001 From: Omar Gerardo Date: Wed, 26 Aug 2026 20:58:41 +0800 Subject: [PATCH 05/10] fix(state): bump the layout version the key-format change required Chunk keys started carrying the bucket content hash without VECTOR_LAYOUT moving off 1, which is the one thing that constant's own doc comment says must never happen. Upgrading from the previous format therefore saw a matching layout, matching bucket count, matching chunkChars and matching content hashes, skipped every write, republished the manifest, and then looked for hash-bearing keys while the store still held index-addressed ones. Every bucket unreadable, and the old keys orphaned because reclaim names them in the new format. Measured: 30 durable vectors to 0 across the upgrade. No test could see it. Every test in the file writes and reads through the same build, so the suite stays self-consistent under any addressing change; deleting the layout comparison entirely left all 42 green. Add the one cross-version fixture: rewrite a stored manifest to an older layout while keeping its content hashes, drop the payload keys, and require the next save to rewrite rather than skip. Also bound chunks in isValidBucketEntry. Number.isInteger(1e9) is true, and the value is the loop bound for both reclaim walks, so one poisoned manifest field queued 200,006 deletes in 634ms. The cap still allows a 20 GB bucket at the default chunk size. Two tests were passengers. The O(1)-writes test asserted only an upper bound on write count, which a wrongly-skipped write also satisfies, so it now loads the result back and requires the new vector to be retrievable. The delete test removed an observation that was the sole occupant of its bucket, so the bucket was reclaimed and no write ever happened; it now removes one that shares a bucket and requires the bucket-mate to survive the rewrite. --- src/state/index-persistence.ts | 16 +++++-- test/index-persistence.test.ts | 77 +++++++++++++++++++++++++++++++++- 2 files changed, 88 insertions(+), 5 deletions(-) diff --git a/src/state/index-persistence.ts b/src/state/index-persistence.ts index b4f9952ea3..86d4d079ec 100644 --- a/src/state/index-persistence.ts +++ b/src/state/index-persistence.ts @@ -21,6 +21,7 @@ const DEFAULT_INDEX_SHARD_CHARS = 2_000_000; // are deterministic and overwritten in place, so nothing can be stranded. const VECTOR_BUCKET_SCOPE = `${KV.bm25Index}:vectors:v2`; const DEFAULT_VECTOR_BUCKETS = 256; +const MAX_BUCKET_CHUNKS = 10_000; type IndexShardManifest = { v: 1; @@ -59,7 +60,10 @@ type VectorBucketEntry = { hash: string; chunks: number }; * well enough to reclaim the keys it named. Bumping `v` instead would make the * old manifest unreadable and strand every key in it. */ -const VECTOR_LAYOUT = 1; +// 2: chunk keys carry the bucket's content hash. 1 addressed them by index +// alone. Anything written under layout 1 is unreadable under 2, which is +// precisely why the mismatch has to force a full rewrite instead of a skip. +const VECTOR_LAYOUT = 2; type VectorBucketManifest = { v: 2; @@ -93,13 +97,17 @@ function isVectorBucketManifest(value: unknown): value is VectorBucketManifest { function isValidBucketEntry(value: unknown): value is VectorBucketEntry { if (!value || typeof value !== "object") return false; const candidate = value as Partial; - // The chunks bound matters beyond shape: it is the loop bound for both the - // reclaim walks below, so a poisoned value would spin a garbage-length loop. + // chunks is the loop bound for both reclaim walks, so it needs a ceiling and + // not just an integer check. Number.isInteger(1e9) is true, and a single + // poisoned manifest field would otherwise queue a billion deletes. The cap is + // deliberately far above any real bucket: at the default 2M chunkChars this + // still allows a 20 GB bucket. return ( typeof candidate.hash === "string" && candidate.hash.length > 0 && Number.isInteger(candidate.chunks) && - (candidate.chunks as number) >= 1 + (candidate.chunks as number) >= 1 && + (candidate.chunks as number) <= MAX_BUCKET_CHUNKS ); } diff --git a/test/index-persistence.test.ts b/test/index-persistence.test.ts index 32cc26ea25..9dfdb4fbf9 100644 --- a/test/index-persistence.test.ts +++ b/test/index-persistence.test.ts @@ -871,6 +871,14 @@ describe("IndexPersistence vector bucketing", () => { await persistence.save(); expect(kv.bucketWrites).toBeLessThanOrEqual(2); + // Bounding the write count alone does not discriminate: skipping writes + // that were needed also lowers it. Read the result back. + const loaded = await persistence.load(); + expect(loaded.vector!.size).toBe(61); + expect( + loaded.vector!.search(new Float32Array([9, 9, 9]), 61) + .some((r) => r.obsId === "obs_new"), + ).toBe(true); }); // Covers the same-instance case too: saveVectorBuckets holds no state between @@ -911,13 +919,22 @@ describe("IndexPersistence vector bucketing", () => { }); await persistence.save(); - vector.remove("obs_7"); + // obs_9 shares a bucket with obs_30, so removing it forces that bucket to be + // REWRITTEN with its surviving member. Removing a bucket's only occupant + // instead would just reclaim it and never exercise a write at all, which is + // what this test is supposed to be about. + vector.remove("obs_9"); kv.resetCount(); await persistence.save(); expect(kv.bucketWrites).toBeLessThanOrEqual(1); const loaded = await persistence.load(); expect(loaded.vector!.size).toBe(59); + // The bucket-mate must survive the rewrite. + expect( + loaded.vector!.search(new Float32Array([30, 31, 32]), 59) + .some((r) => r.obsId === "obs_30"), + ).toBe(true); }); it("round-trips every vector through the bucketed format", async () => { @@ -1439,3 +1456,61 @@ describe("IndexPersistence vector save/load hazards", () => { expect(await loadedSize()).toBe(60); }); }); + +// Every other test in this file writes and reads through the same build, so the +// suite is self-consistent under ANY addressing change and cannot see one. This +// is the only cross-version fixture: it stands in for "a store written by an +// older layout" and pins the guard that forces a rewrite instead of a skip. +describe("IndexPersistence vector layout version", () => { + let kv: ReturnType; + + beforeEach(() => { + vi.useFakeTimers(); + kv = countingKV(); + }); + afterEach(() => vi.useRealTimers()); + + function seeded(count: number): VectorIndex { + const vector = new VectorIndex(); + for (let i = 0; i < count; i++) { + vector.add(`obs_${i}`, "ses_1", new Float32Array([i, i + 1, i + 2])); + } + return vector; + } + + it("rewrites everything when the stored layout version differs", async () => { + const vector = seeded(20); + await new IndexPersistence(kv as never, new SearchIndex(), vector, { + shardChars: 400, + }).save(); + + // Rewrite the manifest as an older layout while KEEPING every content hash, + // then drop the payload keys. Same content, different addressing — exactly + // the shape a content hash cannot detect on its own. A build that trusts the + // hash alone skips every write and then cannot find its own data. + const current = await kv.get( + BM25_SCOPE, + VECTOR_MANIFEST_KEY, + ); + await kv.set(BM25_SCOPE, VECTOR_MANIFEST_KEY, { + ...current!, + layout: current!.layout - 1, + }); + for (const [bucketKey, entry] of Object.entries(current!.shards)) { + for (let i = 0; i < entry.chunks; i++) { + await kv.delete(VECTOR_BUCKET_SCOPE, chunkKey(bucketKey, entry.hash, i)); + } + } + + await new IndexPersistence(kv as never, new SearchIndex(), seeded(20), { + shardChars: 400, + }).save(); + + const loaded = await new IndexPersistence( + kv as never, + new SearchIndex(), + null, + ).load(); + expect(loaded.vector!.size).toBe(20); + }); +}); From bf325f6bcb1058de3ab0a76336a1937759e768f8 Mon Sep 17 00:00:00 2001 From: Omar Gerardo Date: Thu, 27 Aug 2026 01:09:13 +0800 Subject: [PATCH 06/10] fix(state): refuse to publish a vector bucket the loader will reject saveVectorBuckets computed its chunk count with no upper bound while isValidBucketEntry rejects any entry above MAX_BUCKET_CHUNKS at load time. The two sides disagreed and the write side won: a small configured shardChars let a save publish an entry the next boot classified as corrupt, dropping a bucket whose bytes were on disk fully intact and setting vectorLoadIncomplete on every load from then on. Measured against the unmodified source at shardChars 1 with a single bucket: 200 durable vectors to 0, the bucket needing 12,891 chunks against a cap of 10,000, and "vector buckets skipped {missing:0, corrupt:1, total:1}" as the only signal that anything went wrong. Fail the save instead. The previous manifest stays live and readable, so an oversized chunk count is a misconfiguration rather than data loss, and the next save with a sane chunkChars succeeds. Limitation: buckets written earlier in the same pass are left behind by the throw. That is the same shape as a crash mid-save, which this format already accepts. If the corpus has not moved by the next successful save, the content hash is unchanged and those exact keys are reused. If it has moved, they are named by no manifest and leak, because StateKV.list returns values rather than keys and nothing can enumerate them afterwards. Pre-computing every bucket body to validate before writing any of it would avoid the leak, but it would defeat serializeBuckets being a generator, which exists so that only one bucket's base64 is materialised at a time. The test asserts on the reload rather than on a throw, because save() funnels the failure through logFailure and nothing surfaces to the caller. It also asserts its own precondition, that the serialised bucket clears the cap, so a shorter serialisation cannot quietly turn it into a test of nothing. --- src/state/index-persistence.ts | 10 ++++++++++ test/index-persistence.test.ts | 35 ++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/src/state/index-persistence.ts b/src/state/index-persistence.ts index 86d4d079ec..c499c39cec 100644 --- a/src/state/index-persistence.ts +++ b/src/state/index-persistence.ts @@ -402,6 +402,16 @@ export class IndexPersistence { const bucketKey = vectorBucketKey(bucket); const hash = contentHash(body); const chunks = Math.max(1, Math.ceil(body.length / chunkChars)); + // The loader rejects anything above this cap, so publishing it would + // hand the next boot a bucket it drops as corrupt while its bytes sit + // on disk intact. Fail the save instead and keep the previous manifest + // live: a small configured chunkChars is a misconfiguration, not data + // loss, and the next save with a sane one succeeds. + if (chunks > MAX_BUCKET_CHUNKS) { + throw new Error( + `vector bucket ${bucketKey} needs ${chunks} chunks, above MAX_BUCKET_CHUNKS`, + ); + } const priorEntry = isValidBucketEntry(prior[bucketKey]) ? prior[bucketKey] : undefined; diff --git a/test/index-persistence.test.ts b/test/index-persistence.test.ts index 9dfdb4fbf9..153db9fdb7 100644 --- a/test/index-persistence.test.ts +++ b/test/index-persistence.test.ts @@ -10,6 +10,8 @@ const BM25_MANIFEST_KEY = "data:manifest"; const VECTOR_LEGACY_KEY = "vectors"; const VECTOR_MANIFEST_KEY = "vectors:manifest"; const VECTOR_BUCKET_SCOPE = "mem:index:bm25:vectors:v2"; +// Mirrors the cap isValidBucketEntry enforces at load time. +const MAX_BUCKET_CHUNKS = 10_000; // Mirrors the production key builder. Chunk keys carry the bucket's content // hash so a bucket is never overwritten in place. @@ -1114,6 +1116,39 @@ describe("IndexPersistence vector bucketing", () => { expect(loaded.vector!.size).toBeGreaterThan(0); expect(loaded.vector!.size).toBeLessThan(30); }); + + // isValidBucketEntry rejects any entry above MAX_BUCKET_CHUNKS, so a save + // that publishes one hands the next boot a bucket it classifies as corrupt — + // dropping vectors whose bytes are entirely intact, and marking every load + // incomplete from then on. A small configured shardChars is all it takes. + it("fails the save rather than publish a bucket the loader will reject", async () => { + const vector = seeded(200); + // Asserted, not assumed: at one chunk per character the bucket has to clear + // the cap, or a shorter serialisation would leave this testing nothing. + const [, body] = [...vector.serializeBuckets(1)][0]!; + expect(body.length).toBeGreaterThan(MAX_BUCKET_CHUNKS); + + // A good save first, so there is something to lose. + await new IndexPersistence(kv as never, new SearchIndex(), vector, { + shardChars: 400, + vectorBuckets: 1, + }).save(); + + // save() funnels the failure through logFailure, so nothing throws out + // here. What has to hold is that the live manifest still names a readable + // bucket rather than an oversized one the loader will throw away. + await new IndexPersistence(kv as never, new SearchIndex(), vector, { + shardChars: 1, + vectorBuckets: 1, + }).save(); + + const loaded = await new IndexPersistence( + kv as never, + new SearchIndex(), + null, + ).load(); + expect(loaded.vector!.size).toBe(200); + }); }); // A content hash cannot see an addressing change: same bytes, different home. From 99db5013f5479d9f85a54e45ac67b645f049f485 Mon Sep 17 00:00:00 2001 From: Omar Gerardo Date: Thu, 27 Aug 2026 02:31:43 +0800 Subject: [PATCH 07/10] test: hoist the duplicated seeded() vector helper Five byte-identical copies of the same seven-line helper, one per describe block. A single definition beside makeVector covers all of them, and the next test to need it does not add a sixth. --- test/index-persistence.test.ts | 48 ++++++---------------------------- 1 file changed, 8 insertions(+), 40 deletions(-) diff --git a/test/index-persistence.test.ts b/test/index-persistence.test.ts index 153db9fdb7..96ed52c5b7 100644 --- a/test/index-persistence.test.ts +++ b/test/index-persistence.test.ts @@ -88,6 +88,14 @@ function makeVector(id = "obs_1"): VectorIndex { return vector; } +function seeded(count: number): VectorIndex { + const vector = new VectorIndex(); + for (let i = 0; i < count; i++) { + vector.add(`obs_${i}`, "ses_1", new Float32Array([i, i + 1, i + 2])); + } + return vector; +} + async function getBm25Manifest(kv: MockKV): Promise { const manifest = await kv.get( BM25_SCOPE, @@ -851,14 +859,6 @@ describe("IndexPersistence vector bucketing", () => { }); afterEach(() => vi.useRealTimers()); - function seeded(count: number): VectorIndex { - const vector = new VectorIndex(); - for (let i = 0; i < count; i++) { - vector.add(`obs_${i}`, "ses_1", new Float32Array([i, i + 1, i + 2])); - } - return vector; - } - it("writes O(1) buckets when one vector is added, not O(corpus)", async () => { const vector = seeded(60); const persistence = new IndexPersistence(kv as never, new SearchIndex(), vector, { @@ -1163,14 +1163,6 @@ describe("IndexPersistence vector layout changes", () => { }); afterEach(() => vi.useRealTimers()); - function seeded(count: number): VectorIndex { - const vector = new VectorIndex(); - for (let i = 0; i < count; i++) { - vector.add(`obs_${i}`, "ses_1", new Float32Array([i, i + 1, i + 2])); - } - return vector; - } - it("reclaims buckets stranded by a smaller bucket count", async () => { await new IndexPersistence(kv as never, new SearchIndex(), seeded(40), { shardChars: 400, @@ -1274,14 +1266,6 @@ describe("IndexPersistence torn vector save", () => { }); afterEach(() => vi.useRealTimers()); - function seeded(count: number): VectorIndex { - const vector = new VectorIndex(); - for (let i = 0; i < count; i++) { - vector.add(`obs_${i}`, "ses_1", new Float32Array([i, i + 1, i + 2])); - } - return vector; - } - it("loses nothing when a save dies partway through writing buckets", async () => { await new IndexPersistence(kv as never, new SearchIndex(), seeded(40), { shardChars: 400, @@ -1387,14 +1371,6 @@ describe("IndexPersistence vector save/load hazards", () => { }); afterEach(() => vi.useRealTimers()); - function seeded(count: number): VectorIndex { - const vector = new VectorIndex(); - for (let i = 0; i < count; i++) { - vector.add(`obs_${i}`, "ses_1", new Float32Array([i, i + 1, i + 2])); - } - return vector; - } - async function loadedSize(): Promise { const loaded = await new IndexPersistence( kv as never, @@ -1505,14 +1481,6 @@ describe("IndexPersistence vector layout version", () => { }); afterEach(() => vi.useRealTimers()); - function seeded(count: number): VectorIndex { - const vector = new VectorIndex(); - for (let i = 0; i < count; i++) { - vector.add(`obs_${i}`, "ses_1", new Float32Array([i, i + 1, i + 2])); - } - return vector; - } - it("rewrites everything when the stored layout version differs", async () => { const vector = seeded(20); await new IndexPersistence(kv as never, new SearchIndex(), vector, { From 5a1edc7331460faa2abce38fcbcbf9d09453f5ae Mon Sep 17 00:00:00 2001 From: Omar Gerardo Date: Thu, 27 Aug 2026 02:39:46 +0800 Subject: [PATCH 08/10] fix(state): report a vector load that rejected every bucket it read A content check that fails does not heal the way a failed read does. The bytes are intact and the keys are still reachable, but no later load can verify them either. The in-memory index stays empty, so save has nothing to publish and preserves the manifest untouched: bytes and manifest in perfect agreement, vectors unreadable for the life of the store. Nothing noticed, because rebuild is gated on the BM25 index being empty and BM25 restores fine. Surface the rejection from load() and let boot treat it the way it treats an empty BM25 index. Kept off `vectorIndex.size === 0`, which is also the ordinary state of a store whose vectors were never written, and off `missing`, which the existing preserve-and-wait policy already covers because a later boot may resolve it. rebuildIndex clears BM25 before repopulating it, so a false positive here costs a full reindex. The new test drives save() from a load() that rejected its buckets, which is the half the layout-version test does not reach: that one injects a fresh index instead, so it never sees the state where there is nothing to write. --- src/index.ts | 8 ++- src/state/index-persistence.ts | 20 ++++++- test/index-persistence.test.ts | 101 +++++++++++++++++++++++++++++++++ 3 files changed, 127 insertions(+), 2 deletions(-) diff --git a/src/index.ts b/src/index.ts index 717419f7df..0b4a3dccfe 100644 --- a/src/index.ts +++ b/src/index.ts @@ -464,7 +464,13 @@ async function main() { } } - const needsRebuild = bm25Index.size === 0; + // A vector load that rejected every bucket it read is the one failure BM25 + // cannot stand in for: BM25 restores fine, so the store looks healthy, while + // the vectors sit on disk unreadable and nothing re-reads them. Rebuilding is + // the only exit. Kept off `vectorIndex.size === 0`, which is also the ordinary + // state of a store whose vectors were simply never written. + const needsRebuild = + bm25Index.size === 0 || (vectorIndex !== null && loaded?.vectorRejected === true); if (needsRebuild) { // Fire-and-forget. rebuildIndex iterates every observation across diff --git a/src/state/index-persistence.ts b/src/state/index-persistence.ts index c499c39cec..b63e3e892b 100644 --- a/src/state/index-persistence.ts +++ b/src/state/index-persistence.ts @@ -230,6 +230,18 @@ export class IndexPersistence { * unread bucket instead of reclaiming it. */ private vectorLoadIncomplete = false; + /** + * Set when every bucket the manifest named failed its content check. + * + * Unlike a failed read, this does not heal on its own: the bytes are intact + * and the keys are reachable, but no later load can verify them either. The + * in-memory index stays empty, so save has nothing to publish and preserves + * the manifest untouched — bytes and manifest in perfect agreement, vectors + * unreadable forever. Nothing else notices, because rebuild is gated on the + * BM25 index being empty and BM25 loaded fine. Reported so the caller can + * rebuild, which is the only thing that ends this state. + */ + private vectorLoadRejected = false; constructor( private kv: StateKV, @@ -281,6 +293,7 @@ export class IndexPersistence { async load(): Promise<{ bm25: SearchIndex | null; vector: VectorIndex | null; + vectorRejected: boolean; }> { let bm25: SearchIndex | null = null; let vector: VectorIndex | null = null; @@ -301,7 +314,7 @@ export class IndexPersistence { } } - return { bm25, vector }; + return { bm25, vector, vectorRejected: this.vectorLoadRejected }; } stop(): void { @@ -727,6 +740,7 @@ export class IndexPersistence { * the next save rewrites it. */ private async loadVectorBuckets(): Promise { + this.vectorLoadRejected = false; let manifest: unknown; try { manifest = await this.kv.get(KV.bm25Index, VECTOR_MANIFEST_KEY); @@ -783,6 +797,10 @@ export class IndexPersistence { } index.mergeSerialized(body); } + // Deliberately narrower than `missing`. A read that failed may succeed on + // the next boot, and the existing preserve-and-wait policy covers it. A + // content check that failed will fail identically every time. + this.vectorLoadRejected = entries.length > 0 && corrupt === entries.length; if (missing > 0 || corrupt > 0) { // Anything not read is still on disk and still referenced. Mark the load // incomplete so the next save preserves those buckets rather than diff --git a/test/index-persistence.test.ts b/test/index-persistence.test.ts index 96ed52c5b7..628a8b29de 100644 --- a/test/index-persistence.test.ts +++ b/test/index-persistence.test.ts @@ -1,4 +1,5 @@ import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"; +import { createHash } from "node:crypto"; import { IndexPersistence } from "../src/state/index-persistence.js"; import { SearchIndex } from "../src/state/search-index.js"; import { VectorIndex } from "../src/state/vector-index.js"; @@ -1517,3 +1518,103 @@ describe("IndexPersistence vector layout version", () => { expect(loaded.vector!.size).toBe(20); }); }); + +// Stands in for a store written by a build whose contentHash was a different +// function: same bodies, same manifest shape, chunk keys still derived from the +// hash the manifest records — so every chunk is found and every hash fails. +async function rehashStoreUnder( + kv: ReturnType, + algorithm: string, +): Promise { + const manifest = (await kv.get( + BM25_SCOPE, + VECTOR_MANIFEST_KEY, + ))!; + const shards: TestVectorBucketManifest["shards"] = {}; + for (const [bucketKey, entry] of Object.entries(manifest.shards)) { + const parts: string[] = []; + for (let i = 0; i < entry.chunks; i++) { + parts.push( + (await kv.get( + VECTOR_BUCKET_SCOPE, + chunkKey(bucketKey, entry.hash, i), + ))!, + ); + } + const rehashed = createHash(algorithm) + .update(parts.join("")) + .digest("hex"); + for (let i = 0; i < entry.chunks; i++) { + await kv.delete(VECTOR_BUCKET_SCOPE, chunkKey(bucketKey, entry.hash, i)); + await kv.set( + VECTOR_BUCKET_SCOPE, + chunkKey(bucketKey, rehashed, i), + parts[i], + ); + } + shards[bucketKey] = { hash: rehashed, chunks: entry.chunks }; + } + await kv.set(BM25_SCOPE, VECTOR_MANIFEST_KEY, { ...manifest, shards }); +} + +describe("IndexPersistence vector hash-format change", () => { + let kv: ReturnType; + + beforeEach(() => { + vi.useFakeTimers(); + kv = countingKV(); + }); + afterEach(() => vi.useRealTimers()); + + it("recovers when a hash change makes every bucket unverifiable", async () => { + await new IndexPersistence(kv as never, new SearchIndex(), seeded(20), { + shardChars: 400, + }).save(); + + await rehashStoreUnder(kv, "sha512"); + const alien = (await kv.get( + BM25_SCOPE, + VECTOR_MANIFEST_KEY, + ))!; + + // Boot. The live index starts empty and only ever holds what load hands it. + const live = new VectorIndex(); + const persistence = new IndexPersistence( + kv as never, + new SearchIndex(), + live, + { shardChars: 400 }, + ); + const loaded = await persistence.load(); + if (loaded.vector && loaded.vector.size > 0) live.restoreFrom(loaded.vector); + expect(live.size).toBe(0); + + // A debounce flush can land long before any rebuild finishes. It must not + // turn "could not verify" into a delete. + await persistence.save(); + for (const [bucketKey, entry] of Object.entries(alien.shards)) { + for (let i = 0; i < entry.chunks; i++) { + expect( + await kv.get( + VECTOR_BUCKET_SCOPE, + chunkKey(bucketKey, entry.hash, i), + ), + ).not.toBeNull(); + } + } + + // Stands in for rebuildIndex(), which src/index.ts fires on this signal and + // which refills the live index before scheduling a save. Nothing else ever + // re-reads those buckets, so without the signal the vectors stay on disk + // and unreadable for the life of the store. + if (loaded.vectorRejected) live.restoreFrom(seeded(20)); + await persistence.save(); + + const reopened = await new IndexPersistence( + kv as never, + new SearchIndex(), + null, + ).load(); + expect(reopened.vector!.size).toBe(20); + }); +}); From ad38fb623aaba66d4bb6a7e09809b769bbb3667c Mon Sep 17 00:00:00 2001 From: Omar Gerardo Date: Thu, 27 Aug 2026 02:43:47 +0800 Subject: [PATCH 09/10] fix(state): accept a bucket hashed by any scheme this format has published MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The set of hashes a load accepts has to be wider than the one a save publishes, and it was not. Chunk keys are derived from the hash the manifest records, so a build whose content hash differs from the stored one finds every chunk and then rejects every one of them. That is not a rejection a later save can undo: the bucket never reaches memory, so there is nothing to re-serialise, and the rewrite that a layout bump is supposed to force writes nothing at all. Bumping VECTOR_LAYOUT alone therefore does not carry a hash change across — the loader has to be able to read what is already on disk first. Accept any hash in the published set, published one first so the common path still costs a single digest. This is also what makes a rollback safe: a build one step behind can now read a store the newer one wrote. --- src/state/index-persistence.ts | 31 ++++++++++++++++++++++++++----- test/index-persistence.test.ts | 21 +++++++++++++++++++++ 2 files changed, 47 insertions(+), 5 deletions(-) diff --git a/src/state/index-persistence.ts b/src/state/index-persistence.ts index b63e3e892b..6c83bd9197 100644 --- a/src/state/index-persistence.ts +++ b/src/state/index-persistence.ts @@ -41,8 +41,8 @@ type IndexShardManifest = { * size, since a bucket grows with the corpus and the engine rejects an * oversized `state::set`. */ -// No `chars`: the sha1 already proves both content and length. v1 needed a -// length check only because it had no hash to check against. +// No `chars`: the content hash already proves both content and length. v1 +// needed a length check only because it had no hash to check against. type VectorBucketEntry = { hash: string; chunks: number }; /** @@ -158,11 +158,32 @@ function bucketChunkKeys(bucketKey: string, entry: VectorBucketEntry): string[] return keys; } -// sha1 over the bucket body. This one IS a content check — a collision means a +// The hash of a bucket body. This one IS a content check — a collision means a // changed bucket is silently never persisted — so it must not be the 32-bit // hash used for bucket assignment. +const PUBLISHED_BUCKET_HASH = "sha1"; + +/** + * Every hash a stored bucket may legitimately carry, published one first. + * + * The set a load accepts is deliberately wider than the one a save publishes. + * A bucket's chunk keys are derived from the hash the manifest records, so a + * build that only accepts its own hash finds every chunk and then rejects every + * one of them — and a rejected bucket cannot be rewritten, because its contents + * never reach memory to be re-serialised. Widening the accepted set is what + * turns a change of published hash into an ordinary rewrite, and it is also + * what lets a rollback read a store the newer build wrote. + */ +const ACCEPTED_BUCKET_HASHES = [PUBLISHED_BUCKET_HASH, "sha256"]; + function contentHash(value: string): string { - return createHash("sha1").update(value).digest("hex"); + return createHash(PUBLISHED_BUCKET_HASH).update(value).digest("hex"); +} + +function bucketBodyMatches(body: string, expected: string): boolean { + return ACCEPTED_BUCKET_HASHES.some( + (algorithm) => createHash(algorithm).update(body).digest("hex") === expected, + ); } type IndexPersistenceOptions = { @@ -791,7 +812,7 @@ export class IndexPersistence { // check it would be silently parsed as empty and its vectors lost with // no signal. Skipping it instead costs only that bucket, and the next // save rewrites it because the hash still disagrees. - if (contentHash(body) !== entry.hash) { + if (!bucketBodyMatches(body, entry.hash)) { corrupt++; continue; } diff --git a/test/index-persistence.test.ts b/test/index-persistence.test.ts index 628a8b29de..76517e4ba4 100644 --- a/test/index-persistence.test.ts +++ b/test/index-persistence.test.ts @@ -1566,6 +1566,27 @@ describe("IndexPersistence vector hash-format change", () => { }); afterEach(() => vi.useRealTimers()); + // Both hashes a bucket may legitimately carry. Whichever one this build + // publishes, the other stands for a store written either side of the + // migration — an older build's, or a newer build's after a rollback. + for (const algorithm of ["sha1", "sha256"]) { + it(`reads a bucket whose manifest records a ${algorithm} hash`, async () => { + await new IndexPersistence(kv as never, new SearchIndex(), seeded(20), { + shardChars: 400, + }).save(); + + await rehashStoreUnder(kv, algorithm); + + const loaded = await new IndexPersistence( + kv as never, + new SearchIndex(), + null, + ).load(); + expect(loaded.vector!.size).toBe(20); + expect(loaded.vectorRejected).toBe(false); + }); + } + it("recovers when a hash change makes every bucket unverifiable", async () => { await new IndexPersistence(kv as never, new SearchIndex(), seeded(20), { shardChars: 400, From bf21c2dec856d49835d858f22fe3317d047e4a68 Mon Sep 17 00:00:00 2001 From: Omar Gerardo Date: Thu, 27 Aug 2026 02:52:07 +0800 Subject: [PATCH 10/10] fix(state): publish vector bucket hashes as sha256 sha1 stays in the accepted set so existing stores are read and migrated in place on the next save, with no re-embedding. It is no longer written. Not fingerprintId. vectorChunkKey truncates the hash to 12 characters, and fingerprintId returns `_` plus 16 hex characters, so a "vec" prefix would spend four of those twelve on a constant: 8 hex characters of entropy, 32 bits, against 48 from raw hex. Content addressing is what makes a torn save survivable, and cutting its distinguishing bits by a factor of 65536 to reuse a helper is not a trade worth taking. VECTOR_LAYOUT goes to 3 because the constant's contract says to bump it on a hash change, and because the layout number is then a readable marker of which scheme wrote a manifest. It is not what carries the migration: the stored sha1 hash cannot equal the computed sha256 one, so the per-bucket comparison already forces every bucket to be rewritten. Reverting the bump alone leaves the suite green, which is worth saying plainly given that mistaking the layout bump for the mechanism is the failure this whole sequence is about. No literal createHash("sha1") or createHash("md5") call remains under src/. --- src/state/index-persistence.ts | 16 +++++---- test/index-persistence.test.ts | 59 ++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 6 deletions(-) diff --git a/src/state/index-persistence.ts b/src/state/index-persistence.ts index 6c83bd9197..3adde35711 100644 --- a/src/state/index-persistence.ts +++ b/src/state/index-persistence.ts @@ -60,10 +60,11 @@ type VectorBucketEntry = { hash: string; chunks: number }; * well enough to reclaim the keys it named. Bumping `v` instead would make the * old manifest unreadable and strand every key in it. */ -// 2: chunk keys carry the bucket's content hash. 1 addressed them by index -// alone. Anything written under layout 1 is unreadable under 2, which is -// precisely why the mismatch has to force a full rewrite instead of a skip. -const VECTOR_LAYOUT = 2; +// 3: bucket hashes are sha256. 2: chunk keys carry the bucket's content hash. +// 1 addressed them by index alone. A bucket written under an earlier layout is +// unreachable under a later one, which is precisely why the mismatch has to +// force a full rewrite instead of a skip. +const VECTOR_LAYOUT = 3; type VectorBucketManifest = { v: 2; @@ -128,6 +129,9 @@ function vectorBucketKey(bucket: number): string { * * Addressing by content instead means the old bucket stays readable until the * new manifest names the new one, so a save that dies partway loses nothing. + * + * The 12 characters must be raw hex. A prefixed identifier would spend some of + * them on a constant and leave fewer distinguishing bits than the key needs. */ function vectorChunkKey( bucketKey: string, @@ -161,7 +165,7 @@ function bucketChunkKeys(bucketKey: string, entry: VectorBucketEntry): string[] // The hash of a bucket body. This one IS a content check — a collision means a // changed bucket is silently never persisted — so it must not be the 32-bit // hash used for bucket assignment. -const PUBLISHED_BUCKET_HASH = "sha1"; +const PUBLISHED_BUCKET_HASH = "sha256"; /** * Every hash a stored bucket may legitimately carry, published one first. @@ -174,7 +178,7 @@ const PUBLISHED_BUCKET_HASH = "sha1"; * turns a change of published hash into an ordinary rewrite, and it is also * what lets a rollback read a store the newer build wrote. */ -const ACCEPTED_BUCKET_HASHES = [PUBLISHED_BUCKET_HASH, "sha256"]; +const ACCEPTED_BUCKET_HASHES = [PUBLISHED_BUCKET_HASH, "sha1"]; function contentHash(value: string): string { return createHash(PUBLISHED_BUCKET_HASH).update(value).digest("hex"); diff --git a/test/index-persistence.test.ts b/test/index-persistence.test.ts index 76517e4ba4..75d5a20666 100644 --- a/test/index-persistence.test.ts +++ b/test/index-persistence.test.ts @@ -1587,6 +1587,65 @@ describe("IndexPersistence vector hash-format change", () => { }); } + it("migrates a store off the previous bucket hash without re-embedding", async () => { + await new IndexPersistence(kv as never, new SearchIndex(), seeded(20), { + shardChars: 400, + }).save(); + + // Stands in for a store this branch's earlier builds wrote: sha1 bucket + // hashes, under the layout that published them. + await rehashStoreUnder(kv, "sha1"); + const current = (await kv.get( + BM25_SCOPE, + VECTOR_MANIFEST_KEY, + ))!; + await kv.set(BM25_SCOPE, VECTOR_MANIFEST_KEY, { + ...current, + layout: current.layout - 1, + }); + const stale = (await kv.get( + BM25_SCOPE, + VECTOR_MANIFEST_KEY, + ))!; + + const live = new VectorIndex(); + const persistence = new IndexPersistence( + kv as never, + new SearchIndex(), + live, + { shardChars: 400 }, + ); + const loaded = await persistence.load(); + expect(loaded.vector!.size).toBe(20); + live.restoreFrom(loaded.vector!); + + await persistence.save(); + + const migrated = (await kv.get( + BM25_SCOPE, + VECTOR_MANIFEST_KEY, + ))!; + expect(migrated.layout).toBe(stale.layout + 1); + for (const entry of Object.values(migrated.shards)) { + // sha256 hex. sha1 would be 40. + expect(entry.hash).toHaveLength(64); + } + for (const [bucketKey, entry] of Object.entries(stale.shards)) { + for (let i = 0; i < entry.chunks; i++) { + expect( + await kv.get(VECTOR_BUCKET_SCOPE, chunkKey(bucketKey, entry.hash, i)), + ).toBeNull(); + } + } + + const reopened = await new IndexPersistence( + kv as never, + new SearchIndex(), + null, + ).load(); + expect(reopened.vector!.size).toBe(20); + }); + it("recovers when a hash change makes every bucket unverifiable", async () => { await new IndexPersistence(kv as never, new SearchIndex(), seeded(20), { shardChars: 400,