diff --git a/src/index.ts b/src/index.ts index 717419f7df..0b4a3dccfe 100644 --- a/src/index.ts +++ b/src/index.ts @@ -464,7 +464,13 @@ async function main() { } } - const needsRebuild = bm25Index.size === 0; + // A vector load that rejected every bucket it read is the one failure BM25 + // cannot stand in for: BM25 restores fine, so the store looks healthy, while + // the vectors sit on disk unreadable and nothing re-reads them. Rebuilding is + // the only exit. Kept off `vectorIndex.size === 0`, which is also the ordinary + // state of a store whose vectors were simply never written. + const needsRebuild = + bm25Index.size === 0 || (vectorIndex !== null && loaded?.vectorRejected === true); if (needsRebuild) { // Fire-and-forget. rebuildIndex iterates every observation across diff --git a/src/state/index-persistence.ts b/src/state/index-persistence.ts index 6df0e2fdaf..3adde35711 100644 --- a/src/state/index-persistence.ts +++ b/src/state/index-persistence.ts @@ -1,3 +1,4 @@ +import { createHash } from "node:crypto"; import { SearchIndex } from "./search-index.js"; import { VectorIndex } from "./vector-index.js"; import type { StateKV } from "./kv.js"; @@ -13,10 +14,15 @@ const BM25_MANIFEST_KEY = "data:manifest"; const BM25_SHARD_SCOPE_PREFIX = `${KV.bm25Index}:bm25:`; const VECTOR_KEY = "vectors"; const VECTOR_MANIFEST_KEY = "vectors:manifest"; -const VECTOR_SHARD_SCOPE_PREFIX = `${KV.bm25Index}:vectors:`; const INDEX_SHARD_KEY = "data"; const DEFAULT_INDEX_SHARD_CHARS = 2_000_000; +// Fixed scope for bucketed vector shards. No generation component: bucket keys +// are deterministic and overwritten in place, so nothing can be stranded. +const VECTOR_BUCKET_SCOPE = `${KV.bm25Index}:vectors:v2`; +const DEFAULT_VECTOR_BUCKETS = 256; +const MAX_BUCKET_CHUNKS = 10_000; + type IndexShardManifest = { v: 1; generation?: string; @@ -24,11 +30,181 @@ type IndexShardManifest = { chars: number; }; +/** + * Bucketed vector manifest. + * + * `shards` maps bucket key -> content hash and chunk count, non-empty buckets + * only. + * + * Hashes live in the manifest, not process memory: restarts must not force a + * full rewrite. Buckets bound how much is rewritten; chunks bound the payload + * size, since a bucket grows with the corpus and the engine rejects an + * oversized `state::set`. + */ +// No `chars`: the content hash already proves both content and length. v1 +// needed a length check only because it had no hash to check against. +type VectorBucketEntry = { hash: string; chunks: number }; + +/** + * Version of the *addressing scheme* — which bucket an obsId lands in and which + * keys that bucket occupies. Bump it whenever that mapping changes, including a + * change to the bucket hash function or the key format. + * + * This exists because a content hash cannot see an addressing change: the bytes + * are identical, they just belong somewhere else now. Without this, such a + * change silently skips every write while the manifest claims new locations, + * and the vectors are lost on the next load. + * + * It is deliberately separate from `v`. `v` describes the manifest's shape, and + * keeping it stable is what lets a newer build still parse an older manifest + * well enough to reclaim the keys it named. Bumping `v` instead would make the + * old manifest unreadable and strand every key in it. + */ +// 3: bucket hashes are sha256. 2: chunk keys carry the bucket's content hash. +// 1 addressed them by index alone. A bucket written under an earlier layout is +// unreachable under a later one, which is precisely why the mismatch has to +// force a full rewrite instead of a skip. +const VECTOR_LAYOUT = 3; + +type VectorBucketManifest = { + v: 2; + layout: number; + buckets: number; + chunkChars: number; + shards: Record; + // Keys superseded by this manifest but not yet deleted. Published *with* the + // manifest so the work survives a crash: deletes run after the publish, and + // dying partway would otherwise strand the remainder with nothing able to + // name them. The next save drains whatever is left. + reclaim?: Array<{ scope: string; key: string }>; +}; + +// `layout` and `chunkChars` are not required here on purpose. A manifest +// missing them simply fails the layout comparison, which triggers a full +// rewrite and a full reclaim — the correct, self-healing response — whereas +// rejecting the manifest outright would strand the keys it names. +function isVectorBucketManifest(value: unknown): value is VectorBucketManifest { + if (!value || typeof value !== "object") return false; + const candidate = value as Partial; + return ( + candidate.v === 2 && + Number.isInteger(candidate.buckets) && + (candidate.buckets as number) >= 1 && + !!candidate.shards && + typeof candidate.shards === "object" + ); +} + +function isValidBucketEntry(value: unknown): value is VectorBucketEntry { + if (!value || typeof value !== "object") return false; + const candidate = value as Partial; + // chunks is the loop bound for both reclaim walks, so it needs a ceiling and + // not just an integer check. Number.isInteger(1e9) is true, and a single + // poisoned manifest field would otherwise queue a billion deletes. The cap is + // deliberately far above any real bucket: at the default 2M chunkChars this + // still allows a 20 GB bucket. + return ( + typeof candidate.hash === "string" && + candidate.hash.length > 0 && + Number.isInteger(candidate.chunks) && + (candidate.chunks as number) >= 1 && + (candidate.chunks as number) <= MAX_BUCKET_CHUNKS + ); +} + +function vectorBucketKey(bucket: number): string { + return `b${String(bucket).padStart(4, "0")}`; +} + +/** + * Chunk keys carry the bucket's content hash, so a bucket's bytes are never + * overwritten in place. + * + * This is what makes a torn write survivable. Writing in place looks safe + * because the manifest is published last, but it is not: the half-written + * bucket no longer matches the hash the *old* manifest recorded, load drops it, + * and nothing restores it — rebuild is gated on the BM25 index being empty + * (src/index.ts, src/functions/search.ts), never on the vector index. The next + * save then serialises the index without those vectors and persists the loss. + * + * Addressing by content instead means the old bucket stays readable until the + * new manifest names the new one, so a save that dies partway loses nothing. + * + * The 12 characters must be raw hex. A prefixed identifier would spend some of + * them on a constant and leave fewer distinguishing bits than the key needs. + */ +function vectorChunkKey( + bucketKey: string, + hash: string, + chunk: number, +): string { + return `${bucketKey}:${hash.slice(0, 12)}:${String(chunk).padStart(5, "0")}`; +} + +function isReclaimTarget( + value: unknown, +): value is { scope: string; key: string } { + if (!value || typeof value !== "object") return false; + const candidate = value as { scope?: unknown; key?: unknown }; + return ( + typeof candidate.scope === "string" && + candidate.scope.length > 0 && + typeof candidate.key === "string" && + candidate.key.length > 0 + ); +} + +function bucketChunkKeys(bucketKey: string, entry: VectorBucketEntry): string[] { + const keys: string[] = []; + for (let i = 0; i < entry.chunks; i++) { + keys.push(vectorChunkKey(bucketKey, entry.hash, i)); + } + return keys; +} + +// The hash of a bucket body. This one IS a content check — a collision means a +// changed bucket is silently never persisted — so it must not be the 32-bit +// hash used for bucket assignment. +const PUBLISHED_BUCKET_HASH = "sha256"; + +/** + * Every hash a stored bucket may legitimately carry, published one first. + * + * The set a load accepts is deliberately wider than the one a save publishes. + * A bucket's chunk keys are derived from the hash the manifest records, so a + * build that only accepts its own hash finds every chunk and then rejects every + * one of them — and a rejected bucket cannot be rewritten, because its contents + * never reach memory to be re-serialised. Widening the accepted set is what + * turns a change of published hash into an ordinary rewrite, and it is also + * what lets a rollback read a store the newer build wrote. + */ +const ACCEPTED_BUCKET_HASHES = [PUBLISHED_BUCKET_HASH, "sha1"]; + +function contentHash(value: string): string { + return createHash(PUBLISHED_BUCKET_HASH).update(value).digest("hex"); +} + +function bucketBodyMatches(body: string, expected: string): boolean { + return ACCEPTED_BUCKET_HASHES.some( + (algorithm) => createHash(algorithm).update(body).digest("hex") === expected, + ); +} + type IndexPersistenceOptions = { shardChars?: number; createGeneration?: () => string; + vectorBuckets?: number; }; +function vectorBucketCount(options: IndexPersistenceOptions): number { + const configured = options.vectorBuckets; + if (typeof configured !== "number" || !Number.isFinite(configured)) { + return DEFAULT_VECTOR_BUCKETS; + } + const whole = Math.floor(configured); + return whole >= 1 ? whole : DEFAULT_VECTOR_BUCKETS; +} + function shardChars(options: IndexPersistenceOptions): number { const configured = options.shardChars; if (typeof configured !== "number" || !Number.isFinite(configured)) { @@ -68,6 +244,29 @@ function isValidShardDescriptor( export class IndexPersistence { private timer: ReturnType | null = null; private lastFailureLogAt = 0; + /** + * Set when a vector load could not read everything the manifest named. + * + * `.catch(() => null)` on a read makes "I could not read this" look identical + * to "this does not exist", and reclaim treats absence as authority to + * delete. One timed-out read at boot would otherwise leave the in-memory + * index short, and the next save would serialise that gap and permanently + * delete the buckets behind it. While this is set, save preserves every + * unread bucket instead of reclaiming it. + */ + private vectorLoadIncomplete = false; + /** + * Set when every bucket the manifest named failed its content check. + * + * Unlike a failed read, this does not heal on its own: the bytes are intact + * and the keys are reachable, but no later load can verify them either. The + * in-memory index stays empty, so save has nothing to publish and preserves + * the manifest untouched — bytes and manifest in perfect agreement, vectors + * unreadable forever. Nothing else notices, because rebuild is gated on the + * BM25 index being empty and BM25 loaded fine. Reported so the caller can + * rebuild, which is the only thing that ends this state. + */ + private vectorLoadRejected = false; constructor( private kv: StateKV, @@ -92,11 +291,25 @@ export class IndexPersistence { clearTimeout(this.timer); this.timer = null; } + // Vectors first. This is the save that has been failing in production — it + // is the larger of the two and it runs second, so a BM25 save that consumes + // the engine's budget starves it. Going first also makes a vector delete + // durable before anything else can get in the way, which is what + // flushIndexSave is awaited for on the delete paths. + // + // Each index gets its own try. One try around both would let a failure in + // whichever runs first stop the other from persisting at all — harmless + // when BM25 led and vectors trailed, but reversing the order without this + // would make a vector failure silently block BM25 too. + if (this.vector) { + try { + await this.saveVectorBuckets(this.vector); + } catch (err) { + this.logFailure(err); + } + } try { await this.saveBm25Index(this.bm25.serialize()); - if (this.vector) { - await this.saveVectorIndex(this.vector.serialize()); - } } catch (err) { this.logFailure(err); } @@ -105,6 +318,7 @@ export class IndexPersistence { async load(): Promise<{ bm25: SearchIndex | null; vector: VectorIndex | null; + vectorRejected: boolean; }> { let bm25: SearchIndex | null = null; let vector: VectorIndex | null = null; @@ -114,12 +328,18 @@ export class IndexPersistence { bm25 = SearchIndex.deserialize(bm25Data); } - const vecData = await this.loadVectorData(); - if (vecData && typeof vecData === "string") { - vector = VectorIndex.deserialize(vecData); + // v2 (bucketed) is read directly into an index; the v1 offset-chunked + // path still exists so an upgrade loses nothing and a downgrade is only a + // rebuild, not data loss. + vector = await this.loadVectorBuckets(); + if (!vector) { + const vecData = await this.loadVectorData(); + if (vecData && typeof vecData === "string") { + vector = VectorIndex.deserialize(vecData); + } } - return { bm25, vector }; + return { bm25, vector, vectorRejected: this.vectorLoadRejected }; } stop(): void { @@ -157,13 +377,198 @@ export class IndexPersistence { ); } - private async saveVectorIndex(serialized: string): Promise { - await this.saveShardedIndex( - serialized, + /** + * Persist the vector index as fixed, independently-addressed buckets, + * writing only the buckets whose contents actually changed. + * + * Replaces an offset-chunked format that was O(entire corpus) per change; + * see the commit message for the production measurements. + * + * Cross-bucket atomicity is deliberately given up here. It buys nothing + * today, because the atomic save never completes. Recovery is convergent + * instead: a bucket written before a crash disagrees with the manifest that + * was never published, so the next save sees the hash mismatch and rewrites + * it. Vectors are immutable per obsId, so a surviving older bucket is still + * correct rather than stale. + */ + private async saveVectorBuckets(vector: VectorIndex): Promise { + // Deliberately unguarded. Swallowing a read failure here would report "no + // previous manifest", which is indistinguishable from a genuinely absent + // one — and that mistake strands the entire previous generation, because + // the manifest is the only thing that can name it. One transient engine + // timeout would be enough. Let it throw; runSave logs and the debounce + // retries in seconds. + const previous = await this.kv.get( + KV.bm25Index, VECTOR_MANIFEST_KEY, - VECTOR_KEY, - VECTOR_SHARD_SCOPE_PREFIX, ); + const buckets = vectorBucketCount(this.options); + const chunkChars = shardChars(this.options); + + const previousV2 = isVectorBucketManifest(previous) ? previous : null; + + // Every input that decides *where* a vector's bytes live. If any of them + // moved, a matching content hash proves nothing — the bytes are the same + // but their home is not — so nothing on disk may be skipped. + const layoutMatches = + !!previousV2 && + previousV2.layout === VECTOR_LAYOUT && + previousV2.buckets === buckets && + previousV2.chunkChars === chunkChars; + + // One map, used for two things: comparison (only when the layout matches) + // and reclaim (always). Reclaim must see the previous manifest even on a + // layout change, because those keys are exactly the ones about to become + // unreachable, and nothing can enumerate them afterwards. + const prior = previousV2?.shards ?? {}; + + const nextShards: Record = {}; + // Nothing is deleted until the manifest naming its replacement is live. + // Deleting first is how a failed publish destroys data the still-current + // manifest points at — the old manifest survives the failure, so anything + // it names has to survive with it. + const pendingDeletes: Array<{ scope: string; key: string }> = []; + + // Deletes a previous save published but died before finishing. Carrying + // them forward is the whole reason the list is in the manifest. + for (const target of previousV2?.reclaim ?? []) { + if (isReclaimTarget(target)) pendingDeletes.push(target); + } + let written = 0; + + for (const [bucket, body] of vector.serializeBuckets(buckets)) { + const bucketKey = vectorBucketKey(bucket); + const hash = contentHash(body); + const chunks = Math.max(1, Math.ceil(body.length / chunkChars)); + // The loader rejects anything above this cap, so publishing it would + // hand the next boot a bucket it drops as corrupt while its bytes sit + // on disk intact. Fail the save instead and keep the previous manifest + // live: a small configured chunkChars is a misconfiguration, not data + // loss, and the next save with a sane one succeeds. + if (chunks > MAX_BUCKET_CHUNKS) { + throw new Error( + `vector bucket ${bucketKey} needs ${chunks} chunks, above MAX_BUCKET_CHUNKS`, + ); + } + const priorEntry = isValidBucketEntry(prior[bucketKey]) + ? prior[bucketKey] + : undefined; + nextShards[bucketKey] = { hash, chunks }; + if (layoutMatches && priorEntry?.hash === hash) continue; + + for (let i = 0; i < chunks; i++) { + await this.kv.set( + VECTOR_BUCKET_SCOPE, + vectorChunkKey(bucketKey, hash, i), + body.slice(i * chunkChars, (i + 1) * chunkChars), + ); + } + written++; + // The bucket's old content lives under different keys entirely, so it is + // still intact right now and is reclaimed only after the publish. + if (priorEntry) { + for (const key of bucketChunkKeys(bucketKey, priorEntry)) { + pendingDeletes.push({ scope: VECTOR_BUCKET_SCOPE, key }); + } + } + } + + // Buckets the new manifest does not name: emptied since the last save, or + // left behind by a layout change that remapped them elsewhere. Either way + // this is the last moment anything can still name them. + for (const [bucketKey, priorEntry] of Object.entries(prior)) { + if (bucketKey in nextShards) continue; + if (!isValidBucketEntry(priorEntry)) continue; + if (this.vectorLoadIncomplete) { + // This bucket is missing from memory because we could not read it, not + // because anything was deleted. Keep naming it so it stays reachable + // and a later clean load can recover it. + nextShards[bucketKey] = priorEntry; + continue; + } + for (const key of bucketChunkKeys(bucketKey, priorEntry)) { + pendingDeletes.push({ scope: VECTOR_BUCKET_SCOPE, key }); + } + } + + // One-time migration off the offset-chunked format. A v1 manifest names its + // own shards, so this needs no enumeration — which matters, because + // StateKV.list returns values, not keys, and cannot be used here. + const legacy = previous as IndexShardManifest | null; + if (legacy?.v === 1 && Array.isArray(legacy.shards)) { + for (const shard of legacy.shards) { + if (isValidShardDescriptor(shard)) { + pendingDeletes.push({ scope: shard.scope, key: shard.key }); + } + } + } + + // Never delete a key the new manifest still names. Old and new keys can + // collide legitimately: identical content re-chunked keeps the same bucket + // and hash, so chunk 0's key is unchanged even though the old entry listed + // more chunks. Reclaiming that entry blind would delete a bucket the + // manifest is actively pointing at. + const liveKeys = new Set(); + for (const [bucketKey, entry] of Object.entries(nextShards)) { + for (const key of bucketChunkKeys(bucketKey, entry)) liveKeys.add(key); + } + const reclaimTargets = pendingDeletes.filter( + (target) => + !(target.scope === VECTOR_BUCKET_SCOPE && liveKeys.has(target.key)), + ); + + // Genuinely nothing to do. Republishing an identical manifest every + // debounce costs a write and two audit rows for no change. + if (written === 0 && reclaimTargets.length === 0 && previousV2) return; + + const nextManifest: VectorBucketManifest = { + v: 2, + layout: VECTOR_LAYOUT, + buckets, + chunkChars, + shards: nextShards, + ...(reclaimTargets.length > 0 ? { reclaim: reclaimTargets } : {}), + }; + await this.kv.set( + KV.bm25Index, + VECTOR_MANIFEST_KEY, + nextManifest, + ); + + // Safe from here: the new manifest is live and names none of these. + const undeleted: Array<{ scope: string; key: string }> = []; + for (const target of reclaimTargets) { + const gone = await this.deleteKey( + target.scope, + target.key, + "vector_bucket_reclaim", + ); + if (!gone) undeleted.push(target); + } + if (reclaimTargets.length > 0) { + // Record the drain, keeping anything that did not actually go. Clearing + // the list on a failed delete would strand those keys permanently, and + // not clearing it at all would let the list grow without bound. + const { reclaim: _drained, ...rest } = nextManifest; + await this.kv.set(KV.bm25Index, VECTOR_MANIFEST_KEY, { + ...rest, + ...(undeleted.length > 0 ? { reclaim: undeleted } : {}), + }); + } + + await this.auditIndexPersistence( + "vector_bucket_publish", + [statePath(KV.bm25Index, VECTOR_MANIFEST_KEY)], + { + manifestKey: VECTOR_MANIFEST_KEY, + buckets, + written, + unchanged: Object.keys(nextShards).length - written, + reclaimed: reclaimTargets.length, + }, + ); + + await this.deleteKey(KV.bm25Index, VECTOR_KEY, "legacy_cleanup"); } private async saveShardedIndex( @@ -280,11 +685,13 @@ export class IndexPersistence { ); } + /** Returns whether the key is actually gone. Callers that track reclaim work + * must keep a failed delete queued rather than reporting it done. */ private async deleteKey( scope: string, key: string, reason: string, - ): Promise { + ): Promise { let result = "deleted"; let error: string | undefined; try { @@ -300,6 +707,7 @@ export class IndexPersistence { result, error, }); + return result === "deleted"; } private async deleteShards( @@ -346,6 +754,92 @@ export class IndexPersistence { return this.loadShardedData(VECTOR_KEY, VECTOR_MANIFEST_KEY, "vector"); } + /** + * Read the bucketed vector index, or null when the store is still on the v1 + * format so the caller can fall back. + * + * A missing or unreadable bucket is skipped rather than failing the whole + * load. Unlike an offset-chunked shard — where a hole corrupts the single + * JSON document and the index must be rejected wholesale — each bucket is an + * independent document, so losing one costs exactly the vectors it held and + * the next save rewrites it. + */ + private async loadVectorBuckets(): Promise { + this.vectorLoadRejected = false; + let manifest: unknown; + try { + manifest = await this.kv.get(KV.bm25Index, VECTOR_MANIFEST_KEY); + } catch (err) { + // A read that failed is not a store that is empty. Falling through to the + // v1 path here finds nothing (v1 was migrated away), load returns a null + // vector index, and the next save reclaims every bucket the manifest + // still names. One transient timeout, whole index gone. + logger.warn("index persistence: vector manifest read failed", { + message: errorMessage(err), + }); + this.vectorLoadIncomplete = true; + return new VectorIndex(); + } + if (!isVectorBucketManifest(manifest)) return null; + + const entries = Object.entries(manifest.shards); + const index = new VectorIndex(); + let missing = 0; + let corrupt = 0; + for (const [bucketKey, entry] of entries) { + if (!isValidBucketEntry(entry)) { + corrupt++; + continue; + } + const parts: string[] = []; + let complete = true; + for (let i = 0; i < entry.chunks; i++) { + const chunk = await this.kv + .get( + VECTOR_BUCKET_SCOPE, + vectorChunkKey(bucketKey, entry.hash, i), + ) + .catch(() => null); + if (typeof chunk !== "string") { + complete = false; + break; + } + parts.push(chunk); + } + if (!complete) { + missing++; + continue; + } + const body = parts.join(""); + // Verifying the hash is what makes a torn write safe. A bucket half + // rewritten before a crash reassembles into invalid JSON; without this + // check it would be silently parsed as empty and its vectors lost with + // no signal. Skipping it instead costs only that bucket, and the next + // save rewrites it because the hash still disagrees. + if (!bucketBodyMatches(body, entry.hash)) { + corrupt++; + continue; + } + index.mergeSerialized(body); + } + // Deliberately narrower than `missing`. A read that failed may succeed on + // the next boot, and the existing preserve-and-wait policy covers it. A + // content check that failed will fail identically every time. + this.vectorLoadRejected = entries.length > 0 && corrupt === entries.length; + if (missing > 0 || corrupt > 0) { + // Anything not read is still on disk and still referenced. Mark the load + // incomplete so the next save preserves those buckets rather than + // treating their absence from memory as a deletion. + this.vectorLoadIncomplete = true; + logger.warn("index persistence: vector buckets skipped", { + missing, + corrupt, + total: entries.length, + }); + } + return index; + } + private async loadShardedData( legacyKey: string, manifestKey: string, @@ -364,6 +858,12 @@ export class IndexPersistence { // `manifest.v` with TypeError. Treat both null and undefined as // "no manifest" and fall through to the legacy path. The shape // check stays so a malformed-but-present row still fails closed. + // A bucketed manifest is handled by loadVectorBuckets, not here. Without + // this, a transient manifest-read failure that sends the vector load down + // the v1 path would report a perfectly valid manifest as invalid — the + // wrong signal to hand an operator during exactly the incident this format + // exists to fix. + if (isVectorBucketManifest(manifest.value)) return null; if ( manifest.value != null && typeof manifest.value === "object" diff --git a/src/state/vector-index.ts b/src/state/vector-index.ts index d4b8bda760..7e5b6543d6 100644 --- a/src/state/vector-index.ts +++ b/src/state/vector-index.ts @@ -20,6 +20,25 @@ function base64ToFloat32(b64: string): Float32Array { ); } +// FNV-1a. Only needs to spread obsIds evenly across buckets and be stable +// across processes — it is never a content check, so a non-cryptographic +// 32-bit hash is the right size. Math.imul keeps the multiply in int32. +function fnv1a32(str: string): number { + let hash = 0x811c9dc5; + for (let i = 0; i < str.length; i++) { + hash ^= str.charCodeAt(i); + hash = Math.imul(hash, 0x01000193); + } + return hash >>> 0; +} + +// Deterministic and stable across processes: bucket keys are reused in place, +// so nothing can be stranded. Changing this is an addressing change — bump +// VECTOR_LAYOUT in index-persistence.ts if you do. +function vectorBucketOf(obsId: string, bucketCount: number): number { + return fnv1a32(obsId) % bucketCount; +} + function cosineSimilarity(a: Float32Array, b: Float32Array): number { if (a.length !== b.length) return 0; let dot = 0; @@ -135,15 +154,68 @@ export class VectorIndex { return JSON.stringify(data); } - static deserialize(json: string): VectorIndex { - const idx = new VectorIndex(); + /** + * Serialise one bucket at a time, in the same row shape `serialize()` emits + * so both formats deserialise through one code path. + * + * A generator rather than a `Map` on purpose: materialising + * every bucket would hold the whole index as strings at once (~266 MB in + * production) inside an already memory-constrained process. Yielding lets the + * caller write and drop each bucket, so the peak is a single bucket. + * + * Grouping first costs one pass and holds ids only, not embeddings. + * Empty buckets are not yielded — the caller reconciles those against the + * previous manifest and deletes them. + */ + *serializeBuckets(bucketCount: number): Generator<[number, string]> { + // Snapshot entry references up front rather than reading the live Map as + // each bucket is yielded. + // + // The caller awaits a KV write between yields, and rebuildIndex() calls + // vectorIndex.clear() synchronously before its first await + // (src/functions/search.ts). A search that triggers a rebuild mid-save + // would therefore empty the Map underneath this loop, every remaining + // bucket would serialise to "[]", and those empty buckets would be hashed, + // written, and published as the bucket's true content — with the manifest + // and disk in perfect agreement, so the load-time hash check cannot see it. + // + // Holding references costs nothing: the Float32Arrays already exist and are + // not copied. Only one bucket's base64 is materialised at a time, which is + // the memory property that matters. + const groups = new Map< + number, + Array<[string, { embedding: Float32Array; sessionId: string }]> + >(); + for (const [obsId, entry] of this.vectors) { + const bucket = vectorBucketOf(obsId, bucketCount); + const rows = groups.get(bucket); + if (rows) rows.push([obsId, entry]); + else groups.set(bucket, [[obsId, entry]]); + } + for (const [bucket, rows] of groups) { + const serialised = rows.map( + ([obsId, entry]) => + [ + obsId, + { + embedding: float32ToBase64(entry.embedding), + sessionId: entry.sessionId, + }, + ] as [string, { embedding: string; sessionId: string }], + ); + yield [bucket, JSON.stringify(serialised)]; + } + } + + /** Merge serialised rows into this index. Malformed rows are skipped. */ + mergeSerialized(json: string): void { let data: unknown; try { data = JSON.parse(json); } catch { - return idx; + return; } - if (!Array.isArray(data)) return idx; + if (!Array.isArray(data)) return; for (const row of data) { try { if (!Array.isArray(row) || row.length < 2) continue; @@ -154,7 +226,7 @@ export class VectorIndex { typeof entry?.sessionId !== "string" ) continue; - idx.vectors.set(obsId, { + this.vectors.set(obsId, { embedding: base64ToFloat32(entry.embedding), sessionId: entry.sessionId, }); @@ -162,6 +234,11 @@ export class VectorIndex { continue; } } + } + + static deserialize(json: string): VectorIndex { + const idx = new VectorIndex(); + idx.mergeSerialized(json); return idx; } } diff --git a/test/index-persistence.test.ts b/test/index-persistence.test.ts index 929791657f..75d5a20666 100644 --- a/test/index-persistence.test.ts +++ b/test/index-persistence.test.ts @@ -1,4 +1,5 @@ import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"; +import { createHash } from "node:crypto"; import { IndexPersistence } from "../src/state/index-persistence.js"; import { SearchIndex } from "../src/state/search-index.js"; import { VectorIndex } from "../src/state/vector-index.js"; @@ -9,6 +10,23 @@ const BM25_LEGACY_KEY = "data"; const BM25_MANIFEST_KEY = "data:manifest"; const VECTOR_LEGACY_KEY = "vectors"; const VECTOR_MANIFEST_KEY = "vectors:manifest"; +const VECTOR_BUCKET_SCOPE = "mem:index:bm25:vectors:v2"; +// Mirrors the cap isValidBucketEntry enforces at load time. +const MAX_BUCKET_CHUNKS = 10_000; + +// Mirrors the production key builder. Chunk keys carry the bucket's content +// hash so a bucket is never overwritten in place. +function chunkKey(bucketKey: string, hash: string, chunk = 0): string { + return `${bucketKey}:${hash.slice(0, 12)}:${String(chunk).padStart(5, "0")}`; +} + +type TestVectorBucketManifest = { + v: 2; + layout: number; + buckets: number; + chunkChars: number; + shards: Record; +}; type TestIndexShardManifest = { v: 1; @@ -71,6 +89,14 @@ function makeVector(id = "obs_1"): VectorIndex { return vector; } +function seeded(count: number): VectorIndex { + const vector = new VectorIndex(); + for (let i = 0; i < count; i++) { + vector.add(`obs_${i}`, "ses_1", new Float32Array([i, i + 1, i + 2])); + } + return vector; +} + async function getBm25Manifest(kv: MockKV): Promise { const manifest = await kv.get( BM25_SCOPE, @@ -256,17 +282,24 @@ describe("IndexPersistence", () => { }); await persistence.save(); - const manifest = await kv.get( + const manifest = await kv.get( BM25_SCOPE, VECTOR_MANIFEST_KEY, ); expect(manifest).not.toBeNull(); - expect(manifest!.generation).toBe("gen_vector"); - expect(manifest!.shards.length).toBeGreaterThan(1); - expect(manifest!.shards[0].scope).toContain(":gen_vector:"); + expect(manifest!.v).toBe(2); + const bucketKeys = Object.keys(manifest!.shards); + expect(bucketKeys.length).toBe(1); + // shardChars: 40 against a 32-float embedding, so the bucket must still be + // chunked. Bucketing bounds the rewrite; chunking bounds the payload. + expect(manifest!.shards[bucketKeys[0]].chunks).toBeGreaterThan(1); await expect(kv.get(BM25_SCOPE, VECTOR_LEGACY_KEY)).resolves.toBeNull(); + // Vector payloads live outside the BM25 scope. await expect( - kv.get(manifest!.shards[0].scope, manifest!.shards[0].key), + kv.get( + VECTOR_BUCKET_SCOPE, + chunkKey(bucketKeys[0], manifest!.shards[bucketKeys[0]].hash), + ), ).resolves.toEqual(expect.any(String)); const loaded = await persistence.load(); @@ -289,12 +322,18 @@ describe("IndexPersistence", () => { createGeneration: () => "gen_empty", }).save(); - const vectorManifest = await kv.get( + const vectorManifest = await kv.get( BM25_SCOPE, VECTOR_MANIFEST_KEY, ); expect(vectorManifest).not.toBeNull(); - expect(vectorManifest!.generation).toBe("gen_empty"); + expect(vectorManifest!.v).toBe(2); + // The cleared index publishes a manifest naming no buckets, and the + // previous bucket is reclaimed rather than left to reload. + expect(Object.keys(vectorManifest!.shards)).toHaveLength(0); + await expect( + kv.list(VECTOR_BUCKET_SCOPE), + ).resolves.toHaveLength(0); const loaded = await new IndexPersistence( kv as never, new SearchIndex(), @@ -791,3 +830,871 @@ describe("IndexPersistence", () => { await expect(persistence.load()).resolves.toBeDefined(); }); }); + +// Counts writes landing on vector bucket payload keys, so assertions are about +// payload rewrites rather than manifest bookkeeping. +function countingKV() { + const inner = mockKV(); + let bucketWrites = 0; + return { + ...inner, + resetCount: () => { + bucketWrites = 0; + }, + get bucketWrites() { + return bucketWrites; + }, + set: async (scope: string, key: string, data: T): Promise => { + if (scope === VECTOR_BUCKET_SCOPE) bucketWrites++; + return inner.set(scope, key, data); + }, + }; +} + +describe("IndexPersistence vector bucketing", () => { + let kv: ReturnType; + + beforeEach(() => { + vi.useFakeTimers(); + kv = countingKV(); + }); + afterEach(() => vi.useRealTimers()); + + it("writes O(1) buckets when one vector is added, not O(corpus)", async () => { + const vector = seeded(60); + const persistence = new IndexPersistence(kv as never, new SearchIndex(), vector, { + shardChars: 400, + }); + + await persistence.save(); + expect(kv.bucketWrites).toBeGreaterThan(5); + + vector.add("obs_new", "ses_1", new Float32Array([9, 9, 9])); + kv.resetCount(); + await persistence.save(); + + expect(kv.bucketWrites).toBeLessThanOrEqual(2); + // Bounding the write count alone does not discriminate: skipping writes + // that were needed also lowers it. Read the result back. + const loaded = await persistence.load(); + expect(loaded.vector!.size).toBe(61); + expect( + loaded.vector!.search(new Float32Array([9, 9, 9]), 61) + .some((r) => r.obsId === "obs_new"), + ).toBe(true); + }); + + // Covers the same-instance case too: saveVectorBuckets holds no state between + // saves, so a fresh instance and a reused one run the identical path. + it("rewrites nothing after a restart when nothing changed", async () => { + const vector = seeded(40); + await new IndexPersistence(kv as never, new SearchIndex(), vector, { + shardChars: 400, + }).save(); + + // Go through the real restart path. Seeding a fresh identical index would + // skip load() entirely, leaving the thing that actually keeps hashes stable + // across a restart — deserialize -> mergeSerialized row order -> identical + // serialised bytes — completely unpinned. + const reloaded = await new IndexPersistence( + kv as never, + new SearchIndex(), + null, + ).load(); + expect(reloaded.vector!.size).toBe(40); + + kv.resetCount(); + const restarted = new IndexPersistence( + kv as never, + new SearchIndex(), + reloaded.vector, + { shardChars: 400 }, + ); + await restarted.save(); + + expect(kv.bucketWrites).toBe(0); + }); + + it("writes only the removed observation's bucket on delete", async () => { + const vector = seeded(60); + const persistence = new IndexPersistence(kv as never, new SearchIndex(), vector, { + shardChars: 400, + }); + await persistence.save(); + + // obs_9 shares a bucket with obs_30, so removing it forces that bucket to be + // REWRITTEN with its surviving member. Removing a bucket's only occupant + // instead would just reclaim it and never exercise a write at all, which is + // what this test is supposed to be about. + vector.remove("obs_9"); + kv.resetCount(); + await persistence.save(); + + expect(kv.bucketWrites).toBeLessThanOrEqual(1); + const loaded = await persistence.load(); + expect(loaded.vector!.size).toBe(59); + // The bucket-mate must survive the rewrite. + expect( + loaded.vector!.search(new Float32Array([30, 31, 32]), 59) + .some((r) => r.obsId === "obs_30"), + ).toBe(true); + }); + + it("round-trips every vector through the bucketed format", async () => { + const vector = seeded(50); + const persistence = new IndexPersistence(kv as never, new SearchIndex(), vector, { + shardChars: 400, + }); + await persistence.save(); + + const loaded = await persistence.load(); + expect(loaded.vector!.size).toBe(50); + expect( + loaded.vector!.search(new Float32Array([7, 8, 9]))[0]?.obsId, + ).toBe("obs_7"); + }); + + it("bounds each payload by shardChars even when a bucket is large", async () => { + // One bucket, many vectors in it: only chunking can bound the payload. + const vector = new VectorIndex(); + for (let i = 0; i < 40; i++) { + vector.add( + `obs_${i}`, + "ses_1", + new Float32Array(Array.from({ length: 32 }, (_, n) => n + i)), + ); + } + const maxChars = 120; + const guarded = { + ...kv, + set: async (scope: string, key: string, data: T): Promise => { + if (typeof data === "string" && data.length > maxChars) { + throw new Error(`oversized state::set payload: ${scope}/${key}`); + } + return kv.set(scope, key, data); + }, + }; + await new IndexPersistence(guarded as never, new SearchIndex(), vector, { + shardChars: maxChars, + vectorBuckets: 1, + }).save(); + + const loaded = await new IndexPersistence( + kv as never, + new SearchIndex(), + null, + ).load(); + expect(loaded.vector!.size).toBe(40); + }); + + it("loads a v1 manifest and migrates it to buckets on the next save", async () => { + // Write the legacy offset-chunked format by hand, exactly as the v1 code + // shaped it, then prove no vectors are lost across the upgrade. + const legacy = seeded(20); + const serialized = legacy.serialize(); + const shards: Array<{ scope: string; key: string; chars: number }> = []; + const chunkChars = 200; + for (let offset = 0; offset < serialized.length; offset += chunkChars) { + const scope = `mem:index:bm25:vectors:gen_v1:${String( + shards.length, + ).padStart(5, "0")}`; + const chunk = serialized.slice(offset, offset + chunkChars); + shards.push({ scope, key: "data", chars: chunk.length }); + await kv.set(scope, "data", chunk); + } + await kv.set(BM25_SCOPE, VECTOR_MANIFEST_KEY, { + v: 1, + generation: "gen_v1", + shards, + chars: serialized.length, + }); + + // Load path still understands v1. + const before = await new IndexPersistence( + kv as never, + new SearchIndex(), + null, + ).load(); + expect(before.vector!.size).toBe(20); + + // First save migrates and reclaims the v1 shards. + await new IndexPersistence(kv as never, new SearchIndex(), seeded(20), { + shardChars: 400, + }).save(); + + const manifest = await kv.get( + BM25_SCOPE, + VECTOR_MANIFEST_KEY, + ); + expect(manifest!.v).toBe(2); + for (const shard of shards) { + await expect(kv.get(shard.scope, shard.key)).resolves.toBeNull(); + } + const after = await new IndexPersistence( + kv as never, + new SearchIndex(), + null, + ).load(); + expect(after.vector!.size).toBe(20); + }); + + it("keeps previous buckets loadable when the manifest publish fails", async () => { + const vector = seeded(30); + await new IndexPersistence(kv as never, new SearchIndex(), vector, { + shardChars: 400, + }).save(); + + const failing = { + ...kv, + set: async (scope: string, key: string, data: T): Promise => { + if (scope === BM25_SCOPE && key === VECTOR_MANIFEST_KEY) { + throw new Error("vector manifest write failed"); + } + return kv.set(scope, key, data); + }, + }; + // Empty index: without deferring deletes until after publish, this would + // reclaim every bucket the still-live manifest names and lose the lot. + await new IndexPersistence( + failing as never, + new SearchIndex(), + new VectorIndex(), + { shardChars: 400 }, + ).save(); + + const loaded = await new IndexPersistence( + kv as never, + new SearchIndex(), + null, + ).load(); + expect(loaded.vector!.size).toBe(30); + }); + + it("skips a bucket whose body disagrees with its manifest hash", async () => { + const vector = seeded(30); + // Large shardChars keeps every bucket to a single chunk, so the stand-in + // body below is the bucket in full rather than a fragment of one. + const persistence = new IndexPersistence(kv as never, new SearchIndex(), vector, { + shardChars: 100_000, + }); + await persistence.save(); + + const manifest = await kv.get( + BM25_SCOPE, + VECTOR_MANIFEST_KEY, + ); + const [tornKey] = Object.keys(manifest!.shards); + const tornHash = manifest!.shards[tornKey].hash; + + // Critically, this stand-in is VALID JSON in the right row shape. An + // invalid body would be dropped by mergeSerialized regardless, so the test + // would pass with the hash check removed and prove nothing. Only content + // that would otherwise load cleanly can show the hash check doing work. + const ghost = JSON.stringify([ + [ + "obs_ghost", + { + embedding: Buffer.from( + new Float32Array([1, 2, 3]).buffer, + ).toString("base64"), + sessionId: "ses_1", + }, + ], + ]); + await kv.set(VECTOR_BUCKET_SCOPE, chunkKey(tornKey, tornHash), ghost); + + const loaded = await new IndexPersistence( + kv as never, + new SearchIndex(), + null, + ).load(); + // The disagreeing bucket is dropped whole rather than trusted. + expect( + loaded.vector!.search(new Float32Array([1, 2, 3]), 50) + .some((r) => r.obsId === "obs_ghost"), + ).toBe(false); + // Every other bucket still loads. + expect(loaded.vector!.size).toBeGreaterThan(0); + expect(loaded.vector!.size).toBeLessThan(30); + }); + + // isValidBucketEntry rejects any entry above MAX_BUCKET_CHUNKS, so a save + // that publishes one hands the next boot a bucket it classifies as corrupt — + // dropping vectors whose bytes are entirely intact, and marking every load + // incomplete from then on. A small configured shardChars is all it takes. + it("fails the save rather than publish a bucket the loader will reject", async () => { + const vector = seeded(200); + // Asserted, not assumed: at one chunk per character the bucket has to clear + // the cap, or a shorter serialisation would leave this testing nothing. + const [, body] = [...vector.serializeBuckets(1)][0]!; + expect(body.length).toBeGreaterThan(MAX_BUCKET_CHUNKS); + + // A good save first, so there is something to lose. + await new IndexPersistence(kv as never, new SearchIndex(), vector, { + shardChars: 400, + vectorBuckets: 1, + }).save(); + + // save() funnels the failure through logFailure, so nothing throws out + // here. What has to hold is that the live manifest still names a readable + // bucket rather than an oversized one the loader will throw away. + await new IndexPersistence(kv as never, new SearchIndex(), vector, { + shardChars: 1, + vectorBuckets: 1, + }).save(); + + const loaded = await new IndexPersistence( + kv as never, + new SearchIndex(), + null, + ).load(); + expect(loaded.vector!.size).toBe(200); + }); +}); + +// A content hash cannot see an addressing change: same bytes, different home. +// These two cover that whole family — the bucket count and the chunk size are +// the two inputs that decide where a vector's bytes live. +describe("IndexPersistence vector layout changes", () => { + let kv: ReturnType; + + beforeEach(() => { + vi.useFakeTimers(); + kv = countingKV(); + }); + afterEach(() => vi.useRealTimers()); + + it("reclaims buckets stranded by a smaller bucket count", async () => { + await new IndexPersistence(kv as never, new SearchIndex(), seeded(40), { + shardChars: 400, + vectorBuckets: 8, + }).save(); + + const wide = await kv.get( + BM25_SCOPE, + VECTOR_MANIFEST_KEY, + ); + const widened = Object.keys(wide!.shards); + // Needs keys that only the 8-bucket layout can produce. + const beyondNarrow = widened.filter((key) => Number(key.slice(1)) >= 4); + expect(beyondNarrow.length).toBeGreaterThan(0); + + await new IndexPersistence(kv as never, new SearchIndex(), seeded(40), { + shardChars: 400, + vectorBuckets: 4, + }).save(); + + // Every key the old layout owned and the new one cannot name must be gone. + for (const bucketKey of beyondNarrow) { + await expect( + kv.get( + VECTOR_BUCKET_SCOPE, + chunkKey(bucketKey, wide!.shards[bucketKey].hash), + ), + ).resolves.toBeNull(); + } + const narrow = await kv.get( + BM25_SCOPE, + VECTOR_MANIFEST_KEY, + ); + expect(narrow!.buckets).toBe(4); + const loaded = await new IndexPersistence( + kv as never, + new SearchIndex(), + null, + ).load(); + expect(loaded.vector!.size).toBe(40); + }); + + it("survives a chunk-size change that leaves content identical", async () => { + // Small chunks first, so buckets span several keys. + await new IndexPersistence(kv as never, new SearchIndex(), seeded(30), { + shardChars: 60, + vectorBuckets: 4, + }).save(); + + const wide = await kv.get( + BM25_SCOPE, + VECTOR_MANIFEST_KEY, + ); + const multiChunk = Object.entries(wide!.shards).find( + ([, entry]) => entry.chunks > 1, + ); + expect(multiChunk).toBeDefined(); + const [shrinkingKey, wideEntry] = multiChunk!; + + // Same vectors, larger chunks. The bucket bodies are byte-identical, so the + // content hash matches and a hash-only skip would write nothing at all — + // while the manifest records the new, smaller chunk count. Load would then + // read too few keys and drop every bucket. + await new IndexPersistence(kv as never, new SearchIndex(), seeded(30), { + shardChars: 100_000, + vectorBuckets: 4, + }).save(); + + // The bucket now needs fewer keys, so its old tail must be reclaimed + // rather than left addressable-by-nobody. + // Chunk 0 keeps its key: same bucket, same content hash, same index. Only + // the tail is no longer addressed, and reclaiming it must not take chunk 0 + // with it. + await expect( + kv.get(VECTOR_BUCKET_SCOPE, chunkKey(shrinkingKey, wideEntry.hash, 0)), + ).resolves.toEqual(expect.any(String)); + for (let i = 1; i < wideEntry.chunks; i++) { + await expect( + kv.get(VECTOR_BUCKET_SCOPE, chunkKey(shrinkingKey, wideEntry.hash, i)), + ).resolves.toBeNull(); + } + + const loaded = await new IndexPersistence( + kv as never, + new SearchIndex(), + null, + ).load(); + expect(loaded.vector!.size).toBe(30); + expect( + loaded.vector!.search(new Float32Array([7, 8, 9]))[0]?.obsId, + ).toBe("obs_7"); + }); +}); + +describe("IndexPersistence torn vector save", () => { + let kv: ReturnType; + + beforeEach(() => { + vi.useFakeTimers(); + kv = countingKV(); + }); + afterEach(() => vi.useRealTimers()); + + it("loses nothing when a save dies partway through writing buckets", async () => { + await new IndexPersistence(kv as never, new SearchIndex(), seeded(40), { + shardChars: 400, + vectorBuckets: 8, + }).save(); + + const before = await new IndexPersistence( + kv as never, + new SearchIndex(), + null, + ).load(); + expect(before.vector!.size).toBe(40); + + // A second save with changed content that dies after some bucket writes and + // before the manifest publish. With in-place keys this would corrupt the + // buckets the live manifest still names, load would drop them, and — since + // rebuild is gated on BM25 being empty, never on the vector index — the + // next save would serialise the absence and lose them for good. + const changed = seeded(40); + for (let i = 0; i < 40; i++) { + changed.add(`obs_${i}`, "ses_1", new Float32Array([i + 99, i, i])); + } + let writes = 0; + const dying = { + ...kv, + set: async (scope: string, key: string, data: T): Promise => { + if (scope === VECTOR_BUCKET_SCOPE && ++writes > 3) { + throw new Error("engine died mid-save"); + } + return kv.set(scope, key, data); + }, + }; + await new IndexPersistence(dying as never, new SearchIndex(), changed, { + shardChars: 400, + vectorBuckets: 8, + }).save(); + + // The old generation is still fully intact and loadable. + const after = await new IndexPersistence( + kv as never, + new SearchIndex(), + null, + ).load(); + expect(after.vector!.size).toBe(40); + expect( + after.vector!.search(new Float32Array([7, 8, 9]), 50)[0]?.obsId, + ).toBe("obs_7"); + }); + + it("resumes a reclaim that was published but never finished", async () => { + await new IndexPersistence(kv as never, new SearchIndex(), seeded(40), { + shardChars: 400, + vectorBuckets: 8, + }).save(); + + // Save again with different content, failing every delete so the reclaim + // list is published but never drained. + const changed = seeded(40); + changed.add("obs_extra", "ses_1", new Float32Array([5, 5, 5])); + const undeletable = { + ...kv, + delete: async (): Promise => { + throw new Error("delete unavailable"); + }, + }; + await new IndexPersistence(undeletable as never, new SearchIndex(), changed, { + shardChars: 400, + vectorBuckets: 8, + }).save(); + + const stalled = await kv.get; + }>(BM25_SCOPE, VECTOR_MANIFEST_KEY); + expect(stalled!.reclaim!.length).toBeGreaterThan(0); + const stranded = stalled!.reclaim!.map((t) => t.key); + + // A later healthy save must finish the job rather than orphan those keys. + await new IndexPersistence(kv as never, new SearchIndex(), changed, { + shardChars: 400, + vectorBuckets: 8, + }).save(); + + for (const key of stranded) { + await expect(kv.get(VECTOR_BUCKET_SCOPE, key)).resolves.toBeNull(); + } + const loaded = await new IndexPersistence( + kv as never, + new SearchIndex(), + null, + ).load(); + expect(loaded.vector!.size).toBe(41); + }); +}); + +// A failed read is not an empty store, and a mid-save mutation is not a +// deletion. Both were measured to destroy data before these guards existed. +describe("IndexPersistence vector save/load hazards", () => { + let kv: ReturnType; + + beforeEach(() => { + vi.useFakeTimers(); + kv = countingKV(); + }); + afterEach(() => vi.useRealTimers()); + + async function loadedSize(): Promise { + const loaded = await new IndexPersistence( + kv as never, + new SearchIndex(), + null, + ).load(); + return loaded.vector?.size ?? 0; + } + + it("does not persist empty buckets when the index is cleared mid-save", async () => { + const vector = seeded(60); + // rebuildIndex() calls vectorIndex.clear() synchronously before its first + // await, so a search-triggered rebuild lands between two bucket writes. + let clearedAt = 0; + const clearing = { + ...kv, + set: async (scope: string, key: string, data: T): Promise => { + if (scope === VECTOR_BUCKET_SCOPE && ++clearedAt === 2) vector.clear(); + return kv.set(scope, key, data); + }, + }; + + await new IndexPersistence(clearing as never, new SearchIndex(), vector, { + shardChars: 400, + vectorBuckets: 8, + }).save(); + + // Reading the live Map lazily would serialise every later bucket as "[]", + // hash it, publish it as truth, and reclaim the real content behind it. + expect(await loadedSize()).toBe(60); + }); + + it("does not wipe the index when the manifest read fails at load", async () => { + await new IndexPersistence(kv as never, new SearchIndex(), seeded(60), { + shardChars: 400, + vectorBuckets: 8, + }).save(); + + const blindKv = { + ...kv, + get: async (scope: string, key: string): Promise => { + if (scope === BM25_SCOPE && key === VECTOR_MANIFEST_KEY) { + throw new Error("manifest read timed out"); + } + return kv.get(scope, key); + }, + }; + // Same instance loads then saves, exactly as src/index.ts does. + const persistence = new IndexPersistence( + blindKv as never, + new SearchIndex(), + new VectorIndex(), + { shardChars: 400, vectorBuckets: 8 }, + ); + await persistence.load(); + await persistence.save(); + + expect(await loadedSize()).toBe(60); + }); + + it("does not delete a bucket whose chunk read failed at load", async () => { + await new IndexPersistence(kv as never, new SearchIndex(), seeded(60), { + shardChars: 400, + vectorBuckets: 8, + }).save(); + + const manifest = await kv.get( + BM25_SCOPE, + VECTOR_MANIFEST_KEY, + ); + const [victim] = Object.keys(manifest!.shards); + const victimChunk = chunkKey(victim, manifest!.shards[victim].hash, 0); + + const flakyKv = { + ...kv, + get: async (scope: string, key: string): Promise => { + if (scope === VECTOR_BUCKET_SCOPE && key === victimChunk) { + throw new Error("chunk read timed out"); + } + return kv.get(scope, key); + }, + }; + const persistence = new IndexPersistence( + flakyKv as never, + new SearchIndex(), + new VectorIndex(), + { shardChars: 400, vectorBuckets: 8 }, + ); + const partial = await persistence.load(); + expect(partial.vector!.size).toBeLessThan(60); + + // The blip must not be converted into a permanent delete. + await persistence.save(); + expect(await loadedSize()).toBe(60); + }); +}); + +// Every other test in this file writes and reads through the same build, so the +// suite is self-consistent under ANY addressing change and cannot see one. This +// is the only cross-version fixture: it stands in for "a store written by an +// older layout" and pins the guard that forces a rewrite instead of a skip. +describe("IndexPersistence vector layout version", () => { + let kv: ReturnType; + + beforeEach(() => { + vi.useFakeTimers(); + kv = countingKV(); + }); + afterEach(() => vi.useRealTimers()); + + it("rewrites everything when the stored layout version differs", async () => { + const vector = seeded(20); + await new IndexPersistence(kv as never, new SearchIndex(), vector, { + shardChars: 400, + }).save(); + + // Rewrite the manifest as an older layout while KEEPING every content hash, + // then drop the payload keys. Same content, different addressing — exactly + // the shape a content hash cannot detect on its own. A build that trusts the + // hash alone skips every write and then cannot find its own data. + const current = await kv.get( + BM25_SCOPE, + VECTOR_MANIFEST_KEY, + ); + await kv.set(BM25_SCOPE, VECTOR_MANIFEST_KEY, { + ...current!, + layout: current!.layout - 1, + }); + for (const [bucketKey, entry] of Object.entries(current!.shards)) { + for (let i = 0; i < entry.chunks; i++) { + await kv.delete(VECTOR_BUCKET_SCOPE, chunkKey(bucketKey, entry.hash, i)); + } + } + + await new IndexPersistence(kv as never, new SearchIndex(), seeded(20), { + shardChars: 400, + }).save(); + + const loaded = await new IndexPersistence( + kv as never, + new SearchIndex(), + null, + ).load(); + expect(loaded.vector!.size).toBe(20); + }); +}); + +// Stands in for a store written by a build whose contentHash was a different +// function: same bodies, same manifest shape, chunk keys still derived from the +// hash the manifest records — so every chunk is found and every hash fails. +async function rehashStoreUnder( + kv: ReturnType, + algorithm: string, +): Promise { + const manifest = (await kv.get( + BM25_SCOPE, + VECTOR_MANIFEST_KEY, + ))!; + const shards: TestVectorBucketManifest["shards"] = {}; + for (const [bucketKey, entry] of Object.entries(manifest.shards)) { + const parts: string[] = []; + for (let i = 0; i < entry.chunks; i++) { + parts.push( + (await kv.get( + VECTOR_BUCKET_SCOPE, + chunkKey(bucketKey, entry.hash, i), + ))!, + ); + } + const rehashed = createHash(algorithm) + .update(parts.join("")) + .digest("hex"); + for (let i = 0; i < entry.chunks; i++) { + await kv.delete(VECTOR_BUCKET_SCOPE, chunkKey(bucketKey, entry.hash, i)); + await kv.set( + VECTOR_BUCKET_SCOPE, + chunkKey(bucketKey, rehashed, i), + parts[i], + ); + } + shards[bucketKey] = { hash: rehashed, chunks: entry.chunks }; + } + await kv.set(BM25_SCOPE, VECTOR_MANIFEST_KEY, { ...manifest, shards }); +} + +describe("IndexPersistence vector hash-format change", () => { + let kv: ReturnType; + + beforeEach(() => { + vi.useFakeTimers(); + kv = countingKV(); + }); + afterEach(() => vi.useRealTimers()); + + // Both hashes a bucket may legitimately carry. Whichever one this build + // publishes, the other stands for a store written either side of the + // migration — an older build's, or a newer build's after a rollback. + for (const algorithm of ["sha1", "sha256"]) { + it(`reads a bucket whose manifest records a ${algorithm} hash`, async () => { + await new IndexPersistence(kv as never, new SearchIndex(), seeded(20), { + shardChars: 400, + }).save(); + + await rehashStoreUnder(kv, algorithm); + + const loaded = await new IndexPersistence( + kv as never, + new SearchIndex(), + null, + ).load(); + expect(loaded.vector!.size).toBe(20); + expect(loaded.vectorRejected).toBe(false); + }); + } + + it("migrates a store off the previous bucket hash without re-embedding", async () => { + await new IndexPersistence(kv as never, new SearchIndex(), seeded(20), { + shardChars: 400, + }).save(); + + // Stands in for a store this branch's earlier builds wrote: sha1 bucket + // hashes, under the layout that published them. + await rehashStoreUnder(kv, "sha1"); + const current = (await kv.get( + BM25_SCOPE, + VECTOR_MANIFEST_KEY, + ))!; + await kv.set(BM25_SCOPE, VECTOR_MANIFEST_KEY, { + ...current, + layout: current.layout - 1, + }); + const stale = (await kv.get( + BM25_SCOPE, + VECTOR_MANIFEST_KEY, + ))!; + + const live = new VectorIndex(); + const persistence = new IndexPersistence( + kv as never, + new SearchIndex(), + live, + { shardChars: 400 }, + ); + const loaded = await persistence.load(); + expect(loaded.vector!.size).toBe(20); + live.restoreFrom(loaded.vector!); + + await persistence.save(); + + const migrated = (await kv.get( + BM25_SCOPE, + VECTOR_MANIFEST_KEY, + ))!; + expect(migrated.layout).toBe(stale.layout + 1); + for (const entry of Object.values(migrated.shards)) { + // sha256 hex. sha1 would be 40. + expect(entry.hash).toHaveLength(64); + } + for (const [bucketKey, entry] of Object.entries(stale.shards)) { + for (let i = 0; i < entry.chunks; i++) { + expect( + await kv.get(VECTOR_BUCKET_SCOPE, chunkKey(bucketKey, entry.hash, i)), + ).toBeNull(); + } + } + + const reopened = await new IndexPersistence( + kv as never, + new SearchIndex(), + null, + ).load(); + expect(reopened.vector!.size).toBe(20); + }); + + it("recovers when a hash change makes every bucket unverifiable", async () => { + await new IndexPersistence(kv as never, new SearchIndex(), seeded(20), { + shardChars: 400, + }).save(); + + await rehashStoreUnder(kv, "sha512"); + const alien = (await kv.get( + BM25_SCOPE, + VECTOR_MANIFEST_KEY, + ))!; + + // Boot. The live index starts empty and only ever holds what load hands it. + const live = new VectorIndex(); + const persistence = new IndexPersistence( + kv as never, + new SearchIndex(), + live, + { shardChars: 400 }, + ); + const loaded = await persistence.load(); + if (loaded.vector && loaded.vector.size > 0) live.restoreFrom(loaded.vector); + expect(live.size).toBe(0); + + // A debounce flush can land long before any rebuild finishes. It must not + // turn "could not verify" into a delete. + await persistence.save(); + for (const [bucketKey, entry] of Object.entries(alien.shards)) { + for (let i = 0; i < entry.chunks; i++) { + expect( + await kv.get( + VECTOR_BUCKET_SCOPE, + chunkKey(bucketKey, entry.hash, i), + ), + ).not.toBeNull(); + } + } + + // Stands in for rebuildIndex(), which src/index.ts fires on this signal and + // which refills the live index before scheduling a save. Nothing else ever + // re-reads those buckets, so without the signal the vectors stay on disk + // and unreadable for the life of the store. + if (loaded.vectorRejected) live.restoreFrom(seeded(20)); + await persistence.save(); + + const reopened = await new IndexPersistence( + kv as never, + new SearchIndex(), + null, + ).load(); + expect(reopened.vector!.size).toBe(20); + }); +});