diff --git a/.changeset/skip-unchanged-reindex.md b/.changeset/skip-unchanged-reindex.md new file mode 100644 index 0000000..683f23b --- /dev/null +++ b/.changeset/skip-unchanged-reindex.md @@ -0,0 +1,9 @@ +--- +"@chendpoc/pi-memory": patch +--- + +Skip re-embedding unchanged memory files on reindex, and stop blocking session startup on the initial reindex. + +Sidecar `reindex` now computes a content fingerprint (sha256 over sorted chunk ids + contents + embedder identity) and short-circuits when it matches the last full reindex, returning `skipped: "fingerprint_match"` instead of re-embedding every chunk. The first `runNow()` in `bootstrapSidecar` is fire-and-forget so `session_start` no longer waits on a full re-embed; preflight already falls back to MEMORY.md while the index rebuilds. + +Measured with a 128-chunk memory file and the ollama `bge-m3` embedder: repeated reindex calls went from ~7.3-7.8s each to ~2ms, and session startup cost attributable to pi-memory dropped by roughly 7s per attach / new chat. diff --git a/src/extension/lifecycle.ts b/src/extension/lifecycle.ts index ba5638a..531371d 100644 --- a/src/extension/lifecycle.ts +++ b/src/extension/lifecycle.ts @@ -46,7 +46,11 @@ export async function bootstrapSidecar(opts: { }); const unsubSyncToSidecar = opts.store.onSyncToSidecar(() => reindexScheduler.schedule()); - await reindexScheduler.runNow(); + // Fire-and-forget: the initial runNow re-embeds the whole memory file when it + // changed, which with a remote embedder can take seconds. Session startup + // (session_start) must not block on it — preflight already falls back to + // MEMORY.md while the index is being rebuilt. + void reindexScheduler.runNow(); return { reindexScheduler, unsubSyncToSidecar }; } diff --git a/src/sidecar/protocol.ts b/src/sidecar/protocol.ts index da919e4..db77010 100644 --- a/src/sidecar/protocol.ts +++ b/src/sidecar/protocol.ts @@ -30,7 +30,13 @@ export type SidecarResponse = | { type: "pong" } | { type: "stats_ok" } & IndexStats | { type: "result"; request_id: string; results: MemoryEntry[] } - | { type: "reindex_ok"; request_id: string; indexed: number; index_generation: number } + | { + type: "reindex_ok"; + request_id: string; + indexed: number; + index_generation: number; + skipped?: "fingerprint_match"; + } | { type: "error"; request_id?: string; error: string }; export function isErrorResponse(res: SidecarResponse): res is Extract { diff --git a/src/sidecar/server/reindex.ts b/src/sidecar/server/reindex.ts index fd8d48f..d496182 100644 --- a/src/sidecar/server/reindex.ts +++ b/src/sidecar/server/reindex.ts @@ -1,4 +1,8 @@ +import { createHash } from "node:crypto"; + import type { IndexDocument, SidecarResponse } from "../protocol.js"; +import { computeDocumentsFingerprint } from "./vec/fingerprint.js"; +import { getEmbedder } from "./vec/embedder.js"; import { getVecStore } from "./vec/store.js"; export type ReindexContext = { @@ -11,7 +15,23 @@ export async function handleReindex( documents: IndexDocument[] = [], ): Promise> { const store = getVecStore(ctx.dbPath); + // Skip re-embedding when the document set is unchanged since the last full + // reindex (same ids + contents + embedder identity). Session startup calls + // reindex on every attach; re-embedding an unchanged memory file dominated + // attach latency with remote embedders (e.g. ~7s for 128 chunks via ollama). + const embedder = getEmbedder(); + const fingerprint = computeDocumentsFingerprint(documents, embedder); + if (documents.length > 0 && store.getLastReindexFingerprint() === fingerprint) { + return { + type: "reindex_ok", + request_id: requestId, + indexed: 0, + index_generation: store.getIndexGeneration(), + skipped: "fingerprint_match", + }; + } const outcome = await store.reindex(documents); + store.setLastReindexFingerprint(fingerprint); return { type: "reindex_ok", request_id: requestId, diff --git a/src/sidecar/server/vec/fingerprint.ts b/src/sidecar/server/vec/fingerprint.ts new file mode 100644 index 0000000..a52fad1 --- /dev/null +++ b/src/sidecar/server/vec/fingerprint.ts @@ -0,0 +1,26 @@ +import { createHash } from "node:crypto"; + +import type { IndexDocument } from "../../protocol.js"; +import type { Embedder } from "../../../adapters/embed/types.js"; +import { getLastReindexFingerprint, setLastReindexFingerprint } from "./schema.js"; + +/** + * Content fingerprint of a document set: sha256 over sorted (id, content) pairs + * plus the embedder identity. Any change to ids, contents, or the embedder + * (model/provider/dim) produces a different fingerprint, so a match means the + * stored chunks + embeddings already reflect exactly this input. + */ +export function computeDocumentsFingerprint(documents: IndexDocument[], embedder: Embedder): string { + const hash = createHash("sha256"); + const sorted = [...documents].sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0)); + for (const doc of sorted) { + hash.update(doc.id); + hash.update("\u0000"); + hash.update(doc.content); + hash.update("\u0000"); + } + hash.update(`${embedder.provider}\u0000${embedder.model}\u0000${String(embedder.dim)}`); + return hash.digest("hex"); +} + +export { getLastReindexFingerprint, setLastReindexFingerprint }; diff --git a/src/sidecar/server/vec/schema.ts b/src/sidecar/server/vec/schema.ts index 69d05ab..228869a 100644 --- a/src/sidecar/server/vec/schema.ts +++ b/src/sidecar/server/vec/schema.ts @@ -84,3 +84,19 @@ export function writeEmbeddingMeta(db: VecDatabase, embedder: Embedder): void { upsert.run("embedding_provider", embedder.provider); upsert.run("embedding_dim", String(embedder.dim)); } + +const LAST_REINDEX_FINGERPRINT_KEY = "last_reindex_fingerprint"; + +export function getLastReindexFingerprint(db: VecDatabase): string | null { + const raw = db.prepare("SELECT value FROM meta WHERE key = ?").pluck().get(LAST_REINDEX_FINGERPRINT_KEY) as + | string + | undefined; + return raw ?? null; +} + +export function setLastReindexFingerprint(db: VecDatabase, fingerprint: string): void { + db.prepare("INSERT INTO meta(key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value").run( + LAST_REINDEX_FINGERPRINT_KEY, + fingerprint, + ); +} diff --git a/src/sidecar/server/vec/store.ts b/src/sidecar/server/vec/store.ts index 924c1b9..80b0d83 100644 --- a/src/sidecar/server/vec/store.ts +++ b/src/sidecar/server/vec/store.ts @@ -9,8 +9,10 @@ import { embeddingMetaMatches, getChunkCount, getIndexGeneration, + getLastReindexFingerprint, getStoredEmbeddingMeta, initVecSchema, + setLastReindexFingerprint, type ReindexOutcome, type StoredEmbeddingMeta, } from "./schema.js"; @@ -41,6 +43,14 @@ export class VecStore { return getIndexGeneration(this.db); } + getLastReindexFingerprint(): string | null { + return getLastReindexFingerprint(this.db); + } + + setLastReindexFingerprint(fingerprint: string): void { + setLastReindexFingerprint(this.db, fingerprint); + } + getChunkCount(): number { return getChunkCount(this.db); } diff --git a/test/reindexFingerprint.test.ts b/test/reindexFingerprint.test.ts new file mode 100644 index 0000000..dee8782 --- /dev/null +++ b/test/reindexFingerprint.test.ts @@ -0,0 +1,113 @@ +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; + +import { resetEmbedderForTests } from "../src/sidecar/server/vec/embedder.js"; +import { resetVecStoreForTests } from "../src/sidecar/server/vec/store.js"; +import { computeDocumentsFingerprint } from "../src/sidecar/server/vec/fingerprint.js"; +import { getEmbedder } from "../src/adapters/embed/factory.js"; +import { handleReindex } from "../src/sidecar/server/reindex.js"; +import type { IndexDocument } from "../src/sidecar/protocol.js"; + +const DOC_A: IndexDocument = { + id: "pref-1", + content: "User prefers TypeScript strict mode", + source: "MEMORY.md", + timestamp: "2026-07-04T00:00:00.000Z", +}; + +const DOC_B: IndexDocument = { + id: "todo-1", + content: "Ship sidecar vector search MVP", + source: "MEMORY.md", + timestamp: "2026-07-04T01:00:00.000Z", +}; + +describe("computeDocumentsFingerprint", () => { + const embedder = getEmbedder(); + + it("is order-insensitive for the same document set", () => { + const a = computeDocumentsFingerprint([DOC_A, DOC_B], embedder); + const b = computeDocumentsFingerprint([DOC_B, DOC_A], embedder); + expect(a).toBe(b); + }); + + it("changes when content changes", () => { + const base = computeDocumentsFingerprint([DOC_A, DOC_B], embedder); + const mutated = computeDocumentsFingerprint([{ ...DOC_A, content: "changed" }, DOC_B], embedder); + expect(mutated).not.toBe(base); + }); + + it("changes when the document set grows or shrinks", () => { + const base = computeDocumentsFingerprint([DOC_A], embedder); + const grown = computeDocumentsFingerprint([DOC_A, DOC_B], embedder); + expect(grown).not.toBe(base); + expect(computeDocumentsFingerprint([], embedder)).not.toBe(base); + }); +}); + +describe("handleReindex fingerprint skip", () => { + let tmpDir: string; + let dbPath: string; + + afterEach(() => { + resetVecStoreForTests(); + resetEmbedderForTests(); + if (tmpDir) rmSync(tmpDir, { recursive: true, force: true }); + }); + + function freshCtx(): { dbPath: string } { + tmpDir = mkdtempSync(join(tmpdir(), "pi-memory-reindex-fp-")); + return { dbPath: join(tmpDir, "memory.db") }; + } + + it("skips re-embedding when the same documents are reindexed", async () => { + const ctx = freshCtx(); + + const first = await handleReindex("r1", ctx, [DOC_A, DOC_B]); + expect(first.indexed).toBe(2); + expect(first.skipped).toBeUndefined(); + + const second = await handleReindex("r2", ctx, [DOC_A, DOC_B]); + expect(second.skipped).toBe("fingerprint_match"); + expect(second.indexed).toBe(0); + expect(second.index_generation).toBe(first.index_generation); + + // Data is still queryable after the skipped round. + const store = (await import("../src/sidecar/server/vec/store.js")).getVecStore(ctx.dbPath); + const results = await store.query("TypeScript strict mode"); + expect(results.length).toBeGreaterThan(0); + }); + + it("rebuilds when content changes, then skips again", async () => { + const ctx = freshCtx(); + + await handleReindex("r1", ctx, [DOC_A, DOC_B]); + const mutated: IndexDocument = { ...DOC_A, content: "User prefers Rust over TypeScript now" }; + const rebuilt = await handleReindex("r2", ctx, [mutated, DOC_B]); + expect(rebuilt.skipped).toBeUndefined(); + expect(rebuilt.indexed).toBe(2); + + const again = await handleReindex("r3", ctx, [mutated, DOC_B]); + expect(again.skipped).toBe("fingerprint_match"); + + const store = (await import("../src/sidecar/server/vec/store.js")).getVecStore(ctx.dbPath); + const results = await store.query("Rust over TypeScript"); + expect(results.length).toBeGreaterThan(0); + }); + + it("does not skip for an empty document set (never fingerprints empty)", async () => { + const ctx = freshCtx(); + + await handleReindex("r1", ctx, [DOC_A]); + // Empty reindex is the explicit "clear/shrink" path and must stay effective. + const cleared = await handleReindex("r2", ctx, []); + expect(cleared.skipped).toBeUndefined(); + + // And a subsequent non-empty reindex re-embeds (fingerprint was not stored on empty). + const reembed = await handleReindex("r3", ctx, [DOC_A]); + expect(reembed.skipped).toBeUndefined(); + expect(reembed.indexed).toBe(1); + }); +});