Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions .changeset/skip-unchanged-reindex.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
---
"@chendpoc/pi-memory": patch
---

Skip re-embedding unchanged memory files on reindex, and stop blocking session startup on the initial reindex.

Sidecar `reindex` now computes a content fingerprint (sha256 over sorted chunk ids + contents + embedder identity) and short-circuits when it matches the last full reindex, returning `skipped: "fingerprint_match"` instead of re-embedding every chunk. The first `runNow()` in `bootstrapSidecar` is fire-and-forget so `session_start` no longer waits on a full re-embed; preflight already falls back to MEMORY.md while the index rebuilds.

Measured with a 128-chunk memory file and the ollama `bge-m3` embedder: repeated reindex calls went from ~7.3-7.8s each to ~2ms, and session startup cost attributable to pi-memory dropped by roughly 7s per attach / new chat.
6 changes: 5 additions & 1 deletion src/extension/lifecycle.ts
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,11 @@ export async function bootstrapSidecar(opts: {
});

const unsubSyncToSidecar = opts.store.onSyncToSidecar(() => reindexScheduler.schedule());
await reindexScheduler.runNow();
// Fire-and-forget: the initial runNow re-embeds the whole memory file when it
// changed, which with a remote embedder can take seconds. Session startup
// (session_start) must not block on it — preflight already falls back to
// MEMORY.md while the index is being rebuilt.
void reindexScheduler.runNow();

return { reindexScheduler, unsubSyncToSidecar };
}
Expand Down
8 changes: 7 additions & 1 deletion src/sidecar/protocol.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,13 @@ export type SidecarResponse =
| { type: "pong" }
| { type: "stats_ok" } & IndexStats
| { type: "result"; request_id: string; results: MemoryEntry[] }
| { type: "reindex_ok"; request_id: string; indexed: number; index_generation: number }
| {
type: "reindex_ok";
request_id: string;
indexed: number;
index_generation: number;
skipped?: "fingerprint_match";
}
| { type: "error"; request_id?: string; error: string };

export function isErrorResponse(res: SidecarResponse): res is Extract<SidecarResponse, { type: "error" }> {
Expand Down
20 changes: 20 additions & 0 deletions src/sidecar/server/reindex.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,8 @@
import { createHash } from "node:crypto";

import type { IndexDocument, SidecarResponse } from "../protocol.js";
import { computeDocumentsFingerprint } from "./vec/fingerprint.js";
import { getEmbedder } from "./vec/embedder.js";
import { getVecStore } from "./vec/store.js";

export type ReindexContext = {
Expand All @@ -11,7 +15,23 @@ export async function handleReindex(
documents: IndexDocument[] = [],
): Promise<Extract<SidecarResponse, { type: "reindex_ok" }>> {
const store = getVecStore(ctx.dbPath);
// Skip re-embedding when the document set is unchanged since the last full
// reindex (same ids + contents + embedder identity). Session startup calls
// reindex on every attach; re-embedding an unchanged memory file dominated
// attach latency with remote embedders (e.g. ~7s for 128 chunks via ollama).
const embedder = getEmbedder();
const fingerprint = computeDocumentsFingerprint(documents, embedder);
if (documents.length > 0 && store.getLastReindexFingerprint() === fingerprint) {
return {
type: "reindex_ok",
request_id: requestId,
indexed: 0,
index_generation: store.getIndexGeneration(),
skipped: "fingerprint_match",
};
}
const outcome = await store.reindex(documents);
store.setLastReindexFingerprint(fingerprint);
return {
type: "reindex_ok",
request_id: requestId,
Expand Down
26 changes: 26 additions & 0 deletions src/sidecar/server/vec/fingerprint.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
import { createHash } from "node:crypto";

import type { IndexDocument } from "../../protocol.js";
import type { Embedder } from "../../../adapters/embed/types.js";
import { getLastReindexFingerprint, setLastReindexFingerprint } from "./schema.js";

/**
* Content fingerprint of a document set: sha256 over sorted (id, content) pairs
* plus the embedder identity. Any change to ids, contents, or the embedder
* (model/provider/dim) produces a different fingerprint, so a match means the
* stored chunks + embeddings already reflect exactly this input.
*/
export function computeDocumentsFingerprint(documents: IndexDocument[], embedder: Embedder): string {
const hash = createHash("sha256");
const sorted = [...documents].sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0));
for (const doc of sorted) {
hash.update(doc.id);
hash.update("\u0000");
hash.update(doc.content);
hash.update("\u0000");
}
hash.update(`${embedder.provider}\u0000${embedder.model}\u0000${String(embedder.dim)}`);
return hash.digest("hex");
}

export { getLastReindexFingerprint, setLastReindexFingerprint };
16 changes: 16 additions & 0 deletions src/sidecar/server/vec/schema.ts
Original file line number Diff line number Diff line change
Expand Up @@ -84,3 +84,19 @@ export function writeEmbeddingMeta(db: VecDatabase, embedder: Embedder): void {
upsert.run("embedding_provider", embedder.provider);
upsert.run("embedding_dim", String(embedder.dim));
}

const LAST_REINDEX_FINGERPRINT_KEY = "last_reindex_fingerprint";

export function getLastReindexFingerprint(db: VecDatabase): string | null {
const raw = db.prepare("SELECT value FROM meta WHERE key = ?").pluck().get(LAST_REINDEX_FINGERPRINT_KEY) as
| string
| undefined;
return raw ?? null;
}

export function setLastReindexFingerprint(db: VecDatabase, fingerprint: string): void {
db.prepare("INSERT INTO meta(key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value").run(
LAST_REINDEX_FINGERPRINT_KEY,
fingerprint,
);
}
10 changes: 10 additions & 0 deletions src/sidecar/server/vec/store.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,10 @@ import {
embeddingMetaMatches,
getChunkCount,
getIndexGeneration,
getLastReindexFingerprint,
getStoredEmbeddingMeta,
initVecSchema,
setLastReindexFingerprint,
type ReindexOutcome,
type StoredEmbeddingMeta,
} from "./schema.js";
Expand Down Expand Up @@ -41,6 +43,14 @@ export class VecStore {
return getIndexGeneration(this.db);
}

getLastReindexFingerprint(): string | null {
return getLastReindexFingerprint(this.db);
}

setLastReindexFingerprint(fingerprint: string): void {
setLastReindexFingerprint(this.db, fingerprint);
}

getChunkCount(): number {
return getChunkCount(this.db);
}
Expand Down
113 changes: 113 additions & 0 deletions test/reindexFingerprint.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterEach, describe, expect, it } from "vitest";

import { resetEmbedderForTests } from "../src/sidecar/server/vec/embedder.js";
import { resetVecStoreForTests } from "../src/sidecar/server/vec/store.js";
import { computeDocumentsFingerprint } from "../src/sidecar/server/vec/fingerprint.js";
import { getEmbedder } from "../src/adapters/embed/factory.js";
import { handleReindex } from "../src/sidecar/server/reindex.js";
import type { IndexDocument } from "../src/sidecar/protocol.js";

const DOC_A: IndexDocument = {
id: "pref-1",
content: "User prefers TypeScript strict mode",
source: "MEMORY.md",
timestamp: "2026-07-04T00:00:00.000Z",
};

const DOC_B: IndexDocument = {
id: "todo-1",
content: "Ship sidecar vector search MVP",
source: "MEMORY.md",
timestamp: "2026-07-04T01:00:00.000Z",
};

describe("computeDocumentsFingerprint", () => {
const embedder = getEmbedder();

it("is order-insensitive for the same document set", () => {
const a = computeDocumentsFingerprint([DOC_A, DOC_B], embedder);
const b = computeDocumentsFingerprint([DOC_B, DOC_A], embedder);
expect(a).toBe(b);
});

it("changes when content changes", () => {
const base = computeDocumentsFingerprint([DOC_A, DOC_B], embedder);
const mutated = computeDocumentsFingerprint([{ ...DOC_A, content: "changed" }, DOC_B], embedder);
expect(mutated).not.toBe(base);
});

it("changes when the document set grows or shrinks", () => {
const base = computeDocumentsFingerprint([DOC_A], embedder);
const grown = computeDocumentsFingerprint([DOC_A, DOC_B], embedder);
expect(grown).not.toBe(base);
expect(computeDocumentsFingerprint([], embedder)).not.toBe(base);
});
});

describe("handleReindex fingerprint skip", () => {
let tmpDir: string;
let dbPath: string;

afterEach(() => {
resetVecStoreForTests();
resetEmbedderForTests();
if (tmpDir) rmSync(tmpDir, { recursive: true, force: true });
});

function freshCtx(): { dbPath: string } {
tmpDir = mkdtempSync(join(tmpdir(), "pi-memory-reindex-fp-"));
return { dbPath: join(tmpDir, "memory.db") };
}

it("skips re-embedding when the same documents are reindexed", async () => {
const ctx = freshCtx();

const first = await handleReindex("r1", ctx, [DOC_A, DOC_B]);
expect(first.indexed).toBe(2);
expect(first.skipped).toBeUndefined();

const second = await handleReindex("r2", ctx, [DOC_A, DOC_B]);
expect(second.skipped).toBe("fingerprint_match");
expect(second.indexed).toBe(0);
expect(second.index_generation).toBe(first.index_generation);

// Data is still queryable after the skipped round.
const store = (await import("../src/sidecar/server/vec/store.js")).getVecStore(ctx.dbPath);
const results = await store.query("TypeScript strict mode");
expect(results.length).toBeGreaterThan(0);
});

it("rebuilds when content changes, then skips again", async () => {
const ctx = freshCtx();

await handleReindex("r1", ctx, [DOC_A, DOC_B]);
const mutated: IndexDocument = { ...DOC_A, content: "User prefers Rust over TypeScript now" };
const rebuilt = await handleReindex("r2", ctx, [mutated, DOC_B]);
expect(rebuilt.skipped).toBeUndefined();
expect(rebuilt.indexed).toBe(2);

const again = await handleReindex("r3", ctx, [mutated, DOC_B]);
expect(again.skipped).toBe("fingerprint_match");

const store = (await import("../src/sidecar/server/vec/store.js")).getVecStore(ctx.dbPath);
const results = await store.query("Rust over TypeScript");
expect(results.length).toBeGreaterThan(0);
});

it("does not skip for an empty document set (never fingerprints empty)", async () => {
const ctx = freshCtx();

await handleReindex("r1", ctx, [DOC_A]);
// Empty reindex is the explicit "clear/shrink" path and must stay effective.
const cleared = await handleReindex("r2", ctx, []);
expect(cleared.skipped).toBeUndefined();

// And a subsequent non-empty reindex re-embeds (fingerprint was not stored on empty).
const reembed = await handleReindex("r3", ctx, [DOC_A]);
expect(reembed.skipped).toBeUndefined();
expect(reembed.indexed).toBe(1);
});
});