diff --git a/CLAUDE.md b/CLAUDE.md index ed70cf3ed4..07afc76354 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -241,6 +241,8 @@ Concurrent deletion locking is outside this contract. `MEMORY_WRITE_LOCK` (`src/state/memory-utils.ts`) serializes Memory saves, deletes, relations, evolution, and auto-forget demotion. Before demoting, auto-forget rechecks both live rows under the lock and reports only completed demotions; dry runs report snapshot matches. +Mesh receive, mesh sync and heal hold the same lock across each Memory write and its index update. +Heal rechecks the stored supersession edge under the lock before demoting. Consolidation checks the predecessor again under the lock after the provider call. Release the lock before graph eviction. Import and snapshot restore keep their own replacement semantics. Eviction decrements image references after deleting the row and logs a failed release without @@ -425,8 +427,9 @@ Memories, not Lessons. and it is not yet injected; it goes directly above the first passed Memory it shadows, within the 3 results, and skips the gate. - **De-indexing** — every path that sets a Memory `isLatest: false` (`mem::remember`, `mem::evolve`, - `mem::consolidate`, `mem::auto-forget`) calls `deindexSupersededMemory()`; evolve and - consolidate index the new version with `indexNewMemoryVersion()` (`src/functions/search.ts`). + `mem::consolidate`, `mem::auto-forget`, `mem::heal`, mesh receive and sync) calls + `deindexSupersededMemory()`; every other path that writes a latest Memory (evolve, consolidate, + `mem::auto-page`, mesh) indexes it with `indexNewMemoryVersion()` (`src/functions/search.ts`). - **Keyed Semantic Facts** — see the `SemanticMemory` consistency rule; a keyed fact whose text matches an unkeyed one keys it when every Session that fact cites is this project's, so the corpus written before keys can be superseded without one project closing another's fact. diff --git a/docs/supersession.md b/docs/supersession.md index ababb2d9cb..34693da930 100644 --- a/docs/supersession.md +++ b/docs/supersession.md @@ -93,7 +93,10 @@ when the newest claim was already given, an older one is not admitted in its pla removes the row from BM25 and the vector index. `mem::evolve` and the evolve branch of `mem::consolidate` add the new version through `indexNewMemoryVersion()`, so a replaced Memory is neither returned nor the only hit (T5, T6). A Memory `mem::consolidate` creates fresh (no -predecessor) is still not added to the live index until the next boot. A daemon restart rebuilds +predecessor), one `mem::auto-page` archives and a latest one mesh receive or sync accepts enter the +live index the same way; `mem::heal` and a non-latest mesh row leave it through +`deindexSupersededMemory()` (`test/memory-write-indexing.test.ts`, unscoped smart-search). +Fresh consolidation and auto-page do not supersede a Memory. A daemon restart rebuilds BM25 from content and skips non-latest Memories. `mem::evict` deletes a non-latest Memory after `nonLatestMemoryMaxDays`; `AGENTMEMORY_NONLATEST_FILTER` stays off by default. diff --git a/src/functions/consolidate.ts b/src/functions/consolidate.ts index debc0ff0e7..5a05c36a45 100644 --- a/src/functions/consolidate.ts +++ b/src/functions/consolidate.ts @@ -237,6 +237,7 @@ export function registerConsolidateFunction( ...(scopedProject !== undefined && { project: scopedProject }), }; await kv.set(KV.memories, memory.id, memory); + await indexNewMemoryVersion(memory); existingMemories.push(memory); writtenTitles.add(memory.title.toLowerCase()); await recordAudit(kv, "remember", "mem::consolidate", [memory.id], { diff --git a/src/functions/diagnostics.ts b/src/functions/diagnostics.ts index 7c638f0b23..38ada5ef3f 100644 --- a/src/functions/diagnostics.ts +++ b/src/functions/diagnostics.ts @@ -2,6 +2,8 @@ import type { ISdk } from "../engine/types.js"; import type { StateKV } from "../state/kv.js"; import { KV } from "../state/schema.js"; import { withKeyedLock } from "../state/keyed-mutex.js"; +import { MEMORY_WRITE_LOCK } from "../state/memory-utils.js"; +import { deindexSupersededMemory } from "./search.js"; import { recordAudit } from "./audit.js"; import { storeAcceptsWrite } from "../health/store-probe.js"; import { readMissedInjections } from "../hooks/_missed-injection.js"; @@ -768,7 +770,8 @@ export function registerDiagnosticsFunction(sdk: ISdk, kv: StateKV): void { } for (const memory of memories) { - if (memory.isLatest && supersededBy.has(memory.id)) { + const successorId = supersededBy.get(memory.id); + if (memory.isLatest && successorId && successorId !== memory.id) { if (dryRun) { details.push( `[dry-run] Would set isLatest=false on memory "${memory.title}" (${memory.id})`, @@ -776,22 +779,23 @@ export function registerDiagnosticsFunction(sdk: ISdk, kv: StateKV): void { fixed++; continue; } - const didFix = await withKeyedLock( - `mem:memory:${memory.id}`, - async () => { - const fresh = await kv.get(KV.memories, memory.id); - if (!fresh || !fresh.isLatest) return false; - fresh.isLatest = false; - fresh.updatedAt = new Date().toISOString(); - await kv.set(KV.memories, fresh.id, fresh); - await recordAudit(kv, "heal", "mem::heal", [fresh.id], { - entityType: "memory", - reason: "superseded-memory-mark-non-latest", - action: "update", - }); - return true; - }, - ); + const didFix = await withKeyedLock(MEMORY_WRITE_LOCK, async () => { + const [fresh, successor] = await Promise.all([ + kv.get(KV.memories, memory.id), + kv.get(KV.memories, successorId), + ]); + if (!fresh?.isLatest || !successor?.supersedes?.includes(fresh.id)) return false; + await kv.set(KV.memories, fresh.id, { + ...fresh, isLatest: false, updatedAt: new Date().toISOString(), + }); + deindexSupersededMemory(fresh.id); + await recordAudit(kv, "heal", "mem::heal", [fresh.id], { + entityType: "memory", + reason: "superseded-memory-mark-non-latest", + action: "update", + }); + return true; + }); if (didFix) { details.push( `Set isLatest=false on memory "${memory.title}" (${memory.id})`, diff --git a/src/functions/mesh.ts b/src/functions/mesh.ts index 494bb92cf8..8681a9605d 100644 --- a/src/functions/mesh.ts +++ b/src/functions/mesh.ts @@ -2,11 +2,13 @@ import type { ISdk } from "../engine/types.js"; import type { StateKV } from "../state/kv.js"; import { KV, generateId } from "../state/schema.js"; import { withKeyedLock } from "../state/keyed-mutex.js"; +import { MEMORY_WRITE_LOCK } from "../state/memory-utils.js"; import { indexGraphEdge, indexGraphNode, graphLegDisabled, } from "../state/graph-indexes.js"; +import { deindexSupersededMemory, indexNewMemoryVersion } from "./search.js"; import { recordAudit } from "./audit.js"; import { capRecordProvenance } from "./graph-provenance.js"; import { GRAPH_WRITE_LOCK } from "./graph.js"; @@ -82,7 +84,7 @@ async function lwwMergeList( kv: StateKV, scope: string, items: T[] | undefined, - lockPrefix: string, + lockKey: (id: string) => string, tsField: "updatedAt" | "createdAt", onWrite?: (item: T) => Promise, ): Promise { @@ -92,27 +94,24 @@ async function lwwMergeList( if (!item.id || typeof item.id !== "string") continue; const ts = (item as Record)[tsField]; if (typeof ts !== "string" || Number.isNaN(new Date(ts).getTime())) continue; - const wrote = await withKeyedLock(`${lockPrefix}:${item.id}`, async () => { + const wrote = await withKeyedLock(lockKey(item.id), async () => { const existing = await kv.get(scope, item.id); - if (!existing) { - await kv.set(scope, item.id, item); - return true; - } - const existingTs = (existing as Record)[tsField] as string; - if (new Date(ts) > new Date(existingTs)) { - await kv.set(scope, item.id, item); - return true; - } - return false; - }); - if (wrote) { - count++; + const existingTs = existing && (existing as Record)[tsField] as string | undefined; + if (existing && !(new Date(ts) > new Date(existingTs!))) return false; + await kv.set(scope, item.id, item); if (onWrite) await onWrite(item); - } + return true; + }); + if (wrote) count++; } return count; } +async function reindexMergedMemory(memory: Memory): Promise { + deindexSupersededMemory(memory.id); + if (memory.isLatest) await indexNewMemoryVersion(memory); +} + function graphNodeTs(node: GraphNode): string { return node.updatedAt || node.createdAt; } @@ -354,9 +353,9 @@ export function registerMeshFunction( } let accepted = 0; - accepted += await lwwMergeList(kv, KV.memories, data.memories, "mem:memory", "updatedAt"); - accepted += await lwwMergeList(kv, KV.semantic, data.semantic, "mem:semantic", "updatedAt"); - accepted += await lwwMergeList(kv, KV.procedural, data.procedural, "mem:procedural", "updatedAt"); + accepted += await lwwMergeList(kv, KV.memories, data.memories, () => MEMORY_WRITE_LOCK, "updatedAt", reindexMergedMemory); + accepted += await lwwMergeList(kv, KV.semantic, data.semantic, (id) => `mem:semantic:${id}`, "updatedAt"); + accepted += await lwwMergeList(kv, KV.procedural, data.procedural, (id) => `mem:procedural:${id}`, "updatedAt"); if (data.relations && Array.isArray(data.relations)) { for (const rel of data.relations) { if (!rel.sourceId || !rel.targetId || !rel.type) continue; @@ -380,7 +379,7 @@ export function registerMeshFunction( kv, KV.graphEdges, data.graphEdges?.map(capRecordProvenance), - "mem:gedge", + (id) => `mem:gedge:${id}`, "createdAt", (edge) => indexGraphEdge(kv, edge), )), @@ -519,13 +518,13 @@ async function applySyncData( let applied = 0; if (scopes.includes("memories")) { - applied += await lwwMergeList(kv, KV.memories, data.memories, "mem:memory", "updatedAt"); + applied += await lwwMergeList(kv, KV.memories, data.memories, () => MEMORY_WRITE_LOCK, "updatedAt", reindexMergedMemory); } if (scopes.includes("semantic")) { - applied += await lwwMergeList(kv, KV.semantic, data.semantic, "mem:semantic", "updatedAt"); + applied += await lwwMergeList(kv, KV.semantic, data.semantic, (id) => `mem:semantic:${id}`, "updatedAt"); } if (scopes.includes("procedural")) { - applied += await lwwMergeList(kv, KV.procedural, data.procedural, "mem:procedural", "updatedAt"); + applied += await lwwMergeList(kv, KV.procedural, data.procedural, (id) => `mem:procedural:${id}`, "updatedAt"); } if (scopes.includes("relations") && data.relations) { for (const rel of data.relations) { @@ -554,7 +553,7 @@ async function applySyncData( kv, KV.graphEdges, data.graphEdges?.map(capRecordProvenance), - "mem:gedge", + (id) => `mem:gedge:${id}`, "createdAt", (edge) => indexGraphEdge(kv, edge), ), diff --git a/src/functions/working-memory.ts b/src/functions/working-memory.ts index 822deb342c..43b7ced06d 100644 --- a/src/functions/working-memory.ts +++ b/src/functions/working-memory.ts @@ -2,6 +2,7 @@ import type { ISdk } from "../engine/types.js"; import type { Memory } from "../types.js"; import { KV, generateId } from "../state/schema.js"; import { StateKV } from "../state/kv.js"; +import { indexNewMemoryVersion } from "./search.js"; import { recordAudit } from "./audit.js"; import { recordAccessBatch } from "./access-tracker.js"; import { logger } from "../logger.js"; @@ -231,6 +232,7 @@ export function registerWorkingMemoryFunctions( isLatest: true, }; await kv.set(KV.memories, archivalMemory.id, archivalMemory); + await indexNewMemoryVersion(archivalMemory); await kv.delete(CORE_SCOPE, entry.id); totalTokens -= tokens; diff --git a/test/memory-write-indexing.test.ts b/test/memory-write-indexing.test.ts new file mode 100644 index 0000000000..857f31fc37 --- /dev/null +++ b/test/memory-write-indexing.test.ts @@ -0,0 +1,230 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; + +vi.mock("../src/logger.js", () => ({ + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, +})); + +import { registerConsolidateFunction } from "../src/functions/consolidate.js"; +import { registerWorkingMemoryFunctions } from "../src/functions/working-memory.js"; +import { registerDiagnosticsFunction } from "../src/functions/diagnostics.js"; +import { registerMeshFunction } from "../src/functions/mesh.js"; +import { registerSmartSearchFunction } from "../src/functions/smart-search.js"; +import { HybridSearch } from "../src/state/hybrid-search.js"; +import { getSearchIndex } from "../src/functions/search.js"; +import { withKeyedLock } from "../src/state/keyed-mutex.js"; +import { MEMORY_WRITE_LOCK, memoryToIndexDoc } from "../src/state/memory-utils.js"; +import { KV } from "../src/state/schema.js"; +import { mockKV } from "./helpers/mocks.js"; +import type { CompactSearchResult, CompressedObservation, Memory, MemoryProvider, Session } from "../src/types.js"; + +function mockSdk() { + const functions = new Map(); + return { + registerFunction: (idOrOpts: string | { id: string }, handler: Function) => { + functions.set(typeof idOrOpts === "string" ? idOrOpts : idOrOpts.id, handler); + }, + registerTrigger: () => {}, + trigger: async (idOrInput: string | { function_id: string; payload: unknown }, data?: unknown) => { + const id = typeof idOrInput === "string" ? idOrInput : idOrInput.function_id; + const payload = typeof idOrInput === "string" ? data : idOrInput.payload; + return functions.get(id)!(payload); + }, + }; +} + +function memory(id: string, overrides: Partial = {}): Memory { + return { + id, + createdAt: "2026-01-01T00:00:00.000Z", + updatedAt: "2026-01-01T00:00:00.000Z", + type: "fact", + title: `title ${id}`, + content: `zebracorn ${id}`, + concepts: [], + files: [], + sessionIds: [], + strength: 5, + version: 1, + isLatest: true, + ...overrides, + }; +} + +const indexed = (id: string) => getSearchIndex().has(id); + +function searchIds(sdk: ReturnType, kv: ReturnType) { + const hybrid = new HybridSearch(getSearchIndex(), null, null, kv as never, 1, 0, 0); + registerSmartSearchFunction(sdk as never, kv as never, (query) => hybrid.prepareSearch(query)); + return async (query: string) => { + const { results } = await sdk.trigger("mem::smart-search", { query }) as { results: CompactSearchResult[] }; + return results.map(({ obsId }) => obsId); + }; +} + + +describe("Memory write paths keep the live search index current", () => { + beforeEach(() => { + const index = getSearchIndex(); + index.clear(); + for (let i = 0; i < 40; i++) { + index.add(memoryToIndexDoc(memory(`filler_${i}`, { content: `database migration batch ${i}` })), "memory"); + } + }); + afterEach(() => vi.unstubAllGlobals()); + + it.each(["creates", "evolves"])("mem::consolidate indexes a Memory it %s", async (mode) => { + const sdk = mockSdk(); + const kv = mockKV(); + const session: Session = { + id: "s1", + project: "api", + cwd: "/srv/api", + startedAt: new Date().toISOString(), + status: "completed", + observationCount: 3, + }; + await kv.set(KV.sessions, session.id, session); + for (let i = 0; i < 3; i++) { + const o: CompressedObservation = { + id: `o${i}`, + sessionId: "s1", + timestamp: new Date().toISOString(), + type: "decision", + title: `observation ${i}`, + facts: [], + narrative: `narrative ${i}`, + concepts: ["auth"], + files: [], + importance: 8, + }; + await kv.set(KV.observations("s1"), o.id, o); + } + const provider = { + name: "mock", + compress: async () => + `patternauth rulecontent7`, + embed: vi.fn(), + embedBatch: vi.fn(), + dimensions: 384, + compressionModel: "mock-model", + } as unknown as MemoryProvider; + const search = searchIds(sdk, kv); + const old = memory("mem_old", { type: "pattern", title: "auth rule", project: "api" }); + if (mode === "evolves") { + await kv.set(KV.memories, old.id, old); + getSearchIndex().add(memoryToIndexDoc(old), "memory"); + expect(await search("auth rule")).toEqual([old.id]); + } + registerConsolidateFunction(sdk as never, kv as never, provider); + await sdk.trigger("mem::consolidate", { project: "api", minObservations: 1 }); + const created = (await kv.list(KV.memories)).find((row) => row.id !== old.id)!; + expect(created).toBeDefined(); + expect(await search("auth rule")).toEqual([created.id]); + if (mode === "evolves") expect(indexed(old.id)).toBe(false); + }); + + it("mem::auto-page indexes the Memory it pages out", async () => { + const sdk = mockSdk(); + const kv = mockKV(); + registerWorkingMemoryFunctions(sdk as never, kv as never, 10); + await sdk.trigger("mem::core-add", { content: "a long pinned entry that exceeds the tiny budget", importance: 3 }); + await sdk.trigger("mem::auto-page", {}); + const [paged] = await kv.list(KV.memories); + expect(paged).toBeDefined(); + expect(await searchIds(sdk, kv)("pinned entry")).toContain(paged.id); + }); + + it("mem::heal removes a Memory it marks non-latest from the index", async () => { + const sdk = mockSdk(); + const kv = mockKV(); + registerDiagnosticsFunction(sdk as never, kv as never); + const old = memory("mem_old"); + const next = memory("mem_new", { supersedes: ["mem_old"], version: 2 }); + await kv.set(KV.memories, old.id, old); + await kv.set(KV.memories, next.id, next); + getSearchIndex().add(memoryToIndexDoc(old), "memory"); + getSearchIndex().add(memoryToIndexDoc(next), "memory"); + const search = searchIds(sdk, kv); + expect(await search("zebracorn")).toContain(old.id); + await sdk.trigger("mem::heal", { categories: ["memories"] }); + expect((await kv.get(KV.memories, "mem_old"))!.isLatest).toBe(false); + expect(await search("zebracorn")).toEqual([next.id]); + expect(indexed(old.id)).toBe(false); + }); + + it("keeps a Memory indexed when heal's successor disappears after listing", async () => { + const sdk = mockSdk(); + const kv = mockKV(); + registerDiagnosticsFunction(sdk as never, kv as never); + const old = memory("mem_old"); + const next = memory("mem_new", { supersedes: [old.id] }); + await kv.set(KV.memories, old.id, old); + await kv.set(KV.memories, next.id, next); + getSearchIndex().add(memoryToIndexDoc(old), "memory"); + let heal!: Promise; + try { + await withKeyedLock(MEMORY_WRITE_LOCK, async () => { + heal = sdk.trigger("mem::heal", { categories: ["memories"] }); + await new Promise((resolve) => setImmediate(resolve)); + await kv.delete(KV.memories, next.id); + }); + } finally { + await heal; + } + expect(await heal).toMatchObject({ fixed: 0, skipped: 1 }); + expect(await searchIds(sdk, kv)("zebracorn")).toEqual([old.id]); + }); + + it("keeps mesh Memory application behind the Memory write lock", async () => { + const sdk = mockSdk(); + const kv = mockKV(); + registerMeshFunction(sdk as never, kv as never, "token"); + const incoming = memory("mem_incoming"); + let receive!: Promise; + + try { + await withKeyedLock(MEMORY_WRITE_LOCK, async () => { + receive = sdk.trigger("mem::mesh-receive", { memories: [incoming] }); + await new Promise((resolve) => setImmediate(resolve)); + expect(await kv.get(KV.memories, incoming.id)).toBeNull(); + expect(indexed(incoming.id)).toBe(false); + }); + } finally { + await receive; + } + expect(await kv.get(KV.memories, incoming.id)).toEqual(incoming); + expect(indexed(incoming.id)).toBe(true); + }); + + it.each(["receive", "sync"])("mesh %s applies live-index updates and rejects stale rows", async (path) => { + const sdk = mockSdk(); + const kv = mockKV(); + registerMeshFunction(sdk as never, kv as never, "token"); + const old = memory("mem_old"); + const closed = { ...old, isLatest: false, updatedAt: "2026-02-01T00:00:00.000Z" }; + const fresh = memory("mem_fresh", { updatedAt: "2026-02-01T00:00:00.000Z" }); + await kv.set(KV.memories, old.id, old); + getSearchIndex().add(memoryToIndexDoc(old), "memory"); + const search = searchIds(sdk, kv); + expect(await search("zebracorn")).toEqual([old.id]); + + const registration = path === "sync" ? await sdk.trigger("mem::mesh-register", { + url: "https://peer.example.com", name: "test peer", + }) as { peer: { id: string } } : undefined; + const apply = async (memories: Memory[]) => { + if (!registration) { + await sdk.trigger("mem::mesh-receive", { memories }); + return; + } + vi.stubGlobal("fetch", async () => Response.json({ memories })); + const result = await sdk.trigger("mem::mesh-sync", { peerId: registration.peer.id, direction: "pull" }); + expect(result).toMatchObject({ success: true, results: [{ errors: [] }] }); + }; + await apply([closed, fresh]); + expect(await search("zebracorn")).toEqual([fresh.id]); + expect(indexed(old.id)).toBe(false); + await apply([old]); + expect(await search("zebracorn")).toEqual([fresh.id]); + expect((await kv.get(KV.memories, old.id))!.isLatest).toBe(false); + }); +});