Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 13 additions & 1 deletion CLAUDE.md
Original file line number Diff line number Diff line change
Expand Up @@ -237,6 +237,18 @@ Auto-forget and retention release a Memory's image reference after its indexed d
A failed release is logged without blocking deletion bookkeeping; it may leave an unused image.
Concurrent deletion locking is outside this contract.

### Memory writes and provider recovery
`MEMORY_WRITE_LOCK` (`src/state/memory-utils.ts`) serializes Memory saves, deletes,
relations, evolution, and auto-forget demotion. Before demoting, auto-forget rechecks both
live rows under the lock and reports only completed demotions; dry runs report snapshot matches.
Consolidation checks the predecessor again under the lock after the provider call. Release the
lock before graph eviction. Import and snapshot restore keep their own replacement semantics.
Eviction decrements image references after deleting the row and logs a failed release without
skipping bookkeeping.
`ResilientProvider` admits one half-open probe per operation after acquiring a provider slot.
The reservation lasts until the call settles; busy and content-filter responses release it,
and probe generations discard stale completions. An open circuit rejects before waiting for a slot.

### Replace import
`mem::import` validates section shapes, size limits and keyed fields used by import indexing
before any store mutation. Rows without string keys are dropped and counted in `invalid`, so
Expand Down Expand Up @@ -469,7 +481,7 @@ short forms count only beside an ordinal, a year, or after "on". The in-process
BM25 from content at boot, so there is no migration.

### Search and consolidation
`memoryToObservation()` carries the saved Memory type as optional `memoryType` on full, compact and narrative `mem::search` results and smart-search results; `memory_recall` rendering is unchanged. Graph retrieval keeps an Observation's best score among paths returned from different start nodes. `mem::consolidate` serializes its own runs, counts failed model calls toward its ten-call budget, writes a successor before demoting its predecessor, and updates the index before the predecessor audit. A failed demotion can leave two latest versions; cross-writer snapshot races are not covered by this lock.
`memoryToObservation()` carries the saved Memory type as optional `memoryType` on full, compact and narrative `mem::search` results and smart-search results; `memory_recall` rendering is unchanged. Graph retrieval keeps an Observation's best score among paths returned from different start nodes. `mem::consolidate` serializes its own runs, counts failed model calls toward its ten-call budget, writes a successor before demoting its predecessor, and updates the index before the predecessor audit. A failed demotion can leave two latest versions; the write lock does not make both writes transactional.

### Search on the event loop
Search runs on the daemon's one event loop, which also answers `/livez` and the hooks. BM25
Expand Down
124 changes: 59 additions & 65 deletions src/functions/auto-forget.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,10 @@ import { KV } from "../state/schema.js";
import { StateKV } from "../state/kv.js";
import { recordAudit } from "./audit.js";
import { deleteIndexed, deindexSupersededMemory } from "./search.js";
import { withKeyedLock } from "../state/keyed-mutex.js";
import { lowerObservationCounts } from "./observe.js";
import { evictGraphForSources } from "./graph-eviction.js";
import { refersToDifferentDates } from "../state/memory-utils.js";
import { refersToDifferentDates, MEMORY_WRITE_LOCK } from "../state/memory-utils.js";
import { logger } from "../logger.js";

const MS_PER_DAY = 24 * 60 * 60 * 1000;
Expand All @@ -18,6 +19,22 @@ function sameSupersessionScope(a: Memory, b: Memory): boolean {
a.agentId === b.agentId;
}

function contradictionTokens(content: string): Set<string> {
return new Set(content.toLowerCase().split(/\s+/).filter((word) => word.length > 2));
}

function contradictionSimilarity(a: Memory, b: Memory, setA: Set<string>, setB: Set<string>): number {
if (!setA.size || !setB.size) return 0;
let intersection = 0;
for (const word of setA) {
if (setB.has(word)) intersection++;
}
const similarity = intersection / (setA.size + setB.size - intersection);
// Daily reports for different dates are not contradictions, even if their text nearly matches.
return similarity > CONTRADICTION_THRESHOLD && sameSupersessionScope(a, b) &&
!refersToDifferentDates(a.content, b.content) ? similarity : 0;
}

interface AutoForgetResult {
ttlExpired: string[];
contradictions: Array<{
Expand Down Expand Up @@ -53,12 +70,16 @@ export function registerAutoForgetFunction(sdk: ISdk, kv: StateKV): void {
result.ttlExpired.push(mem.id);
deletedIds.add(mem.id);
if (!dryRun) {
await deleteIndexed(kv, KV.memories, mem.id);
if (mem.imageRef) {
await decrementImageRef(kv, sdk, mem.imageRef).catch((error) => {
logger.warn("Auto-forget could not release image reference", { memoryId: mem.id, error: String(error) });
});
}
await withKeyedLock(MEMORY_WRITE_LOCK, async () => {
const current = await kv.get<Memory>(KV.memories, mem.id);
if (!current) return;
await deleteIndexed(kv, KV.memories, mem.id);
if (current.imageRef) {
await decrementImageRef(kv, sdk, current.imageRef).catch((error) => {
logger.warn("Auto-forget could not release image reference", { memoryId: current.id, error: String(error) });
});
}
});
evictedSourceIds.push(mem.id);
await recordAudit(kv, "delete", "mem::auto-forget", [mem.id], {
resource: "memory",
Expand All @@ -78,19 +99,7 @@ export function registerAutoForgetFunction(sdk: ISdk, kv: StateKV): void {
)
.slice(0, 1000);

const tokenCache = new Map<string, Set<string>>();
for (const mem of latestMemories) {
tokenCache.set(
mem.id,
new Set(
mem.content
.toLowerCase()
.split(/\s+/)
.filter((t) => t.length > 2),
),
);
}

const tokenCache = new Map(latestMemories.map((mem) => [mem.id, contradictionTokens(mem.content)]));
const memById = new Map(latestMemories.map((m) => [m.id, m]));
const conceptIndex = new Map<string, string[]>();
for (const mem of latestMemories) {
Expand All @@ -113,53 +122,38 @@ export function registerAutoForgetFunction(sdk: ISdk, kv: StateKV): void {
if (compared.has(key)) continue;
compared.add(key);

const setA = tokenCache.get(memIds[i])!;
const setB = tokenCache.get(memIds[j])!;
let intersection = 0;
if (setA.size === 0 && setB.size === 0) continue;
if (setA.size === 0 || setB.size === 0) continue;
for (const word of setA) {
if (setB.has(word)) intersection++;
const memA = memById.get(memIds[i])!;
const memB = memById.get(memIds[j])!;
if (memA.id === memB.id) continue;
const similarity = contradictionSimilarity(memA, memB, tokenCache.get(memA.id)!, tokenCache.get(memB.id)!);
if (!similarity) continue;
if (dryRun) {
result.contradictions.push({ memoryA: memA.id, memoryB: memB.id, similarity });
continue;
}
const sim =
intersection / (setA.size + setB.size - intersection);

if (sim > CONTRADICTION_THRESHOLD) {
const memA = memById.get(memIds[i])!;
const memB = memById.get(memIds[j])!;
if (!sameSupersessionScope(memA, memB)) continue;
// Two daily reports of two different days are not a
// contradiction. Without this, auto-forget silently re-demotes
// exactly the rows a lineage repair just restored - measured
// 2026-08-10, one hour after the step 5 repair - and unlike
// remember it records NO supersedes edge, so a lineage census
// cannot even see what it did.
if (refersToDifferentDates(memA.content, memB.content)) {
continue;
}
result.contradictions.push({
memoryA: memA.id,
memoryB: memB.id,
similarity: sim,
});

if (!dryRun) {
const older =
new Date(memA.createdAt).getTime() <
new Date(memB.createdAt).getTime()
? memA
: memB;
older.isLatest = false;
await kv.set(KV.memories, older.id, older);
deindexSupersededMemory(older.id);
await recordAudit(kv, "forget", "mem::auto-forget", [older.id], {
resource: "memory",
reason: "auto-forget contradiction",
olderId: older.id,
similarity: sim,
});
}
}
const demotion = await withKeyedLock(MEMORY_WRITE_LOCK, async () => {
const [a, b] = await Promise.all([
kv.get<Memory>(KV.memories, memA.id),
kv.get<Memory>(KV.memories, memB.id),
]);
if (!a || !b || a.isLatest === false || b.isLatest === false) return;
const conceptsB = new Set(b.concepts?.map((concept) => concept.toLowerCase()));
if (!a.concepts?.some((concept) => conceptsB.has(concept.toLowerCase()))) return;
const similarity = contradictionSimilarity(a, b, contradictionTokens(a.content), contradictionTokens(b.content));
if (!similarity) return;
const older = new Date(a.createdAt).getTime() < new Date(b.createdAt).getTime() ? a : b;
await kv.set(KV.memories, older.id, { ...older, isLatest: false });
deindexSupersededMemory(older.id);
return { olderId: older.id, similarity };
});
if (!demotion) continue;
result.contradictions.push({ memoryA: memA.id, memoryB: memB.id, similarity: demotion.similarity });
await recordAudit(kv, "forget", "mem::auto-forget", [demotion.olderId], {
resource: "memory",
reason: "auto-forget contradiction",
...demotion,
});
}
}
}
Expand Down
119 changes: 62 additions & 57 deletions src/functions/consolidate.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import type {
} from "../types.js";
import { KV, generateId } from "../state/schema.js";
import { StateKV } from "../state/kv.js";
import { MEMORY_WRITE_LOCK } from "../state/memory-utils.js";
import { withKeyedLock } from "../state/keyed-mutex.js";
import { recordAudit } from "./audit.js";
import { deindexSupersededMemory, indexNewMemoryVersion } from "./search.js";
Expand Down Expand Up @@ -113,7 +114,7 @@ export function registerConsolidateFunction(
}

let consolidated = 0;
const existingMemories = await kv.list<Memory>(KV.memories);
const existingMemories = structuredClone(await kv.list<Memory>(KV.memories));
const writtenTitles = new Set<string>();
const scopedProject =
typeof data.project === "string" && data.project.trim().length > 0
Expand Down Expand Up @@ -185,62 +186,66 @@ export function registerConsolidateFunction(
m.title.toLowerCase() === parsed.title.toLowerCase(),
);

if (existingMatch) {
const evolved: Memory = {
id: generateId("mem"),
createdAt: now,
updatedAt: now,
...parsed,
version: (existingMatch.version || 1) + 1,
parentId: existingMatch.id,
supersedes: [
existingMatch.id,
...(existingMatch.supersedes || []),
],
sourceObservationIds: [
...new Set([...(existingMatch.sourceObservationIds ?? []), ...obsIds]),
],
isLatest: true,
...(scopedProject !== undefined && { project: scopedProject }),
};
await kv.set(KV.memories, evolved.id, evolved);
existingMatch.isLatest = false;
await kv.set(KV.memories, existingMatch.id, existingMatch);
existingMemories.push(evolved);
writtenTitles.add(evolved.title.toLowerCase());
deindexSupersededMemory(existingMatch.id);
await indexNewMemoryVersion(evolved);
await recordAudit(kv, "evolve", "mem::consolidate", [existingMatch.id], {
action: "mark_non_latest",
concept,
});
await recordAudit(kv, "evolve", "mem::consolidate", [evolved.id], {
action: "evolve_memory",
oldId: existingMatch.id,
newId: evolved.id,
concept,
});
consolidated++;
} else {
const memory: Memory = {
id: generateId("mem"),
createdAt: now,
updatedAt: now,
...parsed,
sourceObservationIds: obsIds,
version: 1,
isLatest: true,
...(scopedProject !== undefined && { project: scopedProject }),
};
await kv.set(KV.memories, memory.id, memory);
existingMemories.push(memory);
writtenTitles.add(memory.title.toLowerCase());
await recordAudit(kv, "remember", "mem::consolidate", [memory.id], {
action: "create_memory",
concept,
});
consolidated++;
}
await withKeyedLock(MEMORY_WRITE_LOCK, async () => {
if (existingMatch) {
const current = await kv.get<Memory>(KV.memories, existingMatch.id);
if (!current || !evolvable(current) || current.version !== existingMatch.version || current.updatedAt !== existingMatch.updatedAt) return;
const evolved: Memory = {
id: generateId("mem"),
createdAt: now,
updatedAt: now,
...parsed,
version: (current.version || 1) + 1,
parentId: existingMatch.id,
supersedes: [
existingMatch.id,
...(current.supersedes || []),
],
sourceObservationIds: [
...new Set([...(current.sourceObservationIds ?? []), ...obsIds]),
],
isLatest: true,
...(scopedProject !== undefined && { project: scopedProject }),
};
await kv.set(KV.memories, evolved.id, evolved);
existingMatch.isLatest = false;
await kv.set(KV.memories, current.id, { ...current, isLatest: false });
existingMemories.push(evolved);
writtenTitles.add(evolved.title.toLowerCase());
deindexSupersededMemory(existingMatch.id);
await indexNewMemoryVersion(evolved);
await recordAudit(kv, "evolve", "mem::consolidate", [existingMatch.id], {
action: "mark_non_latest",
concept,
});
await recordAudit(kv, "evolve", "mem::consolidate", [evolved.id], {
action: "evolve_memory",
oldId: existingMatch.id,
newId: evolved.id,
concept,
});
consolidated++;
} else {
const memory: Memory = {
id: generateId("mem"),
createdAt: now,
updatedAt: now,
...parsed,
sourceObservationIds: obsIds,
version: 1,
isLatest: true,
...(scopedProject !== undefined && { project: scopedProject }),
};
await kv.set(KV.memories, memory.id, memory);
existingMemories.push(memory);
writtenTitles.add(memory.title.toLowerCase());
await recordAudit(kv, "remember", "mem::consolidate", [memory.id], {
action: "create_memory",
concept,
});
consolidated++;
}
});
} catch (err) {
logger.warn("Consolidation failed for concept", {
concept,
Expand Down
Loading