Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 12 additions & 1 deletion CLAUDE.md
Original file line number Diff line number Diff line change
Expand Up @@ -257,6 +257,16 @@ before any store mutation. Rows without string keys are dropped and counted in `
legacy keyless Session stubs do not prevent restoring the rest. Access logs and legacy Memory
session IDs retain their normal normalization. This preflight does not make replacement transactional.

### Facet cleanup
Ordinary hard deletes of Observations, Memories and Lessons use `deleteWithFacets()`
(`src/functions/facets.ts`), which serializes deletion with Facet tag and untag mutations and
best-effort removes Facets for the IDs actually deleted. It caches a per-`StateKV` target-to-key
lookup, rebuilding in pages when `KV.facets`'s write generation changes; successful cleanup
updates the cache without a rescan. A later tag on a missing ID remains allowed. Import and
snapshot restore hold the same Facet lock around their Facet mutations, while retaining their
replacement semantics; this is not an orphan sweep. Memory writers take `MEMORY_WRITE_LOCK`
before the Facet lock.

### Relation backups
Full exports and snapshots carry Memory relations with their original storage keys, including legacy
`rel_` keys. Replace import and snapshot restore replace that scope when the backup contains it;
Expand All @@ -265,7 +275,8 @@ validate relation entries before replacing any scope, preserve present endpoint
(including explicit nulls and non-strings accepted by mesh), retain string relation types, map
non-string types to `related`, and omit nonnumeric or nonfinite confidence from backup data without
changing the source row. `StateKV.listPage` returns complete SQLite keys, including embedded NULs,
so those keys remain valid paging cursors and can be restored exactly.
so those keys remain valid paging cursors and can be restored exactly. Its first page includes the
empty key; a supplied cursor, including `""`, is exclusive.

### Graph Eviction
Every path that hard-deletes Observations or Memories (`mem::evict`, `mem::observe` session-cap
Expand Down
8 changes: 5 additions & 3 deletions src/engine/inproc/state.ts
Original file line number Diff line number Diff line change
Expand Up @@ -305,9 +305,11 @@ export class SqliteState {
}

listPage(scope: string, after: string | undefined, limit: number): Array<{ key: string; value: unknown }> {
const rows = this.db
.prepare("SELECT CAST(key AS BLOB) AS key, value FROM kv WHERE scope = ? AND key > ? ORDER BY kv.key LIMIT ?")
.all(scope, after ?? "", limit) as Array<{ key: Uint8Array; value: string }>;
const rows = (after === undefined
? this.db.prepare("SELECT CAST(key AS BLOB) AS key, value FROM kv WHERE scope = ? ORDER BY kv.key LIMIT ?")
.all(scope, limit)
: this.db.prepare("SELECT CAST(key AS BLOB) AS key, value FROM kv WHERE scope = ? AND key > ? ORDER BY kv.key LIMIT ?")
.all(scope, after, limit)) as Array<{ key: Uint8Array; value: string }>;
return rows.map((r) => ({ key: Buffer.from(r.key).toString("utf8"), value: JSON.parse(r.value) }));
}

Expand Down
12 changes: 10 additions & 2 deletions src/functions/auto-forget.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import { deleteWithFacets } from "./facets.js";
import type { ISdk } from "../engine/types.js";
import type { Memory, CompressedObservation, Session } from "../types.js";
import { KV } from "../state/schema.js";
Expand Down Expand Up @@ -73,7 +74,10 @@ export function registerAutoForgetFunction(sdk: ISdk, kv: StateKV): void {
await withKeyedLock(MEMORY_WRITE_LOCK, async () => {
const current = await kv.get<Memory>(KV.memories, mem.id);
if (!current) return;
await deleteIndexed(kv, KV.memories, mem.id);
await deleteWithFacets(kv, async (removed) => {
await deleteIndexed(kv, KV.memories, mem.id);
removed.push(mem.id);
});
if (current.imageRef) {
await decrementImageRef(kv, sdk, current.imageRef).catch((error) => {
logger.warn("Auto-forget could not release image reference", { memoryId: current.id, error: String(error) });
Expand Down Expand Up @@ -181,7 +185,11 @@ export function registerAutoForgetFunction(sdk: ISdk, kv: StateKV): void {
if (!dryRun) {
let deletedOk = false;
try {
await deleteIndexed(kv, KV.observations(sessions[i].id), obs.id);
await deleteWithFacets(kv, async (removed) => {
const existed = await kv.get(KV.observations(sessions[i].id), obs.id);
await deleteIndexed(kv, KV.observations(sessions[i].id), obs.id);
if (existed) removed.push(obs.id);
});
deletedOk = true;
} catch {
deletedOk = false;
Expand Down
23 changes: 19 additions & 4 deletions src/functions/evict.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import { deleteWithFacets } from "./facets.js";
import type { ISdk } from "../engine/types.js";
import type {
Session,
Expand Down Expand Up @@ -275,7 +276,11 @@ export function registerEvictFunction(sdk: ISdk, kv: StateKV): void {
lowImportanceIds.add(o.id);
} else {
try {
await deleteIndexed(kv, KV.observations(session.id), o.id);
await deleteWithFacets(kv, async (removed) => {
const existed = await kv.get(KV.observations(session.id), o.id);
await deleteIndexed(kv, KV.observations(session.id), o.id);
if (existed) removed.push(o.id);
});
stats.lowImportanceObs++;
lowImportanceIds.add(o.id);
countRemoval(session.id);
Expand Down Expand Up @@ -321,7 +326,11 @@ export function registerEvictFunction(sdk: ISdk, kv: StateKV): void {
} else {
for (const o of toEvict) {
try {
await deleteIndexed(kv, KV.observations(o.sessionId), o.id);
await deleteWithFacets(kv, async (removed) => {
const existed = await kv.get(KV.observations(o.sessionId), o.id);
await deleteIndexed(kv, KV.observations(o.sessionId), o.id);
if (existed) removed.push(o.id);
});
stats.capEvictions++;
countRemoval(o.sessionId);
evictedSourceIds.push(o.id);
Expand Down Expand Up @@ -363,7 +372,10 @@ export function registerEvictFunction(sdk: ISdk, kv: StateKV): void {
const removed = await withKeyedLock(MEMORY_WRITE_LOCK, async () => {
const current = await kv.get<Memory>(KV.memories, mem.id);
if (!current) return false;
await deleteIndexed(kv, KV.memories, mem.id);
await deleteWithFacets(kv, async (removed) => {
await deleteIndexed(kv, KV.memories, mem.id);
removed.push(mem.id);
});
if (current.imageRef) {
await decrementImageRef(kv, sdk, current.imageRef).catch((err: unknown) => {
logger.warn("Eviction could not release image reference", {
Expand Down Expand Up @@ -410,7 +422,10 @@ export function registerEvictFunction(sdk: ISdk, kv: StateKV): void {
const removed = await withKeyedLock(MEMORY_WRITE_LOCK, async () => {
const current = await kv.get<Memory>(KV.memories, mem.id);
if (!current) return false;
await deleteIndexed(kv, KV.memories, mem.id);
await deleteWithFacets(kv, async (removed) => {
await deleteIndexed(kv, KV.memories, mem.id);
removed.push(mem.id);
});
if (current.imageRef) {
await decrementImageRef(kv, sdk, current.imageRef).catch((err: unknown) => {
logger.warn("Eviction could not release image reference", {
Expand Down
25 changes: 15 additions & 10 deletions src/functions/export-import.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import { withFacetLock } from "./facets.js";
import type { ISdk } from "../engine/types.js";
import type {
Session,
Expand Down Expand Up @@ -491,10 +492,12 @@ export function registerExportImportFunction(sdk: ISdk, kv: StateKV): void {
await kv.list<Signal>(KV.signals).catch(() => []),
(s) => kv.delete(KV.signals, s.id),
);
await runChunked(
await kv.list<Facet>(KV.facets).catch(() => []),
(f) => kv.delete(KV.facets, f.id),
);
await withFacetLock(async () => {
await runChunked(
await kv.list<Facet>(KV.facets).catch(() => []),
(f) => kv.delete(KV.facets, f.id),
);
});
await runChunked(
await kv.list<Lesson>(KV.lessons).catch(() => []),
(l) => kv.delete(KV.lessons, l.id),
Expand Down Expand Up @@ -687,12 +690,14 @@ export function registerExportImportFunction(sdk: ISdk, kv: StateKV): void {
});
}
if (importData.facets) {
await runChunked(importData.facets, async (facet) => {
if (strategy === "skip") {
const existing = await kv.get(KV.facets, facet.id).catch(() => null);
if (existing) { stats.skipped++; return; }
}
await kv.set(KV.facets, facet.id, facet);
await withFacetLock(async () => {
await runChunked(importData.facets!, async (facet) => {
if (strategy === "skip") {
const existing = await kv.get(KV.facets, facet.id).catch(() => null);
if (existing) { stats.skipped++; return; }
}
await kv.set(KV.facets, facet.id, facet);
});
});
}
if (importData.lessons) {
Expand Down
169 changes: 134 additions & 35 deletions src/functions/facets.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,103 @@ import type { ISdk } from "../engine/types.js";
import type { StateKV } from "../state/kv.js";
import { KV, generateId } from "../state/schema.js";
import type { Facet } from "../types.js";
import { withKeyedLock } from "../state/keyed-mutex.js";
import { logger } from "../logger.js";
import { scrubFields } from "./privacy.js";

const FACET_LOCK = "mem:facet:mutate";

export function withFacetLock<T>(fn: () => Promise<T>): Promise<T> {
return withKeyedLock(FACET_LOCK, fn);
}

type FacetIndex = { generation: number; targets: Map<string, Set<string>> };
const facetIndexes = new WeakMap<StateKV, FacetIndex>();

async function facetIndex(kv: StateKV): Promise<FacetIndex> {
for (let attempt = 0; attempt < 3; attempt++) {
const generation = await kv.writeGeneration(KV.facets);
const cached = facetIndexes.get(kv);
if (cached?.generation === generation) return cached;
const targets = new Map<string, Set<string>>();
let after: string | undefined;
for (;;) {
const page = await kv.listPage<Facet>(KV.facets, after, 256);
for (const row of page) {
let keys = targets.get(row.value.targetId);
if (!keys) targets.set(row.value.targetId, keys = new Set());
keys.add(row.key);
}
if (page.length < 256) break;
after = page[page.length - 1].key;
}
if (generation === await kv.writeGeneration(KV.facets)) {
const index = { generation, targets };
facetIndexes.set(kv, index);
return index;
}
}
throw new Error("Facet scope changed during index rebuild");
}

async function cleanupFacets(kv: StateKV, deleted: string[]): Promise<void> {
for (let attempt = 0; attempt < 3; attempt++) {
const index = await facetIndex(kv);
let stale = false;
for (const target of new Set(deleted)) {
const keys = index.targets.get(target);
if (!keys) continue;
for (const key of keys) {
const facet = await kv.get<Facet>(KV.facets, key);
if (await kv.writeGeneration(KV.facets) !== index.generation) {
stale = true;
break;
}
if (facet?.targetId !== target) {
keys.delete(key);
continue;
}
try {
await kv.delete(KV.facets, key);
keys.delete(key);
const generation = await kv.writeGeneration(KV.facets);
if (generation !== index.generation + 1) {
stale = true;
break;
}
index.generation = generation;
} catch (error) {
facetIndexes.delete(kv);
logger.warn("Facet cleanup failed", { facetId: key, error: String(error) });
}
}
if (stale) break;
if (keys.size === 0) index.targets.delete(target);
}
if (!stale) return;
facetIndexes.delete(kv);
}
throw new Error("Facet scope changed during cleanup");
}

export async function deleteWithFacets<T>(kv: StateKV, remove: (deleted: string[]) => Promise<T>): Promise<T> {
return withFacetLock(async () => {
const deleted: string[] = [];
try {
return await remove(deleted);
} finally {
if (deleted.length) {
try {
await cleanupFacets(kv, deleted);
} catch (error) {
facetIndexes.delete(kv);
logger.warn("Facet cleanup failed", { error: String(error) });
}
}
}
});
}

export function registerFacetsFunction(sdk: ISdk, kv: StateKV): void {
sdk.registerFunction("mem::facet-tag",
async (data: {
Expand Down Expand Up @@ -44,28 +139,30 @@ export function registerFacetsFunction(sdk: ISdk, kv: StateKV): void {
const dimension = data.dimension.trim();
const value = data.value.trim();

const existing = await kv.list<Facet>(KV.facets);
const duplicate = existing.find(
(f) =>
f.targetId === data.targetId &&
f.dimension === dimension &&
f.value === value,
);
if (duplicate) {
return { success: true, facet: duplicate, skipped: true };
}

const facet: Facet = {
id: generateId("fct"),
targetId: data.targetId,
targetType: data.targetType as Facet["targetType"],
dimension,
value,
createdAt: new Date().toISOString(),
};
return withFacetLock(async () => {
const existing = await kv.list<Facet>(KV.facets);
const duplicate = existing.find(
(f) =>
f.targetId === data.targetId &&
f.dimension === dimension &&
f.value === value,
);
if (duplicate) {
return { success: true, facet: duplicate, skipped: true };
}

const facet: Facet = {
id: generateId("fct"),
targetId: data.targetId,
targetType: data.targetType as Facet["targetType"],
dimension,
value,
createdAt: new Date().toISOString(),
};

await kv.set(KV.facets, facet.id, facet);
return { success: true, facet };
await kv.set(KV.facets, facet.id, facet);
return { success: true, facet };
});
},
);

Expand All @@ -82,22 +179,24 @@ export function registerFacetsFunction(sdk: ISdk, kv: StateKV): void {
return { success: false, error: "dimension is required" };
}

const all = await kv.list<Facet>(KV.facets);
const matches = all.filter((f) => {
if (f.targetId !== data.targetId || f.dimension !== data.dimension) {
return false;
}
if (data.value !== undefined) {
return f.value === data.value;
}
return true;
});
return withFacetLock(async () => {
const all = await kv.list<Facet>(KV.facets);
const matches = all.filter((f) => {
if (f.targetId !== data.targetId || f.dimension !== data.dimension) {
return false;
}
if (data.value !== undefined) {
return f.value === data.value;
}
return true;
});

for (const f of matches) {
await kv.delete(KV.facets, f.id);
}
for (const f of matches) {
await kv.delete(KV.facets, f.id);
}

return { success: true, removed: matches.length };
return { success: true, removed: matches.length };
});
},
);

Expand Down
Loading