Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions CLAUDE.md
Original file line number Diff line number Diff line change
Expand Up @@ -257,6 +257,16 @@ before any store mutation. Rows without string keys are dropped and counted in `
legacy keyless Session stubs do not prevent restoring the rest. Access logs and legacy Memory
session IDs retain their normal normalization. This preflight does not make replacement transactional.

### Relation backups
Full exports and snapshots carry Memory relations with their original storage keys, including legacy
`rel_` keys. Replace import and snapshot restore replace that scope when the backup contains it;
older backups without a `relations` section leave the scope untouched. Import and snapshot restore
validate relation entries before replacing any scope, preserve present endpoint values as stored
(including explicit nulls and non-strings accepted by mesh), retain string relation types, map
non-string types to `related`, and omit nonnumeric or nonfinite confidence from backup data without
changing the source row. `StateKV.listPage` returns complete SQLite keys, including embedded NULs,
so those keys remain valid paging cursors and can be restored exactly.

### Graph Eviction
Every path that hard-deletes Observations or Memories (`mem::evict`, `mem::observe` session-cap
eviction, `mem::auto-forget`, `mem::forget`, `mem::retention-evict`, `mem::governance-delete`,
Expand Down
6 changes: 3 additions & 3 deletions src/engine/inproc/state.ts
Original file line number Diff line number Diff line change
Expand Up @@ -306,9 +306,9 @@ export class SqliteState {

listPage(scope: string, after: string | undefined, limit: number): Array<{ key: string; value: unknown }> {
const rows = this.db
.prepare("SELECT key, value FROM kv WHERE scope = ? AND key > ? ORDER BY key LIMIT ?")
.all(scope, after ?? "", limit) as Array<{ key: string; value: string }>;
return rows.map((r) => ({ key: r.key, value: JSON.parse(r.value) }));
.prepare("SELECT CAST(key AS BLOB) AS key, value FROM kv WHERE scope = ? AND key > ? ORDER BY kv.key LIMIT ?")
.all(scope, after ?? "", limit) as Array<{ key: Uint8Array; value: string }>;
return rows.map((r) => ({ key: Buffer.from(r.key).toString("utf8"), value: JSON.parse(r.value) }));
}

listNewest(
Expand Down
52 changes: 51 additions & 1 deletion src/functions/export-import.ts
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,7 @@ export const DURABLE_STORES: readonly DurableStore[] = [
{ field: "semanticMemories", scope: KV.semantic, keyOf: byId },
{ field: "proceduralMemories", scope: KV.procedural, keyOf: byId },
{ field: "facets", scope: KV.facets, keyOf: byId },
{ field: "relations", scope: KV.relations, keyOf: (row) => String(row.key) },
{ field: "lessons", scope: KV.lessons, keyOf: byId },
{ field: "insights", scope: KV.insights, keyOf: byId },
{ field: "signals", scope: KV.signals, keyOf: byId },
Expand All @@ -95,10 +96,34 @@ function readableStores(): DurableStore[] {
// fields are absent rather than empty.
export async function readDurableStores(kv: StateKV): Promise<DurableRows> {
const readable = readableStores();
const lists = await Promise.all(readable.map((s) => kv.list(s.scope)));
const lists = await Promise.all(readable.map((s) =>
s.field === "relations" ? relationEntries(kv) : kv.list(s.scope),
));
return Object.fromEntries(readable.map((s, i) => [s.field, lists[i]]));
}

export function normalizeRelation(value: unknown): unknown {
if (!value || typeof value !== "object" || Array.isArray(value)) return value;
const row = value as Record<string, unknown>;
const normalized = { ...row };
if ("type" in normalized && typeof normalized.type !== "string") normalized.type = "related";
if ("confidence" in normalized && (typeof normalized.confidence !== "number" || !Number.isFinite(normalized.confidence))) {
delete normalized.confidence;
}
return normalized;
}

export async function relationEntries(kv: StateKV): Promise<Array<{ key: string; value: unknown }>> {
const entries: Array<{ key: string; value: unknown }> = [];
let after: string | undefined;
for (;;) {
const page = await kv.listPage(KV.relations, after, 1000);
entries.push(...page.map(({ key, value }) => ({ key, value: normalizeRelation(value) })));
if (page.length < 1000) return entries;
after = page[page.length - 1].key;
}
}

function nonEmpty<T>(rows: T[] | undefined): T[] | undefined {
return rows && rows.length > 0 ? rows : undefined;
}
Expand Down Expand Up @@ -128,13 +153,28 @@ function isKeyedRow(row: unknown, key: string): boolean {
);
}

export function validRelationEntries(value: unknown): boolean {
if (value === undefined) return true;
if (!Array.isArray(value)) return false;
const keys = new Set<string>();
return value.every((entry) => {
if (!entry || typeof entry !== "object" || typeof entry.key !== "string" || !entry.key ||
keys.has(entry.key) || !entry.value || typeof entry.value !== "object" || Array.isArray(entry.value) ||
!("sourceId" in entry.value) || !("targetId" in entry.value) ||
!("type" in entry.value)) return false;
keys.add(entry.key);
return true;
});
}

// A replace import deletes every store before it writes, so a malformed
// section must fail first, leaving the store untouched.
function invalidImportSection(importData: ExportData): string | null {
for (const [section] of KEYED_SECTIONS) {
const rows: unknown = importData[section];
if (rows != null && !Array.isArray(rows)) return `${section} must be an array`;
}
if (!validRelationEntries(importData.relations)) return "relations must contain keyed relation rows";
const accessLogs: unknown = importData.accessLogs;
if (accessLogs != null && !Array.isArray(accessLogs)) return "accessLogs must be an array";
if (Array.isArray(accessLogs) && accessLogs.length > MAX_ACCESS_LOGS) {
Expand Down Expand Up @@ -283,6 +323,7 @@ export function registerExportImportFunction(sdk: ISdk, kv: StateKV): void {
semanticMemories: nonEmpty(stores.semanticMemories),
proceduralMemories: nonEmpty(stores.proceduralMemories),
facets: nonEmpty(stores.facets),
relations: stores.relations,
lessons: nonEmpty(stores.lessons),
insights: nonEmpty(stores.insights),
signals: nonEmpty(stores.signals),
Expand Down Expand Up @@ -459,6 +500,9 @@ export function registerExportImportFunction(sdk: ISdk, kv: StateKV): void {
(l) => kv.delete(KV.lessons, l.id),
);
resetLessonIndex();
if (importData.relations) {
await runChunked(await relationEntries(kv), ({ key }) => kv.delete(KV.relations, key));
}
await runChunked(
await kv.list<Insight>(KV.insights).catch(() => []),
(i) => kv.delete(KV.insights, i.id),
Expand Down Expand Up @@ -636,6 +680,12 @@ export function registerExportImportFunction(sdk: ISdk, kv: StateKV): void {
await kv.set(KV.signals, signal.id, signal);
});
}
if (importData.relations) {
await runChunked(importData.relations, async ({ key, value }) => {
if (strategy === "skip" && await kv.get(KV.relations, key)) { stats.skipped++; return; }
await kv.set(KV.relations, key, normalizeRelation(value));
});
}
if (importData.facets) {
await runChunked(importData.facets, async (facet) => {
if (strategy === "skip") {
Expand Down
12 changes: 10 additions & 2 deletions src/functions/snapshot.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,9 @@ import { graphWritesDisabled } from "./graph.js";
import {
DURABLE_STORES,
readDurableStores,
relationEntries,
normalizeRelation,
validRelationEntries,
runChunked,
type DurableStore,
type DurableStoreField,
Expand Down Expand Up @@ -104,7 +107,9 @@ async function replaceStore(
const rows =
store.field === "lessons" ? captured.filter((r) => r.deleted !== true) : captured;
const keep = new Set(rows.map(store.keyOf));
const existing = await kv.list<SnapshotRow>(store.scope);
const existing = store.field === "relations"
? await relationEntries(kv)
: await kv.list<SnapshotRow>(store.scope);
const stale = existing.map(store.keyOf).filter((key) => !keep.has(key));

await runChunked(stale, (key) =>
Expand All @@ -123,7 +128,7 @@ async function replaceStore(
await kv.set(store.scope, edge.id, edge);
await indexGraphEdge(kv, edge);
} else {
await kv.set(store.scope, store.keyOf(row), row);
await kv.set(store.scope, store.keyOf(row), store.field === "relations" ? normalizeRelation(row.value) : row);
}
});

Expand Down Expand Up @@ -322,6 +327,9 @@ export function registerSnapshotFunction(
]);
const content = readFileSync(join(snapshotDir, "state.json"), "utf-8");
const state = JSON.parse(content) as SnapshotState;
if (!validRelationEntries(state.relations)) {
throw new Error("relations must contain keyed relation rows");
}

const counts: Record<string, StoreCounts> = {};
const notCaptured: string[] = [];
Expand Down
1 change: 1 addition & 0 deletions src/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -452,6 +452,7 @@ export interface ExportData {
proceduralMemories?: ProceduralMemory[];
signals?: Signal[];
facets?: Facet[];
relations?: Array<{ key: string; value: MemoryRelation }>;
lessons?: Lesson[];
insights?: Insight[];
accessLogs?: AccessLogExport[];
Expand Down
122 changes: 122 additions & 0 deletions test/export-import.test.ts
Original file line number Diff line number Diff line change
@@ -1,10 +1,18 @@
import { describe, it, expect, beforeEach, vi } from "vitest";
import { mkdtempSync, mkdirSync, rmSync } from "node:fs";
import { join } from "node:path";
import { registerSnapshotFunction } from "../src/functions/snapshot.js";
import { pagedReads } from "./helpers/mocks.js";

vi.mock("../src/logger.js", () => ({
logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() },
}));

import { registerExportImportFunction } from "../src/functions/export-import.js";
import { registerRelationsFunction } from "../src/functions/relations.js";
import { registerMeshFunction } from "../src/functions/mesh.js";
import { SqliteState, stateFunctions } from "../src/engine/inproc/state.js";
import { StateKV } from "../src/state/kv.js";
import { VERSION } from "../src/version.js";
import { getSearchIndex } from "../src/functions/search.js";
import type {
Expand Down Expand Up @@ -33,6 +41,7 @@ function mockKV() {
const entries = store.get(scope);
return entries ? (Array.from(entries.values()) as T[]) : [];
},
...pagedReads(store),
bytes: async (scope: string): Promise<number> => {
const entries = store.get(scope);
return entries ? JSON.stringify(Array.from(entries.values())).length : 0;
Expand Down Expand Up @@ -328,6 +337,118 @@ describe("Export/Import Functions", () => {
expect(oldSession).toBeNull();
});

it("round-trips legacy-keyed relations through replace without retaining destination relations", async () => {
const relation = { sourceId: "mem_1", targetId: "mem_2", type: "related", createdAt: "2026-01-01" };
await kv.set("mem:relations", "rel_legacy", relation);
const exported = await sdk.trigger("mem::export", {});
const destination = mockKV();
const target = mockSdk();
registerExportImportFunction(target as never, destination as never);
await destination.set("mem:relations", "rel_stale", { ...relation, targetId: "stale" });
expect(await target.trigger("mem::import", { exportData: exported, strategy: "replace" })).toMatchObject({ success: true });
expect(await destination.list("mem:relations")).toEqual([relation]);
expect(await destination.get("mem:relations", "rel_legacy")).toEqual(relation);
});

it("round-trips relation types accepted by the writer", async () => {
await kv.set("mem:memories", "mem_2", { ...testMemory, id: "mem_2" });
registerRelationsFunction(sdk as never, kv as never);
const created = await sdk.trigger("mem::relate", { sourceId: "mem_1", targetId: "mem_2", type: "custom" });
expect(created.success).toBe(true);
const exported = await sdk.trigger("mem::export", {});
expect(await sdk.trigger("mem::import", { exportData: exported, strategy: "replace" })).toMatchObject({ success: true });
expect(await kv.get("mem:relations", created.relationId)).toMatchObject({ type: "custom" });
});

it("round-trips a non-string type accepted by the relation writer", async () => {
await kv.set("mem:memories", "mem_2", { ...testMemory, id: "mem_2" });
registerRelationsFunction(sdk as never, kv as never);
const created = await sdk.trigger("mem::relate", { sourceId: "mem_1", targetId: "mem_2", type: 42 });
expect(created.success).toBe(true);
const exported = await sdk.trigger("mem::export", {});
expect(exported.relations[0].value.type).toBe("related");
expect(await kv.get("mem:relations", created.relationId)).toMatchObject({ type: 42 });
exported.relations[0].value.type = 42;
expect(await sdk.trigger("mem::import", { exportData: exported, strategy: "replace" })).toMatchObject({ success: true });
expect(await kv.get("mem:relations", created.relationId)).toMatchObject({ type: "related" });
});

it("round-trips opaque endpoints accepted by mesh without changing their values or key", async () => {
registerMeshFunction(sdk as never, kv as never);
const relation = { sourceId: 42, targetId: { id: "target" }, type: "related" };
expect(await sdk.trigger("mem::mesh-receive", { relations: [relation] })).toMatchObject({ success: true, accepted: 1 });
const exported = await sdk.trigger("mem::export", {});
expect(await sdk.trigger("mem::import", { exportData: exported, strategy: "replace" })).toMatchObject({ success: true });
expect(await kv.get("mem:relations", "42:[object Object]:related")).toEqual(relation);
});

it("replaces real SQLite relations using exact NUL-containing storage keys", async () => {
const state = new SqliteState(":memory:");
try {
const handlers = stateFunctions(state);
const realKV = new StateKV({ trigger: async ({ function_id, payload }: { function_id: string; payload: unknown }) => handlers[function_id](payload) } as never);
const realSdk = mockSdk();
registerExportImportFunction(realSdk as never, realKV);
const relation = { sourceId: null, targetId: "b", type: "related" };
await realKV.set("mem:relations", "rel\u0000original", relation);
const exported = await realSdk.trigger("mem::export", {});
expect(exported.relations[0].key).toBe("rel\u0000original");
await realKV.set("mem:relations", "rel\u0000stale", { ...relation, targetId: "stale" });
expect(await realSdk.trigger("mem::import", { exportData: exported, strategy: "replace" })).toMatchObject({ success: true });
expect(await realKV.get("mem:relations", "rel\u0000original")).toEqual(relation);
expect(await realKV.get("mem:relations", "rel\u0000stale")).toBeNull();
expect(await realKV.list("mem:relations")).toEqual([relation]);
} finally {
state.close();
}
});

it("preserves explicit null endpoints in persisted relations", async () => {
const relation = { sourceId: null, targetId: "b", type: "related" };
await kv.set("mem:relations", "rel_null", relation);
const exported = await sdk.trigger("mem::export", {});
expect(await sdk.trigger("mem::import", { exportData: exported, strategy: "replace" })).toMatchObject({ success: true });
expect(await kv.get("mem:relations", "rel_null")).toEqual(relation);
});

it("omits invalid relation confidence in exports and imports without changing the source", async () => {
const value = { sourceId: "a", targetId: "b", type: "custom", createdAt: "2026-01-01", confidence: "bad" };
await kv.set("mem:relations", "rel_bad", value);
const exported = await sdk.trigger("mem::export", {});
expect(exported.relations[0].value).not.toHaveProperty("confidence");
expect(await kv.get("mem:relations", "rel_bad")).toEqual(value);
exported.relations[0].value.confidence = Infinity;
expect(await sdk.trigger("mem::import", { exportData: exported, strategy: "replace" })).toMatchObject({ success: true });
expect(await kv.get("mem:relations", "rel_bad")).not.toHaveProperty("confidence");
});

it("leaves relations untouched when replacing from an older export without them", async () => {
const relation = { sourceId: "a", targetId: "b", type: "related", createdAt: "2026-01-01" };
await kv.set("mem:relations", "rel_old", relation);
const oldExport = { version: VERSION, exportedAt: "2026-01-01", sessions: [], observations: {}, memories: [], summaries: [] };
expect(await sdk.trigger("mem::import", { exportData: oldExport, strategy: "replace" })).toMatchObject({ success: true });
expect(await kv.get("mem:relations", "rel_old")).toEqual(relation);
});

it("restores captured legacy-keyed relations and removes later relations", async () => {
mkdirSync("tmp", { recursive: true });
const dir = mkdtempSync(join(process.cwd(), "tmp", "snapshot-relations-"));
try {
registerSnapshotFunction(sdk as never, kv as never, dir);
const relation = { sourceId: "mem_1", targetId: "mem_2", type: "related", createdAt: "2026-01-01" };
await kv.set("mem:relations", "rel_legacy", relation);
const created = await sdk.trigger("mem::snapshot-create", {});
expect(created.success).toBe(true);
await kv.delete("mem:relations", "rel_legacy");
await kv.set("mem:relations", "rel_stale", { ...relation, targetId: "stale" });
expect(await sdk.trigger("mem::snapshot-restore", { commitHash: created.snapshot.commitHash })).toMatchObject({ success: true });
expect(await kv.list("mem:relations")).toEqual([relation]);
expect(await kv.get("mem:relations", "rel_legacy")).toEqual(relation);
} finally {
rmSync(dir, { recursive: true, force: true });
}
});

it("export then import round-trip preserves data", async () => {
const exported = (await sdk.trigger("mem::export", {})) as ExportData;

Expand Down Expand Up @@ -424,6 +545,7 @@ describe("Export/Import Functions", () => {
});

it.each([
["relations are malformed", { relations: [{ key: "rel_1", value: null }] }],
["accessLogs is not an array", { accessLogs: {} }],
["lessons is not an array", { lessons: {} }],
["insights is not an array", { insights: "x" }],
Expand Down
16 changes: 16 additions & 0 deletions test/inproc-state.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,22 @@ describe("inproc state store", () => {
expect(next).toEqual([{ key: "c", value: "c" }, { key: "d", value: "d" }]);
});

it("pages keys containing NUL without truncating the cursor", async () => {
const fns = stateFunctions(store);
for (const key of ["a", "a\u0000b", "a\u0000c", "b"]) {
await fns["state::set"]({ scope: "s", key, value: key });
}
const keys: string[] = [];
let after: string | undefined;
for (let i = 0; i < 4; i++) {
const page = await fns["state::list-page"]({ scope: "s", after, limit: 1 }) as Array<{ key: string }>;
expect(page).toHaveLength(1);
keys.push(page[0].key);
after = page[0].key;
}
expect(keys).toEqual(["a", "a\u0000b", "a\u0000c", "b"]);
});

it("counts exactly the current rows in one scope", async () => {
const fns = stateFunctions(store);
expect(await fns["state::count"]({ scope: "s" })).toBe(0);
Expand Down
2 changes: 2 additions & 0 deletions test/snapshot-prune.test.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { describe, it, expect, beforeEach, afterEach } from "vitest";
import { pagedReads } from "./helpers/mocks.js";
import { execFileSync } from "node:child_process";
import { mkdirSync, mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
Expand All @@ -19,6 +20,7 @@ function mockKV() {
delete: async (scope: string, key: string): Promise<void> => {
store.get(scope)?.delete(key);
},
...pagedReads(store),
list: async <T>(scope: string): Promise<T[]> => {
const entries = store.get(scope);
return entries ? (Array.from(entries.values()) as T[]) : [];
Expand Down
Loading