diff --git a/CLAUDE.md b/CLAUDE.md index 07afc76354..acf1d49a97 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -257,6 +257,16 @@ before any store mutation. Rows without string keys are dropped and counted in ` legacy keyless Session stubs do not prevent restoring the rest. Access logs and legacy Memory session IDs retain their normal normalization. This preflight does not make replacement transactional. +### Relation backups +Full exports and snapshots carry Memory relations with their original storage keys, including legacy +`rel_` keys. Replace import and snapshot restore replace that scope when the backup contains it; +older backups without a `relations` section leave the scope untouched. Import and snapshot restore +validate relation entries before replacing any scope, preserve present endpoint values as stored +(including explicit nulls and non-strings accepted by mesh), retain string relation types, map +non-string types to `related`, and omit nonnumeric or nonfinite confidence from backup data without +changing the source row. `StateKV.listPage` returns complete SQLite keys, including embedded NULs, +so those keys remain valid paging cursors and can be restored exactly. + ### Graph Eviction Every path that hard-deletes Observations or Memories (`mem::evict`, `mem::observe` session-cap eviction, `mem::auto-forget`, `mem::forget`, `mem::retention-evict`, `mem::governance-delete`, diff --git a/src/engine/inproc/state.ts b/src/engine/inproc/state.ts index b6e31ab29f..3ef2666a8b 100644 --- a/src/engine/inproc/state.ts +++ b/src/engine/inproc/state.ts @@ -306,9 +306,9 @@ export class SqliteState { listPage(scope: string, after: string | undefined, limit: number): Array<{ key: string; value: unknown }> { const rows = this.db - .prepare("SELECT key, value FROM kv WHERE scope = ? AND key > ? ORDER BY key LIMIT ?") - .all(scope, after ?? "", limit) as Array<{ key: string; value: string }>; - return rows.map((r) => ({ key: r.key, value: JSON.parse(r.value) })); + .prepare("SELECT CAST(key AS BLOB) AS key, value FROM kv WHERE scope = ? AND key > ? ORDER BY kv.key LIMIT ?") + .all(scope, after ?? "", limit) as Array<{ key: Uint8Array; value: string }>; + return rows.map((r) => ({ key: Buffer.from(r.key).toString("utf8"), value: JSON.parse(r.value) })); } listNewest( diff --git a/src/functions/export-import.ts b/src/functions/export-import.ts index b1981f00f6..060377469c 100644 --- a/src/functions/export-import.ts +++ b/src/functions/export-import.ts @@ -77,6 +77,7 @@ export const DURABLE_STORES: readonly DurableStore[] = [ { field: "semanticMemories", scope: KV.semantic, keyOf: byId }, { field: "proceduralMemories", scope: KV.procedural, keyOf: byId }, { field: "facets", scope: KV.facets, keyOf: byId }, + { field: "relations", scope: KV.relations, keyOf: (row) => String(row.key) }, { field: "lessons", scope: KV.lessons, keyOf: byId }, { field: "insights", scope: KV.insights, keyOf: byId }, { field: "signals", scope: KV.signals, keyOf: byId }, @@ -95,10 +96,34 @@ function readableStores(): DurableStore[] { // fields are absent rather than empty. export async function readDurableStores(kv: StateKV): Promise { const readable = readableStores(); - const lists = await Promise.all(readable.map((s) => kv.list(s.scope))); + const lists = await Promise.all(readable.map((s) => + s.field === "relations" ? relationEntries(kv) : kv.list(s.scope), + )); return Object.fromEntries(readable.map((s, i) => [s.field, lists[i]])); } +export function normalizeRelation(value: unknown): unknown { + if (!value || typeof value !== "object" || Array.isArray(value)) return value; + const row = value as Record; + const normalized = { ...row }; + if ("type" in normalized && typeof normalized.type !== "string") normalized.type = "related"; + if ("confidence" in normalized && (typeof normalized.confidence !== "number" || !Number.isFinite(normalized.confidence))) { + delete normalized.confidence; + } + return normalized; +} + +export async function relationEntries(kv: StateKV): Promise> { + const entries: Array<{ key: string; value: unknown }> = []; + let after: string | undefined; + for (;;) { + const page = await kv.listPage(KV.relations, after, 1000); + entries.push(...page.map(({ key, value }) => ({ key, value: normalizeRelation(value) }))); + if (page.length < 1000) return entries; + after = page[page.length - 1].key; + } +} + function nonEmpty(rows: T[] | undefined): T[] | undefined { return rows && rows.length > 0 ? rows : undefined; } @@ -128,6 +153,20 @@ function isKeyedRow(row: unknown, key: string): boolean { ); } +export function validRelationEntries(value: unknown): boolean { + if (value === undefined) return true; + if (!Array.isArray(value)) return false; + const keys = new Set(); + return value.every((entry) => { + if (!entry || typeof entry !== "object" || typeof entry.key !== "string" || !entry.key || + keys.has(entry.key) || !entry.value || typeof entry.value !== "object" || Array.isArray(entry.value) || + !("sourceId" in entry.value) || !("targetId" in entry.value) || + !("type" in entry.value)) return false; + keys.add(entry.key); + return true; + }); +} + // A replace import deletes every store before it writes, so a malformed // section must fail first, leaving the store untouched. function invalidImportSection(importData: ExportData): string | null { @@ -135,6 +174,7 @@ function invalidImportSection(importData: ExportData): string | null { const rows: unknown = importData[section]; if (rows != null && !Array.isArray(rows)) return `${section} must be an array`; } + if (!validRelationEntries(importData.relations)) return "relations must contain keyed relation rows"; const accessLogs: unknown = importData.accessLogs; if (accessLogs != null && !Array.isArray(accessLogs)) return "accessLogs must be an array"; if (Array.isArray(accessLogs) && accessLogs.length > MAX_ACCESS_LOGS) { @@ -283,6 +323,7 @@ export function registerExportImportFunction(sdk: ISdk, kv: StateKV): void { semanticMemories: nonEmpty(stores.semanticMemories), proceduralMemories: nonEmpty(stores.proceduralMemories), facets: nonEmpty(stores.facets), + relations: stores.relations, lessons: nonEmpty(stores.lessons), insights: nonEmpty(stores.insights), signals: nonEmpty(stores.signals), @@ -459,6 +500,9 @@ export function registerExportImportFunction(sdk: ISdk, kv: StateKV): void { (l) => kv.delete(KV.lessons, l.id), ); resetLessonIndex(); + if (importData.relations) { + await runChunked(await relationEntries(kv), ({ key }) => kv.delete(KV.relations, key)); + } await runChunked( await kv.list(KV.insights).catch(() => []), (i) => kv.delete(KV.insights, i.id), @@ -636,6 +680,12 @@ export function registerExportImportFunction(sdk: ISdk, kv: StateKV): void { await kv.set(KV.signals, signal.id, signal); }); } + if (importData.relations) { + await runChunked(importData.relations, async ({ key, value }) => { + if (strategy === "skip" && await kv.get(KV.relations, key)) { stats.skipped++; return; } + await kv.set(KV.relations, key, normalizeRelation(value)); + }); + } if (importData.facets) { await runChunked(importData.facets, async (facet) => { if (strategy === "skip") { diff --git a/src/functions/snapshot.ts b/src/functions/snapshot.ts index 63487eff89..e82a3ff59d 100644 --- a/src/functions/snapshot.ts +++ b/src/functions/snapshot.ts @@ -19,6 +19,9 @@ import { graphWritesDisabled } from "./graph.js"; import { DURABLE_STORES, readDurableStores, + relationEntries, + normalizeRelation, + validRelationEntries, runChunked, type DurableStore, type DurableStoreField, @@ -104,7 +107,9 @@ async function replaceStore( const rows = store.field === "lessons" ? captured.filter((r) => r.deleted !== true) : captured; const keep = new Set(rows.map(store.keyOf)); - const existing = await kv.list(store.scope); + const existing = store.field === "relations" + ? await relationEntries(kv) + : await kv.list(store.scope); const stale = existing.map(store.keyOf).filter((key) => !keep.has(key)); await runChunked(stale, (key) => @@ -123,7 +128,7 @@ async function replaceStore( await kv.set(store.scope, edge.id, edge); await indexGraphEdge(kv, edge); } else { - await kv.set(store.scope, store.keyOf(row), row); + await kv.set(store.scope, store.keyOf(row), store.field === "relations" ? normalizeRelation(row.value) : row); } }); @@ -322,6 +327,9 @@ export function registerSnapshotFunction( ]); const content = readFileSync(join(snapshotDir, "state.json"), "utf-8"); const state = JSON.parse(content) as SnapshotState; + if (!validRelationEntries(state.relations)) { + throw new Error("relations must contain keyed relation rows"); + } const counts: Record = {}; const notCaptured: string[] = []; diff --git a/src/types.ts b/src/types.ts index e53a7ab1f0..e10d9e7ad5 100644 --- a/src/types.ts +++ b/src/types.ts @@ -452,6 +452,7 @@ export interface ExportData { proceduralMemories?: ProceduralMemory[]; signals?: Signal[]; facets?: Facet[]; + relations?: Array<{ key: string; value: MemoryRelation }>; lessons?: Lesson[]; insights?: Insight[]; accessLogs?: AccessLogExport[]; diff --git a/test/export-import.test.ts b/test/export-import.test.ts index 7f7e172857..db581697ef 100644 --- a/test/export-import.test.ts +++ b/test/export-import.test.ts @@ -1,10 +1,18 @@ import { describe, it, expect, beforeEach, vi } from "vitest"; +import { mkdtempSync, mkdirSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { registerSnapshotFunction } from "../src/functions/snapshot.js"; +import { pagedReads } from "./helpers/mocks.js"; vi.mock("../src/logger.js", () => ({ logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, })); import { registerExportImportFunction } from "../src/functions/export-import.js"; +import { registerRelationsFunction } from "../src/functions/relations.js"; +import { registerMeshFunction } from "../src/functions/mesh.js"; +import { SqliteState, stateFunctions } from "../src/engine/inproc/state.js"; +import { StateKV } from "../src/state/kv.js"; import { VERSION } from "../src/version.js"; import { getSearchIndex } from "../src/functions/search.js"; import type { @@ -33,6 +41,7 @@ function mockKV() { const entries = store.get(scope); return entries ? (Array.from(entries.values()) as T[]) : []; }, + ...pagedReads(store), bytes: async (scope: string): Promise => { const entries = store.get(scope); return entries ? JSON.stringify(Array.from(entries.values())).length : 0; @@ -328,6 +337,118 @@ describe("Export/Import Functions", () => { expect(oldSession).toBeNull(); }); + it("round-trips legacy-keyed relations through replace without retaining destination relations", async () => { + const relation = { sourceId: "mem_1", targetId: "mem_2", type: "related", createdAt: "2026-01-01" }; + await kv.set("mem:relations", "rel_legacy", relation); + const exported = await sdk.trigger("mem::export", {}); + const destination = mockKV(); + const target = mockSdk(); + registerExportImportFunction(target as never, destination as never); + await destination.set("mem:relations", "rel_stale", { ...relation, targetId: "stale" }); + expect(await target.trigger("mem::import", { exportData: exported, strategy: "replace" })).toMatchObject({ success: true }); + expect(await destination.list("mem:relations")).toEqual([relation]); + expect(await destination.get("mem:relations", "rel_legacy")).toEqual(relation); + }); + + it("round-trips relation types accepted by the writer", async () => { + await kv.set("mem:memories", "mem_2", { ...testMemory, id: "mem_2" }); + registerRelationsFunction(sdk as never, kv as never); + const created = await sdk.trigger("mem::relate", { sourceId: "mem_1", targetId: "mem_2", type: "custom" }); + expect(created.success).toBe(true); + const exported = await sdk.trigger("mem::export", {}); + expect(await sdk.trigger("mem::import", { exportData: exported, strategy: "replace" })).toMatchObject({ success: true }); + expect(await kv.get("mem:relations", created.relationId)).toMatchObject({ type: "custom" }); + }); + + it("round-trips a non-string type accepted by the relation writer", async () => { + await kv.set("mem:memories", "mem_2", { ...testMemory, id: "mem_2" }); + registerRelationsFunction(sdk as never, kv as never); + const created = await sdk.trigger("mem::relate", { sourceId: "mem_1", targetId: "mem_2", type: 42 }); + expect(created.success).toBe(true); + const exported = await sdk.trigger("mem::export", {}); + expect(exported.relations[0].value.type).toBe("related"); + expect(await kv.get("mem:relations", created.relationId)).toMatchObject({ type: 42 }); + exported.relations[0].value.type = 42; + expect(await sdk.trigger("mem::import", { exportData: exported, strategy: "replace" })).toMatchObject({ success: true }); + expect(await kv.get("mem:relations", created.relationId)).toMatchObject({ type: "related" }); + }); + + it("round-trips opaque endpoints accepted by mesh without changing their values or key", async () => { + registerMeshFunction(sdk as never, kv as never); + const relation = { sourceId: 42, targetId: { id: "target" }, type: "related" }; + expect(await sdk.trigger("mem::mesh-receive", { relations: [relation] })).toMatchObject({ success: true, accepted: 1 }); + const exported = await sdk.trigger("mem::export", {}); + expect(await sdk.trigger("mem::import", { exportData: exported, strategy: "replace" })).toMatchObject({ success: true }); + expect(await kv.get("mem:relations", "42:[object Object]:related")).toEqual(relation); + }); + + it("replaces real SQLite relations using exact NUL-containing storage keys", async () => { + const state = new SqliteState(":memory:"); + try { + const handlers = stateFunctions(state); + const realKV = new StateKV({ trigger: async ({ function_id, payload }: { function_id: string; payload: unknown }) => handlers[function_id](payload) } as never); + const realSdk = mockSdk(); + registerExportImportFunction(realSdk as never, realKV); + const relation = { sourceId: null, targetId: "b", type: "related" }; + await realKV.set("mem:relations", "rel\u0000original", relation); + const exported = await realSdk.trigger("mem::export", {}); + expect(exported.relations[0].key).toBe("rel\u0000original"); + await realKV.set("mem:relations", "rel\u0000stale", { ...relation, targetId: "stale" }); + expect(await realSdk.trigger("mem::import", { exportData: exported, strategy: "replace" })).toMatchObject({ success: true }); + expect(await realKV.get("mem:relations", "rel\u0000original")).toEqual(relation); + expect(await realKV.get("mem:relations", "rel\u0000stale")).toBeNull(); + expect(await realKV.list("mem:relations")).toEqual([relation]); + } finally { + state.close(); + } + }); + + it("preserves explicit null endpoints in persisted relations", async () => { + const relation = { sourceId: null, targetId: "b", type: "related" }; + await kv.set("mem:relations", "rel_null", relation); + const exported = await sdk.trigger("mem::export", {}); + expect(await sdk.trigger("mem::import", { exportData: exported, strategy: "replace" })).toMatchObject({ success: true }); + expect(await kv.get("mem:relations", "rel_null")).toEqual(relation); + }); + + it("omits invalid relation confidence in exports and imports without changing the source", async () => { + const value = { sourceId: "a", targetId: "b", type: "custom", createdAt: "2026-01-01", confidence: "bad" }; + await kv.set("mem:relations", "rel_bad", value); + const exported = await sdk.trigger("mem::export", {}); + expect(exported.relations[0].value).not.toHaveProperty("confidence"); + expect(await kv.get("mem:relations", "rel_bad")).toEqual(value); + exported.relations[0].value.confidence = Infinity; + expect(await sdk.trigger("mem::import", { exportData: exported, strategy: "replace" })).toMatchObject({ success: true }); + expect(await kv.get("mem:relations", "rel_bad")).not.toHaveProperty("confidence"); + }); + + it("leaves relations untouched when replacing from an older export without them", async () => { + const relation = { sourceId: "a", targetId: "b", type: "related", createdAt: "2026-01-01" }; + await kv.set("mem:relations", "rel_old", relation); + const oldExport = { version: VERSION, exportedAt: "2026-01-01", sessions: [], observations: {}, memories: [], summaries: [] }; + expect(await sdk.trigger("mem::import", { exportData: oldExport, strategy: "replace" })).toMatchObject({ success: true }); + expect(await kv.get("mem:relations", "rel_old")).toEqual(relation); + }); + + it("restores captured legacy-keyed relations and removes later relations", async () => { + mkdirSync("tmp", { recursive: true }); + const dir = mkdtempSync(join(process.cwd(), "tmp", "snapshot-relations-")); + try { + registerSnapshotFunction(sdk as never, kv as never, dir); + const relation = { sourceId: "mem_1", targetId: "mem_2", type: "related", createdAt: "2026-01-01" }; + await kv.set("mem:relations", "rel_legacy", relation); + const created = await sdk.trigger("mem::snapshot-create", {}); + expect(created.success).toBe(true); + await kv.delete("mem:relations", "rel_legacy"); + await kv.set("mem:relations", "rel_stale", { ...relation, targetId: "stale" }); + expect(await sdk.trigger("mem::snapshot-restore", { commitHash: created.snapshot.commitHash })).toMatchObject({ success: true }); + expect(await kv.list("mem:relations")).toEqual([relation]); + expect(await kv.get("mem:relations", "rel_legacy")).toEqual(relation); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + it("export then import round-trip preserves data", async () => { const exported = (await sdk.trigger("mem::export", {})) as ExportData; @@ -424,6 +545,7 @@ describe("Export/Import Functions", () => { }); it.each([ + ["relations are malformed", { relations: [{ key: "rel_1", value: null }] }], ["accessLogs is not an array", { accessLogs: {} }], ["lessons is not an array", { lessons: {} }], ["insights is not an array", { insights: "x" }], diff --git a/test/inproc-state.test.ts b/test/inproc-state.test.ts index b5f5452a3a..93329705b6 100644 --- a/test/inproc-state.test.ts +++ b/test/inproc-state.test.ts @@ -123,6 +123,22 @@ describe("inproc state store", () => { expect(next).toEqual([{ key: "c", value: "c" }, { key: "d", value: "d" }]); }); + it("pages keys containing NUL without truncating the cursor", async () => { + const fns = stateFunctions(store); + for (const key of ["a", "a\u0000b", "a\u0000c", "b"]) { + await fns["state::set"]({ scope: "s", key, value: key }); + } + const keys: string[] = []; + let after: string | undefined; + for (let i = 0; i < 4; i++) { + const page = await fns["state::list-page"]({ scope: "s", after, limit: 1 }) as Array<{ key: string }>; + expect(page).toHaveLength(1); + keys.push(page[0].key); + after = page[0].key; + } + expect(keys).toEqual(["a", "a\u0000b", "a\u0000c", "b"]); + }); + it("counts exactly the current rows in one scope", async () => { const fns = stateFunctions(store); expect(await fns["state::count"]({ scope: "s" })).toBe(0); diff --git a/test/snapshot-prune.test.ts b/test/snapshot-prune.test.ts index 92704a9ba5..bf92fe4e61 100644 --- a/test/snapshot-prune.test.ts +++ b/test/snapshot-prune.test.ts @@ -1,4 +1,5 @@ import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { pagedReads } from "./helpers/mocks.js"; import { execFileSync } from "node:child_process"; import { mkdirSync, mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; @@ -19,6 +20,7 @@ function mockKV() { delete: async (scope: string, key: string): Promise => { store.get(scope)?.delete(key); }, + ...pagedReads(store), list: async (scope: string): Promise => { const entries = store.get(scope); return entries ? (Array.from(entries.values()) as T[]) : []; diff --git a/test/snapshot.test.ts b/test/snapshot.test.ts index 3263dc9e26..7890eb5713 100644 --- a/test/snapshot.test.ts +++ b/test/snapshot.test.ts @@ -1,4 +1,5 @@ import { describe, it, expect, beforeEach, vi } from "vitest"; +import { pagedReads } from "./helpers/mocks.js"; vi.mock("../src/logger.js", () => ({ logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, @@ -65,6 +66,7 @@ function mockKV() { delete: async (scope: string, key: string): Promise => { store.get(scope)?.delete(key); }, + ...pagedReads(store), list: async (scope: string): Promise => { const entries = store.get(scope); return entries ? (Array.from(entries.values()) as T[]) : []; @@ -257,6 +259,21 @@ describe("snapshot-restore replaces each captured store", () => { getSearchIndex().add(memoryToIndexDoc(memory("mem_1", "Original"))); }); + it("rejects malformed relation entries before replacing any captured store", async () => { + snapshotFile.content = JSON.stringify({ sessions: [], memories: [], observations: {}, relations: [{ key: "bad", value: null }] }); + await kv.set("mem:relations", "rel_existing", { sourceId: "a", targetId: "b", type: "custom" }); + const result = await sdk.trigger("mem::snapshot-restore", { commitHash: "abc1234" }); + expect(result).toMatchObject({ success: false }); + expect(await kv.get("mem:memories", "mem_1")).not.toBeNull(); + expect(await kv.get("mem:relations", "rel_existing")).not.toBeNull(); + }); + + it("normalizes invalid type and confidence when restoring a captured relation", async () => { + snapshotFile.content = JSON.stringify({ relations: [{ key: "rel_old", value: { sourceId: "a", targetId: "b", type: 42, confidence: "bad" } }] }); + expect(await sdk.trigger("mem::snapshot-restore", { commitHash: "abc1234" })).toMatchObject({ success: true }); + expect(await kv.get("mem:relations", "rel_old")).toEqual({ sourceId: "a", targetId: "b", type: "related" }); + }); + it("removes rows added after the snapshot and brings back snapshot-time rows", async () => { await sdk.trigger("mem::snapshot-create", { message: "before" }); @@ -416,6 +433,7 @@ describe("snapshot-create reentrancy guard", () => { return data; }, delete: async () => {}, + listPage: async () => [], list: async (scope: string): Promise => { listCalls++; // Park the first snapshot inside its initial list() so a second