From cc17c90bfab444c25e11f9df5ccc042d6c504d1d Mon Sep 17 00:00:00 2001 From: neal <130699270+possiblyneal@users.noreply.github.com> Date: Sat, 21 Nov 2026 15:31:15 -0500 Subject: [PATCH 1/6] Preserve memory relations in backups Generated-By: gpt-6.1-terra --- CLAUDE.md | 6 +++++ src/functions/export-import.ts | 32 ++++++++++++++++++++++- src/functions/snapshot.ts | 7 +++-- src/types.ts | 1 + test/export-import.test.ts | 48 ++++++++++++++++++++++++++++++++++ test/snapshot-prune.test.ts | 4 +++ test/snapshot.test.ts | 5 ++++ 7 files changed, 100 insertions(+), 3 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 07afc76354..000d516e71 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -257,6 +257,12 @@ before any store mutation. Rows without string keys are dropped and counted in ` legacy keyless Session stubs do not prevent restoring the rest. Access logs and legacy Memory session IDs retain their normal normalization. This preflight does not make replacement transactional. +### Relation backups +Full exports and snapshots carry Memory relations with their original storage keys, including legacy +`rel_` keys. Replace import and snapshot restore replace that scope when the backup contains it; +older backups without a `relations` section leave the scope untouched. Import validates relation +entries before a replace can delete anything. + ### Graph Eviction Every path that hard-deletes Observations or Memories (`mem::evict`, `mem::observe` session-cap eviction, `mem::auto-forget`, `mem::forget`, `mem::retention-evict`, `mem::governance-delete`, diff --git a/src/functions/export-import.ts b/src/functions/export-import.ts index b1981f00f6..601afe0e50 100644 --- a/src/functions/export-import.ts +++ b/src/functions/export-import.ts @@ -77,6 +77,7 @@ export const DURABLE_STORES: readonly DurableStore[] = [ { field: "semanticMemories", scope: KV.semantic, keyOf: byId }, { field: "proceduralMemories", scope: KV.procedural, keyOf: byId }, { field: "facets", scope: KV.facets, keyOf: byId }, + { field: "relations", scope: KV.relations, keyOf: (row) => String(row.key) }, { field: "lessons", scope: KV.lessons, keyOf: byId }, { field: "insights", scope: KV.insights, keyOf: byId }, { field: "signals", scope: KV.signals, keyOf: byId }, @@ -95,10 +96,23 @@ function readableStores(): DurableStore[] { // fields are absent rather than empty. export async function readDurableStores(kv: StateKV): Promise { const readable = readableStores(); - const lists = await Promise.all(readable.map((s) => kv.list(s.scope))); + const lists = await Promise.all(readable.map((s) => + s.field === "relations" ? relationEntries(kv) : kv.list(s.scope), + )); return Object.fromEntries(readable.map((s, i) => [s.field, lists[i]])); } +export async function relationEntries(kv: StateKV): Promise> { + const entries: Array<{ key: string; value: unknown }> = []; + let after: string | undefined; + for (;;) { + const page = await kv.listPage(KV.relations, after, 1000); + entries.push(...page); + if (page.length < 1000) return entries; + after = page[page.length - 1].key; + } +} + function nonEmpty(rows: T[] | undefined): T[] | undefined { return rows && rows.length > 0 ? rows : undefined; } @@ -135,6 +149,12 @@ function invalidImportSection(importData: ExportData): string | null { const rows: unknown = importData[section]; if (rows != null && !Array.isArray(rows)) return `${section} must be an array`; } + const relations: unknown = importData.relations; + if (relations != null && (!Array.isArray(relations) || relations.some((entry) => + !entry || typeof entry.key !== "string" || !entry.key || !entry.value || + typeof entry.value.sourceId !== "string" || typeof entry.value.targetId !== "string" || + !["supersedes", "extends", "derives", "contradicts", "related"].includes(entry.value.type) + ))) return "relations must contain keyed relation rows"; const accessLogs: unknown = importData.accessLogs; if (accessLogs != null && !Array.isArray(accessLogs)) return "accessLogs must be an array"; if (Array.isArray(accessLogs) && accessLogs.length > MAX_ACCESS_LOGS) { @@ -283,6 +303,7 @@ export function registerExportImportFunction(sdk: ISdk, kv: StateKV): void { semanticMemories: nonEmpty(stores.semanticMemories), proceduralMemories: nonEmpty(stores.proceduralMemories), facets: nonEmpty(stores.facets), + relations: stores.relations, lessons: nonEmpty(stores.lessons), insights: nonEmpty(stores.insights), signals: nonEmpty(stores.signals), @@ -459,6 +480,9 @@ export function registerExportImportFunction(sdk: ISdk, kv: StateKV): void { (l) => kv.delete(KV.lessons, l.id), ); resetLessonIndex(); + if (importData.relations) { + await runChunked(await relationEntries(kv), ({ key }) => kv.delete(KV.relations, key)); + } await runChunked( await kv.list(KV.insights).catch(() => []), (i) => kv.delete(KV.insights, i.id), @@ -636,6 +660,12 @@ export function registerExportImportFunction(sdk: ISdk, kv: StateKV): void { await kv.set(KV.signals, signal.id, signal); }); } + if (importData.relations) { + await runChunked(importData.relations, async ({ key, value }) => { + if (strategy === "skip" && await kv.get(KV.relations, key)) { stats.skipped++; return; } + await kv.set(KV.relations, key, value); + }); + } if (importData.facets) { await runChunked(importData.facets, async (facet) => { if (strategy === "skip") { diff --git a/src/functions/snapshot.ts b/src/functions/snapshot.ts index 63487eff89..0d35c6cc4f 100644 --- a/src/functions/snapshot.ts +++ b/src/functions/snapshot.ts @@ -19,6 +19,7 @@ import { graphWritesDisabled } from "./graph.js"; import { DURABLE_STORES, readDurableStores, + relationEntries, runChunked, type DurableStore, type DurableStoreField, @@ -104,7 +105,9 @@ async function replaceStore( const rows = store.field === "lessons" ? captured.filter((r) => r.deleted !== true) : captured; const keep = new Set(rows.map(store.keyOf)); - const existing = await kv.list(store.scope); + const existing = store.field === "relations" + ? await relationEntries(kv) + : await kv.list(store.scope); const stale = existing.map(store.keyOf).filter((key) => !keep.has(key)); await runChunked(stale, (key) => @@ -123,7 +126,7 @@ async function replaceStore( await kv.set(store.scope, edge.id, edge); await indexGraphEdge(kv, edge); } else { - await kv.set(store.scope, store.keyOf(row), row); + await kv.set(store.scope, store.keyOf(row), store.field === "relations" ? row.value : row); } }); diff --git a/src/types.ts b/src/types.ts index e53a7ab1f0..e10d9e7ad5 100644 --- a/src/types.ts +++ b/src/types.ts @@ -452,6 +452,7 @@ export interface ExportData { proceduralMemories?: ProceduralMemory[]; signals?: Signal[]; facets?: Facet[]; + relations?: Array<{ key: string; value: MemoryRelation }>; lessons?: Lesson[]; insights?: Insight[]; accessLogs?: AccessLogExport[]; diff --git a/test/export-import.test.ts b/test/export-import.test.ts index 7f7e172857..f2cb7d9fe3 100644 --- a/test/export-import.test.ts +++ b/test/export-import.test.ts @@ -1,4 +1,7 @@ import { describe, it, expect, beforeEach, vi } from "vitest"; +import { mkdtempSync, mkdirSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { registerSnapshotFunction } from "../src/functions/snapshot.js"; vi.mock("../src/logger.js", () => ({ logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, @@ -33,6 +36,10 @@ function mockKV() { const entries = store.get(scope); return entries ? (Array.from(entries.values()) as T[]) : []; }, + listPage: async (scope: string, after: string | undefined, limit: number): Promise> => + Array.from(store.get(scope)?.entries() ?? []).sort(([a], [b]) => a.localeCompare(b)) + .filter(([key]) => after === undefined || key > after).slice(0, limit) + .map(([key, value]) => ({ key, value: value as T })), bytes: async (scope: string): Promise => { const entries = store.get(scope); return entries ? JSON.stringify(Array.from(entries.values())).length : 0; @@ -328,6 +335,46 @@ describe("Export/Import Functions", () => { expect(oldSession).toBeNull(); }); + it("round-trips legacy-keyed relations through replace without retaining destination relations", async () => { + const relation = { sourceId: "mem_1", targetId: "mem_2", type: "related", createdAt: "2026-01-01" }; + await kv.set("mem:relations", "rel_legacy", relation); + const exported = await sdk.trigger("mem::export", {}); + const destination = mockKV(); + const target = mockSdk(); + registerExportImportFunction(target as never, destination as never); + await destination.set("mem:relations", "rel_stale", { ...relation, targetId: "stale" }); + expect(await target.trigger("mem::import", { exportData: exported, strategy: "replace" })).toMatchObject({ success: true }); + expect(await destination.list("mem:relations")).toEqual([relation]); + expect(await destination.get("mem:relations", "rel_legacy")).toEqual(relation); + }); + + it("leaves relations untouched when replacing from an older export without them", async () => { + const relation = { sourceId: "a", targetId: "b", type: "related", createdAt: "2026-01-01" }; + await kv.set("mem:relations", "rel_old", relation); + const oldExport = { version: VERSION, exportedAt: "2026-01-01", sessions: [], observations: {}, memories: [], summaries: [] }; + expect(await sdk.trigger("mem::import", { exportData: oldExport, strategy: "replace" })).toMatchObject({ success: true }); + expect(await kv.get("mem:relations", "rel_old")).toEqual(relation); + }); + + it("restores captured legacy-keyed relations and removes later relations", async () => { + mkdirSync("tmp", { recursive: true }); + const dir = mkdtempSync(join(process.cwd(), "tmp", "snapshot-relations-")); + try { + registerSnapshotFunction(sdk as never, kv as never, dir); + const relation = { sourceId: "mem_1", targetId: "mem_2", type: "related", createdAt: "2026-01-01" }; + await kv.set("mem:relations", "rel_legacy", relation); + const created = await sdk.trigger("mem::snapshot-create", {}); + expect(created.success).toBe(true); + await kv.delete("mem:relations", "rel_legacy"); + await kv.set("mem:relations", "rel_stale", { ...relation, targetId: "stale" }); + expect(await sdk.trigger("mem::snapshot-restore", { commitHash: created.snapshot.commitHash })).toMatchObject({ success: true }); + expect(await kv.list("mem:relations")).toEqual([relation]); + expect(await kv.get("mem:relations", "rel_legacy")).toEqual(relation); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + it("export then import round-trip preserves data", async () => { const exported = (await sdk.trigger("mem::export", {})) as ExportData; @@ -424,6 +471,7 @@ describe("Export/Import Functions", () => { }); it.each([ + ["relations are malformed", { relations: [{ key: "rel_1", value: null }] }], ["accessLogs is not an array", { accessLogs: {} }], ["lessons is not an array", { lessons: {} }], ["insights is not an array", { insights: "x" }], diff --git a/test/snapshot-prune.test.ts b/test/snapshot-prune.test.ts index 92704a9ba5..a35972f337 100644 --- a/test/snapshot-prune.test.ts +++ b/test/snapshot-prune.test.ts @@ -19,6 +19,10 @@ function mockKV() { delete: async (scope: string, key: string): Promise => { store.get(scope)?.delete(key); }, + listPage: async (scope: string, after: string | undefined, limit: number): Promise> => + Array.from(store.get(scope)?.entries() ?? []).sort(([a], [b]) => a.localeCompare(b)) + .filter(([key]) => after === undefined || key > after).slice(0, limit) + .map(([key, value]) => ({ key, value: value as T })), list: async (scope: string): Promise => { const entries = store.get(scope); return entries ? (Array.from(entries.values()) as T[]) : []; diff --git a/test/snapshot.test.ts b/test/snapshot.test.ts index 3263dc9e26..4b5fd2fdf0 100644 --- a/test/snapshot.test.ts +++ b/test/snapshot.test.ts @@ -65,6 +65,10 @@ function mockKV() { delete: async (scope: string, key: string): Promise => { store.get(scope)?.delete(key); }, + listPage: async (scope: string, after: string | undefined, limit: number): Promise> => + Array.from(store.get(scope)?.entries() ?? []).sort(([a], [b]) => a.localeCompare(b)) + .filter(([key]) => after === undefined || key > after).slice(0, limit) + .map(([key, value]) => ({ key, value: value as T })), list: async (scope: string): Promise => { const entries = store.get(scope); return entries ? (Array.from(entries.values()) as T[]) : []; @@ -416,6 +420,7 @@ describe("snapshot-create reentrancy guard", () => { return data; }, delete: async () => {}, + listPage: async () => [], list: async (scope: string): Promise => { listCalls++; // Park the first snapshot inside its initial list() so a second From 2b820231a9b23f6a4c2e805d723b847b71c90ad4 Mon Sep 17 00:00:00 2001 From: neal <130699270+possiblyneal@users.noreply.github.com> Date: Sat, 21 Nov 2026 15:54:41 -0500 Subject: [PATCH 2/6] Validate and normalize relation backups Generated-By: gpt-6.1-terra --- CLAUDE.md | 5 +++-- src/functions/export-import.ts | 33 +++++++++++++++++++++++++-------- src/functions/snapshot.ts | 7 ++++++- test/export-import.test.ts | 28 ++++++++++++++++++++++++---- test/snapshot-prune.test.ts | 6 ++---- test/snapshot.test.ts | 21 +++++++++++++++++---- 6 files changed, 77 insertions(+), 23 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 000d516e71..edbd547645 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -260,8 +260,9 @@ session IDs retain their normal normalization. This preflight does not make repl ### Relation backups Full exports and snapshots carry Memory relations with their original storage keys, including legacy `rel_` keys. Replace import and snapshot restore replace that scope when the backup contains it; -older backups without a `relations` section leave the scope untouched. Import validates relation -entries before a replace can delete anything. +older backups without a `relations` section leave the scope untouched. Import and snapshot restore +validate relation entries before replacing any scope, accept any string relation type, and omit +nonnumeric or nonfinite confidence from backup data without changing the source row. ### Graph Eviction Every path that hard-deletes Observations or Memories (`mem::evict`, `mem::observe` session-cap diff --git a/src/functions/export-import.ts b/src/functions/export-import.ts index 601afe0e50..b3eae0fcce 100644 --- a/src/functions/export-import.ts +++ b/src/functions/export-import.ts @@ -102,12 +102,20 @@ export async function readDurableStores(kv: StateKV): Promise { return Object.fromEntries(readable.map((s, i) => [s.field, lists[i]])); } +export function normalizeRelation(value: unknown): unknown { + if (!value || typeof value !== "object" || Array.isArray(value)) return value; + const row = value as Record; + if (!("confidence" in row) || (typeof row.confidence === "number" && Number.isFinite(row.confidence))) return value; + const { confidence: _confidence, ...rest } = row; + return rest; +} + export async function relationEntries(kv: StateKV): Promise> { const entries: Array<{ key: string; value: unknown }> = []; let after: string | undefined; for (;;) { const page = await kv.listPage(KV.relations, after, 1000); - entries.push(...page); + entries.push(...page.map(({ key, value }) => ({ key, value: normalizeRelation(value) }))); if (page.length < 1000) return entries; after = page[page.length - 1].key; } @@ -142,6 +150,20 @@ function isKeyedRow(row: unknown, key: string): boolean { ); } +export function validRelationEntries(value: unknown): boolean { + if (value === undefined) return true; + if (!Array.isArray(value)) return false; + const keys = new Set(); + return value.every((entry) => { + if (!entry || typeof entry !== "object" || typeof entry.key !== "string" || !entry.key || + keys.has(entry.key) || !entry.value || typeof entry.value !== "object" || Array.isArray(entry.value) || + typeof entry.value.sourceId !== "string" || typeof entry.value.targetId !== "string" || + typeof entry.value.type !== "string") return false; + keys.add(entry.key); + return true; + }); +} + // A replace import deletes every store before it writes, so a malformed // section must fail first, leaving the store untouched. function invalidImportSection(importData: ExportData): string | null { @@ -149,12 +171,7 @@ function invalidImportSection(importData: ExportData): string | null { const rows: unknown = importData[section]; if (rows != null && !Array.isArray(rows)) return `${section} must be an array`; } - const relations: unknown = importData.relations; - if (relations != null && (!Array.isArray(relations) || relations.some((entry) => - !entry || typeof entry.key !== "string" || !entry.key || !entry.value || - typeof entry.value.sourceId !== "string" || typeof entry.value.targetId !== "string" || - !["supersedes", "extends", "derives", "contradicts", "related"].includes(entry.value.type) - ))) return "relations must contain keyed relation rows"; + if (!validRelationEntries(importData.relations)) return "relations must contain keyed relation rows"; const accessLogs: unknown = importData.accessLogs; if (accessLogs != null && !Array.isArray(accessLogs)) return "accessLogs must be an array"; if (Array.isArray(accessLogs) && accessLogs.length > MAX_ACCESS_LOGS) { @@ -663,7 +680,7 @@ export function registerExportImportFunction(sdk: ISdk, kv: StateKV): void { if (importData.relations) { await runChunked(importData.relations, async ({ key, value }) => { if (strategy === "skip" && await kv.get(KV.relations, key)) { stats.skipped++; return; } - await kv.set(KV.relations, key, value); + await kv.set(KV.relations, key, normalizeRelation(value)); }); } if (importData.facets) { diff --git a/src/functions/snapshot.ts b/src/functions/snapshot.ts index 0d35c6cc4f..e82a3ff59d 100644 --- a/src/functions/snapshot.ts +++ b/src/functions/snapshot.ts @@ -20,6 +20,8 @@ import { DURABLE_STORES, readDurableStores, relationEntries, + normalizeRelation, + validRelationEntries, runChunked, type DurableStore, type DurableStoreField, @@ -126,7 +128,7 @@ async function replaceStore( await kv.set(store.scope, edge.id, edge); await indexGraphEdge(kv, edge); } else { - await kv.set(store.scope, store.keyOf(row), store.field === "relations" ? row.value : row); + await kv.set(store.scope, store.keyOf(row), store.field === "relations" ? normalizeRelation(row.value) : row); } }); @@ -325,6 +327,9 @@ export function registerSnapshotFunction( ]); const content = readFileSync(join(snapshotDir, "state.json"), "utf-8"); const state = JSON.parse(content) as SnapshotState; + if (!validRelationEntries(state.relations)) { + throw new Error("relations must contain keyed relation rows"); + } const counts: Record = {}; const notCaptured: string[] = []; diff --git a/test/export-import.test.ts b/test/export-import.test.ts index f2cb7d9fe3..3fa5496777 100644 --- a/test/export-import.test.ts +++ b/test/export-import.test.ts @@ -2,12 +2,14 @@ import { describe, it, expect, beforeEach, vi } from "vitest"; import { mkdtempSync, mkdirSync, rmSync } from "node:fs"; import { join } from "node:path"; import { registerSnapshotFunction } from "../src/functions/snapshot.js"; +import { pagedReads } from "./helpers/mocks.js"; vi.mock("../src/logger.js", () => ({ logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, })); import { registerExportImportFunction } from "../src/functions/export-import.js"; +import { registerRelationsFunction } from "../src/functions/relations.js"; import { VERSION } from "../src/version.js"; import { getSearchIndex } from "../src/functions/search.js"; import type { @@ -36,10 +38,7 @@ function mockKV() { const entries = store.get(scope); return entries ? (Array.from(entries.values()) as T[]) : []; }, - listPage: async (scope: string, after: string | undefined, limit: number): Promise> => - Array.from(store.get(scope)?.entries() ?? []).sort(([a], [b]) => a.localeCompare(b)) - .filter(([key]) => after === undefined || key > after).slice(0, limit) - .map(([key, value]) => ({ key, value: value as T })), + ...pagedReads(store), bytes: async (scope: string): Promise => { const entries = store.get(scope); return entries ? JSON.stringify(Array.from(entries.values())).length : 0; @@ -348,6 +347,27 @@ describe("Export/Import Functions", () => { expect(await destination.get("mem:relations", "rel_legacy")).toEqual(relation); }); + it("round-trips relation types accepted by the writer", async () => { + await kv.set("mem:memories", "mem_2", { ...testMemory, id: "mem_2" }); + registerRelationsFunction(sdk as never, kv as never); + const created = await sdk.trigger("mem::relate", { sourceId: "mem_1", targetId: "mem_2", type: "custom" }); + expect(created.success).toBe(true); + const exported = await sdk.trigger("mem::export", {}); + expect(await sdk.trigger("mem::import", { exportData: exported, strategy: "replace" })).toMatchObject({ success: true }); + expect(await kv.get("mem:relations", created.relationId)).toMatchObject({ type: "custom" }); + }); + + it("omits invalid relation confidence in exports and imports without changing the source", async () => { + const value = { sourceId: "a", targetId: "b", type: "custom", createdAt: "2026-01-01", confidence: "bad" }; + await kv.set("mem:relations", "rel_bad", value); + const exported = await sdk.trigger("mem::export", {}); + expect(exported.relations[0].value).not.toHaveProperty("confidence"); + expect(await kv.get("mem:relations", "rel_bad")).toEqual(value); + exported.relations[0].value.confidence = Infinity; + expect(await sdk.trigger("mem::import", { exportData: exported, strategy: "replace" })).toMatchObject({ success: true }); + expect(await kv.get("mem:relations", "rel_bad")).not.toHaveProperty("confidence"); + }); + it("leaves relations untouched when replacing from an older export without them", async () => { const relation = { sourceId: "a", targetId: "b", type: "related", createdAt: "2026-01-01" }; await kv.set("mem:relations", "rel_old", relation); diff --git a/test/snapshot-prune.test.ts b/test/snapshot-prune.test.ts index a35972f337..bf92fe4e61 100644 --- a/test/snapshot-prune.test.ts +++ b/test/snapshot-prune.test.ts @@ -1,4 +1,5 @@ import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { pagedReads } from "./helpers/mocks.js"; import { execFileSync } from "node:child_process"; import { mkdirSync, mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; @@ -19,10 +20,7 @@ function mockKV() { delete: async (scope: string, key: string): Promise => { store.get(scope)?.delete(key); }, - listPage: async (scope: string, after: string | undefined, limit: number): Promise> => - Array.from(store.get(scope)?.entries() ?? []).sort(([a], [b]) => a.localeCompare(b)) - .filter(([key]) => after === undefined || key > after).slice(0, limit) - .map(([key, value]) => ({ key, value: value as T })), + ...pagedReads(store), list: async (scope: string): Promise => { const entries = store.get(scope); return entries ? (Array.from(entries.values()) as T[]) : []; diff --git a/test/snapshot.test.ts b/test/snapshot.test.ts index 4b5fd2fdf0..6f31ad1a43 100644 --- a/test/snapshot.test.ts +++ b/test/snapshot.test.ts @@ -1,4 +1,5 @@ import { describe, it, expect, beforeEach, vi } from "vitest"; +import { pagedReads } from "./helpers/mocks.js"; vi.mock("../src/logger.js", () => ({ logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, @@ -65,10 +66,7 @@ function mockKV() { delete: async (scope: string, key: string): Promise => { store.get(scope)?.delete(key); }, - listPage: async (scope: string, after: string | undefined, limit: number): Promise> => - Array.from(store.get(scope)?.entries() ?? []).sort(([a], [b]) => a.localeCompare(b)) - .filter(([key]) => after === undefined || key > after).slice(0, limit) - .map(([key, value]) => ({ key, value: value as T })), + ...pagedReads(store), list: async (scope: string): Promise => { const entries = store.get(scope); return entries ? (Array.from(entries.values()) as T[]) : []; @@ -261,6 +259,21 @@ describe("snapshot-restore replaces each captured store", () => { getSearchIndex().add(memoryToIndexDoc(memory("mem_1", "Original"))); }); + it("rejects malformed relation entries before replacing any captured store", async () => { + snapshotFile.content = JSON.stringify({ sessions: [], memories: [], observations: {}, relations: [{ key: "bad", value: null }] }); + await kv.set("mem:relations", "rel_existing", { sourceId: "a", targetId: "b", type: "custom" }); + const result = await sdk.trigger("mem::snapshot-restore", { commitHash: "abc1234" }); + expect(result).toMatchObject({ success: false }); + expect(await kv.get("mem:memories", "mem_1")).not.toBeNull(); + expect(await kv.get("mem:relations", "rel_existing")).not.toBeNull(); + }); + + it("omits invalid confidence when restoring a captured relation", async () => { + snapshotFile.content = JSON.stringify({ relations: [{ key: "rel_old", value: { sourceId: "a", targetId: "b", type: "custom", confidence: "bad" } }] }); + expect(await sdk.trigger("mem::snapshot-restore", { commitHash: "abc1234" })).toMatchObject({ success: true }); + expect(await kv.get("mem:relations", "rel_old")).toEqual({ sourceId: "a", targetId: "b", type: "custom" }); + }); + it("removes rows added after the snapshot and brings back snapshot-time rows", async () => { await sdk.trigger("mem::snapshot-create", { message: "before" }); From 49ee2ca76ceff5889ca2225c0bbab1538392ffe8 Mon Sep 17 00:00:00 2001 From: neal <130699270+possiblyneal@users.noreply.github.com> Date: Sat, 21 Nov 2026 15:57:54 -0500 Subject: [PATCH 3/6] Normalize legacy non-string relation types in backups Generated-By: gpt-6.1-terra --- CLAUDE.md | 5 +++-- src/functions/export-import.ts | 11 +++++++---- test/export-import.test.ts | 13 +++++++++++++ test/snapshot.test.ts | 6 +++--- 4 files changed, 26 insertions(+), 9 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index edbd547645..f298328fb9 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -261,8 +261,9 @@ session IDs retain their normal normalization. This preflight does not make repl Full exports and snapshots carry Memory relations with their original storage keys, including legacy `rel_` keys. Replace import and snapshot restore replace that scope when the backup contains it; older backups without a `relations` section leave the scope untouched. Import and snapshot restore -validate relation entries before replacing any scope, accept any string relation type, and omit -nonnumeric or nonfinite confidence from backup data without changing the source row. +validate relation entries before replacing any scope, retain string relation types, map non-string +types to `related`, and omit nonnumeric or nonfinite confidence from backup data without changing +the source row. ### Graph Eviction Every path that hard-deletes Observations or Memories (`mem::evict`, `mem::observe` session-cap diff --git a/src/functions/export-import.ts b/src/functions/export-import.ts index b3eae0fcce..7566352d9f 100644 --- a/src/functions/export-import.ts +++ b/src/functions/export-import.ts @@ -105,9 +105,12 @@ export async function readDurableStores(kv: StateKV): Promise { export function normalizeRelation(value: unknown): unknown { if (!value || typeof value !== "object" || Array.isArray(value)) return value; const row = value as Record; - if (!("confidence" in row) || (typeof row.confidence === "number" && Number.isFinite(row.confidence))) return value; - const { confidence: _confidence, ...rest } = row; - return rest; + const normalized = { ...row }; + if ("type" in normalized && typeof normalized.type !== "string") normalized.type = "related"; + if ("confidence" in normalized && (typeof normalized.confidence !== "number" || !Number.isFinite(normalized.confidence))) { + delete normalized.confidence; + } + return normalized; } export async function relationEntries(kv: StateKV): Promise> { @@ -158,7 +161,7 @@ export function validRelationEntries(value: unknown): boolean { if (!entry || typeof entry !== "object" || typeof entry.key !== "string" || !entry.key || keys.has(entry.key) || !entry.value || typeof entry.value !== "object" || Array.isArray(entry.value) || typeof entry.value.sourceId !== "string" || typeof entry.value.targetId !== "string" || - typeof entry.value.type !== "string") return false; + !("type" in entry.value)) return false; keys.add(entry.key); return true; }); diff --git a/test/export-import.test.ts b/test/export-import.test.ts index 3fa5496777..fd7798691b 100644 --- a/test/export-import.test.ts +++ b/test/export-import.test.ts @@ -357,6 +357,19 @@ describe("Export/Import Functions", () => { expect(await kv.get("mem:relations", created.relationId)).toMatchObject({ type: "custom" }); }); + it("round-trips a non-string type accepted by the relation writer", async () => { + await kv.set("mem:memories", "mem_2", { ...testMemory, id: "mem_2" }); + registerRelationsFunction(sdk as never, kv as never); + const created = await sdk.trigger("mem::relate", { sourceId: "mem_1", targetId: "mem_2", type: 42 }); + expect(created.success).toBe(true); + const exported = await sdk.trigger("mem::export", {}); + expect(exported.relations[0].value.type).toBe("related"); + expect(await kv.get("mem:relations", created.relationId)).toMatchObject({ type: 42 }); + exported.relations[0].value.type = 42; + expect(await sdk.trigger("mem::import", { exportData: exported, strategy: "replace" })).toMatchObject({ success: true }); + expect(await kv.get("mem:relations", created.relationId)).toMatchObject({ type: "related" }); + }); + it("omits invalid relation confidence in exports and imports without changing the source", async () => { const value = { sourceId: "a", targetId: "b", type: "custom", createdAt: "2026-01-01", confidence: "bad" }; await kv.set("mem:relations", "rel_bad", value); diff --git a/test/snapshot.test.ts b/test/snapshot.test.ts index 6f31ad1a43..7890eb5713 100644 --- a/test/snapshot.test.ts +++ b/test/snapshot.test.ts @@ -268,10 +268,10 @@ describe("snapshot-restore replaces each captured store", () => { expect(await kv.get("mem:relations", "rel_existing")).not.toBeNull(); }); - it("omits invalid confidence when restoring a captured relation", async () => { - snapshotFile.content = JSON.stringify({ relations: [{ key: "rel_old", value: { sourceId: "a", targetId: "b", type: "custom", confidence: "bad" } }] }); + it("normalizes invalid type and confidence when restoring a captured relation", async () => { + snapshotFile.content = JSON.stringify({ relations: [{ key: "rel_old", value: { sourceId: "a", targetId: "b", type: 42, confidence: "bad" } }] }); expect(await sdk.trigger("mem::snapshot-restore", { commitHash: "abc1234" })).toMatchObject({ success: true }); - expect(await kv.get("mem:relations", "rel_old")).toEqual({ sourceId: "a", targetId: "b", type: "custom" }); + expect(await kv.get("mem:relations", "rel_old")).toEqual({ sourceId: "a", targetId: "b", type: "related" }); }); it("removes rows added after the snapshot and brings back snapshot-time rows", async () => { From 6b372e8ff4cabc111d3b93478bf4a08ec5794d32 Mon Sep 17 00:00:00 2001 From: neal <130699270+possiblyneal@users.noreply.github.com> Date: Sat, 21 Nov 2026 16:00:47 -0500 Subject: [PATCH 4/6] Preserve mesh relation endpoints in backups Generated-By: gpt-6.1-terra --- CLAUDE.md | 6 +++--- src/functions/export-import.ts | 2 +- test/export-import.test.ts | 10 ++++++++++ 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index f298328fb9..194f91e3ef 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -261,9 +261,9 @@ session IDs retain their normal normalization. This preflight does not make repl Full exports and snapshots carry Memory relations with their original storage keys, including legacy `rel_` keys. Replace import and snapshot restore replace that scope when the backup contains it; older backups without a `relations` section leave the scope untouched. Import and snapshot restore -validate relation entries before replacing any scope, retain string relation types, map non-string -types to `related`, and omit nonnumeric or nonfinite confidence from backup data without changing -the source row. +validate relation entries before replacing any scope, preserve truthy endpoint values as stored +(including non-strings accepted by mesh), retain string relation types, map non-string types to +`related`, and omit nonnumeric or nonfinite confidence from backup data without changing the source row. ### Graph Eviction Every path that hard-deletes Observations or Memories (`mem::evict`, `mem::observe` session-cap diff --git a/src/functions/export-import.ts b/src/functions/export-import.ts index 7566352d9f..7bac36b20c 100644 --- a/src/functions/export-import.ts +++ b/src/functions/export-import.ts @@ -160,7 +160,7 @@ export function validRelationEntries(value: unknown): boolean { return value.every((entry) => { if (!entry || typeof entry !== "object" || typeof entry.key !== "string" || !entry.key || keys.has(entry.key) || !entry.value || typeof entry.value !== "object" || Array.isArray(entry.value) || - typeof entry.value.sourceId !== "string" || typeof entry.value.targetId !== "string" || + !entry.value.sourceId || !entry.value.targetId || !("type" in entry.value)) return false; keys.add(entry.key); return true; diff --git a/test/export-import.test.ts b/test/export-import.test.ts index fd7798691b..1a111d1264 100644 --- a/test/export-import.test.ts +++ b/test/export-import.test.ts @@ -10,6 +10,7 @@ vi.mock("../src/logger.js", () => ({ import { registerExportImportFunction } from "../src/functions/export-import.js"; import { registerRelationsFunction } from "../src/functions/relations.js"; +import { registerMeshFunction } from "../src/functions/mesh.js"; import { VERSION } from "../src/version.js"; import { getSearchIndex } from "../src/functions/search.js"; import type { @@ -370,6 +371,15 @@ describe("Export/Import Functions", () => { expect(await kv.get("mem:relations", created.relationId)).toMatchObject({ type: "related" }); }); + it("round-trips opaque endpoints accepted by mesh without changing their values or key", async () => { + registerMeshFunction(sdk as never, kv as never); + const relation = { sourceId: 42, targetId: { id: "target" }, type: "related" }; + expect(await sdk.trigger("mem::mesh-receive", { relations: [relation] })).toMatchObject({ success: true, accepted: 1 }); + const exported = await sdk.trigger("mem::export", {}); + expect(await sdk.trigger("mem::import", { exportData: exported, strategy: "replace" })).toMatchObject({ success: true }); + expect(await kv.get("mem:relations", "42:[object Object]:related")).toEqual(relation); + }); + it("omits invalid relation confidence in exports and imports without changing the source", async () => { const value = { sourceId: "a", targetId: "b", type: "custom", createdAt: "2026-01-01", confidence: "bad" }; await kv.set("mem:relations", "rel_bad", value); From 557cd61ee359220a9a6bffee8a79ed00162fa021 Mon Sep 17 00:00:00 2001 From: neal <130699270+possiblyneal@users.noreply.github.com> Date: Sat, 21 Nov 2026 16:04:15 -0500 Subject: [PATCH 5/6] Preserve SQLite backup keys and null endpoints Generated-By: gpt-6.1-terra --- CLAUDE.md | 8 +++++--- src/engine/inproc/state.ts | 6 +++--- src/functions/export-import.ts | 2 +- test/export-import.test.ts | 31 +++++++++++++++++++++++++++++++ test/inproc-state.test.ts | 16 ++++++++++++++++ 5 files changed, 56 insertions(+), 7 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 194f91e3ef..acf1d49a97 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -261,9 +261,11 @@ session IDs retain their normal normalization. This preflight does not make repl Full exports and snapshots carry Memory relations with their original storage keys, including legacy `rel_` keys. Replace import and snapshot restore replace that scope when the backup contains it; older backups without a `relations` section leave the scope untouched. Import and snapshot restore -validate relation entries before replacing any scope, preserve truthy endpoint values as stored -(including non-strings accepted by mesh), retain string relation types, map non-string types to -`related`, and omit nonnumeric or nonfinite confidence from backup data without changing the source row. +validate relation entries before replacing any scope, preserve present endpoint values as stored +(including explicit nulls and non-strings accepted by mesh), retain string relation types, map +non-string types to `related`, and omit nonnumeric or nonfinite confidence from backup data without +changing the source row. `StateKV.listPage` returns complete SQLite keys, including embedded NULs, +so those keys remain valid paging cursors and can be restored exactly. ### Graph Eviction Every path that hard-deletes Observations or Memories (`mem::evict`, `mem::observe` session-cap diff --git a/src/engine/inproc/state.ts b/src/engine/inproc/state.ts index b6e31ab29f..8466b000a9 100644 --- a/src/engine/inproc/state.ts +++ b/src/engine/inproc/state.ts @@ -306,9 +306,9 @@ export class SqliteState { listPage(scope: string, after: string | undefined, limit: number): Array<{ key: string; value: unknown }> { const rows = this.db - .prepare("SELECT key, value FROM kv WHERE scope = ? AND key > ? ORDER BY key LIMIT ?") - .all(scope, after ?? "", limit) as Array<{ key: string; value: string }>; - return rows.map((r) => ({ key: r.key, value: JSON.parse(r.value) })); + .prepare("SELECT CAST(key AS BLOB) AS key, value FROM kv WHERE scope = ? AND key > ? ORDER BY key LIMIT ?") + .all(scope, after ?? "", limit) as Array<{ key: Uint8Array; value: string }>; + return rows.map((r) => ({ key: Buffer.from(r.key).toString("utf8"), value: JSON.parse(r.value) })); } listNewest( diff --git a/src/functions/export-import.ts b/src/functions/export-import.ts index 7bac36b20c..060377469c 100644 --- a/src/functions/export-import.ts +++ b/src/functions/export-import.ts @@ -160,7 +160,7 @@ export function validRelationEntries(value: unknown): boolean { return value.every((entry) => { if (!entry || typeof entry !== "object" || typeof entry.key !== "string" || !entry.key || keys.has(entry.key) || !entry.value || typeof entry.value !== "object" || Array.isArray(entry.value) || - !entry.value.sourceId || !entry.value.targetId || + !("sourceId" in entry.value) || !("targetId" in entry.value) || !("type" in entry.value)) return false; keys.add(entry.key); return true; diff --git a/test/export-import.test.ts b/test/export-import.test.ts index 1a111d1264..db581697ef 100644 --- a/test/export-import.test.ts +++ b/test/export-import.test.ts @@ -11,6 +11,8 @@ vi.mock("../src/logger.js", () => ({ import { registerExportImportFunction } from "../src/functions/export-import.js"; import { registerRelationsFunction } from "../src/functions/relations.js"; import { registerMeshFunction } from "../src/functions/mesh.js"; +import { SqliteState, stateFunctions } from "../src/engine/inproc/state.js"; +import { StateKV } from "../src/state/kv.js"; import { VERSION } from "../src/version.js"; import { getSearchIndex } from "../src/functions/search.js"; import type { @@ -380,6 +382,35 @@ describe("Export/Import Functions", () => { expect(await kv.get("mem:relations", "42:[object Object]:related")).toEqual(relation); }); + it("replaces real SQLite relations using exact NUL-containing storage keys", async () => { + const state = new SqliteState(":memory:"); + try { + const handlers = stateFunctions(state); + const realKV = new StateKV({ trigger: async ({ function_id, payload }: { function_id: string; payload: unknown }) => handlers[function_id](payload) } as never); + const realSdk = mockSdk(); + registerExportImportFunction(realSdk as never, realKV); + const relation = { sourceId: null, targetId: "b", type: "related" }; + await realKV.set("mem:relations", "rel\u0000original", relation); + const exported = await realSdk.trigger("mem::export", {}); + expect(exported.relations[0].key).toBe("rel\u0000original"); + await realKV.set("mem:relations", "rel\u0000stale", { ...relation, targetId: "stale" }); + expect(await realSdk.trigger("mem::import", { exportData: exported, strategy: "replace" })).toMatchObject({ success: true }); + expect(await realKV.get("mem:relations", "rel\u0000original")).toEqual(relation); + expect(await realKV.get("mem:relations", "rel\u0000stale")).toBeNull(); + expect(await realKV.list("mem:relations")).toEqual([relation]); + } finally { + state.close(); + } + }); + + it("preserves explicit null endpoints in persisted relations", async () => { + const relation = { sourceId: null, targetId: "b", type: "related" }; + await kv.set("mem:relations", "rel_null", relation); + const exported = await sdk.trigger("mem::export", {}); + expect(await sdk.trigger("mem::import", { exportData: exported, strategy: "replace" })).toMatchObject({ success: true }); + expect(await kv.get("mem:relations", "rel_null")).toEqual(relation); + }); + it("omits invalid relation confidence in exports and imports without changing the source", async () => { const value = { sourceId: "a", targetId: "b", type: "custom", createdAt: "2026-01-01", confidence: "bad" }; await kv.set("mem:relations", "rel_bad", value); diff --git a/test/inproc-state.test.ts b/test/inproc-state.test.ts index b5f5452a3a..93329705b6 100644 --- a/test/inproc-state.test.ts +++ b/test/inproc-state.test.ts @@ -123,6 +123,22 @@ describe("inproc state store", () => { expect(next).toEqual([{ key: "c", value: "c" }, { key: "d", value: "d" }]); }); + it("pages keys containing NUL without truncating the cursor", async () => { + const fns = stateFunctions(store); + for (const key of ["a", "a\u0000b", "a\u0000c", "b"]) { + await fns["state::set"]({ scope: "s", key, value: key }); + } + const keys: string[] = []; + let after: string | undefined; + for (let i = 0; i < 4; i++) { + const page = await fns["state::list-page"]({ scope: "s", after, limit: 1 }) as Array<{ key: string }>; + expect(page).toHaveLength(1); + keys.push(page[0].key); + after = page[0].key; + } + expect(keys).toEqual(["a", "a\u0000b", "a\u0000c", "b"]); + }); + it("counts exactly the current rows in one scope", async () => { const fns = stateFunctions(store); expect(await fns["state::count"]({ scope: "s" })).toBe(0); From 27f42d595fc77a89dcfbc7c7975c2479dd24ea3b Mon Sep 17 00:00:00 2001 From: neal <130699270+possiblyneal@users.noreply.github.com> Date: Sat, 21 Nov 2026 16:08:33 -0500 Subject: [PATCH 6/6] fix(sqlite): keep key paging in index order Sort by the stored TEXT column rather than the projected BLOB alias. SQLite can then use the key index for ordering while returning all key bytes, including embedded NULs. Refs: #443 Generated-By: gpt-6.2-sol --- src/engine/inproc/state.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/engine/inproc/state.ts b/src/engine/inproc/state.ts index 8466b000a9..3ef2666a8b 100644 --- a/src/engine/inproc/state.ts +++ b/src/engine/inproc/state.ts @@ -306,7 +306,7 @@ export class SqliteState { listPage(scope: string, after: string | undefined, limit: number): Array<{ key: string; value: unknown }> { const rows = this.db - .prepare("SELECT CAST(key AS BLOB) AS key, value FROM kv WHERE scope = ? AND key > ? ORDER BY key LIMIT ?") + .prepare("SELECT CAST(key AS BLOB) AS key, value FROM kv WHERE scope = ? AND key > ? ORDER BY kv.key LIMIT ?") .all(scope, after ?? "", limit) as Array<{ key: Uint8Array; value: string }>; return rows.map((r) => ({ key: Buffer.from(r.key).toString("utf8"), value: JSON.parse(r.value) })); }