From 6715febcdc2770e6bc77336c62a20a5908937244 Mon Sep 17 00:00:00 2001 From: David Waltermire Date: Fri, 25 Sep 2026 00:25:38 -0400 Subject: [PATCH 01/11] fix(graph): bound and attribute node and edge provenance Each merge in persistGraphDelta added every observation id in the batch to every node and edge it touched, with no limit. Hot nodes grew without bound. On one store, 98% of the graph's bytes were these ids: 426 per node and 362 per edge on average. Nodes took 323 MB and edges 398 MB. Every kv.list traversal of the graph carried them. - A merge now adds the incoming record's own sourceObservationIds, not the whole batch. Heuristic extraction knows the exact source, so that attribution is now kept. - Each record keeps its newest 32 ids (MAX_GRAPH_SOURCE_OBSERVATIONS). A re-seen id moves to the newest end. New records are trimmed on first write. Retrieval returns at most 20 results, so 32 covers it. - The temporal-graph node merge uses the same rule. - New mem::graph-compact and POST /agentmemory/graph/compact trim records written before the cap. They walk the name and edge-key indexes and read and write one key at a time, so they never list the nodes or edges scope. An optional scope, offset and limit runs one slice per call, so each call finishes between worker reconnects. Re-running a slice is safe. - Compaction takes the graph:persist lock per record, so a trim cannot overwrite a merge that lands during it. persistGraphDelta no longer takes the batch id list. Measured on a live store: compaction trimmed 18,324 nodes and 32,654 edges and removed 18.9 million ids in 45 s with no retries. Nodes went from 323 to 30 MB and edges from 398 to 42 MB. Fixes #1374 Signed-off-by: David Waltermire --- AGENTS.md | 2 +- README.md | 2 +- .../skills/agentmemory-rest-api/REFERENCE.md | 3 +- src/functions/graph-import.ts | 1 - src/functions/graph.ts | 191 ++++++- src/functions/temporal-graph.ts | 12 +- src/index.ts | 2 +- src/triggers/api.ts | 28 ++ test/graph-compact.test.ts | 469 ++++++++++++++++++ test/graph-provenance-cap.test.ts | 288 +++++++++++ 10 files changed, 968 insertions(+), 30 deletions(-) create mode 100644 test/graph-compact.test.ts create mode 100644 test/graph-provenance-cap.test.ts diff --git a/AGENTS.md b/AGENTS.md index 9145dfa2ca..e65ccef47d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -119,7 +119,7 @@ Hook scripts in `src/hooks/` are standalone Node.js scripts (no iii-sdk import). ## Current Stats (v0.9.29) - 54 MCP tools (8 visible by default, `AGENTMEMORY_TOOLS=all` for all) -- 134 REST endpoints +- 135 REST endpoints - 6 MCP resources, 3 MCP prompts - 12 hooks, 17 skills - 260+ iii functions diff --git a/README.md b/README.md index 120ac49fa0..60f98155d7 100644 --- a/README.md +++ b/README.md @@ -1671,7 +1671,7 @@ Create `~/.agentmemory/.env`:

API

-134 endpoints on port `3111`. The REST API binds to `127.0.0.1` by default. Protected endpoints require `Authorization: Bearer ` when `AGENTMEMORY_SECRET` is set, and mesh sync endpoints require `AGENTMEMORY_SECRET` on both peers. +135 endpoints on port `3111`. The REST API binds to `127.0.0.1` by default. Protected endpoints require `Authorization: Bearer ` when `AGENTMEMORY_SECRET` is set, and mesh sync endpoints require `AGENTMEMORY_SECRET` on both peers.
Key endpoints diff --git a/plugin/skills/agentmemory-rest-api/REFERENCE.md b/plugin/skills/agentmemory-rest-api/REFERENCE.md index 75671c946e..7de5301098 100644 --- a/plugin/skills/agentmemory-rest-api/REFERENCE.md +++ b/plugin/skills/agentmemory-rest-api/REFERENCE.md @@ -5,7 +5,7 @@ Generated from `src/triggers/api.ts`. Do not edit the block below by hand; run ` The REST API is the primary surface. All paths are under `http://localhost:3111` (override with `--port`). When `AGENTMEMORY_SECRET` is set, send `Authorization: Bearer $AGENTMEMORY_SECRET`; localhost is otherwise open. -134 registered endpoints: +135 registered endpoints: | Method | Path | | --- | --- | @@ -55,6 +55,7 @@ The REST API is the primary surface. All paths are under `http://localhost:3111` | POST | `/agentmemory/governance/bulk-delete` | | DELETE | `/agentmemory/governance/memories` | | POST | `/agentmemory/graph/build` | +| POST | `/agentmemory/graph/compact` | | POST | `/agentmemory/graph/extract` | | POST | `/agentmemory/graph/import-graphify` | | GET | `/agentmemory/graph/node` | diff --git a/src/functions/graph-import.ts b/src/functions/graph-import.ts index 502526e8ba..bc595c9db7 100644 --- a/src/functions/graph-import.ts +++ b/src/functions/graph-import.ts @@ -224,7 +224,6 @@ export function registerGraphImportFunction(sdk: IIIClient, kv: StateKV): void { kv, parsed.nodes, parsed.edges, - [], ); await recordAudit(kv, "import", "mem::graph::import-graphify", [], { diff --git a/src/functions/graph.ts b/src/functions/graph.ts index 38add02bd4..0bd365838b 100644 --- a/src/functions/graph.ts +++ b/src/functions/graph.ts @@ -279,35 +279,47 @@ function snapshotPushEdgeIfBothInTop( } } +// Provenance is bounded. Every merge used to union the whole batch's +// observation ids into every node and edge it touched, which grew hot +// nodes without limit: on one store 98% of the graph's bytes were these +// ids, and every kv.list traversal carried them. Retrieval reads at most +// 20 results, so a record never needs more than its newest sources. +export const MAX_GRAPH_SOURCE_OBSERVATIONS = 32; + +// Union in arrival order, with a re-seen id moved to the newest end, then +// keep the newest MAX_GRAPH_SOURCE_OBSERVATIONS. +export function boundSources(existing: string[], incoming: string[]): string[] { + const merged = new Set(existing); + for (const id of incoming) { + merged.delete(id); + merged.add(id); + } + return [...merged].slice(-MAX_GRAPH_SOURCE_OBSERVATIONS); +} + function mergeNode( existing: GraphNode, incoming: GraphNode, - obsIds: string[], capturedAt: string, ): GraphNode { return { ...existing, - sourceObservationIds: [ - ...new Set([ - ...existing.sourceObservationIds, - ...incoming.sourceObservationIds, - ...obsIds, - ]), - ], + sourceObservationIds: boundSources( + existing.sourceObservationIds ?? [], + incoming.sourceObservationIds ?? [], + ), properties: { ...existing.properties, ...incoming.properties }, updatedAt: capturedAt, }; } -function mergeEdge( - existing: GraphEdge, - obsIds: string[], -): GraphEdge { +function mergeEdge(existing: GraphEdge, incoming: GraphEdge): GraphEdge { return { ...existing, - sourceObservationIds: [ - ...new Set([...existing.sourceObservationIds, ...obsIds]), - ], + sourceObservationIds: boundSources( + existing.sourceObservationIds ?? [], + incoming.sourceObservationIds ?? [], + ), }; } @@ -559,7 +571,6 @@ export async function persistGraphDelta( kv: StateKV, nodes: GraphNode[], edges: GraphEdge[], - obsIds: string[], ): Promise<{ newNodeCount: number; newEdgeCount: number }> { return withKeyedLock("graph:persist", async () => { const snap = (await readSnapshot(kv)) ?? emptySnapshot(); @@ -600,7 +611,7 @@ export async function persistGraphDelta( if (existing) { idRemap.set(node.id, existing.id); - const merged = mergeNode(existing, node, obsIds, capturedAt); + const merged = mergeNode(existing, node, capturedAt); await kv.set(KV.graphNodes, existing.id, merged); // Update topNodes entry if present so a stale clone isn't // returned from the snapshot fast path. @@ -610,6 +621,7 @@ export async function persistGraphDelta( snapMutated = true; } } else { + node.sourceObservationIds = boundSources([], node.sourceObservationIds ?? []); await kv.set(KV.graphNodes, node.id, node); await kv.set(KV.graphNameIndex, indexKey, node.id); await kv.set(KV.graphNodeDegree, node.id, 0); @@ -650,7 +662,7 @@ export async function persistGraphDelta( } if (existing) { - const merged = mergeEdge(existing, obsIds); + const merged = mergeEdge(existing, edge); await kv.set(KV.graphEdges, existing.id, merged); // Replace cached topEdges entry too if present. const topIdx = snap.topEdges.findIndex((e) => e.id === existing!.id); @@ -659,6 +671,7 @@ export async function persistGraphDelta( snapMutated = true; } } else { + edge.sourceObservationIds = boundSources([], edge.sourceObservationIds ?? []); await kv.set(KV.graphEdges, edge.id, edge); await kv.set(KV.graphEdgeKey, eKey, edge.id); snap.stats.totalEdges += 1; @@ -688,6 +701,133 @@ export async function persistGraphDelta( }); } +export interface GraphCompactResult { + nodesScanned: number; + nodesTrimmed: number; + edgesScanned: number; + edgesTrimmed: number; + idsRemoved: number; + snapshotTrimmed: boolean; + // Records in the sliced index, and where the next slice starts; null + // when this call reached the end. A full run always ends with null. + total?: number; + nextOffset: number | null; +} + +export type GraphCompactScope = "nodes" | "edges" | "snapshot"; + +export interface GraphCompactOptions { + scope?: GraphCompactScope; + offset?: number; + limit?: number; +} + +const COMPACT_SCOPES: readonly GraphCompactScope[] = ["nodes", "edges", "snapshot"]; + +// Trims provenance written before the cap existed. Records are reached +// through the name and edge-key indexes, whose values are short ids, and +// then read and written one key at a time. The nodes and edges scopes are +// never listed: that payload is what drops the worker on a large store. +// A record missing from both indexes is not reached; it is also never +// merged again, so it cannot grow. +// +// With no options it does everything in one call. With a scope it does one +// slice, so each invocation stays short enough to finish between worker +// reconnects; the caller walks nextOffset until it is null. Re-running any +// slice is safe because trimmed records are skipped. +export async function compactGraphProvenance( + kv: StateKV, + opts: GraphCompactOptions = {}, +): Promise { + const { scope } = opts; + if (scope !== undefined && !COMPACT_SCOPES.includes(scope)) { + throw new Error(`unknown compact scope: ${String(scope)}`); + } + const offset = opts.offset ?? 0; + const limit = opts.limit ?? Number.POSITIVE_INFINITY; + if (!Number.isInteger(offset) || offset < 0) { + throw new Error(`compact offset must be a non-negative integer: ${String(opts.offset)}`); + } + if (!(limit === Number.POSITIVE_INFINITY || (Number.isInteger(limit) && limit > 0))) { + throw new Error(`compact limit must be a positive integer: ${String(opts.limit)}`); + } + + const result: GraphCompactResult = { + nodesScanned: 0, + nodesTrimmed: 0, + edgesScanned: 0, + edgesTrimmed: 0, + idsRemoved: 0, + snapshotTrimmed: false, + nextOffset: null, + }; + + const trimScope = async ( + indexScope: string, + recordScope: string, + ): Promise<{ scanned: number; trimmed: number }> => { + const allIds = [...new Set(await kv.list(indexScope))].filter( + (id): id is string => typeof id === "string", + ); + const end = Math.min(allIds.length, offset + limit); + result.total = allIds.length; + result.nextOffset = end < allIds.length ? end : null; + let scanned = 0; + let trimmed = 0; + for (const id of allIds.slice(offset, end)) { + await withKeyedLock("graph:persist", async () => { + const record = await kv.get(recordScope, id); + if (!record) return; + scanned += 1; + const sources = record.sourceObservationIds ?? []; + if (sources.length <= MAX_GRAPH_SOURCE_OBSERVATIONS) return; + const bounded = boundSources([], sources); + result.idsRemoved += sources.length - bounded.length; + await kv.set(recordScope, id, { ...record, sourceObservationIds: bounded }); + trimmed += 1; + }); + } + return { scanned, trimmed }; + }; + + const trimSnapshot = () => + withKeyedLock("graph:persist", async () => { + const snap = await readSnapshot(kv); + if (!snap) return; + const trimList = (list: R[]) => + list.map((r) => { + const sources = r.sourceObservationIds ?? []; + if (sources.length <= MAX_GRAPH_SOURCE_OBSERVATIONS) return r; + result.snapshotTrimmed = true; + return { ...r, sourceObservationIds: boundSources([], sources) }; + }); + const topNodes = trimList(snap.topNodes); + const topEdges = trimList(snap.topEdges); + if (result.snapshotTrimmed) { + await kv.set(KV.graphSnapshot, SNAPSHOT_KEY, { ...snap, topNodes, topEdges }); + } + }); + + if (scope === undefined || scope === "nodes") { + const n = await trimScope(KV.graphNameIndex, KV.graphNodes); + result.nodesScanned = n.scanned; + result.nodesTrimmed = n.trimmed; + } + if (scope === undefined || scope === "edges") { + const e = await trimScope(KV.graphEdgeKey, KV.graphEdges); + result.edgesScanned = e.scanned; + result.edgesTrimmed = e.trimmed; + } + if (scope === undefined || scope === "snapshot") { + await trimSnapshot(); + } + if (scope === undefined) { + delete result.total; + result.nextOffset = null; + } + return result; +} + export function registerGraphFunction( sdk: IIIClient, kv: StateKV, @@ -751,7 +891,6 @@ export function registerGraphFunction( kv, nodes, edges, - obsIds, ); await recordAudit(kv, "observe", "mem::graph-extract", obsIds, { @@ -1135,6 +1274,20 @@ export function registerGraphFunction( // read by any post-#816 code path. Cleanup is deferred to a future // chunked-vacuum job; #816's broken vacuum-via-list strategy is // what we are leaving behind here. + sdk.registerFunction("mem::graph-compact", async (data?: GraphCompactOptions) => { + const started = Date.now(); + try { + const result = await compactGraphProvenance(kv, data ?? {}); + const tookMs = Date.now() - started; + logger.info("Graph provenance compacted", { ...result, tookMs }); + return { success: true, ...result, tookMs }; + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + logger.error("Graph provenance compaction failed", { error: msg }); + return { success: false, error: msg }; + } + }); + sdk.registerFunction("mem::graph-reset", async () => { const started = Date.now(); // Stamp resetAt=now on the empty snapshot. Future diff --git a/src/functions/temporal-graph.ts b/src/functions/temporal-graph.ts index 18d13c1fe5..4ee0c4c1be 100644 --- a/src/functions/temporal-graph.ts +++ b/src/functions/temporal-graph.ts @@ -8,6 +8,7 @@ import type { MemoryProvider, } from "../types.js"; import { KV, generateId } from "../state/schema.js"; +import { boundSources } from "./graph.js"; import type { StateKV } from "../state/kv.js"; import { withKeyedLock } from "../state/keyed-mutex.js"; import { logger } from "../logger.js"; @@ -201,12 +202,10 @@ export function registerTemporalGraphFunctions( const oldId = node.id; const merged = { ...existing, - sourceObservationIds: [ - ...new Set([ - ...existing.sourceObservationIds, - ...obsIds, - ]), - ], + sourceObservationIds: boundSources( + existing.sourceObservationIds ?? [], + node.sourceObservationIds ?? [], + ), properties: { ...existing.properties, ...node.properties }, updatedAt: new Date().toISOString(), aliases: [ @@ -221,6 +220,7 @@ export function registerTemporalGraphFunctions( node.id = existing.id; idRemap.set(oldId, existing.id); } else { + node.sourceObservationIds = boundSources([], node.sourceObservationIds ?? []); await kv.set(KV.graphNodes, node.id, node); existingNodes.push(node); } diff --git a/src/index.ts b/src/index.ts index e85f16217f..914a72a567 100644 --- a/src/index.ts +++ b/src/index.ts @@ -536,7 +536,7 @@ async function main() { `Ready. ${embeddingProvider ? "Triple-stream (BM25+Vector+Graph)" : "BM25+Graph"} search active.`, ); bootLog( - `REST API: 134 endpoints at http://localhost:${config.restPort}/agentmemory/*`, + `REST API: 135 endpoints at http://localhost:${config.restPort}/agentmemory/*`, ); bootLog( `MCP surface (opt-in via \`npx @agentmemory/mcp\`): ${getAllTools().length} tools · 6 resources · 3 prompts`, diff --git a/src/triggers/api.ts b/src/triggers/api.ts index 8ae72d43c5..5a229dfef2 100644 --- a/src/triggers/api.ts +++ b/src/triggers/api.ts @@ -2017,6 +2017,34 @@ export function registerApiTriggers( config: { api_path: "/agentmemory/graph/reset", http_method: "POST" }, }); + // Trims graph provenance written before the per-record cap, walking the + // small name and edge-key indexes instead of listing the graph. + sdk.registerFunction("api::graph-compact", + async (req: HttpRequest): Promise => { + const authErr = checkAuth(req, secret); + if (authErr) return authErr; + try { + const body = (req.body ?? {}) as Record; + const result = await sdk.trigger({ + function_id: "mem::graph-compact", + payload: { + scope: body.scope, + offset: body.offset, + limit: body.limit, + }, + }); + return { status_code: 200, body: result }; + } catch { + return graphDisabledResponse(); + } + }, + ); + sdk.registerTrigger({ + type: "http", + function_id: "api::graph-compact", + config: { api_path: "/agentmemory/graph/compact", http_method: "POST" }, + }); + sdk.registerFunction("api::graph-extract", async (req: HttpRequest<{ observations: unknown[] }>): Promise => { const authErr = checkAuth(req, secret); diff --git a/test/graph-compact.test.ts b/test/graph-compact.test.ts new file mode 100644 index 0000000000..6b32a910c2 --- /dev/null +++ b/test/graph-compact.test.ts @@ -0,0 +1,469 @@ +import { describe, it, expect, vi } from "vitest"; + +vi.mock("../src/logger.js", () => ({ + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, +})); + +import { + compactGraphProvenance, + persistGraphDelta, + registerGraphFunction, + MAX_GRAPH_SOURCE_OBSERVATIONS, +} from "../src/functions/graph.js"; +import type { GraphEdge, GraphNode } from "../src/types.js"; + +// Compaction trims provenance written before the cap existed. It must +// reach records through the small name and edge-key indexes only: listing +// the nodes or edges scope is the payload that drops the worker. + +// slowFirstNodeRead delays only the FIRST read of a node record, after the +// value is captured. That forces the interleaving that loses a merge when +// writers are not serialized: compaction reads, a merge writes, then +// compaction writes back its stale copy. +function mockKV(slowFirstNodeReadMs = 0) { + const store = new Map>(); + const calls = { list: [] as string[], set: [] as string[] }; + let nodeReads = 0; + return { + calls, + get: async (scope: string, key: string): Promise => { + const v = store.get(scope)?.get(key); + if ( + scope === "mem:graph:nodes" && + nodeReads++ === 0 && + slowFirstNodeReadMs + ) { + await new Promise((r) => setTimeout(r, slowFirstNodeReadMs)); + } + return (v === undefined ? null : structuredClone(v)) as T | null; + }, + set: async (scope: string, key: string, data: T): Promise => { + calls.set.push(`${scope}/${key}`); + if (!store.has(scope)) store.set(scope, new Map()); + store.get(scope)!.set(key, structuredClone(data)); + return data; + }, + delete: async (scope: string, key: string): Promise => { + store.get(scope)?.delete(key); + }, + list: async (scope: string): Promise => { + calls.list.push(scope); + const entries = store.get(scope); + return entries + ? (Array.from(entries.values()).map((v) => structuredClone(v)) as T[]) + : []; + }, + }; +} +type KV = ReturnType; + +const ids = (prefix: string, n: number) => + Array.from({ length: n }, (_, i) => `${prefix}_${i}`); + +async function seedNode(kv: KV, id: string, name: string, sources: string[]) { + const n: GraphNode = { + id, + type: "concept", + name, + properties: { keep: "me" }, + sourceObservationIds: sources, + createdAt: "2026-09-01T00:00:00.000Z", + aliases: ["alias"], + }; + await kv.set("mem:graph:nodes", id, n); + await kv.set("mem:graph:name-index", `concept|${name}`, id); + return n; +} + +async function seedEdge( + kv: KV, + id: string, + src: string, + tgt: string, + sources: string[], +) { + const e: GraphEdge = { + id, + type: "related_to", + sourceNodeId: src, + targetNodeId: tgt, + weight: 0.7, + sourceObservationIds: sources, + createdAt: "2026-09-01T00:00:00.000Z", + }; + await kv.set("mem:graph:edges", id, e); + await kv.set("mem:graph:edge-key", `${src}|${tgt}|related_to`, id); + return e; +} + +const node = (kv: KV, id: string) => kv.get("mem:graph:nodes", id); +const edge = (kv: KV, id: string) => kv.get("mem:graph:edges", id); + +describe("compactGraphProvenance", () => { + it("trims an oversized node to the newest ids and keeps every other field", async () => { + const kv = mockKV(); + const many = ids("obs", 500); + const before = await seedNode(kv, "gn_1", "hub", many); + await compactGraphProvenance(kv as never); + const after = (await node(kv, "gn_1"))!; + expect(after.sourceObservationIds).toEqual( + many.slice(-MAX_GRAPH_SOURCE_OBSERVATIONS), + ); + expect({ ...after, sourceObservationIds: [] }).toEqual({ + ...before, + sourceObservationIds: [], + }); + }); + + it("trims an oversized edge the same way", async () => { + const kv = mockKV(); + const many = ids("obs", 300); + const before = await seedEdge(kv, "ge_1", "gn_a", "gn_b", many); + await compactGraphProvenance(kv as never); + const after = (await edge(kv, "ge_1"))!; + expect(after.sourceObservationIds).toEqual( + many.slice(-MAX_GRAPH_SOURCE_OBSERVATIONS), + ); + expect({ ...after, sourceObservationIds: [] }).toEqual({ + ...before, + sourceObservationIds: [], + }); + }); + + it("does not rewrite records already within the cap", async () => { + const kv = mockKV(); + await seedNode( + kv, + "gn_small", + "small", + ids("obs", MAX_GRAPH_SOURCE_OBSERVATIONS), + ); + await seedEdge(kv, "ge_small", "gn_a", "gn_b", ["obs_1"]); + kv.calls.set.length = 0; + await compactGraphProvenance(kv as never); + expect( + kv.calls.set.filter( + (c) => + c.startsWith("mem:graph:nodes/") || c.startsWith("mem:graph:edges/"), + ), + ).toEqual([]); + }); + + it("never lists the nodes or edges scope", async () => { + const kv = mockKV(); + await seedNode(kv, "gn_1", "hub", ids("obs", 100)); + await seedEdge(kv, "ge_1", "gn_1", "gn_2", ids("obs", 100)); + await compactGraphProvenance(kv as never); + expect(kv.calls.list).not.toContain("mem:graph:nodes"); + expect(kv.calls.list).not.toContain("mem:graph:edges"); + }); + + it("reports what it scanned and trimmed", async () => { + const kv = mockKV(); + await seedNode(kv, "gn_1", "big", ids("obs", 100)); + await seedNode(kv, "gn_2", "small", ["obs_1"]); + await seedEdge(kv, "ge_1", "gn_1", "gn_2", ids("obs", 40)); + const r = await compactGraphProvenance(kv as never); + expect(r).toMatchObject({ + nodesScanned: 2, + nodesTrimmed: 1, + edgesScanned: 1, + edgesTrimmed: 1, + idsRemoved: + 100 - + MAX_GRAPH_SOURCE_OBSERVATIONS + + (40 - MAX_GRAPH_SOURCE_OBSERVATIONS), + }); + }); + + it("skips an index entry whose record is missing", async () => { + const kv = mockKV(); + await kv.set("mem:graph:name-index", "concept|ghost", "gn_missing"); + await kv.set("mem:graph:edge-key", "a|b|related_to", "ge_missing"); + const r = await compactGraphProvenance(kv as never); + expect(r).toMatchObject({ + nodesScanned: 0, + edgesScanned: 0, + nodesTrimmed: 0, + edgesTrimmed: 0, + }); + }); + + it("visits a record once when two index keys point at it", async () => { + const kv = mockKV(); + await seedNode(kv, "gn_1", "hub", ids("obs", 100)); + await kv.set("mem:graph:name-index", "concept|hub-alias", "gn_1"); + const r = await compactGraphProvenance(kv as never); + expect(r.nodesScanned).toBe(1); + expect(r.nodesTrimmed).toBe(1); + }); + + it("trims the cached snapshot's top nodes and edges", async () => { + const kv = mockKV(); + const n = await seedNode(kv, "gn_1", "hub", ids("obs", 100)); + const e = await seedEdge(kv, "ge_1", "gn_1", "gn_2", ids("obs", 80)); + await kv.set("mem:graph:snapshot", "current", { + version: 1, + dirty: false, + topNodes: [n], + topEdges: [e], + topDegrees: { gn_1: 1 }, + stats: { totalNodes: 1, totalEdges: 1, nodesByType: {}, edgesByType: {} }, + updatedAt: "2026-09-01T00:00:00.000Z", + }); + const r = await compactGraphProvenance(kv as never); + const snap = (await kv.get<{ + topNodes: GraphNode[]; + topEdges: GraphEdge[]; + }>("mem:graph:snapshot", "current"))!; + expect(snap.topNodes[0]!.sourceObservationIds.length).toBe( + MAX_GRAPH_SOURCE_OBSERVATIONS, + ); + expect(snap.topEdges[0]!.sourceObservationIds.length).toBe( + MAX_GRAPH_SOURCE_OBSERVATIONS, + ); + expect(r.snapshotTrimmed).toBe(true); + }); + + it("an empty graph is a no-op", async () => { + const kv = mockKV(); + const r = await compactGraphProvenance(kv as never); + expect(r).toMatchObject({ + nodesScanned: 0, + edgesScanned: 0, + idsRemoved: 0, + snapshotTrimmed: false, + }); + }); + + it("a merge running during compaction is not lost", async () => { + const kv = mockKV(20); + await seedNode(kv, "gn_1", "hub", ids("obs", 100)); + const incoming: GraphNode = { + id: "gn_fresh", + type: "concept", + name: "hub", + properties: { added: "during-compaction" }, + sourceObservationIds: ["obs_during"], + createdAt: "2026-09-24T00:00:00.000Z", + }; + await Promise.all([ + compactGraphProvenance(kv as never), + persistGraphDelta(kv as never, [incoming], []), + ]); + const after = (await node(kv, "gn_1"))!; + expect(after.sourceObservationIds).toContain("obs_during"); + expect(after.properties).toMatchObject({ + keep: "me", + added: "during-compaction", + }); + expect(after.sourceObservationIds.length).toBeLessThanOrEqual( + MAX_GRAPH_SOURCE_OBSERVATIONS, + ); + }); + + it("is registered as mem::graph-compact", async () => { + const kv = mockKV(); + await seedNode(kv, "gn_1", "hub", ids("obs", 50)); + const functions = new Map(); + const sdk = { + registerFunction: ( + idOrOpts: string | { id: string }, + handler: Function, + ) => + functions.set( + typeof idOrOpts === "string" ? idOrOpts : idOrOpts.id, + handler, + ), + registerTrigger: () => {}, + }; + registerGraphFunction( + sdk as never, + kv as never, + { name: "noop", compress: vi.fn(), summarize: vi.fn() } as never, + ); + const fn = functions.get("mem::graph-compact"); + expect(fn).toBeDefined(); + const r = await fn!({}); + expect(r).toMatchObject({ success: true, nodesTrimmed: 1 }); + }); +}); + +describe("api::graph-compact endpoint", () => { + it("is an authenticated POST at /agentmemory/graph/compact that runs mem::graph-compact", async () => { + const { readFileSync } = await import("node:fs"); + const api = readFileSync("src/triggers/api.ts", "utf-8"); + expect(api).toMatch( + /registerFunction\("api::graph-compact",[\s\S]*?checkAuth\(req, secret\)[\s\S]*?function_id: "mem::graph-compact"/, + ); + expect(api).toMatch( + /function_id: "api::graph-compact",\s*config: \{ api_path: "\/agentmemory\/graph\/compact", http_method: "POST" \}/, + ); + }); +}); + +describe("chunked compaction", () => { + it("processes one slice of a scope and returns where the next slice starts", async () => { + const kv = mockKV(); + for (let i = 0; i < 5; i++) + await seedNode(kv, `gn_${i}`, `n${i}`, ids(`obs${i}`, 50)); + const first = await compactGraphProvenance(kv as never, { + scope: "nodes", + offset: 0, + limit: 2, + }); + expect(first).toMatchObject({ + nodesScanned: 2, + nodesTrimmed: 2, + total: 5, + nextOffset: 2, + }); + const trimmed = []; + for (let i = 0; i < 5; i++) + trimmed.push((await node(kv, `gn_${i}`))!.sourceObservationIds.length); + expect(trimmed).toEqual([ + MAX_GRAPH_SOURCE_OBSERVATIONS, + MAX_GRAPH_SOURCE_OBSERVATIONS, + 50, + 50, + 50, + ]); + }); + + it("walking every slice trims everything and ends with nextOffset null", async () => { + const kv = mockKV(); + for (let i = 0; i < 5; i++) + await seedNode(kv, `gn_${i}`, `n${i}`, ids(`obs${i}`, 50)); + let offset: number | null = 0; + let calls = 0; + while (offset !== null && calls < 10) { + const r = await compactGraphProvenance(kv as never, { + scope: "nodes", + offset, + limit: 2, + }); + offset = r.nextOffset === undefined ? -1 : r.nextOffset; + calls += 1; + } + expect(calls).toBe(3); + for (let i = 0; i < 5; i++) { + expect((await node(kv, `gn_${i}`))!.sourceObservationIds.length).toBe( + MAX_GRAPH_SOURCE_OBSERVATIONS, + ); + } + }); + + it("an edges slice touches edges only", async () => { + const kv = mockKV(); + const n = await seedNode(kv, "gn_1", "hub", ids("obs", 50)); + await seedEdge(kv, "ge_1", "gn_1", "gn_2", ids("obs", 50)); + await kv.set("mem:graph:snapshot", "current", { + version: 1, + dirty: false, + topNodes: [n], + topEdges: [], + topDegrees: {}, + stats: { totalNodes: 1, totalEdges: 1, nodesByType: {}, edgesByType: {} }, + updatedAt: "2026-09-01T00:00:00.000Z", + }); + const r = await compactGraphProvenance(kv as never, { + scope: "edges", + offset: 0, + limit: 10, + }); + expect(r).toMatchObject({ + edgesTrimmed: 1, + nodesScanned: 0, + nextOffset: null, + snapshotTrimmed: false, + }); + expect((await node(kv, "gn_1"))!.sourceObservationIds.length).toBe(50); + const snap = (await kv.get<{ topNodes: GraphNode[] }>( + "mem:graph:snapshot", + "current", + ))!; + expect(snap.topNodes[0]!.sourceObservationIds.length).toBe(50); + }); + + it("the snapshot scope trims only the snapshot", async () => { + const kv = mockKV(); + const n = await seedNode(kv, "gn_1", "hub", ids("obs", 50)); + await kv.set("mem:graph:snapshot", "current", { + version: 1, + dirty: false, + topNodes: [n], + topEdges: [], + topDegrees: {}, + stats: { totalNodes: 1, totalEdges: 0, nodesByType: {}, edgesByType: {} }, + updatedAt: "2026-09-01T00:00:00.000Z", + }); + const r = await compactGraphProvenance(kv as never, { scope: "snapshot" }); + expect(r).toMatchObject({ + snapshotTrimmed: true, + nodesScanned: 0, + nextOffset: null, + }); + expect((await node(kv, "gn_1"))!.sourceObservationIds.length).toBe(50); + }); + + it("an offset past the end scans nothing and ends", async () => { + const kv = mockKV(); + await seedNode(kv, "gn_1", "hub", ids("obs", 50)); + const r = await compactGraphProvenance(kv as never, { + scope: "nodes", + offset: 10, + limit: 5, + }); + expect(r).toMatchObject({ nodesScanned: 0, total: 1, nextOffset: null }); + }); + + it("rejects an unknown scope or a bad offset or limit", async () => { + const kv = mockKV(); + await expect( + compactGraphProvenance(kv as never, { scope: "bogus" as never }), + ).rejects.toThrow(/scope/); + await expect( + compactGraphProvenance(kv as never, { + scope: "nodes", + offset: -1, + limit: 5, + }), + ).rejects.toThrow(/offset/); + await expect( + compactGraphProvenance(kv as never, { + scope: "nodes", + offset: 0, + limit: 0, + }), + ).rejects.toThrow(/limit/); + }); + + it("mem::graph-compact passes the slice through", async () => { + const kv = mockKV(); + for (let i = 0; i < 3; i++) + await seedNode(kv, `gn_${i}`, `n${i}`, ids(`obs${i}`, 50)); + const functions = new Map(); + const sdk = { + registerFunction: ( + idOrOpts: string | { id: string }, + handler: Function, + ) => + functions.set( + typeof idOrOpts === "string" ? idOrOpts : idOrOpts.id, + handler, + ), + registerTrigger: () => {}, + }; + registerGraphFunction( + sdk as never, + kv as never, + { name: "noop", compress: vi.fn(), summarize: vi.fn() } as never, + ); + const r = await functions.get("mem::graph-compact")!({ + scope: "nodes", + offset: 0, + limit: 1, + }); + expect(r).toMatchObject({ success: true, nodesScanned: 1, nextOffset: 1 }); + }); +}); diff --git a/test/graph-provenance-cap.test.ts b/test/graph-provenance-cap.test.ts new file mode 100644 index 0000000000..44f436df76 --- /dev/null +++ b/test/graph-provenance-cap.test.ts @@ -0,0 +1,288 @@ +import { describe, it, expect, vi } from "vitest"; + +vi.mock("../src/logger.js", () => ({ + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, +})); + +import { + persistGraphDelta, + MAX_GRAPH_SOURCE_OBSERVATIONS, +} from "../src/functions/graph.js"; +import type { GraphEdge, GraphNode } from "../src/types.js"; + +// Each merge used to union every observation id in the batch into every +// node and edge it touched, with no bound. On one store that made 98% of +// the graph's bytes provenance ids (426 per node on average), and every +// traversal carried them. + +function mockKV() { + const store = new Map>(); + return { + get: async (scope: string, key: string): Promise => + (store.get(scope)?.get(key) as T) ?? null, + set: async (scope: string, key: string, data: T): Promise => { + if (!store.has(scope)) store.set(scope, new Map()); + store.get(scope)!.set(key, data); + return data; + }, + delete: async (scope: string, key: string): Promise => { + store.get(scope)?.delete(key); + }, + list: async (scope: string): Promise => { + const entries = store.get(scope); + return entries ? (Array.from(entries.values()) as T[]) : []; + }, + }; +} + +let seq = 0; +function node(name: string, sources: string[]): GraphNode { + seq += 1; + return { + id: `gn_${seq}`, + type: "concept", + name, + properties: {}, + sourceObservationIds: sources, + createdAt: "2026-09-24T00:00:00.000Z", + }; +} + +function edge(src: string, tgt: string, sources: string[]): GraphEdge { + seq += 1; + return { + id: `ge_${seq}`, + type: "related_to", + sourceNodeId: src, + targetNodeId: tgt, + weight: 0.5, + sourceObservationIds: sources, + createdAt: "2026-09-24T00:00:00.000Z", + }; +} + +const ids = (prefix: string, n: number) => + Array.from({ length: n }, (_, i) => `${prefix}_${i}`); + +async function onlyNode(kv: ReturnType, name: string) { + const all = await kv.list("mem:graph:nodes"); + const hits = all.filter((n) => n.name === name); + expect(hits.length).toBe(1); + return hits[0]!; +} + +async function onlyEdge(kv: ReturnType) { + const all = await kv.list("mem:graph:edges"); + expect(all.length).toBe(1); + return all[0]!; +} + +describe("graph provenance is bounded and attributed", () => { + it("the cap is at least the largest retrieval result size", () => { + expect(MAX_GRAPH_SOURCE_OBSERVATIONS).toBeGreaterThanOrEqual(20); + }); + + it("a node merged many times never exceeds the cap", async () => { + const kv = mockKV(); + for (let batch = 0; batch < 20; batch++) { + const batchIds = ids(`obs_b${batch}`, 10); + await persistGraphDelta(kv as never, [node("CLAUDE.md", batchIds)], []); + } + const n = await onlyNode(kv, "CLAUDE.md"); + expect(n.sourceObservationIds.length).toBe(MAX_GRAPH_SOURCE_OBSERVATIONS); + }); + + it("the cap keeps the NEWEST ids, in arrival order", async () => { + const kv = mockKV(); + const all: string[] = []; + for (let batch = 0; batch < 10; batch++) { + const batchIds = ids(`obs_b${batch}`, 10); + all.push(...batchIds); + await persistGraphDelta(kv as never, [node("hub", batchIds)], []); + } + const n = await onlyNode(kv, "hub"); + expect(n.sourceObservationIds).toEqual( + all.slice(-MAX_GRAPH_SOURCE_OBSERVATIONS), + ); + }); + + it("a re-seen id moves to the newest end and survives trimming", async () => { + const kv = mockKV(); + await persistGraphDelta(kv as never, [node("hub", ["obs_old"])], []); + const filler = ids("obs_fill", MAX_GRAPH_SOURCE_OBSERVATIONS - 1); + await persistGraphDelta(kv as never, [node("hub", filler)], []); + await persistGraphDelta(kv as never, [node("hub", ["obs_old"])], []); + await persistGraphDelta(kv as never, [node("hub", ["obs_new"])], []); + const n = await onlyNode(kv, "hub"); + expect(n.sourceObservationIds).toContain("obs_old"); + expect(n.sourceObservationIds.at(-1)).toBe("obs_new"); + expect(n.sourceObservationIds.at(-2)).toBe("obs_old"); + expect(n.sourceObservationIds).not.toContain("obs_fill_0"); + }); + + it("ids are never duplicated", async () => { + const kv = mockKV(); + for (let i = 0; i < 3; i++) { + await persistGraphDelta( + kv as never, + [node("hub", ["obs_a", "obs_b"])], + [], + ); + } + const n = await onlyNode(kv, "hub"); + expect(n.sourceObservationIds).toEqual(["obs_a", "obs_b"]); + }); + + it("a brand-new node carrying more than the cap is trimmed on first write", async () => { + const kv = mockKV(); + const many = ids("obs", MAX_GRAPH_SOURCE_OBSERVATIONS + 15); + await persistGraphDelta(kv as never, [node("fresh", many)], []); + const n = await onlyNode(kv, "fresh"); + expect(n.sourceObservationIds).toEqual( + many.slice(-MAX_GRAPH_SOURCE_OBSERVATIONS), + ); + }); + + it("a merged node gains ITS OWN sources, not every id in the batch", async () => { + const kv = mockKV(); + await persistGraphDelta(kv as never, [node("a.ts", ["obs_1"])], []); + // A batch of obs_2..obs_4 in which only obs_3 mentioned a.ts. + await persistGraphDelta( + kv as never, + [node("a.ts", ["obs_3"]), node("b.ts", ["obs_2", "obs_4"])], + [], + ); + const a = await onlyNode(kv, "a.ts"); + expect(a.sourceObservationIds).toEqual(["obs_1", "obs_3"]); + }); + + it("a merged edge gains its own sources, bounded, not the batch", async () => { + const kv = mockKV(); + const a = node("a", ["obs_0"]); + const b = node("b", ["obs_0"]); + await persistGraphDelta(kv as never, [a, b], [edge(a.id, b.id, ["obs_0"])]); + for (let batch = 1; batch <= 10; batch++) { + // A batch of ten observations; only the first one links a and b. + const batchIds = ids(`obs_b${batch}`, 10); + const a2 = node("a", batchIds); + const b2 = node("b", batchIds); + await persistGraphDelta( + kv as never, + [a2, b2], + [edge(a2.id, b2.id, [batchIds[0]!])], + ); + } + const e = await onlyEdge(kv); + expect(e.sourceObservationIds).toEqual([ + "obs_0", + ...Array.from({ length: 10 }, (_, i) => `obs_b${i + 1}_0`), + ]); + }); + + it("a merged edge is capped", async () => { + const kv = mockKV(); + const a = node("a", ["obs_0"]); + const b = node("b", ["obs_0"]); + await persistGraphDelta(kv as never, [a, b], [edge(a.id, b.id, ["obs_0"])]); + const many = ids("obs_e", MAX_GRAPH_SOURCE_OBSERVATIONS * 2); + const a2 = node("a", ["obs_x"]); + const b2 = node("b", ["obs_x"]); + await persistGraphDelta(kv as never, [a2, b2], [edge(a2.id, b2.id, many)]); + const e = await onlyEdge(kv); + expect(e.sourceObservationIds).toEqual( + many.slice(-MAX_GRAPH_SOURCE_OBSERVATIONS), + ); + }); + + it("a brand-new edge carrying more than the cap is trimmed on first write", async () => { + const kv = mockKV(); + const a = node("a", ["obs_0"]); + const b = node("b", ["obs_0"]); + const many = ids("obs", MAX_GRAPH_SOURCE_OBSERVATIONS + 5); + await persistGraphDelta(kv as never, [a, b], [edge(a.id, b.id, many)]); + const e = await onlyEdge(kv); + expect(e.sourceObservationIds).toEqual( + many.slice(-MAX_GRAPH_SOURCE_OBSERVATIONS), + ); + }); + + it("a record written before this change with an oversized list is trimmed on its next merge", async () => { + const kv = mockKV(); + const legacy = node("legacy", ids("obs_legacy", 500)); + await kv.set("mem:graph:nodes", legacy.id, legacy); + await kv.set("mem:graph:name-index", "concept|legacy", legacy.id); + await persistGraphDelta(kv as never, [node("legacy", ["obs_now"])], []); + const n = await onlyNode(kv, "legacy"); + expect(n.sourceObservationIds.length).toBe(MAX_GRAPH_SOURCE_OBSERVATIONS); + expect(n.sourceObservationIds.at(-1)).toBe("obs_now"); + }); +}); + +describe("temporal graph provenance is bounded", () => { + function mockSdk() { + const functions = new Map(); + return { + registerFunction: ( + idOrOpts: string | { id: string }, + handler: Function, + ) => { + functions.set( + typeof idOrOpts === "string" ? idOrOpts : idOrOpts.id, + handler, + ); + }, + registerTrigger: () => {}, + trigger: async (id: string, data: unknown) => functions.get(id)!(data), + }; + } + + it("a temporal node merged many times keeps only the newest ids", async () => { + const { registerTemporalGraphFunctions } = await import( + "../src/functions/temporal-graph.js" + ); + const response = ` + + engineer + + +`; + const provider = { + name: "test", + compress: vi.fn().mockResolvedValue(response), + summarize: vi.fn().mockResolvedValue(response), + }; + const sdk = mockSdk(); + const kv = mockKV(); + registerTemporalGraphFunctions( + sdk as never, + kv as never, + provider as never, + ); + + const all: string[] = []; + for (let batch = 0; batch < 10; batch++) { + const observations = ids(`obs_t${batch}`, 10).map((id) => { + all.push(id); + return { + id, + title: "t", + narrative: "Alice works at Acme", + concepts: [], + files: [], + type: "conversation", + timestamp: "2026-09-24T00:00:00Z", + }; + }); + const r = (await sdk.trigger("mem::temporal-graph-extract", { + observations, + })) as { + success: boolean; + }; + expect(r.success).toBe(true); + } + const n = await onlyNode(kv, "Alice"); + expect(n.sourceObservationIds).toEqual( + all.slice(-MAX_GRAPH_SOURCE_OBSERVATIONS), + ); + }); +}); From 82b9e48bee3fd06926ae92b03809b34d6ca186be Mon Sep 17 00:00:00 2001 From: David Waltermire Date: Tue, 29 Sep 2026 22:39:21 -0400 Subject: [PATCH 02/11] fix(api): validate graph/compact input and report its real failures Review found two defects in POST /agentmemory/graph/compact. Invalid input returned HTTP 200. mem::graph-compact reports a bad scope, offset or limit as { success: false }, so the route passed that through with 200. The route now rejects an unknown scope, a negative or non-integer offset, and a non-positive or non-integer limit with 400, before invoking compaction. It accepts JSON numbers only, so "5" is rejected. Other success:false results still return 200, because that envelope also carries operational failures. Every failure was reported as "graph disabled". The catch returned graphDisabledResponse(), a 503 with GRAPH_EXTRACTION_ENABLED advice, but compaction is registered whether or not graph extraction is enabled. A timeout, such as an unsliced run on a large graph, now returns 504 and says to run in slices. Any other failure returns 500. Signed-off-by: David Waltermire --- src/triggers/api.ts | 34 ++++++++++++----- test/graph-compact.test.ts | 75 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 100 insertions(+), 9 deletions(-) diff --git a/src/triggers/api.ts b/src/triggers/api.ts index 5a229dfef2..22a806bb62 100644 --- a/src/triggers/api.ts +++ b/src/triggers/api.ts @@ -1,4 +1,4 @@ -import { TriggerAction, type IIIClient } from "iii-sdk"; +import { InvocationError, TriggerAction, type IIIClient } from "iii-sdk"; import type { HttpRequest } from "@iii-dev/helpers/http"; import { randomBytes } from "node:crypto"; import type { Session, CompressedObservation, HookPayload, CommitLink, SessionSummary, HealthSnapshot, AuditQueryResult, AuditMigrationState } from "../types.js"; @@ -2023,19 +2023,35 @@ export function registerApiTriggers( async (req: HttpRequest): Promise => { const authErr = checkAuth(req, secret); if (authErr) return authErr; + // Validate here: mem::graph-compact reports bad input as success:false, + // which would reach the caller as HTTP 200. JSON numbers only. + const body = (req.body ?? {}) as Record; + const { scope, offset, limit } = body; + if (scope !== undefined && scope !== "nodes" && scope !== "edges" && scope !== "snapshot") { + return { status_code: 400, body: { error: "scope must be nodes, edges or snapshot" } }; + } + if (offset !== undefined && !(Number.isInteger(offset) && (offset as number) >= 0)) { + return { status_code: 400, body: { error: "offset must be a non-negative integer" } }; + } + if (limit !== undefined && !(Number.isInteger(limit) && (limit as number) >= 1)) { + return { status_code: 400, body: { error: "limit must be a positive integer" } }; + } try { - const body = (req.body ?? {}) as Record; const result = await sdk.trigger({ function_id: "mem::graph-compact", - payload: { - scope: body.scope, - offset: body.offset, - limit: body.limit, - }, + payload: { scope, offset, limit }, }); return { status_code: 200, body: result }; - } catch { - return graphDisabledResponse(); + } catch (err) { + // Compaction is registered whether or not graph extraction is on, so + // a failure here is never the graph-disabled case. + if (err instanceof InvocationError && err.code === "TIMEOUT") { + return { + status_code: 504, + body: { error: "Graph compaction timed out; pass scope, offset and limit to run it in slices" }, + }; + } + return { status_code: 500, body: { error: "Graph compaction failed" } }; } }, ); diff --git a/test/graph-compact.test.ts b/test/graph-compact.test.ts index 6b32a910c2..9cc4ac2335 100644 --- a/test/graph-compact.test.ts +++ b/test/graph-compact.test.ts @@ -467,3 +467,78 @@ describe("chunked compaction", () => { expect(r).toMatchObject({ success: true, nodesScanned: 1, nextOffset: 1 }); }); }); + +describe("api::graph-compact responses", () => { + async function route( + trigger: (req: { + function_id: string; + payload?: unknown; + }) => Promise, + ) { + const { registerApiTriggers } = await import("../src/triggers/api.js"); + const fns = new Map(); + const sdk = { + registerFunction: (id: string, h: Function) => fns.set(id, h), + registerTrigger: () => {}, + trigger, + }; + registerApiTriggers(sdk as never, mockKV() as never); + return (body: unknown) => + fns.get("api::graph-compact")!({ headers: {}, body }) as Promise<{ + status_code: number; + body: Record; + }>; + } + + const ok = async () => ({ success: true, nodesTrimmed: 0 }); + + it("rejects invalid slice parameters with 400 before invoking compaction", async () => { + let calls = 0; + const post = await route(async () => { + calls += 1; + return ok(); + }); + for (const body of [ + { scope: "bogus" }, + { scope: "nodes", offset: -1 }, + { scope: "nodes", offset: 1.5 }, + { scope: "nodes", offset: "5" }, + { scope: "nodes", limit: 0 }, + { scope: "nodes", limit: "5" }, + ]) { + const r = await post(body); + expect(r.status_code, JSON.stringify(body)).toBe(400); + } + expect(calls).toBe(0); + }); + + it("passes valid parameters through and returns 200", async () => { + let seen: unknown; + const post = await route(async (req) => { + seen = req.payload; + return ok(); + }); + const r = await post({ scope: "edges", offset: 0, limit: 1000 }); + expect(r.status_code).toBe(200); + expect(seen).toEqual({ scope: "edges", offset: 0, limit: 1000 }); + expect((await post({})).status_code).toBe(200); + }); + + it("returns 504 when the invocation times out", async () => { + const { InvocationError } = await import("iii-sdk"); + const post = await route(async () => { + throw new InvocationError({ code: "TIMEOUT", message: "timed out" }); + }); + const r = await post({}); + expect(r.status_code).toBe(504); + }); + + it("returns 500 for other failures, not the graph-disabled advice", async () => { + const post = await route(async () => { + throw new Error("worker gone"); + }); + const r = await post({}); + expect(r.status_code).toBe(500); + expect(JSON.stringify(r.body)).not.toMatch(/GRAPH_EXTRACTION_ENABLED/); + }); +}); From b00eaed19c28c063c7c0a3a05f6c256e6f9fc37f Mon Sep 17 00:00:00 2001 From: David Waltermire Date: Wed, 30 Sep 2026 01:08:00 -0400 Subject: [PATCH 03/11] fix(api): return 500 when graph compaction reports a failure mem::graph-compact catches a storage error and returns { success: false, error } instead of throwing. sdk.trigger therefore resolves, and POST /agentmemory/graph/compact answered 200. The route validates its input first, so a success:false result reaching this point is an operational failure. It now returns 500 with the generic "Graph compaction failed" message. The detailed error stays in the server-side log, which mem::graph-compact already writes. Signed-off-by: David Waltermire --- src/triggers/api.ts | 5 +++++ test/graph-compact.test.ts | 12 ++++++++++++ 2 files changed, 17 insertions(+) diff --git a/src/triggers/api.ts b/src/triggers/api.ts index 22a806bb62..72882d0b80 100644 --- a/src/triggers/api.ts +++ b/src/triggers/api.ts @@ -2041,6 +2041,11 @@ export function registerApiTriggers( function_id: "mem::graph-compact", payload: { scope, offset, limit }, }); + // Input was validated above, so success:false here is an operational + // failure (mem::graph-compact logs the detail). Do not answer 200. + if ((result as { success?: boolean } | null)?.success === false) { + return { status_code: 500, body: { error: "Graph compaction failed" } }; + } return { status_code: 200, body: result }; } catch (err) { // Compaction is registered whether or not graph extraction is on, so diff --git a/test/graph-compact.test.ts b/test/graph-compact.test.ts index 9cc4ac2335..67b3044367 100644 --- a/test/graph-compact.test.ts +++ b/test/graph-compact.test.ts @@ -524,6 +524,18 @@ describe("api::graph-compact responses", () => { expect((await post({})).status_code).toBe(200); }); + it("returns 500 when compaction reports its own failure, without leaking the error", async () => { + // mem::graph-compact catches a storage error and returns success:false, + // so sdk.trigger resolves; the route must not answer 200. + const post = await route(async () => ({ + success: false, + error: "kv write failed: disk full", + })); + const r = await post({ scope: "nodes", offset: 0, limit: 10 }); + expect(r.status_code).toBe(500); + expect(r.body).toEqual({ error: "Graph compaction failed" }); + }); + it("returns 504 when the invocation times out", async () => { const { InvocationError } = await import("iii-sdk"); const post = await route(async () => { From 93143edb5ed7cc1b1b60d83f054922f3b81c1de1 Mon Sep 17 00:00:00 2001 From: David Waltermire Date: Wed, 30 Sep 2026 21:00:24 -0400 Subject: [PATCH 04/11] docs: add doc comments to the provenance and compaction helpers The explanations above boundSources and compactGraphProvenance are now doc comments, word for word. mergeNode, mergeEdge and the two compaction inner helpers get one-line doc comments, and so do the test helpers. This answers the docstring-coverage pre-merge check. No behavior changes. Signed-off-by: David Waltermire --- src/functions/graph.ts | 34 +++++++++++++++++++------------ test/graph-compact.test.ts | 7 +++++++ test/graph-provenance-cap.test.ts | 7 +++++++ 3 files changed, 35 insertions(+), 13 deletions(-) diff --git a/src/functions/graph.ts b/src/functions/graph.ts index 0bd365838b..68b3263618 100644 --- a/src/functions/graph.ts +++ b/src/functions/graph.ts @@ -286,8 +286,10 @@ function snapshotPushEdgeIfBothInTop( // 20 results, so a record never needs more than its newest sources. export const MAX_GRAPH_SOURCE_OBSERVATIONS = 32; -// Union in arrival order, with a re-seen id moved to the newest end, then -// keep the newest MAX_GRAPH_SOURCE_OBSERVATIONS. +/** + * Union in arrival order, with a re-seen id moved to the newest end, then + * keep the newest MAX_GRAPH_SOURCE_OBSERVATIONS. + */ export function boundSources(existing: string[], incoming: string[]): string[] { const merged = new Set(existing); for (const id of incoming) { @@ -297,6 +299,7 @@ export function boundSources(existing: string[], incoming: string[]): string[] { return [...merged].slice(-MAX_GRAPH_SOURCE_OBSERVATIONS); } +/** Merges an extracted node into the stored one: bounded own sources, newer properties. */ function mergeNode( existing: GraphNode, incoming: GraphNode, @@ -313,6 +316,7 @@ function mergeNode( }; } +/** Merges an extracted edge into the stored one, keeping its bounded own sources. */ function mergeEdge(existing: GraphEdge, incoming: GraphEdge): GraphEdge { return { ...existing, @@ -724,17 +728,19 @@ export interface GraphCompactOptions { const COMPACT_SCOPES: readonly GraphCompactScope[] = ["nodes", "edges", "snapshot"]; -// Trims provenance written before the cap existed. Records are reached -// through the name and edge-key indexes, whose values are short ids, and -// then read and written one key at a time. The nodes and edges scopes are -// never listed: that payload is what drops the worker on a large store. -// A record missing from both indexes is not reached; it is also never -// merged again, so it cannot grow. -// -// With no options it does everything in one call. With a scope it does one -// slice, so each invocation stays short enough to finish between worker -// reconnects; the caller walks nextOffset until it is null. Re-running any -// slice is safe because trimmed records are skipped. +/** + * Trims provenance written before the cap existed. Records are reached + * through the name and edge-key indexes, whose values are short ids, and + * then read and written one key at a time. The nodes and edges scopes are + * never listed: that payload is what drops the worker on a large store. + * A record missing from both indexes is not reached; it is also never + * merged again, so it cannot grow. + * + * With no options it does everything in one call. With a scope it does one + * slice, so each invocation stays short enough to finish between worker + * reconnects; the caller walks nextOffset until it is null. Re-running any + * slice is safe because trimmed records are skipped. + */ export async function compactGraphProvenance( kv: StateKV, opts: GraphCompactOptions = {}, @@ -762,6 +768,7 @@ export async function compactGraphProvenance( nextOffset: null, }; + /** Trims one slice of records reached through an index scope, one locked key at a time. */ const trimScope = async ( indexScope: string, recordScope: string, @@ -790,6 +797,7 @@ export async function compactGraphProvenance( return { scanned, trimmed }; }; + /** Trims the cached snapshot's top nodes and edges under the same lock. */ const trimSnapshot = () => withKeyedLock("graph:persist", async () => { const snap = await readSnapshot(kv); diff --git a/test/graph-compact.test.ts b/test/graph-compact.test.ts index 67b3044367..0f2ed1de66 100644 --- a/test/graph-compact.test.ts +++ b/test/graph-compact.test.ts @@ -20,6 +20,7 @@ import type { GraphEdge, GraphNode } from "../src/types.js"; // value is captured. That forces the interleaving that loses a merge when // writers are not serialized: compaction reads, a merge writes, then // compaction writes back its stale copy. +/** In-memory StateKV that records list and set calls; can delay the first node read to force a race. */ function mockKV(slowFirstNodeReadMs = 0) { const store = new Map>(); const calls = { list: [] as string[], set: [] as string[] }; @@ -57,9 +58,11 @@ function mockKV(slowFirstNodeReadMs = 0) { } type KV = ReturnType; +/** `n` distinct observation ids with a prefix. */ const ids = (prefix: string, n: number) => Array.from({ length: n }, (_, i) => `${prefix}_${i}`); +/** Stores a node and its name-index entry. */ async function seedNode(kv: KV, id: string, name: string, sources: string[]) { const n: GraphNode = { id, @@ -75,6 +78,7 @@ async function seedNode(kv: KV, id: string, name: string, sources: string[]) { return n; } +/** Stores an edge and its edge-key entry. */ async function seedEdge( kv: KV, id: string, @@ -96,7 +100,9 @@ async function seedEdge( return e; } +/** Reads a stored node by id. */ const node = (kv: KV, id: string) => kv.get("mem:graph:nodes", id); +/** Reads a stored edge by id. */ const edge = (kv: KV, id: string) => kv.get("mem:graph:edges", id); describe("compactGraphProvenance", () => { @@ -469,6 +475,7 @@ describe("chunked compaction", () => { }); describe("api::graph-compact responses", () => { + /** Registers the API handlers with a stub trigger and returns a caller for api::graph-compact. */ async function route( trigger: (req: { function_id: string; diff --git a/test/graph-provenance-cap.test.ts b/test/graph-provenance-cap.test.ts index 44f436df76..29b0afae2d 100644 --- a/test/graph-provenance-cap.test.ts +++ b/test/graph-provenance-cap.test.ts @@ -15,6 +15,7 @@ import type { GraphEdge, GraphNode } from "../src/types.js"; // the graph's bytes provenance ids (426 per node on average), and every // traversal carried them. +/** In-memory StateKV: one Map per scope. */ function mockKV() { const store = new Map>(); return { @@ -36,6 +37,7 @@ function mockKV() { } let seq = 0; +/** A concept node with the given name and sources. */ function node(name: string, sources: string[]): GraphNode { seq += 1; return { @@ -48,6 +50,7 @@ function node(name: string, sources: string[]): GraphNode { }; } +/** A related_to edge between two node ids with the given sources. */ function edge(src: string, tgt: string, sources: string[]): GraphEdge { seq += 1; return { @@ -61,9 +64,11 @@ function edge(src: string, tgt: string, sources: string[]): GraphEdge { }; } +/** `n` distinct observation ids with a prefix. */ const ids = (prefix: string, n: number) => Array.from({ length: n }, (_, i) => `${prefix}_${i}`); +/** The single stored node with this name; fails if there is not exactly one. */ async function onlyNode(kv: ReturnType, name: string) { const all = await kv.list("mem:graph:nodes"); const hits = all.filter((n) => n.name === name); @@ -71,6 +76,7 @@ async function onlyNode(kv: ReturnType, name: string) { return hits[0]!; } +/** The single stored edge; fails if there is not exactly one. */ async function onlyEdge(kv: ReturnType) { const all = await kv.list("mem:graph:edges"); expect(all.length).toBe(1); @@ -219,6 +225,7 @@ describe("graph provenance is bounded and attributed", () => { }); describe("temporal graph provenance is bounded", () => { + /** SDK stub that records registered functions so they can be triggered directly. */ function mockSdk() { const functions = new Map(); return { From 477dd0dfa9f60685f30524f2bfc31960a7cd4fe0 Mon Sep 17 00:00:00 2001 From: David Waltermire Date: Wed, 30 Sep 2026 21:16:37 -0400 Subject: [PATCH 05/11] docs: add doc comments to the graph function registrars Each comment names the functions its registrar exposes. Signed-off-by: David Waltermire --- src/functions/graph.ts | 4 ++++ src/functions/temporal-graph.ts | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/src/functions/graph.ts b/src/functions/graph.ts index 68b3263618..eda6fba6ad 100644 --- a/src/functions/graph.ts +++ b/src/functions/graph.ts @@ -836,6 +836,10 @@ export async function compactGraphProvenance( return result; } +/** + * Registers the knowledge-graph functions: mem::graph-extract, mem::graph-query, + * mem::graph-stats, mem::graph-compact and mem::graph-reset. + */ export function registerGraphFunction( sdk: IIIClient, kv: StateKV, diff --git a/src/functions/temporal-graph.ts b/src/functions/temporal-graph.ts index 4ee0c4c1be..e40be10ddc 100644 --- a/src/functions/temporal-graph.ts +++ b/src/functions/temporal-graph.ts @@ -151,6 +151,10 @@ function parseTemporalGraphXml( return { nodes, edges }; } +/** + * Registers the temporal-graph functions: mem::temporal-graph-extract, + * mem::temporal-query and mem::differential-state. + */ export function registerTemporalGraphFunctions( sdk: IIIClient, kv: StateKV, From a909039ab751afebf6dd147f38195a7267af7ad0 Mon Sep 17 00:00:00 2001 From: David Waltermire Date: Wed, 30 Sep 2026 21:41:39 -0400 Subject: [PATCH 06/11] fix(graph): bound temporal edge provenance before it is written The temporal parser gives every edge the whole batch's observation ids, and the edge write stored them without the cap that nodes already had. A batch of more than 32 observations wrote an edge over the cap. Signed-off-by: David Waltermire --- src/functions/temporal-graph.ts | 1 + test/graph-provenance-cap.test.ts | 50 +++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/src/functions/temporal-graph.ts b/src/functions/temporal-graph.ts index e40be10ddc..111c726368 100644 --- a/src/functions/temporal-graph.ts +++ b/src/functions/temporal-graph.ts @@ -259,6 +259,7 @@ export function registerTemporalGraphFunctions( edge.version = (existingEdge.version || 1) + 1; } + edge.sourceObservationIds = boundSources([], edge.sourceObservationIds ?? []); await kv.set(KV.graphEdges, edge.id, edge); existingEdges.push(edge); } diff --git a/test/graph-provenance-cap.test.ts b/test/graph-provenance-cap.test.ts index 29b0afae2d..f8bf16df78 100644 --- a/test/graph-provenance-cap.test.ts +++ b/test/graph-provenance-cap.test.ts @@ -292,4 +292,54 @@ describe("temporal graph provenance is bounded", () => { all.slice(-MAX_GRAPH_SOURCE_OBSERVATIONS), ); }); + + it("a temporal edge from a batch larger than the cap keeps only the newest ids", async () => { + const { registerTemporalGraphFunctions } = await import( + "../src/functions/temporal-graph.js" + ); + const response = ` + + + + + + + +`; + const provider = { + name: "test", + compress: vi.fn().mockResolvedValue(response), + summarize: vi.fn().mockResolvedValue(response), + }; + const sdk = mockSdk(); + const kv = mockKV(); + registerTemporalGraphFunctions( + sdk as never, + kv as never, + provider as never, + ); + + const batch = ids("obs_e", MAX_GRAPH_SOURCE_OBSERVATIONS + 8); + for (let round = 0; round < 2; round++) { + const r = (await sdk.trigger("mem::temporal-graph-extract", { + observations: batch.map((id) => ({ + id, + title: "t", + narrative: "Alice works on Acme", + concepts: [], + files: [], + type: "conversation", + timestamp: "2026-09-24T00:00:00Z", + })), + })) as { success: boolean }; + expect(r.success).toBe(true); + } + const edges = await kv.list("mem:graph:edges"); + expect(edges).toHaveLength(2); + for (const e of edges) { + expect(e.sourceObservationIds).toEqual( + batch.slice(-MAX_GRAPH_SOURCE_OBSERVATIONS), + ); + } + }); }); From 05138d31162573515ee34a476227ac8eb2b9ec85 Mon Sep 17 00:00:00 2001 From: Rohit Ghumare Date: Thu, 1 Oct 2026 13:22:54 +0530 Subject: [PATCH 07/11] chore(graph): drop code comments added with provenance compaction The repo keeps code free of comments; the behaviour is covered by tests and the endpoint is documented in the README and REST reference. --- src/functions/graph.ts | 32 ------------------------------- src/functions/temporal-graph.ts | 4 ---- src/triggers/api.ts | 8 -------- test/graph-compact.test.ts | 17 ---------------- test/graph-provenance-cap.test.ts | 14 -------------- 5 files changed, 75 deletions(-) diff --git a/src/functions/graph.ts b/src/functions/graph.ts index 662af70edf..16559500af 100644 --- a/src/functions/graph.ts +++ b/src/functions/graph.ts @@ -301,17 +301,8 @@ function snapshotPushEdgeIfBothInTop( } } -// Provenance is bounded. Every merge used to union the whole batch's -// observation ids into every node and edge it touched, which grew hot -// nodes without limit: on one store 98% of the graph's bytes were these -// ids, and every kv.list traversal carried them. Retrieval reads at most -// 20 results, so a record never needs more than its newest sources. export const MAX_GRAPH_SOURCE_OBSERVATIONS = 32; -/** - * Union in arrival order, with a re-seen id moved to the newest end, then - * keep the newest MAX_GRAPH_SOURCE_OBSERVATIONS. - */ export function boundSources(existing: string[], incoming: string[]): string[] { const merged = new Set(existing); for (const id of incoming) { @@ -321,7 +312,6 @@ export function boundSources(existing: string[], incoming: string[]): string[] { return [...merged].slice(-MAX_GRAPH_SOURCE_OBSERVATIONS); } -/** Merges an extracted node into the stored one: bounded own sources, newer properties. */ function mergeNode( existing: GraphNode, incoming: GraphNode, @@ -338,7 +328,6 @@ function mergeNode( }; } -/** Merges an extracted edge into the stored one, keeping its bounded own sources. */ function mergeEdge(existing: GraphEdge, incoming: GraphEdge): GraphEdge { return { ...existing, @@ -733,8 +722,6 @@ export interface GraphCompactResult { edgesTrimmed: number; idsRemoved: number; snapshotTrimmed: boolean; - // Records in the sliced index, and where the next slice starts; null - // when this call reached the end. A full run always ends with null. total?: number; nextOffset: number | null; } @@ -749,19 +736,6 @@ export interface GraphCompactOptions { const COMPACT_SCOPES: readonly GraphCompactScope[] = ["nodes", "edges", "snapshot"]; -/** - * Trims provenance written before the cap existed. Records are reached - * through the name and edge-key indexes, whose values are short ids, and - * then read and written one key at a time. The nodes and edges scopes are - * never listed: that payload is what drops the worker on a large store. - * A record missing from both indexes is not reached; it is also never - * merged again, so it cannot grow. - * - * With no options it does everything in one call. With a scope it does one - * slice, so each invocation stays short enough to finish between worker - * reconnects; the caller walks nextOffset until it is null. Re-running any - * slice is safe because trimmed records are skipped. - */ export async function compactGraphProvenance( kv: StateKV, opts: GraphCompactOptions = {}, @@ -789,7 +763,6 @@ export async function compactGraphProvenance( nextOffset: null, }; - /** Trims one slice of records reached through an index scope, one locked key at a time. */ const trimScope = async ( indexScope: string, recordScope: string, @@ -818,7 +791,6 @@ export async function compactGraphProvenance( return { scanned, trimmed }; }; - /** Trims the cached snapshot's top nodes and edges under the same lock. */ const trimSnapshot = () => withKeyedLock("graph:persist", async () => { const snap = await readSnapshot(kv); @@ -857,10 +829,6 @@ export async function compactGraphProvenance( return result; } -/** - * Registers the knowledge-graph functions: mem::graph-extract, mem::graph-query, - * mem::graph-stats, mem::graph-compact and mem::graph-reset. - */ export function registerGraphFunction( sdk: IIIClient, kv: StateKV, diff --git a/src/functions/temporal-graph.ts b/src/functions/temporal-graph.ts index 111c726368..d79dd04f20 100644 --- a/src/functions/temporal-graph.ts +++ b/src/functions/temporal-graph.ts @@ -151,10 +151,6 @@ function parseTemporalGraphXml( return { nodes, edges }; } -/** - * Registers the temporal-graph functions: mem::temporal-graph-extract, - * mem::temporal-query and mem::differential-state. - */ export function registerTemporalGraphFunctions( sdk: IIIClient, kv: StateKV, diff --git a/src/triggers/api.ts b/src/triggers/api.ts index d01d991d51..c10868aab6 100644 --- a/src/triggers/api.ts +++ b/src/triggers/api.ts @@ -2027,14 +2027,10 @@ export function registerApiTriggers( config: { api_path: "/agentmemory/graph/reset", http_method: "POST" }, }); - // Trims graph provenance written before the per-record cap, walking the - // small name and edge-key indexes instead of listing the graph. sdk.registerFunction("api::graph-compact", async (req: HttpRequest): Promise => { const authErr = checkAuth(req, secret); if (authErr) return authErr; - // Validate here: mem::graph-compact reports bad input as success:false, - // which would reach the caller as HTTP 200. JSON numbers only. const body = (req.body ?? {}) as Record; const { scope, offset, limit } = body; if (scope !== undefined && scope !== "nodes" && scope !== "edges" && scope !== "snapshot") { @@ -2051,15 +2047,11 @@ export function registerApiTriggers( function_id: "mem::graph-compact", payload: { scope, offset, limit }, }); - // Input was validated above, so success:false here is an operational - // failure (mem::graph-compact logs the detail). Do not answer 200. if ((result as { success?: boolean } | null)?.success === false) { return { status_code: 500, body: { error: "Graph compaction failed" } }; } return { status_code: 200, body: result }; } catch (err) { - // Compaction is registered whether or not graph extraction is on, so - // a failure here is never the graph-disabled case. if (err instanceof InvocationError && err.code === "TIMEOUT") { return { status_code: 504, diff --git a/test/graph-compact.test.ts b/test/graph-compact.test.ts index 0f2ed1de66..4cd175276a 100644 --- a/test/graph-compact.test.ts +++ b/test/graph-compact.test.ts @@ -12,15 +12,6 @@ import { } from "../src/functions/graph.js"; import type { GraphEdge, GraphNode } from "../src/types.js"; -// Compaction trims provenance written before the cap existed. It must -// reach records through the small name and edge-key indexes only: listing -// the nodes or edges scope is the payload that drops the worker. - -// slowFirstNodeRead delays only the FIRST read of a node record, after the -// value is captured. That forces the interleaving that loses a merge when -// writers are not serialized: compaction reads, a merge writes, then -// compaction writes back its stale copy. -/** In-memory StateKV that records list and set calls; can delay the first node read to force a race. */ function mockKV(slowFirstNodeReadMs = 0) { const store = new Map>(); const calls = { list: [] as string[], set: [] as string[] }; @@ -58,11 +49,9 @@ function mockKV(slowFirstNodeReadMs = 0) { } type KV = ReturnType; -/** `n` distinct observation ids with a prefix. */ const ids = (prefix: string, n: number) => Array.from({ length: n }, (_, i) => `${prefix}_${i}`); -/** Stores a node and its name-index entry. */ async function seedNode(kv: KV, id: string, name: string, sources: string[]) { const n: GraphNode = { id, @@ -78,7 +67,6 @@ async function seedNode(kv: KV, id: string, name: string, sources: string[]) { return n; } -/** Stores an edge and its edge-key entry. */ async function seedEdge( kv: KV, id: string, @@ -100,9 +88,7 @@ async function seedEdge( return e; } -/** Reads a stored node by id. */ const node = (kv: KV, id: string) => kv.get("mem:graph:nodes", id); -/** Reads a stored edge by id. */ const edge = (kv: KV, id: string) => kv.get("mem:graph:edges", id); describe("compactGraphProvenance", () => { @@ -475,7 +461,6 @@ describe("chunked compaction", () => { }); describe("api::graph-compact responses", () => { - /** Registers the API handlers with a stub trigger and returns a caller for api::graph-compact. */ async function route( trigger: (req: { function_id: string; @@ -532,8 +517,6 @@ describe("api::graph-compact responses", () => { }); it("returns 500 when compaction reports its own failure, without leaking the error", async () => { - // mem::graph-compact catches a storage error and returns success:false, - // so sdk.trigger resolves; the route must not answer 200. const post = await route(async () => ({ success: false, error: "kv write failed: disk full", diff --git a/test/graph-provenance-cap.test.ts b/test/graph-provenance-cap.test.ts index f8bf16df78..8def25bf72 100644 --- a/test/graph-provenance-cap.test.ts +++ b/test/graph-provenance-cap.test.ts @@ -10,12 +10,6 @@ import { } from "../src/functions/graph.js"; import type { GraphEdge, GraphNode } from "../src/types.js"; -// Each merge used to union every observation id in the batch into every -// node and edge it touched, with no bound. On one store that made 98% of -// the graph's bytes provenance ids (426 per node on average), and every -// traversal carried them. - -/** In-memory StateKV: one Map per scope. */ function mockKV() { const store = new Map>(); return { @@ -37,7 +31,6 @@ function mockKV() { } let seq = 0; -/** A concept node with the given name and sources. */ function node(name: string, sources: string[]): GraphNode { seq += 1; return { @@ -50,7 +43,6 @@ function node(name: string, sources: string[]): GraphNode { }; } -/** A related_to edge between two node ids with the given sources. */ function edge(src: string, tgt: string, sources: string[]): GraphEdge { seq += 1; return { @@ -64,11 +56,9 @@ function edge(src: string, tgt: string, sources: string[]): GraphEdge { }; } -/** `n` distinct observation ids with a prefix. */ const ids = (prefix: string, n: number) => Array.from({ length: n }, (_, i) => `${prefix}_${i}`); -/** The single stored node with this name; fails if there is not exactly one. */ async function onlyNode(kv: ReturnType, name: string) { const all = await kv.list("mem:graph:nodes"); const hits = all.filter((n) => n.name === name); @@ -76,7 +66,6 @@ async function onlyNode(kv: ReturnType, name: string) { return hits[0]!; } -/** The single stored edge; fails if there is not exactly one. */ async function onlyEdge(kv: ReturnType) { const all = await kv.list("mem:graph:edges"); expect(all.length).toBe(1); @@ -152,7 +141,6 @@ describe("graph provenance is bounded and attributed", () => { it("a merged node gains ITS OWN sources, not every id in the batch", async () => { const kv = mockKV(); await persistGraphDelta(kv as never, [node("a.ts", ["obs_1"])], []); - // A batch of obs_2..obs_4 in which only obs_3 mentioned a.ts. await persistGraphDelta( kv as never, [node("a.ts", ["obs_3"]), node("b.ts", ["obs_2", "obs_4"])], @@ -168,7 +156,6 @@ describe("graph provenance is bounded and attributed", () => { const b = node("b", ["obs_0"]); await persistGraphDelta(kv as never, [a, b], [edge(a.id, b.id, ["obs_0"])]); for (let batch = 1; batch <= 10; batch++) { - // A batch of ten observations; only the first one links a and b. const batchIds = ids(`obs_b${batch}`, 10); const a2 = node("a", batchIds); const b2 = node("b", batchIds); @@ -225,7 +212,6 @@ describe("graph provenance is bounded and attributed", () => { }); describe("temporal graph provenance is bounded", () => { - /** SDK stub that records registered functions so they can be triggered directly. */ function mockSdk() { const functions = new Map(); return { From d84e5da80ba55b1ed723df9d36908edf0c9e979a Mon Sep 17 00:00:00 2001 From: Rohit Ghumare Date: Thu, 1 Oct 2026 13:26:37 +0530 Subject: [PATCH 08/11] fix(graph): walk compaction slices in a stable order and audit the run Sliced compaction paged over kv.list(index) by offset across calls. The Redis backend returns list entries in a different order on each call, so a full walk could end with nextOffset null while records were never visited. Sort the ids before slicing; ids carry a base36 timestamp, so records inserted during a walk sort to the end. mem::graph-compact now records a graph_compact audit entry with the counts whenever it removes ids, and its registration sits above the graph-reset notes it was splitting. --- src/functions/graph.ts | 47 ++++++++++------- src/types.ts | 1 + src/viewer/index.html | 3 +- test/graph-compact.test.ts | 101 +++++++++++++++++++++++++++++++++++++ 4 files changed, 133 insertions(+), 19 deletions(-) diff --git a/src/functions/graph.ts b/src/functions/graph.ts index 16559500af..5caafdcc1c 100644 --- a/src/functions/graph.ts +++ b/src/functions/graph.ts @@ -15,7 +15,7 @@ import { buildGraphExtractionPrompt, } from "../prompts/graph-extraction.js"; import { isGraphExtractionEnabled } from "../config.js"; -import { recordAudit } from "./audit.js"; +import { recordAudit, safeAudit } from "./audit.js"; import { logger } from "../logger.js"; // #753: keep the response payload below the iii state channel ceiling. @@ -767,9 +767,9 @@ export async function compactGraphProvenance( indexScope: string, recordScope: string, ): Promise<{ scanned: number; trimmed: number }> => { - const allIds = [...new Set(await kv.list(indexScope))].filter( - (id): id is string => typeof id === "string", - ); + const allIds = [...new Set(await kv.list(indexScope))] + .filter((id): id is string => typeof id === "string") + .sort(); const end = Math.min(allIds.length, offset + limit); result.total = allIds.length; result.nextOffset = end < allIds.length ? end : null; @@ -1254,6 +1254,31 @@ export function registerGraphFunction( } }); + sdk.registerFunction("mem::graph-compact", async (data?: GraphCompactOptions) => { + const started = Date.now(); + try { + const result = await compactGraphProvenance(kv, data ?? {}); + const tookMs = Date.now() - started; + logger.info("Graph provenance compacted", { ...result, tookMs }); + if (result.idsRemoved > 0) { + await safeAudit(kv, "graph_compact", "mem::graph-compact", [], { + scope: data?.scope ?? "all", + offset: data?.offset, + limit: data?.limit, + nodesTrimmed: result.nodesTrimmed, + edgesTrimmed: result.edgesTrimmed, + idsRemoved: result.idsRemoved, + snapshotTrimmed: result.snapshotTrimmed, + }); + } + return { success: true, ...result, tookMs }; + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + logger.error("Graph provenance compaction failed", { error: msg }); + return { success: false, error: msg }; + } + }); + // #814 v2 + #825: clean-restart escape hatch for corpora of any // size, including the legacy 75K+ case that crashes kv.list. // @@ -1275,20 +1300,6 @@ export function registerGraphFunction( // read by any post-#816 code path. Cleanup is deferred to a future // chunked-vacuum job; #816's broken vacuum-via-list strategy is // what we are leaving behind here. - sdk.registerFunction("mem::graph-compact", async (data?: GraphCompactOptions) => { - const started = Date.now(); - try { - const result = await compactGraphProvenance(kv, data ?? {}); - const tookMs = Date.now() - started; - logger.info("Graph provenance compacted", { ...result, tookMs }); - return { success: true, ...result, tookMs }; - } catch (err) { - const msg = err instanceof Error ? err.message : String(err); - logger.error("Graph provenance compaction failed", { error: msg }); - return { success: false, error: msg }; - } - }); - sdk.registerFunction("mem::graph-reset", async () => { const started = Date.now(); // Stamp resetAt=now on the empty snapshot. Future diff --git a/src/types.ts b/src/types.ts index a8674e2f36..d2f5c3ba03 100644 --- a/src/types.ts +++ b/src/types.ts @@ -655,6 +655,7 @@ export interface AuditEntry { | "slot_create" | "slot_delete" | "slot_reflect" + | "graph_compact" | "audit_migrate"; userId?: string; functionId: string; diff --git a/src/viewer/index.html b/src/viewer/index.html index 7a2e071932..0484361059 100644 --- a/src/viewer/index.html +++ b/src/viewer/index.html @@ -6714,7 +6714,8 @@

agentmemory

skill_extract: 'Extracted a skill', core_add: 'Added core memory', core_remove: 'Removed core memory', auto_page: 'Paged memory out', vision_embed: 'Embedded an image', slot_append: 'Appended to a slot', slot_replace: 'Replaced a slot', slot_create: 'Created a slot', - slot_delete: 'Deleted a slot', slot_reflect: 'Reflected into a slot' + slot_delete: 'Deleted a slot', slot_reflect: 'Reflected into a slot', + graph_compact: 'Compacted graph provenance' }; function auditVerb(op) { diff --git a/test/graph-compact.test.ts b/test/graph-compact.test.ts index 4cd175276a..b330376af4 100644 --- a/test/graph-compact.test.ts +++ b/test/graph-compact.test.ts @@ -460,6 +460,107 @@ describe("chunked compaction", () => { }); }); +function shuffledListKV() { + const kv = mockKV(); + let seed = 7; + const rand = () => { + seed = (seed * 1103515245 + 12345) % 2147483648; + return seed / 2147483648; + }; + const list = kv.list; + return { + ...kv, + list: async (scope: string): Promise => { + const items = await list(scope); + for (let i = items.length - 1; i > 0; i--) { + const j = Math.floor(rand() * (i + 1)); + [items[i], items[j]] = [items[j], items[i]]; + } + return items; + }, + }; +} + +function registerCompact(kv: KV) { + const functions = new Map(); + const sdk = { + registerFunction: (idOrOpts: string | { id: string }, handler: Function) => + functions.set(typeof idOrOpts === "string" ? idOrOpts : idOrOpts.id, handler), + registerTrigger: () => {}, + }; + registerGraphFunction( + sdk as never, + kv as never, + { name: "noop", compress: vi.fn(), summarize: vi.fn() } as never, + ); + return functions.get("mem::graph-compact")!; +} + +describe("compaction with an unstable list order", () => { + it("a full sliced walk leaves no node or edge over the cap", async () => { + const kv = shuffledListKV(); + const nid = (i: number) => `gn_${String(i).padStart(2, "0")}`; + for (let i = 0; i < 40; i++) await seedNode(kv, nid(i), `n${i}`, ids(`obs${i}`, 50)); + for (let i = 0; i < 39; i++) + await seedEdge(kv, `ge_${String(i).padStart(2, "0")}`, nid(i), nid(i + 1), ids(`eobs${i}`, 50)); + for (const scope of ["nodes", "edges"] as const) { + let offset: number | null = 0; + let calls = 0; + while (offset !== null && calls < 100) { + const r = await compactGraphProvenance(kv as never, { scope, offset, limit: 3 }); + offset = r.nextOffset; + calls += 1; + } + expect(offset).toBeNull(); + } + const nodes = await kv.list("mem:graph:nodes"); + const edges = await kv.list("mem:graph:edges"); + expect(nodes).toHaveLength(40); + expect(edges).toHaveLength(39); + const over = (r: { sourceObservationIds: string[] }) => + r.sourceObservationIds.length > MAX_GRAPH_SOURCE_OBSERVATIONS; + expect(nodes.filter(over)).toEqual([]); + expect(edges.filter(over)).toEqual([]); + }); +}); + +describe("mem::graph-compact audit", () => { + const auditEntries = async (kv: KV) => { + const scopes = [...new Set(kv.calls.set.map((k) => k.slice(0, k.lastIndexOf("/"))))].filter( + (s) => /^mem:audit:\d{4}-\d{2}$/.test(s), + ); + const rows = await Promise.all( + scopes.map((s) => + kv.list<{ operation: string; functionId: string; details: Record }>(s), + ), + ); + return rows.flat(); + }; + + it("records an audit entry with the counts when ids are removed", async () => { + const kv = mockKV(); + await seedNode(kv, "gn_1", "hub", ids("obs", 50)); + await seedEdge(kv, "ge_1", "gn_1", "gn_2", ids("eobs", 40)); + const r = await registerCompact(kv)({}); + expect(r).toMatchObject({ success: true, idsRemoved: 26 }); + const entries = await auditEntries(kv); + expect(entries).toHaveLength(1); + expect(entries[0]).toMatchObject({ + operation: "graph_compact", + functionId: "mem::graph-compact", + details: { scope: "all", nodesTrimmed: 1, edgesTrimmed: 1, idsRemoved: 26 }, + }); + }); + + it("records nothing when there is nothing to trim", async () => { + const kv = mockKV(); + await seedNode(kv, "gn_1", "hub", ids("obs", 5)); + const r = await registerCompact(kv)({}); + expect(r).toMatchObject({ success: true, idsRemoved: 0 }); + expect(await auditEntries(kv)).toEqual([]); + }); +}); + describe("api::graph-compact responses", () => { async function route( trigger: (req: { From fdfbf5d79d78271f4066a9400fac9db7d80dfa6b Mon Sep 17 00:00:00 2001 From: Rohit Ghumare Date: Thu, 1 Oct 2026 13:28:01 +0530 Subject: [PATCH 09/11] fix(graph): bound provenance on import, mesh sync and snapshot restore mem::import, mem::mesh-receive / mesh pull and mem::snapshot-restore wrote graph nodes and edges as received. They do not write the name or edge-key index, so compaction cannot reach those rows, and a pre-fix export or peer brought the bloat straight back. Each path now keeps the newest 32 source observation ids, the same bound extraction uses. --- src/functions/export-import.ts | 5 +-- src/functions/graph.ts | 6 ++++ src/functions/mesh.ts | 9 +++--- src/functions/snapshot.ts | 3 +- test/export-import.test.ts | 59 ++++++++++++++++++++++++++++++++++ test/mesh.test.ts | 42 ++++++++++++++++++++++++ test/snapshot.test.ts | 35 ++++++++++++++++++++ 7 files changed, 152 insertions(+), 7 deletions(-) diff --git a/src/functions/export-import.ts b/src/functions/export-import.ts index 1758a94c58..2a14ea2737 100644 --- a/src/functions/export-import.ts +++ b/src/functions/export-import.ts @@ -40,6 +40,7 @@ import { indexRecords } from "./search.js"; import { resetLessonIndex } from "./lessons.js"; import { logger } from "../logger.js"; import { budgetImportedObservationSources } from "./observation-source-budget.js"; +import { boundRecordSources } from "./graph.js"; // Bounded-concurrency chunk size for the import delete/write loops. A // "replace" or "merge" of a large export (up to MAX_TOTAL_OBSERVATIONS, @@ -516,7 +517,7 @@ export function registerExportImportFunction(sdk: IIIClient, kv: StateKV): void const existing = await kv.get(KV.graphNodes, node.id).catch(() => null); if (existing) { stats.skipped++; return; } } - await kv.set(KV.graphNodes, node.id, node); + await kv.set(KV.graphNodes, node.id, boundRecordSources(node)); }), ); } @@ -527,7 +528,7 @@ export function registerExportImportFunction(sdk: IIIClient, kv: StateKV): void const existing = await kv.get(KV.graphEdges, edge.id).catch(() => null); if (existing) { stats.skipped++; return; } } - await kv.set(KV.graphEdges, edge.id, edge); + await kv.set(KV.graphEdges, edge.id, boundRecordSources(edge)); }), ); } diff --git a/src/functions/graph.ts b/src/functions/graph.ts index 5caafdcc1c..0f5b494905 100644 --- a/src/functions/graph.ts +++ b/src/functions/graph.ts @@ -312,6 +312,12 @@ export function boundSources(existing: string[], incoming: string[]): string[] { return [...merged].slice(-MAX_GRAPH_SOURCE_OBSERVATIONS); } +export function boundRecordSources(record: R): R { + const sources = record?.sourceObservationIds; + if (!Array.isArray(sources) || sources.length <= MAX_GRAPH_SOURCE_OBSERVATIONS) return record; + return { ...record, sourceObservationIds: boundSources([], sources as string[]) }; +} + function mergeNode( existing: GraphNode, incoming: GraphNode, diff --git a/src/functions/mesh.ts b/src/functions/mesh.ts index 0dd7e95753..444e5105a3 100644 --- a/src/functions/mesh.ts +++ b/src/functions/mesh.ts @@ -3,6 +3,7 @@ import type { StateKV } from "../state/kv.js"; import { KV, generateId } from "../state/schema.js"; import { withKeyedLock } from "../state/keyed-mutex.js"; import { recordAudit } from "./audit.js"; +import { boundRecordSources } from "./graph.js"; import type { MeshPeer, Memory, @@ -122,11 +123,11 @@ async function lwwMergeGraphNodes( const wrote = await withKeyedLock("graph:persist", async () => { const existing = await kv.get(KV.graphNodes, item.id); if (!existing) { - await kv.set(KV.graphNodes, item.id, item); + await kv.set(KV.graphNodes, item.id, boundRecordSources(item)); return true; } if (new Date(ts) > new Date(graphNodeTs(existing))) { - await kv.set(KV.graphNodes, item.id, item); + await kv.set(KV.graphNodes, item.id, boundRecordSources(item)); return true; } return false; @@ -149,11 +150,11 @@ async function lwwMergeGraphEdges( const wrote = await withKeyedLock("graph:persist", async () => { const existing = await kv.get(KV.graphEdges, item.id); if (!existing) { - await kv.set(KV.graphEdges, item.id, item); + await kv.set(KV.graphEdges, item.id, boundRecordSources(item)); return true; } if (new Date(ts) > new Date(existing.createdAt)) { - await kv.set(KV.graphEdges, item.id, item); + await kv.set(KV.graphEdges, item.id, boundRecordSources(item)); return true; } return false; diff --git a/src/functions/snapshot.ts b/src/functions/snapshot.ts index 532c448a0c..d43b60f429 100644 --- a/src/functions/snapshot.ts +++ b/src/functions/snapshot.ts @@ -15,6 +15,7 @@ import type { StateKV } from "../state/kv.js"; import { addSessionToProjectIndex } from "../state/session-index.js"; import { withKeyedLock } from "../state/keyed-mutex.js"; import { recordAudit } from "./audit.js"; +import { boundRecordSources } from "./graph.js"; import { VERSION } from "../version.js"; import { logger } from "../logger.js"; @@ -220,7 +221,7 @@ export function registerSnapshotFunction( if (state.graphNodes) { for (const node of state.graphNodes) { await withKeyedLock("graph:persist", () => - kv.set(KV.graphNodes, node.id, node), + kv.set(KV.graphNodes, node.id, boundRecordSources(node)), ); } } diff --git a/test/export-import.test.ts b/test/export-import.test.ts index d33aacb1e6..8dfe5fbb40 100644 --- a/test/export-import.test.ts +++ b/test/export-import.test.ts @@ -309,3 +309,62 @@ describe("Export/Import Functions", () => { expect(result.error).toContain("Unsupported export version"); }); }); + +const bloatedIds = (prefix: string, n: number) => + Array.from({ length: n }, (_, i) => `${prefix}_${String(i).padStart(3, "0")}`); + +describe("import bounds graph provenance", () => { + it("caps sourceObservationIds on imported graph nodes and edges", async () => { + const sdk = mockSdk(); + const kv = mockKV(); + registerExportImportFunction(sdk as never, kv as never); + const exportData = { + version: "0.9.28", + exportedAt: new Date().toISOString(), + sessions: [], + observations: {}, + memories: [], + summaries: [], + graphNodes: [ + { + id: "gn_bloat", + type: "file", + name: "src/hot.ts", + properties: {}, + sourceObservationIds: bloatedIds("obs", 400), + createdAt: "2026-03-01T00:00:00Z", + }, + { + id: "gn_small", + type: "file", + name: "src/cold.ts", + properties: {}, + sourceObservationIds: ["obs_x"], + createdAt: "2026-03-01T00:00:00Z", + }, + ], + graphEdges: [ + { + id: "ge_bloat", + type: "related_to", + sourceNodeId: "gn_bloat", + targetNodeId: "gn_small", + weight: 0.5, + sourceObservationIds: bloatedIds("eobs", 100), + createdAt: "2026-03-01T00:00:00Z", + }, + ], + } as unknown as ExportData; + const result = (await sdk.trigger("mem::import", { + exportData, + strategy: "merge", + })) as { success: boolean }; + expect(result.success).toBe(true); + const n = await kv.get<{ sourceObservationIds: string[] }>("mem:graph:nodes", "gn_bloat"); + const s = await kv.get<{ sourceObservationIds: string[] }>("mem:graph:nodes", "gn_small"); + const e = await kv.get<{ sourceObservationIds: string[] }>("mem:graph:edges", "ge_bloat"); + expect(n!.sourceObservationIds).toEqual(bloatedIds("obs", 400).slice(-32)); + expect(s!.sourceObservationIds).toEqual(["obs_x"]); + expect(e!.sourceObservationIds).toEqual(bloatedIds("eobs", 100).slice(-32)); + }); +}); diff --git a/test/mesh.test.ts b/test/mesh.test.ts index e08db07840..5c5ecfd629 100644 --- a/test/mesh.test.ts +++ b/test/mesh.test.ts @@ -753,3 +753,45 @@ describe("Mesh Functions", () => { }); }); }); + +const bloatedIds = (prefix: string, n: number) => + Array.from({ length: n }, (_, i) => `${prefix}_${String(i).padStart(3, "0")}`); + +describe("mesh-receive bounds graph provenance", () => { + it("caps sourceObservationIds on received nodes and edges", async () => { + const sdk = mockSdk(); + const kv = mockKV(); + registerMeshFunction(sdk as never, kv as never); + const node: GraphNode = { + id: "gn_peer", + type: "file", + name: "src/hot.ts", + properties: {}, + sourceObservationIds: bloatedIds("obs", 500), + createdAt: "2026-03-01T00:00:00Z", + }; + const small: GraphNode = { ...node, id: "gn_small", sourceObservationIds: ["obs_a", "obs_b"] }; + const edge: GraphEdge = { + id: "ge_peer", + type: "related_to", + sourceNodeId: "gn_peer", + targetNodeId: "gn_small", + weight: 0.5, + sourceObservationIds: bloatedIds("eobs", 300), + createdAt: "2026-03-01T00:00:00Z", + }; + const result = (await sdk.trigger("mem::mesh-receive", { + graphNodes: [node, small], + graphEdges: [edge], + })) as { success: boolean; accepted: number }; + expect(result.success).toBe(true); + expect(result.accepted).toBe(3); + const storedNode = await kv.get("mem:graph:nodes", "gn_peer"); + const storedSmall = await kv.get("mem:graph:nodes", "gn_small"); + const storedEdge = await kv.get("mem:graph:edges", "ge_peer"); + expect(storedNode!.sourceObservationIds).toEqual(bloatedIds("obs", 500).slice(-32)); + expect(storedNode!.name).toBe("src/hot.ts"); + expect(storedSmall!.sourceObservationIds).toEqual(["obs_a", "obs_b"]); + expect(storedEdge!.sourceObservationIds).toEqual(bloatedIds("eobs", 300).slice(-32)); + }); +}); diff --git a/test/snapshot.test.ts b/test/snapshot.test.ts index 5fc51feab3..70969cfa62 100644 --- a/test/snapshot.test.ts +++ b/test/snapshot.test.ts @@ -258,3 +258,38 @@ describe("snapshot-create reentrancy guard", () => { expect(r3.snapshot).toBeDefined(); }); }); + +const bloatedIds = (prefix: string, n: number) => + Array.from({ length: n }, (_, i) => `${prefix}_${String(i).padStart(3, "0")}`); + +describe("snapshot-restore bounds graph provenance", () => { + it("caps sourceObservationIds on restored graph nodes", async () => { + const sdk = mockSdk(); + const kv = mockKV(); + registerSnapshotFunction(sdk as never, kv as never, "/tmp/agentmemory-snapshots"); + vi.mocked(readFileSync).mockReturnValueOnce( + JSON.stringify({ + version: "0.4.0", + sessions: [], + memories: [], + graphNodes: [ + { + id: "gn_bloat", + type: "file", + name: "src/hot.ts", + properties: {}, + sourceObservationIds: bloatedIds("obs", 250), + createdAt: "2026-03-01T00:00:00Z", + }, + ], + }), + ); + const result = (await sdk.trigger("mem::snapshot-restore", { + commitHash: "abc1234", + })) as { success: boolean }; + expect(result.success).toBe(true); + const n = await kv.get<{ name: string; sourceObservationIds: string[] }>("mem:graph:nodes", "gn_bloat"); + expect(n!.name).toBe("src/hot.ts"); + expect(n!.sourceObservationIds).toEqual(bloatedIds("obs", 250).slice(-32)); + }); +}); From 683ce83da484b1acc05f65091ee0b382f6ba0419 Mon Sep 17 00:00:00 2001 From: Rohit Ghumare Date: Thu, 1 Oct 2026 13:29:38 +0530 Subject: [PATCH 10/11] fix(graph): accept loosely typed records in boundRecordSources snapshot-restore passes parsed JSON records typed as { id } & Record, which tsc rejected against the narrower generic bound. --- src/functions/graph.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/functions/graph.ts b/src/functions/graph.ts index 0f5b494905..cf551727c7 100644 --- a/src/functions/graph.ts +++ b/src/functions/graph.ts @@ -312,8 +312,8 @@ export function boundSources(existing: string[], incoming: string[]): string[] { return [...merged].slice(-MAX_GRAPH_SOURCE_OBSERVATIONS); } -export function boundRecordSources(record: R): R { - const sources = record?.sourceObservationIds; +export function boundRecordSources(record: R): R { + const sources = (record as { sourceObservationIds?: unknown } | null)?.sourceObservationIds; if (!Array.isArray(sources) || sources.length <= MAX_GRAPH_SOURCE_OBSERVATIONS) return record; return { ...record, sourceObservationIds: boundSources([], sources as string[]) }; } From ebe12f9acea27516c7610fe289351ca3b1f80f14 Mon Sep 17 00:00:00 2001 From: Rohit Ghumare Date: Thu, 1 Oct 2026 13:29:38 +0530 Subject: [PATCH 11/11] docs: explain POST /agentmemory/graph/compact Add the endpoint to the key endpoints table and describe when to run it, the scope/offset/limit slicing loop, and the final scope "snapshot" call a sliced run needs. --- README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/README.md b/README.md index 60f98155d7..41cbda22c5 100644 --- a/README.md +++ b/README.md @@ -1691,6 +1691,7 @@ Create `~/.agentmemory/.env`: | `GET` | `/agentmemory/export` | Export all data | | `POST` | `/agentmemory/import` | Import from JSON | | `POST` | `/agentmemory/graph/query` | Knowledge graph query | +| `POST` | `/agentmemory/graph/compact` | Trim oversized graph provenance | | `POST` | `/agentmemory/team/share` | Share with team | | `GET` | `/agentmemory/audit` | Audit trail | @@ -1698,6 +1699,19 @@ Full endpoint list: [`src/triggers/api.ts`](src/triggers/api.ts)
+**Compacting graph provenance.** Each knowledge graph node and edge keeps the ids of the newest 32 observations it came from. Stores written before that cap can hold thousands of ids per hot node, which makes graph search and the viewer slow or drops the worker. Run `POST /agentmemory/graph/compact` once after upgrading if graph queries are slow or time out. It trims every node, edge and the cached snapshot to the cap, never lists the full graph, and is safe to re-run. When it trims ids it writes a `graph_compact` audit entry. + +```bash +curl -X POST http://localhost:3111/agentmemory/graph/compact -H "Content-Type: application/json" -d '{}' +``` + +On a large store, or when the call returns 504, run it in slices. Send `scope` (`nodes` or `edges`), `offset` and `limit`, then call again with the returned `nextOffset` until it is `null`. Do this for `nodes` and for `edges`, and finish with one `{"scope":"snapshot"}` call, because a sliced run does not touch the cached snapshot. + +```bash +curl -X POST http://localhost:3111/agentmemory/graph/compact -H "Content-Type: application/json" -d '{"scope":"nodes","offset":0,"limit":200}' +curl -X POST http://localhost:3111/agentmemory/graph/compact -H "Content-Type: application/json" -d '{"scope":"snapshot"}' +``` + ---

Development