From d5611a67246347b6a9d5f19fa5d8ebe198a2db26 Mon Sep 17 00:00:00 2001 From: Jenq Shiin Haw <1343528669@qq.com> Date: Fri, 10 Jul 2026 13:58:23 +0800 Subject: [PATCH 1/6] chore(runtime): snapshot p0 freeze baseline Signed-off-by: Jenq Shiin Haw <1343528669@qq.com> --- src/config.ts | 27 +++++++++++++++------------ src/functions/image-quota-cleanup.ts | 2 +- src/functions/observe.ts | 24 +++++++++--------------- src/index.ts | 10 +++++----- src/mcp/tools-registry.ts | 8 ++++---- src/triggers/api.ts | 12 ++---------- 6 files changed, 36 insertions(+), 47 deletions(-) diff --git a/src/config.ts b/src/config.ts index 3348ac2267..f68da2e31d 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,7 +1,6 @@ import { existsSync, readFileSync } from "node:fs"; import { join } from "node:path"; import { homedir } from "node:os"; -import pc from "picocolors"; import type { AgentMemoryConfig, ProviderConfig, @@ -127,11 +126,15 @@ function detectProvider(env: Record): ProviderConfig { const allowAgentSdk = env["AGENTMEMORY_ALLOW_AGENT_SDK"] === "true"; if (!allowAgentSdk) { process.stderr.write( - pc.dim( - "[agentmemory] No LLM provider key set — running zero-LLM (BM25 + on-device embeddings). " + - "Set ANTHROPIC_API_KEY (or GEMINI/OPENAI/OPENROUTER/MINIMAX) in ~/.agentmemory/.env for LLM compression and summaries. " + - "Agent-SDK fallback stays off by default to avoid a Stop-hook recursion loop; opt in with AGENTMEMORY_AUTO_COMPRESS=true + AGENTMEMORY_ALLOW_AGENT_SDK=true.\n", - ), + "[agentmemory] No LLM provider key found " + + "(ANTHROPIC_API_KEY, GEMINI_API_KEY, OPENROUTER_API_KEY, MINIMAX_API_KEY, OPENAI_API_KEY). " + + "LLM-backed compression and summarization are DISABLED — using no-op provider. " + + "This is the safe default: the agent-sdk fallback used to spawn Claude Agent SDK " + + "child sessions which inherit Claude Code's plugin hooks and cause infinite Stop-hook " + + "recursion (#149 follow-up). To opt in to the agent-sdk fallback anyway, set both " + + "AGENTMEMORY_AUTO_COMPRESS=true AND AGENTMEMORY_ALLOW_AGENT_SDK=true — but be aware " + + "it will burn your Claude Pro allocation and may still recurse if you use it from " + + "inside Claude Code itself.\n", ); return { provider: "noop", @@ -143,7 +146,7 @@ function detectProvider(env: Record): ProviderConfig { process.stderr.write( "[agentmemory] WARNING: agent-sdk fallback enabled via AGENTMEMORY_ALLOW_AGENT_SDK=true. " + "This spawns @anthropic-ai/claude-agent-sdk child sessions that can trigger the Stop-hook " + - "recursion loop. A SDK-child env marker is set to block re-entry, " + + "recursion loop (#149 follow-up). A SDK-child env marker is set to block re-entry, " + "but prefer setting a real API key in ~/.agentmemory/.env instead.\n", ); return { @@ -161,7 +164,7 @@ export function loadConfig(): AgentMemoryConfig { // Port quartet: REST is the anchor; streams/engine derive from it // unless individually overridden. Default anchor 3111 yields the // canonical 3112 streams / 49134 engine pair, but `III_REST_PORT=3211` - // auto-picks 3212 + 49234 so a second instance doesn't collide. + // auto-picks 3212 + 49234 so a second instance doesn't collide (#750). const restPort = parseInt(env["III_REST_PORT"] || "3111", 10) || 3111; const streamsPort = parseInt(env["III_STREAM_PORT"] || env["III_STREAMS_PORT"] || "", 10) || @@ -333,7 +336,7 @@ export function getGraphBatchSize(): number { return safeParseInt(getMergedEnv()["GRAPH_EXTRACTION_BATCH_SIZE"], 10); } -// window for the smart-search followup-rate diagnostic. A second +// #771: window for the smart-search followup-rate diagnostic. A second // search arriving within this many seconds (with disjoint results) // counts as a "follow-up" — a directional signal that the first result // set didn't satisfy. Long values overcount (legitimate refinement @@ -373,7 +376,7 @@ function hasLLMProviderConfigured(env: Record): bool ); } -// Per-observation LLM compression is OFF by default as of 0.8.8. +// Per-observation LLM compression is OFF by default as of 0.8.8 (see #138). // When disabled, observations are captured and indexed via a synthetic // (zero-LLM) compression path so recall/search still works. Users who want // richer LLM-generated summaries can set AGENTMEMORY_AUTO_COMPRESS=true in @@ -384,7 +387,7 @@ export function isAutoCompressEnabled(): boolean { } // Hook-level context injection into Claude Code's conversation is OFF by -// default as of 0.8.10. When disabled, pre-tool-use and +// default as of 0.8.10 (see #143). When disabled, pre-tool-use and // session-start hooks still POST observations for background capture, but // never write context to stdout — so Claude Code doesn't inject an extra // ~4000-char blob into every tool turn. 0.8.8 stopped the agentmemory-side @@ -443,7 +446,7 @@ export function loadFallbackConfig(): FallbackConfig { "[agentmemory] Ignoring FALLBACK_PROVIDERS entry 'agent-sdk' " + "(AGENTMEMORY_ALLOW_AGENT_SDK is not 'true'). The agent-sdk " + "fallback can spawn Claude Agent SDK child sessions that trigger " + - "the Stop-hook recursion loop. Opt in explicitly " + + "the Stop-hook recursion loop (#149 follow-up). Opt in explicitly " + "with AGENTMEMORY_ALLOW_AGENT_SDK=true if this is intentional.\n", ); return false; diff --git a/src/functions/image-quota-cleanup.ts b/src/functions/image-quota-cleanup.ts index d07dfa9698..b32bf1f1df 100644 --- a/src/functions/image-quota-cleanup.ts +++ b/src/functions/image-quota-cleanup.ts @@ -4,6 +4,7 @@ import { StateKV } from "../state/kv.js"; import { readdir, stat } from "node:fs/promises"; import { join } from "node:path"; import { IMAGES_DIR, getMaxBytes, deleteImage } from "../utils/image-store.js"; +import { getImageRefCount } from "./image-refs.js"; import { withKeyedLock } from "../state/keyed-mutex.js"; import { logger } from "../logger.js"; @@ -55,7 +56,6 @@ export function registerImageQuotaCleanup(sdk: ISdk, kv: StateKV): void { await withKeyedLock(`imgRef:${f.filePath}`, async () => { let refCount: number; try { - const { getImageRefCount } = await import("./image-refs.js"); refCount = await getImageRefCount(kv, f.filePath); } catch (err) { // Fail-closed: if we cannot determine refCount we must NOT diff --git a/src/functions/observe.ts b/src/functions/observe.ts index 8ad4ba0ff5..2bf13d5476 100644 --- a/src/functions/observe.ts +++ b/src/functions/observe.ts @@ -10,7 +10,6 @@ import { buildSyntheticCompression } from "./compress-synthetic.js"; import { getSearchIndex, vectorIndexAddGuarded } from "./search.js"; import { getAgentId } from "../config.js"; import { logger } from "../logger.js"; -import { saveImageToDisk } from "../utils/image-store.js"; export function extractImage(d: unknown): string | undefined { if (!d) return undefined; @@ -152,6 +151,7 @@ export function registerObserveFunction( } if (pendingImageData && (pendingImageData.startsWith("data:image/") || pendingImageData.startsWith("iVBORw0KGgo") || pendingImageData.startsWith("/9j/"))) { + const { saveImageToDisk } = await import("../utils/image-store.js"); const { filePath, bytesWritten } = await saveImageToDisk(pendingImageData); raw.imageData = filePath; const { incrementImageRef } = await import("./image-refs.js"); @@ -180,19 +180,13 @@ export function registerObserveFunction( } catch (error) { if (raw.imageData) { - // Roll back the ref taken above. decrementImageRef deletes the file - // only when no other observation still references it (deduped images - // survive) and emits the disk-size delta itself — deleting the file - // directly here would orphan shared images and leave a stale ref. - // If the rollback itself fails, log it but still surface the - // original write error (the more useful failure to diagnose). - try { - const { decrementImageRef } = await import("./image-refs.js"); - await decrementImageRef(kv, sdk, raw.imageData); - } catch (rollbackError) { - logger.error("Failed to roll back image ref after observation write failure", { - imageRef: raw.imageData, - error: rollbackError instanceof Error ? rollbackError.message : String(rollbackError), + const { deleteImage } = await import("../utils/image-store.js"); + const { deletedBytes } = await deleteImage(raw.imageData); + if (deletedBytes > 0) { + sdk.trigger({ + function_id: "mem::disk-size-delta", + payload: { deltaBytes: -deletedBytes }, + action: TriggerAction.Void(), }); } } @@ -280,7 +274,7 @@ export function registerObserveFunction( }); } - // Per-observation LLM compression is opt-in as of 0.8.8. + // Per-observation LLM compression is opt-in as of 0.8.8 (#138). // Default path: build a zero-LLM synthetic compression so recall // and BM25 search still work without burning the user's Claude // token allocation on every tool invocation. diff --git a/src/index.ts b/src/index.ts index 1e623eae81..4233e8a67e 100644 --- a/src/index.ts +++ b/src/index.ts @@ -276,21 +276,21 @@ async function main() { if (isAutoCompressEnabled()) { bootLog( - `WARNING: AGENTMEMORY_AUTO_COMPRESS=true — every PostToolUse observation will be sent to your LLM provider for compression. This spends API tokens proportional to your session tool-use frequency. Set AGENTMEMORY_AUTO_COMPRESS=false to disable.`, + `WARNING: AGENTMEMORY_AUTO_COMPRESS=true — every PostToolUse observation will be sent to your LLM provider for compression. This spends API tokens proportional to your session tool-use frequency (see #138). Set AGENTMEMORY_AUTO_COMPRESS=false to disable.`, ); } else { bootLog( - `Auto-compress: OFF (default) — observations indexed via zero-LLM synthetic compression. Set AGENTMEMORY_AUTO_COMPRESS=true to opt-in to LLM-powered summaries (uses your API key).`, + `Auto-compress: OFF (default, #138) — observations indexed via zero-LLM synthetic compression. Set AGENTMEMORY_AUTO_COMPRESS=true to opt-in to LLM-powered summaries (uses your API key).`, ); } if (isContextInjectionEnabled()) { bootLog( - `WARNING: AGENTMEMORY_INJECT_CONTEXT=true — the PreToolUse and SessionStart hooks will inject up to ~4000 chars of memory context into every tool turn. On Claude Pro this burns session tokens proportional to your tool-call frequency. Set AGENTMEMORY_INJECT_CONTEXT=false to disable.`, + `WARNING: AGENTMEMORY_INJECT_CONTEXT=true — the PreToolUse and SessionStart hooks will inject up to ~4000 chars of memory context into every tool turn. On Claude Pro this burns session tokens proportional to your tool-call frequency (see #143). Set AGENTMEMORY_INJECT_CONTEXT=false to disable.`, ); } else { bootLog( - `Context injection: OFF (default) — hooks capture observations but do not inject context into Claude Code's conversation. Set AGENTMEMORY_INJECT_CONTEXT=true to opt-in (warning: expect your Claude Pro allocation to drain faster).`, + `Context injection: OFF (default, #143) — hooks capture observations but do not inject context into Claude Code's conversation. Set AGENTMEMORY_INJECT_CONTEXT=true to opt-in (warning: expect your Claude Pro allocation to drain faster).`, ); } @@ -498,7 +498,7 @@ async function main() { } if (backfilled > 0) { bootLog( - `Backfilled ${backfilled} memories into BM25 (legacy index gap)`, + `Backfilled ${backfilled} memories into BM25 (legacy gap before #257)`, ); indexPersistence.scheduleSave(); } diff --git a/src/mcp/tools-registry.ts b/src/mcp/tools-registry.ts index c4df3499c7..c86d899bbc 100644 --- a/src/mcp/tools-registry.ts +++ b/src/mcp/tools-registry.ts @@ -925,7 +925,7 @@ export const V010_SLOTS_TOOLS: McpToolDef[] = [ }, ]; -export const ESSENTIAL_TOOLS = new Set([ +const ESSENTIAL_TOOLS = new Set([ "memory_save", "memory_recall", "memory_consolidate", @@ -950,9 +950,9 @@ export function getAllTools(): McpToolDef[] { } // default switched from "core" (8 essential tools) to "all" -// (full 53-tool surface). README and plugin manifests have always -// advertised 53 tools "in proxy mode"; the old default left OpenCode / -// Claude Code users seeing 8 with no indication the other tools existed. +// (full 51-tool surface). README and plugin manifests have always +// advertised 51 tools "in proxy mode"; the old default left OpenCode / +// Claude Code users seeing 8 with no indication the other 43 existed. // Users who want the lean essentials can still set AGENTMEMORY_TOOLS=core. export function getVisibleTools(): McpToolDef[] { const mode = process.env["AGENTMEMORY_TOOLS"] || "all"; diff --git a/src/triggers/api.ts b/src/triggers/api.ts index 7b6c2bf2b8..4a214a3acd 100644 --- a/src/triggers/api.ts +++ b/src/triggers/api.ts @@ -1,5 +1,5 @@ import { TriggerAction, type ISdk, type ApiRequest } from "iii-sdk"; -import type { Session, CompressedObservation, HookPayload, CommitLink, SessionSummary } from "../types.js"; +import type { Session, CompressedObservation, HookPayload, CommitLink } from "../types.js"; import { withKeyedLock } from "../state/keyed-mutex.js"; import { KV } from "../state/schema.js"; import { StateKV } from "../state/kv.js"; @@ -823,15 +823,7 @@ export function registerApiTriggers( const filtered = filterAgentId ? sessions.filter((s) => s.agentId === filterAgentId) : sessions; - const summaries = await Promise.all( - filtered.map((s) => - kv.get(KV.summaries, s.id).catch(() => null), - ), - ); - const withSummary = filtered.map((s, i) => - summaries[i] ? { ...s, summary: summaries[i] } : s, - ); - return { status_code: 200, body: { sessions: withSummary } }; + return { status_code: 200, body: { sessions: filtered } }; }, ); sdk.registerTrigger({ From 7ac195c955dcc96bc979254966a32bdac7d6da28 Mon Sep 17 00:00:00 2001 From: Jenq Shiin Haw <1343528669@qq.com> Date: Fri, 10 Jul 2026 15:00:06 +0800 Subject: [PATCH 2/6] feat(memory): add durable candidates MVP Signed-off-by: Jenq Shiin Haw <1343528669@qq.com> --- AGENTS.md | 2 +- README.md | 2 +- docs/continuity-schema-note.md | 43 ++ plugin/scripts/notification.mjs | 6 +- plugin/scripts/post-commit.mjs | 5 +- plugin/scripts/post-tool-failure.mjs | 6 +- plugin/scripts/post-tool-use.mjs | 6 +- plugin/scripts/pre-compact.mjs | 6 +- plugin/scripts/pre-tool-use.mjs | 4 +- plugin/scripts/prompt-submit.mjs | 6 +- plugin/scripts/session-end.mjs | 4 +- plugin/scripts/session-start.mjs | 6 +- plugin/scripts/stop.mjs | 4 +- plugin/scripts/subagent-start.mjs | 6 +- plugin/scripts/subagent-stop.mjs | 6 +- plugin/scripts/task-completed.mjs | 6 +- src/eval/schemas.ts | 38 ++ src/functions/durable-candidate-utils.ts | 270 +++++++++ src/functions/durable-candidates.ts | 673 +++++++++++++++++++++++ src/functions/remember.ts | 38 +- src/functions/replay.ts | 4 +- src/functions/summarize.ts | 344 +++++++----- src/index.ts | 4 +- src/mcp/tools-registry.ts | 2 +- src/prompts/summary.ts | 65 ++- src/state/memory-utils.ts | 1 + src/state/schema.ts | 1 + src/triggers/api.ts | 279 ++++++++++ src/types.ts | 41 +- test/durable-candidate-utils.test.ts | 86 +++ test/durable-candidates.test.ts | 326 +++++++++++ test/summarize.test.ts | 81 +++ 32 files changed, 2162 insertions(+), 209 deletions(-) create mode 100644 docs/continuity-schema-note.md create mode 100644 src/functions/durable-candidate-utils.ts create mode 100644 src/functions/durable-candidates.ts create mode 100644 test/durable-candidate-utils.test.ts create mode 100644 test/durable-candidates.test.ts diff --git a/AGENTS.md b/AGENTS.md index 873ef10fc1..2b3d32f771 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -117,7 +117,7 @@ Hook scripts in `src/hooks/` are standalone Node.js scripts (no iii-sdk import). ## Current Stats (v0.9.16) - 53 MCP tools (8 visible by default, `AGENTMEMORY_TOOLS=all` for all) -- 128 REST endpoints +- 132 REST endpoints - 6 MCP resources, 3 MCP prompts - 12 hooks, 15 skills - 50+ iii functions diff --git a/README.md b/README.md index d04da3712f..9451b082b5 100644 --- a/README.md +++ b/README.md @@ -1501,7 +1501,7 @@ Create `~/.agentmemory/.env`:

API

-128 endpoints on port `3111`. The REST API binds to `127.0.0.1` by default. Protected endpoints require `Authorization: Bearer ` when `AGENTMEMORY_SECRET` is set, and mesh sync endpoints require `AGENTMEMORY_SECRET` on both peers. +132 endpoints on port `3111`. The REST API binds to `127.0.0.1` by default. Protected endpoints require `Authorization: Bearer ` when `AGENTMEMORY_SECRET` is set, and mesh sync endpoints require `AGENTMEMORY_SECRET` on both peers.
Key endpoints diff --git a/docs/continuity-schema-note.md b/docs/continuity-schema-note.md new file mode 100644 index 0000000000..75a32faf3b --- /dev/null +++ b/docs/continuity-schema-note.md @@ -0,0 +1,43 @@ +# Continuity Schema Note + +P1 keeps continuity out of the runtime path on purpose. + +## Scope in P1 + +- archived sessions may produce `durableCandidates[]` +- only explicit promote writes `Memory` +- backfill defaults to `dryRun` +- archive processing does not promote or recall + +## Candidate -> Memory lifecycle + +1. session summary stores `durableCandidates[]` +2. operator or tool lists candidates from `KV.summaries` +3. explicit promote writes `Memory` +4. promoted memory stores: + - `title` + - `content` + - `type` + - `concepts` + - `files` + - `sourceObservationIds` + - `sessionIds` + - `sourceCandidateId` + - `confidence` + - `strength` + +## Continuity follow-up for P2+ + +Potential continuity rows should remain a separate schema from `Memory` so they can model: + +- open loops or pending threads +- expected next actions +- handoff-specific state +- freshness / expiry +- recall priority separate from durable importance + +That design note is intentionally deferred until after P1 proves: + +- candidates can be generated from archived sessions +- promote is idempotent +- backfill can preview and then write candidates without auto-promoting diff --git a/plugin/scripts/notification.mjs b/plugin/scripts/notification.mjs index 3967158c92..95786fb409 100755 --- a/plugin/scripts/notification.mjs +++ b/plugin/scripts/notification.mjs @@ -1,7 +1,6 @@ #!/usr/bin/env node import { execSync } from "node:child_process"; import { basename } from "node:path"; - //#region src/hooks/_project.ts function resolveProject(cwd) { const explicit = process.env["AGENTMEMORY_PROJECT_NAME"]; @@ -21,7 +20,6 @@ function resolveProject(cwd) { } catch {} return basename(dir); } - //#endregion //#region src/hooks/notification.ts function isSdkChildContext(payload) { @@ -70,7 +68,7 @@ async function main() { setTimeout(() => process.exit(0), 500).unref(); } main(); - //#endregion -export { }; +export {}; + //# sourceMappingURL=notification.mjs.map \ No newline at end of file diff --git a/plugin/scripts/post-commit.mjs b/plugin/scripts/post-commit.mjs index 8552cd614e..5318184f85 100755 --- a/plugin/scripts/post-commit.mjs +++ b/plugin/scripts/post-commit.mjs @@ -1,7 +1,6 @@ #!/usr/bin/env node import { execFile } from "node:child_process"; import { promisify } from "node:util"; - //#region src/hooks/post-commit.ts const exec = promisify(execFile); function isSdkChildContext(payload) { @@ -96,7 +95,7 @@ async function main() { } catch {} } main(); - //#endregion -export { }; +export {}; + //# sourceMappingURL=post-commit.mjs.map \ No newline at end of file diff --git a/plugin/scripts/post-tool-failure.mjs b/plugin/scripts/post-tool-failure.mjs index 6fdad8d9d9..4562a7474b 100755 --- a/plugin/scripts/post-tool-failure.mjs +++ b/plugin/scripts/post-tool-failure.mjs @@ -1,7 +1,6 @@ #!/usr/bin/env node import { execSync } from "node:child_process"; import { basename } from "node:path"; - //#region src/hooks/_project.ts function resolveProject(cwd) { const explicit = process.env["AGENTMEMORY_PROJECT_NAME"]; @@ -21,7 +20,6 @@ function resolveProject(cwd) { } catch {} return basename(dir); } - //#endregion //#region src/hooks/post-tool-failure.ts function isSdkChildContext(payload) { @@ -71,7 +69,7 @@ async function main() { setTimeout(() => process.exit(0), 500).unref(); } main(); - //#endregion -export { }; +export {}; + //# sourceMappingURL=post-tool-failure.mjs.map \ No newline at end of file diff --git a/plugin/scripts/post-tool-use.mjs b/plugin/scripts/post-tool-use.mjs index b4aef9c942..1103a54f38 100755 --- a/plugin/scripts/post-tool-use.mjs +++ b/plugin/scripts/post-tool-use.mjs @@ -1,7 +1,6 @@ #!/usr/bin/env node import { execSync } from "node:child_process"; import { basename } from "node:path"; - //#region src/hooks/_project.ts function resolveProject(cwd) { const explicit = process.env["AGENTMEMORY_PROJECT_NAME"]; @@ -21,7 +20,6 @@ function resolveProject(cwd) { } catch {} return basename(dir); } - //#endregion //#region src/hooks/post-tool-use.ts function isSdkChildContext(payload) { @@ -116,7 +114,7 @@ function truncate(value, max) { return value; } main(); - //#endregion -export { }; +export {}; + //# sourceMappingURL=post-tool-use.mjs.map \ No newline at end of file diff --git a/plugin/scripts/pre-compact.mjs b/plugin/scripts/pre-compact.mjs index 0afdcb0b4e..bf7ce2b763 100755 --- a/plugin/scripts/pre-compact.mjs +++ b/plugin/scripts/pre-compact.mjs @@ -1,7 +1,6 @@ #!/usr/bin/env node import { execSync } from "node:child_process"; import { basename } from "node:path"; - //#region src/hooks/_project.ts function resolveProject(cwd) { const explicit = process.env["AGENTMEMORY_PROJECT_NAME"]; @@ -21,7 +20,6 @@ function resolveProject(cwd) { } catch {} return basename(dir); } - //#endregion //#region src/hooks/pre-compact.ts function isSdkChildContext(payload) { @@ -74,7 +72,7 @@ async function main() { } catch {} } main(); - //#endregion -export { }; +export {}; + //# sourceMappingURL=pre-compact.mjs.map \ No newline at end of file diff --git a/plugin/scripts/pre-tool-use.mjs b/plugin/scripts/pre-tool-use.mjs index d70c166ed2..a0d6e95832 100755 --- a/plugin/scripts/pre-tool-use.mjs +++ b/plugin/scripts/pre-tool-use.mjs @@ -78,7 +78,7 @@ async function main() { } catch {} } main(); - //#endregion -export { }; +export {}; + //# sourceMappingURL=pre-tool-use.mjs.map \ No newline at end of file diff --git a/plugin/scripts/prompt-submit.mjs b/plugin/scripts/prompt-submit.mjs index 1a4147e6ab..112bcbcc3a 100755 --- a/plugin/scripts/prompt-submit.mjs +++ b/plugin/scripts/prompt-submit.mjs @@ -1,7 +1,6 @@ #!/usr/bin/env node import { execSync } from "node:child_process"; import { basename } from "node:path"; - //#region src/hooks/_project.ts function resolveProject(cwd) { const explicit = process.env["AGENTMEMORY_PROJECT_NAME"]; @@ -21,7 +20,6 @@ function resolveProject(cwd) { } catch {} return basename(dir); } - //#endregion //#region src/hooks/prompt-submit.ts function isSdkChildContext(payload) { @@ -63,7 +61,7 @@ async function main() { setTimeout(() => process.exit(0), 500).unref(); } main(); - //#endregion -export { }; +export {}; + //# sourceMappingURL=prompt-submit.mjs.map \ No newline at end of file diff --git a/plugin/scripts/session-end.mjs b/plugin/scripts/session-end.mjs index 019149c309..0ba564be58 100755 --- a/plugin/scripts/session-end.mjs +++ b/plugin/scripts/session-end.mjs @@ -54,7 +54,7 @@ async function main() { setTimeout(() => process.exit(0), 1500).unref(); } main(); - //#endregion -export { }; +export {}; + //# sourceMappingURL=session-end.mjs.map \ No newline at end of file diff --git a/plugin/scripts/session-start.mjs b/plugin/scripts/session-start.mjs index 51b70eb4c2..bad7ffc0ba 100755 --- a/plugin/scripts/session-start.mjs +++ b/plugin/scripts/session-start.mjs @@ -1,7 +1,6 @@ #!/usr/bin/env node import { execSync } from "node:child_process"; import { basename } from "node:path"; - //#region src/hooks/_project.ts function resolveProject(cwd) { const explicit = process.env["AGENTMEMORY_PROJECT_NAME"]; @@ -21,7 +20,6 @@ function resolveProject(cwd) { } catch {} return basename(dir); } - //#endregion //#region src/hooks/session-start.ts function isSdkChildContext(payload) { @@ -81,7 +79,7 @@ async function main() { } catch {} } main(); - //#endregion -export { }; +export {}; + //# sourceMappingURL=session-start.mjs.map \ No newline at end of file diff --git a/plugin/scripts/stop.mjs b/plugin/scripts/stop.mjs index 03d30c64f3..63051c8309 100755 --- a/plugin/scripts/stop.mjs +++ b/plugin/scripts/stop.mjs @@ -38,7 +38,7 @@ async function main() { setTimeout(() => process.exit(0), 1500).unref(); } main(); - //#endregion -export { }; +export {}; + //# sourceMappingURL=stop.mjs.map \ No newline at end of file diff --git a/plugin/scripts/subagent-start.mjs b/plugin/scripts/subagent-start.mjs index 2359a1c62f..685b783493 100755 --- a/plugin/scripts/subagent-start.mjs +++ b/plugin/scripts/subagent-start.mjs @@ -1,7 +1,6 @@ #!/usr/bin/env node import { execSync } from "node:child_process"; import { basename } from "node:path"; - //#region src/hooks/_project.ts function resolveProject(cwd) { const explicit = process.env["AGENTMEMORY_PROJECT_NAME"]; @@ -21,7 +20,6 @@ function resolveProject(cwd) { } catch {} return basename(dir); } - //#endregion //#region src/hooks/subagent-start.ts function isSdkChildContext(payload) { @@ -69,7 +67,7 @@ async function main() { setTimeout(() => process.exit(0), 500).unref(); } main(); - //#endregion -export { }; +export {}; + //# sourceMappingURL=subagent-start.mjs.map \ No newline at end of file diff --git a/plugin/scripts/subagent-stop.mjs b/plugin/scripts/subagent-stop.mjs index 2ba1b002c9..715d8207cd 100755 --- a/plugin/scripts/subagent-stop.mjs +++ b/plugin/scripts/subagent-stop.mjs @@ -1,7 +1,6 @@ #!/usr/bin/env node import { execSync } from "node:child_process"; import { basename } from "node:path"; - //#region src/hooks/_project.ts function resolveProject(cwd) { const explicit = process.env["AGENTMEMORY_PROJECT_NAME"]; @@ -21,7 +20,6 @@ function resolveProject(cwd) { } catch {} return basename(dir); } - //#endregion //#region src/hooks/subagent-stop.ts function isSdkChildContext(payload) { @@ -70,7 +68,7 @@ async function main() { setTimeout(() => process.exit(0), 500).unref(); } main(); - //#endregion -export { }; +export {}; + //# sourceMappingURL=subagent-stop.mjs.map \ No newline at end of file diff --git a/plugin/scripts/task-completed.mjs b/plugin/scripts/task-completed.mjs index 478f0688e8..9ca9520e20 100755 --- a/plugin/scripts/task-completed.mjs +++ b/plugin/scripts/task-completed.mjs @@ -1,7 +1,6 @@ #!/usr/bin/env node import { execSync } from "node:child_process"; import { basename } from "node:path"; - //#region src/hooks/_project.ts function resolveProject(cwd) { const explicit = process.env["AGENTMEMORY_PROJECT_NAME"]; @@ -21,7 +20,6 @@ function resolveProject(cwd) { } catch {} return basename(dir); } - //#endregion //#region src/hooks/task-completed.ts function isSdkChildContext(payload) { @@ -69,7 +67,7 @@ async function main() { setTimeout(() => process.exit(0), 500).unref(); } main(); - //#endregion -export { }; +export {}; + //# sourceMappingURL=task-completed.mjs.map \ No newline at end of file diff --git a/src/eval/schemas.ts b/src/eval/schemas.ts index bf5bfca4ed..e6997a5811 100644 --- a/src/eval/schemas.ts +++ b/src/eval/schemas.ts @@ -32,6 +32,15 @@ const ObservationTypeEnum = z.enum([ "other", ]); +const DurableCandidateTypeEnum = z.enum([ + "pattern", + "preference", + "architecture", + "bug", + "workflow", + "fact", +]); + export const ObserveInputSchema = z.object({ hookType: HookTypeEnum, sessionId: z.string().min(1), @@ -58,6 +67,26 @@ export const SummaryOutputSchema = z.object({ keyDecisions: z.array(z.string()), filesModified: z.array(z.string()), concepts: z.array(z.string()), + durableCandidates: z + .array( + z.object({ + id: z.string().min(1), + sessionId: z.string().min(1), + project: z.string().min(1).optional(), + type: DurableCandidateTypeEnum, + title: z.string().min(1), + content: z.string().min(1), + concepts: z.array(z.string()), + files: z.array(z.string()), + sourceObservationIds: z.array(z.string()), + confidence: z.number().min(0).max(1), + promotionReason: z.string().min(1).optional(), + createdAt: z.string().min(1), + promotedMemoryId: z.string().min(1).optional(), + promotedAt: z.string().min(1).optional(), + }), + ) + .optional(), }); export const SearchInputSchema = z.object({ @@ -73,11 +102,20 @@ export const ContextInputSchema = z.object({ export const RememberInputSchema = z.object({ content: z.string().min(1), + title: z.string().min(1).optional(), type: z .enum(["pattern", "preference", "architecture", "bug", "workflow", "fact"]) .optional(), concepts: z.array(z.string()).optional(), files: z.array(z.string()).optional(), + ttlDays: z.number().positive().optional(), + sourceObservationIds: z.array(z.string()).optional(), + sessionIds: z.array(z.string()).optional(), + sourceCandidateId: z.string().min(1).optional(), + confidence: z.number().min(0).max(1).optional(), + strength: z.number().int().min(1).max(10).optional(), + agentId: z.string().min(1).optional(), + project: z.string().min(1).optional(), }); export const SmartSearchInputSchema = z.object({ diff --git a/src/functions/durable-candidate-utils.ts b/src/functions/durable-candidate-utils.ts new file mode 100644 index 0000000000..e25c3b45d5 --- /dev/null +++ b/src/functions/durable-candidate-utils.ts @@ -0,0 +1,270 @@ +import { getXmlChildren, getXmlTag } from "../prompts/xml.js"; +import { fingerprintId } from "../state/schema.js"; +import type { + DurableCandidate, + DurableCandidateType, + SessionSummary, +} from "../types.js"; + +export const DURABLE_CANDIDATE_MIN_CONFIDENCE = 0.55; +export const DURABLE_PROMOTE_MIN_CONFIDENCE = 0.7; +export const DURABLE_CANDIDATE_MAX_CONFIDENCE_WITHOUT_EVIDENCE = 0.6; + +const DURABLE_CANDIDATE_TYPES = new Set([ + "pattern", + "preference", + "architecture", + "bug", + "workflow", + "fact", +]); + +const RELATIVE_TIME_PATTERNS: RegExp[] = [ + /\b(today|yesterday|tomorrow|this session|last session|just now|recently)\b/gi, + /\b(earlier today|later today|last time)\b/gi, + /(刚才|昨天|上次|这次会话|本次会话)/g, +]; + +function clamp01(value: number): number { + if (!Number.isFinite(value)) return 0; + if (value < 0) return 0; + if (value > 1) return 1; + return value; +} + +function normalizeStringList(values: string[]): string[] { + const out = new Set(); + for (const value of values) { + if (typeof value !== "string") continue; + const trimmed = value.trim(); + if (!trimmed) continue; + out.add(trimmed); + } + return Array.from(out); +} + +function getXmlBlocks(xml: string, parentTag: string, childTag: string): string[] { + const parent = getXmlTag(xml, parentTag); + if (!parent) return []; + const out: string[] = []; + const re = new RegExp(`<${childTag}>([\\s\\S]*?)`, "g"); + let match: RegExpExecArray | null; + while ((match = re.exec(parent)) !== null) { + out.push(match[1].trim()); + } + return out; +} + +function asDurableCandidateType(value: string | undefined): DurableCandidateType { + if (value && DURABLE_CANDIDATE_TYPES.has(value as DurableCandidateType)) { + return value as DurableCandidateType; + } + return "fact"; +} + +export function normalizeDurableText(value: string): string { + let normalized = value.trim(); + for (const pattern of RELATIVE_TIME_PATTERNS) { + normalized = normalized.replace(pattern, " "); + } + normalized = normalized + .replace(/\s+/g, " ") + .replace(/[!?;,]+$/g, "") + .replace(/\.+$/g, "") + .trim(); + return normalized.toLowerCase(); +} + +export function buildDurableCandidateId(input: { + sessionId: string; + type: DurableCandidateType; + title: string; + content: string; + sourceObservationIds: string[]; +}): string { + const canonical = [ + input.sessionId.trim(), + normalizeDurableText(input.type), + normalizeDurableText(input.title), + normalizeDurableText(input.content), + normalizeStringList(input.sourceObservationIds).sort().join(","), + ].join("\n"); + return fingerprintId("cand", canonical); +} + +export function bucketCandidateConfidence(confidence: number): string { + if (confidence < DURABLE_PROMOTE_MIN_CONFIDENCE) return "0.55-0.69"; + if (confidence < 0.85) return "0.70-0.84"; + return "0.85-1.00"; +} + +export interface MaterializeDurableCandidateInput { + sessionId: string; + project?: string; + type?: string; + title?: string; + content?: string; + concepts?: string[]; + files?: string[]; + sourceObservationIds?: string[]; + confidence?: number; + promotionReason?: string; + createdAt: string; + validObservationIds?: Set; +} + +export function materializeDurableCandidate( + input: MaterializeDurableCandidateInput, +): DurableCandidate | null { + const rawContent = typeof input.content === "string" ? input.content.trim() : ""; + if (!rawContent) return null; + + const rawTitle = + typeof input.title === "string" && input.title.trim() + ? input.title.trim() + : rawContent; + const validObservationIds = input.validObservationIds; + const sourceObservationIds = normalizeStringList( + (input.sourceObservationIds || []).filter((id) => + !validObservationIds || validObservationIds.has(id), + ), + ); + + const cappedConfidence = + sourceObservationIds.length === 0 + ? Math.min( + clamp01(typeof input.confidence === "number" ? input.confidence : 0), + DURABLE_CANDIDATE_MAX_CONFIDENCE_WITHOUT_EVIDENCE, + ) + : clamp01(typeof input.confidence === "number" ? input.confidence : 0); + + if (cappedConfidence < DURABLE_CANDIDATE_MIN_CONFIDENCE) { + return null; + } + + const type = asDurableCandidateType( + typeof input.type === "string" ? input.type.trim() : undefined, + ); + const title = rawTitle.replace(/\s+/g, " ").trim().slice(0, 160); + const content = rawContent.replace(/\s+/g, " ").trim(); + const concepts = normalizeStringList(input.concepts || []); + const files = normalizeStringList(input.files || []); + const promotionReason = + typeof input.promotionReason === "string" && input.promotionReason.trim() + ? input.promotionReason.trim().replace(/\s+/g, " ") + : undefined; + + return { + id: buildDurableCandidateId({ + sessionId: input.sessionId, + type, + title, + content, + sourceObservationIds, + }), + sessionId: input.sessionId, + ...(input.project ? { project: input.project } : {}), + type, + title, + content, + concepts, + files, + sourceObservationIds, + confidence: cappedConfidence, + ...(promotionReason ? { promotionReason } : {}), + createdAt: input.createdAt, + }; +} + +export function parseDurableCandidatesXml( + xml: string, + input: { + sessionId: string; + project?: string; + createdAt: string; + validObservationIds?: Set; + }, +): DurableCandidate[] { + const merged = new Map(); + for (const block of getXmlBlocks(xml, "durableCandidates", "candidate")) { + const candidate = materializeDurableCandidate({ + sessionId: input.sessionId, + project: input.project, + createdAt: input.createdAt, + validObservationIds: input.validObservationIds, + type: getXmlTag(block, "type"), + title: getXmlTag(block, "title"), + content: getXmlTag(block, "content"), + concepts: getXmlChildren(block, "concepts", "concept"), + files: getXmlChildren(block, "files", "file"), + sourceObservationIds: getXmlChildren( + block, + "sourceObservationIds", + "id", + ), + confidence: Number(getXmlTag(block, "confidence")), + promotionReason: getXmlTag(block, "promotionReason"), + }); + if (!candidate) continue; + const existing = merged.get(candidate.id); + if (!existing) { + merged.set(candidate.id, candidate); + continue; + } + merged.set(candidate.id, { + ...existing, + confidence: Math.max(existing.confidence, candidate.confidence), + concepts: normalizeStringList([...existing.concepts, ...candidate.concepts]), + files: normalizeStringList([...existing.files, ...candidate.files]), + sourceObservationIds: normalizeStringList([ + ...existing.sourceObservationIds, + ...candidate.sourceObservationIds, + ]), + promotionReason: + existing.promotionReason || candidate.promotionReason, + }); + } + return Array.from(merged.values()); +} + +export function mergeDurableCandidates( + existing: DurableCandidate[] = [], + incoming: DurableCandidate[] = [], +): DurableCandidate[] { + const merged = new Map(); + for (const candidate of [...existing, ...incoming]) { + const current = merged.get(candidate.id); + if (!current) { + merged.set(candidate.id, candidate); + continue; + } + merged.set(candidate.id, { + ...current, + ...candidate, + confidence: Math.max(current.confidence, candidate.confidence), + concepts: normalizeStringList([...current.concepts, ...candidate.concepts]), + files: normalizeStringList([...current.files, ...candidate.files]), + sourceObservationIds: normalizeStringList([ + ...current.sourceObservationIds, + ...candidate.sourceObservationIds, + ]), + promotedMemoryId: candidate.promotedMemoryId ?? current.promotedMemoryId, + promotedAt: candidate.promotedAt ?? current.promotedAt, + promotionReason: current.promotionReason ?? candidate.promotionReason, + }); + } + return Array.from(merged.values()); +} + +export function updateSummaryCandidate( + summary: SessionSummary, + candidate: DurableCandidate, +): SessionSummary { + return { + ...summary, + durableCandidates: mergeDurableCandidates( + (summary.durableCandidates || []).filter((item) => item.id !== candidate.id), + [candidate], + ), + }; +} diff --git a/src/functions/durable-candidates.ts b/src/functions/durable-candidates.ts new file mode 100644 index 0000000000..299512913e --- /dev/null +++ b/src/functions/durable-candidates.ts @@ -0,0 +1,673 @@ +import { createHash } from "node:crypto"; +import { homedir } from "node:os"; +import { lstat, readFile } from "node:fs/promises"; +import { join, resolve } from "node:path"; +import type { ISdk } from "iii-sdk"; +import type { + ArchiveImportRecord, + DurableCandidate, + Session, + SessionSummary, +} from "../types.js"; +import { StateKV } from "../state/kv.js"; +import { KV, fingerprintId } from "../state/schema.js"; +import { + DURABLE_PROMOTE_MIN_CONFIDENCE, + bucketCandidateConfidence, + updateSummaryCandidate, +} from "./durable-candidate-utils.js"; +import { summarizeSession } from "./summarize.js"; +import { + MAX_FILES_DEFAULT, + MAX_FILES_UPPER_BOUND, + findJsonlFiles, + isSensitive, + isSymlink, +} from "./replay.js"; +import { parseJsonlText } from "../replay/jsonl-parser.js"; +import type { MemoryProvider } from "../types.js"; +import { safeAudit } from "./audit.js"; +import { logger } from "../logger.js"; + +const ARCHIVE_ROOT_DEFAULT = join(homedir(), ".codex", "archived_sessions"); + +function hashText(text: string): string { + return createHash("sha256").update(text).digest("hex"); +} + +function isSubPath(path: string, root: string): boolean { + const normalizedPath = resolve(path).toLowerCase(); + const normalizedRoot = resolve(root).toLowerCase(); + return ( + normalizedPath === normalizedRoot || + normalizedPath.startsWith(`${normalizedRoot}\\`) || + normalizedPath.startsWith(`${normalizedRoot}/`) + ); +} + +function asStringArray(value: unknown): string[] | null { + if (!Array.isArray(value)) return null; + const out: string[] = []; + for (const item of value) { + if (typeof item !== "string") return null; + const trimmed = item.trim(); + if (!trimmed) continue; + out.push(trimmed); + } + return out; +} + +function asPositiveInt(value: unknown): number | null | undefined { + if (value === undefined) return undefined; + if (typeof value !== "number" || !Number.isInteger(value) || value < 1) { + return null; + } + return value; +} + +function asOptionalBoolean(value: unknown): boolean | undefined | null { + if (value === undefined) return undefined; + if (typeof value !== "boolean") return null; + return value; +} + +function findCandidateInSummaries( + summaries: SessionSummary[], + candidateId: string, +): { summary: SessionSummary; candidate: DurableCandidate } | null { + for (const summary of summaries) { + for (const candidate of summary.durableCandidates || []) { + if (candidate.id === candidateId) { + return { summary, candidate }; + } + } + } + return null; +} + +function collectDurableCandidates( + summaries: SessionSummary[], +): DurableCandidate[] { + return summaries + .flatMap((summary) => summary.durableCandidates || []) + .sort((a, b) => { + const byCreatedAt = (b.createdAt || "").localeCompare(a.createdAt || ""); + if (byCreatedAt !== 0) return byCreatedAt; + return b.confidence - a.confidence; + }); +} + +async function readArchiveTargets( + rawPath: string, + maxFiles?: number, +): Promise< + | { + success: true; + files: string[]; + discovered: number; + truncated: boolean; + traversalCapped: boolean; + rootPath: string; + maxFiles: number; + } + | { success: false; error: string } +> { + const expanded = rawPath.startsWith("~") + ? join(homedir(), rawPath.slice(1)) + : rawPath; + const abs = resolve(expanded); + if (isSensitive(abs)) { + return { success: false, error: "refusing to process sensitive-looking path" }; + } + if (await isSymlink(abs)) { + return { success: false, error: "symlinks are not supported" }; + } + + let stat; + try { + stat = await lstat(abs); + } catch { + return { success: false, error: "path not found" }; + } + + const effectiveMaxFiles = + maxFiles === undefined + ? MAX_FILES_DEFAULT + : Math.min(maxFiles, MAX_FILES_UPPER_BOUND); + + if (stat.isDirectory()) { + const found = await findJsonlFiles(abs, effectiveMaxFiles); + return { + success: true, + files: found.files, + discovered: found.discovered, + truncated: found.truncated, + traversalCapped: found.traversalCapped, + rootPath: abs, + maxFiles: effectiveMaxFiles, + }; + } + + if (stat.isFile() && abs.endsWith(".jsonl")) { + return { + success: true, + files: [abs], + discovered: 1, + truncated: false, + traversalCapped: false, + rootPath: abs, + maxFiles: effectiveMaxFiles, + }; + } + + return { success: false, error: "path must be a .jsonl file or directory" }; +} + +export function registerDurableCandidateFunctions( + sdk: ISdk, + kv: StateKV, + provider: MemoryProvider, +): void { + sdk.registerFunction( + "mem::durable-candidates::list", + async (data: { + sessionId?: string; + project?: string; + type?: string; + promoted?: boolean; + minConfidence?: number; + limit?: number; + } = {}) => { + const summaries = await kv.list(KV.summaries); + let candidates = collectDurableCandidates(summaries); + if (typeof data.sessionId === "string" && data.sessionId.trim()) { + candidates = candidates.filter( + (candidate) => candidate.sessionId === data.sessionId?.trim(), + ); + } + if (typeof data.project === "string" && data.project.trim()) { + candidates = candidates.filter( + (candidate) => candidate.project === data.project?.trim(), + ); + } + if (typeof data.type === "string" && data.type.trim()) { + candidates = candidates.filter( + (candidate) => candidate.type === data.type?.trim(), + ); + } + if (typeof data.promoted === "boolean") { + candidates = candidates.filter((candidate) => + data.promoted + ? Boolean(candidate.promotedMemoryId) + : !candidate.promotedMemoryId, + ); + } + if ( + typeof data.minConfidence === "number" && + Number.isFinite(data.minConfidence) + ) { + candidates = candidates.filter( + (candidate) => candidate.confidence >= data.minConfidence, + ); + } + if (typeof data.limit === "number" && Number.isInteger(data.limit) && data.limit > 0) { + candidates = candidates.slice(0, data.limit); + } + return { + success: true, + source: "summaries-scan", + total: candidates.length, + candidates, + }; + }, + ); + + sdk.registerFunction( + "mem::durable-candidates::promote", + async (data: { + candidateId?: string; + dryRun?: boolean; + force?: boolean; + }) => { + const candidateId = + typeof data?.candidateId === "string" ? data.candidateId.trim() : ""; + if (!candidateId) { + return { success: false, error: "candidateId is required" }; + } + + const summaries = await kv.list(KV.summaries); + const found = findCandidateInSummaries(summaries, candidateId); + if (!found) { + return { success: false, error: "candidate_not_found" }; + } + + const { summary, candidate } = found; + const dryRun = data?.dryRun === true; + const force = data?.force === true; + const requiresForce = + candidate.confidence < DURABLE_PROMOTE_MIN_CONFIDENCE || + candidate.sourceObservationIds.length === 0; + + if (candidate.promotedMemoryId) { + const existingPromoted = await kv.get(KV.memories, candidate.promotedMemoryId); + if (existingPromoted) { + return { + success: true, + promoted: false, + dryRun, + skipped: true, + reason: "already_promoted", + candidate, + memoryId: candidate.promotedMemoryId, + }; + } + } + + const memories = await kv.list(KV.memories); + const existingBySourceCandidate = memories.find( + (memory) => memory.sourceCandidateId === candidate.id, + ); + if (existingBySourceCandidate) { + const promotedAt = candidate.promotedAt || existingBySourceCandidate.createdAt; + if (!dryRun) { + await kv.set( + KV.summaries, + summary.sessionId, + updateSummaryCandidate(summary, { + ...candidate, + promotedMemoryId: existingBySourceCandidate.id, + promotedAt, + }), + ); + } + return { + success: true, + promoted: false, + dryRun, + skipped: true, + reason: "existing_memory_for_source_candidate", + candidate: { + ...candidate, + promotedMemoryId: existingBySourceCandidate.id, + promotedAt, + }, + memoryId: existingBySourceCandidate.id, + }; + } + + if (requiresForce && !force) { + return { + success: false, + error: "force_required", + candidate, + requiresForce: true, + }; + } + + if (dryRun) { + return { + success: true, + dryRun: true, + promoted: false, + requiresForce, + wouldCreateMemory: true, + candidate, + }; + } + + const rememberResult = (await sdk.trigger({ + function_id: "mem::remember", + payload: { + title: candidate.title, + content: candidate.content, + type: candidate.type, + concepts: candidate.concepts, + files: candidate.files, + sourceObservationIds: candidate.sourceObservationIds, + sessionIds: [candidate.sessionId], + sourceCandidateId: candidate.id, + project: candidate.project, + confidence: candidate.confidence, + strength: Math.max(1, Math.min(10, Math.round(candidate.confidence * 10))), + }, + })) as { success?: boolean; memory?: import("../types.js").Memory; error?: string }; + + if (!rememberResult?.success || !rememberResult.memory) { + return { + success: false, + error: rememberResult?.error || "remember_failed", + }; + } + + const promotedAt = new Date().toISOString(); + const updatedSummary = updateSummaryCandidate(summary, { + ...candidate, + promotedMemoryId: rememberResult.memory.id, + promotedAt, + }); + await kv.set(KV.summaries, summary.sessionId, updatedSummary); + + return { + success: true, + promoted: true, + candidate: updatedSummary.durableCandidates?.find( + (item) => item.id === candidate.id, + ), + memory: rememberResult.memory, + }; + }, + ); + + sdk.registerFunction( + "mem::durable-candidates::backfill", + async (data: { + dryRun?: boolean; + limit?: number; + sessionIds?: string[]; + } = {}) => { + const dryRun = data.dryRun !== false; + const limit = + typeof data.limit === "number" && Number.isInteger(data.limit) && data.limit > 0 + ? data.limit + : undefined; + const requestedSessionIds = Array.isArray(data.sessionIds) + ? new Set(data.sessionIds.map((id) => id.trim()).filter(Boolean)) + : null; + + const sessions = await kv.list(KV.sessions); + const summaries = await kv.list(KV.summaries); + const memories = await kv.list(KV.memories); + const summaryBySessionId = new Map( + summaries.map((summary) => [summary.sessionId, summary] as const), + ); + + const scope = { + sessions: sessions.length, + observations: sessions.reduce( + (sum, session) => sum + (session.observationCount || 0), + 0, + ), + summaries: summaries.length, + memories: memories.length, + }; + + const eligibleSessions: Array<{ + sessionId: string; + project: string; + observationCount: number; + }> = []; + const skippedSessions: Array<{ sessionId: string; reason: string }> = []; + + for (const session of sessions) { + if (requestedSessionIds && !requestedSessionIds.has(session.id)) { + continue; + } + if (!session.id || !session.project) { + skippedSessions.push({ + sessionId: session.id || "", + reason: "invalid_row", + }); + continue; + } + if (session.status === "active") { + skippedSessions.push({ sessionId: session.id, reason: "active" }); + continue; + } + const observations = await kv.list(KV.observations(session.id)); + if (observations.length === 0) { + skippedSessions.push({ sessionId: session.id, reason: "no_observations" }); + continue; + } + const existingSummary = summaryBySessionId.get(session.id); + if ((existingSummary?.durableCandidates?.length || 0) > 0) { + skippedSessions.push({ + sessionId: session.id, + reason: "already_has_candidates", + }); + continue; + } + eligibleSessions.push({ + sessionId: session.id, + project: session.project, + observationCount: observations.length, + }); + } + + const limitedEligibleSessions = + limit !== undefined ? eligibleSessions.slice(0, limit) : eligibleSessions; + const candidatePreview = { + total: 0, + byType: {} as Record, + byConfidence: {} as Record, + }; + const processedSessions: Array<{ + sessionId: string; + candidateCount: number; + }> = []; + + for (const eligible of limitedEligibleSessions) { + const result = await summarizeSession(kv, provider, { + sessionId: eligible.sessionId, + persistSummary: !dryRun, + mergeDurableCandidatesOnly: true, + }); + if (!result.success) { + skippedSessions.push({ + sessionId: eligible.sessionId, + reason: result.error, + }); + continue; + } + + const candidates = result.summary.durableCandidates || []; + processedSessions.push({ + sessionId: eligible.sessionId, + candidateCount: candidates.length, + }); + candidatePreview.total += candidates.length; + for (const candidate of candidates) { + candidatePreview.byType[candidate.type] = + (candidatePreview.byType[candidate.type] || 0) + 1; + const bucket = bucketCandidateConfidence(candidate.confidence); + candidatePreview.byConfidence[bucket] = + (candidatePreview.byConfidence[bucket] || 0) + 1; + } + } + + if (!dryRun && processedSessions.length > 0) { + await safeAudit( + kv, + "compress", + "mem::durable-candidates::backfill", + processedSessions.map((item) => item.sessionId), + { + dryRun, + processedSessions: processedSessions.length, + candidateCount: candidatePreview.total, + }, + ); + } + + return { + success: true, + dryRun, + startedAt: new Date().toISOString(), + scope, + eligibleSessions: limitedEligibleSessions, + skippedSessions, + processedSessions, + candidatePreview, + wouldMutate: dryRun ? processedSessions.length > 0 : false, + }; + }, + ); + + sdk.registerFunction( + "mem::archive::process", + async (data: { + path?: string; + maxFiles?: number; + force?: boolean; + allowNonArchivePath?: boolean; + } = {}) => { + const rawPath = + typeof data.path === "string" && data.path.trim() + ? data.path.trim() + : ARCHIVE_ROOT_DEFAULT; + const maxFiles = + data.maxFiles === undefined + ? undefined + : Math.min(data.maxFiles, MAX_FILES_UPPER_BOUND); + const allowNonArchivePath = data.allowNonArchivePath === true; + const fileSelection = await readArchiveTargets(rawPath, maxFiles); + if (!fileSelection.success) { + return { success: false, error: fileSelection.error }; + } + + if (!allowNonArchivePath && !isSubPath(fileSelection.rootPath, ARCHIVE_ROOT_DEFAULT)) { + return { + success: false, + error: `archive path must live under ${ARCHIVE_ROOT_DEFAULT}`, + }; + } + + const processed: Array<{ + archivePath: string; + sessionId: string; + durableCandidateCount: number; + idempotencyKey: string; + fileHash: string; + }> = []; + const skipped: Array<{ + archivePath: string; + sessionId?: string; + reason: string; + }> = []; + + for (const file of fileSelection.files) { + let text: string; + try { + text = await readFile(file, "utf-8"); + } catch (err) { + skipped.push({ + archivePath: file, + reason: err instanceof Error ? err.message : "read_failed", + }); + continue; + } + + const parsed = parseJsonlText(text); + if (parsed.observations.length === 0) { + skipped.push({ + archivePath: file, + sessionId: parsed.sessionId, + reason: "no_observations", + }); + continue; + } + + const fileHash = hashText(text); + const idempotencyKey = fingerprintId( + "arch", + `${file}\n${fileHash}\n${parsed.sessionId}`, + ); + const existingImport = await kv.get( + KV.archiveImports, + idempotencyKey, + ); + if (existingImport && data.force !== true) { + skipped.push({ + archivePath: file, + sessionId: parsed.sessionId, + reason: "already_processed", + }); + continue; + } + + const importResult = (await sdk.trigger({ + function_id: "mem::replay::import-jsonl", + payload: { path: file, maxFiles: 1 }, + })) as { success?: boolean; error?: string }; + if (!importResult?.success) { + skipped.push({ + archivePath: file, + sessionId: parsed.sessionId, + reason: importResult?.error || "import_failed", + }); + continue; + } + + const session = await kv.get(KV.sessions, parsed.sessionId); + if (session) { + const observationCount = ( + await kv.list(KV.observations(parsed.sessionId)) + ).length; + session.observationCount = observationCount; + session.status = "completed"; + session.endedAt = parsed.endedAt || session.endedAt; + await kv.set(KV.sessions, parsed.sessionId, session); + } + + const summaryResult = await summarizeSession(kv, provider, { + sessionId: parsed.sessionId, + persistSummary: true, + }); + if (!summaryResult.success) { + skipped.push({ + archivePath: file, + sessionId: parsed.sessionId, + reason: summaryResult.error, + }); + continue; + } + + const durableCandidateCount = + summaryResult.summary.durableCandidates?.length || 0; + await kv.set(KV.archiveImports, idempotencyKey, { + id: idempotencyKey, + archivePath: file, + fileHash, + sessionId: parsed.sessionId, + processedAt: new Date().toISOString(), + durableCandidateCount, + summaryCreated: true, + source: "archive-process", + }); + processed.push({ + archivePath: file, + sessionId: parsed.sessionId, + durableCandidateCount, + idempotencyKey, + fileHash, + }); + } + + if (processed.length > 0) { + await safeAudit( + kv, + "import", + "mem::archive::process", + processed.map((item) => item.sessionId), + { + path: fileSelection.rootPath, + files: processed.length, + discovered: fileSelection.discovered, + truncated: fileSelection.truncated, + }, + ); + } + + return { + success: true, + source: "archive-process", + archiveRoot: fileSelection.rootPath, + discovered: fileSelection.discovered, + truncated: fileSelection.truncated, + traversalCapped: fileSelection.traversalCapped, + maxFiles: fileSelection.maxFiles, + processed, + skipped, + }; + }, + ); +} diff --git a/src/functions/remember.ts b/src/functions/remember.ts index 5735b4f238..8558a19d07 100644 --- a/src/functions/remember.ts +++ b/src/functions/remember.ts @@ -14,11 +14,16 @@ export function registerRememberFunction(sdk: ISdk, kv: StateKV): void { sdk.registerFunction("mem::remember", async (data: { content: string; + title?: string; type?: string; concepts?: string[]; files?: string[]; ttlDays?: number; sourceObservationIds?: string[]; + sessionIds?: string[]; + sourceCandidateId?: string; + confidence?: number; + strength?: number; agentId?: string; project?: string; }) => { @@ -38,6 +43,9 @@ export function registerRememberFunction(sdk: ISdk, kv: StateKV): void { if (data.sourceObservationIds && !Array.isArray(data.sourceObservationIds)) { return { success: false, error: "sourceObservationIds must be an array" }; } + if (data.sessionIds && !Array.isArray(data.sessionIds)) { + return { success: false, error: "sessionIds must be an array" }; + } const validTypes = new Set([ "pattern", "preference", @@ -94,24 +102,48 @@ export function registerRememberFunction(sdk: ISdk, kv: StateKV): void { typeof data.agentId === "string" && data.agentId.trim().length > 0 ? data.agentId.trim().slice(0, 128) : getAgentId(); + const title = + typeof data.title === "string" && data.title.trim().length > 0 + ? data.title.trim().slice(0, 120) + : data.content.slice(0, 80); + const sessionIds = (data.sessionIds || []).filter( + (id): id is string => typeof id === "string" && id.trim().length > 0, + ); + const confidence = + typeof data.confidence === "number" && Number.isFinite(data.confidence) + ? Math.max(0, Math.min(1, data.confidence)) + : undefined; + const strength = + typeof data.strength === "number" && Number.isFinite(data.strength) + ? Math.max(1, Math.min(10, Math.round(data.strength))) + : confidence !== undefined + ? Math.max(1, Math.min(10, Math.round(confidence * 10))) + : 7; + const sourceCandidateId = + typeof data.sourceCandidateId === "string" && + data.sourceCandidateId.trim().length > 0 + ? data.sourceCandidateId.trim() + : undefined; const memory: Memory = { id: generateId("mem"), createdAt: now, updatedAt: now, type: memType, - title: data.content.slice(0, 80), + title, content: data.content, concepts: data.concepts || [], files: data.files || [], - sessionIds: [], - strength: 7, + sessionIds, + strength, + ...(confidence !== undefined && { confidence }), version: supersededId ? supersededVersion + 1 : 1, parentId: supersededId, supersedes: supersededId ? [supersededId] : [], sourceObservationIds: (data.sourceObservationIds || []).filter( (id): id is string => typeof id === "string" && id.length > 0, ), + ...(sourceCandidateId ? { sourceCandidateId } : {}), isLatest: true, ...(callAgentId ? { agentId: callAgentId } : {}), ...(project !== undefined && { project }), diff --git a/src/functions/replay.ts b/src/functions/replay.ts index e44ca8cc9d..79b40311d1 100644 --- a/src/functions/replay.ts +++ b/src/functions/replay.ts @@ -37,7 +37,7 @@ export function isSensitive(path: string): boolean { return SENSITIVE_PATH_PATTERNS.some((re) => re.test(path)); } -async function isSymlink(path: string): Promise { +export async function isSymlink(path: string): Promise { try { const st = await lstat(path); return st.isSymbolicLink(); @@ -203,7 +203,7 @@ async function loadObservations( return rows.map((r) => (isRawShape(r) ? r : rawFromCompressed(r as CompressedObservation))); } -async function findJsonlFiles( +export async function findJsonlFiles( root: string, limit = 200, ): Promise<{ diff --git a/src/functions/summarize.ts b/src/functions/summarize.ts index 4c501ca8cf..e7ac808566 100644 --- a/src/functions/summarize.ts +++ b/src/functions/summarize.ts @@ -19,6 +19,7 @@ import { validateOutput } from "../eval/validator.js"; import { scoreSummary } from "../eval/quality.js"; import type { MetricsStore } from "../eval/metrics-store.js"; import { safeAudit } from "./audit.js"; +import { parseDurableCandidatesXml } from "./durable-candidate-utils.js"; import { logger } from "../logger.js"; // Per-chunk observation budget when a session is too large to fit in one @@ -71,7 +72,7 @@ async function summarizeChunkWithRetry( SUMMARY_SYSTEM, buildSummaryPrompt(chunk), ); - const parsed = parseSummaryXml(xml, sessionId, project, chunk.length); + const parsed = parseSummaryXml(xml, sessionId, project, chunk); if (parsed) return parsed; logger.warn("Summarize chunk parse failed", { sessionId, @@ -173,6 +174,7 @@ async function produceSummaryXml( keyDecisions: p.keyDecisions, filesModified: p.filesModified, concepts: p.concepts, + durableCandidates: p.durableCandidates || [], obsRangeStart: originalIdx * chunkSize + 1, obsRangeEnd: Math.min((originalIdx + 1) * chunkSize, compressed.length), }; @@ -207,192 +209,234 @@ function parseSummaryXml( xml: string, sessionId: string, project: string, - obsCount: number, + observations: CompressedObservation[], ): SessionSummary | null { const cleaned = stripXmlWrappers(xml); const title = getXmlTag(cleaned, "title"); if (!title) return null; + const createdAt = new Date().toISOString(); return { sessionId, project, - createdAt: new Date().toISOString(), + createdAt, title, narrative: getXmlTag(cleaned, "narrative"), keyDecisions: getXmlChildren(cleaned, "decisions", "decision"), filesModified: getXmlChildren(cleaned, "files", "file"), concepts: getXmlChildren(cleaned, "concepts", "concept"), - observationCount: obsCount, + observationCount: observations.length, + durableCandidates: parseDurableCandidatesXml(cleaned, { + sessionId, + project, + createdAt, + validObservationIds: new Set(observations.map((obs) => obs.id)), + }), }; } -export function registerSummarizeFunction( - sdk: ISdk, +export interface SummarizeSessionOptions { + sessionId: string; + persistSummary?: boolean; + mergeDurableCandidatesOnly?: boolean; + metricsStore?: MetricsStore; +} + +export async function summarizeSession( kv: StateKV, provider: MemoryProvider, - metricsStore?: MetricsStore, -): void { - sdk.registerFunction("mem::summarize", - async (data: { sessionId: string } | undefined) => { - const startMs = Date.now(); - if (!data || typeof data.sessionId !== "string" || !data.sessionId.trim()) { - return { success: false, error: "sessionId is required" }; - } - const sessionId = data.sessionId.trim(); + options: SummarizeSessionOptions, +): Promise< + | { success: true; summary: SessionSummary; qualityScore: number } + | { success: false; error: string; reason?: string } +> { + const startMs = Date.now(); + const sessionId = options.sessionId.trim(); + const persistSummary = options.persistSummary !== false; + const mergeDurableCandidatesOnly = options.mergeDurableCandidatesOnly === true; + const metricsStore = options.metricsStore; - const session = await kv.get(KV.sessions, sessionId); - if (!session) { - logger.warn("Session not found for summarize", { - sessionId, - }); - return { success: false, error: "session_not_found" }; - } + const session = await kv.get(KV.sessions, sessionId); + if (!session) { + logger.warn("Session not found for summarize", { + sessionId, + }); + return { success: false, error: "session_not_found" }; + } - const observations = await kv.list( - KV.observations(sessionId), - ); - const compressed = observations.filter((o) => o.title); + const observations = await kv.list( + KV.observations(sessionId), + ); + const compressed = observations.filter((o) => o.title); + + if (compressed.length === 0) { + logger.info("No observations to summarize", { + sessionId, + }); + return { success: false, error: "no_observations" }; + } - if (compressed.length === 0) { - logger.info("No observations to summarize", { + if (provider.name === "noop") { + logger.info("Summarize skipped — no LLM provider configured", { + sessionId, + }); + return { + success: false, + error: "no_provider", + reason: + "No LLM provider key set; Summarize is a no-op. Set ANTHROPIC_API_KEY (or GEMINI/OPENROUTER/MINIMAX) in ~/.agentmemory/.env to enable.", + }; + } + + try { + let summary: SessionSummary | null = null; + let response = ""; + let mode = "single"; + let chunks = 1; + for (let attempt = 1; attempt <= 2; attempt++) { + const produced = await produceSummaryXml( + provider, + compressed, + sessionId, + session.project, + ); + response = produced.response; + mode = produced.mode; + chunks = produced.chunks; + if (!response || !response.trim()) { + logger.warn("Empty provider response on summarize", { sessionId, + provider: provider.name, + mode, + chunks, + observationCount: compressed.length, + attempt, }); - return { success: false, error: "no_observations" }; + continue; } + summary = parseSummaryXml( + response, + sessionId, + session.project, + compressed, + ); + if (summary) break; + logger.warn("Failed to parse summary XML", { sessionId, attempt }); + } - if (provider.name === "noop") { - logger.info("Summarize skipped — no LLM provider configured", { - sessionId, - }); - return { - success: false, - error: "no_provider", - reason: - "No LLM provider key set; Summarize is a no-op. Set ANTHROPIC_API_KEY (or GEMINI/OPENROUTER/MINIMAX) in ~/.agentmemory/.env to enable.", - }; + if (!response || !response.trim()) { + const latencyMs = Date.now() - startMs; + if (metricsStore) { + await metricsStore.record("mem::summarize", latencyMs, false); } + return { success: false, error: "empty_provider_response" }; + } - try { - // #783: chunk-level produceSummaryXml retries internally, but - // the final merge used to parse once and bail. Wrap the - // produce-and-parse pair in the same 2-attempt loop so a - // markdown-wrapped or otherwise wrapped response gets a - // second roll-of-the-dice instead of dropping the summary. - let summary: SessionSummary | null = null; - let response = ""; - let mode = "single"; - let chunks = 1; - for (let attempt = 1; attempt <= 2; attempt++) { - const produced = await produceSummaryXml( - provider, - compressed, - sessionId, - session.project, - ); - response = produced.response; - mode = produced.mode; - chunks = produced.chunks; - if (!response || !response.trim()) { - logger.warn("Empty provider response on summarize", { - sessionId, - provider: provider.name, - mode, - chunks, - observationCount: compressed.length, - attempt, - }); - continue; - } - summary = parseSummaryXml( - response, - sessionId, - session.project, - compressed.length, - ); - if (summary) break; - logger.warn("Failed to parse summary XML", { sessionId, attempt }); - } + if (!summary) { + const latencyMs = Date.now() - startMs; + if (metricsStore) { + await metricsStore.record("mem::summarize", latencyMs, false); + } + return { success: false, error: "parse_failed" }; + } - if (!response || !response.trim()) { - const latencyMs = Date.now() - startMs; - if (metricsStore) { - await metricsStore.record("mem::summarize", latencyMs, false); - } - return { success: false, error: "empty_provider_response" }; - } + const summaryForValidation = { + title: summary.title, + narrative: summary.narrative, + keyDecisions: summary.keyDecisions, + filesModified: summary.filesModified, + concepts: summary.concepts, + durableCandidates: summary.durableCandidates, + }; + const validation = validateOutput( + SummaryOutputSchema, + summaryForValidation, + "mem::summarize", + ); - if (!summary) { - const latencyMs = Date.now() - startMs; - if (metricsStore) { - await metricsStore.record("mem::summarize", latencyMs, false); - } - return { success: false, error: "parse_failed" }; - } + if (!validation.valid) { + const latencyMs = Date.now() - startMs; + if (metricsStore) { + await metricsStore.record("mem::summarize", latencyMs, false); + } + logger.warn("Summary validation failed", { + sessionId, + errors: validation.result.errors, + }); + return { success: false, error: "validation_failed" }; + } - const summaryForValidation = { - title: summary.title, - narrative: summary.narrative, - keyDecisions: summary.keyDecisions, - filesModified: summary.filesModified, - concepts: summary.concepts, - }; - const validation = validateOutput( - SummaryOutputSchema, - summaryForValidation, - "mem::summarize", - ); + const qualityScore = scoreSummary(summaryForValidation); + let summaryToStore = summary; - if (!validation.valid) { - const latencyMs = Date.now() - startMs; - if (metricsStore) { - await metricsStore.record("mem::summarize", latencyMs, false); - } - logger.warn("Summary validation failed", { - sessionId, - errors: validation.result.errors, - }); - return { success: false, error: "validation_failed" }; + if (persistSummary) { + if (mergeDurableCandidatesOnly) { + const existingSummary = await kv.get(KV.summaries, sessionId); + if (existingSummary) { + summaryToStore = { + ...existingSummary, + observationCount: summary.observationCount, + durableCandidates: summary.durableCandidates, + }; } + } + await kv.set(KV.summaries, sessionId, summaryToStore); + await safeAudit(kv, "compress", "mem::summarize", [sessionId], { + title: summaryToStore.title, + observationCount: compressed.length, + }); + } - const qualityScore = scoreSummary(summaryForValidation); - - await kv.set(KV.summaries, sessionId, summary); - await safeAudit(kv, "compress", "mem::summarize", [sessionId], { - title: summary.title, - observationCount: compressed.length, - }); + const latencyMs = Date.now() - startMs; + if (metricsStore) { + await metricsStore.record( + "mem::summarize", + latencyMs, + true, + qualityScore, + ); + } - const latencyMs = Date.now() - startMs; - if (metricsStore) { - await metricsStore.record( - "mem::summarize", - latencyMs, - true, - qualityScore, - ); - } + logger.info("Session summarized", { + sessionId, + title: summaryToStore.title, + decisions: summaryToStore.keyDecisions.length, + durableCandidates: summaryToStore.durableCandidates?.length || 0, + qualityScore, + valid: validation.valid, + }); - logger.info("Session summarized", { - sessionId, - title: summary.title, - decisions: summary.keyDecisions.length, - qualityScore, - valid: validation.valid, - }); + return { success: true, summary: summaryToStore, qualityScore }; + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + const latencyMs = Date.now() - startMs; + if (metricsStore) { + await metricsStore.record("mem::summarize", latencyMs, false); + } + logger.error("Summarize failed", { + sessionId, + error: msg, + }); + return { success: false, error: msg }; + } +} - return { success: true, summary, qualityScore }; - } catch (err) { - const msg = err instanceof Error ? err.message : String(err); - const latencyMs = Date.now() - startMs; - if (metricsStore) { - await metricsStore.record("mem::summarize", latencyMs, false); - } - logger.error("Summarize failed", { - sessionId, - error: msg, - }); - return { success: false, error: msg }; +export function registerSummarizeFunction( + sdk: ISdk, + kv: StateKV, + provider: MemoryProvider, + metricsStore?: MetricsStore, +): void { + sdk.registerFunction("mem::summarize", + async (data: { sessionId: string } | undefined) => { + if (!data || typeof data.sessionId !== "string" || !data.sessionId.trim()) { + return { success: false, error: "sessionId is required" }; } + return summarizeSession(kv, provider, { + sessionId: data.sessionId, + persistSummary: true, + metricsStore, + }); }, ); } diff --git a/src/index.ts b/src/index.ts index 4233e8a67e..58cc7ce336 100644 --- a/src/index.ts +++ b/src/index.ts @@ -88,6 +88,7 @@ import { registerTemporalGraphFunctions } from "./functions/temporal-graph.js"; import { registerRetentionFunctions } from "./functions/retention.js"; import { registerCompressFileFunction } from "./functions/compress-file.js"; import { registerReplayFunctions } from "./functions/replay.js"; +import { registerDurableCandidateFunctions } from "./functions/durable-candidates.js"; import { registerApiTriggers } from "./triggers/api.js"; import { registerEventTriggers } from "./triggers/events.js"; import { registerMcpEndpoints } from "./mcp/server.js"; @@ -332,6 +333,7 @@ async function main() { registerRetentionFunctions(sdk, kv); registerCompressFileFunction(sdk, kv, provider); registerReplayFunctions(sdk, kv); + registerDurableCandidateFunctions(sdk, kv, provider); bootLog( `v0.6 advanced retrieval: sliding-window, query-expansion, temporal-graph, retention-scoring`, ); @@ -518,7 +520,7 @@ async function main() { `Ready. ${embeddingProvider ? "Triple-stream (BM25+Vector+Graph)" : "BM25+Graph"} search active.`, ); bootLog( - `REST API: 128 endpoints at http://localhost:${config.restPort}/agentmemory/*`, + `REST API: 132 endpoints at http://localhost:${config.restPort}/agentmemory/*`, ); bootLog( `MCP surface (opt-in via \`npx @agentmemory/mcp\`): ${getAllTools().length} tools · 6 resources · 3 prompts`, diff --git a/src/mcp/tools-registry.ts b/src/mcp/tools-registry.ts index c86d899bbc..b892d711d4 100644 --- a/src/mcp/tools-registry.ts +++ b/src/mcp/tools-registry.ts @@ -925,7 +925,7 @@ export const V010_SLOTS_TOOLS: McpToolDef[] = [ }, ]; -const ESSENTIAL_TOOLS = new Set([ +export const ESSENTIAL_TOOLS = new Set([ "memory_save", "memory_recall", "memory_consolidate", diff --git a/src/prompts/summary.ts b/src/prompts/summary.ts index bd04021277..6103cfd5ee 100644 --- a/src/prompts/summary.ts +++ b/src/prompts/summary.ts @@ -14,15 +14,39 @@ Output EXACTLY this XML format with no additional text: key concept from session + + + pattern|preference|architecture|bug|workflow|fact + Stable memory title + Durable fact or convention worth keeping across sessions + + searchable concept + + + path/to/file + + + obs_123 + + 0.70 + Why this may deserve explicit promote later + + Rules: - Focus on outcomes, not individual tool calls - Highlight decisions and their rationale - List all files that were created or modified -- Concepts should be searchable terms for future context retrieval` +- Concepts should be searchable terms for future context retrieval +- Durable candidates are only cross-session facts, decisions, conventions, workflows, or bugs likely worth keeping +- Do not emit low-value, ephemeral, or purely session-local notes +- Emit only candidates with confidence >= 0.55 +- Use exact source observation ids from the prompt when available +- If a candidate has no reliable source observation id, leave empty and keep confidence <= 0.60` export function buildSummaryPrompt(observations: Array<{ + id: string type: string title: string facts: string[] @@ -32,7 +56,7 @@ export function buildSummaryPrompt(observations: Array<{ }>): string { const lines = observations.map((obs, i) => { const facts = obs.facts.map((f) => ` - ${f}`).join('\n') - return `[${i + 1}] ${obs.type}: ${obs.title}\n${obs.narrative}\nFacts:\n${facts}\nFiles: ${obs.files.join(', ')}` + return `[${i + 1}] ${obs.type}: ${obs.title}\nObservation ID: ${obs.id}\n${obs.narrative}\nFacts:\n${facts}\nFiles: ${obs.files.join(', ')}\nConcepts: ${obs.concepts.join(', ')}` }) return `Session observations (${observations.length} total):\n\n${lines.join('\n\n---\n\n')}` } @@ -53,13 +77,33 @@ Output EXACTLY this XML format with no additional text: key concept from session + + + pattern|preference|architecture|bug|workflow|fact + Stable memory title + Durable fact or convention worth keeping across sessions + + searchable concept + + + path/to/file + + + obs_123 + + 0.70 + Why this may deserve explicit promote later + + Rules: - Synthesize a single narrative that reflects the whole arc, not a chunk-by-chunk recap - Preserve every distinct decision across chunks - Union (deduplicate) all files and concepts -- Title should capture the session's overall outcome` +- Title should capture the session's overall outcome +- Merge durable candidates by meaning, keep only the best ones, and preserve exact observation ids +- Emit only durable candidates that still clear confidence >= 0.55 after merging` export function buildReducePrompt(partials: Array<{ title: string @@ -67,6 +111,14 @@ export function buildReducePrompt(partials: Array<{ keyDecisions: string[] filesModified: string[] concepts: string[] + durableCandidates?: Array<{ + type: string + title: string + content: string + confidence: number + sourceObservationIds: string[] + promotionReason?: string + }> obsRangeStart: number obsRangeEnd: number }>): string { @@ -74,6 +126,9 @@ export function buildReducePrompt(partials: Array<{ const decisions = p.keyDecisions.map((d) => ` - ${d}`).join('\n') const files = p.filesModified.map((f) => ` - ${f}`).join('\n') const concepts = p.concepts.join(', ') + const durableCandidates = (p.durableCandidates ?? []).map((candidate) => + ` - [${candidate.type}] ${candidate.title} | confidence=${candidate.confidence.toFixed(2)} | sources=${candidate.sourceObservationIds.join(', ') || ''} | content=${candidate.content}${candidate.promotionReason ? ` | reason=${candidate.promotionReason}` : ''}`, + ).join('\n') return `[Chunk ${i + 1} of ${partials.length} — obs ${p.obsRangeStart}-${p.obsRangeEnd}] Title: ${p.title} Narrative: ${p.narrative} @@ -81,7 +136,9 @@ Decisions: ${decisions} Files: ${files} -Concepts: ${concepts}` +Concepts: ${concepts} +Durable candidates: +${durableCandidates || ' - '}` }) return `Partial summaries (${partials.length} chunks of one session, chronological):\n\n${sections.join('\n\n---\n\n')}` } diff --git a/src/state/memory-utils.ts b/src/state/memory-utils.ts index aa0bcc5b8a..3ab1eedc64 100644 --- a/src/state/memory-utils.ts +++ b/src/state/memory-utils.ts @@ -20,5 +20,6 @@ export function memoryToObservation(memory: Memory): CompressedObservation { concepts: memory.concepts, files: memory.files, importance: memory.strength, + ...(memory.confidence !== undefined && { confidence: memory.confidence }), }; } diff --git a/src/state/schema.ts b/src/state/schema.ts index cb29d41ad8..f8a2b0c934 100644 --- a/src/state/schema.ts +++ b/src/state/schema.ts @@ -5,6 +5,7 @@ export const KV = { observations: (sessionId: string) => `mem:obs:${sessionId}`, memories: "mem:memories", summaries: "mem:summaries", + archiveImports: "mem:archive-imports", config: "mem:config", metrics: "mem:metrics", health: "mem:health", diff --git a/src/triggers/api.ts b/src/triggers/api.ts index 4a214a3acd..802bb4aaf9 100644 --- a/src/triggers/api.ts +++ b/src/triggers/api.ts @@ -557,6 +557,73 @@ export function registerApiTriggers( config: { api_path: "/agentmemory/replay/import-jsonl", http_method: "POST" }, }); + sdk.registerFunction("api::archive::process", + async ( + req: ApiRequest<{ + path?: string; + maxFiles?: number; + force?: boolean; + allowNonArchivePath?: boolean; + }>, + ): Promise => { + const authErr = checkAuth(req, secret); + if (authErr) return authErr; + const body = (req.body ?? {}) as Record; + const payload: { + path?: string; + maxFiles?: number; + force?: boolean; + allowNonArchivePath?: boolean; + } = {}; + if (body.path !== undefined) { + if (typeof body.path !== "string" || body.path.trim().length === 0) { + return { + status_code: 400, + body: { error: "path must be a non-empty string" }, + }; + } + payload.path = body.path.trim(); + } + if (body.maxFiles !== undefined) { + const n = body.maxFiles as number; + if (!Number.isInteger(n) || n < 1 || n > MAX_FILES_UPPER_BOUND) { + return { + status_code: 400, + body: { + error: `maxFiles must be an integer between 1 and ${MAX_FILES_UPPER_BOUND}`, + }, + }; + } + payload.maxFiles = n; + } + if (body.force !== undefined) { + if (typeof body.force !== "boolean") { + return { status_code: 400, body: { error: "force must be a boolean" } }; + } + payload.force = body.force; + } + if (body.allowNonArchivePath !== undefined) { + if (typeof body.allowNonArchivePath !== "boolean") { + return { + status_code: 400, + body: { error: "allowNonArchivePath must be a boolean" }, + }; + } + payload.allowNonArchivePath = body.allowNonArchivePath; + } + const result = await sdk.trigger({ + function_id: "mem::archive::process", + payload, + }); + return { status_code: 202, body: result }; + }, + ); + sdk.registerTrigger({ + type: "http", + function_id: "api::archive::process", + config: { api_path: "/agentmemory/archive/process", http_method: "POST" }, + }); + sdk.registerFunction("api::session::start", async ( req: ApiRequest<{ sessionId: string; project: string; cwd: string }>, @@ -949,11 +1016,16 @@ export function registerApiTriggers( async ( req: ApiRequest<{ content: string; + title?: string; type?: string; concepts?: string[]; files?: string[]; ttlDays?: number; sourceObservationIds?: string[]; + sessionIds?: string[]; + sourceCandidateId?: string; + confidence?: number; + strength?: number; project?: string; }>, ): Promise => { @@ -972,15 +1044,80 @@ export function registerApiTriggers( ) { return { status_code: 400, body: { error: "project must be a non-empty string" } }; } + if ( + req.body.title !== undefined && + (typeof req.body.title !== "string" || !req.body.title.trim()) + ) { + return { status_code: 400, body: { error: "title must be a non-empty string" } }; + } + for (const [field, value] of [ + ["concepts", req.body.concepts], + ["files", req.body.files], + ["sourceObservationIds", req.body.sourceObservationIds], + ["sessionIds", req.body.sessionIds], + ] as const) { + if ( + value !== undefined && + (!Array.isArray(value) || !value.every((item: unknown) => typeof item === "string")) + ) { + return { + status_code: 400, + body: { error: `${field} must be an array of strings` }, + }; + } + } + if ( + req.body.ttlDays !== undefined && + (typeof req.body.ttlDays !== "number" || !Number.isFinite(req.body.ttlDays) || req.body.ttlDays <= 0) + ) { + return { status_code: 400, body: { error: "ttlDays must be a positive number" } }; + } + if ( + req.body.sourceCandidateId !== undefined && + (typeof req.body.sourceCandidateId !== "string" || !req.body.sourceCandidateId.trim()) + ) { + return { + status_code: 400, + body: { error: "sourceCandidateId must be a non-empty string" }, + }; + } + if ( + req.body.confidence !== undefined && + (typeof req.body.confidence !== "number" || + !Number.isFinite(req.body.confidence) || + req.body.confidence < 0 || + req.body.confidence > 1) + ) { + return { + status_code: 400, + body: { error: "confidence must be a number between 0 and 1" }, + }; + } + if ( + req.body.strength !== undefined && + (!Number.isInteger(req.body.strength) || + req.body.strength < 1 || + req.body.strength > 10) + ) { + return { + status_code: 400, + body: { error: "strength must be an integer between 1 and 10" }, + }; + } const result = await sdk.trigger({ function_id: "mem::remember", payload: { content: req.body.content, + ...(req.body.title !== undefined && { title: req.body.title }), ...(req.body.type !== undefined && { type: req.body.type }), ...(req.body.concepts !== undefined && { concepts: req.body.concepts }), ...(req.body.files !== undefined && { files: req.body.files }), ...(req.body.ttlDays !== undefined && { ttlDays: req.body.ttlDays }), ...(req.body.sourceObservationIds !== undefined && { sourceObservationIds: req.body.sourceObservationIds }), + ...(req.body.sessionIds !== undefined && { sessionIds: req.body.sessionIds }), + ...(req.body.sourceCandidateId !== undefined && { sourceCandidateId: req.body.sourceCandidateId }), + ...(req.body.confidence !== undefined && { confidence: req.body.confidence }), + ...(req.body.strength !== undefined && { strength: req.body.strength }), ...(req.body.project !== undefined && { project: req.body.project }), }, }); @@ -993,6 +1130,148 @@ export function registerApiTriggers( config: { api_path: "/agentmemory/remember", http_method: "POST" }, }); + sdk.registerFunction("api::durable-candidates", + async (req: ApiRequest): Promise => { + const authErr = checkAuth(req, secret); + if (authErr) return authErr; + const promotedRaw = req.query_params?.["promoted"]; + const promoted = + promotedRaw === undefined + ? undefined + : promotedRaw === "true" + ? true + : promotedRaw === "false" + ? false + : null; + if (promoted === null) { + return { + status_code: 400, + body: { error: "promoted must be true or false when provided" }, + }; + } + const minConfidence = parseOptionalFiniteNumber( + req.query_params?.["minConfidence"], + ); + if (minConfidence === null) { + return { + status_code: 400, + body: { error: "invalid numeric parameter: minConfidence" }, + }; + } + const limit = parseOptionalPositiveInt(req.query_params?.["limit"]); + if (limit === null) { + return { + status_code: 400, + body: { error: "invalid numeric parameter: limit" }, + }; + } + const result = await sdk.trigger({ + function_id: "mem::durable-candidates::list", + payload: { + ...(req.query_params?.["sessionId"] && { + sessionId: req.query_params["sessionId"], + }), + ...(req.query_params?.["project"] && { + project: req.query_params["project"], + }), + ...(req.query_params?.["type"] && { type: req.query_params["type"] }), + ...(promoted !== undefined && { promoted }), + ...(minConfidence !== undefined && { minConfidence }), + ...(limit !== undefined && { limit }), + }, + }); + return { status_code: 200, body: result }; + }, + ); + sdk.registerTrigger({ + type: "http", + function_id: "api::durable-candidates", + config: { api_path: "/agentmemory/durable-candidates", http_method: "GET" }, + }); + + sdk.registerFunction("api::durable-candidates::promote", + async ( + req: ApiRequest<{ candidateId?: string; dryRun?: boolean; force?: boolean }>, + ): Promise => { + const authErr = checkAuth(req, secret); + if (authErr) return authErr; + const body = (req.body ?? {}) as Record; + const candidateId = asNonEmptyString(body.candidateId); + if (!candidateId) { + return { status_code: 400, body: { error: "candidateId is required" } }; + } + if (body.dryRun !== undefined && typeof body.dryRun !== "boolean") { + return { status_code: 400, body: { error: "dryRun must be a boolean" } }; + } + if (body.force !== undefined && typeof body.force !== "boolean") { + return { status_code: 400, body: { error: "force must be a boolean" } }; + } + const result = await sdk.trigger({ + function_id: "mem::durable-candidates::promote", + payload: { + candidateId, + ...(body.dryRun !== undefined && { dryRun: body.dryRun }), + ...(body.force !== undefined && { force: body.force }), + }, + }); + return { status_code: 200, body: result }; + }, + ); + sdk.registerTrigger({ + type: "http", + function_id: "api::durable-candidates::promote", + config: { + api_path: "/agentmemory/durable-candidates/promote", + http_method: "POST", + }, + }); + + sdk.registerFunction("api::durable-candidates::backfill", + async ( + req: ApiRequest<{ dryRun?: boolean; limit?: number; sessionIds?: string[] }>, + ): Promise => { + const authErr = checkAuth(req, secret); + if (authErr) return authErr; + const body = (req.body ?? {}) as Record; + if (body.dryRun !== undefined && typeof body.dryRun !== "boolean") { + return { status_code: 400, body: { error: "dryRun must be a boolean" } }; + } + if ( + body.limit !== undefined && + (!Number.isInteger(body.limit) || (body.limit as number) < 1) + ) { + return { status_code: 400, body: { error: "limit must be a positive integer" } }; + } + if ( + body.sessionIds !== undefined && + (!Array.isArray(body.sessionIds) || + !(body.sessionIds as unknown[]).every((item) => typeof item === "string")) + ) { + return { + status_code: 400, + body: { error: "sessionIds must be an array of strings" }, + }; + } + const result = await sdk.trigger({ + function_id: "mem::durable-candidates::backfill", + payload: { + ...(body.dryRun !== undefined && { dryRun: body.dryRun }), + ...(body.limit !== undefined && { limit: body.limit }), + ...(body.sessionIds !== undefined && { sessionIds: body.sessionIds }), + }, + }); + return { status_code: 200, body: result }; + }, + ); + sdk.registerTrigger({ + type: "http", + function_id: "api::durable-candidates::backfill", + config: { + api_path: "/agentmemory/durable-candidates/backfill", + http_method: "POST", + }, + }); + sdk.registerFunction("api::forget", async ( req: ApiRequest<{ diff --git a/src/types.ts b/src/types.ts index 6797dfaf99..9f79e046a3 100644 --- a/src/types.ts +++ b/src/types.ts @@ -80,22 +80,49 @@ export type ObservationType = | "image" | "other"; +export type DurableCandidateType = + | "pattern" + | "preference" + | "architecture" + | "bug" + | "workflow" + | "fact"; + +export interface DurableCandidate { + id: string; + sessionId: string; + project?: string; + type: DurableCandidateType; + title: string; + content: string; + concepts: string[]; + files: string[]; + sourceObservationIds: string[]; + confidence: number; + promotionReason?: string; + createdAt: string; + promotedMemoryId?: string; + promotedAt?: string; +} + export interface Memory { id: string; createdAt: string; updatedAt: string; - type: "pattern" | "preference" | "architecture" | "bug" | "workflow" | "fact"; + type: DurableCandidateType; title: string; content: string; concepts: string[]; files: string[]; sessionIds: string[]; strength: number; + confidence?: number; version: number; parentId?: string; supersedes?: string[]; relatedIds?: string[]; sourceObservationIds?: string[]; + sourceCandidateId?: string; isLatest: boolean; forgetAfter?: string; imageRef?: string; @@ -114,6 +141,18 @@ export interface SessionSummary { filesModified: string[]; concepts: string[]; observationCount: number; + durableCandidates?: DurableCandidate[]; +} + +export interface ArchiveImportRecord { + id: string; + archivePath: string; + fileHash: string; + sessionId: string; + processedAt: string; + durableCandidateCount: number; + summaryCreated: boolean; + source: "archive-process"; } export type HookType = diff --git a/test/durable-candidate-utils.test.ts b/test/durable-candidate-utils.test.ts new file mode 100644 index 0000000000..3e2d35d051 --- /dev/null +++ b/test/durable-candidate-utils.test.ts @@ -0,0 +1,86 @@ +import { describe, expect, it } from "vitest"; +import { + DURABLE_CANDIDATE_MIN_CONFIDENCE, + buildDurableCandidateId, + materializeDurableCandidate, + normalizeDurableText, +} from "../src/functions/durable-candidate-utils.js"; + +describe("durable candidate normalization", () => { + it("removes relative time phrases but preserves concrete dates and versions", () => { + const normalized = normalizeDurableText( + "Yesterday we pinned v0.9.27 on 2026-07-10 in PR #579", + ); + + expect(normalized).not.toContain("yesterday"); + expect(normalized).toContain("2026-07-10"); + expect(normalized).toContain("v0.9.27"); + expect(normalized).toContain("pr #579"); + }); + + it("builds a stable id across whitespace and relative-time variations", () => { + const a = buildDurableCandidateId({ + sessionId: "sess_1", + type: "workflow", + title: "Archive promote flow", + content: "Today archive candidates are promoted manually.", + sourceObservationIds: ["obs_b", "obs_a"], + }); + const b = buildDurableCandidateId({ + sessionId: "sess_1", + type: "workflow", + title: " archive promote flow ", + content: "archive candidates are promoted manually yesterday", + sourceObservationIds: ["obs_a", "obs_b"], + }); + + expect(a).toBe(b); + }); +}); + +describe("materializeDurableCandidate", () => { + it("caps no-evidence candidates to 0.6 and keeps them force-only", () => { + const candidate = materializeDurableCandidate({ + sessionId: "sess_1", + type: "fact", + title: "No evidence", + content: "Long-lived fact without source observation ids.", + confidence: 0.95, + createdAt: "2026-07-10T00:00:00.000Z", + }); + + expect(candidate).not.toBeNull(); + expect(candidate?.confidence).toBe(0.6); + expect(candidate?.sourceObservationIds).toEqual([]); + }); + + it("drops candidates below the minimum generation threshold", () => { + const candidate = materializeDurableCandidate({ + sessionId: "sess_1", + type: "fact", + title: "Too weak", + content: "This should not survive thresholding.", + confidence: DURABLE_CANDIDATE_MIN_CONFIDENCE - 0.01, + createdAt: "2026-07-10T00:00:00.000Z", + sourceObservationIds: ["obs_1"], + }); + + expect(candidate).toBeNull(); + }); + + it("filters hallucinated observation ids against the known session ids", () => { + const candidate = materializeDurableCandidate({ + sessionId: "sess_1", + type: "architecture", + title: "Candidate", + content: "Only real observation ids should survive.", + confidence: 0.8, + createdAt: "2026-07-10T00:00:00.000Z", + sourceObservationIds: ["obs_real", "obs_fake"], + validObservationIds: new Set(["obs_real"]), + }); + + expect(candidate).not.toBeNull(); + expect(candidate?.sourceObservationIds).toEqual(["obs_real"]); + }); +}); diff --git a/test/durable-candidates.test.ts b/test/durable-candidates.test.ts new file mode 100644 index 0000000000..ef2674cd63 --- /dev/null +++ b/test/durable-candidates.test.ts @@ -0,0 +1,326 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; + +vi.mock("../src/logger.js", () => ({ + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, +})); + +vi.mock("../src/functions/audit.js", () => ({ + safeAudit: vi.fn(), +})); + +import { registerDurableCandidateFunctions } from "../src/functions/durable-candidates.js"; +import { materializeDurableCandidate } from "../src/functions/durable-candidate-utils.js"; +import type { + CompressedObservation, + MemoryProvider, + Session, + SessionSummary, +} from "../src/types.js"; + +function mockKV() { + const store = new Map>(); + return { + get: async (scope: string, key: string): Promise => + (store.get(scope)?.get(key) as T) ?? null, + set: async (scope: string, key: string, data: T): Promise => { + if (!store.has(scope)) store.set(scope, new Map()); + store.get(scope)!.set(key, data); + return data; + }, + delete: async (scope: string, key: string): Promise => { + store.get(scope)?.delete(key); + }, + list: async (scope: string): Promise => { + const entries = store.get(scope); + return entries ? (Array.from(entries.values()) as T[]) : []; + }, + }; +} + +function mockSdk() { + const functions = new Map(); + return { + functions, + registerFunction: (id: string, handler: Function) => { + functions.set(id, handler); + }, + registerTrigger: vi.fn(), + trigger: async (input: { function_id: string; payload: unknown }) => { + const fn = functions.get(input.function_id); + if (!fn) throw new Error(`No function registered for ${input.function_id}`); + return fn(input.payload); + }, + }; +} + +function makeObservation(sessionId: string, id = "obs_1"): CompressedObservation { + return { + id, + sessionId, + timestamp: "2026-07-10T00:00:00.000Z", + type: "decision", + title: "Archive flow decision", + facts: ["Explicit promote is required for durable memories."], + narrative: + "The session introduced durable candidates that stay out of long-term memory until an explicit promote step happens.", + concepts: ["durable-candidates", "promotion"], + files: ["src/functions/durable-candidates.ts"], + importance: 8, + }; +} + +function makeProvider(xml: string): MemoryProvider { + return { + name: "test", + compress: async () => "", + summarize: async () => xml, + }; +} + +function makeSummaryXml(sessionId: string, obsId: string): string { + return ` +Durable lifecycle +This session defined a durable candidate lifecycle, kept archive imports idempotent, and reserved long-term memory writes for explicit promote. +Archive import creates candidates only. +src/functions/durable-candidates.ts +durable-candidates + + + workflow + Explicit promote only + Archived sessions should create durable candidates and only write Memory through explicit promote. + durable-candidates + src/functions/durable-candidates.ts + ${obsId} + 0.82 + Cross-session workflow policy for future recalls. + + +`; +} + +describe("durable candidates lifecycle", () => { + let sdk: ReturnType; + let kv: ReturnType; + const tempDirs = new Set(); + + beforeEach(() => { + sdk = mockSdk(); + kv = mockKV(); + }); + + afterEach(() => { + for (const dir of tempDirs) { + rmSync(dir, { recursive: true, force: true }); + } + tempDirs.clear(); + }); + + it("promote skips when a memory already exists for sourceCandidateId and backfills promotedMemoryId", async () => { + const candidate = materializeDurableCandidate({ + sessionId: "sess_promote", + project: "agentmemory", + type: "workflow", + title: "Explicit promote only", + content: + "Archived sessions should create durable candidates and only write Memory through explicit promote.", + concepts: ["durable-candidates"], + files: ["src/functions/durable-candidates.ts"], + sourceObservationIds: ["obs_1"], + confidence: 0.82, + promotionReason: "Cross-session workflow policy.", + createdAt: "2026-07-10T00:00:00.000Z", + }); + expect(candidate).not.toBeNull(); + + const summary: SessionSummary = { + sessionId: "sess_promote", + project: "agentmemory", + createdAt: "2026-07-10T00:00:00.000Z", + title: "Summary", + narrative: "Narrative long enough to count as a real summary in tests.", + keyDecisions: [], + filesModified: [], + concepts: [], + observationCount: 1, + durableCandidates: [candidate!], + }; + await kv.set("mem:summaries", summary.sessionId, summary); + await kv.set("mem:memories", "mem_existing", { + id: "mem_existing", + createdAt: "2026-07-10T00:05:00.000Z", + updatedAt: "2026-07-10T00:05:00.000Z", + type: "workflow", + title: candidate!.title, + content: candidate!.content, + concepts: candidate!.concepts, + files: candidate!.files, + sessionIds: [candidate!.sessionId], + strength: 8, + confidence: candidate!.confidence, + version: 1, + isLatest: true, + sourceCandidateId: candidate!.id, + }); + + registerDurableCandidateFunctions( + sdk as never, + kv as never, + makeProvider(makeSummaryXml("sess_promote", "obs_1")), + ); + + const result: any = await sdk.trigger({ + function_id: "mem::durable-candidates::promote", + payload: { candidateId: candidate!.id }, + }); + + expect(result.success).toBe(true); + expect(result.skipped).toBe(true); + expect(result.memoryId).toBe("mem_existing"); + + const stored = await kv.get("mem:summaries", "sess_promote"); + expect(stored?.durableCandidates?.[0]?.promotedMemoryId).toBe("mem_existing"); + }); + + it("backfill dry-run reports candidate preview without mutating summaries", async () => { + const session: Session = { + id: "sess_backfill_dry", + project: "agentmemory", + cwd: "C:/repo", + startedAt: "2026-07-10T00:00:00.000Z", + endedAt: "2026-07-10T00:10:00.000Z", + status: "completed", + observationCount: 1, + }; + await kv.set("mem:sessions", session.id, session); + await kv.set("mem:obs:sess_backfill_dry", "obs_1", makeObservation(session.id)); + + registerDurableCandidateFunctions( + sdk as never, + kv as never, + makeProvider(makeSummaryXml(session.id, "obs_1")), + ); + + const result: any = await sdk.trigger({ + function_id: "mem::durable-candidates::backfill", + payload: { dryRun: true }, + }); + + expect(result.success).toBe(true); + expect(result.dryRun).toBe(true); + expect(result.candidatePreview.total).toBe(1); + expect(result.wouldMutate).toBe(true); + expect(await kv.get("mem:summaries", session.id)).toBeNull(); + }); + + it("backfill real run respects limit and writes candidates only", async () => { + const sessions: Session[] = [ + { + id: "sess_backfill_real_1", + project: "agentmemory", + cwd: "C:/repo", + startedAt: "2026-07-10T00:00:00.000Z", + endedAt: "2026-07-10T00:10:00.000Z", + status: "completed", + observationCount: 1, + }, + { + id: "sess_backfill_real_2", + project: "agentmemory", + cwd: "C:/repo", + startedAt: "2026-07-10T00:20:00.000Z", + endedAt: "2026-07-10T00:30:00.000Z", + status: "completed", + observationCount: 1, + }, + ]; + for (const session of sessions) { + await kv.set("mem:sessions", session.id, session); + await kv.set( + `mem:obs:${session.id}`, + "obs_1", + makeObservation(session.id), + ); + } + + registerDurableCandidateFunctions( + sdk as never, + kv as never, + makeProvider(makeSummaryXml("sess_backfill_real_1", "obs_1")), + ); + + const result: any = await sdk.trigger({ + function_id: "mem::durable-candidates::backfill", + payload: { dryRun: false, limit: 1 }, + }); + + expect(result.success).toBe(true); + expect(result.processedSessions).toHaveLength(1); + expect( + (await kv.get("mem:summaries", "sess_backfill_real_1")) + ?.durableCandidates?.length, + ).toBe(1); + expect(await kv.get("mem:summaries", "sess_backfill_real_2")).toBeNull(); + expect((await kv.list("mem:memories")).length).toBe(0); + }); + + it("archive/process is idempotent on archivePath + fileHash + sessionId", async () => { + const tempDir = mkdtempSync(join(tmpdir(), "agentmemory-archive-")); + tempDirs.add(tempDir); + const archivePath = join(tempDir, "session.jsonl"); + writeFileSync( + archivePath, + `${JSON.stringify({ + type: "user", + sessionId: "sess_archive", + cwd: "/tmp/agentmemory", + timestamp: "2026-07-10T00:00:00.000Z", + message: { role: "user", content: [{ type: "text", text: "Ship archive processing." }] }, + })}\n`, + "utf-8", + ); + + sdk.registerFunction("mem::replay::import-jsonl", async () => { + const session: Session = { + id: "sess_archive", + project: "agentmemory", + cwd: "/tmp/agentmemory", + startedAt: "2026-07-10T00:00:00.000Z", + endedAt: "2026-07-10T00:01:00.000Z", + status: "completed", + observationCount: 1, + }; + await kv.set("mem:sessions", session.id, session); + await kv.set( + "mem:obs:sess_archive", + "obs_1", + makeObservation(session.id), + ); + return { success: true }; + }); + + registerDurableCandidateFunctions( + sdk as never, + kv as never, + makeProvider(makeSummaryXml("sess_archive", "obs_1")), + ); + + const first: any = await sdk.trigger({ + function_id: "mem::archive::process", + payload: { path: archivePath, allowNonArchivePath: true }, + }); + const second: any = await sdk.trigger({ + function_id: "mem::archive::process", + payload: { path: archivePath, allowNonArchivePath: true }, + }); + + expect(first.success).toBe(true); + expect(first.processed).toHaveLength(1); + expect(first.processed[0].sessionId).toBe("sess_archive"); + expect(first.processed[0].durableCandidateCount).toBe(1); + expect(second.skipped[0].reason).toBe("already_processed"); + }); +}); diff --git a/test/summarize.test.ts b/test/summarize.test.ts index 4723da8781..30fbd4dd0b 100644 --- a/test/summarize.test.ts +++ b/test/summarize.test.ts @@ -11,6 +11,10 @@ vi.mock("../src/state/schema.js", () => ({ observations: (sessionId: string) => `obs:${sessionId}`, audit: "audit", }, + fingerprintId: (prefix: string, content: string) => { + const { createHash } = require("node:crypto"); + return `${prefix}_${createHash("sha256").update(content).digest("hex").slice(0, 16)}`; + }, })); vi.mock("../src/eval/schemas.js", () => ({ @@ -108,16 +112,41 @@ function summaryXml(opts: { decisions?: string[]; files?: string[]; concepts?: string[]; + durableCandidates?: Array<{ + type: string; + title: string; + content: string; + concepts?: string[]; + files?: string[]; + sourceObservationIds?: string[]; + confidence: number; + promotionReason?: string; + }>; }): string { const d = (opts.decisions ?? []).map((x) => `${x}`).join(""); const f = (opts.files ?? []).map((x) => `${x}`).join(""); const c = (opts.concepts ?? []).map((x) => `${x}`).join(""); + const durableCandidates = (opts.durableCandidates ?? []) + .map((candidate) => { + const candidateConcepts = (candidate.concepts ?? []) + .map((value) => `${value}`) + .join(""); + const candidateFiles = (candidate.files ?? []) + .map((value) => `${value}`) + .join(""); + const candidateSourceObservationIds = (candidate.sourceObservationIds ?? []) + .map((value) => `${value}`) + .join(""); + return `${candidate.type}${candidate.title}${candidate.content}${candidateConcepts}${candidateFiles}${candidateSourceObservationIds}${candidate.confidence}${candidate.promotionReason ? `${candidate.promotionReason}` : ""}`; + }) + .join(""); return ` ${opts.title} ${opts.narrative ?? "narrative"} ${d} ${f} ${c} +${durableCandidates} `; } @@ -479,4 +508,56 @@ describe("mem::summarize chunking", () => { expect(result.success).toBe(false); expect(result.error).toBe("parse_failed"); }); + + it("parses durable candidates, drops low-confidence rows, and caps no-evidence confidence", async () => { + const provider = makeProvider([ + summaryXml({ + title: "durables", + narrative: "This session established a stable archive promotion workflow for future reuse.", + durableCandidates: [ + { + type: "workflow", + title: "Explicit promote only", + content: "Archived sessions should create durable candidates first and only write Memory through explicit promote.", + concepts: ["durable-candidates"], + files: ["src/functions/durable-candidates.ts"], + sourceObservationIds: ["obs_0"], + confidence: 0.82, + promotionReason: "Cross-session workflow policy", + }, + { + type: "fact", + title: "Weak idea", + content: "This one should be filtered out.", + confidence: 0.4, + }, + { + type: "fact", + title: "No evidence candidate", + content: "Candidate without evidence ids should remain force-only.", + confidence: 0.95, + }, + ], + }), + ]); + const { handler, kv } = await setupHandler({ + sessionId: "ses_durable", + obsCount: 1, + provider, + }); + + const result: any = await handler({ sessionId: "ses_durable" }); + + expect(result.success).toBe(true); + const stored: any = await kv.get("summaries", "ses_durable"); + expect(stored?.durableCandidates).toHaveLength(2); + expect(stored?.durableCandidates[0].sourceObservationIds).toEqual(["obs_0"]); + expect( + stored?.durableCandidates.find((item: any) => item.title === "No evidence candidate") + ?.confidence, + ).toBe(0.6); + expect( + stored?.durableCandidates.some((item: any) => item.title === "Weak idea"), + ).toBe(false); + }); }); From 6c7ab5cffa6188d083202bd8ce087f93255cd6b0 Mon Sep 17 00:00:00 2001 From: Jenq Shiin Haw <1343528669@qq.com> Date: Sat, 11 Jul 2026 13:34:58 +0800 Subject: [PATCH 3/6] docs(handoff): capture p1 durable candidates state Signed-off-by: Jenq Shiin Haw <1343528669@qq.com> --- ...1-durable-candidates-handoff-2026-07-11.md | 115 ++++++++++++++++++ 1 file changed, 115 insertions(+) create mode 100644 docs/p1-durable-candidates-handoff-2026-07-11.md diff --git a/docs/p1-durable-candidates-handoff-2026-07-11.md b/docs/p1-durable-candidates-handoff-2026-07-11.md new file mode 100644 index 0000000000..4e0c1671b2 --- /dev/null +++ b/docs/p1-durable-candidates-handoff-2026-07-11.md @@ -0,0 +1,115 @@ +# P1 Durable Candidates Handoff (2026-07-11) + +## Code baseline + +- Repo: `C:\Users\86185\dev\agentmemory-src` +- Branch: `feature/p1-durable-candidates` +- HEAD: `ecb2487` (`feat(memory): add durable candidates MVP`) +- Tag: `p1-durable-candidates-mvp` + +Related plugin/runtime baseline outside this repo: + +- Plugin repo: `C:\Users\86185\plugins\agentmemory` +- Tag: `p1-plugin-surface` + +## What landed in P1 + +- `DurableCandidate` MVP schema is present, including `title`, + `promotionReason`, `confidence`, `sourceObservationIds`, and promote + bookkeeping. +- `/agentmemory/archive/process` generates `durableCandidates[]` only. + It does not promote and does not recall. +- Promote is explicit and idempotent. It checks existing + `Memory.sourceCandidateId` and backfills `promotedMemoryId` when needed. +- Backfill supports `dryRun`, `limit`, and candidates-only writes. +- Continuity remains doc-only in P1. See + `docs/continuity-schema-note.md`. + +## Read-only archive inventory snapshot + +Inventory target: + +- `C:\Users\86185\.codex\archived_sessions` + +Inventory guardrails: + +- no `/archive/process` +- no KV writes +- no promote + +Snapshot taken at: + +- `2026-07-11T05:33:32.191Z` + +Results: + +- Archive files: `44` +- Parseable files: `44` +- Bad files: `0` +- Unique session ids: `44` +- Duplicate session ids: `0` +- Observations: `1057` +- Observation counting rule: + `event_msg:user_message + event_msg:task_complete` +- Diagnostic `response_item` total: `14334` + +Expected baseline: + +- Expected: `29 sessions / 965 observations` +- Observed: `44 sessions / 1057 observations` +- Delta: `+15 sessions / +92 observations` + +Conclusion: + +- The current archive corpus snapshot does not reproduce the expected + `29 / 965` baseline. + +## Current live-store comparison + +Live-store snapshot captured during the same inventory run: + +- `12 sessions / 367 observations / 1 memory` + +Archive-vs-live session-id overlap: + +- Already present in live store: `0` +- Missing from live store: `44` +- Live-only sessions: `12` + +Note: + +- A nearby `agentmemory status` check reported `12 / 365 / 1`. The live store + is drifting slightly between snapshots, but in both cases it is clearly not + the expected `29 / 965` corpus. + +## What this means + +- P1 code surface is in place. +- The blocking question is now corpus alignment, not runtime feature coverage. +- We should not run real backfill or promote on the archive corpus until the + `29 / 965` expectation is either reproduced or explicitly revised. + +## Recommended next steps + +1. Confirm which archive subset is supposed to define the `29 / 965` baseline. +2. Confirm which live data directory/runtime should contain that baseline. +3. Re-run the read-only archive inventory until the gap is explained. +4. Only after that, run `POST /agentmemory/durable-candidates/backfill` with + `{"dryRun": true}`. +5. If the dry-run looks sane, run a limited real backfill with `limit=1`. +6. Promote exactly one high-confidence candidate and verify repeated promote + skips correctly. + +## Do not do yet + +- Do not run `/archive/process` on the raw archive corpus. +- Do not run full real backfill. +- Do not auto-promote. +- Do not connect continuity/handoff state into runtime for P1. + +## Repro commands + +```powershell +agentmemory status +node C:\Users\86185\dev\agentmemory\scripts\archive-corpus-inventory.mjs +``` From 285fdf841d8145ceff0682cf6e22ed7afff44624 Mon Sep 17 00:00:00 2001 From: Jenq Shiin Haw <1343528669@qq.com> Date: Sat, 11 Jul 2026 22:54:15 +0800 Subject: [PATCH 4/6] feat(archive): finalize canonical store import Signed-off-by: Jenq Shiin Haw <1343528669@qq.com> --- package.json | 2 +- .../skills/agentmemory-rest-api/REFERENCE.md | 6 +- scripts/build-runtime-assets.mjs | 78 +++++ src/functions/durable-candidates.ts | 280 ++++++++++++++---- src/functions/replay.ts | 51 ++-- src/functions/summarize.ts | 10 +- src/replay/jsonl-parser.ts | 136 ++++++--- src/triggers/api.ts | 31 +- src/types.ts | 26 +- test/durable-candidates.test.ts | 219 +++++++++++++- test/replay-import-key.test.ts | 53 ++++ test/replay.test.ts | 46 +++ 12 files changed, 801 insertions(+), 137 deletions(-) create mode 100644 scripts/build-runtime-assets.mjs diff --git a/package.json b/package.json index 4e2bc8c4c3..7cdbba409b 100644 --- a/package.json +++ b/package.json @@ -17,7 +17,7 @@ "agentmemory": "dist/cli.mjs" }, "scripts": { - "build": "tsdown && (cp iii-config.yaml dist/ 2>/dev/null || true) && (cp iii-config.docker.yaml dist/ 2>/dev/null || true) && (cp docker-compose.yml dist/ 2>/dev/null || true) && (cp .env.example dist/ 2>/dev/null || true) && mkdir -p dist/viewer && cp src/viewer/index.html dist/viewer/ && cp src/viewer/favicon.svg dist/viewer/", + "build": "tsdown && node scripts/build-runtime-assets.mjs", "dev": "tsx src/index.ts", "start": "node dist/cli.mjs", "migrate": "node dist/functions/migrate.js", diff --git a/plugin/skills/agentmemory-rest-api/REFERENCE.md b/plugin/skills/agentmemory-rest-api/REFERENCE.md index d36171defe..ed8bbf7c92 100644 --- a/plugin/skills/agentmemory-rest-api/REFERENCE.md +++ b/plugin/skills/agentmemory-rest-api/REFERENCE.md @@ -5,7 +5,7 @@ Generated from `src/triggers/api.ts`. Do not edit the block below by hand; run ` The REST API is the primary surface. All paths are under `http://localhost:3111` (override with `--port`). When `AGENTMEMORY_SECRET` is set, send `Authorization: Bearer $AGENTMEMORY_SECRET`; localhost is otherwise open. -117 registered endpoints: +121 registered endpoints: | Method | Path | | --- | --- | @@ -13,6 +13,7 @@ The REST API is the primary surface. All paths are under `http://localhost:3111` | POST | `/agentmemory/actions/edges` | | GET | `/agentmemory/actions/get` | | POST | `/agentmemory/actions/update` | +| POST | `/agentmemory/archive/process` | | GET | `/agentmemory/audit` | | POST | `/agentmemory/auto-forget` | | GET | `/agentmemory/branch/detect` | @@ -35,6 +36,9 @@ The REST API is the primary surface. All paths are under `http://localhost:3111` | POST | `/agentmemory/diagnostics` | | GET | `/agentmemory/diagnostics/followup` | | POST | `/agentmemory/diagnostics/heal` | +| GET | `/agentmemory/durable-candidates` | +| POST | `/agentmemory/durable-candidates/backfill` | +| POST | `/agentmemory/durable-candidates/promote` | | POST | `/agentmemory/enrich` | | POST | `/agentmemory/evict` | | POST | `/agentmemory/evolve` | diff --git a/scripts/build-runtime-assets.mjs b/scripts/build-runtime-assets.mjs new file mode 100644 index 0000000000..b87b5391b4 --- /dev/null +++ b/scripts/build-runtime-assets.mjs @@ -0,0 +1,78 @@ +import { createHash } from "node:crypto"; +import { cp, mkdir, readFile, stat, writeFile } from "node:fs/promises"; +import { fileURLToPath } from "node:url"; +import { basename, dirname, join, relative } from "node:path"; +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; + +const execFileAsync = promisify(execFile); +const root = join(dirname(fileURLToPath(import.meta.url)), ".."); +const dist = join(root, "dist"); +const assets = [ + ["iii-config.yaml", "iii-config.yaml"], + ["iii-config.docker.yaml", "iii-config.docker.yaml"], + ["docker-compose.yml", "docker-compose.yml"], + [".env.example", ".env.example"], + ["src/viewer/index.html", "viewer/index.html"], + ["src/viewer/favicon.svg", "viewer/favicon.svg"], +]; + +async function copyIfPresent(sourceRelative, destinationRelative) { + const source = join(root, sourceRelative); + try { + await stat(source); + } catch { + return; + } + const destination = join(dist, destinationRelative); + await mkdir(dirname(destination), { recursive: true }); + await cp(source, destination, { force: true, recursive: true }); +} + +async function sourceCommit() { + try { + const { stdout } = await execFileAsync("git", ["rev-parse", "HEAD"], { cwd: root }); + return stdout.trim(); + } catch { + return "unknown"; + } +} + +async function sourceDirty() { + try { + const { stdout } = await execFileAsync("git", ["status", "--porcelain"], { cwd: root }); + return stdout.trim().length > 0; + } catch { + return null; + } +} + +async function hashRuntimeFiles() { + const hash = createHash("sha256"); + const files = ["index.mjs", "cli.mjs", "iii-config.yaml", "viewer/index.html"]; + for (const file of files) { + const path = join(dist, file); + try { + const content = await readFile(path); + hash.update(relative(dist, path)); + hash.update(content); + } catch { + hash.update(`missing:${basename(path)}`); + } + } + return hash.digest("hex"); +} + +for (const [source, destination] of assets) { + await copyIfPresent(source, destination); +} + +const packageJson = JSON.parse(await readFile(join(root, "package.json"), "utf8")); +const buildInfo = { + version: packageJson.version, + sourceCommit: await sourceCommit(), + sourceDirty: await sourceDirty(), + builtAt: new Date().toISOString(), + artifactHash: await hashRuntimeFiles(), +}; +await writeFile(join(dist, "build-info.json"), `${JSON.stringify(buildInfo, null, 2)}\n`, "utf8"); diff --git a/src/functions/durable-candidates.ts b/src/functions/durable-candidates.ts index 299512913e..85439af9d2 100644 --- a/src/functions/durable-candidates.ts +++ b/src/functions/durable-candidates.ts @@ -1,6 +1,6 @@ import { createHash } from "node:crypto"; import { homedir } from "node:os"; -import { lstat, readFile } from "node:fs/promises"; +import { lstat, readFile, realpath } from "node:fs/promises"; import { join, resolve } from "node:path"; import type { ISdk } from "iii-sdk"; import type { @@ -16,7 +16,7 @@ import { bucketCandidateConfidence, updateSummaryCandidate, } from "./durable-candidate-utils.js"; -import { summarizeSession } from "./summarize.js"; +import { DURABLE_CANDIDATE_SCHEMA_VERSION, summarizeSession } from "./summarize.js"; import { MAX_FILES_DEFAULT, MAX_FILES_UPPER_BOUND, @@ -99,6 +99,7 @@ function collectDurableCandidates( async function readArchiveTargets( rawPath: string, + archiveRoot: string, maxFiles?: number, ): Promise< | { @@ -116,6 +117,10 @@ async function readArchiveTargets( ? join(homedir(), rawPath.slice(1)) : rawPath; const abs = resolve(expanded); + const allowedRoot = resolve(archiveRoot); + if (!isSubPath(abs, allowedRoot)) { + return { success: false, error: `archive path must live under ${allowedRoot}` }; + } if (isSensitive(abs)) { return { success: false, error: "refusing to process sensitive-looking path" }; } @@ -130,6 +135,17 @@ async function readArchiveTargets( return { success: false, error: "path not found" }; } + let realAllowedRoot: string; + let realPath: string; + try { + [realAllowedRoot, realPath] = await Promise.all([realpath(allowedRoot), realpath(abs)]); + } catch { + return { success: false, error: "archive path cannot be resolved safely" }; + } + if (!isSubPath(realPath, realAllowedRoot)) { + return { success: false, error: "archive path resolves outside the allowed root" }; + } + const effectiveMaxFiles = maxFiles === undefined ? MAX_FILES_DEFAULT @@ -137,6 +153,19 @@ async function readArchiveTargets( if (stat.isDirectory()) { const found = await findJsonlFiles(abs, effectiveMaxFiles); + const unsafeFile = await Promise.all( + found.files.map(async (file) => { + if (await isSymlink(file)) return true; + try { + return !isSubPath(await realpath(file), realAllowedRoot); + } catch { + return true; + } + }), + ); + if (unsafeFile.some(Boolean)) { + return { success: false, error: "archive contains a symlink or path outside the allowed root" }; + } return { success: true, files: found.files, @@ -167,7 +196,9 @@ export function registerDurableCandidateFunctions( sdk: ISdk, kv: StateKV, provider: MemoryProvider, + options: { archiveRoot?: string } = {}, ): void { + const archiveRoot = options.archiveRoot || ARCHIVE_ROOT_DEFAULT; sdk.registerFunction( "mem::durable-candidates::list", async (data: { @@ -228,6 +259,8 @@ export function registerDurableCandidateFunctions( candidateId?: string; dryRun?: boolean; force?: boolean; + forceReason?: string; + promotedBy?: string; }) => { const candidateId = typeof data?.candidateId === "string" ? data.candidateId.trim() : ""; @@ -244,6 +277,10 @@ export function registerDurableCandidateFunctions( const { summary, candidate } = found; const dryRun = data?.dryRun === true; const force = data?.force === true; + const forceReason = + typeof data?.forceReason === "string" ? data.forceReason.trim() : ""; + const promotedBy = + typeof data?.promotedBy === "string" ? data.promotedBy.trim() : ""; const requiresForce = candidate.confidence < DURABLE_PROMOTE_MIN_CONFIDENCE || candidate.sourceObservationIds.length === 0; @@ -304,6 +341,15 @@ export function registerDurableCandidateFunctions( }; } + if (requiresForce && (!forceReason || !promotedBy)) { + return { + success: false, + error: "force_metadata_required", + requiresForce: true, + required: ["forceReason", "promotedBy"], + }; + } + if (dryRun) { return { success: true, @@ -344,8 +390,14 @@ export function registerDurableCandidateFunctions( ...candidate, promotedMemoryId: rememberResult.memory.id, promotedAt, + ...(requiresForce && { forceReason, promotedBy }), }); await kv.set(KV.summaries, summary.sessionId, updatedSummary); + await safeAudit(kv, "remember", "mem::durable-candidates::promote", [candidate.sessionId], { + candidateId: candidate.id, + force: requiresForce, + ...(requiresForce && { forceReason, promotedBy }), + }); return { success: true, @@ -377,9 +429,17 @@ export function registerDurableCandidateFunctions( const sessions = await kv.list(KV.sessions); const summaries = await kv.list(KV.summaries); const memories = await kv.list(KV.memories); + const archiveImports = await kv.list(KV.archiveImports); const summaryBySessionId = new Map( summaries.map((summary) => [summary.sessionId, summary] as const), ); + const archiveBySessionId = new Map(); + for (const record of archiveImports) { + const previous = archiveBySessionId.get(record.sessionId); + if (!previous || record.updatedAt > previous.updatedAt) { + archiveBySessionId.set(record.sessionId, record); + } + } const scope = { sessions: sessions.length, @@ -395,6 +455,11 @@ export function registerDurableCandidateFunctions( sessionId: string; project: string; observationCount: number; + reason: + | "summary_failed" + | "candidate_schema_outdated" + | "live_session_without_archive" + | "candidate_schema_missing"; }> = []; const skippedSessions: Array<{ sessionId: string; reason: string }> = []; @@ -419,17 +484,40 @@ export function registerDurableCandidateFunctions( continue; } const existingSummary = summaryBySessionId.get(session.id); - if ((existingSummary?.durableCandidates?.length || 0) > 0) { + const archiveImport = archiveBySessionId.get(session.id); + if ( + existingSummary?.durableCandidatesVersion === DURABLE_CANDIDATE_SCHEMA_VERSION && + (existingSummary.durableCandidates?.length || 0) > 0 + ) { skippedSessions.push({ sessionId: session.id, reason: "already_has_candidates", }); continue; } + if ( + existingSummary?.durableCandidatesVersion === DURABLE_CANDIDATE_SCHEMA_VERSION && + Array.isArray(existingSummary.durableCandidates) + ) { + skippedSessions.push({ + sessionId: session.id, + reason: "candidate_schema_current", + }); + continue; + } + const reason = + archiveImport?.status === "failed" && archiveImport.failureStage === "summary" + ? "summary_failed" + : existingSummary?.durableCandidatesVersion !== undefined + ? "candidate_schema_outdated" + : !archiveImport + ? "live_session_without_archive" + : "candidate_schema_missing"; eligibleSessions.push({ sessionId: session.id, project: session.project, observationCount: observations.length, + reason, }); } @@ -508,7 +596,6 @@ export function registerDurableCandidateFunctions( path?: string; maxFiles?: number; force?: boolean; - allowNonArchivePath?: boolean; } = {}) => { const rawPath = typeof data.path === "string" && data.path.trim() @@ -518,19 +605,11 @@ export function registerDurableCandidateFunctions( data.maxFiles === undefined ? undefined : Math.min(data.maxFiles, MAX_FILES_UPPER_BOUND); - const allowNonArchivePath = data.allowNonArchivePath === true; - const fileSelection = await readArchiveTargets(rawPath, maxFiles); + const fileSelection = await readArchiveTargets(rawPath, archiveRoot, maxFiles); if (!fileSelection.success) { return { success: false, error: fileSelection.error }; } - if (!allowNonArchivePath && !isSubPath(fileSelection.rootPath, ARCHIVE_ROOT_DEFAULT)) { - return { - success: false, - error: `archive path must live under ${ARCHIVE_ROOT_DEFAULT}`, - }; - } - const processed: Array<{ archivePath: string; sessionId: string; @@ -557,82 +636,183 @@ export function registerDurableCandidateFunctions( } const parsed = parseJsonlText(text); - if (parsed.observations.length === 0) { - skipped.push({ - archivePath: file, - sessionId: parsed.sessionId, - reason: "no_observations", - }); - continue; - } const fileHash = hashText(text); const idempotencyKey = fingerprintId( "arch", - `${file}\n${fileHash}\n${parsed.sessionId}`, + `${parsed.sessionId}\n${fileHash}`, ); const existingImport = await kv.get( KV.archiveImports, idempotencyKey, ); - if (existingImport && data.force !== true) { + if ( + existingImport && + (existingImport.status === "completed" || existingImport.summaryCreated === true) + ) { skipped.push({ archivePath: file, sessionId: parsed.sessionId, - reason: "already_processed", + reason: "already_completed", }); continue; } - const importResult = (await sdk.trigger({ - function_id: "mem::replay::import-jsonl", - payload: { path: file, maxFiles: 1 }, - })) as { success?: boolean; error?: string }; - if (!importResult?.success) { - skipped.push({ - archivePath: file, - sessionId: parsed.sessionId, - reason: importResult?.error || "import_failed", - }); - continue; - } - - const session = await kv.get(KV.sessions, parsed.sessionId); - if (session) { - const observationCount = ( + const startedAt = new Date().toISOString(); + let record: ArchiveImportRecord = { + ...existingImport, + id: idempotencyKey, + archivePath: file, + fileHash, + sessionId: parsed.sessionId, + status: existingImport?.status || "discovered", + createdAt: existingImport?.createdAt || startedAt, + updatedAt: startedAt, + attempts: (existingImport?.attempts || 0) + 1, + parsedObservationCount: parsed.observations.length, + importedObservationCount: + existingImport?.importedObservationCount || 0, + source: "archive-process", + }; + await kv.set(KV.archiveImports, idempotencyKey, record); + + const observationsAlreadyImported = + record.status === "observations_imported" || + record.status === "summarizing" || + (record.status === "failed" && record.failureStage === "summary"); + if (!observationsAlreadyImported) { + record = { + ...record, + status: "importing_observations", + failureStage: undefined, + lastError: undefined, + updatedAt: new Date().toISOString(), + }; + await kv.set(KV.archiveImports, idempotencyKey, record); + + const importResult = (await sdk.trigger({ + function_id: "mem::replay::import-jsonl", + payload: { path: file, maxFiles: 1 }, + })) as { success?: boolean; error?: string }; + if (!importResult?.success) { + record = { + ...record, + status: "failed", + failureStage: "observations", + lastError: importResult?.error || "import_failed", + updatedAt: new Date().toISOString(), + }; + await kv.set(KV.archiveImports, idempotencyKey, record); + skipped.push({ + archivePath: file, + sessionId: parsed.sessionId, + reason: record.lastError || "import_failed", + }); + continue; + } + + const session = await kv.get(KV.sessions, parsed.sessionId); + if (!session) { + record = { + ...record, + status: "failed", + failureStage: "observations", + lastError: "session_not_found_after_import", + updatedAt: new Date().toISOString(), + }; + await kv.set(KV.archiveImports, idempotencyKey, record); + skipped.push({ + archivePath: file, + sessionId: parsed.sessionId, + reason: record.lastError || "session_not_found_after_import", + }); + continue; + } + + session.observationCount = ( await kv.list(KV.observations(parsed.sessionId)) ).length; - session.observationCount = observationCount; session.status = "completed"; session.endedAt = parsed.endedAt || session.endedAt; await kv.set(KV.sessions, parsed.sessionId, session); + + record = { + ...record, + status: "observations_imported", + importedObservationCount: parsed.observations.length, + failureStage: undefined, + lastError: undefined, + updatedAt: new Date().toISOString(), + }; + await kv.set(KV.archiveImports, idempotencyKey, record); } + if (parsed.observations.length === 0) { + const completedAt = new Date().toISOString(); + record = { + ...record, + status: "completed", + durableCandidateCount: 0, + summaryCreated: false, + completedAt, + failureStage: undefined, + lastError: undefined, + updatedAt: completedAt, + }; + await kv.set(KV.archiveImports, idempotencyKey, record); + processed.push({ + archivePath: file, + sessionId: parsed.sessionId, + durableCandidateCount: 0, + idempotencyKey, + fileHash, + }); + continue; + } + + record = { + ...record, + status: "summarizing", + failureStage: undefined, + lastError: undefined, + updatedAt: new Date().toISOString(), + }; + await kv.set(KV.archiveImports, idempotencyKey, record); const summaryResult = await summarizeSession(kv, provider, { sessionId: parsed.sessionId, persistSummary: true, }); if (!summaryResult.success) { + record = { + ...record, + status: "failed", + failureStage: "summary", + lastError: summaryResult.error, + updatedAt: new Date().toISOString(), + }; + await kv.set(KV.archiveImports, idempotencyKey, record); skipped.push({ archivePath: file, sessionId: parsed.sessionId, - reason: summaryResult.error, + reason: record.lastError || "summary_failed", }); continue; } const durableCandidateCount = summaryResult.summary.durableCandidates?.length || 0; - await kv.set(KV.archiveImports, idempotencyKey, { - id: idempotencyKey, - archivePath: file, - fileHash, - sessionId: parsed.sessionId, - processedAt: new Date().toISOString(), + const completedAt = new Date().toISOString(); + record = { + ...record, + status: "completed", durableCandidateCount, summaryCreated: true, - source: "archive-process", - }); + completedAt, + failureStage: undefined, + lastError: undefined, + updatedAt: completedAt, + }; + await kv.set(KV.archiveImports, idempotencyKey, record); processed.push({ archivePath: file, sessionId: parsed.sessionId, diff --git a/src/functions/replay.ts b/src/functions/replay.ts index 79b40311d1..2e16f96236 100644 --- a/src/functions/replay.ts +++ b/src/functions/replay.ts @@ -382,7 +382,6 @@ export function registerReplayFunctions(sdk: ISdk, kv: StateKV): void { } const parsed = parseJsonlText(text, generateId("sess")); - if (parsed.observations.length === 0) continue; const firstPromptObs = parsed.observations.find( (o) => typeof o.userPrompt === "string" && o.userPrompt.trim().length > 0, @@ -392,19 +391,19 @@ export function registerReplayFunctions(sdk: ISdk, kv: StateKV): void { : undefined; const existing = await kv.get(KV.sessions, parsed.sessionId); + let session: Session; if (existing) { - existing.observationCount = - (existing.observationCount || 0) + parsed.observations.length; + session = existing; if (parsed.endedAt > (existing.endedAt || "")) { - existing.endedAt = parsed.endedAt; + session.endedAt = parsed.endedAt; } - if (existing.status === "active") existing.status = "completed"; - const existingTags = existing.tags || []; + if (session.status === "active") session.status = "completed"; + const existingTags = session.tags || []; if (!existingTags.includes("jsonl-import")) { - existing.tags = [...existingTags, "jsonl-import"]; + session.tags = [...existingTags, "jsonl-import"]; } - if (!existing.firstPrompt && firstPrompt) { - existing.firstPrompt = firstPrompt; + if (!session.firstPrompt && firstPrompt) { + session.firstPrompt = firstPrompt; } // #775: re-key on parsed.sessionId, not existing.id. Older // session rows may be missing the `id` field; existing.id @@ -415,25 +414,24 @@ export function registerReplayFunctions(sdk: ISdk, kv: StateKV): void { // legacy row killed the entire batch. parsed.sessionId is // always populated (parseJsonlText has a three-level // fallback) and is what we just used to read the row. - if (!existing.id) existing.id = parsed.sessionId; - await kv.set(KV.sessions, parsed.sessionId, existing); + if (!session.id) session.id = parsed.sessionId; } else { - const session: Session = { + session = { id: parsed.sessionId, project: parsed.project, cwd: parsed.cwd, startedAt: parsed.startedAt, endedAt: parsed.endedAt, status: "completed", - observationCount: parsed.observations.length, + observationCount: 0, tags: ["jsonl-import"], firstPrompt, }; - await kv.set(KV.sessions, session.id, session); } const searchIndex = getSearchIndex(); const compressed: CompressedObservation[] = []; + const observationsBefore = await kv.list(KV.observations(parsed.sessionId)); await Promise.all( parsed.observations.map(async (obs) => { const synthetic = buildSyntheticCompression(obs); @@ -442,17 +440,24 @@ export function registerReplayFunctions(sdk: ISdk, kv: StateKV): void { searchIndex.add(synthetic); }), ); - observationCount += parsed.observations.length; + const storedObservations = await kv.list(KV.observations(parsed.sessionId)); + const addedObservations = Math.max(0, storedObservations.length - observationsBefore.length); + session.observationCount = storedObservations.length; + await kv.set(KV.sessions, parsed.sessionId, session); + + observationCount += addedObservations; sessionIds.push(parsed.sessionId); - await deriveCrystalAndLessons( - kv, - parsed.sessionId, - parsed.project, - parsed.observations, - compressed, - firstPrompt, - ); + if (addedObservations > 0) { + await deriveCrystalAndLessons( + kv, + parsed.sessionId, + parsed.project, + parsed.observations, + compressed, + firstPrompt, + ); + } } await safeAudit(kv, "import", "mem::replay::import-jsonl", sessionIds, { diff --git a/src/functions/summarize.ts b/src/functions/summarize.ts index e7ac808566..7f2d9a9da4 100644 --- a/src/functions/summarize.ts +++ b/src/functions/summarize.ts @@ -36,6 +36,7 @@ const CHUNK_CONCURRENCY_DEFAULT = 6; // Bail on the merged summary if more than this fraction of chunks fail // to parse — a half-blind narrative is worse than a clean error. const MAX_SKIP_RATIO = 0.5; +export const DURABLE_CANDIDATE_SCHEMA_VERSION = 1; function getChunkSize(): number { const raw = process.env.SUMMARIZE_CHUNK_SIZE; @@ -367,7 +368,12 @@ export async function summarizeSession( } const qualityScore = scoreSummary(summaryForValidation); - let summaryToStore = summary; + const durableCandidatesGeneratedAt = new Date().toISOString(); + let summaryToStore: SessionSummary = { + ...summary, + durableCandidatesVersion: DURABLE_CANDIDATE_SCHEMA_VERSION, + durableCandidatesGeneratedAt, + }; if (persistSummary) { if (mergeDurableCandidatesOnly) { @@ -377,6 +383,8 @@ export async function summarizeSession( ...existingSummary, observationCount: summary.observationCount, durableCandidates: summary.durableCandidates, + durableCandidatesVersion: DURABLE_CANDIDATE_SCHEMA_VERSION, + durableCandidatesGeneratedAt, }; } } diff --git a/src/replay/jsonl-parser.ts b/src/replay/jsonl-parser.ts index 5060c34513..b6bd856f29 100644 --- a/src/replay/jsonl-parser.ts +++ b/src/replay/jsonl-parser.ts @@ -1,5 +1,6 @@ +import { createHash } from "node:crypto"; import type { HookType, RawObservation } from "../types.js"; -import { generateId } from "../state/schema.js"; +import { fingerprintId } from "../state/schema.js"; interface JsonlEntry { type?: string; @@ -7,6 +8,7 @@ interface JsonlEntry { sessionId?: string; timestamp?: string; cwd?: string; + payload?: Record; message?: { role?: string; content?: unknown; @@ -15,6 +17,11 @@ interface JsonlEntry { [k: string]: unknown; } +interface ParsedEntry { + entry: JsonlEntry; + line: number; +} + export interface ParsedTranscript { sessionId: string; project: string; @@ -26,7 +33,7 @@ export interface ParsedTranscript { function deriveProject(cwd: string): string { if (!cwd) return "unknown"; - const parts = cwd.split("/").filter(Boolean); + const parts = cwd.split(/[\\/]/).filter(Boolean); return parts[parts.length - 1] || "unknown"; } @@ -78,15 +85,32 @@ function extractToolResults(content: unknown): Array<{ toolUseId: string; output return out; } +function stableObservationId( + sessionId: string, + line: number, + kind: string, + payload: unknown, +): string { + const payloadHash = createHash("sha256").update(JSON.stringify(payload)).digest("hex"); + return fingerprintId("obs", `${sessionId}\n${line}\n${kind}\n${payloadHash}`); +} + +function payloadString(payload: Record, key: string): string { + const value = payload[key]; + return typeof value === "string" ? value : ""; +} + export function parseJsonlText(text: string, fallbackSessionId?: string): ParsedTranscript { - const lines = text.split("\n").filter((l) => l.trim().length > 0); - const entries: JsonlEntry[] = []; - for (const line of lines) { + const entries: ParsedEntry[] = []; + for (const [index, line] of text.split("\n").entries()) { + if (!line.trim()) continue; try { const parsed = JSON.parse(line); - if (parsed && typeof parsed === "object") entries.push(parsed as JsonlEntry); + if (parsed && typeof parsed === "object") { + entries.push({ entry: parsed as JsonlEntry, line: index + 1 }); + } } catch { - // skip malformed lines + // Malformed transcript lines cannot produce a trustworthy observation. } } @@ -95,25 +119,63 @@ export function parseJsonlText(text: string, fallbackSessionId?: string): Parsed let firstTs = ""; let lastTs = ""; - const observations: RawObservation[] = []; - - for (const entry of entries) { + for (const { entry } of entries) { if (entry.sessionId && !sessionId) sessionId = entry.sessionId; if (entry.cwd && !cwd) cwd = entry.cwd; - const ts = entry.timestamp || new Date().toISOString(); - if (!firstTs) firstTs = ts; - lastTs = ts; + if (entry.type === "session_meta" && entry.payload) { + if (!sessionId && typeof entry.payload.id === "string") sessionId = entry.payload.id; + if (!cwd && typeof entry.payload.cwd === "string") cwd = entry.payload.cwd; + if (!firstTs && typeof entry.payload.timestamp === "string") firstTs = entry.payload.timestamp; + } + if (entry.timestamp) { + if (!firstTs) firstTs = entry.timestamp; + lastTs = entry.timestamp; + } + } + + const effectiveSessionId = sessionId || fallbackSessionId || fingerprintId("sess", text); + const observations: RawObservation[] = []; + const nowIso = new Date().toISOString(); + for (const { entry, line } of entries) { + const ts = entry.timestamp || nowIso; const role = entry.message?.role; const content = entry.message?.content; + if (entry.type === "event_msg" && entry.payload?.type === "user_message") { + const prompt = payloadString(entry.payload, "message"); + if (prompt.trim()) { + observations.push({ + id: stableObservationId(effectiveSessionId, line, "codex:user_message", entry.payload), + sessionId: effectiveSessionId, + timestamp: ts, + hookType: "prompt_submit", + userPrompt: prompt, + raw: entry, + }); + } + continue; + } + + if (entry.type === "event_msg" && entry.payload?.type === "task_complete") { + observations.push({ + id: stableObservationId(effectiveSessionId, line, "codex:task_complete", entry.payload), + sessionId: effectiveSessionId, + timestamp: ts, + hookType: "task_completed", + assistantResponse: payloadString(entry.payload, "last_agent_message") || undefined, + raw: entry, + }); + continue; + } + if (entry.type === "user" && role === "user") { const toolResults = extractToolResults(content); if (toolResults.length > 0) { - for (const result of toolResults) { + for (const [resultIndex, result] of toolResults.entries()) { observations.push({ - id: generateId("obs"), - sessionId: sessionId || "imported", + id: stableObservationId(effectiveSessionId, line, `claude:tool_result:${resultIndex}`, result), + sessionId: effectiveSessionId, timestamp: ts, hookType: (result.isError ? "post_tool_failure" : "post_tool_use") as HookType, toolName: undefined, @@ -123,59 +185,51 @@ export function parseJsonlText(text: string, fallbackSessionId?: string): Parsed }); } } else { - const text = toText(content); - if (text.trim().length > 0) { + const prompt = toText(content); + if (prompt.trim()) { observations.push({ - id: generateId("obs"), - sessionId: sessionId || "imported", + id: stableObservationId(effectiveSessionId, line, "claude:user", entry), + sessionId: effectiveSessionId, timestamp: ts, - hookType: "prompt_submit" as HookType, - userPrompt: text, + hookType: "prompt_submit", + userPrompt: prompt, raw: entry, }); } } } else if (entry.type === "assistant" && role === "assistant") { - const text = toText(content); + const response = toText(content); const tools = extractToolUses(content); - if (text.trim().length > 0) { + if (response.trim()) { observations.push({ - id: generateId("obs"), - sessionId: sessionId || "imported", + id: stableObservationId(effectiveSessionId, line, "claude:assistant", entry), + sessionId: effectiveSessionId, timestamp: ts, - hookType: "stop" as HookType, - assistantResponse: text, + hookType: "stop", + assistantResponse: response, raw: entry, }); } - for (const tool of tools) { + for (const [toolIndex, tool] of tools.entries()) { observations.push({ - id: generateId("obs"), - sessionId: sessionId || "imported", + id: stableObservationId(effectiveSessionId, line, `claude:tool_use:${toolIndex}`, tool), + sessionId: effectiveSessionId, timestamp: ts, - hookType: "pre_tool_use" as HookType, + hookType: "pre_tool_use", toolName: tool.name, toolInput: tool.input, raw: { toolUseId: tool.id, entry }, }); } - } else if (entry.type === "summary" || entry.type === "system") { - // ignore meta entries } } - const effectiveSessionId = sessionId || fallbackSessionId || generateId("sess"); - for (const obs of observations) { - if (obs.sessionId === "imported") obs.sessionId = effectiveSessionId; - } - - const nowIso = new Date().toISOString(); return { sessionId: effectiveSessionId, project: deriveProject(cwd), cwd: cwd || process.cwd(), startedAt: firstTs || nowIso, - endedAt: lastTs || nowIso, + endedAt: lastTs || firstTs || nowIso, observations, }; } diff --git a/src/triggers/api.ts b/src/triggers/api.ts index 802bb4aaf9..2143b237fb 100644 --- a/src/triggers/api.ts +++ b/src/triggers/api.ts @@ -563,7 +563,6 @@ export function registerApiTriggers( path?: string; maxFiles?: number; force?: boolean; - allowNonArchivePath?: boolean; }>, ): Promise => { const authErr = checkAuth(req, secret); @@ -573,7 +572,6 @@ export function registerApiTriggers( path?: string; maxFiles?: number; force?: boolean; - allowNonArchivePath?: boolean; } = {}; if (body.path !== undefined) { if (typeof body.path !== "string" || body.path.trim().length === 0) { @@ -602,15 +600,6 @@ export function registerApiTriggers( } payload.force = body.force; } - if (body.allowNonArchivePath !== undefined) { - if (typeof body.allowNonArchivePath !== "boolean") { - return { - status_code: 400, - body: { error: "allowNonArchivePath must be a boolean" }, - }; - } - payload.allowNonArchivePath = body.allowNonArchivePath; - } const result = await sdk.trigger({ function_id: "mem::archive::process", payload, @@ -1191,7 +1180,13 @@ export function registerApiTriggers( sdk.registerFunction("api::durable-candidates::promote", async ( - req: ApiRequest<{ candidateId?: string; dryRun?: boolean; force?: boolean }>, + req: ApiRequest<{ + candidateId?: string; + dryRun?: boolean; + force?: boolean; + forceReason?: string; + promotedBy?: string; + }>, ): Promise => { const authErr = checkAuth(req, secret); if (authErr) return authErr; @@ -1206,12 +1201,24 @@ export function registerApiTriggers( if (body.force !== undefined && typeof body.force !== "boolean") { return { status_code: 400, body: { error: "force must be a boolean" } }; } + const forceReason = + body.forceReason === undefined ? undefined : asNonEmptyString(body.forceReason); + if (body.forceReason !== undefined && !forceReason) { + return { status_code: 400, body: { error: "forceReason must be a non-empty string" } }; + } + const promotedBy = + body.promotedBy === undefined ? undefined : asNonEmptyString(body.promotedBy); + if (body.promotedBy !== undefined && !promotedBy) { + return { status_code: 400, body: { error: "promotedBy must be a non-empty string" } }; + } const result = await sdk.trigger({ function_id: "mem::durable-candidates::promote", payload: { candidateId, ...(body.dryRun !== undefined && { dryRun: body.dryRun }), ...(body.force !== undefined && { force: body.force }), + ...(forceReason !== undefined && { forceReason }), + ...(promotedBy !== undefined && { promotedBy }), }, }); return { status_code: 200, body: result }; diff --git a/src/types.ts b/src/types.ts index 9f79e046a3..4304061ce8 100644 --- a/src/types.ts +++ b/src/types.ts @@ -103,6 +103,8 @@ export interface DurableCandidate { createdAt: string; promotedMemoryId?: string; promotedAt?: string; + promotedBy?: string; + forceReason?: string; } export interface Memory { @@ -142,16 +144,34 @@ export interface SessionSummary { concepts: string[]; observationCount: number; durableCandidates?: DurableCandidate[]; + durableCandidatesVersion?: number; + durableCandidatesGeneratedAt?: string; } +export type ArchiveImportStatus = + | "discovered" + | "importing_observations" + | "observations_imported" + | "summarizing" + | "completed" + | "failed"; + export interface ArchiveImportRecord { id: string; archivePath: string; fileHash: string; sessionId: string; - processedAt: string; - durableCandidateCount: number; - summaryCreated: boolean; + status: ArchiveImportStatus; + createdAt: string; + updatedAt: string; + attempts: number; + parsedObservationCount: number; + importedObservationCount: number; + durableCandidateCount?: number; + summaryCreated?: boolean; + completedAt?: string; + failureStage?: "observations" | "summary"; + lastError?: string; source: "archive-process"; } diff --git a/test/durable-candidates.test.ts b/test/durable-candidates.test.ts index ef2674cd63..34da9609d0 100644 --- a/test/durable-candidates.test.ts +++ b/test/durable-candidates.test.ts @@ -1,5 +1,5 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { mkdirSync, mkdtempSync, renameSync, rmSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { tmpdir } from "node:os"; @@ -267,7 +267,7 @@ describe("durable candidates lifecycle", () => { expect((await kv.list("mem:memories")).length).toBe(0); }); - it("archive/process is idempotent on archivePath + fileHash + sessionId", async () => { + it("archive/process is idempotent on sessionId + fileHash", async () => { const tempDir = mkdtempSync(join(tmpdir(), "agentmemory-archive-")); tempDirs.add(tempDir); const archivePath = join(tempDir, "session.jsonl"); @@ -306,21 +306,230 @@ describe("durable candidates lifecycle", () => { sdk as never, kv as never, makeProvider(makeSummaryXml("sess_archive", "obs_1")), + { archiveRoot: tempDir }, ); const first: any = await sdk.trigger({ function_id: "mem::archive::process", - payload: { path: archivePath, allowNonArchivePath: true }, + payload: { path: archivePath }, }); + const movedDir = join(tempDir, "moved"); + mkdirSync(movedDir); + const movedPath = join(movedDir, "session.jsonl"); + renameSync(archivePath, movedPath); const second: any = await sdk.trigger({ function_id: "mem::archive::process", - payload: { path: archivePath, allowNonArchivePath: true }, + payload: { path: movedPath, force: true }, }); expect(first.success).toBe(true); expect(first.processed).toHaveLength(1); expect(first.processed[0].sessionId).toBe("sess_archive"); expect(first.processed[0].durableCandidateCount).toBe(1); - expect(second.skipped[0].reason).toBe("already_processed"); + expect(second.skipped[0].reason).toBe("already_completed"); + const ledger = await kv.get("mem:archive-imports", first.processed[0].idempotencyKey); + expect(ledger.status).toBe("completed"); + expect(ledger.attempts).toBe(1); + }); + + it("completes a zero-observation archive without attempting a summary", async () => { + const tempDir = mkdtempSync(join(tmpdir(), "agentmemory-archive-empty-")); + tempDirs.add(tempDir); + const archivePath = join(tempDir, "empty.jsonl"); + writeFileSync( + archivePath, + [ + JSON.stringify({ + type: "session_meta", + payload: { + id: "sess_archive_empty", + cwd: "C:\\work\\agentmemory", + timestamp: "2026-07-10T00:00:00.000Z", + }, + }), + JSON.stringify({ type: "event_msg", payload: { type: "task_started" } }), + ].join("\n") + "\n", + "utf-8", + ); + + sdk.registerFunction("mem::replay::import-jsonl", async () => { + await kv.set("mem:sessions", "sess_archive_empty", { + id: "sess_archive_empty", + project: "agentmemory", + cwd: "C:\\work\\agentmemory", + startedAt: "2026-07-10T00:00:00.000Z", + endedAt: "2026-07-10T00:00:00.000Z", + status: "completed", + observationCount: 0, + } satisfies Session); + return { success: true }; + }); + const summarize = vi.fn(); + registerDurableCandidateFunctions( + sdk as never, + kv as never, + { name: "test", compress: async () => "", summarize }, + { archiveRoot: tempDir }, + ); + + const result: any = await sdk.trigger({ + function_id: "mem::archive::process", + payload: { path: archivePath }, + }); + + expect(result.processed).toHaveLength(1); + expect(result.processed[0]).toMatchObject({ + sessionId: "sess_archive_empty", + durableCandidateCount: 0, + }); + expect(summarize).not.toHaveBeenCalled(); + const ledger = await kv.get("mem:archive-imports", result.processed[0].idempotencyKey); + expect(ledger).toMatchObject({ + status: "completed", + summaryCreated: false, + parsedObservationCount: 0, + importedObservationCount: 0, + }); + }); + + it("retries a failed archive summary without replaying observations", async () => { + const tempDir = mkdtempSync(join(tmpdir(), "agentmemory-archive-retry-")); + tempDirs.add(tempDir); + const archivePath = join(tempDir, "session.jsonl"); + writeFileSync( + archivePath, + `${JSON.stringify({ + type: "user", + sessionId: "sess_archive_retry", + cwd: "/tmp/agentmemory", + timestamp: "2026-07-10T00:00:00.000Z", + message: { role: "user", content: [{ type: "text", text: "Retry summary." }] }, + })}\n`, + "utf-8", + ); + + let replayCalls = 0; + let failSummary = true; + sdk.registerFunction("mem::replay::import-jsonl", async () => { + replayCalls += 1; + const session: Session = { + id: "sess_archive_retry", + project: "agentmemory", + cwd: "/tmp/agentmemory", + startedAt: "2026-07-10T00:00:00.000Z", + endedAt: "2026-07-10T00:01:00.000Z", + status: "completed", + observationCount: 1, + }; + await kv.set("mem:sessions", session.id, session); + await kv.set("mem:obs:sess_archive_retry", "obs_1", makeObservation(session.id)); + return { success: true }; + }); + const provider: MemoryProvider = { + name: "test", + compress: async () => "", + summarize: async () => { + if (failSummary) throw new Error("summary_unavailable"); + return makeSummaryXml("sess_archive_retry", "obs_1"); + }, + }; + registerDurableCandidateFunctions(sdk as never, kv as never, provider, { + archiveRoot: tempDir, + }); + + const first: any = await sdk.trigger({ + function_id: "mem::archive::process", + payload: { path: archivePath }, + }); + const afterFailure = await kv.list("mem:archive-imports"); + expect(first.skipped[0].reason).toBe("summary_unavailable"); + expect(afterFailure[0].status).toBe("failed"); + expect(afterFailure[0].failureStage).toBe("summary"); + + failSummary = false; + const second: any = await sdk.trigger({ + function_id: "mem::archive::process", + payload: { path: archivePath, force: true }, + }); + + expect(second.processed).toHaveLength(1); + expect(replayCalls).toBe(1); + expect((await kv.list("mem:archive-imports"))[0].status).toBe("completed"); + }); + + it("requires force metadata for a low-confidence candidate", async () => { + const candidate = materializeDurableCandidate({ + sessionId: "sess_force", + project: "agentmemory", + type: "workflow", + title: "Needs review", + content: "This candidate has limited supporting evidence.", + concepts: ["review"], + files: [], + sourceObservationIds: ["obs_1"], + confidence: 0.6, + createdAt: "2026-07-10T00:00:00.000Z", + }); + expect(candidate).not.toBeNull(); + await kv.set("mem:summaries", "sess_force", { + sessionId: "sess_force", + project: "agentmemory", + createdAt: "2026-07-10T00:00:00.000Z", + title: "Summary", + narrative: "Narrative long enough to count as a valid test summary.", + keyDecisions: [], + filesModified: [], + concepts: [], + observationCount: 1, + durableCandidates: [candidate!], + } satisfies SessionSummary); + registerDurableCandidateFunctions( + sdk as never, + kv as never, + makeProvider(makeSummaryXml("sess_force", "obs_1")), + ); + + const denied: any = await sdk.trigger({ + function_id: "mem::durable-candidates::promote", + payload: { candidateId: candidate!.id, force: true }, + }); + const preview: any = await sdk.trigger({ + function_id: "mem::durable-candidates::promote", + payload: { + candidateId: candidate!.id, + force: true, + dryRun: true, + forceReason: "Reviewed during migration.", + promotedBy: "operator", + }, + }); + + expect(denied.error).toBe("force_metadata_required"); + expect(preview.success).toBe(true); + expect(preview.dryRun).toBe(true); + }); + + it("rejects archive paths outside the configured archive root", async () => { + const archiveRoot = mkdtempSync(join(tmpdir(), "agentmemory-archive-root-")); + const outsideRoot = mkdtempSync(join(tmpdir(), "agentmemory-archive-outside-")); + tempDirs.add(archiveRoot); + tempDirs.add(outsideRoot); + const outsidePath = join(outsideRoot, "session.jsonl"); + writeFileSync(outsidePath, "{}\n", "utf-8"); + + registerDurableCandidateFunctions( + sdk as never, + kv as never, + makeProvider(makeSummaryXml("sess_archive", "obs_1")), + { archiveRoot }, + ); + + const result: any = await sdk.trigger({ + function_id: "mem::archive::process", + payload: { path: outsidePath }, + }); + + expect(result.success).toBe(false); + expect(result.error).toContain("archive path must live under"); }); }); diff --git a/test/replay-import-key.test.ts b/test/replay-import-key.test.ts index 9a4c64675c..ee850bbc47 100644 --- a/test/replay-import-key.test.ts +++ b/test/replay-import-key.test.ts @@ -151,4 +151,57 @@ describe("import-jsonl re-key on parsed.sessionId (#775)", () => { .filter((c) => c.scope === KV.sessions && c.key === "sess-fresh"); expect(sessionWrites.length).toBe(1); }); + + it("re-imports stable observations without inflating the session count", async () => { + writeFixture("sess-idempotent"); + const kv = mockKV(); + const sdk = mockSdk(kv); + registerReplayFunctions(sdk, kv as never); + + const first = (await sdk.trigger("mem::replay::import-jsonl", { + path: tmpRoot, + })) as { success: boolean; observations?: number }; + const second = (await sdk.trigger("mem::replay::import-jsonl", { + path: tmpRoot, + })) as { success: boolean; observations?: number }; + + expect(first).toMatchObject({ success: true, observations: 2 }); + expect(second).toMatchObject({ success: true, observations: 0 }); + expect(await kv.list(KV.observations("sess-idempotent"))).toHaveLength(2); + expect((await kv.get(KV.sessions, "sess-idempotent"))?.observationCount).toBe(2); + }); + + it("creates a zero-observation session for a Codex archive", async () => { + const dir = join(tmpRoot, "empty"); + require("node:fs").mkdirSync(dir, { recursive: true }); + writeFileSync( + join(dir, "sess-codex-empty.jsonl"), + [ + JSON.stringify({ + type: "session_meta", + payload: { + id: "sess-codex-empty", + cwd: "C:\\work\\agentmemory", + timestamp: "2026-07-10T00:00:00.000Z", + }, + }), + JSON.stringify({ type: "event_msg", payload: { type: "task_started" } }), + ].join("\n") + "\n", + ); + + const kv = mockKV(); + const sdk = mockSdk(kv); + registerReplayFunctions(sdk, kv as never); + + const result = (await sdk.trigger("mem::replay::import-jsonl", { + path: dir, + })) as { success: boolean; observations?: number }; + + expect(result).toMatchObject({ success: true, observations: 0 }); + expect(await kv.get(KV.sessions, "sess-codex-empty")).toMatchObject({ + id: "sess-codex-empty", + observationCount: 0, + status: "completed", + }); + }); }); diff --git a/test/replay.test.ts b/test/replay.test.ts index f5dfb9a6e7..116af43795 100644 --- a/test/replay.test.ts +++ b/test/replay.test.ts @@ -97,6 +97,52 @@ describe("parseJsonlText", () => { const out = parseJsonlText(text, "fb-used"); expect(out.sessionId).toBe("fb-used"); }); + + it("parses Codex archive primary events with stable observation ids", () => { + const text = [ + JSON.stringify({ + type: "session_meta", + timestamp: "2026-07-10T00:00:00.000Z", + payload: { + id: "codex-session", + timestamp: "2026-07-10T00:00:00.000Z", + cwd: "C:\\work\\agentmemory", + }, + }), + JSON.stringify({ + type: "response_item", + timestamp: "2026-07-10T00:00:01.000Z", + payload: { type: "message", role: "assistant", content: [] }, + }), + JSON.stringify({ + type: "event_msg", + timestamp: "2026-07-10T00:00:02.000Z", + payload: { type: "user_message", message: "Import this archive." }, + }), + "not-json", + JSON.stringify({ + type: "event_msg", + timestamp: "2026-07-10T00:00:03.000Z", + payload: { type: "task_complete", last_agent_message: "Done." }, + }), + ].join("\n"); + + const first = parseJsonlText(text); + const second = parseJsonlText(text); + + expect(first.sessionId).toBe("codex-session"); + expect(first.project).toBe("agentmemory"); + expect(first.cwd).toBe("C:\\work\\agentmemory"); + expect(first.observations.map((observation) => observation.hookType)).toEqual([ + "prompt_submit", + "task_completed", + ]); + expect(first.observations[0].userPrompt).toBe("Import this archive."); + expect(first.observations[1].assistantResponse).toBe("Done."); + expect(first.observations.map((observation) => observation.id)).toEqual( + second.observations.map((observation) => observation.id), + ); + }); }); describe("projectTimeline", () => { From 9e57966baba6b97a5fdb2b3208bbba64ef55ac2c Mon Sep 17 00:00:00 2001 From: Jenq Shiin Haw <1343528669@qq.com> Date: Sun, 12 Jul 2026 13:17:22 +0800 Subject: [PATCH 5/6] feat(recall): add scoped traceable context recall Signed-off-by: Jenq Shiin Haw <1343528669@qq.com> --- .env.example | 8 + .gitignore | 1 + AGENTS.md | 4 +- README.md | 25 +- eval/recall/README.md | 8 + eval/recall/fixtures/sanitized.json | 17 + eval/recall/score.ts | 51 +++ package.json | 1 + plugin/plugin.json | 2 +- plugin/scripts/pre-compact.mjs | 9 +- plugin/scripts/pre-tool-use.mjs | 1 + plugin/scripts/prompt-submit.mjs | 27 +- plugin/skills/agentmemory-config/REFERENCE.md | 15 +- .../skills/agentmemory-mcp-tools/REFERENCE.md | 4 +- .../skills/agentmemory-rest-api/REFERENCE.md | 6 +- plugin/skills/recall-debug/SKILL.md | 46 ++ plugin/skills/why-memory/SKILL.md | 44 ++ src/cli/connect/index.ts | 2 +- src/config.ts | 198 +++++++++ src/functions/context.ts | 35 +- src/functions/durable-candidates.ts | 47 +- src/functions/enrich.ts | 27 +- src/functions/remember.ts | 30 +- src/functions/slots.ts | 10 + src/hooks/pre-compact.ts | 13 +- src/hooks/pre-tool-use.ts | 1 + src/hooks/prompt-submit.ts | 39 +- src/index.ts | 13 +- src/mcp/server.ts | 35 ++ src/mcp/tools-registry.ts | 10 + src/recall/core.ts | 420 ++++++++++++++++++ src/recall/identity.ts | 50 +++ src/recall/ledger.ts | 95 ++++ src/recall/scope.ts | 69 +++ src/recall/tokens.ts | 26 ++ src/recall/trace-store.ts | 81 ++++ src/recall/vector-health.ts | 77 ++++ src/state/hybrid-search.ts | 73 ++- src/state/schema.ts | 4 + src/triggers/api.ts | 187 +++++++- src/types.ts | 168 +++++++ src/viewer/index.html | 34 +- test/durable-candidates.test.ts | 22 + test/recall-benchmark-score.test.ts | 13 + test/recall-core.test.ts | 154 +++++++ test/remember-project-scope.test.ts | 12 +- test/vector-retrieval-health.test.ts | 27 ++ 47 files changed, 2184 insertions(+), 57 deletions(-) create mode 100644 eval/recall/README.md create mode 100644 eval/recall/fixtures/sanitized.json create mode 100644 eval/recall/score.ts create mode 100644 plugin/skills/recall-debug/SKILL.md create mode 100644 plugin/skills/why-memory/SKILL.md create mode 100644 src/recall/core.ts create mode 100644 src/recall/identity.ts create mode 100644 src/recall/ledger.ts create mode 100644 src/recall/scope.ts create mode 100644 src/recall/tokens.ts create mode 100644 src/recall/trace-store.ts create mode 100644 src/recall/vector-health.ts create mode 100644 test/recall-benchmark-score.test.ts create mode 100644 test/recall-core.test.ts create mode 100644 test/vector-retrieval-health.test.ts diff --git a/.env.example b/.env.example index 77ca0f3a38..e3c06e46bd 100644 --- a/.env.example +++ b/.env.example @@ -97,6 +97,14 @@ # VECTOR_WEIGHT=0.6 # Hybrid search weight for vector leg # AGENTMEMORY_GRAPH_WEIGHT=0.2 # Graph traversal bonus on smart-search ranking # TOKEN_BUDGET=2000 # Max tokens injected via mem::context per session +# Recall context now uses ~/.agentmemory/config.toml for structured budgets. +# Environment overrides still win over config.toml: +# AGENTMEMORY_RECALL_MAX_CONTEXT_TOKENS=800 +# AGENTMEMORY_RECALL_RESERVED_BOOTSTRAP_TOKENS=200 +# AGENTMEMORY_RECALL_MAX_SEMANTIC_TOKENS=600 +# AGENTMEMORY_RECALL_UNKNOWN_AUTO_INJECTION=false +# AGENTMEMORY_RECALL_UNKNOWN_EXPLICIT_SEARCH=true +# AGENTMEMORY_RECALL_TRACE_MAX_DROPPED_ITEMS_PER_REASON=20 # MAX_OBS_PER_SESSION=500 # Per-session observation cap before consolidation kicks in # SUMMARIZE_CHUNK_SIZE=400 # When mem::summarize sees a session larger than this, it chunks observations and map-reduces (chunk-summarize → reduce-merge) to stay within the LLM's context window. Default 400 ≈ 50k tokens per chunk at ~110 tok/obs. Native sessions are capped by MAX_OBS_PER_SESSION; chunking primarily matters for bulk-imported jsonl sessions, which bypass that cap. # SUMMARIZE_CHUNK_CONCURRENCY=6 # Parallel chunk LLM calls during chunked summarize. Default 6 fits ~100-chunk sessions under iii's 180s function-invocation timeout at typical ~8s/call. High-throughput providers (Novita, DeepInfra, DeepSeek) commonly allow 100+ concurrent — bump this for very large imported sessions. diff --git a/.gitignore b/.gitignore index ba6af995b7..cad90a74f9 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,7 @@ agentmemory-debug/ # Lock files — never commit (see feedback_no_lockfiles memory) package-lock.json +eval/recall/fixtures/local.json pnpm-lock.yaml yarn.lock integrations/hermes/__pycache__/ diff --git a/AGENTS.md b/AGENTS.md index 2b3d32f771..e2c72218b8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -117,8 +117,8 @@ Hook scripts in `src/hooks/` are standalone Node.js scripts (no iii-sdk import). ## Current Stats (v0.9.16) - 53 MCP tools (8 visible by default, `AGENTMEMORY_TOOLS=all` for all) -- 132 REST endpoints +- 136 REST endpoints - 6 MCP resources, 3 MCP prompts -- 12 hooks, 15 skills +- 12 hooks, 17 skills - 50+ iii functions - 950+ tests diff --git a/README.md b/README.md index 9451b082b5..0b1e0ced7a 100644 --- a/README.md +++ b/README.md @@ -514,7 +514,7 @@ Implementation details live in `src/cli.ts` (see `runUpgrade` around the `src/cl ### Claude Code (one block, paste it) ```text -Install agentmemory: run `npx @agentmemory/agentmemory` in a separate terminal to start the memory server. Then run `/plugin marketplace add rohitg00/agentmemory` and `/plugin install agentmemory` — the plugin registers all 12 hooks, 15 skills, AND auto-wires the `@agentmemory/mcp` stdio server via its `.mcp.json`, so you get 53 MCP tools (memory_smart_search, memory_save, memory_sessions, memory_governance_delete, etc.) without any extra config step. Verify with `curl http://localhost:3111/agentmemory/health`. The real-time viewer is at http://localhost:3113. +Install agentmemory: run `npx @agentmemory/agentmemory` in a separate terminal to start the memory server. Then run `/plugin marketplace add rohitg00/agentmemory` and `/plugin install agentmemory` — the plugin registers all 12 hooks, 17 skills, AND auto-wires the `@agentmemory/mcp` stdio server via its `.mcp.json`, so you get 53 MCP tools (memory_smart_search, memory_save, memory_sessions, memory_governance_delete, etc.) without any extra config step. Verify with `curl http://localhost:3111/agentmemory/health`. The real-time viewer is at http://localhost:3113. ``` #### Claude Code without the plugin install (MCP-standalone path) @@ -625,7 +625,7 @@ Start the memory server: `npx @agentmemory/agentmemory` #### Native skills via `npx skills add` (50+ agents) -agentmemory ships 15 skills in the Claude-Code-style `/SKILL.md` format: 8 invocable action skills (`remember`, `recall`, `recap`, `handoff`, `forget`, `commit-context`, `commit-history`, `session-history`) and 7 reference skills the agent loads on demand (`agentmemory-mcp-tools`, `agentmemory-rest-api`, `agentmemory-config`, `agentmemory-agents`, `agentmemory-hooks`, `agentmemory-architecture`, `write-agentmemory-skill`). The reference skills carry data tables generated from source, so they never drift. The [`skills`](https://npmjs.com/package/skills) CLI by vercel-labs auto-installs them into the calling agent's native skill directory across 50+ agents (Claude Code, Cursor, Cline, Continue, Droid, Warp, Codex, Antigravity, Kiro, OpenCode, Goose, Roo, Trae, Windsurf, and more): +agentmemory ships 17 skills in the Claude-Code-style `/SKILL.md` format: 10 invocable action skills (`remember`, `recall`, `recap`, `handoff`, `forget`, `commit-context`, `commit-history`, `session-history`, `recall-debug`, `why-memory`) and 7 reference skills the agent loads on demand (`agentmemory-mcp-tools`, `agentmemory-rest-api`, `agentmemory-config`, `agentmemory-agents`, `agentmemory-hooks`, `agentmemory-architecture`, `write-agentmemory-skill`). The reference skills carry data tables generated from source, so they never drift. The [`skills`](https://npmjs.com/package/skills) CLI by vercel-labs auto-installs them into the calling agent's native skill directory across 50+ agents (Claude Code, Cursor, Cline, Continue, Droid, Warp, Codex, Antigravity, Kiro, OpenCode, Goose, Roo, Trae, Windsurf, and more): ```bash npx skills add rohitg00/agentmemory -y # auto-detects the calling agent @@ -668,7 +668,7 @@ The agentmemory entry is the **same MCP server block** across every host that us | **GitHub Copilot CLI (full plugin)** | Copilot plugin install | `copilot plugin install rohitg00/agentmemory:plugin` for the plugin from the GitHub subdir. | | **OpenClaw** | OpenClaw MCP config | Same `mcpServers` block, or use the deeper [memory plugin](integrations/openclaw/). | | **Codex CLI (MCP only)** | `.codex/config.toml` | TOML shape: `codex mcp add agentmemory -- npx -y @agentmemory/mcp`, or add `[mcp_servers.agentmemory]` manually. | -| **Codex CLI (full plugin)** | Codex plugin marketplace | `codex plugin marketplace add rohitg00/agentmemory` then `codex plugin add agentmemory@agentmemory`. Registers MCP + 6 lifecycle hooks (SessionStart, UserPromptSubmit, PreToolUse, PostToolUse, PreCompact, Stop) + 15 skills. On Codex Desktop, also run `agentmemory connect codex --with-hooks` until [openai/codex#16430](https://github.com/openai/codex/issues/16430) lands — plugin hooks are currently silent there. | +| **Codex CLI (full plugin)** | Codex plugin marketplace | `codex plugin marketplace add rohitg00/agentmemory` then `codex plugin add agentmemory@agentmemory`. Registers MCP + 6 lifecycle hooks (SessionStart, UserPromptSubmit, PreToolUse, PostToolUse, PreCompact, Stop) + 17 skills. On Codex Desktop, also run `agentmemory connect codex --with-hooks` until [openai/codex#16430](https://github.com/openai/codex/issues/16430) lands — plugin hooks are currently silent there. | | **OpenCode (MCP only)** | `opencode.json` | Different shape — top-level `mcp` key, command as array: `{"mcp": {"agentmemory": {"type": "local", "command": ["npx", "-y", "@agentmemory/mcp"], "enabled": true}}}`. | | **OpenCode (full plugin)** | `plugin/opencode/` | 22 auto-capture hooks covering session lifecycle, messages, tools, errors. Two slash commands (`/recall`, `/remember`). Copy `plugin/opencode/` into your OpenCode workspace and add the plugin entry to `opencode.json`. See [`plugin/opencode/README.md`](plugin/opencode/README.md) for the full hook table + gap analysis. | | **pi** | `~/.pi/agent/extensions/agentmemory` | Copy [`integrations/pi`](integrations/pi/) and restart pi. | @@ -959,7 +959,7 @@ npm install @xenova/transformers

MCP Server

-53 tools, 6 resources, 3 prompts, and 15 skills, the most comprehensive MCP memory toolkit for any agent. +53 tools, 6 resources, 3 prompts, and 17 skills, the most comprehensive MCP memory toolkit for any agent. > **MCP shim vs full server:** the published `@agentmemory/mcp` package is a thin shim. It exposes the full 53-tool surface **only when it can reach a running agentmemory server** via `AGENTMEMORY_URL` (proxy mode). With no server reachable, the shim falls back to a 7-tool local set (`memory_save`, `memory_recall`, `memory_smart_search`, `memory_sessions`, `memory_export`, `memory_audit`, `memory_governance_delete`). The `AGENTMEMORY_TOOLS=core|all` env var is a *server-side* flag — setting it in the shim's `env` block has no effect. If you see only 7 tools in Cursor / OpenCode / Gemini CLI, start `npx @agentmemory/agentmemory` (or the Docker stack) and set `AGENTMEMORY_URL=http://localhost:3111`. @@ -1437,6 +1437,21 @@ Create `~/.agentmemory/.env`: # VECTOR_WEIGHT=0.6 # TOKEN_BUDGET=2000 +# Structured recall budget: ~/.agentmemory/config.toml +# [recall_budget] +# max_context_tokens = 800 +# reserved_bootstrap_tokens = 200 +# max_semantic_tokens = 600 +# max_memories = 5 +# max_session_summaries = 1 +# max_observations = 3 +# max_continuity_items = 1 +# [recall_scope] +# unknown_auto_injection = false +# unknown_explicit_search = true +# [recall_trace] +# max_dropped_items_per_reason = 20 + # Auth # AGENTMEMORY_SECRET=your-secret @@ -1501,7 +1516,7 @@ Create `~/.agentmemory/.env`:

API

-132 endpoints on port `3111`. The REST API binds to `127.0.0.1` by default. Protected endpoints require `Authorization: Bearer ` when `AGENTMEMORY_SECRET` is set, and mesh sync endpoints require `AGENTMEMORY_SECRET` on both peers. +136 endpoints on port `3111`. The REST API binds to `127.0.0.1` by default. Protected endpoints require `Authorization: Bearer ` when `AGENTMEMORY_SECRET` is set, and mesh sync endpoints require `AGENTMEMORY_SECRET` on both peers.
Key endpoints diff --git a/eval/recall/README.md b/eval/recall/README.md new file mode 100644 index 0000000000..b9934fae26 --- /dev/null +++ b/eval/recall/README.md @@ -0,0 +1,8 @@ +# Recall benchmark + +`fixtures/sanitized.json` contains committed, sanitized project cases. A local, +gitignored fixture may add real project cases with the same schema. Score each +case from its persisted recall trace with `scoreRecallBenchmark`. + +Hard gates are zero scope contamination, zero budget violations, zero duplicate +or stale injection. Precision and hit rate are reported for regression tracking. diff --git a/eval/recall/fixtures/sanitized.json b/eval/recall/fixtures/sanitized.json new file mode 100644 index 0000000000..0950d34a8b --- /dev/null +++ b/eval/recall/fixtures/sanitized.json @@ -0,0 +1,17 @@ +[ + {"id":"pps-pair-cache","query":"PPS7000 weighted benchmark 的 pair cache 在哪里?","projectId":"pps7000","expectedMemoryIds":["mem_pps_pair_cache"],"forbiddenMemoryIds":["mem_ic_plan"],"maxAcceptableTokens":800}, + {"id":"pps-stage-one","query":"weighted stage1 的实现约束是什么?","projectId":"pps7000","expectedMemoryIds":["mem_pps_weighted_stage1"],"forbiddenMemoryIds":["mem_ic_plan"],"maxAcceptableTokens":800}, + {"id":"pps-docs","query":"这个项目的文档应该怎么写?","projectId":"pps7000","expectedMemoryIds":["mem_docs_preference"],"forbiddenMemoryIds":["mem_gat_formula"],"maxAcceptableTokens":800}, + {"id":"gat-formula","query":"GAT edge weight 的公式是什么?","projectId":"study-heavy-industrial-chain","expectedMemoryIds":["mem_gat_formula"],"forbiddenMemoryIds":["mem_ic_plan"],"maxAcceptableTokens":800}, + {"id":"ic-plan","query":"Ivan's Childhood 当前实现计划是什么?","projectId":"ivans-childhood","expectedMemoryIds":["mem_ic_plan"],"forbiddenMemoryIds":["mem_pps_pair_cache"],"maxAcceptableTokens":800}, + {"id":"pps-cache-key","query":"pair cache key 包含哪些字段?","projectId":"pps7000","expectedMemoryIds":["mem_pps_pair_cache"],"forbiddenMemoryIds":["mem_ic_plan"],"maxAcceptableTokens":800}, + {"id":"gat-weighted","query":"edge weight 进入 attention 的位置?","projectId":"study-heavy-industrial-chain","expectedMemoryIds":["mem_gat_formula"],"forbiddenMemoryIds":["mem_docs_preference"],"maxAcceptableTokens":800}, + {"id":"pps-benchmark","query":"PPS7000 benchmark 的 weighted stage 如何运行?","projectId":"pps7000","expectedMemoryIds":["mem_pps_weighted_stage1"],"forbiddenMemoryIds":["mem_ic_plan"],"maxAcceptableTokens":800}, + {"id":"docs-style","query":"README 更新时遵循什么写作偏好?","projectId":"pps7000","expectedMemoryIds":["mem_docs_preference"],"forbiddenMemoryIds":["mem_gat_formula"],"maxAcceptableTokens":800}, + {"id":"ic-next","query":"Ivan 项目的下一步实现工作是什么?","projectId":"ivans-childhood","expectedMemoryIds":["mem_ic_plan"],"forbiddenMemoryIds":["mem_pps_weighted_stage1"],"maxAcceptableTokens":800}, + {"id":"pps-no-ic","query":"PPS7000 pair cache 的目录?","projectId":"pps7000","expectedMemoryIds":["mem_pps_pair_cache"],"forbiddenMemoryIds":["mem_ic_plan"],"maxAcceptableTokens":800}, + {"id":"gat-no-docs","query":"图神经网络边权公式?","projectId":"study-heavy-industrial-chain","expectedMemoryIds":["mem_gat_formula"],"forbiddenMemoryIds":["mem_docs_preference"],"maxAcceptableTokens":800}, + {"id":"pps-score","query":"weighted benchmark 的评分缓存复用规则?","projectId":"pps7000","expectedMemoryIds":["mem_pps_pair_cache","mem_pps_weighted_stage1"],"forbiddenMemoryIds":["mem_ic_plan"],"maxAcceptableTokens":800}, + {"id":"docs-project","query":"本项目的技术文档采用什么结构?","projectId":"pps7000","expectedMemoryIds":["mem_docs_preference"],"forbiddenMemoryIds":["mem_ic_plan"],"maxAcceptableTokens":800}, + {"id":"ic-isolation","query":"Ivan's Childhood 的计划和约束?","projectId":"ivans-childhood","expectedMemoryIds":["mem_ic_plan"],"forbiddenMemoryIds":["mem_pps_pair_cache","mem_gat_formula"],"maxAcceptableTokens":800} +] diff --git a/eval/recall/score.ts b/eval/recall/score.ts new file mode 100644 index 0000000000..0b56f24726 --- /dev/null +++ b/eval/recall/score.ts @@ -0,0 +1,51 @@ +export interface RecallBenchmarkCase { + id: string; + query: string; + projectId: string; + expectedMemoryIds: string[]; + forbiddenMemoryIds: string[]; + maxAcceptableTokens: number; +} + +export interface RecallBenchmarkResult { + selectedIds: string[]; + injectedTokens: number; + duplicateIds?: string[]; + staleIds?: string[]; +} + +export function scoreRecallBenchmark( + cases: RecallBenchmarkCase[], + results: Record, +) { + let expected = 0; + let correct = 0; + let selected = 0; + let contamination = 0; + let budgetViolations = 0; + let duplicates = 0; + let stale = 0; + let tokens = 0; + for (const testCase of cases) { + const result = results[testCase.id]; + if (!result) continue; + const chosen = new Set(result.selectedIds); + expected += testCase.expectedMemoryIds.length; + correct += testCase.expectedMemoryIds.filter((id) => chosen.has(id)).length; + selected += result.selectedIds.length; + contamination += testCase.forbiddenMemoryIds.filter((id) => chosen.has(id)).length; + budgetViolations += result.injectedTokens > testCase.maxAcceptableTokens ? 1 : 0; + duplicates += result.duplicateIds?.length || 0; + stale += result.staleIds?.length || 0; + tokens += result.injectedTokens; + } + return { + precision: selected ? correct / selected : 0, + hitRate: expected ? correct / expected : 0, + crossProjectContaminationRate: selected ? contamination / selected : 0, + averageInjectedTokens: cases.length ? tokens / cases.length : 0, + duplicateInjectionRate: selected ? duplicates / selected : 0, + staleMemoryRate: selected ? stale / selected : 0, + budgetViolations, + }; +} diff --git a/package.json b/package.json index 7cdbba409b..89eccf157d 100644 --- a/package.json +++ b/package.json @@ -66,6 +66,7 @@ "dotenv": "^17.4.2", "iii-sdk": "0.11.2", "picocolors": "^1.1.1", + "smol-toml": "1.7.0", "zod": "^4.0.0" }, "optionalDependencies": { diff --git a/plugin/plugin.json b/plugin/plugin.json index 9d0751a7ab..20ffae334a 100644 --- a/plugin/plugin.json +++ b/plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "agentmemory", "version": "0.9.27", - "description": "Persistent memory for AI coding agents -- captures tool usage, compresses via LLM, injects context into future sessions. 12 hooks, 53 MCP tools, 15 skills, real-time viewer.", + "description": "Persistent memory for AI coding agents -- captures tool usage, compresses via LLM, injects context into future sessions. 12 hooks, 53 MCP tools, 17 skills, real-time viewer.", "author": { "name": "Rohit Ghumare", "url": "https://github.com/rohitg00" diff --git a/plugin/scripts/pre-compact.mjs b/plugin/scripts/pre-compact.mjs index bf7ce2b763..c40b6e3e9d 100755 --- a/plugin/scripts/pre-compact.mjs +++ b/plugin/scripts/pre-compact.mjs @@ -55,13 +55,20 @@ async function main() { }); } catch {} try { + await fetch(`${REST_URL}/agentmemory/recall/context-epoch`, { + method: "POST", + headers: authHeaders(), + body: JSON.stringify({ sessionId }), + signal: AbortSignal.timeout(1e3) + }); const res = await fetch(`${REST_URL}/agentmemory/context`, { method: "POST", headers: authHeaders(), body: JSON.stringify({ sessionId, project, - budget: 1500 + cwd: data.cwd || process.cwd(), + outputMode: "bootstrap" }), signal: AbortSignal.timeout(5e3) }); diff --git a/plugin/scripts/pre-tool-use.mjs b/plugin/scripts/pre-tool-use.mjs index a0d6e95832..078b898bac 100755 --- a/plugin/scripts/pre-tool-use.mjs +++ b/plugin/scripts/pre-tool-use.mjs @@ -67,6 +67,7 @@ async function main() { files, terms, toolName, + cwd: typeof data.cwd === "string" ? data.cwd : process.cwd(), ...project !== void 0 && { project } }), signal: AbortSignal.timeout(2e3) diff --git a/plugin/scripts/prompt-submit.mjs b/plugin/scripts/prompt-submit.mjs index 112bcbcc3a..44a5a0ec9b 100755 --- a/plugin/scripts/prompt-submit.mjs +++ b/plugin/scripts/prompt-submit.mjs @@ -29,6 +29,7 @@ function isSdkChildContext(payload) { } const REST_URL = process.env["AGENTMEMORY_URL"] || "http://localhost:3111"; const SECRET = process.env["AGENTMEMORY_SECRET"] || ""; +const INJECT_CONTEXT = process.env["AGENTMEMORY_INJECT_CONTEXT"] === "true"; function authHeaders() { const h = { "Content-Type": "application/json" }; if (SECRET) h["Authorization"] = `Bearer ${SECRET}`; @@ -45,20 +46,40 @@ async function main() { } if (isSdkChildContext(data)) return; const sessionId = data.session_id || data.sessionId || "unknown"; + const project = resolveProject(data.cwd); + const prompt = typeof data.prompt === "string" ? data.prompt : typeof data.userPrompt === "string" ? data.userPrompt : ""; fetch(`${REST_URL}/agentmemory/observe`, { method: "POST", headers: authHeaders(), body: JSON.stringify({ hookType: "prompt_submit", sessionId, - project: resolveProject(data.cwd), + project, cwd: data.cwd || process.cwd(), timestamp: (/* @__PURE__ */ new Date()).toISOString(), - data: { prompt: data.prompt ?? data.userPrompt } + data: { prompt } }), signal: AbortSignal.timeout(3e3) }).catch(() => {}); - setTimeout(() => process.exit(0), 500).unref(); + if (!INJECT_CONTEXT || !prompt) return; + try { + const response = await fetch(`${REST_URL}/agentmemory/context`, { + method: "POST", + headers: authHeaders(), + body: JSON.stringify({ + sessionId, + project, + cwd: data.cwd || process.cwd(), + query: prompt, + outputMode: "prompt_injection" + }), + signal: AbortSignal.timeout(1500) + }); + if (response.ok) { + const result = await response.json(); + if (typeof result.context === "string" && result.context) process.stdout.write(result.context); + } + } catch {} } main(); //#endregion diff --git a/plugin/skills/agentmemory-config/REFERENCE.md b/plugin/skills/agentmemory-config/REFERENCE.md index 08c873a444..296c16794e 100644 --- a/plugin/skills/agentmemory-config/REFERENCE.md +++ b/plugin/skills/agentmemory-config/REFERENCE.md @@ -3,7 +3,7 @@ Generated by scanning `src/` for `AGENTMEMORY_*` usage. Do not edit the block below by hand; run `npm run skills:gen` after adding or removing a variable. Internal markers ending in two underscores are excluded. -Configuration is read from the environment and from `~/.agentmemory/.env` (no `export` prefix). 34 recognized variables: +Configuration is read from the environment and from `~/.agentmemory/.env` (no `export` prefix). 47 recognized variables: - `AGENTMEMORY_AGENT_SCOPE` - `AGENTMEMORY_ALLOW_AGENT_SDK` @@ -27,6 +27,19 @@ Configuration is read from the environment and from `~/.agentmemory/.env` (no `e - `AGENTMEMORY_PROBE_TIMEOUT_MS` - `AGENTMEMORY_PROJECT_NAME` - `AGENTMEMORY_PROVIDER` +- `AGENTMEMORY_RECALL_MAX_CONTEXT_TOKENS` +- `AGENTMEMORY_RECALL_MAX_CONTINUITY_ITEMS` +- `AGENTMEMORY_RECALL_MAX_MEMORIES` +- `AGENTMEMORY_RECALL_MAX_OBSERVATIONS` +- `AGENTMEMORY_RECALL_MAX_SEMANTIC_TOKENS` +- `AGENTMEMORY_RECALL_MAX_SESSION_SUMMARIES` +- `AGENTMEMORY_RECALL_REINJECTION_TURN_WINDOW` +- `AGENTMEMORY_RECALL_RESERVED_BOOTSTRAP_TOKENS` +- `AGENTMEMORY_RECALL_TRACE_MAX_DROPPED_ITEMS_PER_REASON` +- `AGENTMEMORY_RECALL_TRACE_MAX_TRACES` +- `AGENTMEMORY_RECALL_TRACE_RETENTION_DAYS` +- `AGENTMEMORY_RECALL_UNKNOWN_AUTO_INJECTION` +- `AGENTMEMORY_RECALL_UNKNOWN_EXPLICIT_SEARCH` - `AGENTMEMORY_REFLECT` - `AGENTMEMORY_SDK_CHILD` - `AGENTMEMORY_SECRET` diff --git a/plugin/skills/agentmemory-mcp-tools/REFERENCE.md b/plugin/skills/agentmemory-mcp-tools/REFERENCE.md index 0c556fc77f..f53cc577c7 100644 --- a/plugin/skills/agentmemory-mcp-tools/REFERENCE.md +++ b/plugin/skills/agentmemory-mcp-tools/REFERENCE.md @@ -35,7 +35,7 @@ agentmemory exposes 53 MCP tools. 8 are in the lean core set (`--tools core` or | `memory_obsidian_export` | | `vaultDir`: string, `types`: string | Export memories, lessons, and crystals as Obsidian-compatible Markdown files with YAML frontmatter and wikilinks for graph view. | | `memory_patterns` | | `project`: string | Detect recurring patterns across sessions. | | `memory_profile` | | `project`*: string, `refresh`: string | User/project profile with top concepts and file patterns. | -| `memory_recall` | yes | `query`*: string, `limit`: number, `format`: string, `token_budget`: number | Search past session observations for relevant context. Use when you need to recall what happened in previous sessions, find past decisions, or look up how a file was modified before. | +| `memory_recall` | yes | `query`*: string, `limit`: number, `format`: string, `token_budget`: number, `outputMode`: string, `project`: string, `repoId`: string, `checkoutId`: string | Search past session observations for relevant context. Use when you need to recall what happened in previous sessions, find past decisions, or look up how a file was modified before. | | `memory_reflect` | yes | `project`: string, `maxClusters`: number | Traverse the knowledge graph, group related memories by concept clusters, and synthesize higher-order insights via LLM. Returns new and reinforced insights. | | `memory_relations` | | `memoryId`*: string, `maxHops`: number, `minConfidence`: number | Query the memory relationship graph. | | `memory_routine_run` | | `routineId`*: string, `project`: string, `initiatedBy`: string | Instantiate a frozen workflow routine, creating actions for each step with proper dependencies. | @@ -53,7 +53,7 @@ agentmemory exposes 53 MCP tools. 8 are in the lean core set (`--tools core` or | `memory_slot_get` | | `label`*: string | Read a single slot by label. | | `memory_slot_list` | | none | List all memory slots (pinned + project + global). Slots are editable, size-limited memory units the agent can read and modify across sessions. | | `memory_slot_replace` | | `label`*: string, `content`*: string | Replace slot content in place. Fails if content exceeds sizeLimit. | -| `memory_smart_search` | yes | `query`*: string, `expandIds`: string, `limit`: number | Hybrid semantic+keyword search with progressive disclosure. | +| `memory_smart_search` | yes | `query`*: string, `expandIds`: string, `limit`: number, `project`: string, `repoId`: string, `checkoutId`: string | Hybrid semantic+keyword search with progressive disclosure. | | `memory_snapshot_create` | | `message`: string | Create a git-versioned snapshot of current memory state. | | `memory_team_feed` | | `limit`: number | Get recent shared items from all team members. | | `memory_team_share` | | `itemId`*: string, `itemType`*: string | Share a memory or observation with team members. | diff --git a/plugin/skills/agentmemory-rest-api/REFERENCE.md b/plugin/skills/agentmemory-rest-api/REFERENCE.md index ed8bbf7c92..49c4349f16 100644 --- a/plugin/skills/agentmemory-rest-api/REFERENCE.md +++ b/plugin/skills/agentmemory-rest-api/REFERENCE.md @@ -5,7 +5,7 @@ Generated from `src/triggers/api.ts`. Do not edit the block below by hand; run ` The REST API is the primary surface. All paths are under `http://localhost:3111` (override with `--port`). When `AGENTMEMORY_SECRET` is set, send `Authorization: Bearer $AGENTMEMORY_SECRET`; localhost is otherwise open. -121 registered endpoints: +125 registered endpoints: | Method | Path | | --- | --- | @@ -39,6 +39,7 @@ The REST API is the primary surface. All paths are under `http://localhost:3111` | GET | `/agentmemory/durable-candidates` | | POST | `/agentmemory/durable-candidates/backfill` | | POST | `/agentmemory/durable-candidates/promote` | +| POST | `/agentmemory/durable-candidates/recommend` | | POST | `/agentmemory/enrich` | | POST | `/agentmemory/evict` | | POST | `/agentmemory/evolve` | @@ -85,6 +86,9 @@ The REST API is the primary surface. All paths are under `http://localhost:3111` | POST | `/agentmemory/patterns` | | GET | `/agentmemory/procedural` | | GET | `/agentmemory/profile` | +| POST | `/agentmemory/recall/context-epoch` | +| GET | `/agentmemory/recall/debug` | +| GET | `/agentmemory/recall/debug/:traceId` | | POST | `/agentmemory/reflect` | | POST | `/agentmemory/relations` | | POST | `/agentmemory/remember` | diff --git a/plugin/skills/recall-debug/SKILL.md b/plugin/skills/recall-debug/SKILL.md new file mode 100644 index 0000000000..e8394ea616 --- /dev/null +++ b/plugin/skills/recall-debug/SKILL.md @@ -0,0 +1,46 @@ +--- +name: recall-debug +description: Inspect agentmemory recall traces to explain selected and dropped context. Use when the user asks why a memory was recalled, why context was omitted, or whether retrieval degraded. +argument-hint: "[trace id, memory id, or query]" +user-invocable: true +--- + +## Quick start + +Call `memory_recall` with `outputMode: "structured"`, then inspect its trace id +with the recall debug endpoint or `memory_why`. + +## Why + +Use persisted evidence, not inferred explanations. A trace is authoritative for +channel health, scope decisions, budget drops, and selected context. + +## Workflow + +1. Resolve the supplied trace id or memory id. +2. Read the recall trace and lead with entry point, output mode, and token use. +3. Explain each selected item using its reason and channel scores. +4. Summarize dropped counts and show the relevant top sample for each reason. +5. State vector fallback whenever vector status is degraded or disabled. + +## Anti-patterns + +WRONG: call a fresh search and claim it explains an earlier injection. + +RIGHT: inspect the stored trace that produced the earlier injection. + +## Checklist + +- Selected and dropped reasons come from the trace. +- Scope mismatch is called out explicitly. +- Token estimator and final token count are reported. +- No raw query is reconstructed from its fingerprint. + +## See also + +- `recall`: retrieve relevant memory. +- `remember`: save an explicit durable memory. + +## Troubleshooting + +See ../_shared/TROUBLESHOOTING.md if the recall debug endpoint is unavailable. diff --git a/plugin/skills/why-memory/SKILL.md b/plugin/skills/why-memory/SKILL.md new file mode 100644 index 0000000000..54263e6cd1 --- /dev/null +++ b/plugin/skills/why-memory/SKILL.md @@ -0,0 +1,44 @@ +--- +name: why-memory +description: Explain a memory's recent recall history and scope decisions from agentmemory traces. Use when the user asks why a specific memory was or was not recalled. +argument-hint: "" +user-invocable: true +--- + +## Quick start + +Find the memory id, then query recall debug with `itemId` set to that id. + +## Why + +Memory provenance and recall history answer different questions. Inspect both the +memory record and its traces before drawing a conclusion. + +## Workflow + +1. Require a memory id; do not guess one from title text. +2. Read the memory record and recall stats. +3. Query recall debug filtered by that item id. +4. Separate selected traces from scope mismatch, stale, duplicate, and budget drops. +5. Show source sessions and observations when available. + +## Anti-patterns + +WRONG: describe a memory as global because it was saved manually. + +RIGHT: report its explicit scope and origin from the memory record. + +## Checklist + +- Scope and origin are shown. +- Recent selected and dropped evidence is separated. +- Unknown scope is never described as global. + +## See also + +- `recall-debug`: inspect one recall invocation. +- `recall`: retrieve context for a query. + +## Troubleshooting + +See ../_shared/TROUBLESHOOTING.md if memory lookup is unavailable. diff --git a/src/cli/connect/index.ts b/src/cli/connect/index.ts index 0d9412ca64..2289be7f55 100644 --- a/src/cli/connect/index.ts +++ b/src/cli/connect/index.ts @@ -201,7 +201,7 @@ function summarize( ); if (wiredAny) { p.log.info( - "Next: install agentmemory's 15 skills into the same agent(s) so they know when to call the tools:\n npx skills add rohitg00/agentmemory -y", + "Next: install agentmemory's 17 skills into the same agent(s) so they know when to call the tools:\n npx skills add rohitg00/agentmemory -y", ); } diff --git a/src/config.ts b/src/config.ts index f68da2e31d..3162e86210 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,6 +1,7 @@ import { existsSync, readFileSync } from "node:fs"; import { join } from "node:path"; import { homedir } from "node:os"; +import { parse as parseToml } from "smol-toml"; import type { AgentMemoryConfig, ProviderConfig, @@ -8,6 +9,7 @@ import type { FallbackConfig, ClaudeBridgeConfig, TeamConfig, + RecallConfig, } from "./types.js"; function safeParseInt(value: string | undefined, fallback: number): number { @@ -18,6 +20,31 @@ function safeParseInt(value: string | undefined, fallback: number): number { const DATA_DIR = join(homedir(), ".agentmemory"); const ENV_FILE = join(DATA_DIR, ".env"); +const CONFIG_FILE = join(DATA_DIR, "config.toml"); + +const DEFAULT_RECALL_CONFIG: RecallConfig = { + budget: { + maxContextTokens: 800, + reservedBootstrapTokens: 200, + maxSemanticTokens: 600, + maxMemories: 5, + maxSessionSummaries: 1, + maxObservations: 3, + maxContinuityItems: 1, + }, + scope: { + unknownAutoInjection: false, + unknownExplicitSearch: true, + }, + trace: { + retentionDays: 30, + maxTraces: 10_000, + maxDroppedItemsPerReason: 20, + }, + injection: { + reinjectionTurnWindow: 8, + }, +}; let warnPremiumModelShown = false; @@ -45,6 +72,176 @@ function loadEnvFile(): Record { return vars; } +function loadTomlFile(): Record { + if (!existsSync(CONFIG_FILE)) return {}; + try { + const parsed = parseToml(readFileSync(CONFIG_FILE, "utf-8")); + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { + throw new Error("top-level document must be a table"); + } + return parsed as Record; + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + throw new Error(`Invalid ${CONFIG_FILE}: ${message}`); + } +} + +function nestedTomlValue( + source: Record, + section: string, + key: string, +): unknown { + const table = source[section]; + if (!table || typeof table !== "object" || Array.isArray(table)) return undefined; + return (table as Record)[key]; +} + +function positiveConfigInt( + value: unknown, + fallback: number, + label: string, +): number { + if (value === undefined) return fallback; + if (typeof value !== "number" || !Number.isInteger(value) || value < 1) { + throw new Error(`${label} must be a positive integer`); + } + return value; +} + +function booleanConfigValue( + value: unknown, + fallback: boolean, + label: string, +): boolean { + if (value === undefined) return fallback; + if (typeof value !== "boolean") throw new Error(`${label} must be a boolean`); + return value; +} + +function envOrToml( + env: Record, + envKey: string, + toml: Record, + section: string, + key: string, +): unknown { + if (env[envKey] !== undefined) return Number(env[envKey]); + return nestedTomlValue(toml, section, key); +} + +function envOrTomlBoolean( + env: Record, + envKey: string, + toml: Record, + section: string, + key: string, +): unknown { + if (env[envKey] !== undefined) { + if (env[envKey] === "true") return true; + if (env[envKey] === "false") return false; + return env[envKey]; + } + return nestedTomlValue(toml, section, key); +} + +export function loadRecallConfig( + overrides?: Record, +): RecallConfig { + const env = getMergedEnv(overrides); + const toml = loadTomlFile(); + const budget = DEFAULT_RECALL_CONFIG.budget; + const scope = DEFAULT_RECALL_CONFIG.scope; + const trace = DEFAULT_RECALL_CONFIG.trace; + const injection = DEFAULT_RECALL_CONFIG.injection; + const maxContextTokens = positiveConfigInt( + env["AGENTMEMORY_RECALL_MAX_CONTEXT_TOKENS"] !== undefined + ? Number(env["AGENTMEMORY_RECALL_MAX_CONTEXT_TOKENS"]) + : env["TOKEN_BUDGET"] !== undefined + ? Number(env["TOKEN_BUDGET"]) + : nestedTomlValue(toml, "recall_budget", "max_context_tokens"), + budget.maxContextTokens, + "recall_budget.max_context_tokens", + ); + const result: RecallConfig = { + budget: { + maxContextTokens, + reservedBootstrapTokens: positiveConfigInt( + envOrToml(env, "AGENTMEMORY_RECALL_RESERVED_BOOTSTRAP_TOKENS", toml, "recall_budget", "reserved_bootstrap_tokens"), + budget.reservedBootstrapTokens, + "recall_budget.reserved_bootstrap_tokens", + ), + maxSemanticTokens: positiveConfigInt( + envOrToml(env, "AGENTMEMORY_RECALL_MAX_SEMANTIC_TOKENS", toml, "recall_budget", "max_semantic_tokens"), + budget.maxSemanticTokens, + "recall_budget.max_semantic_tokens", + ), + maxMemories: positiveConfigInt( + envOrToml(env, "AGENTMEMORY_RECALL_MAX_MEMORIES", toml, "recall_budget", "max_memories"), + budget.maxMemories, + "recall_budget.max_memories", + ), + maxSessionSummaries: positiveConfigInt( + envOrToml(env, "AGENTMEMORY_RECALL_MAX_SESSION_SUMMARIES", toml, "recall_budget", "max_session_summaries"), + budget.maxSessionSummaries, + "recall_budget.maxSessionSummaries", + ), + maxObservations: positiveConfigInt( + envOrToml(env, "AGENTMEMORY_RECALL_MAX_OBSERVATIONS", toml, "recall_budget", "max_observations"), + budget.maxObservations, + "recall_budget.max_observations", + ), + maxContinuityItems: positiveConfigInt( + envOrToml(env, "AGENTMEMORY_RECALL_MAX_CONTINUITY_ITEMS", toml, "recall_budget", "max_continuity_items"), + budget.maxContinuityItems, + "recall_budget.max_continuity_items", + ), + }, + scope: { + unknownAutoInjection: booleanConfigValue( + envOrTomlBoolean(env, "AGENTMEMORY_RECALL_UNKNOWN_AUTO_INJECTION", toml, "recall_scope", "unknown_auto_injection"), + scope.unknownAutoInjection, + "recall_scope.unknown_auto_injection", + ), + unknownExplicitSearch: booleanConfigValue( + envOrTomlBoolean(env, "AGENTMEMORY_RECALL_UNKNOWN_EXPLICIT_SEARCH", toml, "recall_scope", "unknown_explicit_search"), + scope.unknownExplicitSearch, + "recall_scope.unknown_explicit_search", + ), + }, + trace: { + retentionDays: positiveConfigInt( + envOrToml(env, "AGENTMEMORY_RECALL_TRACE_RETENTION_DAYS", toml, "recall_trace", "retention_days"), + trace.retentionDays, + "recall_trace.retention_days", + ), + maxTraces: positiveConfigInt( + envOrToml(env, "AGENTMEMORY_RECALL_TRACE_MAX_TRACES", toml, "recall_trace", "max_traces"), + trace.maxTraces, + "recall_trace.max_traces", + ), + maxDroppedItemsPerReason: positiveConfigInt( + envOrToml(env, "AGENTMEMORY_RECALL_TRACE_MAX_DROPPED_ITEMS_PER_REASON", toml, "recall_trace", "max_dropped_items_per_reason"), + trace.maxDroppedItemsPerReason, + "recall_trace.max_dropped_items_per_reason", + ), + }, + injection: { + reinjectionTurnWindow: positiveConfigInt( + envOrToml(env, "AGENTMEMORY_RECALL_REINJECTION_TURN_WINDOW", toml, "recall_injection", "reinjection_turn_window"), + injection.reinjectionTurnWindow, + "recall_injection.reinjection_turn_window", + ), + }, + }; + if (result.budget.reservedBootstrapTokens > result.budget.maxContextTokens) { + throw new Error("recall_budget.reserved_bootstrap_tokens must not exceed max_context_tokens"); + } + if (result.budget.maxSemanticTokens > result.budget.maxContextTokens) { + throw new Error("recall_budget.max_semantic_tokens must not exceed max_context_tokens"); + } + return result; +} + function hasRealValue(v: string | undefined): v is string { return typeof v === "string" && v.trim().length > 0; } @@ -184,6 +381,7 @@ export function loadConfig(): AgentMemoryConfig { maxObservationsPerSession: safeParseInt(env["MAX_OBS_PER_SESSION"], 500), compressionModel: provider.model, dataDir: DATA_DIR, + recall: loadRecallConfig(), }; } diff --git a/src/functions/context.ts b/src/functions/context.ts index 1e6102cf37..6129ad2f97 100644 --- a/src/functions/context.ts +++ b/src/functions/context.ts @@ -17,6 +17,7 @@ import { listPinnedSlots, renderPinnedContext, } from "./slots.js"; +import type { RecallCore } from "../recall/core.js"; function estimateTokens(text: string): number { return Math.ceil(text.length / 3); @@ -34,9 +35,41 @@ export function registerContextFunction( sdk: ISdk, kv: StateKV, tokenBudget: number, + recallCore?: RecallCore, ): void { sdk.registerFunction("mem::context", - async (data: { sessionId: string; project: string; budget?: number }) => { + async (data: { + sessionId: string; + project: string; + budget?: number; + query?: string; + projectId?: string; + repoId?: string; + checkoutId?: string; + outputMode?: "bootstrap" | "prompt_injection" | "rendered_context" | "ranked_results"; + entryPoint?: "context" | "search" | "smart_search" | "memory_recall" | "prompt" | "session_start" | "enrich"; + debug?: boolean; + }) => { + if (recallCore) { + const result = await recallCore.recall({ + entryPoint: data.entryPoint || "context", + outputMode: data.outputMode || "rendered_context", + query: data.query, + sessionId: data.sessionId, + projectId: data.projectId || data.project, + repoId: data.repoId, + checkoutId: data.checkoutId, + ...(data.budget ? { budget: { maxContextTokens: data.budget } } : {}), + debug: data.debug, + }); + return { + context: result.context, + blocks: result.results.length, + tokens: result.trace.finalContextTokenCount, + traceId: result.trace.id, + ...(data.debug ? { trace: result.trace } : {}), + }; + } const budget = data.budget || tokenBudget; const blocks: ContextBlock[] = []; diff --git a/src/functions/durable-candidates.ts b/src/functions/durable-candidates.ts index 85439af9d2..e12d7e72b3 100644 --- a/src/functions/durable-candidates.ts +++ b/src/functions/durable-candidates.ts @@ -6,11 +6,13 @@ import type { ISdk } from "iii-sdk"; import type { ArchiveImportRecord, DurableCandidate, + DurableCandidateRecommendation, + Memory, Session, SessionSummary, } from "../types.js"; import { StateKV } from "../state/kv.js"; -import { KV, fingerprintId } from "../state/schema.js"; +import { KV, fingerprintId, jaccardSimilarity } from "../state/schema.js"; import { DURABLE_PROMOTE_MIN_CONFIDENCE, bucketCandidateConfidence, @@ -97,6 +99,12 @@ function collectDurableCandidates( }); } +function isPlanOrTodo(candidate: DurableCandidate): boolean { + return /\b(todo|to[- ]do|plan|planned|next step|follow[- ]up)\b/i.test( + `${candidate.title}\n${candidate.content}`, + ); +} + async function readArchiveTargets( rawPath: string, archiveRoot: string, @@ -253,6 +261,43 @@ export function registerDurableCandidateFunctions( }, ); + sdk.registerFunction( + "mem::durable-candidates::recommend", + async (data: { candidateId?: string; project?: string } = {}) => { + const summaries = await kv.list(KV.summaries); + const all = collectDurableCandidates(summaries) + .filter((candidate) => !candidate.promotedMemoryId) + .filter((candidate) => !data.candidateId || candidate.id === data.candidateId) + .filter((candidate) => !data.project || candidate.project === data.project); + const memories = await kv.list(KV.memories); + const recommendations: DurableCandidateRecommendation[] = []; + for (const candidate of all) { + const reasons: string[] = []; + if (candidate.confidence >= 0.9) reasons.push("confidence >= 0.90"); + if (candidate.project) reasons.push("project scope explicit"); + if (candidate.sourceObservationIds.length >= 3) reasons.push("3 evidence observations"); + const hasConflict = memories.some((memory) => + memory.isLatest !== false && + memory.project === candidate.project && + jaccardSimilarity(memory.content.toLowerCase(), candidate.content.toLowerCase()) > 0.7, + ); + if (!hasConflict) reasons.push("no conflicting memory"); + if (!isPlanOrTodo(candidate)) reasons.push("not a plan or todo"); + const eligible = reasons.length === 5; + const recommendation: DurableCandidateRecommendation = { + candidateId: candidate.id, + recommendation: eligible ? "auto_promote_eligible" : "not_eligible", + reasons, + wouldPromote: false, + evaluatedAt: new Date().toISOString(), + }; + await kv.set(KV.durableRecommendations, candidate.id, recommendation); + recommendations.push(recommendation); + } + return { success: true, recommendations }; + }, + ); + sdk.registerFunction( "mem::durable-candidates::promote", async (data: { diff --git a/src/functions/enrich.ts b/src/functions/enrich.ts index 032ec97b47..2df1547a60 100644 --- a/src/functions/enrich.ts +++ b/src/functions/enrich.ts @@ -3,6 +3,7 @@ import type { Memory } from "../types.js"; import { KV } from "../state/schema.js"; import { StateKV } from "../state/kv.js"; import { logger } from "../logger.js"; +import type { RecallCore } from "../recall/core.js"; const MAX_CONTEXT_LENGTH = 4000; @@ -15,7 +16,7 @@ function escapeXml(s: string): string { .replace(/'/g, "'"); } -export function registerEnrichFunction(sdk: ISdk, kv: StateKV): void { +export function registerEnrichFunction(sdk: ISdk, kv: StateKV, recallCore?: RecallCore): void { sdk.registerFunction("mem::enrich", async (data: { sessionId: string; @@ -23,12 +24,36 @@ export function registerEnrichFunction(sdk: ISdk, kv: StateKV): void { terms?: string[]; toolName?: string; project?: string; + repoId?: string; + checkoutId?: string; }) => { const project = typeof data.project === "string" && data.project.trim().length > 0 ? data.project.trim() : undefined; + if (recallCore) { + const query = [ + ...data.files.map((file) => file.split("/").pop() || file), + ...(data.terms || []), + ].filter(Boolean).join(" "); + if (!query) return { context: "", truncated: false }; + const result = await recallCore.recall({ + entryPoint: "enrich", + outputMode: "prompt_injection", + query, + sessionId: data.sessionId, + projectId: project, + repoId: data.repoId, + checkoutId: data.checkoutId, + }); + return { + context: result.context, + truncated: false, + traceId: result.trace.id, + }; + } + const parts: string[] = []; const fileContextPromise = sdk diff --git a/src/functions/remember.ts b/src/functions/remember.ts index 8558a19d07..40939ab2eb 100644 --- a/src/functions/remember.ts +++ b/src/functions/remember.ts @@ -9,6 +9,7 @@ import { recordAudit } from "./audit.js"; import { getSearchIndex, vectorIndexAddGuarded, vectorIndexRemove, flushIndexSave } from "./search.js"; import { getAgentId } from "../config.js"; import { logger } from "../logger.js"; +import { normalizeScope, sameScope } from "../recall/scope.js"; export function registerRememberFunction(sdk: ISdk, kv: StateKV): void { sdk.registerFunction("mem::remember", @@ -26,6 +27,9 @@ export function registerRememberFunction(sdk: ISdk, kv: StateKV): void { strength?: number; agentId?: string; project?: string; + scope?: Memory["scope"]; + origin?: Memory["origin"]; + checkoutId?: string; }) => { if ( !data.content || @@ -66,6 +70,19 @@ export function registerRememberFunction(sdk: ISdk, kv: StateKV): void { typeof data.project === "string" && data.project.trim().length > 0 ? data.project.trim() : undefined; + const requestedScope = normalizeScope(data.scope); + const scope = + requestedScope.level !== "unknown" + ? requestedScope + : project + ? { level: "project" as const, projectId: project } + : requestedScope; + if (scope.level === "user" && memType !== "preference") { + return { + success: false, + error: "user scope is restricted to explicit preference memories", + }; + } return withKeyedLock("mem:remember", async () => { const existingMemories = await kv.list(KV.memories); @@ -79,7 +96,7 @@ export function registerRememberFunction(sdk: ISdk, kv: StateKV): void { // Both sides must have an explicit project for the guard to engage; // an unscoped memory (legacy, no project field) is treated as a // wildcard so pre-existing data is not stranded. - if (project && existing.project && existing.project !== project) { + if (!sameScope(normalizeScope(existing.scope), scope)) { continue; } const similarity = jaccardSimilarity( @@ -124,6 +141,12 @@ export function registerRememberFunction(sdk: ISdk, kv: StateKV): void { data.sourceCandidateId.trim().length > 0 ? data.sourceCandidateId.trim() : undefined; + const origin = + data.origin === "system" || data.origin === "candidate_promoted" + ? data.origin + : sourceCandidateId + ? "candidate_promoted" + : "manual"; const memory: Memory = { id: generateId("mem"), @@ -147,6 +170,11 @@ export function registerRememberFunction(sdk: ISdk, kv: StateKV): void { isLatest: true, ...(callAgentId ? { agentId: callAgentId } : {}), ...(project !== undefined && { project }), + scope, + origin, + ...(typeof data.checkoutId === "string" && data.checkoutId.trim() + ? { checkoutId: data.checkoutId.trim() } + : {}), }; if (data.ttlDays && typeof data.ttlDays === "number" && data.ttlDays > 0) { diff --git a/src/functions/slots.ts b/src/functions/slots.ts index 47b49d496f..8f475b33b5 100644 --- a/src/functions/slots.ts +++ b/src/functions/slots.ts @@ -234,6 +234,8 @@ export function registerSlotsFunctions(sdk: ISdk, kv: StateKV): void { description?: string; pinned?: boolean; scope?: SlotScope; + projectId?: string; + repoId?: string; }) => { const label = validateLabel(data?.label); if (!label) return { success: false, error: "label required (lowercase, starts with letter, [a-z0-9_])" }; @@ -249,6 +251,12 @@ export function registerSlotsFunctions(sdk: ISdk, kv: StateKV): void { } const description = typeof data?.description === "string" ? data.description : ""; const pinned = typeof data?.pinned === "boolean" ? data.pinned : true; + const projectId = typeof data?.projectId === "string" && data.projectId.trim() + ? data.projectId.trim() + : undefined; + const repoId = typeof data?.repoId === "string" && data.repoId.trim() + ? data.repoId.trim() + : undefined; return withKeyedLock(`slot:${label}`, async () => { // Duplicate check is scope-local so a project slot can shadow a // global slot with the same label — matches the read precedence. @@ -265,6 +273,8 @@ export function registerSlotsFunctions(sdk: ISdk, kv: StateKV): void { scope, createdAt: ts, updatedAt: ts, + ...(projectId ? { projectId } : {}), + ...(repoId ? { repoId } : {}), }; await kv.set(scopeKv(scope), label, slot); await recordAudit(kv, "slot_create", "mem::slot-create", [label], { diff --git a/src/hooks/pre-compact.ts b/src/hooks/pre-compact.ts index 8bb2660c48..e9e962cfa9 100644 --- a/src/hooks/pre-compact.ts +++ b/src/hooks/pre-compact.ts @@ -48,10 +48,21 @@ async function main() { } try { + await fetch(`${REST_URL}/agentmemory/recall/context-epoch`, { + method: "POST", + headers: authHeaders(), + body: JSON.stringify({ sessionId }), + signal: AbortSignal.timeout(1000), + }); const res = await fetch(`${REST_URL}/agentmemory/context`, { method: "POST", headers: authHeaders(), - body: JSON.stringify({ sessionId, project, budget: 1500 }), + body: JSON.stringify({ + sessionId, + project, + cwd: (data.cwd as string | undefined) || process.cwd(), + outputMode: "bootstrap", + }), signal: AbortSignal.timeout(5000), }); diff --git a/src/hooks/pre-tool-use.ts b/src/hooks/pre-tool-use.ts index 277d7799be..e3e8b75451 100644 --- a/src/hooks/pre-tool-use.ts +++ b/src/hooks/pre-tool-use.ts @@ -107,6 +107,7 @@ async function main() { files, terms, toolName, + cwd: typeof data.cwd === "string" ? data.cwd : process.cwd(), ...(project !== undefined && { project }), }), signal: AbortSignal.timeout(2000), diff --git a/src/hooks/prompt-submit.ts b/src/hooks/prompt-submit.ts index 2da8ea6a41..14843eb162 100644 --- a/src/hooks/prompt-submit.ts +++ b/src/hooks/prompt-submit.ts @@ -9,6 +9,7 @@ function isSdkChildContext(payload: unknown): boolean { const REST_URL = process.env["AGENTMEMORY_URL"] || "http://localhost:3111"; const SECRET = process.env["AGENTMEMORY_SECRET"] || ""; +const INJECT_CONTEXT = process.env["AGENTMEMORY_INJECT_CONTEXT"] === "true"; function authHeaders(): Record { const h: Record = { "Content-Type": "application/json" }; @@ -33,20 +34,50 @@ async function main() { const sessionId = ((data.session_id || data.sessionId) as string) || "unknown"; - fetch(`${REST_URL}/agentmemory/observe`, { + const project = resolveProject(data.cwd as string | undefined); + const prompt = typeof data.prompt === "string" + ? data.prompt + : typeof data.userPrompt === "string" + ? data.userPrompt + : ""; + const observe = fetch(`${REST_URL}/agentmemory/observe`, { method: "POST", headers: authHeaders(), body: JSON.stringify({ hookType: "prompt_submit", sessionId, - project: resolveProject(data.cwd as string | undefined), + project, cwd: (data.cwd as string | undefined) || process.cwd(), timestamp: new Date().toISOString(), - data: { prompt: data.prompt ?? data.userPrompt }, + data: { prompt }, }), signal: AbortSignal.timeout(3000), }).catch(() => {}); - setTimeout(() => process.exit(0), 500).unref(); + if (!INJECT_CONTEXT || !prompt) { + void observe; + return; + } + try { + const response = await fetch(`${REST_URL}/agentmemory/context`, { + method: "POST", + headers: authHeaders(), + body: JSON.stringify({ + sessionId, + project, + cwd: (data.cwd as string | undefined) || process.cwd(), + query: prompt, + outputMode: "prompt_injection", + }), + signal: AbortSignal.timeout(1500), + }); + if (response.ok) { + const result = await response.json() as { context?: unknown }; + if (typeof result.context === "string" && result.context) { + process.stdout.write(result.context); + } + } + } catch {} + void observe; } main(); diff --git a/src/index.ts b/src/index.ts index 58cc7ce336..51241e820b 100644 --- a/src/index.ts +++ b/src/index.ts @@ -23,6 +23,7 @@ import { StateKV } from "./state/kv.js"; import { KV } from "./state/schema.js"; import { VectorIndex } from "./state/vector-index.js"; import { HybridSearch } from "./state/hybrid-search.js"; +import { RecallCore } from "./recall/core.js"; import { IndexPersistence } from "./state/index-persistence.js"; import { registerPrivacyFunction } from "./functions/privacy.js"; import { registerObserveFunction } from "./functions/observe.js"; @@ -243,7 +244,6 @@ async function main() { registerDiskSizeManager(sdk, kv); registerCompressFunction(sdk, kv, provider, metricsStore); registerSearchFunction(sdk, kv); - registerContextFunction(sdk, kv, config.tokenBudget); registerSummarizeFunction(sdk, kv, provider, metricsStore); registerMigrateFunction(sdk, kv); registerFileIndexFunction(sdk, kv); @@ -257,7 +257,6 @@ async function main() { registerProfileFunction(sdk, kv); registerAutoForgetFunction(sdk, kv); registerExportImportFunction(sdk, kv); - registerEnrichFunction(sdk, kv); const claudeBridgeConfig = loadClaudeBridgeConfig(); if (claudeBridgeConfig.enabled) { @@ -366,6 +365,14 @@ async function main() { graphWeight, ); + const recallCore = new RecallCore( + kv, + config.recall, + (query, limit) => hybridSearch.searchWithTrace(query, limit), + ); + registerContextFunction(sdk, kv, config.tokenBudget, recallCore); + registerEnrichFunction(sdk, kv, recallCore); + registerSmartSearchFunction(sdk, kv, (query, limit) => hybridSearch.search(query, limit), ); @@ -520,7 +527,7 @@ async function main() { `Ready. ${embeddingProvider ? "Triple-stream (BM25+Vector+Graph)" : "BM25+Graph"} search active.`, ); bootLog( - `REST API: 132 endpoints at http://localhost:${config.restPort}/agentmemory/*`, + `REST API: 136 endpoints at http://localhost:${config.restPort}/agentmemory/*`, ); bootLog( `MCP surface (opt-in via \`npx @agentmemory/mcp\`): ${getAllTools().length} tools · 6 resources · 3 prompts`, diff --git a/src/mcp/server.ts b/src/mcp/server.ts index dbca07d9b8..404664b3e5 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -121,6 +121,25 @@ export function registerMcpEndpoints( typeof args.agentId === "string" && args.agentId.trim().length > 0 ? (args.agentId as string).trim() : undefined; + const outputMode = typeof args.outputMode === "string" ? args.outputMode.trim().toLowerCase() : undefined; + if (outputMode === "structured" || outputMode === "context") { + const result = await sdk.trigger({ function_id: "mem::context", payload: { + sessionId: typeof args.sessionId === "string" ? args.sessionId : "explicit-recall", + project: typeof args.project === "string" ? args.project : "", + projectId: typeof args.project === "string" ? args.project : undefined, + repoId: typeof args.repoId === "string" ? args.repoId : undefined, + checkoutId: typeof args.checkoutId === "string" ? args.checkoutId : undefined, + query: args.query, + entryPoint: "memory_recall", + outputMode: outputMode === "structured" ? "ranked_results" : "rendered_context", + ...(tokenBudget ? { budget: tokenBudget } : {}), + debug: true, + } }); + return { + status_code: 200, + body: { content: [{ type: "text", text: JSON.stringify(result, null, 2) }] }, + }; + } const result = await sdk.trigger({ function_id: "mem::search", payload: { query: args.query, limit: typeof args.limit === "number" ? args.limit : 10, @@ -273,12 +292,28 @@ export function registerMcpEndpoints( } const expandIds = parseCsvList(args.expandIds).slice(0, 20); const limit = Math.max(1, Math.min(100, asNumber(args.limit, 10) ?? 10)); + if (typeof args.project === "string" && args.project.trim()) { + const result = await sdk.trigger({ function_id: "mem::context", payload: { + sessionId: typeof args.sessionId === "string" ? args.sessionId : "smart-search", + project: args.project, + projectId: args.project, + repoId: typeof args.repoId === "string" ? args.repoId : undefined, + checkoutId: typeof args.checkoutId === "string" ? args.checkoutId : undefined, + query: args.query, + entryPoint: "smart_search", + outputMode: "ranked_results", + debug: true, + limit, + } }); + return { status_code: 200, body: { content: [{ type: "text", text: JSON.stringify(result, null, 2) }] } }; + } const result = await sdk.trigger({ function_id: "mem::smart-search", payload: { query: args.query, expandIds, limit, + project: typeof args.project === "string" ? args.project : undefined, }, }); return { diff --git a/src/mcp/tools-registry.ts b/src/mcp/tools-registry.ts index b892d711d4..4bcfb9921e 100644 --- a/src/mcp/tools-registry.ts +++ b/src/mcp/tools-registry.ts @@ -32,6 +32,13 @@ export const CORE_TOOLS: McpToolDef[] = [ type: "number", description: "Optional token budget to trim returned results", }, + outputMode: { + type: "string", + description: "structured returns ranked traceable results; context returns token-packed rendered context", + }, + project: { type: "string", description: "Project scope identifier" }, + repoId: { type: "string", description: "Repository identity fingerprint" }, + checkoutId: { type: "string", description: "Checkout identity fingerprint" }, }, required: ["query"], }, @@ -130,6 +137,9 @@ export const CORE_TOOLS: McpToolDef[] = [ description: "Comma-separated observation IDs to expand", }, limit: { type: "number", description: "Max results (default 10)" }, + project: { type: "string", description: "Project scope identifier" }, + repoId: { type: "string", description: "Repository identity fingerprint" }, + checkoutId: { type: "string", description: "Checkout identity fingerprint" }, }, required: ["query"], }, diff --git a/src/recall/core.ts b/src/recall/core.ts new file mode 100644 index 0000000000..920293b136 --- /dev/null +++ b/src/recall/core.ts @@ -0,0 +1,420 @@ +import type { + HybridSearchResult, + Memory, + ProjectProfile, + RecallConfig, + RecallDecision, + RecallItemKind, + RecallItemTrace, + RecallRequest, + RecallScope, + RecallTrace, + Session, + SessionSummary, + MemorySlot, + RetrievalChannelStatus, +} from "../types.js"; +import { KV, generateId } from "../state/schema.js"; +import type { StateKV } from "../state/kv.js"; +import { evaluateScope, memoryScope } from "./scope.js"; +import { countTokens, getTokenEstimator } from "./tokens.js"; +import { persistRecallTrace, redactQuery } from "./trace-store.js"; +import type { HybridSearchResponse } from "../state/hybrid-search.js"; +import { currentRecallTurn, isDuplicateInjection, recordInjection } from "./ledger.js"; + +interface RecallCandidate { + id: string; + kind: RecallItemKind; + subkind?: RecallItemTrace["subkind"]; + content: string; + score: number; + bm25Score?: number; + vectorScore?: number; + graphScore?: number; + recencyScore: number; + scope: RecallScope; + sourceSessionIds?: string[]; + sourceObservationIds?: string[]; + isLatest?: boolean; + forgetAfter?: string; + version?: number; + bootstrap?: boolean; +} + +export interface RecallResult { + context: string; + results: RecallItemTrace[]; + trace: RecallTrace; +} + +export type HybridRecall = (query: string, limit: number) => Promise; + +function unavailableRetrievalMode(): { + bm25: RetrievalChannelStatus; + vector: RetrievalChannelStatus; + graph: RetrievalChannelStatus; +} { + return { + bm25: { status: "healthy", attempted: false }, + vector: { status: "disabled", attempted: false, reason: "hybrid retrieval was not configured" }, + graph: { status: "disabled", attempted: false, reason: "hybrid retrieval was not configured" }, + }; +} + +function lexicalScore(query: string, text: string): number { + const terms = query.toLowerCase().split(/\s+/).filter((term) => term.length > 1); + if (terms.length === 0) return 0; + const haystack = text.toLowerCase(); + const hits = terms.filter((term) => haystack.includes(term)).length; + return hits / terms.length; +} + +function recencyScore(timestamp: string): number { + const ageDays = Math.max(0, Date.now() - new Date(timestamp).getTime()) / 86_400_000; + return Math.max(0, 0.03 * (1 / (1 + ageDays / 30))); +} + +function renderProfile(profile: ProjectProfile): string { + const parts: string[] = []; + if (profile.summary) parts.push(profile.summary); + if (profile.conventions.length > 0) parts.push(`Conventions: ${profile.conventions.join("; ")}`); + if (profile.commonErrors.length > 0) parts.push(`Common errors: ${profile.commonErrors.join("; ")}`); + return parts.join("\n"); +} + +function traceItem(candidate: RecallCandidate, decision: RecallDecision, reason: string, score = candidate.score): RecallItemTrace { + return { + id: candidate.id, + kind: candidate.kind, + ...(candidate.subkind ? { subkind: candidate.subkind } : {}), + score, + ...(candidate.bm25Score !== undefined ? { bm25Score: candidate.bm25Score } : {}), + ...(candidate.vectorScore !== undefined ? { vectorScore: candidate.vectorScore } : {}), + ...(candidate.graphScore !== undefined ? { graphScore: candidate.graphScore } : {}), + recencyScore: candidate.recencyScore, + tokenCount: countTokens(candidate.content), + reason, + ...(candidate.sourceSessionIds ? { sourceSessionIds: candidate.sourceSessionIds } : {}), + ...(candidate.sourceObservationIds ? { sourceObservationIds: candidate.sourceObservationIds } : {}), + decision, + }; +} + +function normalizeDuplicate(content: string): string { + return content.toLowerCase().replace(/\s+/g, " ").trim(); +} + +export class RecallCore { + constructor( + private readonly kv: StateKV, + private readonly config: RecallConfig, + private readonly hybridRecall?: HybridRecall, + ) {} + + async recall(request: RecallRequest): Promise { + const estimator = getTokenEstimator(request.tokenizerHint); + const redactedQuery = await redactQuery(this.kv, request.query); + const injectionState = request.outputMode === "prompt_injection" + ? await currentRecallTurn(this.kv, request.sessionId, request.entryPoint === "prompt") + : null; + const dropped: RecallItemTrace[] = []; + const droppedCounts: Partial> = {}; + const drop = (candidate: RecallCandidate, decision: RecallDecision, reason: string) => { + droppedCounts[decision] = (droppedCounts[decision] || 0) + 1; + const sameDecision = dropped.filter((item) => item.decision === decision); + if (sameDecision.length < this.config.trace.maxDroppedItemsPerReason) { + dropped.push(traceItem(candidate, decision, reason)); + } + }; + + const bootstrap = request.outputMode === "bootstrap" || request.outputMode === "prompt_injection" || request.outputMode === "rendered_context" + ? await this.collectBootstrap(request) + : []; + const semanticResponse = request.outputMode === "bootstrap" + ? { candidates: [], retrievalMode: unavailableRetrievalMode() } + : await this.collectSemantic(request); + const semantic = semanticResponse.candidates; + const allCandidates = [...bootstrap, ...semantic]; + + const automatic = request.outputMode === "prompt_injection" || request.outputMode === "bootstrap" || request.entryPoint === "enrich"; + const allowUnknown = automatic + ? this.config.scope.unknownAutoInjection + : this.config.scope.unknownExplicitSearch; + const surviving: RecallCandidate[] = []; + for (const candidate of allCandidates) { + if (candidate.isLatest === false) { + drop(candidate, "superseded", "dropped because memory has been superseded"); + continue; + } + if (candidate.forgetAfter && new Date(candidate.forgetAfter).getTime() <= Date.now()) { + drop(candidate, "stale", "dropped because memory has expired"); + continue; + } + const scope = evaluateScope(candidate.scope, request, allowUnknown); + if (!scope.eligible) { + drop(candidate, "scope_mismatch", scope.reason); + continue; + } + candidate.score += scope.score + candidate.recencyScore + this.kindBoost(candidate); + surviving.push(candidate); + } + + const bootstrapCandidates = surviving + .filter((candidate) => candidate.bootstrap) + .sort((left, right) => this.bootstrapOrder(left) - this.bootstrapOrder(right)); + const semanticCandidates = surviving + .filter((candidate) => !candidate.bootstrap) + .sort((left, right) => right.score - left.score); + const selected: RecallItemTrace[] = []; + const selectedContent: string[] = []; + const usedContents = new Set(); + const maxContext = request.budget?.maxContextTokens ?? this.config.budget.maxContextTokens; + const opening = request.outputMode === "bootstrap" + ? `` + : ``; + const closing = request.outputMode === "bootstrap" ? "" : ""; + const wrapperTokens = countTokens(`${opening}\n${closing}`); + let remaining = Math.max(0, maxContext - wrapperTokens); + let bootstrapContinuity = 0; + + for (const candidate of bootstrapCandidates) { + const item = traceItem(candidate, "selected", "selected as mandatory bootstrap"); + if (candidate.kind === "continuity" && bootstrapContinuity >= this.config.budget.maxContinuityItems) { + drop(candidate, "low_score", "dropped because the configured continuity limit was reached"); + continue; + } + if (usedContents.has(normalizeDuplicate(candidate.content))) { + drop(candidate, "duplicate", "dropped because equivalent context was already selected"); + continue; + } + const packedTokens = countTokens(`${selectedContent.length > 0 ? "\n\n" : ""}${candidate.content}`); + if (packedTokens > remaining) { + drop(candidate, "over_budget", "dropped because mandatory bootstrap exceeded hard context budget"); + continue; + } + selected.push(item); + selectedContent.push(candidate.content); + usedContents.add(normalizeDuplicate(candidate.content)); + remaining -= packedTokens; + if (candidate.kind === "continuity") bootstrapContinuity += 1; + } + + if (request.outputMode !== "bootstrap") { + const semanticLimit = Math.min( + request.budget?.maxSemanticTokens ?? this.config.budget.maxSemanticTokens, + remaining, + ); + let semanticUsed = 0; + const kindCounts: Partial> = {}; + const rankedLimit = Math.max(1, request.limit || 20); + for (const candidate of semanticCandidates) { + const item = traceItem(candidate, "selected", "selected by hybrid retrieval and bounded boosts"); + if (request.outputMode === "ranked_results" && selected.length >= rankedLimit) { + drop(candidate, "low_score", "dropped because the requested result limit was reached"); + continue; + } + if (usedContents.has(normalizeDuplicate(candidate.content))) { + drop(candidate, "duplicate", "dropped because equivalent context was already selected"); + continue; + } + if (await isDuplicateInjection( + this.kv, + injectionState, + candidate.id, + candidate.version, + redactedQuery.queryFingerprint, + this.config.injection, + )) { + drop(candidate, "duplicate", "dropped because this item/version/query was injected in the current context epoch"); + continue; + } + if (request.outputMode !== "ranked_results" && !this.withinKindCap(candidate, kindCounts)) { + drop(candidate, "low_score", "dropped because the configured kind limit was reached"); + continue; + } + const packedTokens = countTokens(`${selectedContent.length > 0 ? "\n\n" : ""}${candidate.content}`); + if (request.outputMode !== "ranked_results" && (packedTokens > remaining || semanticUsed + packedTokens > semanticLimit)) { + drop(candidate, "over_budget", "dropped because semantic context budget was exhausted"); + continue; + } + selected.push(item); + selectedContent.push(candidate.content); + usedContents.add(normalizeDuplicate(candidate.content)); + kindCounts[candidate.kind] = (kindCounts[candidate.kind] || 0) + 1; + semanticUsed += packedTokens; + remaining -= request.outputMode === "ranked_results" ? 0 : packedTokens; + await recordInjection( + this.kv, + injectionState, + candidate.id, + candidate.version, + redactedQuery.queryFingerprint, + ); + } + } + + const context = request.outputMode === "ranked_results" || selectedContent.length === 0 + ? "" + : `${opening}\n${selectedContent.join("\n\n")}\n${closing}`; + const trace: RecallTrace = { + id: generateId("rtr"), + timestamp: new Date().toISOString(), + entryPoint: request.entryPoint, + outputMode: request.outputMode, + ...(request.projectId ? { projectId: request.projectId } : {}), + ...(request.repoId ? { repoId: request.repoId } : {}), + ...(request.checkoutId ? { checkoutId: request.checkoutId } : {}), + ...redactedQuery, + selected, + dropped, + droppedCountsByDecision: droppedCounts, + totalCandidateCount: allCandidates.length, + selectedTokenCount: selected.reduce((total, item) => total + item.tokenCount, 0), + finalContextTokenCount: context ? countTokens(context) : 0, + tokenEstimator: estimator, + retrievalMode: semanticResponse.retrievalMode, + }; + await persistRecallTrace(this.kv, trace, this.config.trace); + return { context, results: selected, trace }; + } + + private async collectBootstrap(request: RecallRequest): Promise { + if (!request.projectId) return []; + const candidates: RecallCandidate[] = []; + const profile = await this.kv.get(KV.profiles, request.projectId).catch(() => null); + if (profile) { + const content = renderProfile(profile); + if (content) { + candidates.push({ + id: `profile:${request.projectId}`, + kind: "memory", + subkind: "project_rule", + content: `## Project Rules\n${content}`, + score: 1, + recencyScore: recencyScore(profile.updatedAt), + scope: { level: "project", projectId: request.projectId, ...(request.repoId ? { repoId: request.repoId } : {}) }, + bootstrap: true, + }); + } + } + const [projectSlots, globalSlots] = await Promise.all([ + this.kv.list(KV.slots).catch(() => []), + this.kv.list(KV.globalSlots).catch(() => []), + ]); + for (const slot of projectSlots) { + if (!slot.pinned || slot.projectId !== request.projectId) continue; + if (slot.repoId && request.repoId && slot.repoId !== request.repoId) continue; + if (!slot.content.trim()) continue; + candidates.push({ + id: `slot:${slot.label}:${slot.projectId}`, + kind: slot.label === "pending_items" ? "continuity" : "memory", + ...(slot.label === "pending_items" ? {} : { subkind: "project_rule" as const }), + content: `## ${slot.label}\n${slot.content}`, + score: 1, + recencyScore: recencyScore(slot.updatedAt), + scope: { level: "project", projectId: slot.projectId, ...(slot.repoId ? { repoId: slot.repoId } : {}) }, + bootstrap: true, + }); + } + for (const slot of globalSlots) { + if (!slot.pinned || !slot.repoId || slot.repoId !== request.repoId || !slot.content.trim()) continue; + candidates.push({ + id: `slot:${slot.label}:${slot.repoId}`, + kind: "memory", + subkind: "repo_instruction", + content: `## ${slot.label}\n${slot.content}`, + score: 1, + recencyScore: recencyScore(slot.updatedAt), + scope: { level: "repo", repoId: slot.repoId }, + bootstrap: true, + }); + } + return candidates; + } + + private async collectSemantic(request: RecallRequest): Promise<{ + candidates: RecallCandidate[]; + retrievalMode: ReturnType; + }> { + if (!request.query?.trim()) return { candidates: [], retrievalMode: unavailableRetrievalMode() }; + const candidates: RecallCandidate[] = []; + const sessionCache = new Map(); + const sessionFor = async (sessionId: string) => { + if (!sessionCache.has(sessionId)) sessionCache.set(sessionId, await this.kv.get(KV.sessions, sessionId)); + return sessionCache.get(sessionId) || null; + }; + const hybrid = this.hybridRecall + ? await this.hybridRecall(request.query, Math.max((request.limit || 20) * 5, 50)) + : []; + const hits = Array.isArray(hybrid) ? hybrid : hybrid.results; + const retrievalMode = Array.isArray(hybrid) + ? { + bm25: { status: "healthy" as const, attempted: true }, + vector: { status: "disabled" as const, attempted: false, reason: "legacy hybrid callback did not report vector health" }, + graph: { status: "disabled" as const, attempted: false, reason: "legacy hybrid callback did not report graph health" }, + } + : hybrid.retrievalMode; + for (const hit of hits) { + const memory = await this.kv.get(KV.memories, hit.observation.id).catch(() => null); + const session = memory ? null : await sessionFor(hit.sessionId); + const isMemory = Boolean(memory); + const content = isMemory + ? `## ${memory!.title}\n${memory!.content}` + : `## ${hit.observation.title}\n${hit.observation.narrative}`; + candidates.push({ + id: hit.observation.id, + kind: isMemory ? "memory" : "observation", + ...(isMemory ? { subkind: "durable_memory" as const } : {}), + content, + score: hit.combinedScore, + bm25Score: hit.bm25Score, + vectorScore: hit.vectorScore, + graphScore: hit.graphScore, + recencyScore: recencyScore(isMemory ? memory!.updatedAt : hit.observation.timestamp), + scope: isMemory ? memoryScope(memory!) : { level: "project", projectId: session?.project || "" }, + ...(isMemory ? { sourceSessionIds: memory!.sessionIds, sourceObservationIds: memory!.sourceObservationIds } : { sourceSessionIds: [hit.sessionId], sourceObservationIds: [hit.observation.id] }), + ...(isMemory ? { isLatest: memory!.isLatest, forgetAfter: memory!.forgetAfter, version: memory!.version } : {}), + }); + } + const summaries = await this.kv.list(KV.summaries).catch(() => []); + for (const summary of summaries) { + if (summary.project !== request.projectId) continue; + const content = `## ${summary.title}\n${summary.narrative}\nDecisions: ${summary.keyDecisions.join("; ")}`; + const score = lexicalScore(request.query, content); + if (score <= 0) continue; + candidates.push({ + id: `summary:${summary.sessionId}`, + kind: "summary", + content, + score, + recencyScore: recencyScore(summary.createdAt), + scope: { level: "project", projectId: summary.project }, + sourceSessionIds: [summary.sessionId], + }); + } + return { candidates, retrievalMode }; + } + + private kindBoost(candidate: RecallCandidate): number { + if (candidate.kind === "memory") return 0.03; + if (candidate.kind === "summary") return 0.01; + return 0; + } + + private bootstrapOrder(candidate: RecallCandidate): number { + if (candidate.subkind === "project_rule") return 0; + if (candidate.subkind === "repo_instruction") return 1; + return 2; + } + + private withinKindCap(candidate: RecallCandidate, counts: Partial>): boolean { + const cap = candidate.kind === "memory" + ? this.config.budget.maxMemories + : candidate.kind === "summary" + ? this.config.budget.maxSessionSummaries + : candidate.kind === "observation" + ? this.config.budget.maxObservations + : this.config.budget.maxContinuityItems; + return (counts[candidate.kind] || 0) < cap; + } +} diff --git a/src/recall/identity.ts b/src/recall/identity.ts new file mode 100644 index 0000000000..5b45fabce8 --- /dev/null +++ b/src/recall/identity.ts @@ -0,0 +1,50 @@ +import { createHash } from "node:crypto"; +import { execFileSync } from "node:child_process"; +import { resolve } from "node:path"; + +export interface RecallIdentity { + projectId: string; + repoId?: string; + checkoutId: string; +} + +function fingerprint(value: string): string { + return createHash("sha256").update(value).digest("hex").slice(0, 24); +} + +function normalizeRemote(value: string): string { + const trimmed = value + .trim() + .replace(/^[a-z]+:\/\/[^@/]+@/i, (prefix) => prefix.replace(/\/\/[^@/]+@/, "//")) + .replace(/\.git$/i, "") + .replace(/\/$/, ""); + const ssh = trimmed.match(/^[^@\s]+@([^:]+):(.+)$/); + return (ssh ? `${ssh[1]}/${ssh[2]}` : trimmed).toLowerCase(); +} + +function gitValue(cwd: string, args: string[]): string | undefined { + try { + const value = execFileSync("git", args, { + cwd, + stdio: ["ignore", "pipe", "ignore"], + timeout: 500, + }).toString().trim(); + return value || undefined; + } catch { + return undefined; + } +} + +export function resolveRecallIdentity( + cwd: string, + projectId: string, +): RecallIdentity { + const checkoutRoot = gitValue(cwd, ["rev-parse", "--show-toplevel"]) || cwd; + const normalizedRoot = resolve(checkoutRoot).replace(/\\/g, "/").toLowerCase(); + const remote = gitValue(checkoutRoot, ["remote", "get-url", "origin"]); + return { + projectId, + repoId: remote ? fingerprint(normalizeRemote(remote)) : fingerprint(normalizedRoot), + checkoutId: fingerprint(normalizedRoot), + }; +} diff --git a/src/recall/ledger.ts b/src/recall/ledger.ts new file mode 100644 index 0000000000..fa32403353 --- /dev/null +++ b/src/recall/ledger.ts @@ -0,0 +1,95 @@ +import type { RecallInjectionConfig } from "../types.js"; +import { KV } from "../state/schema.js"; +import type { StateKV } from "../state/kv.js"; + +interface SessionRecallState { + sessionId: string; + turn: number; + contextEpoch: number; +} + +export interface InjectionLedgerEntry { + sessionId: string; + contextEpoch: number; + queryFingerprint?: string; + itemId: string; + itemVersion?: number; + injectedAtTurn: number; + injectedAt: string; +} + +function stateKey(sessionId: string): string { + return `recall-session:${sessionId}`; +} + +function ledgerKey(sessionId: string, itemId: string): string { + return `${sessionId}:${itemId}`; +} + +export async function currentRecallTurn( + kv: StateKV, + sessionId: string | undefined, + increment: boolean, +): Promise { + if (!sessionId) return null; + const current = (await kv.get(KV.state, stateKey(sessionId))) || { + sessionId, + turn: 0, + contextEpoch: 0, + }; + const next = increment ? { ...current, turn: current.turn + 1 } : current; + if (increment) await kv.set(KV.state, stateKey(sessionId), next); + return next; +} + +export async function advanceRecallContextEpoch( + kv: StateKV, + sessionId: string, +): Promise { + const current = (await currentRecallTurn(kv, sessionId, false)) || { + sessionId, + turn: 0, + contextEpoch: 0, + }; + const next = { ...current, contextEpoch: current.contextEpoch + 1 }; + await kv.set(KV.state, stateKey(sessionId), next); + return next; +} + +export async function isDuplicateInjection( + kv: StateKV, + state: SessionRecallState | null, + itemId: string, + itemVersion: number | undefined, + queryFingerprint: string | undefined, + config: RecallInjectionConfig, +): Promise { + if (!state) return false; + const prior = await kv.get(KV.injectionLedger, ledgerKey(state.sessionId, itemId)); + return Boolean( + prior && + prior.contextEpoch === state.contextEpoch && + prior.itemVersion === itemVersion && + prior.queryFingerprint === queryFingerprint && + state.turn - prior.injectedAtTurn < config.reinjectionTurnWindow, + ); +} + +export async function recordInjection( + kv: StateKV, + state: SessionRecallState | null, + itemId: string, + itemVersion: number | undefined, + queryFingerprint: string | undefined, +): Promise { + if (!state) return; + await kv.set(KV.injectionLedger, ledgerKey(state.sessionId, itemId), { + sessionId: state.sessionId, + contextEpoch: state.contextEpoch, + queryFingerprint, + itemId, + itemVersion, + injectedAtTurn: state.turn, + injectedAt: new Date().toISOString(), + }); +} diff --git a/src/recall/scope.ts b/src/recall/scope.ts new file mode 100644 index 0000000000..7144dedd4b --- /dev/null +++ b/src/recall/scope.ts @@ -0,0 +1,69 @@ +import type { Memory, RecallRequest, RecallScope } from "../types.js"; + +export interface ScopeDecision { + eligible: boolean; + score: number; + reason: string; +} + +export function normalizeScope(value: unknown): RecallScope { + if (!value || typeof value !== "object" || Array.isArray(value)) { + return { level: "unknown" }; + } + const raw = value as Partial; + if ( + raw.level !== "project" && + raw.level !== "repo" && + raw.level !== "user" && + raw.level !== "unknown" + ) { + return { level: "unknown" }; + } + return { + level: raw.level, + ...(typeof raw.projectId === "string" && raw.projectId.trim() + ? { projectId: raw.projectId.trim() } + : {}), + ...(typeof raw.repoId === "string" && raw.repoId.trim() + ? { repoId: raw.repoId.trim() } + : {}), + }; +} + +export function memoryScope(memory: Memory): RecallScope { + return normalizeScope(memory.scope); +} + +export function evaluateScope( + scope: RecallScope, + request: Pick, + allowUnknown: boolean, +): ScopeDecision { + if (scope.level === "user") { + return { eligible: true, score: 0, reason: "selected because explicit user scope" }; + } + if (scope.level === "unknown") { + if (!allowUnknown) { + return { eligible: false, score: -0.1, reason: "dropped because scope is unknown for automatic injection" }; + } + return { eligible: true, score: -0.1, reason: "selected with legacy unknown-scope penalty" }; + } + if (scope.level === "repo") { + if (!scope.repoId || !request.repoId || scope.repoId !== request.repoId) { + return { eligible: false, score: 0, reason: "dropped because repo scope mismatched" }; + } + return { eligible: true, score: 0.03, reason: "selected because repo matched" }; + } + if (!scope.projectId || !request.projectId || scope.projectId !== request.projectId) { + return { eligible: false, score: 0, reason: "dropped because project scope mismatched" }; + } + if (scope.repoId && request.repoId && scope.repoId !== request.repoId) { + return { eligible: false, score: 0, reason: "dropped because project repo scope mismatched" }; + } + return { eligible: true, score: 0.06, reason: "selected because project scope matched" }; +} + +export function sameScope(left: RecallScope, right: RecallScope): boolean { + if (left.level !== right.level) return false; + return left.projectId === right.projectId && left.repoId === right.repoId; +} diff --git a/src/recall/tokens.ts b/src/recall/tokens.ts new file mode 100644 index 0000000000..edbd58db80 --- /dev/null +++ b/src/recall/tokens.ts @@ -0,0 +1,26 @@ +import type { RecallTokenEstimator } from "../types.js"; + +const CJK_CHAR = /[\u3040-\u30ff\u3400-\u9fff\uf900-\ufaff]/u; + +export function getTokenEstimator(_hint?: string): RecallTokenEstimator { + return { + name: "conservative-unicode", + version: "1", + estimated: true, + }; +} + +export function countTokens(text: string): number { + let tokens = 0; + let asciiRun = 0; + for (const char of text) { + if (CJK_CHAR.test(char)) { + tokens += Math.ceil(asciiRun / 4); + asciiRun = 0; + tokens += 1; + } else { + asciiRun += char.length; + } + } + return tokens + Math.ceil(asciiRun / 4); +} diff --git a/src/recall/trace-store.ts b/src/recall/trace-store.ts new file mode 100644 index 0000000000..67c15a1469 --- /dev/null +++ b/src/recall/trace-store.ts @@ -0,0 +1,81 @@ +import { createHmac, randomBytes } from "node:crypto"; +import type { RecallItemStats, RecallTrace, RecallTraceConfig } from "../types.js"; +import { KV } from "../state/schema.js"; +import type { StateKV } from "../state/kv.js"; +import { stripPrivateData } from "../functions/privacy.js"; + +const QUERY_SALT_KEY = "recall-query-salt"; + +async function querySalt(kv: StateKV): Promise { + const existing = await kv.get(KV.state, QUERY_SALT_KEY).catch(() => null); + if (existing) return existing; + const salt = randomBytes(32).toString("base64url"); + await kv.set(KV.state, QUERY_SALT_KEY, salt); + return salt; +} + +export async function redactQuery( + kv: StateKV, + query: string | undefined, +): Promise> { + if (!query) return { redactionKinds: [] }; + const redacted = stripPrivateData(query); + const redactionKinds: string[] = []; + if (redacted.includes("[REDACTED_SECRET]")) redactionKinds.push("secret"); + if (redacted.includes("[REDACTED]")) redactionKinds.push("private_tag"); + const salt = await querySalt(kv); + return { + query: redacted, + queryFingerprint: createHmac("sha256", salt).update(query).digest("hex"), + redactionKinds, + }; +} + +function emptyStats(itemId: string): RecallItemStats { + return { + itemId, + recallCount: 0, + averageScore: 0, + scopeMismatchCount: 0, + }; +} + +export async function persistRecallTrace( + kv: StateKV, + trace: RecallTrace, + config: RecallTraceConfig, +): Promise { + await kv.set(KV.recallTraces, trace.id, trace); + const now = new Date(trace.timestamp).getTime(); + const traces = await kv.list(KV.recallTraces).catch(() => []); + const cutoff = now - config.retentionDays * 86_400_000; + const retained = traces + .filter((entry) => new Date(entry.timestamp).getTime() >= cutoff) + .sort((a, b) => new Date(b.timestamp).getTime() - new Date(a.timestamp).getTime()); + const allowed = new Set(retained.slice(0, config.maxTraces).map((entry) => entry.id)); + await Promise.all( + traces + .filter((entry) => !allowed.has(entry.id)) + .map((entry) => kv.delete(KV.recallTraces, entry.id)), + ); + + await Promise.all(trace.selected.map(async (item) => { + const current = (await kv.get(KV.recallStats, item.id)) || emptyStats(item.id); + const count = current.recallCount + 1; + await kv.set(KV.recallStats, item.id, { + ...current, + recallCount: count, + averageScore: ((current.averageScore * current.recallCount) + item.score) / count, + lastRecalledAt: trace.timestamp, + recentQuery: trace.query, + }); + })); + const mismatches = trace.dropped.filter((item) => item.decision === "scope_mismatch"); + await Promise.all(mismatches.map(async (item) => { + const current = (await kv.get(KV.recallStats, item.id)) || emptyStats(item.id); + await kv.set(KV.recallStats, item.id, { + ...current, + scopeMismatchCount: current.scopeMismatchCount + 1, + }); + })); +} diff --git a/src/recall/vector-health.ts b/src/recall/vector-health.ts new file mode 100644 index 0000000000..e5fbb6e8dc --- /dev/null +++ b/src/recall/vector-health.ts @@ -0,0 +1,77 @@ +import type { RetrievalChannelStatus } from "../types.js"; + +type VectorFailureKind = "rate_limited" | "authentication" | "timeout" | "index_unavailable" | "invalid_request" | "provider_error"; + +function classify(error: unknown): VectorFailureKind { + const message = error instanceof Error ? error.message : String(error); + if (/\b429\b|quota|rate.?limit/i.test(message)) return "rate_limited"; + if (/\b401\b|\b403\b|auth|api.?key|invalid model/i.test(message)) return "authentication"; + if (/timeout|abort/i.test(message)) return "timeout"; + if (/dimension|index unavailable|index missing/i.test(message)) return "index_unavailable"; + if (/\b400\b|invalid request/i.test(message)) return "invalid_request"; + return "provider_error"; +} + +export class VectorRetrievalHealth { + private state: "closed" | "open" | "half_open" | "latched_disabled" = "closed"; + private failures = 0; + private openedAt = 0; + private probeInFlight = false; + private reason?: string; + + constructor( + private readonly failureThreshold = 3, + private readonly recoveryMs = 30_000, + ) {} + + begin(configured: boolean, indexAvailable: boolean): RetrievalChannelStatus { + if (!configured) return { status: "disabled", attempted: false, reason: "embedding provider is not configured" }; + if (!indexAvailable) return { status: "degraded", attempted: false, reason: "vector index is unavailable", fallback: "BM25/graph" }; + if (this.state === "latched_disabled") { + return { status: "disabled", attempted: false, reason: this.reason || "embedding configuration is invalid", fallback: "BM25/graph" }; + } + if (this.state === "open") { + if (Date.now() - this.openedAt < this.recoveryMs) { + return { status: "degraded", attempted: false, reason: this.reason || "vector circuit is open", fallback: "BM25/graph" }; + } + if (this.probeInFlight) { + return { status: "degraded", attempted: false, reason: "vector half-open probe is in flight", fallback: "BM25/graph" }; + } + this.state = "half_open"; + } + if (this.state === "half_open") { + if (this.probeInFlight) { + return { status: "degraded", attempted: false, reason: "vector half-open probe is in flight", fallback: "BM25/graph" }; + } + this.probeInFlight = true; + } + return { status: "healthy", attempted: true }; + } + + success(): void { + this.state = "closed"; + this.failures = 0; + this.openedAt = 0; + this.probeInFlight = false; + this.reason = undefined; + } + + failure(error: unknown): RetrievalChannelStatus { + const kind = classify(error); + this.probeInFlight = false; + this.reason = `embedding ${kind.replace(/_/g, " ")}`; + if (kind === "authentication") { + this.state = "latched_disabled"; + return { status: "disabled", attempted: true, reason: this.reason, fallback: "BM25/graph" }; + } + if (kind === "invalid_request") { + return { status: "degraded", attempted: true, reason: this.reason, fallback: "BM25/graph" }; + } + this.failures += 1; + if (this.state === "half_open" || this.failures >= this.failureThreshold) { + this.state = "open"; + this.openedAt = Date.now(); + } + return { status: "degraded", attempted: true, reason: this.reason, fallback: "BM25/graph" }; + } +} diff --git a/src/state/hybrid-search.ts b/src/state/hybrid-search.ts index d234a3efca..f7b44043b0 100644 --- a/src/state/hybrid-search.ts +++ b/src/state/hybrid-search.ts @@ -6,6 +6,7 @@ import type { CompressedObservation, Memory, QueryExpansion, + RetrievalChannelStatus, } from "../types.js"; import { memoryToObservation } from "./memory-utils.js"; import type { StateKV } from "./kv.js"; @@ -16,11 +17,23 @@ import { } from "../functions/graph-retrieval.js"; import { extractEntitiesFromQuery } from "../functions/query-expansion.js"; import { rerank } from "./reranker.js"; +import { VectorRetrievalHealth } from "../recall/vector-health.js"; +import { isGraphExtractionEnabled } from "../config.js"; const RRF_K = 60; +export interface HybridSearchResponse { + results: HybridSearchResult[]; + retrievalMode: { + bm25: RetrievalChannelStatus; + vector: RetrievalChannelStatus; + graph: RetrievalChannelStatus; + }; +} + export class HybridSearch { private graphRetrieval: GraphRetrieval; + private vectorHealth = new VectorRetrievalHealth(); constructor( private bm25: SearchIndex, @@ -36,6 +49,10 @@ export class HybridSearch { } async search(query: string, limit = 20): Promise { + return (await this.tripleStreamSearch(query, limit)).results; + } + + async searchWithTrace(query: string, limit = 20): Promise { return this.tripleStreamSearch(query, limit); } @@ -60,8 +77,8 @@ export class HybridSearch { ); const merged = new Map(); - for (const results of resultSets) { - for (const r of results) { + for (const response of resultSets) { + for (const r of response.results) { const existing = merged.get(r.observation.id); if (!existing || r.combinedScore > existing.combinedScore) { merged.set(r.observation.id, r); @@ -78,8 +95,9 @@ export class HybridSearch { query: string, limit: number, entityHints?: string[], - ): Promise { + ): Promise { const bm25Results = this.bm25.search(query, limit * 2); + const bm25Mode: RetrievalChannelStatus = { status: "healthy", attempted: true }; let vectorResults: Array<{ obsId: string; @@ -88,12 +106,17 @@ export class HybridSearch { }> = []; let queryEmbedding: Float32Array | null = null; - if (this.vector && this.embeddingProvider && this.vector.size > 0) { + let vectorMode = this.vectorHealth.begin( + Boolean(this.embeddingProvider), + Boolean(this.vector && this.vector.size > 0), + ); + if (vectorMode.attempted && this.vector && this.embeddingProvider) { try { queryEmbedding = await this.embeddingProvider.embed(query); vectorResults = this.vector.search(queryEmbedding, limit * 2); - } catch { - // fall through to BM25-only + this.vectorHealth.success(); + } catch (err) { + vectorMode = this.vectorHealth.failure(err); } } @@ -102,15 +125,23 @@ export class HybridSearch { ? entityHints : extractEntitiesFromQuery(query); let graphResults: GraphRetrievalResult[] = []; - if (entities.length > 0) { + let graphMode: RetrievalChannelStatus = isGraphExtractionEnabled() + ? { status: "healthy", attempted: entities.length > 0 } + : { status: "disabled", attempted: false, reason: "graph extraction is disabled" }; + if (entities.length > 0 && isGraphExtractionEnabled()) { try { graphResults = await this.graphRetrieval.searchByEntities( entities, 2, limit, ); - } catch { - // graph search is best-effort + } catch (err) { + graphMode = { + status: "disabled", + attempted: true, + reason: err instanceof Error ? err.message : "graph retrieval failed", + fallback: "BM25/vector", + }; } } @@ -120,8 +151,13 @@ export class HybridSearch { const expansionResults = await this.graphRetrieval.expandFromChunks(topVectorObs, 1, 5); graphResults = [...graphResults, ...expansionResults]; - } catch { - // expansion is best-effort + } catch (err) { + graphMode = { + status: "disabled", + attempted: true, + reason: err instanceof Error ? err.message : "graph expansion failed", + fallback: "BM25/vector", + }; } } @@ -230,13 +266,22 @@ export class HybridSearch { const head = enriched.slice(0, rerankWindow); const tail = enriched.slice(rerankWindow); const reranked = await rerank(query, head, rerankWindow); - return reranked.concat(tail).slice(0, limit); + return { + results: reranked.concat(tail).slice(0, limit), + retrievalMode: { bm25: bm25Mode, vector: vectorMode, graph: graphMode }, + }; } catch { - return enriched.slice(0, limit); + return { + results: enriched.slice(0, limit), + retrievalMode: { bm25: bm25Mode, vector: vectorMode, graph: graphMode }, + }; } } - return enriched.slice(0, limit); + return { + results: enriched.slice(0, limit), + retrievalMode: { bm25: bm25Mode, vector: vectorMode, graph: graphMode }, + }; } private diversifyBySession( diff --git a/src/state/schema.ts b/src/state/schema.ts index f8a2b0c934..facbf2a294 100644 --- a/src/state/schema.ts +++ b/src/state/schema.ts @@ -73,6 +73,10 @@ export const KV = { // #771: tracks the most recent smart-search call per session, used by // the followup-rate diagnostic. Key = sessionId. TTL-swept hourly. recentSearches: "mem:recent-searches", + recallTraces: "mem:recall-traces", + recallStats: "mem:recall-stats", + injectionLedger: "mem:injection-ledger", + durableRecommendations: "mem:durable-recommendations", } as const; export const STREAM = { diff --git a/src/triggers/api.ts b/src/triggers/api.ts index 2143b237fb..91ed282582 100644 --- a/src/triggers/api.ts +++ b/src/triggers/api.ts @@ -1,5 +1,5 @@ import { TriggerAction, type ISdk, type ApiRequest } from "iii-sdk"; -import type { Session, CompressedObservation, HookPayload, CommitLink } from "../types.js"; +import type { Session, CompressedObservation, HookPayload, CommitLink, RecallTrace } from "../types.js"; import { withKeyedLock } from "../state/keyed-mutex.js"; import { KV } from "../state/schema.js"; import { StateKV } from "../state/kv.js"; @@ -11,6 +11,8 @@ import { timingSafeCompare } from "../auth.js"; import { isSlotsEnabled, isReflectEnabled } from "../functions/slots.js"; import { renderViewerDocument } from "../viewer/document.js"; import { getBoundViewerPort, getViewerSkipped } from "../viewer/server.js"; +import { resolveRecallIdentity } from "../recall/identity.js"; +import { advanceRecallContextEpoch } from "../recall/ledger.js"; import { MAX_FILES_UPPER_BOUND } from "../functions/replay.js"; import { logger } from "../logger.js"; import { @@ -342,11 +344,45 @@ export function registerApiTriggers( body: { error: "budget must be a positive integer" }, }; } - const payload: { sessionId: string; project: string; budget?: number } = { + const outputMode = body.outputMode; + if (outputMode !== undefined && outputMode !== "bootstrap" && outputMode !== "prompt_injection" && outputMode !== "rendered_context") { + return { status_code: 400, body: { error: "outputMode must be bootstrap, prompt_injection, or rendered_context" } }; + } + const optionalString = (key: string): string | undefined => + typeof body[key] === "string" && body[key].trim() ? body[key].trim() : undefined; + const payload: { + sessionId: string; + project: string; + budget?: number; + query?: string; + projectId?: string; + repoId?: string; + checkoutId?: string; + cwd?: string; + outputMode?: "bootstrap" | "prompt_injection" | "rendered_context"; + debug?: boolean; + } = { sessionId, project, }; if (budget !== undefined) payload.budget = budget; + for (const key of ["query", "projectId", "repoId", "checkoutId"] as const) { + const value = optionalString(key); + if (value !== undefined) payload[key] = value; + } + const cwd = optionalString("cwd"); + if (cwd !== undefined) payload.cwd = cwd; + if (cwd && (!payload.projectId || !payload.repoId || !payload.checkoutId)) { + const identity = resolveRecallIdentity(cwd, payload.projectId || project); + payload.projectId ||= identity.projectId; + payload.repoId ||= identity.repoId; + payload.checkoutId ||= identity.checkoutId; + } + if (outputMode !== undefined) payload.outputMode = outputMode; + if (body.debug !== undefined) { + if (typeof body.debug !== "boolean") return { status_code: 400, body: { error: "debug must be a boolean" } }; + payload.debug = body.debug; + } const result = await sdk.trigger({ function_id: "mem::context", payload }); return { status_code: 200, body: result }; }, @@ -361,6 +397,69 @@ export function registerApiTriggers( }, }); + sdk.registerFunction("api::recall-debug", + async (req: ApiRequest): Promise => { + const authErr = checkAuth(req, secret); + if (authErr) return authErr; + const params = req.query_params || {}; + const rawLimit = Number(params["limit"] || 50); + const limit = Number.isInteger(rawLimit) && rawLimit > 0 ? Math.min(rawLimit, 200) : 50; + const traces = await kv.list(KV.recallTraces); + const filtered = traces + .filter((trace) => !params["projectId"] || trace.projectId === params["projectId"]) + .filter((trace) => !params["repoId"] || trace.repoId === params["repoId"]) + .filter((trace) => !params["entryPoint"] || trace.entryPoint === params["entryPoint"]) + .filter((trace) => { + const itemId = params["itemId"]; + return !itemId || trace.selected.some((item) => item.id === itemId) || trace.dropped.some((item) => item.id === itemId); + }) + .sort((left, right) => right.timestamp.localeCompare(left.timestamp)) + .slice(0, limit); + return { status_code: 200, body: { traces: filtered, total: filtered.length } }; + }, + ); + sdk.registerTrigger({ + type: "http", + function_id: "api::recall-debug", + config: { api_path: "/agentmemory/recall/debug", http_method: "GET" }, + }); + + sdk.registerFunction("api::recall-debug-by-id", + async (req: ApiRequest): Promise => { + const authErr = checkAuth(req, secret); + if (authErr) return authErr; + const traceId = req.path_params?.["traceId"]; + if (!traceId || typeof traceId !== "string") { + return { status_code: 400, body: { error: "traceId path parameter is required" } }; + } + const trace = await kv.get(KV.recallTraces, traceId); + return trace + ? { status_code: 200, body: { trace } } + : { status_code: 404, body: { error: "recall trace not found" } }; + }, + ); + sdk.registerTrigger({ + type: "http", + function_id: "api::recall-debug-by-id", + config: { api_path: "/agentmemory/recall/debug/:traceId", http_method: "GET" }, + }); + + sdk.registerFunction("api::recall-context-epoch", + async (req: ApiRequest<{ sessionId?: string }>): Promise => { + const authErr = checkAuth(req, secret); + if (authErr) return authErr; + const sessionId = asNonEmptyString((req.body as Record | undefined)?.sessionId); + if (!sessionId) return { status_code: 400, body: { error: "sessionId is required" } }; + const state = await advanceRecallContextEpoch(kv, sessionId); + return { status_code: 200, body: state }; + }, + ); + sdk.registerTrigger({ + type: "http", + function_id: "api::recall-context-epoch", + config: { api_path: "/agentmemory/recall/context-epoch", http_method: "POST" }, + }); + sdk.registerFunction("api::search", async ( req: ApiRequest<{ @@ -638,6 +737,7 @@ export function registerApiTriggers( ? body.agentId.trim().slice(0, 128) : undefined; const agentId = requestAgentId ?? getAgentId(); + const identity = resolveRecallIdentity(cwd, project); const session: Session = { id: sessionId, project, @@ -648,15 +748,24 @@ export function registerApiTriggers( ...(title ? { summary: title.slice(0, 200) } : {}), ...(title ? { firstPrompt: title.slice(0, 200) } : {}), ...(agentId ? { agentId } : {}), + ...(identity.repoId ? { repoId: identity.repoId } : {}), + checkoutId: identity.checkoutId, }; await kv.set(KV.sessions, sessionId, session); const contextResult = await sdk.trigger< - { sessionId: string; project: string }, - { context: string } - >({ function_id: "mem::context", payload: { sessionId, project } }); + { sessionId: string; project: string; projectId: string; repoId?: string; checkoutId: string; outputMode: "bootstrap" }, + { context: string; traceId?: string } + >({ function_id: "mem::context", payload: { + sessionId, + project, + projectId: identity.projectId, + ...(identity.repoId ? { repoId: identity.repoId } : {}), + checkoutId: identity.checkoutId, + outputMode: "bootstrap", + } }); return { status_code: 200, - body: { session, context: contextResult.context }, + body: { session, context: contextResult.context, ...(contextResult.traceId ? { traceId: contextResult.traceId } : {}) }, }; }, ); @@ -945,6 +1054,9 @@ export function registerApiTriggers( terms?: string[]; toolName?: string; project?: string; + repoId?: string; + checkoutId?: string; + cwd?: string; }>, ): Promise => { const authErr = checkAuth(req, secret); @@ -1016,6 +1128,9 @@ export function registerApiTriggers( confidence?: number; strength?: number; project?: string; + scope?: { level?: string; projectId?: string; repoId?: string }; + origin?: string; + checkoutId?: string; }>, ): Promise => { const authErr = checkAuth(req, secret); @@ -1033,6 +1148,8 @@ export function registerApiTriggers( ) { return { status_code: 400, body: { error: "project must be a non-empty string" } }; } + const cwd = typeof req.body.cwd === "string" && req.body.cwd.trim() ? req.body.cwd.trim() : undefined; + const identity = cwd ? resolveRecallIdentity(cwd, req.body.project || "") : undefined; if ( req.body.title !== undefined && (typeof req.body.title !== "string" || !req.body.title.trim()) @@ -1093,6 +1210,28 @@ export function registerApiTriggers( body: { error: "strength must be an integer between 1 and 10" }, }; } + const scope = req.body.scope; + if (scope !== undefined) { + if (!scope || typeof scope !== "object" || Array.isArray(scope)) { + return { status_code: 400, body: { error: "scope must be an object" } }; + } + const rawScope = scope as Record; + if (!["project", "repo", "user", "unknown"].includes(String(rawScope.level))) { + return { status_code: 400, body: { error: "scope.level must be project, repo, user, or unknown" } }; + } + if (rawScope.projectId !== undefined && typeof rawScope.projectId !== "string") { + return { status_code: 400, body: { error: "scope.projectId must be a string" } }; + } + if (rawScope.repoId !== undefined && typeof rawScope.repoId !== "string") { + return { status_code: 400, body: { error: "scope.repoId must be a string" } }; + } + } + if (req.body.origin !== undefined && !["manual", "candidate_promoted", "system"].includes(String(req.body.origin))) { + return { status_code: 400, body: { error: "origin must be manual, candidate_promoted, or system" } }; + } + if (req.body.checkoutId !== undefined && typeof req.body.checkoutId !== "string") { + return { status_code: 400, body: { error: "checkoutId must be a string" } }; + } const result = await sdk.trigger({ function_id: "mem::remember", payload: { @@ -1108,6 +1247,11 @@ export function registerApiTriggers( ...(req.body.confidence !== undefined && { confidence: req.body.confidence }), ...(req.body.strength !== undefined && { strength: req.body.strength }), ...(req.body.project !== undefined && { project: req.body.project }), + ...(typeof req.body.repoId === "string" ? { repoId: req.body.repoId } : identity?.repoId ? { repoId: identity.repoId } : {}), + ...(typeof req.body.checkoutId === "string" ? { checkoutId: req.body.checkoutId } : identity ? { checkoutId: identity.checkoutId } : {}), + ...(scope !== undefined && { scope }), + ...(req.body.origin !== undefined && { origin: req.body.origin }), + ...(req.body.checkoutId !== undefined && { checkoutId: req.body.checkoutId }), }, }); return { status_code: 201, body: result }; @@ -1178,6 +1322,26 @@ export function registerApiTriggers( config: { api_path: "/agentmemory/durable-candidates", http_method: "GET" }, }); + sdk.registerFunction("api::durable-candidates::recommend", + async (req: ApiRequest<{ candidateId?: string; project?: string }>): Promise => { + const authErr = checkAuth(req, secret); + if (authErr) return authErr; + const result = await sdk.trigger({ + function_id: "mem::durable-candidates::recommend", + payload: { + ...(typeof req.body?.candidateId === "string" ? { candidateId: req.body.candidateId } : {}), + ...(typeof req.body?.project === "string" ? { project: req.body.project } : {}), + }, + }); + return { status_code: 200, body: result }; + }, + ); + sdk.registerTrigger({ + type: "http", + function_id: "api::durable-candidates::recommend", + config: { api_path: "/agentmemory/durable-candidates/recommend", http_method: "POST" }, + }); + sdk.registerFunction("api::durable-candidates::promote", async ( req: ApiRequest<{ @@ -2199,7 +2363,8 @@ export function registerApiTriggers( if (!memory) { return { status_code: 404, body: { error: `memory not found: ${id}` } }; } - return { status_code: 200, body: { memory } }; + const recallStats = await kv.get(KV.recallStats, id); + return { status_code: 200, body: { memory, recallStats } }; }, ); sdk.registerTrigger({ @@ -2366,6 +2531,12 @@ export function registerApiTriggers( if (body["pinned"] !== undefined && typeof body["pinned"] !== "boolean") { return { status_code: 400, body: { error: "pinned must be a boolean" } }; } + if (body["projectId"] !== undefined && typeof body["projectId"] !== "string") { + return { status_code: 400, body: { error: "projectId must be a string" } }; + } + if (body["repoId"] !== undefined && typeof body["repoId"] !== "string") { + return { status_code: 400, body: { error: "repoId must be a string" } }; + } if ( body["scope"] !== undefined && body["scope"] !== "project" && @@ -2385,6 +2556,8 @@ export function registerApiTriggers( if (typeof body["description"] === "string") payload["description"] = body["description"]; if (sizeLimit !== undefined) payload["sizeLimit"] = sizeLimit; if (typeof body["pinned"] === "boolean") payload["pinned"] = body["pinned"]; + if (typeof body["projectId"] === "string") payload["projectId"] = body["projectId"]; + if (typeof body["repoId"] === "string") payload["repoId"] = body["repoId"]; if (body["scope"] === "project" || body["scope"] === "global") payload["scope"] = body["scope"]; const result = await sdk.trigger({ function_id: "mem::slot-create", payload }); const resp = result as { success?: boolean; error?: string }; diff --git a/src/types.ts b/src/types.ts index 4304061ce8..5c5626dc1a 100644 --- a/src/types.ts +++ b/src/types.ts @@ -12,6 +12,8 @@ export interface Session { summary?: string; commitShas?: string[]; agentId?: string; + repoId?: string; + checkoutId?: string; } export interface CommitLink { @@ -107,6 +109,14 @@ export interface DurableCandidate { forceReason?: string; } +export interface DurableCandidateRecommendation { + candidateId: string; + recommendation: "auto_promote_eligible" | "not_eligible"; + reasons: string[]; + wouldPromote: false; + evaluatedAt: string; +} + export interface Memory { id: string; createdAt: string; @@ -131,6 +141,9 @@ export interface Memory { imageData?: string; agentId?: string; project?: string; + scope?: RecallScope; + origin?: MemoryOrigin; + checkoutId?: string; } export interface SessionSummary { @@ -224,6 +237,159 @@ export interface AgentMemoryConfig { maxObservationsPerSession: number; compressionModel: string; dataDir: string; + recall: RecallConfig; +} + +export interface RecallBudget { + maxContextTokens: number; + reservedBootstrapTokens: number; + maxSemanticTokens: number; + maxMemories: number; + maxSessionSummaries: number; + maxObservations: number; + maxContinuityItems: number; +} + +export interface RecallScopeConfig { + unknownAutoInjection: boolean; + unknownExplicitSearch: boolean; +} + +export interface RecallTraceConfig { + retentionDays: number; + maxTraces: number; + maxDroppedItemsPerReason: number; +} + +export interface RecallInjectionConfig { + reinjectionTurnWindow: number; +} + +export interface RecallConfig { + budget: RecallBudget; + scope: RecallScopeConfig; + trace: RecallTraceConfig; + injection: RecallInjectionConfig; +} + +export type RecallScopeLevel = "project" | "repo" | "user" | "unknown"; + +export interface RecallScope { + level: RecallScopeLevel; + projectId?: string; + repoId?: string; +} + +export type MemoryOrigin = "manual" | "candidate_promoted" | "system"; + +export type RecallEntryPoint = + | "session_start" + | "prompt" + | "context" + | "enrich" + | "search" + | "smart_search" + | "memory_recall"; + +export type RecallOutputMode = + | "bootstrap" + | "prompt_injection" + | "ranked_results" + | "rendered_context"; + +export interface RecallRequest { + entryPoint: RecallEntryPoint; + outputMode: RecallOutputMode; + query?: string; + sessionId?: string; + projectId?: string; + repoId?: string; + checkoutId?: string; + limit?: number; + budget?: Partial; + debug?: boolean; + tokenizerHint?: string; +} + +export type RecallItemKind = + | "memory" + | "summary" + | "observation" + | "continuity"; + +export type RecallDecision = + | "selected" + | "low_score" + | "over_budget" + | "duplicate" + | "stale" + | "scope_mismatch" + | "superseded"; + +export interface RecallItemTrace { + id: string; + kind: RecallItemKind; + subkind?: "project_rule" | "repo_instruction" | "lesson" | "durable_memory"; + score: number; + bm25Score?: number; + vectorScore?: number; + graphScore?: number; + recencyScore?: number; + scopeScore?: number; + tokenCount: number; + reason: string; + sourceSessionIds?: string[]; + sourceObservationIds?: string[]; + decision: RecallDecision; +} + +export type RetrievalChannelHealth = "healthy" | "degraded" | "disabled"; + +export interface RetrievalChannelStatus { + status: RetrievalChannelHealth; + attempted: boolean; + reason?: string; + fallback?: string; +} + +export interface RecallTokenEstimator { + name: string; + version: string; + estimated: boolean; +} + +export interface RecallTrace { + id: string; + timestamp: string; + entryPoint: RecallEntryPoint; + outputMode: RecallOutputMode; + projectId?: string; + repoId?: string; + checkoutId?: string; + query?: string; + queryFingerprint?: string; + redactionKinds: string[]; + selected: RecallItemTrace[]; + dropped: RecallItemTrace[]; + droppedCountsByDecision: Partial>; + totalCandidateCount: number; + selectedTokenCount: number; + finalContextTokenCount: number; + tokenEstimator: RecallTokenEstimator; + retrievalMode: { + bm25: RetrievalChannelStatus; + vector: RetrievalChannelStatus; + graph: RetrievalChannelStatus; + }; +} + +export interface RecallItemStats { + itemId: string; + recallCount: number; + lastRecalledAt?: string; + recentQuery?: string; + averageScore: number; + scopeMismatchCount: number; } export interface SearchResult { @@ -292,6 +458,8 @@ export interface MemorySlot { scope: "project" | "global"; createdAt: string; updatedAt: string; + projectId?: string; + repoId?: string; } export interface EmbeddingProvider { diff --git a/src/viewer/index.html b/src/viewer/index.html index 305397e935..6c1ed21409 100644 --- a/src/viewer/index.html +++ b/src/viewer/index.html @@ -991,6 +991,7 @@

agentmemory

+ @@ -1008,6 +1009,7 @@

agentmemory

+
@@ -1105,7 +1107,7 @@

agentmemory

task: '☑', other: '📄' }; var CB_STATE_COLORS = { closed: 'badge-green', open: 'badge-red', 'half-open': 'badge-yellow' }; - var TAB_IDS = ['dashboard', 'graph', 'memories', 'timeline', 'sessions', 'lessons', 'actions', 'crystals', 'audit', 'activity', 'profile', 'replay']; + var TAB_IDS = ['dashboard', 'graph', 'memories', 'recall-traces', 'timeline', 'sessions', 'lessons', 'actions', 'crystals', 'audit', 'activity', 'profile', 'replay']; var VIEWER_TOKEN_STORAGE_KEY = 'agentmemory-viewer-token'; var state = { @@ -1113,6 +1115,7 @@

agentmemory

dashboard: { loaded: false, health: null, sessions: [], memories: [], graphStats: null, recentAudit: [], lessons: [], crystals: [] }, graph: { loaded: false, nodes: [], edges: [], stats: null, filters: {}, selectedNode: null, queryError: null, truncated: false, totalNodes: 0, totalEdges: 0 }, memories: { loaded: false, items: [], search: '', typeFilter: '' }, + recallTraces: { loaded: false, items: [] }, timeline: { loaded: false, observations: [], sessionId: '', minImportance: 0, page: 0, pageSize: 50 }, sessions: { loaded: false, items: [], selectedId: null }, audit: { loaded: false, entries: [], opFilter: '' }, @@ -1327,6 +1330,7 @@

agentmemory

case 'dashboard': if (!state.dashboard.loaded) await loadDashboard(); break; case 'graph': if (!state.graph.loaded) await loadGraph(); break; case 'memories': if (!state.memories.loaded) await loadMemories(); break; + case 'recall-traces': if (!state.recallTraces.loaded) await loadRecallTraces(); break; case 'timeline': if (!state.timeline.loaded) await loadTimeline(); break; case 'sessions': if (!state.sessions.loaded) await loadSessions(); break; case 'lessons': if (!state.lessons.loaded) await loadLessons(); break; @@ -2406,6 +2410,34 @@

agentmemory

renderMemories(); } + async function loadRecallTraces() { + var el = document.getElementById('view-recall-traces'); + el.innerHTML = '
Loading recall traces...
'; + var result = await apiGet('recall/debug?limit=100'); + var traces = (result && result.traces) || []; + state.recallTraces.items = traces; + state.recallTraces.loaded = true; + if (!traces.length) { + el.innerHTML = '
No recall traces yet.
'; + return; + } + var rows = traces.map(function(trace) { + var mode = trace.retrievalMode || {}; + var vector = mode.vector || {}; + var selected = (trace.selected || []).length; + var dropped = trace.droppedCountsByDecision || {}; + return '' + + '' + esc(trace.timestamp || '') + '' + + '' + esc(trace.entryPoint || '') + ' / ' + esc(trace.outputMode || '') + '' + + '' + esc(trace.query || '[no query]') + '' + + '' + selected + ' selected, ' + Object.keys(dropped).map(function(k) { return esc(k) + ': ' + dropped[k]; }).join(', ') + '' + + '' + (trace.finalContextTokenCount || 0) + ' tokens' + + '' + esc(vector.status || 'disabled') + (vector.reason ? ': ' + esc(vector.reason) : '') + '' + + ''; + }).join(''); + el.innerHTML = '

Recall Traces

' + rows + '
TimeEntryQueryDecisionBudgetVector
'; + } + function renderMemories() { var el = document.getElementById('view-memories'); var items = state.memories.items; diff --git a/test/durable-candidates.test.ts b/test/durable-candidates.test.ts index 34da9609d0..00b65f547a 100644 --- a/test/durable-candidates.test.ts +++ b/test/durable-candidates.test.ts @@ -13,6 +13,7 @@ vi.mock("../src/functions/audit.js", () => ({ import { registerDurableCandidateFunctions } from "../src/functions/durable-candidates.js"; import { materializeDurableCandidate } from "../src/functions/durable-candidate-utils.js"; +import { KV } from "../src/state/schema.js"; import type { CompressedObservation, MemoryProvider, @@ -119,6 +120,27 @@ describe("durable candidates lifecycle", () => { tempDirs.clear(); }); + it("records a shadow recommendation without writing a memory", async () => { + registerDurableCandidateFunctions(sdk as never, kv as never, makeProvider("")); + const candidate = { + id: "cand_shadow", sessionId: "s1", project: "project-a", type: "fact" as const, + title: "Stable cache invariant", content: "The cache key must include stage, pair id, and benchmark revision.", + concepts: [], files: [], sourceObservationIds: ["obs1", "obs2", "obs3"], confidence: 0.95, + createdAt: "2026-07-10T00:00:00.000Z", + }; + await kv.set(KV.summaries, "s1", { + sessionId: "s1", project: "project-a", createdAt: "2026-07-10T00:00:00.000Z", + title: "summary", narrative: "", keyDecisions: [], filesModified: [], concepts: [], observationCount: 3, + durableCandidates: [candidate], + }); + + const result = await sdk.functions.get("mem::durable-candidates::recommend")!({ candidateId: "cand_shadow" }); + + expect(result.recommendations[0]).toMatchObject({ recommendation: "auto_promote_eligible", wouldPromote: false }); + expect(await kv.list(KV.memories)).toEqual([]); + expect(await kv.get(KV.durableRecommendations, "cand_shadow")).toBeTruthy(); + }); + it("promote skips when a memory already exists for sourceCandidateId and backfills promotedMemoryId", async () => { const candidate = materializeDurableCandidate({ sessionId: "sess_promote", diff --git a/test/recall-benchmark-score.test.ts b/test/recall-benchmark-score.test.ts new file mode 100644 index 0000000000..447d1b5807 --- /dev/null +++ b/test/recall-benchmark-score.test.ts @@ -0,0 +1,13 @@ +import { describe, expect, it } from "vitest"; +import { scoreRecallBenchmark } from "../eval/recall/score.js"; + +describe("recall benchmark scoring", () => { + it("measures contamination and token violations independently from hit rate", () => { + const score = scoreRecallBenchmark([ + { id: "a", query: "q", projectId: "p", expectedMemoryIds: ["good"], forbiddenMemoryIds: ["bad"], maxAcceptableTokens: 10 }, + ], { + a: { selectedIds: ["good", "bad"], injectedTokens: 11, duplicateIds: [], staleIds: [] }, + }); + expect(score).toMatchObject({ hitRate: 1, precision: 0.5, crossProjectContaminationRate: 0.5, budgetViolations: 1 }); + }); +}); diff --git a/test/recall-core.test.ts b/test/recall-core.test.ts new file mode 100644 index 0000000000..a5ff99a74f --- /dev/null +++ b/test/recall-core.test.ts @@ -0,0 +1,154 @@ +import { describe, expect, it } from "vitest"; +import { RecallCore } from "../src/recall/core.js"; +import type { HybridSearchResult, RecallConfig } from "../src/types.js"; +import { KV } from "../src/state/schema.js"; + +function makeKv() { + const store = new Map>(); + return { + get: async (scope: string, key: string): Promise => + (store.get(scope)?.get(key) as T) ?? null, + set: async (scope: string, key: string, value: T): Promise => { + if (!store.has(scope)) store.set(scope, new Map()); + store.get(scope)!.set(key, value); + return value; + }, + delete: async (scope: string, key: string): Promise => { + store.get(scope)?.delete(key); + }, + list: async (scope: string): Promise => + (Array.from(store.get(scope)?.values() || []) as T[]), + store, + }; +} + +const config: RecallConfig = { + budget: { + maxContextTokens: 80, + reservedBootstrapTokens: 20, + maxSemanticTokens: 60, + maxMemories: 5, + maxSessionSummaries: 1, + maxObservations: 3, + maxContinuityItems: 1, + }, + scope: { unknownAutoInjection: false, unknownExplicitSearch: true }, + trace: { retentionDays: 30, maxTraces: 100, maxDroppedItemsPerReason: 20 }, + injection: { reinjectionTurnWindow: 8 }, +}; + +function hit(id: string, score: number): HybridSearchResult { + return { + observation: { + id, + sessionId: "memory", + timestamp: "2026-07-01T00:00:00.000Z", + type: "decision", + title: id, + facts: [], + narrative: `${id} relevant implementation detail`, + concepts: [], + files: [], + importance: 7, + }, + bm25Score: score, + vectorScore: 0, + graphScore: 0, + combinedScore: score, + sessionId: "memory", + }; +} + +describe("RecallCore", () => { + it("blocks unknown memories from automatic prompt injection and records why", async () => { + const kv = makeKv(); + await kv.set(KV.memories, "mem_scoped", { + id: "mem_scoped", createdAt: "2026-07-01T00:00:00.000Z", updatedAt: "2026-07-01T00:00:00.000Z", + type: "fact", title: "scoped", content: "PPS7000 pair cache uses a staged key", concepts: [], files: [], sessionIds: [], strength: 7, version: 1, isLatest: true, + scope: { level: "project", projectId: "pps", repoId: "repo-a" }, origin: "manual", + }); + await kv.set(KV.memories, "mem_legacy", { + id: "mem_legacy", createdAt: "2026-07-01T00:00:00.000Z", updatedAt: "2026-07-01T00:00:00.000Z", + type: "fact", title: "legacy", content: "Ivan plan is not for PPS", concepts: [], files: [], sessionIds: [], strength: 7, version: 1, isLatest: true, + }); + const core = new RecallCore(kv as never, config, async () => [hit("mem_scoped", 0.8), hit("mem_legacy", 0.9)]); + + const result = await core.recall({ + entryPoint: "prompt", outputMode: "prompt_injection", query: "PPS7000 pair cache", + projectId: "pps", repoId: "repo-a", sessionId: "s1", + }); + + expect(result.context).toContain("PPS7000 pair cache"); + expect(result.context).not.toContain("Ivan plan"); + expect(result.trace.droppedCountsByDecision.scope_mismatch).toBe(1); + expect(result.trace.dropped[0]).toMatchObject({ id: "mem_legacy", decision: "scope_mismatch" }); + }); + + it("does not apply context token budget to ranked search results", async () => { + const kv = makeKv(); + for (const id of ["mem_one", "mem_two"]) { + await kv.set(KV.memories, id, { + id, createdAt: "2026-07-01T00:00:00.000Z", updatedAt: "2026-07-01T00:00:00.000Z", + type: "fact", title: id, content: "a deliberately long but relevant memory value for structured search", concepts: [], files: [], sessionIds: [], strength: 7, version: 1, isLatest: true, + scope: { level: "project", projectId: "pps" }, origin: "manual", + }); + } + const core = new RecallCore(kv as never, { ...config, budget: { ...config.budget, maxContextTokens: 1 } }, async () => [hit("mem_one", 0.8), hit("mem_two", 0.7)]); + const result = await core.recall({ entryPoint: "search", outputMode: "ranked_results", query: "relevant", projectId: "pps", limit: 2 }); + + expect(result.results.map((item) => item.id)).toEqual(["mem_one", "mem_two"]); + expect(result.context).toBe(""); + }); + + it("suppresses duplicate automatic injection only inside the current epoch and turn window", async () => { + const kv = makeKv(); + await kv.set(KV.memories, "mem_one", { + id: "mem_one", createdAt: "2026-07-01T00:00:00.000Z", updatedAt: "2026-07-01T00:00:00.000Z", + type: "fact", title: "one", content: "relevant durable implementation memory", concepts: [], files: [], sessionIds: [], strength: 7, version: 1, isLatest: true, + scope: { level: "project", projectId: "pps" }, origin: "manual", + }); + const core = new RecallCore(kv as never, config, async () => [hit("mem_one", 0.8)]); + const request = { entryPoint: "prompt" as const, outputMode: "prompt_injection" as const, query: "relevant", projectId: "pps", sessionId: "session" }; + + const first = await core.recall(request); + const second = await core.recall(request); + + expect(first.trace.selectedTokenCount).toBeGreaterThan(0); + expect(second.trace.selectedTokenCount).toBe(0); + expect(second.trace.droppedCountsByDecision.duplicate).toBe(1); + }); + + it("loads explicitly scoped bootstrap rules separately from semantic recall", async () => { + const kv = makeKv(); + await kv.set(KV.slots, "project_context", { + label: "project_context", content: "Run the project validation before publishing.", sizeLimit: 1000, + description: "", pinned: true, readOnly: false, scope: "project", projectId: "pps", repoId: "repo-a", + createdAt: "2026-07-01T00:00:00.000Z", updatedAt: "2026-07-01T00:00:00.000Z", + }); + await kv.set(KV.slots, "pending_items", { + label: "pending_items", content: "Validate the pair cache fixture.", sizeLimit: 1000, + description: "", pinned: true, readOnly: false, scope: "project", projectId: "pps", repoId: "repo-a", + createdAt: "2026-07-01T00:00:00.000Z", updatedAt: "2026-07-01T00:00:00.000Z", + }); + const core = new RecallCore(kv as never, config); + const result = await core.recall({ entryPoint: "session_start", outputMode: "bootstrap", projectId: "pps", repoId: "repo-a", sessionId: "s1" }); + + expect(result.context).toContain("Run the project validation"); + expect(result.context).toContain("Validate the pair cache fixture"); + expect(result.trace.selected.map((item) => item.kind)).toContain("continuity"); + }); + + it("counts wrappers and separators inside the hard rendered-context budget", async () => { + const kv = makeKv(); + await kv.set(KV.memories, "mem_budget", { + id: "mem_budget", createdAt: "2026-07-01T00:00:00.000Z", updatedAt: "2026-07-01T00:00:00.000Z", + type: "fact", title: "budget", content: "A long enough memory body to exercise conservative token packing.", concepts: [], files: [], sessionIds: [], strength: 7, version: 1, isLatest: true, + scope: { level: "project", projectId: "pps" }, origin: "manual", + }); + const core = new RecallCore(kv as never, { ...config, budget: { ...config.budget, maxContextTokens: 12, maxSemanticTokens: 12 } }, async () => [hit("mem_budget", 0.8)]); + const result = await core.recall({ entryPoint: "context", outputMode: "rendered_context", query: "budget", projectId: "pps" }); + + expect(result.trace.finalContextTokenCount).toBeLessThanOrEqual(12); + expect(result.trace.droppedCountsByDecision.over_budget).toBe(1); + }); +}); diff --git a/test/remember-project-scope.test.ts b/test/remember-project-scope.test.ts index bda699f5ae..152471861f 100644 --- a/test/remember-project-scope.test.ts +++ b/test/remember-project-scope.test.ts @@ -203,7 +203,7 @@ describe("mem::remember — cross-project dedup isolation", () => { expect(original?.isLatest).toBe(false); }); - it("allows an unscoped memory to be superseded by a scoped one (legacy compat)", async () => { + it("does not supersede an unknown-scope legacy memory from a scoped write", async () => { const sdk = mockSdk(); const kv = mockKV(); registerRememberFunction(sdk as never, kv as never); @@ -217,7 +217,7 @@ describe("mem::remember — cross-project dedup isolation", () => { }, }) as { memory: { id: string } }; - // New scoped memory — should supersede the legacy unscoped one + // New scoped memory must stay isolated from an unknown-scope legacy row. const scoped = await sdk.trigger({ function_id: "mem::remember", payload: { @@ -227,10 +227,10 @@ describe("mem::remember — cross-project dedup isolation", () => { }, }) as { memory: { supersedes: string[] } }; - expect(scoped.memory.supersedes).toContain(legacy.memory.id); + expect(scoped.memory.supersedes).not.toContain(legacy.memory.id); }); - it("allows a scoped memory to be superseded by an unscoped one (legacy compat)", async () => { + it("does not let an unknown-scope write supersede a scoped memory", async () => { const sdk = mockSdk(); const kv = mockKV(); registerRememberFunction(sdk as never, kv as never); @@ -244,7 +244,7 @@ describe("mem::remember — cross-project dedup isolation", () => { }, }) as { memory: { id: string } }; - // Unscoped write — should still supersede since one side has no project + // Unknown scope must not cross the project isolation boundary. const unscoped = await sdk.trigger({ function_id: "mem::remember", payload: { @@ -253,6 +253,6 @@ describe("mem::remember — cross-project dedup isolation", () => { }, }) as { memory: { supersedes: string[] } }; - expect(unscoped.memory.supersedes).toContain(scoped.memory.id); + expect(unscoped.memory.supersedes).not.toContain(scoped.memory.id); }); }); diff --git a/test/vector-retrieval-health.test.ts b/test/vector-retrieval-health.test.ts new file mode 100644 index 0000000000..565026b719 --- /dev/null +++ b/test/vector-retrieval-health.test.ts @@ -0,0 +1,27 @@ +import { describe, expect, it, vi } from "vitest"; +import { VectorRetrievalHealth } from "../src/recall/vector-health.js"; + +describe("VectorRetrievalHealth", () => { + it("opens after transient failures and allows only one half-open probe", () => { + vi.useFakeTimers(); + const health = new VectorRetrievalHealth(2, 1000); + expect(health.begin(true, true)).toMatchObject({ status: "healthy", attempted: true }); + health.failure(new Error("429 quota exceeded")); + expect(health.begin(true, true)).toMatchObject({ status: "healthy", attempted: true }); + health.failure(new Error("429 quota exceeded")); + expect(health.begin(true, true)).toMatchObject({ status: "degraded", attempted: false }); + vi.advanceTimersByTime(1000); + expect(health.begin(true, true)).toMatchObject({ status: "healthy", attempted: true }); + expect(health.begin(true, true)).toMatchObject({ status: "degraded", attempted: false }); + health.success(); + expect(health.begin(true, true)).toMatchObject({ status: "healthy", attempted: true }); + vi.useRealTimers(); + }); + + it("latches permanent authentication failures", () => { + const health = new VectorRetrievalHealth(); + health.begin(true, true); + expect(health.failure(new Error("401 invalid API key"))).toMatchObject({ status: "disabled" }); + expect(health.begin(true, true)).toMatchObject({ status: "disabled", attempted: false }); + }); +}); From 2245597286253a158b3d00401ad8838595698828 Mon Sep 17 00:00:00 2001 From: Jenq Shiin Haw <1343528669@qq.com> Date: Sun, 12 Jul 2026 14:08:52 +0800 Subject: [PATCH 6/6] docs: sync stats to P2 recall + P1 durable baseline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Update all documentation to reflect the 6-commit feature branch: - AGENTS.md: stats bump (184 src files, ~42,400 LOC, 1,500+ tests, 296 functions, 58 KV scopes), new recall code pattern section - README.md: stat line update, skills catalog (10 invocable + 7 reference), scoped traceable recall section, recall TOML config, durable candidates env vars, stale skill counts (15→17, 8→10 invocable) - plugin.json: refresh description with recall capabilities - Skill reference docs regenerated via npm run skills:gen All 6 tool-count-consistency tests pass. Signed-off-by: Jenq Shiin Haw <1343528669@qq.com> --- AGENTS.md | 34 ++++++++++++++++--- README.md | 83 ++++++++++++++++++++++++++++++++++++++++++---- plugin/plugin.json | 2 +- 3 files changed, 107 insertions(+), 12 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index e2c72218b8..2e080ba01a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -90,6 +90,30 @@ case "memory_your_tool": { } ``` +### Recall Pattern + +Recall is centralized through `RecallCore` (`src/recall/core.ts`). All search paths (`context`, `enrich`, `prompt-submit`, `pre-compact`, MCP `memory_recall`, `memory_smart_search`) route through it instead of calling `hybridSearch` or `kv.list` directly. This ensures every recall produces a traceable `RecallTrace` with selected/dropped items and reasons. + +```typescript +const recallCore = new RecallCore(kv, config.recall, hybridSearch.searchWithTrace); +// context generation +registerContextFunction(sdk, kv, config.tokenBudget, recallCore); +// enrichment (PreToolUse) +registerEnrichFunction(sdk, kv, recallCore); +// MCP tools: memory_recall + memory_smart_search with outputMode=structured/context +// route through mem::context which delegates to recallCore +``` + +**When adding new search/recal paths, you MUST:** +1. Route through `RecallCore` — never bypass it with direct `hybridSearch` or `kv.list` +2. Wire recall trace persistence via `persistRecallTrace()` in `src/recall/trace-store.ts` +3. Register new debug endpoints in `src/triggers/api.ts` (`api::recall-debug` / `api::recall-debug-by-id`) +4. Add corresponding skill docs if the recall path is user-facing + +**When adding new KV scopes for recall traces:** +1. `src/state/schema.ts` — add to the KV object (e.g. `recallTraces: { kv: "memory" }`) +2. `src/types.ts` — add the corresponding interface (`RecallTrace`, `RecallItemTrace`, etc.) + ### Hook Scripts Hook scripts in `src/hooks/` are standalone Node.js scripts (no iii-sdk import). They read JSON from stdin, make HTTP calls to the REST API, and exit. There are two patterns depending on whether Claude Code consumes the script's stdout: @@ -114,11 +138,11 @@ Hook scripts in `src/hooks/` are standalone Node.js scripts (no iii-sdk import). - Test files go in `test/` with `.test.ts` extension - Follow existing patterns in `test/crystallize.test.ts` for function tests -## Current Stats (v0.9.16) +## Current Stats (v0.9.27) -- 53 MCP tools (8 visible by default, `AGENTMEMORY_TOOLS=all` for all) +- 53 MCP tools (core: memory_recall, memory_save, memory_compress_file, memory_file_history, memory_patterns, memory_sessions, memory_smart_search, memory_timeline, memory_profile, memory_export, memory_relations; `AGENTMEMORY_TOOLS=all` for 53) - 136 REST endpoints - 6 MCP resources, 3 MCP prompts -- 12 hooks, 17 skills -- 50+ iii functions -- 950+ tests +- 12 hooks, 17 skills (10 invocable + 7 reference) +- 296 registered functions (148 mem::, 137 api::, 6 mcp::, 5 event::) +- 184 source files · ~42,400 LOC · 1,500+ tests · 58 KV scopes diff --git a/README.md b/README.md index 0b1e0ced7a..7e8a0bd7fc 100644 --- a/README.md +++ b/README.md @@ -100,7 +100,7 @@ agentmemory # start the memory server on :3 agentmemory demo # seed sample sessions + prove recall agentmemory demo --serve # one command: boot server, run demo, tear down (no second terminal) agentmemory connect claude-code # wire MCP into your agent (also: copilot-cli, codex, cursor, gemini-cli, ...) -npx skills add rohitg00/agentmemory -y # install 15 native skills (8 you can invoke, 7 reference) so your agent knows when to use the tools +npx skills add rohitg00/agentmemory -y # install 17 native skills (10 invocable, 7 reference) so your agent knows when to use the tools ``` Or via `npx` (no install): @@ -545,7 +545,7 @@ The Codex plugin ships from the same `plugin/` directory as the Claude Code plug - `@agentmemory/mcp` as an MCP server (proxies all 53 tools when `AGENTMEMORY_URL` points at a running agentmemory server; falls back to 7 tools locally when no server is reachable) - 6 lifecycle hooks: `SessionStart`, `UserPromptSubmit`, `PreToolUse`, `PostToolUse`, `PreCompact`, `Stop` -- 8 invocable skills: `/recall`, `/remember`, `/session-history`, `/forget`, `/recap`, `/handoff`, `/commit-context`, `/commit-history`, plus 7 reference skills the agent loads on demand (MCP tools, REST API, config, agents, hooks, architecture, and the skill-authoring guide) +- 10 invocable skills: `/recall`, `/remember`, `/session-history`, `/forget`, `/recap`, `/handoff`, `/commit-context`, `/commit-history`, `/recall-debug`, `/why-memory`, plus 7 reference skills the agent loads on demand (MCP tools, REST API, config, agents, hooks, architecture, and the skill-authoring guide) Codex's hook engine injects `CLAUDE_PLUGIN_ROOT` into hook subprocesses (per [`codex-rs/hooks/src/engine/discovery.rs`](https://github.com/openai/codex/blob/main/codex-rs/hooks/src/engine/discovery.rs)), so the same hook scripts work across both hosts without duplication. Subagent / SessionEnd / Notification / TaskCompleted / PostToolUseFailure events are Claude-Code-only and are not registered for Codex. @@ -638,7 +638,7 @@ This is **complementary** to `agentmemory connect `: - `agentmemory connect ` writes the MCP server config so the tools are available. - `npx skills add rohitg00/agentmemory` installs the skills so the agent knows when to call them. -For the few agents the skills CLI doesn't cover yet (Zed v1.3.x and below), drop the 15 SKILL.md files under the agent's native skill directory yourself — same format works everywhere. +For the few agents the skills CLI doesn't cover yet (Zed v1.3.x and below), drop the 17 SKILL.md files under the agent's native skill directory yourself — same format works everywhere. #### Standard MCP block @@ -676,7 +676,7 @@ The agentmemory entry is the **same MCP server block** across every host that us | **Qwen Code** | `~/.qwen/settings.json` | `agentmemory connect qwen` writes the standard `mcpServers` block. Hook payload is field-compatible with Claude Code, so the existing 12-hook scripts work without modification — wire them via the `hooks` section in the same `settings.json`. | | **Antigravity** (replaces Gemini CLI) | `mcp_config.json` (in Antigravity's User dir) | `agentmemory connect antigravity` writes the standard `mcpServers` block. macOS: `~/Library/Application Support/Antigravity/User/`. Linux: `~/.config/Antigravity/User/`. Use after the 2026-06-18 Gemini CLI sunset. | | **Kiro** | `~/.kiro/settings/mcp.json` | `agentmemory connect kiro` writes the user-level config. Workspace overrides go in `.kiro/settings/mcp.json` next to your code. | -| **Warp** | `~/.warp/.mcp.json` | `agentmemory connect warp` writes the standard `mcpServers` block. Warp also auto-discovers skills from `.claude/skills/` — once the Claude Code plugin is installed the 8 agentmemory skills (`remember`, `recall`, `recap`, `handoff`, `forget`, `commit-context`, `commit-history`, `session-history`) appear natively in Warp's slash-command palette. | +| **Warp** | `~/.warp/.mcp.json` | `agentmemory connect warp` writes the standard `mcpServers` block. Warp also auto-discovers skills from `.claude/skills/` — once the Claude Code plugin is installed the 10 agentmemory skills (`remember`, `recall`, `recap`, `handoff`, `forget`, `commit-context`, `commit-history`, `session-history`, `recall-debug`, `why-memory`) appear natively in Warp's slash-command palette. | | **Cline (CLI)** | `~/.cline/mcp.json` | `agentmemory connect cline` writes the standard `mcpServers` block. VS Code extension users: paste the same block via Cline Settings → MCP Servers → Edit JSON. | | **Continue.dev** | `~/.continue/config.yaml` (preferred) or `config.json` (legacy) | `agentmemory connect continue` creates `config.yaml` from scratch when neither exists, or modifies existing `config.json`. **If you already have `config.yaml`** the adapter prints the exact block to paste under `mcpServers:` — it won't silently rewrite your yaml because preserving comments and anchors safely needs a YAML parser the package doesn't ship. Continue uses array form (not object) for `mcpServers`. | | **Zed** | `~/.config/zed/settings.json` | `agentmemory connect zed` writes under `context_servers` (Zed's key, NOT `mcpServers`). Remote MCP servers can be wired via `{"url": "..."}` instead. | @@ -955,6 +955,50 @@ npm install @xenova/transformers | Cohere | `embed-english-v3.0` | Free trial | General purpose | | OpenRouter | Any model | Varies | Multi-model proxy | +### Scoped Traceable Recall + +Every recall (search, context injection, enrichment) is centralized through `RecallCore` (`src/recall/core.ts`) and produces a `RecallTrace` recording exactly which items were selected or dropped — and why. Traces are queryable via the `recall-debug` APIs and skills. + +| Trace field | What it records | +|---|---| +| `selected` | Items injected into context, with scores and source | +| `dropped` | Items considered but excluded, with a structured reason | +| `budget` | Token budget applied at recall time | +| `scope` | Resolved project/repo/checkout identity fingerprint | +| `entryPoint` | Which code path triggered the recall (context, enrich, smart_search, etc.) | + +**Recall config** (`~/.agentmemory/config.toml`): + +```toml +[recall_budget] +max_context_tokens = 800 +reserved_bootstrap_tokens = 200 +max_semantic_tokens = 600 +max_memories = 5 +max_session_summaries = 1 +max_observations = 3 +max_continuity_items = 1 + +[recall_scope] +unknown_auto_injection = false +unknown_explicit_search = true + +[recall_trace] +retention_days = 30 +max_traces = 10000 +max_dropped_items_per_reason = 20 + +[recall_injection] +reinjection_turn_window = 8 +``` + +**Recall debug endpoints:** + +| Method | Path | Description | +|--------|------|-------------| +| `GET` | `/agentmemory/recall/debug` | Query recent recall traces, filterable by `projectId`, `repoId`, `entryPoint`, `itemId` | +| `GET` | `/agentmemory/recall/debug/{traceId}` | Fetch a single recall trace by id | + ---

MCP Server

@@ -1023,7 +1067,7 @@ npm install @xenova/transformers
-### 6 Resources · 3 Prompts · 4 Skills +### 6 Resources · 3 Prompts · 10 Skills + 7 Reference | Type | Name | Description | |------|------|-------------| @@ -1031,13 +1075,21 @@ npm install @xenova/transformers | Resource | `agentmemory://project/{name}/profile` | Per-project intelligence | | Resource | `agentmemory://memories/latest` | Latest 10 active memories | | Resource | `agentmemory://graph/stats` | Knowledge graph statistics | +| Resource | `agentmemory://recall/ledger/{projectId}` | Recall injection ledger (dedup guard) | +| Resource | `agentmemory://recall/trace/{traceId}` | Single recall trace by id | | Prompt | `recall_context` | Search + return context messages | | Prompt | `session_handoff` | Handoff data between agents | | Prompt | `detect_patterns` | Analyze recurring patterns | | Skill | `/recall` | Search memory | | Skill | `/remember` | Save to long-term memory | +| Skill | `/recap` | Recap recent session activity | +| Skill | `/handoff` | Handoff context between agents/sessions | | Skill | `/session-history` | Recent session summaries | | Skill | `/forget` | Delete observations/sessions | +| Skill | `/commit-context` | Show commit-level context for a change | +| Skill | `/commit-history` | Show commit history for a file | +| Skill | `/recall-debug` | Inspect a single recall invocation — why items were selected or dropped | +| Skill | `/why-memory` | Explain a memory's recent recall history and scope decisions | ### Standalone MCP @@ -1211,7 +1263,7 @@ Full registry: [workers.iii.dev](https://workers.iii.dev). Every worker there co | Prometheus / Grafana | iii OTEL + health monitor | | Custom plugin systems | `iii worker add ` | -**174 source files · ~37,800 LOC · 1,423+ tests · 258 functions · 44 KV scopes** — all on three primitives. No `agentmemory plugin install`. The plugin system is iii itself. +**184 source files · ~42,400 LOC · 1,500+ tests · 296 functions · 58 KV scopes** — all on three primitives. No `agentmemory plugin install`. The plugin system is iii itself. --- @@ -1450,7 +1502,26 @@ Create `~/.agentmemory/.env`: # unknown_auto_injection = false # unknown_explicit_search = true # [recall_trace] +# retention_days = 30 +# max_traces = 10000 # max_dropped_items_per_reason = 20 +# [recall_injection] +# reinjection_turn_window = 8 + +# Durable candidates (P1): archive processing extracts candidate +# memories from past sessions. Promote is explicit and idempotent. +# Features: +# AGENTMEMORY_DURABLE_CANDIDATES_ENABLED=false # OFF by default +# POST /agentmemory/archive/process # generate candidates +# POST /agentmemory/durable-candidates/backfill # backfill from existing observations +# POST /agentmemory/durable-candidates/promote # promote a candidate to memory + +# Vector index maintenance +# AGENTMEMORY_DROP_STALE_INDEX=false # OFF by default. When on, stale + # (dimension-mismatched) vectors + # in the persisted index are + # silently dropped on load instead + # of rejecting the entire index. # Auth # AGENTMEMORY_SECRET=your-secret diff --git a/plugin/plugin.json b/plugin/plugin.json index 20ffae334a..22ed75a207 100644 --- a/plugin/plugin.json +++ b/plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "agentmemory", "version": "0.9.27", - "description": "Persistent memory for AI coding agents -- captures tool usage, compresses via LLM, injects context into future sessions. 12 hooks, 53 MCP tools, 17 skills, real-time viewer.", + "description": "Persistent memory for AI coding agents -- captures tool usage, compresses via LLM, injects context into future sessions. 12 hooks, 53 MCP tools, 17 skills (10 invocable + 7 reference), real-time viewer, scoped traceable recall.", "author": { "name": "Rohit Ghumare", "url": "https://github.com/rohitg00"