diff --git a/AGENTS.md b/AGENTS.md index ead439f1b3..762b9b7455 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -117,7 +117,7 @@ Hook scripts in `src/hooks/` are standalone Node.js scripts (no iii-sdk import). ## Current Stats (v0.9.28) - 53 MCP tools (8 visible by default, `AGENTMEMORY_TOOLS=all` for all) -- 128 REST endpoints +- 129 REST endpoints - 6 MCP resources, 3 MCP prompts - 12 hooks, 15 skills - 260+ iii functions diff --git a/README.md b/README.md index 48f4045cd6..4c6261f980 100644 --- a/README.md +++ b/README.md @@ -1498,7 +1498,7 @@ Create `~/.agentmemory/.env`:

API

-128 endpoints on port `3111`. The REST API binds to `127.0.0.1` by default. Protected endpoints require `Authorization: Bearer ` when `AGENTMEMORY_SECRET` is set, and mesh sync endpoints require `AGENTMEMORY_SECRET` on both peers. +129 endpoints on port `3111`. The REST API binds to `127.0.0.1` by default. Protected endpoints require `Authorization: Bearer ` when `AGENTMEMORY_SECRET` is set, and mesh sync endpoints require `AGENTMEMORY_SECRET` on both peers.
Key endpoints diff --git a/docs/recipes/pairings.md b/docs/recipes/pairings.md index 093abf8554..3337fe1313 100644 --- a/docs/recipes/pairings.md +++ b/docs/recipes/pairings.md @@ -80,6 +80,15 @@ Recipe with agentmemory: This is the broadest sweep across artifacts that live alongside the code. agentmemory then captures everything the agent does while exploring that graph — the questions you asked, the conclusions, the decisions — so the next session opens with both the graph and the conversation history available. +agentmemory can also import the graph directly. With `GRAPH_EXTRACTION_ENABLED=true`: + +```bash +curl -X POST localhost:3111/agentmemory/graph/import-graphify \ + -H 'content-type: application/json' -d '{"cwd": "'"$PWD"'"}' +``` + +This merges Graphify's structural entities and relationships (with their EXTRACTED/INFERRED confidence carried over as edge weight) into agentmemory's knowledge graph, so graph retrieval and context injection see codebase structure the developer never touched in a session. Re-importing after `graphify update .` merges instead of duplicating. + ## How the four projects line up Four planes, four consumers, four update models. None of them try to do what the others do. diff --git a/integrations/filesystem-watcher/watcher.mjs b/integrations/filesystem-watcher/watcher.mjs index d68b2b8d16..a73d178f35 100644 --- a/integrations/filesystem-watcher/watcher.mjs +++ b/integrations/filesystem-watcher/watcher.mjs @@ -248,6 +248,18 @@ export class FilesystemWatcher { const failures = []; for (const root of this.roots) { try { + // Validate the root before handing it to fs.watch: on Linux with + // Node 24+, fs.watch on a nonexistent path no longer throws + // synchronously, so a missing root would otherwise count as + // "attached" and be watched-in-name-only. An explicit stat keeps + // the failure deterministic across Node versions and platforms. + const st = statSync(root, { throwIfNoEntry: false }); + if (!st) { + throw new Error("no such directory"); + } + if (!st.isDirectory()) { + throw new Error("not a directory"); + } const handle = watch( root, { recursive: true, persistent: true }, diff --git a/plugin/scripts/session-end.mjs b/plugin/scripts/session-end.mjs index 811c1c2620..f19d80addf 100755 --- a/plugin/scripts/session-end.mjs +++ b/plugin/scripts/session-end.mjs @@ -30,23 +30,6 @@ async function main() { body: JSON.stringify({ sessionId }), signal: AbortSignal.timeout(3e4) }).catch(() => {}); - if (process.env["CONSOLIDATION_ENABLED"] === "true") { - fetch(`${REST_URL}/agentmemory/crystals/auto`, { - method: "POST", - headers: authHeaders(), - body: JSON.stringify({ olderThanDays: 0 }), - signal: AbortSignal.timeout(6e4) - }).catch(() => {}); - fetch(`${REST_URL}/agentmemory/consolidate-pipeline`, { - method: "POST", - headers: authHeaders(), - body: JSON.stringify({ - tier: "all", - force: true - }), - signal: AbortSignal.timeout(12e4) - }).catch(() => {}); - } if (process.env["CLAUDE_MEMORY_BRIDGE"] === "true") fetch(`${REST_URL}/agentmemory/claude-bridge/sync`, { method: "POST", headers: authHeaders(), diff --git a/plugin/skills/agentmemory-config/REFERENCE.md b/plugin/skills/agentmemory-config/REFERENCE.md index e56e8743e1..cbd485d3bb 100644 --- a/plugin/skills/agentmemory-config/REFERENCE.md +++ b/plugin/skills/agentmemory-config/REFERENCE.md @@ -3,12 +3,13 @@ Generated by scanning `src/` for `AGENTMEMORY_*` usage. Do not edit the block below by hand; run `npm run skills:gen` after adding or removing a variable. Internal markers ending in two underscores are excluded. -Configuration is read from the environment and from `~/.agentmemory/.env` (no `export` prefix). 35 recognized variables: +Configuration is read from the environment and from `~/.agentmemory/.env` (no `export` prefix). 36 recognized variables: - `AGENTMEMORY_AGENT_SCOPE` - `AGENTMEMORY_ALLOW_AGENT_SDK` - `AGENTMEMORY_AUTO_COMPRESS` - `AGENTMEMORY_COMMIT_SHA` +- `AGENTMEMORY_CONSOLIDATION_COOLDOWN_MS` - `AGENTMEMORY_COPILOT_MCP_BLOCK` - `AGENTMEMORY_CWD` - `AGENTMEMORY_DATA_DIR` diff --git a/plugin/skills/agentmemory-rest-api/REFERENCE.md b/plugin/skills/agentmemory-rest-api/REFERENCE.md index d36171defe..77e0a5e4d3 100644 --- a/plugin/skills/agentmemory-rest-api/REFERENCE.md +++ b/plugin/skills/agentmemory-rest-api/REFERENCE.md @@ -5,7 +5,7 @@ Generated from `src/triggers/api.ts`. Do not edit the block below by hand; run ` The REST API is the primary surface. All paths are under `http://localhost:3111` (override with `--port`). When `AGENTMEMORY_SECRET` is set, send `Authorization: Bearer $AGENTMEMORY_SECRET`; localhost is otherwise open. -117 registered endpoints: +118 registered endpoints: | Method | Path | | --- | --- | @@ -52,6 +52,7 @@ The REST API is the primary surface. All paths are under `http://localhost:3111` | DELETE | `/agentmemory/governance/memories` | | POST | `/agentmemory/graph/build` | | POST | `/agentmemory/graph/extract` | +| POST | `/agentmemory/graph/import-graphify` | | POST | `/agentmemory/graph/query` | | POST | `/agentmemory/graph/reset` | | POST | `/agentmemory/graph/snapshot-rebuild` | diff --git a/src/cli.ts b/src/cli.ts index d763eada3d..2ae20f08b1 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -58,6 +58,7 @@ import { renderSplash } from "./cli/splash.js"; import { isFirstRun, readPrefs, resetPrefs, writePrefs } from "./cli/preferences.js"; import { runOnboarding } from "./cli/onboarding.js"; import { setBootVerbose } from "./logger.js"; +import { hydrateProcessEnvFromFile } from "./config.js"; import { VERSION } from "./version.js"; import { getAllTools, ESSENTIAL_TOOLS } from "./mcp/tools-registry.js"; import { knownAgents } from "./cli/connect/index.js"; @@ -83,6 +84,12 @@ setBootVerbose(IS_VERBOSE); const IS_RESET = args.includes("--reset"); +// Fold ~/.agentmemory/.env into process.env before any port/URL read +// (getRestPort/getBaseUrl/getStreamPort/getEnginePort) or the --port / +// --instance / --tools handlers below. Only-if-unset, so a real +// process.env value — including one just set by a CLI flag — still wins. +hydrateProcessEnvFromFile(); + // --version / -V early exit. Print VERSION + exit before any side effects // (engine boot, env load, dir mkdir). `-v` is taken by --verbose so we // reserve `-V` (capital) for version per POSIX convention. diff --git a/src/cli/connect/guidelines.ts b/src/cli/connect/guidelines.ts new file mode 100644 index 0000000000..d1b95593a0 --- /dev/null +++ b/src/cli/connect/guidelines.ts @@ -0,0 +1,196 @@ +import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { homedir } from "node:os"; +import { dirname, join } from "node:path"; + +// Activating memory for agents that have no auto-capture hooks: write a short +// "use the memory tools" guideline into each agent's native rules mechanism so +// the agent proactively calls memory_recall / memory_save. Paths + formats are +// verified against each agent's OFFICIAL docs (cited per entry below). Claude +// Code / Codex are omitted here: they already auto-capture via lifecycle hooks. + +const START = ""; +const END = ""; + +const GUIDELINE_BODY = `## Agent memory (agentmemory) + +You have persistent long-term memory via the agentmemory MCP server. Tools: \`memory_recall\`, \`memory_smart_search\`, \`memory_save\`, \`memory_sessions\`. + +- At the START of a task, call \`memory_recall\` (or \`memory_smart_search\`) with the task context to load relevant past decisions, fixes, and preferences before asking the user to repeat anything. +- When you learn something durable (a decision, a fix, a gotcha, a user preference, a project convention), call \`memory_save\` to persist it. +- Prefer recalling over re-deriving, and save concise reusable facts rather than transcripts.`; + +// "block" -> upsert a marked block inside a shared instructions file +// "mdc" -> Cursor project rule: dedicated .mdc with alwaysApply frontmatter +// "steering" -> Kiro steering file: dedicated .md with inclusion:always frontmatter +// "rule" -> dedicated always-on markdown rule file (own file, safe to own) +type GuidelineFormat = "block" | "mdc" | "steering" | "rule"; + +type GuidelineTarget = { + // Preferred user-global rules file (absolute). Omitted when the agent has no + // clean global rules FILE (UI-only or project-scoped), forcing the project + // fallback. Global is preferred so memory activates across all repos. + globalPath?: string; + // Project fallback, relative to cwd. + projectPath: string; + format: GuidelineFormat; + scope: "global" | "project"; + source: string; // official doc URL the path/format was verified against +}; + +export function guidelineTargets( + home: string = homedir(), +): Record { + return { + // No global rules FILE (User Rules are UI-only) -> project .cursor/rules/*.mdc + cursor: { + projectPath: join(".cursor", "rules", "agentmemory.mdc"), + format: "mdc", + scope: "project", + source: "https://cursor.com/docs/rules", + }, + // Cline's global rules dir is a non-standard ~/Documents path; the dedicated + // project rule (no frontmatter = always active) is the verified stable form. + cline: { + projectPath: join(".clinerules", "agentmemory.md"), + format: "rule", + scope: "project", + source: "https://docs.cline.bot/customization/cline-rules", + }, + // Continue does not auto-read AGENTS.md; rules live in .continue/rules/*.md. + continue: { + projectPath: join(".continue", "rules", "00-agentmemory.md"), + format: "rule", + scope: "project", + source: "https://docs.continue.dev/customize/deep-dives/rules", + }, + zed: { + globalPath: join(home, ".config", "zed", "AGENTS.md"), + projectPath: "AGENTS.md", + format: "block", + scope: "global", + source: "https://zed.dev/docs/ai/instructions", + }, + // Warp global Rules are UI-managed (no file path) -> project AGENTS.md. + warp: { + projectPath: "AGENTS.md", + format: "block", + scope: "project", + source: "https://docs.warp.dev/agent-platform/capabilities/rules", + }, + kiro: { + globalPath: join(home, ".kiro", "steering", "agentmemory.md"), + projectPath: join(".kiro", "steering", "agentmemory.md"), + format: "steering", + scope: "global", + source: "https://kiro.dev/docs/steering", + }, + "gemini-cli": { + globalPath: join(home, ".gemini", "GEMINI.md"), + projectPath: "GEMINI.md", + format: "block", + scope: "global", + source: "https://github.com/google-gemini/gemini-cli/blob/main/docs/cli/gemini-md.md", + }, + qwen: { + globalPath: join(home, ".qwen", "QWEN.md"), + projectPath: "QWEN.md", + format: "block", + scope: "global", + source: "https://github.com/QwenLM/qwen-code/blob/main/docs/users/features/memory.md", + }, + opencode: { + globalPath: join(home, ".config", "opencode", "AGENTS.md"), + projectPath: "AGENTS.md", + format: "block", + scope: "global", + source: "https://opencode.ai/docs/rules", + }, + droid: { + globalPath: join(home, ".factory", "AGENTS.md"), + projectPath: "AGENTS.md", + format: "block", + scope: "global", + source: "https://docs.factory.ai/cli/configuration/agents-md", + }, + // Antigravity does NOT read AGENTS.md; global rules live in ~/.gemini/GEMINI.md. + antigravity: { + globalPath: join(home, ".gemini", "GEMINI.md"), + projectPath: join(".agents", "rules", "agentmemory.md"), + format: "block", + scope: "global", + source: "https://antigravity.google/docs/rules-workflows", + }, + "copilot-cli": { + globalPath: join(home, ".copilot", "copilot-instructions.md"), + projectPath: join(".github", "copilot-instructions.md"), + format: "block", + scope: "global", + source: + "https://docs.github.com/en/copilot/how-tos/copilot-cli/customize-copilot/add-custom-instructions", + }, + }; +} + +function renderDedicated(format: GuidelineFormat): string { + if (format === "mdc") { + return `---\ndescription: agentmemory long-term memory usage\nalwaysApply: true\n---\n\n${GUIDELINE_BODY}\n`; + } + if (format === "steering") { + return `---\ninclusion: always\n---\n\n${GUIDELINE_BODY}\n`; + } + // plain dedicated rule (Cline / Continue) + return `${GUIDELINE_BODY}\n`; +} + +// Insert or replace the marked block in a shared instructions file, preserving +// any surrounding user content. Idempotent: re-running updates only our block. +function upsertBlock(existing: string): string { + const block = `${START}\n${GUIDELINE_BODY}\n${END}`; + const startIdx = existing.indexOf(START); + const endIdx = existing.indexOf(END); + if (startIdx !== -1 && endIdx !== -1 && endIdx > startIdx) { + return ( + existing.slice(0, startIdx) + block + existing.slice(endIdx + END.length) + ); + } + // Malformed marker state: a lone or reversed marker. Appending here would + // let a later run pair the orphan marker with the appended block's twin and + // cut the user's content between them. Leave the file untouched. + if (startIdx !== -1 || endIdx !== -1) { + return existing; + } + const sep = existing.length === 0 || existing.endsWith("\n") ? "" : "\n"; + const lead = existing.length === 0 ? "" : "\n"; + return `${existing}${sep}${lead}${block}\n`; +} + +export type GuidelineResult = + | { kind: "written"; path: string; scope: string; source: string } + | { kind: "unchanged"; path: string; scope: string } + | { kind: "would-write"; path: string; scope: string } + | { kind: "no-target" }; + +export function writeGuideline( + agentName: string, + opts: { cwd: string; home?: string; dryRun?: boolean } = { cwd: process.cwd() }, +): GuidelineResult { + const home = opts.home ?? homedir(); + const target = guidelineTargets(home)[agentName]; + if (!target) return { kind: "no-target" }; + + const path = target.globalPath ?? join(opts.cwd, target.projectPath); + const dedicated = target.format !== "block"; + + const existing = existsSync(path) ? readFileSync(path, "utf8") : ""; + const next = dedicated ? renderDedicated(target.format) : upsertBlock(existing); + + if (existing === next) { + return { kind: "unchanged", path, scope: target.scope }; + } + if (opts.dryRun) { + return { kind: "would-write", path, scope: target.scope }; + } + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, next, "utf8"); + return { kind: "written", path, scope: target.scope, source: target.source }; +} diff --git a/src/cli/connect/index.ts b/src/cli/connect/index.ts index 0d9412ca64..0014e7d4d1 100644 --- a/src/cli/connect/index.ts +++ b/src/cli/connect/index.ts @@ -2,6 +2,7 @@ import { platform } from "node:os"; import * as p from "@clack/prompts"; import pc from "picocolors"; import type { ConnectAdapter, ConnectOptions, ConnectResult } from "./types.js"; +import { writeGuideline } from "./guidelines.js"; import { adapter as antigravity } from "./antigravity.js"; import { adapter as claudeCode } from "./claude-code.js"; import { adapter as cline } from "./cline.js"; @@ -56,6 +57,7 @@ function parseFlags(args: string[]): { force: boolean; all: boolean; withHooks: boolean; + guidelines: boolean; positional: string[]; } { const positional: string[] = []; @@ -63,14 +65,16 @@ function parseFlags(args: string[]): { let force = false; let all = false; let withHooks = false; + let guidelines = true; // memory-usage guideline is written by default for (const a of args) { if (a === "--dry-run") dryRun = true; else if (a === "--force") force = true; else if (a === "--all") all = true; else if (a === "--with-hooks") withHooks = true; + else if (a === "--no-guidelines") guidelines = false; else if (!a.startsWith("-")) positional.push(a); } - return { dryRun, force, all, withHooks, positional }; + return { dryRun, force, all, withHooks, guidelines, positional }; } export async function runAdapter( @@ -88,7 +92,33 @@ export async function runAdapter( p.log.message(adapter.protocolNote); } try { - return await adapter.install(opts); + const result = await adapter.install(opts); + // After MCP/hooks are wired, activate memory for hook-less agents by + // writing a memory-usage guideline into their native rules file. Best + // effort: never fail the connect over the guideline. + if ( + opts.guidelines !== false && + (result.kind === "installed" || result.kind === "already-wired") + ) { + try { + const g = writeGuideline(adapter.name, { + cwd: process.cwd(), + dryRun: opts.dryRun, + }); + if (g.kind === "written") { + p.log.message( + ` ${pc.dim("guideline")} ${g.scope} → ${g.path} (memory auto-use)`, + ); + } else if (g.kind === "would-write") { + p.log.message(` ${pc.dim("[dry-run] guideline")} → ${g.path}`); + } + } catch (gerr) { + p.log.warn( + `${adapter.displayName}: guideline not written (${gerr instanceof Error ? gerr.message : String(gerr)})`, + ); + } + } + return result; } catch (err) { p.log.error( `${adapter.displayName}: ${err instanceof Error ? err.message : String(err)}`, @@ -98,7 +128,8 @@ export async function runAdapter( } export async function runConnect(args: string[]): Promise { - const { dryRun, force, all, withHooks, positional } = parseFlags(args); + const { dryRun, force, all, withHooks, guidelines, positional } = + parseFlags(args); const allowWindowsAdapter = positional.length === 1 && positional[0]?.toLowerCase() === "copilot-cli"; if (platform() === "win32" && !allowWindowsAdapter) { @@ -110,7 +141,7 @@ export async function runConnect(args: string[]): Promise { return; } - const opts: ConnectOptions = { dryRun, force, withHooks }; + const opts: ConnectOptions = { dryRun, force, withHooks, guidelines }; p.intro("agentmemory connect"); diff --git a/src/cli/connect/types.ts b/src/cli/connect/types.ts index 169dde1382..bae29622c3 100644 --- a/src/cli/connect/types.ts +++ b/src/cli/connect/types.ts @@ -8,6 +8,13 @@ export type ConnectOptions = { * hooks from dispatching on Codex Desktop. No-op for other adapters. */ withHooks?: boolean; + /** + * When true (default), after wiring the agent's MCP/hooks, also write a + * memory-usage guideline into the agent's native rules file so hook-less + * agents proactively call memory_recall / memory_save. Disabled with + * `--no-guidelines`. No-op for agents without a guideline target. + */ + guidelines?: boolean; }; export type ConnectAdapter = { diff --git a/src/config.ts b/src/config.ts index 3348ac2267..e2b520613b 100644 --- a/src/config.ts +++ b/src/config.ts @@ -22,8 +22,20 @@ const ENV_FILE = join(DATA_DIR, ".env"); let warnPremiumModelShown = false; +// Parsed ~/.agentmemory/.env, memoized for the process lifetime. getMergedEnv() +// runs on every config getter (~20 of them), so without this cache a single +// request would readFileSync + reparse the file dozens of times. The file is +// boot-static, so read it from disk once and reuse the result. Tests that +// mutate the file between cases reset the module (clearing this via reload) or +// call __resetEnvFileCache(). +let envFileCache: Record | undefined; + function loadEnvFile(): Record { - if (!existsSync(ENV_FILE)) return {}; + if (envFileCache) return envFileCache; + if (!existsSync(ENV_FILE)) { + envFileCache = {}; + return envFileCache; + } const content = readFileSync(ENV_FILE, "utf-8"); const vars: Record = {}; for (const line of content.split("\n")) { @@ -43,13 +55,34 @@ function loadEnvFile(): Record { } vars[key] = val; } - return vars; + envFileCache = vars; + return envFileCache; +} + +// Test hook: clears the memoized .env so the next loadEnvFile() re-reads disk +// within the same module instance. vi.resetModules() reloads this module and +// resets the cache on its own; this exists for tests that mutate the file +// without a module reload. +export function __resetEnvFileCache(): void { + envFileCache = undefined; } function hasRealValue(v: string | undefined): v is string { return typeof v === "string" && v.trim().length > 0; } +// Hydrate ~/.agentmemory/.env into process.env at boot. loadEnvFile() is +// otherwise only consumed via getMergedEnv(), which the many modules that +// read raw process.env["X"] never call — so .env-only values were silently +// ignored by them. Copy the file's vars into process.env, but only when the +// key is currently unset so a real process.env value still wins (this +// preserves the {...fileEnv, ...process.env} precedence getMergedEnv uses). +export function hydrateProcessEnvFromFile(): void { + for (const [k, v] of Object.entries(loadEnvFile())) { + if (process.env[k] === undefined) process.env[k] = v; + } +} + function detectProvider(env: Record): ProviderConfig { const maxTokens = parseInt(env["MAX_TOKENS"] || "4096", 10); @@ -312,15 +345,30 @@ export function isAgentScopeIsolated(): boolean { return loadAgentScope()?.mode === "isolated"; } +// Floor for the git-snapshot timer. A zero/negative SNAPSHOT_INTERVAL would +// make setInterval fire on roughly every event-loop tick, saturating the +// worker with back-to-back full-state snapshots + git commits. Anything below +// this floor is treated as a misconfiguration and falls back to the default. +const SNAPSHOT_INTERVAL_DEFAULT_SECONDS = 3600; +const MIN_SNAPSHOT_INTERVAL_SECONDS = 1; + export function loadSnapshotConfig(): { enabled: boolean; interval: number; dir: string; } { const env = getMergedEnv(); + const rawInterval = safeParseInt( + env["SNAPSHOT_INTERVAL"], + SNAPSHOT_INTERVAL_DEFAULT_SECONDS, + ); + const interval = + rawInterval >= MIN_SNAPSHOT_INTERVAL_SECONDS + ? rawInterval + : SNAPSHOT_INTERVAL_DEFAULT_SECONDS; return { enabled: env["SNAPSHOT_ENABLED"] === "true", - interval: safeParseInt(env["SNAPSHOT_INTERVAL"], 3600), + interval, dir: env["SNAPSHOT_DIR"] || join(homedir(), ".agentmemory", "snapshots"), }; } @@ -400,6 +448,21 @@ export function getConsolidationDecayDays(): number { return safeParseInt(getMergedEnv()["CONSOLIDATION_DECAY_DAYS"], 30); } +// Cooldown between corpus consolidations triggered by session stop. The Stop +// hook fires per agent turn and posts /session/end, so without this every turn +// would kick a full LLM semantic-merge + reflect + crystallize. Debounced to at +// most once per window. Set to 0 to disable the debounce (consolidate on every +// stop). Default 5 minutes. +const CONSOLIDATION_COOLDOWN_DEFAULT_MS = 300000; + +export function getConsolidationCooldownMs(): number { + const raw = safeParseInt( + getMergedEnv()["AGENTMEMORY_CONSOLIDATION_COOLDOWN_MS"], + CONSOLIDATION_COOLDOWN_DEFAULT_MS, + ); + return raw >= 0 ? raw : CONSOLIDATION_COOLDOWN_DEFAULT_MS; +} + export function isStandaloneMcp(): boolean { return getMergedEnv()["STANDALONE_MCP"] === "true"; } diff --git a/src/functions/evict.ts b/src/functions/evict.ts index 626f4ff1df..722606951b 100644 --- a/src/functions/evict.ts +++ b/src/functions/evict.ts @@ -8,6 +8,7 @@ import type { } from "../types.js"; import { KV } from "../state/schema.js"; import { StateKV } from "../state/kv.js"; +import { isConsolidationEnabled } from "../config.js"; import { recordAudit } from "./audit.js"; import { deleteAccessLog } from "./access-tracker.js"; import { logger } from "../logger.js"; @@ -60,7 +61,9 @@ async function recoverStaleSession( try { const result = await sdk.trigger({ function_id: "event::session::stopped", - payload: { sessionId }, + // Suppress the per-session consolidation fan-out: eviction runs a + // single corpus-wide consolidation pass after all recoveries instead. + payload: { sessionId, skipConsolidation: true }, }); if (!isValidRecoveryResult(result)) { logger.warn("Stale session recovery failed", { @@ -80,10 +83,20 @@ async function recoverStaleSession( } async function runRecoveredSessionConsolidation(sdk: ISdk): Promise { + // Same gate as the session-stop path: keyless installs must not fire + // no-op LLM consolidation from an eviction sweep either. + if (!isConsolidationEnabled()) return; try { await sdk.trigger({ function_id: "mem::consolidate-pipeline", - payload: { tier: "all" }, + payload: { tier: "all", force: true }, + }); + // One crystallization pass for the batch (the per-session fan-out was + // suppressed with skipConsolidation), keeping recovered sessions + // consistent with normally-stopped ones without the N-fold amplification. + await sdk.trigger({ + function_id: "mem::auto-crystallize", + payload: { olderThanDays: 0 }, }); } catch (err) { logger.warn("Recovered session consolidation failed", { diff --git a/src/functions/export-import.ts b/src/functions/export-import.ts index 8b3bc7cc07..2e30e070fe 100644 --- a/src/functions/export-import.ts +++ b/src/functions/export-import.ts @@ -29,8 +29,32 @@ import { KV } from "../state/schema.js"; import { StateKV } from "../state/kv.js"; import { VERSION } from "../version.js"; import { recordAudit } from "./audit.js"; +import { indexRecords } from "./search.js"; import { logger } from "../logger.js"; +// Bounded-concurrency chunk size for the import delete/write loops. A +// "replace" or "merge" of a large export (up to MAX_TOTAL_OBSERVATIONS, +// ~500k) would otherwise issue hundreds of thousands of sequential state +// round-trips and blow the 180s function timeout, leaving partial state. +// 20 keeps per-chunk fan-out low enough not to overwhelm the state +// backend while collapsing wallclock by ~20x versus the serial path. +const IMPORT_CHUNK_SIZE = 20; + +// Run `fn` over `items` in fixed-size chunks, awaiting each chunk before +// starting the next. Preserves ordering guarantees across chunks (chunk N +// fully settles before chunk N+1 begins) while parallelizing within a +// chunk. Errors propagate — a failing item rejects the whole import, same +// as the original serial loops. +async function runChunked( + items: readonly T[], + fn: (item: T) => Promise, +): Promise { + for (let i = 0; i < items.length; i += IMPORT_CHUNK_SIZE) { + const chunk = items.slice(i, i + IMPORT_CHUNK_SIZE); + await Promise.all(chunk.map(fn)); + } +} + export function registerExportImportFunction(sdk: ISdk, kv: StateKV): void { sdk.registerFunction("mem::export", async (data?: { maxSessions?: number; offset?: number }) => { @@ -265,114 +289,145 @@ export function registerExportImportFunction(sdk: ISdk, kv: StateKV): void { if (strategy === "replace") { const existing = await kv.list(KV.sessions); - for (const session of existing) { + // Collect observation deletes across all sessions, then run them in + // one bounded pass: a runChunked nested inside a runChunked callback + // multiplies in-flight deletes to chunk-size squared. + const obsDeletes: Array<{ sessionId: string; obsId: string }> = []; + await runChunked(existing, async (session) => { await kv.delete(KV.sessions, session.id); const obs = await kv .list(KV.observations(session.id)) .catch(() => []); for (const o of obs) { - await kv.delete(KV.observations(session.id), o.id); + obsDeletes.push({ sessionId: session.id, obsId: o.id }); } - } - const existingMem = await kv.list(KV.memories); - for (const m of existingMem) { - await kv.delete(KV.memories, m.id); - } - const existingSummaries = await kv.list(KV.summaries); - for (const s of existingSummaries) { - await kv.delete(KV.summaries, s.sessionId); - } - for (const a of await kv.list(KV.actions).catch(() => [])) { - await kv.delete(KV.actions, a.id); - } - for (const e of await kv.list(KV.actionEdges).catch(() => [])) { - await kv.delete(KV.actionEdges, e.id); - } - for (const r of await kv.list(KV.routines).catch(() => [])) { - await kv.delete(KV.routines, r.id); - } - for (const s of await kv.list(KV.signals).catch(() => [])) { - await kv.delete(KV.signals, s.id); - } - for (const c of await kv.list(KV.checkpoints).catch(() => [])) { - await kv.delete(KV.checkpoints, c.id); - } - for (const s of await kv.list(KV.sentinels).catch(() => [])) { - await kv.delete(KV.sentinels, s.id); - } - for (const s of await kv.list(KV.sketches).catch(() => [])) { - await kv.delete(KV.sketches, s.id); - } - for (const c of await kv.list(KV.crystals).catch(() => [])) { - await kv.delete(KV.crystals, c.id); - } - for (const f of await kv.list(KV.facets).catch(() => [])) { - await kv.delete(KV.facets, f.id); - } - for (const l of await kv.list(KV.lessons).catch(() => [])) { - await kv.delete(KV.lessons, l.id); - } - for (const i of await kv.list(KV.insights).catch(() => [])) { - await kv.delete(KV.insights, i.id); - } - for (const n of await kv.list<{ id: string }>(KV.graphNodes).catch(() => [])) { - await kv.delete(KV.graphNodes, n.id); - } - for (const e of await kv.list<{ id: string }>(KV.graphEdges).catch(() => [])) { - await kv.delete(KV.graphEdges, e.id); - } - for (const s of await kv.list<{ id: string }>(KV.semantic).catch(() => [])) { - await kv.delete(KV.semantic, s.id); - } - for (const p of await kv.list<{ id: string }>(KV.procedural).catch(() => [])) { - await kv.delete(KV.procedural, p.id); - } - for (const profile of await kv.list(KV.profiles).catch(() => [])) { - await kv.delete(KV.profiles, profile.project); - } - for (const a of await kv.list(KV.accessLog).catch(() => [])) { - await kv.delete(KV.accessLog, a.memoryId); - } + }); + await runChunked(obsDeletes, (d) => + kv.delete(KV.observations(d.sessionId), d.obsId), + ); + await runChunked(await kv.list(KV.memories), (m) => + kv.delete(KV.memories, m.id), + ); + await runChunked( + await kv.list(KV.summaries), + (s) => kv.delete(KV.summaries, s.sessionId), + ); + await runChunked(await kv.list(KV.actions).catch(() => []), (a) => + kv.delete(KV.actions, a.id), + ); + await runChunked( + await kv.list(KV.actionEdges).catch(() => []), + (e) => kv.delete(KV.actionEdges, e.id), + ); + await runChunked( + await kv.list(KV.routines).catch(() => []), + (r) => kv.delete(KV.routines, r.id), + ); + await runChunked( + await kv.list(KV.signals).catch(() => []), + (s) => kv.delete(KV.signals, s.id), + ); + await runChunked( + await kv.list(KV.checkpoints).catch(() => []), + (c) => kv.delete(KV.checkpoints, c.id), + ); + await runChunked( + await kv.list(KV.sentinels).catch(() => []), + (s) => kv.delete(KV.sentinels, s.id), + ); + await runChunked( + await kv.list(KV.sketches).catch(() => []), + (s) => kv.delete(KV.sketches, s.id), + ); + await runChunked( + await kv.list(KV.crystals).catch(() => []), + (c) => kv.delete(KV.crystals, c.id), + ); + await runChunked( + await kv.list(KV.facets).catch(() => []), + (f) => kv.delete(KV.facets, f.id), + ); + await runChunked( + await kv.list(KV.lessons).catch(() => []), + (l) => kv.delete(KV.lessons, l.id), + ); + await runChunked( + await kv.list(KV.insights).catch(() => []), + (i) => kv.delete(KV.insights, i.id), + ); + await runChunked( + await kv.list<{ id: string }>(KV.graphNodes).catch(() => []), + (n) => kv.delete(KV.graphNodes, n.id), + ); + await runChunked( + await kv.list<{ id: string }>(KV.graphEdges).catch(() => []), + (e) => kv.delete(KV.graphEdges, e.id), + ); + await runChunked( + await kv.list<{ id: string }>(KV.semantic).catch(() => []), + (s) => kv.delete(KV.semantic, s.id), + ); + await runChunked( + await kv.list<{ id: string }>(KV.procedural).catch(() => []), + (p) => kv.delete(KV.procedural, p.id), + ); + await runChunked( + await kv.list(KV.profiles).catch(() => []), + (profile) => kv.delete(KV.profiles, profile.project), + ); + await runChunked( + await kv.list(KV.accessLog).catch(() => []), + (a) => kv.delete(KV.accessLog, a.memoryId), + ); } - for (const session of importData.sessions) { + // Records actually written this run, accumulated for search + // indexing after the KV writes settle. Skipped (already-present) + // and merge-overwritten rows are already in the index or will be + // re-added below, so re-indexing them is harmless; we only skip the + // ones the "skip" strategy declined to write. + const indexObs: CompressedObservation[] = []; + const indexMems: Memory[] = []; + + await runChunked(importData.sessions, async (session) => { if (strategy === "skip") { const existing = await kv .get(KV.sessions, session.id) .catch(() => null); if (existing) { stats.skipped++; - continue; + return; } } await kv.set(KV.sessions, session.id, session); stats.sessions++; - } + }); for (const [sessionId, obs] of Object.entries(importData.observations)) { - for (const o of obs) { + await runChunked(obs, async (o) => { if (strategy === "skip") { const existing = await kv .get(KV.observations(sessionId), o.id) .catch(() => null); if (existing) { stats.skipped++; - continue; + return; } } await kv.set(KV.observations(sessionId), o.id, o); stats.observations++; - } + indexObs.push(o); + }); } - for (const memory of importData.memories) { + await runChunked(importData.memories, async (memory) => { if (strategy === "skip") { const existing = await kv .get(KV.memories, memory.id) .catch(() => null); if (existing) { stats.skipped++; - continue; + return; } } // Older exports + hand-edited dumps can omit this field. @@ -381,171 +436,172 @@ export function registerExportImportFunction(sdk: ISdk, kv: StateKV): void { } await kv.set(KV.memories, memory.id, memory); stats.memories++; - } + indexMems.push(memory); + }); - for (const summary of importData.summaries) { + await runChunked(importData.summaries, async (summary) => { if (strategy === "skip") { const existing = await kv .get(KV.summaries, summary.sessionId) .catch(() => null); if (existing) { stats.skipped++; - continue; + return; } } await kv.set(KV.summaries, summary.sessionId, summary); stats.summaries++; - } + }); if (importData.graphNodes) { - for (const node of importData.graphNodes) { + await runChunked(importData.graphNodes, async (node) => { if (strategy === "skip") { const existing = await kv.get(KV.graphNodes, node.id).catch(() => null); - if (existing) { stats.skipped++; continue; } + if (existing) { stats.skipped++; return; } } await kv.set(KV.graphNodes, node.id, node); - } + }); } if (importData.graphEdges) { - for (const edge of importData.graphEdges) { + await runChunked(importData.graphEdges, async (edge) => { if (strategy === "skip") { const existing = await kv.get(KV.graphEdges, edge.id).catch(() => null); - if (existing) { stats.skipped++; continue; } + if (existing) { stats.skipped++; return; } } await kv.set(KV.graphEdges, edge.id, edge); - } + }); } if (importData.semanticMemories) { - for (const sem of importData.semanticMemories) { + await runChunked(importData.semanticMemories, async (sem) => { if (strategy === "skip") { const existing = await kv.get(KV.semantic, sem.id).catch(() => null); - if (existing) { stats.skipped++; continue; } + if (existing) { stats.skipped++; return; } } await kv.set(KV.semantic, sem.id, sem); - } + }); } if (importData.proceduralMemories) { - for (const proc of importData.proceduralMemories) { + await runChunked(importData.proceduralMemories, async (proc) => { if (strategy === "skip") { const existing = await kv.get(KV.procedural, proc.id).catch(() => null); - if (existing) { stats.skipped++; continue; } + if (existing) { stats.skipped++; return; } } await kv.set(KV.procedural, proc.id, proc); - } + }); } if (importData.profiles) { - for (const profile of importData.profiles) { + await runChunked(importData.profiles, async (profile) => { if (strategy === "skip") { const existing = await kv .get(KV.profiles, profile.project) .catch(() => null); if (existing) { stats.skipped++; - continue; + return; } } await kv.set(KV.profiles, profile.project, profile); - } + }); } if (importData.actions) { - for (const action of importData.actions) { + await runChunked(importData.actions, async (action) => { if (strategy === "skip") { const existing = await kv.get(KV.actions, action.id).catch(() => null); - if (existing) { stats.skipped++; continue; } + if (existing) { stats.skipped++; return; } } await kv.set(KV.actions, action.id, action); - } + }); } if (importData.actionEdges) { - for (const edge of importData.actionEdges) { + await runChunked(importData.actionEdges, async (edge) => { if (strategy === "skip") { const existing = await kv.get(KV.actionEdges, edge.id).catch(() => null); - if (existing) { stats.skipped++; continue; } + if (existing) { stats.skipped++; return; } } await kv.set(KV.actionEdges, edge.id, edge); - } + }); } if (importData.routines) { - for (const routine of importData.routines) { + await runChunked(importData.routines, async (routine) => { if (strategy === "skip") { const existing = await kv.get(KV.routines, routine.id).catch(() => null); - if (existing) { stats.skipped++; continue; } + if (existing) { stats.skipped++; return; } } await kv.set(KV.routines, routine.id, routine); - } + }); } if (importData.signals) { - for (const signal of importData.signals) { + await runChunked(importData.signals, async (signal) => { if (strategy === "skip") { const existing = await kv.get(KV.signals, signal.id).catch(() => null); - if (existing) { stats.skipped++; continue; } + if (existing) { stats.skipped++; return; } } await kv.set(KV.signals, signal.id, signal); - } + }); } if (importData.checkpoints) { - for (const checkpoint of importData.checkpoints) { + await runChunked(importData.checkpoints, async (checkpoint) => { if (strategy === "skip") { const existing = await kv.get(KV.checkpoints, checkpoint.id).catch(() => null); - if (existing) { stats.skipped++; continue; } + if (existing) { stats.skipped++; return; } } await kv.set(KV.checkpoints, checkpoint.id, checkpoint); - } + }); } if (importData.sentinels) { - for (const sentinel of importData.sentinels) { + await runChunked(importData.sentinels, async (sentinel) => { if (strategy === "skip") { const existing = await kv.get(KV.sentinels, sentinel.id).catch(() => null); - if (existing) { stats.skipped++; continue; } + if (existing) { stats.skipped++; return; } } await kv.set(KV.sentinels, sentinel.id, sentinel); - } + }); } if (importData.sketches) { - for (const sketch of importData.sketches) { + await runChunked(importData.sketches, async (sketch) => { if (strategy === "skip") { const existing = await kv.get(KV.sketches, sketch.id).catch(() => null); - if (existing) { stats.skipped++; continue; } + if (existing) { stats.skipped++; return; } } await kv.set(KV.sketches, sketch.id, sketch); - } + }); } if (importData.crystals) { - for (const crystal of importData.crystals) { + await runChunked(importData.crystals, async (crystal) => { if (strategy === "skip") { const existing = await kv.get(KV.crystals, crystal.id).catch(() => null); - if (existing) { stats.skipped++; continue; } + if (existing) { stats.skipped++; return; } } await kv.set(KV.crystals, crystal.id, crystal); - } + }); } if (importData.facets) { - for (const facet of importData.facets) { + await runChunked(importData.facets, async (facet) => { if (strategy === "skip") { const existing = await kv.get(KV.facets, facet.id).catch(() => null); - if (existing) { stats.skipped++; continue; } + if (existing) { stats.skipped++; return; } } await kv.set(KV.facets, facet.id, facet); - } + }); } if (importData.lessons) { - for (const lesson of importData.lessons) { + await runChunked(importData.lessons, async (lesson) => { if (strategy === "skip") { const existing = await kv.get(KV.lessons, lesson.id).catch(() => null); - if (existing) { stats.skipped++; continue; } + if (existing) { stats.skipped++; return; } } await kv.set(KV.lessons, lesson.id, lesson); - } + }); } if (importData.insights) { - for (const insight of importData.insights) { + await runChunked(importData.insights, async (insight) => { if (strategy === "skip") { const existing = await kv.get(KV.insights, insight.id).catch(() => null); - if (existing) { stats.skipped++; continue; } + if (existing) { stats.skipped++; return; } } await kv.set(KV.insights, insight.id, insight); - } + }); } if (importData.accessLogs) { if (!Array.isArray(importData.accessLogs)) { @@ -560,20 +616,37 @@ export function registerExportImportFunction(sdk: ISdk, kv: StateKV): void { const memoryIds = new Set( importData.memories.map((m) => m.id), ); - for (const raw of importData.accessLogs) { + await runChunked(importData.accessLogs, async (raw) => { const log = normalizeAccessLog(raw); - if (!log.memoryId || !memoryIds.has(log.memoryId)) continue; + if (!log.memoryId || !memoryIds.has(log.memoryId)) return; if (strategy === "skip") { const existing = await kv .get(KV.accessLog, log.memoryId) .catch(() => null); if (existing) { stats.skipped++; - continue; + return; } } await kv.set(KV.accessLog, log.memoryId, log); - } + }); + } + + // Imported rows are now in KV but invisible to search: the boot + // rebuild gate only fires when BM25 is empty, so on any existing + // install (non-empty persisted index) imported observations and + // memories never surface via mem::search / smart-search until a + // manual rebuild. Add them to BM25 (synchronous) and enqueue the + // vector embeddings in batches (one embedBatch call per chunk) + // rather than one giant Promise.all over 500k docs. Indexing + // failures are logged, not fatal — the KV writes already committed + // and the restart rebuild is the backstop. + try { + await indexRecords(indexObs, indexMems); + } catch (err) { + logger.warn("Import indexing failed; restart rebuild will recover", { + error: err instanceof Error ? err.message : String(err), + }); } logger.info("Import complete", { strategy, ...stats }); diff --git a/src/functions/graph-import.ts b/src/functions/graph-import.ts new file mode 100644 index 0000000000..520decc671 --- /dev/null +++ b/src/functions/graph-import.ts @@ -0,0 +1,265 @@ +import { readFile, stat } from "node:fs/promises"; +import { join } from "node:path"; +import type { ISdk } from "iii-sdk"; +import type { GraphEdge, GraphEdgeType, GraphNode, GraphNodeType } from "../types.js"; +import { KV, generateId } from "../state/schema.js"; +import type { StateKV } from "../state/kv.js"; +import { persistGraphDelta } from "./graph.js"; +import { recordAudit } from "./audit.js"; +import { logger } from "../logger.js"; + +// Import graphify's structural knowledge graph (graphify-out/graph.json) into +// the memory graph, so context injection and graph retrieval see codebase +// structure the developer never touched in a session. graphify's extraction is +// deterministic AST analysis; ours is session-derived. The two meet here. +// +// Idempotency comes from persistGraphDelta's (type, name) name-index upsert: +// re-importing after `graphify update .` merges into existing nodes instead of +// duplicating. Provenance is kept on properties.source so imported structure +// is distinguishable from session-derived entities. + +// Bounds protect the KV store and the iii invocation budget from very large +// graphs (graphify caps graph.json at 512MiB; a 100k-node import would blow +// the invocation window). Truncation is reported loudly in the result. +const MAX_FILE_BYTES = 32 * 1024 * 1024; +const MAX_NODES = 5000; +const MAX_EDGES = 20000; + +// graphify file_type enum: code|document|paper|image|rationale|concept. +// Mapped to the closest memory-graph node type; code symbols carrying a file +// extension in the label are files, the rest are treated as functions. +function mapNodeType(fileType: unknown, label: string): GraphNodeType { + switch (fileType) { + case "rationale": + return "decision"; + case "document": + case "paper": + case "image": + case "concept": + return "concept"; + case "code": + return /\.[a-z0-9]{1,10}$/i.test(label) ? "file" : "function"; + default: + return /\.[a-z0-9]{1,10}$/i.test(label) ? "file" : "concept"; + } +} + +// graphify relations observed in its extractors: calls, imports, uses, +// requires, inherits, references, source. Anything unrecognized stays a +// generic related_to edge rather than being dropped. +function mapEdgeType(relation: unknown): GraphEdgeType { + switch (relation) { + case "imports": + case "source": + return "imports"; + case "calls": + case "uses": + return "uses"; + case "requires": + case "inherits": + case "depends_on": + return "depends_on"; + default: + return "related_to"; + } +} + +// graphify edge confidence tags: EXTRACTED (deterministic AST) beats +// INFERRED (heuristic) beats AMBIGUOUS. +function mapConfidence(confidence: unknown): number { + switch (confidence) { + case "EXTRACTED": + return 0.9; + case "INFERRED": + return 0.6; + case "AMBIGUOUS": + return 0.3; + default: + return 0.5; + } +} + +type RawGraph = { + nodes?: unknown; + links?: unknown; + edges?: unknown; +}; + +export type GraphifyImportResult = { + success: boolean; + error?: string; + path?: string; + nodesRead?: number; + edgesRead?: number; + nodesImported?: number; + edgesImported?: number; + newNodes?: number; + newEdges?: number; + skippedEdges?: number; + truncated?: { nodes: number; edges: number } | null; +}; + +export function parseGraphifyGraph(raw: string): { + nodes: GraphNode[]; + edges: GraphEdge[]; + nodesRead: number; + edgesRead: number; + skippedEdges: number; + truncated: { nodes: number; edges: number } | null; +} { + const parsed = JSON.parse(raw) as RawGraph; + const rawNodes = Array.isArray(parsed.nodes) ? parsed.nodes : []; + // graphify's clustered output stores edges under "links" (NetworkX + // node_link); --no-cluster graphs store them under "edges". Accept both. + const rawEdges = Array.isArray(parsed.links) + ? parsed.links + : Array.isArray(parsed.edges) + ? parsed.edges + : []; + + const now = new Date().toISOString(); + const truncatedNodes = Math.max(0, rawNodes.length - MAX_NODES); + const truncatedEdges = Math.max(0, rawEdges.length - MAX_EDGES); + + const nodes: GraphNode[] = []; + const byGraphifyId = new Map(); + for (const entry of rawNodes.slice(0, MAX_NODES)) { + if (!entry || typeof entry !== "object") continue; + const n = entry as Record; + const graphifyId = typeof n.id === "string" || typeof n.id === "number" ? String(n.id) : null; + const label = + typeof n.label === "string" && n.label.trim() + ? n.label.trim() + : graphifyId; + if (!graphifyId || !label) continue; + + const properties: Record = { source: "graphify" }; + if (typeof n.source_file === "string") properties.sourceFile = n.source_file; + if (n.community !== undefined) properties.community = n.community; + if (typeof n.file_type === "string") properties.fileType = n.file_type; + + const node: GraphNode = { + id: generateId("gn"), + type: mapNodeType(n.file_type, label), + name: label, + properties, + sourceObservationIds: [], + createdAt: now, + }; + nodes.push(node); + byGraphifyId.set(graphifyId, node); + } + + const edges: GraphEdge[] = []; + let skippedEdges = 0; + for (const entry of rawEdges.slice(0, MAX_EDGES)) { + if (!entry || typeof entry !== "object") { + skippedEdges++; + continue; + } + const e = entry as Record; + const source = e.source !== undefined ? String(e.source) : null; + const target = e.target !== undefined ? String(e.target) : null; + const sourceNode = source ? byGraphifyId.get(source) : undefined; + const targetNode = target ? byGraphifyId.get(target) : undefined; + if (!sourceNode || !targetNode) { + // Endpoint outside the imported node set (dropped by the cap, or a + // dangling reference in the file). Skipped, counted, never guessed. + skippedEdges++; + continue; + } + edges.push({ + id: generateId("ge"), + type: mapEdgeType(e.relation ?? e.type), + sourceNodeId: sourceNode.id, + targetNodeId: targetNode.id, + weight: mapConfidence(e.confidence), + sourceObservationIds: [], + createdAt: now, + }); + } + + return { + nodes, + edges, + nodesRead: rawNodes.length, + edgesRead: rawEdges.length, + skippedEdges, + truncated: + truncatedNodes > 0 || truncatedEdges > 0 + ? { nodes: truncatedNodes, edges: truncatedEdges } + : null, + }; +} + +export function registerGraphImportFunction(sdk: ISdk, kv: StateKV): void { + sdk.registerFunction( + "mem::graph::import-graphify", + async (data?: { path?: string; cwd?: string }): Promise => { + const explicitPath = typeof data?.path === "string" ? data.path : undefined; + const cwd = typeof data?.cwd === "string" ? data.cwd : process.cwd(); + const path = explicitPath ?? join(cwd, "graphify-out", "graph.json"); + + try { + let size: number; + try { + size = (await stat(path)).size; + } catch { + return { + success: false, + error: `graph.json not found at ${path}. Run graphify first, or pass an explicit path.`, + path, + }; + } + if (size > MAX_FILE_BYTES) { + return { + success: false, + error: `graph.json is ${Math.round(size / 1024 / 1024)}MB, over the ${MAX_FILE_BYTES / 1024 / 1024}MB import cap.`, + path, + }; + } + + const parsed = parseGraphifyGraph(await readFile(path, "utf-8")); + const { newNodeCount, newEdgeCount } = await persistGraphDelta( + kv, + parsed.nodes, + parsed.edges, + [], + ); + + await recordAudit(kv, "import", "mem::graph::import-graphify", [], { + path, + nodesImported: parsed.nodes.length, + edgesImported: parsed.edges.length, + newNodes: newNodeCount, + newEdges: newEdgeCount, + }); + + logger.info("graphify graph imported", { + path, + nodes: parsed.nodes.length, + edges: parsed.edges.length, + newNodes: newNodeCount, + newEdges: newEdgeCount, + }); + + return { + success: true, + path, + nodesRead: parsed.nodesRead, + edgesRead: parsed.edgesRead, + nodesImported: parsed.nodes.length, + edgesImported: parsed.edges.length, + newNodes: newNodeCount, + newEdges: newEdgeCount, + skippedEdges: parsed.skippedEdges, + truncated: parsed.truncated, + }; + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + logger.error("graphify import failed", { path, error: msg }); + return { success: false, error: msg, path }; + } + }, + ); +} diff --git a/src/functions/graph.ts b/src/functions/graph.ts index bb0d727224..86332a3039 100644 --- a/src/functions/graph.ts +++ b/src/functions/graph.ts @@ -450,6 +450,148 @@ function parseGraphXml( return { nodes, edges }; } +// Shared persistence for a batch of extracted/imported nodes and edges. +// Factored out of mem::graph-extract so structural importers (graphify) +// reuse the exact same name-index upsert, degree bookkeeping, and snapshot +// maintenance — which also makes re-imports idempotent: an existing +// (type, name) resolves through the name index and merges instead of +// duplicating. +// +// #814 v2: targeted name-index lookups replace the O(n) scan over +// `kv.list(KV.graphNodes)`. At 75K nodes the list payload +// exceeds the iii heartbeat budget and the worker dies before merge can +// complete. Each name-index entry is a single small kv.get/set pair. +export async function persistGraphDelta( + kv: StateKV, + nodes: GraphNode[], + edges: GraphEdge[], + obsIds: string[], +): Promise<{ newNodeCount: number; newEdgeCount: number }> { + const snap = (await readSnapshot(kv)) ?? emptySnapshot(); + const capturedAt = new Date().toISOString(); + let newNodeCount = 0; + let newEdgeCount = 0; + // Merge-only batches mutate cached topNodes/topEdges entries without + // changing the counts; track that separately so the snapshot still persists. + let snapMutated = false; + const newEdgesForTopCheck: GraphEdge[] = []; + // When a freshly-minted node merges into an existing row via the name + // index, edges in the same batch still reference the fresh id. Remap edge + // endpoints to the persisted ids so edges never dangle and re-runs hit the + // same edge-index key instead of duplicating. + const idRemap = new Map(); + + for (const node of nodes) { + const indexKey = nameIndexKey(node.type, node.name); + const existingId = await kv.get(KV.graphNameIndex, indexKey); + + let existing: GraphNode | null = null; + if (existingId) { + existing = await kv.get(KV.graphNodes, existingId); + // #825 follow-up: name-index lookups can resolve into + // pre-reset rows. Drop them so extract writes a fresh + // node + index entry instead of silently reconnecting + // to a legacy orphan (which would keep the snapshot at + // 0 forever after a reset). + if ( + existing && + snap.resetAt && + typeof existing.createdAt === "string" && + existing.createdAt < snap.resetAt + ) { + existing = null; + } + } + + if (existing) { + idRemap.set(node.id, existing.id); + const merged = mergeNode(existing, node, obsIds, capturedAt); + await kv.set(KV.graphNodes, existing.id, merged); + // Update topNodes entry if present so a stale clone isn't + // returned from the snapshot fast path. + const topIdx = snap.topNodes.findIndex((n) => n.id === existing!.id); + if (topIdx !== -1) { + snap.topNodes[topIdx] = merged; + snapMutated = true; + } + } else { + await kv.set(KV.graphNodes, node.id, node); + await kv.set(KV.graphNameIndex, indexKey, node.id); + await kv.set(KV.graphNodeDegree, node.id, 0); + snap.stats.totalNodes += 1; + snap.stats.nodesByType[node.type] = + (snap.stats.nodesByType[node.type] ?? 0) + 1; + newNodeCount += 1; + if (snap.topNodes.length < SNAPSHOT_TOP_NODES) { + // Degree 0 still beats an empty slot — sit at the tail + // until edges arrive and promote. + snap.topNodes.push(node); + snap.topDegrees[node.id] = 0; + } + } + } + + for (const rawEdge of edges) { + const edge: GraphEdge = { + ...rawEdge, + sourceNodeId: idRemap.get(rawEdge.sourceNodeId) ?? rawEdge.sourceNodeId, + targetNodeId: idRemap.get(rawEdge.targetNodeId) ?? rawEdge.targetNodeId, + }; + const eKey = edgeIndexKey(edge.sourceNodeId, edge.targetNodeId, edge.type); + const existingId = await kv.get(KV.graphEdgeKey, eKey); + + let existing: GraphEdge | null = null; + if (existingId) { + existing = await kv.get(KV.graphEdges, existingId); + // Same #825 orphan check as the node path above. + if ( + existing && + snap.resetAt && + typeof existing.createdAt === "string" && + existing.createdAt < snap.resetAt + ) { + existing = null; + } + } + + if (existing) { + const merged = mergeEdge(existing, obsIds); + await kv.set(KV.graphEdges, existing.id, merged); + // Replace cached topEdges entry too if present. + const topIdx = snap.topEdges.findIndex((e) => e.id === existing!.id); + if (topIdx !== -1) { + snap.topEdges[topIdx] = merged; + snapMutated = true; + } + } else { + await kv.set(KV.graphEdges, edge.id, edge); + await kv.set(KV.graphEdgeKey, eKey, edge.id); + snap.stats.totalEdges += 1; + snap.stats.edgesByType[edge.type] = + (snap.stats.edgesByType[edge.type] ?? 0) + 1; + newEdgeCount += 1; + await applyDegreeDelta(kv, snap, edge.sourceNodeId, +1); + await applyDegreeDelta(kv, snap, edge.targetNodeId, +1); + newEdgesForTopCheck.push(edge); + } + } + + // Push newly-added edges into snapshot.topEdges if both + // endpoints are in the top-N (post-degree-delta). Done after + // all degree updates so the topIds set is stable. + for (const edge of newEdgesForTopCheck) { + snapshotPushEdgeIfBothInTop(snap, edge); + } + + if (newNodeCount > 0 || newEdgeCount > 0 || snapMutated) { + snap.updatedAt = capturedAt; + snap.dirty = false; + await kv.set(KV.graphSnapshot, SNAPSHOT_KEY, snap); + } + + return { newNodeCount, newEdgeCount }; +} + export function registerGraphFunction( sdk: ISdk, kv: StateKV, @@ -480,123 +622,12 @@ export function registerGraphFunction( const obsIds = data.observations.map((o) => o.id); const { nodes, edges } = parseGraphXml(response, obsIds); - // #814 v2: targeted name-index lookups replace the O(n) scan - // over `kv.list(KV.graphNodes)`. At 75K nodes the - // list payload exceeds the iii heartbeat budget and the worker - // dies before merge can complete. Each name-index entry is a - // single small kv.get/set pair. - const snap = (await readSnapshot(kv)) ?? emptySnapshot(); - const capturedAt = new Date().toISOString(); - let newNodeCount = 0; - let newEdgeCount = 0; - const newEdgesForTopCheck: GraphEdge[] = []; - - for (const node of nodes) { - const indexKey = nameIndexKey(node.type, node.name); - const existingId = await kv.get( - KV.graphNameIndex, - indexKey, - ); - - let existing: GraphNode | null = null; - if (existingId) { - existing = await kv.get(KV.graphNodes, existingId); - // #825 follow-up: name-index lookups can resolve into - // pre-reset rows. Drop them so extract writes a fresh - // node + index entry instead of silently reconnecting - // to a legacy orphan (which would keep the snapshot at - // 0 forever after a reset). - if ( - existing && - snap.resetAt && - typeof existing.createdAt === "string" && - existing.createdAt < snap.resetAt - ) { - existing = null; - } - } - - if (existing) { - const merged = mergeNode(existing, node, obsIds, capturedAt); - await kv.set(KV.graphNodes, existing.id, merged); - // Update topNodes entry if present so a stale clone isn't - // returned from the snapshot fast path. - const topIdx = snap.topNodes.findIndex( - (n) => n.id === existing!.id, - ); - if (topIdx !== -1) snap.topNodes[topIdx] = merged; - } else { - await kv.set(KV.graphNodes, node.id, node); - await kv.set(KV.graphNameIndex, indexKey, node.id); - await kv.set(KV.graphNodeDegree, node.id, 0); - snap.stats.totalNodes += 1; - snap.stats.nodesByType[node.type] = - (snap.stats.nodesByType[node.type] ?? 0) + 1; - newNodeCount += 1; - if (snap.topNodes.length < SNAPSHOT_TOP_NODES) { - // Degree 0 still beats an empty slot — sit at the tail - // until edges arrive and promote. - snap.topNodes.push(node); - snap.topDegrees[node.id] = 0; - } - } - } - - for (const edge of edges) { - const eKey = edgeIndexKey( - edge.sourceNodeId, - edge.targetNodeId, - edge.type, - ); - const existingId = await kv.get(KV.graphEdgeKey, eKey); - - let existing: GraphEdge | null = null; - if (existingId) { - existing = await kv.get(KV.graphEdges, existingId); - // Same #825 orphan check as the node path above. - if ( - existing && - snap.resetAt && - typeof existing.createdAt === "string" && - existing.createdAt < snap.resetAt - ) { - existing = null; - } - } - - if (existing) { - const merged = mergeEdge(existing, obsIds); - await kv.set(KV.graphEdges, existing.id, merged); - // Replace cached topEdges entry too if present. - const topIdx = snap.topEdges.findIndex( - (e) => e.id === existing!.id, - ); - if (topIdx !== -1) snap.topEdges[topIdx] = merged; - } else { - await kv.set(KV.graphEdges, edge.id, edge); - await kv.set(KV.graphEdgeKey, eKey, edge.id); - snap.stats.totalEdges += 1; - snap.stats.edgesByType[edge.type] = - (snap.stats.edgesByType[edge.type] ?? 0) + 1; - newEdgeCount += 1; - await applyDegreeDelta(kv, snap, edge.sourceNodeId, +1); - await applyDegreeDelta(kv, snap, edge.targetNodeId, +1); - newEdgesForTopCheck.push(edge); - } - } - - // Push newly-added edges into snapshot.topEdges if both - // endpoints are in the top-N (post-degree-delta). Done after - // all degree updates so the topIds set is stable. - for (const edge of newEdgesForTopCheck) { - snapshotPushEdgeIfBothInTop(snap, edge); - } - - if (newNodeCount > 0 || newEdgeCount > 0) { - snap.updatedAt = capturedAt; - snap.dirty = false; - await kv.set(KV.graphSnapshot, SNAPSHOT_KEY, snap); - } + const { newNodeCount, newEdgeCount } = await persistGraphDelta( + kv, + nodes, + edges, + obsIds, + ); await recordAudit(kv, "observe", "mem::graph-extract", obsIds, { nodesExtracted: nodes.length, diff --git a/src/functions/patterns.ts b/src/functions/patterns.ts index 8d1365fc5b..ef6821cbee 100644 --- a/src/functions/patterns.ts +++ b/src/functions/patterns.ts @@ -29,37 +29,54 @@ export function registerPatternsFunction(sdk: ISdk, kv: StateKV): void { { count: number; sessions: Set } >(); - for (const session of filtered) { - const observations = await kv.list( - KV.observations(session.id), + // Bounded fan-out: load observations for up to 10 sessions in + // parallel per batch (like consolidate), then fold each session's + // observations into the shared maps serially so the accumulation + // stays race-free. Parallelizing the kv.list I/O without exceeding + // the invocation pool cuts wall time versus the old serial loop. + for (let batch = 0; batch < filtered.length; batch += 10) { + const chunk = filtered.slice(batch, batch + 10); + const loaded = await Promise.all( + chunk.map(async (session) => ({ + session, + observations: await kv.list( + KV.observations(session.id), + ), + })), ); - if (!observations.length) continue; - - const sessionFiles = new Set(); - for (const obs of observations) { - if (!obs.files) continue; - for (const f of obs.files) { - sessionFiles.add(f); - if (!fileSessionMap.has(f)) fileSessionMap.set(f, new Set()); - fileSessionMap.get(f)!.add(session.id); - } - if (obs.type === "error" && obs.title) { - const key = obs.title.toLowerCase(); - if (!errorPatterns.has(key)) { - errorPatterns.set(key, { count: 0, sessions: new Set() }); + for (const { session, observations } of loaded) { + if (!observations.length) continue; + + const sessionFiles = new Set(); + for (const obs of observations) { + if (!obs.files) continue; + for (const f of obs.files) { + sessionFiles.add(f); + if (!fileSessionMap.has(f)) fileSessionMap.set(f, new Set()); + fileSessionMap.get(f)!.add(session.id); + } + + if (obs.type === "error" && obs.title) { + const key = obs.title.toLowerCase(); + if (!errorPatterns.has(key)) { + errorPatterns.set(key, { count: 0, sessions: new Set() }); + } + const ep = errorPatterns.get(key)!; + ep.count++; + ep.sessions.add(session.id); } - const ep = errorPatterns.get(key)!; - ep.count++; - ep.sessions.add(session.id); } - } - const fileList = [...sessionFiles].sort(); - for (let i = 0; i < fileList.length; i++) { - for (let j = i + 1; j < fileList.length; j++) { - const pair = `${fileList[i]}::${fileList[j]}`; - fileCoOccurrences.set(pair, (fileCoOccurrences.get(pair) || 0) + 1); + const fileList = [...sessionFiles].sort(); + for (let i = 0; i < fileList.length; i++) { + for (let j = i + 1; j < fileList.length; j++) { + const pair = `${fileList[i]}::${fileList[j]}`; + fileCoOccurrences.set( + pair, + (fileCoOccurrences.get(pair) || 0) + 1, + ); + } } } } diff --git a/src/functions/remember.ts b/src/functions/remember.ts index 5735b4f238..5cbfb6931d 100644 --- a/src/functions/remember.ts +++ b/src/functions/remember.ts @@ -10,6 +10,15 @@ import { getSearchIndex, vectorIndexAddGuarded, vectorIndexRemove, flushIndexSav import { getAgentId } from "../config.js"; import { logger } from "../logger.js"; +// Slicing by UTF-16 code unit can cut an astral character (emoji, some CJK +// extensions) mid surrogate pair, leaving a lone high surrogate that renders +// as a replacement glyph. Drop a dangling trailing high surrogate so the +// title stays valid. +function safeSlice(text: string, length: number): string { + const sliced = text.slice(0, length); + return /[\uD800-\uDBFF]$/.test(sliced) ? sliced.slice(0, -1) : sliced; +} + export function registerRememberFunction(sdk: ISdk, kv: StateKV): void { sdk.registerFunction("mem::remember", async (data: { @@ -100,7 +109,7 @@ export function registerRememberFunction(sdk: ISdk, kv: StateKV): void { createdAt: now, updatedAt: now, type: memType, - title: data.content.slice(0, 80), + title: safeSlice(data.content, 80), content: data.content, concepts: data.concepts || [], files: data.files || [], diff --git a/src/functions/replay.ts b/src/functions/replay.ts index e44ca8cc9d..e91850503e 100644 --- a/src/functions/replay.ts +++ b/src/functions/replay.ts @@ -15,7 +15,7 @@ import { parseJsonlText } from "../replay/jsonl-parser.js"; import { projectTimeline, type Timeline } from "../replay/timeline.js"; import { safeAudit } from "./audit.js"; import { buildSyntheticCompression } from "./compress-synthetic.js"; -import { getSearchIndex } from "./search.js"; +import { indexRecords } from "./search.js"; import { logger } from "../logger.js"; export const MAX_FILES_DEFAULT = 200; @@ -432,16 +432,23 @@ export function registerReplayFunctions(sdk: ISdk, kv: StateKV): void { await kv.set(KV.sessions, session.id, session); } - const searchIndex = getSearchIndex(); const compressed: CompressedObservation[] = []; await Promise.all( parsed.observations.map(async (obs) => { const synthetic = buildSyntheticCompression(obs); compressed.push(synthetic); await kv.set(KV.observations(parsed.sessionId), obs.id, synthetic); - searchIndex.add(synthetic); }), ); + // BM25 + vector in one path so jsonl-imported observations are + // reachable by semantic search, not just keyword. + try { + await indexRecords(compressed, []); + } catch (err) { + logger.warn("Import indexing failed; restart rebuild will recover", { + error: err instanceof Error ? err.message : String(err), + }); + } observationCount += parsed.observations.length; sessionIds.push(parsed.sessionId); diff --git a/src/functions/search.ts b/src/functions/search.ts index df699a1a3f..9bcda6ae01 100644 --- a/src/functions/search.ts +++ b/src/functions/search.ts @@ -14,6 +14,15 @@ let index: SearchIndex | null = null let vectorIndex: VectorIndex | null = null let currentEmbeddingProvider: EmbeddingProvider | null = null +// Dedupes the lazy cold-start rebuild kicked off from the mem::search +// request path. A full rebuildIndex walks every observation across every +// session, so N concurrent queries against an empty index would each +// launch their own rebuild and saturate the engine invocation pool. The +// first query with an empty index starts one rebuild and shares its +// promise; concurrent queries await the same rebuild instead of spawning +// duplicates. The boot-time rebuild in index.ts is unaffected. +let rebuildPromise: Promise | null = null + export function getSearchIndex(): SearchIndex { if (!index) index = new SearchIndex() return index @@ -218,19 +227,20 @@ function getRebuildEmbedBatchSize(): number { return Number.isFinite(n) && n > 0 ? n : DEFAULT_REBUILD_EMBED_BATCH } -export async function rebuildIndex(kv: StateKV): Promise { +// Shared BM25 + batched-vector indexing for a set of records. The full +// rebuild and every import path (export-import, jsonl replay) funnel +// through this so they index identically and none can silently skip the +// vector side. It does NOT clear the index — callers that rebuild clear +// first; importers add. When no embedding provider is configured it skips +// the vector enqueue entirely, so a keyless install never allocates embed +// jobs it would immediately discard. +export async function indexRecords( + observations: CompressedObservation[], + memories: Memory[], +): Promise { const idx = getSearchIndex() - idx.clear() - - // BM25 clear above wipes stale doc entries; the vector index has the - // symmetric concern — memories/observations deleted between runs - // would leave orphan embeddings here forever. Clear both before the - // repopulation loops run, so BM25 and vector stay in sync. - vectorIndex?.clear() - + const vectorEnabled = Boolean(vectorIndex && currentEmbeddingProvider) const batchSize = getRebuildEmbedBatchSize() - // Accumulator for the batched embed flush. BM25 add is synchronous and - // doesn't need batching — only the vector path benefits. type EmbedJob = { id: string sessionId: string @@ -238,36 +248,62 @@ export async function rebuildIndex(kv: StateKV): Promise { context: { kind: "memory" | "observation" | "synthetic"; logId: string } } const pending: EmbedJob[] = [] - let count = 0 - const flush = async (): Promise => { if (pending.length === 0) return await vectorIndexAddBatchGuarded(pending) pending.length = 0 } const enqueue = async (job: EmbedJob): Promise => { + if (!vectorEnabled) return pending.push(job) if (pending.length >= batchSize) await flush() } + let count = 0 + for (const memory of memories) { + if (memory.isLatest === false) continue + if (!memory.title || !memory.content) continue + idx.add(memoryToObservation(memory)) + await enqueue({ + id: memory.id, + sessionId: memory.sessionIds?.[0] ?? 'memory', + text: memory.title + ' ' + memory.content, + context: { kind: "memory", logId: memory.id }, + }) + count++ + } + for (const obs of observations) { + if (!obs.title || !obs.narrative) continue + idx.add(obs) + await enqueue({ + id: obs.id, + sessionId: obs.sessionId, + text: obs.title + ' ' + obs.narrative, + context: { kind: "observation", logId: obs.id }, + }) + count++ + } + await flush() + return count +} + +export async function rebuildIndex(kv: StateKV): Promise { + const idx = getSearchIndex() + idx.clear() + + // BM25 clear above wipes stale doc entries; the vector index has the + // symmetric concern — memories/observations deleted between runs + // would leave orphan embeddings here forever. Clear both before the + // repopulation loops run, so BM25 and vector stay in sync. + vectorIndex?.clear() + // Memories live in their own KV scope outside per-session observation // scopes, so they need a separate walk. Without this, mem::remember // entries vanish from BM25 on every restart even after the live-write - // fix in remember.ts (#257). + // fix in remember.ts. + let memories: Memory[] = [] try { - const memories = await kv.list(KV.memories) - for (const memory of memories) { - if (memory.isLatest === false) continue - if (!memory.title || !memory.content) continue - idx.add(memoryToObservation(memory)) - await enqueue({ - id: memory.id, - sessionId: memory.sessionIds?.[0] ?? 'memory', - text: memory.title + ' ' + memory.content, - context: { kind: "memory", logId: memory.id }, - }) - count++ - } + memories = await kv.list(KV.memories) } catch (err) { logger.warn('rebuildIndex: failed to load memories', { error: err instanceof Error ? err.message : String(err), @@ -275,13 +311,10 @@ export async function rebuildIndex(kv: StateKV): Promise { } const sessions = await kv.list(KV.sessions) - if (!sessions.length) { - await flush() - return count - } - - const obsPerSession: CompressedObservation[][] = [] const failedSessions: string[] = [] + // Index each session chunk as it loads instead of accumulating every + // observation first, so peak memory stays bounded to one chunk. + let indexed = 0 for (let batch = 0; batch < sessions.length; batch += 10) { const chunk = sessions.slice(batch, batch + 10) const results = await Promise.all( @@ -294,29 +327,17 @@ export async function rebuildIndex(kv: StateKV): Promise { } }) ) - obsPerSession.push(...results) + const chunkObs = results.flat() + if (chunkObs.length > 0) { + indexed += await indexRecords(chunkObs, []) + } } if (failedSessions.length > 0) { logger.warn('rebuildIndex: failed to load observations for sessions', { failedSessions }) } - for (const observations of obsPerSession) { - for (const obs of observations) { - if (obs.title && obs.narrative) { - idx.add(obs) - await enqueue({ - id: obs.id, - sessionId: obs.sessionId, - text: obs.title + ' ' + obs.narrative, - context: { kind: "observation", logId: obs.id }, - }) - count++ - } - } - } - // Drain the last partial batch. - await flush() - return count + indexed += await indexRecords([], memories) + return indexed } export function registerSearchFunction(sdk: ISdk, kv: StateKV): void { @@ -397,8 +418,25 @@ export function registerSearchFunction(sdk: ISdk, kv: StateKV): void { } if (idx.size === 0) { - const count = await rebuildIndex(kv) - logger.info('Search index rebuilt', { entries: count }) + // Share one rebuild across concurrent cold-start queries so they + // don't each walk the whole corpus and saturate the pool. + if (!rebuildPromise) { + rebuildPromise = rebuildIndex(kv) + .then((count) => { + logger.info('Search index rebuilt', { entries: count }) + return count + }) + .catch((err) => { + logger.warn('Index rebuild failed', { + error: err instanceof Error ? err.message : String(err), + }) + return 0 + }) + .finally(() => { + rebuildPromise = null + }) + } + await rebuildPromise } // When filtering by project/cwd, over-fetch from the index so the diff --git a/src/functions/snapshot.ts b/src/functions/snapshot.ts index 40430ce50f..99c45c061f 100644 --- a/src/functions/snapshot.ts +++ b/src/functions/snapshot.ts @@ -41,8 +41,18 @@ export function registerSnapshotFunction( kv: StateKV, snapshotDir: string, ): void { - sdk.registerFunction("mem::snapshot-create", + // Serialize snapshots: the periodic timer, REST (api::snapshot-create), and + // MCP can all trigger this concurrently. Two runs writing state.json and + // committing in the same git repo at once race on the index lock. An + // overlapping call is a no-op success; the winner captures current state. + let snapshotInFlight = false; + + sdk.registerFunction("mem::snapshot-create", async (data?: { message?: string }) => { + if (snapshotInFlight) { + return { success: true, message: "Snapshot already in progress" }; + } + snapshotInFlight = true; try { await ensureGitRepo(snapshotDir); @@ -124,6 +134,8 @@ export function registerSnapshotFunction( const msg = err instanceof Error ? err.message : String(err); logger.error("Snapshot failed", { error: msg }); return { success: false, error: msg }; + } finally { + snapshotInFlight = false; } }, ); diff --git a/src/hooks/session-end.ts b/src/hooks/session-end.ts index db149abdcc..c1f6cc9845 100644 --- a/src/hooks/session-end.ts +++ b/src/hooks/session-end.ts @@ -40,22 +40,6 @@ async function main() { signal: AbortSignal.timeout(30000), }).catch(() => {}); - if (process.env["CONSOLIDATION_ENABLED"] === "true") { - fetch(`${REST_URL}/agentmemory/crystals/auto`, { - method: "POST", - headers: authHeaders(), - body: JSON.stringify({ olderThanDays: 0 }), - signal: AbortSignal.timeout(60000), - }).catch(() => {}); - - fetch(`${REST_URL}/agentmemory/consolidate-pipeline`, { - method: "POST", - headers: authHeaders(), - body: JSON.stringify({ tier: "all", force: true }), - signal: AbortSignal.timeout(120000), - }).catch(() => {}); - } - if (process.env["CLAUDE_MEMORY_BRIDGE"] === "true") { fetch(`${REST_URL}/agentmemory/claude-bridge/sync`, { method: "POST", diff --git a/src/index.ts b/src/index.ts index 1e623eae81..5f66d76c95 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,5 +1,6 @@ -import { registerWorker } from "iii-sdk"; +import { registerWorker, TriggerAction } from "iii-sdk"; import { + hydrateProcessEnvFromFile, loadConfig, getEnvVar, loadEmbeddingConfig, @@ -57,6 +58,7 @@ import { registerExportImportFunction } from "./functions/export-import.js"; import { registerEnrichFunction } from "./functions/enrich.js"; import { registerClaudeBridgeFunction } from "./functions/claude-bridge.js"; import { registerGraphFunction } from "./functions/graph.js"; +import { registerGraphImportFunction } from "./functions/graph-import.js"; import { registerConsolidationPipelineFunction } from "./functions/consolidation-pipeline.js"; import { registerTeamFunction } from "./functions/team.js"; import { registerGovernanceFunction } from "./functions/governance.js"; @@ -158,6 +160,10 @@ process.on("unhandledRejection", (reason) => { }); async function main() { + // Fold ~/.agentmemory/.env into process.env before anything reads config + // or raw process.env. Only-if-unset, so real process.env still wins. + hydrateProcessEnvFromFile(); + const config = loadConfig(); const embeddingConfig = loadEmbeddingConfig(); const fallbackConfig = loadFallbackConfig(); @@ -268,6 +274,7 @@ async function main() { if (isGraphExtractionEnabled()) { registerGraphFunction(sdk, kv, provider); + registerGraphImportFunction(sdk, kv); bootLog(`Knowledge graph: extraction enabled`); } @@ -347,6 +354,21 @@ async function main() { const snapshotConfig = loadSnapshotConfig(); if (snapshotConfig.enabled) { registerSnapshotFunction(sdk, kv, snapshotConfig.dir); + // The boot line promised "every s" but nothing ever fired + // mem::snapshot-create. Drive it on a periodic timer (unref'd so it + // never keeps the process alive), mirroring the auto-forget timer. + // mem::snapshot-create serializes overlapping runs internally (git-lock + // safety), so the timer can stay a simple fire-and-forget tick. + const snapshotTimer = setInterval(() => { + sdk + .trigger({ + function_id: "mem::snapshot-create", + payload: {}, + action: TriggerAction.Void(), + }) + .catch(() => {}); + }, snapshotConfig.interval * 1000); + snapshotTimer.unref(); bootLog( `Git snapshots: ${snapshotConfig.dir} (every ${snapshotConfig.interval}s)`, ); @@ -518,7 +540,7 @@ async function main() { `Ready. ${embeddingProvider ? "Triple-stream (BM25+Vector+Graph)" : "BM25+Graph"} search active.`, ); bootLog( - `REST API: 128 endpoints at http://localhost:${config.restPort}/agentmemory/*`, + `REST API: 129 endpoints at http://localhost:${config.restPort}/agentmemory/*`, ); bootLog( `MCP surface (opt-in via \`npx @agentmemory/mcp\`): ${getAllTools().length} tools · 6 resources · 3 prompts`, diff --git a/src/providers/_fetch.ts b/src/providers/_fetch.ts index ed9b5e8961..3b02d9352d 100644 --- a/src/providers/_fetch.ts +++ b/src/providers/_fetch.ts @@ -1,15 +1,52 @@ import { getEnvVar } from "../config.js"; -export function fetchWithTimeout( +// Bounded retry for transient rate-limit / unavailable responses. Attempts is +// total tries (initial + retries). Retries are bounded by a TOTAL elapsed +// deadline — not per-attempt — so the worst case never blows past the caller's +// timeout budget or the iii invocation timeout. A single retry delay is capped +// low so a hostile Retry-After header can't dominate the budget. +const MAX_ATTEMPTS = 3; +const MAX_RETRY_DELAY_MS = 5000; +// Absolute ceiling on the total budget, kept well under the iii 180s invocation +// timeout so retries + sleeps + per-attempt timeouts can never overrun it. +const HARD_BUDGET_CAP_MS = 170000; +// A retry only makes sense if there's room for at least a token attempt after +// the sleep; without this floor we'd sleep, fire, and get instantly cut off. +const MIN_ATTEMPT_FLOOR_MS = 100; +const RETRY_STATUS = new Set([429, 503]); + +const sleep = (ms: number): Promise => + new Promise((resolve) => setTimeout(resolve, ms)); + +/** + * Parse a Retry-After header into a delay in milliseconds. Supports both the + * integer-seconds form and the HTTP-date form. Returns undefined when absent + * or unparseable, so the caller falls back to exponential backoff. Negative + * or past values clamp to 0. + */ +function parseRetryAfter(header: string | null): number | undefined { + if (!header) return undefined; + const trimmed = header.trim(); + if (trimmed === "") return undefined; + + const seconds = Number(trimmed); + if (Number.isFinite(seconds)) { + return Math.max(0, seconds * 1000); + } + + const date = Date.parse(trimmed); + if (Number.isFinite(date)) { + return Math.max(0, date - Date.now()); + } + + return undefined; +} + +async function fetchOnce( url: string, init: RequestInit, - timeoutMs?: number, + ms: number, ): Promise { - const parsed = - timeoutMs ?? - Number.parseInt(getEnvVar("AGENTMEMORY_LLM_TIMEOUT_MS") ?? "60000", 10); - const ms = Number.isFinite(parsed) && parsed > 0 ? parsed : 60000; - const ctl = new AbortController(); const signal = init.signal ? AbortSignal.any([init.signal, ctl.signal]) @@ -17,3 +54,53 @@ export function fetchWithTimeout( const t = setTimeout(() => ctl.abort(), ms); return fetch(url, { ...init, signal }).finally(() => clearTimeout(t)); } + +export async function fetchWithTimeout( + url: string, + init: RequestInit, + timeoutMs?: number, +): Promise { + const parsed = + timeoutMs ?? + Number.parseInt(getEnvVar("AGENTMEMORY_LLM_TIMEOUT_MS") ?? "60000", 10); + const ms = Number.isFinite(parsed) && parsed > 0 ? parsed : 60000; + + // The caller's timeout is the TOTAL budget for all attempts + sleeps, hard + // capped so we never approach the iii invocation timeout. + const budgetMs = Math.min(ms, HARD_BUDGET_CAP_MS); + const start = Date.now(); + + // The first attempt must honor the capped budget too — passing raw `ms` + // here would let a large caller timeout hang past HARD_BUDGET_CAP_MS (and + // the iii 180s invocation timeout) before any retry logic runs. + let response: Response = await fetchOnce(url, init, budgetMs); + for (let attempt = 1; attempt < MAX_ATTEMPTS; attempt++) { + if (!RETRY_STATUS.has(response.status)) return response; + + const retryAfter = parseRetryAfter(response.headers.get("Retry-After")); + // Exponential backoff fallback when no Retry-After: 500ms, 1000ms, ... + const backoff = 500 * 2 ** (attempt - 1); + const delay = Math.min(retryAfter ?? backoff, MAX_RETRY_DELAY_MS); + + // Stop retrying if the sleep plus a minimal attempt would overrun the total + // budget — a hostile Retry-After that alone exceeds the remaining budget + // returns the last response instead of stalling the caller. + const elapsed = Date.now() - start; + const remaining = budgetMs - elapsed; + if (delay + MIN_ATTEMPT_FLOOR_MS > remaining) return response; + + // This response is being discarded for a retry; release its body so the + // underlying connection is returned to the pool instead of leaking. + await response.body?.cancel().catch(() => {}); + await sleep(delay); + + // Cap the per-attempt timeout to whatever budget is left so a late attempt + // can't push total elapsed past the deadline. + const attemptMs = Math.max( + MIN_ATTEMPT_FLOOR_MS, + Math.min(ms, budgetMs - (Date.now() - start)), + ); + response = await fetchOnce(url, init, attemptMs); + } + return response; +} diff --git a/src/providers/embedding/_dimensions.ts b/src/providers/embedding/_dimensions.ts new file mode 100644 index 0000000000..f264e8a900 --- /dev/null +++ b/src/providers/embedding/_dimensions.ts @@ -0,0 +1,48 @@ +/** + * Shared embedding-dimension logic for OpenAI-compatible providers. + * + * OpenAI and OpenRouter expose the same underlying embedding models, so they + * share one dimension table and one resolver. OpenRouter namespaces model ids + * (e.g. "openai/text-embedding-3-small"); the lookup strips a leading + * "provider/" prefix so both bare and namespaced keys resolve to the same + * dimensions. + * + * The dimension guard (index.ts) throws on mismatch, so a wrong value here + * breaks every embed call — keep entries accurate. Callers pass the relevant + * env-var name (OPENAI_EMBEDDING_DIMENSIONS / OPENROUTER_EMBEDDING_DIMENSIONS) + * so error messages point at the knob the operator actually set. + */ +const MODEL_DIMENSIONS: Record = { + "text-embedding-3-small": 1536, + "text-embedding-3-large": 3072, + "text-embedding-ada-002": 1536, +}; + +const DEFAULT_DIMENSIONS = 1536; + +function lookupModelDimensions(model: string): number | undefined { + if (model in MODEL_DIMENSIONS) return MODEL_DIMENSIONS[model]; + const slash = model.indexOf("/"); + if (slash === -1) return undefined; + const bare = model.slice(slash + 1); + return MODEL_DIMENSIONS[bare]; +} + +export function resolveDimensions( + model: string, + override: string | undefined, + envName: string, +): number { + if (override !== undefined && override.trim().length > 0) { + const parsed = parseInt(override, 10); + if (!Number.isFinite(parsed) || parsed <= 0) { + throw new Error( + `${envName} must be a positive integer, got: ${override}`, + ); + } + return parsed; + } + return lookupModelDimensions(model) ?? DEFAULT_DIMENSIONS; +} + +export { MODEL_DIMENSIONS, DEFAULT_DIMENSIONS }; diff --git a/src/providers/embedding/openai.ts b/src/providers/embedding/openai.ts index 7384d51372..50344626a4 100644 --- a/src/providers/embedding/openai.ts +++ b/src/providers/embedding/openai.ts @@ -8,35 +8,10 @@ import { detectAzure, normalizeBaseUrl, } from "../_openai-shared.js"; +import { resolveDimensions } from "./_dimensions.js"; const DEFAULT_MODEL = "text-embedding-3-small"; -/** - * Known OpenAI embedding model dimensions. Extend as new models ship. - * Override in any case via OPENAI_EMBEDDING_DIMENSIONS for custom or - * self-hosted OpenAI-compatible endpoints returning non-standard sizes. - */ -const MODEL_DIMENSIONS: Record = { - "text-embedding-3-small": 1536, - "text-embedding-3-large": 3072, - "text-embedding-ada-002": 1536, -}; - -const DEFAULT_DIMENSIONS = MODEL_DIMENSIONS[DEFAULT_MODEL] ?? 1536; - -function resolveDimensions(model: string, override: string | undefined): number { - if (override !== undefined && override.trim().length > 0) { - const parsed = parseInt(override, 10); - if (!Number.isFinite(parsed) || parsed <= 0) { - throw new Error( - `OPENAI_EMBEDDING_DIMENSIONS must be a positive integer, got: ${override}`, - ); - } - return parsed; - } - return MODEL_DIMENSIONS[model] ?? DEFAULT_DIMENSIONS; -} - /** * OpenAI-compatible embedding provider. * @@ -71,7 +46,7 @@ function resolveDimensions(model: string, override: string | undefined): number * OPENAI_EMBEDDING_MODEL — model name (default: text-embedding-3-small) * OPENAI_EMBEDDING_DIMENSIONS — override reported dimensions (required for * custom / self-hosted models not in the - * MODEL_DIMENSIONS table above) + * shared MODEL_DIMENSIONS table) */ export class OpenAIEmbeddingProvider implements EmbeddingProvider { readonly name = "openai"; @@ -107,6 +82,7 @@ export class OpenAIEmbeddingProvider implements EmbeddingProvider { this.dimensions = resolveDimensions( this.model, getEnvVar("OPENAI_EMBEDDING_DIMENSIONS"), + "OPENAI_EMBEDDING_DIMENSIONS", ); this.isAzure = detectAzure(this.baseUrl); this.azureApiVersion = diff --git a/src/providers/embedding/openrouter.ts b/src/providers/embedding/openrouter.ts index 46999e559c..c4aa0bcf5c 100644 --- a/src/providers/embedding/openrouter.ts +++ b/src/providers/embedding/openrouter.ts @@ -1,21 +1,27 @@ import type { EmbeddingProvider } from "../../types.js"; import { getEnvVar } from "../../config.js"; import { fetchWithTimeout } from "../_fetch.js"; +import { resolveDimensions } from "./_dimensions.js"; const API_URL = "https://openrouter.ai/api/v1/embeddings"; +const DEFAULT_MODEL = "openai/text-embedding-3-small"; + export class OpenRouterEmbeddingProvider implements EmbeddingProvider { readonly name = "openrouter"; - readonly dimensions = 1536; + readonly dimensions: number; private apiKey: string; private model: string; constructor(apiKey?: string) { this.apiKey = apiKey || getEnvVar("OPENROUTER_API_KEY") || ""; if (!this.apiKey) throw new Error("OPENROUTER_API_KEY is required"); - this.model = - getEnvVar("OPENROUTER_EMBEDDING_MODEL") || - "openai/text-embedding-3-small"; + this.model = getEnvVar("OPENROUTER_EMBEDDING_MODEL") || DEFAULT_MODEL; + this.dimensions = resolveDimensions( + this.model, + getEnvVar("OPENROUTER_EMBEDDING_DIMENSIONS"), + "OPENROUTER_EMBEDDING_DIMENSIONS", + ); } async embed(text: string): Promise { diff --git a/src/state/schema.ts b/src/state/schema.ts index cb29d41ad8..773f147ce6 100644 --- a/src/state/schema.ts +++ b/src/state/schema.ts @@ -1,4 +1,5 @@ import { createHash } from "node:crypto"; +import { hasCjk, segmentCjk } from "./cjk-segmenter.js"; export const KV = { sessions: "mem:sessions", @@ -91,11 +92,60 @@ export function fingerprintId(prefix: string, content: string): string { return `${prefix}_${hash.slice(0, 16)}`; } +// CJK/Japanese/Thai text carries no inter-word whitespace, so a plain +// split(/\s+/) collapses a whole sentence into one token and the two token +// sets never overlap — dedup silently stops working. When either input +// contains CJK we segment the CJK runs (word tokens via the shared +// segmenter, plus character bigram shingles so near-identical strings still +// overlap even when the optional segmenter deps fall back to whole-string). +// ASCII text keeps the original word-level behavior. +function jaccardTokens(text: string): Set { + if (!hasCjk(text)) { + return new Set(text.split(/\s+/).filter((t) => t.length > 2)); + } + const tokens = new Set(); + for (const raw of text.split(/\s+/)) { + if (!raw) continue; + if (hasCjk(raw)) { + for (const seg of segmentCjk(raw)) { + if (seg) tokens.add(seg); + } + // Character bigram shingles keep the CJK signal even when the + // segmenter returns the whole run: overlapping bigrams make + // near-identical strings match while unrelated ones ("北京" vs + // "上海") share none. Never drop short CJK tokens. + const chars = Array.from(raw); + if (chars.length === 1) { + tokens.add(chars[0]); + } else { + for (let i = 0; i < chars.length - 1; i++) { + tokens.add(chars[i] + chars[i + 1]); + } + } + } else if (raw.length > 2) { + tokens.add(raw); + } + } + return tokens; +} + export function jaccardSimilarity(a: string, b: string): number { - const setA = new Set(a.split(/\s+/).filter((t) => t.length > 2)); - const setB = new Set(b.split(/\s+/).filter((t) => t.length > 2)); - if (setA.size === 0 && setB.size === 0) return 1; - if (setA.size === 0 || setB.size === 0) return 0; + const na = a.normalize("NFC"); + const nb = b.normalize("NFC"); + const setA = jaccardTokens(na); + const setB = jaccardTokens(nb); + // An empty token set carries no signal for the overlap metric. Very short + // ASCII text ("AI", "a b", "go") tokenizes to nothing because words <=2 + // chars are dropped. Fall back to exact normalized equality so re-saving + // the identical short memory still supersedes, while unrelated short + // strings ("AI" vs "ML") correctly score 0. Returning 1 unconditionally + // here (as an "both empty" shortcut once did) would let any short memory + // falsely supersede another, silently marking a real memory not-latest. + if (setA.size === 0 || setB.size === 0) { + return na.trim().replace(/\s+/g, " ") === nb.trim().replace(/\s+/g, " ") + ? 1 + : 0; + } let intersection = 0; for (const word of setA) { if (setB.has(word)) intersection++; diff --git a/src/triggers/api.ts b/src/triggers/api.ts index 99fb990840..ce339ae05f 100644 --- a/src/triggers/api.ts +++ b/src/triggers/api.ts @@ -851,11 +851,21 @@ export function registerApiTriggers( const filtered = filterAgentId ? sessions.filter((s) => s.agentId === filterAgentId) : sessions; - const summaries = await Promise.all( - filtered.map((s) => - kv.get(KV.summaries, s.id).catch(() => null), - ), - ); + // Bounded fan-out: each kv.get is a full engine invocation, so + // Promise.all over hundreds of sessions saturates the invocation + // pool. Batch in chunks of 10 (parallel within a chunk, sequential + // across chunks); the summaries array stays index-aligned with + // `filtered`. + const summaries: Array = []; + for (let batch = 0; batch < filtered.length; batch += 10) { + const chunk = filtered.slice(batch, batch + 10); + const results = await Promise.all( + chunk.map((s) => + kv.get(KV.summaries, s.id).catch(() => null), + ), + ); + summaries.push(...results); + } const withSummary = filtered.map((s, i) => summaries[i] ? { ...s, summary: summaries[i] } : s, ); @@ -1641,6 +1651,43 @@ export function registerApiTriggers( config: { api_path: "/agentmemory/graph/build", http_method: "POST" }, }); + // Import graphify's structural graph (graphify-out/graph.json) into the + // memory graph. Deterministic, no LLM call; idempotent via the graph + // name-index upsert. + sdk.registerFunction("api::graph-import-graphify", + async (req: ApiRequest<{ path?: string; cwd?: string }>): Promise => { + const authErr = checkAuth(req, secret); + if (authErr) return authErr; + const { path, cwd } = req.body ?? {}; + if ( + (path !== undefined && typeof path !== "string") || + (cwd !== undefined && typeof cwd !== "string") + ) { + return { + status_code: 400, + body: { error: "path and cwd must be strings when provided" }, + }; + } + try { + const result = await sdk.trigger({ + function_id: "mem::graph::import-graphify", + payload: { + ...(path !== undefined ? { path } : {}), + ...(cwd !== undefined ? { cwd } : {}), + }, + }); + return { status_code: 200, body: result }; + } catch { + return graphDisabledResponse(); + } + }, + ); + sdk.registerTrigger({ + type: "http", + function_id: "api::graph-import-graphify", + config: { api_path: "/agentmemory/graph/import-graphify", http_method: "POST" }, + }); + sdk.registerFunction("api::consolidate-pipeline", async (req: ApiRequest<{ tier?: string }>): Promise => { const authErr = checkAuth(req, secret); diff --git a/src/triggers/events.ts b/src/triggers/events.ts index e75d615b58..65db703513 100644 --- a/src/triggers/events.ts +++ b/src/triggers/events.ts @@ -3,9 +3,44 @@ import type { CompressedObservation, HookPayload, Session } from "../types.js"; import { KV, STREAM } from "../state/schema.js"; import { StateKV } from "../state/kv.js"; import { isReflectEnabled } from "../functions/slots.js"; -import { getAgentId, isGraphExtractionEnabled } from "../config.js"; +import { + getAgentId, + getConsolidationCooldownMs, + isConsolidationEnabled, + isGraphExtractionEnabled, +} from "../config.js"; import { logger } from "../logger.js"; +// Global marker recording when corpus consolidation last ran, used to debounce +// the per-turn session-stop fan-out. +const CONSOLIDATION_MARKER_KEY = "consolidation:lastRun"; + +async function consolidationDueUnserialized(kv: StateKV): Promise { + const cooldownMs = getConsolidationCooldownMs(); + if (cooldownMs <= 0) return true; // debounce disabled + const now = Date.now(); + const marker = await kv + .get<{ at?: number }>(KV.config, CONSOLIDATION_MARKER_KEY) + .catch(() => null); + const lastAt = typeof marker?.at === "number" ? marker.at : 0; + if (now - lastAt < cooldownMs) return false; + await kv.set(KV.config, CONSOLIDATION_MARKER_KEY, { at: now }).catch(() => {}); + return true; +} + +// Concurrent session-stop events would otherwise interleave the marker +// read-check-write above and both pass the cooldown. Serialize the whole +// check through an in-process chain so exactly one concurrent caller wins. +let consolidationCheckChain: Promise = Promise.resolve(); + +function consolidationDue(kv: StateKV): Promise { + const result = consolidationCheckChain.then(() => + consolidationDueUnserialized(kv), + ); + consolidationCheckChain = result.catch(() => false); + return result; +} + export function registerEventTriggers(sdk: ISdk, kv: StateKV): void { sdk.registerFunction( "event::session::started", @@ -59,21 +94,19 @@ export function registerEventTriggers(sdk: ISdk, kv: StateKV): void { config: { topic: "agentmemory.observation" }, }); - sdk.registerFunction("event::session::stopped", async (data: { sessionId: string }) => { + sdk.registerFunction("event::session::stopped", async (data: { sessionId: string; skipConsolidation?: boolean }) => { const summary = await sdk.trigger({ function_id: "mem::summarize", payload: data }); + const fireVoid = (function_id: string, payload: unknown) => + sdk + .trigger({ function_id, payload, action: TriggerAction.Void() }) + .catch((err) => + logger.warn(function_id + " trigger failed", { + sessionId: data.sessionId, + error: err instanceof Error ? err.message : String(err), + }), + ); if (isReflectEnabled()) { - try { - sdk.trigger({ - function_id: "mem::slot-reflect", - payload: { sessionId: data.sessionId }, - action: TriggerAction.Void(), - }); - } catch (err) { - logger.warn("slot-reflect trigger failed", { - sessionId: data.sessionId, - error: err instanceof Error ? err.message : String(err), - }); - } + fireVoid("mem::slot-reflect", { sessionId: data.sessionId }); } if (isGraphExtractionEnabled()) { try { @@ -95,6 +128,28 @@ export function registerEventTriggers(sdk: ISdk, kv: StateKV): void { }); } } + // Crystals + lessons consolidation. The stop lifecycle is the single + // source of truth: event::session::stopped fires for ALL agents (the + // client-side session-end hook no longer drives consolidation directly). + // Gated so keyless/zero-LLM users don't fire no-op LLM calls. + // + // skipConsolidation suppresses the fan-out when this handler is driven + // by eviction's stale-session recovery: evict calls session::stopped + // once per recovered session, then runs ONE final consolidation pass. + // Without this guard, N recovered sessions launch N concurrent forced + // full-corpus consolidations plus N crystallizations. + // + // Debounce: /session/end is posted by the per-turn Stop hook, so this + // handler fires on every agent turn. consolidate-pipeline + auto-crystallize + // are full-corpus LLM work with no internal "nothing changed" guard, so + // firing them every turn is a cost/latency storm for connected agents. + // Bound the global corpus consolidation to once per cooldown window. + if (isConsolidationEnabled() && !data.skipConsolidation) { + if (await consolidationDue(kv)) { + fireVoid("mem::consolidate-pipeline", { tier: "all", force: true }); + fireVoid("mem::auto-crystallize", { olderThanDays: 0 }); + } + } return summary; }); sdk.registerTrigger({ diff --git a/src/viewer/server.ts b/src/viewer/server.ts index c80e67f0af..3f983ba250 100644 --- a/src/viewer/server.ts +++ b/src/viewer/server.ts @@ -158,7 +158,7 @@ function json( function readBody(req: IncomingMessage): Promise { return new Promise((resolve, reject) => { - let data = ""; + const chunks: Buffer[] = []; let size = 0; req.on("data", (chunk: Buffer) => { size += chunk.length; @@ -167,9 +167,9 @@ function readBody(req: IncomingMessage): Promise { reject(new Error("too large")); return; } - data += chunk.toString(); + chunks.push(chunk); }); - req.on("end", () => resolve(data)); + req.on("end", () => resolve(Buffer.concat(chunks).toString("utf8"))); req.on("error", reject); }); } diff --git a/test/connect-guidelines.test.ts b/test/connect-guidelines.test.ts new file mode 100644 index 0000000000..9172723609 --- /dev/null +++ b/test/connect-guidelines.test.ts @@ -0,0 +1,156 @@ +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { mkdtempSync, rmSync, existsSync, readFileSync, writeFileSync, mkdirSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { writeGuideline, guidelineTargets } from "../src/cli/connect/guidelines.js"; + +let home: string; +let cwd: string; + +beforeEach(() => { + home = mkdtempSync(join(tmpdir(), "am-guide-home-")); + cwd = mkdtempSync(join(tmpdir(), "am-guide-cwd-")); +}); + +afterEach(() => { + rmSync(home, { recursive: true, force: true }); + rmSync(cwd, { recursive: true, force: true }); +}); + +describe("writeGuideline", () => { + it("writes a Cursor .mdc project rule with alwaysApply frontmatter", () => { + const r = writeGuideline("cursor", { cwd, home }); + expect(r.kind).toBe("written"); + const path = join(cwd, ".cursor", "rules", "agentmemory.mdc"); + expect(existsSync(path)).toBe(true); + const body = readFileSync(path, "utf8"); + expect(body).toContain("alwaysApply: true"); + expect(body).toContain("memory_recall"); + expect(body).toContain("memory_save"); + }); + + it("writes a Kiro steering file with inclusion: always (global)", () => { + const r = writeGuideline("kiro", { cwd, home }); + expect(r.kind).toBe("written"); + if (r.kind === "written") expect(r.scope).toBe("global"); + const path = join(home, ".kiro", "steering", "agentmemory.md"); + expect(existsSync(path)).toBe(true); + expect(readFileSync(path, "utf8")).toContain("inclusion: always"); + }); + + it("writes a marked block into the agent's global AGENTS.md (Zed)", () => { + const r = writeGuideline("zed", { cwd, home }); + expect(r.kind).toBe("written"); + const path = join(home, ".config", "zed", "AGENTS.md"); + const body = readFileSync(path, "utf8"); + expect(body).toContain(""); + expect(body).toContain(""); + }); + + it("prefers the global path when the target defines one (Droid)", () => { + const r = writeGuideline("droid", { cwd, home }); + expect(r.kind).toBe("written"); + expect(existsSync(join(home, ".factory", "AGENTS.md"))).toBe(true); + // must NOT have written into the project cwd + expect(existsSync(join(cwd, "AGENTS.md"))).toBe(false); + }); + + it("falls back to a project path when the agent has no global rules file (Warp)", () => { + const r = writeGuideline("warp", { cwd, home }); + expect(r.kind).toBe("written"); + if (r.kind === "written") expect(r.scope).toBe("project"); + expect(existsSync(join(cwd, "AGENTS.md"))).toBe(true); + }); + + it("preserves surrounding user content and updates only its own block", () => { + const path = join(home, ".factory", "AGENTS.md"); + mkdirSync(join(home, ".factory"), { recursive: true }); + writeFileSync(path, "# My rules\n\nUse tabs, not spaces.\n", "utf8"); + + writeGuideline("droid", { cwd, home }); + const first = readFileSync(path, "utf8"); + expect(first).toContain("# My rules"); + expect(first).toContain("Use tabs, not spaces."); + expect(first).toContain(""); + + // Re-running is idempotent (no duplicate block, reports unchanged). + const again = writeGuideline("droid", { cwd, home }); + expect(again.kind).toBe("unchanged"); + const second = readFileSync(path, "utf8"); + expect(second).toBe(first); + expect(second.match(/agentmemory:start/g)?.length).toBe(1); + }); + + it("refuses to touch a file with a lone or reversed marker", () => { + const path = join(home, ".factory", "AGENTS.md"); + mkdirSync(join(home, ".factory"), { recursive: true }); + // Lone START marker: appending would let a later run pair this orphan + // with the appended block's END and cut the user's content in between. + const lone = "# Rules\n\nuser notes here\n"; + writeFileSync(path, lone, "utf8"); + const r = writeGuideline("droid", { cwd, home }); + expect(r.kind).toBe("unchanged"); + expect(readFileSync(path, "utf8")).toBe(lone); + + // Reversed pair: same refusal. + const reversed = + "\nmiddle\n\n"; + writeFileSync(path, reversed, "utf8"); + const r2 = writeGuideline("droid", { cwd, home }); + expect(r2.kind).toBe("unchanged"); + expect(readFileSync(path, "utf8")).toBe(reversed); + }); + + it("is idempotent for dedicated files (second run unchanged)", () => { + expect(writeGuideline("cursor", { cwd, home }).kind).toBe("written"); + expect(writeGuideline("cursor", { cwd, home }).kind).toBe("unchanged"); + }); + + it("dry-run reports would-write without creating the file", () => { + const r = writeGuideline("kiro", { cwd, home, dryRun: true }); + expect(r.kind).toBe("would-write"); + expect(existsSync(join(home, ".kiro", "steering", "agentmemory.md"))).toBe(false); + }); + + it("returns no-target for agents that already auto-capture (claude-code)", () => { + expect(writeGuideline("claude-code", { cwd, home }).kind).toBe("no-target"); + expect(writeGuideline("codex", { cwd, home }).kind).toBe("no-target"); + }); + + it("Gemini and Antigravity share ~/.gemini/GEMINI.md idempotently", () => { + writeGuideline("gemini-cli", { cwd, home }); + const r2 = writeGuideline("antigravity", { cwd, home }); + // Antigravity targets the same GEMINI.md; the block already exists. + expect(r2.kind).toBe("unchanged"); + const body = readFileSync(join(home, ".gemini", "GEMINI.md"), "utf8"); + expect(body.match(/agentmemory:start/g)?.length).toBe(1); + }); +}); + +describe("guidelineTargets coverage", () => { + it("covers every MCP-only / partial agent and no others", () => { + const names = Object.keys(guidelineTargets("/home/x")).sort(); + expect(names).toEqual( + [ + "antigravity", + "cline", + "continue", + "copilot-cli", + "cursor", + "droid", + "gemini-cli", + "kiro", + "opencode", + "qwen", + "warp", + "zed", + ].sort(), + ); + }); + + it("every target cites an official source URL", () => { + for (const t of Object.values(guidelineTargets("/home/x"))) { + expect(t.source).toMatch(/^https:\/\//); + } + }); +}); diff --git a/test/embedding-dimensions.test.ts b/test/embedding-dimensions.test.ts new file mode 100644 index 0000000000..51b3b084a1 --- /dev/null +++ b/test/embedding-dimensions.test.ts @@ -0,0 +1,101 @@ +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { resolveDimensions } from "../src/providers/embedding/_dimensions.js"; +import { OpenRouterEmbeddingProvider } from "../src/providers/embedding/openrouter.js"; +import { OpenAIEmbeddingProvider } from "../src/providers/embedding/openai.js"; + +describe("resolveDimensions", () => { + const ENV = "OPENROUTER_EMBEDDING_DIMENSIONS"; + + it("resolves namespaced OpenRouter model ids to their real dimensions", () => { + expect(resolveDimensions("openai/text-embedding-3-large", undefined, ENV)).toBe(3072); + expect(resolveDimensions("openai/text-embedding-3-small", undefined, ENV)).toBe(1536); + expect(resolveDimensions("openai/text-embedding-ada-002", undefined, ENV)).toBe(1536); + }); + + it("resolves bare model ids to their real dimensions", () => { + expect(resolveDimensions("text-embedding-3-large", undefined, ENV)).toBe(3072); + expect(resolveDimensions("text-embedding-3-small", undefined, ENV)).toBe(1536); + expect(resolveDimensions("text-embedding-ada-002", undefined, ENV)).toBe(1536); + }); + + it("lets a valid override win over the model-derived dimensions", () => { + expect(resolveDimensions("openai/text-embedding-3-large", "1024", ENV)).toBe(1024); + expect(resolveDimensions("text-embedding-3-small", "768", ENV)).toBe(768); + }); + + it("throws with the given env name on invalid override values", () => { + for (const bad of ["abc", "0", "-5"]) { + expect(() => resolveDimensions("text-embedding-3-large", bad, ENV)).toThrow( + new RegExp(`${ENV} must be a positive integer, got: ${bad}`), + ); + } + }); + + it("uses the supplied env name in the error message", () => { + expect(() => resolveDimensions("text-embedding-3-large", "abc", "OPENAI_EMBEDDING_DIMENSIONS")).toThrow( + /OPENAI_EMBEDDING_DIMENSIONS must be a positive integer, got: abc/, + ); + }); + + it("falls back to the default (1536) for unknown models", () => { + expect(resolveDimensions("mystery-self-hosted-model", undefined, ENV)).toBe(1536); + expect(resolveDimensions("someprovider/unknown-model", undefined, ENV)).toBe(1536); + }); +}); + +describe("OpenRouterEmbeddingProvider dimension regression", () => { + const originalEnv = { ...process.env }; + + beforeEach(() => { + process.env = { ...originalEnv }; + delete process.env["OPENROUTER_EMBEDDING_MODEL"]; + delete process.env["OPENROUTER_EMBEDDING_DIMENSIONS"]; + }); + + afterEach(() => { + process.env = originalEnv; + }); + + it("reports 3072 for openai/text-embedding-3-large with no override (guard would throw on the old hardcoded 1536)", () => { + process.env["OPENROUTER_EMBEDDING_MODEL"] = "openai/text-embedding-3-large"; + const provider = new OpenRouterEmbeddingProvider("test-key"); + expect(provider.dimensions).toBe(3072); + }); + + it("defaults to 1536 for openai/text-embedding-3-small", () => { + const provider = new OpenRouterEmbeddingProvider("test-key"); + expect(provider.dimensions).toBe(1536); + }); + + it("lets OPENROUTER_EMBEDDING_DIMENSIONS override the model-derived dimensions", () => { + process.env["OPENROUTER_EMBEDDING_MODEL"] = "openai/text-embedding-3-large"; + process.env["OPENROUTER_EMBEDDING_DIMENSIONS"] = "1024"; + const provider = new OpenRouterEmbeddingProvider("test-key"); + expect(provider.dimensions).toBe(1024); + }); +}); + +describe("OpenAIEmbeddingProvider defaults unchanged", () => { + const originalEnv = { ...process.env }; + + beforeEach(() => { + process.env = { ...originalEnv }; + delete process.env["OPENAI_EMBEDDING_MODEL"]; + delete process.env["OPENAI_EMBEDDING_DIMENSIONS"]; + }); + + afterEach(() => { + process.env = originalEnv; + }); + + it("defaults to 1536 for text-embedding-3-small", () => { + const provider = new OpenAIEmbeddingProvider("test-key"); + expect(provider.dimensions).toBe(1536); + }); + + it("reports 3072 for text-embedding-3-large", () => { + process.env["OPENAI_EMBEDDING_MODEL"] = "text-embedding-3-large"; + const provider = new OpenAIEmbeddingProvider("test-key"); + expect(provider.dimensions).toBe(3072); + }); +}); diff --git a/test/env-loader.test.ts b/test/env-loader.test.ts index 17ff6a8e41..4c31bebab4 100644 --- a/test/env-loader.test.ts +++ b/test/env-loader.test.ts @@ -3,6 +3,19 @@ import { mkdtempSync, writeFileSync, mkdirSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; +const readFileSyncCalls: unknown[][] = []; + +vi.mock("node:fs", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + readFileSync: (...args: unknown[]) => { + readFileSyncCalls.push(args); + return (actual.readFileSync as (...a: unknown[]) => unknown)(...args); + }, + }; +}); + const ORIGINAL_HOME = process.env["HOME"]; const ORIGINAL_USERPROFILE = process.env["USERPROFILE"]; @@ -90,3 +103,106 @@ describe("loadEnvFile", () => { expect(cfg.isDropStaleIndexEnabled()).toBe(true); }); }); + +describe("hydrateProcessEnvFromFile", () => { + const TOUCHED = ["HYDRATE_ONLY", "HYDRATE_WINS"]; + + beforeEach(() => { + sandboxHome = mkdtempSync(join(tmpdir(), "agentmemory-hydrate-")); + process.env["HOME"] = sandboxHome; + process.env["USERPROFILE"] = sandboxHome; + for (const k of TOUCHED) delete process.env[k]; + }); + + afterEach(() => { + if (ORIGINAL_HOME === undefined) delete process.env["HOME"]; + else process.env["HOME"] = ORIGINAL_HOME; + if (ORIGINAL_USERPROFILE === undefined) delete process.env["USERPROFILE"]; + else process.env["USERPROFILE"] = ORIGINAL_USERPROFILE; + for (const k of TOUCHED) delete process.env[k]; + rmSync(sandboxHome, { recursive: true, force: true }); + }); + + it("copies .env-only keys into process.env", async () => { + writeEnv("HYDRATE_ONLY=from-file"); + const cfg = await freshConfig(); + expect(process.env["HYDRATE_ONLY"]).toBeUndefined(); + cfg.hydrateProcessEnvFromFile(); + expect(process.env["HYDRATE_ONLY"]).toBe("from-file"); + }); + + it("does not overwrite a value already set in process.env", async () => { + writeEnv("HYDRATE_WINS=from-file"); + process.env["HYDRATE_WINS"] = "from-process"; + const cfg = await freshConfig(); + cfg.hydrateProcessEnvFromFile(); + expect(process.env["HYDRATE_WINS"]).toBe("from-process"); + }); + + it("exposes a .env-only key via getEnvVar and, after hydrate, via raw process.env", async () => { + writeEnv("HYDRATE_ONLY=from-file"); + const cfg = await freshConfig(); + expect(cfg.getEnvVar("HYDRATE_ONLY")).toBe("from-file"); + expect(process.env["HYDRATE_ONLY"]).toBeUndefined(); + cfg.hydrateProcessEnvFromFile(); + expect(process.env["HYDRATE_ONLY"]).toBe("from-file"); + }); +}); + +describe("loadEnvFile cache", () => { + beforeEach(() => { + sandboxHome = mkdtempSync(join(tmpdir(), "agentmemory-cache-")); + process.env["HOME"] = sandboxHome; + process.env["USERPROFILE"] = sandboxHome; + delete process.env["CACHED_VAR"]; + readFileSyncCalls.length = 0; + }); + + afterEach(() => { + if (ORIGINAL_HOME === undefined) delete process.env["HOME"]; + else process.env["HOME"] = ORIGINAL_HOME; + if (ORIGINAL_USERPROFILE === undefined) delete process.env["USERPROFILE"]; + else process.env["USERPROFILE"] = ORIGINAL_USERPROFILE; + delete process.env["CACHED_VAR"]; + rmSync(sandboxHome, { recursive: true, force: true }); + }); + + it("reads the .env file from disk only once across many getMergedEnv/getEnvVar calls", async () => { + writeEnv("CACHED_VAR=cached"); + const cfg = await freshConfig(); + const envPath = join(sandboxHome, ".agentmemory", ".env"); + readFileSyncCalls.length = 0; + + for (let i = 0; i < 25; i++) { + cfg.getEnvVar("CACHED_VAR"); + cfg.loadEmbeddingConfig(); + cfg.isConsolidationEnabled(); + } + + const envReads = readFileSyncCalls.filter(([p]) => p === envPath); + expect(envReads).toHaveLength(1); + expect(cfg.getEnvVar("CACHED_VAR")).toBe("cached"); + }); + + it("sees updated .env content after vi.resetModules reloads the module", async () => { + writeEnv("CACHED_VAR=first"); + const first = await freshConfig(); + expect(first.getEnvVar("CACHED_VAR")).toBe("first"); + + writeEnv("CACHED_VAR=second"); + const second = await freshConfig(); + expect(second.getEnvVar("CACHED_VAR")).toBe("second"); + }); + + it("re-reads disk after __resetEnvFileCache within the same module instance", async () => { + writeEnv("CACHED_VAR=first"); + const cfg = await freshConfig(); + expect(cfg.getEnvVar("CACHED_VAR")).toBe("first"); + + writeEnv("CACHED_VAR=second"); + expect(cfg.getEnvVar("CACHED_VAR")).toBe("first"); + + cfg.__resetEnvFileCache(); + expect(cfg.getEnvVar("CACHED_VAR")).toBe("second"); + }); +}); diff --git a/test/events-consolidation.test.ts b/test/events-consolidation.test.ts new file mode 100644 index 0000000000..038566a8c3 --- /dev/null +++ b/test/events-consolidation.test.ts @@ -0,0 +1,324 @@ +import { describe, it, expect, beforeEach, vi } from "vitest"; +import { readFileSync } from "node:fs"; + +vi.mock("../src/logger.js", () => ({ + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, +})); + +vi.mock("../src/config.js", () => ({ + getAgentId: vi.fn(() => undefined), + isConsolidationEnabled: vi.fn(() => true), + isGraphExtractionEnabled: vi.fn(() => false), + getConsolidationCooldownMs: vi.fn(() => 300000), +})); + +vi.mock("../src/functions/slots.js", () => ({ + isReflectEnabled: vi.fn(() => false), +})); + +import { registerEventTriggers } from "../src/triggers/events.js"; +import { + isConsolidationEnabled, + isGraphExtractionEnabled, + getConsolidationCooldownMs, +} from "../src/config.js"; +import { isReflectEnabled } from "../src/functions/slots.js"; +import { logger } from "../src/logger.js"; + +// event::session::stopped is the single source of truth for consolidation. +// It fans out mem::summarize (awaited) plus fire-and-forget void triggers for +// slot-reflect / consolidate-pipeline / auto-crystallize, each gated by config. +// The client session-end hook no longer POSTs crystals/auto or +// consolidate-pipeline, so these no longer double-fire for Claude Code. + +function mockKV() { + return { + get: vi.fn(async () => null), + set: vi.fn(async (_scope: string, _key: string, data: unknown) => data), + delete: vi.fn(async () => {}), + update: vi.fn(async () => {}), + list: vi.fn(async () => []), + }; +} + +type StoppedHandler = (data: { + sessionId: string; + skipConsolidation?: boolean; +}) => Promise; + +// Builds a spy-backed sdk. `trigger` resolves for mem::summarize with a fake +// summary; void triggers resolve unless `rejectFor` matches the function_id, +// in which case they reject (to exercise fireVoid's .catch()). +function mockSdk(opts?: { rejectFor?: string }) { + const handlers = new Map(); + const trigger = vi.fn( + async (input: { function_id: string; payload?: unknown; action?: unknown }) => { + if (opts?.rejectFor && input.function_id === opts.rejectFor) { + throw new Error(`boom: ${input.function_id}`); + } + if (input.function_id === "mem::summarize") { + return { summary: "session summary", sessionId: "ses_1" }; + } + return { ok: true }; + }, + ); + return { + sdk: { + registerFunction: (id: string, handler: StoppedHandler) => handlers.set(id, handler), + registerTrigger: () => {}, + trigger, + }, + handlers, + trigger, + }; +} + +function functionIds(trigger: ReturnType): string[] { + return trigger.mock.calls.map((c) => (c[0] as { function_id: string }).function_id); +} + +describe("event::session::stopped consolidation fan-out", () => { + beforeEach(() => { + vi.mocked(isConsolidationEnabled).mockReturnValue(true); + vi.mocked(isGraphExtractionEnabled).mockReturnValue(false); + vi.mocked(isReflectEnabled).mockReturnValue(false); + vi.mocked(logger.warn).mockClear(); + }); + + it("fires consolidate-pipeline and auto-crystallize when consolidation enabled", async () => { + vi.mocked(isConsolidationEnabled).mockReturnValue(true); + const { sdk, handlers, trigger } = mockSdk(); + registerEventTriggers(sdk as never, mockKV() as never); + + const stopped = handlers.get("event::session::stopped")!; + await stopped({ sessionId: "ses_1" }); + + const ids = functionIds(trigger); + expect(ids).toContain("mem::summarize"); + expect(ids).toContain("mem::consolidate-pipeline"); + expect(ids).toContain("mem::auto-crystallize"); + + const consolidateCall = trigger.mock.calls.find( + (c) => (c[0] as { function_id: string }).function_id === "mem::consolidate-pipeline", + ); + expect((consolidateCall![0] as { payload: unknown }).payload).toEqual({ + tier: "all", + force: true, + }); + const crystallizeCall = trigger.mock.calls.find( + (c) => (c[0] as { function_id: string }).function_id === "mem::auto-crystallize", + ); + expect((crystallizeCall![0] as { payload: unknown }).payload).toEqual({ + olderThanDays: 0, + }); + }); + + it("skips consolidate-pipeline and auto-crystallize when consolidation disabled but still summarizes", async () => { + vi.mocked(isConsolidationEnabled).mockReturnValue(false); + const { sdk, handlers, trigger } = mockSdk(); + registerEventTriggers(sdk as never, mockKV() as never); + + const stopped = handlers.get("event::session::stopped")!; + await stopped({ sessionId: "ses_1" }); + + const ids = functionIds(trigger); + expect(ids).toContain("mem::summarize"); + expect(ids).not.toContain("mem::consolidate-pipeline"); + expect(ids).not.toContain("mem::auto-crystallize"); + }); + + it("suppresses the consolidation fan-out when skipConsolidation is set (eviction recovery path)", async () => { + // Regression: mem::evict calls event::session::stopped once per recovered + // stale session, then runs ONE final consolidation pass. Without the + // skipConsolidation guard, N recovered sessions would launch N concurrent + // forced full-corpus consolidations + N crystallizations. + vi.mocked(isConsolidationEnabled).mockReturnValue(true); + const { sdk, handlers, trigger } = mockSdk(); + registerEventTriggers(sdk as never, mockKV() as never); + + const stopped = handlers.get("event::session::stopped")!; + await stopped({ sessionId: "ses_1", skipConsolidation: true }); + + const ids = functionIds(trigger); + // Per-session work still happens... + expect(ids).toContain("mem::summarize"); + // ...but the corpus-wide fan-out is deferred to evict's single pass. + expect(ids).not.toContain("mem::consolidate-pipeline"); + expect(ids).not.toContain("mem::auto-crystallize"); + }); + + it("still fans out when skipConsolidation is explicitly false (normal stop)", async () => { + vi.mocked(isConsolidationEnabled).mockReturnValue(true); + const { sdk, handlers, trigger } = mockSdk(); + registerEventTriggers(sdk as never, mockKV() as never); + + await handlers.get("event::session::stopped")!({ + sessionId: "ses_1", + skipConsolidation: false, + }); + + const ids = functionIds(trigger); + expect(ids).toContain("mem::consolidate-pipeline"); + expect(ids).toContain("mem::auto-crystallize"); + }); + + it("respects the cooldown across repeated stops but this suite's non-persistent KV fires each single call", async () => { + // Sanity: with the default cooldown and a fresh (marker-less) KV, a single + // stop still consolidates. The real debounce is exercised below with a + // persistent KV. + vi.mocked(getConsolidationCooldownMs).mockReturnValue(300000); + const { sdk, handlers, trigger } = mockSdk(); + registerEventTriggers(sdk as never, mockKV() as never); + await handlers.get("event::session::stopped")!({ sessionId: "ses_1" }); + expect(functionIds(trigger)).toContain("mem::consolidate-pipeline"); + }); + + it("respects isReflectEnabled gating for slot-reflect", async () => { + vi.mocked(isReflectEnabled).mockReturnValue(false); + const off = mockSdk(); + registerEventTriggers(off.sdk as never, mockKV() as never); + await off.handlers.get("event::session::stopped")!({ sessionId: "ses_1" }); + expect(functionIds(off.trigger)).not.toContain("mem::slot-reflect"); + + vi.mocked(isReflectEnabled).mockReturnValue(true); + const on = mockSdk(); + registerEventTriggers(on.sdk as never, mockKV() as never); + await on.handlers.get("event::session::stopped")!({ sessionId: "ses_1" }); + expect(functionIds(on.trigger)).toContain("mem::slot-reflect"); + }); + + it("does not throw and still returns the summary when consolidate-pipeline trigger rejects", async () => { + vi.mocked(isConsolidationEnabled).mockReturnValue(true); + const { sdk, handlers } = mockSdk({ rejectFor: "mem::consolidate-pipeline" }); + registerEventTriggers(sdk as never, mockKV() as never); + + const stopped = handlers.get("event::session::stopped")!; + const summary = await stopped({ sessionId: "ses_1" }); + + expect(summary).toEqual({ summary: "session summary", sessionId: "ses_1" }); + await Promise.resolve(); + await Promise.resolve(); + expect(logger.warn).toHaveBeenCalledWith( + "mem::consolidate-pipeline trigger failed", + expect.objectContaining({ sessionId: "ses_1" }), + ); + }); +}); + +// The client session-end hook is bundled into a standalone binary that reads +// stdin and POSTs to REST, so it is exercised at the source level: after the +// double-fire fix it must no longer POST the two direct consolidation +// endpoints, leaving event::session::stopped as the only consolidation path. +describe("session-end hook no longer double-fires consolidation", () => { + const src = readFileSync("src/hooks/session-end.ts", "utf-8"); + + it("does not POST /agentmemory/crystals/auto", () => { + expect(src).not.toContain("/agentmemory/crystals/auto"); + }); + + it("does not POST /agentmemory/consolidate-pipeline", () => { + expect(src).not.toContain("/agentmemory/consolidate-pipeline"); + }); + + it("no longer references the CONSOLIDATION_ENABLED gate", () => { + expect(src).not.toContain("CONSOLIDATION_ENABLED"); + }); + + it("still POSTs /agentmemory/session/end (the single source of truth path)", () => { + expect(src).toContain("/agentmemory/session/end"); + }); + + it("keeps the claude-bridge/sync block", () => { + expect(src).toContain("/agentmemory/claude-bridge/sync"); + }); + + it("keeps the null-guard and main().catch() patterns", () => { + expect(src).toContain('if (!data || typeof data !== "object") return;'); + expect(src).toContain("main().catch(() => process.exit(0));"); + }); +}); + +// A KV that actually persists writes, so the debounce marker survives between +// simulated per-turn stops. +function persistentKV() { + const store = new Map>(); + return { + get: vi.fn(async (scope: string, key: string) => store.get(scope)?.get(key) ?? null), + set: vi.fn(async (scope: string, key: string, data: unknown) => { + if (!store.has(scope)) store.set(scope, new Map()); + store.get(scope)!.set(key, data); + return data; + }), + delete: vi.fn(async () => {}), + update: vi.fn(async () => {}), + list: vi.fn(async () => []), + }; +} + +// Regression: the Stop hook posts /session/end on every agent turn, which fires +// event::session::stopped. consolidate-pipeline + auto-crystallize are full +// corpus LLM work with no internal "nothing changed" guard, so firing them per +// turn is a cost/latency storm for connected agents (Claude/Codex/Copilot/ +// Hermes). The debounce bounds corpus consolidation to once per cooldown. +describe("session-stop consolidation debounce", () => { + beforeEach(() => { + vi.mocked(isConsolidationEnabled).mockReturnValue(true); + vi.mocked(isGraphExtractionEnabled).mockReturnValue(false); + vi.mocked(isReflectEnabled).mockReturnValue(false); + vi.mocked(getConsolidationCooldownMs).mockReturnValue(300000); + }); + + it("consolidates at most once across many per-turn stops within the cooldown", async () => { + const { sdk, handlers, trigger } = mockSdk(); + registerEventTriggers(sdk as never, persistentKV() as never); + const stopped = handlers.get("event::session::stopped")!; + + // 5 per-turn Stop hooks in quick succession (all within the cooldown). + for (let i = 0; i < 5; i++) await stopped({ sessionId: "ses_1" }); + + const count = (id: string) => + trigger.mock.calls.filter( + (c) => (c[0] as { function_id: string }).function_id === id, + ).length; + + // Corpus consolidation runs ONCE, not five times. + expect(count("mem::consolidate-pipeline")).toBe(1); + expect(count("mem::auto-crystallize")).toBe(1); + // Per-turn summary capture still runs every turn (the cheap path). + expect(count("mem::summarize")).toBe(5); + }); + + it("consolidates once when stops arrive concurrently (serialized cooldown check)", async () => { + // Regression: without serialization, two stops racing through the marker + // read-check-write both observe the stale marker and both fire. + const { sdk, handlers, trigger } = mockSdk(); + registerEventTriggers(sdk as never, persistentKV() as never); + const stopped = handlers.get("event::session::stopped")!; + + await Promise.all([ + stopped({ sessionId: "ses_1" }), + stopped({ sessionId: "ses_2" }), + stopped({ sessionId: "ses_3" }), + ]); + + const consolidateCount = trigger.mock.calls.filter( + (c) => (c[0] as { function_id: string }).function_id === "mem::consolidate-pipeline", + ).length; + expect(consolidateCount).toBe(1); + }); + + it("consolidates on every stop when the cooldown is disabled (0)", async () => { + vi.mocked(getConsolidationCooldownMs).mockReturnValue(0); + const { sdk, handlers, trigger } = mockSdk(); + registerEventTriggers(sdk as never, persistentKV() as never); + const stopped = handlers.get("event::session::stopped")!; + + await stopped({ sessionId: "ses_1" }); + await stopped({ sessionId: "ses_1" }); + + const consolidateCount = trigger.mock.calls.filter( + (c) => (c[0] as { function_id: string }).function_id === "mem::consolidate-pipeline", + ).length; + expect(consolidateCount).toBe(2); + }); +}); diff --git a/test/evict.test.ts b/test/evict.test.ts index d1aedac20e..690e8a7c1f 100644 --- a/test/evict.test.ts +++ b/test/evict.test.ts @@ -11,6 +11,13 @@ vi.mock("../src/logger.js", () => ({ logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, })); +// The recovered-session consolidation pass is gated on isConsolidationEnabled +// (keyless installs skip it); force it on so these tests exercise the pass. +vi.mock("../src/config.js", async (importOriginal) => ({ + ...(await importOriginal()), + isConsolidationEnabled: () => true, +})); + type Store = Map>; type Handler = (payload: unknown) => unknown | Promise; @@ -126,7 +133,9 @@ describe("mem::evict stale sessions", () => { registerEvictFunction(sdk as never, kv as never); sdk.registerFunction("event::session::stopped", async (payload) => { - expect(payload).toEqual({ sessionId }); + // Recovery must pass skipConsolidation so the per-session fan-out is + // suppressed (evict runs a single corpus-wide pass afterwards). + expect(payload).toEqual({ sessionId, skipConsolidation: true }); expect(await kv.get(KV.sessions, sessionId)).toMatchObject({ id: sessionId, }); @@ -135,6 +144,7 @@ describe("mem::evict stale sessions", () => { sdk.registerFunction("mem::consolidate-pipeline", () => ({ success: true, })); + sdk.registerFunction("mem::auto-crystallize", () => ({ success: true })); const result = (await sdk.trigger({ function_id: "mem::evict", @@ -157,6 +167,53 @@ describe("mem::evict stale sessions", () => { ); }); + it("bounds consolidation to one pass regardless of how many stale sessions are recovered", async () => { + // Regression (P1): before the skipConsolidation guard, N recovered + // sessions each triggered a forced full-corpus consolidate + crystallize + // via the session::stopped fan-out, on top of evict's final pass — an + // N+1 amplification of an expensive LLM path. Recovery must stay O(1). + const ids = ["ses_a", "ses_b", "ses_c"]; + const store: Store = new Map([ + [ + KV.sessions, + new Map(ids.map((id) => [id, makeSession(id)])), + ], + [KV.summaries, new Map()], + [KV.config, new Map()], + [KV.audit, new Map()], + ]); + for (const id of ids) { + store.set( + KV.observations(id), + new Map([["obs_1", makeObservation(id)]]), + ); + } + const kv = mockKV(store); + const { sdk, calls } = mockSdk(); + + registerEvictFunction(sdk as never, kv as never); + const stoppedPayloads: unknown[] = []; + sdk.registerFunction("event::session::stopped", (payload) => { + stoppedPayloads.push(payload); + return { success: true }; + }); + sdk.registerFunction("mem::consolidate-pipeline", () => ({ success: true })); + sdk.registerFunction("mem::auto-crystallize", () => ({ success: true })); + + await sdk.trigger({ function_id: "mem::evict", payload: {} }); + + // session::stopped fires once per recovered session, each suppressing its + // own fan-out... + expect(stoppedPayloads).toHaveLength(3); + for (const p of stoppedPayloads) { + expect(p).toMatchObject({ skipConsolidation: true }); + } + // ...and the corpus-wide consolidation + crystallization run exactly once. + const fnIds = calls.map((c) => c.function_id); + expect(fnIds.filter((f) => f === "mem::consolidate-pipeline")).toHaveLength(1); + expect(fnIds.filter((f) => f === "mem::auto-crystallize")).toHaveLength(1); + }); + it("keeps a stale observed session when recovery fails", async () => { const sessionId = "ses_unrecovered"; const store = storeForObservedSession(sessionId); diff --git a/test/export-import.test.ts b/test/export-import.test.ts index 39c17cab88..a345ca3d5c 100644 --- a/test/export-import.test.ts +++ b/test/export-import.test.ts @@ -5,6 +5,7 @@ vi.mock("../src/logger.js", () => ({ })); import { registerExportImportFunction } from "../src/functions/export-import.js"; +import { getSearchIndex } from "../src/functions/search.js"; import type { Session, CompressedObservation, @@ -108,6 +109,10 @@ describe("Export/Import Functions", () => { beforeEach(async () => { sdk = mockSdk(); kv = mockKV(); + // getSearchIndex() returns a module-level singleton shared across + // tests. Clear it so index assertions here don't see rows added by + // a prior test's import. + getSearchIndex().clear(); registerExportImportFunction(sdk as never, kv as never); await kv.set("mem:sessions", "ses_1", testSession); @@ -151,6 +156,59 @@ describe("Export/Import Functions", () => { expect(allSessions.length).toBe(2); }); + it("import adds imported records to the search index", async () => { + // Regression: mem::import wrote rows to KV but never indexed them. + // On an existing install the boot rebuild gate (bm25.size === 0) is + // false, so imported data stayed invisible to mem::search forever. + const importedObs: CompressedObservation = { + id: "obs_imported", + sessionId: "ses_imported", + timestamp: "2026-03-01T10:00:00Z", + type: "file_edit", + title: "Kubernetes deployment rollout", + facts: ["Scaled replicas"], + narrative: "Adjusted the kubernetes deployment rollout strategy", + concepts: ["k8s"], + files: ["deploy.yaml"], + importance: 6, + }; + const importedMem: Memory = { + ...testMemory, + id: "mem_imported", + title: "Postgres connection pooling", + content: "Use pgbouncer for postgres connection pooling", + }; + const exportData: ExportData = { + version: "0.9.28", + exportedAt: new Date().toISOString(), + sessions: [ + { ...testSession, id: "ses_imported", observationCount: 1 }, + ], + observations: { ses_imported: [importedObs] }, + memories: [importedMem], + summaries: [], + }; + + const result = (await sdk.trigger("mem::import", { + exportData, + strategy: "merge", + })) as { success: boolean; observations: number; memories: number }; + + expect(result.success).toBe(true); + expect(result.observations).toBe(1); + expect(result.memories).toBe(1); + + const idx = getSearchIndex(); + expect(idx.has("obs_imported")).toBe(true); + expect(idx.has("mem_imported")).toBe(true); + + const obsHit = idx.search("kubernetes rollout"); + expect(obsHit.some((r) => r.obsId === "obs_imported")).toBe(true); + + const memHit = idx.search("postgres pooling"); + expect(memHit.some((r) => r.obsId === "mem_imported")).toBe(true); + }); + it("import with skip strategy does not overwrite existing", async () => { const exportData: ExportData = { version: "0.3.0", diff --git a/test/fetch-timeout.test.ts b/test/fetch-timeout.test.ts index 5b2cd7c9c5..03bf9580f3 100644 --- a/test/fetch-timeout.test.ts +++ b/test/fetch-timeout.test.ts @@ -74,6 +74,241 @@ describe("fetchWithTimeout", () => { }); }); +// ───────────────────────────────────────────────────────────── +// Bounded-retry total-deadline tests +// +// The retry wrapper must bound TOTAL elapsed time across every +// attempt + sleep — not per-attempt — so worst case never blows +// past the caller's budget or the iii 180s invocation timeout. +// ───────────────────────────────────────────────────────────── +describe("fetchWithTimeout bounded retry (total deadline)", () => { + // Builds a fetch mock that replays a queue of {status, headers} in order, + // repeating the last entry once the queue is exhausted. Records how many + // times it was invoked so we can assert attempt counts. + function queuedFetch( + responses: Array<{ status: number; headers?: Record }>, + ): { fetch: typeof fetch; calls: () => number } { + let i = 0; + const impl = (async () => { + const spec = responses[Math.min(i, responses.length - 1)]; + i++; + return new Response(null, { + status: spec.status, + headers: spec.headers, + }); + }) as typeof fetch; + return { fetch: impl, calls: () => i }; + } + + afterEach(() => { + vi.useRealTimers(); + vi.restoreAllMocks(); + delete process.env["AGENTMEMORY_LLM_TIMEOUT_MS"]; + }); + + // 0. The FIRST attempt must honor the hard budget cap, not the raw caller + // timeout. Regression: fetchOnce was called with `ms` instead of the + // capped `budgetMs`, so a large caller timeout (e.g. 300s) could hang the + // initial request past HARD_BUDGET_CAP_MS (170s) and the iii 180s + // invocation timeout before any retry logic ran. + it("caps the first attempt at the hard budget, not the raw caller timeout", async () => { + const signals: AbortSignal[] = []; + const capturing = ((_url: string, init?: RequestInit) => { + const signal = init!.signal as AbortSignal; + signals.push(signal); + return new Promise((_resolve, reject) => { + signal.addEventListener("abort", () => + reject(new DOMException("AbortError", "AbortError")), + ); + }); + }) as typeof fetch; + vi.spyOn(globalThis, "fetch").mockImplementation(capturing); + vi.useFakeTimers(); + + // 300s caller timeout — far above the 170s HARD_BUDGET_CAP_MS. + const p = fetchWithTimeout("https://example.com", {}, 300000); + p.catch(() => {}); // observe rejection so it isn't flagged unhandled + + expect(signals).toHaveLength(1); + // Just before the cap the first attempt is still alive. + await vi.advanceTimersByTimeAsync(169999); + expect(signals[0].aborted).toBe(false); + // At the cap it must abort. Pre-fix (raw 300s) it would still be alive. + await vi.advanceTimersByTimeAsync(2); + expect(signals[0].aborted).toBe(true); + + await expect(p).rejects.toThrow(); + }); + + // 1. 429 then 200 → retried exactly once, resolves 200. + it("retries once on 429 then resolves the follow-up 200", async () => { + const q = queuedFetch([{ status: 429 }, { status: 200 }]); + vi.spyOn(globalThis, "fetch").mockImplementation(q.fetch); + vi.useFakeTimers(); + + const p = fetchWithTimeout("https://example.com", {}, 60000); + // Drain the backoff sleep (500ms default) so the retry fires. + await vi.advanceTimersByTimeAsync(500); + const res = await p; + + expect(res.status).toBe(200); + expect(q.calls()).toBe(2); + }); + + // 2. 429 with a hostile Retry-After (100000s) must NOT wait that long. The + // delay collapses to the low per-delay cap (5000ms) so the retry fires + // quickly and total elapsed stays nowhere near 100000s. + it("does not honour a hostile Retry-After literally — caps it and stays within budget", async () => { + const q = queuedFetch([ + { status: 429, headers: { "Retry-After": "100000" } }, // 100000s + { status: 200 }, + ]); + vi.spyOn(globalThis, "fetch").mockImplementation(q.fetch); + vi.useFakeTimers(); + const start = Date.now(); + + const p = fetchWithTimeout("https://example.com", {}, 60000); + // runAllTimers drains every scheduled sleep; if the code honored 100000s + // literally this would advance 100_000_000ms of simulated time. + await vi.runAllTimersAsync(); + const res = await p; + const elapsed = Date.now() - start; + + expect(res.status).toBe(200); + expect(q.calls()).toBe(2); + // The only sleep was the capped 5000ms delay — total elapsed is bounded to + // the cap, nowhere near the 100_000_000ms the header requested. + expect(elapsed).toBeLessThanOrEqual(5000); + expect(elapsed).toBeLessThan(60000); + }); + + // 2c. When the capped delay still cannot fit inside a small budget, we do NOT + // retry and return the last 429 within bound. + it("returns the last 429 without retrying when even the capped delay overruns a small budget", async () => { + const q = queuedFetch([ + { status: 429, headers: { "Retry-After": "100000" } }, + { status: 200 }, + ]); + vi.spyOn(globalThis, "fetch").mockImplementation(q.fetch); + vi.useFakeTimers(); + const start = Date.now(); + + // Budget 1000ms: capped delay 5000ms + floor 100ms > remaining, no retry. + const p = fetchWithTimeout("https://example.com", {}, 1000); + await vi.runAllTimersAsync(); + const res = await p; + const elapsed = Date.now() - start; + + expect(res.status).toBe(429); + expect(q.calls()).toBe(1); + expect(elapsed).toBeLessThan(1000); + }); + + // 2b. Retry-After present but LARGER than the low per-delay cap collapses to + // MAX_RETRY_DELAY_MS (5000), still bounded, still retries when budget + // allows. + it("caps an oversized Retry-After to the max delay and still retries within budget", async () => { + const q = queuedFetch([ + { status: 503, headers: { "Retry-After": "60" } }, // 60s requested + { status: 200 }, + ]); + vi.spyOn(globalThis, "fetch").mockImplementation(q.fetch); + vi.useFakeTimers(); + + const p = fetchWithTimeout("https://example.com", {}, 60000); + // Requested 60s, but the honored delay is capped at 5000ms. + await vi.advanceTimersByTimeAsync(5000); + const res = await p; + + expect(res.status).toBe(200); + expect(q.calls()).toBe(2); + }); + + // 3. Persistent 503 → stops after the bounded attempts AND within the total + // deadline. Attempt count is bounded at MAX_ATTEMPTS (3) and total + // simulated elapsed stays under the budget. + it("stops persistent 503 at the attempt cap and within the deadline", async () => { + const q = queuedFetch([{ status: 503 }]); + vi.spyOn(globalThis, "fetch").mockImplementation(q.fetch); + vi.useFakeTimers(); + const start = Date.now(); + + const p = fetchWithTimeout("https://example.com", {}, 60000); + // Two retries with 500ms + 1000ms exponential backoff. + await vi.advanceTimersByTimeAsync(500); + await vi.advanceTimersByTimeAsync(1000); + const res = await p; + const elapsed = Date.now() - start; + + expect(res.status).toBe(503); + // Initial attempt + 2 retries = MAX_ATTEMPTS. + expect(q.calls()).toBe(3); + expect(elapsed).toBeLessThan(60000); + // Total simulated sleep was only the two backoffs. + expect(elapsed).toBeLessThanOrEqual(1500); + }); + + // 3b. A tiny budget forces us to bail before even the first retry sleep — + // the sleep + minimal attempt floor already overruns the remaining + // budget, so we return the first 503 without retrying. + it("does not retry when the budget is too small to fit a retry", async () => { + const q = queuedFetch([{ status: 503 }, { status: 200 }]); + vi.spyOn(globalThis, "fetch").mockImplementation(q.fetch); + vi.useFakeTimers(); + + // Budget of 200ms: backoff 500ms + floor 100ms > remaining, so no retry. + const p = fetchWithTimeout("https://example.com", {}, 200); + await vi.runAllTimersAsync(); + const res = await p; + + expect(res.status).toBe(503); + expect(q.calls()).toBe(1); + }); + + // 5. Retry-After as an HTTP-date is parsed, capped to the max delay, and the + // retry fires within budget. + it("parses an HTTP-date Retry-After and caps the honored delay", async () => { + // 2s in the future — under the max delay cap, so honored as-is. + const future = new Date(Date.now() + 2000).toUTCString(); + const q = queuedFetch([ + { status: 429, headers: { "Retry-After": future } }, + { status: 200 }, + ]); + vi.spyOn(globalThis, "fetch").mockImplementation(q.fetch); + vi.useFakeTimers(); + + const p = fetchWithTimeout("https://example.com", {}, 60000); + await vi.advanceTimersByTimeAsync(2000); + const res = await p; + + expect(res.status).toBe(200); + expect(q.calls()).toBe(2); + }); + + // 5b. A far-future HTTP-date collapses to the capped delay (5000ms), which + // still exceeds a small budget → no retry, last 503 returned in bound. + it("does not retry on a far-future HTTP-date Retry-After that overruns a small budget", async () => { + const farFuture = new Date(Date.now() + 3600_000).toUTCString(); // 1h out + const q = queuedFetch([ + { status: 503, headers: { "Retry-After": farFuture } }, + { status: 200 }, + ]); + vi.spyOn(globalThis, "fetch").mockImplementation(q.fetch); + vi.useFakeTimers(); + const start = Date.now(); + + // Budget 1000ms: capped delay 5000ms + floor > remaining, so no retry. + const p = fetchWithTimeout("https://example.com", {}, 1000); + await vi.runAllTimersAsync(); + const res = await p; + const elapsed = Date.now() - start; + + expect(res.status).toBe(503); + expect(q.calls()).toBe(1); + expect(elapsed).toBeLessThan(1000); + }); +}); + // ───────────────────────────────────────────────────────────── // Provider hang regression tests // Each provider must call fetchWithTimeout, which honours the diff --git a/test/fs-watcher.test.ts b/test/fs-watcher.test.ts index 48c1b09484..8375c6864b 100644 --- a/test/fs-watcher.test.ts +++ b/test/fs-watcher.test.ts @@ -98,6 +98,10 @@ describe("FilesystemWatcher", { retry: 2 }, () => { }); it("throws if no watched roots could be attached", () => { + // Regression: on Linux with Node 24+, fs.watch on a nonexistent path no + // longer throws synchronously, so the watcher must stat roots itself or a + // missing root silently counts as attached (caught by the Node 24/26 CI + // matrix rows). const w = new FilesystemWatcher({ roots: ["/definitely/does/not/exist/xyz123"], baseUrl: "http://localhost:3111", @@ -106,6 +110,17 @@ describe("FilesystemWatcher", { retry: 2 }, () => { expect(() => w.start()).toThrow(/could not watch any of the configured roots/); }); + it("rejects a root that is a file, not a directory", () => { + const filePath = join(root, "not-a-dir.txt"); + writeFileSync(filePath, "plain file"); + const w = new FilesystemWatcher({ + roots: [filePath], + baseUrl: "http://localhost:3111", + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + }); + expect(() => w.start()).toThrow(/could not watch any of the configured roots/); + }); + it("ignores paths that match the default ignore set", async () => { mkdirSync(join(root, "node_modules"), { recursive: true }); const w = new FilesystemWatcher({ diff --git a/test/graph-import.test.ts b/test/graph-import.test.ts new file mode 100644 index 0000000000..e342cc8ab5 --- /dev/null +++ b/test/graph-import.test.ts @@ -0,0 +1,218 @@ +import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"; +import { mkdtempSync, rmSync, writeFileSync, mkdirSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +vi.mock("../src/logger.js", () => ({ + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + bootLog: vi.fn(), +})); + +import { parseGraphifyGraph, registerGraphImportFunction } from "../src/functions/graph-import.js"; +import { KV } from "../src/state/schema.js"; +import type { GraphNode, GraphEdge } from "../src/types.js"; + +// graphify's clustered graph.json is NetworkX node_link: nodes carry +// label/source_file/community/file_type, links carry +// source/target/relation/confidence. --no-cluster output stores the edge +// array under "edges" instead of "links". +const FIXTURE = { + nodes: [ + { id: "n1", label: "extract", source_file: "extract.py", community: 0, file_type: "code" }, + { id: "n2", label: "cluster.py", source_file: "cluster.py", community: 0, file_type: "code" }, + { id: "n3", label: "retry rationale", community: 1, file_type: "rationale" }, + { id: "n4", label: "architecture overview", file_type: "document" }, + ], + links: [ + { source: "n1", target: "n2", relation: "imports", confidence: "EXTRACTED" }, + { source: "n1", target: "n3", relation: "references", confidence: "INFERRED" }, + { source: "n2", target: "zz-missing", relation: "calls", confidence: "EXTRACTED" }, + ], +}; + +function mockKV() { + const store = new Map>(); + return { + get: async (scope: string, key: string): Promise => + (store.get(scope)?.get(key) as T) ?? null, + set: async (scope: string, key: string, data: T): Promise => { + if (!store.has(scope)) store.set(scope, new Map()); + store.get(scope)!.set(key, data); + return data; + }, + delete: async (scope: string, key: string): Promise => { + store.get(scope)?.delete(key); + }, + list: async (scope: string): Promise => + Array.from(store.get(scope)?.values() ?? []) as T[], + _store: store, + }; +} + +function mockSdk() { + const fns = new Map(); + return { + registerFunction: (id: string, handler: Function) => fns.set(id, handler), + registerTrigger: () => {}, + trigger: async (input: { function_id: string; payload?: unknown }) => { + const fn = fns.get(input.function_id); + if (!fn) throw new Error(`missing handler: ${input.function_id}`); + return fn(input.payload); + }, + } as never; +} + +describe("parseGraphifyGraph", () => { + it("maps nodes with file_type-aware types and provenance", () => { + const parsed = parseGraphifyGraph(JSON.stringify(FIXTURE)); + expect(parsed.nodesRead).toBe(4); + const byName = new Map(parsed.nodes.map((n) => [n.name, n])); + // code symbol without extension -> function; with extension -> file + expect(byName.get("extract")!.type).toBe("function"); + expect(byName.get("cluster.py")!.type).toBe("file"); + // rationale -> decision, document -> concept + expect(byName.get("retry rationale")!.type).toBe("decision"); + expect(byName.get("architecture overview")!.type).toBe("concept"); + // provenance kept on every imported node + for (const n of parsed.nodes) { + expect(n.properties.source).toBe("graphify"); + } + expect(byName.get("extract")!.properties.sourceFile).toBe("extract.py"); + }); + + it("maps relations to memory edge types and confidence to weight", () => { + const parsed = parseGraphifyGraph(JSON.stringify(FIXTURE)); + // the edge to a missing endpoint is skipped and counted, never guessed + expect(parsed.edges).toHaveLength(2); + expect(parsed.skippedEdges).toBe(1); + const types = parsed.edges.map((e) => e.type).sort(); + expect(types).toEqual(["imports", "related_to"]); + const weights = parsed.edges.map((e) => e.weight).sort((a, b) => a - b); + expect(weights).toEqual([0.6, 0.9]); + }); + + it("maps AMBIGUOUS confidence and unknown file_type defaults", () => { + const fixture = { + nodes: [ + { id: "a", label: "mystery" }, + { id: "b", label: "helper.rs", file_type: "wat" }, + ], + links: [{ source: "a", target: "b", relation: "calls", confidence: "AMBIGUOUS" }], + }; + const parsed = parseGraphifyGraph(JSON.stringify(fixture)); + const byName = new Map(parsed.nodes.map((n) => [n.name, n])); + // no/unknown file_type: extension-looking labels are files, rest concepts + expect(byName.get("mystery")!.type).toBe("concept"); + expect(byName.get("helper.rs")!.type).toBe("file"); + expect(parsed.edges[0].weight).toBe(0.3); + }); + + it("accepts the --no-cluster shape where edges live under `edges`", () => { + const noCluster = { nodes: FIXTURE.nodes, edges: FIXTURE.links }; + const parsed = parseGraphifyGraph(JSON.stringify(noCluster)); + expect(parsed.edges).toHaveLength(2); + expect(parsed.edgesRead).toBe(3); + }); + + it("reports truncation loudly instead of silently capping", () => { + const big = { + nodes: Array.from({ length: 5010 }, (_, i) => ({ id: `n${i}`, label: `sym${i}` })), + links: [], + }; + const parsed = parseGraphifyGraph(JSON.stringify(big)); + expect(parsed.nodes).toHaveLength(5000); + expect(parsed.truncated).toEqual({ nodes: 10, edges: 0 }); + }); +}); + +describe("mem::graph::import-graphify", () => { + let tmp: string; + let kv: ReturnType; + let sdk: ReturnType; + + beforeEach(() => { + tmp = mkdtempSync(join(tmpdir(), "am-graphify-")); + mkdirSync(join(tmp, "graphify-out"), { recursive: true }); + writeFileSync(join(tmp, "graphify-out", "graph.json"), JSON.stringify(FIXTURE)); + kv = mockKV(); + sdk = mockSdk(); + registerGraphImportFunction(sdk, kv as never); + }); + + afterEach(() => { + rmSync(tmp, { recursive: true, force: true }); + }); + + it("imports nodes and edges into the memory graph", async () => { + const result = (await (sdk as any).trigger({ + function_id: "mem::graph::import-graphify", + payload: { cwd: tmp }, + })) as { success: boolean; newNodes: number; newEdges: number; skippedEdges: number }; + + expect(result.success).toBe(true); + expect(result.newNodes).toBe(4); + expect(result.newEdges).toBe(2); + expect(result.skippedEdges).toBe(1); + + const nodes = await kv.list(KV.graphNodes); + const edges = await kv.list(KV.graphEdges); + expect(nodes).toHaveLength(4); + expect(edges).toHaveLength(2); + }); + + it("re-import is idempotent: second run merges instead of duplicating", async () => { + await (sdk as any).trigger({ function_id: "mem::graph::import-graphify", payload: { cwd: tmp } }); + const second = (await (sdk as any).trigger({ + function_id: "mem::graph::import-graphify", + payload: { cwd: tmp }, + })) as { success: boolean; newNodes: number; newEdges: number }; + + expect(second.success).toBe(true); + // Everything resolves through the (type, name) index and merges. + expect(second.newNodes).toBe(0); + expect(second.newEdges).toBe(0); + expect(await kv.list(KV.graphNodes)).toHaveLength(4); + expect(await kv.list(KV.graphEdges)).toHaveLength(2); + + // A merge-only run mutates cached snapshot entries even with zero new + // counts; the persisted snapshot must still reflect current graph data. + const snap = await kv.get<{ + stats: { totalNodes: number; totalEdges: number }; + topNodes: unknown[]; + }>("mem:graph:snapshot", "current"); + expect(snap).not.toBeNull(); + expect(snap!.stats.totalNodes).toBe(4); + expect(snap!.stats.totalEdges).toBe(2); + expect(snap!.topNodes.length).toBeGreaterThan(0); + }); + + it("fails cleanly with a pointer when graph.json is absent", async () => { + const result = (await (sdk as any).trigger({ + function_id: "mem::graph::import-graphify", + payload: { cwd: join(tmp, "nowhere") }, + })) as { success: boolean; error: string }; + expect(result.success).toBe(false); + expect(result.error).toContain("Run graphify first"); + }); + + it("accepts an explicit path", async () => { + const alt = join(tmp, "custom.json"); + writeFileSync(alt, JSON.stringify(FIXTURE)); + const result = (await (sdk as any).trigger({ + function_id: "mem::graph::import-graphify", + payload: { path: alt }, + })) as { success: boolean; nodesImported: number }; + expect(result.success).toBe(true); + expect(result.nodesImported).toBe(4); + }); + + it("rejects malformed JSON without writing anything", async () => { + writeFileSync(join(tmp, "graphify-out", "graph.json"), "{not json"); + const result = (await (sdk as any).trigger({ + function_id: "mem::graph::import-graphify", + payload: { cwd: tmp }, + })) as { success: boolean }; + expect(result.success).toBe(false); + expect(await kv.list(KV.graphNodes)).toHaveLength(0); + }); +}); diff --git a/test/remember-project-scope.test.ts b/test/remember-project-scope.test.ts index bda699f5ae..31bbf43e1a 100644 --- a/test/remember-project-scope.test.ts +++ b/test/remember-project-scope.test.ts @@ -256,3 +256,81 @@ describe("mem::remember — cross-project dedup isolation", () => { expect(unscoped.memory.supersedes).toContain(scoped.memory.id); }); }); + +describe("mem::remember — CJK dedup", () => { + beforeEach(() => { + getSearchIndex().clear(); + setIndexPersistence(null); + }); + + afterEach(() => { + setIndexPersistence(null); + }); + + it("dedups two near-identical CJK memories (new one supersedes old)", async () => { + const sdk = mockSdk(); + const kv = mockKV(); + registerRememberFunction(sdk as never, kv as never); + + const first = await sdk.trigger({ + function_id: "mem::remember", + payload: { + content: "用户认证中间件必须先去除请求头里的 Bearer 前缀然后再校验令牌", + type: "pattern", + }, + }) as { memory: { id: string } }; + + const second = await sdk.trigger({ + function_id: "mem::remember", + payload: { + content: "用户认证中间件必须先去除请求头里的 Bearer 前缀然后校验令牌", + type: "pattern", + }, + }) as { memory: { supersedes: string[] } }; + + expect(second.memory.supersedes).toContain(first.memory.id); + + const original = await kv.get<{ isLatest: boolean }>("mem:memories", first.memory.id); + expect(original?.isLatest).toBe(false); + }); + + it("does NOT supersede two unrelated short CJK memories (北京 vs 上海)", async () => { + const sdk = mockSdk(); + const kv = mockKV(); + registerRememberFunction(sdk as never, kv as never); + + const beijing = await sdk.trigger({ + function_id: "mem::remember", + payload: { content: "北京", type: "fact" }, + }) as { memory: { id: string } }; + + const shanghai = await sdk.trigger({ + function_id: "mem::remember", + payload: { content: "上海", type: "fact" }, + }) as { memory: { supersedes: string[] } }; + + // The old empty-set shortcut returned similarity 1 here and falsely + // chained "上海" as a new version of "北京". + expect(shanghai.memory.supersedes).toHaveLength(0); + + const original = await kv.get<{ isLatest: boolean }>("mem:memories", beijing.memory.id); + expect(original?.isLatest).toBe(true); + }); + + it("preserves a trailing astral character in the title (no lone surrogate)", async () => { + const sdk = mockSdk(); + const kv = mockKV(); + registerRememberFunction(sdk as never, kv as never); + + // 80th UTF-16 code unit falls inside a surrogate pair; the title must + // not end on a lone high surrogate. + const content = "x".repeat(79) + "😀 trailing"; + const result = await sdk.trigger({ + function_id: "mem::remember", + payload: { content, type: "fact" }, + }) as { memory: { title: string } }; + + expect(/[\uD800-\uDBFF]$/.test(result.memory.title)).toBe(false); + expect(result.memory.title.length).toBe(79); + }); +}); diff --git a/test/replay-import-key.test.ts b/test/replay-import-key.test.ts index 9a4c64675c..1c1c991f3e 100644 --- a/test/replay-import-key.test.ts +++ b/test/replay-import-key.test.ts @@ -1,4 +1,4 @@ -import { describe, it, expect, beforeEach, vi } from "vitest"; +import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"; import { mkdtempSync, writeFileSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -9,6 +9,13 @@ vi.mock("../src/logger.js", () => ({ import { registerReplayFunctions } from "../src/functions/replay.js"; import { KV } from "../src/state/schema.js"; +import { + getSearchIndex, + setVectorIndex, + setEmbeddingProvider, +} from "../src/functions/search.js"; +import { VectorIndex } from "../src/state/vector-index.js"; +import type { EmbeddingProvider } from "../src/types.js"; function mockKV() { const store = new Map>(); @@ -152,3 +159,90 @@ describe("import-jsonl re-key on parsed.sessionId (#775)", () => { expect(sessionWrites.length).toBe(1); }); }); + +describe("import-jsonl indexes observations into BM25 AND vector", () => { + const mockEmbedder: EmbeddingProvider = { + name: "test", + dimensions: 3, + embed: async () => new Float32Array([0.1, 0.2, 0.3]), + embedBatch: async (texts: string[]) => + texts.map(() => new Float32Array([0.1, 0.2, 0.3])), + }; + let tmpRoot: string; + let vectorIndex: VectorIndex; + + beforeEach(() => { + tmpRoot = mkdtempSync(join(tmpdir(), "replay-import-index-")); + getSearchIndex().clear(); + vectorIndex = new VectorIndex(); + setVectorIndex(vectorIndex); + setEmbeddingProvider(mockEmbedder); + }); + + afterEach(() => { + setVectorIndex(null); + setEmbeddingProvider(null); + getSearchIndex().clear(); + }); + + function writeFixture(sessionId: string) { + const dir = join(tmpRoot, "proj"); + require("node:fs").mkdirSync(dir, { recursive: true }); + const ts = "2026-04-17T10:00:00.000Z"; + const lines = [ + JSON.stringify({ + type: "user", + uuid: "u1", + sessionId, + timestamp: ts, + cwd: tmpRoot, + message: { role: "user", content: [{ type: "text", text: "how do I fix the flaky retry" }] }, + }), + JSON.stringify({ + type: "assistant", + uuid: "a1", + sessionId, + timestamp: ts, + message: { role: "assistant", content: [{ type: "text", text: "await the fetch before asserting" }] }, + }), + ]; + writeFileSync(join(dir, `${sessionId}.jsonl`), lines.join("\n") + "\n"); + } + + it("populates the vector index (regression: replay used to BM25-add only, leaving imports unsearchable by meaning)", async () => { + writeFixture("sess-index"); + const kv = mockKV(); + const sdk = mockSdk(kv); + registerReplayFunctions(sdk, kv as never); + + expect(vectorIndex.size).toBe(0); + expect(getSearchIndex().size).toBe(0); + + const result = (await sdk.trigger("mem::replay::import-jsonl", { + path: tmpRoot, + })) as { success: boolean; imported?: number }; + + expect(result.success).toBe(true); + // Both lanes must be populated. The old code left vectorIndex at 0. + expect(vectorIndex.size).toBeGreaterThan(0); + expect(getSearchIndex().size).toBeGreaterThan(0); + expect(vectorIndex.size).toBe(getSearchIndex().size); + }); + + it("skips the vector lane cleanly when no embedding provider is configured (keyless install)", async () => { + setVectorIndex(null); + setEmbeddingProvider(null); + writeFixture("sess-keyless"); + const kv = mockKV(); + const sdk = mockSdk(kv); + registerReplayFunctions(sdk, kv as never); + + const result = (await sdk.trigger("mem::replay::import-jsonl", { + path: tmpRoot, + })) as { success: boolean }; + + // BM25 still works; no crash from the absent vector index. + expect(result.success).toBe(true); + expect(getSearchIndex().size).toBeGreaterThan(0); + }); +}); diff --git a/test/schema.test.ts b/test/schema.test.ts index 2d3f65d290..94941c4266 100644 --- a/test/schema.test.ts +++ b/test/schema.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect } from 'vitest' -import { KV, STREAM, generateId } from '../src/state/schema.js' +import { KV, STREAM, generateId, jaccardSimilarity } from '../src/state/schema.js' describe('KV', () => { it('has correct session scope', () => { @@ -40,3 +40,85 @@ describe('generateId', () => { expect(id.length).toBeGreaterThan(15) }) }) + +describe('jaccardSimilarity', () => { + it('returns 1 for identical ASCII strings', () => { + const s = 'always use express-jwt middleware for token validation' + expect(jaccardSimilarity(s, s)).toBe(1) + }) + + it('keeps ASCII word-level behavior', () => { + const a = 'always use express-jwt middleware for token validation' + const b = 'always use express-jwt middleware for request validation' + const score = jaccardSimilarity(a, b) + expect(score).toBeGreaterThan(0.5) + expect(score).toBeLessThan(1) + }) + + it('returns 0 for unrelated ASCII strings', () => { + expect( + jaccardSimilarity('the quick brown fox', 'lorem ipsum dolor sit'), + ).toBe(0) + }) + + it('returns 0 (never 1) when both token sets are empty and inputs differ', () => { + // Two short ASCII strings whose tokens are all filtered out by the + // length gate must not be treated as identical. + expect(jaccardSimilarity('a b', 'x y')).toBe(0) + }) + + it('supersedes identical short memories that tokenize to nothing', () => { + // Regression: words <=2 chars are dropped, so "AI" / "go" / "a b" + // produce empty token sets. Re-saving the exact same short memory must + // still be detected as a duplicate (score 1) via an exact-equality + // fallback, instead of leaking duplicate latest records. + expect(jaccardSimilarity('AI', 'AI')).toBe(1) + expect(jaccardSimilarity('go', 'go')).toBe(1) + // Unrelated short strings must still score 0, not falsely supersede. + expect(jaccardSimilarity('AI', 'ML')).toBe(0) + expect(jaccardSimilarity('go', 'AI')).toBe(0) + }) + + it('treats whitespace-only differences in short text as identical', () => { + // The exact-equality fallback collapses runs of whitespace and trims, + // so cosmetic spacing differences on an otherwise-empty-token memory + // still dedupe. + expect(jaccardSimilarity('a b', 'a b')).toBe(1) + expect(jaccardSimilarity(' AI ', 'AI')).toBe(1) + }) + + it('gives high similarity for near-identical CJK sentences', () => { + const a = '用户认证中间件必须先去除请求头里的 Bearer 前缀然后再校验令牌' + const b = '用户认证中间件必须先去除请求头里的 Bearer 前缀然后校验令牌' + expect(jaccardSimilarity(a, b)).toBeGreaterThan(0.7) + }) + + it('gives low similarity for unrelated short CJK strings', () => { + // "北京" vs "上海" — the old empty-set shortcut returned 1 here and + // falsely superseded an unrelated memory. Must be well below the + // 0.7 supersede threshold. + expect(jaccardSimilarity('北京', '上海')).toBeLessThan(0.7) + expect(jaccardSimilarity('北京', '上海')).toBe(0) + }) + + it('detects a duplicate for identical CJK strings', () => { + expect(jaccardSimilarity('设置认证中间件', '设置认证中间件')).toBe(1) + }) + + it('handles Japanese kana without whitespace', () => { + const a = 'トークンを検証する前に接頭辞を取り除く' + const b = 'トークンを検証する前に接頭辞を削除する' + expect(jaccardSimilarity(a, b)).toBeGreaterThan(0.4) + expect(jaccardSimilarity('東京', '大阪')).toBe(0) + }) + + it('NFC-normalizes before comparing', () => { + // Composed U+00E9 vs decomposed 'e' + U+0301 combining accent for + // "caf\u00e9" must compare equal even though the two strings differ + // byte-for-byte before normalization. + const composed = 'caf\u00e9 latte order' + const decomposed = 'cafe\u0301 latte order' + expect(composed).not.toBe(decomposed) + expect(jaccardSimilarity(composed, decomposed)).toBe(1) + }) +}) diff --git a/test/search.test.ts b/test/search.test.ts index 7bc635ce52..84b8134e3a 100644 --- a/test/search.test.ts +++ b/test/search.test.ts @@ -101,6 +101,10 @@ describe("mem::search", () => { // Module-level SearchIndex singleton would leak across tests; reset. getSearchIndex().clear(); + // mem::search awaits a shared rebuild on a cold index; the explicit call + // here pre-populates the index deterministically so the query assertions + // below never depend on that cold-start path. + await rebuildIndex(kv as never); }); it("returns full format by default", async () => { diff --git a/test/snapshot-config.test.ts b/test/snapshot-config.test.ts new file mode 100644 index 0000000000..ee9949d63c --- /dev/null +++ b/test/snapshot-config.test.ts @@ -0,0 +1,96 @@ +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { loadSnapshotConfig, __resetEnvFileCache } from "../src/config.js"; + +// loadSnapshotConfig reads getMergedEnv(), which merges the on-disk +// ~/.agentmemory/.env under process.env. The tests below delete the +// process.env keys, so a developer machine with SNAPSHOT_* in its real .env +// would leak into the defaults; point HOME at an empty temp dir and reset +// the env-file cache so the file layer is deterministic. +// +// Regression (P1): a zero/negative SNAPSHOT_INTERVAL flowed straight into +// setInterval(fn, interval * 1000). Node clamps a non-positive delay to ~1ms, +// so the git-snapshot timer would fire on nearly every event-loop tick, +// saturating the worker with back-to-back full-state snapshots + commits. +// Non-positive values must fall back to the documented 3600s default. + +const KEYS = ["SNAPSHOT_INTERVAL", "SNAPSHOT_ENABLED", "SNAPSHOT_DIR"] as const; +const DEFAULT_INTERVAL = 3600; + +describe("loadSnapshotConfig interval validation", () => { + const saved: Record = {}; + let sandboxHome: string; + let savedHome: string | undefined; + let savedUserProfile: string | undefined; + + beforeEach(() => { + sandboxHome = mkdtempSync(join(tmpdir(), "am-snapcfg-")); + savedHome = process.env["HOME"]; + savedUserProfile = process.env["USERPROFILE"]; + process.env["HOME"] = sandboxHome; + process.env["USERPROFILE"] = sandboxHome; + __resetEnvFileCache(); + for (const k of KEYS) { + saved[k] = process.env[k]; + delete process.env[k]; + } + }); + + afterEach(() => { + for (const k of KEYS) { + if (saved[k] === undefined) delete process.env[k]; + else process.env[k] = saved[k]; + } + if (savedHome === undefined) delete process.env["HOME"]; + else process.env["HOME"] = savedHome; + if (savedUserProfile === undefined) delete process.env["USERPROFILE"]; + else process.env["USERPROFILE"] = savedUserProfile; + __resetEnvFileCache(); + rmSync(sandboxHome, { recursive: true, force: true }); + }); + + it("falls back to the default when interval is zero", () => { + process.env["SNAPSHOT_INTERVAL"] = "0"; + expect(loadSnapshotConfig().interval).toBe(DEFAULT_INTERVAL); + }); + + it("falls back to the default when interval is negative", () => { + process.env["SNAPSHOT_INTERVAL"] = "-5"; + expect(loadSnapshotConfig().interval).toBe(DEFAULT_INTERVAL); + }); + + it("falls back to the default when interval is non-numeric", () => { + process.env["SNAPSHOT_INTERVAL"] = "not-a-number"; + expect(loadSnapshotConfig().interval).toBe(DEFAULT_INTERVAL); + }); + + it("uses the default when interval is unset", () => { + expect(loadSnapshotConfig().interval).toBe(DEFAULT_INTERVAL); + }); + + it("accepts a valid positive interval unchanged", () => { + process.env["SNAPSHOT_INTERVAL"] = "120"; + expect(loadSnapshotConfig().interval).toBe(120); + }); + + it("accepts the minimum floor of 1 second", () => { + process.env["SNAPSHOT_INTERVAL"] = "1"; + expect(loadSnapshotConfig().interval).toBe(1); + }); + + it("never yields an interval that would clamp setInterval to sub-second", () => { + for (const bad of ["0", "-1", "-3600", "0.5"]) { + process.env["SNAPSHOT_INTERVAL"] = bad; + expect(loadSnapshotConfig().interval).toBeGreaterThanOrEqual(1); + } + }); + + it("reports enabled state from SNAPSHOT_ENABLED", () => { + process.env["SNAPSHOT_ENABLED"] = "true"; + expect(loadSnapshotConfig().enabled).toBe(true); + process.env["SNAPSHOT_ENABLED"] = "false"; + expect(loadSnapshotConfig().enabled).toBe(false); + }); +}); diff --git a/test/snapshot.test.ts b/test/snapshot.test.ts index 198774d26b..6a052cd54c 100644 --- a/test/snapshot.test.ts +++ b/test/snapshot.test.ts @@ -164,3 +164,64 @@ describe("Snapshot Functions", () => { expect(audits.length).toBe(1); }); }); + +describe("snapshot-create reentrancy guard", () => { + // Regression (P2): mem::snapshot-create is triggered by the periodic timer, + // REST (api::snapshot-create), and MCP. Two runs writing state.json and + // committing in the same git repo at once race on the index lock. An + // overlapping call must be a no-op success while the first run finishes. + it("skips an overlapping call and releases the guard on completion", async () => { + let releaseFirst!: () => void; + const firstListGate = new Promise((resolve) => { + releaseFirst = resolve; + }); + let listCalls = 0; + const store = new Map>(); + const gatedKv = { + get: async () => null, + set: async (scope: string, key: string, data: T): Promise => { + if (!store.has(scope)) store.set(scope, new Map()); + store.get(scope)!.set(key, data); + return data; + }, + delete: async () => {}, + list: async (scope: string): Promise => { + listCalls++; + // Park the first snapshot inside its initial list() so a second + // snapshot-create observes the in-flight guard. + if (listCalls === 1) await firstListGate; + return (Array.from(store.get(scope)?.values() ?? []) as T[]) ?? []; + }, + }; + const localSdk = mockSdk(); + registerSnapshotFunction(localSdk as never, gatedKv as never, "/tmp/reentrant"); + + // Start the first snapshot; it parks inside kv.list with the guard held. + const p1 = localSdk.trigger("mem::snapshot-create", { message: "first" }); + await Promise.resolve(); + await Promise.resolve(); + + // Overlapping call: must be rejected as already-in-progress, NOT run git. + const r2 = (await localSdk.trigger("mem::snapshot-create", { + message: "second", + })) as { success: boolean; message?: string; snapshot?: unknown }; + expect(r2).toEqual({ + success: true, + message: "Snapshot already in progress", + }); + expect(r2.snapshot).toBeUndefined(); + + // Release the first run; it completes normally. + releaseFirst(); + const r1 = (await p1) as { success: boolean; snapshot?: unknown }; + expect(r1.success).toBe(true); + expect(r1.snapshot).toBeDefined(); + + // Guard is released: a fresh call runs the full body again. + const r3 = (await localSdk.trigger("mem::snapshot-create", { + message: "third", + })) as { success: boolean; snapshot?: unknown }; + expect(r3.success).toBe(true); + expect(r3.snapshot).toBeDefined(); + }); +});