From 042f3a160c2ce7d92b7a47a6d593276c6afd5242 Mon Sep 17 00:00:00 2001 From: neal Date: Fri, 25 Sep 2026 18:29:54 -0400 Subject: [PATCH 01/17] fix(providers): stop 429s opening the breaker; cap LLM concurrency A busy self-hosted backend (429 queue_full, Retry-After: 30) tripped the summarize breaker after three calls, so every summary for the next 30s failed with circuit_breaker_open. Providers now throw ProviderHttpError carrying the status; ResilientProvider does not count 429/503 as a failure. Retry-After is honored in full when it fits the call's budget instead of being cut to 5s. All LLM calls share one slot pool, AGENTMEMORY_LLM_MAX_CONCURRENCY (default 4). Upstream PR#1259. Generated-By: Claude Opus 5.5 (1M context) --- .env.example | 6 ++ plugin/skills/agentmemory-config/REFERENCE.md | 3 +- src/providers/_fetch.ts | 28 +++++-- src/providers/minimax.ts | 4 +- src/providers/openai.ts | 4 +- src/providers/openrouter.ts | 4 +- src/providers/resilient.ts | 35 +++++++- test/fetch-timeout.test.ts | 48 +++++------ test/resilient-provider.test.ts | 80 ++++++++++++++++++- 9 files changed, 169 insertions(+), 43 deletions(-) diff --git a/.env.example b/.env.example index 1a1c5c6c3a..1db846ca37 100644 --- a/.env.example +++ b/.env.example @@ -55,6 +55,12 @@ # embedding). The OpenAI LLM path also honours the OpenAI-scoped # OPENAI_TIMEOUT_MS alias for back-compat with v0.9.17 (precedence). # AGENTMEMORY_LLM_TIMEOUT_MS=60000 # Default: 60 000 ms (60 s) +# Most LLM calls (compress, summarize, image description) in flight at once +# across the whole process (default 4). Calls over the cap wait for a slot. +# Lower it when a self-hosted backend answers 429 "queue full". A 429 or 503 +# never opens the circuit breaker; the call honors Retry-After within its +# timeout budget instead. +# AGENTMEMORY_LLM_MAX_CONCURRENCY=4 # Opt-in Claude-subscription fallback (spawns @anthropic-ai/claude-agent-sdk # child sessions). Off by default — the agent-sdk fallback can trigger diff --git a/plugin/skills/agentmemory-config/REFERENCE.md b/plugin/skills/agentmemory-config/REFERENCE.md index 7d14662916..f94de4cb50 100644 --- a/plugin/skills/agentmemory-config/REFERENCE.md +++ b/plugin/skills/agentmemory-config/REFERENCE.md @@ -3,7 +3,7 @@ Generated by scanning `src/` for `AGENTMEMORY_*` usage. Do not edit the block below by hand; run `npm run skills:gen` after adding or removing a variable. Internal markers ending in two underscores are excluded. -Configuration is read from the environment and from `~/.agentmemory/.env` (no `export` prefix). 70 recognized variables: +Configuration is read from the environment and from `~/.agentmemory/.env` (no `export` prefix). 71 recognized variables: - `AGENTMEMORY_AGENT_SCOPE` - `AGENTMEMORY_AGENT_SDK_MAX_CONCURRENCY` @@ -45,6 +45,7 @@ Configuration is read from the environment and from `~/.agentmemory/.env` (no `e - `AGENTMEMORY_INDEX_EXCLUDE_TOOL_PREFIXES` - `AGENTMEMORY_INDEX_REBUILD` - `AGENTMEMORY_INJECT_CONTEXT` +- `AGENTMEMORY_LLM_MAX_CONCURRENCY` - `AGENTMEMORY_LLM_NOTHINK` - `AGENTMEMORY_LLM_TIMEOUT_MS` - `AGENTMEMORY_LOG_LEVEL` diff --git a/src/providers/_fetch.ts b/src/providers/_fetch.ts index 3b3b9041b5..a2bf3906fa 100644 --- a/src/providers/_fetch.ts +++ b/src/providers/_fetch.ts @@ -3,15 +3,31 @@ import { getEnvVar } from "../config.js"; // Bounded retry for transient rate-limit / unavailable responses. Attempts is // total tries (initial + retries). Retries are bounded by a TOTAL elapsed // deadline — not per-attempt — so the worst case never blows past the caller's -// timeout budget. A single retry delay is capped low so a hostile Retry-After -// header can't dominate the budget. +// timeout budget. A Retry-After is honored in full when it fits that budget: +// every LLM call is background work, and retrying a busy server sooner than it +// asked only earns another 429. const MAX_ATTEMPTS = 3; -const MAX_RETRY_DELAY_MS = 5000; // A retry only makes sense if there's room for at least a token attempt after // the sleep; without this floor we'd sleep, fire, and get instantly cut off. const MIN_ATTEMPT_FLOOR_MS = 100; const RETRY_STATUS = new Set([429, 503]); +// A non-2xx provider response, carrying its status so callers can tell a busy +// provider from a broken one. SDK errors (Anthropic) carry `status` the same way. +export class ProviderHttpError extends Error { + constructor( + message: string, + readonly status: number, + ) { + super(message); + } +} + +export function isProviderBusy(err: unknown): boolean { + const status = (err as { status?: unknown } | null)?.status; + return typeof status === "number" && RETRY_STATUS.has(status); +} + const sleep = (ms: number): Promise => new Promise((resolve) => setTimeout(resolve, ms)); @@ -73,11 +89,11 @@ export async function fetchWithTimeout( const retryAfter = parseRetryAfter(response.headers.get("Retry-After")); // Exponential backoff fallback when no Retry-After: 500ms, 1000ms, ... const backoff = 500 * 2 ** (attempt - 1); - const delay = Math.min(retryAfter ?? backoff, MAX_RETRY_DELAY_MS); + const delay = retryAfter ?? backoff; // Stop retrying if the sleep plus a minimal attempt would overrun the total - // budget — a hostile Retry-After that alone exceeds the remaining budget - // returns the last response instead of stalling the caller. + // budget — a Retry-After that alone exceeds the remaining budget returns + // the last response instead of stalling the caller. const elapsed = Date.now() - start; const remaining = ms - elapsed; if (delay + MIN_ATTEMPT_FLOOR_MS > remaining) return response; diff --git a/src/providers/minimax.ts b/src/providers/minimax.ts index 77c0dcd27c..22ca125187 100644 --- a/src/providers/minimax.ts +++ b/src/providers/minimax.ts @@ -1,6 +1,6 @@ import type { MemoryProvider } from '../types.js' import { getEnvVar } from '../config.js' -import { fetchWithTimeout } from './_fetch.js' +import { fetchWithTimeout, ProviderHttpError } from './_fetch.js' /** * MiniMax provider using raw fetch to call MiniMax's Anthropic-compatible API. @@ -58,7 +58,7 @@ export class MinimaxProvider implements MemoryProvider { if (!response.ok) { const text = await response.text() - throw new Error(`MiniMax API error ${response.status}: ${text}`) + throw new ProviderHttpError(`MiniMax API error ${response.status}: ${text}`, response.status) } const data = (await response.json()) as { diff --git a/src/providers/openai.ts b/src/providers/openai.ts index 13aeccaa2f..fbfef7abb7 100644 --- a/src/providers/openai.ts +++ b/src/providers/openai.ts @@ -1,6 +1,6 @@ import type { MemoryProvider } from "../types.js"; import { getEnvVar } from "../config.js"; -import { fetchWithTimeout } from "./_fetch.js"; +import { fetchWithTimeout, ProviderHttpError } from "./_fetch.js"; import { DEFAULT_AZURE_API_VERSION, buildAuthHeaders, @@ -158,7 +158,7 @@ export class OpenAIProvider implements MemoryProvider { if (!response.ok) { const text = await response.text(); - throw new Error(`OpenAI API error (${response.status}): ${text}`); + throw new ProviderHttpError(`OpenAI API error (${response.status}): ${text}`, response.status); } const data = (await response.json()) as { diff --git a/src/providers/openrouter.ts b/src/providers/openrouter.ts index 5c47bb0a87..2dd8129594 100644 --- a/src/providers/openrouter.ts +++ b/src/providers/openrouter.ts @@ -1,5 +1,5 @@ import type { MemoryProvider } from "../types.js"; -import { fetchWithTimeout } from "./_fetch.js"; +import { fetchWithTimeout, ProviderHttpError } from "./_fetch.js"; export class OpenRouterProvider implements MemoryProvider { name: string; @@ -54,7 +54,7 @@ export class OpenRouterProvider implements MemoryProvider { if (!response.ok) { const text = await response.text(); - throw new Error(`${this.name} API error (${response.status}): ${text}`); + throw new ProviderHttpError(`${this.name} API error (${response.status}): ${text}`, response.status); } const data = (await response.json()) as Record; diff --git a/src/providers/resilient.ts b/src/providers/resilient.ts index 698c48c2c4..0b1b35d207 100644 --- a/src/providers/resilient.ts +++ b/src/providers/resilient.ts @@ -1,10 +1,20 @@ import type { MemoryProvider, CircuitBreakerState } from "../types.js"; import { CircuitBreaker } from "./circuit-breaker.js"; +import { isProviderBusy } from "./_fetch.js"; +import { getEnvVar } from "../config.js"; export type ProviderOperation = "compress" | "summarize" | "describeImage"; const OPERATIONS: ProviderOperation[] = ["compress", "summarize", "describeImage"]; +const MAX_CONCURRENCY_DEFAULT = 4; + +function maxConcurrency(): number { + const raw = getEnvVar("AGENTMEMORY_LLM_MAX_CONCURRENCY")?.trim(); + const n = raw && /^\d+$/.test(raw) ? Number(raw) : 0; + return n > 0 ? n : MAX_CONCURRENCY_DEFAULT; +} + const SEVERITY: Record = { closed: 0, "half-open": 1, @@ -17,6 +27,8 @@ export class ResilientProvider implements MemoryProvider { private breakers = Object.fromEntries( OPERATIONS.map((op) => [op, new CircuitBreaker()]), ) as Record; + private inFlight = 0; + private waiters: Array<() => void> = []; name: string; // A failed count is a fallback for the caller, not a provider failure, // so it is forwarded outside the breaker. @@ -36,6 +48,22 @@ export class ResilientProvider implements MemoryProvider { } } + // One cap across every operation: a busy provider is busy for all of them, + // and fanning out past it only turns queued work into 429s. + private async acquireSlot(): Promise { + if (this.inFlight < maxConcurrency()) { + this.inFlight++; + return; + } + await new Promise((resolve) => this.waiters.push(resolve)); + } + + private releaseSlot(): void { + const next = this.waiters.shift(); + if (next) next(); + else this.inFlight--; + } + private async call( operation: ProviderOperation, fn: () => Promise, @@ -44,13 +72,18 @@ export class ResilientProvider implements MemoryProvider { if (!breaker.isAllowed) { throw new Error("circuit_breaker_open"); } + await this.acquireSlot(); try { const result = await fn(); breaker.recordSuccess(); return result; } catch (err) { - breaker.recordFailure(); + // A provider that says "busy" is healthy; opening the breaker on it + // would fail unrelated work for the whole cooldown. + if (!isProviderBusy(err)) breaker.recordFailure(); throw err; + } finally { + this.releaseSlot(); } } diff --git a/test/fetch-timeout.test.ts b/test/fetch-timeout.test.ts index f761050196..1d0b24c457 100644 --- a/test/fetch-timeout.test.ts +++ b/test/fetch-timeout.test.ts @@ -184,10 +184,9 @@ describe("fetchWithTimeout bounded retry (total deadline)", () => { expect(q.calls()).toBe(2); }); - // 2. 429 with a hostile Retry-After (100000s) must NOT wait that long. The - // delay collapses to the low per-delay cap (5000ms) so the retry fires - // quickly and total elapsed stays nowhere near 100000s. - it("does not honour a hostile Retry-After literally — caps it and stays within budget", async () => { + // 2. 429 with a hostile Retry-After (100000s) must NOT wait that long: it + // cannot fit the budget, so the 429 comes back without a retry. + it("returns a hostile Retry-After's 429 at once instead of waiting it out", async () => { const q = queuedFetch([ { status: 429, headers: { "Retry-After": "100000" } }, // 100000s { status: 200 }, @@ -197,23 +196,18 @@ describe("fetchWithTimeout bounded retry (total deadline)", () => { const start = Date.now(); const p = fetchWithTimeout("https://example.com", {}, 60000); - // runAllTimers drains every scheduled sleep; if the code honored 100000s - // literally this would advance 100_000_000ms of simulated time. await vi.runAllTimersAsync(); const res = await p; const elapsed = Date.now() - start; - expect(res.status).toBe(200); - expect(q.calls()).toBe(2); - // The only sleep was the capped 5000ms delay — total elapsed is bounded to - // the cap, nowhere near the 100_000_000ms the header requested. - expect(elapsed).toBeLessThanOrEqual(5000); + expect(res.status).toBe(429); + expect(q.calls()).toBe(1); expect(elapsed).toBeLessThan(60000); }); - // 2c. When the capped delay still cannot fit inside a small budget, we do NOT + // 2c. When the requested delay cannot fit inside a small budget, we do NOT // retry and return the last 429 within bound. - it("returns the last 429 without retrying when even the capped delay overruns a small budget", async () => { + it("returns the last 429 without retrying when the delay overruns a small budget", async () => { const q = queuedFetch([ { status: 429, headers: { "Retry-After": "100000" } }, { status: 200 }, @@ -222,7 +216,7 @@ describe("fetchWithTimeout bounded retry (total deadline)", () => { vi.useFakeTimers(); const start = Date.now(); - // Budget 1000ms: capped delay 5000ms + floor 100ms > remaining, no retry. + // Budget 1000ms: delay + floor 100ms > remaining, no retry. const p = fetchWithTimeout("https://example.com", {}, 1000); await vi.runAllTimersAsync(); const res = await p; @@ -233,20 +227,20 @@ describe("fetchWithTimeout bounded retry (total deadline)", () => { expect(elapsed).toBeLessThan(1000); }); - // 2b. Retry-After present but LARGER than the low per-delay cap collapses to - // MAX_RETRY_DELAY_MS (5000), still bounded, still retries when budget - // allows. - it("caps an oversized Retry-After to the max delay and still retries within budget", async () => { + // 2b. A long Retry-After that fits the budget is waited out in full — a busy + // server asking for 30s gets 30s, not an early retry that 429s again. + it("honors a long Retry-After in full when it fits the budget", async () => { const q = queuedFetch([ - { status: 503, headers: { "Retry-After": "60" } }, // 60s requested + { status: 503, headers: { "Retry-After": "30" } }, { status: 200 }, ]); vi.spyOn(globalThis, "fetch").mockImplementation(q.fetch); vi.useFakeTimers(); const p = fetchWithTimeout("https://example.com", {}, 60000); - // Requested 60s, but the honored delay is capped at 5000ms. - await vi.advanceTimersByTimeAsync(5000); + await vi.advanceTimersByTimeAsync(29_999); + expect(q.calls()).toBe(1); + await vi.advanceTimersByTimeAsync(1); const res = await p; expect(res.status).toBe(200); @@ -294,10 +288,8 @@ describe("fetchWithTimeout bounded retry (total deadline)", () => { expect(q.calls()).toBe(1); }); - // 5. Retry-After as an HTTP-date is parsed, capped to the max delay, and the - // retry fires within budget. - it("parses an HTTP-date Retry-After and caps the honored delay", async () => { - // 2s in the future — under the max delay cap, so honored as-is. + // 5. Retry-After as an HTTP-date is parsed and the retry fires within budget. + it("parses an HTTP-date Retry-After", async () => { const future = new Date(Date.now() + 2000).toUTCString(); const q = queuedFetch([ { status: 429, headers: { "Retry-After": future } }, @@ -314,8 +306,8 @@ describe("fetchWithTimeout bounded retry (total deadline)", () => { expect(q.calls()).toBe(2); }); - // 5b. A far-future HTTP-date collapses to the capped delay (5000ms), which - // still exceeds a small budget → no retry, last 503 returned in bound. + // 5b. A far-future HTTP-date exceeds a small budget → no retry, last 503 + // returned in bound. it("does not retry on a far-future HTTP-date Retry-After that overruns a small budget", async () => { const farFuture = new Date(Date.now() + 3600_000).toUTCString(); // 1h out const q = queuedFetch([ @@ -326,7 +318,7 @@ describe("fetchWithTimeout bounded retry (total deadline)", () => { vi.useFakeTimers(); const start = Date.now(); - // Budget 1000ms: capped delay 5000ms + floor > remaining, so no retry. + // Budget 1000ms: delay + floor > remaining, so no retry. const p = fetchWithTimeout("https://example.com", {}, 1000); await vi.runAllTimersAsync(); const res = await p; diff --git a/test/resilient-provider.test.ts b/test/resilient-provider.test.ts index 15c7630dc1..b8c9867cf4 100644 --- a/test/resilient-provider.test.ts +++ b/test/resilient-provider.test.ts @@ -1,6 +1,7 @@ -import { describe, it, expect } from "vitest"; +import { describe, it, expect, afterEach } from "vitest"; import { ResilientProvider } from "../src/providers/resilient.js"; import type { MemoryProvider } from "../src/types.js"; +import { ProviderHttpError } from "../src/providers/_fetch.js"; function fakeProvider(overrides: Partial = {}): MemoryProvider { return { @@ -75,4 +76,81 @@ describe("ResilientProvider per-operation breakers", () => { expect(vision.circuitStates.compress.state).toBe("closed"); await expect(vision.compress("s", "u")).resolves.toBe("compressed"); }); + + it("does not open the breaker when the provider only says it is busy", async () => { + const provider = new ResilientProvider( + fakeProvider({ + summarize: async () => { + throw new ProviderHttpError("OpenAI API error (429): queue_full", 429); + }, + }), + ); + + await failTimes(() => provider.summarize("s", "u"), 5); + + expect(provider.circuitStates.summarize.state).toBe("closed"); + }); + + it("treats an SDK error carrying status 503 as busy", async () => { + const provider = new ResilientProvider( + fakeProvider({ + compress: async () => { + throw Object.assign(new Error("overloaded"), { status: 503 }); + }, + }), + ); + + await failTimes(() => provider.compress("s", "u"), 5); + + expect(provider.circuitStates.compress.state).toBe("closed"); + }); +}); + +describe("ResilientProvider concurrency cap", () => { + afterEach(() => { + delete process.env.AGENTMEMORY_LLM_MAX_CONCURRENCY; + }); + + it("holds calls past AGENTMEMORY_LLM_MAX_CONCURRENCY until a slot frees, across operations", async () => { + process.env.AGENTMEMORY_LLM_MAX_CONCURRENCY = "1"; + let running = 0; + let peak = 0; + const releases: Array<() => void> = []; + const gated = () => + new Promise((resolve) => { + running++; + peak = Math.max(peak, running); + releases.push(() => { + running--; + resolve("ok"); + }); + }); + const provider = new ResilientProvider( + fakeProvider({ compress: gated, summarize: gated }), + ); + + const first = provider.compress("s", "u"); + const second = provider.summarize("s", "u"); + await new Promise((r) => setTimeout(r, 0)); + expect(releases).toHaveLength(1); + + releases[0](); + await first; + await new Promise((r) => setTimeout(r, 0)); + expect(releases).toHaveLength(2); + releases[1](); + await second; + + expect(peak).toBe(1); + }); + + it("frees the slot when a call throws", async () => { + process.env.AGENTMEMORY_LLM_MAX_CONCURRENCY = "1"; + const provider = new ResilientProvider( + fakeProvider({ summarize: async () => { throw new Error("boom"); } }), + ); + + await expect(provider.summarize("s", "u")).rejects.toThrow("boom"); + await expect(provider.compress("s", "u")).resolves.toBe("compressed"); + }); }); From 385ab26392df41023e3f97ce29ed5905331a034e Mon Sep 17 00:00:00 2001 From: neal Date: Fri, 25 Sep 2026 18:31:09 -0400 Subject: [PATCH 02/17] fix(summarize): skip tokenize when the estimate fits one chunk countObservationTokens sent one /tokenize per Observation, 16 at a time, on every summarize. On a broker that schedules tokenize beside generation those calls queued, timed out (1331 in three days) or drew 429s (205), adding load in front of the summary they were measuring. Only 12 of ~1800 summaries needed more than one chunk. The chars/3 estimate overcounts, so when it fits the budget the Session is summarized without measuring. Generated-By: Claude Opus 5.5 (1M context) --- src/functions/summarize.ts | 15 +++++++++++++-- test/summarize.test.ts | 18 ++++++++++++++++++ 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/src/functions/summarize.ts b/src/functions/summarize.ts index 54ff691350..f801769650 100644 --- a/src/functions/summarize.ts +++ b/src/functions/summarize.ts @@ -81,11 +81,22 @@ async function mapWithConcurrency( return out; } +function fitsOneChunk(counts: number[], budget: number): boolean { + const payload = counts.reduce((sum, n) => sum + n + SEPARATOR_TOKENS, 0); + return payload <= budget - PROMPT_OVERHEAD_TOKENS; +} + +// The estimate overcounts, so a Session it already fits in one chunk needs no +// measuring: on a broker that queues tokenize behind generation, one request +// per Observation is load the summary itself has to wait behind. async function countObservationTokens( provider: MemoryProvider, texts: string[], + budget: number, sessionId: string, ): Promise { + const estimates = texts.map(estimateTokens); + if (fitsOneChunk(estimates, budget)) return estimates; if (provider.countTokens) { try { return await mapWithConcurrency(texts, COUNT_CONCURRENCY, (t) => @@ -98,7 +109,7 @@ async function countObservationTokens( }); } } - return texts.map(estimateTokens); + return estimates; } // Greedy in-order packing. An Observation that alone exceeds the budget @@ -184,7 +195,7 @@ async function planChunks( ): Promise { const budget = getChunkTokens(); const texts = compressed.map(renderSummaryObservation); - const counts = await countObservationTokens(provider, texts, sessionId); + const counts = await countObservationTokens(provider, texts, budget, sessionId); const chunks = packChunks(compressed, counts, budget, sessionId); if (chunks.length > 1) { logger.info("Summarize chunking session", { diff --git a/test/summarize.test.ts b/test/summarize.test.ts index f25f8a7cac..560609cd09 100644 --- a/test/summarize.test.ts +++ b/test/summarize.test.ts @@ -544,6 +544,9 @@ describe("mem::summarize chunking", () => { }); it("measures every Observation once, even across the summarize retry", async () => { + // The chars/3 estimate (~31 per Observation with separator) overruns this + // budget, so the Session is measured; the measured 14 each fits exactly. + process.env.SUMMARIZE_CHUNK_TOKENS = budgetFor(5); let counted = 0; const provider = makeProvider(["garbage", summaryXml({ title: "second-attempt" })]); provider.countTokens = async () => { @@ -559,6 +562,21 @@ describe("mem::summarize chunking", () => { expect(counted).toBe(5); }); + it("skips measuring when the estimate already fits one chunk", async () => { + let counted = 0; + const provider = makeProvider([summaryXml({ title: "small" })]); + provider.countTokens = async () => { + counted += 1; + return TOKENS_PER_OBS; + }; + const { handler } = await setupHandler({ sessionId: "ses_fits", obsCount: 5, provider }); + + const result: any = await handler({ sessionId: "ses_fits" }); + + expect(result.success).toBe(true); + expect(counted).toBe(0); + }); + it("returns parse_failed only after both attempts fail", async () => { const provider = makeProvider([ "garbage one", From 895d01c68a81647e06d43a98c7e2c9ea32084625 Mon Sep 17 00:00:00 2001 From: neal Date: Fri, 25 Sep 2026 18:32:16 -0400 Subject: [PATCH 03/17] fix(hooks): send PreToolUse context in the hookSpecificOutput envelope Claude Code writes plain PreToolUse stdout to the debug log and never shows it to the model, so AGENTMEMORY_INJECT_CONTEXT=true fetched /enrich on every file tool call for nothing. When the payload names hook_event_name "PreToolUse" the context now goes out as hookSpecificOutput.additionalContext; hosts that send no event name keep the plain text. Upstream #1278. Generated-By: Claude Opus 5.5 (1M context) --- plugin/scripts/pre-tool-use.mjs | 9 +++++- src/hooks/pre-tool-use.ts | 20 +++++++++++-- test/context-injection.test.ts | 52 ++++++++++++++++++++++++++++++++- 3 files changed, 76 insertions(+), 5 deletions(-) diff --git a/plugin/scripts/pre-tool-use.mjs b/plugin/scripts/pre-tool-use.mjs index 0146f69926..d10072e16e 100755 --- a/plugin/scripts/pre-tool-use.mjs +++ b/plugin/scripts/pre-tool-use.mjs @@ -44,6 +44,13 @@ function isSdkChildContext(payload) { const INJECT_CONTEXT = process.env["AGENTMEMORY_INJECT_CONTEXT"] === "true"; const REST_URL = process.env["AGENTMEMORY_URL"] || "http://localhost:3111"; const SECRET = process.env["AGENTMEMORY_SECRET"] || ""; +function contextPayload(data, context) { + if (data.hook_event_name === "PreToolUse") return JSON.stringify({ hookSpecificOutput: { + hookEventName: "PreToolUse", + additionalContext: context + } }); + return context; +} function authHeaders() { const h = { "Content-Type": "application/json" }; if (SECRET) h["Authorization"] = `Bearer ${SECRET}`; @@ -110,7 +117,7 @@ async function main() { }); if (res.ok) { const result = await res.json(); - if (result.context) process.stdout.write(result.context); + if (result.context) process.stdout.write(contextPayload(data, result.context)); } } catch {} } diff --git a/src/hooks/pre-tool-use.ts b/src/hooks/pre-tool-use.ts index 8e2383fa88..a895af0a4b 100644 --- a/src/hooks/pre-tool-use.ts +++ b/src/hooks/pre-tool-use.ts @@ -13,8 +13,7 @@ function isSdkChildContext(payload: unknown): boolean { // // THIS HOOK IS A NO-OP BY DEFAULT AS OF 0.8.10 (#143). Previously it // fired /agentmemory/enrich on every Edit/Write/Read/Glob/Grep tool call -// and wrote up to 4000 chars of context to stdout. Claude Code reads -// PreToolUse stdout and prepends it to the model's next turn, which meant +// and wrote up to 4000 chars of context to the model's next turn, which meant // agentmemory was silently injecting ~1000 tokens into every tool turn // via the user's Claude Code session. On Claude Pro that burned entire // allocations in a handful of messages (@adrianricardo, #143). @@ -29,6 +28,21 @@ const INJECT_CONTEXT = process.env["AGENTMEMORY_INJECT_CONTEXT"] === "true"; const REST_URL = process.env["AGENTMEMORY_URL"] || "http://localhost:3111"; const SECRET = process.env["AGENTMEMORY_SECRET"] || ""; +// Claude Code drops plain PreToolUse stdout into the debug log; only the +// hookSpecificOutput envelope reaches the model. Hosts that send no +// hook_event_name keep the plain text they have always read. +function contextPayload(data: Record, context: string): string { + if (data.hook_event_name === "PreToolUse") { + return JSON.stringify({ + hookSpecificOutput: { + hookEventName: "PreToolUse", + additionalContext: context, + }, + }); + } + return context; +} + function authHeaders(): Record { const h: Record = { "Content-Type": "application/json" }; if (SECRET) h["Authorization"] = `Bearer ${SECRET}`; @@ -120,7 +134,7 @@ async function main() { if (res.ok) { const result = (await res.json()) as { context?: string }; if (result.context) { - process.stdout.write(result.context); + process.stdout.write(contextPayload(data, result.context)); } } } catch { diff --git a/test/context-injection.test.ts b/test/context-injection.test.ts index 0b1148e80d..92ce71c10f 100644 --- a/test/context-injection.test.ts +++ b/test/context-injection.test.ts @@ -1,4 +1,5 @@ -import { describe, it, expect } from "vitest"; +import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import { createServer, type Server } from "node:http"; import { spawn } from "node:child_process"; import { join } from "node:path"; @@ -113,6 +114,55 @@ describe("pre-tool-use hook — context injection gate (#143)", () => { }); }); +describe("pre-tool-use hook — context envelope (#1278)", () => { + let server: Server; + let url = ""; + + beforeAll(async () => { + server = createServer((_req, res) => { + res.setHeader("Content-Type", "application/json"); + res.end(JSON.stringify({ context: "remembered about foo.ts" })); + }); + await new Promise((r) => server.listen(0, "127.0.0.1", r)); + const addr = server.address(); + url = `http://127.0.0.1:${typeof addr === "object" && addr ? addr.port : 0}`; + }); + + afterAll(() => new Promise((r) => server.close(() => r()))); + + it("wraps context in the hookSpecificOutput envelope for Claude Code", async () => { + const payload = JSON.stringify({ + session_id: "ses_test", + hook_event_name: "PreToolUse", + tool_name: "Read", + tool_input: { file_path: "src/foo.ts" }, + }); + const result = await runHook("pre-tool-use.mjs", payload, { + AGENTMEMORY_INJECT_CONTEXT: "true", + AGENTMEMORY_URL: url, + }); + expect(JSON.parse(result.stdout)).toEqual({ + hookSpecificOutput: { + hookEventName: "PreToolUse", + additionalContext: "remembered about foo.ts", + }, + }); + }); + + it("keeps plain text for hosts that send no hook_event_name", async () => { + const payload = JSON.stringify({ + conversation_id: "ses_test", + toolName: "read", + toolArgs: { path: "src/foo.ts" }, + }); + const result = await runHook("pre-tool-use.mjs", payload, { + AGENTMEMORY_INJECT_CONTEXT: "true", + AGENTMEMORY_URL: url, + }); + expect(result.stdout).toBe("remembered about foo.ts"); + }); +}); + describe("session-start hook — context injection gate (#143)", () => { it("registers the session but writes nothing to stdout when AGENTMEMORY_INJECT_CONTEXT is unset", async () => { // Session registration POST will fail against the unreachable URL, From 3b2545ed3eba63fc57f701120b5135f850a3da13 Mon Sep 17 00:00:00 2001 From: neal Date: Fri, 25 Sep 2026 18:33:00 -0400 Subject: [PATCH 04/17] fix(evict): run the Eviction sweep daily mem::evict was registered but no timer called it, so Eviction only ran when someone POSTed /agentmemory/evict. It now runs every 24h beside the lesson and insight decay sweeps; EVICTION_ENABLED=false opts out. Upstream PR#1287. Generated-By: Claude Opus 5.5 (1M context) --- .env.example | 1 + src/index.ts | 1330 +++++++++++++++++++++++++------------------------- 2 files changed, 671 insertions(+), 660 deletions(-) diff --git a/.env.example b/.env.example index 1db846ca37..2c293eb779 100644 --- a/.env.example +++ b/.env.example @@ -157,6 +157,7 @@ # AGENTMEMORY_INJECT_CONTEXT=true # Inject recalled memories back into agent prompts (#143). Default off — hooks capture observations but do not modify conversation. # CONSOLIDATION_ENABLED=true # Run the 4-tier consolidation pipeline (memories → semantic → procedural). Default off — opt in once you've measured the LLM cost. # CONSOLIDATION_DECAY_DAYS=30 # Age (days) after which non-reinforced memories decay during consolidation +# EVICTION_ENABLED=false # Opt out of the daily Eviction sweep (stale unsummarized sessions, old low-importance observations, per-project cap) # GRAPH_EXTRACTION_ENABLED=true # Extract concept-graph edges on remember; powers the graph-traversal recall path # GRAPH_EXTRACTION_BATCH_SIZE=8 # Memories per graph-extraction batch diff --git a/src/index.ts b/src/index.ts index 32a8f13a89..0fdb5eab6f 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,660 +1,670 @@ -import { TriggerAction } from "./engine/types.js"; -import { - hydrateProcessEnvFromFile, - loadConfig, - getEnvVar, - loadEmbeddingConfig, - loadFallbackConfig, - loadClaudeBridgeConfig, - loadTeamConfig, - loadSnapshotConfig, - isGraphExtractionEnabled, - isAutoCompressEnabled, - isConsolidationEnabled, - isContextInjectionEnabled, - isDropStaleIndexEnabled, - getSqlitePath, -} from "./config.js"; -import { createInprocSdk } from "./engine/inproc/sdk.js"; -import { SqliteVectorStore } from "./engine/inproc/vectors.js"; -import { createIndexFill, registerIndexFillFunction } from "./functions/index-fill.js"; -import { registerMaintenanceFunctions } from "./functions/maintenance.js"; -import { - createProvider, - createFallbackProvider, - createEmbeddingProvider, - createImageEmbeddingProvider, -} from "./providers/index.js"; -import { StateKV } from "./state/kv.js"; -import { KV } from "./state/schema.js"; -import { runStartupMaintenance } from "./state/startup-maintenance.js"; -import { - GRAPH_INDEX_NODE_CEILING, - backfillGraphIndexes, - graphIndexesReady, - graphLegDisabled, -} from "./state/graph-indexes.js"; -import { VectorIndex } from "./state/vector-index.js"; -import { HybridSearch } from "./state/hybrid-search.js"; -import { registerPrivacyFunction } from "./functions/privacy.js"; -import { registerObserveFunction } from "./functions/observe.js"; -import { registerImageQuotaCleanup } from "./functions/image-quota-cleanup.js"; -import { registerVisionSearchFunctions } from "./functions/vision-search.js"; -import { registerSlotsFunctions, isSlotsEnabled, isReflectEnabled } from "./functions/slots.js"; -import { registerDiskSizeManager } from "./functions/disk-size-manager.js"; -import { registerCompressFunction } from "./functions/compress.js"; -import { - registerSearchFunction, - getSearchIndex, - setVectorIndex, - setEmbeddingProvider, - setInprocStores, - rebuildBm25FromContent, -} from "./functions/search.js"; -import { registerContextFunction } from "./functions/context.js"; -import { registerSummarizeFunction } from "./functions/summarize.js"; -import { registerMigrateFunction } from "./functions/migrate.js"; -import { registerFileIndexFunction } from "./functions/file-index.js"; -import { registerConsolidateFunction } from "./functions/consolidate.js"; -import { registerPatternsFunction } from "./functions/patterns.js"; -import { registerRememberFunction } from "./functions/remember.js"; -import { registerEvictFunction } from "./functions/evict.js"; -import { registerRelationsFunction } from "./functions/relations.js"; -import { registerTimelineFunction } from "./functions/timeline.js"; -import { registerSmartSearchFunction } from "./functions/smart-search.js"; -import { registerRecentSearchesSweepFunction } from "./functions/recent-searches-sweep.js"; -import { registerProfileFunction } from "./functions/profile.js"; -import { registerAutoForgetFunction } from "./functions/auto-forget.js"; -import { registerExportImportFunction } from "./functions/export-import.js"; -import { registerEnrichFunction } from "./functions/enrich.js"; -import { registerClaudeBridgeFunction } from "./functions/claude-bridge.js"; -import { registerGraphFunction } from "./functions/graph.js"; -import { registerGraphImportFunction } from "./functions/graph-import.js"; -import { registerConsolidationPipelineFunction } from "./functions/consolidation-pipeline.js"; -import { registerTeamFunction } from "./functions/team.js"; -import { registerGovernanceFunction } from "./functions/governance.js"; -import { registerSnapshotFunction } from "./functions/snapshot.js"; -import { registerActionsFunction } from "./functions/actions.js"; -import { registerFrontierFunction } from "./functions/frontier.js"; -import { registerLeasesFunction } from "./functions/leases.js"; -import { registerRoutinesFunction } from "./functions/routines.js"; -import { registerSignalsFunction } from "./functions/signals.js"; -import { registerCheckpointsFunction } from "./functions/checkpoints.js"; -import { registerFlowCompressFunction } from "./functions/flow-compress.js"; -import { registerMeshFunction } from "./functions/mesh.js"; -import { registerBranchAwareFunction } from "./functions/branch-aware.js"; -import { registerSentinelsFunction } from "./functions/sentinels.js"; -import { registerSketchesFunction } from "./functions/sketches.js"; -import { registerCrystallizeFunction } from "./functions/crystallize.js"; -import { registerDiagnosticsFunction } from "./functions/diagnostics.js"; -import { registerFacetsFunction } from "./functions/facets.js"; -import { registerVerifyFunction } from "./functions/verify.js"; -import { registerCascadeFunction } from "./functions/cascade.js"; -import { registerLessonsFunctions } from "./functions/lessons.js"; -import { registerObsidianExportFunction } from "./functions/obsidian-export.js"; -import { registerReflectFunctions } from "./functions/reflect.js"; -import { registerWorkingMemoryFunctions } from "./functions/working-memory.js"; -import { registerSkillExtractFunctions } from "./functions/skill-extract.js"; -import { registerSlidingWindowFunction } from "./functions/sliding-window.js"; -import { - expandQuery, - registerQueryExpansionFunction, -} from "./functions/query-expansion.js"; -import { registerTemporalGraphFunctions } from "./functions/temporal-graph.js"; -import { registerRetentionFunctions } from "./functions/retention.js"; -import { registerCompressFileFunction } from "./functions/compress-file.js"; -import { registerReplayFunctions } from "./functions/replay.js"; -import { registerApiTriggers } from "./triggers/api.js"; -import { registerEventTriggers } from "./triggers/events.js"; -import { registerMcpEndpoints } from "./mcp/server.js"; -import { getAllTools } from "./mcp/tools-registry.js"; -import { startViewerServer } from "./viewer/server.js"; -import { MetricsStore } from "./eval/metrics-store.js"; -import { DedupMap } from "./functions/dedup.js"; -import { registerHealthMonitor } from "./health/monitor.js"; -import { initMetrics } from "./telemetry/setup.js"; -import { VERSION } from "./version.js"; -import { bootLog } from "./logger.js"; -import { mkdirSync, writeFileSync, unlinkSync } from "node:fs"; -import { dirname, join } from "node:path"; -import { homedir } from "node:os"; - -// #640 + #474: record this process's pid so `agentmemory stop` and -// `agentmemory doctor` can identify the daemon. Without it they can only -// scan the REST port, which cannot tell our `node` process apart from a -// tunnel or a stray dev server holding the same port. -function workerPidfilePath(): string { - return join(homedir(), ".agentmemory", "worker.pid"); -} -function writeWorkerPidfile(): void { - try { - const p = workerPidfilePath(); - mkdirSync(dirname(p), { recursive: true }); - writeFileSync(p, `${process.pid}\n`, { encoding: "utf-8" }); - } catch { - // best-effort; stop still has the port scan fallback - } -} -function clearWorkerPidfile(): void { - try { - unlinkSync(workerPidfilePath()); - } catch {} -} - -// Top-level safety net for state invocation timeouts (issue #204). -// Under sustained write load (e.g. Claude Code hooks across many -// projects) `state::set` can occasionally exceed its 30s timeout. -// We don't want one such timeout to terminate the long-lived memory -// service — the rejection is surfaced to the relevant call site via -// .catch() where it matters; everything else is logged-and-continued. -// Throttle logs to avoid spamming on bursts. -let lastUnhandledLogAt = 0; -process.on("unhandledRejection", (reason) => { - const now = Date.now(); - if (now - lastUnhandledLogAt < 60_000) return; - lastUnhandledLogAt = now; - const r = reason as { code?: string; function_id?: string; message?: string }; - console.warn( - `[agentmemory] unhandledRejection (suppressed):`, - r?.code ? `${r.code} ${r.function_id ?? ""} ${r.message ?? ""}`.trim() : reason, - ); -}); - -async function main() { - // Fold ~/.agentmemory/.env into process.env before anything reads config - // or raw process.env. Only-if-unset, so real process.env still wins. - hydrateProcessEnvFromFile(); - - const config = loadConfig(); - const embeddingConfig = loadEmbeddingConfig(); - const fallbackConfig = loadFallbackConfig(); - - const provider = - fallbackConfig.providers.length > 0 - ? createFallbackProvider(config.provider, fallbackConfig) - : createProvider(config.provider); - - const embeddingProvider = createEmbeddingProvider(); - const imageEmbeddingProvider = createImageEmbeddingProvider(); - - const sqlitePath = getSqlitePath(); - - bootLog(`Starting worker v${VERSION}...`); - bootLog(`Engine: in-process (${sqlitePath})`); - bootLog( - `Provider: ${config.provider.provider} (${config.provider.model})`, - ); - if (embeddingProvider) { - bootLog( - `Embedding provider: ${embeddingProvider.name} (${embeddingProvider.dimensions} dims)`, - ); - } else { - bootLog(`Embedding provider: none (BM25-only mode)`); - } - if (imageEmbeddingProvider) { - bootLog( - `Image embedding provider: ${imageEmbeddingProvider.name} (${imageEmbeddingProvider.dimensions} dims) — vision-search active`, - ); - } - bootLog( - `REST API: http://localhost:${config.restPort}/agentmemory/*`, - ); - bootLog(`Streams: ws://localhost:${config.streamsPort}`); - - // The only runtime: no engine process, no worker bus. The shim binds the - // REST and stream ports itself, so a bind failure has to be fatal here - // rather than leaving a daemon up with no listeners. - const sdk = createInprocSdk({ - restPort: config.restPort, - streamsPort: config.streamsPort, - sqlitePath, - maxBodyBytes: parseInt(getEnvVar("AGENTMEMORY_MAX_BODY_BYTES") || "", 10) || undefined, - }); - await sdk.listening(); - - writeWorkerPidfile(); - - const kv = new StateKV(sdk); - const secret = getEnvVar("AGENTMEMORY_SECRET"); - const metricsStore = new MetricsStore(kv); - const dedupMap = new DedupMap(); - - const vectorIndex = embeddingProvider ? new VectorIndex() : null; - - // Vectors are rows in the same SQLite file as the state. Every map - // mutation writes its rows first and every embedding completion commits - // through the store's content-revalidating transaction (src/engine/inproc/vectors.ts). - // The store exists even without an embedding provider so content deletes - // still take their persisted vector rows with them. - const vectorStore = new SqliteVectorStore(sdk.store); - vectorIndex?.attachStore(vectorStore); - setInprocStores(sdk.store, vectorStore); - - setVectorIndex(vectorIndex); - setEmbeddingProvider(embeddingProvider); - - // The in-process Engine exposes no meter provider of its own, so metrics - // fall back to whatever the OTel global supplies. - initMetrics(undefined); - - registerPrivacyFunction(sdk); - registerObserveFunction(sdk, kv, dedupMap, config.maxObservationsPerSession); - registerImageQuotaCleanup(sdk, kv); - registerVisionSearchFunctions(sdk, kv, imageEmbeddingProvider); - if (isSlotsEnabled()) { - registerSlotsFunctions(sdk, kv); - } - registerDiskSizeManager(sdk, kv); - registerCompressFunction(sdk, kv, provider, metricsStore); - registerSearchFunction(sdk, kv); - registerContextFunction(sdk, kv, config.tokenBudget); - registerSummarizeFunction(sdk, kv, provider, metricsStore); - registerMigrateFunction(sdk, kv); - registerFileIndexFunction(sdk, kv); - registerConsolidateFunction(sdk, kv, provider); - registerPatternsFunction(sdk, kv); - registerRememberFunction(sdk, kv); - registerEvictFunction(sdk, kv); - - registerRelationsFunction(sdk, kv); - registerTimelineFunction(sdk, kv); - registerProfileFunction(sdk, kv); - registerAutoForgetFunction(sdk, kv); - registerExportImportFunction(sdk, kv); - registerEnrichFunction(sdk, kv); - - const claudeBridgeConfig = loadClaudeBridgeConfig(); - if (claudeBridgeConfig.enabled) { - registerClaudeBridgeFunction(sdk, kv, claudeBridgeConfig); - bootLog( - `Claude bridge: syncing to ${claudeBridgeConfig.memoryFilePath}`, - ); - } - - registerGraphFunction(sdk, kv, provider); - registerGraphImportFunction(sdk, kv); - bootLog( - `Knowledge graph: structural extraction on (LLM relations ${isGraphExtractionEnabled() ? "enabled" : "off"})`, - ); - - registerConsolidationPipelineFunction(sdk, kv, provider); - bootLog(`Consolidation pipeline: registered (CONSOLIDATION_ENABLED=${isConsolidationEnabled() ? "true" : "false"})`); - - if (isAutoCompressEnabled()) { - bootLog( - `WARNING: AGENTMEMORY_AUTO_COMPRESS=true — every PostToolUse observation will be sent to your LLM provider for compression. This spends API tokens proportional to your session tool-use frequency. Set AGENTMEMORY_AUTO_COMPRESS=false to disable.`, - ); - } else { - bootLog( - `Auto-compress: OFF (default) — observations indexed via zero-LLM synthetic compression. Set AGENTMEMORY_AUTO_COMPRESS=true to opt-in to LLM-powered summaries (uses your API key).`, - ); - } - - if (isContextInjectionEnabled()) { - bootLog( - `WARNING: AGENTMEMORY_INJECT_CONTEXT=true — the PreToolUse and SessionStart hooks will inject up to ~4000 chars of memory context into every tool turn. On Claude Pro this burns session tokens proportional to your tool-call frequency. Set AGENTMEMORY_INJECT_CONTEXT=false to disable.`, - ); - } else { - bootLog( - `Context injection: OFF (default) — hooks capture observations but do not inject context into Claude Code's conversation. Set AGENTMEMORY_INJECT_CONTEXT=true to opt-in (warning: expect your Claude Pro allocation to drain faster).`, - ); - } - - const teamConfig = loadTeamConfig(); - if (teamConfig) { - registerTeamFunction(sdk, kv, teamConfig); - bootLog( - `Team memory: ${teamConfig.teamId} (${teamConfig.mode})`, - ); - } - - registerGovernanceFunction(sdk, kv); - - registerActionsFunction(sdk, kv); - registerFrontierFunction(sdk, kv); - registerLeasesFunction(sdk, kv); - registerRoutinesFunction(sdk, kv); - registerSignalsFunction(sdk, kv); - registerCheckpointsFunction(sdk, kv); - registerMeshFunction(sdk, kv, secret); - registerBranchAwareFunction(sdk, kv); - registerFlowCompressFunction(sdk, kv, provider); - registerSentinelsFunction(sdk, kv); - registerSketchesFunction(sdk, kv); - registerCrystallizeFunction(sdk, kv, provider); - registerDiagnosticsFunction(sdk, kv); - registerFacetsFunction(sdk, kv); - registerVerifyFunction(sdk, kv); - registerLessonsFunctions(sdk, kv); - registerObsidianExportFunction(sdk, kv); - registerReflectFunctions(sdk, kv, provider); - registerWorkingMemoryFunctions(sdk, kv, config.tokenBudget); - registerSkillExtractFunctions(sdk, kv, provider); - registerCascadeFunction(sdk, kv); - - registerSlidingWindowFunction(sdk, kv, provider); - registerQueryExpansionFunction(sdk, provider); - registerTemporalGraphFunctions(sdk, kv, provider); - registerRetentionFunctions(sdk, kv); - registerCompressFileFunction(sdk, kv, provider); - registerReplayFunctions(sdk, kv); - bootLog( - `v0.6 advanced retrieval: sliding-window, query-expansion, temporal-graph, retention-scoring`, - ); - bootLog( - `Orchestration layer: actions, frontier, leases, routines, signals, checkpoints, flow-compress, mesh, branch-aware, sentinels, sketches, crystallize, diagnostics, facets`, - ); - if (isSlotsEnabled()) { - bootLog( - `Slots: enabled (pinned editable memory). Reflect on Stop hook: ${isReflectEnabled() ? "on" : "off"}`, - ); - } - - const snapshotConfig = loadSnapshotConfig(); - if (snapshotConfig.enabled) { - registerSnapshotFunction(sdk, kv, snapshotConfig.dir); - // The boot line promised "every s" but nothing ever fired - // mem::snapshot-create. Drive it on a periodic timer (unref'd so it - // never keeps the process alive), mirroring the auto-forget timer. - // mem::snapshot-create serializes overlapping runs internally (git-lock - // safety), so the timer can stay a simple fire-and-forget tick. - const snapshotTimer = setInterval(() => { - sdk - .trigger({ - function_id: "mem::snapshot-create", - payload: {}, - action: TriggerAction.Void(), - }) - .catch(() => {}); - }, snapshotConfig.interval * 1000); - snapshotTimer.unref(); - bootLog( - `Git snapshots: ${snapshotConfig.dir} (every ${snapshotConfig.interval}s)`, - ); - } - - const bm25Index = getSearchIndex(); - const graphWeight = parseFloat(getEnvVar("AGENTMEMORY_GRAPH_WEIGHT") || "0.3"); - const hybridSearch = new HybridSearch( - bm25Index, - vectorIndex, - embeddingProvider, - kv, - embeddingConfig.bm25Weight, - embeddingConfig.vectorWeight, - graphWeight, - ); - - // `searchWithExpansion` existed as dead code since the fork point. Probe 0 - // measured why it is worth waking up: the two frozen-fixture targets that - // never reached the candidate pool sit at combined rank 422 and 261 when the - // pool is opened to depth 4000, so BOTH retrieval legs rank them past 100 and - // neither a deeper pool nor a weight change would seat them in a 20-slot - // head. The gap is vocabulary — the query and the memory share almost no - // surface — which is the one thing reformulation addresses. - // - // OFF by default: it costs one LLM round-trip plus N extra searches per - // query. Affordable here (single-user, a handful of searches a day) but not a - // cost to impose on every deployment silently. - const queryExpansionEnabled = - getEnvVar("AGENTMEMORY_QUERY_EXPANSION") === "true"; - registerSmartSearchFunction(sdk, kv, async (query, limit) => - queryExpansionEnabled - ? hybridSearch.searchWithExpansion( - query, - limit, - await expandQuery(provider, query), - ) - : hybridSearch.search(query, limit), - ); - registerRecentSearchesSweepFunction(sdk, kv); - - registerApiTriggers(sdk, kv, secret, metricsStore, provider); - registerEventTriggers(sdk, kv); - registerMcpEndpoints(sdk, kv, secret, metricsStore); - - const healthMonitor = registerHealthMonitor(sdk, kv); - - if (vectorIndex) { - const hydrated = vectorStore.hydrate(vectorIndex); - if (hydrated > 0) bootLog(`Hydrated ${hydrated} vectors from SQLite`); - } - // The dimension guard runs over the rows just hydrated. - const vectorSource = vectorIndex; - if (vectorSource && vectorIndex && vectorSource.size > 0) { - // Persisted vectors carry whatever dimension the provider had when - // they were written. If the active provider declares a different - // dimension — or if the on-disk index contains a mix of dimensions - // (legacy indexes written before the live-API guard in this PR) — - // restoring would silently corrupt search: cosineSimilarity returns - // 0 on cross-dim pairs, so affected observations stop matching - // anything and recall degrades without an error. Walk every stored - // vector instead of trusting the first; refuse to load if anything - // is off. - const activeDim = embeddingProvider?.dimensions ?? 0; - const { mismatches, seenDimensions } = - activeDim > 0 - ? vectorSource.validateDimensions(activeDim) - : { mismatches: [], seenDimensions: new Set() }; - - if (mismatches.length > 0) { - const sample = mismatches - .slice(0, 5) - .map((m) => `${m.obsId} (dim=${m.dim})`) - .join(", "); - const distinct = Array.from(seenDimensions).sort((a, b) => a - b).join(", "); - const dropStale = isDropStaleIndexEnabled(); - if (dropStale) { - console.warn( - `[agentmemory] Persisted vector index has ${mismatches.length} of ` + - `${vectorSource.size} vectors with the wrong dimension. Active ` + - `provider (${embeddingProvider?.name}) declares ${activeDim}; ` + - `dimensions seen on disk: ${distinct}. ` + - `AGENTMEMORY_DROP_STALE_INDEX=true is set — discarding the persisted ` + - `vectors. Live observations will rebuild the index over time.`, - ); - // The rows ARE the index, so discarding means deleting them. - vectorIndex.clear(); - } else { - throw new Error( - `[agentmemory] Refusing to start: persisted vector index has ` + - `${mismatches.length} of ${vectorSource.size} vectors with the ` + - `wrong dimension. Active provider (${embeddingProvider?.name}) ` + - `declares ${activeDim}; dimensions seen on disk: ${distinct}. ` + - `First mismatched obsIds: ${sample}. Loading would silently corrupt ` + - `search (cross-dimension cosine returns 0). Choose one:\n` + - ` - Re-embed the existing index against the new provider, then start.\n` + - ` - Set AGENTMEMORY_DROP_STALE_INDEX=true to discard the persisted ` + - `vectors and rebuild from live observations.\n` + - ` - Switch the embedding provider back to the one that wrote the index.`, - ); - } - } - } - - // BM25 is not persisted: it is rebuilt from the content rows on every boot - // - one ordered walk, vectors untouched (they hydrated above). - { - const t0 = performance.now(); - const r = await rebuildBm25FromContent(sdk.store.db); - bootLog( - `BM25 rebuilt from content: ${r.docs} docs of ${r.rows} rows in ` + - `${Math.round(performance.now() - t0)} ms (read ${Math.round(r.readMs)} ms, ` + - `index ${Math.round(r.indexMs)} ms${r.skipped ? `, ${r.skipped} rows skipped` : ""})`, - ); - } - - // Maintenance routes: mem::backup, and mem::index-debug-legs when - // AGENTMEMORY_INDEX_DEBUG=1. Registered before the gate opens so they - // exist the moment readyz answers 200. - registerMaintenanceFunctions(sdk, sdk.store); - - // Readiness: DB opened, a write probe round-tripped, vectors hydrated, - // BM25 rebuilt -> readyz 200 and the 503 gate opens. The probe uses the - // same key the health monitor writes. A failure here is fatal on purpose: - // systemd restarts a daemon that cannot write its store. - { - const stamp = Date.now(); - sdk.store.set(KV.health, "_probe", { ts: stamp }); - const back = sdk.store.get(KV.health, "_probe") as { ts?: number } | null; - if (back?.ts !== stamp) throw new Error("readiness write probe did not round-trip"); - sdk.setReady(); - bootLog("Ready: /agentmemory/readyz -> 200, routes open"); - } - - // One-time repair of stores written before this fork: graph provenance that - // predates the write-time bound, and the index shards the removed engine - // left behind. Not awaited — readyz already answers 200 — and the pass - // itself yields between chunks, so requests arriving during it are served - // rather than queued. It records its own version, so every later boot is a - // single marker read. - void runStartupMaintenance(sdk.store).catch((err: unknown) => { - console.warn( - `[agentmemory] startup maintenance skipped: ${err instanceof Error ? err.message : String(err)}`, - ); - }); - - // The vector fill/repair pass closes the gap between the content rows and - // the vectors table - missing or stale rows are embedded, orphans pruned. - // Runs once at boot after readiness (not awaited: a large backlog after an - // import must not hold up the listener) and hourly. - if (vectorIndex) { - const fill = createIndexFill(sdk.store, vectorStore, vectorIndex); - registerIndexFillFunction(sdk, fill); - void fill.run(); - setInterval(() => void fill.run(), 3_600_000).unref(); - bootLog("Vector fill pass: at boot and hourly (mem::index-fill-missing)"); - } - - // Backfill the graph read side-indexes for corpora that predate them. - // Mirrors the BM25 memories backfill above: one-time, gated on the - // snapshot's recorded node count so we never enumerate a corpus large - // enough to starve the worker heartbeat. - // - // graph-read-fix local delta (B-mode boot-skip): when the leg is off - // this block is skipped ENTIRELY — it makes the only graph-scope reads - // at boot (the kv.list arming enumeration), and B-mode must issue zero. - // Skipping is safe: with the marker absent the readers fail-closed - // (graphReadable === false) rather than enumerating. - if (!graphLegDisabled()) { - try { - if (!(await graphIndexesReady(kv))) { - const graphSnap = await kv.get( - KV.graphSnapshot, - "current", - ); - const totalNodes = graphSnap?.stats?.totalNodes ?? 0; - if (graphSnap && totalNodes > 0 && totalNodes <= GRAPH_INDEX_NODE_CEILING) { - const [graphNodes, graphEdges] = await Promise.all([ - kv.list(KV.graphNodes), - kv.list(KV.graphEdges), - ]); - await backfillGraphIndexes( - kv, - graphNodes.filter((n) => !n.stale), - graphEdges.filter((e) => !e.stale), - ); - bootLog(`Backfilled graph read indexes (${totalNodes} nodes)`); - } - } - } catch (err) { - console.warn(`[agentmemory] Failed to backfill graph indexes:`, err); - } - } - - // Ready / Endpoints lines are emitted via `bootLog` so they're - // buffered in quiet mode and printed verbatim under --verbose. The - // CLI surfaces a compact summary when it sees the worker reach - // ready state. - bootLog( - `Ready. ${embeddingProvider ? "Triple-stream (BM25+Vector+Graph)" : "BM25+Graph"} search active.`, - ); - bootLog( - `REST API: 132 endpoints at http://localhost:${config.restPort}/agentmemory/*`, - ); - bootLog( - `MCP surface (opt-in via \`npx @agentmemory/mcp\`): ${getAllTools().length} tools · 6 resources · 3 prompts`, - ); - - const viewerPort = config.restPort + 2; - const viewerServer = startViewerServer( - viewerPort, - kv, - sdk, - secret, - config.restPort, - ); - - const autoForgetIntervalMs = parseInt(process.env.AUTO_FORGET_INTERVAL_MS || "3600000", 10); - const consolidationIntervalMs = parseInt(process.env.CONSOLIDATION_INTERVAL_MS || "7200000", 10); - - if (process.env.AUTO_FORGET_ENABLED !== "false") { - const autoForgetTimer = setInterval(async () => { - try { - await sdk.trigger({ function_id: "mem::auto-forget", payload: { dryRun: false } }); - } catch {} - }, autoForgetIntervalMs); - autoForgetTimer.unref(); - bootLog(`Auto-forget: enabled (every ${autoForgetIntervalMs / 60000}m)`); - } - - if (process.env.LESSON_DECAY_ENABLED !== "false") { - const lessonDecayTimer = setInterval(async () => { - try { - await sdk.trigger({ function_id: "mem::lesson-decay-sweep", payload: {} }); - } catch {} - }, 86400000); - lessonDecayTimer.unref(); - bootLog(`Lesson decay sweep: enabled (every 24h)`); - } - - if (process.env.INSIGHT_DECAY_ENABLED !== "false") { - const insightDecayTimer = setInterval(async () => { - try { - await sdk.trigger({ function_id: "mem::insight-decay-sweep", payload: {} }); - } catch {} - }, 86400000); - insightDecayTimer.unref(); - } - - // #771: hourly TTL sweep for the followup-rate diagnostic. The - // recent-searches scope only needs the last entry per session; - // sweeping anything older than the retention window keeps the scope - // from growing unbounded across long-lived deployments. - const recentSearchesSweepTimer = setInterval(async () => { - try { - await sdk.trigger({ - function_id: "mem::diagnostic::recent-searches-sweep", - payload: {}, - }); - } catch {} - }, 60 * 60 * 1000); - recentSearchesSweepTimer.unref(); - - if (isConsolidationEnabled()) { - const consolidationTimer = setInterval(async () => { - try { - await sdk.trigger({ function_id: "mem::consolidate-pipeline", payload: {} }); - } catch {} - }, consolidationIntervalMs); - consolidationTimer.unref(); - bootLog(`Auto-consolidation: enabled (every ${consolidationIntervalMs / 60000}m)`); - } - - const shutdown = async () => { - console.log(`\n[agentmemory] Shutting down...`); - healthMonitor.stop(); - dedupMap.stop(); - await new Promise((resolve) => viewerServer.close(() => resolve())); - await sdk.shutdown(); - clearWorkerPidfile(); - process.exit(0); - }; - process.on("SIGINT", shutdown); - process.on("SIGTERM", shutdown); -} - -main().catch((err) => { - console.error(`[agentmemory] Fatal:`, err); - process.exit(1); -}); +import { TriggerAction } from "./engine/types.js"; +import { + hydrateProcessEnvFromFile, + loadConfig, + getEnvVar, + loadEmbeddingConfig, + loadFallbackConfig, + loadClaudeBridgeConfig, + loadTeamConfig, + loadSnapshotConfig, + isGraphExtractionEnabled, + isAutoCompressEnabled, + isConsolidationEnabled, + isContextInjectionEnabled, + isDropStaleIndexEnabled, + getSqlitePath, +} from "./config.js"; +import { createInprocSdk } from "./engine/inproc/sdk.js"; +import { SqliteVectorStore } from "./engine/inproc/vectors.js"; +import { createIndexFill, registerIndexFillFunction } from "./functions/index-fill.js"; +import { registerMaintenanceFunctions } from "./functions/maintenance.js"; +import { + createProvider, + createFallbackProvider, + createEmbeddingProvider, + createImageEmbeddingProvider, +} from "./providers/index.js"; +import { StateKV } from "./state/kv.js"; +import { KV } from "./state/schema.js"; +import { runStartupMaintenance } from "./state/startup-maintenance.js"; +import { + GRAPH_INDEX_NODE_CEILING, + backfillGraphIndexes, + graphIndexesReady, + graphLegDisabled, +} from "./state/graph-indexes.js"; +import { VectorIndex } from "./state/vector-index.js"; +import { HybridSearch } from "./state/hybrid-search.js"; +import { registerPrivacyFunction } from "./functions/privacy.js"; +import { registerObserveFunction } from "./functions/observe.js"; +import { registerImageQuotaCleanup } from "./functions/image-quota-cleanup.js"; +import { registerVisionSearchFunctions } from "./functions/vision-search.js"; +import { registerSlotsFunctions, isSlotsEnabled, isReflectEnabled } from "./functions/slots.js"; +import { registerDiskSizeManager } from "./functions/disk-size-manager.js"; +import { registerCompressFunction } from "./functions/compress.js"; +import { + registerSearchFunction, + getSearchIndex, + setVectorIndex, + setEmbeddingProvider, + setInprocStores, + rebuildBm25FromContent, +} from "./functions/search.js"; +import { registerContextFunction } from "./functions/context.js"; +import { registerSummarizeFunction } from "./functions/summarize.js"; +import { registerMigrateFunction } from "./functions/migrate.js"; +import { registerFileIndexFunction } from "./functions/file-index.js"; +import { registerConsolidateFunction } from "./functions/consolidate.js"; +import { registerPatternsFunction } from "./functions/patterns.js"; +import { registerRememberFunction } from "./functions/remember.js"; +import { registerEvictFunction } from "./functions/evict.js"; +import { registerRelationsFunction } from "./functions/relations.js"; +import { registerTimelineFunction } from "./functions/timeline.js"; +import { registerSmartSearchFunction } from "./functions/smart-search.js"; +import { registerRecentSearchesSweepFunction } from "./functions/recent-searches-sweep.js"; +import { registerProfileFunction } from "./functions/profile.js"; +import { registerAutoForgetFunction } from "./functions/auto-forget.js"; +import { registerExportImportFunction } from "./functions/export-import.js"; +import { registerEnrichFunction } from "./functions/enrich.js"; +import { registerClaudeBridgeFunction } from "./functions/claude-bridge.js"; +import { registerGraphFunction } from "./functions/graph.js"; +import { registerGraphImportFunction } from "./functions/graph-import.js"; +import { registerConsolidationPipelineFunction } from "./functions/consolidation-pipeline.js"; +import { registerTeamFunction } from "./functions/team.js"; +import { registerGovernanceFunction } from "./functions/governance.js"; +import { registerSnapshotFunction } from "./functions/snapshot.js"; +import { registerActionsFunction } from "./functions/actions.js"; +import { registerFrontierFunction } from "./functions/frontier.js"; +import { registerLeasesFunction } from "./functions/leases.js"; +import { registerRoutinesFunction } from "./functions/routines.js"; +import { registerSignalsFunction } from "./functions/signals.js"; +import { registerCheckpointsFunction } from "./functions/checkpoints.js"; +import { registerFlowCompressFunction } from "./functions/flow-compress.js"; +import { registerMeshFunction } from "./functions/mesh.js"; +import { registerBranchAwareFunction } from "./functions/branch-aware.js"; +import { registerSentinelsFunction } from "./functions/sentinels.js"; +import { registerSketchesFunction } from "./functions/sketches.js"; +import { registerCrystallizeFunction } from "./functions/crystallize.js"; +import { registerDiagnosticsFunction } from "./functions/diagnostics.js"; +import { registerFacetsFunction } from "./functions/facets.js"; +import { registerVerifyFunction } from "./functions/verify.js"; +import { registerCascadeFunction } from "./functions/cascade.js"; +import { registerLessonsFunctions } from "./functions/lessons.js"; +import { registerObsidianExportFunction } from "./functions/obsidian-export.js"; +import { registerReflectFunctions } from "./functions/reflect.js"; +import { registerWorkingMemoryFunctions } from "./functions/working-memory.js"; +import { registerSkillExtractFunctions } from "./functions/skill-extract.js"; +import { registerSlidingWindowFunction } from "./functions/sliding-window.js"; +import { + expandQuery, + registerQueryExpansionFunction, +} from "./functions/query-expansion.js"; +import { registerTemporalGraphFunctions } from "./functions/temporal-graph.js"; +import { registerRetentionFunctions } from "./functions/retention.js"; +import { registerCompressFileFunction } from "./functions/compress-file.js"; +import { registerReplayFunctions } from "./functions/replay.js"; +import { registerApiTriggers } from "./triggers/api.js"; +import { registerEventTriggers } from "./triggers/events.js"; +import { registerMcpEndpoints } from "./mcp/server.js"; +import { getAllTools } from "./mcp/tools-registry.js"; +import { startViewerServer } from "./viewer/server.js"; +import { MetricsStore } from "./eval/metrics-store.js"; +import { DedupMap } from "./functions/dedup.js"; +import { registerHealthMonitor } from "./health/monitor.js"; +import { initMetrics } from "./telemetry/setup.js"; +import { VERSION } from "./version.js"; +import { bootLog } from "./logger.js"; +import { mkdirSync, writeFileSync, unlinkSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { homedir } from "node:os"; + +// #640 + #474: record this process's pid so `agentmemory stop` and +// `agentmemory doctor` can identify the daemon. Without it they can only +// scan the REST port, which cannot tell our `node` process apart from a +// tunnel or a stray dev server holding the same port. +function workerPidfilePath(): string { + return join(homedir(), ".agentmemory", "worker.pid"); +} +function writeWorkerPidfile(): void { + try { + const p = workerPidfilePath(); + mkdirSync(dirname(p), { recursive: true }); + writeFileSync(p, `${process.pid}\n`, { encoding: "utf-8" }); + } catch { + // best-effort; stop still has the port scan fallback + } +} +function clearWorkerPidfile(): void { + try { + unlinkSync(workerPidfilePath()); + } catch {} +} + +// Top-level safety net for state invocation timeouts (issue #204). +// Under sustained write load (e.g. Claude Code hooks across many +// projects) `state::set` can occasionally exceed its 30s timeout. +// We don't want one such timeout to terminate the long-lived memory +// service — the rejection is surfaced to the relevant call site via +// .catch() where it matters; everything else is logged-and-continued. +// Throttle logs to avoid spamming on bursts. +let lastUnhandledLogAt = 0; +process.on("unhandledRejection", (reason) => { + const now = Date.now(); + if (now - lastUnhandledLogAt < 60_000) return; + lastUnhandledLogAt = now; + const r = reason as { code?: string; function_id?: string; message?: string }; + console.warn( + `[agentmemory] unhandledRejection (suppressed):`, + r?.code ? `${r.code} ${r.function_id ?? ""} ${r.message ?? ""}`.trim() : reason, + ); +}); + +async function main() { + // Fold ~/.agentmemory/.env into process.env before anything reads config + // or raw process.env. Only-if-unset, so real process.env still wins. + hydrateProcessEnvFromFile(); + + const config = loadConfig(); + const embeddingConfig = loadEmbeddingConfig(); + const fallbackConfig = loadFallbackConfig(); + + const provider = + fallbackConfig.providers.length > 0 + ? createFallbackProvider(config.provider, fallbackConfig) + : createProvider(config.provider); + + const embeddingProvider = createEmbeddingProvider(); + const imageEmbeddingProvider = createImageEmbeddingProvider(); + + const sqlitePath = getSqlitePath(); + + bootLog(`Starting worker v${VERSION}...`); + bootLog(`Engine: in-process (${sqlitePath})`); + bootLog( + `Provider: ${config.provider.provider} (${config.provider.model})`, + ); + if (embeddingProvider) { + bootLog( + `Embedding provider: ${embeddingProvider.name} (${embeddingProvider.dimensions} dims)`, + ); + } else { + bootLog(`Embedding provider: none (BM25-only mode)`); + } + if (imageEmbeddingProvider) { + bootLog( + `Image embedding provider: ${imageEmbeddingProvider.name} (${imageEmbeddingProvider.dimensions} dims) — vision-search active`, + ); + } + bootLog( + `REST API: http://localhost:${config.restPort}/agentmemory/*`, + ); + bootLog(`Streams: ws://localhost:${config.streamsPort}`); + + // The only runtime: no engine process, no worker bus. The shim binds the + // REST and stream ports itself, so a bind failure has to be fatal here + // rather than leaving a daemon up with no listeners. + const sdk = createInprocSdk({ + restPort: config.restPort, + streamsPort: config.streamsPort, + sqlitePath, + maxBodyBytes: parseInt(getEnvVar("AGENTMEMORY_MAX_BODY_BYTES") || "", 10) || undefined, + }); + await sdk.listening(); + + writeWorkerPidfile(); + + const kv = new StateKV(sdk); + const secret = getEnvVar("AGENTMEMORY_SECRET"); + const metricsStore = new MetricsStore(kv); + const dedupMap = new DedupMap(); + + const vectorIndex = embeddingProvider ? new VectorIndex() : null; + + // Vectors are rows in the same SQLite file as the state. Every map + // mutation writes its rows first and every embedding completion commits + // through the store's content-revalidating transaction (src/engine/inproc/vectors.ts). + // The store exists even without an embedding provider so content deletes + // still take their persisted vector rows with them. + const vectorStore = new SqliteVectorStore(sdk.store); + vectorIndex?.attachStore(vectorStore); + setInprocStores(sdk.store, vectorStore); + + setVectorIndex(vectorIndex); + setEmbeddingProvider(embeddingProvider); + + // The in-process Engine exposes no meter provider of its own, so metrics + // fall back to whatever the OTel global supplies. + initMetrics(undefined); + + registerPrivacyFunction(sdk); + registerObserveFunction(sdk, kv, dedupMap, config.maxObservationsPerSession); + registerImageQuotaCleanup(sdk, kv); + registerVisionSearchFunctions(sdk, kv, imageEmbeddingProvider); + if (isSlotsEnabled()) { + registerSlotsFunctions(sdk, kv); + } + registerDiskSizeManager(sdk, kv); + registerCompressFunction(sdk, kv, provider, metricsStore); + registerSearchFunction(sdk, kv); + registerContextFunction(sdk, kv, config.tokenBudget); + registerSummarizeFunction(sdk, kv, provider, metricsStore); + registerMigrateFunction(sdk, kv); + registerFileIndexFunction(sdk, kv); + registerConsolidateFunction(sdk, kv, provider); + registerPatternsFunction(sdk, kv); + registerRememberFunction(sdk, kv); + registerEvictFunction(sdk, kv); + + registerRelationsFunction(sdk, kv); + registerTimelineFunction(sdk, kv); + registerProfileFunction(sdk, kv); + registerAutoForgetFunction(sdk, kv); + registerExportImportFunction(sdk, kv); + registerEnrichFunction(sdk, kv); + + const claudeBridgeConfig = loadClaudeBridgeConfig(); + if (claudeBridgeConfig.enabled) { + registerClaudeBridgeFunction(sdk, kv, claudeBridgeConfig); + bootLog( + `Claude bridge: syncing to ${claudeBridgeConfig.memoryFilePath}`, + ); + } + + registerGraphFunction(sdk, kv, provider); + registerGraphImportFunction(sdk, kv); + bootLog( + `Knowledge graph: structural extraction on (LLM relations ${isGraphExtractionEnabled() ? "enabled" : "off"})`, + ); + + registerConsolidationPipelineFunction(sdk, kv, provider); + bootLog(`Consolidation pipeline: registered (CONSOLIDATION_ENABLED=${isConsolidationEnabled() ? "true" : "false"})`); + + if (isAutoCompressEnabled()) { + bootLog( + `WARNING: AGENTMEMORY_AUTO_COMPRESS=true — every PostToolUse observation will be sent to your LLM provider for compression. This spends API tokens proportional to your session tool-use frequency. Set AGENTMEMORY_AUTO_COMPRESS=false to disable.`, + ); + } else { + bootLog( + `Auto-compress: OFF (default) — observations indexed via zero-LLM synthetic compression. Set AGENTMEMORY_AUTO_COMPRESS=true to opt-in to LLM-powered summaries (uses your API key).`, + ); + } + + if (isContextInjectionEnabled()) { + bootLog( + `WARNING: AGENTMEMORY_INJECT_CONTEXT=true — the PreToolUse and SessionStart hooks will inject up to ~4000 chars of memory context into every tool turn. On Claude Pro this burns session tokens proportional to your tool-call frequency. Set AGENTMEMORY_INJECT_CONTEXT=false to disable.`, + ); + } else { + bootLog( + `Context injection: OFF (default) — hooks capture observations but do not inject context into Claude Code's conversation. Set AGENTMEMORY_INJECT_CONTEXT=true to opt-in (warning: expect your Claude Pro allocation to drain faster).`, + ); + } + + const teamConfig = loadTeamConfig(); + if (teamConfig) { + registerTeamFunction(sdk, kv, teamConfig); + bootLog( + `Team memory: ${teamConfig.teamId} (${teamConfig.mode})`, + ); + } + + registerGovernanceFunction(sdk, kv); + + registerActionsFunction(sdk, kv); + registerFrontierFunction(sdk, kv); + registerLeasesFunction(sdk, kv); + registerRoutinesFunction(sdk, kv); + registerSignalsFunction(sdk, kv); + registerCheckpointsFunction(sdk, kv); + registerMeshFunction(sdk, kv, secret); + registerBranchAwareFunction(sdk, kv); + registerFlowCompressFunction(sdk, kv, provider); + registerSentinelsFunction(sdk, kv); + registerSketchesFunction(sdk, kv); + registerCrystallizeFunction(sdk, kv, provider); + registerDiagnosticsFunction(sdk, kv); + registerFacetsFunction(sdk, kv); + registerVerifyFunction(sdk, kv); + registerLessonsFunctions(sdk, kv); + registerObsidianExportFunction(sdk, kv); + registerReflectFunctions(sdk, kv, provider); + registerWorkingMemoryFunctions(sdk, kv, config.tokenBudget); + registerSkillExtractFunctions(sdk, kv, provider); + registerCascadeFunction(sdk, kv); + + registerSlidingWindowFunction(sdk, kv, provider); + registerQueryExpansionFunction(sdk, provider); + registerTemporalGraphFunctions(sdk, kv, provider); + registerRetentionFunctions(sdk, kv); + registerCompressFileFunction(sdk, kv, provider); + registerReplayFunctions(sdk, kv); + bootLog( + `v0.6 advanced retrieval: sliding-window, query-expansion, temporal-graph, retention-scoring`, + ); + bootLog( + `Orchestration layer: actions, frontier, leases, routines, signals, checkpoints, flow-compress, mesh, branch-aware, sentinels, sketches, crystallize, diagnostics, facets`, + ); + if (isSlotsEnabled()) { + bootLog( + `Slots: enabled (pinned editable memory). Reflect on Stop hook: ${isReflectEnabled() ? "on" : "off"}`, + ); + } + + const snapshotConfig = loadSnapshotConfig(); + if (snapshotConfig.enabled) { + registerSnapshotFunction(sdk, kv, snapshotConfig.dir); + // The boot line promised "every s" but nothing ever fired + // mem::snapshot-create. Drive it on a periodic timer (unref'd so it + // never keeps the process alive), mirroring the auto-forget timer. + // mem::snapshot-create serializes overlapping runs internally (git-lock + // safety), so the timer can stay a simple fire-and-forget tick. + const snapshotTimer = setInterval(() => { + sdk + .trigger({ + function_id: "mem::snapshot-create", + payload: {}, + action: TriggerAction.Void(), + }) + .catch(() => {}); + }, snapshotConfig.interval * 1000); + snapshotTimer.unref(); + bootLog( + `Git snapshots: ${snapshotConfig.dir} (every ${snapshotConfig.interval}s)`, + ); + } + + const bm25Index = getSearchIndex(); + const graphWeight = parseFloat(getEnvVar("AGENTMEMORY_GRAPH_WEIGHT") || "0.3"); + const hybridSearch = new HybridSearch( + bm25Index, + vectorIndex, + embeddingProvider, + kv, + embeddingConfig.bm25Weight, + embeddingConfig.vectorWeight, + graphWeight, + ); + + // `searchWithExpansion` existed as dead code since the fork point. Probe 0 + // measured why it is worth waking up: the two frozen-fixture targets that + // never reached the candidate pool sit at combined rank 422 and 261 when the + // pool is opened to depth 4000, so BOTH retrieval legs rank them past 100 and + // neither a deeper pool nor a weight change would seat them in a 20-slot + // head. The gap is vocabulary — the query and the memory share almost no + // surface — which is the one thing reformulation addresses. + // + // OFF by default: it costs one LLM round-trip plus N extra searches per + // query. Affordable here (single-user, a handful of searches a day) but not a + // cost to impose on every deployment silently. + const queryExpansionEnabled = + getEnvVar("AGENTMEMORY_QUERY_EXPANSION") === "true"; + registerSmartSearchFunction(sdk, kv, async (query, limit) => + queryExpansionEnabled + ? hybridSearch.searchWithExpansion( + query, + limit, + await expandQuery(provider, query), + ) + : hybridSearch.search(query, limit), + ); + registerRecentSearchesSweepFunction(sdk, kv); + + registerApiTriggers(sdk, kv, secret, metricsStore, provider); + registerEventTriggers(sdk, kv); + registerMcpEndpoints(sdk, kv, secret, metricsStore); + + const healthMonitor = registerHealthMonitor(sdk, kv); + + if (vectorIndex) { + const hydrated = vectorStore.hydrate(vectorIndex); + if (hydrated > 0) bootLog(`Hydrated ${hydrated} vectors from SQLite`); + } + // The dimension guard runs over the rows just hydrated. + const vectorSource = vectorIndex; + if (vectorSource && vectorIndex && vectorSource.size > 0) { + // Persisted vectors carry whatever dimension the provider had when + // they were written. If the active provider declares a different + // dimension — or if the on-disk index contains a mix of dimensions + // (legacy indexes written before the live-API guard in this PR) — + // restoring would silently corrupt search: cosineSimilarity returns + // 0 on cross-dim pairs, so affected observations stop matching + // anything and recall degrades without an error. Walk every stored + // vector instead of trusting the first; refuse to load if anything + // is off. + const activeDim = embeddingProvider?.dimensions ?? 0; + const { mismatches, seenDimensions } = + activeDim > 0 + ? vectorSource.validateDimensions(activeDim) + : { mismatches: [], seenDimensions: new Set() }; + + if (mismatches.length > 0) { + const sample = mismatches + .slice(0, 5) + .map((m) => `${m.obsId} (dim=${m.dim})`) + .join(", "); + const distinct = Array.from(seenDimensions).sort((a, b) => a - b).join(", "); + const dropStale = isDropStaleIndexEnabled(); + if (dropStale) { + console.warn( + `[agentmemory] Persisted vector index has ${mismatches.length} of ` + + `${vectorSource.size} vectors with the wrong dimension. Active ` + + `provider (${embeddingProvider?.name}) declares ${activeDim}; ` + + `dimensions seen on disk: ${distinct}. ` + + `AGENTMEMORY_DROP_STALE_INDEX=true is set — discarding the persisted ` + + `vectors. Live observations will rebuild the index over time.`, + ); + // The rows ARE the index, so discarding means deleting them. + vectorIndex.clear(); + } else { + throw new Error( + `[agentmemory] Refusing to start: persisted vector index has ` + + `${mismatches.length} of ${vectorSource.size} vectors with the ` + + `wrong dimension. Active provider (${embeddingProvider?.name}) ` + + `declares ${activeDim}; dimensions seen on disk: ${distinct}. ` + + `First mismatched obsIds: ${sample}. Loading would silently corrupt ` + + `search (cross-dimension cosine returns 0). Choose one:\n` + + ` - Re-embed the existing index against the new provider, then start.\n` + + ` - Set AGENTMEMORY_DROP_STALE_INDEX=true to discard the persisted ` + + `vectors and rebuild from live observations.\n` + + ` - Switch the embedding provider back to the one that wrote the index.`, + ); + } + } + } + + // BM25 is not persisted: it is rebuilt from the content rows on every boot + // - one ordered walk, vectors untouched (they hydrated above). + { + const t0 = performance.now(); + const r = await rebuildBm25FromContent(sdk.store.db); + bootLog( + `BM25 rebuilt from content: ${r.docs} docs of ${r.rows} rows in ` + + `${Math.round(performance.now() - t0)} ms (read ${Math.round(r.readMs)} ms, ` + + `index ${Math.round(r.indexMs)} ms${r.skipped ? `, ${r.skipped} rows skipped` : ""})`, + ); + } + + // Maintenance routes: mem::backup, and mem::index-debug-legs when + // AGENTMEMORY_INDEX_DEBUG=1. Registered before the gate opens so they + // exist the moment readyz answers 200. + registerMaintenanceFunctions(sdk, sdk.store); + + // Readiness: DB opened, a write probe round-tripped, vectors hydrated, + // BM25 rebuilt -> readyz 200 and the 503 gate opens. The probe uses the + // same key the health monitor writes. A failure here is fatal on purpose: + // systemd restarts a daemon that cannot write its store. + { + const stamp = Date.now(); + sdk.store.set(KV.health, "_probe", { ts: stamp }); + const back = sdk.store.get(KV.health, "_probe") as { ts?: number } | null; + if (back?.ts !== stamp) throw new Error("readiness write probe did not round-trip"); + sdk.setReady(); + bootLog("Ready: /agentmemory/readyz -> 200, routes open"); + } + + // One-time repair of stores written before this fork: graph provenance that + // predates the write-time bound, and the index shards the removed engine + // left behind. Not awaited — readyz already answers 200 — and the pass + // itself yields between chunks, so requests arriving during it are served + // rather than queued. It records its own version, so every later boot is a + // single marker read. + void runStartupMaintenance(sdk.store).catch((err: unknown) => { + console.warn( + `[agentmemory] startup maintenance skipped: ${err instanceof Error ? err.message : String(err)}`, + ); + }); + + // The vector fill/repair pass closes the gap between the content rows and + // the vectors table - missing or stale rows are embedded, orphans pruned. + // Runs once at boot after readiness (not awaited: a large backlog after an + // import must not hold up the listener) and hourly. + if (vectorIndex) { + const fill = createIndexFill(sdk.store, vectorStore, vectorIndex); + registerIndexFillFunction(sdk, fill); + void fill.run(); + setInterval(() => void fill.run(), 3_600_000).unref(); + bootLog("Vector fill pass: at boot and hourly (mem::index-fill-missing)"); + } + + // Backfill the graph read side-indexes for corpora that predate them. + // Mirrors the BM25 memories backfill above: one-time, gated on the + // snapshot's recorded node count so we never enumerate a corpus large + // enough to starve the worker heartbeat. + // + // graph-read-fix local delta (B-mode boot-skip): when the leg is off + // this block is skipped ENTIRELY — it makes the only graph-scope reads + // at boot (the kv.list arming enumeration), and B-mode must issue zero. + // Skipping is safe: with the marker absent the readers fail-closed + // (graphReadable === false) rather than enumerating. + if (!graphLegDisabled()) { + try { + if (!(await graphIndexesReady(kv))) { + const graphSnap = await kv.get( + KV.graphSnapshot, + "current", + ); + const totalNodes = graphSnap?.stats?.totalNodes ?? 0; + if (graphSnap && totalNodes > 0 && totalNodes <= GRAPH_INDEX_NODE_CEILING) { + const [graphNodes, graphEdges] = await Promise.all([ + kv.list(KV.graphNodes), + kv.list(KV.graphEdges), + ]); + await backfillGraphIndexes( + kv, + graphNodes.filter((n) => !n.stale), + graphEdges.filter((e) => !e.stale), + ); + bootLog(`Backfilled graph read indexes (${totalNodes} nodes)`); + } + } + } catch (err) { + console.warn(`[agentmemory] Failed to backfill graph indexes:`, err); + } + } + + // Ready / Endpoints lines are emitted via `bootLog` so they're + // buffered in quiet mode and printed verbatim under --verbose. The + // CLI surfaces a compact summary when it sees the worker reach + // ready state. + bootLog( + `Ready. ${embeddingProvider ? "Triple-stream (BM25+Vector+Graph)" : "BM25+Graph"} search active.`, + ); + bootLog( + `REST API: 132 endpoints at http://localhost:${config.restPort}/agentmemory/*`, + ); + bootLog( + `MCP surface (opt-in via \`npx @agentmemory/mcp\`): ${getAllTools().length} tools · 6 resources · 3 prompts`, + ); + + const viewerPort = config.restPort + 2; + const viewerServer = startViewerServer( + viewerPort, + kv, + sdk, + secret, + config.restPort, + ); + + const autoForgetIntervalMs = parseInt(process.env.AUTO_FORGET_INTERVAL_MS || "3600000", 10); + const consolidationIntervalMs = parseInt(process.env.CONSOLIDATION_INTERVAL_MS || "7200000", 10); + + if (process.env.AUTO_FORGET_ENABLED !== "false") { + const autoForgetTimer = setInterval(async () => { + try { + await sdk.trigger({ function_id: "mem::auto-forget", payload: { dryRun: false } }); + } catch {} + }, autoForgetIntervalMs); + autoForgetTimer.unref(); + bootLog(`Auto-forget: enabled (every ${autoForgetIntervalMs / 60000}m)`); + } + + if (process.env.LESSON_DECAY_ENABLED !== "false") { + const lessonDecayTimer = setInterval(async () => { + try { + await sdk.trigger({ function_id: "mem::lesson-decay-sweep", payload: {} }); + } catch {} + }, 86400000); + lessonDecayTimer.unref(); + bootLog(`Lesson decay sweep: enabled (every 24h)`); + } + + if (process.env.EVICTION_ENABLED !== "false") { + const evictionTimer = setInterval(async () => { + try { + await sdk.trigger({ function_id: "mem::evict", payload: { dryRun: false } }); + } catch {} + }, 86400000); + evictionTimer.unref(); + bootLog(`Eviction sweep: enabled (every 24h)`); + } + + if (process.env.INSIGHT_DECAY_ENABLED !== "false") { + const insightDecayTimer = setInterval(async () => { + try { + await sdk.trigger({ function_id: "mem::insight-decay-sweep", payload: {} }); + } catch {} + }, 86400000); + insightDecayTimer.unref(); + } + + // #771: hourly TTL sweep for the followup-rate diagnostic. The + // recent-searches scope only needs the last entry per session; + // sweeping anything older than the retention window keeps the scope + // from growing unbounded across long-lived deployments. + const recentSearchesSweepTimer = setInterval(async () => { + try { + await sdk.trigger({ + function_id: "mem::diagnostic::recent-searches-sweep", + payload: {}, + }); + } catch {} + }, 60 * 60 * 1000); + recentSearchesSweepTimer.unref(); + + if (isConsolidationEnabled()) { + const consolidationTimer = setInterval(async () => { + try { + await sdk.trigger({ function_id: "mem::consolidate-pipeline", payload: {} }); + } catch {} + }, consolidationIntervalMs); + consolidationTimer.unref(); + bootLog(`Auto-consolidation: enabled (every ${consolidationIntervalMs / 60000}m)`); + } + + const shutdown = async () => { + console.log(`\n[agentmemory] Shutting down...`); + healthMonitor.stop(); + dedupMap.stop(); + await new Promise((resolve) => viewerServer.close(() => resolve())); + await sdk.shutdown(); + clearWorkerPidfile(); + process.exit(0); + }; + process.on("SIGINT", shutdown); + process.on("SIGTERM", shutdown); +} + +main().catch((err) => { + console.error(`[agentmemory] Fatal:`, err); + process.exit(1); +}); From 98d246c979e9f74366a514d69da0c15bf5ad32be Mon Sep 17 00:00:00 2001 From: neal Date: Fri, 25 Sep 2026 18:33:45 -0400 Subject: [PATCH 05/17] fix(lessons): recall every live lesson by default lesson-recall defaulted minConfidence to 0.1 while the decay sweep floors a reinforced lesson at 0.05 and keeps it live, so every correction old enough to have decayed was invisible to default recall. The default is now 0, matching lesson-list; soft-delete remains the gate and confidence still ranks decayed lessons last. Upstream #1065. Generated-By: Claude Opus 5.5 (1M context) --- src/functions/lessons.ts | 764 ++++++++++++++++--------------- src/mcp/tools-registry.ts | 2 +- test/lesson-index-recall.test.ts | 23 + 3 files changed, 407 insertions(+), 382 deletions(-) diff --git a/src/functions/lessons.ts b/src/functions/lessons.ts index 1dbae9123b..a1c58a14bb 100644 --- a/src/functions/lessons.ts +++ b/src/functions/lessons.ts @@ -1,381 +1,383 @@ -import type { ISdk } from "../engine/types.js"; -import type { StateKV } from "../state/kv.js"; -import { KV, fingerprintId } from "../state/schema.js"; -import type { Lesson } from "../types.js"; -import { SearchIndex } from "../state/search-index.js"; -import { lessonToObservation } from "../state/memory-utils.js"; -import { recordAudit } from "./audit.js"; -import { scrubFields } from "./privacy.js"; - -// Dedicated BM25 index for lessons, with the full records cached -// alongside it. Recall previously listed every lesson from KV and -// substring-matched per query — O(corpus) per call with no term -// weighting. Index and record cache are built lazily from one KV list -// (the same cost a single recall used to pay) and kept current -// incrementally on save/delete/decay. Confidence x recency reranking -// stays exactly as before — the index only replaces the relevance term, -// and the record cache keeps recall at zero KV round-trips. -let lessonIndex: SearchIndex | null = null; -const lessonRecords = new Map(); -let lessonIndexBuild: Promise | null = null; -let lessonIndexGeneration = 0; - -export function resetLessonIndex(): void { - lessonIndexGeneration++; - lessonIndex = null; - lessonRecords.clear(); -} - -function noteLessonMutation(): void { - if (!lessonIndex && lessonIndexBuild) resetLessonIndex(); -} - -async function ensureLessonIndex(kv: StateKV): Promise { - if (lessonIndex) return lessonIndex; - if (!lessonIndexBuild) { - const generation = lessonIndexGeneration; - lessonIndexBuild = (async () => { - const idx = new SearchIndex(); - const all = await kv.list(KV.lessons); - if (generation !== lessonIndexGeneration) return; - for (const l of all) { - if (!l.deleted) { - idx.add(lessonToObservation(l)); - lessonRecords.set(l.id, l); - } - } - lessonIndex = idx; - })().finally(() => { - lessonIndexBuild = null; - }); - } - await lessonIndexBuild; - return lessonIndex ?? ensureLessonIndex(kv); -} - -function reinforceLesson(lesson: Lesson): void { - const now = new Date().toISOString(); - lesson.reinforcements++; - lesson.confidence = Math.min( - 1.0, - lesson.confidence + 0.1 * (1 - lesson.confidence), - ); - lesson.lastReinforcedAt = now; - lesson.updatedAt = now; -} - -export function registerLessonsFunctions(sdk: ISdk, kv: StateKV): void { - sdk.registerFunction("mem::lesson-save", - async (data: { - content: string; - context?: string; - confidence?: number; - project?: string; - tags?: string[]; - source?: "crystal" | "manual" | "consolidation"; - sourceIds?: string[]; - }) => { - data = scrubFields(data, "content", "context"); - if (!data.content?.trim()) { - return { success: false, error: "content is required" }; - } - - const fp = fingerprintId("lsn", data.content.trim().toLowerCase()); - const existing = await kv.get(KV.lessons, fp); - - if (existing && !existing.deleted) { - reinforceLesson(existing); - let indexedTextChanged = false; - if (data.context && !existing.context) { - existing.context = data.context; - indexedTextChanged = true; - } - await kv.set(KV.lessons, existing.id, existing); - lessonRecords.set(existing.id, existing); - if (indexedTextChanged && lessonIndex) { - lessonIndex.remove(existing.id); - lessonIndex.add(lessonToObservation(existing)); - } - noteLessonMutation(); - - try { - await recordAudit(kv, "lesson_strengthen", "mem::lesson-save", [ - existing.id, - ]); - } catch {} - - return { - success: true, - action: "strengthened", - lesson: existing, - }; - } - - const confidence = - typeof data.confidence === "number" && - data.confidence >= 0 && - data.confidence <= 1 - ? data.confidence - : 0.5; - - const now = new Date().toISOString(); - const lesson: Lesson = { - id: fp, - content: data.content.trim(), - context: data.context?.trim() || "", - confidence, - reinforcements: 0, - source: data.source || "manual", - sourceIds: data.sourceIds || [], - project: data.project, - tags: data.tags || [], - createdAt: now, - updatedAt: now, - decayRate: 0.05, - }; - - await kv.set(KV.lessons, lesson.id, lesson); - lessonRecords.set(lesson.id, lesson); - if (lessonIndex) lessonIndex.add(lessonToObservation(lesson)); - noteLessonMutation(); - - try { - await recordAudit(kv, "lesson_save", "mem::lesson-save", [lesson.id]); - } catch {} - - return { success: true, action: "created", lesson }; - }, - ); - - sdk.registerFunction("mem::lesson-recall", - async (data: { - query: string; - project?: string; - minConfidence?: number; - limit?: number; - }) => { - if (!data.query?.trim()) { - return { success: false, error: "query is required" }; - } - - const minConfidence = data.minConfidence ?? 0.1; - const limit = data.limit ?? 10; - - const idx = await ensureLessonIndex(kv); - const filtering = !!data.project || minConfidence > 0.1; - const fetchLimit = filtering - ? Math.max(limit * 10, 100) - : Math.max(limit * 5, 50); - const hits = idx.search(data.query, fetchLimit); - const maxHit = hits.length > 0 ? hits[0].score : 0; - - const scored: Array<{ lesson: Lesson; score: number }> = []; - for (let i = 0; i < hits.length; i++) { - const l = lessonRecords.get(hits[i].obsId); - if (!l || l.deleted || l.confidence < minConfidence) continue; - if (data.project && l.project !== data.project) continue; - - const relevance = maxHit > 0 ? hits[i].score / maxHit : 0; - const daysSinceReinforced = l.lastReinforcedAt - ? (Date.now() - new Date(l.lastReinforcedAt).getTime()) / - (1000 * 60 * 60 * 24) - : (Date.now() - new Date(l.createdAt).getTime()) / - (1000 * 60 * 60 * 24); - const recencyBoost = 1 / (1 + daysSinceReinforced * 0.01); - scored.push({ lesson: l, score: l.confidence * relevance * recencyBoost }); - } - - scored.sort( - (a, b) => - b.score - a.score || - (a.lesson.id < b.lesson.id ? -1 : a.lesson.id > b.lesson.id ? 1 : 0), - ); - - try { - await recordAudit(kv, "lesson_recall", "mem::lesson-recall", [], { - query: data.query, - resultCount: scored.length, - }); - } catch {} - - return { - success: true, - lessons: scored.slice(0, limit).map((s) => ({ - ...s.lesson, - score: Math.round(s.score * 1000) / 1000, - })), - }; - }, - ); - - sdk.registerFunction("mem::lesson-list", - async (data: { - project?: string; - source?: string; - minConfidence?: number; - limit?: number; - }) => { - const limit = data.limit ?? 50; - const minConfidence = data.minConfidence ?? 0; - let lessons = await kv.list(KV.lessons); - - lessons = lessons.filter( - (l) => !l.deleted && l.confidence >= minConfidence, - ); - - if (data.project) { - lessons = lessons.filter((l) => l.project === data.project); - } - if (data.source) { - lessons = lessons.filter((l) => l.source === data.source); - } - - lessons.sort((a, b) => b.confidence - a.confidence); - - return { success: true, lessons: lessons.slice(0, limit) }; - }, - ); - - sdk.registerFunction("mem::lesson-strengthen", - async (data: { lessonId: string }) => { - if (!data.lessonId) { - return { success: false, error: "lessonId is required" }; - } - - const lesson = await kv.get(KV.lessons, data.lessonId); - if (!lesson || lesson.deleted) { - return { success: false, error: "lesson not found" }; - } - - reinforceLesson(lesson); - - await kv.set(KV.lessons, lesson.id, lesson); - lessonRecords.set(lesson.id, lesson); - noteLessonMutation(); - - try { - await recordAudit(kv, "lesson_strengthen", "mem::lesson-strengthen", [ - lesson.id, - ]); - } catch {} - - return { success: true, lesson }; - }, - ); - - sdk.registerFunction("mem::lesson-delete", - async (data: { lessonId: string }) => { - if (!data.lessonId) { - return { success: false, error: "lessonId is required" }; - } - - const lesson = await kv.get(KV.lessons, data.lessonId); - if (!lesson || lesson.deleted) { - return { success: false, error: "lesson not found" }; - } - - lesson.deleted = true; - lesson.updatedAt = new Date().toISOString(); - - await kv.set(KV.lessons, lesson.id, lesson); - lessonRecords.delete(lesson.id); - if (lessonIndex) lessonIndex.remove(lesson.id); - noteLessonMutation(); - - try { - await recordAudit(kv, "lesson_delete", "mem::lesson-delete", [ - lesson.id, - ]); - } catch {} - - return { success: true, lesson }; - }, - ); - - sdk.registerFunction("mem::lesson-decay-sweep", - async () => { - const lessons = await kv.list(KV.lessons); - let decayed = 0; - let softDeleted = 0; - const now = Date.now(); - const timestamp = new Date().toISOString(); - const dirty: Lesson[] = []; - const auditEvents: Array<{ - id: string; - action: "decay" | "soft-delete"; - beforeConfidence: number; - afterConfidence: number; - beforeDeleted: boolean; - afterDeleted: boolean; - }> = []; - - for (const lesson of lessons) { - if (lesson.deleted) continue; - - const baseline = lesson.lastDecayedAt || lesson.lastReinforcedAt || lesson.createdAt; - const weeksSinceBaseline = - (now - new Date(baseline).getTime()) / (1000 * 60 * 60 * 24 * 7); - - if (weeksSinceBaseline < 1) continue; - - const decay = lesson.decayRate * weeksSinceBaseline; - const newConfidence = Math.max(0.05, lesson.confidence - decay); - - if (newConfidence !== lesson.confidence) { - const beforeConfidence = lesson.confidence; - const beforeDeleted = !!lesson.deleted; - lesson.confidence = Math.round(newConfidence * 1000) / 1000; - lesson.lastDecayedAt = timestamp; - lesson.updatedAt = timestamp; - - if (lesson.confidence <= 0.1 && lesson.reinforcements === 0) { - lesson.deleted = true; - softDeleted++; - } else { - decayed++; - } - - dirty.push(lesson); - auditEvents.push({ - id: lesson.id, - action: lesson.deleted ? "soft-delete" : "decay", - beforeConfidence, - afterConfidence: lesson.confidence, - beforeDeleted, - afterDeleted: !!lesson.deleted, - }); - } - } - - // Awaited batches, not a fan-out: N un-awaited sets run back to back on - // the event loop under inproc (see StateKV.setMany). - await kv.setMany(KV.lessons, dirty.map((l) => ({ key: l.id, value: l }))); - for (const l of dirty) { - if (l.deleted) { - lessonRecords.delete(l.id); - if (lessonIndex) lessonIndex.remove(l.id); - } else { - lessonRecords.set(l.id, l); - } - } - if (dirty.length > 0) noteLessonMutation(); - for (const event of auditEvents) { - await recordAudit(kv, "lesson_strengthen", "mem::lesson-decay-sweep", [event.id], { - action: event.action, - actor: "system", - reason: "decay-sweep", - before: { - confidence: event.beforeConfidence, - deleted: event.beforeDeleted, - }, - after: { - confidence: event.afterConfidence, - deleted: event.afterDeleted, - }, - }); - } - - return { success: true, decayed, softDeleted, total: lessons.length }; - }, - ); -} +import type { ISdk } from "../engine/types.js"; +import type { StateKV } from "../state/kv.js"; +import { KV, fingerprintId } from "../state/schema.js"; +import type { Lesson } from "../types.js"; +import { SearchIndex } from "../state/search-index.js"; +import { lessonToObservation } from "../state/memory-utils.js"; +import { recordAudit } from "./audit.js"; +import { scrubFields } from "./privacy.js"; + +// Dedicated BM25 index for lessons, with the full records cached +// alongside it. Recall previously listed every lesson from KV and +// substring-matched per query — O(corpus) per call with no term +// weighting. Index and record cache are built lazily from one KV list +// (the same cost a single recall used to pay) and kept current +// incrementally on save/delete/decay. Confidence x recency reranking +// stays exactly as before — the index only replaces the relevance term, +// and the record cache keeps recall at zero KV round-trips. +let lessonIndex: SearchIndex | null = null; +const lessonRecords = new Map(); +let lessonIndexBuild: Promise | null = null; +let lessonIndexGeneration = 0; + +export function resetLessonIndex(): void { + lessonIndexGeneration++; + lessonIndex = null; + lessonRecords.clear(); +} + +function noteLessonMutation(): void { + if (!lessonIndex && lessonIndexBuild) resetLessonIndex(); +} + +async function ensureLessonIndex(kv: StateKV): Promise { + if (lessonIndex) return lessonIndex; + if (!lessonIndexBuild) { + const generation = lessonIndexGeneration; + lessonIndexBuild = (async () => { + const idx = new SearchIndex(); + const all = await kv.list(KV.lessons); + if (generation !== lessonIndexGeneration) return; + for (const l of all) { + if (!l.deleted) { + idx.add(lessonToObservation(l)); + lessonRecords.set(l.id, l); + } + } + lessonIndex = idx; + })().finally(() => { + lessonIndexBuild = null; + }); + } + await lessonIndexBuild; + return lessonIndex ?? ensureLessonIndex(kv); +} + +function reinforceLesson(lesson: Lesson): void { + const now = new Date().toISOString(); + lesson.reinforcements++; + lesson.confidence = Math.min( + 1.0, + lesson.confidence + 0.1 * (1 - lesson.confidence), + ); + lesson.lastReinforcedAt = now; + lesson.updatedAt = now; +} + +export function registerLessonsFunctions(sdk: ISdk, kv: StateKV): void { + sdk.registerFunction("mem::lesson-save", + async (data: { + content: string; + context?: string; + confidence?: number; + project?: string; + tags?: string[]; + source?: "crystal" | "manual" | "consolidation"; + sourceIds?: string[]; + }) => { + data = scrubFields(data, "content", "context"); + if (!data.content?.trim()) { + return { success: false, error: "content is required" }; + } + + const fp = fingerprintId("lsn", data.content.trim().toLowerCase()); + const existing = await kv.get(KV.lessons, fp); + + if (existing && !existing.deleted) { + reinforceLesson(existing); + let indexedTextChanged = false; + if (data.context && !existing.context) { + existing.context = data.context; + indexedTextChanged = true; + } + await kv.set(KV.lessons, existing.id, existing); + lessonRecords.set(existing.id, existing); + if (indexedTextChanged && lessonIndex) { + lessonIndex.remove(existing.id); + lessonIndex.add(lessonToObservation(existing)); + } + noteLessonMutation(); + + try { + await recordAudit(kv, "lesson_strengthen", "mem::lesson-save", [ + existing.id, + ]); + } catch {} + + return { + success: true, + action: "strengthened", + lesson: existing, + }; + } + + const confidence = + typeof data.confidence === "number" && + data.confidence >= 0 && + data.confidence <= 1 + ? data.confidence + : 0.5; + + const now = new Date().toISOString(); + const lesson: Lesson = { + id: fp, + content: data.content.trim(), + context: data.context?.trim() || "", + confidence, + reinforcements: 0, + source: data.source || "manual", + sourceIds: data.sourceIds || [], + project: data.project, + tags: data.tags || [], + createdAt: now, + updatedAt: now, + decayRate: 0.05, + }; + + await kv.set(KV.lessons, lesson.id, lesson); + lessonRecords.set(lesson.id, lesson); + if (lessonIndex) lessonIndex.add(lessonToObservation(lesson)); + noteLessonMutation(); + + try { + await recordAudit(kv, "lesson_save", "mem::lesson-save", [lesson.id]); + } catch {} + + return { success: true, action: "created", lesson }; + }, + ); + + sdk.registerFunction("mem::lesson-recall", + async (data: { + query: string; + project?: string; + minConfidence?: number; + limit?: number; + }) => { + if (!data.query?.trim()) { + return { success: false, error: "query is required" }; + } + + // Soft-delete is the only default gate: a reinforced lesson decays to + // the 0.05 floor and stays live, and confidence already ranks it low. + const minConfidence = data.minConfidence ?? 0; + const limit = data.limit ?? 10; + + const idx = await ensureLessonIndex(kv); + const filtering = !!data.project || minConfidence > 0; + const fetchLimit = filtering + ? Math.max(limit * 10, 100) + : Math.max(limit * 5, 50); + const hits = idx.search(data.query, fetchLimit); + const maxHit = hits.length > 0 ? hits[0].score : 0; + + const scored: Array<{ lesson: Lesson; score: number }> = []; + for (let i = 0; i < hits.length; i++) { + const l = lessonRecords.get(hits[i].obsId); + if (!l || l.deleted || l.confidence < minConfidence) continue; + if (data.project && l.project !== data.project) continue; + + const relevance = maxHit > 0 ? hits[i].score / maxHit : 0; + const daysSinceReinforced = l.lastReinforcedAt + ? (Date.now() - new Date(l.lastReinforcedAt).getTime()) / + (1000 * 60 * 60 * 24) + : (Date.now() - new Date(l.createdAt).getTime()) / + (1000 * 60 * 60 * 24); + const recencyBoost = 1 / (1 + daysSinceReinforced * 0.01); + scored.push({ lesson: l, score: l.confidence * relevance * recencyBoost }); + } + + scored.sort( + (a, b) => + b.score - a.score || + (a.lesson.id < b.lesson.id ? -1 : a.lesson.id > b.lesson.id ? 1 : 0), + ); + + try { + await recordAudit(kv, "lesson_recall", "mem::lesson-recall", [], { + query: data.query, + resultCount: scored.length, + }); + } catch {} + + return { + success: true, + lessons: scored.slice(0, limit).map((s) => ({ + ...s.lesson, + score: Math.round(s.score * 1000) / 1000, + })), + }; + }, + ); + + sdk.registerFunction("mem::lesson-list", + async (data: { + project?: string; + source?: string; + minConfidence?: number; + limit?: number; + }) => { + const limit = data.limit ?? 50; + const minConfidence = data.minConfidence ?? 0; + let lessons = await kv.list(KV.lessons); + + lessons = lessons.filter( + (l) => !l.deleted && l.confidence >= minConfidence, + ); + + if (data.project) { + lessons = lessons.filter((l) => l.project === data.project); + } + if (data.source) { + lessons = lessons.filter((l) => l.source === data.source); + } + + lessons.sort((a, b) => b.confidence - a.confidence); + + return { success: true, lessons: lessons.slice(0, limit) }; + }, + ); + + sdk.registerFunction("mem::lesson-strengthen", + async (data: { lessonId: string }) => { + if (!data.lessonId) { + return { success: false, error: "lessonId is required" }; + } + + const lesson = await kv.get(KV.lessons, data.lessonId); + if (!lesson || lesson.deleted) { + return { success: false, error: "lesson not found" }; + } + + reinforceLesson(lesson); + + await kv.set(KV.lessons, lesson.id, lesson); + lessonRecords.set(lesson.id, lesson); + noteLessonMutation(); + + try { + await recordAudit(kv, "lesson_strengthen", "mem::lesson-strengthen", [ + lesson.id, + ]); + } catch {} + + return { success: true, lesson }; + }, + ); + + sdk.registerFunction("mem::lesson-delete", + async (data: { lessonId: string }) => { + if (!data.lessonId) { + return { success: false, error: "lessonId is required" }; + } + + const lesson = await kv.get(KV.lessons, data.lessonId); + if (!lesson || lesson.deleted) { + return { success: false, error: "lesson not found" }; + } + + lesson.deleted = true; + lesson.updatedAt = new Date().toISOString(); + + await kv.set(KV.lessons, lesson.id, lesson); + lessonRecords.delete(lesson.id); + if (lessonIndex) lessonIndex.remove(lesson.id); + noteLessonMutation(); + + try { + await recordAudit(kv, "lesson_delete", "mem::lesson-delete", [ + lesson.id, + ]); + } catch {} + + return { success: true, lesson }; + }, + ); + + sdk.registerFunction("mem::lesson-decay-sweep", + async () => { + const lessons = await kv.list(KV.lessons); + let decayed = 0; + let softDeleted = 0; + const now = Date.now(); + const timestamp = new Date().toISOString(); + const dirty: Lesson[] = []; + const auditEvents: Array<{ + id: string; + action: "decay" | "soft-delete"; + beforeConfidence: number; + afterConfidence: number; + beforeDeleted: boolean; + afterDeleted: boolean; + }> = []; + + for (const lesson of lessons) { + if (lesson.deleted) continue; + + const baseline = lesson.lastDecayedAt || lesson.lastReinforcedAt || lesson.createdAt; + const weeksSinceBaseline = + (now - new Date(baseline).getTime()) / (1000 * 60 * 60 * 24 * 7); + + if (weeksSinceBaseline < 1) continue; + + const decay = lesson.decayRate * weeksSinceBaseline; + const newConfidence = Math.max(0.05, lesson.confidence - decay); + + if (newConfidence !== lesson.confidence) { + const beforeConfidence = lesson.confidence; + const beforeDeleted = !!lesson.deleted; + lesson.confidence = Math.round(newConfidence * 1000) / 1000; + lesson.lastDecayedAt = timestamp; + lesson.updatedAt = timestamp; + + if (lesson.confidence <= 0.1 && lesson.reinforcements === 0) { + lesson.deleted = true; + softDeleted++; + } else { + decayed++; + } + + dirty.push(lesson); + auditEvents.push({ + id: lesson.id, + action: lesson.deleted ? "soft-delete" : "decay", + beforeConfidence, + afterConfidence: lesson.confidence, + beforeDeleted, + afterDeleted: !!lesson.deleted, + }); + } + } + + // Awaited batches, not a fan-out: N un-awaited sets run back to back on + // the event loop under inproc (see StateKV.setMany). + await kv.setMany(KV.lessons, dirty.map((l) => ({ key: l.id, value: l }))); + for (const l of dirty) { + if (l.deleted) { + lessonRecords.delete(l.id); + if (lessonIndex) lessonIndex.remove(l.id); + } else { + lessonRecords.set(l.id, l); + } + } + if (dirty.length > 0) noteLessonMutation(); + for (const event of auditEvents) { + await recordAudit(kv, "lesson_strengthen", "mem::lesson-decay-sweep", [event.id], { + action: event.action, + actor: "system", + reason: "decay-sweep", + before: { + confidence: event.beforeConfidence, + deleted: event.beforeDeleted, + }, + after: { + confidence: event.afterConfidence, + deleted: event.afterDeleted, + }, + }); + } + + return { success: true, decayed, softDeleted, total: lessons.length }; + }, + ); +} diff --git a/src/mcp/tools-registry.ts b/src/mcp/tools-registry.ts index 849e1f2f36..0288100f35 100644 --- a/src/mcp/tools-registry.ts +++ b/src/mcp/tools-registry.ts @@ -836,7 +836,7 @@ export const V070_TOOLS: McpToolDef[] = [ project: { type: "string", description: "Filter by project" }, minConfidence: { type: "number", - description: "Minimum confidence threshold (default 0.1)", + description: "Minimum confidence threshold (default 0)", }, limit: { type: "number", description: "Max results (default 10)" }, }, diff --git a/test/lesson-index-recall.test.ts b/test/lesson-index-recall.test.ts index d9bb8f0f2d..fc7d5441ac 100644 --- a/test/lesson-index-recall.test.ts +++ b/test/lesson-index-recall.test.ts @@ -61,6 +61,29 @@ describe("lesson recall through the lesson index", () => { expect(res.lessons[0].score).toBeGreaterThan(res.lessons[1].score); }); + it("recalls a reinforced lesson that decayed to the confidence floor", async () => { + const { sdk, kv } = await setup(); + await kv.set("mem:lessons", "lsn_floor", { + id: "lsn_floor", + content: "never force-push a shared branch", + context: "", + confidence: 0.05, + reinforcements: 1, + source: "manual", + sourceIds: [], + tags: ["git"], + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + decayRate: 0.05, + }); + + const res = (await sdk.trigger("mem::lesson-recall", { + query: "force-push shared branch", + })) as { lessons: Array<{ id: string }> }; + + expect(res.lessons.map((l) => l.id)).toEqual(["lsn_floor"]); + }); + it("recalls lessons that existed before the index was built (lazy rebuild)", async () => { const { sdk, kv } = await setup(); await kv.set("mem:lessons", "lsn_pre", { From cc30b8e2e540441a166f8525deb019be02a8a21e Mon Sep 17 00:00:00 2001 From: neal Date: Fri, 25 Sep 2026 18:34:52 -0400 Subject: [PATCH 06/17] fix(audit): prune the audit store to AGENTMEMORY_AUDIT_MAX daily recordAudit wrote a row per state change and nothing ever removed one, so mem:audit grew for the life of the install. A daily sweep now keeps the newest AGENTMEMORY_AUDIT_MAX entries (default 50000) and logs what it removed; AGENTMEMORY_AUDIT_STORE=off still skips the store entirely. Upstream PR#1176. Generated-By: Claude Opus 5.5 (1M context) --- .env.example | 1 + plugin/skills/agentmemory-config/REFERENCE.md | 3 +- src/functions/audit.ts | 356 ++++++++++-------- src/index.ts | 8 + test/audit.test.ts | 25 +- 5 files changed, 225 insertions(+), 168 deletions(-) diff --git a/.env.example b/.env.example index 2c293eb779..b6848ebbf4 100644 --- a/.env.example +++ b/.env.example @@ -238,4 +238,5 @@ # AGENTMEMORY_SQLITE_PATH=/var/lib/agentmemory/agentmemory.sqlite # The store (WAL mode); its directory must exist # AGENTMEMORY_AUDIT_STORE=off # Emit audit entries as `[agentmemory] audit ` log lines instead of KV rows; memory_audit then answers an empty list # AGENTMEMORY_AUDIT_LOG=all # With AUDIT_STORE=off: all | deletions (forget, delete, lesson_delete, slot_delete, core_remove, sketch_discard only) | off +# AGENTMEMORY_AUDIT_MAX=50000 # Newest audit rows the daily prune keeps in KV # AGENTMEMORY_LOG_LEVEL=info # Lowest logger level written: info | warn | error | off. Audit lines bypass it. diff --git a/plugin/skills/agentmemory-config/REFERENCE.md b/plugin/skills/agentmemory-config/REFERENCE.md index f94de4cb50..59cdd5a343 100644 --- a/plugin/skills/agentmemory-config/REFERENCE.md +++ b/plugin/skills/agentmemory-config/REFERENCE.md @@ -3,13 +3,14 @@ Generated by scanning `src/` for `AGENTMEMORY_*` usage. Do not edit the block below by hand; run `npm run skills:gen` after adding or removing a variable. Internal markers ending in two underscores are excluded. -Configuration is read from the environment and from `~/.agentmemory/.env` (no `export` prefix). 71 recognized variables: +Configuration is read from the environment and from `~/.agentmemory/.env` (no `export` prefix). 72 recognized variables: - `AGENTMEMORY_AGENT_SCOPE` - `AGENTMEMORY_AGENT_SDK_MAX_CONCURRENCY` - `AGENTMEMORY_AGENT_SDK_TIMEOUT_MS` - `AGENTMEMORY_ALLOW_AGENT_SDK` - `AGENTMEMORY_AUDIT_LOG` +- `AGENTMEMORY_AUDIT_MAX` - `AGENTMEMORY_AUDIT_STORE` - `AGENTMEMORY_AUTO_COMPRESS` - `AGENTMEMORY_CHUNK_MAX_CHARS` diff --git a/src/functions/audit.ts b/src/functions/audit.ts index 5ba639d4c2..ad9c01d697 100644 --- a/src/functions/audit.ts +++ b/src/functions/audit.ts @@ -1,166 +1,190 @@ -import type { AuditEntry } from "../types.js"; -import { KV, generateId } from "../state/schema.js"; -import type { StateKV } from "../state/kv.js"; -import { logger } from "../logger.js"; - -// Audit coverage policy (issue #125). -// -// Every structural deletion of a memory, observation, session, or -// semantic row MUST call recordAudit. Two shapes are allowed, keyed to -// whether the caller is scoped or bulk: -// -// Scoped deletions — a user-visible, per-call action removing a -// bounded set of items. Emit ONE audit row per call with targetIds -// populated. Examples: mem::governance-delete, mem::forget. -// -// Bulk deletions — automatic sweeps (retention, TTL eviction, -// auto-forget) that can remove hundreds of rows per invocation. -// Emit ONE batched audit row per invocation with targetIds listing -// every removed id and details.evicted holding the count. Per-item -// audit rows would flood the audit log during routine sweeps. -// -// Either shape is required; silent deletes are not acceptable. -// -// operation field: -// - "delete" — permanent removal (governance, retention sweep, evict). -// - "forget" — forget/removal flows. Scoped when emitted by -// mem::forget (user-initiated); bulk-batched when -// emitted by mem::auto-forget (automatic sweep). -// - everything else — see AuditEntry["operation"] union in src/types.ts. -// -// When adding a new deletion path, add an explicit recordAudit call -// BEFORE kv.delete(...) and match one of the two shapes above. -// -// Storage: with AGENTMEMORY_AUDIT_STORE=off the entry is emitted as one -// structured log line (journald -> Loki keeps the evidence: operation, -// functionId, targetIds, details) and nothing is written to KV; the audit -// query then answers an empty list. Any other value keeps the KV store. -// -// Which entries become lines when the store is off: AGENTMEMORY_AUDIT_LOG= -// all (default) | deletions | off. `deletions` keeps every path that removes -// data - the six removal operations plus the removals that hide under a -// broader name: heal with action "delete", mesh peer removal, an import -// that replaces the store, and the soft deletions (a lesson or insight -// decayed to deleted, a sketch discarded by heal) - and drops the -// per-operation chatter (observe, compress, heal repairs ...): a few lines -// a month instead of ~100 KB a day. The details checks mirror the payloads -// at those call sites; a new deletion path needs either a listed operation -// or one of these markers. - -function auditStoreOff(): boolean { - return (process.env.AGENTMEMORY_AUDIT_STORE ?? "").toLowerCase() === "off"; -} - -const DELETION_OPS: ReadonlySet = new Set([ - "forget", "delete", "lesson_delete", "slot_delete", "core_remove", "sketch_discard", -]); - -function isDeletion(operation: AuditEntry["operation"], details: Record): boolean { - if (DELETION_OPS.has(operation)) return true; - if (details.action === "delete" || details.action === "soft-delete" || details.action === "mesh.remove") return true; - if (details.newStatus === "discarded") return true; - if (typeof details.softDeleted === "number" && details.softDeleted > 0) return true; - return operation === "import" && details.strategy === "replace"; -} - -function auditLineWanted(operation: AuditEntry["operation"], details: Record): boolean { - const mode = (process.env.AGENTMEMORY_AUDIT_LOG ?? "all").toLowerCase(); - if (mode === "off") return false; - if (mode === "deletions") return isDeletion(operation, details); - return true; -} - -export async function recordAudit( - kv: StateKV, - operation: AuditEntry["operation"], - functionId: string, - targetIds: string[], - details: Record = {}, - qualityScore?: number, - userId?: string, -): Promise { - const entry: AuditEntry = { - id: generateId("aud"), - timestamp: new Date().toISOString(), - operation, - userId, - functionId, - targetIds, - details, - qualityScore, - }; - if (auditStoreOff()) { - if (!auditLineWanted(operation, details)) return entry; - logger.audit(operation, { - auditId: entry.id, - functionId, - targetIds, - targetCount: targetIds.length, - details, - ...(qualityScore !== undefined ? { qualityScore } : {}), - ...(userId !== undefined ? { userId } : {}), - }); - return entry; - } - await kv.set(KV.audit, entry.id, entry); - return entry; -} - -export async function safeAudit( - kv: StateKV, - operation: AuditEntry["operation"], - functionId: string, - targetIds: string[], - details: Record = {}, - qualityScore?: number, - userId?: string, -): Promise { - try { - await recordAudit(kv, operation, functionId, targetIds, details, qualityScore, userId); - } catch (err) { - try { - logger.warn("audit write failed", { - functionId, - operation, - targetIds, - error: err instanceof Error ? err.message : String(err), - }); - } catch {} - } -} - -export async function queryAudit( - kv: StateKV, - filter?: { - operation?: AuditEntry["operation"]; - dateFrom?: string; - dateTo?: string; - limit?: number; - }, -): Promise { - if (auditStoreOff()) return []; - const all = await kv.list(KV.audit); - let entries = [...all].sort( - (a, b) => new Date(b.timestamp).getTime() - new Date(a.timestamp).getTime(), - ); - - if (filter?.operation) { - entries = entries.filter((e) => e.operation === filter.operation); - } - if (filter?.dateFrom) { - const from = new Date(filter.dateFrom).getTime(); - if (Number.isNaN(from)) { - throw new Error(`Invalid dateFrom: ${filter.dateFrom}`); - } - entries = entries.filter((e) => new Date(e.timestamp).getTime() >= from); - } - if (filter?.dateTo) { - const to = new Date(filter.dateTo).getTime(); - if (Number.isNaN(to)) { - throw new Error(`Invalid dateTo: ${filter.dateTo}`); - } - entries = entries.filter((e) => new Date(e.timestamp).getTime() <= to); - } - - return entries.slice(0, filter?.limit || 100); -} +import type { AuditEntry } from "../types.js"; +import { KV, generateId } from "../state/schema.js"; +import type { StateKV } from "../state/kv.js"; +import { logger } from "../logger.js"; + +// Audit coverage policy (issue #125). +// +// Every structural deletion of a memory, observation, session, or +// semantic row MUST call recordAudit. Two shapes are allowed, keyed to +// whether the caller is scoped or bulk: +// +// Scoped deletions — a user-visible, per-call action removing a +// bounded set of items. Emit ONE audit row per call with targetIds +// populated. Examples: mem::governance-delete, mem::forget. +// +// Bulk deletions — automatic sweeps (retention, TTL eviction, +// auto-forget) that can remove hundreds of rows per invocation. +// Emit ONE batched audit row per invocation with targetIds listing +// every removed id and details.evicted holding the count. Per-item +// audit rows would flood the audit log during routine sweeps. +// +// Either shape is required; silent deletes are not acceptable. +// +// operation field: +// - "delete" — permanent removal (governance, retention sweep, evict). +// - "forget" — forget/removal flows. Scoped when emitted by +// mem::forget (user-initiated); bulk-batched when +// emitted by mem::auto-forget (automatic sweep). +// - everything else — see AuditEntry["operation"] union in src/types.ts. +// +// When adding a new deletion path, add an explicit recordAudit call +// BEFORE kv.delete(...) and match one of the two shapes above. +// +// Storage: with AGENTMEMORY_AUDIT_STORE=off the entry is emitted as one +// structured log line (journald -> Loki keeps the evidence: operation, +// functionId, targetIds, details) and nothing is written to KV; the audit +// query then answers an empty list. Any other value keeps the KV store. +// +// Which entries become lines when the store is off: AGENTMEMORY_AUDIT_LOG= +// all (default) | deletions | off. `deletions` keeps every path that removes +// data - the six removal operations plus the removals that hide under a +// broader name: heal with action "delete", mesh peer removal, an import +// that replaces the store, and the soft deletions (a lesson or insight +// decayed to deleted, a sketch discarded by heal) - and drops the +// per-operation chatter (observe, compress, heal repairs ...): a few lines +// a month instead of ~100 KB a day. The details checks mirror the payloads +// at those call sites; a new deletion path needs either a listed operation +// or one of these markers. + +function auditStoreOff(): boolean { + return (process.env.AGENTMEMORY_AUDIT_STORE ?? "").toLowerCase() === "off"; +} + +const DELETION_OPS: ReadonlySet = new Set([ + "forget", "delete", "lesson_delete", "slot_delete", "core_remove", "sketch_discard", +]); + +function isDeletion(operation: AuditEntry["operation"], details: Record): boolean { + if (DELETION_OPS.has(operation)) return true; + if (details.action === "delete" || details.action === "soft-delete" || details.action === "mesh.remove") return true; + if (details.newStatus === "discarded") return true; + if (typeof details.softDeleted === "number" && details.softDeleted > 0) return true; + return operation === "import" && details.strategy === "replace"; +} + +function auditLineWanted(operation: AuditEntry["operation"], details: Record): boolean { + const mode = (process.env.AGENTMEMORY_AUDIT_LOG ?? "all").toLowerCase(); + if (mode === "off") return false; + if (mode === "deletions") return isDeletion(operation, details); + return true; +} + +export async function recordAudit( + kv: StateKV, + operation: AuditEntry["operation"], + functionId: string, + targetIds: string[], + details: Record = {}, + qualityScore?: number, + userId?: string, +): Promise { + const entry: AuditEntry = { + id: generateId("aud"), + timestamp: new Date().toISOString(), + operation, + userId, + functionId, + targetIds, + details, + qualityScore, + }; + if (auditStoreOff()) { + if (!auditLineWanted(operation, details)) return entry; + logger.audit(operation, { + auditId: entry.id, + functionId, + targetIds, + targetCount: targetIds.length, + details, + ...(qualityScore !== undefined ? { qualityScore } : {}), + ...(userId !== undefined ? { userId } : {}), + }); + return entry; + } + await kv.set(KV.audit, entry.id, entry); + return entry; +} + +export async function safeAudit( + kv: StateKV, + operation: AuditEntry["operation"], + functionId: string, + targetIds: string[], + details: Record = {}, + qualityScore?: number, + userId?: string, +): Promise { + try { + await recordAudit(kv, operation, functionId, targetIds, details, qualityScore, userId); + } catch (err) { + try { + logger.warn("audit write failed", { + functionId, + operation, + targetIds, + error: err instanceof Error ? err.message : String(err), + }); + } catch {} + } +} + +const AUDIT_MAX_DEFAULT = 50_000; + +function auditMax(): number { + const raw = process.env.AGENTMEMORY_AUDIT_MAX?.trim(); + const n = raw && /^\d+$/.test(raw) ? Number(raw) : 0; + return n > 0 ? n : AUDIT_MAX_DEFAULT; +} + +// Retention for the audit store: keeps the newest AGENTMEMORY_AUDIT_MAX +// entries. Pruning the log is not itself audited - that would write the +// rows it exists to bound - so it leaves one log line instead. +export async function pruneAudit(kv: StateKV): Promise { + if (auditStoreOff()) return 0; + const max = auditMax(); + const all = await kv.list(KV.audit); + if (all.length <= max) return 0; + const oldest = [...all] + .sort((a, b) => a.timestamp.localeCompare(b.timestamp)) + .slice(0, all.length - max); + for (const entry of oldest) await kv.delete(KV.audit, entry.id); + logger.info("Audit log pruned", { removed: oldest.length, kept: max }); + return oldest.length; +} + +export async function queryAudit( + kv: StateKV, + filter?: { + operation?: AuditEntry["operation"]; + dateFrom?: string; + dateTo?: string; + limit?: number; + }, +): Promise { + if (auditStoreOff()) return []; + const all = await kv.list(KV.audit); + let entries = [...all].sort( + (a, b) => new Date(b.timestamp).getTime() - new Date(a.timestamp).getTime(), + ); + + if (filter?.operation) { + entries = entries.filter((e) => e.operation === filter.operation); + } + if (filter?.dateFrom) { + const from = new Date(filter.dateFrom).getTime(); + if (Number.isNaN(from)) { + throw new Error(`Invalid dateFrom: ${filter.dateFrom}`); + } + entries = entries.filter((e) => new Date(e.timestamp).getTime() >= from); + } + if (filter?.dateTo) { + const to = new Date(filter.dateTo).getTime(); + if (Number.isNaN(to)) { + throw new Error(`Invalid dateTo: ${filter.dateTo}`); + } + entries = entries.filter((e) => new Date(e.timestamp).getTime() <= to); + } + + return entries.slice(0, filter?.limit || 100); +} diff --git a/src/index.ts b/src/index.ts index 0fdb5eab6f..b61ee8f0a2 100644 --- a/src/index.ts +++ b/src/index.ts @@ -59,6 +59,7 @@ import { registerConsolidateFunction } from "./functions/consolidate.js"; import { registerPatternsFunction } from "./functions/patterns.js"; import { registerRememberFunction } from "./functions/remember.js"; import { registerEvictFunction } from "./functions/evict.js"; +import { pruneAudit } from "./functions/audit.js"; import { registerRelationsFunction } from "./functions/relations.js"; import { registerTimelineFunction } from "./functions/timeline.js"; import { registerSmartSearchFunction } from "./functions/smart-search.js"; @@ -618,6 +619,13 @@ async function main() { bootLog(`Eviction sweep: enabled (every 24h)`); } + const auditPruneTimer = setInterval(async () => { + try { + await pruneAudit(kv); + } catch {} + }, 86400000); + auditPruneTimer.unref(); + if (process.env.INSIGHT_DECAY_ENABLED !== "false") { const insightDecayTimer = setInterval(async () => { try { diff --git a/test/audit.test.ts b/test/audit.test.ts index 12cb66104f..3bb74f15f6 100644 --- a/test/audit.test.ts +++ b/test/audit.test.ts @@ -4,7 +4,7 @@ vi.mock("../src/logger.js", () => ({ logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, })); -import { recordAudit, queryAudit } from "../src/functions/audit.js"; +import { recordAudit, queryAudit, pruneAudit } from "../src/functions/audit.js"; function mockKV() { const store = new Map>(); @@ -103,4 +103,27 @@ describe("Audit Functions", () => { const entries = await queryAudit(kv as never, { limit: 3 }); expect(entries.length).toBe(3); }); + + it("pruneAudit keeps the newest AGENTMEMORY_AUDIT_MAX entries", async () => { + process.env.AGENTMEMORY_AUDIT_MAX = "2"; + try { + for (let i = 0; i < 4; i++) { + await kv.set("mem:audit", `aud_${i}`, { + id: `aud_${i}`, + timestamp: new Date(Date.UTC(2026, 0, 1 + i)).toISOString(), + operation: "observe", + functionId: "mem::observe", + targetIds: [], + details: {}, + }); + } + + expect(await pruneAudit(kv as never)).toBe(2); + + const kept = (await kv.list<{ id: string }>("mem:audit")).map((e) => e.id).sort(); + expect(kept).toEqual(["aud_2", "aud_3"]); + } finally { + delete process.env.AGENTMEMORY_AUDIT_MAX; + } + }); }); From 0a4874af16531360f11c07496421ec4d3cb919e7 Mon Sep 17 00:00:00 2001 From: neal Date: Fri, 25 Sep 2026 18:37:06 -0400 Subject: [PATCH 07/17] fix(replay): derive lessons from whole sentences only A trigger word matched mid-sentence dropped the subject and could invert the lesson; a dot inside a filename cut it short. Lessons are now whole sentences that open with a trigger, with balanced markup, within 220 chars. A re-import no longer bumps reinforcements or lastReinforcedAt, so a re-derived lesson can still decay out (#1292). Generated-By: Claude Opus 5.5 (1M context) --- src/functions/replay.ts | 64 +++++++++++++------- test/replay-lessons.test.ts | 114 ++++++++++++++++++++++++++++++++++++ 2 files changed, 156 insertions(+), 22 deletions(-) create mode 100644 test/replay-lessons.test.ts diff --git a/src/functions/replay.ts b/src/functions/replay.ts index 3d1f5e9906..b4ebb80ce8 100644 --- a/src/functions/replay.ts +++ b/src/functions/replay.ts @@ -63,10 +63,45 @@ function rawFromCompressed(obs: CompressedObservation): RawObservation { }; } -const LESSON_PATTERNS: RegExp[] = [ - /\b(always|never|don'?t|do not|make sure|remember to|note:|caveat:|warning:)\b[^.\n]{10,200}[.!\n]/gi, - /\b(prefer|avoid)\s[^.\n]{10,200}[.!\n]/gi, -]; +// A lesson is a whole sentence that opens with a trigger. A trigger found +// mid-sentence would drop the subject and can invert the meaning (#1292). +const LESSON_START = + /^(always|never|don'?t|do not|make sure|remember to|prefer|avoid|note:|caveat:|warning:)\s/i; +const LIST_MARKER = /^(?:[-*+>]|\d+[.)])\s+/; +const SENTENCE_BREAK = /\n+|(?<=[.!?])\s+/; +const MIN_LESSON_CHARS = 20; +const MAX_LESSON_CHARS = 220; +const MAX_LESSONS = 20; + +function isBalanced(sentence: string): boolean { + const count = (token: string) => sentence.split(token).length - 1; + return ( + count("**") % 2 === 0 && + count("`") % 2 === 0 && + count("[[") === count("]]") + ); +} + +export function extractLessons(texts: string[]): string[] { + const lessons = new Map(); + for (const text of texts) { + for (const raw of text.split(SENTENCE_BREAK)) { + const sentence = raw.trim().replace(LIST_MARKER, "").replace(/\s+/g, " "); + if ( + sentence.length < MIN_LESSON_CHARS || + sentence.length > MAX_LESSON_CHARS || + !LESSON_START.test(sentence) || + !isBalanced(sentence) + ) { + continue; + } + const key = sentence.toLowerCase(); + if (!lessons.has(key)) lessons.set(key, sentence); + if (lessons.size >= MAX_LESSONS) return Array.from(lessons.values()); + } + } + return Array.from(lessons.values()); +} async function deriveCrystalAndLessons( kv: StateKV, @@ -97,22 +132,9 @@ async function deriveCrystalAndLessons( } } - const lessonMatches = new Map(); - for (const text of assistantTexts.concat(userPrompts).slice(0, 200)) { - for (const pat of LESSON_PATTERNS) { - pat.lastIndex = 0; - let m: RegExpExecArray | null; - while ((m = pat.exec(text)) !== null && lessonMatches.size < 40) { - const snippet = m[0].replace(/\s+/g, " ").trim(); - if (snippet.length >= 20 && snippet.length <= 220) { - const key = snippet.toLowerCase(); - if (!lessonMatches.has(key)) lessonMatches.set(key, snippet); - } - } - } - } - - const lessonEntries = Array.from(lessonMatches.values()).slice(0, 20); + const lessonEntries = extractLessons( + assistantTexts.concat(userPrompts).slice(0, 200), + ); const lessonIds: string[] = []; for (const content of lessonEntries) { // Content-addressed ID so re-importing the same JSONL does not @@ -134,9 +156,7 @@ async function deriveCrystalAndLessons( ...existing, sourceIds: mergedSources, tags: mergedTags, - reinforcements: (existing.reinforcements || 0) + 1, updatedAt: createdAt, - lastReinforcedAt: createdAt, }; await kv.set(KV.lessons, lessonId, merged); } else { diff --git a/test/replay-lessons.test.ts b/test/replay-lessons.test.ts new file mode 100644 index 0000000000..917e58802f --- /dev/null +++ b/test/replay-lessons.test.ts @@ -0,0 +1,114 @@ +import { describe, it, expect, vi } from "vitest"; +import { mkdtempSync, mkdirSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +vi.mock("../src/logger.js", () => ({ + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, +})); + +import { extractLessons, registerReplayFunctions } from "../src/functions/replay.js"; +import { KV } from "../src/state/schema.js"; +import type { Lesson } from "../src/types.js"; + +function mockKV() { + const store = new Map>(); + return { + get: async (scope: string, key: string): Promise => + (store.get(scope)?.get(key) as T) ?? null, + set: async (scope: string, key: string, value: T): Promise => { + if (!store.has(scope)) store.set(scope, new Map()); + store.get(scope)!.set(key, value); + return value; + }, + delete: async (scope: string, key: string) => { + store.get(scope)?.delete(key); + }, + list: async (scope: string): Promise => + Array.from(store.get(scope)?.values() ?? []) as T[], + }; +} + +function mockSdk() { + const fns = new Map(); + return { + registerFunction: (id: string, handler: Function) => fns.set(id, handler), + registerTrigger: () => {}, + trigger: async (input: { function_id: string; payload?: unknown }) => + fns.get(input.function_id)?.(input.payload) ?? { success: true }, + } as any; +} + +describe("extractLessons (#1292)", () => { + it("keeps a whole sentence that opens with a trigger", () => { + expect( + extractLessons(["Done. Always run the migration before seeding the db."]), + ).toEqual(["Always run the migration before seeding the db."]); + }); + + it("drops a trigger that sits mid-sentence and would lose its subject", () => { + expect( + extractLessons(["The watchdog must never restart on ROUTE_MISSING_404."]), + ).toEqual([]); + }); + + it("does not stop at a dot inside a filename", () => { + expect( + extractLessons(["Never edit config.ts by hand, regenerate it instead."]), + ).toEqual(["Never edit config.ts by hand, regenerate it instead."]); + }); + + it("accepts a trigger after a list marker", () => { + expect(extractLessons(["- Prefer node:sqlite over a native addon here."])).toEqual([ + "Prefer node:sqlite over a native addon here.", + ]); + }); + + it("ignores triggers inside identifiers and unbalanced markup", () => { + expect( + extractLessons([ + "dont-repeat-yourself]]/some-page is linked here", + "Never mix** the statistical constraint with the other one", + ]), + ).toEqual([]); + }); + + it("rejects a sentence longer than the cap instead of truncating it", () => { + expect(extractLessons([`Always ${"x ".repeat(120)}end.`])).toEqual([]); + }); + + it("returns each lesson once", () => { + expect( + extractLessons(["Avoid global state in hooks.", "avoid global state in hooks."]), + ).toEqual(["Avoid global state in hooks."]); + }); +}); + +describe("import-jsonl lesson re-derivation (#1292)", () => { + it("does not reinforce a lesson just because a re-import matched it again", async () => { + const root = mkdtempSync(join(tmpdir(), "replay-lessons-")); + const dir = join(root, "proj"); + mkdirSync(dir); + const ts = "2026-04-17T10:00:00.000Z"; + writeFileSync( + join(dir, "sess-lesson.jsonl"), + [ + { type: "user", sessionId: "sess-lesson", timestamp: ts, cwd: root, + message: { role: "user", content: [{ type: "text", text: "hello" }] } }, + { type: "assistant", sessionId: "sess-lesson", timestamp: ts, + message: { role: "assistant", content: [{ type: "text", text: "Always run the migration before seeding." }] } }, + ].map((l) => JSON.stringify(l)).join("\n") + "\n", + ); + const kv = mockKV(); + const sdk = mockSdk(); + registerReplayFunctions(sdk, kv as never); + + await sdk.trigger({ function_id: "mem::replay::import-jsonl", payload: { path: root } }); + await sdk.trigger({ function_id: "mem::replay::import-jsonl", payload: { path: root } }); + + const lessons = await kv.list(KV.lessons); + expect(lessons).toHaveLength(1); + expect(lessons[0].reinforcements).toBe(0); + expect(lessons[0].lastReinforcedAt).toBeUndefined(); + }); +}); From b0b25f9e50b2d0f3aa56fb40dabb6bc03d6fdb0b Mon Sep 17 00:00:00 2001 From: neal Date: Fri, 25 Sep 2026 18:37:50 -0400 Subject: [PATCH 08/17] fix(openai): send max_completion_tokens to api.openai.com OpenAI's reasoning models, including the default gpt-5.6-luna, reject max_tokens with a 400, so every compress/summarize call failed on a default config. api.openai.com accepts max_completion_tokens on every model; compatible servers keep max_tokens (#1219). Generated-By: Claude Opus 5.5 (1M context) --- src/providers/openai.ts | 9 +++++++- test/openai-token-param.test.ts | 40 +++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) create mode 100644 test/openai-token-param.test.ts diff --git a/src/providers/openai.ts b/src/providers/openai.ts index fbfef7abb7..299e70f10d 100644 --- a/src/providers/openai.ts +++ b/src/providers/openai.ts @@ -56,6 +56,10 @@ export class OpenAIProvider implements MemoryProvider { private isAzure: boolean; private azureApiVersion: string; private tokenizeUnavailable = false; + // api.openai.com rejects max_tokens on reasoning models and accepts + // max_completion_tokens on every model; compatible servers may only know + // max_tokens (#1219). + private outputTokenParam: "max_tokens" | "max_completion_tokens"; constructor(apiKey: string, model: string, maxTokens: number, baseURL?: string) { this.apiKey = apiKey; @@ -67,6 +71,9 @@ export class OpenAIProvider implements MemoryProvider { this.azureApiVersion = getEnvVar("OPENAI_API_VERSION") || DEFAULT_AZURE_API_VERSION; this.isAzure = detectAzure(this.baseUrl); + this.outputTokenParam = /^https:\/\/api\.openai\.com(\/|$)/.test(this.baseUrl) + ? "max_completion_tokens" + : "max_tokens"; } async compress(systemPrompt: string, userPrompt: string): Promise { @@ -112,7 +119,7 @@ export class OpenAIProvider implements MemoryProvider { const url = buildChatUrl(this.baseUrl, this.isAzure, this.azureApiVersion); const body: Record = { model: this.model, - max_tokens: this.maxTokens, + [this.outputTokenParam]: this.maxTokens, // OpenAI API spec defines `stream` as defaulting to false, so omitting // it should yield a JSON response. Some OpenAI-compatible proxies // (notably 9Router < 0.4.56 — see decolua/9router#1260) default to diff --git a/test/openai-token-param.test.ts b/test/openai-token-param.test.ts new file mode 100644 index 0000000000..7a005567e8 --- /dev/null +++ b/test/openai-token-param.test.ts @@ -0,0 +1,40 @@ +import { describe, it, expect, vi, afterEach } from "vitest"; +import { OpenAIProvider } from "../src/providers/openai.js"; + +function sentBody(fetchSpy: ReturnType): Record { + const [, init] = fetchSpy.mock.calls[0] as [string, RequestInit]; + return JSON.parse(init.body as string); +} + +describe("OpenAIProvider output-token parameter (#1219)", () => { + afterEach(() => { + vi.restoreAllMocks(); + }); + + function stubFetch() { + return vi.spyOn(globalThis, "fetch").mockResolvedValue( + new Response(JSON.stringify({ choices: [{ message: { content: "ok" } }] }), { + status: 200, + headers: { "Content-Type": "application/json" }, + }), + ); + } + + it("sends max_completion_tokens to api.openai.com, which rejects max_tokens on reasoning models", async () => { + const fetchSpy = stubFetch(); + await new OpenAIProvider("key", "gpt-5.6-luna", 4096, "https://api.openai.com").summarize("s", "u"); + + const body = sentBody(fetchSpy); + expect(body.max_completion_tokens).toBe(4096); + expect(body).not.toHaveProperty("max_tokens"); + }); + + it("keeps max_tokens for OpenAI-compatible servers", async () => { + const fetchSpy = stubFetch(); + await new OpenAIProvider("key", "general", 8192, "http://broker:4010/v1").summarize("s", "u"); + + const body = sentBody(fetchSpy); + expect(body.max_tokens).toBe(8192); + expect(body).not.toHaveProperty("max_completion_tokens"); + }); +}); From b082b94df50a49c68ff35bf3064846091a7424c5 Mon Sep 17 00:00:00 2001 From: neal Date: Fri, 25 Sep 2026 18:38:39 -0400 Subject: [PATCH 09/17] fix(search): keep agentmemory's own MCP calls out of the index by default A memory_smart_search observation contains its query verbatim, so it outranked the real answer on the next identical search. Unset AGENTMEMORY_INDEX_EXCLUDE_TOOL_PREFIXES now means both agentmemory MCP prefixes; set it empty to index everything. Rows stay stored (#993). Generated-By: Claude Opus 5.5 (1M context) --- .env.example | 2 +- src/state/index-corpus.ts | 428 +++++++++++++++--------------- test/index-echo-exclusion.test.ts | 301 +++++++++++---------- 3 files changed, 372 insertions(+), 359 deletions(-) diff --git a/.env.example b/.env.example index b6848ebbf4..6ca3307219 100644 --- a/.env.example +++ b/.env.example @@ -139,7 +139,7 @@ # AGENTMEMORY_MEMORY_LAYER_BOOST=1.25 # Multiplicative boost on the fused score for saved memories (id prefix mem_, or sessionId "memory"). Multiplicative so it needs no recalibration when the weights, RRF_K or the fusion mode move. Keep it a tiebreak, not an override — it is bounded only by the supersede de-index and the isLatest skip. # AGENTMEMORY_RETRIEVAL_DEPTH=100 # Minimum candidates fetched from EACH leg before fusion, independent of the caller's page size. Legs were previously fetched at limit*2, which made results non-monotonic in limit: a document deep in the vector leg fell outside the fetch at small limits and forfeited its vector term, and under RRF that is unrecoverable (the vector term is worth up to 0.6/61 while the whole BM25 term tops out at 0.4/61). Curated memories are exactly the long, diffuse documents that sit deep in the vector leg. Nearly free — the vector scan is full regardless of limit; only the sort grows (~+8ms at 25k vectors). This is a CANDIDATE POOL size, not a page size - the caller still receives exactly `limit` rows. Set to 0 to disable the floor and restore the old limit*2 fetch. # AGENTMEMORY_PREFIX_MATCH=max # How the prefix-expansion leg combines. 'max' (default) treats all index terms sharing a query-token prefix as ONE term and takes the best, bounding the contribution by construction. 'sum' restores the historical additive behaviour, in which each distinct sibling added its own full BM25 contribution — a document listing v1.0.0 through v1.40.0 accumulated 40 contributions for the single query token "v1" and outscored exact matches (measured 110.01 at 80 siblings against a ~22 single-term ceiling). -# AGENTMEMORY_INDEX_EXCLUDE_TOOL_PREFIXES=mcp__plugin_agentmemory_agentmemory__ # Comma-separated toolName prefixes whose observations are STORED but never INDEXED. Aimed at retrieval echoes — the daemon's own record of "the agent searched for X" contains the query verbatim, making it a near-perfect match for exactly the query that created it. The records stay fully readable via timeline, session history, the viewer and expandIds. +# AGENTMEMORY_INDEX_EXCLUDE_TOOL_PREFIXES=mcp__plugin_agentmemory_agentmemory__,mcp__agentmemory__ # Default shown; set empty to index everything. Comma-separated toolName prefixes whose observations are STORED but never INDEXED. Aimed at retrieval echoes — the daemon's own record of "the agent searched for X" contains the query verbatim, making it a near-perfect match for exactly the query that created it. The records stay fully readable via timeline, session history, the viewer and expandIds. # --- Memory chunking --------------------------------------------------- # AGENTMEMORY_MEMORY_CHUNKING=true # Embed each saved memory as several vectors, one per ~section, instead of one vector for the whole thing. A long multi-topic memory otherwise averages out to a point near the corpus centroid and matches nothing. Also puts the memory's concepts into the embedded text, which the single-vector path never did. Requires a full index rebuild to take effect on existing memories. diff --git a/src/state/index-corpus.ts b/src/state/index-corpus.ts index d813f30680..42eabe4930 100644 --- a/src/state/index-corpus.ts +++ b/src/state/index-corpus.ts @@ -1,211 +1,217 @@ -// The one place that says which content rows are indexed and what text is -// embedded for them. Every consumer of that rule routes through here so they -// cannot drift apart: -// -// - the live write paths (observe / compress / remember) build their embed -// jobs from it; -// - the inproc vector store re-derives the expected input for an id from the -// CURRENT content row before it commits an embedding, so a completion for -// text that has since been rewritten or deleted is dropped as stale; -// - the boot BM25 rebuild and the vector fill pass enumerate the corpus with -// the same eligibility. -import { createHash } from "node:crypto"; -import type { DatabaseSync } from "node:sqlite"; -import type { CompressedObservation, Memory } from "../types.js"; -import { KV } from "./schema.js"; -import { parentIdOf } from "./vector-index.js"; -import { - MEMORY_SESSION, - isLatestEligible, - memoryChunkJobs, - memoryToIndexDoc, -} from "./memory-utils.js"; - -export type EmbedKind = "memory" | "observation" | "synthetic"; - -export type EmbedJob = { - id: string; - sessionId: string; - text: string; - kind: EmbedKind; -}; - -// Hard cap on embedding input length. Most providers cap input around -// 8k tokens (~32k chars at ~4 chars/token). Truncate defensively so a -// huge memory.content can't 400 the embed call or blow context budget -// on a single doc. 16k chars ≈ 4k tokens, safely under every provider. -const EMBED_MAX_CHARS = 16_000; - -export function clipEmbedInput(text: string): string { - if (text.length <= EMBED_MAX_CHARS) return text; - return text.slice(0, EMBED_MAX_CHARS); -} - -// Fingerprint of the exact text handed to the provider. Stored on the vector -// row so "a vector exists" means "a vector for THIS content exists". -export function embedInputHash(text: string): string { - return createHash("sha256").update(clipEmbedInput(text)).digest("hex"); -} - -// Tool-name prefixes whose observations are STORED but never INDEXED. -// -// Retrieval echoes — the daemon's own record of "the agent ran -// memory_smart_search for " — contain the query verbatim, so they -// are near-perfect BM25 and vector matches for exactly the query that -// created them, and they crowd out the answer (upstream #993). Live -// measurement: for one query, ranks 1, 2 and 6 were all records of prior -// searches, scoring up to 74.89 against the real memory's 42.23 at rank 13. -// -// The KV writes are deliberately untouched. These rows stay in the -// timeline, session history, the viewer and expandIds — they just stop -// competing in search. That is how "observations are kept" is satisfied -// structurally rather than by promise. -let excludeCache: { raw: string; prefixes: string[] } | null = null; - -function excludedToolPrefixes(): string[] { - const raw = process.env.AGENTMEMORY_INDEX_EXCLUDE_TOOL_PREFIXES ?? ""; - if (!excludeCache || excludeCache.raw !== raw) { - excludeCache = { - raw, - prefixes: raw - .split(",") - .map((s) => s.trim()) - .filter((s) => s.length > 0), - }; - } - return excludeCache.prefixes; -} - -export function isIndexExcluded(obs: { toolName?: string }): boolean { - if (typeof obs.toolName !== "string" || !obs.toolName) return false; - const prefixes = excludedToolPrefixes(); - if (prefixes.length === 0) return false; - return prefixes.some((p) => obs.toolName!.startsWith(p)); -} - -// Where the content row for a vector id lives. Memories may carry a real -// session id as their first `sessionIds` entry, so the session alone does not -// identify the scope; the kind does. -export function contentLocation( - kind: EmbedKind, - sessionId: string, - vectorId: string, -): { scope: string; key: string } { - const parent = parentIdOf(vectorId); - return kind === "memory" - ? { scope: KV.memories, key: parent } - : { scope: KV.observations(sessionId), key: parent }; -} - -// The embed jobs a stored row is entitled to right now. An empty array means -// the row is not indexed (missing, superseded, incomplete, or an excluded -// tool), so any vector for it is stale. -// Field checks are `typeof`, not truthiness: a stored row may carry a -// non-string where the type says string (the import handler accepts such -// rows), and string concatenation on an object throws. Such a row is simply -// not indexable. -export function memoryEmbedJobs(memory: Memory | null | undefined): EmbedJob[] { - if (!memory || typeof memory.id !== "string" || !memory.id || !isLatestEligible(memory)) return []; - if (typeof memory.title !== "string" || !memory.title) return []; - if (typeof memory.content !== "string" || !memory.content) return []; - const first = Array.isArray(memory.sessionIds) ? memory.sessionIds[0] : undefined; - const sessionId = typeof first === "string" && first ? first : MEMORY_SESSION; - return memoryChunkJobs(memory).map((job) => ({ - id: job.id, - sessionId, - text: job.text, - kind: "memory" as const, - })); -} - -export function observationEmbedJobs( - obs: CompressedObservation | null | undefined, - kind: "observation" | "synthetic" = "observation", -): EmbedJob[] { - if (!obs || typeof obs.id !== "string" || typeof obs.sessionId !== "string") return []; - if (typeof obs.title !== "string" || !obs.title) return []; - if (typeof obs.narrative !== "string" || !obs.narrative) return []; - if (isIndexExcluded(obs)) return []; - return [{ id: obs.id, sessionId: obs.sessionId, text: obs.title + " " + obs.narrative, kind }]; -} - -export function embedJobsFor(kind: EmbedKind, row: unknown): EmbedJob[] { - return kind === "memory" - ? memoryEmbedJobs(row as Memory | null) - : observationEmbedJobs(row as CompressedObservation | null, kind); -} - -// ---------------------------------------------------------------- corpus - -// SearchIndex spreads `facts`, `concepts` and `files`; an imported or -// hand-written observation may lack them (the import handler accepts such -// rows and only skips their indexing). Normalise here so the rebuild indexes -// the row instead of throwing on it. -const arr = (v: unknown): string[] => (Array.isArray(v) ? v : []); -function indexableObservation(obs: CompressedObservation): CompressedObservation { - return { ...obs, facts: arr(obs.facts), concepts: arr(obs.concepts), files: arr(obs.files) }; -} - -// `mem:obs:` scopes as a half-open range (';' follows ':'), so the -// (scope, seq) index serves it and no session id needs escaping. -const OBS_SCOPE_LO = "mem:obs:"; -const OBS_SCOPE_HI = "mem:obs;"; - -export type CorpusItem = { - kind: "memory" | "observation"; - scope: string; - key: string; - seq: number; - row: Memory | CompressedObservation; - // The BM25 document, or null when the row is not indexed. - doc: CompressedObservation | null; - // The embedding jobs the row is entitled to (empty when not indexed). - jobs: EmbedJob[]; - // Set when building the doc or the jobs threw: the row is treated as not - // indexed and consumers count and log it instead of failing the walk. - error?: string; -}; - -// Walks every memory and every observation on disk - including observation -// scopes whose session record is gone (evict.ts removes sessions but keeps -// their observations, which are content and stay indexed) - in `seq` order, -// and applies the eligibility rules above once for both legs. The boot BM25 -// rebuild and the vector fill pass consume this same iterator, so the two can -// never disagree about what belongs in the index. -export function* enumerateIndexCorpus(db: DatabaseSync): IterableIterator { - const stmt = db.prepare( - "SELECT seq, scope, key, value FROM kv WHERE scope = ? OR (scope >= ? AND scope < ?) ORDER BY seq", - ); - for (const r of stmt.iterate(KV.memories, OBS_SCOPE_LO, OBS_SCOPE_HI) as Iterable<{ - seq: number; - scope: string; - key: string; - value: string; - }>) { - let row: unknown; - try { - row = JSON.parse(r.value); - } catch { - continue; - } - const kind = r.scope === KV.memories ? "memory" : "observation"; - const item: CorpusItem = { kind, scope: r.scope, key: r.key, seq: r.seq, row: row as Memory, doc: null, jobs: [] }; - // One malformed row must never end the walk (the boot rebuild awaits it). - try { - if (kind === "memory") { - const memory = row as Memory; - item.jobs = memoryEmbedJobs(memory); - item.doc = item.jobs.length > 0 ? memoryToIndexDoc(memory) : null; - } else { - const obs = row as CompressedObservation; - item.jobs = observationEmbedJobs(obs); - item.doc = item.jobs.length > 0 ? indexableObservation(obs) : null; - } - } catch (err) { - item.doc = null; - item.jobs = []; - item.error = err instanceof Error ? err.message : String(err); - } - yield item; - } -} +// The one place that says which content rows are indexed and what text is +// embedded for them. Every consumer of that rule routes through here so they +// cannot drift apart: +// +// - the live write paths (observe / compress / remember) build their embed +// jobs from it; +// - the inproc vector store re-derives the expected input for an id from the +// CURRENT content row before it commits an embedding, so a completion for +// text that has since been rewritten or deleted is dropped as stale; +// - the boot BM25 rebuild and the vector fill pass enumerate the corpus with +// the same eligibility. +import { createHash } from "node:crypto"; +import type { DatabaseSync } from "node:sqlite"; +import type { CompressedObservation, Memory } from "../types.js"; +import { KV } from "./schema.js"; +import { parentIdOf } from "./vector-index.js"; +import { + MEMORY_SESSION, + isLatestEligible, + memoryChunkJobs, + memoryToIndexDoc, +} from "./memory-utils.js"; + +export type EmbedKind = "memory" | "observation" | "synthetic"; + +export type EmbedJob = { + id: string; + sessionId: string; + text: string; + kind: EmbedKind; +}; + +// Hard cap on embedding input length. Most providers cap input around +// 8k tokens (~32k chars at ~4 chars/token). Truncate defensively so a +// huge memory.content can't 400 the embed call or blow context budget +// on a single doc. 16k chars ≈ 4k tokens, safely under every provider. +const EMBED_MAX_CHARS = 16_000; + +export function clipEmbedInput(text: string): string { + if (text.length <= EMBED_MAX_CHARS) return text; + return text.slice(0, EMBED_MAX_CHARS); +} + +// Fingerprint of the exact text handed to the provider. Stored on the vector +// row so "a vector exists" means "a vector for THIS content exists". +export function embedInputHash(text: string): string { + return createHash("sha256").update(clipEmbedInput(text)).digest("hex"); +} + +// Tool-name prefixes whose observations are STORED but never INDEXED. +// +// Retrieval echoes — the daemon's own record of "the agent ran +// memory_smart_search for " — contain the query verbatim, so they +// are near-perfect BM25 and vector matches for exactly the query that +// created them, and they crowd out the answer (upstream #993). Live +// measurement: for one query, ranks 1, 2 and 6 were all records of prior +// searches, scoring up to 74.89 against the real memory's 42.23 at rank 13. +// +// The KV writes are deliberately untouched. These rows stay in the +// timeline, session history, the viewer and expandIds — they just stop +// competing in search. That is how "observations are kept" is satisfied +// structurally rather than by promise. +// Unset means agentmemory's own MCP tools, under both the plugin and the +// standalone server name; set it empty to index everything. +const DEFAULT_EXCLUDED_TOOL_PREFIXES = + "mcp__plugin_agentmemory_agentmemory__,mcp__agentmemory__"; +let excludeCache: { raw: string; prefixes: string[] } | null = null; + +function excludedToolPrefixes(): string[] { + const raw = + process.env.AGENTMEMORY_INDEX_EXCLUDE_TOOL_PREFIXES ?? + DEFAULT_EXCLUDED_TOOL_PREFIXES; + if (!excludeCache || excludeCache.raw !== raw) { + excludeCache = { + raw, + prefixes: raw + .split(",") + .map((s) => s.trim()) + .filter((s) => s.length > 0), + }; + } + return excludeCache.prefixes; +} + +export function isIndexExcluded(obs: { toolName?: string }): boolean { + if (typeof obs.toolName !== "string" || !obs.toolName) return false; + const prefixes = excludedToolPrefixes(); + if (prefixes.length === 0) return false; + return prefixes.some((p) => obs.toolName!.startsWith(p)); +} + +// Where the content row for a vector id lives. Memories may carry a real +// session id as their first `sessionIds` entry, so the session alone does not +// identify the scope; the kind does. +export function contentLocation( + kind: EmbedKind, + sessionId: string, + vectorId: string, +): { scope: string; key: string } { + const parent = parentIdOf(vectorId); + return kind === "memory" + ? { scope: KV.memories, key: parent } + : { scope: KV.observations(sessionId), key: parent }; +} + +// The embed jobs a stored row is entitled to right now. An empty array means +// the row is not indexed (missing, superseded, incomplete, or an excluded +// tool), so any vector for it is stale. +// Field checks are `typeof`, not truthiness: a stored row may carry a +// non-string where the type says string (the import handler accepts such +// rows), and string concatenation on an object throws. Such a row is simply +// not indexable. +export function memoryEmbedJobs(memory: Memory | null | undefined): EmbedJob[] { + if (!memory || typeof memory.id !== "string" || !memory.id || !isLatestEligible(memory)) return []; + if (typeof memory.title !== "string" || !memory.title) return []; + if (typeof memory.content !== "string" || !memory.content) return []; + const first = Array.isArray(memory.sessionIds) ? memory.sessionIds[0] : undefined; + const sessionId = typeof first === "string" && first ? first : MEMORY_SESSION; + return memoryChunkJobs(memory).map((job) => ({ + id: job.id, + sessionId, + text: job.text, + kind: "memory" as const, + })); +} + +export function observationEmbedJobs( + obs: CompressedObservation | null | undefined, + kind: "observation" | "synthetic" = "observation", +): EmbedJob[] { + if (!obs || typeof obs.id !== "string" || typeof obs.sessionId !== "string") return []; + if (typeof obs.title !== "string" || !obs.title) return []; + if (typeof obs.narrative !== "string" || !obs.narrative) return []; + if (isIndexExcluded(obs)) return []; + return [{ id: obs.id, sessionId: obs.sessionId, text: obs.title + " " + obs.narrative, kind }]; +} + +export function embedJobsFor(kind: EmbedKind, row: unknown): EmbedJob[] { + return kind === "memory" + ? memoryEmbedJobs(row as Memory | null) + : observationEmbedJobs(row as CompressedObservation | null, kind); +} + +// ---------------------------------------------------------------- corpus + +// SearchIndex spreads `facts`, `concepts` and `files`; an imported or +// hand-written observation may lack them (the import handler accepts such +// rows and only skips their indexing). Normalise here so the rebuild indexes +// the row instead of throwing on it. +const arr = (v: unknown): string[] => (Array.isArray(v) ? v : []); +function indexableObservation(obs: CompressedObservation): CompressedObservation { + return { ...obs, facts: arr(obs.facts), concepts: arr(obs.concepts), files: arr(obs.files) }; +} + +// `mem:obs:` scopes as a half-open range (';' follows ':'), so the +// (scope, seq) index serves it and no session id needs escaping. +const OBS_SCOPE_LO = "mem:obs:"; +const OBS_SCOPE_HI = "mem:obs;"; + +export type CorpusItem = { + kind: "memory" | "observation"; + scope: string; + key: string; + seq: number; + row: Memory | CompressedObservation; + // The BM25 document, or null when the row is not indexed. + doc: CompressedObservation | null; + // The embedding jobs the row is entitled to (empty when not indexed). + jobs: EmbedJob[]; + // Set when building the doc or the jobs threw: the row is treated as not + // indexed and consumers count and log it instead of failing the walk. + error?: string; +}; + +// Walks every memory and every observation on disk - including observation +// scopes whose session record is gone (evict.ts removes sessions but keeps +// their observations, which are content and stay indexed) - in `seq` order, +// and applies the eligibility rules above once for both legs. The boot BM25 +// rebuild and the vector fill pass consume this same iterator, so the two can +// never disagree about what belongs in the index. +export function* enumerateIndexCorpus(db: DatabaseSync): IterableIterator { + const stmt = db.prepare( + "SELECT seq, scope, key, value FROM kv WHERE scope = ? OR (scope >= ? AND scope < ?) ORDER BY seq", + ); + for (const r of stmt.iterate(KV.memories, OBS_SCOPE_LO, OBS_SCOPE_HI) as Iterable<{ + seq: number; + scope: string; + key: string; + value: string; + }>) { + let row: unknown; + try { + row = JSON.parse(r.value); + } catch { + continue; + } + const kind = r.scope === KV.memories ? "memory" : "observation"; + const item: CorpusItem = { kind, scope: r.scope, key: r.key, seq: r.seq, row: row as Memory, doc: null, jobs: [] }; + // One malformed row must never end the walk (the boot rebuild awaits it). + try { + if (kind === "memory") { + const memory = row as Memory; + item.jobs = memoryEmbedJobs(memory); + item.doc = item.jobs.length > 0 ? memoryToIndexDoc(memory) : null; + } else { + const obs = row as CompressedObservation; + item.jobs = observationEmbedJobs(obs); + item.doc = item.jobs.length > 0 ? indexableObservation(obs) : null; + } + } catch (err) { + item.doc = null; + item.jobs = []; + item.error = err instanceof Error ? err.message : String(err); + } + yield item; + } +} diff --git a/test/index-echo-exclusion.test.ts b/test/index-echo-exclusion.test.ts index c6b487095d..826ba9eedb 100644 --- a/test/index-echo-exclusion.test.ts +++ b/test/index-echo-exclusion.test.ts @@ -1,147 +1,154 @@ -import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"; - -vi.mock("../src/logger.js", () => ({ - logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, -})); - -import { - isIndexExcluded, - rebuildIndex, - getSearchIndex, - setVectorIndex, - setEmbeddingProvider, - getVectorIndex, -} from "../src/functions/search.js"; -import { VectorIndex } from "../src/state/vector-index.js"; -import { KV } from "../src/state/schema.js"; -import type { CompressedObservation, Session } from "../src/types.js"; - -const FLAG = "AGENTMEMORY_INDEX_EXCLUDE_TOOL_PREFIXES"; -const ECHO_PREFIX = "mcp__plugin_agentmemory_agentmemory__"; - -function mockKV() { - const store = new Map>(); - return { - get: async (scope: string, key: string): Promise => - (store.get(scope)?.get(key) as T) ?? null, - set: async (scope: string, key: string, data: T): Promise => { - if (!store.has(scope)) store.set(scope, new Map()); - store.get(scope)!.set(key, data); - return data; - }, - delete: async (scope: string, key: string): Promise => { - store.get(scope)?.delete(key); - }, - list: async (scope: string): Promise => - Array.from(store.get(scope)?.values() ?? []) as T[], - }; -} - -function obs(id: string, toolName?: string): CompressedObservation { - return { - id, - sessionId: "ses_1", - timestamp: "2026-01-01T00:00:00Z", - type: "search", - title: toolName ? `Memory search: ${id}` : `Real work ${id}`, - facts: [], - narrative: "residentKey teleport vnet mainpc-rdp", - concepts: [], - files: [], - importance: 5, - ...(toolName ? { toolName } : {}), - }; -} - -describe("index exclusion by toolName prefix", () => { - const saved = process.env[FLAG]; - - beforeEach(() => { - delete process.env[FLAG]; - getSearchIndex().clear(); - setVectorIndex(null); - setEmbeddingProvider(null); - }); - afterEach(() => { - if (saved === undefined) delete process.env[FLAG]; - else process.env[FLAG] = saved; - getSearchIndex().clear(); - setVectorIndex(null); - setEmbeddingProvider(null); - }); - - it("excludes nothing when the flag is unset (default behaviour)", () => { - expect(isIndexExcluded(obs("obs_1", `${ECHO_PREFIX}memory_smart_search`))).toBe(false); - }); - - it("matches on prefix, and only on the configured prefixes", () => { - process.env[FLAG] = ECHO_PREFIX; - expect(isIndexExcluded(obs("o", `${ECHO_PREFIX}memory_smart_search`))).toBe(true); - expect(isIndexExcluded(obs("o", `${ECHO_PREFIX}memory_recall`))).toBe(true); - expect(isIndexExcluded(obs("o", "Bash"))).toBe(false); - expect(isIndexExcluded(obs("o", "mcp__veeam-rag__hybrid_search"))).toBe(false); - // A record with no toolName (every pre-existing row on disk) is never - // matched by this predicate — that is why legacy echoes need a - // separate, explicitly reviewed deletion pass. - expect(isIndexExcluded(obs("o"))).toBe(false); - }); - - it("accepts a comma-separated list and tolerates whitespace", () => { - process.env[FLAG] = ` ${ECHO_PREFIX} , other__ `; - expect(isIndexExcluded(obs("o", `${ECHO_PREFIX}x`))).toBe(true); - expect(isIndexExcluded(obs("o", "other__thing"))).toBe(true); - expect(isIndexExcluded(obs("o", "unrelated"))).toBe(false); - }); - - it("keeps the row in KV while dropping it from BOTH indexes", async () => { - process.env[FLAG] = ECHO_PREFIX; - const kv = mockKV(); - const session: Session = { - id: "ses_1", - project: "demo", - cwd: "/tmp/demo", - startedAt: "2026-01-01T00:00:00Z", - status: "completed", - observationCount: 2, - }; - await kv.set(KV.sessions, session.id, session); - - const echo = obs("obs_echo", `${ECHO_PREFIX}memory_smart_search`); - const real = obs("obs_real"); - await kv.set(KV.observations("ses_1"), echo.id, echo); - await kv.set(KV.observations("ses_1"), real.id, real); - - setVectorIndex(new VectorIndex()); - setEmbeddingProvider({ - name: "test", - dimensions: 3, - embed: async () => new Float32Array([0.1, 0.2, 0.3]), - embedBatch: async (t: string[]) => - t.map(() => new Float32Array([0.1, 0.2, 0.3])), - }); - - const count = await rebuildIndex(kv as never); - - // Indexed: only the real observation. - expect(count).toBe(1); - expect(getSearchIndex().has("obs_real")).toBe(true); - expect(getSearchIndex().has("obs_echo")).toBe(false); - expect( - getVectorIndex()! - .search(new Float32Array([0.1, 0.2, 0.3]), 10) - .map((r) => r.obsId), - ).toEqual(["obs_real"]); - - // Kept: the echo is still fully readable from KV, so timeline, - // session history, the viewer and expandIds all still see it. - const stored = await kv.get( - KV.observations("ses_1"), - "obs_echo", - ); - expect(stored).not.toBeNull(); - expect(stored!.title).toContain("Memory search:"); - expect( - (await kv.list(KV.observations("ses_1"))).length, - ).toBe(2); - }); -}); +import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"; + +vi.mock("../src/logger.js", () => ({ + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, +})); + +import { + isIndexExcluded, + rebuildIndex, + getSearchIndex, + setVectorIndex, + setEmbeddingProvider, + getVectorIndex, +} from "../src/functions/search.js"; +import { VectorIndex } from "../src/state/vector-index.js"; +import { KV } from "../src/state/schema.js"; +import type { CompressedObservation, Session } from "../src/types.js"; + +const FLAG = "AGENTMEMORY_INDEX_EXCLUDE_TOOL_PREFIXES"; +const ECHO_PREFIX = "mcp__plugin_agentmemory_agentmemory__"; + +function mockKV() { + const store = new Map>(); + return { + get: async (scope: string, key: string): Promise => + (store.get(scope)?.get(key) as T) ?? null, + set: async (scope: string, key: string, data: T): Promise => { + if (!store.has(scope)) store.set(scope, new Map()); + store.get(scope)!.set(key, data); + return data; + }, + delete: async (scope: string, key: string): Promise => { + store.get(scope)?.delete(key); + }, + list: async (scope: string): Promise => + Array.from(store.get(scope)?.values() ?? []) as T[], + }; +} + +function obs(id: string, toolName?: string): CompressedObservation { + return { + id, + sessionId: "ses_1", + timestamp: "2026-01-01T00:00:00Z", + type: "search", + title: toolName ? `Memory search: ${id}` : `Real work ${id}`, + facts: [], + narrative: "residentKey teleport vnet mainpc-rdp", + concepts: [], + files: [], + importance: 5, + ...(toolName ? { toolName } : {}), + }; +} + +describe("index exclusion by toolName prefix", () => { + const saved = process.env[FLAG]; + + beforeEach(() => { + delete process.env[FLAG]; + getSearchIndex().clear(); + setVectorIndex(null); + setEmbeddingProvider(null); + }); + afterEach(() => { + if (saved === undefined) delete process.env[FLAG]; + else process.env[FLAG] = saved; + getSearchIndex().clear(); + setVectorIndex(null); + setEmbeddingProvider(null); + }); + + it("excludes agentmemory's own MCP tools when the flag is unset (#993)", () => { + expect(isIndexExcluded(obs("o", `${ECHO_PREFIX}memory_smart_search`))).toBe(true); + expect(isIndexExcluded(obs("o", "mcp__agentmemory__memory_recall"))).toBe(true); + expect(isIndexExcluded(obs("o", "Bash"))).toBe(false); + }); + + it("excludes nothing when the flag is set empty", () => { + process.env[FLAG] = ""; + expect(isIndexExcluded(obs("o", `${ECHO_PREFIX}memory_smart_search`))).toBe(false); + }); + + it("matches on prefix, and only on the configured prefixes", () => { + process.env[FLAG] = ECHO_PREFIX; + expect(isIndexExcluded(obs("o", `${ECHO_PREFIX}memory_smart_search`))).toBe(true); + expect(isIndexExcluded(obs("o", `${ECHO_PREFIX}memory_recall`))).toBe(true); + expect(isIndexExcluded(obs("o", "Bash"))).toBe(false); + expect(isIndexExcluded(obs("o", "mcp__veeam-rag__hybrid_search"))).toBe(false); + // A record with no toolName (every pre-existing row on disk) is never + // matched by this predicate — that is why legacy echoes need a + // separate, explicitly reviewed deletion pass. + expect(isIndexExcluded(obs("o"))).toBe(false); + }); + + it("accepts a comma-separated list and tolerates whitespace", () => { + process.env[FLAG] = ` ${ECHO_PREFIX} , other__ `; + expect(isIndexExcluded(obs("o", `${ECHO_PREFIX}x`))).toBe(true); + expect(isIndexExcluded(obs("o", "other__thing"))).toBe(true); + expect(isIndexExcluded(obs("o", "unrelated"))).toBe(false); + }); + + it("keeps the row in KV while dropping it from BOTH indexes", async () => { + process.env[FLAG] = ECHO_PREFIX; + const kv = mockKV(); + const session: Session = { + id: "ses_1", + project: "demo", + cwd: "/tmp/demo", + startedAt: "2026-01-01T00:00:00Z", + status: "completed", + observationCount: 2, + }; + await kv.set(KV.sessions, session.id, session); + + const echo = obs("obs_echo", `${ECHO_PREFIX}memory_smart_search`); + const real = obs("obs_real"); + await kv.set(KV.observations("ses_1"), echo.id, echo); + await kv.set(KV.observations("ses_1"), real.id, real); + + setVectorIndex(new VectorIndex()); + setEmbeddingProvider({ + name: "test", + dimensions: 3, + embed: async () => new Float32Array([0.1, 0.2, 0.3]), + embedBatch: async (t: string[]) => + t.map(() => new Float32Array([0.1, 0.2, 0.3])), + }); + + const count = await rebuildIndex(kv as never); + + // Indexed: only the real observation. + expect(count).toBe(1); + expect(getSearchIndex().has("obs_real")).toBe(true); + expect(getSearchIndex().has("obs_echo")).toBe(false); + expect( + getVectorIndex()! + .search(new Float32Array([0.1, 0.2, 0.3]), 10) + .map((r) => r.obsId), + ).toEqual(["obs_real"]); + + // Kept: the echo is still fully readable from KV, so timeline, + // session history, the viewer and expandIds all still see it. + const stored = await kv.get( + KV.observations("ses_1"), + "obs_echo", + ); + expect(stored).not.toBeNull(); + expect(stored!.title).toContain("Memory search:"); + expect( + (await kv.list(KV.observations("ses_1"))).length, + ).toBe(2); + }); +}); From 2f99c658f3d62e92a0fba830c1d2fc4cfbe6241c Mon Sep 17 00:00:00 2001 From: neal Date: Fri, 25 Sep 2026 18:39:38 -0400 Subject: [PATCH 10/17] fix(replay): skip Claude Code's Warmup and summary transcripts on import import-jsonl recorded these background jobs as sessions the Operator never worked in. A transcript whose first prompt is Warmup or the conversation-list summary instruction is now skipped (#1064). Generated-By: Claude Opus 5.5 (1M context) --- src/functions/replay.ts | 10 +++++++++ test/replay-import-key.test.ts | 40 ++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/src/functions/replay.ts b/src/functions/replay.ts index b4ebb80ce8..ff51725d13 100644 --- a/src/functions/replay.ts +++ b/src/functions/replay.ts @@ -35,6 +35,15 @@ const SENSITIVE_PATH_PATTERNS: RegExp[] = [ /(^|[\\/])api[_-]?token([\\/_.-]|$)/i, ]; +// Claude Code writes its own background jobs to ~/.claude/projects beside +// real work: a Warmup probe and the conversation-list summary job (#1064). +function isHarnessPrompt(firstPrompt: string): boolean { + return ( + firstPrompt === "Warmup" || + firstPrompt.startsWith("Context: This summary will be shown in a list") + ); +} + export function isSensitive(path: string): boolean { return SENSITIVE_PATH_PATTERNS.some((re) => re.test(path)); } @@ -413,6 +422,7 @@ export function registerReplayFunctions(sdk: ISdk, kv: StateKV): void { const firstPrompt = firstPromptObs?.userPrompt ? firstPromptObs.userPrompt.replace(/\s+/g, " ").trim().slice(0, 200) : undefined; + if (firstPrompt && isHarnessPrompt(firstPrompt)) continue; const existing = await kv.get(KV.sessions, parsed.sessionId); if (existing) { diff --git a/test/replay-import-key.test.ts b/test/replay-import-key.test.ts index 1c1c991f3e..c125966534 100644 --- a/test/replay-import-key.test.ts +++ b/test/replay-import-key.test.ts @@ -246,3 +246,43 @@ describe("import-jsonl indexes observations into BM25 AND vector", () => { expect(getSearchIndex().size).toBeGreaterThan(0); }); }); + +describe("import-jsonl skips Claude Code's own background transcripts (#1064)", () => { + function writeTranscript(root: string, sessionId: string, prompt: string) { + const dir = join(root, "proj"); + require("node:fs").mkdirSync(dir, { recursive: true }); + const ts = "2026-04-17T10:00:00.000Z"; + writeFileSync( + join(dir, `${sessionId}.jsonl`), + [ + { type: "user", uuid: "u1", sessionId, timestamp: ts, cwd: root, + message: { role: "user", content: [{ type: "text", text: prompt }] } }, + { type: "assistant", uuid: "a1", sessionId, timestamp: ts, + message: { role: "assistant", content: [{ type: "text", text: "ok" }] } }, + ].map((l) => JSON.stringify(l)).join("\n") + "\n", + ); + } + + it("imports the real session and neither the Warmup probe nor the summary job", async () => { + const root = mkdtempSync(join(tmpdir(), "replay-harness-")); + writeTranscript(root, "sess-real", "Fix the login bug"); + writeTranscript(root, "sess-warmup", "Warmup"); + writeTranscript( + root, + "sess-summary", + "Context: This summary will be shown in a list to help users and Claude choose which conversations are relevant.", + ); + const kv = mockKV(); + const sdk = mockSdk(kv); + registerReplayFunctions(sdk, kv as never); + + const result = (await sdk.trigger("mem::replay::import-jsonl", { + path: root, + })) as { sessionIds: string[] }; + + expect(result.sessionIds).toEqual(["sess-real"]); + expect(await kv.get(KV.sessions, "sess-warmup")).toBeNull(); + expect(await kv.get(KV.sessions, "sess-summary")).toBeNull(); + rmSync(root, { recursive: true, force: true }); + }); +}); From db9f5db8d3e4ac7f525a3f090e953a5ac13d6bdd Mon Sep 17 00:00:00 2001 From: neal Date: Fri, 25 Sep 2026 18:40:48 -0400 Subject: [PATCH 11/17] fix(observe): evict the least important observation at the session cap At MAX_OBS_PER_SESSION, mem::observe refused every new observation, so the end of each long session was silently lost; ten live sessions sat at exactly 500. The cap now evicts the lowest-importance row, oldest first on a tie, from KV and both indexes, and logs a warning (PR#1174). Generated-By: Claude Opus 5.5 (1M context) --- .env.example | 2 +- src/functions/observe.ts | 779 ++++++++++++++++--------------- test/observe-session-cap.test.ts | 107 +++++ 3 files changed, 519 insertions(+), 369 deletions(-) create mode 100644 test/observe-session-cap.test.ts diff --git a/.env.example b/.env.example index 6ca3307219..b561f1a65e 100644 --- a/.env.example +++ b/.env.example @@ -124,7 +124,7 @@ # VECTOR_WEIGHT=0.6 # Hybrid search weight for vector leg # AGENTMEMORY_GRAPH_WEIGHT=0.2 # Graph traversal bonus on smart-search ranking # TOKEN_BUDGET=2000 # Max tokens injected via mem::context per session -# MAX_OBS_PER_SESSION=500 # Per-session observation cap before consolidation kicks in +# MAX_OBS_PER_SESSION=500 # Per-session observation cap; at the cap the least important observation is evicted to admit the new one # SUMMARIZE_CHUNK_TOKENS=50000 # Prompt-token budget per chunk when mem::summarize map-reduces a large session (chunk-summarize → reduce-merge). Observations are measured with the provider's tokenize route when it has one (llama.cpp-style POST /tokenize) and estimated at chars/3 otherwise; the ~400-token system prompt is reserved inside this budget. A 50k chunk summarizes cold in 60–89s on a local 35B model. # SUMMARIZE_CHUNK_CONCURRENCY=2 # Parallel chunk LLM calls during chunked summarize. A local broker serialises chunks on one GPU, so more concurrency only stretches each call: AGENTMEMORY_LLM_TIMEOUT_MS is sized for 2. Raise it for hosted providers that run chunks in parallel, and re-measure the timeout if you do. diff --git a/src/functions/observe.ts b/src/functions/observe.ts index f9214af090..dfffe9111c 100644 --- a/src/functions/observe.ts +++ b/src/functions/observe.ts @@ -1,368 +1,411 @@ -import { TriggerAction, type ISdk } from "../engine/types.js"; -import type { RawObservation, HookPayload, Origin } from "../types.js"; - -const TOOL_HOOKS = new Set(["pre_tool_use", "post_tool_use", "post_tool_failure"]); -import { KV, STREAM, generateId } from "../state/schema.js"; -import { StateKV } from "../state/kv.js"; -import { stripPrivateData } from "./privacy.js"; -import { DedupMap } from "./dedup.js"; -import { withKeyedLock } from "../state/keyed-mutex.js"; -import { isAutoCompressEnabled } from "../config.js"; -import { buildSyntheticCompression } from "./compress-synthetic.js"; -import { getSearchIndex, vectorIndexAddGuarded, isIndexExcluded } from "./search.js"; -import { getAgentId } from "../config.js"; -import { logger } from "../logger.js"; -import { saveImageToDisk } from "../utils/image-store.js"; - -export function extractImage(d: unknown): string | undefined { - if (!d) return undefined; - if (typeof d === "string") { - if (d.startsWith("data:image/") || d.startsWith("iVBORw0KGgo") || d.startsWith("/9j/")) { - return d; - } - return undefined; - } - if (typeof d === "object" && d !== null) { - const obj = d as Record; - if (typeof obj["image_data"] === "string") return obj["image_data"]; - if (typeof obj["image_path"] === "string") return obj["image_path"]; - if (typeof obj["imageBase64"] === "string") return obj["imageBase64"]; - if (typeof obj["imagePath"] === "string") return obj["imagePath"]; - - for (const key of Object.keys(obj)) { - const match = extractImage(obj[key]); - if (match) return match; - } - } - return undefined; -} - -export function registerObserveFunction( - sdk: ISdk, - kv: StateKV, - dedupMap?: DedupMap, - maxObservationsPerSession?: number, -): void { - sdk.registerFunction("mem::observe", - async (payload: HookPayload) => { - - if ( - !payload?.sessionId || - typeof payload.sessionId !== "string" || - !payload.hookType || - typeof payload.hookType !== "string" || - !payload.timestamp || - typeof payload.timestamp !== "string" - ) { - return { - success: false, - error: - "Invalid payload: sessionId, hookType, and timestamp are required", - }; - } - - const obsId = generateId("obs"); - - let dedupHash: string | undefined; - if (dedupMap) { - const dataIsObject = - typeof payload.data === "object" && payload.data !== null; - const d = dataIsObject - ? (payload.data as Record) - : {}; - const toolName = (d["tool_name"] as string) || payload.hookType; - // Hash the full payload when tool_input is absent so distinct - // events never collapse onto one key. - const dedupInput = - d["tool_input"] !== undefined - ? d["tool_input"] - : dataIsObject - ? d - : payload.data; - dedupHash = dedupMap.computeHash( - payload.sessionId, - toolName, - dedupInput, - ); - if (dedupMap.isDuplicate(dedupHash)) { - return { deduplicated: true, sessionId: payload.sessionId }; - } - } - - let sanitizedRaw: unknown = payload.data; - try { - const jsonStr = JSON.stringify(payload.data); - const sanitized = stripPrivateData(jsonStr); - sanitizedRaw = JSON.parse(sanitized); - } catch { - sanitizedRaw = stripPrivateData(String(payload.data)); - } - - let originChannel: Origin["channel"] = "agent"; - if (payload.hookType === "prompt_submit") originChannel = "user"; - else if (TOOL_HOOKS.has(payload.hookType)) originChannel = "tool"; - const raw: RawObservation = { - id: obsId, - sessionId: payload.sessionId, - timestamp: payload.timestamp, - hookType: payload.hookType, - raw: sanitizedRaw, - origin: { - channel: originChannel, - capturedAt: payload.timestamp, - }, - }; - - let extractedImage: string | undefined; - - if (typeof sanitizedRaw === "object" && sanitizedRaw !== null) { - const d = sanitizedRaw as Record; - if ( - payload.hookType === "post_tool_use" || - payload.hookType === "post_tool_failure" - ) { - raw.toolName = d["tool_name"] as string | undefined; - raw.toolInput = d["tool_input"]; - raw.toolOutput = d["tool_output"] || d["error"]; - if (raw.origin && raw.toolName) raw.origin.detail = raw.toolName; - } - if (payload.hookType === "prompt_submit") { - raw.userPrompt = d["prompt"] as string | undefined; - } - - extractedImage = extractImage(sanitizedRaw); - if (extractedImage) { - raw.modality = (raw.toolInput || raw.toolOutput || raw.userPrompt) ? "mixed" : "image"; - } - } else if (typeof sanitizedRaw === "string") { - extractedImage = extractImage(sanitizedRaw); - if (extractedImage) { - raw.modality = "image"; - } - } - - const pendingImageData = extractedImage; - - return withKeyedLock(`obs:${payload.sessionId}`, async () => { - if (maxObservationsPerSession && maxObservationsPerSession > 0) { - const existing = await kv.list(KV.observations(payload.sessionId)); - if (existing.length >= maxObservationsPerSession) { - return { - success: false, - error: `Session observation limit reached (${maxObservationsPerSession})`, - }; - } - } - - // Existing session is the source of truth for agentId (even - // undefined). Env AGENT_ID only fires when no session row - // exists yet — otherwise an unscoped session would get - // retroactively scoped by a later AGENT_ID export. - const existingSession = await kv.get<{ - agentId?: string; - observationCount?: number; - firstPrompt?: string; - }>(KV.sessions, payload.sessionId); - const inheritedAgentId = existingSession - ? existingSession.agentId - : getAgentId(); - if (inheritedAgentId) { - raw.agentId = inheritedAgentId; - } - - if (pendingImageData && (pendingImageData.startsWith("data:image/") || pendingImageData.startsWith("iVBORw0KGgo") || pendingImageData.startsWith("/9j/"))) { - const { filePath, bytesWritten } = await saveImageToDisk(pendingImageData); - raw.imageData = filePath; - const { incrementImageRef } = await import("./image-refs.js"); - await incrementImageRef(kv, filePath); - sdk.trigger({ - function_id: "mem::disk-size-delta", - payload: { deltaBytes: bytesWritten }, - action: TriggerAction.Void(), - }); - if (process.env["AGENTMEMORY_IMAGE_EMBEDDINGS"] === "true") { - sdk.trigger({ - function_id: "mem::vision-embed", - payload: { - imageRef: filePath, - sessionId: payload.sessionId, - observationId: obsId, - }, - action: TriggerAction.Void(), - }); - } - } - - try { - - await kv.set(KV.observations(payload.sessionId), obsId, raw); - - } catch (error) { - if (raw.imageData) { - // Roll back the ref taken above. decrementImageRef deletes the file - // only when no other observation still references it (deduped images - // survive) and emits the disk-size delta itself — deleting the file - // directly here would orphan shared images and leave a stale ref. - // If the rollback itself fails, log it but still surface the - // original write error (the more useful failure to diagnose). - try { - const { decrementImageRef } = await import("./image-refs.js"); - await decrementImageRef(kv, sdk, raw.imageData); - } catch (rollbackError) { - logger.error("Failed to roll back image ref after observation write failure", { - imageRef: raw.imageData, - error: rollbackError instanceof Error ? rollbackError.message : String(rollbackError), - }); - } - } - throw error; - } - - if (dedupMap && dedupHash) { - dedupMap.record(dedupHash); - } - - await sdk.trigger({ - function_id: "stream::set", - payload: { - stream_name: STREAM.name, - group_id: STREAM.group(payload.sessionId), - item_id: obsId, - data: { type: "raw", observation: raw }, - }, - }); - - await sdk.trigger({ - function_id: "stream::send", - payload: { - stream_name: STREAM.name, - group_id: STREAM.viewerGroup, - id: `raw-${obsId}`, - type: "raw_observation", - data: { type: "raw", observation: raw, sessionId: payload.sessionId }, - }, - action: TriggerAction.Void(), - }); - - const session = existingSession; - if (session) { - const updates: Array<{ type: "set"; path: string; value: unknown }> = [ - { type: "set", path: "updatedAt", value: new Date().toISOString() }, - { - type: "set", - path: "observationCount", - value: (session.observationCount || 0) + 1, - }, - ]; - if (!session.firstPrompt && typeof raw.userPrompt === "string") { - const trimmed = raw.userPrompt.replace(/\s+/g, " ").trim(); - if (trimmed.length > 0) { - updates.push({ - type: "set", - path: "firstPrompt", - value: trimmed.slice(0, 200), - }); - } - } - await kv.update(KV.sessions, payload.sessionId, updates); - } else if ( - typeof payload.project === "string" && - payload.project.trim().length > 0 && - typeof payload.cwd === "string" && - payload.cwd.trim().length > 0 - ) { - // OpenCode (and any plugin that skips POST /session/start) - // can fire observations before the session record exists. Without - // an implicit create, those observations stack up but - // `memory_sessions` never lists them, and summarize bails with - // "Session not found for summarize". Create the session now from - // the observation payload — but only when project + cwd are - // present (HookPayload contract). Older test payloads without - // those fields keep their original no-op behaviour. - const trimmedPrompt = - typeof raw.userPrompt === "string" - ? raw.userPrompt.replace(/\s+/g, " ").trim().slice(0, 200) - : undefined; - const ts = new Date().toISOString(); - await kv.set(KV.sessions, payload.sessionId, { - id: payload.sessionId, - project: payload.project, - cwd: payload.cwd, - startedAt: payload.timestamp ?? ts, - updatedAt: ts, - status: "active", - observationCount: 1, - ...(inheritedAgentId ? { agentId: inheritedAgentId } : {}), - ...(trimmedPrompt && trimmedPrompt.length > 0 - ? { firstPrompt: trimmedPrompt } - : {}), - }); - } - - // Per-observation LLM compression is opt-in as of 0.8.8. - // Default path: build a zero-LLM synthetic compression so recall - // and BM25 search still work without burning the user's Claude - // token allocation on every tool invocation. - if (isAutoCompressEnabled()) { - await sdk.trigger({ - function_id: "mem::compress", - payload: { - observationId: obsId, - sessionId: payload.sessionId, - raw, - }, - action: TriggerAction.Void(), - }); - } else { - const synthetic = buildSyntheticCompression(raw); - await kv.set( - KV.observations(payload.sessionId), - obsId, - synthetic, - ); - // Stored above unconditionally; only the INDEX writes are - // skipped for excluded tools (retrieval echoes). - if (!isIndexExcluded(synthetic)) { - getSearchIndex().add(synthetic); - await vectorIndexAddGuarded( - synthetic.id, - synthetic.sessionId, - synthetic.title + " " + (synthetic.narrative || ""), - { kind: "synthetic", logId: synthetic.id }, - ); - } - await sdk.trigger({ - function_id: "stream::set", - payload: { - stream_name: STREAM.name, - group_id: STREAM.group(payload.sessionId), - item_id: obsId, - data: { type: "compressed", observation: synthetic }, - }, - }); - await sdk.trigger({ - function_id: "stream::set", - payload: { - stream_name: STREAM.name, - group_id: STREAM.viewerGroup, - item_id: obsId, - data: { - type: "compressed", - observation: synthetic, - sessionId: payload.sessionId, - }, - }, - }); - } - - logger.info("Observation captured", { - obsId, - sessionId: payload.sessionId, - hook: payload.hookType, - compress: isAutoCompressEnabled() ? "llm" : "synthetic", - }); - return { observationId: obsId }; - }); - }, - ); -} +import { TriggerAction, type ISdk } from "../engine/types.js"; +import type { RawObservation, HookPayload, Origin } from "../types.js"; + +const TOOL_HOOKS = new Set(["pre_tool_use", "post_tool_use", "post_tool_failure"]); +import { KV, STREAM, generateId } from "../state/schema.js"; +import { StateKV } from "../state/kv.js"; +import { stripPrivateData } from "./privacy.js"; +import { DedupMap } from "./dedup.js"; +import { withKeyedLock } from "../state/keyed-mutex.js"; +import { isAutoCompressEnabled } from "../config.js"; +import { buildSyntheticCompression } from "./compress-synthetic.js"; +import { getSearchIndex, vectorIndexAddGuarded, isIndexExcluded, deleteIndexed } from "./search.js"; +import { decrementImageRef } from "./image-refs.js"; +import { getAgentId } from "../config.js"; +import { logger } from "../logger.js"; +import { saveImageToDisk } from "../utils/image-store.js"; + +export function extractImage(d: unknown): string | undefined { + if (!d) return undefined; + if (typeof d === "string") { + if (d.startsWith("data:image/") || d.startsWith("iVBORw0KGgo") || d.startsWith("/9j/")) { + return d; + } + return undefined; + } + if (typeof d === "object" && d !== null) { + const obj = d as Record; + if (typeof obj["image_data"] === "string") return obj["image_data"]; + if (typeof obj["image_path"] === "string") return obj["image_path"]; + if (typeof obj["imageBase64"] === "string") return obj["imageBase64"]; + if (typeof obj["imagePath"] === "string") return obj["imagePath"]; + + for (const key of Object.keys(obj)) { + const match = extractImage(obj[key]); + if (match) return match; + } + } + return undefined; +} + +// Uncompressed rows carry no importance yet; rank them just below the +// default so a scored row of average value outlives an unscored one. +const UNSCORED_IMPORTANCE = 3; + +// A session at its cap still admits the newest observation: the work at the +// end of a long session is what a later one most often needs. The least +// important rows go first, oldest breaking ties (PR#1174). +async function evictToAdmitOne( + sdk: ISdk, + kv: StateKV, + sessionId: string, + cap: number, +): Promise { + const scope = KV.observations(sessionId); + const existing = await kv.list<{ + id: string; + timestamp?: string; + importance?: number; + imageData?: string; + imageRef?: string; + }>(scope); + const excess = existing.length - cap + 1; + if (excess <= 0) return; + const victims = existing + .sort( + (a, b) => + (a.importance ?? UNSCORED_IMPORTANCE) - (b.importance ?? UNSCORED_IMPORTANCE) || + (a.timestamp ?? "").localeCompare(b.timestamp ?? ""), + ) + .slice(0, excess); + let evicted = 0; + for (const obs of victims) { + try { + await deleteIndexed(kv, scope, obs.id); + evicted++; + } catch { + continue; + } + if (obs.imageData) await decrementImageRef(kv, sdk, obs.imageData); + if (obs.imageRef && obs.imageRef !== obs.imageData) await decrementImageRef(kv, sdk, obs.imageRef); + } + logger.warn("Session observation cap reached; evicted least important", { + sessionId, + cap, + evicted, + }); +} + +export function registerObserveFunction( + sdk: ISdk, + kv: StateKV, + dedupMap?: DedupMap, + maxObservationsPerSession?: number, +): void { + sdk.registerFunction("mem::observe", + async (payload: HookPayload) => { + + if ( + !payload?.sessionId || + typeof payload.sessionId !== "string" || + !payload.hookType || + typeof payload.hookType !== "string" || + !payload.timestamp || + typeof payload.timestamp !== "string" + ) { + return { + success: false, + error: + "Invalid payload: sessionId, hookType, and timestamp are required", + }; + } + + const obsId = generateId("obs"); + + let dedupHash: string | undefined; + if (dedupMap) { + const dataIsObject = + typeof payload.data === "object" && payload.data !== null; + const d = dataIsObject + ? (payload.data as Record) + : {}; + const toolName = (d["tool_name"] as string) || payload.hookType; + // Hash the full payload when tool_input is absent so distinct + // events never collapse onto one key. + const dedupInput = + d["tool_input"] !== undefined + ? d["tool_input"] + : dataIsObject + ? d + : payload.data; + dedupHash = dedupMap.computeHash( + payload.sessionId, + toolName, + dedupInput, + ); + if (dedupMap.isDuplicate(dedupHash)) { + return { deduplicated: true, sessionId: payload.sessionId }; + } + } + + let sanitizedRaw: unknown = payload.data; + try { + const jsonStr = JSON.stringify(payload.data); + const sanitized = stripPrivateData(jsonStr); + sanitizedRaw = JSON.parse(sanitized); + } catch { + sanitizedRaw = stripPrivateData(String(payload.data)); + } + + let originChannel: Origin["channel"] = "agent"; + if (payload.hookType === "prompt_submit") originChannel = "user"; + else if (TOOL_HOOKS.has(payload.hookType)) originChannel = "tool"; + const raw: RawObservation = { + id: obsId, + sessionId: payload.sessionId, + timestamp: payload.timestamp, + hookType: payload.hookType, + raw: sanitizedRaw, + origin: { + channel: originChannel, + capturedAt: payload.timestamp, + }, + }; + + let extractedImage: string | undefined; + + if (typeof sanitizedRaw === "object" && sanitizedRaw !== null) { + const d = sanitizedRaw as Record; + if ( + payload.hookType === "post_tool_use" || + payload.hookType === "post_tool_failure" + ) { + raw.toolName = d["tool_name"] as string | undefined; + raw.toolInput = d["tool_input"]; + raw.toolOutput = d["tool_output"] || d["error"]; + if (raw.origin && raw.toolName) raw.origin.detail = raw.toolName; + } + if (payload.hookType === "prompt_submit") { + raw.userPrompt = d["prompt"] as string | undefined; + } + + extractedImage = extractImage(sanitizedRaw); + if (extractedImage) { + raw.modality = (raw.toolInput || raw.toolOutput || raw.userPrompt) ? "mixed" : "image"; + } + } else if (typeof sanitizedRaw === "string") { + extractedImage = extractImage(sanitizedRaw); + if (extractedImage) { + raw.modality = "image"; + } + } + + const pendingImageData = extractedImage; + + return withKeyedLock(`obs:${payload.sessionId}`, async () => { + if (maxObservationsPerSession && maxObservationsPerSession > 0) { + await evictToAdmitOne(sdk, kv, payload.sessionId, maxObservationsPerSession); + } + + // Existing session is the source of truth for agentId (even + // undefined). Env AGENT_ID only fires when no session row + // exists yet — otherwise an unscoped session would get + // retroactively scoped by a later AGENT_ID export. + const existingSession = await kv.get<{ + agentId?: string; + observationCount?: number; + firstPrompt?: string; + }>(KV.sessions, payload.sessionId); + const inheritedAgentId = existingSession + ? existingSession.agentId + : getAgentId(); + if (inheritedAgentId) { + raw.agentId = inheritedAgentId; + } + + if (pendingImageData && (pendingImageData.startsWith("data:image/") || pendingImageData.startsWith("iVBORw0KGgo") || pendingImageData.startsWith("/9j/"))) { + const { filePath, bytesWritten } = await saveImageToDisk(pendingImageData); + raw.imageData = filePath; + const { incrementImageRef } = await import("./image-refs.js"); + await incrementImageRef(kv, filePath); + sdk.trigger({ + function_id: "mem::disk-size-delta", + payload: { deltaBytes: bytesWritten }, + action: TriggerAction.Void(), + }); + if (process.env["AGENTMEMORY_IMAGE_EMBEDDINGS"] === "true") { + sdk.trigger({ + function_id: "mem::vision-embed", + payload: { + imageRef: filePath, + sessionId: payload.sessionId, + observationId: obsId, + }, + action: TriggerAction.Void(), + }); + } + } + + try { + + await kv.set(KV.observations(payload.sessionId), obsId, raw); + + } catch (error) { + if (raw.imageData) { + // Roll back the ref taken above. decrementImageRef deletes the file + // only when no other observation still references it (deduped images + // survive) and emits the disk-size delta itself — deleting the file + // directly here would orphan shared images and leave a stale ref. + // If the rollback itself fails, log it but still surface the + // original write error (the more useful failure to diagnose). + try { + const { decrementImageRef } = await import("./image-refs.js"); + await decrementImageRef(kv, sdk, raw.imageData); + } catch (rollbackError) { + logger.error("Failed to roll back image ref after observation write failure", { + imageRef: raw.imageData, + error: rollbackError instanceof Error ? rollbackError.message : String(rollbackError), + }); + } + } + throw error; + } + + if (dedupMap && dedupHash) { + dedupMap.record(dedupHash); + } + + await sdk.trigger({ + function_id: "stream::set", + payload: { + stream_name: STREAM.name, + group_id: STREAM.group(payload.sessionId), + item_id: obsId, + data: { type: "raw", observation: raw }, + }, + }); + + await sdk.trigger({ + function_id: "stream::send", + payload: { + stream_name: STREAM.name, + group_id: STREAM.viewerGroup, + id: `raw-${obsId}`, + type: "raw_observation", + data: { type: "raw", observation: raw, sessionId: payload.sessionId }, + }, + action: TriggerAction.Void(), + }); + + const session = existingSession; + if (session) { + const updates: Array<{ type: "set"; path: string; value: unknown }> = [ + { type: "set", path: "updatedAt", value: new Date().toISOString() }, + { + type: "set", + path: "observationCount", + value: (session.observationCount || 0) + 1, + }, + ]; + if (!session.firstPrompt && typeof raw.userPrompt === "string") { + const trimmed = raw.userPrompt.replace(/\s+/g, " ").trim(); + if (trimmed.length > 0) { + updates.push({ + type: "set", + path: "firstPrompt", + value: trimmed.slice(0, 200), + }); + } + } + await kv.update(KV.sessions, payload.sessionId, updates); + } else if ( + typeof payload.project === "string" && + payload.project.trim().length > 0 && + typeof payload.cwd === "string" && + payload.cwd.trim().length > 0 + ) { + // OpenCode (and any plugin that skips POST /session/start) + // can fire observations before the session record exists. Without + // an implicit create, those observations stack up but + // `memory_sessions` never lists them, and summarize bails with + // "Session not found for summarize". Create the session now from + // the observation payload — but only when project + cwd are + // present (HookPayload contract). Older test payloads without + // those fields keep their original no-op behaviour. + const trimmedPrompt = + typeof raw.userPrompt === "string" + ? raw.userPrompt.replace(/\s+/g, " ").trim().slice(0, 200) + : undefined; + const ts = new Date().toISOString(); + await kv.set(KV.sessions, payload.sessionId, { + id: payload.sessionId, + project: payload.project, + cwd: payload.cwd, + startedAt: payload.timestamp ?? ts, + updatedAt: ts, + status: "active", + observationCount: 1, + ...(inheritedAgentId ? { agentId: inheritedAgentId } : {}), + ...(trimmedPrompt && trimmedPrompt.length > 0 + ? { firstPrompt: trimmedPrompt } + : {}), + }); + } + + // Per-observation LLM compression is opt-in as of 0.8.8. + // Default path: build a zero-LLM synthetic compression so recall + // and BM25 search still work without burning the user's Claude + // token allocation on every tool invocation. + if (isAutoCompressEnabled()) { + await sdk.trigger({ + function_id: "mem::compress", + payload: { + observationId: obsId, + sessionId: payload.sessionId, + raw, + }, + action: TriggerAction.Void(), + }); + } else { + const synthetic = buildSyntheticCompression(raw); + await kv.set( + KV.observations(payload.sessionId), + obsId, + synthetic, + ); + // Stored above unconditionally; only the INDEX writes are + // skipped for excluded tools (retrieval echoes). + if (!isIndexExcluded(synthetic)) { + getSearchIndex().add(synthetic); + await vectorIndexAddGuarded( + synthetic.id, + synthetic.sessionId, + synthetic.title + " " + (synthetic.narrative || ""), + { kind: "synthetic", logId: synthetic.id }, + ); + } + await sdk.trigger({ + function_id: "stream::set", + payload: { + stream_name: STREAM.name, + group_id: STREAM.group(payload.sessionId), + item_id: obsId, + data: { type: "compressed", observation: synthetic }, + }, + }); + await sdk.trigger({ + function_id: "stream::set", + payload: { + stream_name: STREAM.name, + group_id: STREAM.viewerGroup, + item_id: obsId, + data: { + type: "compressed", + observation: synthetic, + sessionId: payload.sessionId, + }, + }, + }); + } + + logger.info("Observation captured", { + obsId, + sessionId: payload.sessionId, + hook: payload.hookType, + compress: isAutoCompressEnabled() ? "llm" : "synthetic", + }); + return { observationId: obsId }; + }); + }, + ); +} diff --git a/test/observe-session-cap.test.ts b/test/observe-session-cap.test.ts new file mode 100644 index 0000000000..b0153bc529 --- /dev/null +++ b/test/observe-session-cap.test.ts @@ -0,0 +1,107 @@ +import { describe, it, expect, vi } from "vitest"; + +vi.mock("../src/logger.js", () => ({ + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, +})); + +import { registerObserveFunction } from "../src/functions/observe.js"; +import { getSearchIndex } from "../src/functions/search.js"; +import { KV } from "../src/state/schema.js"; + +function mockKV() { + const store = new Map>(); + return { + get: async (scope: string, key: string): Promise => + (store.get(scope)?.get(key) as T) ?? null, + set: async (scope: string, key: string, data: T): Promise => { + if (!store.has(scope)) store.set(scope, new Map()); + store.get(scope)!.set(key, data); + return data; + }, + delete: async (scope: string, key: string) => { + store.get(scope)?.delete(key); + }, + list: async (scope: string): Promise => + Array.from(store.get(scope)?.values() ?? []) as T[], + }; +} + +function mockSdk() { + const fns = new Map(); + return { + registerFunction: (id: string, fn: Function) => fns.set(id, fn), + registerTrigger: () => {}, + trigger: async (input: { function_id: string; payload?: unknown }) => + fns.get(input.function_id)?.(input.payload) ?? null, + }; +} + +const SESSION = "ses_cap"; + +function stored(id: string, importance: number | undefined, timestamp: string) { + return { + id, + sessionId: SESSION, + timestamp, + title: `obs ${id}`, + type: "file_read", + facts: [], + narrative: "", + concepts: [], + files: [], + ...(importance === undefined ? {} : { importance }), + }; +} + +async function setup(rows: ReturnType[], cap: number) { + const kv = mockKV(); + const sdk = mockSdk(); + for (const r of rows) await kv.set(KV.observations(SESSION), r.id, r); + registerObserveFunction(sdk as never, kv as never, undefined, cap); + const result = (await sdk.trigger({ + function_id: "mem::observe", + payload: { + sessionId: SESSION, + hookType: "post_tool_use", + timestamp: "2026-05-01T00:00:00Z", + data: { tool_name: "Read", tool_input: { file_path: "new.ts" } }, + }, + })) as { observationId?: string; success?: boolean }; + const ids = (await kv.list<{ id: string }>(KV.observations(SESSION))).map((o) => o.id); + return { result, ids }; +} + +describe("mem::observe at MAX_OBS_PER_SESSION (PR#1174)", () => { + it("admits the new observation by evicting the least important one", async () => { + const { result, ids } = await setup( + [ + stored("keep", 8, "2026-01-01T00:00:00Z"), + stored("drop", 2, "2026-01-02T00:00:00Z"), + stored("raw", undefined, "2026-01-03T00:00:00Z"), + ], + 3, + ); + expect(result.observationId).toBeTruthy(); + expect(ids).toHaveLength(3); + expect(ids).toContain("keep"); + expect(ids).toContain("raw"); + expect(ids).not.toContain("drop"); + }); + + it("evicts the older row on an importance tie and drops it from search", async () => { + getSearchIndex().add(stored("older", 5, "2026-01-01T00:00:00Z") as never); + const { ids } = await setup( + [stored("older", 5, "2026-01-01T00:00:00Z"), stored("newer", 5, "2026-01-02T00:00:00Z")], + 2, + ); + expect(ids).not.toContain("older"); + expect(ids).toContain("newer"); + expect(getSearchIndex().has("older")).toBe(false); + }); + + it("evicts nothing under the cap", async () => { + const { ids } = await setup([stored("a", 1, "2026-01-01T00:00:00Z")], 3); + expect(ids).toHaveLength(2); + expect(ids).toContain("a"); + }); +}); From 2e3e215161614ea379aa7fe298a26fd602bf560f Mon Sep 17 00:00:00 2001 From: neal Date: Fri, 25 Sep 2026 18:42:04 -0400 Subject: [PATCH 12/17] docs(triage): record verdicts for the rows fixed or verified on this branch Generated-By: Claude Opus 5.5 (1M context) --- docs/upstream-issue-triage.yaml | 51 ++++++++++++++++++++++----------- docs/upstream-pr-triage.yaml | 24 +++++++++------- 2 files changed, 49 insertions(+), 26 deletions(-) diff --git a/docs/upstream-issue-triage.yaml b/docs/upstream-issue-triage.yaml index 079672b731..2e8a6a1ac5 100644 --- a/docs/upstream-issue-triage.yaml +++ b/docs/upstream-issue-triage.yaml @@ -1247,7 +1247,8 @@ issues: difficulty: med kind: bug note: "Replace-import wipes KV but leaves the search indices stale until restart." - disposition: candidate + disposition: already-fixed + fixed_by: "Replace-import deletes through deleteIndexed and merge/replace index what they write via indexRecords (src/functions/export-import.ts)." url: https://github.com/rohitg00/agentmemory/issues/938 - number: 944 title: "[viewer] Agentmemory Token Savings always 0% — estInjected counts all sessions, not just active" @@ -1325,7 +1326,9 @@ issues: difficulty: med kind: bug note: "Observations containing CJK text hang import and live ingestion indefinitely." - disposition: candidate + disposition: wont-fix + status: "unresolved — no hang found: the CJK tokenizer makes one forward-only pass (src/state/cjk-segmenter.ts) and truncation cuts by string length; no loop over text can stall." + wont_fix_reason: "not reproducible in this tree" url: https://github.com/rohitg00/agentmemory/issues/969 - number: 972 title: "Add an optional HVTracker trust badge to the README" @@ -1421,7 +1424,9 @@ issues: difficulty: low kind: bug note: "post-tool-use records agentmemory's own MCP calls, polluting recall." - disposition: candidate + disposition: already-fixed + status: "fixed-here — b082b94: AGENTMEMORY_INDEX_EXCLUDE_TOOL_PREFIXES defaults to mcp__plugin_agentmemory_agentmemory__,mcp__agentmemory__, so agentmemory's own MCP calls are stored but not indexed; set it empty to index everything." + fixed_by: "b082b94" url: https://github.com/rohitg00/agentmemory/issues/993 fix_prs: [994, 1044] - number: 996 @@ -1508,7 +1513,8 @@ issues: difficulty: low kind: bug note: "ResilientProvider renames noop, so the summarize short-circuit can never fire." - disposition: candidate + disposition: already-fixed + fixed_by: "isNoopProvider unwraps resilient(...) names (src/providers/noop.ts:22-34), and the summarize and graph gates use it." url: https://github.com/rohitg00/agentmemory/issues/1020 fix_prs: [1040] - number: 1022 @@ -1600,7 +1606,8 @@ issues: difficulty: low kind: bug note: "Hooks crash with exit 1 when the payload JSON is literal null (unguarded data access)." - disposition: candidate + disposition: already-fixed + fixed_by: "Every hook returns on a null or non-object payload right after parsing (e.g. src/hooks/post-tool-use.ts:35, src/hooks/pre-tool-use.ts:55)." url: https://github.com/rohitg00/agentmemory/issues/1047 - number: 1048 title: "feat: add support for gemini-embedding-2 and make Gemini embedding model configurable" @@ -1670,7 +1677,9 @@ issues: difficulty: low kind: bug note: "Claude Code summarizer and Warmup transcripts ingested as real phantom sessions." - disposition: candidate + disposition: already-fixed + status: "fixed-here — 2f99c65: import-jsonl skips a transcript whose first prompt is Warmup or the conversation-list summary instruction. Live store had 0 such sessions; hooks already drop SDK child runs via isSdkChildContext." + fixed_by: "2f99c65" url: https://github.com/rohitg00/agentmemory/issues/1064 - number: 1065 title: "lesson-recall default minConfidence (0.1) + decay + auto-prune make most auto-consolidated lessons invisible, with no config override" @@ -1678,8 +1687,9 @@ issues: difficulty: low kind: bug note: "Default minConfidence plus decay hides 438 of 439 lessons from recall, with no override." - disposition: candidate - status: "present-here — mem::lesson-recall defaults minConfidence to 0.1 and drops anything below it (src/functions/lessons.ts:159,173). The decay sweep floors confidence at 0.05 (src/functions/lessons.ts:320), so a decayed lesson lands in 0.05-0.1 and is invisible at the default. Upstream's \"no override\" is wrong — the param is exposed." + disposition: already-fixed + status: "fixed-here — 98d246c: mem::lesson-recall defaults minConfidence to 0, so a lesson decayed to the 0.05 floor is still recalled." + fixed_by: "98d246c" url: https://github.com/rohitg00/agentmemory/issues/1065 - number: 1066 title: "memory_heal claims \"fixable\": true for project-scope backfill but the heal step does nothing ({\"fixed\":0,\"skipped\":0})" @@ -2015,7 +2025,8 @@ issues: difficulty: med kind: bug note: "Imported observations are never added to the BM25 index and have no reindex path." - disposition: candidate + disposition: already-fixed + fixed_by: "Imports index what they write via indexRecords (src/functions/export-import.ts, src/functions/replay.ts); BM25 is rebuilt from content rows on every boot." url: https://github.com/rohitg00/agentmemory/issues/1163 mentioned_by_prs: [1187] - number: 1165 @@ -2103,7 +2114,8 @@ issues: difficulty: med kind: bug note: "Nothing schedules an index save after boot, so runtime indexing is lost unless the process exits cleanly." - disposition: candidate + disposition: already-fixed + fixed_by: "Vectors are SQLite rows written in the store transaction, and BM25 is rebuilt from content rows at boot (src/index.ts), so no index save is needed (ADR 0001)." url: https://github.com/rohitg00/agentmemory/issues/1180 - number: 1181 title: "mem:audit is 71% index_persist rows the audit policy does not cover" @@ -2242,8 +2254,9 @@ issues: difficulty: low kind: bug note: "0.9.29's default OpenAI model is a reasoning model but the provider still sends max_tokens; every compress and summarize 400s." - disposition: candidate - status: "present-here — Default model is gpt-5.6-luna (src/providers/openai.ts:12, src/config.ts:95) and the chat body always sends max_tokens (src/providers/openai.ts:82) while also sending reasoning_effort (src/providers/openai.ts:95) — the two are mutually exclusive on OpenAI's chat-completions API. Nothing maps to max_completion_tokens." + disposition: already-fixed + status: "fixed-here — b0b25f9: OpenAIProvider sends max_completion_tokens when the base URL is api.openai.com and max_tokens to compatible servers." + fixed_by: "b0b25f9" url: https://github.com/rohitg00/agentmemory/issues/1219 fix_prs: [1275] - number: 1221 @@ -2452,7 +2465,9 @@ issues: difficulty: med kind: bug note: "PreToolUse injection uses raw stdout instead of the hook envelope, against a 2s budget on a 6-29s endpoint." - disposition: candidate + disposition: already-fixed + status: "fixed-here — 895d01c: pre-tool-use emits {hookSpecificOutput:{hookEventName: PreToolUse,additionalContext}} when the payload names PreToolUse, plain text otherwise; live /enrich answers in under 0.2s, so the 2s bound stays." + fixed_by: "895d01c" url: https://github.com/rohitg00/agentmemory/issues/1278 - number: 1279 title: "[viewer] Dashboard \"Memories\" stat card caps at 500 — uses .length of a limited fetch instead of the total field" @@ -2469,7 +2484,9 @@ issues: difficulty: med kind: bug note: "deriveCrystalAndLessons slices lessons mid-sentence; 775 of 981 unusable and some invert meaning." - disposition: candidate + disposition: already-fixed + status: "fixed-here — 0a4874a: lessons are whole sentences that open with a trigger, have balanced markup and fit 220 chars; a re-import no longer bumps reinforcements or lastReinforcedAt, so a re-derived lesson can still decay out." + fixed_by: "0a4874a" url: https://github.com/rohitg00/agentmemory/issues/1292 - number: 1297 title: "Feature request: signal deletion / TTL for signals" @@ -2588,7 +2605,8 @@ issues: difficulty: med kind: bug note: "The shutdown index save is lost to a teardown race and boot only reconciles a fully empty index." - disposition: candidate + disposition: already-fixed + fixed_by: "No shutdown index save exists to lose: vectors persist per write and BM25 is rebuilt at boot; a boot and hourly fill pass repairs a partial vector index (src/index.ts)." url: https://github.com/rohitg00/agentmemory/issues/1335 - number: 1338 title: "fix(providers): createFallbackProvider drops baseURL — custom-endpoint fallbacks (DeepSeek/Kimi/etc.) hit the wrong API host" @@ -2623,7 +2641,8 @@ issues: difficulty: low kind: bug note: "Dead noop gate causes a summarize failure storm when no LLM key is set." - disposition: candidate + disposition: already-fixed + fixed_by: "isNoopProvider unwraps resilient(...) names (src/providers/noop.ts:22-34), and the summarize and graph gates use it." url: https://github.com/rohitg00/agentmemory/issues/1341 mentioned_by_prs: [1342] - number: 1360 diff --git a/docs/upstream-pr-triage.yaml b/docs/upstream-pr-triage.yaml index 57262516e8..c9d2a9b11f 100644 --- a/docs/upstream-pr-triage.yaml +++ b/docs/upstream-pr-triage.yaml @@ -287,8 +287,8 @@ prs: paths: - src/functions/evict.ts note: "Evicted observations are never removed from the BM25 or vector index, so search returns ids whose KV lookup is null and silently under-fills pages." - disposition: candidate - status: "present-here — src/functions/evict.ts deletes from KV without touching the BM25 or vector index." + disposition: already-fixed + fixed_by: "Every evict.ts delete goes through deleteIndexed, which removes the row, its vector and its BM25 entry (src/functions/search.ts:153-174)." url: https://github.com/rohitg00/agentmemory/pull/1005 - number: 1039 title: "fix: schedule periodic snapshot timer (#1006)" @@ -311,8 +311,9 @@ prs: paths: - src/functions/observe.ts note: "At MAX_OBS_PER_SESSION every further observation is discarded; evicts the lowest-value one instead of dropping the newest." - disposition: candidate - status: "present-here — At MAX_OBS_PER_SESSION mem::observe refuses every further observation outright (src/functions/observe.ts:147-154). The newest work is dropped rather than the least valuable one evicted." + disposition: already-fixed + status: "fixed-here — db9f5db: at the cap mem::observe evicts the lowest-importance observation, oldest first on a tie, from KV and both indexes, and admits the new one." + fixed_by: "db9f5db" url: https://github.com/rohitg00/agentmemory/pull/1174 - number: 1175 title: "feat(slots): reject stale slot overwrites, keep an undo copy, report headroom" @@ -338,8 +339,9 @@ prs: paths: - src/functions/audit.ts note: "recordAudit has no retention and the KV adapter rewrites the whole scope per write, so audit cost compounds on long-lived installs." - disposition: candidate - status: "present-here — recordAudit has no retention bound." + disposition: already-fixed + status: "fixed-here — cc30b8e: pruneAudit keeps the newest AGENTMEMORY_AUDIT_MAX rows (default 50000), run daily." + fixed_by: "cc30b8e" url: https://github.com/rohitg00/agentmemory/pull/1176 - number: 1210 title: "fix(providers): replace retired Anthropic default model identifiers" @@ -389,8 +391,9 @@ prs: paths: - src/providers/resilient.ts note: "No concurrency bound on the provider path and 429s count as circuit-breaker failures, so load trips the breaker." - disposition: candidate - status: "partly-present-here — 429 and 503 are already retried with Retry-After before reaching the breaker (src/providers/_fetch.ts:13,76-82). A concurrency cap exists, but only on the agent-sdk provider (src/providers/agent-sdk.ts:99) — the HTTP providers have none." + disposition: already-fixed + status: "fixed-here — 042f3a1: a 429/503 that exhausts its retries no longer opens the breaker, Retry-After is honoured uncapped within the timeout budget, and AGENTMEMORY_LLM_MAX_CONCURRENCY (default 4) bounds in-flight LLM calls." + fixed_by: "042f3a1" url: https://github.com/rohitg00/agentmemory/pull/1259 - number: 1260 title: "fix(mcp): read graph stats from the snapshot instead of enumerating the graph" @@ -418,8 +421,9 @@ prs: - src/index.ts - test/evict.test.ts note: "mem::evict is fully implemented but nothing schedules it — Eviction never runs." - disposition: candidate - status: "present-here — mem::evict is registered (src/functions/evict.ts:109) but src/index.ts schedules only auto-forget and consolidate-pipeline." + disposition: already-fixed + status: "fixed-here — 3b2545e: src/index.ts runs mem::evict every 24h unless EVICTION_ENABLED=false." + fixed_by: "3b2545e" url: https://github.com/rohitg00/agentmemory/pull/1287 - number: 1307 title: "fix(sessions): close idle sessions no client ever ended" From 6575de0a573adbc2ca234fa499d25e24141b0609 Mon Sep 17 00:00:00 2001 From: neal Date: Fri, 25 Sep 2026 19:10:13 -0400 Subject: [PATCH 13/17] fix(consolidate): keep scheduled reflect and semantic merge inside one project An unscoped consolidation run (the 2h timer, session stop) merged the 20 newest summaries from every project into one semantic pass and ran reflect over all facts, crystals and concepts, so insights mixed repos and carried no project. It now consolidates each project with summaries newer than the last unscoped run on its own, and mem::reflect with a project only reads that project's facts, crystals, lessons and graph concepts. results.semantic and results.reflect are keyed by project. Refs rohitg00/agentmemory#1344 Generated-By: Claude Opus 5.5 (1M context) --- src/functions/consolidation-pipeline.ts | 191 ++++++++++++++---------- src/functions/reflect.ts | 35 ++++- test/consolidation-pipeline.test.ts | 59 +++++++- test/reflect.test.ts | 31 ++++ 4 files changed, 225 insertions(+), 91 deletions(-) diff --git a/src/functions/consolidation-pipeline.ts b/src/functions/consolidation-pipeline.ts index 53259740c3..ede4c08254 100644 --- a/src/functions/consolidation-pipeline.ts +++ b/src/functions/consolidation-pipeline.ts @@ -70,100 +70,125 @@ export function registerConsolidationPipelineFunction( }, ); - async function run(data?: { tier?: string; project?: string }) { - const tier = data?.tier || "all"; - const decayDays = getConsolidationDecayDays(); - const results: Record = {}; + // Summaries and insights from one project must never feed another's + // consolidation (#1344): an unscoped run consolidates each project that has + // new summaries since the last unscoped run, one project at a time. + let lastUnscopedRunAt = new Date(Date.now() - 24 * 60 * 60 * 1000).toISOString(); - if (tier === "all" || tier === "semantic") { - const summaries = await kv.list(KV.summaries); - const existingSemantic = await kv.list(KV.semantic); + async function consolidateSemantic(summaries: SessionSummary[]) { + if (summaries.length < 5) { + return { skipped: true, reason: "fewer than 5 summaries" }; + } + const existingSemantic = await kv.list(KV.semantic); + const recentSummaries = summaries + .sort( + (a, b) => + new Date(b.createdAt).getTime() - + new Date(a.createdAt).getTime(), + ) + .slice(0, 20); - if (summaries.length >= 5) { - const recentSummaries = summaries - .sort( - (a, b) => - new Date(b.createdAt).getTime() - - new Date(a.createdAt).getTime(), - ) - .slice(0, 20); + const prompt = buildSemanticMergePrompt( + recentSummaries.map((s) => ({ + title: s.title, + narrative: s.narrative, + concepts: s.concepts, + })), + ); - const prompt = buildSemanticMergePrompt( - recentSummaries.map((s) => ({ - title: s.title, - narrative: s.narrative, - concepts: s.concepts, - })), - ); + try { + const response = await provider.summarize( + SEMANTIC_MERGE_SYSTEM, + prompt, + ); - try { - const response = await provider.summarize( - SEMANTIC_MERGE_SYSTEM, - prompt, - ); + const factRegex = /([^<]+)<\/fact>/g; + let match; + let newFacts = 0; + const now = new Date().toISOString(); - const factRegex = /([^<]+)<\/fact>/g; - let match; - let newFacts = 0; - const now = new Date().toISOString(); - - while ((match = factRegex.exec(response)) !== null) { - const parsedConf = parseFloat(match[1]); - const confidence = Number.isNaN(parsedConf) ? 0.5 : parsedConf; - const fact = match[2].trim(); + while ((match = factRegex.exec(response)) !== null) { + const parsedConf = parseFloat(match[1]); + const confidence = Number.isNaN(parsedConf) ? 0.5 : parsedConf; + const fact = match[2].trim(); - const existing = existingSemantic.find( - (s) => s.fact.toLowerCase() === fact.toLowerCase(), - ); - if (existing) { - existing.accessCount++; - existing.lastAccessedAt = now; - existing.updatedAt = now; - existing.confidence = Math.max(existing.confidence, confidence); - await kv.set(KV.semantic, existing.id, existing); - } else { - const sem: SemanticMemory = { - id: generateId("sem"), - fact, - confidence, - sourceSessionIds: recentSummaries.map((s) => s.sessionId), - sourceMemoryIds: [], - accessCount: 1, - lastAccessedAt: now, - strength: confidence, - createdAt: now, - updatedAt: now, - }; - await kv.set(KV.semantic, sem.id, sem); - newFacts++; - } - } - results.semantic = { newFacts, totalSummaries: summaries.length }; - } catch (err) { - const msg = err instanceof Error ? err.message : String(err); - logger.error("Semantic consolidation failed", { error: msg }); - results.semantic = { error: msg }; + const existing = existingSemantic.find( + (s) => s.fact.toLowerCase() === fact.toLowerCase(), + ); + if (existing) { + existing.accessCount++; + existing.lastAccessedAt = now; + existing.updatedAt = now; + existing.confidence = Math.max(existing.confidence, confidence); + await kv.set(KV.semantic, existing.id, existing); + } else { + const sem: SemanticMemory = { + id: generateId("sem"), + fact, + confidence, + sourceSessionIds: recentSummaries.map((s) => s.sessionId), + sourceMemoryIds: [], + accessCount: 1, + lastAccessedAt: now, + strength: confidence, + createdAt: now, + updatedAt: now, + }; + await kv.set(KV.semantic, sem.id, sem); + newFacts++; } - } else { - results.semantic = { - skipped: true, - reason: "fewer than 5 summaries", - }; } + return { newFacts, totalSummaries: summaries.length }; + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + logger.error("Semantic consolidation failed", { error: msg }); + return { error: msg }; } + } - if (tier === "all" || tier === "reflect") { - try { - const reflectResult = await sdk.trigger({ function_id: "mem::reflect", payload: { - maxClusters: 10, - project: data?.project, - } }); - results.reflect = reflectResult; - } catch (err) { - const msg = err instanceof Error ? err.message : String(err); - logger.warn("Reflect tier failed", { error: msg }); - results.reflect = { error: msg }; + async function reflect(project: string) { + try { + return await sdk.trigger({ function_id: "mem::reflect", payload: { + maxClusters: 10, + project, + } }); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + logger.warn("Reflect tier failed", { error: msg, project }); + return { error: msg }; + } + } + + async function run(data?: { tier?: string; project?: string }) { + const tier = data?.tier || "all"; + const decayDays = getConsolidationDecayDays(); + const results: Record = {}; + + const runsSemantic = tier === "all" || tier === "semantic"; + const runsReflect = tier === "all" || tier === "reflect"; + if (runsSemantic || runsReflect) { + const startedAt = new Date().toISOString(); + const summaries = await kv.list(KV.summaries); + const projects = data?.project + ? [data.project] + : [...new Set( + summaries + .filter((s) => s.project && s.createdAt > lastUnscopedRunAt) + .map((s) => s.project), + )]; + const semanticByProject: Record = {}; + const reflectByProject: Record = {}; + for (const project of projects) { + if (runsSemantic) { + semanticByProject[project] = await consolidateSemantic( + summaries.filter((s) => s.project === project), + ); + } + if (runsReflect) reflectByProject[project] = await reflect(project); } + if (runsSemantic) results.semantic = semanticByProject; + if (runsReflect) results.reflect = reflectByProject; + if (!data?.project) lastUnscopedRunAt = startedAt; } if (tier === "all" || tier === "procedural") { diff --git a/src/functions/reflect.ts b/src/functions/reflect.ts index a77b659d6c..3edf9f87fd 100644 --- a/src/functions/reflect.ts +++ b/src/functions/reflect.ts @@ -9,6 +9,7 @@ import type { Lesson, Crystal, MemoryProvider, + Session, } from "../types.js"; import { recordAudit } from "./audit.js"; import { REFLECT_SYSTEM, buildReflectPrompt } from "../prompts/reflect.js"; @@ -184,20 +185,42 @@ export function registerReflectFunctions( ]); let activeLessons = lessons.filter((l) => !l.deleted); + let scopedSemantic = semanticMemories; + let scopedCrystals = crystals; + let scopedNodes = graphNodes; + let scopedEdges = graphEdges; + // One project's clusters must never borrow another's facts, crystals or + // concepts: an insight is stamped with the project it was built for (#1344). if (data?.project) { - activeLessons = activeLessons.filter((l) => l.project === data.project); + const project = data.project; + const sessions = await kv.list(KV.sessions).catch(() => []); + const projectSessionIds = new Set( + sessions.filter((s) => s.project === project).map((s) => s.id), + ); + activeLessons = activeLessons.filter((l) => l.project === project); + scopedSemantic = semanticMemories.filter((m) => + m.sourceSessionIds.some((id) => projectSessionIds.has(id)), + ); + scopedCrystals = crystals.filter((c) => c.project === project); + scopedNodes = graphNodes.filter( + (n) => n.sessionId !== undefined && projectSessionIds.has(n.sessionId), + ); + const nodeIds = new Set(scopedNodes.map((n) => n.id)); + scopedEdges = graphEdges.filter( + (e) => nodeIds.has(e.sourceNodeId) && nodeIds.has(e.targetNodeId), + ); } let conceptClusters = buildGraphClusters( - graphNodes, - graphEdges, + scopedNodes, + scopedEdges, maxClusters, ); const usedFallback = conceptClusters.length === 0; if (usedFallback) { conceptClusters = buildJaccardClusters( - semanticMemories, + scopedSemantic, activeLessons, maxClusters, ); @@ -213,7 +236,7 @@ export function registerReflectFunctions( const conceptSet = new Set(conceptNames.map((c) => c.toLowerCase())); - const clusterFacts = semanticMemories.filter((s) => { + const clusterFacts = scopedSemantic.filter((s) => { const factTerms = s.fact.toLowerCase().split(/\s+/); return factTerms.some((t) => conceptSet.has(t)); }); @@ -225,7 +248,7 @@ export function registerReflectFunctions( ), ); - const clusterCrystals = crystals.filter((c) => + const clusterCrystals = scopedCrystals.filter((c) => (c.lessons || []).some((l) => conceptNames.some((cn) => l.toLowerCase().includes(cn.toLowerCase()), diff --git a/test/consolidation-pipeline.test.ts b/test/consolidation-pipeline.test.ts index 4852191572..c3b4033c95 100644 --- a/test/consolidation-pipeline.test.ts +++ b/test/consolidation-pipeline.test.ts @@ -119,7 +119,7 @@ describe("Consolidation Pipeline", () => { })) as { success: boolean; results: Record }; expect(result.success).toBe(true); - const semantic = result.results.semantic as { skipped: boolean; reason: string }; + const semantic = (result.results.semantic as Record)["test-project"]; expect(semantic.skipped).toBe(true); expect(semantic.reason).toContain("fewer than 5"); expect(provider.summarize).not.toHaveBeenCalled(); @@ -168,7 +168,7 @@ describe("Consolidation Pipeline", () => { })) as { success: boolean; results: Record }; expect(result.success).toBe(true); - const semantic = result.results.semantic as { newFacts: number }; + const semantic = (result.results.semantic as Record)["test-project"]; expect(semantic.newFacts).toBe(1); const stored = await kv.list("mem:semantic"); @@ -337,3 +337,58 @@ describe("Consolidation Pipeline: decay writes and single flight", () => { expect(third.success).toBe(true); }); }); + +describe("Consolidation Pipeline: per-project scope (#1344)", () => { + function summaryFor(project: string, i: number): SessionSummary { + return { ...makeSummary(i), sessionId: `${project}_${i}`, project, narrative: `${project} work ${i}` }; + } + + it("an unscoped run consolidates each project with new summaries on its own", async () => { + const sdk = mockSdk(); + const kv = mockKV(); + const provider = { name: "test", compress: vi.fn(), summarize: vi.fn().mockResolvedValue("") }; + const reflected: unknown[] = []; + sdk.registerFunction("mem::reflect", async (payload: unknown) => { + reflected.push(payload); + return { success: true }; + }); + registerConsolidationPipelineFunction(sdk as never, kv as never, provider as never); + for (let i = 0; i < 5; i++) { + await kv.set("mem:summaries", `alpha_${i}`, summaryFor("alpha", i)); + await kv.set("mem:summaries", `beta_${i}`, summaryFor("beta", i)); + } + + const result = (await sdk.trigger("mem::consolidate-pipeline", {})) as { + results: { semantic: Record }; + }; + + expect(Object.keys(result.results.semantic).sort()).toEqual(["alpha", "beta"]); + expect(reflected).toEqual([ + { maxClusters: 10, project: "alpha" }, + { maxClusters: 10, project: "beta" }, + ]); + for (const [, prompt] of provider.summarize.mock.calls) { + const mentionsAlpha = String(prompt).includes("alpha work"); + const mentionsBeta = String(prompt).includes("beta work"); + expect(mentionsAlpha !== mentionsBeta).toBe(true); + } + }); + + it("a later unscoped run skips projects with no new summaries since the last one", async () => { + const sdk = mockSdk(); + const kv = mockKV(); + const provider = { name: "test", compress: vi.fn(), summarize: vi.fn().mockResolvedValue("") }; + sdk.registerFunction("mem::reflect", async () => ({ success: true })); + registerConsolidationPipelineFunction(sdk as never, kv as never, provider as never); + for (let i = 0; i < 5; i++) await kv.set("mem:summaries", `alpha_${i}`, summaryFor("alpha", i)); + + await sdk.trigger("mem::consolidate-pipeline", {}); + const second = (await sdk.trigger("mem::consolidate-pipeline", {})) as { + results: { semantic: Record; reflect: Record }; + }; + + expect(second.results.semantic).toEqual({}); + expect(second.results.reflect).toEqual({}); + expect(provider.summarize).toHaveBeenCalledTimes(1); + }); +}); diff --git a/test/reflect.test.ts b/test/reflect.test.ts index 3219afc7c5..5246c18aa4 100644 --- a/test/reflect.test.ts +++ b/test/reflect.test.ts @@ -360,3 +360,34 @@ describe("Reflect", () => { }); }); }); + +describe("mem::reflect project scope (#1344)", () => { + it("builds a project's insights only from that project's facts, crystals and concepts", async () => { + const sdk = mockSdk(); + const kv = mockKV(); + const provider = { name: "test", compress: vi.fn(), summarize: vi.fn().mockResolvedValue(XML_RESPONSE) }; + registerReflectFunctions(sdk as never, kv as never, provider as never); + await kv.set("mem:sessions", "ses_a", { id: "ses_a", project: "alpha" }); + await kv.set("mem:sessions", "ses_b", { id: "ses_b", project: "beta" }); + for (const [name, sessionId] of [["security", "ses_a"], ["validation", "ses_a"], ["beta", "ses_b"]]) { + await kv.set("mem:graph:nodes", `node_${name}`, { ...makeConceptNode(name), sessionId }); + } + await kv.set("mem:graph:edges", "edge_1", makeEdge("security", "validation")); + await kv.set("mem:graph:edges", "edge_2", makeEdge("security", "beta")); + const alphaFacts = ["Always validate security inputs", "Testing improves security coverage", "Validation prevents injection"]; + for (const [i, fact] of alphaFacts.entries()) { + await kv.set("mem:semantic", `sem_a${i}`, { ...makeSemantic(fact, `sem_a${i}`), sourceSessionIds: ["ses_a"] }); + } + await kv.set("mem:semantic", "sem_b", { ...makeSemantic("Beta security keys live in vault", "sem_b"), sourceSessionIds: ["ses_b"] }); + await kv.set("mem:crystals", "crys_b", { ...makeCrystal("beta work", ["Beta security rotates weekly"]), project: "beta" }); + + await sdk.trigger("mem::reflect", { project: "alpha" }); + + expect(provider.summarize).toHaveBeenCalled(); + for (const [, prompt] of provider.summarize.mock.calls) { + expect(String(prompt)).not.toMatch(/beta/i); + } + const insights = await kv.list("mem:insights"); + expect(insights.every((i) => i.project === "alpha")).toBe(true); + }); +}); From 1dc4d43a00f4b011fe5921f920343f675867ca5a Mon Sep 17 00:00:00 2001 From: neal Date: Fri, 25 Sep 2026 19:11:22 -0400 Subject: [PATCH 14/17] fix(hooks): name a git worktree's Session after its main checkout resolveProject took the basename of --show-toplevel, so every linked worktree (e.g. orca's /) became its own project and its Memory never reached the main checkout or sibling worktrees. It now takes the parent of --git-common-dir when that is a .git folder, falling back to the toplevel for submodules and bare-repo worktrees. Existing Sessions keep the project they were filed under. Refs rohitg00/agentmemory#728 Generated-By: Claude Opus 5.5 (1M context) --- plugin/scripts/notification.mjs | 7 ++++--- plugin/scripts/post-tool-failure.mjs | 7 ++++--- plugin/scripts/post-tool-use.mjs | 7 ++++--- plugin/scripts/pre-compact.mjs | 7 ++++--- plugin/scripts/prompt-submit.mjs | 7 ++++--- plugin/scripts/session-end.mjs | 7 ++++--- plugin/scripts/session-start.mjs | 7 ++++--- plugin/scripts/subagent-start.mjs | 7 ++++--- plugin/scripts/subagent-stop.mjs | 7 ++++--- plugin/scripts/task-completed.mjs | 7 ++++--- src/hooks/_project.ts | 13 +++++++++---- test/hook-project.test.ts | 10 ++++++++++ 12 files changed, 59 insertions(+), 34 deletions(-) diff --git a/plugin/scripts/notification.mjs b/plugin/scripts/notification.mjs index 0b7e188fca..3c66a6b003 100755 --- a/plugin/scripts/notification.mjs +++ b/plugin/scripts/notification.mjs @@ -1,7 +1,7 @@ #!/usr/bin/env node import { readFileSync } from "node:fs"; import { homedir } from "node:os"; -import { basename, join } from "node:path"; +import { basename, dirname, join, resolve } from "node:path"; import { execSync } from "node:child_process"; //#region src/hooks/_env.ts function parseEnvFile(content) { @@ -41,7 +41,7 @@ function resolveProject(cwd) { if (explicit && explicit.trim()) return explicit.trim(); const dir = cwd && cwd.trim() ? cwd : process.cwd(); try { - const top = execSync("git rev-parse --show-toplevel", { + const [commonDir, top] = execSync("git rev-parse --git-common-dir --show-toplevel", { cwd: dir, stdio: [ "ignore", @@ -49,7 +49,8 @@ function resolveProject(cwd) { "ignore" ], timeout: 500 - }).toString().trim(); + }).toString().trim().split("\n"); + if (commonDir && basename(commonDir) === ".git") return basename(dirname(resolve(dir, commonDir))); if (top) return basename(top); } catch {} return basename(dir); diff --git a/plugin/scripts/post-tool-failure.mjs b/plugin/scripts/post-tool-failure.mjs index 354a620dc4..37e5bedc9e 100755 --- a/plugin/scripts/post-tool-failure.mjs +++ b/plugin/scripts/post-tool-failure.mjs @@ -1,7 +1,7 @@ #!/usr/bin/env node import { readFileSync } from "node:fs"; import { homedir } from "node:os"; -import { basename, join } from "node:path"; +import { basename, dirname, join, resolve } from "node:path"; import { execSync } from "node:child_process"; //#region src/hooks/_env.ts function parseEnvFile(content) { @@ -41,7 +41,7 @@ function resolveProject(cwd) { if (explicit && explicit.trim()) return explicit.trim(); const dir = cwd && cwd.trim() ? cwd : process.cwd(); try { - const top = execSync("git rev-parse --show-toplevel", { + const [commonDir, top] = execSync("git rev-parse --git-common-dir --show-toplevel", { cwd: dir, stdio: [ "ignore", @@ -49,7 +49,8 @@ function resolveProject(cwd) { "ignore" ], timeout: 500 - }).toString().trim(); + }).toString().trim().split("\n"); + if (commonDir && basename(commonDir) === ".git") return basename(dirname(resolve(dir, commonDir))); if (top) return basename(top); } catch {} return basename(dir); diff --git a/plugin/scripts/post-tool-use.mjs b/plugin/scripts/post-tool-use.mjs index 2d8341e67a..c8c4c17f86 100755 --- a/plugin/scripts/post-tool-use.mjs +++ b/plugin/scripts/post-tool-use.mjs @@ -1,7 +1,7 @@ #!/usr/bin/env node import { readFileSync } from "node:fs"; import { homedir } from "node:os"; -import { basename, join } from "node:path"; +import { basename, dirname, join, resolve } from "node:path"; import { execSync } from "node:child_process"; //#region src/hooks/_env.ts function parseEnvFile(content) { @@ -41,7 +41,7 @@ function resolveProject(cwd) { if (explicit && explicit.trim()) return explicit.trim(); const dir = cwd && cwd.trim() ? cwd : process.cwd(); try { - const top = execSync("git rev-parse --show-toplevel", { + const [commonDir, top] = execSync("git rev-parse --git-common-dir --show-toplevel", { cwd: dir, stdio: [ "ignore", @@ -49,7 +49,8 @@ function resolveProject(cwd) { "ignore" ], timeout: 500 - }).toString().trim(); + }).toString().trim().split("\n"); + if (commonDir && basename(commonDir) === ".git") return basename(dirname(resolve(dir, commonDir))); if (top) return basename(top); } catch {} return basename(dir); diff --git a/plugin/scripts/pre-compact.mjs b/plugin/scripts/pre-compact.mjs index c4bd90a4ff..3d5af46c46 100755 --- a/plugin/scripts/pre-compact.mjs +++ b/plugin/scripts/pre-compact.mjs @@ -1,7 +1,7 @@ #!/usr/bin/env node import { readFileSync } from "node:fs"; import { homedir } from "node:os"; -import { basename, join } from "node:path"; +import { basename, dirname, join, resolve } from "node:path"; import { execSync } from "node:child_process"; //#region src/hooks/_env.ts function parseEnvFile(content) { @@ -41,7 +41,7 @@ function resolveProject(cwd) { if (explicit && explicit.trim()) return explicit.trim(); const dir = cwd && cwd.trim() ? cwd : process.cwd(); try { - const top = execSync("git rev-parse --show-toplevel", { + const [commonDir, top] = execSync("git rev-parse --git-common-dir --show-toplevel", { cwd: dir, stdio: [ "ignore", @@ -49,7 +49,8 @@ function resolveProject(cwd) { "ignore" ], timeout: 500 - }).toString().trim(); + }).toString().trim().split("\n"); + if (commonDir && basename(commonDir) === ".git") return basename(dirname(resolve(dir, commonDir))); if (top) return basename(top); } catch {} return basename(dir); diff --git a/plugin/scripts/prompt-submit.mjs b/plugin/scripts/prompt-submit.mjs index 0f9c2eaefe..517d18227a 100755 --- a/plugin/scripts/prompt-submit.mjs +++ b/plugin/scripts/prompt-submit.mjs @@ -1,7 +1,7 @@ #!/usr/bin/env node import { readFileSync } from "node:fs"; import { homedir } from "node:os"; -import { basename, join } from "node:path"; +import { basename, dirname, join, resolve } from "node:path"; import { execSync } from "node:child_process"; //#region src/hooks/_env.ts function parseEnvFile(content) { @@ -41,7 +41,7 @@ function resolveProject(cwd) { if (explicit && explicit.trim()) return explicit.trim(); const dir = cwd && cwd.trim() ? cwd : process.cwd(); try { - const top = execSync("git rev-parse --show-toplevel", { + const [commonDir, top] = execSync("git rev-parse --git-common-dir --show-toplevel", { cwd: dir, stdio: [ "ignore", @@ -49,7 +49,8 @@ function resolveProject(cwd) { "ignore" ], timeout: 500 - }).toString().trim(); + }).toString().trim().split("\n"); + if (commonDir && basename(commonDir) === ".git") return basename(dirname(resolve(dir, commonDir))); if (top) return basename(top); } catch {} return basename(dir); diff --git a/plugin/scripts/session-end.mjs b/plugin/scripts/session-end.mjs index a91c151cc9..72cfef97c4 100755 --- a/plugin/scripts/session-end.mjs +++ b/plugin/scripts/session-end.mjs @@ -1,7 +1,7 @@ #!/usr/bin/env node import { readFileSync } from "node:fs"; import { homedir } from "node:os"; -import { basename, join } from "node:path"; +import { basename, dirname, join, resolve } from "node:path"; import { execSync } from "node:child_process"; //#region src/hooks/_env.ts function parseEnvFile(content) { @@ -41,7 +41,7 @@ function resolveProject(cwd) { if (explicit && explicit.trim()) return explicit.trim(); const dir = cwd && cwd.trim() ? cwd : process.cwd(); try { - const top = execSync("git rev-parse --show-toplevel", { + const [commonDir, top] = execSync("git rev-parse --git-common-dir --show-toplevel", { cwd: dir, stdio: [ "ignore", @@ -49,7 +49,8 @@ function resolveProject(cwd) { "ignore" ], timeout: 500 - }).toString().trim(); + }).toString().trim().split("\n"); + if (commonDir && basename(commonDir) === ".git") return basename(dirname(resolve(dir, commonDir))); if (top) return basename(top); } catch {} return basename(dir); diff --git a/plugin/scripts/session-start.mjs b/plugin/scripts/session-start.mjs index 1221981615..3c07317d02 100755 --- a/plugin/scripts/session-start.mjs +++ b/plugin/scripts/session-start.mjs @@ -1,7 +1,7 @@ #!/usr/bin/env node import { readFileSync } from "node:fs"; import { homedir } from "node:os"; -import { basename, join } from "node:path"; +import { basename, dirname, join, resolve } from "node:path"; import { execSync } from "node:child_process"; //#region src/hooks/_env.ts function parseEnvFile(content) { @@ -41,7 +41,7 @@ function resolveProject(cwd) { if (explicit && explicit.trim()) return explicit.trim(); const dir = cwd && cwd.trim() ? cwd : process.cwd(); try { - const top = execSync("git rev-parse --show-toplevel", { + const [commonDir, top] = execSync("git rev-parse --git-common-dir --show-toplevel", { cwd: dir, stdio: [ "ignore", @@ -49,7 +49,8 @@ function resolveProject(cwd) { "ignore" ], timeout: 500 - }).toString().trim(); + }).toString().trim().split("\n"); + if (commonDir && basename(commonDir) === ".git") return basename(dirname(resolve(dir, commonDir))); if (top) return basename(top); } catch {} return basename(dir); diff --git a/plugin/scripts/subagent-start.mjs b/plugin/scripts/subagent-start.mjs index df43650c82..99db966049 100755 --- a/plugin/scripts/subagent-start.mjs +++ b/plugin/scripts/subagent-start.mjs @@ -1,7 +1,7 @@ #!/usr/bin/env node import { readFileSync } from "node:fs"; import { homedir } from "node:os"; -import { basename, join } from "node:path"; +import { basename, dirname, join, resolve } from "node:path"; import { execSync } from "node:child_process"; //#region src/hooks/_env.ts function parseEnvFile(content) { @@ -41,7 +41,7 @@ function resolveProject(cwd) { if (explicit && explicit.trim()) return explicit.trim(); const dir = cwd && cwd.trim() ? cwd : process.cwd(); try { - const top = execSync("git rev-parse --show-toplevel", { + const [commonDir, top] = execSync("git rev-parse --git-common-dir --show-toplevel", { cwd: dir, stdio: [ "ignore", @@ -49,7 +49,8 @@ function resolveProject(cwd) { "ignore" ], timeout: 500 - }).toString().trim(); + }).toString().trim().split("\n"); + if (commonDir && basename(commonDir) === ".git") return basename(dirname(resolve(dir, commonDir))); if (top) return basename(top); } catch {} return basename(dir); diff --git a/plugin/scripts/subagent-stop.mjs b/plugin/scripts/subagent-stop.mjs index 88ab8609ba..4afb64136b 100755 --- a/plugin/scripts/subagent-stop.mjs +++ b/plugin/scripts/subagent-stop.mjs @@ -1,7 +1,7 @@ #!/usr/bin/env node import { readFileSync } from "node:fs"; import { homedir } from "node:os"; -import { basename, join } from "node:path"; +import { basename, dirname, join, resolve } from "node:path"; import { execSync } from "node:child_process"; //#region src/hooks/_env.ts function parseEnvFile(content) { @@ -41,7 +41,7 @@ function resolveProject(cwd) { if (explicit && explicit.trim()) return explicit.trim(); const dir = cwd && cwd.trim() ? cwd : process.cwd(); try { - const top = execSync("git rev-parse --show-toplevel", { + const [commonDir, top] = execSync("git rev-parse --git-common-dir --show-toplevel", { cwd: dir, stdio: [ "ignore", @@ -49,7 +49,8 @@ function resolveProject(cwd) { "ignore" ], timeout: 500 - }).toString().trim(); + }).toString().trim().split("\n"); + if (commonDir && basename(commonDir) === ".git") return basename(dirname(resolve(dir, commonDir))); if (top) return basename(top); } catch {} return basename(dir); diff --git a/plugin/scripts/task-completed.mjs b/plugin/scripts/task-completed.mjs index df78a6f373..f876edce4a 100755 --- a/plugin/scripts/task-completed.mjs +++ b/plugin/scripts/task-completed.mjs @@ -1,7 +1,7 @@ #!/usr/bin/env node import { readFileSync } from "node:fs"; import { homedir } from "node:os"; -import { basename, join } from "node:path"; +import { basename, dirname, join, resolve } from "node:path"; import { execSync } from "node:child_process"; //#region src/hooks/_env.ts function parseEnvFile(content) { @@ -41,7 +41,7 @@ function resolveProject(cwd) { if (explicit && explicit.trim()) return explicit.trim(); const dir = cwd && cwd.trim() ? cwd : process.cwd(); try { - const top = execSync("git rev-parse --show-toplevel", { + const [commonDir, top] = execSync("git rev-parse --git-common-dir --show-toplevel", { cwd: dir, stdio: [ "ignore", @@ -49,7 +49,8 @@ function resolveProject(cwd) { "ignore" ], timeout: 500 - }).toString().trim(); + }).toString().trim().split("\n"); + if (commonDir && basename(commonDir) === ".git") return basename(dirname(resolve(dir, commonDir))); if (top) return basename(top); } catch {} return basename(dir); diff --git a/src/hooks/_project.ts b/src/hooks/_project.ts index 9f0320c5ce..06ce7a6d16 100644 --- a/src/hooks/_project.ts +++ b/src/hooks/_project.ts @@ -1,19 +1,24 @@ import { execSync } from "node:child_process"; -import { basename } from "node:path"; +import { basename, dirname, resolve } from "node:path"; -// Resolution order: AGENTMEMORY_PROJECT_NAME env → git toplevel basename → cwd basename. +// Resolution order: AGENTMEMORY_PROJECT_NAME env → main checkout basename (so every +// git worktree of a repo shares one project) → git toplevel basename → cwd basename. export function resolveProject(cwd?: string): string { const explicit = process.env["AGENTMEMORY_PROJECT_NAME"]; if (explicit && explicit.trim()) return explicit.trim(); const dir = cwd && cwd.trim() ? cwd : process.cwd(); try { - const top = execSync("git rev-parse --show-toplevel", { + const [commonDir, top] = execSync("git rev-parse --git-common-dir --show-toplevel", { cwd: dir, stdio: ["ignore", "pipe", "ignore"], timeout: 500, }) .toString() - .trim(); + .trim() + .split("\n"); + if (commonDir && basename(commonDir) === ".git") { + return basename(dirname(resolve(dir, commonDir))); + } if (top) return basename(top); } catch {} return basename(dir); diff --git a/test/hook-project.test.ts b/test/hook-project.test.ts index 37d0e424de..9072ba0847 100644 --- a/test/hook-project.test.ts +++ b/test/hook-project.test.ts @@ -66,6 +66,16 @@ describe("resolveProject — hook project basename resolver", () => { expect(resolveProject(nestedDir)).toBe(REPO_NAME); }); + it("names a linked worktree after its main checkout (#728)", () => { + const git = (...args: string[]) => execFileSync("git", args, { cwd: repoDir, stdio: "ignore" }); + git("-c", "user.name=t", "-c", "user.email=t@t", "commit", "--allow-empty", "--quiet", "-m", "init"); + const worktree = join(tmpRoot, "goosefish"); + git("worktree", "add", "--quiet", worktree); + mkdirSync(join(worktree, "src"), { recursive: true }); + expect(resolveProject(worktree)).toBe(REPO_NAME); + expect(resolveProject(join(worktree, "src"))).toBe(REPO_NAME); + }); + it("falls back to basename(cwd) when not in a git repo", () => { // mkdtemp lands under os.tmpdir(), which is not always outside a repository — // TMPDIR pointed at a working directory makes git walk up and find one, and the From 73e2aef0ef2748d4e83072c728f52e854b8b1106 Mon Sep 17 00:00:00 2001 From: neal Date: Fri, 25 Sep 2026 19:11:22 -0400 Subject: [PATCH 15/17] docs(triage): record verdicts for #1344 and #728 Generated-By: Claude Opus 5.5 (1M context) --- docs/upstream-issue-triage.yaml | 4 +++- docs/upstream-pr-triage.yaml | 5 +++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/upstream-issue-triage.yaml b/docs/upstream-issue-triage.yaml index 2e8a6a1ac5..8fa5a9083b 100644 --- a/docs/upstream-issue-triage.yaml +++ b/docs/upstream-issue-triage.yaml @@ -699,7 +699,9 @@ issues: difficulty: low kind: bug note: "Hook resolveProject is not worktree-aware, so each worktree is its own memory silo." - disposition: candidate + disposition: already-fixed + status: "fixed-here — 1dc4d43: resolveProject names a linked worktree after its main checkout (parent of --git-common-dir when it is a .git folder), falling back to the toplevel basename. Sessions already filed under a worktree name are not migrated." + fixed_by: "1dc4d43" url: https://github.com/rohitg00/agentmemory/issues/728 - number: 730 title: "feat: add Nix flake support for one-command installation" diff --git a/docs/upstream-pr-triage.yaml b/docs/upstream-pr-triage.yaml index c9d2a9b11f..6e2326fe71 100644 --- a/docs/upstream-pr-triage.yaml +++ b/docs/upstream-pr-triage.yaml @@ -532,8 +532,9 @@ prs: - test/consolidation-pipeline.test.ts - test/reflect.test.ts note: "Reflection and consolidation are not project-scoped, so concept clusters and insights cross-contaminate between repos." - disposition: candidate - status: "partly-present-here — mem::reflect accepts and filters on project (src/functions/reflect.ts:187,354,385), so the capability is present; it is opt-in, and the scheduled run passes no project." + disposition: already-fixed + status: "fixed-here — 6575de0: an unscoped consolidation run merges and reflects each project with new summaries separately, and mem::reflect with a project reads only that project's facts, crystals, lessons and graph concepts. Existing unscoped insights are left as they are." + fixed_by: "6575de0" url: https://github.com/rohitg00/agentmemory/pull/1344 - number: 1350 title: "fix(providers): clear the circuit breaker failure count on success" From 585f7f0f79d8766f158778255f547e591dad57d9 Mon Sep 17 00:00:00 2001 From: neal Date: Fri, 25 Sep 2026 19:20:20 -0400 Subject: [PATCH 16/17] fix: address PR #57 review findings - Eviction and audit Eviction first run 5 min after boot, then daily, so a daemon restarted more often than daily still sweeps. - Consolidation retries a project whose merge or reflect failed, and a fact another project already holds gains this project's Sessions, so scoped reflect still sees it. - Session cap eviction is audited and logs a failed delete; the audit retention sweep is named Eviction, per CONTEXT.md. - A 429 for an exhausted OpenAI quota opens the breaker instead of retrying. - Summarize skips tokenize only when the character count fits one chunk, so dense (CJK) text is still measured. - Lessons accept a curly apostrophe in "Don't"; reflect no longer swallows a failed Session list. - Restore CRLF on files the branch had converted to LF. - Triage: #969 is already-fixed, not wont-fix; #1278 notes part 3 is open. Generated-By: Claude Opus 5.5 (1M context) --- .env.example | 2 +- docs/upstream-issue-triage.yaml | 6 +- src/functions/audit.ts | 380 +++---- src/functions/consolidation-pipeline.ts | 20 +- src/functions/lessons.ts | 766 ++++++------- src/functions/observe.ts | 833 +++++++------- src/functions/reflect.ts | 2 +- src/functions/replay.ts | 2 +- src/functions/summarize.ts | 9 +- src/hooks/pre-tool-use.ts | 2 +- src/index.ts | 1362 ++++++++++++----------- src/providers/_fetch.ts | 5 +- src/providers/resilient.ts | 5 +- src/state/index-corpus.ts | 434 ++++---- test/audit.test.ts | 6 +- test/consolidation-pipeline.test.ts | 49 + test/index-echo-exclusion.test.ts | 308 ++--- test/observe-session-cap.test.ts | 8 +- test/replay-lessons.test.ts | 6 + test/resilient-provider.test.ts | 14 + test/summarize.test.ts | 15 + 21 files changed, 2178 insertions(+), 2056 deletions(-) diff --git a/.env.example b/.env.example index b561f1a65e..80fc1de8c4 100644 --- a/.env.example +++ b/.env.example @@ -238,5 +238,5 @@ # AGENTMEMORY_SQLITE_PATH=/var/lib/agentmemory/agentmemory.sqlite # The store (WAL mode); its directory must exist # AGENTMEMORY_AUDIT_STORE=off # Emit audit entries as `[agentmemory] audit ` log lines instead of KV rows; memory_audit then answers an empty list # AGENTMEMORY_AUDIT_LOG=all # With AUDIT_STORE=off: all | deletions (forget, delete, lesson_delete, slot_delete, core_remove, sketch_discard only) | off -# AGENTMEMORY_AUDIT_MAX=50000 # Newest audit rows the daily prune keeps in KV +# AGENTMEMORY_AUDIT_MAX=50000 # Newest audit rows the daily Eviction keeps in KV # AGENTMEMORY_LOG_LEVEL=info # Lowest logger level written: info | warn | error | off. Audit lines bypass it. diff --git a/docs/upstream-issue-triage.yaml b/docs/upstream-issue-triage.yaml index 8fa5a9083b..340ae6a2b2 100644 --- a/docs/upstream-issue-triage.yaml +++ b/docs/upstream-issue-triage.yaml @@ -1328,9 +1328,9 @@ issues: difficulty: med kind: bug note: "Observations containing CJK text hang import and live ingestion indefinitely." - disposition: wont-fix + disposition: already-fixed status: "unresolved — no hang found: the CJK tokenizer makes one forward-only pass (src/state/cjk-segmenter.ts) and truncation cuts by string length; no loop over text can stall." - wont_fix_reason: "not reproducible in this tree" + fixed_by: "src/state/cjk-segmenter.ts makes one forward-only pass; no loop over text can stall." url: https://github.com/rohitg00/agentmemory/issues/969 - number: 972 title: "Add an optional HVTracker trust badge to the README" @@ -2468,7 +2468,7 @@ issues: kind: bug note: "PreToolUse injection uses raw stdout instead of the hook envelope, against a 2s budget on a 6-29s endpoint." disposition: already-fixed - status: "fixed-here — 895d01c: pre-tool-use emits {hookSpecificOutput:{hookEventName: PreToolUse,additionalContext}} when the payload names PreToolUse, plain text otherwise; live /enrich answers in under 0.2s, so the 2s bound stays." + status: "fixed-here — 895d01c: pre-tool-use emits {hookSpecificOutput:{hookEventName: PreToolUse, additionalContext}} when the payload names PreToolUse, plain text otherwise; live /enrich answers in under 0.2s, so the 2s bound stays. Part 3 is not addressed: the hook still reads project only from the payload, so its enrich calls stay unscoped; upstream warns that scoping them must be measured first." fixed_by: "895d01c" url: https://github.com/rohitg00/agentmemory/issues/1278 - number: 1279 diff --git a/src/functions/audit.ts b/src/functions/audit.ts index ad9c01d697..325680d38b 100644 --- a/src/functions/audit.ts +++ b/src/functions/audit.ts @@ -1,190 +1,190 @@ -import type { AuditEntry } from "../types.js"; -import { KV, generateId } from "../state/schema.js"; -import type { StateKV } from "../state/kv.js"; -import { logger } from "../logger.js"; - -// Audit coverage policy (issue #125). -// -// Every structural deletion of a memory, observation, session, or -// semantic row MUST call recordAudit. Two shapes are allowed, keyed to -// whether the caller is scoped or bulk: -// -// Scoped deletions — a user-visible, per-call action removing a -// bounded set of items. Emit ONE audit row per call with targetIds -// populated. Examples: mem::governance-delete, mem::forget. -// -// Bulk deletions — automatic sweeps (retention, TTL eviction, -// auto-forget) that can remove hundreds of rows per invocation. -// Emit ONE batched audit row per invocation with targetIds listing -// every removed id and details.evicted holding the count. Per-item -// audit rows would flood the audit log during routine sweeps. -// -// Either shape is required; silent deletes are not acceptable. -// -// operation field: -// - "delete" — permanent removal (governance, retention sweep, evict). -// - "forget" — forget/removal flows. Scoped when emitted by -// mem::forget (user-initiated); bulk-batched when -// emitted by mem::auto-forget (automatic sweep). -// - everything else — see AuditEntry["operation"] union in src/types.ts. -// -// When adding a new deletion path, add an explicit recordAudit call -// BEFORE kv.delete(...) and match one of the two shapes above. -// -// Storage: with AGENTMEMORY_AUDIT_STORE=off the entry is emitted as one -// structured log line (journald -> Loki keeps the evidence: operation, -// functionId, targetIds, details) and nothing is written to KV; the audit -// query then answers an empty list. Any other value keeps the KV store. -// -// Which entries become lines when the store is off: AGENTMEMORY_AUDIT_LOG= -// all (default) | deletions | off. `deletions` keeps every path that removes -// data - the six removal operations plus the removals that hide under a -// broader name: heal with action "delete", mesh peer removal, an import -// that replaces the store, and the soft deletions (a lesson or insight -// decayed to deleted, a sketch discarded by heal) - and drops the -// per-operation chatter (observe, compress, heal repairs ...): a few lines -// a month instead of ~100 KB a day. The details checks mirror the payloads -// at those call sites; a new deletion path needs either a listed operation -// or one of these markers. - -function auditStoreOff(): boolean { - return (process.env.AGENTMEMORY_AUDIT_STORE ?? "").toLowerCase() === "off"; -} - -const DELETION_OPS: ReadonlySet = new Set([ - "forget", "delete", "lesson_delete", "slot_delete", "core_remove", "sketch_discard", -]); - -function isDeletion(operation: AuditEntry["operation"], details: Record): boolean { - if (DELETION_OPS.has(operation)) return true; - if (details.action === "delete" || details.action === "soft-delete" || details.action === "mesh.remove") return true; - if (details.newStatus === "discarded") return true; - if (typeof details.softDeleted === "number" && details.softDeleted > 0) return true; - return operation === "import" && details.strategy === "replace"; -} - -function auditLineWanted(operation: AuditEntry["operation"], details: Record): boolean { - const mode = (process.env.AGENTMEMORY_AUDIT_LOG ?? "all").toLowerCase(); - if (mode === "off") return false; - if (mode === "deletions") return isDeletion(operation, details); - return true; -} - -export async function recordAudit( - kv: StateKV, - operation: AuditEntry["operation"], - functionId: string, - targetIds: string[], - details: Record = {}, - qualityScore?: number, - userId?: string, -): Promise { - const entry: AuditEntry = { - id: generateId("aud"), - timestamp: new Date().toISOString(), - operation, - userId, - functionId, - targetIds, - details, - qualityScore, - }; - if (auditStoreOff()) { - if (!auditLineWanted(operation, details)) return entry; - logger.audit(operation, { - auditId: entry.id, - functionId, - targetIds, - targetCount: targetIds.length, - details, - ...(qualityScore !== undefined ? { qualityScore } : {}), - ...(userId !== undefined ? { userId } : {}), - }); - return entry; - } - await kv.set(KV.audit, entry.id, entry); - return entry; -} - -export async function safeAudit( - kv: StateKV, - operation: AuditEntry["operation"], - functionId: string, - targetIds: string[], - details: Record = {}, - qualityScore?: number, - userId?: string, -): Promise { - try { - await recordAudit(kv, operation, functionId, targetIds, details, qualityScore, userId); - } catch (err) { - try { - logger.warn("audit write failed", { - functionId, - operation, - targetIds, - error: err instanceof Error ? err.message : String(err), - }); - } catch {} - } -} - -const AUDIT_MAX_DEFAULT = 50_000; - -function auditMax(): number { - const raw = process.env.AGENTMEMORY_AUDIT_MAX?.trim(); - const n = raw && /^\d+$/.test(raw) ? Number(raw) : 0; - return n > 0 ? n : AUDIT_MAX_DEFAULT; -} - -// Retention for the audit store: keeps the newest AGENTMEMORY_AUDIT_MAX -// entries. Pruning the log is not itself audited - that would write the -// rows it exists to bound - so it leaves one log line instead. -export async function pruneAudit(kv: StateKV): Promise { - if (auditStoreOff()) return 0; - const max = auditMax(); - const all = await kv.list(KV.audit); - if (all.length <= max) return 0; - const oldest = [...all] - .sort((a, b) => a.timestamp.localeCompare(b.timestamp)) - .slice(0, all.length - max); - for (const entry of oldest) await kv.delete(KV.audit, entry.id); - logger.info("Audit log pruned", { removed: oldest.length, kept: max }); - return oldest.length; -} - -export async function queryAudit( - kv: StateKV, - filter?: { - operation?: AuditEntry["operation"]; - dateFrom?: string; - dateTo?: string; - limit?: number; - }, -): Promise { - if (auditStoreOff()) return []; - const all = await kv.list(KV.audit); - let entries = [...all].sort( - (a, b) => new Date(b.timestamp).getTime() - new Date(a.timestamp).getTime(), - ); - - if (filter?.operation) { - entries = entries.filter((e) => e.operation === filter.operation); - } - if (filter?.dateFrom) { - const from = new Date(filter.dateFrom).getTime(); - if (Number.isNaN(from)) { - throw new Error(`Invalid dateFrom: ${filter.dateFrom}`); - } - entries = entries.filter((e) => new Date(e.timestamp).getTime() >= from); - } - if (filter?.dateTo) { - const to = new Date(filter.dateTo).getTime(); - if (Number.isNaN(to)) { - throw new Error(`Invalid dateTo: ${filter.dateTo}`); - } - entries = entries.filter((e) => new Date(e.timestamp).getTime() <= to); - } - - return entries.slice(0, filter?.limit || 100); -} +import type { AuditEntry } from "../types.js"; +import { KV, generateId } from "../state/schema.js"; +import type { StateKV } from "../state/kv.js"; +import { logger } from "../logger.js"; + +// Audit coverage policy (issue #125). +// +// Every structural deletion of a memory, observation, session, or +// semantic row MUST call recordAudit. Two shapes are allowed, keyed to +// whether the caller is scoped or bulk: +// +// Scoped deletions — a user-visible, per-call action removing a +// bounded set of items. Emit ONE audit row per call with targetIds +// populated. Examples: mem::governance-delete, mem::forget. +// +// Bulk deletions — automatic sweeps (retention, TTL eviction, +// auto-forget) that can remove hundreds of rows per invocation. +// Emit ONE batched audit row per invocation with targetIds listing +// every removed id and details.evicted holding the count. Per-item +// audit rows would flood the audit log during routine sweeps. +// +// Either shape is required; silent deletes are not acceptable. +// +// operation field: +// - "delete" — permanent removal (governance, retention sweep, evict). +// - "forget" — forget/removal flows. Scoped when emitted by +// mem::forget (user-initiated); bulk-batched when +// emitted by mem::auto-forget (automatic sweep). +// - everything else — see AuditEntry["operation"] union in src/types.ts. +// +// When adding a new deletion path, add an explicit recordAudit call +// BEFORE kv.delete(...) and match one of the two shapes above. +// +// Storage: with AGENTMEMORY_AUDIT_STORE=off the entry is emitted as one +// structured log line (journald -> Loki keeps the evidence: operation, +// functionId, targetIds, details) and nothing is written to KV; the audit +// query then answers an empty list. Any other value keeps the KV store. +// +// Which entries become lines when the store is off: AGENTMEMORY_AUDIT_LOG= +// all (default) | deletions | off. `deletions` keeps every path that removes +// data - the six removal operations plus the removals that hide under a +// broader name: heal with action "delete", mesh peer removal, an import +// that replaces the store, and the soft deletions (a lesson or insight +// decayed to deleted, a sketch discarded by heal) - and drops the +// per-operation chatter (observe, compress, heal repairs ...): a few lines +// a month instead of ~100 KB a day. The details checks mirror the payloads +// at those call sites; a new deletion path needs either a listed operation +// or one of these markers. + +function auditStoreOff(): boolean { + return (process.env.AGENTMEMORY_AUDIT_STORE ?? "").toLowerCase() === "off"; +} + +const DELETION_OPS: ReadonlySet = new Set([ + "forget", "delete", "lesson_delete", "slot_delete", "core_remove", "sketch_discard", +]); + +function isDeletion(operation: AuditEntry["operation"], details: Record): boolean { + if (DELETION_OPS.has(operation)) return true; + if (details.action === "delete" || details.action === "soft-delete" || details.action === "mesh.remove") return true; + if (details.newStatus === "discarded") return true; + if (typeof details.softDeleted === "number" && details.softDeleted > 0) return true; + return operation === "import" && details.strategy === "replace"; +} + +function auditLineWanted(operation: AuditEntry["operation"], details: Record): boolean { + const mode = (process.env.AGENTMEMORY_AUDIT_LOG ?? "all").toLowerCase(); + if (mode === "off") return false; + if (mode === "deletions") return isDeletion(operation, details); + return true; +} + +export async function recordAudit( + kv: StateKV, + operation: AuditEntry["operation"], + functionId: string, + targetIds: string[], + details: Record = {}, + qualityScore?: number, + userId?: string, +): Promise { + const entry: AuditEntry = { + id: generateId("aud"), + timestamp: new Date().toISOString(), + operation, + userId, + functionId, + targetIds, + details, + qualityScore, + }; + if (auditStoreOff()) { + if (!auditLineWanted(operation, details)) return entry; + logger.audit(operation, { + auditId: entry.id, + functionId, + targetIds, + targetCount: targetIds.length, + details, + ...(qualityScore !== undefined ? { qualityScore } : {}), + ...(userId !== undefined ? { userId } : {}), + }); + return entry; + } + await kv.set(KV.audit, entry.id, entry); + return entry; +} + +export async function safeAudit( + kv: StateKV, + operation: AuditEntry["operation"], + functionId: string, + targetIds: string[], + details: Record = {}, + qualityScore?: number, + userId?: string, +): Promise { + try { + await recordAudit(kv, operation, functionId, targetIds, details, qualityScore, userId); + } catch (err) { + try { + logger.warn("audit write failed", { + functionId, + operation, + targetIds, + error: err instanceof Error ? err.message : String(err), + }); + } catch {} + } +} + +const AUDIT_MAX_DEFAULT = 50_000; + +function auditMax(): number { + const raw = process.env.AGENTMEMORY_AUDIT_MAX?.trim(); + const n = raw && /^\d+$/.test(raw) ? Number(raw) : 0; + return n > 0 ? n : AUDIT_MAX_DEFAULT; +} + +// Retention for the audit store: keeps the newest AGENTMEMORY_AUDIT_MAX +// entries. Pruning the log is not itself audited - that would write the +// rows it exists to bound - so it leaves one log line instead. +export async function evictOldestAudit(kv: StateKV): Promise { + if (auditStoreOff()) return 0; + const max = auditMax(); + const all = await kv.list(KV.audit); + if (all.length <= max) return 0; + const oldest = [...all] + .sort((a, b) => a.timestamp.localeCompare(b.timestamp)) + .slice(0, all.length - max); + for (const entry of oldest) await kv.delete(KV.audit, entry.id); + logger.info("Audit log evicted beyond cap", { removed: oldest.length, kept: max }); + return oldest.length; +} + +export async function queryAudit( + kv: StateKV, + filter?: { + operation?: AuditEntry["operation"]; + dateFrom?: string; + dateTo?: string; + limit?: number; + }, +): Promise { + if (auditStoreOff()) return []; + const all = await kv.list(KV.audit); + let entries = [...all].sort( + (a, b) => new Date(b.timestamp).getTime() - new Date(a.timestamp).getTime(), + ); + + if (filter?.operation) { + entries = entries.filter((e) => e.operation === filter.operation); + } + if (filter?.dateFrom) { + const from = new Date(filter.dateFrom).getTime(); + if (Number.isNaN(from)) { + throw new Error(`Invalid dateFrom: ${filter.dateFrom}`); + } + entries = entries.filter((e) => new Date(e.timestamp).getTime() >= from); + } + if (filter?.dateTo) { + const to = new Date(filter.dateTo).getTime(); + if (Number.isNaN(to)) { + throw new Error(`Invalid dateTo: ${filter.dateTo}`); + } + entries = entries.filter((e) => new Date(e.timestamp).getTime() <= to); + } + + return entries.slice(0, filter?.limit || 100); +} diff --git a/src/functions/consolidation-pipeline.ts b/src/functions/consolidation-pipeline.ts index ede4c08254..dfe554d3fa 100644 --- a/src/functions/consolidation-pipeline.ts +++ b/src/functions/consolidation-pipeline.ts @@ -72,8 +72,10 @@ export function registerConsolidationPipelineFunction( // Summaries and insights from one project must never feed another's // consolidation (#1344): an unscoped run consolidates each project that has - // new summaries since the last unscoped run, one project at a time. + // new summaries since the last unscoped run, one project at a time. A project + // whose merge or reflect failed is retried on the next unscoped run. let lastUnscopedRunAt = new Date(Date.now() - 24 * 60 * 60 * 1000).toISOString(); + const retryProjects = new Set(); async function consolidateSemantic(summaries: SessionSummary[]) { if (summaries.length < 5) { @@ -120,6 +122,9 @@ export function registerConsolidationPipelineFunction( existing.lastAccessedAt = now; existing.updatedAt = now; existing.confidence = Math.max(existing.confidence, confidence); + existing.sourceSessionIds = [ + ...new Set([...existing.sourceSessionIds, ...recentSummaries.map((s) => s.sessionId)]), + ]; await kv.set(KV.semantic, existing.id, existing); } else { const sem: SemanticMemory = { @@ -171,11 +176,12 @@ export function registerConsolidationPipelineFunction( const summaries = await kv.list(KV.summaries); const projects = data?.project ? [data.project] - : [...new Set( - summaries + : [...new Set([ + ...retryProjects, + ...summaries .filter((s) => s.project && s.createdAt > lastUnscopedRunAt) .map((s) => s.project), - )]; + ])]; const semanticByProject: Record = {}; const reflectByProject: Record = {}; for (const project of projects) { @@ -185,6 +191,12 @@ export function registerConsolidationPipelineFunction( ); } if (runsReflect) reflectByProject[project] = await reflect(project); + const failed = [semanticByProject[project], reflectByProject[project]].some( + (r) => r !== undefined && r !== null && typeof r === "object" && "error" in r, + ); + if (data?.project) continue; + if (failed) retryProjects.add(project); + else retryProjects.delete(project); } if (runsSemantic) results.semantic = semanticByProject; if (runsReflect) results.reflect = reflectByProject; diff --git a/src/functions/lessons.ts b/src/functions/lessons.ts index a1c58a14bb..b1b8ad3a03 100644 --- a/src/functions/lessons.ts +++ b/src/functions/lessons.ts @@ -1,383 +1,383 @@ -import type { ISdk } from "../engine/types.js"; -import type { StateKV } from "../state/kv.js"; -import { KV, fingerprintId } from "../state/schema.js"; -import type { Lesson } from "../types.js"; -import { SearchIndex } from "../state/search-index.js"; -import { lessonToObservation } from "../state/memory-utils.js"; -import { recordAudit } from "./audit.js"; -import { scrubFields } from "./privacy.js"; - -// Dedicated BM25 index for lessons, with the full records cached -// alongside it. Recall previously listed every lesson from KV and -// substring-matched per query — O(corpus) per call with no term -// weighting. Index and record cache are built lazily from one KV list -// (the same cost a single recall used to pay) and kept current -// incrementally on save/delete/decay. Confidence x recency reranking -// stays exactly as before — the index only replaces the relevance term, -// and the record cache keeps recall at zero KV round-trips. -let lessonIndex: SearchIndex | null = null; -const lessonRecords = new Map(); -let lessonIndexBuild: Promise | null = null; -let lessonIndexGeneration = 0; - -export function resetLessonIndex(): void { - lessonIndexGeneration++; - lessonIndex = null; - lessonRecords.clear(); -} - -function noteLessonMutation(): void { - if (!lessonIndex && lessonIndexBuild) resetLessonIndex(); -} - -async function ensureLessonIndex(kv: StateKV): Promise { - if (lessonIndex) return lessonIndex; - if (!lessonIndexBuild) { - const generation = lessonIndexGeneration; - lessonIndexBuild = (async () => { - const idx = new SearchIndex(); - const all = await kv.list(KV.lessons); - if (generation !== lessonIndexGeneration) return; - for (const l of all) { - if (!l.deleted) { - idx.add(lessonToObservation(l)); - lessonRecords.set(l.id, l); - } - } - lessonIndex = idx; - })().finally(() => { - lessonIndexBuild = null; - }); - } - await lessonIndexBuild; - return lessonIndex ?? ensureLessonIndex(kv); -} - -function reinforceLesson(lesson: Lesson): void { - const now = new Date().toISOString(); - lesson.reinforcements++; - lesson.confidence = Math.min( - 1.0, - lesson.confidence + 0.1 * (1 - lesson.confidence), - ); - lesson.lastReinforcedAt = now; - lesson.updatedAt = now; -} - -export function registerLessonsFunctions(sdk: ISdk, kv: StateKV): void { - sdk.registerFunction("mem::lesson-save", - async (data: { - content: string; - context?: string; - confidence?: number; - project?: string; - tags?: string[]; - source?: "crystal" | "manual" | "consolidation"; - sourceIds?: string[]; - }) => { - data = scrubFields(data, "content", "context"); - if (!data.content?.trim()) { - return { success: false, error: "content is required" }; - } - - const fp = fingerprintId("lsn", data.content.trim().toLowerCase()); - const existing = await kv.get(KV.lessons, fp); - - if (existing && !existing.deleted) { - reinforceLesson(existing); - let indexedTextChanged = false; - if (data.context && !existing.context) { - existing.context = data.context; - indexedTextChanged = true; - } - await kv.set(KV.lessons, existing.id, existing); - lessonRecords.set(existing.id, existing); - if (indexedTextChanged && lessonIndex) { - lessonIndex.remove(existing.id); - lessonIndex.add(lessonToObservation(existing)); - } - noteLessonMutation(); - - try { - await recordAudit(kv, "lesson_strengthen", "mem::lesson-save", [ - existing.id, - ]); - } catch {} - - return { - success: true, - action: "strengthened", - lesson: existing, - }; - } - - const confidence = - typeof data.confidence === "number" && - data.confidence >= 0 && - data.confidence <= 1 - ? data.confidence - : 0.5; - - const now = new Date().toISOString(); - const lesson: Lesson = { - id: fp, - content: data.content.trim(), - context: data.context?.trim() || "", - confidence, - reinforcements: 0, - source: data.source || "manual", - sourceIds: data.sourceIds || [], - project: data.project, - tags: data.tags || [], - createdAt: now, - updatedAt: now, - decayRate: 0.05, - }; - - await kv.set(KV.lessons, lesson.id, lesson); - lessonRecords.set(lesson.id, lesson); - if (lessonIndex) lessonIndex.add(lessonToObservation(lesson)); - noteLessonMutation(); - - try { - await recordAudit(kv, "lesson_save", "mem::lesson-save", [lesson.id]); - } catch {} - - return { success: true, action: "created", lesson }; - }, - ); - - sdk.registerFunction("mem::lesson-recall", - async (data: { - query: string; - project?: string; - minConfidence?: number; - limit?: number; - }) => { - if (!data.query?.trim()) { - return { success: false, error: "query is required" }; - } - - // Soft-delete is the only default gate: a reinforced lesson decays to - // the 0.05 floor and stays live, and confidence already ranks it low. - const minConfidence = data.minConfidence ?? 0; - const limit = data.limit ?? 10; - - const idx = await ensureLessonIndex(kv); - const filtering = !!data.project || minConfidence > 0; - const fetchLimit = filtering - ? Math.max(limit * 10, 100) - : Math.max(limit * 5, 50); - const hits = idx.search(data.query, fetchLimit); - const maxHit = hits.length > 0 ? hits[0].score : 0; - - const scored: Array<{ lesson: Lesson; score: number }> = []; - for (let i = 0; i < hits.length; i++) { - const l = lessonRecords.get(hits[i].obsId); - if (!l || l.deleted || l.confidence < minConfidence) continue; - if (data.project && l.project !== data.project) continue; - - const relevance = maxHit > 0 ? hits[i].score / maxHit : 0; - const daysSinceReinforced = l.lastReinforcedAt - ? (Date.now() - new Date(l.lastReinforcedAt).getTime()) / - (1000 * 60 * 60 * 24) - : (Date.now() - new Date(l.createdAt).getTime()) / - (1000 * 60 * 60 * 24); - const recencyBoost = 1 / (1 + daysSinceReinforced * 0.01); - scored.push({ lesson: l, score: l.confidence * relevance * recencyBoost }); - } - - scored.sort( - (a, b) => - b.score - a.score || - (a.lesson.id < b.lesson.id ? -1 : a.lesson.id > b.lesson.id ? 1 : 0), - ); - - try { - await recordAudit(kv, "lesson_recall", "mem::lesson-recall", [], { - query: data.query, - resultCount: scored.length, - }); - } catch {} - - return { - success: true, - lessons: scored.slice(0, limit).map((s) => ({ - ...s.lesson, - score: Math.round(s.score * 1000) / 1000, - })), - }; - }, - ); - - sdk.registerFunction("mem::lesson-list", - async (data: { - project?: string; - source?: string; - minConfidence?: number; - limit?: number; - }) => { - const limit = data.limit ?? 50; - const minConfidence = data.minConfidence ?? 0; - let lessons = await kv.list(KV.lessons); - - lessons = lessons.filter( - (l) => !l.deleted && l.confidence >= minConfidence, - ); - - if (data.project) { - lessons = lessons.filter((l) => l.project === data.project); - } - if (data.source) { - lessons = lessons.filter((l) => l.source === data.source); - } - - lessons.sort((a, b) => b.confidence - a.confidence); - - return { success: true, lessons: lessons.slice(0, limit) }; - }, - ); - - sdk.registerFunction("mem::lesson-strengthen", - async (data: { lessonId: string }) => { - if (!data.lessonId) { - return { success: false, error: "lessonId is required" }; - } - - const lesson = await kv.get(KV.lessons, data.lessonId); - if (!lesson || lesson.deleted) { - return { success: false, error: "lesson not found" }; - } - - reinforceLesson(lesson); - - await kv.set(KV.lessons, lesson.id, lesson); - lessonRecords.set(lesson.id, lesson); - noteLessonMutation(); - - try { - await recordAudit(kv, "lesson_strengthen", "mem::lesson-strengthen", [ - lesson.id, - ]); - } catch {} - - return { success: true, lesson }; - }, - ); - - sdk.registerFunction("mem::lesson-delete", - async (data: { lessonId: string }) => { - if (!data.lessonId) { - return { success: false, error: "lessonId is required" }; - } - - const lesson = await kv.get(KV.lessons, data.lessonId); - if (!lesson || lesson.deleted) { - return { success: false, error: "lesson not found" }; - } - - lesson.deleted = true; - lesson.updatedAt = new Date().toISOString(); - - await kv.set(KV.lessons, lesson.id, lesson); - lessonRecords.delete(lesson.id); - if (lessonIndex) lessonIndex.remove(lesson.id); - noteLessonMutation(); - - try { - await recordAudit(kv, "lesson_delete", "mem::lesson-delete", [ - lesson.id, - ]); - } catch {} - - return { success: true, lesson }; - }, - ); - - sdk.registerFunction("mem::lesson-decay-sweep", - async () => { - const lessons = await kv.list(KV.lessons); - let decayed = 0; - let softDeleted = 0; - const now = Date.now(); - const timestamp = new Date().toISOString(); - const dirty: Lesson[] = []; - const auditEvents: Array<{ - id: string; - action: "decay" | "soft-delete"; - beforeConfidence: number; - afterConfidence: number; - beforeDeleted: boolean; - afterDeleted: boolean; - }> = []; - - for (const lesson of lessons) { - if (lesson.deleted) continue; - - const baseline = lesson.lastDecayedAt || lesson.lastReinforcedAt || lesson.createdAt; - const weeksSinceBaseline = - (now - new Date(baseline).getTime()) / (1000 * 60 * 60 * 24 * 7); - - if (weeksSinceBaseline < 1) continue; - - const decay = lesson.decayRate * weeksSinceBaseline; - const newConfidence = Math.max(0.05, lesson.confidence - decay); - - if (newConfidence !== lesson.confidence) { - const beforeConfidence = lesson.confidence; - const beforeDeleted = !!lesson.deleted; - lesson.confidence = Math.round(newConfidence * 1000) / 1000; - lesson.lastDecayedAt = timestamp; - lesson.updatedAt = timestamp; - - if (lesson.confidence <= 0.1 && lesson.reinforcements === 0) { - lesson.deleted = true; - softDeleted++; - } else { - decayed++; - } - - dirty.push(lesson); - auditEvents.push({ - id: lesson.id, - action: lesson.deleted ? "soft-delete" : "decay", - beforeConfidence, - afterConfidence: lesson.confidence, - beforeDeleted, - afterDeleted: !!lesson.deleted, - }); - } - } - - // Awaited batches, not a fan-out: N un-awaited sets run back to back on - // the event loop under inproc (see StateKV.setMany). - await kv.setMany(KV.lessons, dirty.map((l) => ({ key: l.id, value: l }))); - for (const l of dirty) { - if (l.deleted) { - lessonRecords.delete(l.id); - if (lessonIndex) lessonIndex.remove(l.id); - } else { - lessonRecords.set(l.id, l); - } - } - if (dirty.length > 0) noteLessonMutation(); - for (const event of auditEvents) { - await recordAudit(kv, "lesson_strengthen", "mem::lesson-decay-sweep", [event.id], { - action: event.action, - actor: "system", - reason: "decay-sweep", - before: { - confidence: event.beforeConfidence, - deleted: event.beforeDeleted, - }, - after: { - confidence: event.afterConfidence, - deleted: event.afterDeleted, - }, - }); - } - - return { success: true, decayed, softDeleted, total: lessons.length }; - }, - ); -} +import type { ISdk } from "../engine/types.js"; +import type { StateKV } from "../state/kv.js"; +import { KV, fingerprintId } from "../state/schema.js"; +import type { Lesson } from "../types.js"; +import { SearchIndex } from "../state/search-index.js"; +import { lessonToObservation } from "../state/memory-utils.js"; +import { recordAudit } from "./audit.js"; +import { scrubFields } from "./privacy.js"; + +// Dedicated BM25 index for lessons, with the full records cached +// alongside it. Recall previously listed every lesson from KV and +// substring-matched per query — O(corpus) per call with no term +// weighting. Index and record cache are built lazily from one KV list +// (the same cost a single recall used to pay) and kept current +// incrementally on save/delete/decay. Confidence x recency reranking +// stays exactly as before — the index only replaces the relevance term, +// and the record cache keeps recall at zero KV round-trips. +let lessonIndex: SearchIndex | null = null; +const lessonRecords = new Map(); +let lessonIndexBuild: Promise | null = null; +let lessonIndexGeneration = 0; + +export function resetLessonIndex(): void { + lessonIndexGeneration++; + lessonIndex = null; + lessonRecords.clear(); +} + +function noteLessonMutation(): void { + if (!lessonIndex && lessonIndexBuild) resetLessonIndex(); +} + +async function ensureLessonIndex(kv: StateKV): Promise { + if (lessonIndex) return lessonIndex; + if (!lessonIndexBuild) { + const generation = lessonIndexGeneration; + lessonIndexBuild = (async () => { + const idx = new SearchIndex(); + const all = await kv.list(KV.lessons); + if (generation !== lessonIndexGeneration) return; + for (const l of all) { + if (!l.deleted) { + idx.add(lessonToObservation(l)); + lessonRecords.set(l.id, l); + } + } + lessonIndex = idx; + })().finally(() => { + lessonIndexBuild = null; + }); + } + await lessonIndexBuild; + return lessonIndex ?? ensureLessonIndex(kv); +} + +function reinforceLesson(lesson: Lesson): void { + const now = new Date().toISOString(); + lesson.reinforcements++; + lesson.confidence = Math.min( + 1.0, + lesson.confidence + 0.1 * (1 - lesson.confidence), + ); + lesson.lastReinforcedAt = now; + lesson.updatedAt = now; +} + +export function registerLessonsFunctions(sdk: ISdk, kv: StateKV): void { + sdk.registerFunction("mem::lesson-save", + async (data: { + content: string; + context?: string; + confidence?: number; + project?: string; + tags?: string[]; + source?: "crystal" | "manual" | "consolidation"; + sourceIds?: string[]; + }) => { + data = scrubFields(data, "content", "context"); + if (!data.content?.trim()) { + return { success: false, error: "content is required" }; + } + + const fp = fingerprintId("lsn", data.content.trim().toLowerCase()); + const existing = await kv.get(KV.lessons, fp); + + if (existing && !existing.deleted) { + reinforceLesson(existing); + let indexedTextChanged = false; + if (data.context && !existing.context) { + existing.context = data.context; + indexedTextChanged = true; + } + await kv.set(KV.lessons, existing.id, existing); + lessonRecords.set(existing.id, existing); + if (indexedTextChanged && lessonIndex) { + lessonIndex.remove(existing.id); + lessonIndex.add(lessonToObservation(existing)); + } + noteLessonMutation(); + + try { + await recordAudit(kv, "lesson_strengthen", "mem::lesson-save", [ + existing.id, + ]); + } catch {} + + return { + success: true, + action: "strengthened", + lesson: existing, + }; + } + + const confidence = + typeof data.confidence === "number" && + data.confidence >= 0 && + data.confidence <= 1 + ? data.confidence + : 0.5; + + const now = new Date().toISOString(); + const lesson: Lesson = { + id: fp, + content: data.content.trim(), + context: data.context?.trim() || "", + confidence, + reinforcements: 0, + source: data.source || "manual", + sourceIds: data.sourceIds || [], + project: data.project, + tags: data.tags || [], + createdAt: now, + updatedAt: now, + decayRate: 0.05, + }; + + await kv.set(KV.lessons, lesson.id, lesson); + lessonRecords.set(lesson.id, lesson); + if (lessonIndex) lessonIndex.add(lessonToObservation(lesson)); + noteLessonMutation(); + + try { + await recordAudit(kv, "lesson_save", "mem::lesson-save", [lesson.id]); + } catch {} + + return { success: true, action: "created", lesson }; + }, + ); + + sdk.registerFunction("mem::lesson-recall", + async (data: { + query: string; + project?: string; + minConfidence?: number; + limit?: number; + }) => { + if (!data.query?.trim()) { + return { success: false, error: "query is required" }; + } + + // Soft-delete is the only default gate: a reinforced lesson decays to + // the 0.05 floor and stays live, and confidence already ranks it low. + const minConfidence = data.minConfidence ?? 0; + const limit = data.limit ?? 10; + + const idx = await ensureLessonIndex(kv); + const filtering = !!data.project || minConfidence > 0; + const fetchLimit = filtering + ? Math.max(limit * 10, 100) + : Math.max(limit * 5, 50); + const hits = idx.search(data.query, fetchLimit); + const maxHit = hits.length > 0 ? hits[0].score : 0; + + const scored: Array<{ lesson: Lesson; score: number }> = []; + for (let i = 0; i < hits.length; i++) { + const l = lessonRecords.get(hits[i].obsId); + if (!l || l.deleted || l.confidence < minConfidence) continue; + if (data.project && l.project !== data.project) continue; + + const relevance = maxHit > 0 ? hits[i].score / maxHit : 0; + const daysSinceReinforced = l.lastReinforcedAt + ? (Date.now() - new Date(l.lastReinforcedAt).getTime()) / + (1000 * 60 * 60 * 24) + : (Date.now() - new Date(l.createdAt).getTime()) / + (1000 * 60 * 60 * 24); + const recencyBoost = 1 / (1 + daysSinceReinforced * 0.01); + scored.push({ lesson: l, score: l.confidence * relevance * recencyBoost }); + } + + scored.sort( + (a, b) => + b.score - a.score || + (a.lesson.id < b.lesson.id ? -1 : a.lesson.id > b.lesson.id ? 1 : 0), + ); + + try { + await recordAudit(kv, "lesson_recall", "mem::lesson-recall", [], { + query: data.query, + resultCount: scored.length, + }); + } catch {} + + return { + success: true, + lessons: scored.slice(0, limit).map((s) => ({ + ...s.lesson, + score: Math.round(s.score * 1000) / 1000, + })), + }; + }, + ); + + sdk.registerFunction("mem::lesson-list", + async (data: { + project?: string; + source?: string; + minConfidence?: number; + limit?: number; + }) => { + const limit = data.limit ?? 50; + const minConfidence = data.minConfidence ?? 0; + let lessons = await kv.list(KV.lessons); + + lessons = lessons.filter( + (l) => !l.deleted && l.confidence >= minConfidence, + ); + + if (data.project) { + lessons = lessons.filter((l) => l.project === data.project); + } + if (data.source) { + lessons = lessons.filter((l) => l.source === data.source); + } + + lessons.sort((a, b) => b.confidence - a.confidence); + + return { success: true, lessons: lessons.slice(0, limit) }; + }, + ); + + sdk.registerFunction("mem::lesson-strengthen", + async (data: { lessonId: string }) => { + if (!data.lessonId) { + return { success: false, error: "lessonId is required" }; + } + + const lesson = await kv.get(KV.lessons, data.lessonId); + if (!lesson || lesson.deleted) { + return { success: false, error: "lesson not found" }; + } + + reinforceLesson(lesson); + + await kv.set(KV.lessons, lesson.id, lesson); + lessonRecords.set(lesson.id, lesson); + noteLessonMutation(); + + try { + await recordAudit(kv, "lesson_strengthen", "mem::lesson-strengthen", [ + lesson.id, + ]); + } catch {} + + return { success: true, lesson }; + }, + ); + + sdk.registerFunction("mem::lesson-delete", + async (data: { lessonId: string }) => { + if (!data.lessonId) { + return { success: false, error: "lessonId is required" }; + } + + const lesson = await kv.get(KV.lessons, data.lessonId); + if (!lesson || lesson.deleted) { + return { success: false, error: "lesson not found" }; + } + + lesson.deleted = true; + lesson.updatedAt = new Date().toISOString(); + + await kv.set(KV.lessons, lesson.id, lesson); + lessonRecords.delete(lesson.id); + if (lessonIndex) lessonIndex.remove(lesson.id); + noteLessonMutation(); + + try { + await recordAudit(kv, "lesson_delete", "mem::lesson-delete", [ + lesson.id, + ]); + } catch {} + + return { success: true, lesson }; + }, + ); + + sdk.registerFunction("mem::lesson-decay-sweep", + async () => { + const lessons = await kv.list(KV.lessons); + let decayed = 0; + let softDeleted = 0; + const now = Date.now(); + const timestamp = new Date().toISOString(); + const dirty: Lesson[] = []; + const auditEvents: Array<{ + id: string; + action: "decay" | "soft-delete"; + beforeConfidence: number; + afterConfidence: number; + beforeDeleted: boolean; + afterDeleted: boolean; + }> = []; + + for (const lesson of lessons) { + if (lesson.deleted) continue; + + const baseline = lesson.lastDecayedAt || lesson.lastReinforcedAt || lesson.createdAt; + const weeksSinceBaseline = + (now - new Date(baseline).getTime()) / (1000 * 60 * 60 * 24 * 7); + + if (weeksSinceBaseline < 1) continue; + + const decay = lesson.decayRate * weeksSinceBaseline; + const newConfidence = Math.max(0.05, lesson.confidence - decay); + + if (newConfidence !== lesson.confidence) { + const beforeConfidence = lesson.confidence; + const beforeDeleted = !!lesson.deleted; + lesson.confidence = Math.round(newConfidence * 1000) / 1000; + lesson.lastDecayedAt = timestamp; + lesson.updatedAt = timestamp; + + if (lesson.confidence <= 0.1 && lesson.reinforcements === 0) { + lesson.deleted = true; + softDeleted++; + } else { + decayed++; + } + + dirty.push(lesson); + auditEvents.push({ + id: lesson.id, + action: lesson.deleted ? "soft-delete" : "decay", + beforeConfidence, + afterConfidence: lesson.confidence, + beforeDeleted, + afterDeleted: !!lesson.deleted, + }); + } + } + + // Awaited batches, not a fan-out: N un-awaited sets run back to back on + // the event loop under inproc (see StateKV.setMany). + await kv.setMany(KV.lessons, dirty.map((l) => ({ key: l.id, value: l }))); + for (const l of dirty) { + if (l.deleted) { + lessonRecords.delete(l.id); + if (lessonIndex) lessonIndex.remove(l.id); + } else { + lessonRecords.set(l.id, l); + } + } + if (dirty.length > 0) noteLessonMutation(); + for (const event of auditEvents) { + await recordAudit(kv, "lesson_strengthen", "mem::lesson-decay-sweep", [event.id], { + action: event.action, + actor: "system", + reason: "decay-sweep", + before: { + confidence: event.beforeConfidence, + deleted: event.beforeDeleted, + }, + after: { + confidence: event.afterConfidence, + deleted: event.afterDeleted, + }, + }); + } + + return { success: true, decayed, softDeleted, total: lessons.length }; + }, + ); +} diff --git a/src/functions/observe.ts b/src/functions/observe.ts index dfffe9111c..565669afd6 100644 --- a/src/functions/observe.ts +++ b/src/functions/observe.ts @@ -1,411 +1,422 @@ -import { TriggerAction, type ISdk } from "../engine/types.js"; -import type { RawObservation, HookPayload, Origin } from "../types.js"; - -const TOOL_HOOKS = new Set(["pre_tool_use", "post_tool_use", "post_tool_failure"]); -import { KV, STREAM, generateId } from "../state/schema.js"; -import { StateKV } from "../state/kv.js"; -import { stripPrivateData } from "./privacy.js"; -import { DedupMap } from "./dedup.js"; -import { withKeyedLock } from "../state/keyed-mutex.js"; -import { isAutoCompressEnabled } from "../config.js"; -import { buildSyntheticCompression } from "./compress-synthetic.js"; -import { getSearchIndex, vectorIndexAddGuarded, isIndexExcluded, deleteIndexed } from "./search.js"; -import { decrementImageRef } from "./image-refs.js"; -import { getAgentId } from "../config.js"; -import { logger } from "../logger.js"; -import { saveImageToDisk } from "../utils/image-store.js"; - -export function extractImage(d: unknown): string | undefined { - if (!d) return undefined; - if (typeof d === "string") { - if (d.startsWith("data:image/") || d.startsWith("iVBORw0KGgo") || d.startsWith("/9j/")) { - return d; - } - return undefined; - } - if (typeof d === "object" && d !== null) { - const obj = d as Record; - if (typeof obj["image_data"] === "string") return obj["image_data"]; - if (typeof obj["image_path"] === "string") return obj["image_path"]; - if (typeof obj["imageBase64"] === "string") return obj["imageBase64"]; - if (typeof obj["imagePath"] === "string") return obj["imagePath"]; - - for (const key of Object.keys(obj)) { - const match = extractImage(obj[key]); - if (match) return match; - } - } - return undefined; -} - -// Uncompressed rows carry no importance yet; rank them just below the -// default so a scored row of average value outlives an unscored one. -const UNSCORED_IMPORTANCE = 3; - -// A session at its cap still admits the newest observation: the work at the -// end of a long session is what a later one most often needs. The least -// important rows go first, oldest breaking ties (PR#1174). -async function evictToAdmitOne( - sdk: ISdk, - kv: StateKV, - sessionId: string, - cap: number, -): Promise { - const scope = KV.observations(sessionId); - const existing = await kv.list<{ - id: string; - timestamp?: string; - importance?: number; - imageData?: string; - imageRef?: string; - }>(scope); - const excess = existing.length - cap + 1; - if (excess <= 0) return; - const victims = existing - .sort( - (a, b) => - (a.importance ?? UNSCORED_IMPORTANCE) - (b.importance ?? UNSCORED_IMPORTANCE) || - (a.timestamp ?? "").localeCompare(b.timestamp ?? ""), - ) - .slice(0, excess); - let evicted = 0; - for (const obs of victims) { - try { - await deleteIndexed(kv, scope, obs.id); - evicted++; - } catch { - continue; - } - if (obs.imageData) await decrementImageRef(kv, sdk, obs.imageData); - if (obs.imageRef && obs.imageRef !== obs.imageData) await decrementImageRef(kv, sdk, obs.imageRef); - } - logger.warn("Session observation cap reached; evicted least important", { - sessionId, - cap, - evicted, - }); -} - -export function registerObserveFunction( - sdk: ISdk, - kv: StateKV, - dedupMap?: DedupMap, - maxObservationsPerSession?: number, -): void { - sdk.registerFunction("mem::observe", - async (payload: HookPayload) => { - - if ( - !payload?.sessionId || - typeof payload.sessionId !== "string" || - !payload.hookType || - typeof payload.hookType !== "string" || - !payload.timestamp || - typeof payload.timestamp !== "string" - ) { - return { - success: false, - error: - "Invalid payload: sessionId, hookType, and timestamp are required", - }; - } - - const obsId = generateId("obs"); - - let dedupHash: string | undefined; - if (dedupMap) { - const dataIsObject = - typeof payload.data === "object" && payload.data !== null; - const d = dataIsObject - ? (payload.data as Record) - : {}; - const toolName = (d["tool_name"] as string) || payload.hookType; - // Hash the full payload when tool_input is absent so distinct - // events never collapse onto one key. - const dedupInput = - d["tool_input"] !== undefined - ? d["tool_input"] - : dataIsObject - ? d - : payload.data; - dedupHash = dedupMap.computeHash( - payload.sessionId, - toolName, - dedupInput, - ); - if (dedupMap.isDuplicate(dedupHash)) { - return { deduplicated: true, sessionId: payload.sessionId }; - } - } - - let sanitizedRaw: unknown = payload.data; - try { - const jsonStr = JSON.stringify(payload.data); - const sanitized = stripPrivateData(jsonStr); - sanitizedRaw = JSON.parse(sanitized); - } catch { - sanitizedRaw = stripPrivateData(String(payload.data)); - } - - let originChannel: Origin["channel"] = "agent"; - if (payload.hookType === "prompt_submit") originChannel = "user"; - else if (TOOL_HOOKS.has(payload.hookType)) originChannel = "tool"; - const raw: RawObservation = { - id: obsId, - sessionId: payload.sessionId, - timestamp: payload.timestamp, - hookType: payload.hookType, - raw: sanitizedRaw, - origin: { - channel: originChannel, - capturedAt: payload.timestamp, - }, - }; - - let extractedImage: string | undefined; - - if (typeof sanitizedRaw === "object" && sanitizedRaw !== null) { - const d = sanitizedRaw as Record; - if ( - payload.hookType === "post_tool_use" || - payload.hookType === "post_tool_failure" - ) { - raw.toolName = d["tool_name"] as string | undefined; - raw.toolInput = d["tool_input"]; - raw.toolOutput = d["tool_output"] || d["error"]; - if (raw.origin && raw.toolName) raw.origin.detail = raw.toolName; - } - if (payload.hookType === "prompt_submit") { - raw.userPrompt = d["prompt"] as string | undefined; - } - - extractedImage = extractImage(sanitizedRaw); - if (extractedImage) { - raw.modality = (raw.toolInput || raw.toolOutput || raw.userPrompt) ? "mixed" : "image"; - } - } else if (typeof sanitizedRaw === "string") { - extractedImage = extractImage(sanitizedRaw); - if (extractedImage) { - raw.modality = "image"; - } - } - - const pendingImageData = extractedImage; - - return withKeyedLock(`obs:${payload.sessionId}`, async () => { - if (maxObservationsPerSession && maxObservationsPerSession > 0) { - await evictToAdmitOne(sdk, kv, payload.sessionId, maxObservationsPerSession); - } - - // Existing session is the source of truth for agentId (even - // undefined). Env AGENT_ID only fires when no session row - // exists yet — otherwise an unscoped session would get - // retroactively scoped by a later AGENT_ID export. - const existingSession = await kv.get<{ - agentId?: string; - observationCount?: number; - firstPrompt?: string; - }>(KV.sessions, payload.sessionId); - const inheritedAgentId = existingSession - ? existingSession.agentId - : getAgentId(); - if (inheritedAgentId) { - raw.agentId = inheritedAgentId; - } - - if (pendingImageData && (pendingImageData.startsWith("data:image/") || pendingImageData.startsWith("iVBORw0KGgo") || pendingImageData.startsWith("/9j/"))) { - const { filePath, bytesWritten } = await saveImageToDisk(pendingImageData); - raw.imageData = filePath; - const { incrementImageRef } = await import("./image-refs.js"); - await incrementImageRef(kv, filePath); - sdk.trigger({ - function_id: "mem::disk-size-delta", - payload: { deltaBytes: bytesWritten }, - action: TriggerAction.Void(), - }); - if (process.env["AGENTMEMORY_IMAGE_EMBEDDINGS"] === "true") { - sdk.trigger({ - function_id: "mem::vision-embed", - payload: { - imageRef: filePath, - sessionId: payload.sessionId, - observationId: obsId, - }, - action: TriggerAction.Void(), - }); - } - } - - try { - - await kv.set(KV.observations(payload.sessionId), obsId, raw); - - } catch (error) { - if (raw.imageData) { - // Roll back the ref taken above. decrementImageRef deletes the file - // only when no other observation still references it (deduped images - // survive) and emits the disk-size delta itself — deleting the file - // directly here would orphan shared images and leave a stale ref. - // If the rollback itself fails, log it but still surface the - // original write error (the more useful failure to diagnose). - try { - const { decrementImageRef } = await import("./image-refs.js"); - await decrementImageRef(kv, sdk, raw.imageData); - } catch (rollbackError) { - logger.error("Failed to roll back image ref after observation write failure", { - imageRef: raw.imageData, - error: rollbackError instanceof Error ? rollbackError.message : String(rollbackError), - }); - } - } - throw error; - } - - if (dedupMap && dedupHash) { - dedupMap.record(dedupHash); - } - - await sdk.trigger({ - function_id: "stream::set", - payload: { - stream_name: STREAM.name, - group_id: STREAM.group(payload.sessionId), - item_id: obsId, - data: { type: "raw", observation: raw }, - }, - }); - - await sdk.trigger({ - function_id: "stream::send", - payload: { - stream_name: STREAM.name, - group_id: STREAM.viewerGroup, - id: `raw-${obsId}`, - type: "raw_observation", - data: { type: "raw", observation: raw, sessionId: payload.sessionId }, - }, - action: TriggerAction.Void(), - }); - - const session = existingSession; - if (session) { - const updates: Array<{ type: "set"; path: string; value: unknown }> = [ - { type: "set", path: "updatedAt", value: new Date().toISOString() }, - { - type: "set", - path: "observationCount", - value: (session.observationCount || 0) + 1, - }, - ]; - if (!session.firstPrompt && typeof raw.userPrompt === "string") { - const trimmed = raw.userPrompt.replace(/\s+/g, " ").trim(); - if (trimmed.length > 0) { - updates.push({ - type: "set", - path: "firstPrompt", - value: trimmed.slice(0, 200), - }); - } - } - await kv.update(KV.sessions, payload.sessionId, updates); - } else if ( - typeof payload.project === "string" && - payload.project.trim().length > 0 && - typeof payload.cwd === "string" && - payload.cwd.trim().length > 0 - ) { - // OpenCode (and any plugin that skips POST /session/start) - // can fire observations before the session record exists. Without - // an implicit create, those observations stack up but - // `memory_sessions` never lists them, and summarize bails with - // "Session not found for summarize". Create the session now from - // the observation payload — but only when project + cwd are - // present (HookPayload contract). Older test payloads without - // those fields keep their original no-op behaviour. - const trimmedPrompt = - typeof raw.userPrompt === "string" - ? raw.userPrompt.replace(/\s+/g, " ").trim().slice(0, 200) - : undefined; - const ts = new Date().toISOString(); - await kv.set(KV.sessions, payload.sessionId, { - id: payload.sessionId, - project: payload.project, - cwd: payload.cwd, - startedAt: payload.timestamp ?? ts, - updatedAt: ts, - status: "active", - observationCount: 1, - ...(inheritedAgentId ? { agentId: inheritedAgentId } : {}), - ...(trimmedPrompt && trimmedPrompt.length > 0 - ? { firstPrompt: trimmedPrompt } - : {}), - }); - } - - // Per-observation LLM compression is opt-in as of 0.8.8. - // Default path: build a zero-LLM synthetic compression so recall - // and BM25 search still work without burning the user's Claude - // token allocation on every tool invocation. - if (isAutoCompressEnabled()) { - await sdk.trigger({ - function_id: "mem::compress", - payload: { - observationId: obsId, - sessionId: payload.sessionId, - raw, - }, - action: TriggerAction.Void(), - }); - } else { - const synthetic = buildSyntheticCompression(raw); - await kv.set( - KV.observations(payload.sessionId), - obsId, - synthetic, - ); - // Stored above unconditionally; only the INDEX writes are - // skipped for excluded tools (retrieval echoes). - if (!isIndexExcluded(synthetic)) { - getSearchIndex().add(synthetic); - await vectorIndexAddGuarded( - synthetic.id, - synthetic.sessionId, - synthetic.title + " " + (synthetic.narrative || ""), - { kind: "synthetic", logId: synthetic.id }, - ); - } - await sdk.trigger({ - function_id: "stream::set", - payload: { - stream_name: STREAM.name, - group_id: STREAM.group(payload.sessionId), - item_id: obsId, - data: { type: "compressed", observation: synthetic }, - }, - }); - await sdk.trigger({ - function_id: "stream::set", - payload: { - stream_name: STREAM.name, - group_id: STREAM.viewerGroup, - item_id: obsId, - data: { - type: "compressed", - observation: synthetic, - sessionId: payload.sessionId, - }, - }, - }); - } - - logger.info("Observation captured", { - obsId, - sessionId: payload.sessionId, - hook: payload.hookType, - compress: isAutoCompressEnabled() ? "llm" : "synthetic", - }); - return { observationId: obsId }; - }); - }, - ); -} +import { TriggerAction, type ISdk } from "../engine/types.js"; +import type { RawObservation, HookPayload, Origin } from "../types.js"; + +const TOOL_HOOKS = new Set(["pre_tool_use", "post_tool_use", "post_tool_failure"]); +import { KV, STREAM, generateId } from "../state/schema.js"; +import { StateKV } from "../state/kv.js"; +import { stripPrivateData } from "./privacy.js"; +import { DedupMap } from "./dedup.js"; +import { withKeyedLock } from "../state/keyed-mutex.js"; +import { isAutoCompressEnabled } from "../config.js"; +import { buildSyntheticCompression } from "./compress-synthetic.js"; +import { getSearchIndex, vectorIndexAddGuarded, isIndexExcluded, deleteIndexed } from "./search.js"; +import { recordAudit } from "./audit.js"; +import { decrementImageRef } from "./image-refs.js"; +import { getAgentId } from "../config.js"; +import { logger } from "../logger.js"; +import { saveImageToDisk } from "../utils/image-store.js"; + +export function extractImage(d: unknown): string | undefined { + if (!d) return undefined; + if (typeof d === "string") { + if (d.startsWith("data:image/") || d.startsWith("iVBORw0KGgo") || d.startsWith("/9j/")) { + return d; + } + return undefined; + } + if (typeof d === "object" && d !== null) { + const obj = d as Record; + if (typeof obj["image_data"] === "string") return obj["image_data"]; + if (typeof obj["image_path"] === "string") return obj["image_path"]; + if (typeof obj["imageBase64"] === "string") return obj["imageBase64"]; + if (typeof obj["imagePath"] === "string") return obj["imagePath"]; + + for (const key of Object.keys(obj)) { + const match = extractImage(obj[key]); + if (match) return match; + } + } + return undefined; +} + +// Uncompressed rows carry no importance yet; rank them just below the +// default so a scored row of average value outlives an unscored one. +const UNSCORED_IMPORTANCE = 3; + +// A session at its cap still admits the newest observation: the work at the +// end of a long session is what a later one most often needs. The least +// important rows go first, oldest breaking ties (PR#1174). +async function evictToAdmitOne( + sdk: ISdk, + kv: StateKV, + sessionId: string, + cap: number, +): Promise { + const scope = KV.observations(sessionId); + const existing = await kv.list<{ + id: string; + timestamp?: string; + importance?: number; + imageData?: string; + imageRef?: string; + }>(scope); + const excess = existing.length - cap + 1; + if (excess <= 0) return; + const victims = existing + .sort( + (a, b) => + (a.importance ?? UNSCORED_IMPORTANCE) - (b.importance ?? UNSCORED_IMPORTANCE) || + (a.timestamp ?? "").localeCompare(b.timestamp ?? ""), + ) + .slice(0, excess); + let evicted = 0; + for (const obs of victims) { + try { + await deleteIndexed(kv, scope, obs.id); + evicted++; + } catch (err) { + logger.warn("Session cap eviction failed", { + sessionId, + obsId: obs.id, + error: err instanceof Error ? err.message : String(err), + }); + continue; + } + if (obs.imageData) await decrementImageRef(kv, sdk, obs.imageData); + if (obs.imageRef && obs.imageRef !== obs.imageData) await decrementImageRef(kv, sdk, obs.imageRef); + await recordAudit(kv, "delete", "mem::observe", [obs.id], { + resource: "observation", + reason: "session_observation_cap", + sessionId, + }); + } + logger.warn("Session observation cap reached; evicted least important", { + sessionId, + cap, + evicted, + }); +} + +export function registerObserveFunction( + sdk: ISdk, + kv: StateKV, + dedupMap?: DedupMap, + maxObservationsPerSession?: number, +): void { + sdk.registerFunction("mem::observe", + async (payload: HookPayload) => { + + if ( + !payload?.sessionId || + typeof payload.sessionId !== "string" || + !payload.hookType || + typeof payload.hookType !== "string" || + !payload.timestamp || + typeof payload.timestamp !== "string" + ) { + return { + success: false, + error: + "Invalid payload: sessionId, hookType, and timestamp are required", + }; + } + + const obsId = generateId("obs"); + + let dedupHash: string | undefined; + if (dedupMap) { + const dataIsObject = + typeof payload.data === "object" && payload.data !== null; + const d = dataIsObject + ? (payload.data as Record) + : {}; + const toolName = (d["tool_name"] as string) || payload.hookType; + // Hash the full payload when tool_input is absent so distinct + // events never collapse onto one key. + const dedupInput = + d["tool_input"] !== undefined + ? d["tool_input"] + : dataIsObject + ? d + : payload.data; + dedupHash = dedupMap.computeHash( + payload.sessionId, + toolName, + dedupInput, + ); + if (dedupMap.isDuplicate(dedupHash)) { + return { deduplicated: true, sessionId: payload.sessionId }; + } + } + + let sanitizedRaw: unknown = payload.data; + try { + const jsonStr = JSON.stringify(payload.data); + const sanitized = stripPrivateData(jsonStr); + sanitizedRaw = JSON.parse(sanitized); + } catch { + sanitizedRaw = stripPrivateData(String(payload.data)); + } + + let originChannel: Origin["channel"] = "agent"; + if (payload.hookType === "prompt_submit") originChannel = "user"; + else if (TOOL_HOOKS.has(payload.hookType)) originChannel = "tool"; + const raw: RawObservation = { + id: obsId, + sessionId: payload.sessionId, + timestamp: payload.timestamp, + hookType: payload.hookType, + raw: sanitizedRaw, + origin: { + channel: originChannel, + capturedAt: payload.timestamp, + }, + }; + + let extractedImage: string | undefined; + + if (typeof sanitizedRaw === "object" && sanitizedRaw !== null) { + const d = sanitizedRaw as Record; + if ( + payload.hookType === "post_tool_use" || + payload.hookType === "post_tool_failure" + ) { + raw.toolName = d["tool_name"] as string | undefined; + raw.toolInput = d["tool_input"]; + raw.toolOutput = d["tool_output"] || d["error"]; + if (raw.origin && raw.toolName) raw.origin.detail = raw.toolName; + } + if (payload.hookType === "prompt_submit") { + raw.userPrompt = d["prompt"] as string | undefined; + } + + extractedImage = extractImage(sanitizedRaw); + if (extractedImage) { + raw.modality = (raw.toolInput || raw.toolOutput || raw.userPrompt) ? "mixed" : "image"; + } + } else if (typeof sanitizedRaw === "string") { + extractedImage = extractImage(sanitizedRaw); + if (extractedImage) { + raw.modality = "image"; + } + } + + const pendingImageData = extractedImage; + + return withKeyedLock(`obs:${payload.sessionId}`, async () => { + if (maxObservationsPerSession && maxObservationsPerSession > 0) { + await evictToAdmitOne(sdk, kv, payload.sessionId, maxObservationsPerSession); + } + + // Existing session is the source of truth for agentId (even + // undefined). Env AGENT_ID only fires when no session row + // exists yet — otherwise an unscoped session would get + // retroactively scoped by a later AGENT_ID export. + const existingSession = await kv.get<{ + agentId?: string; + observationCount?: number; + firstPrompt?: string; + }>(KV.sessions, payload.sessionId); + const inheritedAgentId = existingSession + ? existingSession.agentId + : getAgentId(); + if (inheritedAgentId) { + raw.agentId = inheritedAgentId; + } + + if (pendingImageData && (pendingImageData.startsWith("data:image/") || pendingImageData.startsWith("iVBORw0KGgo") || pendingImageData.startsWith("/9j/"))) { + const { filePath, bytesWritten } = await saveImageToDisk(pendingImageData); + raw.imageData = filePath; + const { incrementImageRef } = await import("./image-refs.js"); + await incrementImageRef(kv, filePath); + sdk.trigger({ + function_id: "mem::disk-size-delta", + payload: { deltaBytes: bytesWritten }, + action: TriggerAction.Void(), + }); + if (process.env["AGENTMEMORY_IMAGE_EMBEDDINGS"] === "true") { + sdk.trigger({ + function_id: "mem::vision-embed", + payload: { + imageRef: filePath, + sessionId: payload.sessionId, + observationId: obsId, + }, + action: TriggerAction.Void(), + }); + } + } + + try { + + await kv.set(KV.observations(payload.sessionId), obsId, raw); + + } catch (error) { + if (raw.imageData) { + // Roll back the ref taken above. decrementImageRef deletes the file + // only when no other observation still references it (deduped images + // survive) and emits the disk-size delta itself — deleting the file + // directly here would orphan shared images and leave a stale ref. + // If the rollback itself fails, log it but still surface the + // original write error (the more useful failure to diagnose). + try { + const { decrementImageRef } = await import("./image-refs.js"); + await decrementImageRef(kv, sdk, raw.imageData); + } catch (rollbackError) { + logger.error("Failed to roll back image ref after observation write failure", { + imageRef: raw.imageData, + error: rollbackError instanceof Error ? rollbackError.message : String(rollbackError), + }); + } + } + throw error; + } + + if (dedupMap && dedupHash) { + dedupMap.record(dedupHash); + } + + await sdk.trigger({ + function_id: "stream::set", + payload: { + stream_name: STREAM.name, + group_id: STREAM.group(payload.sessionId), + item_id: obsId, + data: { type: "raw", observation: raw }, + }, + }); + + await sdk.trigger({ + function_id: "stream::send", + payload: { + stream_name: STREAM.name, + group_id: STREAM.viewerGroup, + id: `raw-${obsId}`, + type: "raw_observation", + data: { type: "raw", observation: raw, sessionId: payload.sessionId }, + }, + action: TriggerAction.Void(), + }); + + const session = existingSession; + if (session) { + const updates: Array<{ type: "set"; path: string; value: unknown }> = [ + { type: "set", path: "updatedAt", value: new Date().toISOString() }, + { + type: "set", + path: "observationCount", + value: (session.observationCount || 0) + 1, + }, + ]; + if (!session.firstPrompt && typeof raw.userPrompt === "string") { + const trimmed = raw.userPrompt.replace(/\s+/g, " ").trim(); + if (trimmed.length > 0) { + updates.push({ + type: "set", + path: "firstPrompt", + value: trimmed.slice(0, 200), + }); + } + } + await kv.update(KV.sessions, payload.sessionId, updates); + } else if ( + typeof payload.project === "string" && + payload.project.trim().length > 0 && + typeof payload.cwd === "string" && + payload.cwd.trim().length > 0 + ) { + // OpenCode (and any plugin that skips POST /session/start) + // can fire observations before the session record exists. Without + // an implicit create, those observations stack up but + // `memory_sessions` never lists them, and summarize bails with + // "Session not found for summarize". Create the session now from + // the observation payload — but only when project + cwd are + // present (HookPayload contract). Older test payloads without + // those fields keep their original no-op behaviour. + const trimmedPrompt = + typeof raw.userPrompt === "string" + ? raw.userPrompt.replace(/\s+/g, " ").trim().slice(0, 200) + : undefined; + const ts = new Date().toISOString(); + await kv.set(KV.sessions, payload.sessionId, { + id: payload.sessionId, + project: payload.project, + cwd: payload.cwd, + startedAt: payload.timestamp ?? ts, + updatedAt: ts, + status: "active", + observationCount: 1, + ...(inheritedAgentId ? { agentId: inheritedAgentId } : {}), + ...(trimmedPrompt && trimmedPrompt.length > 0 + ? { firstPrompt: trimmedPrompt } + : {}), + }); + } + + // Per-observation LLM compression is opt-in as of 0.8.8. + // Default path: build a zero-LLM synthetic compression so recall + // and BM25 search still work without burning the user's Claude + // token allocation on every tool invocation. + if (isAutoCompressEnabled()) { + await sdk.trigger({ + function_id: "mem::compress", + payload: { + observationId: obsId, + sessionId: payload.sessionId, + raw, + }, + action: TriggerAction.Void(), + }); + } else { + const synthetic = buildSyntheticCompression(raw); + await kv.set( + KV.observations(payload.sessionId), + obsId, + synthetic, + ); + // Stored above unconditionally; only the INDEX writes are + // skipped for excluded tools (retrieval echoes). + if (!isIndexExcluded(synthetic)) { + getSearchIndex().add(synthetic); + await vectorIndexAddGuarded( + synthetic.id, + synthetic.sessionId, + synthetic.title + " " + (synthetic.narrative || ""), + { kind: "synthetic", logId: synthetic.id }, + ); + } + await sdk.trigger({ + function_id: "stream::set", + payload: { + stream_name: STREAM.name, + group_id: STREAM.group(payload.sessionId), + item_id: obsId, + data: { type: "compressed", observation: synthetic }, + }, + }); + await sdk.trigger({ + function_id: "stream::set", + payload: { + stream_name: STREAM.name, + group_id: STREAM.viewerGroup, + item_id: obsId, + data: { + type: "compressed", + observation: synthetic, + sessionId: payload.sessionId, + }, + }, + }); + } + + logger.info("Observation captured", { + obsId, + sessionId: payload.sessionId, + hook: payload.hookType, + compress: isAutoCompressEnabled() ? "llm" : "synthetic", + }); + return { observationId: obsId }; + }); + }, + ); +} diff --git a/src/functions/reflect.ts b/src/functions/reflect.ts index 3edf9f87fd..5fae62eb8a 100644 --- a/src/functions/reflect.ts +++ b/src/functions/reflect.ts @@ -193,7 +193,7 @@ export function registerReflectFunctions( // concepts: an insight is stamped with the project it was built for (#1344). if (data?.project) { const project = data.project; - const sessions = await kv.list(KV.sessions).catch(() => []); + const sessions = await kv.list(KV.sessions); const projectSessionIds = new Set( sessions.filter((s) => s.project === project).map((s) => s.id), ); diff --git a/src/functions/replay.ts b/src/functions/replay.ts index ff51725d13..6432799b5f 100644 --- a/src/functions/replay.ts +++ b/src/functions/replay.ts @@ -75,7 +75,7 @@ function rawFromCompressed(obs: CompressedObservation): RawObservation { // A lesson is a whole sentence that opens with a trigger. A trigger found // mid-sentence would drop the subject and can invert the meaning (#1292). const LESSON_START = - /^(always|never|don'?t|do not|make sure|remember to|prefer|avoid|note:|caveat:|warning:)\s/i; + /^(always|never|don['’]?t|do not|make sure|remember to|prefer|avoid|note:|caveat:|warning:)\s/i; const LIST_MARKER = /^(?:[-*+>]|\d+[.)])\s+/; const SENTENCE_BREAK = /\n+|(?<=[.!?])\s+/; const MIN_LESSON_CHARS = 20; diff --git a/src/functions/summarize.ts b/src/functions/summarize.ts index f801769650..6b2b2f1013 100644 --- a/src/functions/summarize.ts +++ b/src/functions/summarize.ts @@ -86,9 +86,10 @@ function fitsOneChunk(counts: number[], budget: number): boolean { return payload <= budget - PROMPT_OVERHEAD_TOKENS; } -// The estimate overcounts, so a Session it already fits in one chunk needs no -// measuring: on a broker that queues tokenize behind generation, one request -// per Observation is load the summary itself has to wait behind. +// A token is never shorter than one character, so a Session whose character +// count already fits one chunk needs no measuring: on a broker that queues +// tokenize behind generation, one request per Observation is load the summary +// itself has to wait behind. async function countObservationTokens( provider: MemoryProvider, texts: string[], @@ -96,7 +97,7 @@ async function countObservationTokens( sessionId: string, ): Promise { const estimates = texts.map(estimateTokens); - if (fitsOneChunk(estimates, budget)) return estimates; + if (fitsOneChunk(texts.map((t) => t.length), budget)) return estimates; if (provider.countTokens) { try { return await mapWithConcurrency(texts, COUNT_CONCURRENCY, (t) => diff --git a/src/hooks/pre-tool-use.ts b/src/hooks/pre-tool-use.ts index a895af0a4b..0900c10fc0 100644 --- a/src/hooks/pre-tool-use.ts +++ b/src/hooks/pre-tool-use.ts @@ -13,7 +13,7 @@ function isSdkChildContext(payload: unknown): boolean { // // THIS HOOK IS A NO-OP BY DEFAULT AS OF 0.8.10 (#143). Previously it // fired /agentmemory/enrich on every Edit/Write/Read/Glob/Grep tool call -// and wrote up to 4000 chars of context to the model's next turn, which meant +// and wrote up to 4000 chars of context to stdout for the model's next turn, which meant // agentmemory was silently injecting ~1000 tokens into every tool turn // via the user's Claude Code session. On Claude Pro that burned entire // allocations in a handful of messages (@adrianricardo, #143). diff --git a/src/index.ts b/src/index.ts index b61ee8f0a2..1f1d104db9 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,678 +1,684 @@ -import { TriggerAction } from "./engine/types.js"; -import { - hydrateProcessEnvFromFile, - loadConfig, - getEnvVar, - loadEmbeddingConfig, - loadFallbackConfig, - loadClaudeBridgeConfig, - loadTeamConfig, - loadSnapshotConfig, - isGraphExtractionEnabled, - isAutoCompressEnabled, - isConsolidationEnabled, - isContextInjectionEnabled, - isDropStaleIndexEnabled, - getSqlitePath, -} from "./config.js"; -import { createInprocSdk } from "./engine/inproc/sdk.js"; -import { SqliteVectorStore } from "./engine/inproc/vectors.js"; -import { createIndexFill, registerIndexFillFunction } from "./functions/index-fill.js"; -import { registerMaintenanceFunctions } from "./functions/maintenance.js"; -import { - createProvider, - createFallbackProvider, - createEmbeddingProvider, - createImageEmbeddingProvider, -} from "./providers/index.js"; -import { StateKV } from "./state/kv.js"; -import { KV } from "./state/schema.js"; -import { runStartupMaintenance } from "./state/startup-maintenance.js"; -import { - GRAPH_INDEX_NODE_CEILING, - backfillGraphIndexes, - graphIndexesReady, - graphLegDisabled, -} from "./state/graph-indexes.js"; -import { VectorIndex } from "./state/vector-index.js"; -import { HybridSearch } from "./state/hybrid-search.js"; -import { registerPrivacyFunction } from "./functions/privacy.js"; -import { registerObserveFunction } from "./functions/observe.js"; -import { registerImageQuotaCleanup } from "./functions/image-quota-cleanup.js"; -import { registerVisionSearchFunctions } from "./functions/vision-search.js"; -import { registerSlotsFunctions, isSlotsEnabled, isReflectEnabled } from "./functions/slots.js"; -import { registerDiskSizeManager } from "./functions/disk-size-manager.js"; -import { registerCompressFunction } from "./functions/compress.js"; -import { - registerSearchFunction, - getSearchIndex, - setVectorIndex, - setEmbeddingProvider, - setInprocStores, - rebuildBm25FromContent, -} from "./functions/search.js"; -import { registerContextFunction } from "./functions/context.js"; -import { registerSummarizeFunction } from "./functions/summarize.js"; -import { registerMigrateFunction } from "./functions/migrate.js"; -import { registerFileIndexFunction } from "./functions/file-index.js"; -import { registerConsolidateFunction } from "./functions/consolidate.js"; -import { registerPatternsFunction } from "./functions/patterns.js"; -import { registerRememberFunction } from "./functions/remember.js"; -import { registerEvictFunction } from "./functions/evict.js"; -import { pruneAudit } from "./functions/audit.js"; -import { registerRelationsFunction } from "./functions/relations.js"; -import { registerTimelineFunction } from "./functions/timeline.js"; -import { registerSmartSearchFunction } from "./functions/smart-search.js"; -import { registerRecentSearchesSweepFunction } from "./functions/recent-searches-sweep.js"; -import { registerProfileFunction } from "./functions/profile.js"; -import { registerAutoForgetFunction } from "./functions/auto-forget.js"; -import { registerExportImportFunction } from "./functions/export-import.js"; -import { registerEnrichFunction } from "./functions/enrich.js"; -import { registerClaudeBridgeFunction } from "./functions/claude-bridge.js"; -import { registerGraphFunction } from "./functions/graph.js"; -import { registerGraphImportFunction } from "./functions/graph-import.js"; -import { registerConsolidationPipelineFunction } from "./functions/consolidation-pipeline.js"; -import { registerTeamFunction } from "./functions/team.js"; -import { registerGovernanceFunction } from "./functions/governance.js"; -import { registerSnapshotFunction } from "./functions/snapshot.js"; -import { registerActionsFunction } from "./functions/actions.js"; -import { registerFrontierFunction } from "./functions/frontier.js"; -import { registerLeasesFunction } from "./functions/leases.js"; -import { registerRoutinesFunction } from "./functions/routines.js"; -import { registerSignalsFunction } from "./functions/signals.js"; -import { registerCheckpointsFunction } from "./functions/checkpoints.js"; -import { registerFlowCompressFunction } from "./functions/flow-compress.js"; -import { registerMeshFunction } from "./functions/mesh.js"; -import { registerBranchAwareFunction } from "./functions/branch-aware.js"; -import { registerSentinelsFunction } from "./functions/sentinels.js"; -import { registerSketchesFunction } from "./functions/sketches.js"; -import { registerCrystallizeFunction } from "./functions/crystallize.js"; -import { registerDiagnosticsFunction } from "./functions/diagnostics.js"; -import { registerFacetsFunction } from "./functions/facets.js"; -import { registerVerifyFunction } from "./functions/verify.js"; -import { registerCascadeFunction } from "./functions/cascade.js"; -import { registerLessonsFunctions } from "./functions/lessons.js"; -import { registerObsidianExportFunction } from "./functions/obsidian-export.js"; -import { registerReflectFunctions } from "./functions/reflect.js"; -import { registerWorkingMemoryFunctions } from "./functions/working-memory.js"; -import { registerSkillExtractFunctions } from "./functions/skill-extract.js"; -import { registerSlidingWindowFunction } from "./functions/sliding-window.js"; -import { - expandQuery, - registerQueryExpansionFunction, -} from "./functions/query-expansion.js"; -import { registerTemporalGraphFunctions } from "./functions/temporal-graph.js"; -import { registerRetentionFunctions } from "./functions/retention.js"; -import { registerCompressFileFunction } from "./functions/compress-file.js"; -import { registerReplayFunctions } from "./functions/replay.js"; -import { registerApiTriggers } from "./triggers/api.js"; -import { registerEventTriggers } from "./triggers/events.js"; -import { registerMcpEndpoints } from "./mcp/server.js"; -import { getAllTools } from "./mcp/tools-registry.js"; -import { startViewerServer } from "./viewer/server.js"; -import { MetricsStore } from "./eval/metrics-store.js"; -import { DedupMap } from "./functions/dedup.js"; -import { registerHealthMonitor } from "./health/monitor.js"; -import { initMetrics } from "./telemetry/setup.js"; -import { VERSION } from "./version.js"; -import { bootLog } from "./logger.js"; -import { mkdirSync, writeFileSync, unlinkSync } from "node:fs"; -import { dirname, join } from "node:path"; -import { homedir } from "node:os"; - -// #640 + #474: record this process's pid so `agentmemory stop` and -// `agentmemory doctor` can identify the daemon. Without it they can only -// scan the REST port, which cannot tell our `node` process apart from a -// tunnel or a stray dev server holding the same port. -function workerPidfilePath(): string { - return join(homedir(), ".agentmemory", "worker.pid"); -} -function writeWorkerPidfile(): void { - try { - const p = workerPidfilePath(); - mkdirSync(dirname(p), { recursive: true }); - writeFileSync(p, `${process.pid}\n`, { encoding: "utf-8" }); - } catch { - // best-effort; stop still has the port scan fallback - } -} -function clearWorkerPidfile(): void { - try { - unlinkSync(workerPidfilePath()); - } catch {} -} - -// Top-level safety net for state invocation timeouts (issue #204). -// Under sustained write load (e.g. Claude Code hooks across many -// projects) `state::set` can occasionally exceed its 30s timeout. -// We don't want one such timeout to terminate the long-lived memory -// service — the rejection is surfaced to the relevant call site via -// .catch() where it matters; everything else is logged-and-continued. -// Throttle logs to avoid spamming on bursts. -let lastUnhandledLogAt = 0; -process.on("unhandledRejection", (reason) => { - const now = Date.now(); - if (now - lastUnhandledLogAt < 60_000) return; - lastUnhandledLogAt = now; - const r = reason as { code?: string; function_id?: string; message?: string }; - console.warn( - `[agentmemory] unhandledRejection (suppressed):`, - r?.code ? `${r.code} ${r.function_id ?? ""} ${r.message ?? ""}`.trim() : reason, - ); -}); - -async function main() { - // Fold ~/.agentmemory/.env into process.env before anything reads config - // or raw process.env. Only-if-unset, so real process.env still wins. - hydrateProcessEnvFromFile(); - - const config = loadConfig(); - const embeddingConfig = loadEmbeddingConfig(); - const fallbackConfig = loadFallbackConfig(); - - const provider = - fallbackConfig.providers.length > 0 - ? createFallbackProvider(config.provider, fallbackConfig) - : createProvider(config.provider); - - const embeddingProvider = createEmbeddingProvider(); - const imageEmbeddingProvider = createImageEmbeddingProvider(); - - const sqlitePath = getSqlitePath(); - - bootLog(`Starting worker v${VERSION}...`); - bootLog(`Engine: in-process (${sqlitePath})`); - bootLog( - `Provider: ${config.provider.provider} (${config.provider.model})`, - ); - if (embeddingProvider) { - bootLog( - `Embedding provider: ${embeddingProvider.name} (${embeddingProvider.dimensions} dims)`, - ); - } else { - bootLog(`Embedding provider: none (BM25-only mode)`); - } - if (imageEmbeddingProvider) { - bootLog( - `Image embedding provider: ${imageEmbeddingProvider.name} (${imageEmbeddingProvider.dimensions} dims) — vision-search active`, - ); - } - bootLog( - `REST API: http://localhost:${config.restPort}/agentmemory/*`, - ); - bootLog(`Streams: ws://localhost:${config.streamsPort}`); - - // The only runtime: no engine process, no worker bus. The shim binds the - // REST and stream ports itself, so a bind failure has to be fatal here - // rather than leaving a daemon up with no listeners. - const sdk = createInprocSdk({ - restPort: config.restPort, - streamsPort: config.streamsPort, - sqlitePath, - maxBodyBytes: parseInt(getEnvVar("AGENTMEMORY_MAX_BODY_BYTES") || "", 10) || undefined, - }); - await sdk.listening(); - - writeWorkerPidfile(); - - const kv = new StateKV(sdk); - const secret = getEnvVar("AGENTMEMORY_SECRET"); - const metricsStore = new MetricsStore(kv); - const dedupMap = new DedupMap(); - - const vectorIndex = embeddingProvider ? new VectorIndex() : null; - - // Vectors are rows in the same SQLite file as the state. Every map - // mutation writes its rows first and every embedding completion commits - // through the store's content-revalidating transaction (src/engine/inproc/vectors.ts). - // The store exists even without an embedding provider so content deletes - // still take their persisted vector rows with them. - const vectorStore = new SqliteVectorStore(sdk.store); - vectorIndex?.attachStore(vectorStore); - setInprocStores(sdk.store, vectorStore); - - setVectorIndex(vectorIndex); - setEmbeddingProvider(embeddingProvider); - - // The in-process Engine exposes no meter provider of its own, so metrics - // fall back to whatever the OTel global supplies. - initMetrics(undefined); - - registerPrivacyFunction(sdk); - registerObserveFunction(sdk, kv, dedupMap, config.maxObservationsPerSession); - registerImageQuotaCleanup(sdk, kv); - registerVisionSearchFunctions(sdk, kv, imageEmbeddingProvider); - if (isSlotsEnabled()) { - registerSlotsFunctions(sdk, kv); - } - registerDiskSizeManager(sdk, kv); - registerCompressFunction(sdk, kv, provider, metricsStore); - registerSearchFunction(sdk, kv); - registerContextFunction(sdk, kv, config.tokenBudget); - registerSummarizeFunction(sdk, kv, provider, metricsStore); - registerMigrateFunction(sdk, kv); - registerFileIndexFunction(sdk, kv); - registerConsolidateFunction(sdk, kv, provider); - registerPatternsFunction(sdk, kv); - registerRememberFunction(sdk, kv); - registerEvictFunction(sdk, kv); - - registerRelationsFunction(sdk, kv); - registerTimelineFunction(sdk, kv); - registerProfileFunction(sdk, kv); - registerAutoForgetFunction(sdk, kv); - registerExportImportFunction(sdk, kv); - registerEnrichFunction(sdk, kv); - - const claudeBridgeConfig = loadClaudeBridgeConfig(); - if (claudeBridgeConfig.enabled) { - registerClaudeBridgeFunction(sdk, kv, claudeBridgeConfig); - bootLog( - `Claude bridge: syncing to ${claudeBridgeConfig.memoryFilePath}`, - ); - } - - registerGraphFunction(sdk, kv, provider); - registerGraphImportFunction(sdk, kv); - bootLog( - `Knowledge graph: structural extraction on (LLM relations ${isGraphExtractionEnabled() ? "enabled" : "off"})`, - ); - - registerConsolidationPipelineFunction(sdk, kv, provider); - bootLog(`Consolidation pipeline: registered (CONSOLIDATION_ENABLED=${isConsolidationEnabled() ? "true" : "false"})`); - - if (isAutoCompressEnabled()) { - bootLog( - `WARNING: AGENTMEMORY_AUTO_COMPRESS=true — every PostToolUse observation will be sent to your LLM provider for compression. This spends API tokens proportional to your session tool-use frequency. Set AGENTMEMORY_AUTO_COMPRESS=false to disable.`, - ); - } else { - bootLog( - `Auto-compress: OFF (default) — observations indexed via zero-LLM synthetic compression. Set AGENTMEMORY_AUTO_COMPRESS=true to opt-in to LLM-powered summaries (uses your API key).`, - ); - } - - if (isContextInjectionEnabled()) { - bootLog( - `WARNING: AGENTMEMORY_INJECT_CONTEXT=true — the PreToolUse and SessionStart hooks will inject up to ~4000 chars of memory context into every tool turn. On Claude Pro this burns session tokens proportional to your tool-call frequency. Set AGENTMEMORY_INJECT_CONTEXT=false to disable.`, - ); - } else { - bootLog( - `Context injection: OFF (default) — hooks capture observations but do not inject context into Claude Code's conversation. Set AGENTMEMORY_INJECT_CONTEXT=true to opt-in (warning: expect your Claude Pro allocation to drain faster).`, - ); - } - - const teamConfig = loadTeamConfig(); - if (teamConfig) { - registerTeamFunction(sdk, kv, teamConfig); - bootLog( - `Team memory: ${teamConfig.teamId} (${teamConfig.mode})`, - ); - } - - registerGovernanceFunction(sdk, kv); - - registerActionsFunction(sdk, kv); - registerFrontierFunction(sdk, kv); - registerLeasesFunction(sdk, kv); - registerRoutinesFunction(sdk, kv); - registerSignalsFunction(sdk, kv); - registerCheckpointsFunction(sdk, kv); - registerMeshFunction(sdk, kv, secret); - registerBranchAwareFunction(sdk, kv); - registerFlowCompressFunction(sdk, kv, provider); - registerSentinelsFunction(sdk, kv); - registerSketchesFunction(sdk, kv); - registerCrystallizeFunction(sdk, kv, provider); - registerDiagnosticsFunction(sdk, kv); - registerFacetsFunction(sdk, kv); - registerVerifyFunction(sdk, kv); - registerLessonsFunctions(sdk, kv); - registerObsidianExportFunction(sdk, kv); - registerReflectFunctions(sdk, kv, provider); - registerWorkingMemoryFunctions(sdk, kv, config.tokenBudget); - registerSkillExtractFunctions(sdk, kv, provider); - registerCascadeFunction(sdk, kv); - - registerSlidingWindowFunction(sdk, kv, provider); - registerQueryExpansionFunction(sdk, provider); - registerTemporalGraphFunctions(sdk, kv, provider); - registerRetentionFunctions(sdk, kv); - registerCompressFileFunction(sdk, kv, provider); - registerReplayFunctions(sdk, kv); - bootLog( - `v0.6 advanced retrieval: sliding-window, query-expansion, temporal-graph, retention-scoring`, - ); - bootLog( - `Orchestration layer: actions, frontier, leases, routines, signals, checkpoints, flow-compress, mesh, branch-aware, sentinels, sketches, crystallize, diagnostics, facets`, - ); - if (isSlotsEnabled()) { - bootLog( - `Slots: enabled (pinned editable memory). Reflect on Stop hook: ${isReflectEnabled() ? "on" : "off"}`, - ); - } - - const snapshotConfig = loadSnapshotConfig(); - if (snapshotConfig.enabled) { - registerSnapshotFunction(sdk, kv, snapshotConfig.dir); - // The boot line promised "every s" but nothing ever fired - // mem::snapshot-create. Drive it on a periodic timer (unref'd so it - // never keeps the process alive), mirroring the auto-forget timer. - // mem::snapshot-create serializes overlapping runs internally (git-lock - // safety), so the timer can stay a simple fire-and-forget tick. - const snapshotTimer = setInterval(() => { - sdk - .trigger({ - function_id: "mem::snapshot-create", - payload: {}, - action: TriggerAction.Void(), - }) - .catch(() => {}); - }, snapshotConfig.interval * 1000); - snapshotTimer.unref(); - bootLog( - `Git snapshots: ${snapshotConfig.dir} (every ${snapshotConfig.interval}s)`, - ); - } - - const bm25Index = getSearchIndex(); - const graphWeight = parseFloat(getEnvVar("AGENTMEMORY_GRAPH_WEIGHT") || "0.3"); - const hybridSearch = new HybridSearch( - bm25Index, - vectorIndex, - embeddingProvider, - kv, - embeddingConfig.bm25Weight, - embeddingConfig.vectorWeight, - graphWeight, - ); - - // `searchWithExpansion` existed as dead code since the fork point. Probe 0 - // measured why it is worth waking up: the two frozen-fixture targets that - // never reached the candidate pool sit at combined rank 422 and 261 when the - // pool is opened to depth 4000, so BOTH retrieval legs rank them past 100 and - // neither a deeper pool nor a weight change would seat them in a 20-slot - // head. The gap is vocabulary — the query and the memory share almost no - // surface — which is the one thing reformulation addresses. - // - // OFF by default: it costs one LLM round-trip plus N extra searches per - // query. Affordable here (single-user, a handful of searches a day) but not a - // cost to impose on every deployment silently. - const queryExpansionEnabled = - getEnvVar("AGENTMEMORY_QUERY_EXPANSION") === "true"; - registerSmartSearchFunction(sdk, kv, async (query, limit) => - queryExpansionEnabled - ? hybridSearch.searchWithExpansion( - query, - limit, - await expandQuery(provider, query), - ) - : hybridSearch.search(query, limit), - ); - registerRecentSearchesSweepFunction(sdk, kv); - - registerApiTriggers(sdk, kv, secret, metricsStore, provider); - registerEventTriggers(sdk, kv); - registerMcpEndpoints(sdk, kv, secret, metricsStore); - - const healthMonitor = registerHealthMonitor(sdk, kv); - - if (vectorIndex) { - const hydrated = vectorStore.hydrate(vectorIndex); - if (hydrated > 0) bootLog(`Hydrated ${hydrated} vectors from SQLite`); - } - // The dimension guard runs over the rows just hydrated. - const vectorSource = vectorIndex; - if (vectorSource && vectorIndex && vectorSource.size > 0) { - // Persisted vectors carry whatever dimension the provider had when - // they were written. If the active provider declares a different - // dimension — or if the on-disk index contains a mix of dimensions - // (legacy indexes written before the live-API guard in this PR) — - // restoring would silently corrupt search: cosineSimilarity returns - // 0 on cross-dim pairs, so affected observations stop matching - // anything and recall degrades without an error. Walk every stored - // vector instead of trusting the first; refuse to load if anything - // is off. - const activeDim = embeddingProvider?.dimensions ?? 0; - const { mismatches, seenDimensions } = - activeDim > 0 - ? vectorSource.validateDimensions(activeDim) - : { mismatches: [], seenDimensions: new Set() }; - - if (mismatches.length > 0) { - const sample = mismatches - .slice(0, 5) - .map((m) => `${m.obsId} (dim=${m.dim})`) - .join(", "); - const distinct = Array.from(seenDimensions).sort((a, b) => a - b).join(", "); - const dropStale = isDropStaleIndexEnabled(); - if (dropStale) { - console.warn( - `[agentmemory] Persisted vector index has ${mismatches.length} of ` + - `${vectorSource.size} vectors with the wrong dimension. Active ` + - `provider (${embeddingProvider?.name}) declares ${activeDim}; ` + - `dimensions seen on disk: ${distinct}. ` + - `AGENTMEMORY_DROP_STALE_INDEX=true is set — discarding the persisted ` + - `vectors. Live observations will rebuild the index over time.`, - ); - // The rows ARE the index, so discarding means deleting them. - vectorIndex.clear(); - } else { - throw new Error( - `[agentmemory] Refusing to start: persisted vector index has ` + - `${mismatches.length} of ${vectorSource.size} vectors with the ` + - `wrong dimension. Active provider (${embeddingProvider?.name}) ` + - `declares ${activeDim}; dimensions seen on disk: ${distinct}. ` + - `First mismatched obsIds: ${sample}. Loading would silently corrupt ` + - `search (cross-dimension cosine returns 0). Choose one:\n` + - ` - Re-embed the existing index against the new provider, then start.\n` + - ` - Set AGENTMEMORY_DROP_STALE_INDEX=true to discard the persisted ` + - `vectors and rebuild from live observations.\n` + - ` - Switch the embedding provider back to the one that wrote the index.`, - ); - } - } - } - - // BM25 is not persisted: it is rebuilt from the content rows on every boot - // - one ordered walk, vectors untouched (they hydrated above). - { - const t0 = performance.now(); - const r = await rebuildBm25FromContent(sdk.store.db); - bootLog( - `BM25 rebuilt from content: ${r.docs} docs of ${r.rows} rows in ` + - `${Math.round(performance.now() - t0)} ms (read ${Math.round(r.readMs)} ms, ` + - `index ${Math.round(r.indexMs)} ms${r.skipped ? `, ${r.skipped} rows skipped` : ""})`, - ); - } - - // Maintenance routes: mem::backup, and mem::index-debug-legs when - // AGENTMEMORY_INDEX_DEBUG=1. Registered before the gate opens so they - // exist the moment readyz answers 200. - registerMaintenanceFunctions(sdk, sdk.store); - - // Readiness: DB opened, a write probe round-tripped, vectors hydrated, - // BM25 rebuilt -> readyz 200 and the 503 gate opens. The probe uses the - // same key the health monitor writes. A failure here is fatal on purpose: - // systemd restarts a daemon that cannot write its store. - { - const stamp = Date.now(); - sdk.store.set(KV.health, "_probe", { ts: stamp }); - const back = sdk.store.get(KV.health, "_probe") as { ts?: number } | null; - if (back?.ts !== stamp) throw new Error("readiness write probe did not round-trip"); - sdk.setReady(); - bootLog("Ready: /agentmemory/readyz -> 200, routes open"); - } - - // One-time repair of stores written before this fork: graph provenance that - // predates the write-time bound, and the index shards the removed engine - // left behind. Not awaited — readyz already answers 200 — and the pass - // itself yields between chunks, so requests arriving during it are served - // rather than queued. It records its own version, so every later boot is a - // single marker read. - void runStartupMaintenance(sdk.store).catch((err: unknown) => { - console.warn( - `[agentmemory] startup maintenance skipped: ${err instanceof Error ? err.message : String(err)}`, - ); - }); - - // The vector fill/repair pass closes the gap between the content rows and - // the vectors table - missing or stale rows are embedded, orphans pruned. - // Runs once at boot after readiness (not awaited: a large backlog after an - // import must not hold up the listener) and hourly. - if (vectorIndex) { - const fill = createIndexFill(sdk.store, vectorStore, vectorIndex); - registerIndexFillFunction(sdk, fill); - void fill.run(); - setInterval(() => void fill.run(), 3_600_000).unref(); - bootLog("Vector fill pass: at boot and hourly (mem::index-fill-missing)"); - } - - // Backfill the graph read side-indexes for corpora that predate them. - // Mirrors the BM25 memories backfill above: one-time, gated on the - // snapshot's recorded node count so we never enumerate a corpus large - // enough to starve the worker heartbeat. - // - // graph-read-fix local delta (B-mode boot-skip): when the leg is off - // this block is skipped ENTIRELY — it makes the only graph-scope reads - // at boot (the kv.list arming enumeration), and B-mode must issue zero. - // Skipping is safe: with the marker absent the readers fail-closed - // (graphReadable === false) rather than enumerating. - if (!graphLegDisabled()) { - try { - if (!(await graphIndexesReady(kv))) { - const graphSnap = await kv.get( - KV.graphSnapshot, - "current", - ); - const totalNodes = graphSnap?.stats?.totalNodes ?? 0; - if (graphSnap && totalNodes > 0 && totalNodes <= GRAPH_INDEX_NODE_CEILING) { - const [graphNodes, graphEdges] = await Promise.all([ - kv.list(KV.graphNodes), - kv.list(KV.graphEdges), - ]); - await backfillGraphIndexes( - kv, - graphNodes.filter((n) => !n.stale), - graphEdges.filter((e) => !e.stale), - ); - bootLog(`Backfilled graph read indexes (${totalNodes} nodes)`); - } - } - } catch (err) { - console.warn(`[agentmemory] Failed to backfill graph indexes:`, err); - } - } - - // Ready / Endpoints lines are emitted via `bootLog` so they're - // buffered in quiet mode and printed verbatim under --verbose. The - // CLI surfaces a compact summary when it sees the worker reach - // ready state. - bootLog( - `Ready. ${embeddingProvider ? "Triple-stream (BM25+Vector+Graph)" : "BM25+Graph"} search active.`, - ); - bootLog( - `REST API: 132 endpoints at http://localhost:${config.restPort}/agentmemory/*`, - ); - bootLog( - `MCP surface (opt-in via \`npx @agentmemory/mcp\`): ${getAllTools().length} tools · 6 resources · 3 prompts`, - ); - - const viewerPort = config.restPort + 2; - const viewerServer = startViewerServer( - viewerPort, - kv, - sdk, - secret, - config.restPort, - ); - - const autoForgetIntervalMs = parseInt(process.env.AUTO_FORGET_INTERVAL_MS || "3600000", 10); - const consolidationIntervalMs = parseInt(process.env.CONSOLIDATION_INTERVAL_MS || "7200000", 10); - - if (process.env.AUTO_FORGET_ENABLED !== "false") { - const autoForgetTimer = setInterval(async () => { - try { - await sdk.trigger({ function_id: "mem::auto-forget", payload: { dryRun: false } }); - } catch {} - }, autoForgetIntervalMs); - autoForgetTimer.unref(); - bootLog(`Auto-forget: enabled (every ${autoForgetIntervalMs / 60000}m)`); - } - - if (process.env.LESSON_DECAY_ENABLED !== "false") { - const lessonDecayTimer = setInterval(async () => { - try { - await sdk.trigger({ function_id: "mem::lesson-decay-sweep", payload: {} }); - } catch {} - }, 86400000); - lessonDecayTimer.unref(); - bootLog(`Lesson decay sweep: enabled (every 24h)`); - } - - if (process.env.EVICTION_ENABLED !== "false") { - const evictionTimer = setInterval(async () => { - try { - await sdk.trigger({ function_id: "mem::evict", payload: { dryRun: false } }); - } catch {} - }, 86400000); - evictionTimer.unref(); - bootLog(`Eviction sweep: enabled (every 24h)`); - } - - const auditPruneTimer = setInterval(async () => { - try { - await pruneAudit(kv); - } catch {} - }, 86400000); - auditPruneTimer.unref(); - - if (process.env.INSIGHT_DECAY_ENABLED !== "false") { - const insightDecayTimer = setInterval(async () => { - try { - await sdk.trigger({ function_id: "mem::insight-decay-sweep", payload: {} }); - } catch {} - }, 86400000); - insightDecayTimer.unref(); - } - - // #771: hourly TTL sweep for the followup-rate diagnostic. The - // recent-searches scope only needs the last entry per session; - // sweeping anything older than the retention window keeps the scope - // from growing unbounded across long-lived deployments. - const recentSearchesSweepTimer = setInterval(async () => { - try { - await sdk.trigger({ - function_id: "mem::diagnostic::recent-searches-sweep", - payload: {}, - }); - } catch {} - }, 60 * 60 * 1000); - recentSearchesSweepTimer.unref(); - - if (isConsolidationEnabled()) { - const consolidationTimer = setInterval(async () => { - try { - await sdk.trigger({ function_id: "mem::consolidate-pipeline", payload: {} }); - } catch {} - }, consolidationIntervalMs); - consolidationTimer.unref(); - bootLog(`Auto-consolidation: enabled (every ${consolidationIntervalMs / 60000}m)`); - } - - const shutdown = async () => { - console.log(`\n[agentmemory] Shutting down...`); - healthMonitor.stop(); - dedupMap.stop(); - await new Promise((resolve) => viewerServer.close(() => resolve())); - await sdk.shutdown(); - clearWorkerPidfile(); - process.exit(0); - }; - process.on("SIGINT", shutdown); - process.on("SIGTERM", shutdown); -} - -main().catch((err) => { - console.error(`[agentmemory] Fatal:`, err); - process.exit(1); -}); +import { TriggerAction } from "./engine/types.js"; +import { + hydrateProcessEnvFromFile, + loadConfig, + getEnvVar, + loadEmbeddingConfig, + loadFallbackConfig, + loadClaudeBridgeConfig, + loadTeamConfig, + loadSnapshotConfig, + isGraphExtractionEnabled, + isAutoCompressEnabled, + isConsolidationEnabled, + isContextInjectionEnabled, + isDropStaleIndexEnabled, + getSqlitePath, +} from "./config.js"; +import { createInprocSdk } from "./engine/inproc/sdk.js"; +import { SqliteVectorStore } from "./engine/inproc/vectors.js"; +import { createIndexFill, registerIndexFillFunction } from "./functions/index-fill.js"; +import { registerMaintenanceFunctions } from "./functions/maintenance.js"; +import { + createProvider, + createFallbackProvider, + createEmbeddingProvider, + createImageEmbeddingProvider, +} from "./providers/index.js"; +import { StateKV } from "./state/kv.js"; +import { KV } from "./state/schema.js"; +import { runStartupMaintenance } from "./state/startup-maintenance.js"; +import { + GRAPH_INDEX_NODE_CEILING, + backfillGraphIndexes, + graphIndexesReady, + graphLegDisabled, +} from "./state/graph-indexes.js"; +import { VectorIndex } from "./state/vector-index.js"; +import { HybridSearch } from "./state/hybrid-search.js"; +import { registerPrivacyFunction } from "./functions/privacy.js"; +import { registerObserveFunction } from "./functions/observe.js"; +import { registerImageQuotaCleanup } from "./functions/image-quota-cleanup.js"; +import { registerVisionSearchFunctions } from "./functions/vision-search.js"; +import { registerSlotsFunctions, isSlotsEnabled, isReflectEnabled } from "./functions/slots.js"; +import { registerDiskSizeManager } from "./functions/disk-size-manager.js"; +import { registerCompressFunction } from "./functions/compress.js"; +import { + registerSearchFunction, + getSearchIndex, + setVectorIndex, + setEmbeddingProvider, + setInprocStores, + rebuildBm25FromContent, +} from "./functions/search.js"; +import { registerContextFunction } from "./functions/context.js"; +import { registerSummarizeFunction } from "./functions/summarize.js"; +import { registerMigrateFunction } from "./functions/migrate.js"; +import { registerFileIndexFunction } from "./functions/file-index.js"; +import { registerConsolidateFunction } from "./functions/consolidate.js"; +import { registerPatternsFunction } from "./functions/patterns.js"; +import { registerRememberFunction } from "./functions/remember.js"; +import { registerEvictFunction } from "./functions/evict.js"; +import { evictOldestAudit } from "./functions/audit.js"; +import { registerRelationsFunction } from "./functions/relations.js"; +import { registerTimelineFunction } from "./functions/timeline.js"; +import { registerSmartSearchFunction } from "./functions/smart-search.js"; +import { registerRecentSearchesSweepFunction } from "./functions/recent-searches-sweep.js"; +import { registerProfileFunction } from "./functions/profile.js"; +import { registerAutoForgetFunction } from "./functions/auto-forget.js"; +import { registerExportImportFunction } from "./functions/export-import.js"; +import { registerEnrichFunction } from "./functions/enrich.js"; +import { registerClaudeBridgeFunction } from "./functions/claude-bridge.js"; +import { registerGraphFunction } from "./functions/graph.js"; +import { registerGraphImportFunction } from "./functions/graph-import.js"; +import { registerConsolidationPipelineFunction } from "./functions/consolidation-pipeline.js"; +import { registerTeamFunction } from "./functions/team.js"; +import { registerGovernanceFunction } from "./functions/governance.js"; +import { registerSnapshotFunction } from "./functions/snapshot.js"; +import { registerActionsFunction } from "./functions/actions.js"; +import { registerFrontierFunction } from "./functions/frontier.js"; +import { registerLeasesFunction } from "./functions/leases.js"; +import { registerRoutinesFunction } from "./functions/routines.js"; +import { registerSignalsFunction } from "./functions/signals.js"; +import { registerCheckpointsFunction } from "./functions/checkpoints.js"; +import { registerFlowCompressFunction } from "./functions/flow-compress.js"; +import { registerMeshFunction } from "./functions/mesh.js"; +import { registerBranchAwareFunction } from "./functions/branch-aware.js"; +import { registerSentinelsFunction } from "./functions/sentinels.js"; +import { registerSketchesFunction } from "./functions/sketches.js"; +import { registerCrystallizeFunction } from "./functions/crystallize.js"; +import { registerDiagnosticsFunction } from "./functions/diagnostics.js"; +import { registerFacetsFunction } from "./functions/facets.js"; +import { registerVerifyFunction } from "./functions/verify.js"; +import { registerCascadeFunction } from "./functions/cascade.js"; +import { registerLessonsFunctions } from "./functions/lessons.js"; +import { registerObsidianExportFunction } from "./functions/obsidian-export.js"; +import { registerReflectFunctions } from "./functions/reflect.js"; +import { registerWorkingMemoryFunctions } from "./functions/working-memory.js"; +import { registerSkillExtractFunctions } from "./functions/skill-extract.js"; +import { registerSlidingWindowFunction } from "./functions/sliding-window.js"; +import { + expandQuery, + registerQueryExpansionFunction, +} from "./functions/query-expansion.js"; +import { registerTemporalGraphFunctions } from "./functions/temporal-graph.js"; +import { registerRetentionFunctions } from "./functions/retention.js"; +import { registerCompressFileFunction } from "./functions/compress-file.js"; +import { registerReplayFunctions } from "./functions/replay.js"; +import { registerApiTriggers } from "./triggers/api.js"; +import { registerEventTriggers } from "./triggers/events.js"; +import { registerMcpEndpoints } from "./mcp/server.js"; +import { getAllTools } from "./mcp/tools-registry.js"; +import { startViewerServer } from "./viewer/server.js"; +import { MetricsStore } from "./eval/metrics-store.js"; +import { DedupMap } from "./functions/dedup.js"; +import { registerHealthMonitor } from "./health/monitor.js"; +import { initMetrics } from "./telemetry/setup.js"; +import { VERSION } from "./version.js"; +import { bootLog } from "./logger.js"; +import { mkdirSync, writeFileSync, unlinkSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { homedir } from "node:os"; + +const FIRST_SWEEP_DELAY_MS = 5 * 60_000; + +// #640 + #474: record this process's pid so `agentmemory stop` and +// `agentmemory doctor` can identify the daemon. Without it they can only +// scan the REST port, which cannot tell our `node` process apart from a +// tunnel or a stray dev server holding the same port. +function workerPidfilePath(): string { + return join(homedir(), ".agentmemory", "worker.pid"); +} +function writeWorkerPidfile(): void { + try { + const p = workerPidfilePath(); + mkdirSync(dirname(p), { recursive: true }); + writeFileSync(p, `${process.pid}\n`, { encoding: "utf-8" }); + } catch { + // best-effort; stop still has the port scan fallback + } +} +function clearWorkerPidfile(): void { + try { + unlinkSync(workerPidfilePath()); + } catch {} +} + +// Top-level safety net for state invocation timeouts (issue #204). +// Under sustained write load (e.g. Claude Code hooks across many +// projects) `state::set` can occasionally exceed its 30s timeout. +// We don't want one such timeout to terminate the long-lived memory +// service — the rejection is surfaced to the relevant call site via +// .catch() where it matters; everything else is logged-and-continued. +// Throttle logs to avoid spamming on bursts. +let lastUnhandledLogAt = 0; +process.on("unhandledRejection", (reason) => { + const now = Date.now(); + if (now - lastUnhandledLogAt < 60_000) return; + lastUnhandledLogAt = now; + const r = reason as { code?: string; function_id?: string; message?: string }; + console.warn( + `[agentmemory] unhandledRejection (suppressed):`, + r?.code ? `${r.code} ${r.function_id ?? ""} ${r.message ?? ""}`.trim() : reason, + ); +}); + +async function main() { + // Fold ~/.agentmemory/.env into process.env before anything reads config + // or raw process.env. Only-if-unset, so real process.env still wins. + hydrateProcessEnvFromFile(); + + const config = loadConfig(); + const embeddingConfig = loadEmbeddingConfig(); + const fallbackConfig = loadFallbackConfig(); + + const provider = + fallbackConfig.providers.length > 0 + ? createFallbackProvider(config.provider, fallbackConfig) + : createProvider(config.provider); + + const embeddingProvider = createEmbeddingProvider(); + const imageEmbeddingProvider = createImageEmbeddingProvider(); + + const sqlitePath = getSqlitePath(); + + bootLog(`Starting worker v${VERSION}...`); + bootLog(`Engine: in-process (${sqlitePath})`); + bootLog( + `Provider: ${config.provider.provider} (${config.provider.model})`, + ); + if (embeddingProvider) { + bootLog( + `Embedding provider: ${embeddingProvider.name} (${embeddingProvider.dimensions} dims)`, + ); + } else { + bootLog(`Embedding provider: none (BM25-only mode)`); + } + if (imageEmbeddingProvider) { + bootLog( + `Image embedding provider: ${imageEmbeddingProvider.name} (${imageEmbeddingProvider.dimensions} dims) — vision-search active`, + ); + } + bootLog( + `REST API: http://localhost:${config.restPort}/agentmemory/*`, + ); + bootLog(`Streams: ws://localhost:${config.streamsPort}`); + + // The only runtime: no engine process, no worker bus. The shim binds the + // REST and stream ports itself, so a bind failure has to be fatal here + // rather than leaving a daemon up with no listeners. + const sdk = createInprocSdk({ + restPort: config.restPort, + streamsPort: config.streamsPort, + sqlitePath, + maxBodyBytes: parseInt(getEnvVar("AGENTMEMORY_MAX_BODY_BYTES") || "", 10) || undefined, + }); + await sdk.listening(); + + writeWorkerPidfile(); + + const kv = new StateKV(sdk); + const secret = getEnvVar("AGENTMEMORY_SECRET"); + const metricsStore = new MetricsStore(kv); + const dedupMap = new DedupMap(); + + const vectorIndex = embeddingProvider ? new VectorIndex() : null; + + // Vectors are rows in the same SQLite file as the state. Every map + // mutation writes its rows first and every embedding completion commits + // through the store's content-revalidating transaction (src/engine/inproc/vectors.ts). + // The store exists even without an embedding provider so content deletes + // still take their persisted vector rows with them. + const vectorStore = new SqliteVectorStore(sdk.store); + vectorIndex?.attachStore(vectorStore); + setInprocStores(sdk.store, vectorStore); + + setVectorIndex(vectorIndex); + setEmbeddingProvider(embeddingProvider); + + // The in-process Engine exposes no meter provider of its own, so metrics + // fall back to whatever the OTel global supplies. + initMetrics(undefined); + + registerPrivacyFunction(sdk); + registerObserveFunction(sdk, kv, dedupMap, config.maxObservationsPerSession); + registerImageQuotaCleanup(sdk, kv); + registerVisionSearchFunctions(sdk, kv, imageEmbeddingProvider); + if (isSlotsEnabled()) { + registerSlotsFunctions(sdk, kv); + } + registerDiskSizeManager(sdk, kv); + registerCompressFunction(sdk, kv, provider, metricsStore); + registerSearchFunction(sdk, kv); + registerContextFunction(sdk, kv, config.tokenBudget); + registerSummarizeFunction(sdk, kv, provider, metricsStore); + registerMigrateFunction(sdk, kv); + registerFileIndexFunction(sdk, kv); + registerConsolidateFunction(sdk, kv, provider); + registerPatternsFunction(sdk, kv); + registerRememberFunction(sdk, kv); + registerEvictFunction(sdk, kv); + + registerRelationsFunction(sdk, kv); + registerTimelineFunction(sdk, kv); + registerProfileFunction(sdk, kv); + registerAutoForgetFunction(sdk, kv); + registerExportImportFunction(sdk, kv); + registerEnrichFunction(sdk, kv); + + const claudeBridgeConfig = loadClaudeBridgeConfig(); + if (claudeBridgeConfig.enabled) { + registerClaudeBridgeFunction(sdk, kv, claudeBridgeConfig); + bootLog( + `Claude bridge: syncing to ${claudeBridgeConfig.memoryFilePath}`, + ); + } + + registerGraphFunction(sdk, kv, provider); + registerGraphImportFunction(sdk, kv); + bootLog( + `Knowledge graph: structural extraction on (LLM relations ${isGraphExtractionEnabled() ? "enabled" : "off"})`, + ); + + registerConsolidationPipelineFunction(sdk, kv, provider); + bootLog(`Consolidation pipeline: registered (CONSOLIDATION_ENABLED=${isConsolidationEnabled() ? "true" : "false"})`); + + if (isAutoCompressEnabled()) { + bootLog( + `WARNING: AGENTMEMORY_AUTO_COMPRESS=true — every PostToolUse observation will be sent to your LLM provider for compression. This spends API tokens proportional to your session tool-use frequency. Set AGENTMEMORY_AUTO_COMPRESS=false to disable.`, + ); + } else { + bootLog( + `Auto-compress: OFF (default) — observations indexed via zero-LLM synthetic compression. Set AGENTMEMORY_AUTO_COMPRESS=true to opt-in to LLM-powered summaries (uses your API key).`, + ); + } + + if (isContextInjectionEnabled()) { + bootLog( + `WARNING: AGENTMEMORY_INJECT_CONTEXT=true — the PreToolUse and SessionStart hooks will inject up to ~4000 chars of memory context into every tool turn. On Claude Pro this burns session tokens proportional to your tool-call frequency. Set AGENTMEMORY_INJECT_CONTEXT=false to disable.`, + ); + } else { + bootLog( + `Context injection: OFF (default) — hooks capture observations but do not inject context into Claude Code's conversation. Set AGENTMEMORY_INJECT_CONTEXT=true to opt-in (warning: expect your Claude Pro allocation to drain faster).`, + ); + } + + const teamConfig = loadTeamConfig(); + if (teamConfig) { + registerTeamFunction(sdk, kv, teamConfig); + bootLog( + `Team memory: ${teamConfig.teamId} (${teamConfig.mode})`, + ); + } + + registerGovernanceFunction(sdk, kv); + + registerActionsFunction(sdk, kv); + registerFrontierFunction(sdk, kv); + registerLeasesFunction(sdk, kv); + registerRoutinesFunction(sdk, kv); + registerSignalsFunction(sdk, kv); + registerCheckpointsFunction(sdk, kv); + registerMeshFunction(sdk, kv, secret); + registerBranchAwareFunction(sdk, kv); + registerFlowCompressFunction(sdk, kv, provider); + registerSentinelsFunction(sdk, kv); + registerSketchesFunction(sdk, kv); + registerCrystallizeFunction(sdk, kv, provider); + registerDiagnosticsFunction(sdk, kv); + registerFacetsFunction(sdk, kv); + registerVerifyFunction(sdk, kv); + registerLessonsFunctions(sdk, kv); + registerObsidianExportFunction(sdk, kv); + registerReflectFunctions(sdk, kv, provider); + registerWorkingMemoryFunctions(sdk, kv, config.tokenBudget); + registerSkillExtractFunctions(sdk, kv, provider); + registerCascadeFunction(sdk, kv); + + registerSlidingWindowFunction(sdk, kv, provider); + registerQueryExpansionFunction(sdk, provider); + registerTemporalGraphFunctions(sdk, kv, provider); + registerRetentionFunctions(sdk, kv); + registerCompressFileFunction(sdk, kv, provider); + registerReplayFunctions(sdk, kv); + bootLog( + `v0.6 advanced retrieval: sliding-window, query-expansion, temporal-graph, retention-scoring`, + ); + bootLog( + `Orchestration layer: actions, frontier, leases, routines, signals, checkpoints, flow-compress, mesh, branch-aware, sentinels, sketches, crystallize, diagnostics, facets`, + ); + if (isSlotsEnabled()) { + bootLog( + `Slots: enabled (pinned editable memory). Reflect on Stop hook: ${isReflectEnabled() ? "on" : "off"}`, + ); + } + + const snapshotConfig = loadSnapshotConfig(); + if (snapshotConfig.enabled) { + registerSnapshotFunction(sdk, kv, snapshotConfig.dir); + // The boot line promised "every s" but nothing ever fired + // mem::snapshot-create. Drive it on a periodic timer (unref'd so it + // never keeps the process alive), mirroring the auto-forget timer. + // mem::snapshot-create serializes overlapping runs internally (git-lock + // safety), so the timer can stay a simple fire-and-forget tick. + const snapshotTimer = setInterval(() => { + sdk + .trigger({ + function_id: "mem::snapshot-create", + payload: {}, + action: TriggerAction.Void(), + }) + .catch(() => {}); + }, snapshotConfig.interval * 1000); + snapshotTimer.unref(); + bootLog( + `Git snapshots: ${snapshotConfig.dir} (every ${snapshotConfig.interval}s)`, + ); + } + + const bm25Index = getSearchIndex(); + const graphWeight = parseFloat(getEnvVar("AGENTMEMORY_GRAPH_WEIGHT") || "0.3"); + const hybridSearch = new HybridSearch( + bm25Index, + vectorIndex, + embeddingProvider, + kv, + embeddingConfig.bm25Weight, + embeddingConfig.vectorWeight, + graphWeight, + ); + + // `searchWithExpansion` existed as dead code since the fork point. Probe 0 + // measured why it is worth waking up: the two frozen-fixture targets that + // never reached the candidate pool sit at combined rank 422 and 261 when the + // pool is opened to depth 4000, so BOTH retrieval legs rank them past 100 and + // neither a deeper pool nor a weight change would seat them in a 20-slot + // head. The gap is vocabulary — the query and the memory share almost no + // surface — which is the one thing reformulation addresses. + // + // OFF by default: it costs one LLM round-trip plus N extra searches per + // query. Affordable here (single-user, a handful of searches a day) but not a + // cost to impose on every deployment silently. + const queryExpansionEnabled = + getEnvVar("AGENTMEMORY_QUERY_EXPANSION") === "true"; + registerSmartSearchFunction(sdk, kv, async (query, limit) => + queryExpansionEnabled + ? hybridSearch.searchWithExpansion( + query, + limit, + await expandQuery(provider, query), + ) + : hybridSearch.search(query, limit), + ); + registerRecentSearchesSweepFunction(sdk, kv); + + registerApiTriggers(sdk, kv, secret, metricsStore, provider); + registerEventTriggers(sdk, kv); + registerMcpEndpoints(sdk, kv, secret, metricsStore); + + const healthMonitor = registerHealthMonitor(sdk, kv); + + if (vectorIndex) { + const hydrated = vectorStore.hydrate(vectorIndex); + if (hydrated > 0) bootLog(`Hydrated ${hydrated} vectors from SQLite`); + } + // The dimension guard runs over the rows just hydrated. + const vectorSource = vectorIndex; + if (vectorSource && vectorIndex && vectorSource.size > 0) { + // Persisted vectors carry whatever dimension the provider had when + // they were written. If the active provider declares a different + // dimension — or if the on-disk index contains a mix of dimensions + // (legacy indexes written before the live-API guard in this PR) — + // restoring would silently corrupt search: cosineSimilarity returns + // 0 on cross-dim pairs, so affected observations stop matching + // anything and recall degrades without an error. Walk every stored + // vector instead of trusting the first; refuse to load if anything + // is off. + const activeDim = embeddingProvider?.dimensions ?? 0; + const { mismatches, seenDimensions } = + activeDim > 0 + ? vectorSource.validateDimensions(activeDim) + : { mismatches: [], seenDimensions: new Set() }; + + if (mismatches.length > 0) { + const sample = mismatches + .slice(0, 5) + .map((m) => `${m.obsId} (dim=${m.dim})`) + .join(", "); + const distinct = Array.from(seenDimensions).sort((a, b) => a - b).join(", "); + const dropStale = isDropStaleIndexEnabled(); + if (dropStale) { + console.warn( + `[agentmemory] Persisted vector index has ${mismatches.length} of ` + + `${vectorSource.size} vectors with the wrong dimension. Active ` + + `provider (${embeddingProvider?.name}) declares ${activeDim}; ` + + `dimensions seen on disk: ${distinct}. ` + + `AGENTMEMORY_DROP_STALE_INDEX=true is set — discarding the persisted ` + + `vectors. Live observations will rebuild the index over time.`, + ); + // The rows ARE the index, so discarding means deleting them. + vectorIndex.clear(); + } else { + throw new Error( + `[agentmemory] Refusing to start: persisted vector index has ` + + `${mismatches.length} of ${vectorSource.size} vectors with the ` + + `wrong dimension. Active provider (${embeddingProvider?.name}) ` + + `declares ${activeDim}; dimensions seen on disk: ${distinct}. ` + + `First mismatched obsIds: ${sample}. Loading would silently corrupt ` + + `search (cross-dimension cosine returns 0). Choose one:\n` + + ` - Re-embed the existing index against the new provider, then start.\n` + + ` - Set AGENTMEMORY_DROP_STALE_INDEX=true to discard the persisted ` + + `vectors and rebuild from live observations.\n` + + ` - Switch the embedding provider back to the one that wrote the index.`, + ); + } + } + } + + // BM25 is not persisted: it is rebuilt from the content rows on every boot + // - one ordered walk, vectors untouched (they hydrated above). + { + const t0 = performance.now(); + const r = await rebuildBm25FromContent(sdk.store.db); + bootLog( + `BM25 rebuilt from content: ${r.docs} docs of ${r.rows} rows in ` + + `${Math.round(performance.now() - t0)} ms (read ${Math.round(r.readMs)} ms, ` + + `index ${Math.round(r.indexMs)} ms${r.skipped ? `, ${r.skipped} rows skipped` : ""})`, + ); + } + + // Maintenance routes: mem::backup, and mem::index-debug-legs when + // AGENTMEMORY_INDEX_DEBUG=1. Registered before the gate opens so they + // exist the moment readyz answers 200. + registerMaintenanceFunctions(sdk, sdk.store); + + // Readiness: DB opened, a write probe round-tripped, vectors hydrated, + // BM25 rebuilt -> readyz 200 and the 503 gate opens. The probe uses the + // same key the health monitor writes. A failure here is fatal on purpose: + // systemd restarts a daemon that cannot write its store. + { + const stamp = Date.now(); + sdk.store.set(KV.health, "_probe", { ts: stamp }); + const back = sdk.store.get(KV.health, "_probe") as { ts?: number } | null; + if (back?.ts !== stamp) throw new Error("readiness write probe did not round-trip"); + sdk.setReady(); + bootLog("Ready: /agentmemory/readyz -> 200, routes open"); + } + + // One-time repair of stores written before this fork: graph provenance that + // predates the write-time bound, and the index shards the removed engine + // left behind. Not awaited — readyz already answers 200 — and the pass + // itself yields between chunks, so requests arriving during it are served + // rather than queued. It records its own version, so every later boot is a + // single marker read. + void runStartupMaintenance(sdk.store).catch((err: unknown) => { + console.warn( + `[agentmemory] startup maintenance skipped: ${err instanceof Error ? err.message : String(err)}`, + ); + }); + + // The vector fill/repair pass closes the gap between the content rows and + // the vectors table - missing or stale rows are embedded, orphans pruned. + // Runs once at boot after readiness (not awaited: a large backlog after an + // import must not hold up the listener) and hourly. + if (vectorIndex) { + const fill = createIndexFill(sdk.store, vectorStore, vectorIndex); + registerIndexFillFunction(sdk, fill); + void fill.run(); + setInterval(() => void fill.run(), 3_600_000).unref(); + bootLog("Vector fill pass: at boot and hourly (mem::index-fill-missing)"); + } + + // Backfill the graph read side-indexes for corpora that predate them. + // Mirrors the BM25 memories backfill above: one-time, gated on the + // snapshot's recorded node count so we never enumerate a corpus large + // enough to starve the worker heartbeat. + // + // graph-read-fix local delta (B-mode boot-skip): when the leg is off + // this block is skipped ENTIRELY — it makes the only graph-scope reads + // at boot (the kv.list arming enumeration), and B-mode must issue zero. + // Skipping is safe: with the marker absent the readers fail-closed + // (graphReadable === false) rather than enumerating. + if (!graphLegDisabled()) { + try { + if (!(await graphIndexesReady(kv))) { + const graphSnap = await kv.get( + KV.graphSnapshot, + "current", + ); + const totalNodes = graphSnap?.stats?.totalNodes ?? 0; + if (graphSnap && totalNodes > 0 && totalNodes <= GRAPH_INDEX_NODE_CEILING) { + const [graphNodes, graphEdges] = await Promise.all([ + kv.list(KV.graphNodes), + kv.list(KV.graphEdges), + ]); + await backfillGraphIndexes( + kv, + graphNodes.filter((n) => !n.stale), + graphEdges.filter((e) => !e.stale), + ); + bootLog(`Backfilled graph read indexes (${totalNodes} nodes)`); + } + } + } catch (err) { + console.warn(`[agentmemory] Failed to backfill graph indexes:`, err); + } + } + + // Ready / Endpoints lines are emitted via `bootLog` so they're + // buffered in quiet mode and printed verbatim under --verbose. The + // CLI surfaces a compact summary when it sees the worker reach + // ready state. + bootLog( + `Ready. ${embeddingProvider ? "Triple-stream (BM25+Vector+Graph)" : "BM25+Graph"} search active.`, + ); + bootLog( + `REST API: 132 endpoints at http://localhost:${config.restPort}/agentmemory/*`, + ); + bootLog( + `MCP surface (opt-in via \`npx @agentmemory/mcp\`): ${getAllTools().length} tools · 6 resources · 3 prompts`, + ); + + const viewerPort = config.restPort + 2; + const viewerServer = startViewerServer( + viewerPort, + kv, + sdk, + secret, + config.restPort, + ); + + const autoForgetIntervalMs = parseInt(process.env.AUTO_FORGET_INTERVAL_MS || "3600000", 10); + const consolidationIntervalMs = parseInt(process.env.CONSOLIDATION_INTERVAL_MS || "7200000", 10); + + if (process.env.AUTO_FORGET_ENABLED !== "false") { + const autoForgetTimer = setInterval(async () => { + try { + await sdk.trigger({ function_id: "mem::auto-forget", payload: { dryRun: false } }); + } catch {} + }, autoForgetIntervalMs); + autoForgetTimer.unref(); + bootLog(`Auto-forget: enabled (every ${autoForgetIntervalMs / 60000}m)`); + } + + if (process.env.LESSON_DECAY_ENABLED !== "false") { + const lessonDecayTimer = setInterval(async () => { + try { + await sdk.trigger({ function_id: "mem::lesson-decay-sweep", payload: {} }); + } catch {} + }, 86400000); + lessonDecayTimer.unref(); + bootLog(`Lesson decay sweep: enabled (every 24h)`); + } + + if (process.env.EVICTION_ENABLED !== "false") { + // First sweep shortly after boot: a daemon restarted more often than daily + // would otherwise never reach its first 24h tick. + const runEviction = async () => { + try { + await sdk.trigger({ function_id: "mem::evict", payload: { dryRun: false } }); + } catch {} + }; + setTimeout(runEviction, FIRST_SWEEP_DELAY_MS).unref(); + setInterval(runEviction, 86400000).unref(); + bootLog(`Eviction sweep: enabled (5 min after boot, then every 24h)`); + } + + const runAuditEviction = async () => { + try { + await evictOldestAudit(kv); + } catch {} + }; + setTimeout(runAuditEviction, FIRST_SWEEP_DELAY_MS).unref(); + setInterval(runAuditEviction, 86400000).unref(); + + if (process.env.INSIGHT_DECAY_ENABLED !== "false") { + const insightDecayTimer = setInterval(async () => { + try { + await sdk.trigger({ function_id: "mem::insight-decay-sweep", payload: {} }); + } catch {} + }, 86400000); + insightDecayTimer.unref(); + } + + // #771: hourly TTL sweep for the followup-rate diagnostic. The + // recent-searches scope only needs the last entry per session; + // sweeping anything older than the retention window keeps the scope + // from growing unbounded across long-lived deployments. + const recentSearchesSweepTimer = setInterval(async () => { + try { + await sdk.trigger({ + function_id: "mem::diagnostic::recent-searches-sweep", + payload: {}, + }); + } catch {} + }, 60 * 60 * 1000); + recentSearchesSweepTimer.unref(); + + if (isConsolidationEnabled()) { + const consolidationTimer = setInterval(async () => { + try { + await sdk.trigger({ function_id: "mem::consolidate-pipeline", payload: {} }); + } catch {} + }, consolidationIntervalMs); + consolidationTimer.unref(); + bootLog(`Auto-consolidation: enabled (every ${consolidationIntervalMs / 60000}m)`); + } + + const shutdown = async () => { + console.log(`\n[agentmemory] Shutting down...`); + healthMonitor.stop(); + dedupMap.stop(); + await new Promise((resolve) => viewerServer.close(() => resolve())); + await sdk.shutdown(); + clearWorkerPidfile(); + process.exit(0); + }; + process.on("SIGINT", shutdown); + process.on("SIGTERM", shutdown); +} + +main().catch((err) => { + console.error(`[agentmemory] Fatal:`, err); + process.exit(1); +}); diff --git a/src/providers/_fetch.ts b/src/providers/_fetch.ts index a2bf3906fa..ac36e55ee8 100644 --- a/src/providers/_fetch.ts +++ b/src/providers/_fetch.ts @@ -23,9 +23,12 @@ export class ProviderHttpError extends Error { } } +// OpenAI answers an exhausted quota with 429 too; that never clears by waiting, +// so it counts as broken and opens the breaker. export function isProviderBusy(err: unknown): boolean { const status = (err as { status?: unknown } | null)?.status; - return typeof status === "number" && RETRY_STATUS.has(status); + if (typeof status !== "number" || !RETRY_STATUS.has(status)) return false; + return !/insufficient_quota/.test(err instanceof Error ? err.message : ""); } const sleep = (ms: number): Promise => diff --git a/src/providers/resilient.ts b/src/providers/resilient.ts index 0b1b35d207..88a7ff98bf 100644 --- a/src/providers/resilient.ts +++ b/src/providers/resilient.ts @@ -48,8 +48,9 @@ export class ResilientProvider implements MemoryProvider { } } - // One cap across every operation: a busy provider is busy for all of them, - // and fanning out past it only turns queued work into 429s. + // One cap across every generating operation: a busy provider is busy for all + // of them, and fanning out past it only turns queued work into 429s. + // countTokens stays outside it; summarize bounds those with COUNT_CONCURRENCY. private async acquireSlot(): Promise { if (this.inFlight < maxConcurrency()) { this.inFlight++; diff --git a/src/state/index-corpus.ts b/src/state/index-corpus.ts index 42eabe4930..4fe960a188 100644 --- a/src/state/index-corpus.ts +++ b/src/state/index-corpus.ts @@ -1,217 +1,217 @@ -// The one place that says which content rows are indexed and what text is -// embedded for them. Every consumer of that rule routes through here so they -// cannot drift apart: -// -// - the live write paths (observe / compress / remember) build their embed -// jobs from it; -// - the inproc vector store re-derives the expected input for an id from the -// CURRENT content row before it commits an embedding, so a completion for -// text that has since been rewritten or deleted is dropped as stale; -// - the boot BM25 rebuild and the vector fill pass enumerate the corpus with -// the same eligibility. -import { createHash } from "node:crypto"; -import type { DatabaseSync } from "node:sqlite"; -import type { CompressedObservation, Memory } from "../types.js"; -import { KV } from "./schema.js"; -import { parentIdOf } from "./vector-index.js"; -import { - MEMORY_SESSION, - isLatestEligible, - memoryChunkJobs, - memoryToIndexDoc, -} from "./memory-utils.js"; - -export type EmbedKind = "memory" | "observation" | "synthetic"; - -export type EmbedJob = { - id: string; - sessionId: string; - text: string; - kind: EmbedKind; -}; - -// Hard cap on embedding input length. Most providers cap input around -// 8k tokens (~32k chars at ~4 chars/token). Truncate defensively so a -// huge memory.content can't 400 the embed call or blow context budget -// on a single doc. 16k chars ≈ 4k tokens, safely under every provider. -const EMBED_MAX_CHARS = 16_000; - -export function clipEmbedInput(text: string): string { - if (text.length <= EMBED_MAX_CHARS) return text; - return text.slice(0, EMBED_MAX_CHARS); -} - -// Fingerprint of the exact text handed to the provider. Stored on the vector -// row so "a vector exists" means "a vector for THIS content exists". -export function embedInputHash(text: string): string { - return createHash("sha256").update(clipEmbedInput(text)).digest("hex"); -} - -// Tool-name prefixes whose observations are STORED but never INDEXED. -// -// Retrieval echoes — the daemon's own record of "the agent ran -// memory_smart_search for " — contain the query verbatim, so they -// are near-perfect BM25 and vector matches for exactly the query that -// created them, and they crowd out the answer (upstream #993). Live -// measurement: for one query, ranks 1, 2 and 6 were all records of prior -// searches, scoring up to 74.89 against the real memory's 42.23 at rank 13. -// -// The KV writes are deliberately untouched. These rows stay in the -// timeline, session history, the viewer and expandIds — they just stop -// competing in search. That is how "observations are kept" is satisfied -// structurally rather than by promise. -// Unset means agentmemory's own MCP tools, under both the plugin and the -// standalone server name; set it empty to index everything. -const DEFAULT_EXCLUDED_TOOL_PREFIXES = - "mcp__plugin_agentmemory_agentmemory__,mcp__agentmemory__"; -let excludeCache: { raw: string; prefixes: string[] } | null = null; - -function excludedToolPrefixes(): string[] { - const raw = - process.env.AGENTMEMORY_INDEX_EXCLUDE_TOOL_PREFIXES ?? - DEFAULT_EXCLUDED_TOOL_PREFIXES; - if (!excludeCache || excludeCache.raw !== raw) { - excludeCache = { - raw, - prefixes: raw - .split(",") - .map((s) => s.trim()) - .filter((s) => s.length > 0), - }; - } - return excludeCache.prefixes; -} - -export function isIndexExcluded(obs: { toolName?: string }): boolean { - if (typeof obs.toolName !== "string" || !obs.toolName) return false; - const prefixes = excludedToolPrefixes(); - if (prefixes.length === 0) return false; - return prefixes.some((p) => obs.toolName!.startsWith(p)); -} - -// Where the content row for a vector id lives. Memories may carry a real -// session id as their first `sessionIds` entry, so the session alone does not -// identify the scope; the kind does. -export function contentLocation( - kind: EmbedKind, - sessionId: string, - vectorId: string, -): { scope: string; key: string } { - const parent = parentIdOf(vectorId); - return kind === "memory" - ? { scope: KV.memories, key: parent } - : { scope: KV.observations(sessionId), key: parent }; -} - -// The embed jobs a stored row is entitled to right now. An empty array means -// the row is not indexed (missing, superseded, incomplete, or an excluded -// tool), so any vector for it is stale. -// Field checks are `typeof`, not truthiness: a stored row may carry a -// non-string where the type says string (the import handler accepts such -// rows), and string concatenation on an object throws. Such a row is simply -// not indexable. -export function memoryEmbedJobs(memory: Memory | null | undefined): EmbedJob[] { - if (!memory || typeof memory.id !== "string" || !memory.id || !isLatestEligible(memory)) return []; - if (typeof memory.title !== "string" || !memory.title) return []; - if (typeof memory.content !== "string" || !memory.content) return []; - const first = Array.isArray(memory.sessionIds) ? memory.sessionIds[0] : undefined; - const sessionId = typeof first === "string" && first ? first : MEMORY_SESSION; - return memoryChunkJobs(memory).map((job) => ({ - id: job.id, - sessionId, - text: job.text, - kind: "memory" as const, - })); -} - -export function observationEmbedJobs( - obs: CompressedObservation | null | undefined, - kind: "observation" | "synthetic" = "observation", -): EmbedJob[] { - if (!obs || typeof obs.id !== "string" || typeof obs.sessionId !== "string") return []; - if (typeof obs.title !== "string" || !obs.title) return []; - if (typeof obs.narrative !== "string" || !obs.narrative) return []; - if (isIndexExcluded(obs)) return []; - return [{ id: obs.id, sessionId: obs.sessionId, text: obs.title + " " + obs.narrative, kind }]; -} - -export function embedJobsFor(kind: EmbedKind, row: unknown): EmbedJob[] { - return kind === "memory" - ? memoryEmbedJobs(row as Memory | null) - : observationEmbedJobs(row as CompressedObservation | null, kind); -} - -// ---------------------------------------------------------------- corpus - -// SearchIndex spreads `facts`, `concepts` and `files`; an imported or -// hand-written observation may lack them (the import handler accepts such -// rows and only skips their indexing). Normalise here so the rebuild indexes -// the row instead of throwing on it. -const arr = (v: unknown): string[] => (Array.isArray(v) ? v : []); -function indexableObservation(obs: CompressedObservation): CompressedObservation { - return { ...obs, facts: arr(obs.facts), concepts: arr(obs.concepts), files: arr(obs.files) }; -} - -// `mem:obs:` scopes as a half-open range (';' follows ':'), so the -// (scope, seq) index serves it and no session id needs escaping. -const OBS_SCOPE_LO = "mem:obs:"; -const OBS_SCOPE_HI = "mem:obs;"; - -export type CorpusItem = { - kind: "memory" | "observation"; - scope: string; - key: string; - seq: number; - row: Memory | CompressedObservation; - // The BM25 document, or null when the row is not indexed. - doc: CompressedObservation | null; - // The embedding jobs the row is entitled to (empty when not indexed). - jobs: EmbedJob[]; - // Set when building the doc or the jobs threw: the row is treated as not - // indexed and consumers count and log it instead of failing the walk. - error?: string; -}; - -// Walks every memory and every observation on disk - including observation -// scopes whose session record is gone (evict.ts removes sessions but keeps -// their observations, which are content and stay indexed) - in `seq` order, -// and applies the eligibility rules above once for both legs. The boot BM25 -// rebuild and the vector fill pass consume this same iterator, so the two can -// never disagree about what belongs in the index. -export function* enumerateIndexCorpus(db: DatabaseSync): IterableIterator { - const stmt = db.prepare( - "SELECT seq, scope, key, value FROM kv WHERE scope = ? OR (scope >= ? AND scope < ?) ORDER BY seq", - ); - for (const r of stmt.iterate(KV.memories, OBS_SCOPE_LO, OBS_SCOPE_HI) as Iterable<{ - seq: number; - scope: string; - key: string; - value: string; - }>) { - let row: unknown; - try { - row = JSON.parse(r.value); - } catch { - continue; - } - const kind = r.scope === KV.memories ? "memory" : "observation"; - const item: CorpusItem = { kind, scope: r.scope, key: r.key, seq: r.seq, row: row as Memory, doc: null, jobs: [] }; - // One malformed row must never end the walk (the boot rebuild awaits it). - try { - if (kind === "memory") { - const memory = row as Memory; - item.jobs = memoryEmbedJobs(memory); - item.doc = item.jobs.length > 0 ? memoryToIndexDoc(memory) : null; - } else { - const obs = row as CompressedObservation; - item.jobs = observationEmbedJobs(obs); - item.doc = item.jobs.length > 0 ? indexableObservation(obs) : null; - } - } catch (err) { - item.doc = null; - item.jobs = []; - item.error = err instanceof Error ? err.message : String(err); - } - yield item; - } -} +// The one place that says which content rows are indexed and what text is +// embedded for them. Every consumer of that rule routes through here so they +// cannot drift apart: +// +// - the live write paths (observe / compress / remember) build their embed +// jobs from it; +// - the inproc vector store re-derives the expected input for an id from the +// CURRENT content row before it commits an embedding, so a completion for +// text that has since been rewritten or deleted is dropped as stale; +// - the boot BM25 rebuild and the vector fill pass enumerate the corpus with +// the same eligibility. +import { createHash } from "node:crypto"; +import type { DatabaseSync } from "node:sqlite"; +import type { CompressedObservation, Memory } from "../types.js"; +import { KV } from "./schema.js"; +import { parentIdOf } from "./vector-index.js"; +import { + MEMORY_SESSION, + isLatestEligible, + memoryChunkJobs, + memoryToIndexDoc, +} from "./memory-utils.js"; + +export type EmbedKind = "memory" | "observation" | "synthetic"; + +export type EmbedJob = { + id: string; + sessionId: string; + text: string; + kind: EmbedKind; +}; + +// Hard cap on embedding input length. Most providers cap input around +// 8k tokens (~32k chars at ~4 chars/token). Truncate defensively so a +// huge memory.content can't 400 the embed call or blow context budget +// on a single doc. 16k chars ≈ 4k tokens, safely under every provider. +const EMBED_MAX_CHARS = 16_000; + +export function clipEmbedInput(text: string): string { + if (text.length <= EMBED_MAX_CHARS) return text; + return text.slice(0, EMBED_MAX_CHARS); +} + +// Fingerprint of the exact text handed to the provider. Stored on the vector +// row so "a vector exists" means "a vector for THIS content exists". +export function embedInputHash(text: string): string { + return createHash("sha256").update(clipEmbedInput(text)).digest("hex"); +} + +// Tool-name prefixes whose observations are STORED but never INDEXED. +// +// Retrieval echoes — the daemon's own record of "the agent ran +// memory_smart_search for " — contain the query verbatim, so they +// are near-perfect BM25 and vector matches for exactly the query that +// created them, and they crowd out the answer (upstream #993). Live +// measurement: for one query, ranks 1, 2 and 6 were all records of prior +// searches, scoring up to 74.89 against the real memory's 42.23 at rank 13. +// +// The KV writes are deliberately untouched. These rows stay in the +// timeline, session history, the viewer and expandIds — they just stop +// competing in search. That is how "observations are kept" is satisfied +// structurally rather than by promise. +// Unset means agentmemory's own MCP tools, under both the plugin and the +// standalone server name; set it empty to index everything. +const DEFAULT_EXCLUDED_TOOL_PREFIXES = + "mcp__plugin_agentmemory_agentmemory__,mcp__agentmemory__"; +let excludeCache: { raw: string; prefixes: string[] } | null = null; + +function excludedToolPrefixes(): string[] { + const raw = + process.env.AGENTMEMORY_INDEX_EXCLUDE_TOOL_PREFIXES ?? + DEFAULT_EXCLUDED_TOOL_PREFIXES; + if (!excludeCache || excludeCache.raw !== raw) { + excludeCache = { + raw, + prefixes: raw + .split(",") + .map((s) => s.trim()) + .filter((s) => s.length > 0), + }; + } + return excludeCache.prefixes; +} + +export function isIndexExcluded(obs: { toolName?: string }): boolean { + if (typeof obs.toolName !== "string" || !obs.toolName) return false; + const prefixes = excludedToolPrefixes(); + if (prefixes.length === 0) return false; + return prefixes.some((p) => obs.toolName!.startsWith(p)); +} + +// Where the content row for a vector id lives. Memories may carry a real +// session id as their first `sessionIds` entry, so the session alone does not +// identify the scope; the kind does. +export function contentLocation( + kind: EmbedKind, + sessionId: string, + vectorId: string, +): { scope: string; key: string } { + const parent = parentIdOf(vectorId); + return kind === "memory" + ? { scope: KV.memories, key: parent } + : { scope: KV.observations(sessionId), key: parent }; +} + +// The embed jobs a stored row is entitled to right now. An empty array means +// the row is not indexed (missing, superseded, incomplete, or an excluded +// tool), so any vector for it is stale. +// Field checks are `typeof`, not truthiness: a stored row may carry a +// non-string where the type says string (the import handler accepts such +// rows), and string concatenation on an object throws. Such a row is simply +// not indexable. +export function memoryEmbedJobs(memory: Memory | null | undefined): EmbedJob[] { + if (!memory || typeof memory.id !== "string" || !memory.id || !isLatestEligible(memory)) return []; + if (typeof memory.title !== "string" || !memory.title) return []; + if (typeof memory.content !== "string" || !memory.content) return []; + const first = Array.isArray(memory.sessionIds) ? memory.sessionIds[0] : undefined; + const sessionId = typeof first === "string" && first ? first : MEMORY_SESSION; + return memoryChunkJobs(memory).map((job) => ({ + id: job.id, + sessionId, + text: job.text, + kind: "memory" as const, + })); +} + +export function observationEmbedJobs( + obs: CompressedObservation | null | undefined, + kind: "observation" | "synthetic" = "observation", +): EmbedJob[] { + if (!obs || typeof obs.id !== "string" || typeof obs.sessionId !== "string") return []; + if (typeof obs.title !== "string" || !obs.title) return []; + if (typeof obs.narrative !== "string" || !obs.narrative) return []; + if (isIndexExcluded(obs)) return []; + return [{ id: obs.id, sessionId: obs.sessionId, text: obs.title + " " + obs.narrative, kind }]; +} + +export function embedJobsFor(kind: EmbedKind, row: unknown): EmbedJob[] { + return kind === "memory" + ? memoryEmbedJobs(row as Memory | null) + : observationEmbedJobs(row as CompressedObservation | null, kind); +} + +// ---------------------------------------------------------------- corpus + +// SearchIndex spreads `facts`, `concepts` and `files`; an imported or +// hand-written observation may lack them (the import handler accepts such +// rows and only skips their indexing). Normalise here so the rebuild indexes +// the row instead of throwing on it. +const arr = (v: unknown): string[] => (Array.isArray(v) ? v : []); +function indexableObservation(obs: CompressedObservation): CompressedObservation { + return { ...obs, facts: arr(obs.facts), concepts: arr(obs.concepts), files: arr(obs.files) }; +} + +// `mem:obs:` scopes as a half-open range (';' follows ':'), so the +// (scope, seq) index serves it and no session id needs escaping. +const OBS_SCOPE_LO = "mem:obs:"; +const OBS_SCOPE_HI = "mem:obs;"; + +export type CorpusItem = { + kind: "memory" | "observation"; + scope: string; + key: string; + seq: number; + row: Memory | CompressedObservation; + // The BM25 document, or null when the row is not indexed. + doc: CompressedObservation | null; + // The embedding jobs the row is entitled to (empty when not indexed). + jobs: EmbedJob[]; + // Set when building the doc or the jobs threw: the row is treated as not + // indexed and consumers count and log it instead of failing the walk. + error?: string; +}; + +// Walks every memory and every observation on disk - including observation +// scopes whose session record is gone (evict.ts removes sessions but keeps +// their observations, which are content and stay indexed) - in `seq` order, +// and applies the eligibility rules above once for both legs. The boot BM25 +// rebuild and the vector fill pass consume this same iterator, so the two can +// never disagree about what belongs in the index. +export function* enumerateIndexCorpus(db: DatabaseSync): IterableIterator { + const stmt = db.prepare( + "SELECT seq, scope, key, value FROM kv WHERE scope = ? OR (scope >= ? AND scope < ?) ORDER BY seq", + ); + for (const r of stmt.iterate(KV.memories, OBS_SCOPE_LO, OBS_SCOPE_HI) as Iterable<{ + seq: number; + scope: string; + key: string; + value: string; + }>) { + let row: unknown; + try { + row = JSON.parse(r.value); + } catch { + continue; + } + const kind = r.scope === KV.memories ? "memory" : "observation"; + const item: CorpusItem = { kind, scope: r.scope, key: r.key, seq: r.seq, row: row as Memory, doc: null, jobs: [] }; + // One malformed row must never end the walk (the boot rebuild awaits it). + try { + if (kind === "memory") { + const memory = row as Memory; + item.jobs = memoryEmbedJobs(memory); + item.doc = item.jobs.length > 0 ? memoryToIndexDoc(memory) : null; + } else { + const obs = row as CompressedObservation; + item.jobs = observationEmbedJobs(obs); + item.doc = item.jobs.length > 0 ? indexableObservation(obs) : null; + } + } catch (err) { + item.doc = null; + item.jobs = []; + item.error = err instanceof Error ? err.message : String(err); + } + yield item; + } +} diff --git a/test/audit.test.ts b/test/audit.test.ts index 3bb74f15f6..214ec7fbfe 100644 --- a/test/audit.test.ts +++ b/test/audit.test.ts @@ -4,7 +4,7 @@ vi.mock("../src/logger.js", () => ({ logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, })); -import { recordAudit, queryAudit, pruneAudit } from "../src/functions/audit.js"; +import { recordAudit, queryAudit, evictOldestAudit } from "../src/functions/audit.js"; function mockKV() { const store = new Map>(); @@ -104,7 +104,7 @@ describe("Audit Functions", () => { expect(entries.length).toBe(3); }); - it("pruneAudit keeps the newest AGENTMEMORY_AUDIT_MAX entries", async () => { + it("evictOldestAudit keeps the newest AGENTMEMORY_AUDIT_MAX entries", async () => { process.env.AGENTMEMORY_AUDIT_MAX = "2"; try { for (let i = 0; i < 4; i++) { @@ -118,7 +118,7 @@ describe("Audit Functions", () => { }); } - expect(await pruneAudit(kv as never)).toBe(2); + expect(await evictOldestAudit(kv as never)).toBe(2); const kept = (await kv.list<{ id: string }>("mem:audit")).map((e) => e.id).sort(); expect(kept).toEqual(["aud_2", "aud_3"]); diff --git a/test/consolidation-pipeline.test.ts b/test/consolidation-pipeline.test.ts index c3b4033c95..ea0ad8b1a2 100644 --- a/test/consolidation-pipeline.test.ts +++ b/test/consolidation-pipeline.test.ts @@ -391,4 +391,53 @@ describe("Consolidation Pipeline: per-project scope (#1344)", () => { expect(second.results.reflect).toEqual({}); expect(provider.summarize).toHaveBeenCalledTimes(1); }); + + it("retries a project whose reflect failed on the next unscoped run", async () => { + const sdk = mockSdk(); + const kv = mockKV(); + const provider = { name: "test", compress: vi.fn(), summarize: vi.fn().mockResolvedValue("") }; + let calls = 0; + sdk.registerFunction("mem::reflect", async () => { + calls++; + if (calls === 1) throw new Error("provider busy"); + return { success: true }; + }); + registerConsolidationPipelineFunction(sdk as never, kv as never, provider as never); + for (let i = 0; i < 5; i++) await kv.set("mem:summaries", `alpha_${i}`, summaryFor("alpha", i)); + + await sdk.trigger("mem::consolidate-pipeline", { tier: "reflect" }); + const second = (await sdk.trigger("mem::consolidate-pipeline", { tier: "reflect" })) as { + results: { reflect: Record }; + }; + const third = (await sdk.trigger("mem::consolidate-pipeline", { tier: "reflect" })) as { + results: { reflect: Record }; + }; + + expect(second.results.reflect).toEqual({ alpha: { success: true } }); + expect(third.results.reflect).toEqual({}); + }); + + it("credits a fact another project already holds to this project's Sessions too", async () => { + const sdk = mockSdk(); + const kv = mockKV(); + const provider = { + name: "test", + compress: vi.fn(), + summarize: vi.fn().mockResolvedValue(`Use node:sqlite`), + }; + sdk.registerFunction("mem::reflect", async () => ({ success: true })); + registerConsolidationPipelineFunction(sdk as never, kv as never, provider as never); + for (let i = 0; i < 5; i++) { + await kv.set("mem:summaries", `alpha_${i}`, summaryFor("alpha", i)); + await kv.set("mem:summaries", `beta_${i}`, summaryFor("beta", i)); + } + + await sdk.trigger("mem::consolidate-pipeline", { tier: "semantic" }); + + const facts = await kv.list("mem:semantic"); + expect(facts).toHaveLength(1); + expect(facts[0].sourceSessionIds).toEqual( + expect.arrayContaining(["alpha_0", "beta_0"]), + ); + }); }); diff --git a/test/index-echo-exclusion.test.ts b/test/index-echo-exclusion.test.ts index 826ba9eedb..4b60c07e31 100644 --- a/test/index-echo-exclusion.test.ts +++ b/test/index-echo-exclusion.test.ts @@ -1,154 +1,154 @@ -import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"; - -vi.mock("../src/logger.js", () => ({ - logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, -})); - -import { - isIndexExcluded, - rebuildIndex, - getSearchIndex, - setVectorIndex, - setEmbeddingProvider, - getVectorIndex, -} from "../src/functions/search.js"; -import { VectorIndex } from "../src/state/vector-index.js"; -import { KV } from "../src/state/schema.js"; -import type { CompressedObservation, Session } from "../src/types.js"; - -const FLAG = "AGENTMEMORY_INDEX_EXCLUDE_TOOL_PREFIXES"; -const ECHO_PREFIX = "mcp__plugin_agentmemory_agentmemory__"; - -function mockKV() { - const store = new Map>(); - return { - get: async (scope: string, key: string): Promise => - (store.get(scope)?.get(key) as T) ?? null, - set: async (scope: string, key: string, data: T): Promise => { - if (!store.has(scope)) store.set(scope, new Map()); - store.get(scope)!.set(key, data); - return data; - }, - delete: async (scope: string, key: string): Promise => { - store.get(scope)?.delete(key); - }, - list: async (scope: string): Promise => - Array.from(store.get(scope)?.values() ?? []) as T[], - }; -} - -function obs(id: string, toolName?: string): CompressedObservation { - return { - id, - sessionId: "ses_1", - timestamp: "2026-01-01T00:00:00Z", - type: "search", - title: toolName ? `Memory search: ${id}` : `Real work ${id}`, - facts: [], - narrative: "residentKey teleport vnet mainpc-rdp", - concepts: [], - files: [], - importance: 5, - ...(toolName ? { toolName } : {}), - }; -} - -describe("index exclusion by toolName prefix", () => { - const saved = process.env[FLAG]; - - beforeEach(() => { - delete process.env[FLAG]; - getSearchIndex().clear(); - setVectorIndex(null); - setEmbeddingProvider(null); - }); - afterEach(() => { - if (saved === undefined) delete process.env[FLAG]; - else process.env[FLAG] = saved; - getSearchIndex().clear(); - setVectorIndex(null); - setEmbeddingProvider(null); - }); - - it("excludes agentmemory's own MCP tools when the flag is unset (#993)", () => { - expect(isIndexExcluded(obs("o", `${ECHO_PREFIX}memory_smart_search`))).toBe(true); - expect(isIndexExcluded(obs("o", "mcp__agentmemory__memory_recall"))).toBe(true); - expect(isIndexExcluded(obs("o", "Bash"))).toBe(false); - }); - - it("excludes nothing when the flag is set empty", () => { - process.env[FLAG] = ""; - expect(isIndexExcluded(obs("o", `${ECHO_PREFIX}memory_smart_search`))).toBe(false); - }); - - it("matches on prefix, and only on the configured prefixes", () => { - process.env[FLAG] = ECHO_PREFIX; - expect(isIndexExcluded(obs("o", `${ECHO_PREFIX}memory_smart_search`))).toBe(true); - expect(isIndexExcluded(obs("o", `${ECHO_PREFIX}memory_recall`))).toBe(true); - expect(isIndexExcluded(obs("o", "Bash"))).toBe(false); - expect(isIndexExcluded(obs("o", "mcp__veeam-rag__hybrid_search"))).toBe(false); - // A record with no toolName (every pre-existing row on disk) is never - // matched by this predicate — that is why legacy echoes need a - // separate, explicitly reviewed deletion pass. - expect(isIndexExcluded(obs("o"))).toBe(false); - }); - - it("accepts a comma-separated list and tolerates whitespace", () => { - process.env[FLAG] = ` ${ECHO_PREFIX} , other__ `; - expect(isIndexExcluded(obs("o", `${ECHO_PREFIX}x`))).toBe(true); - expect(isIndexExcluded(obs("o", "other__thing"))).toBe(true); - expect(isIndexExcluded(obs("o", "unrelated"))).toBe(false); - }); - - it("keeps the row in KV while dropping it from BOTH indexes", async () => { - process.env[FLAG] = ECHO_PREFIX; - const kv = mockKV(); - const session: Session = { - id: "ses_1", - project: "demo", - cwd: "/tmp/demo", - startedAt: "2026-01-01T00:00:00Z", - status: "completed", - observationCount: 2, - }; - await kv.set(KV.sessions, session.id, session); - - const echo = obs("obs_echo", `${ECHO_PREFIX}memory_smart_search`); - const real = obs("obs_real"); - await kv.set(KV.observations("ses_1"), echo.id, echo); - await kv.set(KV.observations("ses_1"), real.id, real); - - setVectorIndex(new VectorIndex()); - setEmbeddingProvider({ - name: "test", - dimensions: 3, - embed: async () => new Float32Array([0.1, 0.2, 0.3]), - embedBatch: async (t: string[]) => - t.map(() => new Float32Array([0.1, 0.2, 0.3])), - }); - - const count = await rebuildIndex(kv as never); - - // Indexed: only the real observation. - expect(count).toBe(1); - expect(getSearchIndex().has("obs_real")).toBe(true); - expect(getSearchIndex().has("obs_echo")).toBe(false); - expect( - getVectorIndex()! - .search(new Float32Array([0.1, 0.2, 0.3]), 10) - .map((r) => r.obsId), - ).toEqual(["obs_real"]); - - // Kept: the echo is still fully readable from KV, so timeline, - // session history, the viewer and expandIds all still see it. - const stored = await kv.get( - KV.observations("ses_1"), - "obs_echo", - ); - expect(stored).not.toBeNull(); - expect(stored!.title).toContain("Memory search:"); - expect( - (await kv.list(KV.observations("ses_1"))).length, - ).toBe(2); - }); -}); +import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"; + +vi.mock("../src/logger.js", () => ({ + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, +})); + +import { + isIndexExcluded, + rebuildIndex, + getSearchIndex, + setVectorIndex, + setEmbeddingProvider, + getVectorIndex, +} from "../src/functions/search.js"; +import { VectorIndex } from "../src/state/vector-index.js"; +import { KV } from "../src/state/schema.js"; +import type { CompressedObservation, Session } from "../src/types.js"; + +const FLAG = "AGENTMEMORY_INDEX_EXCLUDE_TOOL_PREFIXES"; +const ECHO_PREFIX = "mcp__plugin_agentmemory_agentmemory__"; + +function mockKV() { + const store = new Map>(); + return { + get: async (scope: string, key: string): Promise => + (store.get(scope)?.get(key) as T) ?? null, + set: async (scope: string, key: string, data: T): Promise => { + if (!store.has(scope)) store.set(scope, new Map()); + store.get(scope)!.set(key, data); + return data; + }, + delete: async (scope: string, key: string): Promise => { + store.get(scope)?.delete(key); + }, + list: async (scope: string): Promise => + Array.from(store.get(scope)?.values() ?? []) as T[], + }; +} + +function obs(id: string, toolName?: string): CompressedObservation { + return { + id, + sessionId: "ses_1", + timestamp: "2026-01-01T00:00:00Z", + type: "search", + title: toolName ? `Memory search: ${id}` : `Real work ${id}`, + facts: [], + narrative: "residentKey teleport vnet mainpc-rdp", + concepts: [], + files: [], + importance: 5, + ...(toolName ? { toolName } : {}), + }; +} + +describe("index exclusion by toolName prefix", () => { + const saved = process.env[FLAG]; + + beforeEach(() => { + delete process.env[FLAG]; + getSearchIndex().clear(); + setVectorIndex(null); + setEmbeddingProvider(null); + }); + afterEach(() => { + if (saved === undefined) delete process.env[FLAG]; + else process.env[FLAG] = saved; + getSearchIndex().clear(); + setVectorIndex(null); + setEmbeddingProvider(null); + }); + + it("excludes agentmemory's own MCP tools when the flag is unset (#993)", () => { + expect(isIndexExcluded(obs("o", `${ECHO_PREFIX}memory_smart_search`))).toBe(true); + expect(isIndexExcluded(obs("o", "mcp__agentmemory__memory_recall"))).toBe(true); + expect(isIndexExcluded(obs("o", "Bash"))).toBe(false); + }); + + it("excludes nothing when the flag is set empty", () => { + process.env[FLAG] = ""; + expect(isIndexExcluded(obs("o", `${ECHO_PREFIX}memory_smart_search`))).toBe(false); + }); + + it("matches on prefix, and only on the configured prefixes", () => { + process.env[FLAG] = ECHO_PREFIX; + expect(isIndexExcluded(obs("o", `${ECHO_PREFIX}memory_smart_search`))).toBe(true); + expect(isIndexExcluded(obs("o", `${ECHO_PREFIX}memory_recall`))).toBe(true); + expect(isIndexExcluded(obs("o", "Bash"))).toBe(false); + expect(isIndexExcluded(obs("o", "mcp__veeam-rag__hybrid_search"))).toBe(false); + // A record with no toolName (every pre-existing row on disk) is never + // matched by this predicate — that is why legacy echoes need a + // separate, explicitly reviewed deletion pass. + expect(isIndexExcluded(obs("o"))).toBe(false); + }); + + it("accepts a comma-separated list and tolerates whitespace", () => { + process.env[FLAG] = ` ${ECHO_PREFIX} , other__ `; + expect(isIndexExcluded(obs("o", `${ECHO_PREFIX}x`))).toBe(true); + expect(isIndexExcluded(obs("o", "other__thing"))).toBe(true); + expect(isIndexExcluded(obs("o", "unrelated"))).toBe(false); + }); + + it("keeps the row in KV while dropping it from BOTH indexes", async () => { + process.env[FLAG] = ECHO_PREFIX; + const kv = mockKV(); + const session: Session = { + id: "ses_1", + project: "demo", + cwd: "/tmp/demo", + startedAt: "2026-01-01T00:00:00Z", + status: "completed", + observationCount: 2, + }; + await kv.set(KV.sessions, session.id, session); + + const echo = obs("obs_echo", `${ECHO_PREFIX}memory_smart_search`); + const real = obs("obs_real"); + await kv.set(KV.observations("ses_1"), echo.id, echo); + await kv.set(KV.observations("ses_1"), real.id, real); + + setVectorIndex(new VectorIndex()); + setEmbeddingProvider({ + name: "test", + dimensions: 3, + embed: async () => new Float32Array([0.1, 0.2, 0.3]), + embedBatch: async (t: string[]) => + t.map(() => new Float32Array([0.1, 0.2, 0.3])), + }); + + const count = await rebuildIndex(kv as never); + + // Indexed: only the real observation. + expect(count).toBe(1); + expect(getSearchIndex().has("obs_real")).toBe(true); + expect(getSearchIndex().has("obs_echo")).toBe(false); + expect( + getVectorIndex()! + .search(new Float32Array([0.1, 0.2, 0.3]), 10) + .map((r) => r.obsId), + ).toEqual(["obs_real"]); + + // Kept: the echo is still fully readable from KV, so timeline, + // session history, the viewer and expandIds all still see it. + const stored = await kv.get( + KV.observations("ses_1"), + "obs_echo", + ); + expect(stored).not.toBeNull(); + expect(stored!.title).toContain("Memory search:"); + expect( + (await kv.list(KV.observations("ses_1"))).length, + ).toBe(2); + }); +}); diff --git a/test/observe-session-cap.test.ts b/test/observe-session-cap.test.ts index b0153bc529..232903fa07 100644 --- a/test/observe-session-cap.test.ts +++ b/test/observe-session-cap.test.ts @@ -68,12 +68,12 @@ async function setup(rows: ReturnType[], cap: number) { }, })) as { observationId?: string; success?: boolean }; const ids = (await kv.list<{ id: string }>(KV.observations(SESSION))).map((o) => o.id); - return { result, ids }; + return { result, ids, kv }; } describe("mem::observe at MAX_OBS_PER_SESSION (PR#1174)", () => { it("admits the new observation by evicting the least important one", async () => { - const { result, ids } = await setup( + const { result, ids, kv } = await setup( [ stored("keep", 8, "2026-01-01T00:00:00Z"), stored("drop", 2, "2026-01-02T00:00:00Z"), @@ -86,6 +86,10 @@ describe("mem::observe at MAX_OBS_PER_SESSION (PR#1174)", () => { expect(ids).toContain("keep"); expect(ids).toContain("raw"); expect(ids).not.toContain("drop"); + const audits = await kv.list<{ operation: string; functionId: string; targetIds: string[] }>(KV.audit); + expect(audits).toContainEqual( + expect.objectContaining({ operation: "delete", functionId: "mem::observe", targetIds: ["drop"] }), + ); }); it("evicts the older row on an importance tie and drops it from search", async () => { diff --git a/test/replay-lessons.test.ts b/test/replay-lessons.test.ts index 917e58802f..904ec33071 100644 --- a/test/replay-lessons.test.ts +++ b/test/replay-lessons.test.ts @@ -46,6 +46,12 @@ describe("extractLessons (#1292)", () => { ).toEqual(["Always run the migration before seeding the db."]); }); + it("accepts a curly apostrophe in the trigger", () => { + expect(extractLessons(["Don’t commit the generated bundle by hand."])).toEqual([ + "Don’t commit the generated bundle by hand.", + ]); + }); + it("drops a trigger that sits mid-sentence and would lose its subject", () => { expect( extractLessons(["The watchdog must never restart on ROUTE_MISSING_404."]), diff --git a/test/resilient-provider.test.ts b/test/resilient-provider.test.ts index b8c9867cf4..6cf2cca1c7 100644 --- a/test/resilient-provider.test.ts +++ b/test/resilient-provider.test.ts @@ -91,6 +91,20 @@ describe("ResilientProvider per-operation breakers", () => { expect(provider.circuitStates.summarize.state).toBe("closed"); }); + it("opens the breaker on a 429 that reports an exhausted quota", async () => { + const provider = new ResilientProvider( + fakeProvider({ + summarize: async () => { + throw new ProviderHttpError('OpenAI API error (429): {"error":{"code":"insufficient_quota"}}', 429); + }, + }), + ); + + await failTimes(() => provider.summarize("s", "u"), 5); + + expect(provider.circuitStates.summarize.state).toBe("open"); + }); + it("treats an SDK error carrying status 503 as busy", async () => { const provider = new ResilientProvider( fakeProvider({ diff --git a/test/summarize.test.ts b/test/summarize.test.ts index 560609cd09..3a1b6c9d4e 100644 --- a/test/summarize.test.ts +++ b/test/summarize.test.ts @@ -562,6 +562,21 @@ describe("mem::summarize chunking", () => { expect(counted).toBe(5); }); + it("still measures when chars/3 fits but the characters do not (dense text)", async () => { + process.env.SUMMARIZE_CHUNK_TOKENS = String(PROMPT_OVERHEAD + 200); + let counted = 0; + const provider = makeProvider([summaryXml({ title: "dense" })]); + provider.countTokens = async () => { + counted += 1; + return TOKENS_PER_OBS; + }; + const { handler } = await setupHandler({ sessionId: "ses_dense", obsCount: 5, provider }); + + await handler({ sessionId: "ses_dense" }); + + expect(counted).toBe(5); + }); + it("skips measuring when the estimate already fits one chunk", async () => { let counted = 0; const provider = makeProvider([summaryXml({ title: "small" })]); From 15eb8fcf0b8d701617f90628b37ec968ccd87d5c Mon Sep 17 00:00:00 2001 From: neal Date: Fri, 25 Sep 2026 19:22:33 -0400 Subject: [PATCH 17/17] fix: address PR #57 review signoff notes Generated-By: Claude Opus 5.5 (1M context) --- docs/upstream-issue-triage.yaml | 2 +- src/functions/audit.ts | 2 +- src/functions/observe.ts | 4 ++-- src/functions/summarize.ts | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/upstream-issue-triage.yaml b/docs/upstream-issue-triage.yaml index 340ae6a2b2..8665322735 100644 --- a/docs/upstream-issue-triage.yaml +++ b/docs/upstream-issue-triage.yaml @@ -1329,7 +1329,7 @@ issues: kind: bug note: "Observations containing CJK text hang import and live ingestion indefinitely." disposition: already-fixed - status: "unresolved — no hang found: the CJK tokenizer makes one forward-only pass (src/state/cjk-segmenter.ts) and truncation cuts by string length; no loop over text can stall." + status: "fixed-here — no hang found: the CJK tokenizer makes one forward-only pass (src/state/cjk-segmenter.ts) and truncation cuts by string length; no loop over text can stall." fixed_by: "src/state/cjk-segmenter.ts makes one forward-only pass; no loop over text can stall." url: https://github.com/rohitg00/agentmemory/issues/969 - number: 972 diff --git a/src/functions/audit.ts b/src/functions/audit.ts index 325680d38b..c382c3a539 100644 --- a/src/functions/audit.ts +++ b/src/functions/audit.ts @@ -138,7 +138,7 @@ function auditMax(): number { } // Retention for the audit store: keeps the newest AGENTMEMORY_AUDIT_MAX -// entries. Pruning the log is not itself audited - that would write the +// entries. Evicting from the log is not itself audited - that would write the // rows it exists to bound - so it leaves one log line instead. export async function evictOldestAudit(kv: StateKV): Promise { if (auditStoreOff()) return 0; diff --git a/src/functions/observe.ts b/src/functions/observe.ts index 565669afd6..709804eda7 100644 --- a/src/functions/observe.ts +++ b/src/functions/observe.ts @@ -10,7 +10,7 @@ import { withKeyedLock } from "../state/keyed-mutex.js"; import { isAutoCompressEnabled } from "../config.js"; import { buildSyntheticCompression } from "./compress-synthetic.js"; import { getSearchIndex, vectorIndexAddGuarded, isIndexExcluded, deleteIndexed } from "./search.js"; -import { recordAudit } from "./audit.js"; +import { safeAudit } from "./audit.js"; import { decrementImageRef } from "./image-refs.js"; import { getAgentId } from "../config.js"; import { logger } from "../logger.js"; @@ -84,7 +84,7 @@ async function evictToAdmitOne( } if (obs.imageData) await decrementImageRef(kv, sdk, obs.imageData); if (obs.imageRef && obs.imageRef !== obs.imageData) await decrementImageRef(kv, sdk, obs.imageRef); - await recordAudit(kv, "delete", "mem::observe", [obs.id], { + await safeAudit(kv, "delete", "mem::observe", [obs.id], { resource: "observation", reason: "session_observation_cap", sessionId, diff --git a/src/functions/summarize.ts b/src/functions/summarize.ts index 6b2b2f1013..38fc9b6d5e 100644 --- a/src/functions/summarize.ts +++ b/src/functions/summarize.ts @@ -86,8 +86,8 @@ function fitsOneChunk(counts: number[], budget: number): boolean { return payload <= budget - PROMPT_OVERHEAD_TOKENS; } -// A token is never shorter than one character, so a Session whose character -// count already fits one chunk needs no measuring: on a broker that queues +// A token is rarely shorter than one character, so a Session whose character +// count already fits one chunk is safe to skip measuring: on a broker that queues // tokenize behind generation, one request per Observation is load the summary // itself has to wait behind. async function countObservationTokens(