diff --git a/.env.example b/.env.example index b8d9e51ede..eead65e84e 100644 --- a/.env.example +++ b/.env.example @@ -130,6 +130,7 @@ # MAX_OBS_PER_SESSION=500 # Hard per-session cap: later observations of that session are refused; only capture-worker refusals are parked as dead letters (0 = no cap) # SUMMARIZE_CHUNK_SIZE=400 # When mem::summarize sees a session larger than this, it chunks observations and map-reduces (chunk-summarize → reduce-merge) to stay within the LLM's context window. Default 400 ≈ 50k tokens per chunk at ~110 tok/obs. Native sessions are capped by MAX_OBS_PER_SESSION; chunking primarily matters for bulk-imported jsonl sessions, which bypass that cap. # SUMMARIZE_CHUNK_CONCURRENCY=6 # Parallel chunk LLM calls during chunked summarize. Default 6 fits ~100-chunk sessions under iii's 180s function-invocation timeout at typical ~8s/call. High-throughput providers (Novita, DeepInfra, DeepSeek) commonly allow 100+ concurrent — bump this for very large imported sessions. +# AGENTMEMORY_SUMMARY_TIMEOUT_MS=120000 # Total time budget in whole milliseconds for one mem::summarize run, shared by every chunk call. Default 120000 (2 min). Raise it for slow or local LLMs (Ollama, LM Studio, vLLM) on large chunked sessions that fail with summary_timeout. Session-end and recovery summaries wait this budget plus 60 s; POST /agentmemory/summarize is still cut off by the 180 s HTTP worker timeout, but the summary keeps running and is saved. # ----------------------------------------------------------------------------- # 5. Behaviour flags diff --git a/deploy/coolify/docker-compose.yml b/deploy/coolify/docker-compose.yml index a229cb7692..224494001b 100644 --- a/deploy/coolify/docker-compose.yml +++ b/deploy/coolify/docker-compose.yml @@ -97,6 +97,7 @@ services: AGENTMEMORY_GRAPH_WEIGHT: "${AGENTMEMORY_GRAPH_WEIGHT:-}" SUMMARIZE_CHUNK_SIZE: "${SUMMARIZE_CHUNK_SIZE:-}" SUMMARIZE_CHUNK_CONCURRENCY: "${SUMMARIZE_CHUNK_CONCURRENCY:-}" + AGENTMEMORY_SUMMARY_TIMEOUT_MS: "${AGENTMEMORY_SUMMARY_TIMEOUT_MS:-}" CONSOLIDATION_DECAY_DAYS: "${CONSOLIDATION_DECAY_DAYS:-}" CONSOLIDATION_INTERVAL_MS: "${CONSOLIDATION_INTERVAL_MS:-}" AGENTMEMORY_CONSOLIDATION_COOLDOWN_MS: "${AGENTMEMORY_CONSOLIDATION_COOLDOWN_MS:-}" diff --git a/deploy/runtime/worker-compose.yaml b/deploy/runtime/worker-compose.yaml index c1c1092887..cc25dbbb3d 100644 --- a/deploy/runtime/worker-compose.yaml +++ b/deploy/runtime/worker-compose.yaml @@ -156,6 +156,7 @@ containers: AGENTMEMORY_GRAPH_WEIGHT: "${AGENTMEMORY_GRAPH_WEIGHT:-}" SUMMARIZE_CHUNK_SIZE: "${SUMMARIZE_CHUNK_SIZE:-}" SUMMARIZE_CHUNK_CONCURRENCY: "${SUMMARIZE_CHUNK_CONCURRENCY:-}" + AGENTMEMORY_SUMMARY_TIMEOUT_MS: "${AGENTMEMORY_SUMMARY_TIMEOUT_MS:-}" CONSOLIDATION_DECAY_DAYS: "${CONSOLIDATION_DECAY_DAYS:-}" CONSOLIDATION_INTERVAL_MS: "${CONSOLIDATION_INTERVAL_MS:-}" AGENTMEMORY_CONSOLIDATION_COOLDOWN_MS: "${AGENTMEMORY_CONSOLIDATION_COOLDOWN_MS:-}" diff --git a/docker-compose.yml b/docker-compose.yml index cd236d8243..f71608a0fc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -103,6 +103,7 @@ services: AGENTMEMORY_GRAPH_WEIGHT: "${AGENTMEMORY_GRAPH_WEIGHT:-}" SUMMARIZE_CHUNK_SIZE: "${SUMMARIZE_CHUNK_SIZE:-}" SUMMARIZE_CHUNK_CONCURRENCY: "${SUMMARIZE_CHUNK_CONCURRENCY:-}" + AGENTMEMORY_SUMMARY_TIMEOUT_MS: "${AGENTMEMORY_SUMMARY_TIMEOUT_MS:-}" CONSOLIDATION_DECAY_DAYS: "${CONSOLIDATION_DECAY_DAYS:-}" CONSOLIDATION_INTERVAL_MS: "${CONSOLIDATION_INTERVAL_MS:-}" AGENTMEMORY_CONSOLIDATION_COOLDOWN_MS: "${AGENTMEMORY_CONSOLIDATION_COOLDOWN_MS:-}" diff --git a/plugin/skills/agentmemory-config/REFERENCE.md b/plugin/skills/agentmemory-config/REFERENCE.md index 891f4d156c..e1a56c17af 100644 --- a/plugin/skills/agentmemory-config/REFERENCE.md +++ b/plugin/skills/agentmemory-config/REFERENCE.md @@ -3,7 +3,7 @@ Generated by scanning `src/` for `AGENTMEMORY_*` usage. Do not edit the block below by hand; run `npm run skills:gen` after adding or removing a variable. Internal markers ending in two underscores are excluded. -Configuration is read from the environment and from `~/.agentmemory/.env` (no `export` prefix). 76 recognized variables: +Configuration is read from the environment and from `~/.agentmemory/.env` (no `export` prefix). 77 recognized variables: - `AGENTMEMORY_AGENT_SCOPE` - `AGENTMEMORY_ALLOW_AGENT_SDK` @@ -70,6 +70,7 @@ Configuration is read from the environment and from `~/.agentmemory/.env` (no `e - `AGENTMEMORY_SLOTS` - `AGENTMEMORY_STATE_BACKEND` - `AGENTMEMORY_STATE_SAVE_INTERVAL_MS` +- `AGENTMEMORY_SUMMARY_TIMEOUT_MS` - `AGENTMEMORY_SUPPRESS_COST_WARNING` - `AGENTMEMORY_TOOLS` - `AGENTMEMORY_URL` diff --git a/src/cli/compose-project.ts b/src/cli/compose-project.ts index 687284f219..c7326cf1bb 100644 --- a/src/cli/compose-project.ts +++ b/src/cli/compose-project.ts @@ -78,7 +78,8 @@ const AGENTMEMORY_WORKER_ENV_ALLOWLIST = new Set([ "AGENTMEMORY_CAPTURE_SPOOL", "AGENTMEMORY_CAPTURE_SPOOL_DIR", "AGENTMEMORY_CAPTURE_SPOOL_MAX_BYTES", "AGENTMEMORY_CAPTURE_SPOOL_MAX_AGE_HOURS", "AGENTMEMORY_IMPORT_ROOT", "AGENTMEMORY_EXPORT_ROOT", "AGENTMEMORY_GRAPH_WEIGHT", - "SUMMARIZE_CHUNK_SIZE", "SUMMARIZE_CHUNK_CONCURRENCY", "CONSOLIDATION_DECAY_DAYS", + "SUMMARIZE_CHUNK_SIZE", "SUMMARIZE_CHUNK_CONCURRENCY", "AGENTMEMORY_SUMMARY_TIMEOUT_MS", + "CONSOLIDATION_DECAY_DAYS", "CONSOLIDATION_INTERVAL_MS", "AGENTMEMORY_CONSOLIDATION_COOLDOWN_MS", "AGENTMEMORY_FOLLOWUP_WINDOW_SECONDS", "GRAPH_EXTRACTION_BATCH_SIZE", "AGENTMEMORY_LLM_NOTHINK", "AGENTMEMORY_REFLECT", "AGENTMEMORY_IMAGE_EMBEDDINGS", "AGENTMEMORY_IMAGE_STORE_MAX_BYTES", diff --git a/src/functions/summarize.ts b/src/functions/summarize.ts index 5e984bac1c..54c6eedf39 100644 --- a/src/functions/summarize.ts +++ b/src/functions/summarize.ts @@ -25,13 +25,12 @@ import { logger } from "../logger.js"; import { scrubRecord } from "./privacy.js"; import { registerSummaryRecoveryFunction } from "./summary-recovery.js"; import { withKeyedLock } from "../state/keyed-mutex.js"; -import { summaryFingerprint, summaryObservations } from "./summary-state.js"; +import { getSummaryTimeoutMs, summaryFingerprint, summaryObservations } from "./summary-state.js"; // Per-chunk observation budget when a session is too large to fit in one // LLM call. Default ≈ 50k input tokens per chunk at ~110 tok/obs — fits // comfortably in 128k-window models. Override via SUMMARIZE_CHUNK_SIZE. const CHUNK_SIZE_DEFAULT = 400; -export const SUMMARY_TIMEOUT_MS = 120_000; // Concurrent chunk calls share the summary deadline. Increase this for // large sessions only when the configured provider supports the load. const CHUNK_CONCURRENCY_DEFAULT = 6; @@ -316,11 +315,12 @@ export function registerSummarizeFunction( }; } + const timeoutMs = getSummaryTimeoutMs(); const boundedProvider: MemoryProvider = { name: provider.name, compress: (system, prompt) => provider.compress(system, prompt), summarize: async (system, prompt) => { - const remaining = SUMMARY_TIMEOUT_MS - (Date.now() - startMs); + const remaining = timeoutMs - (Date.now() - startMs); if (remaining <= 0) throw new Error("summary_timeout"); let timer: ReturnType | undefined; try { diff --git a/src/functions/summary-recovery.ts b/src/functions/summary-recovery.ts index 9daaa1c5c4..6a6348f899 100644 --- a/src/functions/summary-recovery.ts +++ b/src/functions/summary-recovery.ts @@ -6,7 +6,7 @@ import { withKeyedLock } from "../state/keyed-mutex.js"; import { logger } from "../logger.js"; import { compressionInFlight } from "./compress.js"; import { compressionRaw, needsCompression, type StoredObservation } from "./compression-state.js"; -import { summaryCovers, summaryObservations } from "./summary-state.js"; +import { getSummaryTriggerTimeoutMs, summaryCovers, summaryObservations } from "./summary-state.js"; export const SUMMARY_RETRY_MS = 30_000; export const SUMMARY_RETRY_MAX_MS = 60 * 60_000; @@ -49,7 +49,7 @@ export function registerSummaryRecoveryFunction(sdk: IIIClient, kv: StateKV, has const remaining = observations.filter(needsCompression); if (remaining.length === 0) { const result = await sdk.trigger<{ sessionId: string }, { success?: boolean; incomplete?: boolean }>({ - function_id: "mem::summarize", payload: { sessionId }, + function_id: "mem::summarize", payload: { sessionId }, timeoutMs: getSummaryTriggerTimeoutMs(), }); if (!result?.success || result.incomplete) lastError = "summary_failed"; } else if (remaining.every((row) => !compressionRaw(row))) { diff --git a/src/functions/summary-state.ts b/src/functions/summary-state.ts index 37321b5c59..63400332b3 100644 --- a/src/functions/summary-state.ts +++ b/src/functions/summary-state.ts @@ -4,6 +4,21 @@ import type { StateKV } from "../state/kv.js"; import { KV } from "../state/schema.js"; import { isKeyedLockHeld, withKeyedLock } from "../state/keyed-mutex.js"; +export const SUMMARY_TIMEOUT_MS = 120_000; +const SUMMARY_TRIGGER_HEADROOM_MS = 60_000; + +const MAX_TIMER_MS = 2_147_483_647; + +export function getSummaryTimeoutMs(): number { + const raw = process.env.AGENTMEMORY_SUMMARY_TIMEOUT_MS?.trim() ?? ""; + const n = /^\d+$/.test(raw) ? Number(raw) : 0; + return n >= 1 && n <= MAX_TIMER_MS - SUMMARY_TRIGGER_HEADROOM_MS ? n : SUMMARY_TIMEOUT_MS; +} + +export function getSummaryTriggerTimeoutMs(): number { + return getSummaryTimeoutMs() + SUMMARY_TRIGGER_HEADROOM_MS; +} + export function summaryObservations(observations: CompressedObservation[]): CompressedObservation[] { return observations.filter((row) => row.title).sort((a, b) => (a.compressedAt ?? a.timestamp).localeCompare(b.compressedAt ?? b.timestamp) || a.id.localeCompare(b.id)); diff --git a/src/triggers/api.ts b/src/triggers/api.ts index a00f754ef4..55b28106de 100644 --- a/src/triggers/api.ts +++ b/src/triggers/api.ts @@ -58,7 +58,7 @@ import { isValidEventId } from "../capture/event-id.js"; import { getCaptureController, type CaptureResult } from "../functions/capture.js"; import { withoutObservationSource } from "../functions/observation-source.js"; import { clearSessionFinalize, scheduleSessionFinalize } from "../functions/session-finalize.js"; -import { requestSummaryRefresh } from "../functions/summary-state.js"; +import { getSummaryTriggerTimeoutMs, requestSummaryRefresh } from "../functions/summary-state.js"; import { registerCompressionApi } from "./compression.js"; import { getFinalizeIdleMs, @@ -1243,6 +1243,7 @@ export function registerApiTriggers( const result = await sdk.trigger({ function_id: "mem::summarize", payload: { sessionId }, + timeoutMs: getSummaryTriggerTimeoutMs(), }); return { status_code: 200, body: result }; }, diff --git a/src/triggers/events.ts b/src/triggers/events.ts index 4c887f5f9c..e11187cd75 100644 --- a/src/triggers/events.ts +++ b/src/triggers/events.ts @@ -12,6 +12,7 @@ import { isConsolidationEnabled, } from "../config.js"; import { logger } from "../logger.js"; +import { getSummaryTriggerTimeoutMs } from "../functions/summary-state.js"; import { noteMemoryChange, noteSessionChange } from "../state/viewer-counts.js"; // Global marker recording when corpus consolidation last ran, used to debounce @@ -49,6 +50,7 @@ export function registerEventTriggers(sdk: IIIClient, kv: StateKV): void { const summary = await sdk.trigger({ function_id: "mem::summarize", payload: data, + timeoutMs: getSummaryTriggerTimeoutMs(), }); if (summary?.unchanged === true) return summary; const fireVoid = (function_id: string, payload: unknown) => diff --git a/test/events-consolidation.test.ts b/test/events-consolidation.test.ts index 0c082ae2bb..b4c337d4c9 100644 --- a/test/events-consolidation.test.ts +++ b/test/events-consolidation.test.ts @@ -114,6 +114,23 @@ describe("event::session::stopped consolidation fan-out", () => { }); }); + it("waits on mem::summarize for longer than the configured summary budget", async () => { + const previous = process.env.AGENTMEMORY_SUMMARY_TIMEOUT_MS; + process.env.AGENTMEMORY_SUMMARY_TIMEOUT_MS = "600000"; + try { + const { sdk, handlers, trigger } = mockSdk(); + registerEventTriggers(sdk as never, mockKV() as never); + await handlers.get("event::session::stopped")!({ sessionId: "ses_1" }); + const summarizeCall = trigger.mock.calls.find( + (c) => (c[0] as { function_id: string }).function_id === "mem::summarize", + ); + expect((summarizeCall![0] as { timeoutMs?: number }).timeoutMs).toBeGreaterThan(600_000); + } finally { + if (previous === undefined) delete process.env.AGENTMEMORY_SUMMARY_TIMEOUT_MS; + else process.env.AGENTMEMORY_SUMMARY_TIMEOUT_MS = previous; + } + }); + it("skips consolidate-pipeline and auto-crystallize when consolidation disabled but still summarizes", async () => { vi.mocked(isConsolidationEnabled).mockReturnValue(false); const { sdk, handlers, trigger } = mockSdk(); diff --git a/test/summarize.test.ts b/test/summarize.test.ts index 224bab21f8..649a6cc850 100644 --- a/test/summarize.test.ts +++ b/test/summarize.test.ts @@ -496,3 +496,68 @@ describe("mem::summarize without an LLM provider", () => { expect(record).not.toHaveBeenCalled(); }); }); + +describe("mem::summarize deadline", () => { + const ORIGINAL_ENV = { ...process.env }; + + beforeEach(() => { + delete process.env.SUMMARIZE_CHUNK_SIZE; + delete process.env.SUMMARIZE_CHUNK_CONCURRENCY; + delete process.env.AGENTMEMORY_SUMMARY_TIMEOUT_MS; + }); + + afterEach(() => { + vi.useRealTimers(); + process.env = { ...ORIGINAL_ENV }; + }); + + function slowProvider(delayMs: number, responses: string[]) { + const provider = makeProvider(responses); + const inner = provider.summarize; + provider.summarize = async (system: string, user: string) => { + await new Promise((resolve) => setTimeout(resolve, delayMs)); + return inner(system, user); + }; + return provider; + } + + async function runChunked(delayMs: number) { + process.env.SUMMARIZE_CHUNK_SIZE = "10"; + process.env.SUMMARIZE_CHUNK_CONCURRENCY = "1"; + const responses = [ + ...Array.from({ length: 4 }, (_, i) => summaryXml({ title: `chunk ${i}` })), + summaryXml({ title: "merged" }), + ]; + const provider = slowProvider(delayMs, responses); + const { handler } = await setupHandler({ sessionId: "ses_slow", obsCount: 40, provider }); + vi.useFakeTimers(); + const pending = handler({ sessionId: "ses_slow" }) as Promise; + await vi.advanceTimersByTimeAsync(10 * delayMs); + return { result: await pending, provider }; + } + + it("keeps the default 120 s budget for a multi-chunk session", async () => { + const { result } = await runChunked(30_000); + expect(result).toMatchObject({ success: false }); + expect(result.error).toMatch(/summary_timeout|too_many_chunks_skipped/); + }); + + it("AGENTMEMORY_SUMMARY_TIMEOUT_MS extends the budget so slow chunked sessions finish", async () => { + process.env.AGENTMEMORY_SUMMARY_TIMEOUT_MS = "600000"; + const { result, provider } = await runChunked(30_000); + expect(result).toMatchObject({ success: true }); + expect(result.summary.title).toBe("merged"); + expect(provider.calls).toHaveLength(5); + }); + + it("ignores an invalid AGENTMEMORY_SUMMARY_TIMEOUT_MS", async () => { + const { getSummaryTimeoutMs, getSummaryTriggerTimeoutMs } = await import("../src/functions/summary-state.js"); + for (const bad of ["nope", "600s", "1e6", "0.5", "0", "2592000000"]) { + process.env.AGENTMEMORY_SUMMARY_TIMEOUT_MS = bad; + expect(getSummaryTimeoutMs()).toBe(120_000); + } + process.env.AGENTMEMORY_SUMMARY_TIMEOUT_MS = "300000"; + expect(getSummaryTimeoutMs()).toBe(300_000); + expect(getSummaryTriggerTimeoutMs()).toBeGreaterThan(300_000); + }); +});