Skip to content
Merged
3 changes: 3 additions & 0 deletions .env.example
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,9 @@
# AGENTMEMORY_DROP_STALE_INDEX=true # Drop on-disk BM25 / vector index on startup if dim guard fires (#248). Recovery toggle for stuck-state debugging.
# AGENTMEMORY_IMAGE_EMBEDDINGS=true # Enable image embeddings when an image provider is present (experimental).
# AGENTMEMORY_INDEX_SAVE_INTERVAL_MS=600000 # Minimum time between search index saves; shutdown and deletes still save immediately.
# AGENTMEMORY_VECTOR_BUCKET_SIZE=500 # Vectors held per storage bucket before a new bucket opens; a save only rewrites buckets it changed.
# AGENTMEMORY_VECTOR_BACKFILL_MAX=500 # Cap on vectors embedded per boot to backfill ones missing since the last save.
# AGENTMEMORY_VECTOR_BACKFILL=all # Required to backfill an entire store's vectors at once (no prior vector save, or a stale legacy manifest); unset means that case waits for an explicit opt-in.

# -----------------------------------------------------------------------------
# 6. CLI / runtime knobs
Expand Down
5 changes: 4 additions & 1 deletion plugin/skills/agentmemory-config/REFERENCE.md
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
Generated by scanning `src/` for `AGENTMEMORY_*` usage. Do not edit the block below by hand; run `npm run skills:gen` after adding or removing a variable. Internal markers ending in two underscores are excluded.

<!-- AUTOGEN:env START - generated by scripts/skills/generate.ts, do not edit by hand -->
Configuration is read from the environment and from `~/.agentmemory/.env` (no `export` prefix). 44 recognized variables:
Configuration is read from the environment and from `~/.agentmemory/.env` (no `export` prefix). 47 recognized variables:

- `AGENTMEMORY_AGENT_SCOPE`
- `AGENTMEMORY_ALLOW_AGENT_SDK`
Expand Down Expand Up @@ -46,6 +46,9 @@ Configuration is read from the environment and from `~/.agentmemory/.env` (no `e
- `AGENTMEMORY_TOOLS`
- `AGENTMEMORY_URL`
- `AGENTMEMORY_USE_DOCKER`
- `AGENTMEMORY_VECTOR_BACKFILL`
- `AGENTMEMORY_VECTOR_BACKFILL_MAX`
- `AGENTMEMORY_VECTOR_BUCKET_SIZE`
- `AGENTMEMORY_VERBOSE`
- `AGENTMEMORY_VIEWER_HOST`
- `AGENTMEMORY_VIEWER_URL`
Expand Down
18 changes: 18 additions & 0 deletions src/config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -477,6 +477,24 @@ export function getIndexSaveIntervalMs(): number {
return raw > 0 ? raw : INDEX_SAVE_INTERVAL_DEFAULT_MS;
}

export const VECTOR_BUCKET_SIZE_DEFAULT = 500;

export function getVectorBucketSize(): number {
const raw = safeParseInt(getMergedEnv()["AGENTMEMORY_VECTOR_BUCKET_SIZE"], VECTOR_BUCKET_SIZE_DEFAULT);
return raw > 0 ? raw : VECTOR_BUCKET_SIZE_DEFAULT;
}

export const VECTOR_BACKFILL_MAX_DEFAULT = 500;

export function getVectorBackfillMax(): number {
const raw = safeParseInt(getMergedEnv()["AGENTMEMORY_VECTOR_BACKFILL_MAX"], VECTOR_BACKFILL_MAX_DEFAULT);
return raw > 0 ? raw : VECTOR_BACKFILL_MAX_DEFAULT;
}

export function isVectorBackfillAllEnabled(): boolean {
return getMergedEnv()["AGENTMEMORY_VECTOR_BACKFILL"] === "all";
}

export function isStandaloneMcp(): boolean {
return getMergedEnv()["STANDALONE_MCP"] === "true";
}
Expand Down
168 changes: 163 additions & 5 deletions src/functions/search.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,12 @@ import type { EmbeddingProvider } from '../types.js'
import { memoryToObservation } from '../state/memory-utils.js'
import { recordAccessBatch } from './access-tracker.js'
import { logger } from "../logger.js";
import { getAgentId, isAgentScopeIsolated } from "../config.js";
import {
getAgentId,
getVectorBackfillMax,
isAgentScopeIsolated,
isVectorBackfillAllEnabled,
} from "../config.js";

let index: SearchIndex | null = null
let vectorIndex: VectorIndex | null = null
Expand Down Expand Up @@ -45,6 +50,19 @@ export function isMemoryIndexReady(): boolean {
return memoryIndexReady
}

let bm25RebuildIncomplete = false
export function isBm25RebuildIncomplete(): boolean {
return bm25RebuildIncomplete
}

let pendingVectorBackfill = 0
export function getPendingVectorBackfillCount(): number {
return pendingVectorBackfill
}
export function setPendingVectorBackfillCount(n: number): void {
pendingVectorBackfill = Math.max(0, n)
}

export function getSearchIndex(): SearchIndex {
if (!index) index = new SearchIndex()
return index
Expand Down Expand Up @@ -399,6 +417,146 @@ export async function rebuildIndex(kv: StateKV): Promise<number> {
return indexed
}

export type VectorBackfillJob = {
id: string
sessionId: string
text: string
context: { kind: "memory" | "observation" | "synthetic"; logId: string }
}

const SESSIONS_LIST_RETRY_ATTEMPTS = 3
const SESSIONS_LIST_RETRY_BASE_MS = 250

function delay(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms))
}

async function listSessionsWithRetry(kv: StateKV): Promise<Session[] | null> {
let lastErr: unknown
for (let attempt = 0; attempt < SESSIONS_LIST_RETRY_ATTEMPTS; attempt++) {
try {
return await kv.list<Session>(KV.sessions)
} catch (err) {
lastErr = err
if (attempt < SESSIONS_LIST_RETRY_ATTEMPTS - 1) {
await delay(SESSIONS_LIST_RETRY_BASE_MS * (attempt + 1))
}
}
}
logger.warn('rebuildKeywordIndex: failed to load sessions after retries', {
attempts: SESSIONS_LIST_RETRY_ATTEMPTS,
error: lastErr instanceof Error ? lastErr.message : String(lastErr),
})
return null
}

export async function rebuildKeywordIndex(
kv: StateKV,
vectorBackfillSince?: string | null,
): Promise<{ documents: number; vectorJobs: VectorBackfillJob[]; fullBackfillPending: number }> {
const idx = getSearchIndex()
idx.clear()
memoryIndexReady = false
bm25RebuildIncomplete = false
const vi = vectorIndex
const backfillEligible = Boolean(vi && currentEmbeddingProvider) && vectorBackfillSince !== undefined
const wholeStoreBackfill = vectorBackfillSince === null
const backfillGated = backfillEligible && wholeStoreBackfill && !isVectorBackfillAllEnabled()
const backfillActive = backfillEligible && !backfillGated
const cutoff = typeof vectorBackfillSince === 'string' ? Date.parse(vectorBackfillSince) : Number.NaN
const backfillCap = backfillActive ? getVectorBackfillMax() : 0
const vectorJobs: VectorBackfillJob[] = []
let fullBackfillPending = 0
const consider = (
id: string,
sessionId: string,
text: string,
timestamp: string | undefined,
kind: "memory" | "observation",
): void => {
if (!backfillEligible || vi?.has(id)) return
if (!Number.isNaN(cutoff) && !(Date.parse(timestamp ?? '') > cutoff)) return
if (backfillGated) {
fullBackfillPending++
return
}
if (!backfillActive || vectorJobs.length >= backfillCap) return
vectorJobs.push({ id, sessionId, text, context: { kind, logId: id } })
}

let documents = 0
let memoriesLoaded = false
try {
const memories = await kv.list<Memory>(KV.memories)
memoriesLoaded = true
for (const memory of memories) {
if (memory.isLatest === false) continue
if (!memory.title || !memory.content) continue
idx.add(memoryToObservation(memory))
consider(memory.id, memory.sessionIds?.[0] ?? 'memory', memory.title + ' ' + memory.content, memory.createdAt, "memory")
documents++
}
} catch (err) {
logger.warn('rebuildKeywordIndex: failed to load memories', {
error: err instanceof Error ? err.message : String(err),
})
}

const sessions = await listSessionsWithRetry(kv)
if (sessions === null) {
bm25RebuildIncomplete = true
if (memoriesLoaded) memoryIndexReady = true
return { documents, vectorJobs, fullBackfillPending }
}
const failedSessions: string[] = []
for (let batch = 0; batch < sessions.length; batch += 10) {
const chunk = sessions.slice(batch, batch + 10)
const results = await Promise.all(
chunk.map(async (s) => {
try {
return await kv.list<CompressedObservation>(KV.observations(s.id))
} catch {
failedSessions.push(s.id)
return [] as CompressedObservation[]
}
})
)
for (const obs of results.flat()) {
if (!obs.title || !obs.narrative) continue
idx.add(obs)
consider(obs.id, obs.sessionId, obs.title + ' ' + obs.narrative, obs.timestamp, "observation")
documents++
}
}
if (failedSessions.length > 0) {
bm25RebuildIncomplete = true
logger.warn('rebuildKeywordIndex: failed to load observations for sessions', { failedSessions })
}
if (memoriesLoaded) memoryIndexReady = true
return { documents, vectorJobs, fullBackfillPending }
}

const BACKFILL_SAVE_EVERY_BATCHES = 10

export async function backfillVectors(jobs: VectorBackfillJob[]): Promise<number> {
const batchSize = getRebuildEmbedBatchSize()
let added = 0
let batchesSinceSave = 0
setPendingVectorBackfillCount(jobs.length)
for (let offset = 0; offset < jobs.length; offset += batchSize) {
const { ok } = await vectorIndexAddBatchGuarded(jobs.slice(offset, offset + batchSize))
added += ok
setPendingVectorBackfillCount(jobs.length - Math.min(jobs.length, offset + batchSize))
batchesSinceSave++
if (batchesSinceSave >= BACKFILL_SAVE_EVERY_BATCHES) {
await flushIndexSave()
batchesSinceSave = 0
}
}
if (added > 0) await flushIndexSave()
return added
}

export function registerSearchFunction(sdk: IIIClient, kv: StateKV): void {
sdk.registerFunction(
'mem::search',
Expand Down Expand Up @@ -480,10 +638,10 @@ export function registerSearchFunction(sdk: IIIClient, kv: StateKV): void {
// Share one rebuild across concurrent cold-start queries so they
// don't each walk the whole corpus and saturate the pool.
if (!rebuildPromise) {
rebuildPromise = rebuildIndex(kv)
.then((count) => {
logger.info('Search index rebuilt', { entries: count })
return count
rebuildPromise = rebuildKeywordIndex(kv)
.then((result) => {
logger.info('Search index rebuilt', { entries: result.documents })
return result.documents
})
.catch((err) => {
logger.warn('Index rebuild failed', {
Expand Down
90 changes: 51 additions & 39 deletions src/functions/status.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { V8_MAX_STRING_CHARS, type IndexLegStatus, type IndexPersistenceStatus } from "../state/index-persistence.js";
import type { IndexLegStatus, IndexPersistenceStatus } from "../state/index-persistence.js";

export type StatusLevel = "ok" | "info" | "warn" | "error";

Expand Down Expand Up @@ -57,6 +57,8 @@ export interface StatusInputs {
observationsIndexed: number;
missingObservations: number | null;
sessions: number | null;
bm25Incomplete: boolean;
pendingVectorBackfill: number;
};
graph: GraphStatsInput | null;
graphExtractionEnabled: boolean;
Expand Down Expand Up @@ -85,7 +87,6 @@ export interface StatusReport {
const FAILURE_RATE_THRESHOLD = 0.2;
const FAILURE_MIN_CALLS = 5;
const GRAPH_SNAPSHOT_STALE_SECONDS = 24 * 60 * 60;
const VECTOR_STRING_WARN_RATIO = 0.8;
const LEVEL_RANK: Record<StatusLevel, number> = { ok: 0, info: 1, warn: 2, error: 3 };

const FUNCTION_FIXES: Record<string, string> = {
Expand Down Expand Up @@ -173,42 +174,54 @@ export function evaluateStatus(input: StatusInputs): StatusReport {
});
}

if (input.index.bm25Incomplete) {
problems.push({
level: "error",
code: "bm25-rebuild-incomplete",
message: "The keyword index could not load every session at boot, so some observations are missing from search until the next successful rebuild.",
fix: "Check the server log for the session listing failure. Restarting or the next cold-start search retries automatically.",
});
}

if (input.index.pendingVectorBackfill > 0) {
problems.push({
level: "info",
code: "index-vector-backfill-pending",
message: `${input.index.pendingVectorBackfill} documents are waiting for a vector embedding.`,
fix: "This runs in the background and is capped per boot by AGENTMEMORY_VECTOR_BACKFILL_MAX. If it is stuck at a nonzero count with no embedding provider errors, set AGENTMEMORY_VECTOR_BACKFILL=all to run a full backfill.",
});
}

const persistence = input.indexPersistence ?? null;
if (persistence) {
const legs: Array<[string, IndexLegStatus | null]> = [
["BM25", persistence.bm25],
["vector", persistence.vector],
];
for (const [label, leg] of legs) {
if (!leg) continue;
if (leg.lastError) {
problems.push({
level: "error",
code: "index-save-failing",
message: `The ${label} search index could not be saved: ${leg.lastError}. Search keeps working from memory, but changes since the last save are lost on restart.`,
fix: "Check the server log for the failing state write. The next save retries automatically.",
});
}
const dirtyAge = secondsBetween(input.now, leg.dirtySince ?? undefined);
if (!leg.lastError && dirtyAge !== null && dirtyAge * 1000 > 2 * persistence.saveIntervalMs) {
problems.push({
level: "warn",
code: "index-save-stale",
message: `The ${label} search index has unsaved changes from ${formatDuration(dirtyAge)} ago.`,
fix: "Saves run at most once per AGENTMEMORY_INDEX_SAVE_INTERVAL_MS. Check the server log for save errors or a save that never finishes.",
});
}
const vectorLeg = persistence?.vector ?? null;
if (persistence && vectorLeg) {
if (vectorLeg.lastError) {
problems.push({
level: "error",
code: "index-save-failing",
message: `The vector index could not be saved: ${vectorLeg.lastError}. Search keeps working from memory, but vectors added since the last save are lost on restart.`,
fix: "Check the server log for the failing state write. The next save retries automatically.",
});
}
const vectorChars = persistence.vector?.serializedChars ?? null;
if (vectorChars !== null && vectorChars > V8_MAX_STRING_CHARS * VECTOR_STRING_WARN_RATIO) {
const dirtyAge = secondsBetween(input.now, vectorLeg.dirtySince ?? undefined);
if (!vectorLeg.lastError && dirtyAge !== null && dirtyAge * 1000 > 2 * persistence.saveIntervalMs) {
problems.push({
level: "warn",
code: "vector-index-near-string-limit",
message: `The saved vector index is ${Math.round((vectorChars / V8_MAX_STRING_CHARS) * 100)}% of the largest string Node.js can hold. Past that limit it can no longer be saved or loaded.`,
fix: "Upgrade agentmemory once bucketed vector storage ships, or reduce the index (a smaller embedding model, or forget unused observations) before it reaches the limit.",
code: "index-save-stale",
message: `The vector index has unsaved changes from ${formatDuration(dirtyAge)} ago.`,
fix: "Saves run at most once per AGENTMEMORY_INDEX_SAVE_INTERVAL_MS. Check the server log for save errors or a save that never finishes.",
});
}
}
if (persistence?.vectorCountShortfall) {
const { expected, loaded } = persistence.vectorCountShortfall;
problems.push({
level: "warn",
code: "index-vector-count-shortfall",
message: `Only ${loaded} of ${expected} vectors loaded from the last save; a bounded backfill re-embeds the rest.`,
fix: "No action needed — missing vectors are queued for backfill up to the per-boot cap.",
});
}

let graph: StatusReport["graph"] = null;
if (input.graph) {
Expand Down Expand Up @@ -300,16 +313,13 @@ function legSummary(report: StatusReport, leg: IndexLegStatus): string {
function indexPersistenceRows(report: StatusReport): string {
const persistence = report.indexPersistence;
if (!persistence) return "";
let rows = row("BM25 save", escapeHtml(legSummary(report, persistence.bm25)));
let rows = row("BM25 index", "rebuilt from stored content at boot");
if (persistence.vector) {
rows += row("Vector save", escapeHtml(legSummary(report, persistence.vector)));
const chars = persistence.vector.serializedChars;
if (chars !== null) {
rows += row(
"Vector index size",
escapeHtml(`${chars.toLocaleString("en-US")} characters (${Math.round((chars / V8_MAX_STRING_CHARS) * 100)}% of the Node.js string limit)`),
);
}
rows += row(
"Vector storage",
escapeHtml(`${persistence.buckets} buckets, ${persistence.pendingChanges} unsaved changes`),
);
}
return rows;
}
Expand Down Expand Up @@ -396,6 +406,8 @@ ${row("Vector documents", escapeHtml(idx.vectorDocuments ?? "vector search off")
${row("Observations indexed", escapeHtml(idx.observationsIndexed))}
${row("Missing from index", escapeHtml(idx.missingObservations ?? "not checked"))}
${row("Sessions", escapeHtml(idx.sessions ?? "unknown"))}
${row("BM25 rebuild", idx.bm25Incomplete ? '<span class="warn">incomplete</span>' : "complete")}
${row("Pending vector backfill", escapeHtml(idx.pendingVectorBackfill))}
${indexPersistenceRows(report)}
</table>
<h2>Knowledge graph</h2><table>
Expand Down
Loading
Loading