diff --git a/.env.example b/.env.example index 98e21b906f..90169e8bb4 100644 --- a/.env.example +++ b/.env.example @@ -129,6 +129,8 @@ # AGENTMEMORY_REFLECT=true # Periodically auto-synthesize lessons from memories # AGENTMEMORY_DROP_STALE_INDEX=true # Drop on-disk BM25 / vector index on startup if dim guard fires (#248). Recovery toggle for stuck-state debugging. # AGENTMEMORY_IMAGE_EMBEDDINGS=true # Enable image embeddings when an image provider is present (experimental). +# AGENTMEMORY_AUDIT_RETENTION_MONTHS=6 # Drop whole audit-log month scopes older than N months on a daily sweep. Default 0 = keep audit history forever. +# AGENTMEMORY_AUDIT_MIGRATE_MAX_BYTES=33554432 # Largest legacy mem:audit scope file the one-time migration will read locally. Default 32 MiB; bigger or unreadable stores are left frozen. # AGENTMEMORY_INDEX_SAVE_INTERVAL_MS=600000 # Minimum time between search index saves; shutdown and deletes still save immediately. # AGENTMEMORY_VECTOR_BUCKET_SIZE=500 # Vectors held per storage bucket before a new bucket opens; a save only rewrites buckets it changed. # AGENTMEMORY_VECTOR_BACKFILL_MAX=500 # Cap on vectors embedded per boot to backfill ones missing since the last save. diff --git a/deploy/coolify/entrypoint.sh b/deploy/coolify/entrypoint.sh index c21ec1d898..808f7fe131 100755 --- a/deploy/coolify/entrypoint.sh +++ b/deploy/coolify/entrypoint.sh @@ -16,6 +16,7 @@ set -eu DATA_DIR="${AGENTMEMORY_DATA_DIR:-/data}" +export AGENTMEMORY_DATA_DIR="$DATA_DIR" HMAC_FILE="${AGENTMEMORY_HMAC_FILE:-/data/.hmac}" RUN_AS="node:node" III_CONFIG="/opt/agentmemory/node_modules/@agentmemory/agentmemory/dist/iii-config.yaml" diff --git a/deploy/fly/entrypoint.sh b/deploy/fly/entrypoint.sh index 5b20f45113..befe7106d4 100755 --- a/deploy/fly/entrypoint.sh +++ b/deploy/fly/entrypoint.sh @@ -16,6 +16,7 @@ set -eu DATA_DIR="${AGENTMEMORY_DATA_DIR:-/data}" +export AGENTMEMORY_DATA_DIR="$DATA_DIR" HMAC_FILE="${AGENTMEMORY_HMAC_FILE:-/data/.hmac}" RUN_AS="node:node" III_CONFIG="/opt/agentmemory/node_modules/@agentmemory/agentmemory/dist/iii-config.yaml" diff --git a/deploy/railway/entrypoint.sh b/deploy/railway/entrypoint.sh index c21ec1d898..808f7fe131 100755 --- a/deploy/railway/entrypoint.sh +++ b/deploy/railway/entrypoint.sh @@ -16,6 +16,7 @@ set -eu DATA_DIR="${AGENTMEMORY_DATA_DIR:-/data}" +export AGENTMEMORY_DATA_DIR="$DATA_DIR" HMAC_FILE="${AGENTMEMORY_HMAC_FILE:-/data/.hmac}" RUN_AS="node:node" III_CONFIG="/opt/agentmemory/node_modules/@agentmemory/agentmemory/dist/iii-config.yaml" diff --git a/deploy/render/entrypoint.sh b/deploy/render/entrypoint.sh index c21ec1d898..808f7fe131 100755 --- a/deploy/render/entrypoint.sh +++ b/deploy/render/entrypoint.sh @@ -16,6 +16,7 @@ set -eu DATA_DIR="${AGENTMEMORY_DATA_DIR:-/data}" +export AGENTMEMORY_DATA_DIR="$DATA_DIR" HMAC_FILE="${AGENTMEMORY_HMAC_FILE:-/data/.hmac}" RUN_AS="node:node" III_CONFIG="/opt/agentmemory/node_modules/@agentmemory/agentmemory/dist/iii-config.yaml" diff --git a/plugin/skills/agentmemory-config/REFERENCE.md b/plugin/skills/agentmemory-config/REFERENCE.md index 086ab4df02..f853443634 100644 --- a/plugin/skills/agentmemory-config/REFERENCE.md +++ b/plugin/skills/agentmemory-config/REFERENCE.md @@ -3,11 +3,13 @@ Generated by scanning `src/` for `AGENTMEMORY_*` usage. Do not edit the block below by hand; run `npm run skills:gen` after adding or removing a variable. Internal markers ending in two underscores are excluded. -Configuration is read from the environment and from `~/.agentmemory/.env` (no `export` prefix). 48 recognized variables: +Configuration is read from the environment and from `~/.agentmemory/.env` (no `export` prefix). 50 recognized variables: - `AGENTMEMORY_AGENT_SCOPE` - `AGENTMEMORY_ALLOW_AGENT_SDK` - `AGENTMEMORY_AUDIT_INDEX_PERSIST` +- `AGENTMEMORY_AUDIT_MIGRATE_MAX_BYTES` +- `AGENTMEMORY_AUDIT_RETENTION_MONTHS` - `AGENTMEMORY_AUTO_COMPRESS` - `AGENTMEMORY_COMMIT_SHA` - `AGENTMEMORY_CONSOLIDATION_COOLDOWN_MS` diff --git a/src/cli/engine-config.ts b/src/cli/engine-config.ts index b48f09dfa5..f5d6d5a4d6 100644 --- a/src/cli/engine-config.ts +++ b/src/cli/engine-config.ts @@ -28,6 +28,11 @@ export function configuredPersistDir(renderedConfig: string): string | null { return null; } +export function configuredSaveIntervalMs(renderedConfig: string): number | null { + const match = renderedConfig.match(/save_interval_ms:\s*(\d+)/); + return match ? parseInt(match[1]!, 10) : null; +} + export function persistedBuiltinConfigDirs( engineCwd: string, configPath: string, diff --git a/src/config.ts b/src/config.ts index 36f8c7b07c..f4dcb218bd 100644 --- a/src/config.ts +++ b/src/config.ts @@ -235,6 +235,17 @@ export function isDropStaleIndexEnabled(): boolean { return getMergedEnv()["AGENTMEMORY_DROP_STALE_INDEX"] === "true"; } +export function getAuditRetentionMonths(): number { + return safeParseInt(getMergedEnv()["AGENTMEMORY_AUDIT_RETENTION_MONTHS"], 0); +} + +export function getAuditMigrateMaxBytes(): number { + return safeParseInt( + getMergedEnv()["AGENTMEMORY_AUDIT_MIGRATE_MAX_BYTES"], + 32 * 1024 * 1024, + ); +} + export function detectLlmProviderKind( env: Record = getMergedEnv(), ): "llm" | "noop" { diff --git a/src/functions/audit.ts b/src/functions/audit.ts index b5d48620d5..c629ff0303 100644 --- a/src/functions/audit.ts +++ b/src/functions/audit.ts @@ -1,7 +1,13 @@ -import type { AuditEntry } from "../types.js"; +import { stat, readFile } from "node:fs/promises"; +import { join } from "node:path"; +import type { AuditEntry, AuditMigrationState, AuditQueryResult } from "../types.js"; import { KV, generateId } from "../state/schema.js"; import type { StateKV } from "../state/kv.js"; import { logger } from "../logger.js"; +import { withKeyedLock } from "../state/keyed-mutex.js"; +import { getEnvVar, getAuditMigrateMaxBytes } from "../config.js"; +import { runtimeConfigPath } from "../cli/engine-launch.js"; +import { configuredSaveIntervalMs } from "../cli/engine-config.js"; // Audit coverage policy (issue #125). // @@ -31,6 +37,142 @@ import { logger } from "../logger.js"; // When adding a new deletion path, add an explicit recordAudit call // BEFORE kv.delete(...) and match one of the two shapes above. +const AUDIT_MIGRATE_FUNCTION_ID = "mem::audit-migrate"; +const AUDIT_MIGRATION_COPY_CONCURRENCY = 32; +const AUDIT_MIGRATION_DELETE_CONCURRENCY = 8; +const AUDIT_MONTHS_LOCK = KV.auditMonths; +const AUDIT_MIGRATION_LOCK = "mem:audit:migration-lock"; +export const AUDIT_MIGRATION_STATE_KEY = "migration"; +const DEFAULT_SAVE_INTERVAL_MS = 5000; +const AUDIT_MIGRATION_DELETE_WAIT_INTERVALS = 2; +const STATE_STORE_DIR_NAME = "state_store.db"; + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function encodeStateIndexSegment(index: string): string { + let out = ""; + for (const byte of Buffer.from(index, "utf8")) { + const ch = String.fromCharCode(byte); + out += /^[A-Za-z0-9\-_.]$/.test(ch) + ? ch + : `%${byte.toString(16).toUpperCase().padStart(2, "0")}`; + } + return out; +} + +function legacyAuditStateFilePath(dataDir: string): string { + return join(dataDir, STATE_STORE_DIR_NAME, `${encodeStateIndexSegment(KV.audit)}.bin`); +} + +export type LegacyAuditProbe = + | { safe: true; empty: true } + | { safe: true; empty: false; sizeBytes: number } + | { safe: false; reason: "too-large"; sizeBytes: number } + | { safe: false; reason: "unreadable" }; + +export async function probeLegacyAuditFile(): Promise { + const dataDir = getEnvVar("AGENTMEMORY_DATA_DIR"); + if (!dataDir) return { safe: false, reason: "unreadable" }; + + const storeDir = join(dataDir, STATE_STORE_DIR_NAME); + try { + await stat(storeDir); + } catch { + return { safe: false, reason: "unreadable" }; + } + + let fileStat; + try { + fileStat = await stat(legacyAuditStateFilePath(dataDir)); + } catch (err) { + if ((err as NodeJS.ErrnoException).code === "ENOENT") { + return { safe: true, empty: true }; + } + return { safe: false, reason: "unreadable" }; + } + + if (fileStat.size > getAuditMigrateMaxBytes()) { + return { safe: false, reason: "too-large", sizeBytes: fileStat.size }; + } + return { safe: true, empty: false, sizeBytes: fileStat.size }; +} + +async function auditMigrationDeleteDelayMs(): Promise { + const dataDir = getEnvVar("AGENTMEMORY_DATA_DIR"); + if (!dataDir) return DEFAULT_SAVE_INTERVAL_MS * AUDIT_MIGRATION_DELETE_WAIT_INTERVALS; + try { + const rendered = await readFile(runtimeConfigPath(dataDir), "utf-8"); + const interval = configuredSaveIntervalMs(rendered) ?? DEFAULT_SAVE_INTERVAL_MS; + return interval * AUDIT_MIGRATION_DELETE_WAIT_INTERVALS; + } catch { + return DEFAULT_SAVE_INTERVAL_MS * AUDIT_MIGRATION_DELETE_WAIT_INTERVALS; + } +} + +async function readAuditMigrationState(kv: StateKV): Promise { + return kv.get(KV.auditMonths, AUDIT_MIGRATION_STATE_KEY); +} + +async function writeAuditMigrationState( + kv: StateKV, + state: AuditMigrationState, +): Promise { + await kv.set(KV.auditMonths, AUDIT_MIGRATION_STATE_KEY, state); +} + +export function auditMonthOf(timestamp: string): string { + const parsed = new Date(timestamp); + const iso = Number.isNaN(parsed.getTime()) + ? new Date().toISOString() + : parsed.toISOString(); + return iso.slice(0, 7); +} + +function monthStartMs(month: string): number { + const [year, mon] = month.split("-").map(Number); + return Date.UTC(year, mon - 1, 1); +} + +function monthEndMs(month: string): number { + const [year, mon] = month.split("-").map(Number); + return Date.UTC(year, mon, 1) - 1; +} + +function monthNMonthsAgo(count: number): string { + const now = new Date(); + const cutoff = new Date( + Date.UTC(now.getUTCFullYear(), now.getUTCMonth() - count, 1), + ); + return cutoff.toISOString().slice(0, 7); +} + +async function readAuditMonthIndex(kv: StateKV): Promise { + const index = await kv.get<{ months: string[] }>(KV.auditMonths, "index"); + return index?.months ?? []; +} + +async function markAuditMonth(kv: StateKV, month: string): Promise { + await withKeyedLock(AUDIT_MONTHS_LOCK, async () => { + const months = await readAuditMonthIndex(kv); + if (months.includes(month)) return; + const next = [...months, month].sort(); + await kv.set(KV.auditMonths, "index", { months: next }); + }); +} + +export async function listAuditMonthsDesc(kv: StateKV): Promise { + const months = await readAuditMonthIndex(kv); + return [...months].sort().reverse(); +} + +let auditRecordedListener: ((entry: AuditEntry) => void) | null = null; + +export function setAuditRecordedListener(listener: ((entry: AuditEntry) => void) | null): void { + auditRecordedListener = listener; +} + export async function recordAudit( kv: StateKV, operation: AuditEntry["operation"], @@ -50,7 +192,12 @@ export async function recordAudit( details, qualityScore, }; - await kv.set(KV.audit, entry.id, entry); + const month = auditMonthOf(entry.timestamp); + await kv.set(KV.auditMonth(month), entry.id, entry); + await markAuditMonth(kv, month); + try { + auditRecordedListener?.(entry); + } catch {} return entry; } @@ -86,37 +233,290 @@ export async function queryAudit( query?: string; limit?: number; }, -): Promise { - const all = await kv.list(KV.audit); - let entries = [...all].sort( - (a, b) => new Date(b.timestamp).getTime() - new Date(a.timestamp).getTime(), - ); +): Promise { + const limit = filter?.limit || 100; - if (filter?.operation) { - entries = entries.filter((e) => e.operation === filter.operation); - } + let fromMs: number | undefined; if (filter?.dateFrom) { - const from = new Date(filter.dateFrom).getTime(); - if (Number.isNaN(from)) { + fromMs = new Date(filter.dateFrom).getTime(); + if (Number.isNaN(fromMs)) { throw new Error(`Invalid dateFrom: ${filter.dateFrom}`); } - entries = entries.filter((e) => new Date(e.timestamp).getTime() >= from); } + const query = filter?.query?.trim().toLowerCase(); + + let toMs: number | undefined; if (filter?.dateTo) { - const to = new Date(filter.dateTo).getTime(); - if (Number.isNaN(to)) { + toMs = new Date(filter.dateTo).getTime(); + if (Number.isNaN(toMs)) { throw new Error(`Invalid dateTo: ${filter.dateTo}`); } - entries = entries.filter((e) => new Date(e.timestamp).getTime() <= to); } - const query = filter?.query?.trim().toLowerCase(); - if (query) { - entries = entries.filter((e) => - [e.functionId, ...(e.targetIds || [])].some((v) => + + const matches = (entry: AuditEntry): boolean => { + if (filter?.operation && entry.operation !== filter.operation) return false; + const t = new Date(entry.timestamp).getTime(); + if (fromMs !== undefined && t < fromMs) return false; + if (toMs !== undefined && t > toMs) return false; + if ( + query && + ![entry.functionId, ...(entry.targetIds || [])].some((v) => String(v || "").toLowerCase().includes(query), - ), + ) + ) { + return false; + } + return true; + }; + + const seenIds = new Set(); + const collected: AuditEntry[] = []; + const addIfNew = (row: AuditEntry): void => { + if (seenIds.has(row.id)) return; + seenIds.add(row.id); + if (matches(row)) collected.push(row); + }; + + const months = await listAuditMonthsDesc(kv); + for (const month of months) { + if (collected.length >= limit) break; + if (toMs !== undefined && monthStartMs(month) > toMs) continue; + if (fromMs !== undefined && monthEndMs(month) < fromMs) break; + const rows = await kv.list(KV.auditMonth(month)); + for (const row of rows) addIfNew(row); + } + + let legacyFrozen = false; + let legacyFrozenBytes: number | undefined; + + if (collected.length < limit) { + const migrationState = await readAuditMigrationState(kv); + if (migrationState?.safeToListLegacy) { + const legacyRows = await kv.list(KV.audit); + for (const row of legacyRows) addIfNew(row); + } else if (migrationState?.status === "too-large" || migrationState?.status === "unreadable") { + legacyFrozen = true; + legacyFrozenBytes = migrationState.legacySizeBytes; + } + } + + collected.sort( + (a, b) => new Date(b.timestamp).getTime() - new Date(a.timestamp).getTime(), + ); + return { + entries: collected.slice(0, limit), + legacyFrozen, + legacyFrozenBytes, + }; +} + +async function inParallel( + items: T[], + concurrency: number, + work: (item: T) => Promise, +): Promise { + let next = 0; + const runners = Array.from({ length: Math.min(concurrency, items.length) }, async () => { + while (next < items.length) { + const item = items[next++]; + await work(item); + } + }); + await Promise.all(runners); +} + +export interface AuditMigrationDeps { + probe?: () => Promise; + deleteDelayMs?: () => Promise; +} + +export async function startAuditMigration( + kv: StateKV, + deps: AuditMigrationDeps = {}, +): Promise { + const runProbe = deps.probe ?? probeLegacyAuditFile; + const runDeleteDelayMs = deps.deleteDelayMs ?? auditMigrationDeleteDelayMs; + + const marker = await readAuditMigrationState(kv); + if (marker?.status === "done") return; + + const probe = await runProbe(); + + if (probe.safe && probe.empty) { + await writeAuditMigrationState(kv, { + status: "done", + safeToListLegacy: true, + legacySizeBytes: 0, + migrated: marker?.migrated ?? 0, + purged: marker?.purged ?? 0, + summaryWritten: marker?.summaryWritten ?? true, + checkedAt: new Date().toISOString(), + }); + return; + } + + if (!probe.safe) { + await writeAuditMigrationState(kv, { + status: probe.reason, + safeToListLegacy: false, + legacySizeBytes: probe.reason === "too-large" ? probe.sizeBytes : marker?.legacySizeBytes, + migrated: marker?.migrated ?? 0, + purged: marker?.purged ?? 0, + summaryWritten: marker?.summaryWritten ?? false, + checkedAt: new Date().toISOString(), + }); + logger.warn("legacy audit log left frozen; migration only reads it locally and under a size cap", { + reason: probe.reason, + sizeBytes: probe.reason === "too-large" ? probe.sizeBytes : undefined, + }); + return; + } + + let migrated = marker?.migrated ?? 0; + let purged = marker?.purged ?? 0; + + if (marker?.status !== "copied") { + let legacy: AuditEntry[]; + try { + legacy = await kv.list(KV.audit); + } catch (err) { + logger.warn("audit migration could not read the legacy audit log", { + error: err instanceof Error ? err.message : String(err), + }); + return; + } + if (legacy.length === 0) { + await writeAuditMigrationState(kv, { + status: "done", + safeToListLegacy: true, + legacySizeBytes: 0, + migrated, + purged, + summaryWritten: marker?.summaryWritten ?? true, + checkedAt: new Date().toISOString(), + }); + return; + } + + const seen = new Set(); + const deduped = legacy.filter((entry) => { + if (seen.has(entry.id)) return false; + seen.add(entry.id); + return true; + }); + const kept = deduped.filter((entry) => entry.operation !== "index_persist"); + purged = deduped.length - kept.length; + migrated = kept.length; + + try { + const months = new Set(); + await inParallel(kept, AUDIT_MIGRATION_COPY_CONCURRENCY, async (entry) => { + const month = auditMonthOf(entry.timestamp); + await kv.set(KV.auditMonth(month), entry.id, entry); + months.add(month); + }); + for (const month of months) await markAuditMonth(kv, month); + } catch (err) { + logger.warn( + "audit migration stopped before finishing the copy pass; it resumes on the next boot", + { error: err instanceof Error ? err.message : String(err) }, + ); + return; + } + + await writeAuditMigrationState(kv, { + status: "copied", + safeToListLegacy: false, + legacySizeBytes: probe.sizeBytes, + migrated, + purged, + summaryWritten: false, + checkedAt: new Date().toISOString(), + }); + } + + await sleep(await runDeleteDelayMs()); + + let remaining: AuditEntry[]; + try { + remaining = await kv.list(KV.audit); + } catch (err) { + logger.warn( + "audit migration could not re-read the legacy audit log before deleting; it resumes on the next boot", + { error: err instanceof Error ? err.message : String(err) }, ); + return; } - return entries.slice(0, filter?.limit || 100); + try { + await inParallel( + remaining.slice().reverse(), + AUDIT_MIGRATION_DELETE_CONCURRENCY, + async (entry) => { + await kv.delete(KV.audit, entry.id); + }, + ); + } catch (err) { + logger.warn( + "audit migration stopped before finishing the delete pass; it resumes on the next boot", + { error: err instanceof Error ? err.message : String(err) }, + ); + return; + } + + await withKeyedLock(AUDIT_MIGRATION_LOCK, async () => { + const current = await readAuditMigrationState(kv); + if (current?.summaryWritten) return; + if (purged > 0) { + await safeAudit(kv, "audit_migrate", AUDIT_MIGRATE_FUNCTION_ID, [], { + reason: "index_persist rows predate opt-in auditing", + purged, + migrated, + }); + } + await writeAuditMigrationState(kv, { + status: "done", + safeToListLegacy: true, + legacySizeBytes: 0, + migrated, + purged, + summaryWritten: true, + checkedAt: new Date().toISOString(), + }); + }); + + logger.info("audit log moved to monthly scopes", { migrated, purged }); +} + +export async function runAuditRetentionSweep( + kv: StateKV, + retentionMonths: number, +): Promise<{ droppedMonths: string[]; droppedRows: number }> { + if (!Number.isFinite(retentionMonths) || retentionMonths <= 0) { + return { droppedMonths: [], droppedRows: 0 }; + } + + const months = await readAuditMonthIndex(kv); + const cutoff = monthNMonthsAgo(retentionMonths); + const toDrop = months.filter((month) => month < cutoff).sort(); + if (toDrop.length === 0) { + return { droppedMonths: [], droppedRows: 0 }; + } + + let droppedRows = 0; + for (const month of toDrop) { + const rows = await kv.list(KV.auditMonth(month)); + for (const row of rows.slice().reverse()) { + await kv.delete(KV.auditMonth(month), row.id); + droppedRows++; + } + } + + await withKeyedLock(AUDIT_MONTHS_LOCK, async () => { + const current = await readAuditMonthIndex(kv); + const remaining = current.filter((month) => !toDrop.includes(month)).sort(); + await kv.set(KV.auditMonths, "index", { months: remaining }); + }); + + return { droppedMonths: toDrop, droppedRows }; } diff --git a/src/functions/retention.ts b/src/functions/retention.ts index fac16796f1..b718cbf50c 100644 --- a/src/functions/retention.ts +++ b/src/functions/retention.ts @@ -13,9 +13,10 @@ import { deleteAccessLog, normalizeAccessLog, } from "./access-tracker.js"; -import { recordAudit } from "./audit.js"; +import { recordAudit, runAuditRetentionSweep } from "./audit.js"; import { getSearchIndex, vectorIndexRemove, flushIndexSave } from "./search.js"; import { logger } from "../logger.js"; +import { getAuditRetentionMonths } from "../config.js"; const DEFAULT_DECAY: DecayConfig = { lambda: 0.01, @@ -409,4 +410,22 @@ export function registerRetentionFunctions( return { success: true, evicted, evictedEpisodic, evictedSemantic }; }, ); + + sdk.registerFunction("mem::audit-retention-sweep", + async (data?: { retentionMonths?: number }) => { + const retentionMonths = + typeof data?.retentionMonths === "number" && + Number.isFinite(data.retentionMonths) + ? data.retentionMonths + : getAuditRetentionMonths(); + + const result = await runAuditRetentionSweep(kv, retentionMonths); + + if (result.droppedRows > 0) { + logger.info("Audit retention sweep dropped old month scopes", result); + } + + return { success: true, ...result }; + }, + ); } diff --git a/src/functions/status.ts b/src/functions/status.ts index 234d748866..aa82287c10 100644 --- a/src/functions/status.ts +++ b/src/functions/status.ts @@ -67,6 +67,7 @@ export interface StatusInputs { }; graph: GraphStatsInput | null; graphExtractionEnabled: boolean; + auditLegacy: { status: string; sizeBytes?: number } | null; indexPersistence?: IndexPersistenceStatus | null; } @@ -353,6 +354,17 @@ export function evaluateStatus(input: StatusInputs): StatusReport { } } + if (input.auditLegacy?.status === "too-large" || input.auditLegacy?.status === "unreadable") { + const size = input.auditLegacy.sizeBytes; + const sizeText = typeof size === "number" ? ` (${Math.round(size / (1024 * 1024))} MiB)` : ""; + problems.push({ + level: "info", + code: "audit-legacy-frozen", + message: `An older audit log${sizeText} was left in place instead of being migrated into monthly scopes, so it is no longer rewritten but its rows do not show up in audit queries.`, + fix: "Safe to ignore. To remove it, stop agentmemory, delete mem%3Aaudit.bin from the state store directory, then start it again.", + }); + } + const status = problems.reduce( (worst, p) => (LEVEL_RANK[p.level] > LEVEL_RANK[worst] ? p.level : worst), "ok", diff --git a/src/index.ts b/src/index.ts index df92cb9357..1111adf12b 100644 --- a/src/index.ts +++ b/src/index.ts @@ -14,6 +14,7 @@ import { getConsolidationIntervalMs, isContextInjectionEnabled, isDropStaleIndexEnabled, + getAuditRetentionMonths, } from "./config.js"; import { createProvider, @@ -95,6 +96,7 @@ import { registerSlidingWindowFunction } from "./functions/sliding-window.js"; import { registerQueryExpansionFunction } from "./functions/query-expansion.js"; import { registerTemporalGraphFunctions } from "./functions/temporal-graph.js"; import { registerRetentionFunctions } from "./functions/retention.js"; +import { startAuditMigration } from "./functions/audit.js"; import { registerCompressFileFunction } from "./functions/compress-file.js"; import { registerReplayFunctions } from "./functions/replay.js"; import { registerApiTriggers } from "./triggers/api.js"; @@ -475,6 +477,8 @@ async function main() { } } + const auditMigration = startAuditMigration(kv).catch(() => {}); + const vectorCountShortfall = Boolean(loaded?.vector) && loaded?.expectedCount !== undefined && @@ -548,6 +552,22 @@ async function main() { bootLog(`Auto-forget: enabled (every ${autoForgetIntervalMs / 60000}m)`); } + const auditRetentionMonths = getAuditRetentionMonths(); + if (auditRetentionMonths > 0) { + const runAuditSweep = async () => { + try { + await auditMigration; + await sdk.trigger({ function_id: "mem::audit-retention-sweep", payload: {} }); + } catch {} + }; + void runAuditSweep(); + const auditRetentionTimer = setInterval(runAuditSweep, 86400000); + auditRetentionTimer.unref(); + bootLog( + `Audit retention sweep: enabled (drop month scopes older than ${auditRetentionMonths}mo, every 24h)`, + ); + } + if (process.env.LESSON_DECAY_ENABLED !== "false") { const lessonDecayTimer = setInterval(async () => { try { diff --git a/src/mcp/standalone.ts b/src/mcp/standalone.ts index 5a4a891b23..46c44bf371 100644 --- a/src/mcp/standalone.ts +++ b/src/mcp/standalone.ts @@ -6,6 +6,7 @@ import { getAllTools } from "./tools-registry.js"; import { getStandalonePersistPath } from "../config.js"; import { VERSION } from "../version.js"; import { generateId } from "../state/schema.js"; +import { queryAudit } from "../functions/audit.js"; import { resolveHandle, invalidateHandle, @@ -350,14 +351,10 @@ async function handleLocal( } case "memory_audit": { - const entries = await kvInstance.list("mem:audit"); - const limit = v.limit ?? 50; - return textResponse( - { - entries: (entries as Array>).slice(0, limit), - }, - true, - ); + const result = await queryAudit(kvInstance as never, { + limit: v.limit ?? 50, + }); + return textResponse(result, true); } default: diff --git a/src/state/schema.ts b/src/state/schema.ts index 2e1588cbf3..207be7df7f 100644 --- a/src/state/schema.ts +++ b/src/state/schema.ts @@ -45,6 +45,8 @@ export const KV = { `mem:team:${teamId}:users:${userId}`, teamProfile: (teamId: string) => `mem:team:${teamId}:profile`, audit: "mem:audit", + auditMonth: (month: string) => `mem:audit:${month}`, + auditMonths: "mem:audit:months", actions: "mem:actions", actionEdges: "mem:action-edges", leases: "mem:leases", diff --git a/src/triggers/api.ts b/src/triggers/api.ts index 5fa9c7a15a..f41bdb0b31 100644 --- a/src/triggers/api.ts +++ b/src/triggers/api.ts @@ -1,7 +1,8 @@ import { TriggerAction, type IIIClient } from "iii-sdk"; import type { HttpRequest } from "@iii-dev/helpers/http"; import { randomBytes } from "node:crypto"; -import type { Session, CompressedObservation, HookPayload, CommitLink, SessionSummary, HealthSnapshot } from "../types.js"; +import type { Session, CompressedObservation, HookPayload, CommitLink, SessionSummary, HealthSnapshot, AuditQueryResult, AuditMigrationState } from "../types.js"; +import { AUDIT_MIGRATION_STATE_KEY } from "../functions/audit.js"; import { withKeyedLock } from "../state/keyed-mutex.js"; import { KV } from "../state/schema.js"; import { checkPayloadFrameSize } from "../state/frame-guard.js"; @@ -322,7 +323,7 @@ export function createStatusReporter(sdk: IIIClient, kv: StateKV, deps: StatusRe const cached = scan.last && options.scanMaxAgeMs !== undefined && Date.now() - scan.last.at < options.scanMaxAgeMs ? scan.last.value : null; - const [health, functionMetrics, graph, unindexed] = await Promise.all([ + const [health, functionMetrics, graph, unindexed, auditMigrationState] = await Promise.all([ options.health !== undefined ? Promise.resolve(options.health) : valueWithin(getLatestHealth(kv), STATUS_CHECK_TIMEOUT_MS), deps.metricsStore ? valueWithin(deps.metricsStore.getAll(), STATUS_CHECK_TIMEOUT_MS) : Promise.resolve([]), valueWithin( @@ -330,6 +331,10 @@ export function createStatusReporter(sdk: IIIClient, kv: StateKV, deps: StatusRe STATUS_CHECK_TIMEOUT_MS, ), cached ? Promise.resolve(cached) : valueWithin(scan.run(), STATUS_CHECK_TIMEOUT_MS), + valueWithin( + kv.get(KV.auditMonths, AUDIT_MIGRATION_STATE_KEY), + STATUS_CHECK_TIMEOUT_MS, + ), ]); const observationsIndexed = [...idx.observationCountsBySession().values()].reduce((a, n) => a + n, 0); const documentKinds = idx.documentKindCounts(); @@ -376,6 +381,9 @@ export function createStatusReporter(sdk: IIIClient, kv: StateKV, deps: StatusRe }, graph, graphExtractionEnabled: isGraphExtractionEnabled(), + auditLegacy: auditMigrationState + ? { status: auditMigrationState.status, sizeBytes: auditMigrationState.legacySizeBytes } + : null, indexPersistence: getIndexPersistenceStatus(), }); }; @@ -2224,14 +2232,22 @@ export function registerApiTriggers( return { status_code: 400, body: { error: `invalid date: ${name}` } }; } } - const entries = await sdk.trigger({ function_id: "mem::audit-query", payload: { + const result = await sdk.trigger({ function_id: "mem::audit-query", payload: { operation: asNonEmptyString(params["operation"]), dateFrom, dateTo, query: asNonEmptyString(params["q"]), limit: Math.min(Math.max(parsedLimit ?? 50, 1), 1000), } }); - return { status_code: 200, body: { entries, success: true } }; + return { + status_code: 200, + body: { + entries: result.entries, + legacyFrozen: result.legacyFrozen, + legacyFrozenBytes: result.legacyFrozenBytes, + success: true, + }, + }; }, ); sdk.registerTrigger({ diff --git a/src/triggers/viewer-streams.ts b/src/triggers/viewer-streams.ts index c959589679..2d07a84c43 100644 --- a/src/triggers/viewer-streams.ts +++ b/src/triggers/viewer-streams.ts @@ -4,7 +4,6 @@ import type { StateKV } from "../state/kv.js"; import { KV } from "../state/schema.js"; import type { Action, - AuditEntry, Crystal, GraphSnapshot, HealthSnapshot, @@ -37,6 +36,7 @@ import { detectLlmProviderKind, loadConfig } from "../config.js"; import { getBoundViewerPort, getViewerSkipped } from "../viewer/server.js"; import { buildConfigFlags, checkAuth, createConsolidationStatusReader, createStatusReporter } from "./api.js"; import { isStateDelete, sendViewerEvent } from "./events.js"; +import { setAuditRecordedListener } from "../functions/audit.js"; type Response = { status_code: number; @@ -190,7 +190,9 @@ export function registerViewerStreamTriggers( registerRowStream({ functionId: "event::viewer::crystal-changed", scope: KV.crystals, entity: "crystal", countKind: "crystals" }); registerRowStream({ functionId: "event::viewer::semantic-changed", scope: KV.semantic, entity: "semantic", countKind: "semantic" }); registerRowStream({ functionId: "event::viewer::procedural-changed", scope: KV.procedural, entity: "procedural", countKind: "procedural" }); - registerRowStream({ functionId: "event::viewer::audit-changed", scope: KV.audit, entity: "audit" }); + setAuditRecordedListener((entry) => { + void sendViewerEvent(sdk, eventId("audit", entry.id), "audit.created", { id: entry.id, audit: entry }).catch(() => {}); + }); let lastGraphSnapshotEventAt = 0; sdk.registerFunction( diff --git a/src/types.ts b/src/types.ts index dff2bc599d..909675acab 100644 --- a/src/types.ts +++ b/src/types.ts @@ -640,7 +640,8 @@ export interface AuditEntry { | "slot_replace" | "slot_create" | "slot_delete" - | "slot_reflect"; + | "slot_reflect" + | "audit_migrate"; userId?: string; functionId: string; targetIds: string[]; @@ -648,6 +649,26 @@ export interface AuditEntry { qualityScore?: number; } +export interface AuditMonthIndex { + months: string[]; +} + +export interface AuditMigrationState { + status: "too-large" | "unreadable" | "copied" | "done"; + safeToListLegacy: boolean; + legacySizeBytes?: number; + migrated: number; + purged: number; + summaryWritten: boolean; + checkedAt: string; +} + +export interface AuditQueryResult { + entries: AuditEntry[]; + legacyFrozen: boolean; + legacyFrozenBytes?: number; +} + export interface GovernanceFilter { type?: string[]; dateFrom?: string; diff --git a/test/audit-retention.test.ts b/test/audit-retention.test.ts new file mode 100644 index 0000000000..9aebd1d13a --- /dev/null +++ b/test/audit-retention.test.ts @@ -0,0 +1,746 @@ +import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"; +import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; + +vi.mock("../src/logger.js", () => ({ + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, +})); + +import { + recordAudit, + queryAudit, + auditMonthOf, + listAuditMonthsDesc, + startAuditMigration, + runAuditRetentionSweep, + probeLegacyAuditFile, + AUDIT_MIGRATION_STATE_KEY, +} from "../src/functions/audit.js"; +import { KV } from "../src/state/schema.js"; +import { mockKV } from "./helpers/mocks.js"; +import { getAuditRetentionMonths } from "../src/config.js"; +import type { AuditEntry, AuditMigrationState } from "../src/types.js"; + +function seedEntry( + kv: ReturnType, + scope: string, + timestamp: string, + overrides: Partial = {}, +): AuditEntry { + const entry: AuditEntry = { + id: overrides.id ?? `aud_${Math.random().toString(36).slice(2)}`, + timestamp, + operation: overrides.operation ?? "delete", + functionId: overrides.functionId ?? "mem::test", + targetIds: overrides.targetIds ?? [], + details: overrides.details ?? {}, + userId: overrides.userId, + qualityScore: overrides.qualityScore, + }; + kv.store.set( + scope, + (kv.store.get(scope) ?? new Map()).set(entry.id, entry), + ); + return entry; +} + +async function seedMonthIndex( + kv: ReturnType, + months: string[], +): Promise { + await kv.set(KV.auditMonths, "index", { months }); +} + +async function seedMigrationState( + kv: ReturnType, + state: AuditMigrationState, +): Promise { + await kv.set(KV.auditMonths, AUDIT_MIGRATION_STATE_KEY, state); +} + +const SMALL_STORE_PROBE = async () => + ({ safe: true, empty: false, sizeBytes: 1024 }) as const; +const EMPTY_STORE_PROBE = async () => ({ safe: true, empty: true }) as const; +const NO_DELAY = async () => 0; + +describe("audit month buckets", () => { + let kv: ReturnType; + + beforeEach(() => { + kv = mockKV(); + }); + + it("recordAudit writes into the UTC month scope of the entry timestamp", async () => { + const entry = await recordAudit(kv as never, "delete", "mem::test", ["a"]); + const month = auditMonthOf(entry.timestamp); + + const rows = await kv.list(KV.auditMonth(month)); + expect(rows).toHaveLength(1); + expect(rows[0].id).toBe(entry.id); + + const legacy = await kv.list(KV.audit); + expect(legacy).toHaveLength(0); + }); + + it("recordAudit registers the month in the month index exactly once", async () => { + await recordAudit(kv as never, "delete", "mem::test", ["a"]); + await recordAudit(kv as never, "delete", "mem::test", ["b"]); + + const months = await listAuditMonthsDesc(kv as never); + expect(months).toHaveLength(1); + expect(months[0]).toBe(auditMonthOf(new Date().toISOString())); + }); + + it("auditMonthOf derives the month in UTC", () => { + expect(auditMonthOf("2026-01-31T23:30:00.000Z")).toBe("2026-01"); + expect(auditMonthOf("2026-02-01T00:00:00.000Z")).toBe("2026-02"); + }); +}); + +describe("queryAudit across months", () => { + let kv: ReturnType; + + beforeEach(() => { + kv = mockKV(); + }); + + it("reads the newest month first and stops once the limit is reached", async () => { + await seedMonthIndex(kv, ["2026-06", "2026-07", "2026-08"]); + seedEntry(kv, KV.auditMonth("2026-06"), "2026-06-15T00:00:00.000Z", { + id: "june_1", + }); + seedEntry(kv, KV.auditMonth("2026-07"), "2026-07-15T00:00:00.000Z", { + id: "july_1", + }); + seedEntry(kv, KV.auditMonth("2026-08"), "2026-08-01T00:00:00.000Z", { + id: "aug_1", + }); + seedEntry(kv, KV.auditMonth("2026-08"), "2026-08-02T00:00:00.000Z", { + id: "aug_2", + }); + + const listSpy = vi.spyOn(kv, "list"); + const { entries } = await queryAudit(kv as never, { limit: 2 }); + + expect(entries.map((e) => e.id).sort()).toEqual(["aug_1", "aug_2"]); + const listedScopes = listSpy.mock.calls.map((call) => call[0]); + expect(listedScopes).toContain(KV.auditMonth("2026-08")); + expect(listedScopes).not.toContain(KV.auditMonth("2026-07")); + expect(listedScopes).not.toContain(KV.auditMonth("2026-06")); + }); + + it("falls back to legacy rows when the migration marker says the legacy scope is safe to read", async () => { + await seedMonthIndex(kv, ["2026-08"]); + seedEntry(kv, KV.auditMonth("2026-08"), "2026-08-01T00:00:00.000Z", { + id: "aug_1", + }); + seedEntry(kv, KV.audit, "2026-05-01T00:00:00.000Z", { id: "legacy_1" }); + await seedMigrationState(kv, { + status: "done", + safeToListLegacy: true, + legacySizeBytes: 0, + migrated: 0, + purged: 0, + summaryWritten: true, + checkedAt: new Date().toISOString(), + }); + + const { entries, legacyFrozen } = await queryAudit(kv as never, { limit: 5 }); + expect(entries.map((e) => e.id).sort()).toEqual(["aug_1", "legacy_1"]); + expect(legacyFrozen).toBe(false); + }); + + it("never falls back to legacy once the month scopes already satisfy the limit", async () => { + await seedMonthIndex(kv, ["2026-08"]); + seedEntry(kv, KV.auditMonth("2026-08"), "2026-08-01T00:00:00.000Z", { + id: "aug_1", + }); + seedEntry(kv, KV.audit, "2026-05-01T00:00:00.000Z", { id: "legacy_1" }); + + const listSpy = vi.spyOn(kv, "list"); + const { entries } = await queryAudit(kv as never, { limit: 1 }); + + expect(entries.map((e) => e.id)).toEqual(["aug_1"]); + const listedScopes = listSpy.mock.calls.map((call) => call[0]); + expect(listedScopes).not.toContain(KV.audit); + }); + + it("does not list the legacy scope, and does not report it frozen, before any migration marker exists", async () => { + await seedMonthIndex(kv, ["2026-08"]); + seedEntry(kv, KV.auditMonth("2026-08"), "2026-08-01T00:00:00.000Z", { + id: "aug_1", + }); + seedEntry(kv, KV.audit, "2026-05-01T00:00:00.000Z", { id: "legacy_1" }); + + const listSpy = vi.spyOn(kv, "list"); + const { entries, legacyFrozen, legacyFrozenBytes } = await queryAudit(kv as never, { + limit: 5, + }); + + expect(entries.map((e) => e.id)).toEqual(["aug_1"]); + expect(legacyFrozen).toBe(false); + expect(legacyFrozenBytes).toBeUndefined(); + const listedScopes = listSpy.mock.calls.map((call) => call[0]); + expect(listedScopes).not.toContain(KV.audit); + }); + + it("reports the legacy scope as frozen when the migration marker says it is unsafe to read", async () => { + await seedMonthIndex(kv, ["2026-08"]); + seedEntry(kv, KV.auditMonth("2026-08"), "2026-08-01T00:00:00.000Z", { + id: "aug_1", + }); + seedEntry(kv, KV.audit, "2026-05-01T00:00:00.000Z", { id: "legacy_1" }); + await seedMigrationState(kv, { + status: "too-large", + safeToListLegacy: false, + legacySizeBytes: 99999999, + migrated: 0, + purged: 0, + summaryWritten: false, + checkedAt: new Date().toISOString(), + }); + + const listSpy = vi.spyOn(kv, "list"); + const { entries, legacyFrozen, legacyFrozenBytes } = await queryAudit(kv as never, { + limit: 5, + }); + + expect(entries.map((e) => e.id)).toEqual(["aug_1"]); + expect(legacyFrozen).toBe(true); + expect(legacyFrozenBytes).toBe(99999999); + const listedScopes = listSpy.mock.calls.map((call) => call[0]); + expect(listedScopes).not.toContain(KV.audit); + }); + + it("does not report the legacy scope frozen once its rows are copied and only the delete pass remains", async () => { + await seedMonthIndex(kv, ["2026-05"]); + seedEntry(kv, KV.auditMonth("2026-05"), "2026-05-01T00:00:00.000Z", { + id: "legacy_1", + }); + seedEntry(kv, KV.audit, "2026-05-01T00:00:00.000Z", { id: "legacy_1" }); + await seedMigrationState(kv, { + status: "copied", + safeToListLegacy: false, + legacySizeBytes: 1024, + migrated: 1, + purged: 0, + summaryWritten: false, + checkedAt: new Date().toISOString(), + }); + + const listSpy = vi.spyOn(kv, "list"); + const { entries, legacyFrozen, legacyFrozenBytes } = await queryAudit(kv as never, { + limit: 5, + }); + + expect(entries.map((e) => e.id)).toEqual(["legacy_1"]); + expect(legacyFrozen).toBe(false); + expect(legacyFrozenBytes).toBeUndefined(); + const listedScopes = listSpy.mock.calls.map((call) => call[0]); + expect(listedScopes).not.toContain(KV.audit); + }); + + it("respects operation and date filters across months", async () => { + await seedMonthIndex(kv, ["2026-07", "2026-08"]); + seedEntry(kv, KV.auditMonth("2026-07"), "2026-07-10T00:00:00.000Z", { + id: "july_delete", + operation: "delete", + }); + seedEntry(kv, KV.auditMonth("2026-08"), "2026-08-10T00:00:00.000Z", { + id: "aug_forget", + operation: "forget", + }); + + const { entries } = await queryAudit(kv as never, { + operation: "delete", + limit: 10, + }); + expect(entries.map((e) => e.id)).toEqual(["july_delete"]); + }); +}); + +describe("probeLegacyAuditFile", () => { + let dir: string; + let originalDataDir: string | undefined; + let originalMaxBytes: string | undefined; + + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), "am-audit-probe-")); + originalDataDir = process.env["AGENTMEMORY_DATA_DIR"]; + originalMaxBytes = process.env["AGENTMEMORY_AUDIT_MIGRATE_MAX_BYTES"]; + }); + + afterEach(() => { + rmSync(dir, { recursive: true, force: true }); + if (originalDataDir === undefined) delete process.env["AGENTMEMORY_DATA_DIR"]; + else process.env["AGENTMEMORY_DATA_DIR"] = originalDataDir; + if (originalMaxBytes === undefined) delete process.env["AGENTMEMORY_AUDIT_MIGRATE_MAX_BYTES"]; + else process.env["AGENTMEMORY_AUDIT_MIGRATE_MAX_BYTES"] = originalMaxBytes; + }); + + it("reports unreadable when AGENTMEMORY_DATA_DIR is not set", async () => { + delete process.env["AGENTMEMORY_DATA_DIR"]; + const result = await probeLegacyAuditFile(); + expect(result).toEqual({ safe: false, reason: "unreadable" }); + }); + + it("reports unreadable when the engine's state store directory is not local", async () => { + process.env["AGENTMEMORY_DATA_DIR"] = join(dir, "does-not-exist"); + const result = await probeLegacyAuditFile(); + expect(result).toEqual({ safe: false, reason: "unreadable" }); + }); + + it("reports empty when the state store exists but the legacy scope file was never written", async () => { + mkdirSync(join(dir, "state_store.db"), { recursive: true }); + process.env["AGENTMEMORY_DATA_DIR"] = dir; + const result = await probeLegacyAuditFile(); + expect(result).toEqual({ safe: true, empty: true }); + }); + + it("reports safe with the measured size when the legacy scope file is under the cap", async () => { + mkdirSync(join(dir, "state_store.db"), { recursive: true }); + const content = JSON.stringify({ aud_1: { id: "aud_1" } }); + writeFileSync(join(dir, "state_store.db", "mem%3Aaudit.bin"), content); + process.env["AGENTMEMORY_DATA_DIR"] = dir; + + const result = await probeLegacyAuditFile(); + expect(result.safe).toBe(true); + if (result.safe && !result.empty) { + expect(result.sizeBytes).toBe(Buffer.byteLength(content)); + } else { + throw new Error("expected a non-empty safe probe result"); + } + }); + + it("reports too-large when the legacy scope file exceeds AGENTMEMORY_AUDIT_MIGRATE_MAX_BYTES", async () => { + mkdirSync(join(dir, "state_store.db"), { recursive: true }); + writeFileSync(join(dir, "state_store.db", "mem%3Aaudit.bin"), "x".repeat(1000)); + process.env["AGENTMEMORY_DATA_DIR"] = dir; + process.env["AGENTMEMORY_AUDIT_MIGRATE_MAX_BYTES"] = "10"; + + const result = await probeLegacyAuditFile(); + expect(result).toEqual({ safe: false, reason: "too-large", sizeBytes: 1000 }); + }); +}); + +describe("startAuditMigration", () => { + let kv: ReturnType; + + beforeEach(() => { + kv = mockKV(); + }); + + it("moves legacy rows into their month scopes and purges index_persist noise", async () => { + seedEntry(kv, KV.audit, "2020-03-01T00:00:00.000Z", { + id: "keep_1", + operation: "delete", + targetIds: ["mem_a"], + }); + seedEntry(kv, KV.audit, "2020-04-01T00:00:00.000Z", { + id: "keep_2", + operation: "forget", + targetIds: ["mem_b"], + }); + seedEntry(kv, KV.audit, "2020-03-05T00:00:00.000Z", { + id: "noise_1", + operation: "index_persist", + }); + seedEntry(kv, KV.audit, "2020-03-06T00:00:00.000Z", { + id: "noise_2", + operation: "index_persist", + }); + + await startAuditMigration(kv as never, { + probe: SMALL_STORE_PROBE, + deleteDelayMs: NO_DELAY, + }); + + const legacyRemaining = await kv.list(KV.audit); + expect(legacyRemaining).toHaveLength(0); + + const marchRows = await kv.list(KV.auditMonth("2020-03")); + expect(marchRows.map((r) => r.id)).toEqual(["keep_1"]); + + const aprilRows = await kv.list(KV.auditMonth("2020-04")); + expect(aprilRows.map((r) => r.id)).toEqual(["keep_2"]); + + const months = await listAuditMonthsDesc(kv as never); + expect(months).toContain("2020-03"); + expect(months).toContain("2020-04"); + + const currentMonth = auditMonthOf(new Date().toISOString()); + const currentMonthRows = await kv.list( + KV.auditMonth(currentMonth), + ); + const summaryRows = currentMonthRows.filter( + (r) => r.operation === "audit_migrate", + ); + expect(summaryRows).toHaveLength(1); + expect(summaryRows[0].details.purged).toBe(2); + expect(summaryRows[0].details.migrated).toBe(2); + + const marker = await kv.get( + KV.auditMonths, + AUDIT_MIGRATION_STATE_KEY, + ); + expect(marker?.status).toBe("done"); + expect(marker?.safeToListLegacy).toBe(true); + }); + + it("records no summary row when there is nothing to purge", async () => { + seedEntry(kv, KV.audit, "2020-03-01T00:00:00.000Z", { + id: "keep_1", + operation: "delete", + }); + + await startAuditMigration(kv as never, { + probe: SMALL_STORE_PROBE, + deleteDelayMs: NO_DELAY, + }); + + const currentMonth = auditMonthOf(new Date().toISOString()); + const currentMonthRows = await kv.list( + KV.auditMonth(currentMonth), + ); + expect( + currentMonthRows.filter((r) => r.operation === "audit_migrate"), + ).toHaveLength(0); + }); + + it("is a no-op on a store with no legacy audit rows", async () => { + await startAuditMigration(kv as never, { + probe: EMPTY_STORE_PROBE, + deleteDelayMs: NO_DELAY, + }); + const months = await listAuditMonthsDesc(kv as never); + expect(months).toHaveLength(0); + const marker = await kv.get( + KV.auditMonths, + AUDIT_MIGRATION_STATE_KEY, + ); + expect(marker?.status).toBe("done"); + }); + + it("leaves the legacy scope untouched and frozen when the probe reports too-large", async () => { + seedEntry(kv, KV.audit, "2020-03-01T00:00:00.000Z", { id: "keep_1" }); + + await startAuditMigration(kv as never, { + probe: async () => ({ safe: false, reason: "too-large", sizeBytes: 99999999 }), + deleteDelayMs: NO_DELAY, + }); + + const legacy = await kv.list(KV.audit); + expect(legacy.map((r) => r.id)).toEqual(["keep_1"]); + + const months = await listAuditMonthsDesc(kv as never); + expect(months).toHaveLength(0); + + const marker = await kv.get( + KV.auditMonths, + AUDIT_MIGRATION_STATE_KEY, + ); + expect(marker).toEqual({ + status: "too-large", + safeToListLegacy: false, + legacySizeBytes: 99999999, + migrated: 0, + purged: 0, + summaryWritten: false, + checkedAt: marker?.checkedAt, + }); + }); + + it("leaves the legacy scope untouched and frozen when the probe reports unreadable", async () => { + seedEntry(kv, KV.audit, "2020-03-01T00:00:00.000Z", { id: "keep_1" }); + + await startAuditMigration(kv as never, { + probe: async () => ({ safe: false, reason: "unreadable" }), + deleteDelayMs: NO_DELAY, + }); + + const legacy = await kv.list(KV.audit); + expect(legacy.map((r) => r.id)).toEqual(["keep_1"]); + + const marker = await kv.get( + KV.auditMonths, + AUDIT_MIGRATION_STATE_KEY, + ); + expect(marker?.status).toBe("unreadable"); + expect(marker?.safeToListLegacy).toBe(false); + }); + + it("waits before deleting so the legacy rows survive until the delay elapses", async () => { + seedEntry(kv, KV.audit, "2020-03-01T00:00:00.000Z", { id: "keep_1" }); + + let releaseDelay: () => void = () => {}; + const delayGate = new Promise((resolve) => { + releaseDelay = resolve; + }); + + const migration = startAuditMigration(kv as never, { + probe: SMALL_STORE_PROBE, + deleteDelayMs: async () => { + await delayGate; + return 0; + }, + }); + + await new Promise((resolve) => setTimeout(resolve, 20)); + + const marker = await kv.get( + KV.auditMonths, + AUDIT_MIGRATION_STATE_KEY, + ); + expect(marker?.status).toBe("copied"); + + const midLegacy = await kv.list(KV.audit); + expect(midLegacy.map((r) => r.id)).toEqual(["keep_1"]); + const midMonthRows = await kv.list(KV.auditMonth("2020-03")); + expect(midMonthRows.map((r) => r.id)).toEqual(["keep_1"]); + + releaseDelay(); + await migration; + + const finalLegacy = await kv.list(KV.audit); + expect(finalLegacy).toHaveLength(0); + }); + + it("deletes legacy rows in reverse insertion order", async () => { + const first = seedEntry(kv, KV.audit, "2020-01-01T00:00:00.000Z", { + id: "first", + }); + const second = seedEntry(kv, KV.audit, "2020-01-02T00:00:00.000Z", { + id: "second", + }); + const third = seedEntry(kv, KV.audit, "2020-01-03T00:00:00.000Z", { + id: "third", + }); + void first; + void second; + void third; + + const deleteOrder: string[] = []; + const orderedKv = { + ...kv, + delete: async (scope: string, key: string) => { + if (scope === KV.audit) deleteOrder.push(key); + return kv.delete(scope, key); + }, + }; + + await startAuditMigration(orderedKv as never, { + probe: SMALL_STORE_PROBE, + deleteDelayMs: NO_DELAY, + }); + + expect(deleteOrder).toEqual(["third", "second", "first"]); + }); + + it("resumes after a crash mid-delete without double-counting or double-writing the summary", async () => { + seedEntry(kv, KV.audit, "2020-03-01T00:00:00.000Z", { + id: "keep_1", + operation: "delete", + }); + seedEntry(kv, KV.audit, "2020-03-02T00:00:00.000Z", { + id: "keep_2", + operation: "delete", + }); + seedEntry(kv, KV.audit, "2020-03-03T00:00:00.000Z", { + id: "noise_1", + operation: "index_persist", + }); + + let deleteCalls = 0; + const crashingKv = { + ...kv, + delete: async (scope: string, key: string) => { + if (scope === KV.audit) { + deleteCalls++; + if (deleteCalls === 2) throw new Error("simulated crash"); + } + return kv.delete(scope, key); + }, + }; + + await startAuditMigration(crashingKv as never, { + probe: SMALL_STORE_PROBE, + deleteDelayMs: NO_DELAY, + }); + + const midLegacy = await kv.list(KV.audit); + expect(midLegacy.length).toBeGreaterThan(0); + + const midMarker = await kv.get( + KV.auditMonths, + AUDIT_MIGRATION_STATE_KEY, + ); + expect(midMarker?.status).toBe("copied"); + expect(midMarker?.summaryWritten).toBe(false); + + await startAuditMigration(kv as never, { + probe: SMALL_STORE_PROBE, + deleteDelayMs: NO_DELAY, + }); + + const finalLegacy = await kv.list(KV.audit); + expect(finalLegacy).toHaveLength(0); + + const marchRows = await kv.list(KV.auditMonth("2020-03")); + expect(marchRows.map((r) => r.id).sort()).toEqual(["keep_1", "keep_2"]); + + const currentMonth = auditMonthOf(new Date().toISOString()); + const currentMonthRows = await kv.list( + KV.auditMonth(currentMonth), + ); + const summaryRows = currentMonthRows.filter( + (r) => r.operation === "audit_migrate", + ); + expect(summaryRows).toHaveLength(1); + expect(summaryRows[0].details.purged).toBe(1); + expect(summaryRows[0].details.migrated).toBe(2); + + const finalMarker = await kv.get( + KV.auditMonths, + AUDIT_MIGRATION_STATE_KEY, + ); + expect(finalMarker?.status).toBe("done"); + expect(finalMarker?.summaryWritten).toBe(true); + }); +}); + +describe("audit retention sweep", () => { + let kv: ReturnType; + + beforeEach(() => { + kv = mockKV(); + }); + + it("defaults to 0 (keep forever) when unset", () => { + expect(getAuditRetentionMonths()).toBe(0); + }); + + it("drops nothing when retentionMonths is 0", async () => { + await seedMonthIndex(kv, ["2020-01"]); + seedEntry(kv, KV.auditMonth("2020-01"), "2020-01-01T00:00:00.000Z"); + + const result = await runAuditRetentionSweep(kv as never, 0); + expect(result).toEqual({ droppedMonths: [], droppedRows: 0 }); + + const months = await listAuditMonthsDesc(kv as never); + expect(months).toEqual(["2020-01"]); + }); + + it("drops whole month scopes older than the retention window", async () => { + const now = new Date(); + const oldMonth = new Date( + Date.UTC(now.getUTCFullYear(), now.getUTCMonth() - 13, 1), + ) + .toISOString() + .slice(0, 7); + const recentMonth = new Date( + Date.UTC(now.getUTCFullYear(), now.getUTCMonth() - 1, 1), + ) + .toISOString() + .slice(0, 7); + + await seedMonthIndex(kv, [oldMonth, recentMonth]); + seedEntry(kv, KV.auditMonth(oldMonth), `${oldMonth}-05T00:00:00.000Z`, { + id: "old_1", + }); + seedEntry( + kv, + KV.auditMonth(recentMonth), + `${recentMonth}-05T00:00:00.000Z`, + { id: "recent_1" }, + ); + + const result = await runAuditRetentionSweep(kv as never, 12); + + expect(result.droppedMonths).toEqual([oldMonth]); + expect(result.droppedRows).toBe(1); + + const remainingMonths = await listAuditMonthsDesc(kv as never); + expect(remainingMonths).toEqual([recentMonth]); + + const oldRows = await kv.list(KV.auditMonth(oldMonth)); + expect(oldRows).toHaveLength(0); + + const recentRows = await kv.list(KV.auditMonth(recentMonth)); + expect(recentRows).toHaveLength(1); + }); + + it("deletes dropped rows in reverse insertion order", async () => { + const oldMonth = "2020-01"; + await seedMonthIndex(kv, [oldMonth]); + seedEntry(kv, KV.auditMonth(oldMonth), "2020-01-01T00:00:00.000Z", { + id: "first", + }); + seedEntry(kv, KV.auditMonth(oldMonth), "2020-01-02T00:00:00.000Z", { + id: "second", + }); + seedEntry(kv, KV.auditMonth(oldMonth), "2020-01-03T00:00:00.000Z", { + id: "third", + }); + + const deleteOrder: string[] = []; + const orderedKv = { + ...kv, + delete: async (scope: string, key: string) => { + deleteOrder.push(key); + return kv.delete(scope, key); + }, + }; + + await runAuditRetentionSweep(orderedKv as never, 1); + expect(deleteOrder).toEqual(["third", "second", "first"]); + }); + + it("does not drop a month added by another writer while the sweep is running", async () => { + const oldMonth = "2020-01"; + const currentMonth = auditMonthOf(new Date().toISOString()); + await seedMonthIndex(kv, [oldMonth]); + seedEntry(kv, KV.auditMonth(oldMonth), `${oldMonth}-05T00:00:00.000Z`, { + id: "old_1", + }); + + let injected = false; + const raceKv = { + ...kv, + list: async (scope: string): Promise => { + if (!injected && scope === KV.auditMonth(oldMonth)) { + injected = true; + await recordAudit(kv as never, "delete", "mem::test", ["concurrent"]); + } + return kv.list(scope); + }, + }; + + const result = await runAuditRetentionSweep(raceKv as never, 1); + expect(result.droppedMonths).toEqual([oldMonth]); + + const months = await listAuditMonthsDesc(kv as never); + expect(months).toEqual([currentMonth]); + }); +}); + +describe("markAuditMonth concurrency", () => { + afterEach(() => { + vi.useRealTimers(); + }); + + it("keeps both months when two writers race different months onto the index", async () => { + const kv = mockKV(); + + vi.useFakeTimers(); + vi.setSystemTime(new Date("2020-01-15T00:00:00.000Z")); + const pA = recordAudit(kv as never, "delete", "mem::test", ["a"]); + vi.setSystemTime(new Date("2020-02-15T00:00:00.000Z")); + const pB = recordAudit(kv as never, "delete", "mem::test", ["b"]); + await Promise.all([pA, pB]); + vi.useRealTimers(); + + const months = await listAuditMonthsDesc(kv as never); + expect([...months].sort()).toEqual(["2020-01", "2020-02"]); + }); +}); diff --git a/test/audit.test.ts b/test/audit.test.ts index b98faea8dd..45ab21d6c6 100644 --- a/test/audit.test.ts +++ b/test/audit.test.ts @@ -60,7 +60,7 @@ describe("Audit Functions", () => { await new Promise((r) => setTimeout(r, 10)); await recordAudit(kv as never, "delete", "fn2", ["b"], {}); - const entries = await queryAudit(kv as never); + const { entries } = await queryAudit(kv as never); expect(entries.length).toBe(2); expect( new Date(entries[0].timestamp).getTime(), @@ -72,7 +72,7 @@ describe("Audit Functions", () => { await recordAudit(kv as never, "delete", "fn2", [], {}); await recordAudit(kv as never, "observe", "fn3", [], {}); - const entries = await queryAudit(kv as never, { operation: "observe" }); + const { entries } = await queryAudit(kv as never, { operation: "observe" }); expect(entries.length).toBe(2); expect(entries.every((e) => e.operation === "observe")).toBe(true); }); @@ -82,13 +82,13 @@ describe("Audit Functions", () => { await new Promise((r) => setTimeout(r, 20)); const late = await recordAudit(kv as never, "delete", "fn2", [], {}); - const entries = await queryAudit(kv as never, { + const { entries } = await queryAudit(kv as never, { dateFrom: late.timestamp, }); expect(entries.length).toBe(1); expect(entries[0].operation).toBe("delete"); - const entriesBefore = await queryAudit(kv as never, { + const { entries: entriesBefore } = await queryAudit(kv as never, { dateTo: early.timestamp, }); expect(entriesBefore.length).toBe(1); @@ -100,9 +100,9 @@ describe("Audit Functions", () => { await recordAudit(kv as never, "observe", "mem::observe", ["obs_9"]); await recordAudit(kv as never, "lesson_save", "mem::lesson-save", ["lsn_1"]); - expect((await queryAudit(kv as never, { query: "MEM_AB" })).map((e) => e.targetIds[0])).toEqual(["mem_abc"]); - expect((await queryAudit(kv as never, { query: "lesson-save" })).map((e) => e.targetIds[0])).toEqual(["lsn_1"]); - expect(await queryAudit(kv as never, { query: "nothing" })).toEqual([]); + expect((await queryAudit(kv as never, { query: "MEM_AB" })).entries.map((e) => e.targetIds[0])).toEqual(["mem_abc"]); + expect((await queryAudit(kv as never, { query: "lesson-save" })).entries.map((e) => e.targetIds[0])).toEqual(["lsn_1"]); + expect((await queryAudit(kv as never, { query: "nothing" })).entries).toEqual([]); }); it("queryAudit respects limit", async () => { @@ -110,7 +110,7 @@ describe("Audit Functions", () => { await recordAudit(kv as never, "observe", `fn${i}`, [], {}); } - const entries = await queryAudit(kv as never, { limit: 3 }); + const { entries } = await queryAudit(kv as never, { limit: 3 }); expect(entries.length).toBe(3); }); }); diff --git a/test/consolidation-pipeline.test.ts b/test/consolidation-pipeline.test.ts index 2bdd77dac3..831e4a8fc9 100644 --- a/test/consolidation-pipeline.test.ts +++ b/test/consolidation-pipeline.test.ts @@ -11,6 +11,7 @@ vi.mock("../src/config.js", () => ({ import { registerConsolidationPipelineFunction } from "../src/functions/consolidation-pipeline.js"; import { isConsolidationEnabled } from "../src/config.js"; +import { currentAuditScope } from "./helpers/mocks.js"; import type { SessionSummary, Memory, SemanticMemory, ProceduralMemory } from "../src/types.js"; function mockKV() { @@ -239,7 +240,7 @@ describe("Consolidation Pipeline", () => { await sdk.trigger("mem::consolidate-pipeline", { tier: "semantic" }); - const audits = await kv.list("mem:audit"); + const audits = await kv.list(currentAuditScope()); expect(audits.length).toBe(1); }); diff --git a/test/evict.test.ts b/test/evict.test.ts index 69786cbd07..59a13f0c83 100644 --- a/test/evict.test.ts +++ b/test/evict.test.ts @@ -6,6 +6,7 @@ import type { } from "../src/types.js"; import { registerEvictFunction } from "../src/functions/evict.js"; import { KV } from "../src/state/schema.js"; +import { currentAuditScope } from "./helpers/mocks.js"; import { indexObservationSession, lookupObservationSession, @@ -159,7 +160,7 @@ describe("mem::evict stale sessions", () => { expect(await kv.get(KV.sessions, sessionId)).toBeNull(); const audits = await kv.list<{ details: { reason: string }; - }>(KV.audit); + }>(currentAuditScope()); expect(audits[0].details.reason).toBe( "stale_session_recovered_then_evicted", ); diff --git a/test/governance.test.ts b/test/governance.test.ts index 6390bbeea6..76334ee329 100644 --- a/test/governance.test.ts +++ b/test/governance.test.ts @@ -247,7 +247,9 @@ describe("Governance Functions", () => { reason: "cleanup", }); - const entries = (await sdk.trigger("mem::audit-query", {})) as AuditEntry[]; + const { entries } = (await sdk.trigger("mem::audit-query", {})) as { + entries: AuditEntry[]; + }; expect(entries.length).toBe(1); expect(entries[0].operation).toBe("delete"); diff --git a/test/helpers/mocks.ts b/test/helpers/mocks.ts index bc829ed5f6..7b3fd335c9 100644 --- a/test/helpers/mocks.ts +++ b/test/helpers/mocks.ts @@ -1,7 +1,12 @@ import { vi } from "vitest"; +import { KV } from "../../src/state/schema.js"; type Handler = (data: unknown) => Promise; +export function currentAuditScope(): string { + return KV.auditMonth(new Date().toISOString().slice(0, 7)); +} + export function mockKV() { const store = new Map>(); return { diff --git a/test/lessons.test.ts b/test/lessons.test.ts index 17213256e1..eeacbaf44a 100644 --- a/test/lessons.test.ts +++ b/test/lessons.test.ts @@ -5,6 +5,7 @@ vi.mock("../src/logger.js", () => ({ })); import { normalizeLessonSourceIds, registerLessonsFunctions } from "../src/functions/lessons.js"; +import { currentAuditScope } from "./helpers/mocks.js"; import type { Lesson } from "../src/types.js"; function mockKV() { @@ -470,7 +471,7 @@ describe("Lessons", () => { await sdk.trigger("mem::lesson-delete", { lessonId: saved.lesson.id }); - const auditRows = (await kv.list("mem:audit")) as Array<{ + const auditRows = (await kv.list(currentAuditScope())) as Array<{ operation: string; targetIds: string[]; }>; diff --git a/test/remember-forget-audit.test.ts b/test/remember-forget-audit.test.ts index f875b7c071..f87ae47451 100644 --- a/test/remember-forget-audit.test.ts +++ b/test/remember-forget-audit.test.ts @@ -14,6 +14,7 @@ import { setIndexPersistence, } from "../src/functions/search.js"; import { memoryToObservation } from "../src/state/memory-utils.js"; +import { currentAuditScope } from "./helpers/mocks.js"; import type { Memory } from "../src/types.js"; function mockKV() { @@ -70,7 +71,7 @@ describe("mem::forget audit coverage (issue #125)", () => { functionId: string; targetIds: string[]; details: Record; - }>("mem:audit"); + }>(currentAuditScope()); expect(auditRows).toHaveLength(1); const [row] = auditRows; expect(row.operation).toBe("forget"); @@ -99,7 +100,7 @@ describe("mem::forget audit coverage (issue #125)", () => { const auditRows = await kv.list<{ targetIds: string[]; details: Record; - }>("mem:audit"); + }>(currentAuditScope()); expect(auditRows).toHaveLength(1); const [row] = auditRows; expect([...row.targetIds].sort()).toEqual(["obs_a", "obs_b"]); @@ -119,7 +120,7 @@ describe("mem::forget audit coverage (issue #125)", () => { payload: { sessionId: undefined, memoryId: undefined }, }); - const auditRows = await kv.list("mem:audit"); + const auditRows = await kv.list(currentAuditScope()); expect(auditRows).toHaveLength(0); }); @@ -153,7 +154,7 @@ describe("mem::forget audit coverage (issue #125)", () => { payload: { memoryId: "lsn_4f9cb07017a7c8ac" }, }); - const auditRows = await kv.list("mem:audit"); + const auditRows = await kv.list(currentAuditScope()); expect(auditRows).toHaveLength(0); }); }); diff --git a/test/retention.test.ts b/test/retention.test.ts index cf8cc681db..d71112e48a 100644 --- a/test/retention.test.ts +++ b/test/retention.test.ts @@ -4,6 +4,7 @@ import { setIndexPersistence, } from "../src/functions/search.js"; import { memoryToObservation } from "../src/state/memory-utils.js"; +import { currentAuditScope } from "./helpers/mocks.js"; import type { Memory, SemanticMemory } from "../src/types.js"; vi.mock("../src/logger.js", () => ({ @@ -358,7 +359,7 @@ describe("RetentionScoring", () => { functionId: string; targetIds: string[]; details: Record; - }>("mem:audit"); + }>(currentAuditScope()); const evictEntries = allEntries.filter( (e) => e.functionId === "mem::retention-evict", ); @@ -391,7 +392,7 @@ describe("RetentionScoring", () => { payload: { threshold: 0.0001 }, }); - const allEntries = await kv.list<{ functionId: string }>("mem:audit"); + const allEntries = await kv.list<{ functionId: string }>(currentAuditScope()); const evictEntries = allEntries.filter( (e) => e.functionId === "mem::retention-evict", ); @@ -417,7 +418,7 @@ describe("RetentionScoring", () => { functionId: string; targetIds: string[]; details: Record; - }>("mem:audit"); + }>(currentAuditScope()); const scoreEntries = allEntries.filter( (e) => e.functionId === "mem::retention-score", ); diff --git a/test/snapshot.test.ts b/test/snapshot.test.ts index 4f11751e7a..5fc51feab3 100644 --- a/test/snapshot.test.ts +++ b/test/snapshot.test.ts @@ -34,6 +34,7 @@ vi.mock("node:fs", () => ({ import { readFileSync } from "node:fs"; import { registerSnapshotFunction } from "../src/functions/snapshot.js"; +import { currentAuditScope } from "./helpers/mocks.js"; import { getProjectSessionIndex } from "../src/state/session-index.js"; import type { Session, Memory, SnapshotMeta } from "../src/types.js"; @@ -192,7 +193,7 @@ describe("Snapshot Functions", () => { it("snapshot-create records an audit entry", async () => { await sdk.trigger("mem::snapshot-create", { message: "Audit test" }); - const audits = await kv.list("mem:audit"); + const audits = await kv.list(currentAuditScope()); expect(audits.length).toBe(1); }); }); diff --git a/test/status.test.ts b/test/status.test.ts index 2448640db3..300394c5c4 100644 --- a/test/status.test.ts +++ b/test/status.test.ts @@ -39,6 +39,7 @@ function inputs(overrides: Partial = {}): StatusInputs { updatedAt: "2026-09-24T11:59:00.000Z", }, graphExtractionEnabled: true, + auditLegacy: null, ...overrides, }; } @@ -142,6 +143,21 @@ describe("evaluateStatus", () => { expect(report.status).toBe("info"); expect(report.problems[0].code).toBe("no-llm-provider"); }); + + it("reports a frozen legacy audit log only when the migration left it in place", () => { + expect(codes(evaluateStatus(inputs({ auditLegacy: { status: "too-large", sizeBytes: 1 } })))).toContain( + "audit-legacy-frozen", + ); + expect(codes(evaluateStatus(inputs({ auditLegacy: { status: "unreadable" } })))).toContain( + "audit-legacy-frozen", + ); + expect(codes(evaluateStatus(inputs({ auditLegacy: { status: "copied", sizeBytes: 1 } })))).not.toContain( + "audit-legacy-frozen", + ); + expect(codes(evaluateStatus(inputs({ auditLegacy: { status: "done" } })))).not.toContain( + "audit-legacy-frozen", + ); + }); }); describe("status without an LLM provider", () => { @@ -397,10 +413,11 @@ describe("status wiring", () => { it("time-boxes every status probe so a slow store cannot hang the page", () => { const reporter = api.slice(api.indexOf("export function createStatusReporter"), api.indexOf("export function createConsolidationStatusReader")); - expect(reporter.match(/valueWithin\(/g)?.length).toBe(4); + expect(reporter.match(/valueWithin\(/g)?.length).toBe(5); expect(reporter).toMatch(/valueWithin\(scan\.run\(\), STATUS_CHECK_TIMEOUT_MS\)/); expect(api).toMatch(/run: singleFlight\(async \(\) => \{\s*const value = await findUnindexedObservations\(kv\);/); const handler = api.slice(api.indexOf('registerFunction("api::status"'), api.indexOf('function_id: "api::status"')); + expect(reporter).toMatch(/valueWithin\(\s*kv\.get\(KV\.auditMonths, AUDIT_MIGRATION_STATE_KEY\),\s*STATUS_CHECK_TIMEOUT_MS,\s*\)/); expect(handler).toContain("const report = await statusReport();"); }); diff --git a/test/viewer-streams.test.ts b/test/viewer-streams.test.ts index 752e17276f..96c0b32a44 100644 --- a/test/viewer-streams.test.ts +++ b/test/viewer-streams.test.ts @@ -3,6 +3,7 @@ import { registerViewerStreamTriggers } from "../src/triggers/viewer-streams.js" import { registerEventTriggers } from "../src/triggers/events.js"; import { getViewerCounts, resetViewerCounts } from "../src/state/viewer-counts.js"; import { KV } from "../src/state/schema.js"; +import { recordAudit, setAuditRecordedListener } from "../src/functions/audit.js"; type Handler = (payload: unknown) => Promise; type Sent = { type: string; group_id: string; data: Record }; @@ -69,6 +70,7 @@ describe("viewer stream triggers", () => { afterEach(() => { vi.useRealTimers(); resetViewerCounts(); + setAuditRecordedListener(null); }); it("registers a state trigger for every scope the viewer shows", () => { @@ -82,7 +84,6 @@ describe("viewer stream triggers", () => { KV.crystals, KV.semantic, KV.procedural, - KV.audit, KV.graphSnapshot, KV.graphNodes, KV.graphEdges, @@ -96,7 +97,7 @@ describe("viewer stream triggers", () => { }); it("pushes created, updated and deleted rows for lessons, actions, crystals, semantic, procedural and audit", async () => { - const { handlers, sent } = setup(); + const { handlers, sent, kv } = setup(); const lesson = { id: "l9", content: "use pnpm", confidence: 0.6 }; await handlers.get("event::viewer::lesson-changed")!({ key: "l9", event_type: "state:created", new_value: lesson }); await handlers.get("event::viewer::lesson-changed")!({ @@ -110,11 +111,7 @@ describe("viewer stream triggers", () => { await handlers.get("event::viewer::crystal-changed")!({ key: "c1", event_type: "state:created", new_value: { id: "c1" } }); await handlers.get("event::viewer::semantic-changed")!({ key: "f1", event_type: "state:created", new_value: { id: "f1" } }); await handlers.get("event::viewer::procedural-changed")!({ key: "p1", event_type: "state:created", new_value: { id: "p1" } }); - await handlers.get("event::viewer::audit-changed")!({ - key: "au1", - event_type: "state:created", - new_value: { id: "au1", operation: "forget", targetIds: ["m1"] }, - }); + await recordAudit(kv as never, "forget", "mem::forget", ["m1"]); expect(sent.map((e) => e.type)).toEqual([ "lesson.created", "lesson.updated",