diff --git a/.env.example b/.env.example index d24b3f654d..4b2df2eef2 100644 --- a/.env.example +++ b/.env.example @@ -1,5 +1,7 @@ -# ======================================================================# agentmemory configuration -# ======================================================================# +# ============================================================================= +# agentmemory configuration +# ============================================================================= +# # Copy this file to `~/.agentmemory/.env` (or to your project root if you # prefer scoped config) and uncomment the lines you want to override. # diff --git a/docs/upgrades/iii-0.26.md b/docs/upgrades/iii-0.26.md index 3d35e42113..750b6706fd 100644 --- a/docs/upgrades/iii-0.26.md +++ b/docs/upgrades/iii-0.26.md @@ -22,7 +22,7 @@ Use the same `--instance`, `--port` and `--data-dir` flags you used to start it. - Observers use `capture` only. Background observers receive the `Observed` payload and read the event from its `input` or `output`; HTTP route handlers still receive the `HttpRequest`. - Failure audit observers capture `["input", "error"]` with an `error` filter for thrown errors, and `["input", "output"]` with an `output.success == false` filter for soft failures. - When the state or http worker has not registered within 30 seconds of boot, `runtime.bindings` in `/agentmemory/status` reports that provider as `pending` instead of falling back. Its observers attach as soon as the worker connects. -- A pipeline step runs as a direct call while its own observer is not active (pending, rejected or suspended), so compression and post-session work keep running. `runtime.bindings.reasons` lists the steps that use direct calls. +- A compression or post-session step runs as a direct call while its own observer is not active (pending, rejected or suspended), so that work keeps running. `runtime.bindings.reasons` lists the steps that use direct calls. Delete cascades, relation reactions, failure audits and post-remember graph extraction have no direct-call path: `runtime.bindings.reasons` lists them separately, and their work waits until their observers are active. - The compression sweep retries a failed compression at most 5 times and sends pending observations before failed ones. ## Custom deployments diff --git a/src/functions/status.ts b/src/functions/status.ts index 8bfe8da91a..7adb7f4b20 100644 --- a/src/functions/status.ts +++ b/src/functions/status.ts @@ -299,7 +299,7 @@ export function evaluateStatus(input: StatusInputs): StatusReport { problems.push({ level: "error", code: "observer-rejected", - message: `${failedObservers} observer${failedObservers === 1 ? " was" : "s were"} rejected by the engine. Rejected route or state observers leave those endpoints or reactions inactive; rejected pipeline observers fall back to direct calls.`, + message: `${failedObservers} observer${failedObservers === 1 ? " was" : "s were"} rejected by the engine. Rejected route or state observers leave those endpoints or reactions inactive. Rejected compression and post-session observers fall back to direct calls; rejected delete-cascade, relation-reaction, failure-audit and post-remember graph observers stop that work until restart.`, fix: "Check runtime.bindings.reasons in /agentmemory/status for the conflicting route, remove the conflict, then restart agentmemory.", }); } diff --git a/src/index.ts b/src/index.ts index 9280f3c70c..3e115806e3 100644 --- a/src/index.ts +++ b/src/index.ts @@ -135,7 +135,7 @@ import { bootLog, bootWarn } from "./logger.js"; import { runtimeMetadataPath } from "./runtime-paths.js"; import { getBindings } from "./runtime/bindings.js"; import { waitForStateWorker, onStateWorkerReturn } from "./runtime/state-worker.js"; -import { singleFlight, shouldRearmIndex, rearmIndexRecovery, backfillFailedOutright } from "./runtime/recovery.js"; +import { singleFlight, shouldRearmIndex, rearmIndexRecovery, backfillFailedOutright, createRetryBackoff } from "./runtime/recovery.js"; import { checkDuplicateWorkers } from "./runtime/duplicates.js"; import { registerPipelineObservers } from "./runtime/pipeline-observers.js"; import { registerPostSessionFunctions } from "./functions/post-session.js"; @@ -489,10 +489,11 @@ async function main() { registerPipelineObservers(sdk); const bindings = getBindings(sdk); + let sweepTimer: ReturnType | null = null; if (isAutoCompressEnabled()) { const sweepMs = compressionSweepMs(); if (sweepMs > 0) { - const sweepTimer = setInterval(() => { void runCompressionSweep(); }, sweepMs); + sweepTimer = setInterval(() => { void runCompressionSweep(); }, sweepMs); sweepTimer.unref(); } } @@ -559,18 +560,22 @@ async function main() { ); } else { console.error( - `[agentmemory] Refusing to start: persisted vector index has ` + + `[agentmemory] Not loading the persisted vector index: it has ` + `${mismatches.length} of ${loaded.vector.size} vectors with the ` + `wrong dimension. Active provider (${embeddingProvider?.name}) ` + `declares ${activeDim}; dimensions seen on disk: ${distinct}. ` + `First mismatched obsIds: ${sample}. Loading would silently corrupt ` + - `search (cross-dimension cosine returns 0). Choose one:\n` + - ` - Re-embed the existing index against the new provider, then start.\n` + + `search (cross-dimension cosine returns 0). The checkpoint is kept ` + + `and vector recovery stays paused; it is not retried. Choose one, then restart:\n` + + ` - Re-embed the existing index against the new provider.\n` + ` - Set AGENTMEMORY_DROP_STALE_INDEX=true to discard the persisted ` + `vectors and rebuild from live observations.\n` + ` - Switch the embedding provider back to the one that wrote the index.`, ); - await indexPersistence.prepareBackfill(null); + indexPersistence.pauseForUnusableCheckpoint( + `persisted vector checkpoint has ${mismatches.length} of ${loaded.vector.size} vectors with the wrong dimension`, + ); + checkpointBackfillSince = undefined; mismatchBlocked = true; } } else if (boot) { @@ -732,23 +737,36 @@ async function main() { capture.start(); })(); + // An unreadable boot checkpoint is retried on the runtime tick with a + // doubling delay; a checkpoint that loads but cannot be used stops retries. + // Neither case is fixed by rebuilding the keyword index, so they do not + // count as a paused recovery that needs one. + const checkpointRetry = createRetryBackoff({ initialMs: 30_000, maxMs: 30 * 60_000 }); + const checkpointUnusable = () => indexPersistence.isPausedForUnavailable() || checkpointRetry.stopped; const rearm = onStateWorkerReturn(sdk, bindings, async () => { try { await rearmIndexRecovery({ retryCheckpoint: async () => { - if (!bootCheckpointUnavailable || !indexPersistence.resumeAfterUnavailable()) return false; + if (!bootCheckpointUnavailable || checkpointRetry.stopped || !indexPersistence.resumeAfterUnavailable()) return false; const retry = await restoreVectorCheckpoint(false); + if (retry.mismatchBlocked) { + checkpointRetry.stop(); + return false; + } checkpointLoaded = retry.loaded; vectorBackfillSince = retry.vectorBackfillSince; - if (!retry.loaded || retry.loaded.state === "unavailable") return false; + if (!retry.loaded || retry.loaded.state === "unavailable") { + checkpointRetry.failed(); + return false; + } vectorIndex?.finishLiveTracking(); return true; }, indexNeedsRearm: () => shouldRearmIndex({ bm25Incomplete: isBm25RebuildIncomplete(), - persistencePaused: indexPersistence.isRecoveryPaused(), + persistencePaused: indexPersistence.isRecoveryPaused() && !checkpointUnusable(), lastRecoveryFailed: indexRecoveryFailed, - checkpointSaveFailed: Boolean(indexPersistence.status().vector?.lastError), + checkpointSaveFailed: indexPersistence.checkpointSaveFailed(), }), runIndexRecovery, }); @@ -760,10 +778,16 @@ async function main() { } catch (err) { bootWarn(`State-worker re-arm failed: ${err instanceof Error ? err.message : String(err)}`); } - }, { pending: () => bootCheckpointUnavailable && indexPersistence.isPausedForUnavailable() }); + }, { + pending: () => bootCheckpointUnavailable && indexPersistence.isPausedForUnavailable() && checkpointRetry.due(), + onWorkersChanged: () => { void checkDuplicateWorkers(sdk); }, + }); + // Worker connect/disconnect events drive the duplicate-worker check, plus one + // check whenever that observer becomes active again. While it is not active + // (pending, rejected, out of conflict retries) the tick checks instead. const rearmTick = setInterval(() => { void rearm.probeTick(); - void checkDuplicateWorkers(sdk); + if (!bindings.runtimeActive("runtime::on-workers-changed")) void checkDuplicateWorkers(sdk); }, 30_000); rearmTick.unref(); void checkDuplicateWorkers(sdk); @@ -908,6 +932,9 @@ async function main() { process.exit(await runStartupCleanup(1).catch(() => 1)); }, SHUTDOWN_HARD_EXIT_MS); hardExit.unref(); + clearInterval(rearmTick); + if (sweepTimer) clearInterval(sweepTimer); + rearm.stop(); healthMonitor.stop(); dedupMap.stop(); capture.stop(); diff --git a/src/runtime/bindings.ts b/src/runtime/bindings.ts index 25b9104aa1..e0e6e9c96f 100644 --- a/src/runtime/bindings.ts +++ b/src/runtime/bindings.ts @@ -35,7 +35,15 @@ interface SdkLike { observe?: (source: ObserveSource, target: string, options?: ObserverOptions) => ObserverHandle; } -type BindingKind = "http" | "state" | "pipeline"; +type BindingKind = "http" | "state" | "pipeline" | "runtime"; + +interface BoundObserver { + handle: ObserverHandle; + kind: BindingKind; + target: string; + // Pipeline work that also runs as a direct call while its observer is not active. + fallback: boolean; +} const instances = new WeakMap(); @@ -47,11 +55,11 @@ const ID_CONFLICT_RETRY_DELAYS_MS = [10_000, 20_000, 40_000, 80_000]; export class Bindings { private readonly sdk: SdkLike; - private readonly bound: Array<{ handle: ObserverHandle; kind: BindingKind; target: string }> = []; + private readonly bound: BoundObserver[] = []; private readonly reasons = new Set(); private readonly inline: boolean; private failedObserverCount = 0; - private readonly boundWatchers = new Set<(entry: { handle: ObserverHandle; kind: BindingKind }) => void>(); + private readonly boundWatchers = new Set<(entry: BoundObserver) => void>(); constructor(sdk: SdkLike) { this.sdk = sdk; @@ -87,6 +95,18 @@ export class Bindings { return this.bound.some((entry) => entry.kind === "pipeline" && entry.target === target && entry.handle.state === "active"); } + onRuntimeHandle(target: string, listener: (handle: ObserverHandle) => void): () => void { + const watch = (entry: BoundObserver) => { + if (entry.kind === "runtime" && entry.target === target) listener(entry.handle); + }; + this.boundWatchers.add(watch); + return () => this.boundWatchers.delete(watch); + } + + runtimeActive(target: string): boolean { + return this.bound.some((entry) => entry.kind === "runtime" && entry.target === target && entry.handle.state === "active"); + } + get modes(): BindingModes { return { state: this.modeOf("state"), http: this.modeOf("http") }; } @@ -107,13 +127,21 @@ export class Bindings { return parsed; } - private observe(kind: BindingKind, label: string, source: ObserveSource, target: string, options?: ObserverOptions, attempt = 0): ObserverHandle | null { + private observe( + kind: BindingKind, + label: string, + source: ObserveSource, + target: string, + options?: ObserverOptions, + extra: { fallback?: boolean; attempt?: number } = {}, + ): ObserverHandle | null { + const attempt = extra.attempt ?? 0; if (typeof this.sdk.observe !== "function") { this.reasons.add("client has no observe(); bindings skipped"); return null; } const handle = this.sdk.observe(source, target, options); - const entry = { handle, kind, target }; + const entry: BoundObserver = { handle, kind, target, fallback: extra.fallback ?? false }; this.bound.push(entry); for (const watch of this.boundWatchers) watch(entry); void handle.ready().catch((err: unknown) => { @@ -125,7 +153,7 @@ export class Bindings { void handle.dispose().catch(() => {}).finally(() => { const index = this.bound.indexOf(entry); if (index >= 0) this.bound.splice(index, 1); - this.observe(kind, label, source, target, options, attempt + 1); + this.observe(kind, label, source, target, options, { ...extra, attempt: attempt + 1 }); }); }, delay); timer.unref?.(); @@ -179,9 +207,23 @@ export class Bindings { ); } - observePipeline(source: ObserveSource, target: string, options?: ObserverOptions): ObserverHandle | null { + /** + * Binds a pipeline step. `fallback: true` (the default) means the caller + * checks `observing(target)` and calls the step directly while the observer + * is not active; pass `fallback: false` for work that only runs through it. + */ + observePipeline( + source: ObserveSource, + target: string, + options?: ObserverOptions, + opts: { fallback?: boolean } = {}, + ): ObserverHandle | null { if (this.inline) return null; - return this.observe("pipeline", `pipeline ${target}`, source, target, options); + return this.observe("pipeline", `pipeline ${target}`, source, target, options, { fallback: opts.fallback ?? true }); + } + + observeRuntime(source: ObserveSource, target: string, options?: ObserverOptions): ObserverHandle | null { + return this.observe("runtime", `runtime ${target}`, source, target, options); } observeStateChanges(filters: Record, target: string, options?: ObserverOptions): ObserverHandle | null { @@ -208,7 +250,7 @@ export class Bindings { const timer = setTimeout(finish, timeoutMs); timer.unref?.(); const watch = ({ handle, kind }: { handle: ObserverHandle; kind: BindingKind }) => { - if (done || kind === "pipeline") return; + if (done || (kind !== "state" && kind !== "http")) return; unsubscribes.push(handle.onStateChange(() => { if (settled()) finish(); })); if (settled()) finish(); }; @@ -223,11 +265,19 @@ export class Bindings { const reasons = new Set(this.reasons); if (modes.state === "pending" && this.handlesOf("state").length > 0) reasons.add("state::on-changed observers pending until the state worker connects"); if (modes.http === "pending" && this.handlesOf("http").length > 0) reasons.add("http::on-request observers pending until the http worker connects"); - if (!this.inline) { - const inactive = this.bound.filter((entry) => entry.kind === "pipeline" && entry.handle.state !== "active"); - if (inactive.length > 0) { - reasons.add(`${inactive.length} pipeline observers are not active, so their work runs as direct calls: ${inactive.map((entry) => entry.target).join(", ")}`); - } + const inactive = this.bound.filter((entry) => entry.handle.state !== "active"); + const targets = (entries: BoundObserver[]) => [...new Set(entries.map((entry) => entry.target))].join(", "); + const directCalls = inactive.filter((entry) => entry.kind === "pipeline" && entry.fallback); + if (!this.inline && directCalls.length > 0) { + reasons.add(`${directCalls.length} pipeline observers are not active, so their work runs as direct calls: ${targets(directCalls)}`); + } + const stalled = inactive.filter((entry) => entry.kind === "pipeline" && !entry.fallback); + if (stalled.length > 0) { + reasons.add(`${stalled.length} observers have no direct-call fallback and are not active, so their work does not run until they activate: ${targets(stalled)}`); + } + const runtime = inactive.filter((entry) => entry.kind === "runtime"); + if (runtime.length > 0) { + reasons.add(`${runtime.length} runtime observers are not active: ${targets(runtime)}`); } for (const kind of ["state", "http"] as const) { if (modes[kind] !== "observe") continue; diff --git a/src/runtime/duplicates.ts b/src/runtime/duplicates.ts index 1af727d32e..b734a8e7a5 100644 --- a/src/runtime/duplicates.ts +++ b/src/runtime/duplicates.ts @@ -8,6 +8,10 @@ export interface DuplicateWorkerInfo { let last: DuplicateWorkerInfo | null = null; let lastKey = ""; +// Checks can overlap (worker events, the fallback tick, a state-worker +// return). A check only publishes when no check started after it already has. +let started = 0; +let applied = 0; export function getDuplicateWorkers(): DuplicateWorkerInfo | null { return last; @@ -16,11 +20,16 @@ export function getDuplicateWorkers(): DuplicateWorkerInfo | null { export function resetDuplicateWorkersForTest(): void { last = null; lastKey = ""; + started = 0; + applied = 0; } export async function checkDuplicateWorkers(sdk: IIIClient): Promise { + const seq = ++started; try { const listed = await sdk.trigger({ function_id: "engine::workers::list", payload: {} }); + if (seq < applied) return last; + applied = seq; const entries = Array.isArray(listed) ? listed : (((listed as { workers?: unknown[] })?.workers ?? []) as unknown[]); diff --git a/src/runtime/pipeline-observers.ts b/src/runtime/pipeline-observers.ts index 891e80ecea..76382d5379 100644 --- a/src/runtime/pipeline-observers.ts +++ b/src/runtime/pipeline-observers.ts @@ -49,6 +49,7 @@ export function registerPipelineObservers(sdk: IIIClient): void { { function_id: "mem::remember", capture: ["input", "output"] }, "mem::post-remember-graph", { id: "agentmemory:pipeline:post-remember-graph", max_in_flight: 1 }, + { fallback: false }, ); } for (const fn of ["mem::compress", "mem::summarize", "mem::graph-extract", "mem::consolidate-pipeline", "mem::reflect", "mem::skill-extract"]) { @@ -57,6 +58,7 @@ export function registerPipelineObservers(sdk: IIIClient): void { { function_id: fn, filters: failure.filters, capture: failure.capture }, "mem::record-failure", { id: `agentmemory:pipeline:failure:${fn}:${failure.suffix}`, max_in_flight: 4, metadata: { source_fn: fn } }, + { fallback: false }, ); } } @@ -64,6 +66,7 @@ export function registerPipelineObservers(sdk: IIIClient): void { { function_id: "mem::graph-extract", filters: { "output.partial": { $eq: true } }, capture: ["input", "output"] }, "mem::record-failure", { id: "agentmemory:pipeline:failure:mem::graph-extract:partial", max_in_flight: 4, metadata: { source_fn: "mem::graph-extract" } }, + { fallback: false }, ); bindings.observeStateChanges( { diff --git a/src/runtime/recovery.ts b/src/runtime/recovery.ts index 28314c9394..b57d3c3bfe 100644 --- a/src/runtime/recovery.ts +++ b/src/runtime/recovery.ts @@ -26,6 +26,36 @@ export async function rearmIndexRecovery(steps: RearmSteps): Promise { if (restored || steps.indexNeedsRearm()) await steps.runIndexRecovery(); } +export interface RetryBackoff { + readonly stopped: boolean; + /** True once the current delay has passed and retries were not stopped. */ + due(): boolean; + /** Records a failed attempt and doubles the delay, up to maxMs. */ + failed(): void; + /** Stops retrying for the life of the process. */ + stop(): void; +} + +export function createRetryBackoff(opts: { initialMs: number; maxMs: number; now?: () => number }): RetryBackoff { + const now = opts.now ?? (() => Date.now()); + let delay = opts.initialMs; + let nextAt = now() + delay; + let stopped = false; + return { + get stopped() { + return stopped; + }, + due: () => !stopped && now() >= nextAt, + failed: () => { + delay = Math.min(delay * 2, opts.maxMs); + nextAt = now() + delay; + }, + stop: () => { + stopped = true; + }, + }; +} + export function backfillFailedOutright(result: { added: number; failed: number }): boolean { return result.failed > 0 && result.added === 0; } diff --git a/src/runtime/state-worker.ts b/src/runtime/state-worker.ts index 3532f451a0..f094fc386f 100644 --- a/src/runtime/state-worker.ts +++ b/src/runtime/state-worker.ts @@ -80,7 +80,11 @@ export function onStateWorkerReturn( sdk: IIIClient, bindings: Bindings, cb: () => void | Promise, - opts: { pending?: () => boolean } = {}, + opts: { + pending?: () => boolean; + /** A worker connected or disconnected, or the observer reporting those events became active again. */ + onWorkersChanged?: (reason: "worker_connected" | "worker_disconnected" | "observer_active") => void; + } = {}, ): { state: StateWorkerReturnState; probeTick: () => Promise; stop: () => void } { const client = sdk as unknown as SdkLike; const state: StateWorkerReturnState = { probeReady: stateWorkerStatus.ready }; @@ -122,9 +126,30 @@ export function onStateWorkerReturn( }; for (const handle of bindings.stateHandles) watchStateHandle(handle, false); bindings.onStateHandle?.((handle) => watchStateHandle(handle, true)); - if (typeof client.observe === "function" && bindings.pipeline === "observe") { + if (typeof client.observe === "function" && typeof bindings.observeRuntime === "function") { + // Events are missed while the observer is not active, however short the gap. + // The first handle is covered by the boot check; a conflict replacement + // replaces one that never delivered, so it counts as a recovery even when + // it is already active. + const reportWorkersChanged = (reason: "worker_connected" | "worker_disconnected" | "observer_active") => { + if (!stopped) opts.onWorkersChanged?.(reason); + }; + let firstRuntimeHandle = true; + bindings.onRuntimeHandle?.("runtime::on-workers-changed", (handle) => { + let wasActive = handle.state === "active"; + if (wasActive && !firstRuntimeHandle) reportWorkersChanged("observer_active"); + firstRuntimeHandle = false; + handle.onStateChange((next) => { + if (next === "active" && !wasActive) reportWorkersChanged("observer_active"); + wasActive = next === "active"; + }); + }); client.registerFunction?.("runtime::on-workers-changed", async (payload: unknown) => { - if (unwrapObserved<{ event?: string } | undefined>(payload, "input")?.event !== "worker_connected") return { handled: false }; + const event = unwrapObserved<{ event?: string } | undefined>(payload, "input")?.event; + if (event !== "worker_connected" && event !== "worker_disconnected") return { handled: false }; + if (stopped) return { handled: false }; + reportWorkersChanged(event); + if (event !== "worker_connected") return { handled: true }; try { await probeOnce(client, 2000); stateWorkerStatus.ready = true; @@ -135,8 +160,12 @@ export function onStateWorkerReturn( } catch {} return { handled: true }; }); - client.observe( - { function_id: "engine::on-workers-changed", capture: ["input"] }, + bindings.observeRuntime( + { + function_id: "engine::on-workers-changed", + filters: { "input.event": { $in: ["worker_connected", "worker_disconnected"] } }, + capture: ["input"], + }, "runtime::on-workers-changed", { id: "agentmemory:runtime:workers-changed" }, ); diff --git a/src/state/index-persistence.ts b/src/state/index-persistence.ts index e6ee1a655c..49a3292d96 100644 --- a/src/state/index-persistence.ts +++ b/src/state/index-persistence.ts @@ -448,6 +448,21 @@ export class IndexPersistence { return this.recoveryPausedForUnavailable; } + /** + * Pauses recovery for a checkpoint that was read but cannot be used, such as + * one written with another embedding dimension. Unlike an unavailable + * checkpoint, reading it again cannot fix it, so it is not retried. + */ + pauseForUnusableCheckpoint(reason: string): void { + this.recoveryPausedForUnavailable = false; + this.pauseRecovery(new Error(reason)); + } + + /** True when the last checkpoint write failed, not when recovery is merely paused. */ + checkpointSaveFailed(): boolean { + return this.leg.lastError !== null && this.leg.lastError !== this.recoveryError; + } + private checkpointRecoveryCutoff(): string | null | undefined { return this.completingBackfill ? null : this.backfillSince; } diff --git a/test/bindings.test.ts b/test/bindings.test.ts index be99a04ca4..fbe724d8ad 100644 --- a/test/bindings.test.ts +++ b/test/bindings.test.ts @@ -2,6 +2,7 @@ import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; import { Bindings, getBindings } from "../src/runtime/bindings.js"; import { registerObserverTarget, unwrapObserved } from "../src/runtime/observed.js"; import { KV } from "../src/state/schema.js"; +import { registerPipelineObservers } from "../src/runtime/pipeline-observers.js"; vi.mock("../src/logger.js", () => ({ logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() }, @@ -346,3 +347,24 @@ describe("observed payloads", () => { expect(await handlers.get("t::in")!({ key: "direct" })).toBe("direct"); }); }); + +describe("observers without a direct-call fallback", () => { + afterEach(() => { vi.unstubAllEnvs(); }); + + it("reports as direct calls only the steps that really fall back", () => { + vi.stubEnv("AGENTMEMORY_AUTO_COMPRESS", "true"); + vi.stubEnv("GRAPH_EXTRACTION_ENABLED", "true"); + vi.stubEnv("AGENTMEMORY_REFLECT", "true"); + const sdk = fakeSdk("pending"); + registerPipelineObservers(sdk as never); + const reasons = getBindings(sdk as never).describe().reasons; + const direct = reasons.find((line) => line.includes("direct calls")) ?? ""; + for (const target of ["mem::compress", "mem::post-session-graph", "mem::post-session-reflect", "mem::post-session-consolidate"]) { + expect(direct).toContain(target); + } + for (const target of ["mem::post-remember-graph", "mem::record-failure", "mem::on-record-deleted", "event::graph::edge-reaction"]) { + expect(direct).not.toContain(target); + expect(reasons.some((line) => line.includes(target) && line.includes("no direct-call fallback"))).toBe(true); + } + }); +}); diff --git a/test/duplicate-workers.test.ts b/test/duplicate-workers.test.ts index 282a3badc5..4424b8d3ba 100644 --- a/test/duplicate-workers.test.ts +++ b/test/duplicate-workers.test.ts @@ -1,6 +1,6 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; import { readFileSync } from "node:fs"; -import { checkDuplicateWorkers, resetDuplicateWorkersForTest } from "../src/runtime/duplicates.js"; +import { checkDuplicateWorkers, getDuplicateWorkers, resetDuplicateWorkersForTest } from "../src/runtime/duplicates.js"; vi.mock("../src/logger.js", () => ({ logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() }, @@ -36,4 +36,21 @@ describe("route bindings in api.ts", () => { const api = readFileSync("src/triggers/api.ts", "utf-8"); expect(api).not.toMatch(/sdk\.registerTrigger\(\{\s*type:\s*"http"/); }); + + it("keeps the newest result when an older check finishes last", async () => { + resetDuplicateWorkersForTest(); + let releaseOlder!: (value: unknown) => void; + const trigger = vi.fn() + .mockImplementationOnce(() => new Promise((resolve) => { releaseOlder = resolve; })) + .mockImplementationOnce(async () => ({ + workers: [{ worker_id: "a", worker_name: "agentmemory" }, { worker_id: "b", worker_name: "agentmemory" }], + })); + const sdk = { trigger } as never; + const older = checkDuplicateWorkers(sdk); + await checkDuplicateWorkers(sdk); + expect(getDuplicateWorkers()?.count).toBe(2); + releaseOlder({ workers: [{ worker_id: "a", worker_name: "agentmemory" }] }); + await older; + expect(getDuplicateWorkers()?.count).toBe(2); + }); }); diff --git a/test/helpers/boot-main.ts b/test/helpers/boot-main.ts new file mode 100644 index 0000000000..d3df8a4e59 --- /dev/null +++ b/test/helpers/boot-main.ts @@ -0,0 +1,123 @@ +import { expect, vi } from "vitest"; +import { mkdtempSync } from "node:fs"; +import { createServer } from "node:net"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { engine } from "./fake-engine.js"; +import { StateKV } from "../../src/state/kv.js"; +import { VectorIndex } from "../../src/state/vector-index.js"; +import { IndexPersistence } from "../../src/state/index-persistence.js"; + +// Boots the real src/index.ts main() against the in-memory engine. The test +// file must mock "iii-sdk" (registerWorker -> engine.client), the embedding +// provider (8 dimensions) and wrap rebuildKeywordIndex in vi.fn. + +export const VECTOR_META = { scope: "mem:index:bm25", key: "vectors:meta" }; + +type Interval = { fn: () => unknown; ms: number; handle: ReturnType }; + +function freePort(): Promise { + return new Promise((resolve, reject) => { + const server = createServer(); + server.once("error", reject); + server.listen(0, "127.0.0.1", () => { + const address = server.address(); + server.close(() => resolve(typeof address === "object" && address ? address.port : 0)); + }); + }); +} + +export async function bootMain(opts: { checkpointDims: number; checkpointUnreadableAtBoot: boolean }) { + const intervals: Interval[] = []; + const cleared = new Set(); + const errors: string[] = []; + let clockOffset = 0; + + const home = mkdtempSync(join(tmpdir(), "am-main-")); + for (const key of Object.keys(process.env)) { + if (/_API_KEY$|^AGENTMEMORY_|^III_/.test(key)) delete process.env[key]; + } + const viewerPort = await freePort(); + Object.assign(process.env, { + HOME: home, + AGENTMEMORY_DATA_DIR: join(home, "data"), + AGENTMEMORY_RUNTIME_DIR: join(home, "run"), + AGENTMEMORY_SECRET: "a".repeat(64), + III_REST_PORT: String(viewerPort - 2), + III_VIEWER_PORT: String(viewerPort), + AGENTMEMORY_AUTO_COMPRESS: "true", + }); + + const realSetInterval = globalThis.setInterval; + const realClearInterval = globalThis.clearInterval; + vi.spyOn(globalThis, "setInterval").mockImplementation(((fn: () => unknown, ms?: number, ...args: unknown[]) => { + const handle = realSetInterval(fn, ms, ...args); + intervals.push({ fn, ms: Number(ms), handle }); + return handle; + }) as typeof setInterval); + vi.spyOn(globalThis, "clearInterval").mockImplementation(((handle?: ReturnType) => { + cleared.add(handle); + realClearInterval(handle); + }) as typeof clearInterval); + const realNow = Date.now.bind(Date); + vi.spyOn(Date, "now").mockImplementation(() => realNow() + clockOffset); + const exitSpy = vi.spyOn(process, "exit").mockImplementation((() => undefined) as never); + const realError = console.error.bind(console); + vi.spyOn(console, "error").mockImplementation((...args: unknown[]) => { + errors.push(args.map(String).join(" ")); + realError(...args); + }); + const loadSpy = vi.spyOn(IndexPersistence.prototype, "load"); + + // A checkpoint written before this start, by a provider with checkpointDims. + const old = new VectorIndex(); + old.add("obs_old", "ses_old", new Float32Array(opts.checkpointDims).fill(0.5)); + await new IndexPersistence(new StateKV(engine.client as never), old).save(); + expect(engine.store.get(VECTOR_META.scope)?.has(VECTOR_META.key)).toBe(true); + loadSpy.mockClear(); + if (opts.checkpointUnreadableAtBoot) { + engine.failGet = (scope, key) => scope === VECTOR_META.scope && key === VECTOR_META.key; + } + + const settle = async (): Promise => { + let last = -1; + let quiet = 0; + for (let i = 0; i < 300 && quiet < 3; i++) { + await new Promise((resolve) => setTimeout(resolve, 10)); + quiet = engine.calls.length === last ? quiet + 1 : 0; + last = engine.calls.length; + } + }; + + const before = new Set(process.listeners("SIGTERM")); + let shutdown: (() => unknown) | undefined; + void import("../../src/index.js"); + await vi.waitFor(() => { + shutdown = process.listeners("SIGTERM").find((listener) => !before.has(listener)) as (() => unknown) | undefined; + expect(shutdown, `main() did not finish booting; exit calls: ${JSON.stringify(exitSpy.mock.calls)}`).toBeDefined(); + }, { timeout: 30_000, interval: 25 }); + await settle(); + + const runtimeTick = (): Interval => { + const entry = intervals.find((interval) => interval.ms === 30_000 && String(interval.fn).includes("probeTick")); + if (!entry) throw new Error("main() did not install the 30 s runtime tick"); + return entry; + }; + + return { + intervals, + cleared, + errors, + exitSpy, + loadSpy, + shutdown: shutdown!, + settle, + runtimeTick, + /** Moves the clock forward and runs one 30 s runtime tick to completion. */ + async tick(advanceMs: number): Promise { + clockOffset += advanceMs; + runtimeTick().fn(); + await settle(); + }, + }; +} diff --git a/test/helpers/fake-engine.ts b/test/helpers/fake-engine.ts new file mode 100644 index 0000000000..cf65e46aa7 --- /dev/null +++ b/test/helpers/fake-engine.ts @@ -0,0 +1,129 @@ +import { applyUpdateOps } from "../../src/state/kv.js"; + +type UpdateOp = Parameters[1][number]; + +type Handler = (payload: unknown, ...rest: unknown[]) => unknown; +type ObserverState = "active" | "pending" | "suspended" | "rejected"; + +export class FakeObserverHandle { + state: ObserverState = "active"; + private readonly listeners = new Set<(state: ObserverState) => void>(); + + constructor(readonly id: string) {} + + ready(): Promise { + return Promise.resolve(); + } + + dispose(): Promise { + return Promise.resolve(); + } + + onStateChange(listener: (state: ObserverState) => void): () => void { + this.listeners.add(listener); + return () => this.listeners.delete(listener); + } + + become(next: ObserverState): void { + this.state = next; + for (const listener of this.listeners) listener(next); + } +} + +export interface ObservedSource { + function_id: string; + args?: Record; + filters?: Record; + capture?: string[]; +} + +/** + * In-memory stand-in for an iii engine plus its state worker. It is enough to + * boot the real src/index.ts main(): state::* calls hit a Map, observers come + * back active, and calls to the worker's own functions run their handlers. + */ +export class FakeEngine { + readonly store = new Map>(); + readonly functions = new Map(); + readonly calls: Array<{ function_id: string; payload: unknown }> = []; + readonly observers: Array<{ source: ObservedSource; target: string; options?: Record; handle: FakeObserverHandle }> = []; + failGet: ((scope: string, key: string) => boolean) | null = null; + workers: Array> = [{ worker_id: "w-self", worker_name: "agentmemory" }]; + + readonly client = { + registerFunction: (id: string, handler: Handler) => { + this.functions.set(id, handler); + return { id, unregister: () => this.functions.delete(id) }; + }, + trigger: (request: { function_id: string; payload?: unknown; action?: unknown }) => this.trigger(request), + observe: (source: ObservedSource, target: string, options?: Record) => { + const handle = new FakeObserverHandle(String(options?.id ?? `observer-${this.observers.length}`)); + this.observers.push({ source, target, options, handle }); + return handle; + }, + on: () => {}, + shutdown: async () => {}, + }; + + callsTo(functionId: string): number { + return this.calls.filter((call) => call.function_id === functionId).length; + } + + private scope(name: string): Map { + let scope = this.store.get(name); + if (!scope) { + scope = new Map(); + this.store.set(name, scope); + } + return scope; + } + + async trigger(request: { function_id: string; payload?: unknown; action?: unknown }): Promise { + const { function_id } = request; + this.calls.push({ function_id, payload: request.payload }); + const p = (request.payload ?? {}) as { scope: string; key: string; value?: unknown; ops?: UpdateOp[] }; + switch (function_id) { + case "state::get": { + if (this.failGet?.(p.scope, p.key)) throw Object.assign(new Error("state read failed"), { code: "io_error" }); + const value = this.scope(p.scope).get(p.key); + return value === undefined ? null : structuredClone(value); + } + case "state::set": { + const scope = this.scope(p.scope); + const old = scope.get(p.key) ?? null; + scope.set(p.key, structuredClone(p.value)); + return { old_value: old, new_value: p.value }; + } + case "state::update": { + const scope = this.scope(p.scope); + const old = scope.get(p.key) ?? null; + const next = applyUpdateOps(structuredClone(old), p.ops ?? []); + scope.set(p.key, next); + return { old_value: old, new_value: next, errors: [] }; + } + case "state::delete": { + const scope = this.scope(p.scope); + const old = scope.get(p.key) ?? null; + scope.delete(p.key); + return { old_value: old }; + } + case "state::list": + return [...this.scope(p.scope).values()].map((value) => structuredClone(value)); + case "state::list_keys": + return [...this.scope(p.scope).keys()]; + case "state::list_groups": + return { groups: [...this.store.keys()] }; + case "engine::workers::list": + return { workers: this.workers.map((worker) => ({ ...worker })) }; + } + const handler = this.functions.get(function_id); + if (!handler) throw Object.assign(new Error(`function_not_found: ${function_id}`), { code: "function_not_found" }); + if (request.action) { + void Promise.resolve().then(() => handler(request.payload)).catch(() => {}); + return undefined; + } + return handler(request.payload); + } +} + +export const engine = new FakeEngine(); diff --git a/test/main-checkpoint-restore.test.ts b/test/main-checkpoint-restore.test.ts new file mode 100644 index 0000000000..fb4f0fd816 --- /dev/null +++ b/test/main-checkpoint-restore.test.ts @@ -0,0 +1,53 @@ +import { describe, it, expect, vi, beforeAll } from "vitest"; +import { engine } from "./helpers/fake-engine.js"; +import { bootMain } from "./helpers/boot-main.js"; +import { getVectorIndex, rebuildKeywordIndex } from "../src/functions/search.js"; + +// Boots the real src/index.ts main() with a matching (8-dimension) vector +// checkpoint that cannot be read at boot, then makes it readable again. + +vi.mock("iii-sdk", async (importOriginal) => { + const actual = await importOriginal(); + const { engine: fake } = await import("./helpers/fake-engine.js"); + return { ...actual, registerWorker: () => fake.client }; +}); + +vi.mock("../src/providers/index.js", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + createEmbeddingProvider: () => ({ + name: "fake-8d", + dimensions: 8, + embed: async () => new Float32Array(8), + embedBatch: async (texts: string[]) => texts.map(() => new Float32Array(8)), + }), + }; +}); + +vi.mock("../src/functions/search.js", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, rebuildKeywordIndex: vi.fn(actual.rebuildKeywordIndex) }; +}); + +let main: Awaited>; + +beforeAll(async () => { + main = await bootMain({ checkpointDims: 8, checkpointUnreadableAtBoot: true }); +}, 60_000); + +describe("real main() checkpoint retry", () => { + it("restores a checkpoint that becomes readable again and rebuilds the index once", async () => { + expect(getVectorIndex()?.has("obs_old")).toBe(false); + await main.tick(30_000); + await main.tick(30_000); + engine.failGet = null; + const rebuildsBefore = vi.mocked(rebuildKeywordIndex).mock.calls.length; + for (let i = 0; i < 4; i++) await main.tick(30_000); + expect(getVectorIndex()?.has("obs_old")).toBe(true); + expect(vi.mocked(rebuildKeywordIndex).mock.calls.length - rebuildsBefore).toBe(1); + const loadsAfterRestore = main.loadSpy.mock.calls.length; + for (let i = 0; i < 4; i++) await main.tick(3_600_000); + expect(main.loadSpy.mock.calls.length).toBe(loadsAfterRestore); + }); +}); diff --git a/test/main-runtime-ticks.test.ts b/test/main-runtime-ticks.test.ts new file mode 100644 index 0000000000..ade8ed601d --- /dev/null +++ b/test/main-runtime-ticks.test.ts @@ -0,0 +1,146 @@ +import { describe, it, expect, vi, beforeAll } from "vitest"; +import { engine } from "./helpers/fake-engine.js"; +import { bootMain } from "./helpers/boot-main.js"; +import { rebuildKeywordIndex } from "../src/functions/search.js"; +import { getDuplicateWorkers } from "../src/runtime/duplicates.js"; + +// Boots the real src/index.ts main() with a vector checkpoint written by a +// 4-dimension provider that cannot be read at boot, then drives the 30 s +// runtime tick and the SIGTERM handler that main() installs. + +vi.mock("iii-sdk", async (importOriginal) => { + const actual = await importOriginal(); + const { engine: fake } = await import("./helpers/fake-engine.js"); + return { ...actual, registerWorker: () => fake.client }; +}); + +vi.mock("../src/providers/index.js", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + createEmbeddingProvider: () => ({ + name: "fake-8d", + dimensions: 8, + embed: async () => new Float32Array(8), + embedBatch: async (texts: string[]) => texts.map(() => new Float32Array(8)), + }), + }; +}); + +vi.mock("../src/functions/search.js", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, rebuildKeywordIndex: vi.fn(actual.rebuildKeywordIndex) }; +}); + +let main: Awaited>; +const rebuilds = () => vi.mocked(rebuildKeywordIndex).mock.calls.length; +const mismatchErrors = () => main.errors.filter((line) => line.includes("wrong dimension")).length; + +beforeAll(async () => { + main = await bootMain({ checkpointDims: 4, checkpointUnreadableAtBoot: true }); +}, 60_000); + +describe("real main() runtime tick", () => { + it("a checkpoint that stays unavailable backs off and does not rebuild BM25 on every tick", async () => { + const loadsBefore = main.loadSpy.mock.calls.length; + const rebuildsBefore = rebuilds(); + for (let i = 0; i < 10; i++) await main.tick(30_000); + const loads = main.loadSpy.mock.calls.length - loadsBefore; + const rebuilt = rebuilds() - rebuildsBefore; + console.info(`[proof 1a] 10 ticks over 5 min with the checkpoint unreadable: ${loads} checkpoint reloads, ${rebuilt} full BM25 rebuilds`); + expect(rebuilt).toBe(0); + expect(loads).toBeLessThanOrEqual(4); + }); + + it("a checkpoint that returns with the wrong dimension is rejected once and not retried", async () => { + engine.failGet = null; + const errorsBefore = mismatchErrors(); + const rebuildsBefore = rebuilds(); + for (let i = 0; i < 5; i++) await main.tick(3_600_000); + const logged = mismatchErrors() - errorsBefore; + const rebuilt = rebuilds() - rebuildsBefore; + console.info(`[proof 1b] 5 ticks after the checkpoint came back with 4-dim vectors: ${logged} "wrong dimension" errors, ${rebuilt} BM25 rebuilds`); + expect(main.errors.some((line) => line.includes("wrong dimension") && line.includes("Refusing to start"))).toBe(false); + expect(logged).toBe(1); + expect(rebuilt).toBeLessThanOrEqual(1); + }); + + it("the 30 s tick does not poll engine::workers::list; worker connect/disconnect events drive the duplicate check", async () => { + const before = engine.callsTo("engine::workers::list"); + for (let i = 0; i < 5; i++) await main.tick(30_000); + const polled = engine.callsTo("engine::workers::list") - before; + console.info(`[proof 3] engine::workers::list calls from 5 idle ticks: ${polled}`); + expect(polled).toBe(0); + + const observer = engine.observers.find((entry) => entry.source.function_id === "engine::on-workers-changed"); + expect(observer).toBeDefined(); + const deliver = (event: string) => engine.functions.get(observer!.target)!({ + observer_id: String(observer!.options?.id), activation_id: `act-${event}`, input: { event }, + }); + engine.workers.push({ worker_id: "w-other", worker_name: "agentmemory" }); + await deliver("worker_connected"); + await main.settle(); + expect(getDuplicateWorkers()?.count).toBe(2); + engine.workers.pop(); + await deliver("worker_disconnected"); + await main.settle(); + expect(getDuplicateWorkers()).toBeNull(); + }); + + it("falls back to the duplicate check on the tick while the workers-changed observer is not active", async () => { + const observer = engine.observers.find((entry) => entry.source.function_id === "engine::on-workers-changed")!; + observer.handle.become("rejected"); + engine.workers.push({ worker_id: "w-late", worker_name: "agentmemory" }); + const before = engine.callsTo("engine::workers::list"); + await main.tick(30_000); + const polled = engine.callsTo("engine::workers::list") - before; + console.info(`[proof review-1] observer rejected, second worker joined, 1 tick: ${polled} workers::list calls, duplicate count ${getDuplicateWorkers()?.count ?? 0}`); + expect(polled).toBe(1); + expect(getDuplicateWorkers()?.count).toBe(2); + + // Once the observer is active again, one catch-up check covers events it + // missed, then the tick stops polling. + engine.workers.pop(); + const activeBefore = engine.callsTo("engine::workers::list"); + observer.handle.become("active"); + await main.tick(30_000); + expect(getDuplicateWorkers()).toBeNull(); + await main.tick(30_000); + await main.tick(30_000); + expect(engine.callsTo("engine::workers::list") - activeBefore).toBe(1); + }); + + it("checks once when the workers-changed observer recovers between ticks", async () => { + const observer = engine.observers.find((entry) => entry.source.function_id === "engine::on-workers-changed")!; + const before = engine.callsTo("engine::workers::list"); + observer.handle.become("pending"); + engine.workers.push({ worker_id: "w-during-outage", worker_name: "agentmemory" }); + observer.handle.become("active"); + await main.settle(); + const checked = engine.callsTo("engine::workers::list") - before; + console.info(`[proof review-3] observer pending then active with no tick between: ${checked} checks, duplicate count ${getDuplicateWorkers()?.count ?? 0}`); + expect(checked).toBe(1); + expect(getDuplicateWorkers()?.count).toBe(2); + engine.workers.pop(); + await engine.functions.get(observer.target)!({ observer_id: "o", activation_id: "cleanup", input: { event: "worker_disconnected" } }); + await main.settle(); + expect(getDuplicateWorkers()).toBeNull(); + }); + + it("shutdown clears the runtime tick and the compression sweep, and a late tick does no work", async () => { + const runtime = main.runtimeTick(); + const sweep = main.intervals.find((interval) => String(interval.fn).includes("runCompressionSweep")); + expect(sweep, "AGENTMEMORY_AUTO_COMPRESS=true installs the sweep").toBeDefined(); + void main.shutdown(); + await vi.waitFor(() => expect(main.exitSpy).toHaveBeenCalled(), { timeout: 20_000, interval: 25 }); + console.info(`[proof 4] after shutdown: runtime tick cleared=${main.cleared.has(runtime.handle)}, sweep cleared=${main.cleared.has(sweep!.handle)}`); + const callsBefore = engine.calls.length; + runtime.fn(); + await main.settle(); + const late = engine.calls.slice(callsBefore).map((call) => call.function_id); + console.info(`[proof 4] engine calls made by a tick after shutdown: ${late.length} ${JSON.stringify([...new Set(late)])}`); + expect(main.cleared.has(runtime.handle)).toBe(true); + expect(main.cleared.has(sweep!.handle)).toBe(true); + expect(late).toEqual([]); + }); +}); diff --git a/test/state-worker-return.test.ts b/test/state-worker-return.test.ts index 72d66b2a29..b141760cb7 100644 --- a/test/state-worker-return.test.ts +++ b/test/state-worker-return.test.ts @@ -2,7 +2,8 @@ import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; import { onStateWorkerReturn, stateWorkerStatus } from "../src/runtime/state-worker.js"; import { IndexPersistence } from "../src/state/index-persistence.js"; import { VectorIndex } from "../src/state/vector-index.js"; -import { rearmIndexRecovery, shouldRearmIndex, backfillFailedOutright } from "../src/runtime/recovery.js"; +import { rearmIndexRecovery, shouldRearmIndex, backfillFailedOutright, createRetryBackoff } from "../src/runtime/recovery.js"; +import { Bindings } from "../src/runtime/bindings.js"; vi.mock("../src/logger.js", () => ({ logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() }, @@ -398,3 +399,137 @@ describe("boot checkpoint load with live writes", () => { expect(pending.get("obs_b")).toBe(false); }); }); + +describe("engine::on-workers-changed observer", () => { + function workersSdk(conflicts = 0) { + const observed: Array<{ source: Record; target: string; options?: Record }> = []; + let remainingConflicts = conflicts; + const sdk = { + observed, + trigger: vi.fn(async () => ({ ok: true })), + registerFunction: vi.fn(), + observe: vi.fn((source: Record, target: string, options?: Record) => { + observed.push({ source, target, options }); + const handle = fakeHandle("active"); + if (source.function_id === "engine::on-workers-changed" && remainingConflicts > 0) { + remainingConflicts--; + handle.ready = () => Promise.reject(Object.assign(new Error("id held"), { code: "OBSERVER_ID_CONFLICT" })); + } + return handle; + }), + }; + return sdk; + } + const workersChanged = (sdk: ReturnType) => + sdk.observed.filter((entry) => entry.source.function_id === "engine::on-workers-changed"); + + afterEach(() => { vi.useRealTimers(); }); + + it("filters worker events in the engine instead of waking on every worker change", () => { + const sdk = workersSdk(); + onStateWorkerReturn(sdk as never, new Bindings(sdk as never), vi.fn()); + expect(workersChanged(sdk)).toHaveLength(1); + expect(workersChanged(sdk)[0]!.source.filters).toEqual({ + "input.event": { $in: ["worker_connected", "worker_disconnected"] }, + }); + }); + + it("is owned by Bindings: reported by describe() and retried after OBSERVER_ID_CONFLICT", async () => { + vi.useFakeTimers(); + const sdk = workersSdk(1); + const bindings = new Bindings(sdk as never); + onStateWorkerReturn(sdk as never, bindings, vi.fn()); + expect(bindings.describe().observers).toBe(1); + await vi.advanceTimersByTimeAsync(10_000); + await vi.advanceTimersByTimeAsync(0); + expect(workersChanged(sdk)).toHaveLength(2); + expect(bindings.describe()).toMatchObject({ observers: 1, failedObservers: 0 }); + }); + + it("treats a conflict replacement that is already active as a recovery", async () => { + vi.useFakeTimers(); + const sdk = workersSdk(1); + const onWorkersChanged = vi.fn(); + onStateWorkerReturn(sdk as never, new Bindings(sdk as never), vi.fn(), { onWorkersChanged }); + await vi.advanceTimersByTimeAsync(10_000); + await vi.advanceTimersByTimeAsync(0); + expect(workersChanged(sdk)).toHaveLength(2); + expect(onWorkersChanged.mock.calls).toEqual([["observer_active"]]); + }); + + it("reports nothing after stop()", async () => { + const sdk = workersSdk(); + const onWorkersChanged = vi.fn(); + const rearm = onStateWorkerReturn(sdk as never, new Bindings(sdk as never), vi.fn(), { onWorkersChanged }); + const handle = sdk.observe.mock.results.find((_, i) => workersChanged(sdk)[0] === sdk.observed[i])!.value as ReturnType; + const target = workersChanged(sdk)[0]!.target; + const handler = sdk.registerFunction.mock.calls.find(([id]) => id === target)![1] as (payload: unknown) => Promise; + rearm.stop(); + handle.emit("pending"); + handle.emit("active"); + await handler({ observer_id: "o", activation_id: "late", input: { event: "worker_connected" } }); + expect(onWorkersChanged).not.toHaveBeenCalled(); + expect(sdk.trigger).not.toHaveBeenCalled(); + }); + + it("passes connect and disconnect events to onWorkersChanged", async () => { + const sdk = workersSdk(); + const onWorkersChanged = vi.fn(); + onStateWorkerReturn(sdk as never, new Bindings(sdk as never), vi.fn(), { onWorkersChanged }); + const target = workersChanged(sdk)[0]!.target; + const handler = sdk.registerFunction.mock.calls.find(([id]) => id === target)![1] as (payload: unknown) => Promise; + await handler({ observer_id: "o", activation_id: "a1", input: { event: "worker_disconnected" } }); + await handler({ observer_id: "o", activation_id: "a2", input: { event: "worker_connected" } }); + expect(onWorkersChanged.mock.calls).toEqual([["worker_disconnected"], ["worker_connected"]]); + }); +}); + +describe("createRetryBackoff", () => { + it("waits the initial delay, doubles after each failure up to the cap, and stays off once stopped", () => { + let now = 0; + const backoff = createRetryBackoff({ initialMs: 30_000, maxMs: 100_000, now: () => now }); + expect(backoff.due()).toBe(false); + now = 30_000; + expect(backoff.due()).toBe(true); + backoff.failed(); + now = 89_999; + expect(backoff.due()).toBe(false); + now = 90_000; + expect(backoff.due()).toBe(true); + backoff.failed(); + now = 190_000; + expect(backoff.due()).toBe(true); + backoff.stop(); + expect(backoff.stopped).toBe(true); + now = 10_000_000; + expect(backoff.due()).toBe(false); + }); +}); + +describe("IndexPersistence pauses that a retry cannot fix", () => { + function store() { + return { + get: vi.fn(async () => { throw new Error("state down"); }), + set: vi.fn(async () => undefined), + delete: vi.fn(async () => undefined), + list: vi.fn(async () => []), + listKeys: vi.fn(async () => []), + }; + } + + it("does not report a paused recovery as a failed checkpoint save", async () => { + const persistence = new IndexPersistence(store() as never, new VectorIndex(), { requireRecoveryPlan: true }); + await persistence.prepareBackfill({ vector: null, state: "unavailable", savedAt: null }); + expect(persistence.status().vector?.lastError).toMatch(/unavailable/); + expect(persistence.checkpointSaveFailed()).toBe(false); + }); + + it("an unusable checkpoint pauses recovery without being retried as unavailable", async () => { + const persistence = new IndexPersistence(store() as never, new VectorIndex(), { requireRecoveryPlan: true }); + await persistence.prepareBackfill({ vector: null, state: "unavailable", savedAt: null }); + persistence.pauseForUnusableCheckpoint("wrong dimension"); + expect(persistence.isPausedForUnavailable()).toBe(false); + expect(persistence.isRecoveryPaused()).toBe(true); + expect(persistence.resumeAfterUnavailable()).toBe(false); + }); +});