From 648883ae55d6b1c577211d67f6eb1bcb8eda4a62 Mon Sep 17 00:00:00 2001 From: guibeira Date: Fri, 13 Nov 2026 09:48:09 -0300 Subject: [PATCH 1/4] fix(env): restore the .env.example header broken by the main merge The merge in 678c67a joined the first header lines into "# ====...# agentmemory configuration". Restore the four header lines from main. Signed-off-by: guibeira --- .env.example | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.env.example b/.env.example index d24b3f654d..4b2df2eef2 100644 --- a/.env.example +++ b/.env.example @@ -1,5 +1,7 @@ -# ======================================================================# agentmemory configuration -# ======================================================================# +# ============================================================================= +# agentmemory configuration +# ============================================================================= +# # Copy this file to `~/.agentmemory/.env` (or to your project root if you # prefer scoped config) and uncomment the lines you want to override. # From f608db8a5a75550bc2c18fa7ec01f38ef0376297 Mon Sep 17 00:00:00 2001 From: guibeira Date: Fri, 13 Nov 2026 09:48:09 -0300 Subject: [PATCH 2/4] fix(runtime): stop checkpoint retry loops, own the workers-changed observer and clear runtime timers - An unreadable boot vector checkpoint was retried on every 30 s tick, and every retry rebuilt the whole BM25 index. The retry re-set its own "unavailable" flag, a paused recovery counted as a reason to rebuild, and the pause message in lastError read as a failed checkpoint save. Retries now back off from 30 s to 30 min, and an unusable checkpoint no longer triggers a keyword rebuild. - A checkpoint that comes back with the wrong embedding dimension logged "Refusing to start" on every tick while the worker kept running. It now pauses recovery once (pauseForUnusableCheckpoint) and is not retried. - The engine::on-workers-changed observer is created through Bindings.observeRuntime, so it gets the OBSERVER_ID_CONFLICT retry and appears in runtime.bindings, and it filters input.event to worker_connected and worker_disconnected in the engine. - The duplicate-worker check runs at boot, when the state worker returns and on worker connect/disconnect events instead of polling engine::workers::list every 30 s. A disconnect now clears the warning. - Shutdown clears the runtime tick and the compression sweep and stops the re-arm. - Status and the 0.26 upgrade guide no longer say that delete cascades, relation reactions, failure audits and post-remember graph extraction run as direct calls; those observers have no direct-call path and are reported separately. Tests boot the real src/index.ts main() against an in-memory engine (test/helpers/fake-engine.ts) and drive the tick and SIGTERM handler it installs. Signed-off-by: guibeira --- docs/upgrades/iii-0.26.md | 2 +- src/functions/status.ts | 2 +- src/index.ts | 55 +++++++++--- src/runtime/bindings.ts | 68 +++++++++++--- src/runtime/pipeline-observers.ts | 3 + src/runtime/recovery.ts | 30 +++++++ src/runtime/state-worker.ts | 17 ++-- src/state/index-persistence.ts | 15 ++++ test/bindings.test.ts | 22 +++++ test/helpers/boot-main.ts | 123 +++++++++++++++++++++++++ test/helpers/fake-engine.ts | 129 +++++++++++++++++++++++++++ test/main-checkpoint-restore.test.ts | 53 +++++++++++ test/main-runtime-ticks.test.ts | 129 +++++++++++++++++++++++++++ test/state-worker-return.test.ts | 111 ++++++++++++++++++++++- 14 files changed, 726 insertions(+), 33 deletions(-) create mode 100644 test/helpers/boot-main.ts create mode 100644 test/helpers/fake-engine.ts create mode 100644 test/main-checkpoint-restore.test.ts create mode 100644 test/main-runtime-ticks.test.ts diff --git a/docs/upgrades/iii-0.26.md b/docs/upgrades/iii-0.26.md index 3d35e42113..750b6706fd 100644 --- a/docs/upgrades/iii-0.26.md +++ b/docs/upgrades/iii-0.26.md @@ -22,7 +22,7 @@ Use the same `--instance`, `--port` and `--data-dir` flags you used to start it. - Observers use `capture` only. Background observers receive the `Observed` payload and read the event from its `input` or `output`; HTTP route handlers still receive the `HttpRequest`. - Failure audit observers capture `["input", "error"]` with an `error` filter for thrown errors, and `["input", "output"]` with an `output.success == false` filter for soft failures. - When the state or http worker has not registered within 30 seconds of boot, `runtime.bindings` in `/agentmemory/status` reports that provider as `pending` instead of falling back. Its observers attach as soon as the worker connects. -- A pipeline step runs as a direct call while its own observer is not active (pending, rejected or suspended), so compression and post-session work keep running. `runtime.bindings.reasons` lists the steps that use direct calls. +- A compression or post-session step runs as a direct call while its own observer is not active (pending, rejected or suspended), so that work keeps running. `runtime.bindings.reasons` lists the steps that use direct calls. Delete cascades, relation reactions, failure audits and post-remember graph extraction have no direct-call path: `runtime.bindings.reasons` lists them separately, and their work waits until their observers are active. - The compression sweep retries a failed compression at most 5 times and sends pending observations before failed ones. ## Custom deployments diff --git a/src/functions/status.ts b/src/functions/status.ts index 8bfe8da91a..7adb7f4b20 100644 --- a/src/functions/status.ts +++ b/src/functions/status.ts @@ -299,7 +299,7 @@ export function evaluateStatus(input: StatusInputs): StatusReport { problems.push({ level: "error", code: "observer-rejected", - message: `${failedObservers} observer${failedObservers === 1 ? " was" : "s were"} rejected by the engine. Rejected route or state observers leave those endpoints or reactions inactive; rejected pipeline observers fall back to direct calls.`, + message: `${failedObservers} observer${failedObservers === 1 ? " was" : "s were"} rejected by the engine. Rejected route or state observers leave those endpoints or reactions inactive. Rejected compression and post-session observers fall back to direct calls; rejected delete-cascade, relation-reaction, failure-audit and post-remember graph observers stop that work until restart.`, fix: "Check runtime.bindings.reasons in /agentmemory/status for the conflicting route, remove the conflict, then restart agentmemory.", }); } diff --git a/src/index.ts b/src/index.ts index 9280f3c70c..1cfd3330e1 100644 --- a/src/index.ts +++ b/src/index.ts @@ -135,7 +135,7 @@ import { bootLog, bootWarn } from "./logger.js"; import { runtimeMetadataPath } from "./runtime-paths.js"; import { getBindings } from "./runtime/bindings.js"; import { waitForStateWorker, onStateWorkerReturn } from "./runtime/state-worker.js"; -import { singleFlight, shouldRearmIndex, rearmIndexRecovery, backfillFailedOutright } from "./runtime/recovery.js"; +import { singleFlight, shouldRearmIndex, rearmIndexRecovery, backfillFailedOutright, createRetryBackoff } from "./runtime/recovery.js"; import { checkDuplicateWorkers } from "./runtime/duplicates.js"; import { registerPipelineObservers } from "./runtime/pipeline-observers.js"; import { registerPostSessionFunctions } from "./functions/post-session.js"; @@ -489,10 +489,11 @@ async function main() { registerPipelineObservers(sdk); const bindings = getBindings(sdk); + let sweepTimer: ReturnType | null = null; if (isAutoCompressEnabled()) { const sweepMs = compressionSweepMs(); if (sweepMs > 0) { - const sweepTimer = setInterval(() => { void runCompressionSweep(); }, sweepMs); + sweepTimer = setInterval(() => { void runCompressionSweep(); }, sweepMs); sweepTimer.unref(); } } @@ -559,18 +560,22 @@ async function main() { ); } else { console.error( - `[agentmemory] Refusing to start: persisted vector index has ` + + `[agentmemory] Not loading the persisted vector index: it has ` + `${mismatches.length} of ${loaded.vector.size} vectors with the ` + `wrong dimension. Active provider (${embeddingProvider?.name}) ` + `declares ${activeDim}; dimensions seen on disk: ${distinct}. ` + `First mismatched obsIds: ${sample}. Loading would silently corrupt ` + - `search (cross-dimension cosine returns 0). Choose one:\n` + - ` - Re-embed the existing index against the new provider, then start.\n` + + `search (cross-dimension cosine returns 0). The checkpoint is kept ` + + `and vector recovery stays paused; it is not retried. Choose one, then restart:\n` + + ` - Re-embed the existing index against the new provider.\n` + ` - Set AGENTMEMORY_DROP_STALE_INDEX=true to discard the persisted ` + `vectors and rebuild from live observations.\n` + ` - Switch the embedding provider back to the one that wrote the index.`, ); - await indexPersistence.prepareBackfill(null); + indexPersistence.pauseForUnusableCheckpoint( + `persisted vector checkpoint has ${mismatches.length} of ${loaded.vector.size} vectors with the wrong dimension`, + ); + checkpointBackfillSince = undefined; mismatchBlocked = true; } } else if (boot) { @@ -732,23 +737,36 @@ async function main() { capture.start(); })(); + // An unreadable boot checkpoint is retried on the runtime tick with a + // doubling delay; a checkpoint that loads but cannot be used stops retries. + // Neither case is fixed by rebuilding the keyword index, so they do not + // count as a paused recovery that needs one. + const checkpointRetry = createRetryBackoff({ initialMs: 30_000, maxMs: 30 * 60_000 }); + const checkpointUnusable = () => indexPersistence.isPausedForUnavailable() || checkpointRetry.stopped; const rearm = onStateWorkerReturn(sdk, bindings, async () => { try { await rearmIndexRecovery({ retryCheckpoint: async () => { - if (!bootCheckpointUnavailable || !indexPersistence.resumeAfterUnavailable()) return false; + if (!bootCheckpointUnavailable || checkpointRetry.stopped || !indexPersistence.resumeAfterUnavailable()) return false; const retry = await restoreVectorCheckpoint(false); + if (retry.mismatchBlocked) { + checkpointRetry.stop(); + return false; + } checkpointLoaded = retry.loaded; vectorBackfillSince = retry.vectorBackfillSince; - if (!retry.loaded || retry.loaded.state === "unavailable") return false; + if (!retry.loaded || retry.loaded.state === "unavailable") { + checkpointRetry.failed(); + return false; + } vectorIndex?.finishLiveTracking(); return true; }, indexNeedsRearm: () => shouldRearmIndex({ bm25Incomplete: isBm25RebuildIncomplete(), - persistencePaused: indexPersistence.isRecoveryPaused(), + persistencePaused: indexPersistence.isRecoveryPaused() && !checkpointUnusable(), lastRecoveryFailed: indexRecoveryFailed, - checkpointSaveFailed: Boolean(indexPersistence.status().vector?.lastError), + checkpointSaveFailed: indexPersistence.checkpointSaveFailed(), }), runIndexRecovery, }); @@ -760,10 +778,20 @@ async function main() { } catch (err) { bootWarn(`State-worker re-arm failed: ${err instanceof Error ? err.message : String(err)}`); } - }, { pending: () => bootCheckpointUnavailable && indexPersistence.isPausedForUnavailable() }); + }, { + pending: () => bootCheckpointUnavailable && indexPersistence.isPausedForUnavailable() && checkpointRetry.due(), + onWorkersChanged: () => { void checkDuplicateWorkers(sdk); }, + }); + // Worker connect/disconnect events drive the duplicate-worker check. While + // that observer is not active (pending, rejected, out of conflict retries) + // the tick checks instead, and once more when it comes back to cover the + // events it missed. + let workersObserverWasActive = true; const rearmTick = setInterval(() => { void rearm.probeTick(); - void checkDuplicateWorkers(sdk); + const workersObserverActive = bindings.runtimeActive("runtime::on-workers-changed"); + if (!workersObserverActive || !workersObserverWasActive) void checkDuplicateWorkers(sdk); + workersObserverWasActive = workersObserverActive; }, 30_000); rearmTick.unref(); void checkDuplicateWorkers(sdk); @@ -908,6 +936,9 @@ async function main() { process.exit(await runStartupCleanup(1).catch(() => 1)); }, SHUTDOWN_HARD_EXIT_MS); hardExit.unref(); + clearInterval(rearmTick); + if (sweepTimer) clearInterval(sweepTimer); + rearm.stop(); healthMonitor.stop(); dedupMap.stop(); capture.stop(); diff --git a/src/runtime/bindings.ts b/src/runtime/bindings.ts index 25b9104aa1..6db05874ac 100644 --- a/src/runtime/bindings.ts +++ b/src/runtime/bindings.ts @@ -35,7 +35,15 @@ interface SdkLike { observe?: (source: ObserveSource, target: string, options?: ObserverOptions) => ObserverHandle; } -type BindingKind = "http" | "state" | "pipeline"; +type BindingKind = "http" | "state" | "pipeline" | "runtime"; + +interface BoundObserver { + handle: ObserverHandle; + kind: BindingKind; + target: string; + // Pipeline work that also runs as a direct call while its observer is not active. + fallback: boolean; +} const instances = new WeakMap(); @@ -47,7 +55,7 @@ const ID_CONFLICT_RETRY_DELAYS_MS = [10_000, 20_000, 40_000, 80_000]; export class Bindings { private readonly sdk: SdkLike; - private readonly bound: Array<{ handle: ObserverHandle; kind: BindingKind; target: string }> = []; + private readonly bound: BoundObserver[] = []; private readonly reasons = new Set(); private readonly inline: boolean; private failedObserverCount = 0; @@ -87,6 +95,10 @@ export class Bindings { return this.bound.some((entry) => entry.kind === "pipeline" && entry.target === target && entry.handle.state === "active"); } + runtimeActive(target: string): boolean { + return this.bound.some((entry) => entry.kind === "runtime" && entry.target === target && entry.handle.state === "active"); + } + get modes(): BindingModes { return { state: this.modeOf("state"), http: this.modeOf("http") }; } @@ -107,13 +119,21 @@ export class Bindings { return parsed; } - private observe(kind: BindingKind, label: string, source: ObserveSource, target: string, options?: ObserverOptions, attempt = 0): ObserverHandle | null { + private observe( + kind: BindingKind, + label: string, + source: ObserveSource, + target: string, + options?: ObserverOptions, + extra: { fallback?: boolean; attempt?: number } = {}, + ): ObserverHandle | null { + const attempt = extra.attempt ?? 0; if (typeof this.sdk.observe !== "function") { this.reasons.add("client has no observe(); bindings skipped"); return null; } const handle = this.sdk.observe(source, target, options); - const entry = { handle, kind, target }; + const entry: BoundObserver = { handle, kind, target, fallback: extra.fallback ?? false }; this.bound.push(entry); for (const watch of this.boundWatchers) watch(entry); void handle.ready().catch((err: unknown) => { @@ -125,7 +145,7 @@ export class Bindings { void handle.dispose().catch(() => {}).finally(() => { const index = this.bound.indexOf(entry); if (index >= 0) this.bound.splice(index, 1); - this.observe(kind, label, source, target, options, attempt + 1); + this.observe(kind, label, source, target, options, { ...extra, attempt: attempt + 1 }); }); }, delay); timer.unref?.(); @@ -179,9 +199,23 @@ export class Bindings { ); } - observePipeline(source: ObserveSource, target: string, options?: ObserverOptions): ObserverHandle | null { + /** + * Binds a pipeline step. `fallback: true` (the default) means the caller + * checks `observing(target)` and calls the step directly while the observer + * is not active; pass `fallback: false` for work that only runs through it. + */ + observePipeline( + source: ObserveSource, + target: string, + options?: ObserverOptions, + opts: { fallback?: boolean } = {}, + ): ObserverHandle | null { if (this.inline) return null; - return this.observe("pipeline", `pipeline ${target}`, source, target, options); + return this.observe("pipeline", `pipeline ${target}`, source, target, options, { fallback: opts.fallback ?? true }); + } + + observeRuntime(source: ObserveSource, target: string, options?: ObserverOptions): ObserverHandle | null { + return this.observe("runtime", `runtime ${target}`, source, target, options); } observeStateChanges(filters: Record, target: string, options?: ObserverOptions): ObserverHandle | null { @@ -208,7 +242,7 @@ export class Bindings { const timer = setTimeout(finish, timeoutMs); timer.unref?.(); const watch = ({ handle, kind }: { handle: ObserverHandle; kind: BindingKind }) => { - if (done || kind === "pipeline") return; + if (done || (kind !== "state" && kind !== "http")) return; unsubscribes.push(handle.onStateChange(() => { if (settled()) finish(); })); if (settled()) finish(); }; @@ -223,11 +257,19 @@ export class Bindings { const reasons = new Set(this.reasons); if (modes.state === "pending" && this.handlesOf("state").length > 0) reasons.add("state::on-changed observers pending until the state worker connects"); if (modes.http === "pending" && this.handlesOf("http").length > 0) reasons.add("http::on-request observers pending until the http worker connects"); - if (!this.inline) { - const inactive = this.bound.filter((entry) => entry.kind === "pipeline" && entry.handle.state !== "active"); - if (inactive.length > 0) { - reasons.add(`${inactive.length} pipeline observers are not active, so their work runs as direct calls: ${inactive.map((entry) => entry.target).join(", ")}`); - } + const inactive = this.bound.filter((entry) => entry.handle.state !== "active"); + const targets = (entries: BoundObserver[]) => [...new Set(entries.map((entry) => entry.target))].join(", "); + const directCalls = inactive.filter((entry) => entry.kind === "pipeline" && entry.fallback); + if (!this.inline && directCalls.length > 0) { + reasons.add(`${directCalls.length} pipeline observers are not active, so their work runs as direct calls: ${targets(directCalls)}`); + } + const stalled = inactive.filter((entry) => entry.kind === "pipeline" && !entry.fallback); + if (stalled.length > 0) { + reasons.add(`${stalled.length} observers have no direct-call fallback and are not active, so their work does not run until they activate: ${targets(stalled)}`); + } + const runtime = inactive.filter((entry) => entry.kind === "runtime"); + if (runtime.length > 0) { + reasons.add(`${runtime.length} runtime observers are not active: ${targets(runtime)}`); } for (const kind of ["state", "http"] as const) { if (modes[kind] !== "observe") continue; diff --git a/src/runtime/pipeline-observers.ts b/src/runtime/pipeline-observers.ts index 891e80ecea..76382d5379 100644 --- a/src/runtime/pipeline-observers.ts +++ b/src/runtime/pipeline-observers.ts @@ -49,6 +49,7 @@ export function registerPipelineObservers(sdk: IIIClient): void { { function_id: "mem::remember", capture: ["input", "output"] }, "mem::post-remember-graph", { id: "agentmemory:pipeline:post-remember-graph", max_in_flight: 1 }, + { fallback: false }, ); } for (const fn of ["mem::compress", "mem::summarize", "mem::graph-extract", "mem::consolidate-pipeline", "mem::reflect", "mem::skill-extract"]) { @@ -57,6 +58,7 @@ export function registerPipelineObservers(sdk: IIIClient): void { { function_id: fn, filters: failure.filters, capture: failure.capture }, "mem::record-failure", { id: `agentmemory:pipeline:failure:${fn}:${failure.suffix}`, max_in_flight: 4, metadata: { source_fn: fn } }, + { fallback: false }, ); } } @@ -64,6 +66,7 @@ export function registerPipelineObservers(sdk: IIIClient): void { { function_id: "mem::graph-extract", filters: { "output.partial": { $eq: true } }, capture: ["input", "output"] }, "mem::record-failure", { id: "agentmemory:pipeline:failure:mem::graph-extract:partial", max_in_flight: 4, metadata: { source_fn: "mem::graph-extract" } }, + { fallback: false }, ); bindings.observeStateChanges( { diff --git a/src/runtime/recovery.ts b/src/runtime/recovery.ts index 28314c9394..b57d3c3bfe 100644 --- a/src/runtime/recovery.ts +++ b/src/runtime/recovery.ts @@ -26,6 +26,36 @@ export async function rearmIndexRecovery(steps: RearmSteps): Promise { if (restored || steps.indexNeedsRearm()) await steps.runIndexRecovery(); } +export interface RetryBackoff { + readonly stopped: boolean; + /** True once the current delay has passed and retries were not stopped. */ + due(): boolean; + /** Records a failed attempt and doubles the delay, up to maxMs. */ + failed(): void; + /** Stops retrying for the life of the process. */ + stop(): void; +} + +export function createRetryBackoff(opts: { initialMs: number; maxMs: number; now?: () => number }): RetryBackoff { + const now = opts.now ?? (() => Date.now()); + let delay = opts.initialMs; + let nextAt = now() + delay; + let stopped = false; + return { + get stopped() { + return stopped; + }, + due: () => !stopped && now() >= nextAt, + failed: () => { + delay = Math.min(delay * 2, opts.maxMs); + nextAt = now() + delay; + }, + stop: () => { + stopped = true; + }, + }; +} + export function backfillFailedOutright(result: { added: number; failed: number }): boolean { return result.failed > 0 && result.added === 0; } diff --git a/src/runtime/state-worker.ts b/src/runtime/state-worker.ts index 3532f451a0..a598129a5e 100644 --- a/src/runtime/state-worker.ts +++ b/src/runtime/state-worker.ts @@ -80,7 +80,7 @@ export function onStateWorkerReturn( sdk: IIIClient, bindings: Bindings, cb: () => void | Promise, - opts: { pending?: () => boolean } = {}, + opts: { pending?: () => boolean; onWorkersChanged?: (event: "worker_connected" | "worker_disconnected") => void } = {}, ): { state: StateWorkerReturnState; probeTick: () => Promise; stop: () => void } { const client = sdk as unknown as SdkLike; const state: StateWorkerReturnState = { probeReady: stateWorkerStatus.ready }; @@ -122,9 +122,12 @@ export function onStateWorkerReturn( }; for (const handle of bindings.stateHandles) watchStateHandle(handle, false); bindings.onStateHandle?.((handle) => watchStateHandle(handle, true)); - if (typeof client.observe === "function" && bindings.pipeline === "observe") { + if (typeof client.observe === "function" && typeof bindings.observeRuntime === "function") { client.registerFunction?.("runtime::on-workers-changed", async (payload: unknown) => { - if (unwrapObserved<{ event?: string } | undefined>(payload, "input")?.event !== "worker_connected") return { handled: false }; + const event = unwrapObserved<{ event?: string } | undefined>(payload, "input")?.event; + if (event !== "worker_connected" && event !== "worker_disconnected") return { handled: false }; + opts.onWorkersChanged?.(event); + if (event !== "worker_connected") return { handled: true }; try { await probeOnce(client, 2000); stateWorkerStatus.ready = true; @@ -135,8 +138,12 @@ export function onStateWorkerReturn( } catch {} return { handled: true }; }); - client.observe( - { function_id: "engine::on-workers-changed", capture: ["input"] }, + bindings.observeRuntime( + { + function_id: "engine::on-workers-changed", + filters: { "input.event": { $in: ["worker_connected", "worker_disconnected"] } }, + capture: ["input"], + }, "runtime::on-workers-changed", { id: "agentmemory:runtime:workers-changed" }, ); diff --git a/src/state/index-persistence.ts b/src/state/index-persistence.ts index e6ee1a655c..49a3292d96 100644 --- a/src/state/index-persistence.ts +++ b/src/state/index-persistence.ts @@ -448,6 +448,21 @@ export class IndexPersistence { return this.recoveryPausedForUnavailable; } + /** + * Pauses recovery for a checkpoint that was read but cannot be used, such as + * one written with another embedding dimension. Unlike an unavailable + * checkpoint, reading it again cannot fix it, so it is not retried. + */ + pauseForUnusableCheckpoint(reason: string): void { + this.recoveryPausedForUnavailable = false; + this.pauseRecovery(new Error(reason)); + } + + /** True when the last checkpoint write failed, not when recovery is merely paused. */ + checkpointSaveFailed(): boolean { + return this.leg.lastError !== null && this.leg.lastError !== this.recoveryError; + } + private checkpointRecoveryCutoff(): string | null | undefined { return this.completingBackfill ? null : this.backfillSince; } diff --git a/test/bindings.test.ts b/test/bindings.test.ts index be99a04ca4..fbe724d8ad 100644 --- a/test/bindings.test.ts +++ b/test/bindings.test.ts @@ -2,6 +2,7 @@ import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; import { Bindings, getBindings } from "../src/runtime/bindings.js"; import { registerObserverTarget, unwrapObserved } from "../src/runtime/observed.js"; import { KV } from "../src/state/schema.js"; +import { registerPipelineObservers } from "../src/runtime/pipeline-observers.js"; vi.mock("../src/logger.js", () => ({ logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() }, @@ -346,3 +347,24 @@ describe("observed payloads", () => { expect(await handlers.get("t::in")!({ key: "direct" })).toBe("direct"); }); }); + +describe("observers without a direct-call fallback", () => { + afterEach(() => { vi.unstubAllEnvs(); }); + + it("reports as direct calls only the steps that really fall back", () => { + vi.stubEnv("AGENTMEMORY_AUTO_COMPRESS", "true"); + vi.stubEnv("GRAPH_EXTRACTION_ENABLED", "true"); + vi.stubEnv("AGENTMEMORY_REFLECT", "true"); + const sdk = fakeSdk("pending"); + registerPipelineObservers(sdk as never); + const reasons = getBindings(sdk as never).describe().reasons; + const direct = reasons.find((line) => line.includes("direct calls")) ?? ""; + for (const target of ["mem::compress", "mem::post-session-graph", "mem::post-session-reflect", "mem::post-session-consolidate"]) { + expect(direct).toContain(target); + } + for (const target of ["mem::post-remember-graph", "mem::record-failure", "mem::on-record-deleted", "event::graph::edge-reaction"]) { + expect(direct).not.toContain(target); + expect(reasons.some((line) => line.includes(target) && line.includes("no direct-call fallback"))).toBe(true); + } + }); +}); diff --git a/test/helpers/boot-main.ts b/test/helpers/boot-main.ts new file mode 100644 index 0000000000..d3df8a4e59 --- /dev/null +++ b/test/helpers/boot-main.ts @@ -0,0 +1,123 @@ +import { expect, vi } from "vitest"; +import { mkdtempSync } from "node:fs"; +import { createServer } from "node:net"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { engine } from "./fake-engine.js"; +import { StateKV } from "../../src/state/kv.js"; +import { VectorIndex } from "../../src/state/vector-index.js"; +import { IndexPersistence } from "../../src/state/index-persistence.js"; + +// Boots the real src/index.ts main() against the in-memory engine. The test +// file must mock "iii-sdk" (registerWorker -> engine.client), the embedding +// provider (8 dimensions) and wrap rebuildKeywordIndex in vi.fn. + +export const VECTOR_META = { scope: "mem:index:bm25", key: "vectors:meta" }; + +type Interval = { fn: () => unknown; ms: number; handle: ReturnType }; + +function freePort(): Promise { + return new Promise((resolve, reject) => { + const server = createServer(); + server.once("error", reject); + server.listen(0, "127.0.0.1", () => { + const address = server.address(); + server.close(() => resolve(typeof address === "object" && address ? address.port : 0)); + }); + }); +} + +export async function bootMain(opts: { checkpointDims: number; checkpointUnreadableAtBoot: boolean }) { + const intervals: Interval[] = []; + const cleared = new Set(); + const errors: string[] = []; + let clockOffset = 0; + + const home = mkdtempSync(join(tmpdir(), "am-main-")); + for (const key of Object.keys(process.env)) { + if (/_API_KEY$|^AGENTMEMORY_|^III_/.test(key)) delete process.env[key]; + } + const viewerPort = await freePort(); + Object.assign(process.env, { + HOME: home, + AGENTMEMORY_DATA_DIR: join(home, "data"), + AGENTMEMORY_RUNTIME_DIR: join(home, "run"), + AGENTMEMORY_SECRET: "a".repeat(64), + III_REST_PORT: String(viewerPort - 2), + III_VIEWER_PORT: String(viewerPort), + AGENTMEMORY_AUTO_COMPRESS: "true", + }); + + const realSetInterval = globalThis.setInterval; + const realClearInterval = globalThis.clearInterval; + vi.spyOn(globalThis, "setInterval").mockImplementation(((fn: () => unknown, ms?: number, ...args: unknown[]) => { + const handle = realSetInterval(fn, ms, ...args); + intervals.push({ fn, ms: Number(ms), handle }); + return handle; + }) as typeof setInterval); + vi.spyOn(globalThis, "clearInterval").mockImplementation(((handle?: ReturnType) => { + cleared.add(handle); + realClearInterval(handle); + }) as typeof clearInterval); + const realNow = Date.now.bind(Date); + vi.spyOn(Date, "now").mockImplementation(() => realNow() + clockOffset); + const exitSpy = vi.spyOn(process, "exit").mockImplementation((() => undefined) as never); + const realError = console.error.bind(console); + vi.spyOn(console, "error").mockImplementation((...args: unknown[]) => { + errors.push(args.map(String).join(" ")); + realError(...args); + }); + const loadSpy = vi.spyOn(IndexPersistence.prototype, "load"); + + // A checkpoint written before this start, by a provider with checkpointDims. + const old = new VectorIndex(); + old.add("obs_old", "ses_old", new Float32Array(opts.checkpointDims).fill(0.5)); + await new IndexPersistence(new StateKV(engine.client as never), old).save(); + expect(engine.store.get(VECTOR_META.scope)?.has(VECTOR_META.key)).toBe(true); + loadSpy.mockClear(); + if (opts.checkpointUnreadableAtBoot) { + engine.failGet = (scope, key) => scope === VECTOR_META.scope && key === VECTOR_META.key; + } + + const settle = async (): Promise => { + let last = -1; + let quiet = 0; + for (let i = 0; i < 300 && quiet < 3; i++) { + await new Promise((resolve) => setTimeout(resolve, 10)); + quiet = engine.calls.length === last ? quiet + 1 : 0; + last = engine.calls.length; + } + }; + + const before = new Set(process.listeners("SIGTERM")); + let shutdown: (() => unknown) | undefined; + void import("../../src/index.js"); + await vi.waitFor(() => { + shutdown = process.listeners("SIGTERM").find((listener) => !before.has(listener)) as (() => unknown) | undefined; + expect(shutdown, `main() did not finish booting; exit calls: ${JSON.stringify(exitSpy.mock.calls)}`).toBeDefined(); + }, { timeout: 30_000, interval: 25 }); + await settle(); + + const runtimeTick = (): Interval => { + const entry = intervals.find((interval) => interval.ms === 30_000 && String(interval.fn).includes("probeTick")); + if (!entry) throw new Error("main() did not install the 30 s runtime tick"); + return entry; + }; + + return { + intervals, + cleared, + errors, + exitSpy, + loadSpy, + shutdown: shutdown!, + settle, + runtimeTick, + /** Moves the clock forward and runs one 30 s runtime tick to completion. */ + async tick(advanceMs: number): Promise { + clockOffset += advanceMs; + runtimeTick().fn(); + await settle(); + }, + }; +} diff --git a/test/helpers/fake-engine.ts b/test/helpers/fake-engine.ts new file mode 100644 index 0000000000..cf65e46aa7 --- /dev/null +++ b/test/helpers/fake-engine.ts @@ -0,0 +1,129 @@ +import { applyUpdateOps } from "../../src/state/kv.js"; + +type UpdateOp = Parameters[1][number]; + +type Handler = (payload: unknown, ...rest: unknown[]) => unknown; +type ObserverState = "active" | "pending" | "suspended" | "rejected"; + +export class FakeObserverHandle { + state: ObserverState = "active"; + private readonly listeners = new Set<(state: ObserverState) => void>(); + + constructor(readonly id: string) {} + + ready(): Promise { + return Promise.resolve(); + } + + dispose(): Promise { + return Promise.resolve(); + } + + onStateChange(listener: (state: ObserverState) => void): () => void { + this.listeners.add(listener); + return () => this.listeners.delete(listener); + } + + become(next: ObserverState): void { + this.state = next; + for (const listener of this.listeners) listener(next); + } +} + +export interface ObservedSource { + function_id: string; + args?: Record; + filters?: Record; + capture?: string[]; +} + +/** + * In-memory stand-in for an iii engine plus its state worker. It is enough to + * boot the real src/index.ts main(): state::* calls hit a Map, observers come + * back active, and calls to the worker's own functions run their handlers. + */ +export class FakeEngine { + readonly store = new Map>(); + readonly functions = new Map(); + readonly calls: Array<{ function_id: string; payload: unknown }> = []; + readonly observers: Array<{ source: ObservedSource; target: string; options?: Record; handle: FakeObserverHandle }> = []; + failGet: ((scope: string, key: string) => boolean) | null = null; + workers: Array> = [{ worker_id: "w-self", worker_name: "agentmemory" }]; + + readonly client = { + registerFunction: (id: string, handler: Handler) => { + this.functions.set(id, handler); + return { id, unregister: () => this.functions.delete(id) }; + }, + trigger: (request: { function_id: string; payload?: unknown; action?: unknown }) => this.trigger(request), + observe: (source: ObservedSource, target: string, options?: Record) => { + const handle = new FakeObserverHandle(String(options?.id ?? `observer-${this.observers.length}`)); + this.observers.push({ source, target, options, handle }); + return handle; + }, + on: () => {}, + shutdown: async () => {}, + }; + + callsTo(functionId: string): number { + return this.calls.filter((call) => call.function_id === functionId).length; + } + + private scope(name: string): Map { + let scope = this.store.get(name); + if (!scope) { + scope = new Map(); + this.store.set(name, scope); + } + return scope; + } + + async trigger(request: { function_id: string; payload?: unknown; action?: unknown }): Promise { + const { function_id } = request; + this.calls.push({ function_id, payload: request.payload }); + const p = (request.payload ?? {}) as { scope: string; key: string; value?: unknown; ops?: UpdateOp[] }; + switch (function_id) { + case "state::get": { + if (this.failGet?.(p.scope, p.key)) throw Object.assign(new Error("state read failed"), { code: "io_error" }); + const value = this.scope(p.scope).get(p.key); + return value === undefined ? null : structuredClone(value); + } + case "state::set": { + const scope = this.scope(p.scope); + const old = scope.get(p.key) ?? null; + scope.set(p.key, structuredClone(p.value)); + return { old_value: old, new_value: p.value }; + } + case "state::update": { + const scope = this.scope(p.scope); + const old = scope.get(p.key) ?? null; + const next = applyUpdateOps(structuredClone(old), p.ops ?? []); + scope.set(p.key, next); + return { old_value: old, new_value: next, errors: [] }; + } + case "state::delete": { + const scope = this.scope(p.scope); + const old = scope.get(p.key) ?? null; + scope.delete(p.key); + return { old_value: old }; + } + case "state::list": + return [...this.scope(p.scope).values()].map((value) => structuredClone(value)); + case "state::list_keys": + return [...this.scope(p.scope).keys()]; + case "state::list_groups": + return { groups: [...this.store.keys()] }; + case "engine::workers::list": + return { workers: this.workers.map((worker) => ({ ...worker })) }; + } + const handler = this.functions.get(function_id); + if (!handler) throw Object.assign(new Error(`function_not_found: ${function_id}`), { code: "function_not_found" }); + if (request.action) { + void Promise.resolve().then(() => handler(request.payload)).catch(() => {}); + return undefined; + } + return handler(request.payload); + } +} + +export const engine = new FakeEngine(); diff --git a/test/main-checkpoint-restore.test.ts b/test/main-checkpoint-restore.test.ts new file mode 100644 index 0000000000..fb4f0fd816 --- /dev/null +++ b/test/main-checkpoint-restore.test.ts @@ -0,0 +1,53 @@ +import { describe, it, expect, vi, beforeAll } from "vitest"; +import { engine } from "./helpers/fake-engine.js"; +import { bootMain } from "./helpers/boot-main.js"; +import { getVectorIndex, rebuildKeywordIndex } from "../src/functions/search.js"; + +// Boots the real src/index.ts main() with a matching (8-dimension) vector +// checkpoint that cannot be read at boot, then makes it readable again. + +vi.mock("iii-sdk", async (importOriginal) => { + const actual = await importOriginal(); + const { engine: fake } = await import("./helpers/fake-engine.js"); + return { ...actual, registerWorker: () => fake.client }; +}); + +vi.mock("../src/providers/index.js", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + createEmbeddingProvider: () => ({ + name: "fake-8d", + dimensions: 8, + embed: async () => new Float32Array(8), + embedBatch: async (texts: string[]) => texts.map(() => new Float32Array(8)), + }), + }; +}); + +vi.mock("../src/functions/search.js", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, rebuildKeywordIndex: vi.fn(actual.rebuildKeywordIndex) }; +}); + +let main: Awaited>; + +beforeAll(async () => { + main = await bootMain({ checkpointDims: 8, checkpointUnreadableAtBoot: true }); +}, 60_000); + +describe("real main() checkpoint retry", () => { + it("restores a checkpoint that becomes readable again and rebuilds the index once", async () => { + expect(getVectorIndex()?.has("obs_old")).toBe(false); + await main.tick(30_000); + await main.tick(30_000); + engine.failGet = null; + const rebuildsBefore = vi.mocked(rebuildKeywordIndex).mock.calls.length; + for (let i = 0; i < 4; i++) await main.tick(30_000); + expect(getVectorIndex()?.has("obs_old")).toBe(true); + expect(vi.mocked(rebuildKeywordIndex).mock.calls.length - rebuildsBefore).toBe(1); + const loadsAfterRestore = main.loadSpy.mock.calls.length; + for (let i = 0; i < 4; i++) await main.tick(3_600_000); + expect(main.loadSpy.mock.calls.length).toBe(loadsAfterRestore); + }); +}); diff --git a/test/main-runtime-ticks.test.ts b/test/main-runtime-ticks.test.ts new file mode 100644 index 0000000000..66e76db9ca --- /dev/null +++ b/test/main-runtime-ticks.test.ts @@ -0,0 +1,129 @@ +import { describe, it, expect, vi, beforeAll } from "vitest"; +import { engine } from "./helpers/fake-engine.js"; +import { bootMain } from "./helpers/boot-main.js"; +import { rebuildKeywordIndex } from "../src/functions/search.js"; +import { getDuplicateWorkers } from "../src/runtime/duplicates.js"; + +// Boots the real src/index.ts main() with a vector checkpoint written by a +// 4-dimension provider that cannot be read at boot, then drives the 30 s +// runtime tick and the SIGTERM handler that main() installs. + +vi.mock("iii-sdk", async (importOriginal) => { + const actual = await importOriginal(); + const { engine: fake } = await import("./helpers/fake-engine.js"); + return { ...actual, registerWorker: () => fake.client }; +}); + +vi.mock("../src/providers/index.js", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + createEmbeddingProvider: () => ({ + name: "fake-8d", + dimensions: 8, + embed: async () => new Float32Array(8), + embedBatch: async (texts: string[]) => texts.map(() => new Float32Array(8)), + }), + }; +}); + +vi.mock("../src/functions/search.js", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, rebuildKeywordIndex: vi.fn(actual.rebuildKeywordIndex) }; +}); + +let main: Awaited>; +const rebuilds = () => vi.mocked(rebuildKeywordIndex).mock.calls.length; +const mismatchErrors = () => main.errors.filter((line) => line.includes("wrong dimension")).length; + +beforeAll(async () => { + main = await bootMain({ checkpointDims: 4, checkpointUnreadableAtBoot: true }); +}, 60_000); + +describe("real main() runtime tick", () => { + it("a checkpoint that stays unavailable backs off and does not rebuild BM25 on every tick", async () => { + const loadsBefore = main.loadSpy.mock.calls.length; + const rebuildsBefore = rebuilds(); + for (let i = 0; i < 10; i++) await main.tick(30_000); + const loads = main.loadSpy.mock.calls.length - loadsBefore; + const rebuilt = rebuilds() - rebuildsBefore; + console.info(`[proof 1a] 10 ticks over 5 min with the checkpoint unreadable: ${loads} checkpoint reloads, ${rebuilt} full BM25 rebuilds`); + expect(rebuilt).toBe(0); + expect(loads).toBeLessThanOrEqual(4); + }); + + it("a checkpoint that returns with the wrong dimension is rejected once and not retried", async () => { + engine.failGet = null; + const errorsBefore = mismatchErrors(); + const rebuildsBefore = rebuilds(); + for (let i = 0; i < 5; i++) await main.tick(3_600_000); + const logged = mismatchErrors() - errorsBefore; + const rebuilt = rebuilds() - rebuildsBefore; + console.info(`[proof 1b] 5 ticks after the checkpoint came back with 4-dim vectors: ${logged} "wrong dimension" errors, ${rebuilt} BM25 rebuilds`); + expect(main.errors.some((line) => line.includes("wrong dimension") && line.includes("Refusing to start"))).toBe(false); + expect(logged).toBe(1); + expect(rebuilt).toBeLessThanOrEqual(1); + }); + + it("the 30 s tick does not poll engine::workers::list; worker connect/disconnect events drive the duplicate check", async () => { + const before = engine.callsTo("engine::workers::list"); + for (let i = 0; i < 5; i++) await main.tick(30_000); + const polled = engine.callsTo("engine::workers::list") - before; + console.info(`[proof 3] engine::workers::list calls from 5 idle ticks: ${polled}`); + expect(polled).toBe(0); + + const observer = engine.observers.find((entry) => entry.source.function_id === "engine::on-workers-changed"); + expect(observer).toBeDefined(); + const deliver = (event: string) => engine.functions.get(observer!.target)!({ + observer_id: String(observer!.options?.id), activation_id: `act-${event}`, input: { event }, + }); + engine.workers.push({ worker_id: "w-other", worker_name: "agentmemory" }); + await deliver("worker_connected"); + await main.settle(); + expect(getDuplicateWorkers()?.count).toBe(2); + engine.workers.pop(); + await deliver("worker_disconnected"); + await main.settle(); + expect(getDuplicateWorkers()).toBeNull(); + }); + + it("falls back to the duplicate check on the tick while the workers-changed observer is not active", async () => { + const observer = engine.observers.find((entry) => entry.source.function_id === "engine::on-workers-changed")!; + observer.handle.become("rejected"); + engine.workers.push({ worker_id: "w-late", worker_name: "agentmemory" }); + const before = engine.callsTo("engine::workers::list"); + await main.tick(30_000); + const polled = engine.callsTo("engine::workers::list") - before; + console.info(`[proof review-1] observer rejected, second worker joined, 1 tick: ${polled} workers::list calls, duplicate count ${getDuplicateWorkers()?.count ?? 0}`); + expect(polled).toBe(1); + expect(getDuplicateWorkers()?.count).toBe(2); + + // Once the observer is active again, one catch-up check covers events it + // missed, then the tick stops polling. + engine.workers.pop(); + observer.handle.become("active"); + const activeBefore = engine.callsTo("engine::workers::list"); + await main.tick(30_000); + expect(getDuplicateWorkers()).toBeNull(); + await main.tick(30_000); + await main.tick(30_000); + expect(engine.callsTo("engine::workers::list") - activeBefore).toBe(1); + }); + + it("shutdown clears the runtime tick and the compression sweep, and a late tick does no work", async () => { + const runtime = main.runtimeTick(); + const sweep = main.intervals.find((interval) => String(interval.fn).includes("runCompressionSweep")); + expect(sweep, "AGENTMEMORY_AUTO_COMPRESS=true installs the sweep").toBeDefined(); + void main.shutdown(); + await vi.waitFor(() => expect(main.exitSpy).toHaveBeenCalled(), { timeout: 20_000, interval: 25 }); + console.info(`[proof 4] after shutdown: runtime tick cleared=${main.cleared.has(runtime.handle)}, sweep cleared=${main.cleared.has(sweep!.handle)}`); + const callsBefore = engine.calls.length; + runtime.fn(); + await main.settle(); + const late = engine.calls.slice(callsBefore).map((call) => call.function_id); + console.info(`[proof 4] engine calls made by a tick after shutdown: ${late.length} ${JSON.stringify([...new Set(late)])}`); + expect(main.cleared.has(runtime.handle)).toBe(true); + expect(main.cleared.has(sweep!.handle)).toBe(true); + expect(late).toEqual([]); + }); +}); diff --git a/test/state-worker-return.test.ts b/test/state-worker-return.test.ts index 72d66b2a29..28e6c6e991 100644 --- a/test/state-worker-return.test.ts +++ b/test/state-worker-return.test.ts @@ -2,7 +2,8 @@ import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; import { onStateWorkerReturn, stateWorkerStatus } from "../src/runtime/state-worker.js"; import { IndexPersistence } from "../src/state/index-persistence.js"; import { VectorIndex } from "../src/state/vector-index.js"; -import { rearmIndexRecovery, shouldRearmIndex, backfillFailedOutright } from "../src/runtime/recovery.js"; +import { rearmIndexRecovery, shouldRearmIndex, backfillFailedOutright, createRetryBackoff } from "../src/runtime/recovery.js"; +import { Bindings } from "../src/runtime/bindings.js"; vi.mock("../src/logger.js", () => ({ logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() }, @@ -398,3 +399,111 @@ describe("boot checkpoint load with live writes", () => { expect(pending.get("obs_b")).toBe(false); }); }); + +describe("engine::on-workers-changed observer", () => { + function workersSdk(conflicts = 0) { + const observed: Array<{ source: Record; target: string; options?: Record }> = []; + let remainingConflicts = conflicts; + const sdk = { + observed, + trigger: vi.fn(async () => ({ ok: true })), + registerFunction: vi.fn(), + observe: vi.fn((source: Record, target: string, options?: Record) => { + observed.push({ source, target, options }); + const handle = fakeHandle("active"); + if (source.function_id === "engine::on-workers-changed" && remainingConflicts > 0) { + remainingConflicts--; + handle.ready = () => Promise.reject(Object.assign(new Error("id held"), { code: "OBSERVER_ID_CONFLICT" })); + } + return handle; + }), + }; + return sdk; + } + const workersChanged = (sdk: ReturnType) => + sdk.observed.filter((entry) => entry.source.function_id === "engine::on-workers-changed"); + + afterEach(() => { vi.useRealTimers(); }); + + it("filters worker events in the engine instead of waking on every worker change", () => { + const sdk = workersSdk(); + onStateWorkerReturn(sdk as never, new Bindings(sdk as never), vi.fn()); + expect(workersChanged(sdk)).toHaveLength(1); + expect(workersChanged(sdk)[0]!.source.filters).toEqual({ + "input.event": { $in: ["worker_connected", "worker_disconnected"] }, + }); + }); + + it("is owned by Bindings: reported by describe() and retried after OBSERVER_ID_CONFLICT", async () => { + vi.useFakeTimers(); + const sdk = workersSdk(1); + const bindings = new Bindings(sdk as never); + onStateWorkerReturn(sdk as never, bindings, vi.fn()); + expect(bindings.describe().observers).toBe(1); + await vi.advanceTimersByTimeAsync(10_000); + await vi.advanceTimersByTimeAsync(0); + expect(workersChanged(sdk)).toHaveLength(2); + expect(bindings.describe()).toMatchObject({ observers: 1, failedObservers: 0 }); + }); + + it("passes connect and disconnect events to onWorkersChanged", async () => { + const sdk = workersSdk(); + const onWorkersChanged = vi.fn(); + onStateWorkerReturn(sdk as never, new Bindings(sdk as never), vi.fn(), { onWorkersChanged }); + const target = workersChanged(sdk)[0]!.target; + const handler = sdk.registerFunction.mock.calls.find(([id]) => id === target)![1] as (payload: unknown) => Promise; + await handler({ observer_id: "o", activation_id: "a1", input: { event: "worker_disconnected" } }); + await handler({ observer_id: "o", activation_id: "a2", input: { event: "worker_connected" } }); + expect(onWorkersChanged.mock.calls).toEqual([["worker_disconnected"], ["worker_connected"]]); + }); +}); + +describe("createRetryBackoff", () => { + it("waits the initial delay, doubles after each failure up to the cap, and stays off once stopped", () => { + let now = 0; + const backoff = createRetryBackoff({ initialMs: 30_000, maxMs: 100_000, now: () => now }); + expect(backoff.due()).toBe(false); + now = 30_000; + expect(backoff.due()).toBe(true); + backoff.failed(); + now = 89_999; + expect(backoff.due()).toBe(false); + now = 90_000; + expect(backoff.due()).toBe(true); + backoff.failed(); + now = 190_000; + expect(backoff.due()).toBe(true); + backoff.stop(); + expect(backoff.stopped).toBe(true); + now = 10_000_000; + expect(backoff.due()).toBe(false); + }); +}); + +describe("IndexPersistence pauses that a retry cannot fix", () => { + function store() { + return { + get: vi.fn(async () => { throw new Error("state down"); }), + set: vi.fn(async () => undefined), + delete: vi.fn(async () => undefined), + list: vi.fn(async () => []), + listKeys: vi.fn(async () => []), + }; + } + + it("does not report a paused recovery as a failed checkpoint save", async () => { + const persistence = new IndexPersistence(store() as never, new VectorIndex(), { requireRecoveryPlan: true }); + await persistence.prepareBackfill({ vector: null, state: "unavailable", savedAt: null }); + expect(persistence.status().vector?.lastError).toMatch(/unavailable/); + expect(persistence.checkpointSaveFailed()).toBe(false); + }); + + it("an unusable checkpoint pauses recovery without being retried as unavailable", async () => { + const persistence = new IndexPersistence(store() as never, new VectorIndex(), { requireRecoveryPlan: true }); + await persistence.prepareBackfill({ vector: null, state: "unavailable", savedAt: null }); + persistence.pauseForUnusableCheckpoint("wrong dimension"); + expect(persistence.isPausedForUnavailable()).toBe(false); + expect(persistence.isRecoveryPaused()).toBe(true); + expect(persistence.resumeAfterUnavailable()).toBe(false); + }); +}); From 0ec2a9b70db4f254a0a54ea6b179f8b3b86f3c3b Mon Sep 17 00:00:00 2001 From: guibeira Date: Fri, 13 Nov 2026 10:20:21 -0300 Subject: [PATCH 3/4] fixup! fix(runtime): stop checkpoint retry loops, own the workers-changed observer and clear runtime timers Run the duplicate-worker check as soon as the workers-changed observer becomes active again, so a short outage between two ticks is covered, and keep the tick check only while that observer is not active. A check now publishes its result only when no check started after it has already finished, so a late response cannot erase a newer warning. Signed-off-by: guibeira Co-authored-by: iii harness (claude-opus-5-5) --- src/index.ts | 12 ++++-------- src/runtime/bindings.ts | 11 ++++++++++- src/runtime/duplicates.ts | 9 +++++++++ src/runtime/state-worker.ts | 14 +++++++++++++- test/duplicate-workers.test.ts | 19 ++++++++++++++++++- test/main-runtime-ticks.test.ts | 19 ++++++++++++++++++- 6 files changed, 72 insertions(+), 12 deletions(-) diff --git a/src/index.ts b/src/index.ts index 1cfd3330e1..3e115806e3 100644 --- a/src/index.ts +++ b/src/index.ts @@ -782,16 +782,12 @@ async function main() { pending: () => bootCheckpointUnavailable && indexPersistence.isPausedForUnavailable() && checkpointRetry.due(), onWorkersChanged: () => { void checkDuplicateWorkers(sdk); }, }); - // Worker connect/disconnect events drive the duplicate-worker check. While - // that observer is not active (pending, rejected, out of conflict retries) - // the tick checks instead, and once more when it comes back to cover the - // events it missed. - let workersObserverWasActive = true; + // Worker connect/disconnect events drive the duplicate-worker check, plus one + // check whenever that observer becomes active again. While it is not active + // (pending, rejected, out of conflict retries) the tick checks instead. const rearmTick = setInterval(() => { void rearm.probeTick(); - const workersObserverActive = bindings.runtimeActive("runtime::on-workers-changed"); - if (!workersObserverActive || !workersObserverWasActive) void checkDuplicateWorkers(sdk); - workersObserverWasActive = workersObserverActive; + if (!bindings.runtimeActive("runtime::on-workers-changed")) void checkDuplicateWorkers(sdk); }, 30_000); rearmTick.unref(); void checkDuplicateWorkers(sdk); diff --git a/src/runtime/bindings.ts b/src/runtime/bindings.ts index 6db05874ac..aafbf87587 100644 --- a/src/runtime/bindings.ts +++ b/src/runtime/bindings.ts @@ -59,7 +59,7 @@ export class Bindings { private readonly reasons = new Set(); private readonly inline: boolean; private failedObserverCount = 0; - private readonly boundWatchers = new Set<(entry: { handle: ObserverHandle; kind: BindingKind }) => void>(); + private readonly boundWatchers = new Set<(entry: BoundObserver) => void>(); constructor(sdk: SdkLike) { this.sdk = sdk; @@ -95,6 +95,15 @@ export class Bindings { return this.bound.some((entry) => entry.kind === "pipeline" && entry.target === target && entry.handle.state === "active"); } + /** Calls listener with every runtime handle bound for target, including conflict replacements. */ + onRuntimeHandle(target: string, listener: (handle: ObserverHandle) => void): () => void { + const watch = (entry: BoundObserver) => { + if (entry.kind === "runtime" && entry.target === target) listener(entry.handle); + }; + this.boundWatchers.add(watch); + return () => this.boundWatchers.delete(watch); + } + runtimeActive(target: string): boolean { return this.bound.some((entry) => entry.kind === "runtime" && entry.target === target && entry.handle.state === "active"); } diff --git a/src/runtime/duplicates.ts b/src/runtime/duplicates.ts index 1af727d32e..b734a8e7a5 100644 --- a/src/runtime/duplicates.ts +++ b/src/runtime/duplicates.ts @@ -8,6 +8,10 @@ export interface DuplicateWorkerInfo { let last: DuplicateWorkerInfo | null = null; let lastKey = ""; +// Checks can overlap (worker events, the fallback tick, a state-worker +// return). A check only publishes when no check started after it already has. +let started = 0; +let applied = 0; export function getDuplicateWorkers(): DuplicateWorkerInfo | null { return last; @@ -16,11 +20,16 @@ export function getDuplicateWorkers(): DuplicateWorkerInfo | null { export function resetDuplicateWorkersForTest(): void { last = null; lastKey = ""; + started = 0; + applied = 0; } export async function checkDuplicateWorkers(sdk: IIIClient): Promise { + const seq = ++started; try { const listed = await sdk.trigger({ function_id: "engine::workers::list", payload: {} }); + if (seq < applied) return last; + applied = seq; const entries = Array.isArray(listed) ? listed : (((listed as { workers?: unknown[] })?.workers ?? []) as unknown[]); diff --git a/src/runtime/state-worker.ts b/src/runtime/state-worker.ts index a598129a5e..51ab836d09 100644 --- a/src/runtime/state-worker.ts +++ b/src/runtime/state-worker.ts @@ -80,7 +80,11 @@ export function onStateWorkerReturn( sdk: IIIClient, bindings: Bindings, cb: () => void | Promise, - opts: { pending?: () => boolean; onWorkersChanged?: (event: "worker_connected" | "worker_disconnected") => void } = {}, + opts: { + pending?: () => boolean; + /** A worker connected or disconnected, or the observer reporting those events became active again. */ + onWorkersChanged?: (reason: "worker_connected" | "worker_disconnected" | "observer_active") => void; + } = {}, ): { state: StateWorkerReturnState; probeTick: () => Promise; stop: () => void } { const client = sdk as unknown as SdkLike; const state: StateWorkerReturnState = { probeReady: stateWorkerStatus.ready }; @@ -123,6 +127,14 @@ export function onStateWorkerReturn( for (const handle of bindings.stateHandles) watchStateHandle(handle, false); bindings.onStateHandle?.((handle) => watchStateHandle(handle, true)); if (typeof client.observe === "function" && typeof bindings.observeRuntime === "function") { + // Events are missed while the observer is not active, however short the gap. + bindings.onRuntimeHandle?.("runtime::on-workers-changed", (handle) => { + let wasActive = handle.state === "active"; + handle.onStateChange((next) => { + if (next === "active" && !wasActive) opts.onWorkersChanged?.("observer_active"); + wasActive = next === "active"; + }); + }); client.registerFunction?.("runtime::on-workers-changed", async (payload: unknown) => { const event = unwrapObserved<{ event?: string } | undefined>(payload, "input")?.event; if (event !== "worker_connected" && event !== "worker_disconnected") return { handled: false }; diff --git a/test/duplicate-workers.test.ts b/test/duplicate-workers.test.ts index 282a3badc5..4424b8d3ba 100644 --- a/test/duplicate-workers.test.ts +++ b/test/duplicate-workers.test.ts @@ -1,6 +1,6 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; import { readFileSync } from "node:fs"; -import { checkDuplicateWorkers, resetDuplicateWorkersForTest } from "../src/runtime/duplicates.js"; +import { checkDuplicateWorkers, getDuplicateWorkers, resetDuplicateWorkersForTest } from "../src/runtime/duplicates.js"; vi.mock("../src/logger.js", () => ({ logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() }, @@ -36,4 +36,21 @@ describe("route bindings in api.ts", () => { const api = readFileSync("src/triggers/api.ts", "utf-8"); expect(api).not.toMatch(/sdk\.registerTrigger\(\{\s*type:\s*"http"/); }); + + it("keeps the newest result when an older check finishes last", async () => { + resetDuplicateWorkersForTest(); + let releaseOlder!: (value: unknown) => void; + const trigger = vi.fn() + .mockImplementationOnce(() => new Promise((resolve) => { releaseOlder = resolve; })) + .mockImplementationOnce(async () => ({ + workers: [{ worker_id: "a", worker_name: "agentmemory" }, { worker_id: "b", worker_name: "agentmemory" }], + })); + const sdk = { trigger } as never; + const older = checkDuplicateWorkers(sdk); + await checkDuplicateWorkers(sdk); + expect(getDuplicateWorkers()?.count).toBe(2); + releaseOlder({ workers: [{ worker_id: "a", worker_name: "agentmemory" }] }); + await older; + expect(getDuplicateWorkers()?.count).toBe(2); + }); }); diff --git a/test/main-runtime-ticks.test.ts b/test/main-runtime-ticks.test.ts index 66e76db9ca..ade8ed601d 100644 --- a/test/main-runtime-ticks.test.ts +++ b/test/main-runtime-ticks.test.ts @@ -101,8 +101,8 @@ describe("real main() runtime tick", () => { // Once the observer is active again, one catch-up check covers events it // missed, then the tick stops polling. engine.workers.pop(); - observer.handle.become("active"); const activeBefore = engine.callsTo("engine::workers::list"); + observer.handle.become("active"); await main.tick(30_000); expect(getDuplicateWorkers()).toBeNull(); await main.tick(30_000); @@ -110,6 +110,23 @@ describe("real main() runtime tick", () => { expect(engine.callsTo("engine::workers::list") - activeBefore).toBe(1); }); + it("checks once when the workers-changed observer recovers between ticks", async () => { + const observer = engine.observers.find((entry) => entry.source.function_id === "engine::on-workers-changed")!; + const before = engine.callsTo("engine::workers::list"); + observer.handle.become("pending"); + engine.workers.push({ worker_id: "w-during-outage", worker_name: "agentmemory" }); + observer.handle.become("active"); + await main.settle(); + const checked = engine.callsTo("engine::workers::list") - before; + console.info(`[proof review-3] observer pending then active with no tick between: ${checked} checks, duplicate count ${getDuplicateWorkers()?.count ?? 0}`); + expect(checked).toBe(1); + expect(getDuplicateWorkers()?.count).toBe(2); + engine.workers.pop(); + await engine.functions.get(observer.target)!({ observer_id: "o", activation_id: "cleanup", input: { event: "worker_disconnected" } }); + await main.settle(); + expect(getDuplicateWorkers()).toBeNull(); + }); + it("shutdown clears the runtime tick and the compression sweep, and a late tick does no work", async () => { const runtime = main.runtimeTick(); const sweep = main.intervals.find((interval) => String(interval.fn).includes("runCompressionSweep")); From df301b5b1d46250ddbc7ef4d459441e2db65b32d Mon Sep 17 00:00:00 2001 From: guibeira Date: Fri, 13 Nov 2026 10:42:26 -0300 Subject: [PATCH 4/4] fixup! fix(runtime): stop checkpoint retry loops, own the workers-changed observer and clear runtime timers Treat a workers-changed conflict replacement that is already active as a recovery, so the events missed during the conflict get one duplicate check, and report no worker changes after stop(). Drop a comment that only restated onRuntimeHandle. Signed-off-by: guibeira Co-authored-by: iii harness (claude-code/claude-opus-5-5) --- src/runtime/bindings.ts | 1 - src/runtime/state-worker.ts | 14 ++++++++++++-- test/state-worker-return.test.ts | 26 ++++++++++++++++++++++++++ 3 files changed, 38 insertions(+), 3 deletions(-) diff --git a/src/runtime/bindings.ts b/src/runtime/bindings.ts index aafbf87587..e0e6e9c96f 100644 --- a/src/runtime/bindings.ts +++ b/src/runtime/bindings.ts @@ -95,7 +95,6 @@ export class Bindings { return this.bound.some((entry) => entry.kind === "pipeline" && entry.target === target && entry.handle.state === "active"); } - /** Calls listener with every runtime handle bound for target, including conflict replacements. */ onRuntimeHandle(target: string, listener: (handle: ObserverHandle) => void): () => void { const watch = (entry: BoundObserver) => { if (entry.kind === "runtime" && entry.target === target) listener(entry.handle); diff --git a/src/runtime/state-worker.ts b/src/runtime/state-worker.ts index 51ab836d09..f094fc386f 100644 --- a/src/runtime/state-worker.ts +++ b/src/runtime/state-worker.ts @@ -128,17 +128,27 @@ export function onStateWorkerReturn( bindings.onStateHandle?.((handle) => watchStateHandle(handle, true)); if (typeof client.observe === "function" && typeof bindings.observeRuntime === "function") { // Events are missed while the observer is not active, however short the gap. + // The first handle is covered by the boot check; a conflict replacement + // replaces one that never delivered, so it counts as a recovery even when + // it is already active. + const reportWorkersChanged = (reason: "worker_connected" | "worker_disconnected" | "observer_active") => { + if (!stopped) opts.onWorkersChanged?.(reason); + }; + let firstRuntimeHandle = true; bindings.onRuntimeHandle?.("runtime::on-workers-changed", (handle) => { let wasActive = handle.state === "active"; + if (wasActive && !firstRuntimeHandle) reportWorkersChanged("observer_active"); + firstRuntimeHandle = false; handle.onStateChange((next) => { - if (next === "active" && !wasActive) opts.onWorkersChanged?.("observer_active"); + if (next === "active" && !wasActive) reportWorkersChanged("observer_active"); wasActive = next === "active"; }); }); client.registerFunction?.("runtime::on-workers-changed", async (payload: unknown) => { const event = unwrapObserved<{ event?: string } | undefined>(payload, "input")?.event; if (event !== "worker_connected" && event !== "worker_disconnected") return { handled: false }; - opts.onWorkersChanged?.(event); + if (stopped) return { handled: false }; + reportWorkersChanged(event); if (event !== "worker_connected") return { handled: true }; try { await probeOnce(client, 2000); diff --git a/test/state-worker-return.test.ts b/test/state-worker-return.test.ts index 28e6c6e991..b141760cb7 100644 --- a/test/state-worker-return.test.ts +++ b/test/state-worker-return.test.ts @@ -446,6 +446,32 @@ describe("engine::on-workers-changed observer", () => { expect(bindings.describe()).toMatchObject({ observers: 1, failedObservers: 0 }); }); + it("treats a conflict replacement that is already active as a recovery", async () => { + vi.useFakeTimers(); + const sdk = workersSdk(1); + const onWorkersChanged = vi.fn(); + onStateWorkerReturn(sdk as never, new Bindings(sdk as never), vi.fn(), { onWorkersChanged }); + await vi.advanceTimersByTimeAsync(10_000); + await vi.advanceTimersByTimeAsync(0); + expect(workersChanged(sdk)).toHaveLength(2); + expect(onWorkersChanged.mock.calls).toEqual([["observer_active"]]); + }); + + it("reports nothing after stop()", async () => { + const sdk = workersSdk(); + const onWorkersChanged = vi.fn(); + const rearm = onStateWorkerReturn(sdk as never, new Bindings(sdk as never), vi.fn(), { onWorkersChanged }); + const handle = sdk.observe.mock.results.find((_, i) => workersChanged(sdk)[0] === sdk.observed[i])!.value as ReturnType; + const target = workersChanged(sdk)[0]!.target; + const handler = sdk.registerFunction.mock.calls.find(([id]) => id === target)![1] as (payload: unknown) => Promise; + rearm.stop(); + handle.emit("pending"); + handle.emit("active"); + await handler({ observer_id: "o", activation_id: "late", input: { event: "worker_connected" } }); + expect(onWorkersChanged).not.toHaveBeenCalled(); + expect(sdk.trigger).not.toHaveBeenCalled(); + }); + it("passes connect and disconnect events to onWorkersChanged", async () => { const sdk = workersSdk(); const onWorkersChanged = vi.fn();