From 9506dd4df5708904cb5c28fd7c74943fa32f4de6 Mon Sep 17 00:00:00 2001 From: Codex Date: Sun, 15 Nov 2026 03:42:39 +0900 Subject: [PATCH] perf: reuse search hydration within verified request lifetime --- .cursor-plugin/plugin.json | 2 +- AGENTS.md | 4 +- CHANGELOG.md | 9 +++ assets/tags/light/stat-tests.svg | 4 +- assets/tags/stat-tests.svg | 4 +- .../SEARCH-REQUEST-HYDRATION.md | 36 +++++++++ integrations/hermes/plugin.yaml | 2 +- integrations/omp/package.json | 2 +- integrations/openclaw/openclaw.plugin.json | 2 +- integrations/openclaw/package.json | 2 +- integrations/openclaw/plugin.yaml | 2 +- package.json | 2 +- packages/mcp/package.json | 4 +- plugin/.claude-plugin/plugin.json | 2 +- plugin/.codex-plugin/plugin.json | 2 +- plugin/.devin-plugin/plugin.json | 4 +- plugin/opencode/plugin.json | 2 +- plugin/plugin.json | 2 +- plugin/scripts/plugin-bridge.mjs | 2 +- scripts/docs-sync.state.json | 4 +- src/functions/export-import.ts | 2 +- src/functions/search.ts | 6 +- src/ops/state-epoch.d.mts | 2 + src/ops/state-epoch.mjs | 8 +- src/state/hybrid-search.ts | 81 +++++++++++++++---- src/state/kv.ts | 3 + src/types.ts | 2 +- src/version.ts | 2 +- test/codex-hook-compat-artifacts.test.ts | 45 ++++++++++- test/search-hydration-cache.test.ts | 78 ++++++++++++++++++ test/state-epoch.test.ts | 7 ++ 31 files changed, 282 insertions(+), 47 deletions(-) create mode 100644 docs/customizations/SEARCH-REQUEST-HYDRATION.md create mode 100644 test/search-hydration-cache.test.ts diff --git a/.cursor-plugin/plugin.json b/.cursor-plugin/plugin.json index de35a75d6d..26b9a18503 100644 --- a/.cursor-plugin/plugin.json +++ b/.cursor-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "agentmemory", - "version": "0.9.33-daniel.75", + "version": "0.9.33-daniel.76", "description": "Persistent memory for Cursor: auto-captures sessions, prompts, and tool use via hooks, recalls with hybrid BM25 + vector + graph search, and exposes 55 memory tools over MCP. Keyless by default, runs entirely on your machine.", "author": { "name": "Rohit Ghumare", diff --git a/AGENTS.md b/AGENTS.md index 1e9e3af79c..9f0028a1a1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -108,7 +108,7 @@ Hook scripts in `src/hooks/` are standalone Node.js scripts (no iii-sdk import). ## Testing -- Before every commit run, in order: `npm run build`, `npm run skills:gen`, `npm run docs:sync`, `npm run skills:check`, `npm test` (7,000+ tests). Commit anything the generators changed. CI runs the same build, skills check and tests. +- Before every commit run, in order: `npm run build`, `npm run skills:gen`, `npm run docs:sync`, `npm run skills:check`, `npm test` (7,100+ tests). Commit anything the generators changed. CI runs the same build, skills check and tests. - Before any npm publish: `npm run release:gate` must pass. It installs the packed tarballs into a clean home and runs capture, recovery, restart, export/import, MCP and status checks against the installed CLI (see CONTRIBUTING.md, Release process) - Mock pattern: `vi.mock("iii-sdk")` with mock `sdk.trigger`, `kv.get/set/list` - Test files go in `test/` with `.test.ts` extension @@ -121,7 +121,7 @@ Hook scripts in `src/hooks/` are standalone Node.js scripts (no iii-sdk import). - 6 MCP resources, 3 MCP prompts - 12 hooks, 17 skills - 355+ iii functions -- 7,000+ tests +- 7,100+ tests ## Personal fork verification isolation diff --git a/CHANGELOG.md b/CHANGELOG.md index 339194fd65..e97008d9a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## 0.9.33-daniel.76 — Request-local search hydration + +- Reuse unchanged hydration within one expanding search request, preserving pinned State identity checks and invalidating on State writes or index publication. +- Bound observers and release them on every request exit; no cross-request cache or backend safety bypass. Synthetic sparse-project RPC reads halve with identical results; production latency is not yet measured. +- Make compatibility-hook test fixtures wait for their actual detached writers before cleanup and propagate child failures; production hook behavior is unchanged. +- Details: `docs/customizations/SEARCH-REQUEST-HYDRATION.md`. + ## 0.9.33-daniel.75 — Additive memory-pressure recovery - Preserve the existing available-memory normal/recovery paths; Linux PSI adds a separately qualified recovery path rather than vetoing previously allowed work. @@ -30,6 +37,7 @@ ## [Unreleased] +## [0.9.33-daniel.76] — 2026-11-14 ## [0.9.33-daniel.75] — 2026-11-14 ## [0.9.33-daniel.74] — 2026-11-14 ## [0.9.33-daniel.73] — 2026-11-14 @@ -700,6 +708,7 @@ Wave release closing several breaking regressions reported against v0.9.26, plus - `/agentmemory:forget` skill still calls `memory_governance_delete` which only touches `KV.memories` and never observations ([#833](https://github.com/rohitg00/agentmemory/issues/833)). Skill rewrite + new `memory_forget` MCP tool tracked separately. - `crypto.randomUUID()` global-only on Node <19 ([#715](https://github.com/rohitg00/agentmemory/issues/715)). Drop-in import fix tracked. +[0.9.33-daniel.76]: https://github.com/rohitg00/agentmemory/compare/v0.9.33-daniel.75...v0.9.33-daniel.76 [0.9.33-daniel.75]: https://github.com/rohitg00/agentmemory/compare/v0.9.33-daniel.74...v0.9.33-daniel.75 [0.9.33-daniel.74]: https://github.com/rohitg00/agentmemory/compare/v0.9.33-daniel.73...v0.9.33-daniel.74 [0.9.33-daniel.73]: https://github.com/rohitg00/agentmemory/compare/v0.9.33-daniel.72...v0.9.33-daniel.73 diff --git a/assets/tags/light/stat-tests.svg b/assets/tags/light/stat-tests.svg index 238a6f2683..57961bb124 100644 --- a/assets/tags/light/stat-tests.svg +++ b/assets/tags/light/stat-tests.svg @@ -1,5 +1,5 @@ - + - 7000+ + 7100+ TESTS PASSING diff --git a/assets/tags/stat-tests.svg b/assets/tags/stat-tests.svg index 0c6b58a33f..6efa018410 100644 --- a/assets/tags/stat-tests.svg +++ b/assets/tags/stat-tests.svg @@ -1,5 +1,5 @@ - + - 7000+ + 7100+ TESTS PASSING diff --git a/docs/customizations/SEARCH-REQUEST-HYDRATION.md b/docs/customizations/SEARCH-REQUEST-HYDRATION.md new file mode 100644 index 0000000000..524579b780 --- /dev/null +++ b/docs/customizations/SEARCH-REQUEST-HYDRATION.md @@ -0,0 +1,36 @@ +# Request-local search hydration reuse + +Sparse project filtering may expand hybrid search through 100, 200, 400, 800, +and 1,000 candidates. Earlier rounds previously fetched the same State records +again. A request-local positive hydration cache avoids those repeated RPCs. + +## Safety and lifetime + +- Reuse requires both State mutation-key observers and the existing pinned State + read-admission predicate. Other backends/clients retain the uncached path. +- Each reused result checks the exact current State/engine identity predicate + before and after its asynchronous continuation. No identity TTL, RPC fence, + writer-lease condition, or mutation protection is removed. +- Index publication and observation/memory key mutation invalidate entries and + associated request-local project/session filtering caches. An invalidation + during hydration cannot publish that obsolete result. +- Failed/missing reads are not retained, including memory fallback after a + failed observation read. There is no cross-request cache or transactional + snapshot promise; concurrent mutations retain read-time search semantics. +- At most 1,000 hydrated entries each observe two State keys. All request + observers are released in `finally`, including authorization errors. + +## Evidence and limits + +A deterministic sparse-project fixture keeps identical result order and +100→200→400→800→1,000 expansion. Total State reads drop from 6,000 to 3,000: +observation hydration 2,500→1,000, memory hydration/filter reads 3,500→2,000. +Identity checks still run on cache hits and may dominate native filesystem cost. +This is an RPC-count improvement, not a measured production latency guarantee. +No production search was repeated to benchmark this change. + +Regression cases include deletion, mutation before index publication, publication +revision changes, mutation during hydration, writer loss before/after an async +cache continuation, failed reads, concurrent requests, absent admission support, +and observer cleanup. Remove this customization when upstream supplies equivalent +request reuse with mutation freshness, pinned admission, and bounded cleanup. diff --git a/integrations/hermes/plugin.yaml b/integrations/hermes/plugin.yaml index e42437977e..4ce25e00aa 100644 --- a/integrations/hermes/plugin.yaml +++ b/integrations/hermes/plugin.yaml @@ -1,5 +1,5 @@ name: agentmemory -version: 0.9.33-daniel.75 +version: 0.9.33-daniel.76 description: "Persistent memory for Hermes Agent with profile-scoped capture, recall, and native memory tools." author: "Rohit Ghumare" homepage: "https://github.com/rohitg00/agentmemory" diff --git a/integrations/omp/package.json b/integrations/omp/package.json index 5ca1f6e163..8961c3ea0d 100644 --- a/integrations/omp/package.json +++ b/integrations/omp/package.json @@ -2,7 +2,7 @@ "name": "agentmemory-omp-extension", "private": true, "type": "module", - "version": "0.9.33-daniel.75", + "version": "0.9.33-daniel.76", "omp": { "name": "agentmemory", "description": "Persistent cross-session memory for omp \u2014 captures sessions, tools, approvals, and compaction; injects smart-search recall.", diff --git a/integrations/openclaw/openclaw.plugin.json b/integrations/openclaw/openclaw.plugin.json index 8278602ad0..c88293775f 100644 --- a/integrations/openclaw/openclaw.plugin.json +++ b/integrations/openclaw/openclaw.plugin.json @@ -3,7 +3,7 @@ "kind": "memory", "name": "agentmemory", "description": "Persistent cross-session memory for OpenClaw via agentmemory.", - "version": "0.9.33-daniel.75", + "version": "0.9.33-daniel.76", "configSchema": { "type": "object", "additionalProperties": false, diff --git a/integrations/openclaw/package.json b/integrations/openclaw/package.json index 0adc98a207..06ff1804d1 100644 --- a/integrations/openclaw/package.json +++ b/integrations/openclaw/package.json @@ -1,6 +1,6 @@ { "name": "agentmemory", - "version": "0.9.33-daniel.75", + "version": "0.9.33-daniel.76", "type": "module", "openclaw": { "extensions": [ diff --git a/integrations/openclaw/plugin.yaml b/integrations/openclaw/plugin.yaml index 594e290941..072f2e8529 100644 --- a/integrations/openclaw/plugin.yaml +++ b/integrations/openclaw/plugin.yaml @@ -1,5 +1,5 @@ name: agentmemory -version: 0.9.33-daniel.75 +version: 0.9.33-daniel.76 description: "Persistent cross-session memory for OpenClaw via agentmemory." author: "Rohit Ghumare" homepage: "https://github.com/rohitg00/agentmemory" diff --git a/package.json b/package.json index 91645790f5..96f885d7d6 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@agentmemory/agentmemory", - "version": "0.9.33-daniel.75", + "version": "0.9.33-daniel.76", "description": "Persistent memory for AI coding agents, powered by iii-engine's three primitives", "type": "module", "main": "dist/index.mjs", diff --git a/packages/mcp/package.json b/packages/mcp/package.json index 6f84c53b96..3e75f12b09 100644 --- a/packages/mcp/package.json +++ b/packages/mcp/package.json @@ -1,6 +1,6 @@ { "name": "@agentmemory/mcp", - "version": "0.9.33-daniel.75", + "version": "0.9.33-daniel.76", "description": "Standalone MCP server for agentmemory — thin shim that re-exposes @agentmemory/agentmemory's MCP entrypoint", "type": "module", "bin": { @@ -28,7 +28,7 @@ "homepage": "https://github.com/rohitg00/agentmemory#readme", "bugs": "https://github.com/rohitg00/agentmemory/issues", "dependencies": { - "@agentmemory/agentmemory": "0.9.33-daniel.75" + "@agentmemory/agentmemory": "0.9.33-daniel.76" }, "publishConfig": { "access": "public", diff --git a/plugin/.claude-plugin/plugin.json b/plugin/.claude-plugin/plugin.json index 17e5986d86..3b2559673d 100644 --- a/plugin/.claude-plugin/plugin.json +++ b/plugin/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "agentmemory", - "version": "0.9.33-daniel.75", + "version": "0.9.33-daniel.76", "description": "Persistent memory for AI coding agents -- captures tool usage, compresses via LLM, injects context into future sessions. 12 hooks, 55 MCP tools, 17 skills, real-time viewer.", "author": { "name": "Rohit Ghumare", diff --git a/plugin/.codex-plugin/plugin.json b/plugin/.codex-plugin/plugin.json index 14f803d3a0..a216716589 100644 --- a/plugin/.codex-plugin/plugin.json +++ b/plugin/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "agentmemory", - "version": "0.9.33-daniel.75", + "version": "0.9.33-daniel.76", "description": "Local persistent memory for coding agents. Recall decisions, resume sessions, learn from corrections, and manage memory through 55 MCP tools and 17 skills. Optional capture through 6 Codex hooks.", "author": { "name": "Rohit Ghumare", diff --git a/plugin/.devin-plugin/plugin.json b/plugin/.devin-plugin/plugin.json index 85d6873d2b..eca59f7edb 100644 --- a/plugin/.devin-plugin/plugin.json +++ b/plugin/.devin-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "agentmemory", "displayName": "agentmemory", - "version": "0.9.33-daniel.75", + "version": "0.9.33-daniel.76", "description": "Persistent memory for Devin CLI and Devin Desktop. Captures every session automatically, recalls with hybrid BM25 + vector + graph search, and exposes 55 memory tools over MCP. Keyless by default, runs on your machine. Not supported in cloud sessions.", "logo": "assets/icon.svg", "author": { @@ -23,7 +23,7 @@ "command": "npx", "args": [ "-y", - "@agentmemory/mcp@0.9.33-daniel.75" + "@agentmemory/mcp@0.9.33-daniel.76" ], "env": { "AGENTMEMORY_URL": "${AGENTMEMORY_URL:-http://localhost:3111}" diff --git a/plugin/opencode/plugin.json b/plugin/opencode/plugin.json index 779fb88ccf..0dab007706 100644 --- a/plugin/opencode/plugin.json +++ b/plugin/opencode/plugin.json @@ -1,6 +1,6 @@ { "name": "agentmemory-capture", - "version": "0.9.33-daniel.75", + "version": "0.9.33-daniel.76", "description": "OpenCode plugin for agentmemory — full Claude Code hook parity: session lifecycle (create/idle/status/compacted/update/diff/delete/error), messages & prompts (chat.message, message.updated user+assistant, message.removed), tool lifecycle (ToolPart states with timing), part tracking (subtask, step-finish, reasoning, file, patch, compaction, agent, retry), file enrichment pipeline, permissions, task tracking (w/ priority), commands, config & model tracking. 22 hooks, 2 slash commands.", "author": { "name": "Rohit Ghumare", diff --git a/plugin/plugin.json b/plugin/plugin.json index f0e0b0b316..72662137a5 100644 --- a/plugin/plugin.json +++ b/plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "agentmemory", - "version": "0.9.33-daniel.75", + "version": "0.9.33-daniel.76", "description": "Persistent memory for AI coding agents -- captures tool usage, compresses via LLM, injects context into future sessions. 12 hooks, 55 MCP tools, 17 skills, real-time viewer.", "author": { "name": "Rohit Ghumare", diff --git a/plugin/scripts/plugin-bridge.mjs b/plugin/scripts/plugin-bridge.mjs index c32a9f6cbb..51538a1eb4 100644 --- a/plugin/scripts/plugin-bridge.mjs +++ b/plugin/scripts/plugin-bridge.mjs @@ -3,7 +3,7 @@ import { readFileSync, realpathSync } from "node:fs"; import { homedir } from "node:os"; import { join } from "node:path"; //#region src/version.ts -const VERSION = "0.9.33-daniel.75"; +const VERSION = "0.9.33-daniel.76"; //#endregion //#region src/secret-store.ts const SECRET_KEY = "AGENTMEMORY_SECRET"; diff --git a/scripts/docs-sync.state.json b/scripts/docs-sync.state.json index 6f96f78b57..47ce788816 100644 --- a/scripts/docs-sync.state.json +++ b/scripts/docs-sync.state.json @@ -1,5 +1,5 @@ { - "version": "0.9.33-daniel.75", + "version": "0.9.33-daniel.76", "facts": { "mcpTools": [ "55" @@ -20,7 +20,7 @@ "12" ], "tests": [ - "7000+" + "7100+" ], "functions": [ "355" diff --git a/src/functions/export-import.ts b/src/functions/export-import.ts index d165d6372a..c198f39d16 100644 --- a/src/functions/export-import.ts +++ b/src/functions/export-import.ts @@ -315,7 +315,7 @@ export function registerExportImportFunction(sdk: IIIClient, state: StateKV): vo const strategy = data.strategy || "merge"; const importData = data.exportData; - const supportedVersions = new Set(["0.3.0", "0.4.0", "0.5.0", "0.6.0", "0.6.1", "0.7.0", "0.7.2", "0.7.3", "0.7.4", "0.7.5", "0.7.6", "0.7.7", "0.7.9", "0.8.0", "0.8.1", "0.8.2", "0.8.3", "0.8.4", "0.8.5", "0.8.6", "0.8.7", "0.8.8", "0.8.9", "0.8.10", "0.8.11", "0.8.12", "0.8.13", "0.9.0", "0.9.1", "0.9.2", "0.9.3", "0.9.4", "0.9.5", "0.9.6", "0.9.7", "0.9.8", "0.9.9", "0.9.10", "0.9.11", "0.9.12", "0.9.13", "0.9.14", "0.9.15", "0.9.16", "0.9.17", "0.9.18", "0.9.19", "0.9.20", "0.9.21", "0.9.22", "0.9.23", "0.9.24", "0.9.25", "0.9.26", "0.9.27", "0.9.28", "0.9.29", "0.9.30", "0.9.31", "0.9.31-daniel.1", "0.9.31-daniel.33", "0.9.31-daniel.34", "0.9.31-daniel.35", "0.9.31-daniel.36", "0.9.31-daniel.37", "0.9.31-daniel.38", "0.9.31-daniel.39", "0.9.31-daniel.40", "0.9.31-daniel.41", "0.9.31-daniel.42", "0.9.31-daniel.43", "0.9.31-daniel.44", "0.9.31-daniel.45", "0.9.31-daniel.46", "0.9.31-daniel.47", "0.9.31-daniel.48", "0.9.31-daniel.49", "0.9.31-daniel.50", "0.9.31-daniel.51", "0.9.31-daniel.52", "0.9.31-daniel.53", "0.9.31-daniel.54", "0.9.32-daniel.55", "0.9.32-daniel.57", "0.9.32-daniel.58", "0.9.32-daniel.59", "0.9.32-daniel.60", "0.9.32-daniel.61", "0.9.32-daniel.62", "0.9.32-daniel.63", "0.9.32-daniel.64", "0.9.32-daniel.65", "0.9.32-daniel.66", "0.9.32-daniel.67", "0.9.32-daniel.68", "0.9.33-daniel.69", "0.9.33-daniel.70", "0.9.33-daniel.71", "0.9.33-daniel.72", "0.9.33-daniel.73", "0.9.33-daniel.74", "0.9.33-daniel.75", "0.9.30-daniel.1", "0.9.30-daniel.10", "0.9.30-daniel.11", "0.9.30-daniel.15", "0.9.30-daniel.16", "0.9.30-daniel.18", "0.9.30-daniel.19", "0.9.30-daniel.2", "0.9.30-daniel.20", "0.9.30-daniel.21", "0.9.30-daniel.22", "0.9.30-daniel.23", "0.9.30-daniel.24", "0.9.30-daniel.25", "0.9.30-daniel.26", "0.9.30-daniel.27", "0.9.30-daniel.28", "0.9.30-daniel.3", "0.9.30-daniel.4", "0.9.30-daniel.5", "0.9.30-daniel.6", "0.9.30-daniel.7", "0.9.30-daniel.8", "0.9.30-daniel.9", "0.9.30-daniel.29", "0.9.30-daniel.30", "0.9.30-daniel.31", "0.9.30-daniel.32"]); + const supportedVersions = new Set(["0.3.0", "0.4.0", "0.5.0", "0.6.0", "0.6.1", "0.7.0", "0.7.2", "0.7.3", "0.7.4", "0.7.5", "0.7.6", "0.7.7", "0.7.9", "0.8.0", "0.8.1", "0.8.2", "0.8.3", "0.8.4", "0.8.5", "0.8.6", "0.8.7", "0.8.8", "0.8.9", "0.8.10", "0.8.11", "0.8.12", "0.8.13", "0.9.0", "0.9.1", "0.9.2", "0.9.3", "0.9.4", "0.9.5", "0.9.6", "0.9.7", "0.9.8", "0.9.9", "0.9.10", "0.9.11", "0.9.12", "0.9.13", "0.9.14", "0.9.15", "0.9.16", "0.9.17", "0.9.18", "0.9.19", "0.9.20", "0.9.21", "0.9.22", "0.9.23", "0.9.24", "0.9.25", "0.9.26", "0.9.27", "0.9.28", "0.9.29", "0.9.30", "0.9.31", "0.9.31-daniel.1", "0.9.31-daniel.33", "0.9.31-daniel.34", "0.9.31-daniel.35", "0.9.31-daniel.36", "0.9.31-daniel.37", "0.9.31-daniel.38", "0.9.31-daniel.39", "0.9.31-daniel.40", "0.9.31-daniel.41", "0.9.31-daniel.42", "0.9.31-daniel.43", "0.9.31-daniel.44", "0.9.31-daniel.45", "0.9.31-daniel.46", "0.9.31-daniel.47", "0.9.31-daniel.48", "0.9.31-daniel.49", "0.9.31-daniel.50", "0.9.31-daniel.51", "0.9.31-daniel.52", "0.9.31-daniel.53", "0.9.31-daniel.54", "0.9.32-daniel.55", "0.9.32-daniel.57", "0.9.32-daniel.58", "0.9.32-daniel.59", "0.9.32-daniel.60", "0.9.32-daniel.61", "0.9.32-daniel.62", "0.9.32-daniel.63", "0.9.32-daniel.64", "0.9.32-daniel.65", "0.9.32-daniel.66", "0.9.32-daniel.67", "0.9.32-daniel.68", "0.9.33-daniel.69", "0.9.33-daniel.70", "0.9.33-daniel.71", "0.9.33-daniel.72", "0.9.33-daniel.73", "0.9.33-daniel.74", "0.9.33-daniel.75", "0.9.33-daniel.76", "0.9.30-daniel.1", "0.9.30-daniel.10", "0.9.30-daniel.11", "0.9.30-daniel.15", "0.9.30-daniel.16", "0.9.30-daniel.18", "0.9.30-daniel.19", "0.9.30-daniel.2", "0.9.30-daniel.20", "0.9.30-daniel.21", "0.9.30-daniel.22", "0.9.30-daniel.23", "0.9.30-daniel.24", "0.9.30-daniel.25", "0.9.30-daniel.26", "0.9.30-daniel.27", "0.9.30-daniel.28", "0.9.30-daniel.3", "0.9.30-daniel.4", "0.9.30-daniel.5", "0.9.30-daniel.6", "0.9.30-daniel.7", "0.9.30-daniel.8", "0.9.30-daniel.9", "0.9.30-daniel.29", "0.9.30-daniel.30", "0.9.30-daniel.31", "0.9.30-daniel.32"]); if (!supportedVersions.has(importData.version)) { return { success: false, diff --git a/src/functions/search.ts b/src/functions/search.ts index bec51700f0..76d17b2bfc 100644 --- a/src/functions/search.ts +++ b/src/functions/search.ts @@ -1,7 +1,7 @@ import { observationIndexEligible } from '../ops/observation-index-policy.mjs'; import type { IIIClient } from 'iii-sdk' import type { GraphObservationAuthorization } from "./graph-retrieval.js"; -import type { HybridSearchRequest } from "../state/hybrid-search.js"; +import { releaseHybridSearchRequest, type HybridSearchRequest } from "../state/hybrid-search.js"; import type { IndexPersistenceStatus } from "../state/index-persistence.js"; import type { CompressedObservation, Memory, SearchResult, Session } from '../types.js' import { KV } from '../state/schema.js' @@ -948,7 +948,8 @@ export function registerSearchFunction(sdk: IIIClient, kv: StateKV): void { observation: CompressedObservation layer: SearchResultLayer } | null>>() - const hybridRequest: HybridSearchRequest = {} + const hybridRequest: HybridSearchRequest = { onHydrationInvalidated: () => { sessionCache.clear(); memoryProjectCache.clear(); observationCache.clear(); } } + try { let enriched: SearchResult[] = [] let useHybridRanker = hybridRanker !== null while (true) { @@ -1051,6 +1052,7 @@ export function registerSearchFunction(sdk: IIIClient, kv: StateKV): void { hasCwdFilter: !!cwdFilter, }) return response + } finally { releaseHybridSearchRequest(hybridRequest); } } ) } diff --git a/src/ops/state-epoch.d.mts b/src/ops/state-epoch.d.mts index 30cc45e43a..cf9f23cbf4 100644 --- a/src/ops/state-epoch.d.mts +++ b/src/ops/state-epoch.d.mts @@ -3,3 +3,5 @@ import type {EngineEpochEvidence} from './engine-epoch.mjs'; export interface StateBootIdentity {version:1;pid:number;start:string;instanceId:string;bootId:string;adapter:string;storePath:string|null} export function createStateBoundEvidence(options:{engineEvidence:EngineEpochEvidence;identity:StateBootIdentity;expectedExecutable:{path:string;sha256:string};storePath:string;procRoot?:string}):EngineEpochEvidence; export function pinStateClient(sdk:IIIClient,prefix:string,current?:()=>boolean,diagnostic?:()=>string|null):IIIClient; + +export function pinnedStateReadCurrent(client:IIIClient):(()=>boolean)|null; diff --git a/src/ops/state-epoch.mjs b/src/ops/state-epoch.mjs index 7813e209f5..db1c7edab2 100644 --- a/src/ops/state-epoch.mjs +++ b/src/ops/state-epoch.mjs @@ -49,9 +49,13 @@ export function createStateBoundEvidence({engineEvidence,identity,expectedExecut const initial=capture();if(!initial)fail(); return {...engineEvidence,capture,diagnostic:()=>lastFailure,current(expected){const now=capture();if(!now)return false;if(!expected?.state)return reject('STATE_EXPECTED_EPOCH_MISSING');if(!engineEvidence.current(expected))return reject(engineEvidence.diagnostic?.()??'ENGINE_EPOCH_CHANGED');if(JSON.stringify(now.state)!==JSON.stringify(expected.state))return reject('STATE_EPOCH_CHANGED');return true;}}; } +// Reuse exactly the pinned predicate for request-local cached read admission. +// Unpinned clients must not opt into hydration reuse. +const pinnedReadChecks=new WeakMap(); +export function pinnedStateReadCurrent(client){return pinnedReadChecks.get(client)??null;} export function pinStateClient(sdk,prefix,current=()=>true,diagnostic=()=>'STATE_EPOCH_NOT_CURRENT'){ if(!/^state::[a-f0-9-]{36}::$/.test(prefix))fail(); - return new Proxy(sdk,{get(target,key){ + const pinned=new Proxy(sdk,{get(target,key){ if(key==='trigger')return async request=>{ const stateCall=request.function_id.startsWith('state::'); if(stateCall&&!current())throw stateDispatchRejected(diagnostic()??'STATE_EPOCH_NOT_CURRENT'); @@ -61,4 +65,6 @@ export function pinStateClient(sdk,prefix,current=()=>true,diagnostic=()=>'STATE }; const value=Reflect.get(target,key);return typeof value==='function'?value.bind(target):value; }}); + pinnedReadChecks.set(pinned,()=>{try{return current()===true;}catch{return false;}}); + return pinned; } diff --git a/src/state/hybrid-search.ts b/src/state/hybrid-search.ts index 2c7eb4e2ad..256622a51c 100644 --- a/src/state/hybrid-search.ts +++ b/src/state/hybrid-search.ts @@ -1,4 +1,4 @@ -import { SearchIndex } from "./search-index.js"; +import { SearchIndex, type SearchIndexRevision } from "./search-index.js"; import type { SearchLayer } from "./search-layer.js"; import { VectorIndex } from "./vector-index.js"; import type { @@ -28,6 +28,16 @@ const MIN_FUSION_POOL = 40; export interface HybridSearchRequest { embedding?: { query: string; value: Promise }; candidateCount?: number; + onHydrationInvalidated?: () => void; +} + +type Hydrated = { observation: CompressedObservation; layer: "observation" | "memory" }; +type HydrationEntry = { value: Promise; revision: SearchIndexRevision; keys: { unchanged(): boolean; release(): void } }; +const requestHydration = new WeakMap }>(); +export function releaseHybridSearchRequest(request: HybridSearchRequest): void { + const cache = requestHydration.get(request); + if (cache) for (const entry of cache.entries.values()) { entry.revision.release(); entry.keys.release(); } + requestHydration.delete(request); } export class HybridSearch { @@ -258,7 +268,14 @@ export class HybridSearch { if (request) request.candidateCount = combined.length; const retrievalDepth = Math.max(limit, 20); const diversified = this.diversifyBySession(combined, combined.length); - const enriched = await this.enrichResults(diversified, retrievalDepth); + let cache = request ? requestHydration.get(request) : undefined; + if (request && this.kv.durability?.watchKeys && this.kv.hydrationReadCurrent) { + if (cache && (cache.owner !== this || cache.query !== query || cache.layer !== targetLayer)) { + releaseHybridSearchRequest(request); request.onHydrationInvalidated?.(); cache = undefined; + } + if (!cache) { cache = { owner: this, query, layer: targetLayer, entries: new Map() }; requestHydration.set(request, cache); } + } + const enriched = await this.enrichResults(diversified, retrievalDepth, cache?.entries, request); return applyReranking ? this.rerankResults(query, enriched, limit) : enriched.slice(0, limit); } @@ -365,13 +382,15 @@ export class HybridSearch { graphContext?: string; }>, limit: number, + cache?: Map, + request?: HybridSearchRequest, ): Promise { const enriched: HybridSearchResult[] = []; let next = 0; while (next < results.length && enriched.length < limit) { const batch = results.slice(next, next + limit); next += batch.length; - enriched.push(...(await this.enrichBatch(batch))); + enriched.push(...(await this.enrichBatch(batch, cache, request))); } return enriched.slice(0, limit); } @@ -386,21 +405,53 @@ export class HybridSearch { combinedScore: number; graphContext?: string; }>, + cache?: Map, + request?: HybridSearchRequest, ): Promise { const observations = await Promise.all( sliced.map(async (r) => { - const obs = await this.kv - .get(KV.observations(r.sessionId), r.obsId) - .catch(() => null); - if (obs) return { observation: obs, layer: "observation" as const }; - // Fallback: indexed entry may originate from mem::remember, which - // writes to KV.memories with a synthetic sessionId ("memory" or the - // memory's first associated session). Coerce the Memory record into - // a CompressedObservation so search/recall surface saved memories. - const mem = await this.kv - .get(KV.memories, r.obsId) - .catch(() => null); - return mem ? { observation: memoryToObservation(mem), layer: "memory" as const } : null; + let failedRead = false; + const load = async (): Promise => { + const obs = await this.kv + .get(KV.observations(r.sessionId), r.obsId) + .catch(() => { failedRead = true; return null; }); + if (obs) return { observation: obs, layer: "observation" as const }; + // Fallback: indexed entry may originate from mem::remember, which + // writes to KV.memories with a synthetic sessionId ("memory" or the + // memory's first associated session). Coerce the Memory record into + // a CompressedObservation so search/recall surface saved memories. + const mem = await this.kv + .get(KV.memories, r.obsId) + .catch(() => { failedRead = true; return null; }); + return mem ? { observation: memoryToObservation(mem), layer: "memory" as const } : null; + }; + if (!cache || !this.kv.durability?.watchKeys || !this.kv.hydrationReadCurrent) return load(); + const key = JSON.stringify([r.sessionId, r.obsId]); + const valid = (entry: HydrationEntry) => entry.revision.isCurrent() && entry.keys.unchanged(); + const discard = (entry: HydrationEntry, changed = false) => { + entry.revision.release(); entry.keys.release(); + if (cache.get(key) === entry) cache.delete(key); + if (changed) request?.onHydrationInvalidated?.(); + }; + const previous = cache.get(key); + if (previous) { + if (valid(previous) && this.kv.hydrationReadCurrent()) { + const value = await previous.value; + if (valid(previous) && this.kv.hydrationReadCurrent()) return value; + } + discard(previous, true); + } + if (cache.size >= 1000) return load(); + const revision = this.bm25.watchRevision(r.obsId); + const keys = this.kv.durability.watchKeys([{scope: KV.observations(r.sessionId), key: r.obsId}, {scope: KV.memories, key: r.obsId}]); + const entry = { revision, keys, value: load() }; + cache.set(key, entry); + try { + const value = await entry.value; + if (!valid(entry)) { discard(entry, true); return null; } + if (!value || failedRead) discard(entry); + return value; + } catch (error) { discard(entry); throw error; } }), ); const enriched: HybridSearchResult[] = []; diff --git a/src/state/kv.ts b/src/state/kv.ts index bc794db4c9..cb1040edd8 100644 --- a/src/state/kv.ts +++ b/src/state/kv.ts @@ -1,3 +1,4 @@ +import {pinnedStateReadCurrent} from '../ops/state-epoch.mjs' import {isStateDispatchRejected,localMutationRejected} from '../ops/state-dispatch-error.mjs' import { lstatSync, realpathSync } from 'node:fs' import { join } from 'node:path' @@ -146,6 +147,8 @@ export class StateKV { get recoveryMutationSafe(){return !this.uncertainMutation&&!!this.writerLease&&fileWriterRecoveryAdmitted(this.writerLease)} readonly recoveryWrite?: ReturnType readonly backend: StateBackend + get hydrationReadCurrent(){return pinnedStateReadCurrent(this.sdk)} + readonly durability: ReturnType | null readonly observationReader: ReturnType | null readonly graphFileList: ReturnType | null diff --git a/src/types.ts b/src/types.ts index d2e1d305d7..56565ff35b 100644 --- a/src/types.ts +++ b/src/types.ts @@ -421,7 +421,7 @@ export interface ExportCollectionPagination { } export interface ExportData { - version: "0.3.0" | "0.4.0" | "0.5.0" | "0.6.0" | "0.6.1" | "0.7.0" | "0.7.2" | "0.7.3" | "0.7.4" | "0.7.5" | "0.7.6" | "0.7.7" | "0.7.9" | "0.8.0" | "0.8.1" | "0.8.2" | "0.8.3" | "0.8.4" | "0.8.5" | "0.8.6" | "0.8.7" | "0.8.8" | "0.8.9" | "0.8.10" | "0.8.11" | "0.8.12" | "0.8.13" | "0.9.0" | "0.9.1" | "0.9.2" | "0.9.3" | "0.9.4" | "0.9.5" | "0.9.6" | "0.9.7" | "0.9.8" | "0.9.9" | "0.9.10" | "0.9.11" | "0.9.12" | "0.9.13" | "0.9.14" | "0.9.15" | "0.9.16" | "0.9.17" | "0.9.18" | "0.9.19" | "0.9.20" | "0.9.21" | "0.9.22" | "0.9.23" | "0.9.24" | "0.9.25" | "0.9.26" | "0.9.27" | "0.9.28" | "0.9.29" | "0.9.30" | "0.9.30-daniel.1" | "0.9.30-daniel.2" | "0.9.30-daniel.3" | "0.9.30-daniel.4" | "0.9.30-daniel.5" | "0.9.30-daniel.6" | "0.9.30-daniel.7" | "0.9.30-daniel.8" | "0.9.30-daniel.9" | "0.9.30-daniel.10" | "0.9.30-daniel.11" | "0.9.30-daniel.15" | "0.9.30-daniel.16" | "0.9.30-daniel.18" | "0.9.30-daniel.19" | "0.9.30-daniel.20" | "0.9.30-daniel.21" | "0.9.30-daniel.22" | "0.9.30-daniel.23" | "0.9.30-daniel.24" | "0.9.30-daniel.25" | "0.9.30-daniel.26" | "0.9.30-daniel.27" | "0.9.30-daniel.28" | "0.9.31" | "0.9.31-daniel.1" | "0.9.31-daniel.33" | "0.9.31-daniel.34" | "0.9.31-daniel.35" | "0.9.31-daniel.36" | "0.9.31-daniel.37" | "0.9.31-daniel.38" | "0.9.31-daniel.39" | "0.9.31-daniel.40" | "0.9.31-daniel.41" | "0.9.31-daniel.42" | "0.9.31-daniel.43" | "0.9.31-daniel.44" | "0.9.31-daniel.45" | "0.9.31-daniel.46" | "0.9.31-daniel.47" | "0.9.31-daniel.48" | "0.9.31-daniel.49" | "0.9.31-daniel.50" | "0.9.31-daniel.51" | "0.9.31-daniel.52" | "0.9.31-daniel.53" | "0.9.31-daniel.54" | "0.9.32-daniel.55" | "0.9.32-daniel.57" | "0.9.32-daniel.58" | "0.9.32-daniel.59" | "0.9.32-daniel.60" | "0.9.32-daniel.61" | "0.9.32-daniel.62" | "0.9.32-daniel.63" | "0.9.32-daniel.64" | "0.9.32-daniel.65" | "0.9.32-daniel.66" | "0.9.32-daniel.67" | "0.9.32-daniel.68" | "0.9.33-daniel.69" | "0.9.33-daniel.70" | "0.9.33-daniel.71" | "0.9.33-daniel.72" | "0.9.33-daniel.73" | "0.9.33-daniel.74" | "0.9.33-daniel.75" | "0.9.30-daniel.29" | "0.9.30-daniel.30" | "0.9.30-daniel.31" | "0.9.30-daniel.32"; + version: "0.3.0" | "0.4.0" | "0.5.0" | "0.6.0" | "0.6.1" | "0.7.0" | "0.7.2" | "0.7.3" | "0.7.4" | "0.7.5" | "0.7.6" | "0.7.7" | "0.7.9" | "0.8.0" | "0.8.1" | "0.8.2" | "0.8.3" | "0.8.4" | "0.8.5" | "0.8.6" | "0.8.7" | "0.8.8" | "0.8.9" | "0.8.10" | "0.8.11" | "0.8.12" | "0.8.13" | "0.9.0" | "0.9.1" | "0.9.2" | "0.9.3" | "0.9.4" | "0.9.5" | "0.9.6" | "0.9.7" | "0.9.8" | "0.9.9" | "0.9.10" | "0.9.11" | "0.9.12" | "0.9.13" | "0.9.14" | "0.9.15" | "0.9.16" | "0.9.17" | "0.9.18" | "0.9.19" | "0.9.20" | "0.9.21" | "0.9.22" | "0.9.23" | "0.9.24" | "0.9.25" | "0.9.26" | "0.9.27" | "0.9.28" | "0.9.29" | "0.9.30" | "0.9.30-daniel.1" | "0.9.30-daniel.2" | "0.9.30-daniel.3" | "0.9.30-daniel.4" | "0.9.30-daniel.5" | "0.9.30-daniel.6" | "0.9.30-daniel.7" | "0.9.30-daniel.8" | "0.9.30-daniel.9" | "0.9.30-daniel.10" | "0.9.30-daniel.11" | "0.9.30-daniel.15" | "0.9.30-daniel.16" | "0.9.30-daniel.18" | "0.9.30-daniel.19" | "0.9.30-daniel.20" | "0.9.30-daniel.21" | "0.9.30-daniel.22" | "0.9.30-daniel.23" | "0.9.30-daniel.24" | "0.9.30-daniel.25" | "0.9.30-daniel.26" | "0.9.30-daniel.27" | "0.9.30-daniel.28" | "0.9.31" | "0.9.31-daniel.1" | "0.9.31-daniel.33" | "0.9.31-daniel.34" | "0.9.31-daniel.35" | "0.9.31-daniel.36" | "0.9.31-daniel.37" | "0.9.31-daniel.38" | "0.9.31-daniel.39" | "0.9.31-daniel.40" | "0.9.31-daniel.41" | "0.9.31-daniel.42" | "0.9.31-daniel.43" | "0.9.31-daniel.44" | "0.9.31-daniel.45" | "0.9.31-daniel.46" | "0.9.31-daniel.47" | "0.9.31-daniel.48" | "0.9.31-daniel.49" | "0.9.31-daniel.50" | "0.9.31-daniel.51" | "0.9.31-daniel.52" | "0.9.31-daniel.53" | "0.9.31-daniel.54" | "0.9.32-daniel.55" | "0.9.32-daniel.57" | "0.9.32-daniel.58" | "0.9.32-daniel.59" | "0.9.32-daniel.60" | "0.9.32-daniel.61" | "0.9.32-daniel.62" | "0.9.32-daniel.63" | "0.9.32-daniel.64" | "0.9.32-daniel.65" | "0.9.32-daniel.66" | "0.9.32-daniel.67" | "0.9.32-daniel.68" | "0.9.33-daniel.69" | "0.9.33-daniel.70" | "0.9.33-daniel.71" | "0.9.33-daniel.72" | "0.9.33-daniel.73" | "0.9.33-daniel.74" | "0.9.33-daniel.75" | "0.9.33-daniel.76" | "0.9.30-daniel.29" | "0.9.30-daniel.30" | "0.9.30-daniel.31" | "0.9.30-daniel.32"; exportedAt: string; sessions: Session[]; observations: Record; diff --git a/src/version.ts b/src/version.ts index baae89d89b..56d22e50a9 100644 --- a/src/version.ts +++ b/src/version.ts @@ -1,2 +1,2 @@ -export const VERSION = "0.9.33-daniel.75"; +export const VERSION = "0.9.33-daniel.76"; export const III_PINNED_VERSION = "0.25.1"; diff --git a/test/codex-hook-compat-artifacts.test.ts b/test/codex-hook-compat-artifacts.test.ts index 4560bd3eca..242fddd024 100644 --- a/test/codex-hook-compat-artifacts.test.ts +++ b/test/codex-hook-compat-artifacts.test.ts @@ -32,6 +32,22 @@ beforeAll(async () => { for (const name of ["session-start.mjs", "pre-tool-use.mjs", "notification.mjs"]) writeFileSync(join(sandbox, directory, name), "stock sentinel " + name); } await buildCodexCompat(sandbox); + // Production hooks detach their retry worker to meet host deadlines. In this + // fixture, retain that real child so hook completion also proves writer exit. + // Do not set CAPTURE_DRAIN_CHILD here: that bypasses normal hook collection. + writeFileSync(join(sandbox, "await-hook-children.mjs"), ` + import childProcess from 'node:child_process'; + import {syncBuiltinESMExports} from 'node:module'; + const spawn=childProcess.spawn; + childProcess.spawn=function(...args){ + const child=spawn(...args); + child.unref=()=>child; + child.once('error',()=>{process.exitCode=1;}); + child.once('close',(code,signal)=>{if(code!==0||signal)process.exitCode=1;}); + return child; + }; + syncBuiltinESMExports(); + `); }, 30_000); afterAll(() => rmSync(sandbox, { recursive: true, force: true })); beforeEach(async () => { @@ -74,8 +90,8 @@ async function client(directory = "plugin/scripts", historical = true) { } async function hook(name: string, input: unknown) { return await new Promise<{ code: number | null; output: string }>((resolveResult, reject) => { - const child = spawn(process.execPath, [join(sandbox, "plugin/scripts/codex-compat", name + ".mjs")], { env: { PATH: process.env.PATH ?? "", HOME: home, AGENTMEMORY_HOME: join(home, ".agentmemory"), AGENTMEMORY_URL: url, AGENTMEMORY_SECRET: "synthetic-secret", AGENTMEMORY_CAPTURE_SPOOL_DIR: join(home, "spool"), AGENTMEMORY_HOOK_INJECT_CONTEXT: "true" }, stdio: ["pipe", "pipe", "pipe"] }); - let output = ""; child.stdout.on("data", data => { output += data; }); child.on("error", reject); child.on("exit", code => resolveResult({ code, output })); child.stdin.end(JSON.stringify(input)); + const child = spawn(process.execPath, ["--import", pathToFileURL(join(sandbox, "await-hook-children.mjs")).href, join(sandbox, "plugin/scripts/codex-compat", name + ".mjs")], { env: { PATH: process.env.PATH ?? "", HOME: home, AGENTMEMORY_HOME: join(home, ".agentmemory"), AGENTMEMORY_URL: url, AGENTMEMORY_SECRET: "synthetic-secret", AGENTMEMORY_CAPTURE_SPOOL_DIR: join(home, "spool"), AGENTMEMORY_HOOK_INJECT_CONTEXT: "true" }, stdio: ["pipe", "pipe", "pipe"] }); + let output = ""; child.stdout.on("data", data => { output += data; }); child.stderr.resume(); child.on("error", reject); child.on("close", code => resolveResult({ code, output })); child.stdin.end(JSON.stringify(input)); }); } async function drain() { @@ -196,6 +212,31 @@ describe("explicit custom compatibility build artifacts", () => { expect(await instance.flush()).toBe(false); expect(records(join(home, "legacy"), n => n.endsWith(".json"))[0].body.data).toEqual(event.data); }); + it("waits for detached hook writers before resolving fixture completion", async () => { + const marker = join(home, "delayed-writer-finished"); + writeFileSync(join(sandbox, "plugin/scripts/codex-compat/fixture-delayed-writer.mjs"), ` + import {spawn} from 'node:child_process'; + const child=spawn(process.execPath,['--input-type=module','-e', + ${JSON.stringify(`await new Promise(r=>setTimeout(r,300));(await import('node:fs')).writeFileSync(${JSON.stringify(marker)},'complete');`)}], + {detached:true,stdio:'ignore'}); + child.unref(); + `); + try { + expect(await hook("fixture-delayed-writer", {})).toEqual({code:0,output:""}); + expect(readFileSync(marker,"utf8")).toBe("complete"); + } finally { + // Keep the negative-control run isolated too: never delete a live writer's home. + await vi.waitFor(()=>expect(readFileSync(marker,"utf8")).toBe("complete")); + } + }); + it("does not report fixture success when a detached writer fails", async () => { + writeFileSync(join(sandbox, "plugin/scripts/codex-compat/fixture-failed-writer.mjs"), ` + import {spawn} from 'node:child_process'; + const child=spawn(process.execPath,['-e','setTimeout(()=>process.exit(7),50)'],{detached:true,stdio:'ignore'}); + child.unref(); + `); + expect(await hook("fixture-failed-writer", {})).toEqual({code:1,output:""}); + }); it("retains PreCompact collection while suppressing unsupported output and context lookup", async () => { const result = await hook("pre-compact", { session_id: "synthetic-precompact", cwd: home, trigger: "auto" }); expect(result).toEqual({ code: 0, output: "" }); diff --git a/test/search-hydration-cache.test.ts b/test/search-hydration-cache.test.ts new file mode 100644 index 0000000000..5102bb1720 --- /dev/null +++ b/test/search-hydration-cache.test.ts @@ -0,0 +1,78 @@ +import {afterEach,expect,it,vi} from 'vitest'; +vi.mock('../src/logger.js',()=>({logger:{info:vi.fn(),warn:vi.fn(),error:vi.fn()}})); +vi.mock('../src/functions/access-tracker.js',()=>({recordAccessBatch:vi.fn()})); +import {getSearchIndex,registerSearchFunction,setHybridRanker} from '../src/functions/search.js'; +import {HybridSearch} from '../src/state/hybrid-search.js'; +import {memoryToObservation} from '../src/state/memory-utils.js'; +import {createStateDurabilityFence} from '../src/ops/state-durability.mjs'; +import {KV} from '../src/state/schema.js'; +const date='2026-01-01T00:00:00Z'; +afterEach(()=>{setHybridRanker(null);getSearchIndex().clear();vi.restoreAllMocks();}); +function fixture(cache:boolean){ + getSearchIndex().clear();let observers=0,peakObservers=0;let roundHook: ((limit:number)=>Promise)|undefined;const records=new Map(),reads={observations:0,memories:0}; + const fence=createStateDurabilityFence({storePath:'/tmp/synthetic-search-no-file-access',completion:'sdk'}); + const watch=fence.watchKeys.bind(fence);fence.watchKeys=(keys:any)=>{const token=watch(keys);observers++;peakObservers=Math.max(peakObservers,observers);let released=false;return{unchanged:token.unchanged,release(){if(!released){released=true;observers--;token.release();}}};}; + const kv:any={...(cache?{durability:fence,hydrationReadCurrent:()=>true}:{}),async get(scope:string,key:string){if(scope===KV.observations('memory'))reads.observations++;if(scope===KV.memories){reads.memories++;return structuredClone(records.get(key)??null);}return null;},async list(){return[];}}; + for(let i=0;i<1000;i++){const m={id:`mem_${String(i).padStart(4,'0')}`,createdAt:date,updatedAt:date,type:'fact',title:'hydration marker',content:'hydration marker',concepts:[],files:[],sessionIds:[],strength:7,version:1,isLatest:true,project:i>=998?'wanted':'other'};records.set(m.id,m);getSearchIndex().add(memoryToObservation(m as any),'memory');} + const hybrid=new HybridSearch(getSearchIndex(),null,null,kv,0.4,0.6,0),limits:number[]=[]; + setHybridRanker(async(...args:any[])=>{limits.push(args[1]);await roundHook?.(args[1]);return (hybrid.search as any)(...args);}); + let handler:any;registerSearchFunction({registerFunction:(_:string,fn:any)=>handler=fn} as any,kv); + return{records,reads,limits,kv,fence,onRound:(fn:(limit:number)=>Promise)=>roundHook=fn,observers:()=>observers,peakObservers:()=>peakObservers,search:()=>handler({query:'hydration marker',project:'wanted',targetLayer:'memory',agentId:'*',limit:5})}; +} +it('reduces repeated hydration across 100-to-1000 sparse-project rounds without changing results',async()=>{ + const baseline=fixture(false);const expected=await baseline.search();expect(baseline.limits).toEqual([100,200,400,800,1000]);expect(baseline.reads).toEqual({observations:2500,memories:3500}); + const cached=fixture(true);const result=await cached.search();expect(result).toEqual(expected);expect(cached.limits).toEqual(baseline.limits);expect(cached.reads).toEqual({observations:1000,memories:2000});expect(cached.observers()).toBe(0);expect(cached.peakObservers()).toBeLessThanOrEqual(1000); +}); +it('never shares cached values across separate search requests',async()=>{ + const f=fixture(true);await f.search();f.reads.observations=0;f.reads.memories=0;f.records.get('mem_0999').content='new current content'; + const r=await f.search();expect(JSON.stringify(r)).toContain('new current content');expect(f.reads).toEqual({observations:1000,memories:2000});expect(f.observers()).toBe(0); +}); + +it('invalidates hydration and project authorization on mutation before index publication',async()=>{ + const f=fixture(true);f.records.get('mem_0000').project='wanted'; + f.onRound(async limit=>{if(limit===200){await f.fence.mutate(KV.memories,'mem_0000',async()=>{f.records.get('mem_0000').project='other';});await f.fence.mutate(KV.memories,'mem_0001',async()=>{f.records.get('mem_0001').project='wanted';});}}); + const result=await f.search();const ids=result.results.map((r:any)=>r.observation.id);expect(ids).not.toContain('mem_0000');expect(ids).toContain('mem_0001');expect(f.observers()).toBe(0); +}); +it('does not retain a fallback hydration after an asynchronous State read failure',async()=>{ + const f=fixture(true),get=f.kv.get;let attempts=0; + f.kv.get=async(scope:string,key:string)=>{const value=await get(scope,key);if(scope===KV.observations('memory')&&key==='mem_0000'&&++attempts===1)throw Error('synthetic temporary failure');return value;}; + await f.search();expect(attempts).toBe(2);expect(f.observers()).toBe(0); +}); +it('keeps concurrent requests separate and releases all watchers',async()=>{ + const f=fixture(true);const [a,b]=await Promise.all([f.search(),f.search()]);expect(a).toEqual(b);expect(f.reads).toEqual({observations:2000,memories:4000});expect(f.observers()).toBe(0); +}); + +it('invalidates a cached entry when index publication replaces its revision',async()=>{ + const f=fixture(true);f.onRound(async limit=>{if(limit===200){f.records.get('mem_0000').project='wanted';getSearchIndex().invalidateRevision('mem_0000');}}); + expect((await f.search()).results.map((r:any)=>r.observation.id)).toContain('mem_0000');expect(f.observers()).toBe(0); +}); +it('does not retain deleted hydration or release watchers only on success',async()=>{ + const f=fixture(true);f.records.get('mem_0000').project='wanted';f.onRound(async limit=>{if(limit===200)await f.fence.mutate(KV.memories,'mem_0000',async()=>{f.records.delete('mem_0000');});}); + expect((await f.search()).results.map((r:any)=>r.observation.id)).not.toContain('mem_0000');expect(f.observers()).toBe(0); +}); +it('releases request watchers when project authorization fails',async()=>{ + const f=fixture(true),get=f.kv.get;f.kv.get=async(scope:string,key:string)=>{if(scope===KV.sessions)throw Error('synthetic authorization read failure');return get(scope,key);}; + await expect(f.search()).rejects.toThrow('synthetic authorization read failure');expect(f.observers()).toBe(0); +}); +it('does not reuse a record changed while its first hydration is pending',async()=>{ + const f=fixture(true);f.records.get('mem_0000').project='wanted';f.records.get('mem_0000').content='obsolete hydration payload';const get=f.kv.get;let changed=false; + f.kv.get=async(scope:string,key:string)=>{const value=await get(scope,key);if(scope===KV.memories&&key==='mem_0000'&&!changed){changed=true;await f.fence.mutate(scope,key,async()=>{f.records.get(key).content='current hydration payload';});}return value;}; + const result=await f.search();expect(JSON.stringify(result)).toContain('current hydration payload');expect(JSON.stringify(result)).not.toContain('obsolete hydration payload');expect(f.observers()).toBe(0); +}); + +it('does not reuse cached records after State writer identity changes',async()=>{ + const f=fixture(true);f.records.get('mem_0000').project='wanted';let current=true;const get=f.kv.get; + f.kv.hydrationReadCurrent=()=>current;f.kv.get=async(...args:any[])=>{if(!current)throw Error('STATE_WRITER_IDENTITY_CHANGED');return get(...args);}; + f.onRound(async limit=>{if(limit===200)current=false;}); + expect((await f.search()).results).toEqual([]);expect(f.observers()).toBe(0); +}); +it('disables reuse without a pinned read admission check',async()=>{ + const f=fixture(true);delete f.kv.hydrationReadCurrent;await f.search();expect(f.reads).toEqual({observations:2500,memories:3500});expect(f.peakObservers()).toBe(0); +}); +it('rechecks identity after an asynchronous cached-value continuation',async()=>{ + const f=fixture(true);f.records.get('mem_0000').project='wanted';let current=true,armed=false;const get=f.kv.get; + f.kv.hydrationReadCurrent=()=>{const result=current;if(armed){armed=false;queueMicrotask(()=>{current=false;});}return result;}; + f.kv.get=async(...args:any[])=>{if(!current)throw Error('STATE_WRITER_IDENTITY_CHANGED');return get(...args);}; + f.onRound(async limit=>{if(limit===200)armed=true;}); + expect((await f.search()).results).toEqual([]);expect(f.observers()).toBe(0); +}); diff --git a/test/state-epoch.test.ts b/test/state-epoch.test.ts index 107883b073..d2f8b45b6c 100644 --- a/test/state-epoch.test.ts +++ b/test/state-epoch.test.ts @@ -76,3 +76,10 @@ it('bounds socket reobservation and still rejects persistent foreign ownership', const f=await fixture();rmSync(join(f.proc,'200/fd/3'));let reads=0;race.afterRead=path=>{if(path===join(f.proc,'200/fd'))reads++;}; expect(()=>f.m.createStateBoundEvidence({...f,storePath:f.store,procRoot:f.proc})).toThrow();expect(reads).toBeLessThanOrEqual(3); }); +it('exposes the exact pinned read predicate without issuing RPCs and fails closed',async()=>{ + const {pinStateClient,pinnedStateReadCurrent}=await import('../src/ops/state-epoch.mjs'); + const trigger=vi.fn();const sdk={trigger} as any;let current=true; + expect(pinnedStateReadCurrent(sdk)).toBeNull(); + const pinned=pinStateClient(sdk,'state::11111111-1111-4111-8111-111111111111::',()=>{if(!current)throw Error('identity unavailable');return true;}); + const check=pinnedStateReadCurrent(pinned)!;expect(check()).toBe(true);current=false;expect(check()).toBe(false);expect(trigger).not.toHaveBeenCalled(); +});