diff --git a/.cursor-plugin/plugin.json b/.cursor-plugin/plugin.json index b845f693e6..b46de0240c 100644 --- a/.cursor-plugin/plugin.json +++ b/.cursor-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "agentmemory", - "version": "0.9.32-daniel.55", + "version": "0.9.33-daniel.73", "description": "Persistent memory for Cursor: auto-captures sessions, prompts, and tool use via hooks, recalls with hybrid BM25 + vector + graph search, and exposes 55 memory tools over MCP. Keyless by default, runs entirely on your machine.", "author": { "name": "Rohit Ghumare", diff --git a/.env.example b/.env.example index d1b668a082..4a8cd7b032 100644 --- a/.env.example +++ b/.env.example @@ -130,6 +130,7 @@ # MAX_OBS_PER_SESSION=500 # Hard per-session cap: later observations of that session are refused; only capture-worker refusals are parked as dead letters (0 = no cap) # SUMMARIZE_CHUNK_SIZE=400 # When mem::summarize sees a session larger than this, it chunks observations and map-reduces (chunk-summarize → reduce-merge) to stay within the LLM's context window. Default 400 ≈ 50k tokens per chunk at ~110 tok/obs. Native sessions are capped by MAX_OBS_PER_SESSION; chunking primarily matters for bulk-imported jsonl sessions, which bypass that cap. # SUMMARIZE_CHUNK_CONCURRENCY=6 # Parallel chunk LLM calls during chunked summarize. Default 6 fits ~100-chunk sessions under iii's 180s function-invocation timeout at typical ~8s/call. High-throughput providers (Novita, DeepInfra, DeepSeek) commonly allow 100+ concurrent — bump this for very large imported sessions. +# AGENTMEMORY_SUMMARY_TIMEOUT_MS=120000 # Total time budget in whole milliseconds for one mem::summarize run, shared by every chunk call. Default 120000 (2 min). Raise it for slow or local LLMs (Ollama, LM Studio, vLLM) on large chunked sessions that fail with summary_timeout. Session-end and recovery summaries wait this budget plus 60 s; POST /agentmemory/summarize is still cut off by the 180 s HTTP worker timeout, but the summary keeps running and is saved. # ----------------------------------------------------------------------------- # 5. Behaviour flags @@ -137,6 +138,7 @@ # AGENTMEMORY_AUTO_COMPRESS=true # Run LLM-written compression on every observation batch. Requires BOTH this flag and a configured provider; a provider alone leaves synthetic compression active. # AGENTMEMORY_INJECT_CONTEXT=true # Opt in to Claude-style hook context injection. Pi/OMP use 1 in their launch environment for file enrichment; native prompt recall is independent. +# AGENTMEMORY_ENRICH_TIMEOUT_MS=2000 # PreToolUse hook wait (default 2000 ms). Set this in the hook process environment; setting it only in this server env file does not change the hook timeout. # CONSOLIDATION_ENABLED=false # Disable the consolidation pipeline. Otherwise enabled when an LLM provider is configured, unless AGENTMEMORY_PROVIDER=noop. # CONSOLIDATION_DECAY_DAYS=30 # Age (days) after which non-reinforced memories decay during consolidation # GRAPH_EXTRACTION_ENABLED=true # Extract concept-graph edges on remember; powers the graph-traversal recall path @@ -168,6 +170,7 @@ # AGENTMEMORY_DEBUG=1 # Trace MCP shim probe + standalone fallback decisions to stderr # AGENTMEMORY_FORCE_PROXY=1 # Skip the MCP shim livez probe and trust AGENTMEMORY_URL (for sandboxed MCP clients that can't reach localhost) # AGENTMEMORY_PROBE_TIMEOUT_MS=2000 # MCP shim livez probe timeout +# AGENTMEMORY_CALL_TIMEOUT_MS=190000 # MCP shim and plugin bridge timeout for each proxied REST call, read from the MCP host environment or this file. Default 190000, a little above the 180 s worker invocation limit, so long calls such as memory_consolidate and memory_reflect end on the server first # AGENTMEMORY_READY_TIMEOUT_MS=300000 # CLI worker readiness deadline, including state loading (5 minutes) # AGENTMEMORY_ENGINE_READY_TIMEOUT_MS=120000 # CLI engine readiness deadline (2 minutes) # Startup deadlines accept whole milliseconds from 1 to 3600000. Shell @@ -215,10 +218,10 @@ # 8. iii engine pin # ----------------------------------------------------------------------------- # -# agentmemory pins iii-engine and its iii-sdk dependencies to v0.25.0. -# The CLI downloads the engine into ~/.agentmemory/bin on macOS/Linux. -# Native Windows requires installing the matching iii.exe manually. -# AGENTMEMORY_III_VERSION=0.25.0 # Override only when deliberately matching a different engine +# agentmemory pins iii-engine and its iii-sdk dependencies to v0.25.1. +# The CLI downloads the engine into ~/.agentmemory/bin, including the Windows ZIP. +# AGENTMEMORY_III_VERSION=0.25.1 # Override only when deliberately matching a different engine +# AGENTMEMORY_LOCAL_DISPATCH=true # iii 0.25 runs calls to the worker's own functions in-process; set false for full engine traces # Engine usage telemetry (iii). Off unless you set it; true opts in. # III_TELEMETRY_ENABLED=false diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000000..e77aaeb170 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +# Unified diff context lines intentionally contain one leading space, including blank context. +deploy/state-worker/compat.patch whitespace=-blank-at-eol diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 97f74c44b2..2806b98182 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -57,7 +57,7 @@ jobs: os: [ubuntu-latest, macos-latest] node-version: [20, 22, 24, 26] steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 with: persist-credentials: false - uses: actions/setup-node@v6 diff --git a/.github/workflows/personal-pr-validation.yml b/.github/workflows/personal-pr-validation.yml new file mode 100644 index 0000000000..25f885b090 --- /dev/null +++ b/.github/workflows/personal-pr-validation.yml @@ -0,0 +1,49 @@ +name: Personal fork PR checks + +# Source-only validation for changes proposed against personal. +# No production credentials, service control, data migration or deployment. +on: + pull_request: + branches: [personal] + types: [opened, synchronize, reopened, ready_for_review] + +permissions: + contents: read + +concurrency: + group: personal-pr-checks-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + source: + name: Source, generated hooks and unit regressions + runs-on: ubuntu-latest + timeout-minutes: 35 + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + - uses: actions/setup-node@v6 + with: + node-version: 22 + - name: Install dependencies + run: npm install --legacy-peer-deps --no-audit --no-fund + - name: Build + run: npm run build + - name: Generate explicit Codex compatibility artifacts + run: npm run build:codex-compat + - name: Reject stale checked-in hook bundles + run: git diff --exit-code -- plugin/scripts + - name: Verify skills and documented facts + run: | + npm run skills:check + npm run docs:check + - name: Eval TypeScript contract + run: npx tsc --noEmit --project tsconfig.eval.json + - name: Unit and non-integration regressions (isolated HOME) + shell: bash + run: | + test_home="$(mktemp -d "${RUNNER_TEMP}/agentmemory-pr-home.XXXXXX")" + trap 'rm -rf "$test_home"' EXIT + export HOME="$test_home" + npm test diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index af230453fc..fec9218700 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -25,7 +25,7 @@ jobs: contents: read id-token: write steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 with: persist-credentials: false @@ -56,7 +56,7 @@ jobs: NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} - if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: npm-release-${{ github.run_id }}-${{ github.run_attempt }} path: ${{ runner.temp }}/npm-release diff --git a/.github/workflows/release-gate.yml b/.github/workflows/release-gate.yml index fb929712a4..c5b24a0d1e 100644 --- a/.github/workflows/release-gate.yml +++ b/.github/workflows/release-gate.yml @@ -34,7 +34,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-latest] + os: [ubuntu-latest, ubuntu-24.04-arm, macos-latest, macos-15-intel] steps: - uses: actions/checkout@v6 with: @@ -49,9 +49,9 @@ jobs: shell: bash run: | engine_pin=$(node --import tsx --input-type=module -e 'import { III_PINNED_VERSION } from "./src/version.ts"; console.log(III_PINNED_VERSION)') - engine_bin="$HOME/.cache/agentmemory-release-gate/iii-${engine_pin}-linux-x64" + engine_bin="$HOME/.cache/agentmemory-release-gate/iii-${engine_pin}-linux-$(node -p process.arch)" test -x "$engine_bin" - AGENTMEMORY_TEST_III="$engine_bin" AGENTMEMORY_REQUIRE_LIVE_TESTS=1 npx vitest run test/compose-supervision-integration.test.ts test/codex-live-daemon.test.ts + AGENTMEMORY_TEST_III="$engine_bin" AGENTMEMORY_REQUIRE_LIVE_TESTS=1 npx vitest run test/compose-supervision-integration.test.ts test/codex-live-daemon.test.ts test/compose-c1-integration.test.ts - run: npm run test:startup:packaged -- --skip-build --out "${{ runner.temp }}/startup-gate" - name: Test packaged vector recovery shell: bash @@ -74,9 +74,9 @@ jobs: archives=("$RUNNER_TEMP"/graph-index-pack/*.tgz) npm run test:graph-index:packaged -- --tarball "${archives[0]}" --out "$RUNNER_TEMP/graph-index-gate" - if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: - name: release-gate-${{ matrix.os }} + name: release-gate-${{ matrix.os }}-${{ github.run_attempt }} path: | ${{ runner.temp }}/release-gate ${{ runner.temp }}/startup-gate @@ -88,9 +88,13 @@ jobs: retention-days: 14 windows-smoke: - name: windows smoke - runs-on: windows-latest + name: windows smoke (${{ matrix.os }}) + runs-on: ${{ matrix.os }} timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + os: [windows-latest, windows-11-arm] steps: - uses: actions/checkout@v6 with: @@ -117,9 +121,9 @@ jobs: node scripts/release-gate/windows-smoke.mjs --tarball $tarballs[0].FullName --out "$env:RUNNER_TEMP\windows-smoke" if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: - name: windows-smoke + name: windows-smoke-${{ matrix.os }}-${{ github.run_attempt }} path: ${{ runner.temp }}/windows-smoke if-no-files-found: ignore retention-days: 14 diff --git a/AGENTS.md b/AGENTS.md index 7811057591..1e9e3af79c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -11,7 +11,7 @@ agentmemory is a persistent memory system for AI coding agents, built on iii-engine's three primitives (Worker/Function/Trigger). Everything goes through `registerFunction`/`registerTrigger`/`sdk.trigger()` — never bypass iii-engine with standalone SQLite or in-process alternatives. -- **Engine**: iii-sdk 0.25.0 with @iii-dev/helpers 0.25.0 (Worker Compose-managed iii-engine 0.25.0 on port 49134; the client type is `IIIClient`, HTTP requests are `HttpRequest` from `@iii-dev/helpers/http`) +- **Engine**: iii-sdk 0.25.1 with @iii-dev/helpers 0.25.1 (Worker Compose-managed iii-engine 0.25.1 on port 49134; the client type is `IIIClient`, HTTP requests are `HttpRequest` from `@iii-dev/helpers/http`) - **State**: iii state worker with a file-backed KV store by default (`./data/state_store.db`); Docker pins `save_interval_ms` to 2000 ms - **Build**: TypeScript → ESM via tsdown, output to `dist/` - **Test**: vitest (`npm test` excludes integration tests) @@ -108,20 +108,20 @@ Hook scripts in `src/hooks/` are standalone Node.js scripts (no iii-sdk import). ## Testing -- Before every commit run, in order: `npm run build`, `npm run skills:gen`, `npm run docs:sync`, `npm run skills:check`, `npm test` (6,300+ tests). Commit anything the generators changed. CI runs the same build, skills check and tests. +- Before every commit run, in order: `npm run build`, `npm run skills:gen`, `npm run docs:sync`, `npm run skills:check`, `npm test` (7,000+ tests). Commit anything the generators changed. CI runs the same build, skills check and tests. - Before any npm publish: `npm run release:gate` must pass. It installs the packed tarballs into a clean home and runs capture, recovery, restart, export/import, MCP and status checks against the installed CLI (see CONTRIBUTING.md, Release process) - Mock pattern: `vi.mock("iii-sdk")` with mock `sdk.trigger`, `kv.get/set/list` - Test files go in `test/` with `.test.ts` extension - Follow existing patterns in `test/crystallize.test.ts` for function tests -## Current Stats (v0.9.32-daniel.55) +## Current Stats (v0.9.32-daniel.67) - 55 MCP tools visible by default; `AGENTMEMORY_TOOLS=core` selects 8 essentials -- 160 REST endpoints +- 161 REST endpoints - 6 MCP resources, 3 MCP prompts - 12 hooks, 17 skills -- 353+ iii functions -- 6,300+ tests +- 355+ iii functions +- 7,000+ tests ## Personal fork verification isolation diff --git a/CHANGELOG.md b/CHANGELOG.md index 67a7429f84..27f72c74d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,69 @@ +## 0.9.32-daniel.68 — Sharded graph retrieval index + +- Store new graph retrieval builds in per-build hash shards to stay below the native file namespace size limit. +- Explicit graph-index migration rebuilds a ready legacy index from canonical graph records and verifies inventory before publishing readiness. The old index data and canonical nodes, edges, and history are preserved. +- Continue to read legacy indexes and resume their existing journals. Newly rebuilt indexes use schema 3 and require the sharding-aware server. + +## 0.9.32-daniel.67 — Health current-state display + +- Count recorded matching SDK resolutions separately from unresolved recovery checkpoints. +- Reflect exact current-source legacy summary coverage in Health through read-only metadata. +- Preserve original recovery records and stored session progress. + +## Unreleased — compact summary acknowledgement + +- Preserve summary bodies and cursors while avoiding old+new response amplification: file-backed server summary writes use the engine state::set_ack success-only response, with a pinned JSON request snapshot. Ordinary state::set/Redis and compatibility-mode StateKV remain unchanged. +- Require read-only acknowledgement capability discovery before writer acquisition and capture admission. Missing/malformed acknowledgement or actual storage failure remains an uncertain mutation; no fallback, timeout extension, message-cap increase or data migration. +- Cause: identical installed COW engine cc4ff876 + SDK0.25 synthetic overwrite of33556506-byte JSON emits67113039-byte old/new result, exceeds SDK64MiB, reconnects and TIMEOUT despite applied value. Current failure session's stored summary33555971 bytes crosses the same boundary. Operational content is not used as a fixture. +- Preserve manual/storage holds when a recovered memory hold would otherwise deadlock three-way recovery. Only a fresh stable-memory check and successful storage fence may remove the memory hold; admission stays closed until explicit vector-verified resume, and automatic retries remain bounded. +- Validation and deployment state: see workspace work/storage-timeout63/REPORT.md. Source implementation is not operating activation. Historical failures and old unknown writes remain preserved. Durable memory recording is pending manual review. + # Changelog ## [Unreleased] +## [0.9.33-daniel.73] — 2026-11-14 +- Stage existing SDK capture queue reference replacements in a private, quota-counted queue workspace, avoiding permanent ext4 directory growth from random temporary names in large queues. Preserve the existing 5 MiB admission and 8 KiB transition budgets, crash files, source originals, and conservative refusal when legacy bootstrap cannot fit. Source fix only; installed services are unchanged. See `docs/customizations/SDK-SPOOL-TRANSITION-WORKSPACE.md`. +- Make Ubuntu CI subprocess fixtures use the active Node executable. Validate finite capture deadlines under contention without weakening exact-once ACK/original preservation checks, and wait for real browser fixture completion instead of a fixed virtual-time DOM dump. Test-only changes; no operating service changes. + +## [0.9.33-daniel.72] — 2026-11-13 +- Reobserve transient State/engine socket ownership snapshot gaps at most three times without replaying RPCs or accepting a changed epoch/foreign owner. Classify remaining proven local pre-dispatch admission failures without latching uncertain writes, and retain bounded first/latest failure diagnostics without payloads. Preserve previous uncertainty and source data; see the operating follow-up in the integration note. + +## [0.9.33-daniel.71] — 2026-11-13 +- Separate pending capture admission capacity from retained SDK source evidence. Preserve SDK originals, unresolved proof/review history and non-SDK admission limits; maintain an admission-only idempotent key Set to avoid double-counting and per-event metadata rescans. Source-only validation: 57 focused tests pass; not deployed by this change. + +## [0.9.33-daniel.70] — 2026-11-13 +## [0.9.33-daniel.69] — 2026-11-13 +### v0.9.33 integration reconciled with .68 + +- Integrate upstream paging/60 MiB payload budget, consolidation provenance/version-head recovery/recurrence, bounded reflection, orphan-facet repair, eviction index cleanup and partial-vector recovery while preserving source/cursor/hold/history contracts. +- Integrate session time filters, scoped native hook output and Cursor metadata, configurable summary deadline, machine-relative CPU metrics, pinned worker versions/provider forwarding, guarded Windows demo stop, and verified .25.1 installer hashes/retries. No operating deployment or hook installation. + + +- Complete the official structured-object redaction call site in the failure hook, retain raw private capture paths, and regenerate shipped hooks. Synthetic regressions reproduce the missing behavior before the port. +- Package the pinned patched State worker and migration tool; verify an explicit byte-preserving native-v3-to-State-v4 migration and a State-sealed, file/directory-synced clean shutdown in isolated data. No production deployment or automatic historical-marker conversion. + +- Bind isolated Compose file writers to selected State executable, immutable boot instance, process/socket and effective store identity; pin storage RPCs to instance-specific handlers and reject epoch changes before/after replies. Preserve native v3 leases; State-bound v4 does not silently promote prior v3 evidence. +- Wait for verified engine metadata before creating a lease, fail without poisoning markers on timeout, and publish Compose metadata atomically. Native shutdown and metadata compatibility remain guarded. These are source-only candidates; registry/Docker deployment and v3-to-v4 operating migration are not approved or verified. + +- Upgrade the isolated candidate SDK/helpers to 0.25.1, retain strict bundled-dependency audits, and pin the verified OpenTelemetry/YAML/JSON5 dependencies for reproducible builds. +- Size the synthetic SDK fixture memory preflight to its independent 1 GiB RSS budget plus 2 GiB host headroom rather than an unrelated 12 GiB reserve. Preserve the 512 MiB child heap, RSS/time/CPU caps and low-memory abort; the final SDK 0.25.1 boundary regression passes. No production memory or persistence guard changes. + +- Preserve the verified .68 and personal source lineage while preparing the existing selective v0.9.33 review. Regenerate hook/skills/docs assets and retain both histories. +- Complete the missing official SessionEnd manifest timeout and add the upstream placeholder-key/plain-http doctor regression without overwriting custom tests. Engine compact ACK, source retention, Graph schema 3 and recovery contracts remain unchanged. +- Source-only preparation; no customization removal, operating activation or durable memory update. See `docs/customizations/UPSTREAM-0.9.33-INTEGRATION.md` and the workspace validation report. + +### .68 and personal integration + +- Preserve the .68 operating behavior and the personal capture proof pages and deployment checks. This source integration does not change the active deployment. +- Integration verification (2026-11-13): build, Codex compatibility, docs and skills checks passed. The regression run passed 6,597 tests with 13 skips. The large SDK fixture stopped at its 12 GiB host-memory precondition on both the integration and .68 baseline, so the regression gate remains pending. Long-term memory review remains pending. +### Official v0.9.33 source-only integration review (unverified) + +- Import 25 upstream implementation/regression file blobs only where personal is unchanged from official v0.9.32; preserve all modified personal files and the modified iii engine/SDK pin. +- Candidate provider, retrieval, CLIP, MCP, Claude connector, core hook redaction and SessionEnd fixes require full tests, generated-client verification and manual parity review before merging or deploying. +- No private customization has been removed, no service/host/data has been touched. See `docs/customizations/UPSTREAM-0.9.33-INTEGRATION.md`. + + ### Bounded capture proof pages - Limit routine authenticated capture/status checks to 32 physical-key SDK proofs per page. Keep whole retained-source counts separate and omit whole-inbox completion counts for partial pages. @@ -21,6 +83,61 @@ - Preserve the exact modified iii0.22.1 KV/save-loop patch and synthetic successor examples for writer exclusion, immutable epoch binding, bounded admission observation and enrollment-added review evidence. Host selectors and operational records are excluded. - Document representative native HTTP/SDK/storage verification and the remaining full capture-status scan/timeout limitation without claiming disk flush or historical unknown resolution. +## [0.9.32-daniel.67] + +- Health counts matching committed SDK recovery records as reviewed history and recognizes verified legacy summary coverage without rewriting session progress. + — 2026-11-11 +## [0.9.32-daniel.66] — 2026-11-10 +- Add one bounded, in-memory final retained-vector proof diagnostic to the existing persistence status. Report only a known retained ID, an enumerated result, fresh/loaded exact-byte and session matches, admission/queue/revision state, and an allowlisted physical-proof error code. Do not retain provider vector bytes, original text, arbitrary error messages or unknown requested IDs. +- Preserve retained publication/capture verification decisions, exact Float32 byte comparison (including negative zero), physical checkpoint and receipt proof, mutation guards, SDK-only completion and existing retry/backoff. This change explains later attempts; it cannot reconstruct the fresh vector or physical race from earlier .65 failures. +- Prepared separately from source `87a3d10026b5a727e059c12d6ed9416cfacf84a5`, with the parent-authorized .66 version metadata synchronized. Commit, packaging and deployment remain pending parent review. Validation and limits: `docs/customizations/RETAINED-PROOF-DIAGNOSTICS.md`. Long-term memory recording remains pending manual review. + +## [0.9.32-daniel.65] — 2026-11-10 +- Complete a retained embedding retry only when a fresh embedding of the current canonical source exactly matches the retained WAL and loaded vector bytes and session, with current native checkpoint/receipt evidence revalidated during publication. Preserve the original WAL and vector mutation guard; changed or missing evidence remains pending. Retain bounded prior retry and commit metadata with the SDK-only completion marker. +- Apply the operator's 2026-11-10 memory tuning: pause below18% available, recover above22% available continuously for120seconds. Keep the2GiB pause floor,3GiB recovery floor,5%/512MiB critical boundary and existing storage/manual holds. +- Implementation and focused regression evidence are recorded in `docs/customizations/RETAINED-EMBEDDING-65.md`. Operating application, selected capture recovery and long-term memory review remain separate acceptance steps. + +## [0.9.32-daniel.64] — 2026-11-08 +## [0.9.32-daniel.63] — 2026-11-07 +- Keep fresh SDK-v2 source retention and reference admission under one lease. Share the native hook's original deadline across lock acquisition and transport, reserving 750ms for final state recording. Preserve retained-source acknowledgement if reference admission fails, retain the default 150ms lock contract without an explicit deadline, and leave exhausted pending work queued before any send. Unknown transport outcomes remain held; existing held/sending records never become a successful pending result merely because time ran out. +- Claim and read the selected body under the same lease, preserving source hashing and owner fencing. No spool schema, quota, original-retention, automatic unknown-send retry or host trust changes. The source/reference publication gap under an OS crash and unbounded kernel filesystem latency are not eliminated. +- Continue successful embedding batches after a one-second yield, only after that batch's publication has settled and other due work remains. Keep batch size five, shared model serialization, existing retry/backoff and 15-second idle/error cadence; rotate attempts so failing head items cannot starve later intents. The publication bound applies to this pump, not to the existing independent add API. +- Reproduced the client lock/deadline failures and scheduler idle/starvation before changes. Client focused95 and independent22 tests, scheduler focused51 and independent13 tests passed in isolated homes. All39 sequential build configurations and SDK/skills/docs checks passed; TypeScript matches the30 existing .62 diagnostics. The ordinary suite passed6,515 with13 conditional skips; its unchanged12GiB SDK boundary fixture refused to start with8.77GB available. This environment limitation and operational rollout are recorded separately in `docs/customizations/CAPTURE-THROUGHPUT-63.md`; production remains on .62b and manual long-term-memory review is pending. + +## [0.9.32-daniel.62] — 2026-11-07 +- Fix deferred observation embedding certification rejecting an unchanged source when another observation in the same session is written. Reuse the existing durability fence's per-key revision observers, including already pending mutations, and release each selected watch after certification, failure or worker stop. Preserve source replacement/deletion/ABA, intent-generation, missing-fence and uncertain SDK outcome refusals. +- Base the .62 server preparation on .61 commit `f8246e56eb666ef0f9cefa9135ccbe4df5b9deb6`. Keep the .61 client queue repair, server mutation fencing and batch/publication scheduling unchanged. Acquire the new-capture watch after viewer/logging side effects so failed display handling preserves the source/inbox without leaking an unstarted watch. The false rejection and lifecycle failure were reproduced before their fixes; focused mutation/fence/capture regression, all39 sequential build configurations, SDK audit, skills/docs checks and independent review passed. The30 TypeScript diagnostics exactly match .61. See `docs/customizations/EMBEDDING-SOURCE-GUARD-62.md`; operational application and manual long-term-memory review remain separate. + +## [0.9.32-daniel.61] — 2026-11-07 +- Fix native SDK-v2 backlog progress at full admission quotas. Existing reference transitions have a bounded additional 8 KiB workspace and completion-manifest replacement 68 KiB; new originals/references cannot consume this workspace. Preserve all originals, crash files and unknown-send holds. +- Measure only the namespace being charged. Direct delivery claims its selected reference without reading the whole pending inbox; pending listings read manifests, and selected source bytes still receive complete hash verification before transmission. Keep owner fencing, known-not-sent backoff and matching SDK acknowledgement. +- Prepare a client-only .61 follow-up to the .60 server. Scoped validation passed 126 distinct tests, including 1,201 actual synthetic deliveries, a saturated 1,240-reference/3,568-original fixture, retained-original hashes, compiled native hooks and concurrent/unknown-send safety. All 39 sequential build configurations, SDK dependency audit, compatibility artifacts, skills and docs checks passed. The 30 TypeScript diagnostics exactly match a fresh .60 baseline. Actual activation remains separate; see `docs/customizations/SDK-QUEUE-PROGRESS-61.md`. Long-term memory recording remains pending manual review. + +## [0.9.32-daniel.60] — 2026-11-07 +- Read retained capture inboxes through the existing local streaming reader with SDK-acknowledged changes overlaid. Project status and bounded previews before retaining rows, so capture admission, review and retry no longer request an oversized shared SDK frame. Read keyed review metadata once for status instead of making one SDK call per retained original. Keep original payloads and existing per-source completion proofs. +- Add an explicitly invoked selected-source recovery entry for held SDK-v2 sources. Revalidate the retained original, claim it atomically, verify the restore reply, and preserve the original after SDK acknowledgement. Ordinary hooks do not invoke restoration; SDK acknowledgement is not a disk durability claim. +- Preserve the installed compatibility hooks in the server-only release. Source validation, package/native acceptance and operational application are recorded separately in `docs/customizations/CAPTURE-INBOX-60.md` and the November 7 incident dossier. + +## [0.9.32-daniel.59] — 2026-11-07 (held recovery candidate) + +- Support explicit authenticated recovery of a bounded set of vector upserts whose pending and checkpoint records agree in the native store. Preserve the original upserts and record their verified coverage separately. Revalidate coverage on restart and distinguish retained records from pending publication in the index status. +- Require the current cooperative writer, stable checkpoint, matching physical keys and vector bytes. Conflicting updates, clears or incomplete evidence leave recovery blocked. Automatic recovery and deployment resume cannot authorize new coverage. +- Preserve standalone disk-mode indexing when the approved file store has no retention records. During capture reconciliation, a lagging checkpoint timestamp can converge for up to 1.5 seconds before the full evidence check. A failed check leaves the capture and original log intact. +- Record the deployment and operational acceptance separately from this source candidate in the November 7 incident report. + +## [0.9.32-daniel.58] — 2026-11-06 (recovery review candidate) + +- Based on the installed .55 Git commit `4605974f337a50ed7e0ef74e3bb635b6e5d5a805` and the scoped .57 held-start changes from `3148e1d0a3df0241b69b2166085b10ed5979b99f`. This is a separate review candidate for the November 6 outage. +- Limit SDK vector checkpoint reads to one bucket at a time and stop on the first read error. Preserve concurrent reads for disk completion. Keep the .57 guard on vector work during an unresolved storage hold and skip full inbox scans by automatic Viewer updates. +- Give unresolved native worker ownership its own exit status (78). The managed native unit stops retrying that refusal and continues to retry transient failures. +- When memory and storage are both held, release only the memory hold after sustained recovery and a fresh post-wait memory sample. Keep the storage hold and require explicit storage recovery. +- Before an explicit SDK storage recovery, require an empty, readable vector pending log. Retain existing intent and refuse activation when there are entries, unconfirmed reads, or in-flight local log writes. +- The candidate passed 335 focused tests across 15 files, including Chromium, CLI startup, memory recovery and vector intent. All 38 build configurations, asset copying and the server SDK audit passed in sequential batches. The parallel build exceeded the configured V8 heap limits of 700 and 1280 MiB. TypeScript reported the same 30 pre-existing diagnostics as .55. +- Production remains on .55. The incident report records operational evidence and outstanding non-clean recovery admission. + +## [0.9.32-daniel.57] — 2026-11-06 +- Prepare scoped storage-hold protection and viewer status load mitigation; SDK completion remains distinct from disk persistence. Not deployed; compound-hold automatic release is not implemented. See docs/customizations/SCOPED-HOLD-RECOVERY-57.md. + ## [0.9.32-daniel.55] — 2026-11-04 ### Prepared integration (not deployed) @@ -573,6 +690,20 @@ Wave release closing several breaking regressions reported against v0.9.26, plus - `/agentmemory:forget` skill still calls `memory_governance_delete` which only touches `KV.memories` and never observations ([#833](https://github.com/rohitg00/agentmemory/issues/833)). Skill rewrite + new `memory_forget` MCP tool tracked separately. - `crypto.randomUUID()` global-only on Node <19 ([#715](https://github.com/rohitg00/agentmemory/issues/715)). Drop-in import fix tracked. +[0.9.33-daniel.73]: https://github.com/rohitg00/agentmemory/compare/v0.9.33-daniel.72...v0.9.33-daniel.73 +[0.9.33-daniel.72]: https://github.com/rohitg00/agentmemory/compare/v0.9.33-daniel.71...v0.9.33-daniel.72 +[0.9.33-daniel.71]: https://github.com/rohitg00/agentmemory/compare/v0.9.33-daniel.70...v0.9.33-daniel.71 +[0.9.33-daniel.70]: https://github.com/rohitg00/agentmemory/compare/v0.9.33-daniel.69...v0.9.33-daniel.70 +[0.9.33-daniel.69]: https://github.com/rohitg00/agentmemory/compare/v0.9.32-daniel.68...v0.9.33-daniel.69 +[0.9.32-daniel.67]: https://github.com/rohitg00/agentmemory/compare/v0.9.32-daniel.66...v0.9.32-daniel.67 +[0.9.32-daniel.66]: https://github.com/rohitg00/agentmemory/compare/v0.9.32-daniel.65...v0.9.32-daniel.66 +[0.9.32-daniel.65]: https://github.com/rohitg00/agentmemory/compare/v0.9.32-daniel.64...v0.9.32-daniel.65 +[0.9.32-daniel.64]: https://github.com/rohitg00/agentmemory/compare/v0.9.32-daniel.63...v0.9.32-daniel.64 +[0.9.32-daniel.63]: https://github.com/rohitg00/agentmemory/compare/v0.9.32-daniel.62...v0.9.32-daniel.63 +[0.9.32-daniel.62]: https://github.com/rohitg00/agentmemory/compare/v0.9.32-daniel.61...v0.9.32-daniel.62 +[0.9.32-daniel.61]: https://github.com/rohitg00/agentmemory/compare/v0.9.32-daniel.60...v0.9.32-daniel.61 +[0.9.32-daniel.60]: https://github.com/rohitg00/agentmemory/compare/v0.9.32-daniel.59...v0.9.32-daniel.60 +[0.9.32-daniel.57]: https://github.com/rohitg00/agentmemory/compare/v0.9.32-daniel.55...v0.9.32-daniel.57 [0.9.32-daniel.55]: https://github.com/rohitg00/agentmemory/compare/v0.9.31-daniel.54...v0.9.32-daniel.55 [0.9.31-daniel.54]: https://github.com/rohitg00/agentmemory/compare/v0.9.31-daniel.51...v0.9.31-daniel.54 diff --git a/INSTALL_FOR_AGENTS.md b/INSTALL_FOR_AGENTS.md index 2aa89fac1f..c5c1947dab 100644 --- a/INSTALL_FOR_AGENTS.md +++ b/INSTALL_FOR_AGENTS.md @@ -4,7 +4,7 @@ You are a coding agent. Follow this runbook top to bottom to install agentmemory ## What you are installing -agentmemory is a local memory server for coding agents. Its CLI starts an iii Worker Compose runtime using the pinned iii 0.25.0 engine and registry workers `state@0.22.20` and `http@0.22.7`. The default listeners are REST/MCP HTTP on port 3111, the viewer on 3113, and the internal engine WebSocket on 49134. The viewer receives live updates as SSE through `GET /agentmemory/viewer/events` on the REST port. +agentmemory is a local memory server for coding agents. Its CLI starts an iii Worker Compose runtime using the pinned iii 0.25.1 engine and registry workers `state@0.22.20` and `http@0.22.7`. The default listeners are REST/MCP HTTP on port 3111, the viewer on 3113, and the internal engine WebSocket on 49134. The viewer receives live updates as SSE through `GET /agentmemory/viewer/events` on the REST port. The CLI downloads the pinned engine into `~/.agentmemory/bin` on macOS/Linux. Native Windows needs a manually installed pinned `iii.exe`; WSL2 follows the Linux path. Persistent state defaults to `~/Library/Application Support/agentmemory` on macOS, `$XDG_DATA_HOME/agentmemory` or `~/.local/share/agentmemory` on Linux, and `%APPDATA%\agentmemory` on Windows. Override the directory with `--data-dir ` or `AGENTMEMORY_DATA_DIR`, and reuse the same override on every restart. `AGENTMEMORY_ENV_FILE` selects a non-default env file. Instance 0 reuses an existing `./data/state_store.db` when present. `--instance N` creates an isolated directory and port set for another runtime. @@ -14,7 +14,7 @@ Default keyless mode needs no API key or cloud account and disables vector embed - Node.js >= 20 with npm and npx. Check with `node -v`, `npm -v`, and `npx -v`. - macOS/Linux automatic iii installation also requires `curl`, a POSIX `sh`, and `tar`. Check with `command -v curl sh tar`. Minimal images such as `node:20-slim` may not include them. -- Windows: WSL2 follows the Linux path. Native Windows requires the pinned iii-engine v0.25.0 binary installed manually. Native automated `connect` supports only `copilot-cli`; configure other Windows agents manually. WSL `connect` applies only to agents installed in that same WSL environment. +- Windows: WSL2 follows the Linux path. Native Windows requires the pinned iii-engine v0.25.1 binary installed manually. Native automated `connect` supports only `copilot-cli`; configure other Windows agents manually. WSL `connect` applies only to agents installed in that same WSL environment. - Default ports 3111 (REST), 3113 (viewer), and 49134 (engine) must be free. Viewer events are carried over REST. ## Running non-interactively @@ -38,12 +38,12 @@ For native Windows, install the engine before continuing: ```powershell # Download the matching archive from the pinned release: -# https://github.com/iii-hq/iii/releases/tag/iii%2Fv0.25.0 +# https://github.com/iii-hq/iii/releases/tag/iii%2Fv0.25.1 # Extract iii.exe to a directory on PATH, such as $HOME\.local\bin, then verify: iii.exe --version ``` -Expect: exactly `0.25.0`. Verify the archive against its matching SHA-256 asset before extracting it. Do not install an unpinned iii release. +Expect: exactly `0.25.1`. Verify the archive against its matching SHA-256 asset before extracting it. Do not install an unpinned iii release. ## 2. Start the server @@ -61,7 +61,7 @@ npx -y @agentmemory/agentmemory@latest --data-dir /absolute/path/to/agentmemory- For a second isolated runtime, add `--instance 1`; it stores data under `instance-1` and defaults to ports 3211 (REST), 3213 (viewer), and 49234 (engine). `--instance N` supports N from 0 to 50. Do not use `--port` alone for concurrent daemons: it changes ports but preserves instance 0's canonical lifecycle ownership. Use `--instance` for isolation. -Expect: iii 0.25.0 starts, the agentmemory worker registers with iii, and the ready panel lists REST, viewer, and engine addresses. First boot can take longer while the engine binary and registry workers are installed. +Expect: iii 0.25.1 starts, the agentmemory worker registers with iii, and the ready panel lists REST, viewer, and engine addresses. First boot can take longer while the engine binary and registry workers are installed. ## 3. Validate the server and runtime ports @@ -216,7 +216,7 @@ The MCP server exposes 55 tools by default (`--tools all`). Use `--tools core` ( - `The engine process started but the REST API never responded.`: inspect the REST, viewer, and engine ports, confirm the pinned iii process stayed alive, and use `--verbose` for captured engine stderr. - Engine version conflict / another iii engine already running: if an unmanaged engine is already listening on the engine port, agentmemory will not adopt it. Stop that engine or choose another `--instance` or `--port`. The CLI keeps its private pinned binary separate from a user-managed `iii` on `PATH`. - Only 7 tools visible in the agent: the MCP shim is in local fallback because it could not reach a server. Start `npx -y @agentmemory/agentmemory@latest`, ensure `AGENTMEMORY_URL` points at it (default `http://localhost:3111`), then reload MCP. -- Native Windows: install the pinned v0.25.0 `iii.exe` manually or use WSL2. Automated `connect` supports only `copilot-cli`; configure other native agents manually. +- Native Windows: install the pinned v0.25.1 `iii.exe` manually or use WSL2. Automated `connect` supports only `copilot-cli`; configure other native agents manually. - Semantic demo query returns zero hits: this is expected with vectors disabled in default keyless mode. Set `EMBEDDING_PROVIDER=local`, restart, allow the first model download to complete, and rerun the demo. ## Report success @@ -224,7 +224,7 @@ The MCP server exposes 55 tools by default (`--tools all`). Use `--tools core` ( Report back to the user: - agentmemory installed, version, and the server running on port 3111 -- iii-engine v0.25.0 running, the selected platform data directory, REST/viewer/engine ports, and the status `enginePinnedVersion` +- iii-engine v0.25.1 running, the selected platform data directory, REST/viewer/engine ports, and the status `enginePinnedVersion` - which agent was wired via `agentmemory connect`, and the tool count the agent now sees - the save and recall round-trip returned the probe memory after a full stop/start - the viewer is available at `http://localhost:3113` diff --git a/README.md b/README.md index 85b3b82732..1e72a651ad 100644 --- a/README.md +++ b/README.md @@ -686,7 +686,7 @@ npm install && npm run build && npm start ### Windows -agentmemory runs on Windows 10/11. WSL2 follows the Linux path and can install the pinned engine automatically. Native Windows requires the pinned iii-engine v0.25.0 binary installed manually; download the matching archive from the [iii v0.25.0 release page](https://github.com/iii-hq/iii/releases/tag/iii%2Fv0.25.0), verify its checksum, and place `iii.exe` on `PATH` (for example `%USERPROFILE%\.local\bin\iii.exe`). The CLI does not extract the Windows ZIP automatically. +agentmemory runs on Windows 10/11. WSL2 follows the Linux path and can install the pinned engine automatically. Native Windows requires the pinned iii-engine v0.25.1 binary installed manually; download the matching archive from the [iii v0.25.1 release page](https://github.com/iii-hq/iii/releases/tag/iii%2Fv0.25.1), verify its checksum, and place `iii.exe` on `PATH` (for example `%USERPROFILE%\.local\bin\iii.exe`). The CLI does not extract the Windows ZIP automatically. Native Windows automated MCP wiring supports only `agentmemory connect copilot-cli`. For other native Windows agents, use the manual MCP block from [Other agents](#other-agents). Run `connect` inside WSL only when the target agent is installed in that same WSL environment. @@ -1221,7 +1221,7 @@ The complete reference is [`.env.example`](.env.example), including provider cre

API

-160 endpoints on port `3111`. The REST API binds to `127.0.0.1` by default. Protected endpoints require `Authorization: Bearer `, and mesh sync endpoints require an explicitly set `AGENTMEMORY_SECRET` on both peers. +161 endpoints on port `3111`. The REST API binds to `127.0.0.1` by default. Protected endpoints require `Authorization: Bearer `, and mesh sync endpoints require an explicitly set `AGENTMEMORY_SECRET` on both peers. **Authentication is on by default.** When `AGENTMEMORY_SECRET` is not set (in the shell or in `~/.agentmemory/.env`), the CLI generates a random secret on first start and stores it at `~/.agentmemory/secret`, mirrored to the selected data directory as `secret`, with mode `0600` (the platform-specific default data directory is described above). Bundled clients read the canonical secret first when they talk to a local server: the CLI, the viewer, the hooks under `plugin/scripts`, the MCP server and the `@agentmemory/mcp` shim, the configs written by `agentmemory connect`, and the bundled OpenCode, Pi, OpenClaw, Hermes and filesystem-watcher integrations. The stored secret is only sent to loopback URLs (`localhost`, `127.0.0.0/8`, `::1`). An explicit `AGENTMEMORY_SECRET` always wins, and remote clients need the secret set in their environment. Repository-based Docker self-host deployments and the `deploy/` entrypoints generate and export their own secret. To call the API by hand: diff --git a/READMEs/README.ar-SA.md b/READMEs/README.ar-SA.md index 8b8b940dec..fafe6e2bee 100644 --- a/READMEs/README.ar-SA.md +++ b/READMEs/README.ar-SA.md @@ -665,7 +665,7 @@ npm install && npm run build && npm start ### Windows -يعمل agentmemory على Windows 10/11. يتبع WSL2 مسار Linux ويمكنه تثبيت المحرك المثبّت تلقائيًا. يتطلب Windows الأصلي تثبيت ملف iii-engine 0.25.0 المثبّت يدويًا؛ نزّل الأرشيف المطابق من [صفحة إصدار iii 0.25.0](https://github.com/iii-hq/iii/releases/tag/iii%2Fv0.25.0)، وتحقق من بصمته، وضع `iii.exe` على `PATH` (مثل `%USERPROFILE%\.local\bin\iii.exe`). لا يستخرج CLI ملف ZIP الخاص بـ Windows تلقائيًا. +يعمل agentmemory على Windows 10/11. يتبع WSL2 مسار Linux ويمكنه تثبيت المحرك المثبّت تلقائيًا. يتطلب Windows الأصلي تثبيت ملف iii-engine 0.25.1 المثبّت يدويًا؛ نزّل الأرشيف المطابق من [صفحة إصدار iii 0.25.1](https://github.com/iii-hq/iii/releases/tag/iii%2Fv0.25.1)، وتحقق من بصمته، وضع `iii.exe` على `PATH` (مثل `%USERPROFILE%\.local\bin\iii.exe`). لا يستخرج CLI ملف ZIP الخاص بـ Windows تلقائيًا. لا يدعم الربط الآلي لـ MCP على Windows الأصلي إلا `agentmemory connect copilot-cli`. أما وكلاء Windows الأصليون الآخرون، فاستخدم كتلة MCP اليدوية من [وكلاء آخرون](#other-agents). شغّل `connect` داخل WSL فقط عندما يكون الوكيل المستهدف مثبتًا في بيئة WSL نفسها. @@ -1161,7 +1161,7 @@ EMBEDDING_PROVIDER=local

API

-160 نقطة نهاية على المنفذ `3111`. ترتبط REST API بـ `127.0.0.1` افتراضيًا. تتطلب نقاط النهاية المحمية ترويسة `Authorization: Bearer `، وتتطلب نقاط نهاية مزامنة mesh ضبط `AGENTMEMORY_SECRET` صريحًا على كلا الطرفين. +161 نقطة نهاية على المنفذ `3111`. ترتبط REST API بـ `127.0.0.1` افتراضيًا. تتطلب نقاط النهاية المحمية ترويسة `Authorization: Bearer `، وتتطلب نقاط نهاية مزامنة mesh ضبط `AGENTMEMORY_SECRET` صريحًا على كلا الطرفين. **المصادقة مُفعّلة افتراضيًا.** إذا لم يُضبط `AGENTMEMORY_SECRET`، ينشئ CLI عند التشغيل الأول سرًا عشوائيًا في `~/.agentmemory/secret` بصلاحيات `0600` وينسخه أيضًا إلى مجلد البيانات المحدد. يقرأ كل عميل مضمّن هذا السر عند الاتصال بخادم محلي: CLI والعارض والخطافات تحت `plugin/scripts` وخادم MCP ووسيط `@agentmemory/mcp` والإعدادات التي يكتبها `agentmemory connect` وتكاملات OpenCode وPi وOpenClaw وHermes ومراقب نظام الملفات المضمّنة. لا يُرسل السر المحفوظ إلا إلى عناوين loopback (`localhost` و`127.0.0.0/8` و`::1`). وتكون الأولوية دائمًا لقيمة `AGENTMEMORY_SECRET` الصريحة، كما يجب ضبط السر في بيئة العملاء البعيدين. تنشئ عمليات الاستضافة الذاتية عبر Docker من المستودع ونقاط دخول [`deploy/`](../deploy/) سرها وتصدّره. لاستدعاء API يدويًا: diff --git a/READMEs/README.bn-BD.md b/READMEs/README.bn-BD.md index 9349ab0c2e..f48b468271 100644 --- a/READMEs/README.bn-BD.md +++ b/READMEs/README.bn-BD.md @@ -656,7 +656,7 @@ npm install && npm run build && npm start ### Windows -agentmemory Windows 10/11-এ চলে। WSL2 Linux-এর পথ অনুসরণ করে এবং পিন করা engine স্বয়ংক্রিয়ভাবে ইনস্টল করতে পারে। নেটিভ Windows-এর জন্য পিন করা iii-engine 0.25.0 বাইনারি ম্যানুয়ালি ইনস্টল করতে হবে; [iii 0.25.0 release page](https://github.com/iii-hq/iii/releases/tag/iii%2Fv0.25.0) থেকে মিলে যাওয়া archive ডাউনলোড করে checksum যাচাই করুন, তারপর `iii.exe` `PATH`-এ রাখুন (যেমন `%USERPROFILE%\.local\bin\iii.exe`)। CLI Windows ZIP স্বয়ংক্রিয়ভাবে extract করে না। +agentmemory Windows 10/11-এ চলে। WSL2 Linux-এর পথ অনুসরণ করে এবং পিন করা engine স্বয়ংক্রিয়ভাবে ইনস্টল করতে পারে। নেটিভ Windows-এর জন্য পিন করা iii-engine 0.25.1 বাইনারি ম্যানুয়ালি ইনস্টল করতে হবে; [iii 0.25.1 release page](https://github.com/iii-hq/iii/releases/tag/iii%2Fv0.25.1) থেকে মিলে যাওয়া archive ডাউনলোড করে checksum যাচাই করুন, তারপর `iii.exe` `PATH`-এ রাখুন (যেমন `%USERPROFILE%\.local\bin\iii.exe`)। CLI Windows ZIP স্বয়ংক্রিয়ভাবে extract করে না। নেটিভ Windows-এ স্বয়ংক্রিয় MCP wiring কেবল `agentmemory connect copilot-cli` সাপোর্ট করে। অন্য নেটিভ Windows এজেন্টের জন্য [Other agents](#other-agents) থেকে ম্যানুয়াল MCP block ব্যবহার করুন। লক্ষ্য এজেন্টটি একই WSL পরিবেশে ইনস্টল থাকলেই কেবল WSL-এর ভিতরে `connect` চালান। @@ -1174,7 +1174,7 @@ EMBEDDING_PROVIDER=local

API

-`3111` পোর্টে 160 এন্ডপয়েন্ট। REST API ডিফল্টভাবে `127.0.0.1`-এ বাইন্ড করে। প্রোটেক্টেড এন্ডপয়েন্টে `Authorization: Bearer ` লাগে, এবং mesh sync এন্ডপয়েন্টে দুই পিয়ারেই স্পষ্টভাবে সেট করা `AGENTMEMORY_SECRET` লাগে। +`3111` পোর্টে 161 এন্ডপয়েন্ট। REST API ডিফল্টভাবে `127.0.0.1`-এ বাইন্ড করে। প্রোটেক্টেড এন্ডপয়েন্টে `Authorization: Bearer ` লাগে, এবং mesh sync এন্ডপয়েন্টে দুই পিয়ারেই স্পষ্টভাবে সেট করা `AGENTMEMORY_SECRET` লাগে। প্রমাণীকরণ ডিফল্টভাবে চালু। `AGENTMEMORY_SECRET` নির্ধারিত না থাকলে CLI প্রথমবার একটি random secret `~/.agentmemory/secret`-এ রাখে এবং নির্বাচিত ডেটা ডিরেক্টরির `secret`-এ কপি করে; অনুমতি `0600`। সঙ্গে থাকা ক্লায়েন্ট লোকাল সংযোগে মূল ফাইলটিকে অগ্রাধিকার দেয় এবং সংরক্ষিত secret শুধু loopback URL-এ পাঠায়। স্পষ্ট `AGENTMEMORY_SECRET` অগ্রাধিকার পায়; রিমোট ক্লায়েন্টকে environment-এ সেট করতে হয়। Docker নিজস্ব secret তৈরি করে। diff --git a/READMEs/README.cs-CZ.md b/READMEs/README.cs-CZ.md index 7944caa5ec..d6fdd2ba63 100644 --- a/READMEs/README.cs-CZ.md +++ b/READMEs/README.cs-CZ.md @@ -663,7 +663,7 @@ npm install && npm run build && npm start ### Windows -agentmemory běží na Windows 10/11. WSL2 používá cestu pro Linux a může pinned engine nainstalovat automaticky. Nativní Windows vyžaduje ručně nainstalovanou binárku iii-engine v0.25.0; stáhněte odpovídající archiv ze [stránky vydání iii v0.25.0](https://github.com/iii-hq/iii/releases/tag/iii%2Fv0.25.0), ověřte kontrolní součet a umístěte `iii.exe` do `PATH` (například `%USERPROFILE%\.local\bin\iii.exe`). CLI ZIP pro Windows automaticky nerozbaluje. +agentmemory běží na Windows 10/11. WSL2 používá cestu pro Linux a může pinned engine nainstalovat automaticky. Nativní Windows vyžaduje ručně nainstalovanou binárku iii-engine v0.25.1; stáhněte odpovídající archiv ze [stránky vydání iii v0.25.1](https://github.com/iii-hq/iii/releases/tag/iii%2Fv0.25.1), ověřte kontrolní součet a umístěte `iii.exe` do `PATH` (například `%USERPROFILE%\.local\bin\iii.exe`). CLI ZIP pro Windows automaticky nerozbaluje. Automatizované zapojení MCP na nativním Windows podporuje pouze `agentmemory connect copilot-cli`. U ostatních nativních agentů použijte manuální blok MCP ze sekce [Další agenti](#other-agents). `connect` ve WSL spouštějte pouze tehdy, pokud je cílový agent nainstalován ve stejném prostředí WSL. @@ -1160,7 +1160,7 @@ EMBEDDING_PROVIDER=local

API

-160 endpointů na portu `3111`. REST API se ve výchozím stavu váže na `127.0.0.1`. Chráněné endpointy vyžadují `Authorization: Bearer `, a endpointy pro mesh sync vyžadují explicitně nastavené `AGENTMEMORY_SECRET` na obou stranách. +161 endpointů na portu `3111`. REST API se ve výchozím stavu váže na `127.0.0.1`. Chráněné endpointy vyžadují `Authorization: Bearer `, a endpointy pro mesh sync vyžadují explicitně nastavené `AGENTMEMORY_SECRET` na obou stranách. **Autentizace je ve výchozím stavu zapnutá.** Pokud `AGENTMEMORY_SECRET` není nastaveno, CLI při prvním spuštění vygeneruje náhodný secret v `~/.agentmemory/secret` s oprávněním `0600` a zkopíruje jej také do vybraného datového adresáře. Všichni zabalení klienti jej načítají odtud při komunikaci s lokálním serverem: CLI, viewer, hooks v `plugin/scripts`, server MCP a shim `@agentmemory/mcp`, konfigurace zapsané pomocí `agentmemory connect` a zabalené integrace OpenCode, Pi, OpenClaw, Hermes a filesystem-watcher. Uložený secret se odesílá pouze na loopback URL (`localhost`, `127.0.0.0/8`, `::1`). Výslovně nastavená `AGENTMEMORY_SECRET` má vždy přednost a vzdálení klienti ji musí mít ve svém prostředí. Self-host nasazení Dockeru z repozitáře a entrypointy `deploy/` generují a exportují vlastní secret. Pro ruční volání API: diff --git a/READMEs/README.da-DK.md b/READMEs/README.da-DK.md index 80575e8682..7b8630dac0 100644 --- a/READMEs/README.da-DK.md +++ b/READMEs/README.da-DK.md @@ -655,7 +655,7 @@ npm install && npm run build && npm start ### Windows -agentmemory kører på Windows 10/11. WSL2 følger Linux-vejen og kan installere den fastlåste engine automatisk. Native Windows kræver, at den fastlåste iii-engine v0.25.0-binary installeres manuelt; download det tilsvarende arkiv fra [iii v0.25.0-udgivelsessiden](https://github.com/iii-hq/iii/releases/tag/iii%2Fv0.25.0), verificér checksum, og placér `iii.exe` på `PATH` (for eksempel `%USERPROFILE%\.local\bin\iii.exe`). CLI’en pakker ikke Windows-ZIP’en ud automatisk. +agentmemory kører på Windows 10/11. WSL2 følger Linux-vejen og kan installere den fastlåste engine automatisk. Native Windows kræver, at den fastlåste iii-engine v0.25.1-binary installeres manuelt; download det tilsvarende arkiv fra [iii v0.25.1-udgivelsessiden](https://github.com/iii-hq/iii/releases/tag/iii%2Fv0.25.1), verificér checksum, og placér `iii.exe` på `PATH` (for eksempel `%USERPROFILE%\.local\bin\iii.exe`). CLI’en pakker ikke Windows-ZIP’en ud automatisk. Automatisk MCP-opsætning på native Windows understøtter kun `agentmemory connect copilot-cli`. Brug den manuelle MCP-blok fra [Andre agenter](#other-agents) til andre native Windows-agenter. Kør kun `connect` i WSL, hvis målagenten er installeret i samme WSL-miljø. @@ -1168,7 +1168,7 @@ Den fulde reference er [`.env.example`](../.env.example) med legitimationsoplysn

API

-160 endpoints på port `3111`. REST API'et binder til `127.0.0.1` som standard. Beskyttede endpoints kræver `Authorization: Bearer `, og mesh-sync-endpoints kræver en eksplicit sat `AGENTMEMORY_SECRET` på begge peers. +161 endpoints på port `3111`. REST API'et binder til `127.0.0.1` som standard. Beskyttede endpoints kræver `Authorization: Bearer `, og mesh-sync-endpoints kræver en eksplicit sat `AGENTMEMORY_SECRET` på begge peers. **Autentificering er slået til som standard.** Hvis `AGENTMEMORY_SECRET` hverken er sat i shellen eller i `~/.agentmemory/.env`, opretter CLI’en en tilfældig hemmelighed ved første start i `~/.agentmemory/secret` og en kopi med navnet `secret` i den valgte datamappe, med filtilstanden `0600`. Alle medfølgende klienter læser den derfra, når de taler med en lokal server: CLI’en, viewer, hooks under `plugin/scripts`, MCP-serveren og `@agentmemory/mcp`-shimmen, konfigurationerne skrevet af `agentmemory connect`, og de medfølgende OpenCode-, Pi-, OpenClaw-, Hermes- og filesystem-watcher-integrationer. Den lagrede secret sendes kun til loopback-URL’er (`localhost`, `127.0.0.0/8`, `::1`). En eksplicit `AGENTMEMORY_SECRET` vinder altid, og eksterne klienter skal have den i deres miljø. Docker-self-hosting fra repositoryet og `deploy/`-entrypoints genererer og eksporterer deres egen secret. Sådan kaldes API’et manuelt: diff --git a/READMEs/README.de-DE.md b/READMEs/README.de-DE.md index 83f9181cd6..1a7d61e246 100644 --- a/READMEs/README.de-DE.md +++ b/READMEs/README.de-DE.md @@ -653,7 +653,7 @@ npm install && npm run build && npm start ### Windows -agentmemory läuft unter Windows 10/11. WSL2 folgt dem Linux-Pfad und kann die gepinnte Engine automatisch installieren. Unter nativem Windows muss das gepinnte iii-engine-Binary v0.25.0 manuell installiert werden: Laden Sie das passende Archiv von der [iii-v0.25.0-Release-Seite](https://github.com/iii-hq/iii/releases/tag/iii%2Fv0.25.0) herunter, prüfen Sie die Prüfsumme und legen Sie `iii.exe` auf den `PATH` (zum Beispiel `%USERPROFILE%\.local\bin\iii.exe`). Die CLI entpackt das Windows-ZIP nicht automatisch. +agentmemory läuft unter Windows 10/11. WSL2 folgt dem Linux-Pfad und kann die gepinnte Engine automatisch installieren. Unter nativem Windows muss das gepinnte iii-engine-Binary v0.25.1 manuell installiert werden: Laden Sie das passende Archiv von der [iii-v0.25.1-Release-Seite](https://github.com/iii-hq/iii/releases/tag/iii%2Fv0.25.1) herunter, prüfen Sie die Prüfsumme und legen Sie `iii.exe` auf den `PATH` (zum Beispiel `%USERPROFILE%\.local\bin\iii.exe`). Die CLI entpackt das Windows-ZIP nicht automatisch. Die automatisierte MCP-Verdrahtung unter nativem Windows unterstützt nur `agentmemory connect copilot-cli`. Für andere native Windows-Agenten verwenden Sie den manuellen MCP-Block unter [Andere Agenten](#other-agents). Führen Sie `connect` in WSL nur aus, wenn der Ziel-Agent in derselben WSL-Umgebung installiert ist. @@ -1166,7 +1166,7 @@ Die vollständige Referenz steht in [`.env.example`](../.env.example), einschlie

API

-160 Endpunkte auf Port `3111`. Die REST API bindet sich standardmäßig an `127.0.0.1`. Geschützte Endpunkte verlangen `Authorization: Bearer `, und Mesh-Sync-Endpunkte erfordern ein explizit gesetztes `AGENTMEMORY_SECRET` auf beiden Peers. +161 Endpunkte auf Port `3111`. Die REST API bindet sich standardmäßig an `127.0.0.1`. Geschützte Endpunkte verlangen `Authorization: Bearer `, und Mesh-Sync-Endpunkte erfordern ein explizit gesetztes `AGENTMEMORY_SECRET` auf beiden Peers. **Authentifizierung ist standardmäßig aktiv.** Wenn `AGENTMEMORY_SECRET` weder in der Shell noch in `~/.agentmemory/.env` gesetzt ist, erzeugt die CLI beim ersten Start ein zufälliges Secret unter `~/.agentmemory/secret` und spiegelt es als `secret` in das ausgewählte Datenverzeichnis, jeweils mit Dateimodus `0600`. Jeder mitgelieferte Client liest es von dort, wenn er mit einem lokalen Server spricht: die CLI, der Viewer, die Hooks unter `plugin/scripts`, der MCP-Server und das `@agentmemory/mcp`-Shim, die von `agentmemory connect` geschriebenen Konfigs sowie die mitgelieferten Integrationen für OpenCode, Pi, OpenClaw, Hermes und den Filesystem-Watcher. Das gespeicherte Secret wird nur an Loopback-URLs gesendet (`localhost`, `127.0.0.0/8`, `::1`). Ein explizites `AGENTMEMORY_SECRET` gewinnt immer, und Remote-Clients benötigen es trotzdem gesetzt. Docker und die `deploy/`-Entrypoints erzeugen und exportieren bereits ihr eigenes Secret. Um die API von Hand aufzurufen: diff --git a/READMEs/README.el-GR.md b/READMEs/README.el-GR.md index 881e722a43..afe64facea 100644 --- a/READMEs/README.el-GR.md +++ b/READMEs/README.el-GR.md @@ -663,7 +663,7 @@ npm install && npm run build && npm start ### Windows -Το agentmemory εκτελείται σε Windows 10/11. Το WSL2 ακολουθεί τη διαδρομή Linux και μπορεί να εγκαταστήσει αυτόματα τον καρφιτσωμένο engine. Σε native Windows απαιτείται χειροκίνητη εγκατάσταση του καρφιτσωμένου binary iii-engine v0.25.0: κατεβάστε το αντίστοιχο archive από τη [σελίδα έκδοσης iii v0.25.0](https://github.com/iii-hq/iii/releases/tag/iii%2Fv0.25.0), επαληθεύστε το checksum και τοποθετήστε το `iii.exe` στο `PATH` (για παράδειγμα `%USERPROFILE%\.local\bin\iii.exe`). Το CLI δεν εξάγει αυτόματα το Windows ZIP. +Το agentmemory εκτελείται σε Windows 10/11. Το WSL2 ακολουθεί τη διαδρομή Linux και μπορεί να εγκαταστήσει αυτόματα τον καρφιτσωμένο engine. Σε native Windows απαιτείται χειροκίνητη εγκατάσταση του καρφιτσωμένου binary iii-engine v0.25.1: κατεβάστε το αντίστοιχο archive από τη [σελίδα έκδοσης iii v0.25.1](https://github.com/iii-hq/iii/releases/tag/iii%2Fv0.25.1), επαληθεύστε το checksum και τοποθετήστε το `iii.exe` στο `PATH` (για παράδειγμα `%USERPROFILE%\.local\bin\iii.exe`). Το CLI δεν εξάγει αυτόματα το Windows ZIP. Η αυτοματοποιημένη ρύθμιση MCP σε native Windows υποστηρίζει μόνο το `agentmemory connect copilot-cli`. Για άλλους native Windows agents, χρησιμοποιήστε το χειροκίνητο block MCP από τους [Άλλους agents](#other-agents). Εκτελέστε `connect` μέσα στο WSL μόνο αν ο agent-στόχος είναι εγκατεστημένος στο ίδιο περιβάλλον WSL. @@ -1160,7 +1160,7 @@ EMBEDDING_PROVIDER=local

API

-160 endpoints στη θύρα `3111`. Το REST API συνδέεται στο `127.0.0.1` από προεπιλογή. Τα προστατευμένα endpoints απαιτούν `Authorization: Bearer `, και τα endpoints mesh sync απαιτούν ένα ρητά ρυθμισμένο `AGENTMEMORY_SECRET` και στους δύο peers. +161 endpoints στη θύρα `3111`. Το REST API συνδέεται στο `127.0.0.1` από προεπιλογή. Τα προστατευμένα endpoints απαιτούν `Authorization: Bearer `, και τα endpoints mesh sync απαιτούν ένα ρητά ρυθμισμένο `AGENTMEMORY_SECRET` και στους δύο peers. **Η πιστοποίηση είναι ενεργή από προεπιλογή.** Όταν δεν ορίζεται `AGENTMEMORY_SECRET`, στην πρώτη εκκίνηση το CLI δημιουργεί τυχαίο μυστικό στο `~/.agentmemory/secret` με δικαιώματα `0600` και το αντιγράφει και στον επιλεγμένο κατάλογο δεδομένων. Κάθε ενσωματωμένος client το διαβάζει από εκεί όταν μιλά με έναν τοπικό διακομιστή: το CLI, το viewer, τα hooks κάτω από το `plugin/scripts`, ο διακομιστής MCP και το shim `@agentmemory/mcp`, οι ρυθμίσεις που γράφει το `agentmemory connect`, και οι ενσωματωμένες ενσωματώσεις OpenCode, Pi, OpenClaw, Hermes, και filesystem-watcher. Το αποθηκευμένο secret αποστέλλεται μόνο σε loopback URLs (`localhost`, `127.0.0.0/8`, `::1`). Ένα ρητό `AGENTMEMORY_SECRET` υπερισχύει πάντα, και οι απομακρυσμένοι clients εξακολουθούν να το χρειάζονται ρυθμισμένο. Το Docker και τα entrypoints του `deploy/` δημιουργούν και εξάγουν ήδη το δικό τους secret. Για να καλέσετε το API χειροκίνητα: diff --git a/READMEs/README.es-ES.md b/READMEs/README.es-ES.md index 38a5639c25..8670f18be4 100644 --- a/READMEs/README.es-ES.md +++ b/READMEs/README.es-ES.md @@ -655,7 +655,7 @@ npm install && npm run build && npm start ### Windows -agentmemory funciona en Windows 10/11. WSL2 sigue la ruta de Linux y puede instalar el motor fijado automáticamente. Windows nativo requiere instalar manualmente el binario fijado de iii-engine v0.25.0; descarga el archivo correspondiente desde la [página de la release iii v0.25.0](https://github.com/iii-hq/iii/releases/tag/iii%2Fv0.25.0), verifica su checksum y coloca `iii.exe` en `PATH` (por ejemplo, `%USERPROFILE%\.local\bin\iii.exe`). La CLI no extrae automáticamente el ZIP de Windows. +agentmemory funciona en Windows 10/11. WSL2 sigue la ruta de Linux y puede instalar el motor fijado automáticamente. Windows nativo requiere instalar manualmente el binario fijado de iii-engine v0.25.1; descarga el archivo correspondiente desde la [página de la release iii v0.25.1](https://github.com/iii-hq/iii/releases/tag/iii%2Fv0.25.1), verifica su checksum y coloca `iii.exe` en `PATH` (por ejemplo, `%USERPROFILE%\.local\bin\iii.exe`). La CLI no extrae automáticamente el ZIP de Windows. La conexión MCP automatizada en Windows nativo solo admite `agentmemory connect copilot-cli`. Para otros agentes nativos de Windows, usa el bloque MCP manual de [Otros agentes](#other-agents). Ejecuta `connect` dentro de WSL solo si el agente de destino está instalado en ese mismo entorno WSL. @@ -1162,7 +1162,7 @@ La referencia completa está en [`.env.example`](../.env.example): credenciales,

API

-160 endpoints en el puerto `3111`. La REST API se enlaza a `127.0.0.1` por defecto. Los endpoints protegidos requieren `Authorization: Bearer `, y los endpoints de mesh sync requieren un `AGENTMEMORY_SECRET` definido explícitamente en ambos peers. +161 endpoints en el puerto `3111`. La REST API se enlaza a `127.0.0.1` por defecto. Los endpoints protegidos requieren `Authorization: Bearer `, y los endpoints de mesh sync requieren un `AGENTMEMORY_SECRET` definido explícitamente en ambos peers. **La autenticación está activada por defecto.** Si `AGENTMEMORY_SECRET` no está definida en la shell ni en `~/.agentmemory/.env`, la CLI genera un secreto aleatorio en el primer inicio en `~/.agentmemory/secret` y lo copia como `secret` en el directorio de datos seleccionado, con permisos `0600`. Todos los clientes incluidos lo leen desde allí al comunicarse con un servidor local: la CLI, el visor, los hooks de `plugin/scripts`, el servidor MCP y el shim `@agentmemory/mcp`, las configuraciones escritas por `agentmemory connect` y las integraciones incluidas de OpenCode, Pi, OpenClaw, Hermes y filesystem-watcher. El secreto guardado solo se envía a URL de loopback (`localhost`, `127.0.0.0/8`, `::1`). Un `AGENTMEMORY_SECRET` explícito siempre tiene prioridad, y los clientes remotos necesitan que el secreto esté definido en su entorno. Los despliegues Docker alojados por cuenta propia desde el repositorio y los entrypoints de `deploy/` generan y exportan su propio secreto. Para llamar a la API manualmente: diff --git a/READMEs/README.fi-FI.md b/READMEs/README.fi-FI.md index 35ace71c3f..929a60e788 100644 --- a/READMEs/README.fi-FI.md +++ b/READMEs/README.fi-FI.md @@ -663,7 +663,7 @@ npm install && npm run build && npm start ### Windows -agentmemory toimii Windows 10/11:ssä. WSL2 noudattaa Linuxin polkua ja voi asentaa kiinnitetyn moottorin automaattisesti. Natiivi Windows vaatii iii-engine v0.25.0:n kiinnitetyn binäärin asentamisen käsin; lataa sopiva paketti [iii v0.25.0 -julkaisusivulta](https://github.com/iii-hq/iii/releases/tag/iii%2Fv0.25.0), tarkista sen tarkistussumma ja sijoita `iii.exe` `PATH`-polkuun (esimerkiksi `%USERPROFILE%\.local\bin\iii.exe`). CLI ei pura Windows-ZIP-pakettia automaattisesti. +agentmemory toimii Windows 10/11:ssä. WSL2 noudattaa Linuxin polkua ja voi asentaa kiinnitetyn moottorin automaattisesti. Natiivi Windows vaatii iii-engine v0.25.1:n kiinnitetyn binäärin asentamisen käsin; lataa sopiva paketti [iii v0.25.1 -julkaisusivulta](https://github.com/iii-hq/iii/releases/tag/iii%2Fv0.25.1), tarkista sen tarkistussumma ja sijoita `iii.exe` `PATH`-polkuun (esimerkiksi `%USERPROFILE%\.local\bin\iii.exe`). CLI ei pura Windows-ZIP-pakettia automaattisesti. Natiivin Windowsin automaattinen MCP-kytkentä tukee vain komentoa `agentmemory connect copilot-cli`. Muiden natiivien Windows-agenttien kanssa käytä manuaalista MCP-lohkoa kohdasta [Muut agentit](#other-agents). Suorita `connect` WSL:ssä vain, kun kohdeagentti on asennettu samaan WSL-ympäristöön. @@ -1160,7 +1160,7 @@ Täydellinen [`.env.example`](../.env.example) kattaa palveluntarjoajat, tallenn

API

-160 endpointia portissa `3111`. REST-rajapinta sitoutuu oletuksena osoitteeseen `127.0.0.1`. Suojatut endpointit vaativat `Authorization: Bearer `, ja mesh-synkronointiendpointit vaativat nimenomaisesti asetetun `AGENTMEMORY_SECRET`-arvon molemmilla vertaisilla. +161 endpointia portissa `3111`. REST-rajapinta sitoutuu oletuksena osoitteeseen `127.0.0.1`. Suojatut endpointit vaativat `Authorization: Bearer `, ja mesh-synkronointiendpointit vaativat nimenomaisesti asetetun `AGENTMEMORY_SECRET`-arvon molemmilla vertaisilla. **Autentikointi on oletuksena päällä.** Jos `AGENTMEMORY_SECRET` puuttuu, CLI luo ensimmäisellä käynnistyksellä satunnaisen salaisuuden tiedostoon `~/.agentmemory/secret` oikeuksilla `0600` ja kopioi sen myös valittuun datahakemistoon. Kaikki mukana toimitetut asiakkaat lukevat salaisuuden sieltä käyttäessään paikallista palvelinta: CLI, katseluohjelma, `plugin/scripts`-hakemiston hookit, MCP-palvelin ja `@agentmemory/mcp`-shim, `agentmemory connect` -komennon luomat asetukset sekä mukana toimitetut OpenCode-, Pi-, OpenClaw-, Hermes- ja filesystem-watcher-integraatiot. Tallennettu salaisuus lähetetään vain loopback-osoitteisiin (`localhost`, `127.0.0.0/8`, `::1`). Nimenomainen `AGENTMEMORY_SECRET` voittaa aina, ja etäasiakkaiden on asetettava salaisuus ympäristöönsä. Repositorion Docker-itseylläpitokäyttöönotot ja `deploy/`-entrypointit luovat ja vievät oman salaisuutensa. Rajapinnan manuaalinen kutsu: diff --git a/READMEs/README.fr-FR.md b/READMEs/README.fr-FR.md index 145ef84eb3..4d02bb7f66 100644 --- a/READMEs/README.fr-FR.md +++ b/READMEs/README.fr-FR.md @@ -655,7 +655,7 @@ npm install && npm run build && npm start ### Windows -agentmemory fonctionne sous Windows 10/11. WSL2 suit le parcours Linux et peut installer automatiquement le moteur épinglé. Sous Windows natif, installez manuellement le binaire iii-engine v0.25.0 épinglé : téléchargez l'archive correspondante depuis la [page de version iii 0.25.0](https://github.com/iii-hq/iii/releases/tag/iii%2Fv0.25.0), vérifiez son checksum et placez `iii.exe` dans `PATH` (par exemple `%USERPROFILE%\.local\bin\iii.exe`). La CLI n'extrait pas automatiquement le ZIP Windows. +agentmemory fonctionne sous Windows 10/11. WSL2 suit le parcours Linux et peut installer automatiquement le moteur épinglé. Sous Windows natif, installez manuellement le binaire iii-engine v0.25.1 épinglé : téléchargez l'archive correspondante depuis la [page de version iii 0.25.1](https://github.com/iii-hq/iii/releases/tag/iii%2Fv0.25.1), vérifiez son checksum et placez `iii.exe` dans `PATH` (par exemple `%USERPROFILE%\.local\bin\iii.exe`). La CLI n'extrait pas automatiquement le ZIP Windows. Sous Windows natif, le câblage MCP automatisé ne prend en charge que `agentmemory connect copilot-cli`. Pour les autres agents natifs Windows, utilisez le bloc MCP manuel de [Autres agents](#other-agents). Lancez `connect` dans WSL uniquement si l'agent cible est installé dans ce même environnement WSL. @@ -1169,7 +1169,7 @@ La référence complète est [`.env.example`](../.env.example) : identifiants de

API

-160 endpoints sur le port `3111`. L'API REST se lie à `127.0.0.1` par défaut. Les endpoints protégés requièrent `Authorization: Bearer `, et les endpoints de mesh sync requièrent un `AGENTMEMORY_SECRET` explicitement défini sur les deux pairs. +161 endpoints sur le port `3111`. L'API REST se lie à `127.0.0.1` par défaut. Les endpoints protégés requièrent `Authorization: Bearer `, et les endpoints de mesh sync requièrent un `AGENTMEMORY_SECRET` explicitement défini sur les deux pairs. **L'authentification est activée par défaut.** Si `AGENTMEMORY_SECRET` n'est défini ni dans le shell ni dans `~/.agentmemory/.env`, la CLI génère un secret aléatoire au premier lancement dans `~/.agentmemory/secret` et en conserve une copie nommée `secret` dans le répertoire de données choisi, avec les permissions `0600`. Chaque client embarqué le lit depuis là quand il parle à un serveur local : la CLI, le visualiseur, les hooks sous `plugin/scripts`, le serveur MCP et le shim `@agentmemory/mcp`, les configs écrites par `agentmemory connect`, et les intégrations embarquées OpenCode, Pi, OpenClaw, Hermes et filesystem-watcher. Le secret stocké n'est envoyé qu'à des URLs loopback (`localhost`, `127.0.0.0/8`, `::1`). Un `AGENTMEMORY_SECRET` explicite prime toujours, et les clients distants en ont toujours besoin. Docker et les points d'entrée `deploy/` génèrent et exportent déjà leur propre secret. Pour appeler l'API à la main : diff --git a/READMEs/README.he-IL.md b/READMEs/README.he-IL.md index c991700ff7..0321bf49a7 100644 --- a/READMEs/README.he-IL.md +++ b/READMEs/README.he-IL.md @@ -665,7 +665,7 @@ npm install && npm run build && npm start ### Windows -agentmemory פועל ב-Windows 10/11. ‏WSL2 פועל במסלול Linux ויכול להתקין את המנוע הנעוץ אוטומטית. ב-Windows מקורי יש להתקין ידנית את הבינארי הנעוץ iii-engine v0.25.0: הורידו את הארכיון המתאים מ-[דף ההפצה של iii 0.25.0](https://github.com/iii-hq/iii/releases/tag/iii%2Fv0.25.0), אמתו את ה-checksum והוסיפו את `iii.exe` ל-`PATH` (למשל `%USERPROFILE%\.local\bin\iii.exe`). ה-CLI אינו מחלץ אוטומטית את ZIP של Windows. +agentmemory פועל ב-Windows 10/11. ‏WSL2 פועל במסלול Linux ויכול להתקין את המנוע הנעוץ אוטומטית. ב-Windows מקורי יש להתקין ידנית את הבינארי הנעוץ iii-engine v0.25.1: הורידו את הארכיון המתאים מ-[דף ההפצה של iii 0.25.1](https://github.com/iii-hq/iii/releases/tag/iii%2Fv0.25.1), אמתו את ה-checksum והוסיפו את `iii.exe` ל-`PATH` (למשל `%USERPROFILE%\.local\bin\iii.exe`). ה-CLI אינו מחלץ אוטומטית את ZIP של Windows. חיבור MCP אוטומטי ב-Windows מקורי נתמך רק עבור `agentmemory connect copilot-cli`. עבור סוכנים מקומיים אחרים ב-Windows, השתמשו בבלוק MCP הידני מתוך [סוכנים אחרים](#other-agents). הפעילו `connect` בתוך WSL רק אם סוכן היעד מותקן באותה סביבת WSL. @@ -1161,7 +1161,7 @@ EMBEDDING_PROVIDER=local

API

-160 endpoints בפורט `3111`. ה-REST API נקשר ל-`127.0.0.1` כברירת מחדל. endpoints מוגנים דורשים `Authorization: Bearer `, ו-endpoints של mesh sync דורשים `AGENTMEMORY_SECRET` מוגדר באופן מפורש בשני ה-peers. +161 endpoints בפורט `3111`. ה-REST API נקשר ל-`127.0.0.1` כברירת מחדל. endpoints מוגנים דורשים `Authorization: Bearer `, ו-endpoints של mesh sync דורשים `AGENTMEMORY_SECRET` מוגדר באופן מפורש בשני ה-peers. **אימות (Authentication) פעיל כברירת מחדל.** אם `AGENTMEMORY_SECRET` לא מוגדר, בהפעלה הראשונה ה-CLI יוצר סוד אקראי ב-`~/.agentmemory/secret` עם הרשאות `0600`, ומעתיק אותו גם לתיקיית הנתונים שנבחרה. כל לקוח מצורף קורא אותו משם כשהוא מדבר עם שרת מקומי: ה-CLI, המציג, ה-hooks תחת `plugin/scripts`, שרת ה-MCP וה-shim `@agentmemory/mcp`, התצורות שנכתבות על ידי `agentmemory connect`, והאינטגרציות המצורפות של OpenCode, Pi, OpenClaw, Hermes וה-filesystem-watcher. ה-secret המאוחסן נשלח רק ל-URLs של loopback (`localhost`, `127.0.0.0/8`, `::1`). `AGENTMEMORY_SECRET` מפורש גובר תמיד, ולקוחות מרוחקים עדיין צריכים אותו מוגדר. Docker וה-entrypoints של `deploy/` כבר מייצרים ומייצאים את ה-secret שלהם. כדי לקרוא ל-API ביד: diff --git a/READMEs/README.hi-IN.md b/READMEs/README.hi-IN.md index 428ec3eea4..150d33b1fe 100644 --- a/READMEs/README.hi-IN.md +++ b/READMEs/README.hi-IN.md @@ -656,7 +656,7 @@ npm install && npm run build && npm start ### Windows -agentmemory Windows 10/11 पर चलता है। WSL2 Linux path अपनाता है और pinned engine को automatically install कर सकता है। Native Windows पर pinned iii-engine v0.25.0 binary manually install करें: [iii v0.25.0 release page](https://github.com/iii-hq/iii/releases/tag/iii%2Fv0.25.0) से सही archive download करें, checksum verify करें और `iii.exe` को `PATH` में रखें (उदाहरण: `%USERPROFILE%\.local\bin\iii.exe`)। CLI Windows ZIP को automatically extract नहीं करता। +agentmemory Windows 10/11 पर चलता है। WSL2 Linux path अपनाता है और pinned engine को automatically install कर सकता है। Native Windows पर pinned iii-engine v0.25.1 binary manually install करें: [iii v0.25.1 release page](https://github.com/iii-hq/iii/releases/tag/iii%2Fv0.25.1) से सही archive download करें, checksum verify करें और `iii.exe` को `PATH` में रखें (उदाहरण: `%USERPROFILE%\.local\bin\iii.exe`)। CLI Windows ZIP को automatically extract नहीं करता। Native Windows में automated MCP wiring केवल `agentmemory connect copilot-cli` को support करती है। अन्य native Windows agents के लिए [Other agents](#other-agents) वाला manual MCP block इस्तेमाल करें। `connect` को WSL में तभी चलाएँ जब target agent भी उसी WSL environment में installed हो। @@ -1175,7 +1175,7 @@ EMBEDDING_PROVIDER=local

API

-Port `3111` पर 160 endpoints। REST API default रूप से `127.0.0.1` से bind होता है। Protected endpoints `Authorization: Bearer ` की आवश्यकता रखते हैं, और mesh sync endpoints दोनों peers पर एक explicitly set `AGENTMEMORY_SECRET` की आवश्यकता रखते हैं। +Port `3111` पर 161 endpoints। REST API default रूप से `127.0.0.1` से bind होता है। Protected endpoints `Authorization: Bearer ` की आवश्यकता रखते हैं, और mesh sync endpoints दोनों peers पर एक explicitly set `AGENTMEMORY_SECRET` की आवश्यकता रखते हैं। प्रमाणीकरण डिफ़ॉल्ट रूप से चालू है। `AGENTMEMORY_SECRET` सेट न होने पर CLI पहली बार एक random secret `~/.agentmemory/secret` में रखता है और चुनी हुई डेटा डायरेक्टरी के `secret` में उसकी प्रति बनाता है; permissions `0600` होती हैं। साथ आए क्लाइंट लोकल कनेक्शन में इस मूल फ़ाइल को प्राथमिकता देते हैं और सेव किया secret सिर्फ loopback URL को भेजते हैं। स्पष्ट `AGENTMEMORY_SECRET` प्राथमिकता रखता है; रिमोट क्लाइंट को इसे environment में सेट करना होता है। Docker अपना secret बनाता है। diff --git a/READMEs/README.hu-HU.md b/READMEs/README.hu-HU.md index ae152ce45f..30a570ed13 100644 --- a/READMEs/README.hu-HU.md +++ b/READMEs/README.hu-HU.md @@ -661,7 +661,7 @@ Az `npm start` ugyanazt a helyi szervert indítja, mint a telepített CLI. A kö ### Windows -Az agentmemory Windows 10/11 rendszeren fut. A WSL2 a Linux útvonalat követi, és automatikusan telepítheti a rögzített motort. Natív Windowson kézzel telepítse a rögzített iii-engine v0.25.0 binárist: töltse le a megfelelő archívumot az [iii v0.25.0 kiadási oldaláról](https://github.com/iii-hq/iii/releases/tag/iii%2Fv0.25.0), ellenőrizze az ellenőrzőösszeget, majd helyezze az `iii.exe` fájlt a `PATH`-ra (például `%USERPROFILE%\.local\bin\iii.exe`). A CLI nem bontja ki automatikusan a Windows ZIP-et. +Az agentmemory Windows 10/11 rendszeren fut. A WSL2 a Linux útvonalat követi, és automatikusan telepítheti a rögzített motort. Natív Windowson kézzel telepítse a rögzített iii-engine v0.25.1 binárist: töltse le a megfelelő archívumot az [iii v0.25.1 kiadási oldaláról](https://github.com/iii-hq/iii/releases/tag/iii%2Fv0.25.1), ellenőrizze az ellenőrzőösszeget, majd helyezze az `iii.exe` fájlt a `PATH`-ra (például `%USERPROFILE%\.local\bin\iii.exe`). A CLI nem bontja ki automatikusan a Windows ZIP-et. A natív Windows automatizált MCP-kapcsolat csak az `agentmemory connect copilot-cli` parancsot támogatja. Más natív Windows-ügynökökhöz használja az [Egyéb ügynökök](#other-agents) kézi MCP-blokkját. A `connect` parancsot csak akkor futtassa WSL-ben, ha a célügynök is ugyanabban a WSL-környezetben van telepítve. @@ -1158,7 +1158,7 @@ A teljes [`.env.example`](../.env.example) leírja a szolgáltatókat, rögzít

API

-160 endpoint a `3111`-es porton. A REST API alapértelmezetten a `127.0.0.1`-hez kötődik. A védett végpontok `Authorization: Bearer `-et igényelnek, és a mesh-sync végpontok mindkét oldalon explicit módon beállított `AGENTMEMORY_SECRET`-et igényelnek. +161 endpoint a `3111`-es porton. A REST API alapértelmezetten a `127.0.0.1`-hez kötődik. A védett végpontok `Authorization: Bearer `-et igényelnek, és a mesh-sync végpontok mindkét oldalon explicit módon beállított `AGENTMEMORY_SECRET`-et igényelnek. **A hitelesítés alapértelmezetten be van kapcsolva.** Ha az `AGENTMEMORY_SECRET` nincs beállítva, a CLI első indításkor véletlen titkot hoz létre a `~/.agentmemory/secret` fájlban `0600` jogosultsággal, és a kiválasztott adatkönyvtárba is átmásolja. Minden becsomagolt kliens onnan olvassa, amikor egy lokális szerverrel beszél: a CLI, a megjelenítő, a `plugin/scripts` alatti hookok, az MCP szerver és a `@agentmemory/mcp` shim, az `agentmemory connect` által írt konfigurációk, és a becsomagolt OpenCode, Pi, OpenClaw, Hermes és fájlrendszer-figyelő integrációk. A tárolt secret csak loopback URL-eknek kerül elküldésre (`localhost`, `127.0.0.0/8`, `::1`). Egy explicit `AGENTMEMORY_SECRET` mindig elsőbbséget kap, és a távoli klienseknek továbbra is be kell állítaniuk. A Docker és a `deploy/` entrypointok már generálnak és exportálnak egy saját secretet. Az API kézi hívásához: diff --git a/READMEs/README.id-ID.md b/READMEs/README.id-ID.md index 1d772bdee5..c84aa08856 100644 --- a/READMEs/README.id-ID.md +++ b/READMEs/README.id-ID.md @@ -656,7 +656,7 @@ npm install && npm run build && npm start ### Windows -agentmemory berjalan di Windows 10/11. WSL2 mengikuti jalur Linux dan dapat memasang engine yang dipatok secara otomatis. Pada Windows native, pasang biner iii-engine v0.25.0 yang dipatok secara manual: unduh arsip yang sesuai dari [halaman rilis iii v0.25.0](https://github.com/iii-hq/iii/releases/tag/iii%2Fv0.25.0), verifikasi checksum, lalu letakkan `iii.exe` di `PATH` (misalnya `%USERPROFILE%\.local\bin\iii.exe`). CLI tidak mengekstrak ZIP Windows secara otomatis. +agentmemory berjalan di Windows 10/11. WSL2 mengikuti jalur Linux dan dapat memasang engine yang dipatok secara otomatis. Pada Windows native, pasang biner iii-engine v0.25.1 yang dipatok secara manual: unduh arsip yang sesuai dari [halaman rilis iii v0.25.1](https://github.com/iii-hq/iii/releases/tag/iii%2Fv0.25.1), verifikasi checksum, lalu letakkan `iii.exe` di `PATH` (misalnya `%USERPROFILE%\.local\bin\iii.exe`). CLI tidak mengekstrak ZIP Windows secara otomatis. Wiring MCP otomatis di Windows native hanya mendukung `agentmemory connect copilot-cli`. Untuk agen Windows native lainnya, gunakan blok MCP manual dari [Agen lainnya](#other-agents). Jalankan `connect` di WSL hanya jika agen target juga dipasang di lingkungan WSL yang sama. @@ -1175,7 +1175,7 @@ Referensi lengkap ada di [`.env.example`](../.env.example), termasuk penyedia, f

API

-160 endpoint di port `3111`. REST API terbind ke `127.0.0.1` secara default. Endpoint yang terlindungi membutuhkan `Authorization: Bearer `, dan endpoint mesh sync membutuhkan `AGENTMEMORY_SECRET` yang diset secara eksplisit di kedua peer. +161 endpoint di port `3111`. REST API terbind ke `127.0.0.1` secara default. Endpoint yang terlindungi membutuhkan `Authorization: Bearer `, dan endpoint mesh sync membutuhkan `AGENTMEMORY_SECRET` yang diset secara eksplisit di kedua peer. Autentikasi aktif secara default. Jika `AGENTMEMORY_SECRET` tidak disetel, CLI membuat rahasia acak saat pertama dijalankan di `~/.agentmemory/secret` dan menyalinnya ke `secret` dalam direktori data terpilih, dengan mode `0600`. Klien bawaan mengutamakan file kanonis ini untuk koneksi lokal dan hanya mengirim rahasia tersimpan ke URL loopback. `AGENTMEMORY_SECRET` eksplisit selalu diutamakan; klien remote harus mengaturnya melalui environment. Deployment Docker membuat rahasianya sendiri. diff --git a/READMEs/README.it-IT.md b/READMEs/README.it-IT.md index 657e478eaf..dea8356f60 100644 --- a/READMEs/README.it-IT.md +++ b/READMEs/README.it-IT.md @@ -655,7 +655,7 @@ npm install && npm run build && npm start ### Windows -agentmemory funziona su Windows 10/11. WSL2 segue il percorso Linux e può installare automaticamente l'engine fissato. Su Windows nativo, installare manualmente il binario iii-engine v0.25.0 fissato: scaricare l'archivio corrispondente dalla [pagina della release iii 0.25.0](https://github.com/iii-hq/iii/releases/tag/iii%2Fv0.25.0), verificarne il checksum e mettere `iii.exe` nel `PATH` (ad esempio `%USERPROFILE%\.local\bin\iii.exe`). La CLI non estrae automaticamente lo ZIP Windows. +agentmemory funziona su Windows 10/11. WSL2 segue il percorso Linux e può installare automaticamente l'engine fissato. Su Windows nativo, installare manualmente il binario iii-engine v0.25.1 fissato: scaricare l'archivio corrispondente dalla [pagina della release iii 0.25.1](https://github.com/iii-hq/iii/releases/tag/iii%2Fv0.25.1), verificarne il checksum e mettere `iii.exe` nel `PATH` (ad esempio `%USERPROFILE%\.local\bin\iii.exe`). La CLI non estrae automaticamente lo ZIP Windows. La configurazione MCP automatica su Windows nativo supporta solo `agentmemory connect copilot-cli`. Per gli altri agenti Windows nativi, usare il blocco MCP manuale da [Altri agenti](#other-agents). Eseguire `connect` in WSL solo se l'agente di destinazione è installato nello stesso ambiente WSL. @@ -1168,7 +1168,7 @@ La documentazione completa è in [`.env.example`](../.env.example): credenziali,

API

-160 endpoint sulla porta `3111`. L'API REST si collega a `127.0.0.1` per impostazione predefinita. Gli endpoint protetti richiedono `Authorization: Bearer `, e gli endpoint di sincronizzazione mesh richiedono un `AGENTMEMORY_SECRET` impostato esplicitamente su entrambi i peer. +161 endpoint sulla porta `3111`. L'API REST si collega a `127.0.0.1` per impostazione predefinita. Gli endpoint protetti richiedono `Authorization: Bearer `, e gli endpoint di sincronizzazione mesh richiedono un `AGENTMEMORY_SECRET` impostato esplicitamente su entrambi i peer. **L'autenticazione è attiva per default.** Se `AGENTMEMORY_SECRET` non è impostato nella shell o in `~/.agentmemory/.env`, al primo avvio la CLI genera un segreto casuale in `~/.agentmemory/secret` e ne conserva una copia chiamata `secret` nella directory dati selezionata, con permessi `0600`. Ogni client incluso lo legge da lì quando parla con un server locale: la CLI, il viewer, gli hook sotto `plugin/scripts`, il server MCP e lo shim `@agentmemory/mcp`, le configurazioni scritte da `agentmemory connect`, e le integrazioni incluse OpenCode, Pi, OpenClaw, Hermes e filesystem-watcher. Il secret memorizzato viene inviato solo a URL loopback (`localhost`, `127.0.0.0/8`, `::1`). Un `AGENTMEMORY_SECRET` esplicito vince sempre, e i client remoti devono comunque averlo impostato. Docker e gli entrypoint di `deploy/` generano ed esportano già il proprio secret. Per chiamare l'API a mano: diff --git a/READMEs/README.ja-JP.md b/READMEs/README.ja-JP.md index 17f7243c49..bc3c349cd1 100644 --- a/READMEs/README.ja-JP.md +++ b/READMEs/README.ja-JP.md @@ -656,7 +656,7 @@ npm install && npm run build && npm start ### Windows -agentmemory は Windows 10/11 で動作します。WSL2 は Linux の経路に従い、ピン留めされたエンジンを自動インストールできます。ネイティブ Windows では、ピン留めされた iii-engine v0.25.0 バイナリを手動でインストールしてください。[iii v0.25.0 リリースページ](https://github.com/iii-hq/iii/releases/tag/iii%2Fv0.25.0)から該当アーカイブをダウンロードして checksum を確認し、`iii.exe` を `PATH` に配置します(例: `%USERPROFILE%\.local\bin\iii.exe`)。CLI は Windows ZIP を自動展開しません。 +agentmemory は Windows 10/11 で動作します。WSL2 は Linux の経路に従い、ピン留めされたエンジンを自動インストールできます。ネイティブ Windows では、ピン留めされた iii-engine v0.25.1 バイナリを手動でインストールしてください。[iii v0.25.1 リリースページ](https://github.com/iii-hq/iii/releases/tag/iii%2Fv0.25.1)から該当アーカイブをダウンロードして checksum を確認し、`iii.exe` を `PATH` に配置します(例: `%USERPROFILE%\.local\bin\iii.exe`)。CLI は Windows ZIP を自動展開しません。 ネイティブ Windows の自動 MCP 接続は `agentmemory connect copilot-cli` のみをサポートします。それ以外のネイティブ Windows エージェントには[その他のエージェント](#other-agents)の手動 MCP ブロックを使用してください。`connect` を WSL 内で実行するのは、対象エージェントも同じ WSL 環境にインストールされている場合に限ります。 @@ -1174,7 +1174,7 @@ EMBEDDING_PROVIDER=local

API

-ポート `3111` 上の 160 エンドポイント。REST API はデフォルトで `127.0.0.1` にバインドします。保護されたエンドポイントは `Authorization: Bearer ` を要求し、mesh sync エンドポイントは両ピアで明示的に設定された `AGENTMEMORY_SECRET` を要求します。 +ポート `3111` 上の 161 エンドポイント。REST API はデフォルトで `127.0.0.1` にバインドします。保護されたエンドポイントは `Authorization: Bearer ` を要求し、mesh sync エンドポイントは両ピアで明示的に設定された `AGENTMEMORY_SECRET` を要求します。 認証はデフォルトで有効です。`AGENTMEMORY_SECRET` が未設定なら、CLI は初回起動時にランダムなシークレットを `~/.agentmemory/secret` に保存し、選択したデータディレクトリの `secret` にも複製します。権限は `0600` です。同梱クライアントはローカル接続時にこの共通ファイルを優先し、保存されたシークレットを送るのはループバック URL だけです。明示的な `AGENTMEMORY_SECRET` が優先され、リモート接続には環境変数で指定します。Docker デプロイは独自のシークレットを生成します。 diff --git a/READMEs/README.ko-KR.md b/READMEs/README.ko-KR.md index 3ca01235cc..d2b99c6a2a 100644 --- a/READMEs/README.ko-KR.md +++ b/READMEs/README.ko-KR.md @@ -654,7 +654,7 @@ npm install && npm run build && npm start ### Windows -agentmemory는 Windows 10/11에서 실행됩니다. WSL2는 Linux 경로를 따르며 고정 엔진을 자동 설치할 수 있습니다. 네이티브 Windows에서는 고정된 iii-engine v0.25.0 바이너리를 수동 설치하십시오: [iii v0.25.0 릴리스 페이지](https://github.com/iii-hq/iii/releases/tag/iii%2Fv0.25.0)에서 맞는 아카이브를 다운로드하고 checksum을 확인한 뒤 `iii.exe`를 `PATH`에 두십시오(예: `%USERPROFILE%\.local\bin\iii.exe`). CLI는 Windows ZIP을 자동으로 압축 해제하지 않습니다. +agentmemory는 Windows 10/11에서 실행됩니다. WSL2는 Linux 경로를 따르며 고정 엔진을 자동 설치할 수 있습니다. 네이티브 Windows에서는 고정된 iii-engine v0.25.1 바이너리를 수동 설치하십시오: [iii v0.25.1 릴리스 페이지](https://github.com/iii-hq/iii/releases/tag/iii%2Fv0.25.1)에서 맞는 아카이브를 다운로드하고 checksum을 확인한 뒤 `iii.exe`를 `PATH`에 두십시오(예: `%USERPROFILE%\.local\bin\iii.exe`). CLI는 Windows ZIP을 자동으로 압축 해제하지 않습니다. 네이티브 Windows 자동 MCP 연결은 `agentmemory connect copilot-cli`만 지원합니다. 다른 네이티브 Windows 에이전트는 [다른 에이전트](#other-agents)의 수동 MCP 블록을 사용하십시오. 대상 에이전트도 같은 WSL 환경에 설치된 경우에만 WSL에서 `connect`를 실행하십시오. @@ -1173,7 +1173,7 @@ EMBEDDING_PROVIDER=local

API

-포트 `3111`의 엔드포인트 160개. REST API는 기본적으로 `127.0.0.1`에 바인딩됩니다. 보호된 엔드포인트는 `Authorization: Bearer `을 요구하며, mesh sync 엔드포인트는 양쪽 피어 모두에 `AGENTMEMORY_SECRET`이 명시적으로 설정되어 있어야 합니다. +포트 `3111`의 엔드포인트 161개. REST API는 기본적으로 `127.0.0.1`에 바인딩됩니다. 보호된 엔드포인트는 `Authorization: Bearer `을 요구하며, mesh sync 엔드포인트는 양쪽 피어 모두에 `AGENTMEMORY_SECRET`이 명시적으로 설정되어 있어야 합니다. 인증은 기본적으로 활성화됩니다. `AGENTMEMORY_SECRET`이 없으면 CLI가 처음 시작할 때 임의의 비밀값을 `~/.agentmemory/secret`에 저장하고 선택한 데이터 디렉터리의 `secret`에도 복사합니다. 권한은 `0600`입니다. 번들 클라이언트는 로컬 연결 시 이 공통 파일을 우선하며 저장된 비밀값은 루프백 URL에만 보냅니다. 명시적인 `AGENTMEMORY_SECRET`이 우선하고 원격 클라이언트는 환경 변수로 설정해야 합니다. Docker 배포는 자체 비밀값을 생성합니다. diff --git a/READMEs/README.nb-NO.md b/READMEs/README.nb-NO.md index 0c45f96aa1..93b1bf5abb 100644 --- a/READMEs/README.nb-NO.md +++ b/READMEs/README.nb-NO.md @@ -653,7 +653,7 @@ npm install && npm run build && npm start ### Windows -agentmemory kjører på Windows 10/11. WSL2 følger Linux-stien og kan installere den fastlåste engine-en automatisk. Native Windows krever at den fastlåste iii-engine v0.25.0-binærfilen installeres manuelt; last ned det tilsvarende arkivet fra [iii v0.25.0-utgivelsessiden](https://github.com/iii-hq/iii/releases/tag/iii%2Fv0.25.0), kontroller kontrollsummen og legg `iii.exe` på `PATH` (for eksempel `%USERPROFILE%\.local\bin\iii.exe`). CLI-en pakker ikke ut Windows-ZIP-filen automatisk. +agentmemory kjører på Windows 10/11. WSL2 følger Linux-stien og kan installere den fastlåste engine-en automatisk. Native Windows krever at den fastlåste iii-engine v0.25.1-binærfilen installeres manuelt; last ned det tilsvarende arkivet fra [iii v0.25.1-utgivelsessiden](https://github.com/iii-hq/iii/releases/tag/iii%2Fv0.25.1), kontroller kontrollsummen og legg `iii.exe` på `PATH` (for eksempel `%USERPROFILE%\.local\bin\iii.exe`). CLI-en pakker ikke ut Windows-ZIP-filen automatisk. Automatisert native Windows-MCP-tilkobling støtter bare `agentmemory connect copilot-cli`. For andre native Windows-agenter bruker du den manuelle MCP-blokken fra [Andre agenter](#other-agents). Kjør `connect` i WSL bare når målagenten er installert i det samme WSL-miljøet. @@ -1166,7 +1166,7 @@ Den fullstendige referansen er [`.env.example`](../.env.example), med legitimasj

API

-160 endepunkter på port `3111`. REST-API-en bindes til `127.0.0.1` som standard. Beskyttede endepunkter krever `Authorization: Bearer `, og mesh-synkroniseringsendepunkter krever en uttrykkelig satt `AGENTMEMORY_SECRET` på begge sider. +161 endepunkter på port `3111`. REST-API-en bindes til `127.0.0.1` som standard. Beskyttede endepunkter krever `Authorization: Bearer `, og mesh-synkroniseringsendepunkter krever en uttrykkelig satt `AGENTMEMORY_SECRET` på begge sider. **Autentisering er på som standard.** Hvis `AGENTMEMORY_SECRET` verken er satt i skallet eller i `~/.agentmemory/.env`, oppretter CLI-en en tilfeldig hemmelighet ved første start i `~/.agentmemory/secret` og en kopi kalt `secret` i den valgte datamappen, med filmodus `0600`. Alle medfølgende klienter leser den derfra når de snakker med en lokal server: CLI-en, viewer, hookene under `plugin/scripts`, MCP-serveren og `@agentmemory/mcp`-shimmen, konfigurasjonene skrevet av `agentmemory connect` og de medfølgende OpenCode-, Pi-, OpenClaw-, Hermes- og filsystemovervåker-integrasjonene. Den lagrede hemmeligheten sendes bare til loopback-URL-er (`localhost`, `127.0.0.0/8`, `::1`). En uttrykkelig `AGENTMEMORY_SECRET` vinner alltid, og eksterne klienter trenger hemmeligheten i miljøet sitt. Docker-basert selvhosting fra repoet og `deploy/`-entrypointene genererer og eksporterer sin egen hemmelighet. Slik kaller du API-et manuelt: diff --git a/READMEs/README.nl-NL.md b/READMEs/README.nl-NL.md index 9d4f9ce74e..235d056bb1 100644 --- a/READMEs/README.nl-NL.md +++ b/READMEs/README.nl-NL.md @@ -651,7 +651,7 @@ npm install && npm run build && npm start ### Windows -agentmemory draait op Windows 10/11. WSL2 volgt het Linux-pad en kan de vastgezette engine automatisch installeren. Native Windows vereist handmatige installatie van de vastgezette iii-engine v0.25.0-binary; download het passende archief van de [iii v0.25.0-releasepagina](https://github.com/iii-hq/iii/releases/tag/iii%2Fv0.25.0), controleer de checksum en plaats `iii.exe` op `PATH` (bijvoorbeeld `%USERPROFILE%\.local\bin\iii.exe`). De CLI pakt de Windows ZIP niet automatisch uit. +agentmemory draait op Windows 10/11. WSL2 volgt het Linux-pad en kan de vastgezette engine automatisch installeren. Native Windows vereist handmatige installatie van de vastgezette iii-engine v0.25.1-binary; download het passende archief van de [iii v0.25.1-releasepagina](https://github.com/iii-hq/iii/releases/tag/iii%2Fv0.25.1), controleer de checksum en plaats `iii.exe` op `PATH` (bijvoorbeeld `%USERPROFILE%\.local\bin\iii.exe`). De CLI pakt de Windows ZIP niet automatisch uit. Geautomatiseerde MCP-koppeling op native Windows ondersteunt alleen `agentmemory connect copilot-cli`. Gebruik voor andere native Windows-agents het handmatige MCP-blok uit [Other agents](#other-agents). Voer `connect` binnen WSL alleen uit wanneer de doelagent in dezelfde WSL-omgeving is geïnstalleerd. @@ -1162,7 +1162,7 @@ De volledige referentie staat in [`.env.example`](../.env.example), met provider

API

-160 endpoints op poort `3111`. De REST API bindt standaard aan `127.0.0.1`. Beschermde endpoints vereisen `Authorization: Bearer `, en mesh-sync-endpoints vereisen een expliciet ingestelde `AGENTMEMORY_SECRET` op beide peers. +161 endpoints op poort `3111`. De REST API bindt standaard aan `127.0.0.1`. Beschermde endpoints vereisen `Authorization: Bearer `, en mesh-sync-endpoints vereisen een expliciet ingestelde `AGENTMEMORY_SECRET` op beide peers. **Authenticatie staat standaard aan.** Als `AGENTMEMORY_SECRET` niet in de shell of in `~/.agentmemory/.env` staat, maakt de CLI bij de eerste start een willekeurig secret in `~/.agentmemory/secret` en een kopie met de naam `secret` in de geselecteerde gegevensmap, met bestandsmodus `0600`. Elke gebundelde client leest het daar wanneer die met een lokale server communiceert: de CLI, viewer, hooks onder `plugin/scripts`, MCP-server en `@agentmemory/mcp`-shim, configs geschreven door `agentmemory connect`, en de gebundelde OpenCode-, Pi-, OpenClaw-, Hermes- en filesystem-watcher-integraties. Het opgeslagen secret wordt alleen verstuurd naar loopback-URL's (`localhost`, `127.0.0.0/8`, `::1`). Een expliciete `AGENTMEMORY_SECRET` wint altijd, en externe clients hebben het secret in hun omgeving nodig. Docker self-host deployments vanuit de repository en de `deploy/`-entrypoints genereren en exporteren hun eigen secret. Zo roep je de API handmatig aan: diff --git a/READMEs/README.pl-PL.md b/READMEs/README.pl-PL.md index 6c49f60af8..f416f34803 100644 --- a/READMEs/README.pl-PL.md +++ b/READMEs/README.pl-PL.md @@ -661,7 +661,7 @@ npm install && npm run build && npm start ### Windows -agentmemory działa w Windows 10/11. WSL2 korzysta ze ścieżki linuksowej i może automatycznie zainstalować przypięty silnik. Natywny Windows wymaga ręcznie zainstalowanego przypiętego pliku binarnego iii-engine v0.25.0; pobierz pasujące archiwum ze [strony wydania iii 0.25.0](https://github.com/iii-hq/iii/releases/tag/iii%2Fv0.25.0), zweryfikuj sumę kontrolną i umieść `iii.exe` w `PATH` (na przykład `%USERPROFILE%\.local\bin\iii.exe`). CLI nie wypakowuje automatycznie archiwum ZIP dla Windows. +agentmemory działa w Windows 10/11. WSL2 korzysta ze ścieżki linuksowej i może automatycznie zainstalować przypięty silnik. Natywny Windows wymaga ręcznie zainstalowanego przypiętego pliku binarnego iii-engine v0.25.1; pobierz pasujące archiwum ze [strony wydania iii 0.25.1](https://github.com/iii-hq/iii/releases/tag/iii%2Fv0.25.1), zweryfikuj sumę kontrolną i umieść `iii.exe` w `PATH` (na przykład `%USERPROFILE%\.local\bin\iii.exe`). CLI nie wypakowuje automatycznie archiwum ZIP dla Windows. Zautomatyzowane podłączanie MCP w natywnym Windows obsługuje tylko `agentmemory connect copilot-cli`. Dla pozostałych natywnych agentów Windows użyj ręcznego bloku MCP z sekcji [Inni agenci](#other-agents). Uruchamiaj `connect` w WSL tylko wtedy, gdy docelowy agent jest zainstalowany w tym samym środowisku WSL. @@ -1158,7 +1158,7 @@ Pełny opis w [`.env.example`](../.env.example) obejmuje dostawców, filtry prze

API

-160 endpointów na porcie `3111`. REST API domyślnie wiąże się z `127.0.0.1`. Chronione endpointy wymagają `Authorization: Bearer `, a endpointy synchronizacji mesh wymagają wyraźnie ustawionego `AGENTMEMORY_SECRET` na obu peerach. +161 endpointów na porcie `3111`. REST API domyślnie wiąże się z `127.0.0.1`. Chronione endpointy wymagają `Authorization: Bearer `, a endpointy synchronizacji mesh wymagają wyraźnie ustawionego `AGENTMEMORY_SECRET` na obu peerach. **Autoryzacja jest domyślnie włączona.** Gdy `AGENTMEMORY_SECRET` nie jest ustawiony, CLI przy pierwszym uruchomieniu generuje losowy sekret w `~/.agentmemory/secret` z uprawnieniami `0600` i kopiuje go także do wybranego katalogu danych. Wszystkie dołączone klienty odczytują go stamtąd, gdy łączą się z lokalnym serwerem: CLI, podgląd, hooki w `plugin/scripts`, serwer MCP i shim `@agentmemory/mcp`, konfiguracje tworzone przez `agentmemory connect` oraz dołączone integracje OpenCode, Pi, OpenClaw, Hermes i obserwator systemu plików. Zapisany sekret jest wysyłany tylko na adresy loopback (`localhost`, `127.0.0.0/8`, `::1`). Jawnie ustawione `AGENTMEMORY_SECRET` ma zawsze pierwszeństwo, a zdalni klienci muszą mieć sekret w swoim środowisku. Wdrożenia self-hosted Docker oparte na repozytorium i punkty wejścia `deploy/` generują i eksportują własny sekret. Aby ręcznie wywołać API: diff --git a/READMEs/README.pt-BR.md b/READMEs/README.pt-BR.md index 2f60bda7a1..e7b8f7394d 100644 --- a/READMEs/README.pt-BR.md +++ b/READMEs/README.pt-BR.md @@ -653,7 +653,7 @@ npm install && npm run build && npm start ### Windows -agentmemory roda no Windows 10/11. O WSL2 segue o caminho Linux e pode instalar o engine fixado automaticamente. O Windows nativo exige a instalação manual do binário iii-engine v0.25.0 fixado; baixe o arquivo correspondente na [página de releases do iii 0.25.0](https://github.com/iii-hq/iii/releases/tag/iii%2Fv0.25.0), confira o checksum e coloque `iii.exe` no `PATH` (por exemplo, `%USERPROFILE%\.local\bin\iii.exe`). O CLI não extrai o ZIP do Windows automaticamente. +agentmemory roda no Windows 10/11. O WSL2 segue o caminho Linux e pode instalar o engine fixado automaticamente. O Windows nativo exige a instalação manual do binário iii-engine v0.25.1 fixado; baixe o arquivo correspondente na [página de releases do iii 0.25.1](https://github.com/iii-hq/iii/releases/tag/iii%2Fv0.25.1), confira o checksum e coloque `iii.exe` no `PATH` (por exemplo, `%USERPROFILE%\.local\bin\iii.exe`). O CLI não extrai o ZIP do Windows automaticamente. A configuração MCP automatizada no Windows nativo dá suporte apenas a `agentmemory connect copilot-cli`. Para os outros agentes nativos do Windows, use o bloco MCP manual em [Outros agentes](#other-agents). Rode `connect` no WSL somente quando o agente de destino estiver instalado no mesmo ambiente WSL. @@ -1166,7 +1166,7 @@ A referência completa está em [`.env.example`](../.env.example): credenciais,

API

-160 endpoints na porta `3111`. A REST API faz bind em `127.0.0.1` por padrão. Endpoints protegidos exigem `Authorization: Bearer `, e endpoints de mesh sync exigem um `AGENTMEMORY_SECRET` explicitamente definido em ambos os peers. +161 endpoints na porta `3111`. A REST API faz bind em `127.0.0.1` por padrão. Endpoints protegidos exigem `Authorization: Bearer `, e endpoints de mesh sync exigem um `AGENTMEMORY_SECRET` explicitamente definido em ambos os peers. **A autenticação vem ativada por padrão.** Quando `AGENTMEMORY_SECRET` não está definida na shell nem em `~/.agentmemory/.env`, a CLI gera um segredo aleatório na primeira execução em `~/.agentmemory/secret` e grava uma cópia chamada `secret` no diretório de dados selecionado, com permissões `0600`. Todos os clientes empacotados o leem de lá ao falar com um servidor local: CLI, viewer, hooks em `plugin/scripts`, servidor MCP e shim `@agentmemory/mcp`, configurações escritas por `agentmemory connect` e integrações empacotadas de OpenCode, Pi, OpenClaw, Hermes e filesystem-watcher. O secret armazenado só é enviado para URLs de loopback (`localhost`, `127.0.0.0/8`, `::1`). Um `AGENTMEMORY_SECRET` explícito sempre prevalece, e clientes remotos precisam ter o secret definido em seu ambiente. Deploys self-hosted com Docker baseados no repositório e os entrypoints de `deploy/` geram e exportam seu próprio secret. Para chamar a API manualmente: diff --git a/READMEs/README.pt-PT.md b/READMEs/README.pt-PT.md index db51d1fc34..9248ab0ebd 100644 --- a/READMEs/README.pt-PT.md +++ b/READMEs/README.pt-PT.md @@ -655,7 +655,7 @@ npm install && npm run build && npm start ### Windows -O agentmemory funciona no Windows 10/11. O WSL2 segue o caminho Linux e pode instalar automaticamente o engine fixado. O Windows nativo exige que o binário iii-engine v0.25.0 fixado seja instalado manualmente; descarregue o ficheiro correspondente na [página de lançamento iii v0.25.0](https://github.com/iii-hq/iii/releases/tag/iii%2Fv0.25.0), verifique o checksum e coloque `iii.exe` no `PATH` (por exemplo, `%USERPROFILE%\.local\bin\iii.exe`). O CLI não extrai automaticamente o ZIP do Windows. +O agentmemory funciona no Windows 10/11. O WSL2 segue o caminho Linux e pode instalar automaticamente o engine fixado. O Windows nativo exige que o binário iii-engine v0.25.1 fixado seja instalado manualmente; descarregue o ficheiro correspondente na [página de lançamento iii v0.25.1](https://github.com/iii-hq/iii/releases/tag/iii%2Fv0.25.1), verifique o checksum e coloque `iii.exe` no `PATH` (por exemplo, `%USERPROFILE%\.local\bin\iii.exe`). O CLI não extrai automaticamente o ZIP do Windows. A ligação MCP automatizada no Windows nativo só suporta `agentmemory connect copilot-cli`. Para os restantes agentes nativos do Windows, use o bloco MCP manual de [Outros agentes](#other-agents). Execute `connect` no WSL apenas quando o agente de destino também estiver instalado nesse mesmo ambiente WSL. @@ -1162,7 +1162,7 @@ A referência completa está em [`.env.example`](../.env.example): credenciais,

API

-160 endpoints na porta `3111`. A REST API faz bind em `127.0.0.1` por predefinição. Os endpoints protegidos exigem `Authorization: Bearer `, e os endpoints de sincronização mesh exigem uma `AGENTMEMORY_SECRET` explicitamente definida em ambos os pares. +161 endpoints na porta `3111`. A REST API faz bind em `127.0.0.1` por predefinição. Os endpoints protegidos exigem `Authorization: Bearer `, e os endpoints de sincronização mesh exigem uma `AGENTMEMORY_SECRET` explicitamente definida em ambos os pares. **A autenticação está ativa por predefinição.** Quando `AGENTMEMORY_SECRET` não está definida na shell nem em `~/.agentmemory/.env`, o CLI gera um segredo aleatório no primeiro arranque em `~/.agentmemory/secret` e guarda uma cópia chamada `secret` no diretório de dados selecionado, com permissões `0600`. Todos os clientes incluídos o leem daí quando comunicam com um servidor local: o CLI, o visualizador, os hooks em `plugin/scripts`, o servidor MCP e o shim `@agentmemory/mcp`, as configurações escritas por `agentmemory connect`, e as integrações incluídas de OpenCode, Pi, OpenClaw, Hermes e filesystem-watcher. O segredo guardado só é enviado para URLs de loopback (`localhost`, `127.0.0.0/8`, `::1`). Uma `AGENTMEMORY_SECRET` explícita prevalece sempre, e os clientes remotos precisam de ter o segredo definido no seu ambiente. As implementações Docker self-host a partir do repositório e os entrypoints de `deploy/` geram e exportam o seu próprio segredo. Para chamar a API manualmente: diff --git a/READMEs/README.ro-RO.md b/READMEs/README.ro-RO.md index 61b5e7be6c..9a8dd48e4f 100644 --- a/READMEs/README.ro-RO.md +++ b/READMEs/README.ro-RO.md @@ -663,7 +663,7 @@ npm install && npm run build && npm start ### Windows -agentmemory rulează pe Windows 10/11. WSL2 urmează calea Linux și poate instala automat engine-ul fixat. Windows nativ necesită instalarea manuală a binarului iii-engine v0.25.0 fixat; descarcă arhiva potrivită de pe [pagina de release iii v0.25.0](https://github.com/iii-hq/iii/releases/tag/iii%2Fv0.25.0), verifică checksum-ul și pune `iii.exe` în `PATH` (de exemplu `%USERPROFILE%\.local\bin\iii.exe`). CLI-ul nu extrage automat arhiva ZIP pentru Windows. +agentmemory rulează pe Windows 10/11. WSL2 urmează calea Linux și poate instala automat engine-ul fixat. Windows nativ necesită instalarea manuală a binarului iii-engine v0.25.1 fixat; descarcă arhiva potrivită de pe [pagina de release iii v0.25.1](https://github.com/iii-hq/iii/releases/tag/iii%2Fv0.25.1), verifică checksum-ul și pune `iii.exe` în `PATH` (de exemplu `%USERPROFILE%\.local\bin\iii.exe`). CLI-ul nu extrage automat arhiva ZIP pentru Windows. Conectarea MCP automatizată pe Windows nativ acceptă doar `agentmemory connect copilot-cli`. Pentru ceilalți agenți nativi Windows, folosește blocul MCP manual din [Alți agenți](#other-agents). Rulează `connect` în WSL doar când agentul țintă este instalat în același mediu WSL. @@ -1154,7 +1154,7 @@ Referința completă [`.env.example`](../.env.example) include furnizori, filtre

API

-160 de endpoint-uri pe portul `3111`. API-ul REST se leagă implicit la `127.0.0.1`. Endpoint-urile protejate necesită `Authorization: Bearer `, iar endpoint-urile de sincronizare mesh necesită un `AGENTMEMORY_SECRET` setat explicit pe ambii parteneri (peers). +161 de endpoint-uri pe portul `3111`. API-ul REST se leagă implicit la `127.0.0.1`. Endpoint-urile protejate necesită `Authorization: Bearer `, iar endpoint-urile de sincronizare mesh necesită un `AGENTMEMORY_SECRET` setat explicit pe ambii parteneri (peers). **Autentificarea este activată implicit.** Dacă `AGENTMEMORY_SECRET` nu este setat, CLI-ul generează la prima pornire un secret aleatoriu în `~/.agentmemory/secret`, cu permisiuni `0600`, și îl copiază și în directorul de date selectat. Toți clienții incluși îl citesc de acolo când comunică cu un server local: CLI-ul, vizualizatorul, hook-urile din `plugin/scripts`, serverul MCP și shim-ul `@agentmemory/mcp`, configurațiile scrise de `agentmemory connect` și integrările incluse OpenCode, Pi, OpenClaw, Hermes și filesystem-watcher. Secretul stocat este trimis doar către URL-uri loopback (`localhost`, `127.0.0.0/8`, `::1`). Un `AGENTMEMORY_SECRET` explicit are întotdeauna prioritate, iar clienții la distanță trebuie să seteze secretul în mediul lor. Implementările Docker self-host pornite din repository și entrypoint-urile `deploy/` își generează și exportă propriul secret. Pentru a apela API-ul manual: diff --git a/READMEs/README.ru-RU.md b/READMEs/README.ru-RU.md index 8059674f2c..a87ea0a92b 100644 --- a/READMEs/README.ru-RU.md +++ b/READMEs/README.ru-RU.md @@ -664,7 +664,7 @@ npm install && npm run build && npm start ### Windows -agentmemory работает на Windows 10/11. В WSL2 используется путь Linux, и закреплённый движок устанавливается автоматически. Для нативной работы в Windows требуется вручную установить закреплённый бинарь iii-engine v0.25.0: скачайте подходящий архив со [страницы релиза iii v0.25.0](https://github.com/iii-hq/iii/releases/tag/iii%2Fv0.25.0), проверьте контрольную сумму и поместите `iii.exe` в `PATH` (например, `%USERPROFILE%\.local\bin\iii.exe`). CLI не распаковывает Windows ZIP автоматически. +agentmemory работает на Windows 10/11. В WSL2 используется путь Linux, и закреплённый движок устанавливается автоматически. Для нативной работы в Windows требуется вручную установить закреплённый бинарь iii-engine v0.25.1: скачайте подходящий архив со [страницы релиза iii v0.25.1](https://github.com/iii-hq/iii/releases/tag/iii%2Fv0.25.1), проверьте контрольную сумму и поместите `iii.exe` в `PATH` (например, `%USERPROFILE%\.local\bin\iii.exe`). CLI не распаковывает Windows ZIP автоматически. Автоматизированное подключение MCP в нативной Windows поддерживает только `agentmemory connect copilot-cli`. Для других нативных Windows-агентов используйте ручной блок MCP из раздела [Другие агенты](#other-agents). Запускайте `connect` внутри WSL только если целевой агент установлен в той же среде WSL. @@ -1160,7 +1160,7 @@ EMBEDDING_PROVIDER=local

API

-160 эндпоинтов на порту `3111`. REST API по умолчанию слушает на `127.0.0.1`. Защищённые эндпоинты требуют `Authorization: Bearer `, а эндпоинты mesh-синхронизации требуют явно заданного `AGENTMEMORY_SECRET` на обоих узлах. +161 эндпоинтов на порту `3111`. REST API по умолчанию слушает на `127.0.0.1`. Защищённые эндпоинты требуют `Authorization: Bearer `, а эндпоинты mesh-синхронизации требуют явно заданного `AGENTMEMORY_SECRET` на обоих узлах. **Аутентификация включена по умолчанию.** Если `AGENTMEMORY_SECRET` не задан, при первом запуске CLI создаёт случайный секрет в `~/.agentmemory/secret` с правами `0600` и копирует его также в выбранный каталог данных. Каждый встроенный клиент читает его оттуда, когда говорит с локальным сервером: CLI, просмотрщик, хуки из `plugin/scripts`, MCP-сервер и shim `@agentmemory/mcp`, конфиги, записанные `agentmemory connect`, и встроенные интеграции OpenCode, Pi, OpenClaw, Hermes и filesystem-watcher. Сохранённый секрет отправляется только на loopback-адреса (`localhost`, `127.0.0.0/8`, `::1`). Явно заданный `AGENTMEMORY_SECRET` всегда выигрывает, и удалённым клиентам он всё равно нужен. Docker и entrypoint'ы `deploy/` уже генерируют и экспортируют собственный секрет. Чтобы вызвать API вручную: diff --git a/READMEs/README.sv-SE.md b/READMEs/README.sv-SE.md index aad166cfeb..d1e7451f8e 100644 --- a/READMEs/README.sv-SE.md +++ b/READMEs/README.sv-SE.md @@ -656,7 +656,7 @@ npm install && npm run build && npm start ### Windows -agentmemory körs på Windows 10/11. WSL2 följer Linux-vägen och kan installera den fastlåsta enginen automatiskt. Native Windows kräver att den fastlåsta iii-engine v0.25.0-binären installeras manuellt: ladda ner rätt arkiv från [iii v0.25.0-utgåvan](https://github.com/iii-hq/iii/releases/tag/iii%2Fv0.25.0), verifiera kontrollsumman och placera `iii.exe` på `PATH` (till exempel `%USERPROFILE%\.local\bin\iii.exe`). CLI:n packar inte upp Windows-ZIP-filen automatiskt. +agentmemory körs på Windows 10/11. WSL2 följer Linux-vägen och kan installera den fastlåsta enginen automatiskt. Native Windows kräver att den fastlåsta iii-engine v0.25.1-binären installeras manuellt: ladda ner rätt arkiv från [iii v0.25.1-utgåvan](https://github.com/iii-hq/iii/releases/tag/iii%2Fv0.25.1), verifiera kontrollsumman och placera `iii.exe` på `PATH` (till exempel `%USERPROFILE%\.local\bin\iii.exe`). CLI:n packar inte upp Windows-ZIP-filen automatiskt. Automatiserad MCP-inkoppling på native Windows stöder bara `agentmemory connect copilot-cli`. För andra native Windows-agenter använder du det manuella MCP-blocket från [Andra agenter](#other-agents). Kör `connect` i WSL endast om målagenten är installerad i samma WSL-miljö. @@ -1168,7 +1168,7 @@ Den fullständiga referensen finns i [`.env.example`](../.env.example): autentis

API

-160 endpoints på port `3111`. REST API:et binder till `127.0.0.1` som standard. Skyddade endpoints kräver `Authorization: Bearer `, och mesh-synk-endpoints kräver en explicit satt `AGENTMEMORY_SECRET` på båda parterna. +161 endpoints på port `3111`. REST API:et binder till `127.0.0.1` som standard. Skyddade endpoints kräver `Authorization: Bearer `, och mesh-synk-endpoints kräver en explicit satt `AGENTMEMORY_SECRET` på båda parterna. **Autentisering är aktiverad som standard.** Om `AGENTMEMORY_SECRET` inte är angiven i skalet eller i `~/.agentmemory/.env` skapar CLI:n en slumpmässig hemlighet vid första starten i `~/.agentmemory/secret` och en kopia med namnet `secret` i den valda datamappen, med filbehörigheten `0600`. Alla medföljande klienter läser den därifrån när de ansluter till en lokal server: CLI:n, viewern, hookarna under `plugin/scripts`, MCP-servern och `@agentmemory/mcp`-shimmen, konfigurationerna skrivna av `agentmemory connect` samt de medföljande integrationerna för OpenCode, Pi, OpenClaw, Hermes och filsystemsbevakaren. Den lagrade hemligheten skickas bara till loopback-URL:er (`localhost`, `127.0.0.0/8`, `::1`). En explicit `AGENTMEMORY_SECRET` har alltid företräde; fjärrklienter behöver hemligheten i sin miljö. Docker-baserad self-hosting från repot och startpunkterna i `deploy/` genererar och exporterar en egen hemlighet. För att anropa API:et för hand: diff --git a/READMEs/README.th-TH.md b/READMEs/README.th-TH.md index 89d3fe06e1..9a9e276e45 100644 --- a/READMEs/README.th-TH.md +++ b/READMEs/README.th-TH.md @@ -648,7 +648,7 @@ npm install && npm run build && npm start ### Windows -agentmemory รองรับ Windows 10/11; WSL2 ใช้ขั้นตอน Linux และติดตั้ง engine อัตโนมัติได้ Windows แบบ native ต้องติดตั้งไบนารี iii-engine v0.25.0 ด้วยตนเอง ดาวน์โหลด archive จาก [หน้า release iii v0.25.0](https://github.com/iii-hq/iii/releases/tag/iii%2Fv0.25.0), ตรวจ checksum และวาง `iii.exe` บน `PATH` (เช่น `%USERPROFILE%\.local\bin\iii.exe`) CLI ไม่แตก Windows ZIP ให้อัตโนมัติ +agentmemory รองรับ Windows 10/11; WSL2 ใช้ขั้นตอน Linux และติดตั้ง engine อัตโนมัติได้ Windows แบบ native ต้องติดตั้งไบนารี iii-engine v0.25.1 ด้วยตนเอง ดาวน์โหลด archive จาก [หน้า release iii v0.25.1](https://github.com/iii-hq/iii/releases/tag/iii%2Fv0.25.1), ตรวจ checksum และวาง `iii.exe` บน `PATH` (เช่น `%USERPROFILE%\.local\bin\iii.exe`) CLI ไม่แตก Windows ZIP ให้อัตโนมัติ การตั้งค่า MCP อัตโนมัติบน Windows รองรับเฉพาะ `agentmemory connect copilot-cli`; สำหรับ agent อื่นให้ใช้ MCP block ใน [Other agents](#other-agents) เรียก `connect` ใน WSL เมื่อ agent เป้าหมายติดตั้งใน WSL เดียวกันเท่านั้น @@ -1161,7 +1161,7 @@ EMBEDDING_PROVIDER=local

API

-160 endpoint บนพอร์ต `3111` REST API จะ bind กับ `127.0.0.1` โดย default endpoint ที่ป้องกันไว้ต้องมี `Authorization: Bearer ` และ endpoint ของ mesh sync ต้องตั้งค่า `AGENTMEMORY_SECRET` ไว้อย่างชัดเจนทั้งสองฝั่ง +161 endpoint บนพอร์ต `3111` REST API จะ bind กับ `127.0.0.1` โดย default endpoint ที่ป้องกันไว้ต้องมี `Authorization: Bearer ` และ endpoint ของ mesh sync ต้องตั้งค่า `AGENTMEMORY_SECRET` ไว้อย่างชัดเจนทั้งสองฝั่ง เปิดการยืนยันตัวตนตามค่าเริ่มต้น หากไม่ตั้ง `AGENTMEMORY_SECRET` CLI จะสร้างรหัสลับสุ่มครั้งแรกที่ `~/.agentmemory/secret` และคัดลอกไปยัง `secret` ในไดเรกทอรีข้อมูลที่เลือก โดยใช้สิทธิ์ `0600` ไคลเอนต์ที่รวมมาจะอ่านไฟล์หลักนี้ก่อนเมื่อเชื่อมต่อในเครื่อง และส่งรหัสลับที่บันทึกไว้เฉพาะ URL แบบ loopback ค่า `AGENTMEMORY_SECRET` ที่ระบุเองมีลำดับความสำคัญสูงกว่า ไคลเอนต์ระยะไกลต้องตั้งผ่าน environment ส่วน Docker สร้างรหัสลับของตนเอง diff --git a/READMEs/README.tl-PH.md b/READMEs/README.tl-PH.md index 1be8927702..37b608f0cc 100644 --- a/READMEs/README.tl-PH.md +++ b/READMEs/README.tl-PH.md @@ -656,7 +656,7 @@ Pinapatakbo ng `npm start` ang kaparehong local server ng naka-install na CLI. N ### Windows -Tumatakbo ang agentmemory sa Windows 10/11. Awtomatikong nag-i-install ng engine ang WSL2 gamit ang Linux path. Sa native Windows, manu-manong i-install ang pinned na iii-engine v0.25.0: i-download ang archive mula sa [iii v0.25.0 release page](https://github.com/iii-hq/iii/releases/tag/iii%2Fv0.25.0), tiyakin ang checksum, at ilagay ang `iii.exe` sa `PATH`. Hindi awtomatikong nag-e-extract ng ZIP ang CLI +Tumatakbo ang agentmemory sa Windows 10/11. Awtomatikong nag-i-install ng engine ang WSL2 gamit ang Linux path. Sa native Windows, manu-manong i-install ang pinned na iii-engine v0.25.1: i-download ang archive mula sa [iii v0.25.1 release page](https://github.com/iii-hq/iii/releases/tag/iii%2Fv0.25.1), tiyakin ang checksum, at ilagay ang `iii.exe` sa `PATH`. Hindi awtomatikong nag-e-extract ng ZIP ang CLI Native Windows MCP automation ay para lamang sa `agentmemory connect copilot-cli`; gamitin ang manual MCP block sa [Other agents](#other-agents) para sa iba. Sa WSL, patakbuhin ang `connect` lamang kung naroon din ang target agent @@ -1170,7 +1170,7 @@ Nasa [`.env.example`](../.env.example) ang buong reference, kasama ang provider,

API

-160 na endpoint sa port `3111`. Nagbibind ang REST API sa `127.0.0.1` bilang default. Kinakailangan ng mga protected endpoint ang `Authorization: Bearer `, at kinakailangan ng mga mesh sync endpoint ang tahasang naka-set na `AGENTMEMORY_SECRET` sa dalawang peer. +161 na endpoint sa port `3111`. Nagbibind ang REST API sa `127.0.0.1` bilang default. Kinakailangan ng mga protected endpoint ang `Authorization: Bearer `, at kinakailangan ng mga mesh sync endpoint ang tahasang naka-set na `AGENTMEMORY_SECRET` sa dalawang peer. Naka-on ang authentication bilang default. Kapag walang `AGENTMEMORY_SECRET`, gumagawa ang CLI ng random secret sa unang start sa `~/.agentmemory/secret` at kinokopya sa `secret` ng napiling data directory, na may mode `0600`. Inuuna ng bundled clients ang canonical file sa local connection at ipinapadala lang ang naka-save na secret sa loopback URL. Mas mataas ang prayoridad ng explicit na `AGENTMEMORY_SECRET`; dapat itong itakda ng remote clients sa environment. Gumagawa ang Docker deployment ng sarili nitong secret. diff --git a/READMEs/README.tr-TR.md b/READMEs/README.tr-TR.md index 16210031f6..9f3cb7a170 100644 --- a/READMEs/README.tr-TR.md +++ b/READMEs/README.tr-TR.md @@ -663,7 +663,7 @@ npm install && npm run build && npm start ### Windows -agentmemory Windows 10/11’de çalışır. WSL2 Linux yolunu izleyerek engine’i otomatik kurabilir. Yerel Windows’ta iii-engine v0.25.0 elle kurulmalıdır: arşivi [iii v0.25.0 sürüm sayfasından](https://github.com/iii-hq/iii/releases/tag/iii%2Fv0.25.0) indirin, checksum’u doğrulayın ve `iii.exe` dosyasını `PATH` üzerine koyun. CLI Windows ZIP’ini otomatik açmaz +agentmemory Windows 10/11’de çalışır. WSL2 Linux yolunu izleyerek engine’i otomatik kurabilir. Yerel Windows’ta iii-engine v0.25.1 elle kurulmalıdır: arşivi [iii v0.25.1 sürüm sayfasından](https://github.com/iii-hq/iii/releases/tag/iii%2Fv0.25.1) indirin, checksum’u doğrulayın ve `iii.exe` dosyasını `PATH` üzerine koyun. CLI Windows ZIP’ini otomatik açmaz Yerel Windows’ta otomatik MCP yapılandırması yalnızca `agentmemory connect copilot-cli` için vardır; diğerleri için [Other agents](#other-agents) bölümünü kullanın. WSL’de `connect` yalnızca hedef agent aynı WSL ortamındaysa çalıştırılmalıdır @@ -1159,7 +1159,7 @@ Tam [`.env.example`](../.env.example) sağlayıcıları, kayıt filtrelerini, sa

API

-`3111` portunda 160 endpoint. REST API, varsayılan olarak `127.0.0.1`'e bağlanır. Korumalı endpoint'ler `Authorization: Bearer ` gerektirir ve mesh sync endpoint'leri her iki peer'da da açıkça ayarlanmış bir `AGENTMEMORY_SECRET` gerektirir. +`3111` portunda 161 endpoint. REST API, varsayılan olarak `127.0.0.1`'e bağlanır. Korumalı endpoint'ler `Authorization: Bearer ` gerektirir ve mesh sync endpoint'leri her iki peer'da da açıkça ayarlanmış bir `AGENTMEMORY_SECRET` gerektirir. **Kimlik doğrulama varsayılan olarak açıktır.** `AGENTMEMORY_SECRET` ayarlanmamışsa CLI ilk başlatmada `~/.agentmemory/secret` dosyasında `0600` izinleriyle rastgele bir sır oluşturur ve seçilen veri dizinine de kopyalar. Paketli istemciler yerel sunucuya bağlanırken buradan okur; secret yalnız loopback URL’lerine (`localhost`, `127.0.0.0/8`, `::1`) gönderilir. Uzak istemciler secret’ı kendi ortamında sağlamalıdır. Docker self-host ve `deploy/` kendi secret’larını üretir. API’yi elle çağırmak için: diff --git a/READMEs/README.uk-UA.md b/READMEs/README.uk-UA.md index cd76b1d0c0..ace84b9baa 100644 --- a/READMEs/README.uk-UA.md +++ b/READMEs/README.uk-UA.md @@ -663,7 +663,7 @@ npm install && npm run build && npm start ### Windows -agentmemory працює у Windows 10/11; WSL2 може автоматично встановити engine. Нативна Windows потребує ручного встановлення iii-engine v0.25.0: завантажте архів зі [сторінки iii v0.25.0](https://github.com/iii-hq/iii/releases/tag/iii%2Fv0.25.0), перевірте checksum і розмістіть `iii.exe` у `PATH`. CLI не розпаковує ZIP автоматично +agentmemory працює у Windows 10/11; WSL2 може автоматично встановити engine. Нативна Windows потребує ручного встановлення iii-engine v0.25.1: завантажте архів зі [сторінки iii v0.25.1](https://github.com/iii-hq/iii/releases/tag/iii%2Fv0.25.1), перевірте checksum і розмістіть `iii.exe` у `PATH`. CLI не розпаковує ZIP автоматично Автоматичне налаштування MCP у нативній Windows підтримує лише `agentmemory connect copilot-cli`; для інших скористайтеся [Other agents](#other-agents). У WSL виконуйте `connect`, лише якщо агент встановлений у тому самому середовищі @@ -1160,7 +1160,7 @@ EMBEDDING_PROVIDER=local

API

-160 ендпоінтів на порту `3111`. REST API за замовчуванням прив'язується до `127.0.0.1`. Захищені ендпоінти вимагають `Authorization: Bearer `, а ендпоінти синхронізації mesh вимагають явно встановленого `AGENTMEMORY_SECRET` на обох сторонах. +161 ендпоінтів на порту `3111`. REST API за замовчуванням прив'язується до `127.0.0.1`. Захищені ендпоінти вимагають `Authorization: Bearer `, а ендпоінти синхронізації mesh вимагають явно встановленого `AGENTMEMORY_SECRET` на обох сторонах. **Автентифікацію ввімкнено типово.** Якщо `AGENTMEMORY_SECRET` не задано, під час першого запуску CLI створює випадковий секрет у `~/.agentmemory/secret` з правами `0600` і копіює його також у вибраний каталог даних. Вбудовані клієнти читають його звідти для локального сервера; secret надсилається лише на loopback URL (`localhost`, `127.0.0.0/8`, `::1`). Віддалені клієнти мають отримати secret через власне середовище. Docker self-host і `deploy/` створюють власні секрети. Щоб викликати API вручну: diff --git a/READMEs/README.ur-PK.md b/READMEs/README.ur-PK.md index 2e204782a2..48b683f324 100644 --- a/READMEs/README.ur-PK.md +++ b/READMEs/README.ur-PK.md @@ -658,7 +658,7 @@ npm install && npm run build && npm start ### Windows -agentmemory Windows 10/11 پر چلتا ہے؛ WSL2 Linux طریقے سے engine خود انسٹال کر سکتا ہے۔ Native Windows پر iii-engine v0.25.0 binary دستی طور پر انسٹال کریں: [iii v0.25.0 release page](https://github.com/iii-hq/iii/releases/tag/iii%2Fv0.25.0) سے archive ڈاؤن لوڈ، checksum کی تصدیق، اور `iii.exe` کو `PATH` پر رکھیں۔ CLI ZIP خود نہیں کھولتا +agentmemory Windows 10/11 پر چلتا ہے؛ WSL2 Linux طریقے سے engine خود انسٹال کر سکتا ہے۔ Native Windows پر iii-engine v0.25.1 binary دستی طور پر انسٹال کریں: [iii v0.25.1 release page](https://github.com/iii-hq/iii/releases/tag/iii%2Fv0.25.1) سے archive ڈاؤن لوڈ، checksum کی تصدیق، اور `iii.exe` کو `PATH` پر رکھیں۔ CLI ZIP خود نہیں کھولتا Native Windows پر MCP خودکار wiring صرف `agentmemory connect copilot-cli` کے لیے ہے؛ دوسرے agents کے لیے [Other agents](#other-agents) کا دستی MCP block استعمال کریں۔ WSL میں `connect` تب چلائیں جب target agent وہیں نصب ہو @@ -1165,7 +1165,7 @@ EMBEDDING_PROVIDER=local

API

-پورٹ `3111` پر 160 اینڈ پوائنٹس۔ REST API ڈیفالٹ کے طور پر `127.0.0.1` سے bind ہوتا ہے۔ محفوظ اینڈ پوائنٹس کو `Authorization: Bearer ` درکار ہے، اور mesh sync اینڈ پوائنٹس کو دونوں peers پر واضح طور پر سیٹ کیا گیا `AGENTMEMORY_SECRET` درکار ہے۔ +پورٹ `3111` پر 161 اینڈ پوائنٹس۔ REST API ڈیفالٹ کے طور پر `127.0.0.1` سے bind ہوتا ہے۔ محفوظ اینڈ پوائنٹس کو `Authorization: Bearer ` درکار ہے، اور mesh sync اینڈ پوائنٹس کو دونوں peers پر واضح طور پر سیٹ کیا گیا `AGENTMEMORY_SECRET` درکار ہے۔ توثیق بطور طے شدہ فعال ہے۔ `AGENTMEMORY_SECRET` مقرر نہ ہو تو CLI پہلی بار بے ترتیب راز `~/.agentmemory/secret` میں محفوظ کرتا ہے اور منتخب ڈیٹا ڈائریکٹری کے `secret` میں اس کی نقل رکھتا ہے؛ اجازت `0600` ہوتی ہے۔ ساتھ آنے والے کلائنٹس مقامی رابطے میں اس اصل فائل کو ترجیح دیتے ہیں اور محفوظ راز صرف loopback URL کو بھیجتے ہیں۔ واضح `AGENTMEMORY_SECRET` مقدم ہے؛ دور کے کلائنٹس اسے environment میں مقرر کریں۔ Docker اپنا راز بناتا ہے۔ diff --git a/READMEs/README.vi-VN.md b/READMEs/README.vi-VN.md index 6ed5a261d8..94e52b1f8a 100644 --- a/READMEs/README.vi-VN.md +++ b/READMEs/README.vi-VN.md @@ -101,7 +101,7 @@ Yêu cầu: - Node.js 20 hoặc mới hơn kèm npm và npx (`node -v`, `npm -v`, và `npx -v`). - Thiết lập tự động trên macOS/Linux cần `curl`, POSIX `sh` và `tar`. -Windows native cần cài thủ công `iii.exe` của iii v0.25.0 đã pin; WSL2 dùng trình cài tự động cho macOS/Linux +Windows native cần cài thủ công `iii.exe` của iii v0.25.1 đã pin; WSL2 dùng trình cài tự động cho macOS/Linux Lệnh cài mới chuẩn (canonical): @@ -656,7 +656,7 @@ npm install && npm run build && npm start ### Windows -agentmemory chạy trên Windows 10/11; WSL2 có thể tự cài engine theo quy trình Linux. Windows native cần cài thủ công iii-engine v0.25.0: tải archive từ [trang phát hành iii v0.25.0](https://github.com/iii-hq/iii/releases/tag/iii%2Fv0.25.0), xác minh checksum và đặt `iii.exe` trong `PATH`. CLI không tự giải nén ZIP Windows +agentmemory chạy trên Windows 10/11; WSL2 có thể tự cài engine theo quy trình Linux. Windows native cần cài thủ công iii-engine v0.25.1: tải archive từ [trang phát hành iii v0.25.1](https://github.com/iii-hq/iii/releases/tag/iii%2Fv0.25.1), xác minh checksum và đặt `iii.exe` trong `PATH`. CLI không tự giải nén ZIP Windows Cấu hình MCP tự động trên Windows native chỉ hỗ trợ `agentmemory connect copilot-cli`; agent khác dùng khối MCP thủ công trong [Other agents](#other-agents). Chỉ chạy `connect` trong WSL nếu agent đích cũng cài trong cùng môi trường WSL @@ -1170,7 +1170,7 @@ Tham chiếu đầy đủ ở [`.env.example`](../.env.example), gồm nhà cung

API

-160 endpoint trên port `3111`. REST API bind vào `127.0.0.1` theo mặc định. Các endpoint được bảo vệ cần `Authorization: Bearer `, và các endpoint mesh sync cần `AGENTMEMORY_SECRET` được đặt rõ ràng trên cả hai phía. +161 endpoint trên port `3111`. REST API bind vào `127.0.0.1` theo mặc định. Các endpoint được bảo vệ cần `Authorization: Bearer `, và các endpoint mesh sync cần `AGENTMEMORY_SECRET` được đặt rõ ràng trên cả hai phía. Xác thực bật mặc định. Nếu chưa đặt `AGENTMEMORY_SECRET`, CLI tạo bí mật ngẫu nhiên lần đầu tại `~/.agentmemory/secret` và sao chép vào `secret` trong thư mục dữ liệu đã chọn, với quyền `0600`. Client đi kèm ưu tiên tệp chính này khi kết nối cục bộ và chỉ gửi bí mật đã lưu đến URL loopback. `AGENTMEMORY_SECRET` đặt tường minh luôn được ưu tiên; client từ xa phải đặt qua biến môi trường. Triển khai Docker tạo bí mật riêng. diff --git a/READMEs/README.zh-CN.md b/READMEs/README.zh-CN.md index db8a614f9e..b68f1e9e95 100644 --- a/READMEs/README.zh-CN.md +++ b/READMEs/README.zh-CN.md @@ -656,7 +656,7 @@ npm install && npm run build && npm start ### Windows -agentmemory 支持 Windows 10/11;WSL2 可按 Linux 流程自动安装 engine。原生 Windows 需手动安装 iii-engine v0.25.0:从 [iii v0.25.0 发布页](https://github.com/iii-hq/iii/releases/tag/iii%2Fv0.25.0) 下载对应归档、验证校验和,并将 `iii.exe` 放到 `PATH`。CLI 不会自动解压 ZIP +agentmemory 支持 Windows 10/11;WSL2 可按 Linux 流程自动安装 engine。原生 Windows 需手动安装 iii-engine v0.25.1:从 [iii v0.25.1 发布页](https://github.com/iii-hq/iii/releases/tag/iii%2Fv0.25.1) 下载对应归档、验证校验和,并将 `iii.exe` 放到 `PATH`。CLI 不会自动解压 ZIP 原生 Windows 仅为 `agentmemory connect copilot-cli` 提供自动 MCP 配置;其他 agent 请使用 [Other agents](#other-agents) 中的手动配置。仅当目标 agent 也安装在同一 WSL 环境时,才在 WSL 中运行 `connect` @@ -1171,7 +1171,7 @@ EMBEDDING_PROVIDER=local

API

-端口 `3111` 上有 160 个端点。REST API 默认绑定到 `127.0.0.1`。受保护的端点需要 `Authorization: Bearer `,mesh 同步端点则需要双方都显式设置 `AGENTMEMORY_SECRET`。 +端口 `3111` 上有 161 个端点。REST API 默认绑定到 `127.0.0.1`。受保护的端点需要 `Authorization: Bearer `,mesh 同步端点则需要双方都显式设置 `AGENTMEMORY_SECRET`。 认证默认开启。未设置 `AGENTMEMORY_SECRET` 时,CLI 首次启动会将随机密钥保存到 `~/.agentmemory/secret`,并镜像到所选数据目录的 `secret`,权限为 `0600`。内置客户端连接本地服务器时优先读取该公共文件,只向回环 URL 发送保存的密钥。显式设置的 `AGENTMEMORY_SECRET` 优先;远程客户端需通过环境变量设置密钥。Docker 部署生成自己的密钥。 diff --git a/READMEs/README.zh-TW.md b/READMEs/README.zh-TW.md index 23e9cf9a06..9088aa07c6 100644 --- a/READMEs/README.zh-TW.md +++ b/READMEs/README.zh-TW.md @@ -654,7 +654,7 @@ npm install && npm run build && npm start ### Windows -agentmemory 可在 Windows 10/11 執行;WSL2 可依 Linux 流程自動安裝 engine。原生 Windows 需手動安裝 iii-engine v0.25.0:從 [iii v0.25.0 發行頁面](https://github.com/iii-hq/iii/releases/tag/iii%2Fv0.25.0) 下載相符封存檔、驗證 checksum,並將 `iii.exe` 放入 `PATH`。CLI 不會自動解壓 ZIP +agentmemory 可在 Windows 10/11 執行;WSL2 可依 Linux 流程自動安裝 engine。原生 Windows 需手動安裝 iii-engine v0.25.1:從 [iii v0.25.1 發行頁面](https://github.com/iii-hq/iii/releases/tag/iii%2Fv0.25.1) 下載相符封存檔、驗證 checksum,並將 `iii.exe` 放入 `PATH`。CLI 不會自動解壓 ZIP 原生 Windows 僅支援 `agentmemory connect copilot-cli` 自動設定 MCP;其他 agent 請使用 [Other agents](#other-agents) 的手動設定。只有目標 agent 也安裝在相同 WSL 環境時,才在 WSL 執行 `connect` @@ -1163,7 +1163,7 @@ EMBEDDING_PROVIDER=local

API

-連接埠 `3111` 上有 160 個端點。REST API 預設綁定 `127.0.0.1`。受保護的端點需要 `Authorization: Bearer `,而網狀同步端點要求兩端都明確設定 `AGENTMEMORY_SECRET`。 +連接埠 `3111` 上有 161 個端點。REST API 預設綁定 `127.0.0.1`。受保護的端點需要 `Authorization: Bearer `,而網狀同步端點要求兩端都明確設定 `AGENTMEMORY_SECRET`。 驗證預設開啟。未設定 `AGENTMEMORY_SECRET` 時,CLI 首次啟動會將隨機密鑰儲存到 `~/.agentmemory/secret`,並鏡像到所選資料目錄的 `secret`,權限為 `0600`。內建客戶端連接本機伺服器時優先讀取此共用檔案,只向回環 URL 傳送儲存的密鑰。明確設定的 `AGENTMEMORY_SECRET` 優先;遠端客戶端需透過環境變數設定密鑰。Docker 部署會產生自己的密鑰。 diff --git a/assets/tags/light/stat-tests.svg b/assets/tags/light/stat-tests.svg index 2e4f03ca44..238a6f2683 100644 --- a/assets/tags/light/stat-tests.svg +++ b/assets/tags/light/stat-tests.svg @@ -1,5 +1,5 @@ - + - 6300+ + 7000+ TESTS PASSING diff --git a/assets/tags/stat-tests.svg b/assets/tags/stat-tests.svg index 94ef4cf9c3..0c6b58a33f 100644 --- a/assets/tags/stat-tests.svg +++ b/assets/tags/stat-tests.svg @@ -1,5 +1,5 @@ - + - 6300+ + 7000+ TESTS PASSING diff --git a/deploy/Dockerfile b/deploy/Dockerfile index 4cfcc37d27..fcf6836419 100644 --- a/deploy/Dockerfile +++ b/deploy/Dockerfile @@ -1,8 +1,20 @@ # Shared build for local Compose and every deployment platform. # Update these four pins together when promoting the runtime. -ARG III_VERSION=0.25.0 -ARG STATE_VERSION=0.22.20 -ARG HTTP_VERSION=0.22.7 +ARG III_VERSION=0.25.1 +ARG STATE_VERSION=0.22.22 +ARG HTTP_VERSION=0.22.8 + +FROM node:24-bookworm-slim AS state-node +FROM rust:1.93-bookworm AS state-build +COPY --from=state-node /usr/local /usr/local +RUN apt-get update && apt-get install -y --no-install-recommends python3 git ca-certificates && rm -rf /var/lib/apt/lists/* +RUN npm install -g pnpm@10.32.1 +WORKDIR /build +COPY deploy/state-worker ./deploy/state-worker +COPY scripts/state-worker ./scripts/state-worker +RUN git init /workers && git -C /workers remote add origin https://github.com/iii-hq/workers \ + && git -C /workers fetch --depth=1 origin a8ed4889f54cb34cb9ffdb6e44190484d90ba950 \ + && python3 scripts/state-worker/build.py --workers-source /workers --output /artifact --work-dir /state-build --online FROM node:24-slim AS agent-build ARG III_VERSION @@ -27,7 +39,7 @@ RUN apt-get update \ && find /var/lib/apt/lists -mindepth 1 -delete COPY --from=iii-engine /app/iii /usr/local/bin/iii -RUN iii --version 2>&1 | grep -qF "${III_VERSION}" \ +RUN iii --version 2>&1 | tr -s " \t" "\n" | grep -qxF "${III_VERSION}" \ || { echo "iiidev/iii:${III_VERSION} does not contain iii ${III_VERSION}" >&2; exit 1; } WORKDIR /opt/agentmemory @@ -40,6 +52,9 @@ COPY --from=agent-build --chown=node:node /opt/agentmemory/dist ./dist COPY --chown=node:node --chmod=644 deploy/runtime/agentmemory.worker.yaml /opt/agentmemory/iii.worker.yaml COPY --chown=node:node --chmod=644 deploy/runtime/worker-compose.yaml /opt/agentmemory/worker-compose.yaml COPY --chmod=0755 deploy/runtime/entrypoint.sh /usr/local/bin/agentmemory-entrypoint +COPY --from=state-build /artifact /opt/agentmemory-state +COPY deploy/runtime/state-bootstrap.mjs /opt/agentmemory/deploy-runtime/state-bootstrap.mjs +COPY deploy/state-worker/pin.json /opt/agentmemory/deploy-runtime/state-pin.json ENV III_VERSION=${III_VERSION} \ STATE_VERSION=${STATE_VERSION} \ diff --git a/deploy/README.md b/deploy/README.md index 7f6d503d3a..82152811ac 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -1,6 +1,6 @@ # Docker deployments -agentmemory runs on iii 0.25.0 with the `state` 0.22.20 and `http` 0.22.7 registry workers. All image builds use one [`Dockerfile`](Dockerfile), with engine and registry worker pins declared once. Update those build arguments together when promoting a release. Workers are fetched from the iii registry at startup; the `worker-compose.yaml` uses the same Worker Compose format as the CLI. +agentmemory runs on iii 0.25.1 with the `state` 0.22.20 and `http` 0.22.7 registry workers. All image builds use one [`Dockerfile`](Dockerfile), with engine and registry worker pins declared once. Update those build arguments together when promoting a release. Workers are fetched from the iii registry at startup; the `worker-compose.yaml` uses the same Worker Compose format as the CLI. The container runs `iii compose --up`: Compose owns the engine and supervises `state`, `http`, and the local `agentmemory` worker. There is no legacy embedded engine config, standalone stream, pubsub, or queue worker. The registry worker artifacts are pinned in the compose manifest. Engine and worker state, agentmemory state, generated secret, Compose cache, and logs live on the persistent `/data` volume. For Docker Compose, the volume key is `iii-data`, normally stored as `_iii-data`; set `AGENTMEMORY_DATA_DIR` to a host path to use a bind mount instead. The state worker continues to use `/data/state_store.db`, so the existing 0.22.1 file-based store is read in place without conversion or deletion. diff --git a/deploy/coolify/docker-compose.yml b/deploy/coolify/docker-compose.yml index a229cb7692..e35233996f 100644 --- a/deploy/coolify/docker-compose.yml +++ b/deploy/coolify/docker-compose.yml @@ -15,6 +15,7 @@ services: VIEWER_ALLOWED_HOSTS: "${VIEWER_ALLOWED_HOSTS:-localhost:3113,127.0.0.1:3113,[::1]:3113}" VIEWER_ALLOWED_ORIGINS: "${VIEWER_ALLOWED_ORIGINS:-http://localhost:3113,http://127.0.0.1:3113}" EMBEDDING_PROVIDER: "${EMBEDDING_PROVIDER:-}" + AGENTMEMORY_LLM_PROVIDER: "${AGENTMEMORY_LLM_PROVIDER:-}" OPENAI_API_KEY: "${OPENAI_API_KEY:-}" OPENAI_API_KEY_FOR_LLM: "${OPENAI_API_KEY_FOR_LLM:-}" OPENAI_BASE_URL: "${OPENAI_BASE_URL:-}" @@ -59,6 +60,7 @@ services: AGENTMEMORY_STATE_BACKEND: "${AGENTMEMORY_STATE_BACKEND:-}" AGENTMEMORY_REDIS_URL: "${AGENTMEMORY_REDIS_URL:-}" AGENTMEMORY_STATE_SAVE_INTERVAL_MS: "${AGENTMEMORY_STATE_SAVE_INTERVAL_MS:-}" + AGENTMEMORY_LOCAL_DISPATCH: "${AGENTMEMORY_LOCAL_DISPATCH:-}" AGENTMEMORY_SECRET: "${AGENTMEMORY_SECRET:-}" AGENT_ID: "${AGENT_ID:-}" TEAM_ID: "${TEAM_ID:-}" @@ -97,6 +99,7 @@ services: AGENTMEMORY_GRAPH_WEIGHT: "${AGENTMEMORY_GRAPH_WEIGHT:-}" SUMMARIZE_CHUNK_SIZE: "${SUMMARIZE_CHUNK_SIZE:-}" SUMMARIZE_CHUNK_CONCURRENCY: "${SUMMARIZE_CHUNK_CONCURRENCY:-}" + AGENTMEMORY_SUMMARY_TIMEOUT_MS: "${AGENTMEMORY_SUMMARY_TIMEOUT_MS:-}" CONSOLIDATION_DECAY_DAYS: "${CONSOLIDATION_DECAY_DAYS:-}" CONSOLIDATION_INTERVAL_MS: "${CONSOLIDATION_INTERVAL_MS:-}" AGENTMEMORY_CONSOLIDATION_COOLDOWN_MS: "${AGENTMEMORY_CONSOLIDATION_COOLDOWN_MS:-}" diff --git a/deploy/fly/README.md b/deploy/fly/README.md index a8b988eb4e..8c2b3e525d 100644 --- a/deploy/fly/README.md +++ b/deploy/fly/README.md @@ -8,7 +8,7 @@ fly volumes create agentmemory_data --region iad --size 1 --app ``` -The shared `deploy/Dockerfile` build runs iii 0.25.0 Worker Compose. `/data` persists engine configuration, state, secret, registry cache, and logs. REST is served over HTTPS on internal port `3111`; the authenticated viewer uses internal port `3113`, exposed only through the TLS listener on public port `8443`. +The shared `deploy/Dockerfile` build runs iii 0.25.1 Worker Compose. `/data` persists engine configuration, state, secret, registry cache, and logs. REST is served over HTTPS on internal port `3111`; the authenticated viewer uses internal port `3113`, exposed only through the TLS listener on public port `8443`. Set `VIEWER_ALLOWED_HOSTS` to `.fly.dev:8443,localhost:3113` and `VIEWER_ALLOWED_ORIGINS` to `https://.fly.dev:8443` before enabling the viewer. Keep `localhost:3113` for Fly's internal viewer healthcheck. Browse to `https://.fly.dev:8443`; enter the bearer secret when prompted. The API/SSE proxy requires bearer auth. Never expose port `49134`. diff --git a/deploy/railway/README.md b/deploy/railway/README.md index fea48d53d9..160d1598a4 100644 --- a/deploy/railway/README.md +++ b/deploy/railway/README.md @@ -1,6 +1,6 @@ # Railway -Connect the GitHub repository with its root directory left at the repository root; the shared Dockerfile needs root-level `package.json` and source files in its build context. In service settings, set the Config-as-Code path to `/deploy/railway/railway.json` (Railway resolves this path independently of the selected root directory). This config selects `deploy/Dockerfile` while the build context remains the repository root. Add a persistent volume mounted at `/data` before starting the service. The service runs iii 0.25.0 Worker Compose and serves the health endpoint at `/agentmemory/livez`. +Connect the GitHub repository with its root directory left at the repository root; the shared Dockerfile needs root-level `package.json` and source files in its build context. In service settings, set the Config-as-Code path to `/deploy/railway/railway.json` (Railway resolves this path independently of the selected root directory). This config selects `deploy/Dockerfile` while the build context remains the repository root. Add a persistent volume mounted at `/data` before starting the service. The service runs iii 0.25.1 Worker Compose and serves the health endpoint at `/agentmemory/livez`. The REST API listens on `3111`; Railway's generated HTTP domain exposes that service. Read the secret with `railway ssh --service agentmemory -- 'cat /data/secret'` and send it as `Authorization: Bearer `. diff --git a/deploy/recovery/README.md b/deploy/recovery/README.md new file mode 100644 index 0000000000..4b731d5166 --- /dev/null +++ b/deploy/recovery/README.md @@ -0,0 +1,24 @@ +# Limited sealed-for-recovery stop (preparation only) + +These tools do not deploy/start a candidate, resume background work, clear uncertainty, remove lifecycle markers, rewrite clean flags, or force-kill. They are not automatically run by the app. + +## Review and staging + +A user-reviewed config explicitly pins `service: agentmemory.service`, `user`, absolute `helper`, `rpcConfig`, fixed root result path `/var/lib/agentmemory-updates/recovery/sealed-for-recovery-COMPOSEPID-START.json`, and `processes` (`compose`, `engine`, `state`, `app`: each pid/start/exe). `reviewedFiles` must include exact hashes of the helper, RPC config and copied source import closure. Stage helper as `deploy/recovery/seal-current.mjs`, preserving relative source imports `src/ops/engine-epoch.mjs`, `state-epoch.mjs`, `state-dispatch-error.mjs`; include each in reviewedFiles. Source files are not included automatically in npm tarballs. The owner explicitly executes the reviewed runner with sudo after checking its published SHA. A separate root-owned source-tree installation is not required; files must be root/daniel-owned, not group/world writable, nonsymlink, with reviewed manifest hashes. RPC code runs as daniel (UID1000) with a minimal environment, not root. This targets accidental lifecycle changes, not a hostile same-user process with authority to modify the store. + +RPC config explicitly supplies sdkPackageJson (for installed iii-sdk/dotenv resolution), namespace, engineUrl, metadata, enginePidFile, store, marker, markerSha256, healthUrl, receipt, and exactly one of secretFile or secretEnvFile. secretEnvFile parses only AGENTMEMORY_SECRET using dotenv; it does not hydrate other environment settings. Use an existing protected secret source, not a new unnecessary token copy. healthUrl must be HTTP 127.0.0.1 /agentmemory/health. Namespace and endpoint must match Compose metadata. No API secret is printed or saved in receipts. + +Before invocation, create the user's receipt directory with restricted access; the runner creates only its fixed protected root result directory. The reviewed one-time command is `sudo /usr/bin/python3 -I -B REVIEWED_PATH/seal-stop.py --execute-reviewed-seal-stop REVIEWED_PATH/config.json`. This is a template, not authorization to run against production. + +## Actions and boundaries + +1. Verify review hashes, exact live process identities, systemd owner, unchanged dirty v4 marker, and absence of prior attempt/receipt. +2. Require pausedRequested, deployment hold, zero active operations and zero active mutations. Record exact original failure count/holds. +3. Verify SDK engine socket/process evidence, State immutable identity/alias and prior epoch; invoke only instance-pinned seal_flush. Recheck epoch and quiescence, preserve failure count/holds, exclusively write/fsync recovery receipt and its directory. +4. Root verifies marker/processes and receipt, requests only normal systemctl stop, records stop result and all-old-PID disappearance. It never claims clean even if stop returned0. Nonzero stop is retained, not hidden; remaining process stops advancement. No candidate start is included. + +Seal is irreversible until State restarts. It certifies persistence of actual committed bytes, not that an uncertain requested operation succeeded. Existing dirty marker and semantic uncertainty remain. The result/receipt files prevent blind repeated execution after ambiguous failures; inspect first. A timeout may leave the State sealed or children alive, requiring inspection, not a retry or kill. + +## Verification scope + +12 isolated JS procedure tests: exact failure/hold preservation, pre-seal admission failures, post-seal identity/receipt/flush/history failures. 3 Python mocked root-runner tests: helper failure never stops, sealed stop preserves nonclean result, remaining PID never triggers force. Production not executed. Actual SDK wiring and root-owned staged closure still require an isolated integration smoke before deployment use. diff --git a/deploy/recovery/seal-current.mjs b/deploy/recovery/seal-current.mjs new file mode 100644 index 0000000000..9c9b0cf570 --- /dev/null +++ b/deploy/recovery/seal-current.mjs @@ -0,0 +1,56 @@ +import {createHash} from 'node:crypto'; +import {existsSync,readFileSync,openSync,writeFileSync,fsyncSync,closeSync,realpathSync} from 'node:fs'; +import {dirname} from 'node:path'; +import {createRequire} from 'node:module'; +import {createEngineEpochEvidence,waitForEngineEpoch} from '../../src/ops/engine-epoch.mjs'; +import {createStateBoundEvidence,pinStateClient} from '../../src/ops/state-epoch.mjs'; +const hash=x=>createHash('sha256').update(x).digest('hex'); +function requireSafe(ok,code){if(!ok)throw Error(code);} +export function requirePausedHealth(h){ + const b=h?.backgroundControl,d=h?.storageDurability; + requireSafe(b?.pausedRequested===true&&b.activeOperations===0&&Array.isArray(b.holds)&&b.holds.includes('deployment')&&d?.activeMutations===0&&Number.isSafeInteger(d.mutationFailures)&&d.mutationFailures>=0,'RECOVERY_NOT_QUIESCENT'); + return {background:{status:b.status,holds:[...b.holds],pausedRequested:true,activeOperations:0},storage:{mutationFailures:d.mutationFailures,activeMutations:0,lastVerification:d.lastVerification??null}}; +} +export async function sealCurrentForRecovery({health,current,seal,identity,markerBytes,expectedMarkerSha256,persist,backgroundRecord}){ + requireSafe(hash(markerBytes)===expectedMarkerSha256,'RECOVERY_MARKER_CHANGED'); + const prior=JSON.parse(markerBytes);requireSafe(prior.version===4&&prior.clean===false&&prior.epoch?.state?.instanceId===identity.instanceId,'RECOVERY_V4_DIRTY_REQUIRED'); + const persistedBefore=await backgroundRecord(); + requireSafe(persistedBefore?.version===2&&persistedBefore.paused===true&&Array.isArray(persistedBefore.holds)&&persistedBefore.holds.includes('deployment'),'RECOVERY_PERSISTED_DEPLOYMENT_HOLD_REQUIRED'); + const before=requirePausedHealth(await health());requireSafe(current(),'RECOVERY_EPOCH_CHANGED'); + const receipt=await seal(); + requireSafe(current(),'RECOVERY_EPOCH_CHANGED_AFTER_SEAL'); + requireSafe(receipt?.version===1&&receipt.instanceId===identity.instanceId&&receipt.sealed===true&&receipt.synced===true&&receipt.storePath===identity.storePath,'RECOVERY_SEAL_UNVERIFIED'); + const persistedAfter=await backgroundRecord();requireSafe(JSON.stringify(persistedAfter)===JSON.stringify(persistedBefore),'RECOVERY_PERSISTED_HOLD_CHANGED'); + const after=requirePausedHealth(await health()); + requireSafe(after.storage.mutationFailures>=before.storage.mutationFailures&&before.background.holds.every(x=>after.background.holds.includes(x)),'RECOVERY_FAILURE_HISTORY_CHANGED'); + requireSafe(current(),'RECOVERY_EPOCH_CHANGED_BEFORE_RECEIPT'); + const proof={version:1,kind:'sealed-for-recovery',at:new Date().toISOString(),storageSealed:true,verifiedClean:false,semanticOutcomeUncertain:true,markerSha256:expectedMarkerSha256,priorEpoch:prior.epoch,before,after,persistedBackground:persistedAfter,stateSeal:receipt}; + await persist(proof);return proof; +} +function persistExclusive(path,value){const fd=openSync(path,'wx',0o600);try{writeFileSync(fd,JSON.stringify(value)+'\n');fsyncSync(fd);}finally{closeSync(fd);}const dir=openSync(dirname(path),'r');try{fsyncSync(dir);}finally{closeSync(dir);}} +async function main(path,inspect=false){ + const c=JSON.parse(readFileSync(path));if(!inspect)requireSafe(!existsSync(c.receipt),'RECOVERY_RECEIPT_EXISTS_INSPECT_ONLY');const require=createRequire(c.sdkPackageJson);const {registerWorker}=require('iii-sdk'); + const md=JSON.parse(readFileSync(c.metadata));requireSafe(md.kind==='compose'&&typeof c.namespace==='string'&&c.namespace===md.namespace&&new URL(c.engineUrl).href===new URL(md.engineUrl).href,'RECOVERY_COMPOSE_NAMESPACE_MISMATCH'); + const sdk=registerWorker(c.engineUrl,{namespace:c.namespace,workerName:'sealed-recovery-proof',enableMetricsReporting:false,otel:{enabled:false}}); + try{ + const e=createEngineEpochEvidence({engineUrl:c.engineUrl,engineStatePath:c.metadata,enginePidPath:c.enginePidFile,peer:()=>{const s=sdk.ws?._socket;return s?{localAddress:s.localAddress,localPort:s.localPort,remoteAddress:s.remoteAddress,remotePort:s.remotePort}:null;}});try{await waitForEngineEpoch(e);}catch(error){throw Error('RECOVERY_ENGINE_EPOCH_UNVERIFIED:'+(e.diagnostic?.()??error.message));} + const identity=await sdk.trigger({function_id:'state::boot_identity',payload:{}});const prefix=`state::${identity.instanceId}::`; + requireSafe(JSON.stringify(await sdk.trigger({function_id:prefix+'boot_identity',payload:{}}))===JSON.stringify(identity),'RECOVERY_ALIAS_CHANGED'); + const bound=createStateBoundEvidence({engineEvidence:e,identity,expectedExecutable:md.stateExecutable,storePath:c.store});const epoch=bound.capture();requireSafe(epoch,'RECOVERY_EPOCH_UNVERIFIED'); + const prior=JSON.parse(readFileSync(c.marker));requireSafe(JSON.stringify(epoch.engine)===JSON.stringify(prior.epoch?.engine)&&epoch.bootId===prior.epoch.bootId&&epoch.host===prior.epoch.host&&identity.pid===prior.epoch.state.pid&&identity.start===prior.epoch.state.start,'RECOVERY_PRIOR_EPOCH_MISMATCH'); + const client=pinStateClient(sdk,prefix,()=>bound.current(epoch),()=>bound.diagnostic()); + const url=new URL(c.healthUrl);requireSafe(url.protocol==='http:'&&url.hostname==='127.0.0.1'&&!url.username&&!url.password&&url.pathname==='/agentmemory/health','RECOVERY_HEALTH_DESTINATION'); + // Only the explicit token file is read; no environment-file loading or hidden credentials. + requireSafe(Boolean(c.secretFile)!==Boolean(c.secretEnvFile),'RECOVERY_EXPLICIT_SECRET_SOURCE_REQUIRED'); + const secret=c.secretFile?readFileSync(c.secretFile,'utf8').trim():require('dotenv').parse(readFileSync(c.secretEnvFile)).AGENTMEMORY_SECRET;requireSafe(typeof secret==='string'&&secret.length>0,'RECOVERY_SECRET_MISSING'); + const backgroundRecord=()=>client.trigger({function_id:'state::get',payload:{scope:'mem:background-control:v1',key:'manual-pause'}}); + const health=async()=>{const r=await fetch(url,{headers:{Authorization:'Bearer '+secret},signal:AbortSignal.timeout(10000)});requireSafe(r.ok,'RECOVERY_HEALTH_HTTP_'+r.status);return r.json();}; + if(inspect){const marker=readFileSync(c.marker,'utf8');requireSafe(hash(marker)===c.markerSha256,'RECOVERY_MARKER_CHANGED');requireSafe(prior.version===4&&prior.clean===false,'RECOVERY_V4_DIRTY_REQUIRED');const persistedBackground=await backgroundRecord();requireSafe(persistedBackground?.version===2&&persistedBackground.paused===true&&persistedBackground.holds?.includes('deployment'),'RECOVERY_PERSISTED_DEPLOYMENT_HOLD_REQUIRED');const status=requirePausedHealth(await health());requireSafe(bound.current(epoch),'RECOVERY_EPOCH_CHANGED');console.log(JSON.stringify({inspected:true,sealed:false,verifiedClean:false,namespace:c.namespace,priorEpoch:prior.epoch,status,persistedBackground}));return;} + await sealCurrentForRecovery({health,backgroundRecord,current:()=>bound.current(epoch),seal:()=>client.trigger({function_id:'state::seal_flush',payload:{},timeoutMs:120000}),identity,markerBytes:readFileSync(c.marker,'utf8'),expectedMarkerSha256:c.markerSha256,persist:x=>persistExclusive(c.receipt,x)}); + console.log(JSON.stringify({sealedForRecovery:true,verifiedClean:false,receipt:c.receipt})); + }finally{await sdk.shutdown();} +} +if(process.argv[1]&&realpathSync(process.argv[1])===realpathSync(new URL(import.meta.url))){ + if(!['--execute-reviewed-seal','--inspect'].includes(process.argv[2])||process.argv.length!==4)throw Error('EXPLICIT_RECOVERY_SEAL_REQUIRED'); + main(process.argv[3],process.argv[2]==='--inspect').catch(e=>{console.error(JSON.stringify({sealedForRecovery:false,error:e.message,inspectBeforeRetry:true}));process.exitCode=78;}); +} diff --git a/deploy/recovery/seal-stop.py b/deploy/recovery/seal-stop.py new file mode 100644 index 0000000000..9d70c6e4c0 --- /dev/null +++ b/deploy/recovery/seal-stop.py @@ -0,0 +1,64 @@ +#!/usr/bin/python3 +"""Reviewed one-shot seal+normal stop only. Never starts, edits units, or kills.""" +import os,sys,json,pathlib,hashlib,subprocess,pwd,time +P=pathlib.Path +ROOT_RECEIPT_DIR=P('/var/lib/agentmemory-updates/recovery') + +def need(ok,code): + if not ok:raise RuntimeError(code) +def sha(p): + with open(p,'rb') as f:return hashlib.file_digest(f,'sha256').hexdigest() +def trusted(path): + p=P(path);s=p.lstat();need(not p.is_symlink() and s.st_uid in (0,1000) and not s.st_mode&0o022 and p.is_file(),'REVIEWED_FILE_REQUIRED') +def proc(pid): + p=P('/proc')/str(pid);raw=(p/'stat').read_text();return {'pid':pid,'start':raw[raw.rindex(') ')+2:].split()[19],'exe':os.path.realpath(p/'exe')} +def matches(x): + try:return proc(x['pid'])==x + except FileNotFoundError:return False +def gone(x): + try:return proc(x['pid'])['start']!=x['start'] + except FileNotFoundError:return True +def exclusive(path,value): + with open(path,'x') as f:os.chmod(path,0o600);json.dump(value,f);f.write('\n');f.flush();os.fsync(f.fileno()) + d=os.open(P(path).parent,os.O_DIRECTORY);os.fsync(d);os.close(d) +def rootReceiptDirectory(path): + path.mkdir(mode=0o700,parents=True,exist_ok=True) + info=path.lstat();need(not path.is_symlink() and info.st_uid==0 and not info.st_mode&0o022,'ROOT_RECEIPT_DIRECTORY_UNSAFE') +def main(path,inspect=False): + need(os.geteuid()==0,'ROOT_REQUIRED');trusted(path);c=json.loads(P(path).read_text()) + need(c['service']=='agentmemory.service' and c['user']=='daniel','FIXED_SERVICE_AND_USER_REQUIRED') + identity=c['processes']['compose'];need(type(identity['pid']) is int and identity['pid']>1 and str(identity['start']).isdigit(),'EXPECTED_COMPOSE_REQUIRED') + resultRoot=ROOT_RECEIPT_DIR;expectedResult=resultRoot/('sealed-for-recovery-'+str(identity['pid'])+'-'+str(identity['start'])+'.json') + need(P(c['result'])==expectedResult,'FIXED_ROOT_RECEIPT_REQUIRED') + for file,digest in c['reviewedFiles'].items():trusted(file);need(sha(file)==digest,'REVIEWED_CLOSURE_CHANGED') + need(c['helper'] in c['reviewedFiles'] and c['rpcConfig'] in c['reviewedFiles'],'REVIEWED_HELPER_REQUIRED') + rpc=json.loads(P(c['rpcConfig']).read_text());need(inspect or not P(c['result']).exists() and not P(rpc['receipt']).exists(),'PREVIOUS_ATTEMPT_INSPECT_REQUIRED') + user=pwd.getpwnam(c['user']);need(user.pw_uid==1000,'SERVICE_UID_REQUIRED') + need(sha(rpc['marker'])==rpc['markerSha256'],'MARKER_CHANGED') + for x in c['processes'].values():need(matches(x),'PROCESS_IDENTITY_CHANGED') + observed=subprocess.check_output(['/usr/bin/systemctl','show',c['service'],'-p','MainPID','--value'],text=True).strip();need(observed==str(c['processes']['compose']['pid']),'SERVICE_OWNER_CHANGED') + # Durable attempt record blocks blindly repeating an ambiguous seal request. + if not inspect: + rootReceiptDirectory(resultRoot) + if not inspect:exclusive(c['result'],{'version':1,'phase':'seal_intent','verifiedClean':False,'priorProcesses':c['processes']}) + def demote():os.initgroups(user.pw_name,user.pw_gid);os.setgid(user.pw_gid);os.setuid(user.pw_uid) + env={'HOME':user.pw_dir,'PATH':'/usr/bin:/bin','III_TELEMETRY_ENABLED':'false'} + p=subprocess.run(['/usr/bin/node',c['helper'],'--inspect' if inspect else '--execute-reviewed-seal',c['rpcConfig']],env=env,preexec_fn=demote,capture_output=True,text=True,timeout=150) + need(p.returncode==0,'SEAL_HELPER_FAILED_INSPECT_RECEIPT') + if inspect: + print(p.stdout.strip());return + proof=json.loads(P(rpc['receipt']).read_text());need(proof.get('kind')=='sealed-for-recovery' and proof.get('storageSealed') is True and proof.get('verifiedClean') is False and proof.get('markerSha256')==rpc['markerSha256'],'RECOVERY_RECEIPT_UNVERIFIED') + need(sha(rpc['marker'])==rpc['markerSha256'],'MARKER_CHANGED_AFTER_SEAL') + for x in c['processes'].values():need(matches(x),'PROCESS_CHANGED_AFTER_SEAL') + for file,digest in c['reviewedFiles'].items():need(sha(file)==digest,'REVIEWED_CLOSURE_CHANGED') + # A nonzero result is evidence, not permission to force or claim clean. + stopped=subprocess.run(['/usr/bin/systemctl','stop',c['service']],capture_output=True,text=True,timeout=700) + status=subprocess.check_output(['/usr/bin/systemctl','show',c['service'],'-p','ActiveState','-p','Result','-p','MainPID'],text=True) + allgone=all(gone(x) for x in c['processes'].values()) + exclusive(c['result']+'.stopped',{'version':1,'phase':'stopped' if allgone else 'stop_incomplete','verifiedClean':False,'semanticOutcomeUncertain':True,'stopExit':stopped.returncode,'serviceStatus':status,'allPriorProcessesGone':allgone,'sealReceiptSha256':sha(rpc['receipt']),'history':{'before':proof['before'],'after':proof['after']}}) + need(allgone,'PRIOR_PROCESSES_REMAIN_NO_FORCE');print(json.dumps({'sealedForRecovery':True,'stopped':True,'verifiedClean':False,'next':'review offline backup and candidate separately'})) +if __name__=='__main__': + try: + need(len(sys.argv)==3 and sys.argv[1] in ('--execute-reviewed-seal-stop','--inspect'),'EXPLICIT_REVIEWED_ACTION_REQUIRED');main(sys.argv[2],sys.argv[1]=='--inspect') + except Exception as e: + print(json.dumps({'completed':False,'error':str(e) if isinstance(e,RuntimeError) else type(e).__name__,'inspectBeforeRetry':True}));sys.exit(78) diff --git a/deploy/recovery/test-seal-stop.py b/deploy/recovery/test-seal-stop.py new file mode 100644 index 0000000000..71e24fcc51 --- /dev/null +++ b/deploy/recovery/test-seal-stop.py @@ -0,0 +1,20 @@ +import unittest,tempfile,pathlib,json,importlib.util,types +from unittest.mock import patch +P=pathlib.Path;s=importlib.util.spec_from_file_location('runner',P(__file__).with_name('seal-stop.py'));m=importlib.util.module_from_spec(s);s.loader.exec_module(m) +class Test(unittest.TestCase): + def exercise(self,helper_ok=True,allgone=True): + with tempfile.TemporaryDirectory() as d: + r=P(d);helper=r/'helper';helper.write_text('synthetic');marker=r/'marker';marker.write_text('unchanged');rpc=r/'rpc';receipt=r/'receipt';rpc.write_text(json.dumps({'marker':str(marker),'markerSha256':m.sha(marker),'receipt':str(receipt)}));config=r/'config';config.write_text(json.dumps({'service':'agentmemory.service','helper':str(helper),'rpcConfig':str(rpc),'result':str(r/'sealed-for-recovery-123-1.json'),'user':'daniel','processes':{'compose':{'pid':123,'start':'1','exe':'/synthetic'}},'reviewedFiles':{str(helper):m.sha(helper),str(rpc):m.sha(rpc)}}));calls=[] + def run(argv,**kwargs): + calls.append(argv) + if argv[0]=='/usr/bin/node' and helper_ok:receipt.write_text(json.dumps({'kind':'sealed-for-recovery','storageSealed':True,'verifiedClean':False,'markerSha256':m.sha(marker),'before':{'mutationFailures':1},'after':{'mutationFailures':1}})) + return types.SimpleNamespace(returncode=0 if helper_ok else 78) + with patch.object(m,'ROOT_RECEIPT_DIR',r),patch.object(m.os,'geteuid',return_value=0),patch.object(m,'trusted'),patch.object(m,'rootReceiptDirectory',return_value=None),patch.object(m,'matches',return_value=True),patch.object(m,'gone',return_value=allgone),patch.object(m.pwd,'getpwnam',return_value=types.SimpleNamespace(pw_uid=1000,pw_gid=1000,pw_name='fixture',pw_dir=d)),patch.object(m.subprocess,'run',side_effect=run),patch.object(m.subprocess,'check_output',side_effect=['123\n','ActiveState=inactive\nResult=exit-code\nMainPID=0\n']): + if helper_ok and allgone:m.main(str(config));self.assertFalse(json.loads((r/'sealed-for-recovery-123-1.json.stopped').read_text())['verifiedClean']) + else: + with self.assertRaises(RuntimeError):m.main(str(config)) + return calls + def test_helper_failure_never_stops_service(self):self.assertFalse(any('stop' in x for x in self.exercise(False))) + def test_exact_seal_allgone_can_preserve_unclean_stop_result(self):self.assertEqual(sum('stop' in x for x in self.exercise()),1) + def test_remaining_process_never_force_killed(self):self.assertFalse(any('kill' in x for x in self.exercise(True,False))) +if __name__=='__main__':unittest.main() diff --git a/deploy/render/README.md b/deploy/render/README.md index a1871bcce7..eb1bfc8adc 100644 --- a/deploy/render/README.md +++ b/deploy/render/README.md @@ -1,6 +1,6 @@ # Render -Create a Blueprint from `deploy/render/render.yaml`. Render builds the shared `deploy/Dockerfile` and provisions a persistent `/data` disk. The web service exposes REST on `3111` and uses `/agentmemory/livez` for health checks. The runtime is iii 0.25.0 Worker Compose with registry workers pinned in `deploy/Dockerfile` and `deploy/runtime/worker-compose.yaml`. +Create a Blueprint from `deploy/render/render.yaml`. Render builds the shared `deploy/Dockerfile` and provisions a persistent `/data` disk. The web service exposes REST on `3111` and uses `/agentmemory/livez` for health checks. The runtime is iii 0.25.1 Worker Compose with registry workers pinned in `deploy/Dockerfile` and `deploy/runtime/worker-compose.yaml`. The viewer listens on container port `3113`. Render's web service exposes only the primary HTTP port; reach the viewer through an SSH tunnel or a separately configured TLS proxy. For a public proxy, set `VIEWER_ALLOWED_HOSTS` and `VIEWER_ALLOWED_ORIGINS` to exact values. Viewer API/SSE routes require the bearer secret. Read it over SSH with `cat /data/secret`. Do not expose the engine WebSocket on `49134`. diff --git a/deploy/runtime/entrypoint.sh b/deploy/runtime/entrypoint.sh index 527409a60d..3a777eef93 100755 --- a/deploy/runtime/entrypoint.sh +++ b/deploy/runtime/entrypoint.sh @@ -114,6 +114,5 @@ fs.writeFileSync(target, input, { mode: 0o600 }); NODE chown node:node "$DATA_DIR/config/iii-observability.yaml" "$WORKER_COMPOSE_FILE" -# Compose owns the 0.25.0 engine and supervises state, http, and agentmemory. -exec gosu node:node /usr/local/bin/iii compose \ - --namespace agentmemory --up --file "$WORKER_COMPOSE_FILE" +# Compose owns the 0.25.1 engine and supervises state, http, and agentmemory. +exec gosu node:node node /opt/agentmemory/deploy-runtime/state-bootstrap.mjs diff --git a/deploy/runtime/state-bootstrap.mjs b/deploy/runtime/state-bootstrap.mjs new file mode 100644 index 0000000000..3cc2d4730e --- /dev/null +++ b/deploy/runtime/state-bootstrap.mjs @@ -0,0 +1,42 @@ +import {readFileSync,writeFileSync,renameSync,mkdirSync,realpathSync,readdirSync,readlinkSync,statSync} from 'node:fs'; +import {join,dirname} from 'node:path';import {fileURLToPath} from 'node:url';import {createHash} from 'node:crypto';import {spawn} from 'node:child_process';import {parse,stringify} from 'yaml'; +export function verifyArtifact(directory,pin){ + const root=realpathSync(directory),m=JSON.parse(readFileSync(join(root,'manifest.json'),'utf8')); + for(const key of ['artifactVersion','upstreamCommit','patchSha256','engineVersion'])if(m[key]!==pin[key])throw Error('STATE_ARTIFACT_PROVENANCE_MISMATCH'); + if(m.schemaVersion!==1||m.binary!=='state'||m.profile!=='release'||m.target!==(process.arch==='x64'?'x86_64-unknown-linux-gnu':process.arch==='arm64'?'aarch64-unknown-linux-gnu':'unsupported'))throw Error('STATE_ARTIFACT_PLATFORM_MISMATCH'); + const path=realpathSync(join(root,'state'));if(!statSync(path).isFile()||!(statSync(path).mode&0o111))throw Error('STATE_ARTIFACT_BINARY_INVALID'); + const sha256=createHash('sha256').update(readFileSync(path)).digest('hex');if(sha256!==m.binarySha256)throw Error('STATE_ARTIFACT_HASH_MISMATCH');return{path,sha256}; +} +export function selectEnginePid(rootPid,exe,port,proc='/proc'){ + const queue=[rootPid],seen=new Set(),matches=[]; + while(queue.length){const pid=queue.shift();if(seen.has(pid))continue;seen.add(pid);if(seen.size>32)throw Error('COMPOSE_PROCESS_TREE_TOO_LARGE'); + try{queue.push(...readFileSync(join(proc,`${pid}/task/${pid}/children`),'utf8').trim().split(/\s+/).filter(Boolean).map(Number));}catch{} + try{if(realpathSync(join(proc,`${pid}/exe`))!==exe)continue;const sockets=new Set(readdirSync(join(proc,`${pid}/fd`)).flatMap(fd=>{try{const m=/^socket:\[(\d+)\]$/.exec(readlinkSync(join(proc,`${pid}/fd/${fd}`)));return m?[m[1]]:[];}catch{return[];}})); + let listening=false;for(const name of ['tcp','tcp6'])for(const line of readFileSync(join(proc,`${pid}/net/${name}`),'utf8').split('\n')){const v=line.trim().split(/\s+/);if(v[3]==='0A'&&parseInt(v[1]?.split(':')[1],16)===port&&sockets.has(v[9]))listening=true;} + if(listening)matches.push(pid); + }catch{} + } + if(matches.length>1)throw Error('COMPOSE_ENGINE_AMBIGUOUS');return matches[0]??null; +} +const atomic=(path,value)=>{const temp=path+'.tmp';writeFileSync(temp,JSON.stringify(value)+'\n',{mode:0o600});renameSync(temp,path);}; +export async function runBootstrap({file,runtimeDir,artifactDir,pinPath,engineBin='/usr/local/bin/iii'}){ + const stateExecutable=verifyArtifact(artifactDir,JSON.parse(readFileSync(pinPath,'utf8'))),exe=realpathSync(engineBin); + const config=parse(readFileSync(file,'utf8'));const url=new URL(config.engine.url);if(url.hostname!=='127.0.0.1'||url.protocol!=='ws:')throw Error('COMPOSE_ENGINE_ENDPOINT_INVALID'); + mkdirSync(runtimeDir,{recursive:true,mode:0o700});const project=join(runtimeDir,'state-project');mkdirSync(project,{recursive:true,mode:0o700}); + writeFileSync(join(project,'iii.worker.yaml'),stringify({iii:'v1',name:'state',language:'rust',scripts:{start:stateExecutable.path}}),{mode:0o600}); + config.containers.state.worker='path://'+project;delete config.containers.state.version; + config.containers.state.scripts={run:"'"+stateExecutable.path.replaceAll("'","'\\''")+"'"}; + config.containers.agentmemory.environment={...config.containers.agentmemory.environment,AGENTMEMORY_RUNTIME_DIR:runtimeDir}; + writeFileSync(file,stringify(config),{mode:0o600}); + const metadata={kind:'compose',engineUrl:url.href.replace(/\/$/,''),namespace:config.namespace,file,stateExecutable};const statePath=join(runtimeDir,'engine-state.json');atomic(statePath,metadata); + const child=spawn(exe,['compose','--namespace',config.namespace,'--up','--file',file],{stdio:'inherit',env:process.env}); + const forward=signal=>{if(child.exitCode===null)child.kill(signal);};process.once('SIGTERM',forward);process.once('SIGINT',forward); + let done=false;const exited=new Promise((resolve,reject)=>{child.once('error',reject);child.once('exit',(code,signal)=>{done=true;resolve(code??(signal?1:0));});}); + let timer;const deadline=Date.now()+120000; + try{while(!done){const pid=selectEnginePid(child.pid,exe,Number(url.port));if(pid&&pid!==metadata.enginePid){metadata.enginePid=pid;atomic(statePath,metadata);}if(!metadata.enginePid&&Date.now()>deadline)throw Error('COMPOSE_ENGINE_METADATA_TIMEOUT');await Promise.race([exited,new Promise(r=>timer=setTimeout(r,100))]);clearTimeout(timer);}return await exited;} + catch(error){forward('SIGTERM');await exited;throw error;} + finally{process.removeListener('SIGTERM',forward);process.removeListener('SIGINT',forward);clearTimeout(timer);} +} +if(process.argv[1]&&realpathSync(process.argv[1])===fileURLToPath(import.meta.url)){ + runBootstrap({file:process.env.WORKER_COMPOSE_FILE,runtimeDir:process.env.AGENTMEMORY_RUNTIME_DIR??join(process.env.AGENTMEMORY_DATA_DIR,'runtime'),artifactDir:process.env.AGENTMEMORY_STATE_WORKER_ARTIFACT??'/opt/agentmemory-state',pinPath:'/opt/agentmemory/deploy-runtime/state-pin.json'}).then(code=>{process.exitCode=code;}).catch(error=>{console.error(error.message);process.exitCode=1;}); +} diff --git a/deploy/runtime/worker-compose.yaml b/deploy/runtime/worker-compose.yaml index c1c1092887..d3f8b91308 100644 --- a/deploy/runtime/worker-compose.yaml +++ b/deploy/runtime/worker-compose.yaml @@ -74,6 +74,7 @@ containers: VIEWER_ALLOWED_HOSTS: "${VIEWER_ALLOWED_HOSTS:-localhost:3113,127.0.0.1:3113,[::1]:3113}" VIEWER_ALLOWED_ORIGINS: "${VIEWER_ALLOWED_ORIGINS:-http://localhost:3113,http://127.0.0.1:3113}" EMBEDDING_PROVIDER: "${EMBEDDING_PROVIDER:-}" + AGENTMEMORY_LLM_PROVIDER: "${AGENTMEMORY_LLM_PROVIDER:-}" OPENAI_API_KEY: "${OPENAI_API_KEY:-}" OPENAI_API_KEY_FOR_LLM: "${OPENAI_API_KEY_FOR_LLM:-}" OPENAI_BASE_URL: "${OPENAI_BASE_URL:-}" @@ -118,6 +119,7 @@ containers: AGENTMEMORY_STATE_BACKEND: "${AGENTMEMORY_STATE_BACKEND:-}" AGENTMEMORY_REDIS_URL: "${AGENTMEMORY_REDIS_URL:-}" AGENTMEMORY_STATE_SAVE_INTERVAL_MS: "${AGENTMEMORY_STATE_SAVE_INTERVAL_MS:-}" + AGENTMEMORY_LOCAL_DISPATCH: "${AGENTMEMORY_LOCAL_DISPATCH:-}" AGENTMEMORY_SECRET: "${AGENTMEMORY_SECRET:-}" AGENT_ID: "${AGENT_ID:-}" TEAM_ID: "${TEAM_ID:-}" @@ -156,6 +158,7 @@ containers: AGENTMEMORY_GRAPH_WEIGHT: "${AGENTMEMORY_GRAPH_WEIGHT:-}" SUMMARIZE_CHUNK_SIZE: "${SUMMARIZE_CHUNK_SIZE:-}" SUMMARIZE_CHUNK_CONCURRENCY: "${SUMMARIZE_CHUNK_CONCURRENCY:-}" + AGENTMEMORY_SUMMARY_TIMEOUT_MS: "${AGENTMEMORY_SUMMARY_TIMEOUT_MS:-}" CONSOLIDATION_DECAY_DAYS: "${CONSOLIDATION_DECAY_DAYS:-}" CONSOLIDATION_INTERVAL_MS: "${CONSOLIDATION_INTERVAL_MS:-}" AGENTMEMORY_CONSOLIDATION_COOLDOWN_MS: "${AGENTMEMORY_CONSOLIDATION_COOLDOWN_MS:-}" diff --git a/deploy/state-worker/README.md b/deploy/state-worker/README.md new file mode 100644 index 0000000000..0e6fa6e8c7 --- /dev/null +++ b/deploy/state-worker/README.md @@ -0,0 +1,73 @@ +# Pinned local State worker + +This is a local compatibility artifact, **not** an official State release. +`pin.json` records the exact upstream commit/tag and local patch digest. The +upstream tag is `state/v0.22.22`; its Cargo package version is `0.22.21`. +Artifact version `0.22.22-daniel.1` is distinct from that original package version. + +The patch retains compact ACK, immutable boot identity/instance aliases, +cooperative file-store lock, and sealed fsync receipt. ACK alone is not durable. +The advisory lock excludes participating patched workers, not older or +uncooperative processes. Seal drains adapter mutations, remains sealed on error, +and must succeed before an app clean-shutdown marker is published. + +## Build from source + +Requires Git, Python3 (tarfile data filter), Rust >=1.89, Node22+, pnpm10.32.1 and +cached Cargo/pnpm dependencies, or explicit `--online` for official registries. +No system toolchain or production service is changed by the builder. + +```sh +git clone https://github.com/iii-hq/workers workers-source +python3 scripts/state-worker/build.py \ + --workers-source ./workers-source \ + --work-dir ./state-build \ + --output ./state-artifact \ + --jobs 2 --online +``` + +The builder reads the pinned Git objects, **not** uncommitted source changes, +applies the verified patch, builds the console UI with frozen pnpm lockfile, +and compiles State with locked Cargo dependencies. A marked workspace can be +rebuilt with `--resume` and a new output directory. It records compiler, Node, +pnpm, source archive, Cargo lock and binary digests in `manifest.json`, and emits +a deterministic-layout tar.gz alongside the output. Compiler/platform changes +can change binary bytes; this is reproducible source provenance, not a promise +of cross-toolchain bit-identical builds. + +## CLI selection + +```sh +export AGENTMEMORY_STATE_WORKER_ARTIFACT="$PWD/state-artifact" +node dist/cli.mjs start +``` + +Use the directory containing `manifest.json` and `state`. The CLI verifies source +pin, patch digest, target and binary digest, then runs that State binary locally +while keeping the HTTP worker registry-pinned. The normal local-workers override +remains available. Without either override, file-backed Compose refuses early: +official registry State does not provide this custom persistence contract. +Do not edit manifests to relabel another binary as this artifact. + +## Container path + +`deploy/Dockerfile` builds the same pinned source/patch in its Rust build stage +and copies the artifact into the runtime image. The entrypoint runs +`state-bootstrap.mjs`, which verifies the artifact, selects the local State +worker, and atomically publishes metadata for the actual engine listener owned +by the tracked Compose process tree. App epoch verification still validates the +socket and executable; bootstrap PID metadata alone is not authority. +`STATE_VERSION` is a legacy template input and does not override this artifact. +A native Docker image build/run must be verified separately from host tests. + +## Large-overwrite smoke + +```sh +node --max-old-space-size=1536 scripts/state-worker/large-overwrite-proof.mjs \ + /path/to/iii ./state-artifact/state /path/to/iii-sdk/dist/index.mjs ./proof-output +``` + +Uses only new synthetic state and an ephemeral loopback port. Writes and +replaces a >33,556,506-byte JSON value, compares digest/metadata, seals+fsyncs, +restarts only its captured test children, then verifies persisted data. It prints +no source payload. The supplied engine/SDK must be the reviewed0.25.1 versions. diff --git a/deploy/state-worker/compat.patch b/deploy/state-worker/compat.patch new file mode 100644 index 0000000000..4631ba168a --- /dev/null +++ b/deploy/state-worker/compat.patch @@ -0,0 +1,917 @@ +diff --git a/state/README.md b/state/README.md +index a9b0f17..5f62e30 100644 +--- a/state/README.md ++++ b/state/README.md +@@ -226,3 +226,48 @@ externalized builtin. A formal benchmark was waived by project decision + | Durability | store dies with the ENGINE process | store dies with the WORKER process (ADR 0001; file_based/redis unchanged) | + | Latency | in-process µs | engine⇄worker WS round-trip (low ms) — formal benchmark waived by project decision 2026-07-06 | + | Telemetry (`track_state_*`) | engine-internal counters | none — out of scope for this migration; lands with the shared worker observability story | ++ ++## Personal compatibility extension: compact write acknowledgement ++ ++This local candidate adds `state::set_ack` with the same input and mutation, ++validation, reserved-scope protection, and event behavior as `state::set`. ++It returns only JSON `true` after success and propagates the same errors. ++`state::set` retains its existing old/new-value response. This avoids echoing ++large source records across the RPC connection when the caller only needs an ++acknowledgement. An ACK is **not** a disk-flush or crash-durability receipt; ++AgentMemory's separate persistence verification must remain enabled. ++ ++This is a custom extension of the `state/v0.22.22` source tag, not an official ++State Worker release. It can be removed only when the official worker provides ++an equivalent bounded response and passes the same compatibility tests. ++ ++### Local boot identity / instance aliases (migration evidence only) ++ ++`state::boot_identity` returns immutable Linux boot facts: `version:1`, `pid`, ++`start` (process start ticks as a decimal string), `instanceId` (boot UUID), ++`bootId`, `adapter`, and canonical `storePath` (null for non-file stores). ++The path is taken from the constructed adapter, not the seed configuration. ++No credentials or complete configuration are returned. Unavailable process or ++path evidence returns `BOOT_IDENTITY_UNAVAILABLE`; it never fabricates evidence. ++ ++The same handlers also register under `state::::` for `set`, ++`set_ack`, `compare-and-set`, `barrier`, `get`, `delete`, `update`, `list`, ++`list_keys`, `list_groups`, and `boot_identity`. These directly share the canonical ++handlers and guards, rather than forwarding through a replaceable canonical RPC. ++Canonical APIs remain unchanged; private namespace claims remain canonical. ++Clients can independently validate the boot identity, then pin these aliases so a ++replacement worker does not silently receive their calls. ++ ++Identity is NOT authentication, a single-writer lock, or a durability receipt. ++Consumers must validate executable/process/socket/store evidence separately. ++This lifecycle defense is against accidental replacement and stale routing, not ++an adversary with the same OS user/root authority or arbitrary engine registration. ++ ++File-backed local workers also hold an exclusive, nonblocking advisory lock on ++`.agentmemory-state.lock` in the canonical store directory before loading state. ++Contention fails startup before registering RPCs. The lock file is never removed ++or truncated; it is separate from AgentMemory's application writer lease. The ++open lock stays owned by the shared flush mutex, including background and pending ++blocking flush tasks, until all writer references end or the process exits. ++This excludes cooperating patched workers only. Older/unpatched processes do not ++honor this advisory lock and must not be run against the same store. +diff --git a/state/src/adapters.rs b/state/src/adapters.rs +index 6b2cba0..68bbe68 100644 +--- a/state/src/adapters.rs ++++ b/state/src/adapters.rs +@@ -40,6 +40,8 @@ pub enum CompareAndSetOutcome { + + #[async_trait] + pub trait StateAdapter: Send + Sync + 'static { ++ /// Actual constructed file store, never the seed or mutable configuration. ++ fn file_store_path(&self) -> anyhow::Result> { Ok(None) } + async fn set(&self, scope: &str, key: &str, value: Value) -> anyhow::Result; + async fn get(&self, scope: &str, key: &str) -> anyhow::Result>; + async fn delete(&self, scope: &str, key: &str) -> anyhow::Result<()>; +@@ -88,24 +90,37 @@ pub trait StateAdapter: Send + Sync + 'static { + async fn flush(&self) -> anyhow::Result<()> { + Ok(()) + } ++ async fn seal_flush(&self) -> anyhow::Result { anyhow::bail!("STATE_SEAL_UNSUPPORTED") } + async fn destroy(&self) -> anyhow::Result<()>; + } + + pub struct KvStoreAdapter { + storage: KvStore, ++ mutation_gate: tokio::sync::RwLock<()>, ++ sealed: std::sync::atomic::AtomicBool, + } + + impl KvStoreAdapter { + pub fn new(config: Option) -> Self { +- Self { +- storage: KvStore::new(config), +- } ++ Self::from_store(KvStore::new(config)) ++ } ++ fn from_store(storage: KvStore) -> Self { ++ Self { storage, mutation_gate: tokio::sync::RwLock::new(()), sealed: std::sync::atomic::AtomicBool::new(false) } ++ } ++ async fn admit(&self) -> anyhow::Result> { ++ let guard = self.mutation_gate.read().await; ++ if self.sealed.load(std::sync::atomic::Ordering::Acquire) { anyhow::bail!("STATE_WRITER_SEALED"); } ++ Ok(guard) + } + } + + #[async_trait] + impl StateAdapter for KvStoreAdapter { ++ fn file_store_path(&self) -> anyhow::Result> { ++ self.storage.file_store_path() ++ } + async fn set(&self, scope: &str, key: &str, value: Value) -> anyhow::Result { ++ let _admission = self.admit().await?; + Ok(self + .storage + .set(scope.to_string(), key.to_string(), value) +@@ -115,6 +130,7 @@ impl StateAdapter for KvStoreAdapter { + Ok(self.storage.get(scope.to_string(), key.to_string()).await) + } + async fn delete(&self, scope: &str, key: &str) -> anyhow::Result<()> { ++ let _admission = self.admit().await?; + self.storage + .remove_shared(scope.to_string(), key.to_string()) + .await; +@@ -126,6 +142,7 @@ impl StateAdapter for KvStoreAdapter { + key: &str, + ops: Vec, + ) -> anyhow::Result { ++ let _admission = self.admit().await?; + Ok(self + .storage + .update(scope.to_string(), key.to_string(), ops) +@@ -138,6 +155,7 @@ impl StateAdapter for KvStoreAdapter { + expected: Option<&Value>, + value: Value, + ) -> anyhow::Result { ++ let _admission = self.admit().await?; + Ok(self + .storage + .compare_and_set(scope.to_string(), key.to_string(), expected, value) +@@ -151,6 +169,7 @@ impl StateAdapter for KvStoreAdapter { + cfg: &crate::barrier::BarrierConfig, + event: &Value, + ) -> anyhow::Result { ++ let _admission = self.admit().await?; + self.storage + .barrier_arrive(scope.to_string(), key.to_string(), cfg, event) + .await +@@ -172,6 +191,13 @@ impl StateAdapter for KvStoreAdapter { + async fn flush(&self) -> anyhow::Result<()> { + self.storage.flush().await + } ++ async fn seal_flush(&self) -> anyhow::Result { ++ let path = self.storage.file_store_path()?.ok_or_else(|| anyhow::anyhow!("STATE_SEAL_UNSUPPORTED"))?; ++ let _exclusive = self.mutation_gate.write().await; ++ self.sealed.store(true, std::sync::atomic::Ordering::Release); ++ self.storage.flush_and_sync().await?; ++ Ok(path) ++ } + async fn destroy(&self) -> anyhow::Result<()> { + Ok(()) + } +@@ -1046,7 +1072,7 @@ impl StateAdapter for RedisAdapter { + /// builtin's adapter registry). `bridge` is intentionally not ported. + pub async fn build_adapter(config: &StateConfig) -> anyhow::Result> { + match config.effective_adapter_name() { +- "kv" => Ok(Arc::new(KvStoreAdapter::new(config.adapter_config()))), ++ "kv" => Ok(Arc::new(KvStoreAdapter::from_store(KvStore::try_new(config.adapter_config())?))), + "redis" => { + let url = config + .adapter +@@ -1067,6 +1093,17 @@ mod tests { + use super::*; + use crate::config::StateConfig; + ++ async fn reopen_after_drop(config: Value) -> KvStore { ++ let deadline = tokio::time::Instant::now() + Duration::from_secs(2); ++ loop { ++ match KvStore::try_new(Some(config.clone())) { ++ Ok(store) => return store, ++ Err(error) if error.to_string().starts_with("STATE_WRITER_LOCK_UNAVAILABLE") && tokio::time::Instant::now() < deadline => tokio::time::sleep(Duration::from_millis(10)).await, ++ Err(error) => panic!("store failed to release after drop: {error}"), ++ } ++ } ++ } ++ + #[test] + fn cas_uses_json_value_equality() { + let stored = serde_json::from_str(r#"{"a":1,"b":2}"#).unwrap(); +@@ -1155,7 +1192,7 @@ mod tests { + // them to disk as one file per scope. + for _ in 0..50 { + if std::fs::read_dir(&dir) +- .map(|d| d.count() >= 2) ++ .map(|d| d.filter_map(Result::ok).filter(|e| e.path().extension().is_some_and(|x| x == "bin")).count() >= 2) + .unwrap_or(false) + { + break; +@@ -1163,7 +1200,8 @@ mod tests { + tokio::time::sleep(Duration::from_millis(100)).await; + } + +- let reloaded = KvStore::new(Some(config)); ++ drop(store); ++ let reloaded = reopen_after_drop(config).await; + assert_eq!( + reloaded + .get("claims".into(), "slot".into()) +@@ -1208,7 +1246,8 @@ mod tests { + a.set("s", "k", serde_json::json!({"v": 1})).await.unwrap(); + a.flush().await.unwrap(); + +- let reloaded = KvStoreAdapter::new(Some(config)); ++ drop(a); ++ let reloaded = KvStoreAdapter::from_store(reopen_after_drop(config).await); + assert_eq!( + reloaded.get("s", "k").await.unwrap().as_deref(), + Some(&serde_json::json!({"v": 1})) +@@ -1216,3 +1255,48 @@ mod tests { + std::fs::remove_dir_all(&dir).ok(); + } + } ++ ++#[cfg(test)] ++mod seal_tests { ++ use super::*; ++ fn fixture() -> (std::path::PathBuf, Arc) { ++ let path=std::env::temp_dir().join(format!("state-seal-{}", uuid::Uuid::new_v4())); ++ let adapter=Arc::new(KvStoreAdapter::new(Some(serde_json::json!({"store_method":"file_based","file_path":path,"save_interval_ms":3600000})))); ++ (path,adapter) ++ } ++ #[tokio::test] ++ async fn seal_waits_for_admitted_work_and_rejects_all_mutators() { ++ let (path,a)=fixture(); ++ a.set("s","k",serde_json::json!(1)).await.unwrap(); ++ let admitted=a.admit().await.unwrap(); ++ let b=a.clone(); let sealing=tokio::spawn(async move { b.seal_flush().await }); ++ tokio::time::sleep(Duration::from_millis(20)).await; ++ assert!(!sealing.is_finished()); drop(admitted); ++ assert_eq!(sealing.await.unwrap().unwrap(),path.canonicalize().unwrap()); ++ assert!(a.set("s","k",serde_json::json!(2)).await.unwrap_err().to_string().contains("SEALED")); ++ assert!(a.delete("s","k").await.is_err()); ++ assert!(a.update("s","k",vec![]).await.is_err()); ++ assert!(a.compare_and_set("s","k",None,serde_json::json!(2)).await.is_err()); ++ let cfg=crate::barrier::BarrierConfig{id:"b".into(),expect:crate::barrier::Expect::Count(1),key_from:None,carry:None}; ++ assert!(a.barrier_arrive("s","k",&cfg,&serde_json::json!({})).await.is_err()); ++ assert_eq!(a.get("s","k").await.unwrap().as_deref(),Some(&serde_json::json!(1))); ++ assert!(a.seal_flush().await.is_ok()); // retry never reopens admission ++ drop(a);tokio::time::sleep(Duration::from_millis(20)).await; ++ let reopened=KvStoreAdapter::new(Some(serde_json::json!({"store_method":"file_based","file_path":path}))); ++ reopened.set("s","k",serde_json::json!(3)).await.unwrap(); ++ reopened.flush().await.unwrap();drop(reopened);tokio::time::sleep(Duration::from_millis(20)).await; ++ std::fs::remove_dir_all(path).unwrap(); ++ } ++ #[tokio::test] ++ async fn sync_failure_stays_sealed_without_success_receipt() { ++ let (path,a)=fixture(); ++ std::fs::create_dir(path.join("invalid.bin")).unwrap(); ++ assert!(a.seal_flush().await.is_err()); ++ assert!(a.set("s","k",serde_json::json!(1)).await.unwrap_err().to_string().contains("SEALED")); ++ std::fs::remove_dir(path.join("invalid.bin")).unwrap(); ++ assert!(a.seal_flush().await.is_ok()); ++ assert!(a.set("s","k",serde_json::json!(1)).await.is_err()); ++ drop(a);tokio::time::sleep(Duration::from_millis(20)).await; ++ std::fs::remove_dir_all(path).unwrap(); ++ } ++} +diff --git a/state/src/boot.rs b/state/src/boot.rs +index e332978..6e7fddc 100644 +--- a/state/src/boot.rs ++++ b/state/src/boot.rs +@@ -88,6 +88,10 @@ pub async fn start(iii: Arc, config: StateConfig) -> anyhow::Result Result { ++ reject_reserved_scope(&self.private, &input.scope)?; ++ if let Some(limit) = self.snapshot().await.max_value_bytes { ++ let size = crate::json_budget::size_within(&input.value, limit) ++ .map_err(|error| Error::Handler(format!("VALUE_SERIALIZATION_ERROR: {error}")))?; ++ if size.is_none() { ++ return Err(Error::Handler(format!( ++ "VALUE_TOO_LARGE: value exceeds the configured max_value_bytes limit of {limit}" ++ ))); ++ } ++ } ++ let result = self ++ .adapter ++ .set(&input.scope, &input.key, input.value) ++ .await ++ .map_err(|e| { ++ Error::Handler(format!("SET_ERROR: Failed to set value: {e}")) ++ })?; ++ let et = if result.old_value.is_none() { ++ StateEventType::Created ++ } else { ++ StateEventType::Updated ++ }; ++ self.emit_change(et, &input.scope, &input.key, ++ result.old_value.as_ref(), &result.new_value) ++ .await; ++ Ok(result) ++ } ++ + async fn snapshot(&self) -> Arc { + self.config.read().await.clone() + } +@@ -355,41 +384,21 @@ fn event( + + pub fn register_functions(iii: &Arc, ctx: Arc) { + register_claim_namespace(iii, &ctx); ++ register_data_functions(iii, ctx, "state"); ++} ++ ++/// Boot-qualified aliases use these exact handlers, never a routable RPC proxy. ++pub fn register_data_functions(iii: &Arc, ctx: Arc, prefix: &str) { + + // state::set — max_value_bytes LIVE guard before the adapter write. + { + let ctx = ctx.clone(); + iii.register_function( +- "state::set", ++ format!("{prefix}::set"), + RegisterFunction::new_async(move |input: StateSetInput| { + let ctx = ctx.clone(); + async move { +- reject_reserved_scope(&ctx.private, &input.scope)?; +- if let Some(limit) = ctx.snapshot().await.max_value_bytes { +- let size = crate::json_budget::size_within(&input.value, limit) +- .map_err(|error| Error::Handler(format!("VALUE_SERIALIZATION_ERROR: {error}")))?; +- if size.is_none() { +- return Err(Error::Handler(format!( +- "VALUE_TOO_LARGE: value exceeds the configured max_value_bytes limit of {limit}" +- ))); +- } +- } +- let result = ctx +- .adapter +- .set(&input.scope, &input.key, input.value) +- .await +- .map_err(|e| { +- Error::Handler(format!("SET_ERROR: Failed to set value: {e}")) +- })?; +- let et = if result.old_value.is_none() { +- StateEventType::Created +- } else { +- StateEventType::Updated +- }; +- ctx.emit_change(et, &input.scope, &input.key, +- result.old_value.as_ref(), &result.new_value) +- .await; +- Ok(result) ++ ctx.set_value(input).await + } + }) + .description( +@@ -398,13 +407,25 @@ pub fn register_functions(iii: &Arc, ctx: Arc) { + ); + } + ++ { ++ let ctx = ctx.clone(); ++ iii.register_function( ++ format!("{prefix}::set_ack"), ++ RegisterFunction::new_async(move |input: StateSetInput| { ++ let ctx = ctx.clone(); ++ async move { ctx.set_value(input).await.map(|_| true) } ++ }) ++ .description("Store a value and return only a successful acknowledgement; not a disk flush receipt"), ++ ); ++ } ++ + // state::compare-and-set — the primitive a claim needs. `get` then `set` + // from outside cannot express "swap only if nobody moved it": two callers + // reading the same value both write, and both believe they won. + { + let ctx = ctx.clone(); + iii.register_function( +- "state::compare-and-set", ++ format!("{prefix}::compare-and-set"), + RegisterFunction::new_async(move |input: CompareAndSetInput| { + let ctx = ctx.clone(); + async move { +@@ -463,7 +484,7 @@ pub fn register_functions(iii: &Arc, ctx: Arc) { + { + let ctx = ctx.clone(); + iii.register_function( +- "state::barrier", ++ format!("{prefix}::barrier"), + RegisterFunction::new_async(move |input: BarrierInput| { + let ctx = ctx.clone(); + async move { +@@ -495,7 +516,7 @@ pub fn register_functions(iii: &Arc, ctx: Arc) { + { + let ctx = ctx.clone(); + iii.register_function( +- "state::get", ++ format!("{prefix}::get"), + RegisterFunction::new_async(move |input: StateGetInput| { + let ctx = ctx.clone(); + async move { +@@ -519,7 +540,7 @@ pub fn register_functions(iii: &Arc, ctx: Arc) { + { + let ctx = ctx.clone(); + iii.register_function( +- "state::delete", ++ format!("{prefix}::delete"), + RegisterFunction::new_async(move |input: StateDeleteInput| { + let ctx = ctx.clone(); + async move { +@@ -562,7 +583,7 @@ pub fn register_functions(iii: &Arc, ctx: Arc) { + { + let ctx = ctx.clone(); + iii.register_function( +- "state::update", ++ format!("{prefix}::update"), + RegisterFunction::new_async(move |input: StateUpdateInput| { + let ctx = ctx.clone(); + async move { +@@ -600,7 +621,7 @@ pub fn register_functions(iii: &Arc, ctx: Arc) { + { + let ctx = ctx.clone(); + iii.register_function( +- "state::list", ++ format!("{prefix}::list"), + RegisterFunction::new_async(move |input: StateGetGroupInput| { + let ctx = ctx.clone(); + async move { +@@ -626,7 +647,7 @@ pub fn register_functions(iii: &Arc, ctx: Arc) { + { + let ctx = ctx.clone(); + iii.register_function( +- "state::list_keys", ++ format!("{prefix}::list_keys"), + RegisterFunction::new_async(move |input: StateGetGroupInput| { + let ctx = ctx.clone(); + async move { +@@ -645,7 +666,7 @@ pub fn register_functions(iii: &Arc, ctx: Arc) { + { + let ctx = ctx.clone(); + iii.register_function( +- "state::list_groups", ++ format!("{prefix}::list_groups"), + RegisterFunction::new_async(move |_input: StateListGroupsInput| { + let ctx = ctx.clone(); + async move { +@@ -1221,6 +1242,16 @@ mod private_namespace_tests { + tokio::time::sleep(std::time::Duration::from_millis(80)).await; + } + ++ #[tokio::test] ++ async fn shared_set_path_propagates_adapter_failure_before_success() { ++ let ctx = test_ctx(Arc::new(FailFirstAdapter::default())); ++ let input = || StateSetInput { scope: "ack-test".into(), key: "k".into(), value: serde_json::json!(42) }; ++ let error = ctx.set_value(input()).await.unwrap_err(); ++ assert!(error.to_string().contains("SET_ERROR")); ++ assert!(error.to_string().contains("disk full")); ++ assert_eq!(ctx.set_value(input()).await.unwrap().new_value, serde_json::json!(42)); ++ } ++ + #[tokio::test] + async fn observers_get_the_document_trigger_handlers_get() { + let invoker = Arc::new(RecordingInvoker::default()); +diff --git a/state/src/lib.rs b/state/src/lib.rs +index 2592580..980fa82 100644 +--- a/state/src/lib.rs ++++ b/state/src/lib.rs +@@ -20,3 +20,5 @@ pub mod ui; + pub mod update_ops; + + mod json_budget; ++ ++pub mod boot_identity; +diff --git a/state/src/store.rs b/state/src/store.rs +index 99ee5e3..e599d99 100644 +--- a/state/src/store.rs ++++ b/state/src/store.rs +@@ -185,7 +185,7 @@ pub struct KvStore { + /// shutdown flush can find the map already empty while the loop's write + /// is still in flight — and process exit then cancels that write, + /// losing precisely the update this store is here to keep. +- flush_lock: Arc>, ++ flush_lock: Arc>>, + /// The boot-configured save cadence (ms, already floored). A reconfigure + /// that clears `save_interval_ms` reverts to this rather than to the global + /// default, so clearing the runtime knob restores the adapter's configured +@@ -193,8 +193,30 @@ pub struct KvStore { + default_interval: u64, + } + ++/// Advisory exclusion for cooperating State processes. Never unlink this file: ++/// the lock belongs to its inode, and replacement could create a second lock. ++fn lock_store(dir: &Path) -> anyhow::Result { ++ let path = dir.join(".agentmemory-state.lock"); ++ match std::fs::symlink_metadata(&path) { ++ Ok(meta) if !meta.file_type().is_file() => anyhow::bail!("STATE_WRITER_LOCK_INVALID"), ++ Ok(_) => {}, ++ Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}, ++ Err(error) => return Err(error.into()), ++ } ++ let file = std::fs::OpenOptions::new().read(true).write(true).create(true).truncate(false).open(path)?; ++ if !file.metadata()?.is_file() { anyhow::bail!("STATE_WRITER_LOCK_INVALID"); } ++ file.try_lock().map_err(|error| anyhow::anyhow!("STATE_WRITER_LOCK_UNAVAILABLE: {error}"))?; ++ Ok(file) ++} ++ + impl KvStore { ++ pub fn file_store_path(&self) -> anyhow::Result> { ++ self.file_store_dir.as_ref().map(|path| std::fs::canonicalize(path).map_err(Into::into)).transpose() ++ } + pub fn new(config: Option) -> Self { ++ Self::try_new(config).expect("failed to open State store") ++ } ++ pub fn try_new(config: Option) -> anyhow::Result { + tracing::debug!("Initializing KvStore with config: {:?}", config); + let store_method = config + .clone() +@@ -231,10 +253,8 @@ impl KvStore { + let file_store_dir = match store_method.as_str() { + "file_based" => { + let dir = file_path; +- if let Err(err) = std::fs::create_dir_all(&dir) { +- tracing::error!(error = ?err, path = %dir.display(), "failed to create storage directory"); +- } +- Some(dir) ++ std::fs::create_dir_all(&dir)?; ++ Some(std::fs::canonicalize(dir)?) + } + "in_memory" => None, + other => { +@@ -243,6 +263,9 @@ impl KvStore { + } + }; + ++ // Take the cooperative process lock BEFORE loading persisted state. ++ // The flush mutex owns the file so spawned blocking flushes retain it. ++ let writer_lock = file_store_dir.as_ref().map(|dir| lock_store(dir)).transpose()?; + let data_from_disk = match &file_store_dir { + Some(dir) => load_store_from_dir(dir), + None => HashMap::new(), +@@ -255,7 +278,7 @@ impl KvStore { + file_store_dir, + dirty, + save_loop_stop: Arc::new(std::sync::Mutex::new(None)), +- flush_lock: Arc::new(tokio::sync::Mutex::new(())), ++ flush_lock: Arc::new(tokio::sync::Mutex::new(writer_lock)), + default_interval: interval, + }; + +@@ -263,7 +286,7 @@ impl KvStore { + // nothing to persist. `spawn_save_loop` is a no-op when not file-backed. + kv.spawn_save_loop(interval); + +- kv ++ Ok(kv) + } + + /// (Re)start the background save loop at `interval_ms`, signalling any prior +@@ -317,7 +340,7 @@ impl KvStore { + async fn save_loop( + store: Store, + dirty: Arc>>, +- flush_lock: Arc>, ++ flush_lock: Arc>>, + polling_interval: u64, + dir: PathBuf, + mut stop_rx: tokio::sync::watch::Receiver, +@@ -352,7 +375,7 @@ impl KvStore { + async fn flush_dirty( + store: &Store, + dirty: &Arc>>, +- flush_lock: &Arc>, ++ flush_lock: &Arc>>, + dir: &Path, + ) -> anyhow::Result<()> { + // Move the lock into the blocking task BEFORE it drains dirty intent. +@@ -410,6 +433,24 @@ impl KvStore { + /// Flush pending dirty scopes to disk immediately — the shutdown path, + /// so a stop right after a write doesn't lose the last save window. + /// No-op for in-memory stores. ++ /// Called only after the adapter seals and drains mutation admission. ++ pub async fn flush_and_sync(&self) -> anyhow::Result<()> { ++ self.flush().await?; ++ let directory = self.file_store_dir.clone().ok_or_else(|| anyhow::anyhow!("STATE_SEAL_UNSUPPORTED"))?; ++ let guard = Arc::clone(&self.flush_lock).lock_owned().await; ++ tokio::task::spawn_blocking(move || { ++ let _guard = guard; ++ for entry in std::fs::read_dir(&directory)? { ++ let entry = entry?; ++ if entry.path().extension() != Some(OsStr::new(KEY_FILE_EXTENSION)) { continue; } ++ if !entry.file_type()?.is_file() { anyhow::bail!("STATE_SYNC_FILE_INVALID"); } ++ std::fs::File::open(entry.path())?.sync_all()?; ++ } ++ std::fs::File::open(&directory)?.sync_all()?; ++ Ok(()) ++ }).await? ++ } ++ + pub async fn flush(&self) -> anyhow::Result<()> { + match &self.file_store_dir { + Some(dir) => Self::flush_dirty(&self.store, &self.dirty, &self.flush_lock, dir).await, +@@ -1153,3 +1194,29 @@ mod cas_tests { + #[cfg(test)] + #[path = "store_memory_tests.rs"] + mod memory_tests; ++ ++#[cfg(test)] ++mod writer_lock_tests { ++ use super::*; ++ #[test] ++ fn writer_lock_excludes_alias_and_releases_on_drop() { ++ let dir = std::env::temp_dir().join(format!("state-lock-{}", uuid::Uuid::new_v4())); ++ std::fs::create_dir(&dir).unwrap(); ++ let first = lock_store(&dir).unwrap(); ++ assert!(lock_store(&dir.join(".")).is_err()); ++ drop(first); ++ let second = lock_store(&dir).unwrap(); drop(second); ++ std::fs::remove_dir_all(dir).unwrap(); ++ } ++ #[cfg(unix)] ++ #[test] ++ fn writer_lock_rejects_symlink_without_touching_target() { ++ let dir = std::env::temp_dir().join(format!("state-lock-{}", uuid::Uuid::new_v4())); ++ std::fs::create_dir(&dir).unwrap(); ++ let target = dir.join("sentinel"); std::fs::write(&target, "untouched").unwrap(); ++ std::os::unix::fs::symlink(&target, dir.join(".agentmemory-state.lock")).unwrap(); ++ assert!(lock_store(&dir).is_err()); ++ assert_eq!(std::fs::read_to_string(target).unwrap(), "untouched"); ++ std::fs::remove_dir_all(dir).unwrap(); ++ } ++} +diff --git a/state/src/store_memory_tests.rs b/state/src/store_memory_tests.rs +index abd55e3..ee04401 100644 +--- a/state/src/store_memory_tests.rs ++++ b/state/src/store_memory_tests.rs +@@ -13,7 +13,7 @@ pub(super) fn manual_store(dir: &Path) -> KvStore { + file_store_dir: Some(dir.to_path_buf()), + dirty: Arc::new(RwLock::new(HashMap::new())), + save_loop_stop: Arc::new(std::sync::Mutex::new(None)), +- flush_lock: Arc::new(tokio::sync::Mutex::new(())), ++ flush_lock: Arc::new(tokio::sync::Mutex::new(None)), + default_interval: 60_000, + } + } +diff --git a/state/tests/e2e_state.rs b/state/tests/e2e_state.rs +index 09b9c63..7f521c1 100644 +--- a/state/tests/e2e_state.rs ++++ b/state/tests/e2e_state.rs +@@ -903,8 +903,10 @@ async fn claim_namespace_lifecycle() { + #[tokio::test] + #[serial] + async fn claims_survive_a_state_restart() { +- let name = unique("e2e-phoenix"); +- let dir = std::env::temp_dir().join(unique("state-e2e-store")); ++ let child = std::env::var("STATE_CLAIM_RESTART_CHILD").ok(); ++ let (name, scope, dir) = if let Some(raw) = &child { ++ let values: (String, String, std::path::PathBuf) = serde_json::from_str(raw).unwrap(); values ++ } else { (unique("e2e-phoenix"), unique("phoenix"), std::env::temp_dir().join(unique("state-e2e-store"))) }; + let file_config = || { + serde_json::from_value::(json!({ + "adapter": {"name": "kv", "config": { +@@ -916,6 +918,7 @@ async fn claims_survive_a_state_restart() { + .expect("file-backed state config") + }; + ++ if child.is_some() { + // First life: claim, then wait for the ledger to flush to disk (the + // claim is the only write, so any file in the dir is the ledger). + let Some(iii) = engine::connect_fresh_named(&name).await else { +@@ -924,7 +927,6 @@ async fn claims_survive_a_state_restart() { + let boot = iii_state::boot::start(iii.clone(), file_config()) + .await + .expect("state worker should boot"); +- let scope = unique("phoenix"); + let claimed = call( + &iii, + "state::claim-namespace", +@@ -935,7 +937,7 @@ async fn claims_survive_a_state_restart() { + assert_eq!(claimed["claimed"], json!(true)); + for _ in 0..50 { + if std::fs::read_dir(&dir) +- .map(|d| d.count() >= 1) ++ .map(|d| d.filter_map(Result::ok).filter(|e| e.path().extension().is_some_and(|x| x == "bin")).count() >= 1) + .unwrap_or(false) + { + break; +@@ -944,7 +946,7 @@ async fn claims_survive_a_state_restart() { + } + assert!( + std::fs::read_dir(&dir) +- .map(|d| d.count() >= 1) ++ .map(|d| d.filter_map(Result::ok).filter(|e| e.path().extension().is_some_and(|x| x == "bin")).count() >= 1) + .unwrap_or(false), + "the claims ledger never flushed to {}", + dir.display() +@@ -952,6 +954,14 @@ async fn claims_survive_a_state_restart() { + boot.shutdown().await; + iii.shutdown_async().await; + ++ return; // Child process exit releases all SDK-held adapter references. ++ } ++ let output = std::process::Command::new(std::env::current_exe().unwrap()) ++ .args(["--exact", "claims_survive_a_state_restart", "--nocapture"]) ++ .env("STATE_CLAIM_RESTART_CHILD", serde_json::to_string(&(&name, &scope, &dir)).unwrap()) ++ .output().unwrap(); ++ assert!(output.status.success(), "child failed: {} {}", String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr)); ++ + // Second life: a fresh connection and a fresh boot over the same store. + // Restoration must re-reserve the scope and re-register the accessors + // without anyone re-claiming. +@@ -1080,3 +1090,112 @@ async fn cas_misses_and_barrier_fan_in_via_bus() { + boot.shutdown().await; + iii.shutdown_async().await; + } ++ ++ ++#[tokio::test] ++#[serial] ++async fn set_ack_returns_only_true_and_preserves_legacy_set() { ++ let iii = engine::connect_fresh().await.expect("explicit engine required"); ++ let boot = iii_state::boot::start(iii.clone(), in_memory_config()).await.unwrap(); ++ let scope = unique("compact-ack"); ++ let large = "x".repeat(1024 * 1024); ++ let ack = call(&iii, "state::set_ack", json!({"scope": scope, "key": "k", "value": large})).await.expect("compact ACK write"); ++ assert_eq!(ack, json!(true)); ++ assert_eq!(serde_json::to_vec(&ack).unwrap().len(), 4); ++ assert_eq!(call(&iii, "state::get", json!({"scope": scope, "key": "k"})).await.unwrap(), json!(large)); ++ let legacy = call(&iii, "state::set", json!({"scope": scope, "key": "k", "value": 2})).await.unwrap(); ++ assert_eq!(legacy["old_value"], json!(large)); ++ assert_eq!(legacy["new_value"], json!(2)); ++ boot.shutdown().await; ++ iii.shutdown_async().await; ++} ++ ++#[tokio::test] ++#[serial] ++async fn set_ack_preserves_size_and_reserved_scope_guards() { ++ let iii = engine::connect_fresh().await.expect("explicit engine required"); ++ let config = StateConfig { max_value_bytes: Some(8), ..in_memory_config() }; ++ let boot = iii_state::boot::start(iii.clone(), config).await.unwrap(); ++ let scope = unique("ack-reject"); ++ let err = call(&iii, "state::set_ack", json!({"scope": scope, "key": "k", "value": "too large"})).await.unwrap_err(); ++ assert!(err.to_string().contains("VALUE_TOO_LARGE"), "{err}"); ++ assert_eq!(call(&iii, "state::get", json!({"scope": scope, "key": "k"})).await.unwrap(), Value::Null); ++ let err = call(&iii, "state::set_ack", json!({"scope": iii_state::functions::CLAIMS_SCOPE, "key": "k", "value": 1})).await.unwrap_err(); ++ assert!(err.to_string().contains("RESERVED"), "{err}"); ++ boot.shutdown().await; ++ iii.shutdown_async().await; ++} ++ ++ ++#[tokio::test] ++#[serial] ++async fn set_ack_concurrent_writes_keep_each_value() { ++ let iii = engine::connect_fresh().await.expect("explicit engine required"); ++ let boot = iii_state::boot::start(iii.clone(), in_memory_config()).await.unwrap(); ++ let scope = unique("ack-concurrent"); ++ let mut tasks = tokio::task::JoinSet::new(); ++ for i in 0..16 { ++ let iii = iii.clone(); ++ let scope = scope.clone(); ++ tasks.spawn(async move { ++ let ack = call(&iii, "state::set_ack", json!({"scope": scope, "key": i.to_string(), "value": i})).await.unwrap(); ++ assert_eq!(ack, json!(true)); ++ }); ++ } ++ while let Some(result) = tasks.join_next().await { result.unwrap(); } ++ for i in 0..16 { ++ assert_eq!(call(&iii, "state::get", json!({"scope": scope, "key": i.to_string()})).await.unwrap(), json!(i)); ++ } ++ boot.shutdown().await; ++ iii.shutdown_async().await; ++} ++ ++#[tokio::test] ++#[serial] ++async fn boot_identity_is_stable_and_does_not_expose_config() { ++ let Some(iii) = engine::connect_fresh().await else { return }; ++ let boot = iii_state::boot::start(iii.clone(), in_memory_config()).await.unwrap(); ++ let first = call(&iii, "state::boot_identity", json!({})).await.unwrap(); ++ assert_eq!(first["version"], 1); ++ assert_eq!(first["pid"], std::process::id()); ++ assert_eq!(first["adapter"], "kv"); ++ assert!(first["storePath"].is_null()); ++ assert!(first["start"].as_str().unwrap().parse::().is_ok()); ++ assert!(Uuid::parse_str(first["instanceId"].as_str().unwrap()).is_ok()); ++ assert!(Uuid::parse_str(first["bootId"].as_str().unwrap()).is_ok()); ++ assert_eq!(first.as_object().unwrap().len(), 7); ++ assert_eq!(first, call(&iii, "state::boot_identity", json!({})).await.unwrap()); ++ boot.shutdown().await; ++ iii.shutdown_async().await; ++} ++ ++#[tokio::test] ++#[serial] ++async fn boot_aliases_keep_the_original_handlers_when_another_worker_registers() { ++ let Some(first) = engine::connect_fresh().await else { return }; ++ let boot1 = iii_state::boot::start(first.clone(), in_memory_config()).await.unwrap(); ++ let identity1 = call(&first, "state::boot_identity", json!({})).await.unwrap(); ++ let prefix1 = format!("state::{}", identity1["instanceId"].as_str().unwrap()); ++ let key = json!({"scope":unique("alias"),"key":"k"}); ++ let mut write = key.clone(); write["value"] = json!({"original":true}); ++ assert_eq!(call(&first, &format!("{prefix1}::set_ack"), write).await.unwrap(), true); ++ let Some(second) = engine::connect_fresh_named(&unique("alias-second")).await else { panic!("second connection unavailable") }; ++ assert_eq!(call(&second, &format!("{prefix1}::boot_identity"), json!({})).await.unwrap(), identity1); ++ let alias = format!("{prefix1}::get"); ++ second.register_function(alias.clone(), RegisterFunction::new_async(|_:Value| async { Ok::<_, Error>(json!({"hijacked":true})) })); ++ tokio::time::sleep(Duration::from_millis(100)).await; ++ let duplicate_result = call(&second, &alias, key.clone()).await.unwrap(); ++ println!("duplicate_alias_observed={duplicate_result}"); ++ assert_eq!(duplicate_result, json!({"original":true})); ++ boot1.shutdown().await; first.shutdown_async().await; ++ tokio::time::sleep(Duration::from_millis(100)).await; ++ // A rejected duplicate must not be revived when the old owner disconnects. ++ assert!(call(&second, &alias, key.clone()).await.is_err()); ++ let boot2 = iii_state::boot::start(second.clone(), in_memory_config()).await.unwrap(); ++ let identity2 = call(&second, "state::boot_identity", json!({})).await.unwrap(); ++ let prefix2 = format!("state::{}", identity2["instanceId"].as_str().unwrap()); ++ assert_ne!(prefix1, prefix2); ++ assert!(call(&second, &format!("{prefix1}::set_ack"), json!({"scope":"unused","key":"k","value":1})).await.is_err()); ++ assert_eq!(call(&second, &format!("{prefix2}::get"), key).await.unwrap(), Value::Null); ++ boot2.shutdown().await; second.shutdown_async().await; ++} +diff --git a/state/src/boot_identity.rs b/state/src/boot_identity.rs +new file mode 100644 +index 0000000..8d93fb6 +--- /dev/null ++++ b/state/src/boot_identity.rs +@@ -0,0 +1,82 @@ ++//! Read-only boot facts. These are evidence inputs, NOT authentication or a disk receipt. ++use std::sync::Arc; ++use iii_sdk::{IIIClient, RegisterFunction}; ++use iii_sdk::errors::Error; ++use serde_json::{Value, json}; ++use crate::adapters::StateAdapter; ++ ++fn process_start(stat: &str) -> anyhow::Result { ++ let (_, fields) = stat.rsplit_once(") ").ok_or_else(|| anyhow::anyhow!("process stat invalid"))?; ++ let start = fields.split_whitespace().nth(19).ok_or_else(|| anyhow::anyhow!("process start unavailable"))?; ++ start.parse::()?; ++ Ok(start.to_owned()) ++} ++ ++fn capture(adapter: &dyn StateAdapter, name: &str) -> anyhow::Result { ++ // Linux evidence is unavailable on other platforms; never invent a start time. ++ let start = process_start(&std::fs::read_to_string("/proc/self/stat")?)?; ++ let boot_id = std::fs::read_to_string("/proc/sys/kernel/random/boot_id")?; ++ let boot_id = uuid::Uuid::parse_str(boot_id.trim())?; ++ let store_path = adapter.file_store_path()?.map(|path| path.into_os_string().into_string()) ++ .transpose().map_err(|_| anyhow::anyhow!("non UTF-8 store path"))?; ++ Ok(json!({"version":1,"pid":std::process::id(),"start":start, ++ "instanceId":uuid::Uuid::new_v4().to_string(),"bootId":boot_id.to_string(), ++ "adapter":name,"storePath":store_path})) ++} ++ ++pub fn register(iii: &Arc, adapter: &Arc, name: &str) -> Option { ++ // Capture once from the built adapter. Mutable config is not authoritative. ++ // Failure disables only evidence; existing State behavior is unchanged. ++ let identity = capture(adapter.as_ref(), name).map_err(|_| "BOOT_IDENTITY_UNAVAILABLE".to_string()); ++ let instance = identity.as_ref().ok().and_then(|value| value["instanceId"].as_str()).map(str::to_owned); ++ let mut seal_ids = vec!["state::seal_flush".to_string()]; ++ if let Some(id) = &instance { seal_ids.push(format!("state::{id}::seal_flush")); } ++ for id in seal_ids { ++ let adapter = adapter.clone(); let identity = identity.clone(); ++ iii.register_function(id, RegisterFunction::new_async(move |_: Value| { ++ let adapter = adapter.clone(); let identity = identity.clone(); ++ async move { ++ let identity = identity.map_err(Error::Handler)?; ++ let path = adapter.seal_flush().await.map_err(|e| Error::Handler(format!("STATE_SEAL_FAILED: {e}")))?; ++ if Some(path.to_string_lossy().as_ref()) != identity["storePath"].as_str() { return Err(Error::Handler("STATE_SEAL_STORE_CHANGED".to_string())); } ++ Ok::(json!({"version":1,"instanceId":identity["instanceId"],"storePath":identity["storePath"],"sealed":true,"synced":true})) ++ } ++ }).description("Seal mutation admission and flush+fsync this instance; never reopens")); ++ } ++ let mut ids = vec!["state::boot_identity".to_string()]; ++ if let Some(id) = &instance { ids.push(format!("state::{id}::boot_identity")); } ++ for id in ids { ++ let identity = identity.clone(); ++ iii.register_function(id, RegisterFunction::new_async(move |_: Value| { ++ let identity = identity.clone(); ++ async move { identity.map_err(Error::Handler) } ++ }).description("Read immutable boot identity; requires independent owner validation")); ++ } ++ instance ++} ++ ++#[cfg(test)] ++mod tests { ++ use super::*; ++ #[test] ++ fn stat_parser_handles_spaces_and_parentheses() { ++ assert_eq!(process_start("42 (worker (name)) S 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 123 0").unwrap(), "123"); ++ assert!(process_start("42 bad").is_err()); ++ assert!(process_start("42 (name) S bad").is_err()); ++ } ++ #[tokio::test] ++ async fn identity_uses_constructed_store_not_changed_configuration() { ++ let directory = std::env::temp_dir().join(format!("state-identity-{}", uuid::Uuid::new_v4())); ++ std::fs::create_dir(&directory).unwrap(); ++ let adapter = crate::adapters::KvStoreAdapter::new(Some(json!({"store_method":"file_based","file_path":directory}))); ++ let before = capture(&adapter, "kv").unwrap(); ++ adapter.reconfigure(&json!({"file_path":"/not/the/store","save_interval_ms":100})).await.unwrap(); ++ let after = capture(&adapter, "kv").unwrap(); ++ assert_eq!(before["storePath"], directory.canonicalize().unwrap().to_str().unwrap()); ++ assert_eq!(before["storePath"], after["storePath"]); ++ assert_ne!(before["instanceId"], after["instanceId"]); ++ assert_eq!(before.as_object().unwrap().len(), 7); ++ adapter.destroy().await.unwrap(); ++ std::fs::remove_dir_all(&directory).unwrap(); ++ } ++} diff --git a/deploy/state-worker/pin.json b/deploy/state-worker/pin.json new file mode 100644 index 0000000000..ddb4abda49 --- /dev/null +++ b/deploy/state-worker/pin.json @@ -0,0 +1,17 @@ +{ + "schemaVersion": 1, + "artifactVersion": "0.22.22-daniel.1", + "upstreamRepository": "https://github.com/iii-hq/workers", + "upstreamTag": "state/v0.22.22", + "upstreamCommit": "a8ed4889f54cb34cb9ffdb6e44190484d90ba950", + "cargoPackageVersion": "0.22.21", + "patchSha256": "504bbb8a3b6b2eda1d4714f33c89f8ad47aa3af74dc61a99ee5efec76b9f4224", + "engineVersion": "0.25.1", + "capabilities": [ + "compact-ack", + "boot-identity-v1", + "instance-aliases", + "cooperative-store-lock", + "sealed-fsync-receipt" + ] +} diff --git a/deploy/systemd/README.md b/deploy/systemd/README.md new file mode 100644 index 0000000000..be8f0b1d60 --- /dev/null +++ b/deploy/systemd/README.md @@ -0,0 +1,25 @@ +# Ubuntu deployment-only Compose supervisor + +Not installed. No production process or state touched. + +`start.py CONFIG.json` runs the pinned package CLI, then publishes the actual Compose daemon PID for systemd Type=forking. Unlike Type=oneshot, systemd monitors that PID after the startup CLI exits. Unlike a foreground bootstrap, this reuses tested packaged CLI config/readiness/State artifact handling rather than duplicating it. + +Config fields: absolute canonical `cli`, `engine` and their `cliSha256`, `engineSha256`; `node`; `dataDir`; `metadata` (candidate CLI engine-state.json); `namespace`; canonical `composeFile`; `pidFile` within the service RuntimeDirectory. Parent deployment must root-publish reviewed launcher/config and verified package closure. Environment comes from reviewed service configuration; no new env file parsing here. + +Verification: exact Compose argv, executable, same uid/cgroup as launcher, fresh process start, matching metadata namespace/config/executable; recheck process and metadata before exclusive atomic PIDFile publication. Existing lifecycle/PIDFile fails closed; launcher never kills or adopts an existing daemon. Failure exit78 must retain metadata and prevent restart loops. A startup failure can leave candidate children alive for inspection; do not blindly retry. + +Template fragment requires real immutable paths and separate replacement of old ExecStartPre and old coldboot dependency. Preserve existing User/Group, home/config, timeout660, KillMode=process, SendSIGKILL=no. Do not leave old .68 guard/coldboot active for v4 Compose. Type=forking main PID is daemon, not worker; Compose owns child-worker restart policy. + +Isolated proof (2026-11-13): 4 Python tests passed, including 5 wrong process identity variants and 4 wrong metadata variants. Actual user-systemd transient Type=forking synthetic executable service became active/running MainPID1008555 equal published PIDFile; stopping unit reached inactive/MainPID0/success. Separate synthetic crash service MainPID1008957 killed by unit-scoped signal; systemd observed failed/Result=signal/ExecMainStatus9. Both synthetic processes exited and transient states cleaned. No live service affected. This proves Linux systemd PID handoff and failure observation, NOT a production system-unit deployment. A subsequent exact .69 packaged CLI smoke is recorded below. + +Restart=on-failure detects daemon death. It does not guarantee unattended recovery from an uncertain writer epoch: retained metadata causes exit78 and deliberate operator inspection. That is existing storage safety, not oneshot's false-active status. A clean stop must clear lifecycle metadata; RuntimeDirectory/PIDFile are managed by systemd. Never remove production metadata merely to satisfy this wrapper. + +## Exact .69 package integration smoke + +Tar SHA256 `a54be01bc6d5c2fa9b0f3142fa8de840afbd9e196b9f65e5de18c04615a5529b`, version `0.9.33-daniel.69`. Fresh synthetic HOME/data, copied verified local engine/State/HTTP, loopback ports, env scrubbed. Candidate node_modules were reused; this is not a fresh dependency-install test. Real user-systemd Type=forking launch and clean v4 restart both reached active/running with MainPID equal PIDFile. Authenticated health/status returned200, inner health healthy, mutationFailures0. Top-level degraded accurately reflected runtime SDK-only durability, not startup failure. Both stops produced completed/verified disk-seal receipts with syncedtrue. + +Important observed upstream stop race: first unit stop Resultsuccess; second ExecStop CLI exited0 and all processes stopped but Compose main exited1. Compose log `ENGINE_EXITED: managed engine exited with 0` confirms CLI stop terminates engine child before signaling daemon. systemd consequently reports Resultexit-code. No SuccessExitStatus1 override added: it would hide real daemon failure. Deployment must distinguish verified seal+all captured processes gone from this stop-status artifact; never equate arbitrary exit1 with success. + +### Final corrected package validation + +The stop-order issue above was reproduced by a failing regression, then corrected: after `compose::down`, revalidate and SIGTERM the daemon first without escalation; only then retire captured Linux residual child PID/start identities. Native compatibility remains separate. Final tar SHA256 `ce993e2111e5be3660cc4eaa444cebf163199caa25a010d52a68c21b339dde6a` passed two actual user-systemd lifecycle cycles with independently installed production dependencies: both start active/running with matching MainPID, authenticated health200/innerhealthy, stop exit0, inactive/MainPID0/Resultsuccess/ExecMainStatus0. Both receipts completed/verified with disk seal syncedtrue, zero mutation failures, distinct State instances, absent lifecycle metadata and exited app. This supersedes the stop-status limitation of the prior tar; no broad exit-status override was used. Production system-unit activation remains a separate authorized deployment step. diff --git a/deploy/systemd/agentmemory.service.fragment b/deploy/systemd/agentmemory.service.fragment new file mode 100644 index 0000000000..22671e8d87 --- /dev/null +++ b/deploy/systemd/agentmemory.service.fragment @@ -0,0 +1,17 @@ +# Template only: fill immutable release paths after approval. Do not install as-is. +[Service] +Type=forking +PIDFile=/run/agentmemory-compose/compose.pid +RuntimeDirectory=agentmemory-compose +RuntimeDirectoryMode=0700 +GuessMainPID=no +ExecStart= +ExecStart=/usr/bin/python3 -I -B APPROVED_DEPLOYMENT_DIR/start.py APPROVED_DEPLOYMENT_DIR/config.json +ExecStop= +ExecStop=/usr/bin/node RELEASE_DIR/dist/cli.mjs stop --data-dir /srv/agentmemory/data +Restart=on-failure +RestartPreventExitStatus=78 +TimeoutStartSec=900 +TimeoutStopSec=660 +KillMode=process +SendSIGKILL=no diff --git a/deploy/systemd/start.py b/deploy/systemd/start.py new file mode 100644 index 0000000000..3d71e71511 --- /dev/null +++ b/deploy/systemd/start.py @@ -0,0 +1,49 @@ +#!/usr/bin/python3 +"""Deployment-only Type=forking starter. No stop, kill, or data rewriting.""" +import hashlib,json,os,pathlib,subprocess,sys,tempfile +P=pathlib.Path + +def digest(path): + with open(path,'rb') as f:return hashlib.file_digest(f,'sha256').hexdigest() +def identity(pid): + root=P('/proc')/str(pid) + raw=(root/'stat').read_text(); fields=raw[raw.rindex(') ')+2:].split() + if fields[0]=='Z':raise RuntimeError('COMPOSE_ZOMBIE') + return (fields[19],os.readlink(root/'exe'),(root/'cmdline').read_bytes().split(b'\0')[:-1],(root/'cgroup').read_bytes(),root.stat().st_uid) +def validate(cfg,state,own_start): + if state.get('kind')!='compose':raise RuntimeError('COMPOSE_METADATA_REQUIRED') + pid=state.get('composePid') + if not isinstance(pid,int) or isinstance(pid,bool) or pid<=1:raise RuntimeError('COMPOSE_PID_INVALID') + if state.get('namespace')!=cfg['namespace'] or os.path.realpath(state.get('file',''))!=cfg['composeFile'] or os.path.realpath(state.get('binPath',''))!=cfg['engine']:raise RuntimeError('COMPOSE_METADATA_MISMATCH') + proof=identity(pid) + expected=[proof[2][0] if proof[2] else b'',b'compose',b'--namespace',os.fsencode(cfg['namespace']),b'--up',b'--file',os.fsencode(cfg['composeFile'])] + if expected[0] not in (os.fsencode(cfg['engine']),os.fsencode('iii:c:'+cfg['namespace'])) or int(proof[0]) The initial staging sections below are the historical PR #3 snapshot, not the final candidate status. The dated continuation sections supersede its engine/SDK and unexecuted-test statements. The latest functional-integration section supersedes the initial selective scope and its deferred feature list; historical evidence below is not rewritten. No production deployment is claimed. + +## Reproducible source pins + +- Official v0.9.32 base: `0da38f1d5067a479a6e5c4653de342ba43ca22b2`. +- Personal original head: `658f3fdc3350a64670b28b50789defca0eef462a` (`0.9.32-daniel.55`). +- Official v0.9.33 tag: `28ffc538b81b265e2f649354ed9f88b0c12ae414` (2026-11-09). +- Staged review branch: `integration/v0.9.33-review`; upstream mirror branch: `sync/upstream-v0.9.33`. + +Exactly **25** upstream v0.9.33 implementation/test files are imported as upstream Git blobs, but only after verifying that each file in the current `personal` tree is byte-for-byte the same blob as the upstream v0.9.32 base. This deliberately does **not** merge the entire v0.9.33 tree or replace any personalized code. The locally changed `test/cli-doctor-fixes.test.ts` is intentionally not overwritten; any missing upstream regression from that file requires a manual minimal port. + +## Staged upstream source candidates — no parity certification yet + +| Change | Imported source and relevant upstream regression | Remaining check | +| --- | --- | --- | +| #1350 / #1277 | Circuit-breaker success reset and content-filter errors not counted as provider outage | Ensure real HTTP 5xx still trip breaker. | +| #1696 | Hybrid search larger fusion pool, saved-memory diversification, refill after stale hits | Personal CJK/project isolation, large-store retrieval and old-memory ordering. | +| #1250 | CLIP text tower to produce comparable image/text embeddings | Same 512-dimension projections, no automatic re-embedding. | +| #1588 | Placeholder provider keys recognized consistently by config and doctor | Preserve explicit OpenAI selection and local plain-http exception. | +| #1703 | Claude `CLAUDE_CONFIG_DIR` path selection | Actual launch environment and native hook/MCP profile equivalence. | +| #1691 | MCP long-call timeout in source REST proxy/plugin bridge | Personal generated `.mjs` and CLI shim parity, no live timeout changes. | +| #1695 / #1701 | Redact before truncation and recognize compound secret names in core filter | Native/global bundled scripts and private capture path not yet updated/verified. | +| #1692 | Give core SessionEnd hook time to finish | Generated client bundles, source-first delivery and native retained originals. | + +## Deferred / non-replaceable without a separate parity gate + +Official iii/SDK 0.25.1, state scope paging, 64 MiB transport changes, engine auto-install and registry worker changes are **not** incorporated. The personal modified iii 0.22.1, SDK 0.25.1, exact KV save-loop patch, SDK write/flush certification, source-generation and writer identity rules stay unchanged. Official features with overlapping names are not proof of equivalent crash durability. The engine upgrade requires isolated real-engine + existing-format migration/rollback-read tests before any change to engine pin. + +Retain personal sourceRaw and incremental summary/cursor, capture inbox/original/retry/quarantine/uncertainty, native six-event hooks and private source retention, embedding publication evidence, background durable holds, health/Viewer, update trust and feedback store unchanged. Official #1698 orphan facet heal, #1601 eviction cleanup, #1687 partial-vector repair, #1545/#1544 consolidation/reflection, #1527 session filters, #1693/#1220 hook pre-tool context, remaining generated hook security and large-store fixes remain **unported or partly covered**. No custom code is retired. `docs/customizations/catalog.json` retains `unverified` decisions. + +## Mandatory review before promotion to personal or a deployment candidate + +1. From this exact source tree, run `npm run build`, `npm run skills:gen`, `npm run docs:sync`, `npm run skills:check`, `npm run docs:check` and a diff check. Commit all generated artifacts only after inspecting them. +2. Use isolated temporary HOME and Vitest default fork pool for `npm test`; run eval typecheck, compare known baseline diagnostics. Tests copied from upstream have **not been executed** while staging this source; passing test names do not constitute proof. +3. Verify bundled native/global Claude/Codex scripts reflect the reviewed source changes; preserve actual client hooks, original-first image/spool, project scope, strict secret redaction and 500/190-second timeout semantics where applicable. Never infer deployed behavior from an unchanged generated bundle. +4. Require release gate, isolated SDK/real-engine/storage cold-start/read-back/incomplete-write tests and existing data-format compatibility for any engine or persistence changes. No production store, settings, service, hook trust or backlog should be touched during this code review. +5. Remove a custom feature only in an independent follow-up after its complete parity and no-data-loss regression matrix passes. Preserve recorded history, old-format readers and uncertainty markers; no automatic old-memory replacement. + +Status: **prepared for source review only; build, tests, docs regeneration, generated-hook parity, live operating verification and deploy are not claimed.** + + +## 2026-11-13 reconciliation on the verified .68 source lineage + +The historical .55 review above is preserved. The new isolated source candidate +combines `08bb147d1c5146ca23d6955418f69880a8a05c18` (.68 + personal +lineage reconciliation) with PR #3 head +`8ecd207a7fc88073f78140a167ec731f80ae68f5`. This is source preparation, +not PR promotion, an engine upgrade, or deployment. The .68 Graph schema 3, +summary compact ACK, SDK retention/recovery and current Health behavior remain. + +- The only textual merge conflicts were the changelog and generated Codex manifest. + Both changelog histories were retained; the manifest was regenerated from source. +- `skills:gen` fixes the previously failing config reference by discovering + `AGENTMEMORY_CALL_TIMEOUT_MS` and `AGENTMEMORY_SESSION_END_WORKER`. +- The one missing official #1588 regression for placeholder OpenAI keys with a + plain-http local endpoint was ported into the personal doctor test without + replacing its additional custom tests. +- The imported SessionEnd regression exposed a missing official manifest change. + Add its 10-second timeout only to the repository Claude hook manifest; no live + client hook settings or Codex timeout policy are changed. +- No customization is certified as retired. The engine/worker gate remains + separate: the official State Worker does not yet provide the required + `state::set_ack` contract. Keep that guard and existing engine pin intact. +- Validation details and limitations are recorded in the adjacent workspace + `upstream33-reconcile-20261113/REPORT.md`; tests are not operating evidence. + +## SDK 0.25.1 isolated continuation + +The candidate retains compact ACK and durability protections. Clean builds pin SDK/helpers 0.25.1 and the already verified OpenTelemetry dependency set; YAML/JSON5 match the bundle audit. No runtime audit has been disabled. The synthetic SDK boundary regression passes with a 3 GiB host preflight (its child is bounded to 1 GiB RSS and a 512 MiB V8 heap), retaining a 1 GiB host low-memory abort plus time/CPU limits. This replaces only the test's disproportional 12 GiB reserve, not operating memory admission. The minimal separate State worker ACK patch remains custom until upstream provides equivalent behavior; a compact ACK alone is not a disk receipt. No production activation or customization retirement is claimed. + +## State-bound Compose candidate (isolated only) + +The candidate uses an explicitly selected local patched State worker. Immutable boot identity and instance-qualified handlers are combined with the CLI-selected executable SHA-256, effective store path, process start/UID/namespaces and established sockets to the verified engine. Storage calls check the bound epoch before and after the response; replacement makes old aliases/binding unusable. The official engine metadata does not provide authenticated registration-to-OS-PID mapping, so this is a local lifecycle/cooperative-writer safety contract, not a hostile same-user security boundary. A same-user process able to modify the data or executable is outside that claim. + +State-bound leases use v4; native leases remain v3. Existing v3 evidence is retained and is not silently upgraded into a State-bound proof. Registry workers without selected executable evidence fail closed. The bounded engine-evidence wait now finishes before lease creation, avoiding unsupported-marker creation after a slow metadata publication. Actual isolated State-only restart, fsync receipt, and real-binary CLI stop were verified; production migration and packaged registry/Docker deployment remain pending. + +The patched file worker additionally acquires a separate lifetime advisory `.agentmemory-state.lock` before loading the store, retaining it through background flush work. This excludes cooperating patched workers on the same canonical store; software ignoring advisory locks remains outside the operating contract. Actual lock contention, abnormal-exit takeover and normal-exit takeover are separately tested. This does not authorize deleting historical application lease markers. + +## Packaged local candidate and preserved native migration + +The later candidate supersedes the earlier migration/shutdown limitations above. `deploy/state-worker/README.md` documents the pinned patched-State source builder and artifact manifest; `docs/customizations/NATIVE-V3-STATE-V4-MIGRATION.md` documents an explicit native prepare/finalize flow that retains exact original v3 bytes and appends a receipt. First v4 admission verifies that receipt and records provenance; unknown native layouts or changed evidence fail closed. No operating store or marker was migrated by this work. + +The selected release worker now seals mutation admission, drains accepted mutations and flushes/fsyncs the namespace files and directory before issuing an instance-bound receipt. A real synthetic native .22.1→release-State migration preserved source/cursor/manual-hold/unknown fields, and subsequent app shutdown produced a verified disk receipt and clean marker, followed by successful restart/readback. The separate real 35,651,694-byte overwrite test retained the final digest and metadata after sealed flush and restart. These are isolated acceptance results, not claims of deployed client/server behavior. + +Docker source-build/bootstrap wiring is present and its real local bootstrap was exercised. Native Docker image build/run is still unverified: the current account cannot access the root-owned daemon socket, and the rootless prerequisite check reports missing `newuidmap`/`newgidmap` (`uidmap`). No permission, system package or security setting was changed to bypass this limit. + + +## Functional integration continuation (2026-11-13) + +The runtime feature gaps listed above are now implemented in this source candidate, rather than being inferred from the version tag: + +- State scope paging uses bounded key/get batches and instance-qualified State RPCs, preserving file writer epoch and source read guards. The actual pinned release State read 405 synthetic records totaling 74,649,600 bytes through five pages, with no whole-scope `state::list` call. The application payload budget now follows the verified 64 MiB transport with 60 MiB headroom; this is a transport limit, not a host-RAM reserve. `collectAll` still materializes its results and is not claimed constant-memory. +- Consolidation uses official current-head selection, provenance/idempotence keys, successor-first publication and interrupted-publication recovery. Procedural recurrence counts source sessions and actual re-saves, not edit versions. Original records, retained source, cursors, retries and unknown extension fields remain intact. +- Official bounded reflection evidence, citation checks, insight response limits/HTTP status and query validation are integrated. The custom extractive lifecycle remains selected where configured: upstream generative synthesis is not equivalent to its no-inferred-facts and opt-out contracts. Original full evidence is retained for mutation revalidation, not replaced with prompt snippets. +- Orphan-facet diagnosis/repair and eviction index removal are integrated. A facet or unindexed observation that reappears during revalidation is preserved. Historical partial-vector gaps trigger full recovery under the existing opt-in/pause/publication gates; failed watch releases remain owned and prevent a false clean shutdown. +- Session time filters, native PreToolUse envelope/project scope/timeout, Cursor subagent metadata, action creator identity, and official engine download retries/checksums are integrated. Actual host startup/client activation is unchanged. +- Compose versions are .25.1 engine, .22.22 State source and .22.8 HTTP; this fork's file-backed State still requires the verified ACK/epoch/seal artifact rather than the unpatched registry State. Provider selectors, summary timeout and local-dispatch setting reach all deployment templates; multiline/C1 YAML values round-trip safely. Windows demo shutdown uses guarded Compose ownership. +- The official configurable summary deadline and call headroom apply to the stock small-summary path. Custom long sessions remain bounded-input/checkpointed work rather than being replaced by an unbounded monolithic request. Machine-relative CPU display is adopted while the existing cgroup-aware capacity metric and drained health publication remain. + +No source-similarity comparison retires retained-source, writer lease, compact ACK, manual-review, graph-generation or recovery safety code. Official helpers replace duplicate behavior where equivalent; retained custom paths have explicit additional contracts. Supporting release checks and operational documentation are integrated, while upstream marketing pages are intentionally not copied as claims about this fork. + +Final combined regression results and remaining platform/package gates are recorded in the continuation report. No production data/service/hooks, durable memories, remote branches or PR state were changed. The package keeps its internal candidate version until release approval; the installed operating version is not inferred from this checkout. + +### Final Ubuntu acceptance + +Final combined regression: 585 files passed, 5 skipped; 6,978 tests passed, 14 skipped; zero failures. Build, generated Codex artifacts, documentation synchronization and 17-skill checks passed. TypeScript reports 29 existing baseline diagnostics, with no new normalized diagnostic; this is not a clean typecheck claim. + +The actual Ubuntu installed-tar gate passed installation/identity, bundled-hook capture and search, installed MCP save/search, verified stop/start data retention, and Viewer/status, using isolated homes/ports/store and the verified release State + HTTP 0.22.8 + iii 0.25.1. The local fixture provider was synthetic; no user's provider or operating data was used. + +The user's latest scope explicitly excludes Docker and Windows environment validation. They are not blockers for this Ubuntu-only request, and no cross-platform support is certified here. Operating server activation and remote Git publication remain separate from these completed isolated checks. + +### Authorized Ubuntu rollout candidate (.69) + +The owner subsequently authorized Ubuntu operating deployment after a recoverable backup, followed by GitHub publication after normalization. Candidate identity is now `0.9.33-daniel.69`; approval is not deployment evidence. The live baseline remains native `.68` until the explicit rollout executes. Docker/Windows validation remains out of scope. + +Deployment preparation preserves the original native migration proof and accepts only the same-epoch clean:false→true transition caused by a verified normal shutdown. Source bytes, cursors and uncertainty history are not reset. The systemd forking wrapper verifies the actual Compose executable, argv, cgroup, UID and process start before publishing its PIDFile; preexisting metadata requires inspection rather than adoption. Existing custom provider/summary/receipt environment keys are retained through the Compose filter. + +Dependency audit reports the preexisting YAML 2.8.1 deeply nested document stack-overflow advisory GHSA-48c2-rrv3-qjmp. This rollout parses generated/local configuration, not an added remote YAML ingestion endpoint. No unrelated dependency auto-fix was applied; the finding is retained for follow-up rather than concealed. + +Final .69 rollout preparation: 6,991 tests passed, 14 skipped (585 files passed, 5 skipped). A real installed-tar/systemd two-cycle test found and then eliminated a normal-stop ordering race: Compose now retires its own engine before verified residual-child cleanup, with no force-kill escalation. Both final cycles report inactive/Result=success/exit0, verified disk seals and retained-state restart. The final tar SHA256 is `ce993e2111e5be3660cc4eaa444cebf163199caa25a010d52a68c21b339dde6a`. These remain isolated proofs; the live deployment requires the explicit privileged runner. + +## Ubuntu operating acceptance, 2026-11-13 UTC + +The owner applied `.71` after a verified clean disk shutdown of `.70`, all prior processes exiting, and offline hash-verified backups. The service resumed through the supported deployment control. The 10,000 retained SDK originals matched their pre-upgrade hashes exactly (no missing or changed rows); all 431 sessions, including the pre-restart synthetic verification session, survived. Existing 38,032 observations and 61 memories were preserved. One synthetic observation was then accepted and returned by project-scoped search; an SDK-v1 duplicate request returned the same observation ID without a second observation. Health reported running/no holds, zero mutation/SDK failures, and vector backfill zero. + +Completion remains explicitly **SDK**, not per-request disk durability. A legacy-format verification request received HTTP 503 `SDK_COMPLETION_ONLY` after acceptance; the actual hook SDK-v1 contract returned HTTP 200 `duplicate`, `completed:true`, `durable:false`. It was not blindly replayed as a new event. Verified disk shutdown and offline retention are separate evidence. + +Final broad regression: 7,020 passed, 18 failed, 14 skipped. Failure investigation found a concurrent rebuild briefly removing CLI output (15), a process-load deadline case (1), a test which loaded the pre-fix lock implementation (1), and an obsolete SDK archive capacity expectation (1). Frozen final output reruns passed 93/93 tests; updated capacity/restore/reconciliation suites passed 86/86. This is combined regression evidence, not a claim that the initial full invocation was green. TypeScript has the same 29 pre-existing normalized diagnostics with no additions; docs/skills consistency checks pass. + +Remaining limitations: the scoped search took about 48 seconds (a preceding 20-second request timed out); legacy audit history remains preserved outside monthly audit queries; explicit capture/index status checks can remain unavailable under load. Retained SDK archives have no new disk quota and are not represented as semantically resolved or safely deletable. Historical spool recovery is separate from new-event admission. Docker and Windows execution tests were expressly excluded by the owner. GitHub CI must be evaluated for the published commit, not inferred from these local results. + +### Post-acceptance publication hold + +A subsequent live health sample reported `mutationFailures:1` while SDK validation failures remained zero and the service still reported healthy/running. Publication was withheld before commit/push. The successful source-preservation/store/search/dedup evidence above remains valid, but full normalization is **not established**. Feature-lifecycle and observation-context warnings are not sufficient to identify the initiating error. Current diagnostics do not retain the first mutation exception, so its exact dispatch/outcome must not be inferred or erased by restarting. Investigation is continuing; no blind queue replay or clean shutdown claim is authorized by the earlier healthy sample alone. + +### .72 socket-snapshot correction and recovery (2026-11-14 UTC) + +Rotated .71 diagnostics subsequently identified `STATE_ENGINE_SOCKET_OWNER_UNVERIFIED` before and after dispatch. Independent isolated reproduction showed a concrete false-negative path: the engine accepts another legitimate connection after its FD enumeration but before TCP rows are read. The captured inode set omits that new engine socket, even while the original verified connection remains intact; the next stable observation succeeds. This proves the non-atomic observation defect, not the exact timing or semantic outcome of every historical live mutation. + +The .72 fix makes at most three complete State observations for only missing-connection/socket-owner mismatch, keeping the same engine epoch and rechecking process/executable/UID/namespaces on each pass. Persistent foreign ownership, changed identity and ambiguous after-dispatch outcomes remain failures. Local before-dispatch admission rejections use process-local evidence rather than trusting a remotely supplied error code; they are not promoted to uncertain dispatched writes. Regression coverage includes `test/state-local-admission.test.ts`. + +The owner applied .72 through the explicit sealed-not-clean recovery path. The .71 failure count of one and its uncertain outcome were preserved in the recovery receipt; no clean marker was fabricated and no original was removed. The original 10,000 retained hashes remain identical (10,001 total after the accepted synthetic event). Supported manual resume returned running with no holds. Final bounded acceptance: seven health samples from 02:01:30 to 02:04:46 UTC were healthy/running with no holds, mutation failures or SDK failures; first/latest mutation errors were null, while SDK completed work increased from 12 to 43. A new SDKv1 synthetic capture returned 201 stored, scoped search returned 200 with its exact ID in 48.993 seconds, and duplicate submission returned 200 with the same ID; independent disk inspection found exactly one record. This certifies the tested Ubuntu capture/search/dedup and bounded stability, not unlimited future stability or legacy-backlog resolution. The earlier .71 acceptance and subsequent publication hold above remain historical evidence, not overwritten success claims. + +Final status HTTP 200 reported 38,034 observations (38,032 original plus two accepted synthetic probes), 61 memories and 431 sessions, missing observations 0, BM25 complete, vector backfill pending 0/idle. Remaining informational codes were `audit-legacy-frozen` and `capture-check-unavailable`; these are not silently treated as completed legacy cleanup. + + +### PR CI portability and fixture timing follow-up (2026-11-14) + +- The first published candidate CI failed 26 tests: 24 subprocess fixtures assumed `/usr/bin/node`, one capture test assumed six contending processes and 6,000 retained files always finish inside three seconds, and one Chromium DOM dump lacked its fixture marker. The browser failure did not retain enough evidence to identify the missing document conclusively; it is not evidence of a production Viewer regression. +- Tests now use the active Node executable. Capture checks retain the original three-second budget, inspect conservative no-send outcomes and on-disk state, and require eventual exact ACK/completed originals with exactly one POST per event. Only proven never-sent cases retry after contention ends; uncertain or sending states still fail. A deliberate lock overrun covers the exhausted-budget path deterministically. +- Browser checks wait for the expected fixture document and its completed proof, with bounded CDP requests and preserved UI assertions, instead of assuming a fixed virtual-time dump is ready. +- These changes affect test fixtures only. Operating `.72`, data, services and configured deadlines are unchanged. Final aggregate and remote CI results are reported for their exact commit; focused results alone do not establish a green PR. +- Final local verification: full isolated-HOME fork-pool suite passed 7,045 tests (14 intentionally skipped), 590 files passed (5 skipped), no failures. Build, skill generation/check, documentation synchronization/check, Codex artifact generation with no generated drift, and eval TypeScript contract passed. Remote CI remains a separate gate. +- Second CI run passed 7,043 tests and exposed two additional fixture races: a just-created but incomplete Chrome endpoint file could become a default-port-80 request, and an 80ms synthetic filesystem fence could conservatively retain publication retry before the test assumed a committed file existed. Endpoint readiness now validates the complete port/endpoint pair (six deterministic tests). Synthetic successful I/O uses a two-second test budget, while a separate injected slow-read regression keeps the 80ms failure path and verifies retry retention, no committed publication and single inference. Runtime budgets remain unchanged. +- After both race fixes, the final isolated full suite passed 7,052 tests (14 skipped), 591 files passed (5 skipped), no failures; all preceding build/generation/documentation/skills/eval checks passed with no generated drift. diff --git a/docs/reflection-evidence.md b/docs/reflection-evidence.md index f02849a42a..9f5f7657c6 100644 --- a/docs/reflection-evidence.md +++ b/docs/reflection-evidence.md @@ -1,6 +1,6 @@ # Reflection evidence -Reflection connects related facts, lessons, and completed work into insights. Each generated insight must cite at least two distinct source records from the supplied cluster. Stored `sourceMemoryIds`, `sourceLessonIds`, and `sourceCrystalIds` contain only the cited records; an uncited cluster member is not evidence for the result. +Reflection connects related facts, lessons, and completed work into insights. Each generated insight must cite between two and twenty distinct source records from the supplied cluster. Stored `sourceMemoryIds`, `sourceLessonIds`, and `sourceCrystalIds` contain only the cited records; an uncited cluster member is not evidence for the result. `sourceMemoryIds` retains its existing field name and contains semantic-fact IDs. Selected records are read again before an insight is stored, and changed or deleted evidence is rejected. Subsequent source changes do not automatically invalidate an already saved insight. @@ -10,6 +10,18 @@ Evidence validation checks that citations identify supplied records. It does not The reflection result includes `rejectedInsights` for invalid or repeated candidate insights and `failedClusters` for unparseable or unterminated responses, provider failures, or persistence failures. An empty `` response is an abstention. A failed cluster sets `success: false`; results from successfully processed clusters remain available. +## Size and result limits + +Reflection selects complete records in store order, with at most 20 facts, 20 lessons and 20 crystals per cluster. Each type has a 16 KiB budget measured as UTF-8 serialized JSON, including source IDs and label overhead. A record exceeding 4 KiB is skipped; later smaller records can still qualify. Records are never truncated, and citation labels are assigned after selection. A cluster must retain at least three records to reach the model. + +Each cluster retains at most 32 distinct concept names, each fitting 128 bytes of serialized JSON. The combined system and user prompt is limited to 64 KiB. Model responses exceeding 64 KiB fail the cluster before parsing. Generated titles fit 256 UTF-8 bytes and insight content fits 4 KiB. Oversized candidates and citations outside the supplied evidence are rejected. Reinforcement replaces source references and concept metadata with the newly validated, bounded evidence. + +`maxClusters` defaults to 10 and is capped at 20. Insight list and search limits default to 50 and 10 respectively and are capped at 500. Supplied counts must be positive safe integers; `minConfidence` must be finite and between 0 and 1. Invalid options return a small error, with HTTP 400 through REST and daemon MCP. + +Insight results and their REST/MCP envelopes are checked against the 60 MiB transport safety budget (`SAFE_PAYLOAD_BYTES`). An oversized response returns a small refusal with HTTP 413, including when escaping JSON inside an MCP text response expands its size. Reduce the limit or narrow the project/search filter. Oversized legacy records are preserved in storage. + +These limits bound model input, newly accepted provenance and outgoing insight responses. They do not bound reflection's clustering work. Reflection reads semantic memories, lessons and crystals in pages, so a scope larger than the transport limit is still read. A scope whose read fails is treated as empty for that run. Other list and export endpoints retain their own contracts. + ## Test the packaged runtime ```sh @@ -24,6 +36,6 @@ To validate an existing artifact: node scripts/release-gate/reflection.mjs --tarball /path/to/package.tgz --out /path/to/receipts ``` -The deterministic local model supplies accepted, forged, repeated, missing, and malformed citations. The gate also updates a cited fact through the import API while generation is in progress and verifies that the stale result is rejected. It exercises REST, daemon MCP, and the packed standalone MCP bridge; verifies project-specific storage identities and reinforcement; and confirms citations survive daemon restart. It also verifies provider failure reporting and checks that installed package files remain unchanged. +The deterministic local model supplies accepted, forged, repeated, missing, excessive, and malformed citations. It checks bounded evidence selection, oversized model output, count validation, large legacy result refusal and MCP escaping overhead. The gate also updates a cited fact through the import API while generation is in progress and verifies that the stale result is rejected. It exercises REST, daemon MCP, and the packed standalone MCP bridge; verifies project-specific storage identities and reinforcement; and confirms citations survive daemon restart. It also verifies provider failure reporting and checks that installed package files remain unchanged. `summary.json`, `wire.json`, and `model-requests.json` record the results and synthetic traffic. Temporary runtime state is removed unless `--keep` is specified. The gate stops only the processes it started. It uses no external model credentials and evaluates the runtime contract rather than model reasoning quality. diff --git a/docs/reliability/compression-recovery.md b/docs/reliability/compression-recovery.md index 8e5fa1d06b..3cc166f9d6 100644 --- a/docs/reliability/compression-recovery.md +++ b/docs/reliability/compression-recovery.md @@ -34,4 +34,4 @@ The authenticated REST equivalents are: Both accept `project`, `agentId`, `limit` and `cursor`; at least `sessionId` or `project` is required. Results contain identifiers, status, safe error codes and counts. They do not return retained source or raw provider errors. A retry result with `success: false` exits the CLI with status 1. Inspect `recovered`, `items[].outcome` and `items[].retryable` for each batch; source-unavailable or concurrently changed rows are not model successes. -Responses also obey the engine's 15 MiB payload allowance. If unusually large imported identifiers exceed it, REST returns HTTP 413 with a small error. Lower `--limit` or select one session. Already completed retries are not repeated when you inspect again. +Responses also obey the engine's 60 MiB payload allowance. If unusually large imported identifiers exceed it, REST returns HTTP 413 with a small error. Lower `--limit` or select one session. Already completed retries are not repeated when you inspect again. diff --git a/docs/reliability/graph-index-migration.md b/docs/reliability/graph-index-migration.md index 389e1126d3..789ffc854f 100644 --- a/docs/reliability/graph-index-migration.md +++ b/docs/reliability/graph-index-migration.md @@ -14,7 +14,7 @@ To build indexes for existing graph data, schedule a maintenance window and expl agentmemory graph-index migrate --allow-legacy-scan --json ``` -**Initial discovery is not bounded.** The pinned engine cannot paginate its state listings. Migration reads each complete legacy node, edge and history scope once, including full record values. Large stores can consume substantial memory and delay other requests during those reads. An HTTP timeout does not cancel an engine allocation already in progress. No ordinary search, status request or startup silently authorizes this scan. +Initial discovery scans each canonical node, edge and history scope. File-backed node and edge scans retain only IDs for the manifest. Other backends can still require a complete state listing. Large stores can delay other requests during discovery. An HTTP timeout does not cancel an engine allocation already in progress. No ordinary search, status request or startup silently authorizes this scan. The command returns a job ID after saving its intent. It runs in the background in an isolated index build within the current graph generation; canonical records are preserved. Capture, imports, mesh updates, temporal changes and snapshot restores maintain that build while migration runs. Status reports discovered and processed totals, readiness and partial coverage. @@ -38,6 +38,8 @@ Progress and recovery journals use the configured state backend. An acknowledged Oversized source fields remain in canonical storage. Bounded projections record partial coverage when they cannot represent a field fully, and status exposes the affected record count. A ready index with partial coverage does not claim every graph fact is searchable. Search budgets and pagination can also truncate a request; callers should check its graph diagnostics. +An explicit migration can replace a ready schema-2 index with schema 3. Schema 3 stores each build in independent shards. The previous index remains on disk. The server verifies the new inventory before declaring it ready. + Indexes add storage and write overhead. A replacement build isolates previous derived records by build ID; it does not reclaim their disk space. Repeated rebuilds and graph generations can therefore accumulate retained data. Bounded records and reads do not imply bounded total storage. The CLI honors `AGENTMEMORY_URL`, local `--port` and `--instance` selection, and the existing credential resolution. Named instances sharing a home directory share its saved API credential by default. To use separate credentials, set a different `AGENTMEMORY_SECRET` for each instance and its clients, or use separate home directories. Equivalent authenticated endpoints are `GET /agentmemory/graph/index`, `POST /agentmemory/graph/index/migrate`, and `POST /agentmemory/graph/index/resume`. Mutation bodies accept only the boolean `allowLegacyScan` flag. Repeated start requests join the running job. Use one AgentMemory graph worker per state store; the writer lock coordinates operations within that worker. diff --git a/docs/superpowers/plans/2026-11-01-record-gate-lifetime.md b/docs/superpowers/plans/2026-11-01-record-gate-lifetime.md index 4c0978b259..72fb774132 100644 --- a/docs/superpowers/plans/2026-11-01-record-gate-lifetime.md +++ b/docs/superpowers/plans/2026-11-01-record-gate-lifetime.md @@ -6,7 +6,7 @@ **Architecture:** Each queued/running/reentrant record use owns a reference. At zero references, require the inner gate to have no reader/writer/queue, disable its ALS, then remove that entry. Keep the record-wide exclusive ALS and unrelated gate contexts alive. -**Tech stack:** existing TypeScript/Node22.23.3, iii-sdk0.25.0/engine0.22.1. +**Tech stack:** existing TypeScript/Node22.23.3, iii-sdk0.25.1/engine0.22.1. **Spec:** delegated 2026-11-01 ALS minimal-fix request and ../observation-work-selection-development/performance50-isolated/ROOT-CAUSE.md. diff --git a/docs/superpowers/plans/2026-11-07-capture-inbox.md b/docs/superpowers/plans/2026-11-07-capture-inbox.md new file mode 100644 index 0000000000..0348023f45 --- /dev/null +++ b/docs/superpowers/plans/2026-11-07-capture-inbox.md @@ -0,0 +1,50 @@ +# Capture Inbox Recovery Implementation Plan + +> **For agentic workers:** Use subagent-driven development for the disjoint inbox, explicit source recovery and deployment tasks; primary agent reviews the integrated candidate. + +**Goal:** Restore actual capture and indexing without oversized inbox SDK responses or loss of retained originals. + +**Architecture:** Reuse the file namespace reader and acknowledged-write overlay, project metadata during streaming, and load only selected originals. Keep explicit source restore on the existing product protocol and apply .60 through certified normal shutdown. + +**Tech Stack:** Existing TypeScript/Node, iii SDK/native engine, Python deployment tools, Vitest fork pool. + +**Spec:** `docs/customizations/CAPTURE-INBOX-60.md`. + +## Global Constraints + +- No production data/config for development tests; synthetic data and private network/mount namespace only. +- No raw-source loss, false disk-completion claims, data rollback, or change to issued .59 artifacts. +- Temporary real HOME and default fork pool for tests. Preserve installed compatibility hooks in the server-only artifact. + +## Review Focus + +- A projected record cannot satisfy a source-digest proof without separately loading and validating that original. +- Mutating returned values must not modify acknowledged overlay state. +- Deletes, concurrent file replacement and SDK write uncertainty cannot yield stale success. +- Thousands of already acknowledged originals must not require an oversized response or prevent health/capture progress. +- A lost restore response and delayed competing response must not acknowledge another attempt or trigger blind resends. + +## Task 1 — Inbox reads + +- [x] Reproduce original native .59 overflow and write failing capture entry regression. +- [ ] Add exact inbox scope to existing local reader; support projected reads through StateKV with unchanged full-list and backend fallback behavior. +- [ ] Use a shared metadata/preview projection in capture admission, sweep, status, retry, list, review and retry jobs. +- [ ] Reuse original completion verifier only after selected source and metadata agree; test invalid proof and changing source. +- [ ] Run focused regressions, then integrated tests/build and packaged native scale acceptance. + +Files: `src/ops/graph-file-list.*`, `src/state/kv.ts`, `src/functions/capture.ts`, `src/capture/{review,retry-jobs}.ts`, a shared capture projection helper, and focused capture/reader tests. + +## Task 2 — Selected source recovery + +- [ ] Reuse existing delivery for selected pending references. +- [ ] Implement explicit held-source claim and restore using `sdk-spool-v2.ts` and `sdk-delivery-v2.ts`; preserve source hashes, owner checks and acknowledgement semantics. +- [ ] Package a standalone operator-imported recovery entry; normal hooks do not invoke it. +- [ ] Test durable intent/response-loss/owner race handling with synthetic sources before operational use. + +## Task 3 — Deployment and acceptance + +- [ ] Update changelog, generate/check docs and skills, test and build, commit and freeze a separate .60 server-only candidate. +- [ ] Prepare normal .59→.60 deployment with exact previous receipt/code/engine bindings and physical retained-WAL coverage; use safe viewer snapshot preflight rather than .59 `/status`. +- [ ] Run actual generated guard/archive/read-back and normal transaction regressions, review and freeze the administrator enrollment command. +- [ ] After administrator activation, confirm exact .60 generation, resume through the official API, recover selected sources and verify actual SDK approval plus per-item physical indexing. +- [ ] Observe capture/embedding progress, SDK reconnects and service restarts; record implementation, deployment, limits and pending memory review separately. diff --git a/docs/upgrades/iii-0.25.md b/docs/upgrades/iii-0.25.md index c91039f2ca..7a3afea7c1 100644 --- a/docs/upgrades/iii-0.25.md +++ b/docs/upgrades/iii-0.25.md @@ -1,21 +1,16 @@ # Upgrading the AgentMemory runtime to iii 0.25 -This release pins the engine, `iii-sdk` and `@iii-dev/helpers` to 0.25.0. Worker Compose starts the existing `state@0.22.20` and `http@0.22.7` registry workers with AgentMemory. Their versions are independent of the engine version. +This integration candidate pins the engine, `iii-sdk` and `@iii-dev/helpers` to 0.25.1 and HTTP to 0.22.8. File-backed State requires the separately verified `0.22.22-daniel.1` artifact, not the unmodified registry worker: compact ACK, instance-bound routing, exclusive store ownership and sealed disk flush are still required. Worker versions are independent of the engine version. This document is preparation guidance, not evidence of production deployment. ## Before upgrading -Stop the running instance with its currently installed CLI before replacing the package: +For this custom fork, use the verified release artifacts and the [native v3 to State v4 migration procedure](../customizations/NATIVE-V3-STATE-V4-MIGRATION.md). Do not replace the package with an arbitrary `@latest`, delete lease markers, or start an unmodified State worker against the existing file store. Configure `AGENTMEMORY_STATE_WORKER_ARTIFACT` with the verified manifest-bearing artifact directory. The migration preserves original data and requires current-process evidence followed by offline receipt validation; a prior backup is not a reason to rewind current data. -```sh -agentmemory stop -npm install -g @agentmemory/agentmemory@latest -agentmemory -agentmemory status -``` +No operational stop, installation or migration is authorized merely by preparing this candidate. Schedule the deployment explicitly after its applicable acceptance gates pass. Use the same `--instance`, `--port` and `--data-dir` flags you used to start it. Preserve the data directory and `~/.agentmemory`, including its secret. The new CLI will not adopt a running engine with a different version or overwrite a user-managed global iii binary. On macOS and Linux it installs the verified engine under `~/.agentmemory/bin`; native Windows still requires manual installation of the pinned binary. -Existing file state remains at `state_store.db`; this upgrade does not convert or delete it. Docker deployments keep their persistent `/data` volume and use the shared `deploy/Dockerfile`. `AGENTMEMORY_USE_DOCKER` is no longer a CLI startup mode. Stop a legacy Docker-managed runtime using the command printed by the CLI, then follow the [deployment guide](../../deploy/README.md). Keep `AGENTMEMORY_STATE_BACKEND=redis` and `AGENTMEMORY_REDIS_URL` together when using Redis. +Existing file state remains at `state_store.db`; the verified lease migration does not rewrite or delete its records. Docker deployments keep their persistent `/data` volume and use the shared `deploy/Dockerfile`. `AGENTMEMORY_USE_DOCKER` is no longer a CLI startup mode. Stop a legacy Docker-managed runtime using the command printed by the CLI, then follow the [deployment guide](../../deploy/README.md). Keep `AGENTMEMORY_STATE_BACKEND=redis` and `AGENTMEMORY_REDIS_URL` together when using Redis. ## Capabilities and compatibility @@ -24,6 +19,7 @@ Existing file state remains at `state_store.db`; this upgrade does not convert o - `/agentmemory/status` reports the configured engine pin as `service.enginePinnedVersion`, replacing the old `engineVersion` field. This is the supported version, not a probe of an arbitrary remote engine. - `registerTrigger` and `registerTriggerType` remain supported by iii 0.25, with deprecation warnings. HTTP and state providers still use these APIs; their warnings do not mean route registration failed. - iii 0.25 can dispatch calls to a worker's own functions locally. Engine-side spans differ for those calls; SDK tracing and state-worker persistence still apply. +- Local dispatch is on by default, so AgentMemory's internal calls to its own functions run in-process and produce only the SDK's execute span, not an engine call span. Set `AGENTMEMORY_LOCAL_DISPATCH=false` for full engine traces at the cost of a round trip per call. ## Retired engine console diff --git a/eval/README.md b/eval/README.md index 621fddc882..eaa9b3ef58 100644 --- a/eval/README.md +++ b/eval/README.md @@ -15,7 +15,7 @@ npm run eval:lifecycle npm run eval:recall ``` -The defaults are local and require no model API keys. The HTTP adapter and lifecycle suite require an installed **iii 0.25.0** binary, matching the version pinned by the CLI. Set `AGENTMEMORY_EVAL_III` to its absolute path, or use the binary installed by AgentMemory at `~/.agentmemory/bin/iii`. +The defaults are local and require no model API keys. The HTTP adapter and lifecycle suite require an installed **iii 0.25.1** binary, matching the version pinned by the CLI. Set `AGENTMEMORY_EVAL_III` to its absolute path, or use the binary installed by AgentMemory at `~/.agentmemory/bin/iii`. Each HTTP case launches the actual built CLI in a new temporary home, state directory, and port block, with an environment that excludes provider credentials. It uses the noop compression provider, with automatic compression, consolidation, and graph extraction disabled. The parent home and existing daemon are not used. Cleanup runs even after a failed query. `AGENTMEMORY_EVAL_KEEP=1` preserves scratch files for diagnosis; the daemon still stops. This sandbox supports macOS/Linux and needs permission to bind local ports. diff --git a/examples/python/README.md b/examples/python/README.md index 2f37613f31..6265cdc3ed 100644 --- a/examples/python/README.md +++ b/examples/python/README.md @@ -10,7 +10,7 @@ This example uses the official Python SDK. ## Install ```bash -pip install iii-sdk==0.25.0 +pip install iii-sdk==0.25.1 ``` ## Quickstart diff --git a/examples/python/observe_and_recall.py b/examples/python/observe_and_recall.py index 9de7dc25a7..4b6626169f 100644 --- a/examples/python/observe_and_recall.py +++ b/examples/python/observe_and_recall.py @@ -4,7 +4,7 @@ agentmemory to render the most relevant context back at a fixed token budget. Prerequisites: - pip install iii-sdk==0.25.0 + pip install iii-sdk==0.25.1 npx -y @agentmemory/agentmemory Run: diff --git a/examples/python/quickstart.py b/examples/python/quickstart.py index 4d1ba6f747..7a2ef3a8bb 100644 --- a/examples/python/quickstart.py +++ b/examples/python/quickstart.py @@ -1,7 +1,7 @@ """Minimal agentmemory usage via iii-sdk. Prerequisites: - pip install iii-sdk==0.25.0 + pip install iii-sdk==0.25.1 npx -y @agentmemory/agentmemory # daemon at ws://127.0.0.1:49134 Run: diff --git a/integrations/hermes/plugin.yaml b/integrations/hermes/plugin.yaml index 9f324efae8..1b8e55890d 100644 --- a/integrations/hermes/plugin.yaml +++ b/integrations/hermes/plugin.yaml @@ -1,5 +1,5 @@ name: agentmemory -version: 0.9.32-daniel.55 +version: 0.9.33-daniel.73 description: "Persistent memory for Hermes Agent with profile-scoped capture, recall, and native memory tools." author: "Rohit Ghumare" homepage: "https://github.com/rohitg00/agentmemory" diff --git a/integrations/omp/package.json b/integrations/omp/package.json index baeb3c68ef..96503114a8 100644 --- a/integrations/omp/package.json +++ b/integrations/omp/package.json @@ -2,7 +2,7 @@ "name": "agentmemory-omp-extension", "private": true, "type": "module", - "version": "0.9.32-daniel.55", + "version": "0.9.33-daniel.73", "omp": { "name": "agentmemory", "description": "Persistent cross-session memory for omp \u2014 captures sessions, tools, approvals, and compaction; injects smart-search recall.", diff --git a/integrations/openclaw/openclaw.plugin.json b/integrations/openclaw/openclaw.plugin.json index dd6be503e6..2a725d1245 100644 --- a/integrations/openclaw/openclaw.plugin.json +++ b/integrations/openclaw/openclaw.plugin.json @@ -3,7 +3,7 @@ "kind": "memory", "name": "agentmemory", "description": "Persistent cross-session memory for OpenClaw via agentmemory.", - "version": "0.9.32-daniel.55", + "version": "0.9.33-daniel.73", "configSchema": { "type": "object", "additionalProperties": false, diff --git a/integrations/openclaw/package.json b/integrations/openclaw/package.json index 8dd109c630..c8d172bd41 100644 --- a/integrations/openclaw/package.json +++ b/integrations/openclaw/package.json @@ -1,6 +1,6 @@ { "name": "agentmemory", - "version": "0.9.32-daniel.55", + "version": "0.9.33-daniel.73", "type": "module", "openclaw": { "extensions": [ diff --git a/integrations/openclaw/plugin.yaml b/integrations/openclaw/plugin.yaml index f809afd7b0..b02e89e7e9 100644 --- a/integrations/openclaw/plugin.yaml +++ b/integrations/openclaw/plugin.yaml @@ -1,5 +1,5 @@ name: agentmemory -version: 0.9.32-daniel.55 +version: 0.9.33-daniel.73 description: "Persistent cross-session memory for OpenClaw via agentmemory." author: "Rohit Ghumare" homepage: "https://github.com/rohitg00/agentmemory" diff --git a/package.json b/package.json index 98751a2508..6f3406a070 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@agentmemory/agentmemory", - "version": "0.9.32-daniel.55", + "version": "0.9.33-daniel.73", "description": "Persistent memory for AI coding agents, powered by iii-engine's three primitives", "type": "module", "main": "dist/index.mjs", @@ -92,12 +92,12 @@ "@anthropic-ai/sdk": "^0.100.1", "@clack/prompts": "^1.2.0", "dotenv": "^18.0.3", - "@iii-dev/helpers": "0.25.0", - "iii-sdk": "0.25.0", + "@iii-dev/helpers": "0.25.1", + "iii-sdk": "0.25.1", "picocolors": "^1.1.1", "zod": "^4.0.0", - "yaml": "^2.8.1", - "json5": "^2.2.3" + "yaml": "2.8.1", + "json5": "2.2.3" }, "optionalDependencies": { "@huggingface/transformers": "^4.2.0", @@ -112,12 +112,28 @@ "typebox": "^1.4.3", "typescript": "^6.0.3", "vitest": "^4.1.6", - "ws": "^8.21.0" + "ws": "^8.21.0", + "@opentelemetry/api": "1.9.1" }, "overrides": { "qs": "^6.15.2", "ws": "^8.21.0", - "protobufjs": "^7.5.8" + "protobufjs": "^7.5.8", + "@opentelemetry/api": "1.9.1", + "@opentelemetry/api-logs": "0.222.0", + "@opentelemetry/context-async-hooks": "2.12.0", + "@opentelemetry/core": "2.12.0", + "@opentelemetry/instrumentation": "0.222.0", + "@opentelemetry/otlp-transformer": "0.222.0", + "@opentelemetry/propagator-b3": "1.30.1", + "@opentelemetry/propagator-jaeger": "1.30.1", + "@opentelemetry/resources": "2.12.0", + "@opentelemetry/sdk-logs": "0.222.0", + "@opentelemetry/sdk-metrics": "2.12.0", + "@opentelemetry/sdk-trace": "2.11.0", + "@opentelemetry/sdk-trace-base": "2.12.0", + "@opentelemetry/sdk-trace-node": "2.12.0", + "@opentelemetry/semantic-conventions": "1.43.0" }, "engines": { "node": ">=20.0.0" diff --git a/packages/mcp/package.json b/packages/mcp/package.json index 7889acb965..10fcad23ba 100644 --- a/packages/mcp/package.json +++ b/packages/mcp/package.json @@ -1,6 +1,6 @@ { "name": "@agentmemory/mcp", - "version": "0.9.32-daniel.55", + "version": "0.9.33-daniel.73", "description": "Standalone MCP server for agentmemory — thin shim that re-exposes @agentmemory/agentmemory's MCP entrypoint", "type": "module", "bin": { @@ -28,7 +28,7 @@ "homepage": "https://github.com/rohitg00/agentmemory#readme", "bugs": "https://github.com/rohitg00/agentmemory/issues", "dependencies": { - "@agentmemory/agentmemory": "0.9.32-daniel.55" + "@agentmemory/agentmemory": "0.9.33-daniel.73" }, "publishConfig": { "access": "public", diff --git a/plugin/.claude-plugin/plugin.json b/plugin/.claude-plugin/plugin.json index 7dcb2cadb1..c4c91bb45e 100644 --- a/plugin/.claude-plugin/plugin.json +++ b/plugin/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "agentmemory", - "version": "0.9.32-daniel.55", + "version": "0.9.33-daniel.73", "description": "Persistent memory for AI coding agents -- captures tool usage, compresses via LLM, injects context into future sessions. 12 hooks, 55 MCP tools, 17 skills, real-time viewer.", "author": { "name": "Rohit Ghumare", diff --git a/plugin/.codex-plugin/plugin.json b/plugin/.codex-plugin/plugin.json index d836982238..b69f03b19f 100644 --- a/plugin/.codex-plugin/plugin.json +++ b/plugin/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "agentmemory", - "version": "0.9.32-daniel.55", + "version": "0.9.33-daniel.73", "description": "Local persistent memory for coding agents. Recall decisions, resume sessions, learn from corrections, and manage memory through 55 MCP tools and 17 skills. Optional capture through 6 Codex hooks.", "author": { "name": "Rohit Ghumare", diff --git a/plugin/.devin-plugin/plugin.json b/plugin/.devin-plugin/plugin.json index 3aefe96fdc..7d25963140 100644 --- a/plugin/.devin-plugin/plugin.json +++ b/plugin/.devin-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "agentmemory", "displayName": "agentmemory", - "version": "0.9.32-daniel.55", + "version": "0.9.33-daniel.73", "description": "Persistent memory for Devin CLI and Devin Desktop. Captures every session automatically, recalls with hybrid BM25 + vector + graph search, and exposes 55 memory tools over MCP. Keyless by default, runs on your machine. Not supported in cloud sessions.", "logo": "assets/icon.svg", "author": { @@ -23,7 +23,7 @@ "command": "npx", "args": [ "-y", - "@agentmemory/mcp@0.9.32-daniel.55" + "@agentmemory/mcp@0.9.33-daniel.73" ], "env": { "AGENTMEMORY_URL": "${AGENTMEMORY_URL:-http://localhost:3111}" diff --git a/plugin/hooks/hooks.json b/plugin/hooks/hooks.json index 4c7aa13abf..345afd5f8e 100644 --- a/plugin/hooks/hooks.json +++ b/plugin/hooks/hooks.json @@ -128,7 +128,8 @@ { "type": "command", "command": "node", - "args": ["${CLAUDE_PLUGIN_ROOT}/scripts/session-end.mjs"] + "args": ["${CLAUDE_PLUGIN_ROOT}/scripts/session-end.mjs"], + "timeout": 10 } ] } diff --git a/plugin/opencode/plugin.json b/plugin/opencode/plugin.json index 0bd579176b..6d7e6595c9 100644 --- a/plugin/opencode/plugin.json +++ b/plugin/opencode/plugin.json @@ -1,6 +1,6 @@ { "name": "agentmemory-capture", - "version": "0.9.32-daniel.55", + "version": "0.9.33-daniel.73", "description": "OpenCode plugin for agentmemory — full Claude Code hook parity: session lifecycle (create/idle/status/compacted/update/diff/delete/error), messages & prompts (chat.message, message.updated user+assistant, message.removed), tool lifecycle (ToolPart states with timing), part tracking (subtask, step-finish, reasoning, file, patch, compaction, agent, retry), file enrichment pipeline, permissions, task tracking (w/ priority), commands, config & model tracking. 22 hooks, 2 slash commands.", "author": { "name": "Rohit Ghumare", diff --git a/plugin/plugin.json b/plugin/plugin.json index 977a1ae530..2cc3e3f815 100644 --- a/plugin/plugin.json +++ b/plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "agentmemory", - "version": "0.9.32-daniel.55", + "version": "0.9.33-daniel.73", "description": "Persistent memory for AI coding agents -- captures tool usage, compresses via LLM, injects context into future sessions. 12 hooks, 55 MCP tools, 17 skills, real-time viewer.", "author": { "name": "Rohit Ghumare", diff --git a/plugin/scripts/_capture.mjs b/plugin/scripts/_capture.mjs index 8ecb006c0e..88443c9783 100644 --- a/plugin/scripts/_capture.mjs +++ b/plugin/scripts/_capture.mjs @@ -1,6 +1,6 @@ import { performance } from "node:perf_hooks"; -import { chmodSync, closeSync, constants, existsSync, fstatSync, fsyncSync, lstatSync, mkdirSync, openSync, readFileSync, readdirSync, renameSync, statSync, unlinkSync, utimesSync, writeFileSync, writeSync } from "node:fs"; import { createHash, randomUUID } from "node:crypto"; +import { chmodSync, closeSync, constants, existsSync, fstatSync, fsyncSync, lstatSync, mkdirSync, openSync, readFileSync, readdirSync, renameSync, statSync, unlinkSync, utimesSync, writeFileSync, writeSync } from "node:fs"; import { homedir, platform } from "node:os"; import { dirname, isAbsolute, join, parse, relative, resolve, sep } from "node:path"; import { isDeepStrictEqual } from "node:util"; @@ -11,6 +11,8 @@ const BLOCK = 4096; const INPUT_LIMIT = 8 * MiB; const ORIGINAL_LIMIT = 256 * MiB; const QUEUE_LIMIT = 5 * MiB; +const QUEUE_TRANSITION_RESERVE = 2 * BLOCK; +const ACK_TRANSITION_RESERVE = 64 * 1024 + BLOCK; const CHUNK_LIMIT = 256 * 1024; const ID = /^[a-f0-9]{64}$/; function fail(code) { @@ -140,19 +142,23 @@ function limit(value, maximum) { /** Linux-only durable source storage. The lock covers all targets and every quota decision. * Source/staging files are never reclaimed. Disk charges are conservative logical bytes, * rounded to 4 KiB and at least stat.blocks*512; they are not filesystem hard quotas. -* The queue includes a 4 KiB lease plus queue/root directory charges (12 KiB minimum). +* The queue includes a 4 KiB lease plus queue/root/workspace directory charges (16 KiB for initialized stores). * Writes additionally reserve a parent-directory block, and 256 bytes per new chunk-directory entry. +* Admission cannot consume transition workspace: existing refs may use an additional 8 KiB, +* and completed manifests an additional 68 KiB. Preserved crash temporaries count toward both. * /usr/bin/flock is required; no lock fallback, lease deletion, or source cleanup exists. */ var SdkSpoolV2 = class { dir; originals; queue; lease; + queueWorkspace; maxOriginalBytes; maxQueueBytes; maxInputBytes; chunkBytes; onPhase; + deadline; constructor(options = {}) { const path = options.dir ?? process.env.AGENTMEMORY_SDK_CAPTURE_V2_DIR ?? join(homedir(), ".local", "share", "agentmemory", "capture-spool-sdk-v2"); if (!isAbsolute(path)) fail("STORE_OPTIONS_INVALID"); @@ -160,12 +166,15 @@ var SdkSpoolV2 = class { this.originals = join(this.dir, "originals"); this.queue = join(this.dir, "queue"); this.lease = join(this.queue, "lease.lock"); + this.queueWorkspace = join(this.queue, ".transitions"); this.maxOriginalBytes = limit(options.maxOriginalBytes, ORIGINAL_LIMIT); this.maxQueueBytes = limit(options.maxQueueBytes, QUEUE_LIMIT); if (this.maxOriginalBytes < BLOCK || this.maxQueueBytes < 3 * BLOCK) fail("STORE_OPTIONS_INVALID"); this.maxInputBytes = limit(options.maxInputBytes, INPUT_LIMIT); this.chunkBytes = limit(options.chunkBytes, CHUNK_LIMIT); this.onPhase = options.onPhase; + if (options.deadline !== void 0 && (!Number.isFinite(options.deadline) || options.deadline < 0)) fail("STORE_OPTIONS_INVALID"); + this.deadline = options.deadline; } phase(name) { this.onPhase?.(name); @@ -184,10 +193,22 @@ var SdkSpoolV2 = class { queueBytes: measure(this.queue) + BLOCK }; } - budget(kind, additional) { - const usage = this.rawUsage(); - if (kind === "original" && usage.originalBytes + additional > this.maxOriginalBytes) fail("ORIGINALS_FULL"); - if (kind === "queue" && usage.queueBytes + additional > this.maxQueueBytes) fail("QUEUE_FULL"); + budget(kind, additional, transition = false) { + const usage = kind === "original" ? measure(this.originals) : measure(this.queue) + BLOCK; + const maximum = kind === "original" ? this.maxOriginalBytes : this.maxQueueBytes; + const reserve = transition ? kind === "original" ? ACK_TRANSITION_RESERVE : QUEUE_TRANSITION_RESERVE : 0; + if (usage + additional > maximum + reserve) fail(kind === "original" ? "ORIGINALS_FULL" : "QUEUE_FULL"); + } + prepareQueueWorkspace() { + try { + checkDirectory(this.queueWorkspace); + return; + } catch (error) { + if (code(error) !== "ENOENT") throw error; + } + this.budget("queue", 2 * BLOCK, true); + mkdirSync(this.queueWorkspace, { mode: 448 }); + syncDirectory$1(this.queue); } writeFile(path, bytes) { const fd = openSync(path, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 384); @@ -199,16 +220,25 @@ var SdkSpoolV2 = class { closeSync(fd); } } - atomicJson(path, value, kind, beforePublish) { + atomicJson(path, value, kind, beforePublish, transition = false) { const bytes = Buffer.from(canonical$1(value)); - if (kind) this.budget(kind, charge(bytes.length) + BLOCK); - const temp = path + "." + randomUUID() + ".tmp"; + if (kind && bytes.length > (kind === "queue" ? BLOCK : 64 * 1024)) fail("SOURCE_UNVERIFIED"); + const stagedQueue = kind === "queue" && transition; + if (stagedQueue) { + this.prepareQueueWorkspace(); + if (readdirSync(this.queueWorkspace).length) fail("QUEUE_FULL"); + } + if (kind) this.budget(kind, charge(bytes.length) + BLOCK, transition); + const temp = stagedQueue ? join(this.queueWorkspace, "pending.tmp") : path + "." + randomUUID() + ".tmp"; this.writeFile(temp, bytes); if (beforePublish) this.phase(beforePublish); + if (stagedQueue) this.phase("queue_transition_staged"); renameSync(temp, path); syncDirectory$1(join(path, "..")); + if (stagedQueue) syncDirectory$1(this.queueWorkspace); } async locked(operation) { + const operationDeadline = this.deadline; let fd; try { if (process.platform !== "linux") fail("STORE_LOCK_UNSUPPORTED"); @@ -217,8 +247,9 @@ var SdkSpoolV2 = class { const stat = fstatSync(fd); privateStat(stat, false); if (stat.size !== 0) fail("STORE_UNSAFE"); - const deadline = performance.now() + 150; + const deadline = operationDeadline ?? performance.now() + 150; while (true) { + if (operationDeadline !== void 0 && performance.now() >= deadline) fail("STORE_BUSY"); const lock = spawnSync("/usr/bin/flock", [ "--exclusive", "--nonblock", @@ -241,6 +272,8 @@ var SdkSpoolV2 = class { } fsyncSync(fd); syncDirectory$1(this.queue); + if (readdirSync(this.queue).every((name) => name === "lease.lock")) this.prepareQueueWorkspace(); + if (operationDeadline !== void 0 && performance.now() >= deadline) fail("STORE_BUSY"); return operation(); } catch (error) { if (typeof code(error) === "string" && /^(SOURCE_|ORIGINALS_|QUEUE_|STORE_)/.test(code(error))) throw error; @@ -255,13 +288,22 @@ var SdkSpoolV2 = class { conflictKey(target, eventId, body) { return digest$1(this.key(target, eventId) + "\0conflict\0" + sourceFingerprint(body)); } - source(target, sourceId) { + manifest(target, sourceId) { if (!ID.test(sourceId)) fail("SOURCE_UNVERIFIED"); try { const folder = join(this.originals, sourceId); checkDirectory(folder); const manifest = JSON.parse(regularRead(join(folder, "manifest.json"), 64 * 1024).toString()); if (!manifest || manifest.v !== 2 || manifest.sourceId !== sourceId || manifest.targetHash !== digest$1(target) || typeof manifest.eventId !== "string" || !Number.isSafeInteger(manifest.bytes) || manifest.bytes <= 0 || manifest.bytes > this.maxInputBytes || !ID.test(manifest.hash) || typeof manifest.conflict !== "boolean" || typeof manifest.completed !== "boolean" || !Array.isArray(manifest.chunks) || !manifest.chunks.length || manifest.chunks.length > this.maxInputBytes) fail("SOURCE_UNVERIFIED"); + return manifest; + } catch (error) { + if (code(error) === "STORE_UNSAFE") throw error; + fail("SOURCE_UNVERIFIED"); + } + } + source(target, sourceId) { + try { + const manifest = this.manifest(target, sourceId), folder = join(this.originals, sourceId); let bytes = 0; const chunks = manifest.chunks.map((chunk, index) => { if (chunk.name !== String(index).padStart(6, "0") + ".chunk" || !Number.isSafeInteger(chunk.bytes) || chunk.bytes <= 0 || chunk.bytes > CHUNK_LIMIT || !ID.test(chunk.hash)) fail("SOURCE_UNVERIFIED"); @@ -285,7 +327,7 @@ var SdkSpoolV2 = class { fail("SOURCE_UNVERIFIED"); } } - async retain(target, eventId, body) { + prepareSource(target, eventId, body) { let normalized; let bytes; try { @@ -296,79 +338,114 @@ var SdkSpoolV2 = class { fail("SOURCE_UNVERIFIED"); } if (bytes.length > this.maxInputBytes) fail("SOURCE_TOO_LARGE"); - return this.locked(() => { - let sourceId = this.key(target, eventId); - let conflict = false; + return { + normalized, + bytes + }; + } + retainSource(target, eventId, normalized, bytes) { + let sourceId = this.key(target, eventId); + let conflict = false; + try { + lstatSync(join(this.originals, sourceId)); + const prior = this.source(target, sourceId); + if (sameSource(prior.body, normalized)) return { + sourceId, + duplicate: true, + conflict: false, + completed: prior.manifest.completed, + body: prior.body + }; + conflict = true; + sourceId = this.conflictKey(target, eventId, normalized); try { lstatSync(join(this.originals, sourceId)); - const prior = this.source(target, sourceId); - if (sameSource(prior.body, normalized)) return { + const priorConflict = this.source(target, sourceId); + if (!priorConflict.manifest.conflict || !sameSource(priorConflict.body, normalized)) fail("SOURCE_UNVERIFIED"); + return { sourceId, duplicate: true, - conflict: false, - completed: prior.manifest.completed, - body: prior.body + conflict: true, + completed: false, + body: priorConflict.body }; - conflict = true; - sourceId = this.conflictKey(target, eventId, normalized); - try { - lstatSync(join(this.originals, sourceId)); - const priorConflict = this.source(target, sourceId); - if (!priorConflict.manifest.conflict || !sameSource(priorConflict.body, normalized)) fail("SOURCE_UNVERIFIED"); - return { - sourceId, - duplicate: true, - conflict: true, - completed: false, - body: priorConflict.body - }; - } catch (error) { - if (code(error) !== "ENOENT") throw error; - } } catch (error) { if (code(error) !== "ENOENT") throw error; } - const chunks = []; - for (let offset = 0; offset < bytes.length; offset += this.chunkBytes) chunks.push(bytes.subarray(offset, offset + this.chunkBytes)); - const manifest = { - v: 2, - sourceId, - targetHash: digest$1(target), - eventId, - bytes: bytes.length, - hash: digest$1(bytes), - chunks: chunks.map((value, index) => ({ - name: String(index).padStart(6, "0") + ".chunk", - bytes: value.length, - hash: digest$1(value) - })), - conflict, - completed: false + } catch (error) { + if (code(error) !== "ENOENT") throw error; + } + const chunks = []; + for (let offset = 0; offset < bytes.length; offset += this.chunkBytes) chunks.push(bytes.subarray(offset, offset + this.chunkBytes)); + const manifest = { + v: 2, + sourceId, + targetHash: digest$1(target), + eventId, + bytes: bytes.length, + hash: digest$1(bytes), + chunks: chunks.map((value, index) => ({ + name: String(index).padStart(6, "0") + ".chunk", + bytes: value.length, + hash: digest$1(value) + })), + conflict, + completed: false + }; + const manifestBytes = canonical$1(manifest); + if (Buffer.byteLength(manifestBytes) > 64 * 1024) fail("SOURCE_TOO_LARGE"); + this.budget("original", BLOCK + charge((chunks.length + 1) * 256) + charge(Buffer.byteLength(manifestBytes)) + chunks.reduce((sum, value) => sum + charge(value.length), 0)); + const stage = join(this.originals, ".stage-" + randomUUID()); + mkdirSync(stage, { mode: 448 }); + syncDirectory$1(this.originals); + for (let index = 0; index < chunks.length; index++) { + this.writeFile(join(stage, manifest.chunks[index].name), chunks[index]); + this.phase("chunk_fsynced"); + } + this.writeFile(join(stage, "manifest.json"), manifestBytes); + this.phase("manifest_fsynced"); + syncDirectory$1(stage); + this.phase("source_before_publish"); + renameSync(stage, join(this.originals, sourceId)); + syncDirectory$1(this.originals); + this.phase("source_published"); + const verified = this.source(target, sourceId); + return { + sourceId, + duplicate: false, + conflict, + completed: false, + body: verified.body + }; + } + async retain(target, eventId, body) { + const { normalized, bytes } = this.prepareSource(target, eventId, body); + return this.locked(() => this.retainSource(target, eventId, normalized, bytes)); + } + async retainAndEnqueue(target, eventId, body) { + const { normalized, bytes } = this.prepareSource(target, eventId, body); + return this.locked(() => { + const retained = this.retainSource(target, eventId, normalized, bytes); + if (retained.conflict) return { + retained, + referenceAcknowledged: false }; - const manifestBytes = canonical$1(manifest); - if (Buffer.byteLength(manifestBytes) > 64 * 1024) fail("SOURCE_TOO_LARGE"); - this.budget("original", BLOCK + charge((chunks.length + 1) * 256) + charge(Buffer.byteLength(manifestBytes)) + chunks.reduce((sum, value) => sum + charge(value.length), 0)); - const stage = join(this.originals, ".stage-" + randomUUID()); - mkdirSync(stage, { mode: 448 }); - syncDirectory$1(this.originals); - for (let index = 0; index < chunks.length; index++) { - this.writeFile(join(stage, manifest.chunks[index].name), chunks[index]); - this.phase("chunk_fsynced"); + if (retained.completed) return { + retained, + referenceAcknowledged: true + }; + try { + this.enqueueSource(target, retained.sourceId); + } catch (error) { + return { + retained, + referenceAcknowledged: false, + error + }; } - this.writeFile(join(stage, "manifest.json"), manifestBytes); - this.phase("manifest_fsynced"); - syncDirectory$1(stage); - this.phase("source_before_publish"); - renameSync(stage, join(this.originals, sourceId)); - syncDirectory$1(this.originals); - this.phase("source_published"); - const verified = this.source(target, sourceId); return { - sourceId, - duplicate: false, - conflict, - completed: false, - body: verified.body + retained, + referenceAcknowledged: true }; }); } @@ -394,30 +471,31 @@ var SdkSpoolV2 = class { fail("SOURCE_UNVERIFIED"); } } - writeRef(ref) { - this.atomicJson(this.refPath(ref.sourceId), ref, "queue"); + writeRef(ref, transition = true) { + this.atomicJson(this.refPath(ref.sourceId), ref, "queue", void 0, transition); this.phase("reference_published"); } + enqueueSource(target, sourceId) { + const { manifest } = this.source(target, sourceId); + if (manifest.conflict) fail("SOURCE_CONFLICT"); + if (manifest.completed) return; + try { + this.readRef(target, sourceId); + return; + } catch (error) { + if (code(error) !== "ENOENT") throw error; + } + this.writeRef({ + v: 2, + targetHash: digest$1(target), + sourceId, + state: "pending", + attempts: 0, + nextAttemptAt: 0 + }, false); + } async enqueue(target, sourceId) { - return this.locked(() => { - const { manifest } = this.source(target, sourceId); - if (manifest.conflict) fail("SOURCE_CONFLICT"); - if (manifest.completed) return; - try { - this.readRef(target, sourceId); - return; - } catch (error) { - if (code(error) !== "ENOENT") throw error; - } - this.writeRef({ - v: 2, - targetHash: digest$1(target), - sourceId, - state: "pending", - attempts: 0, - nextAttemptAt: 0 - }); - }); + return this.locked(() => this.enqueueSource(target, sourceId)); } async pending(target) { return this.locked(() => { @@ -432,7 +510,7 @@ var SdkSpoolV2 = class { this.writeRef(ref); } try { - if (this.source(target, ref.sourceId).manifest.completed) continue; + if (this.manifest(target, ref.sourceId).completed) continue; } catch { ref.state = "held"; } @@ -446,19 +524,39 @@ var SdkSpoolV2 = class { })); }); } - async markSending(target, sourceId) { + async markSending(target, sourceId, expectedHeldSource) { + await this.claimSource(target, sourceId, expectedHeldSource); + } + async claimForSend(target, sourceId, sendDeadline) { + return this.claimSource(target, sourceId, void 0, sendDeadline); + } + async claimSource(target, sourceId, expectedHeldSource, sendDeadline) { + if (sendDeadline !== void 0 && !Number.isFinite(sendDeadline)) fail("STORE_OPTIONS_INVALID"); return this.locked(() => { const source = this.source(target, sourceId); if (source.manifest.conflict || source.manifest.completed) fail("SOURCE_HELD"); const ref = this.readRef(target, sourceId); - if (ref.state !== "pending") fail("SOURCE_HELD"); - if (ref.nextAttemptAt > Date.now()) fail("SOURCE_BACKOFF"); + if (expectedHeldSource) { + if (ref.state !== "held" || ref.owner || !isDeepStrictEqual(source.body, expectedHeldSource)) fail("SOURCE_HELD"); + } else { + if (ref.state === "sending") { + if (alive(ref.owner)) fail("SOURCE_SENDING"); + ref.state = "held"; + ref.reason = "UNKNOWN_SEND"; + delete ref.owner; + this.writeRef(ref); + } + if (ref.state !== "pending") fail("SOURCE_HELD"); + if (ref.nextAttemptAt > Date.now()) fail("SOURCE_BACKOFF"); + } const owner = ownerIdentity(); if (!owner) fail("STORE_BUSY"); + if (sendDeadline !== void 0 && performance.now() >= sendDeadline) fail("SOURCE_SEND_BUDGET_EXHAUSTED"); ref.state = "sending"; ref.owner = owner; this.writeRef(ref); this.phase("send_intent_fsynced"); + return source.body; }); } async acknowledge(target, sourceId) { @@ -466,9 +564,10 @@ var SdkSpoolV2 = class { const source = this.source(target, sourceId); if (source.manifest.conflict) fail("SOURCE_CONFLICT"); if (!source.manifest.completed) { - if (this.readRef(target, sourceId).state !== "sending") fail("SOURCE_HELD"); + const ref = this.readRef(target, sourceId); + if (ref.state !== "sending" || !isDeepStrictEqual(ref.owner, ownerIdentity())) fail("SOURCE_HELD"); source.manifest.completed = true; - this.atomicJson(join(this.originals, sourceId, "manifest.json"), source.manifest, "original", "ack_before_publish"); + this.atomicJson(join(this.originals, sourceId, "manifest.json"), source.manifest, "original", "ack_before_publish", true); this.phase("ack_fsynced"); } try { @@ -482,7 +581,7 @@ var SdkSpoolV2 = class { async retry(target, sourceId, _reason) { return this.locked(() => { const ref = this.readRef(target, sourceId); - if (ref.state !== "sending" || !ref.owner || !alive(ref.owner)) fail("SOURCE_HELD"); + if (ref.state !== "sending" || !isDeepStrictEqual(ref.owner, ownerIdentity())) fail("SOURCE_HELD"); ref.attempts = Math.min(1e6, ref.attempts + 1); ref.nextAttemptAt = Date.now() + Math.min(3e5, 1e3 * 2 ** Math.min(9, ref.attempts - 1)); ref.state = "pending"; @@ -494,6 +593,7 @@ var SdkSpoolV2 = class { async hold(target, sourceId, _reason) { return this.locked(() => { const ref = this.readRef(target, sourceId); + if (ref.state !== "sending" || !isDeepStrictEqual(ref.owner, ownerIdentity())) fail("SOURCE_HELD"); ref.state = "held"; ref.reason = "HELD"; delete ref.owner; @@ -525,7 +625,8 @@ function safeCode(error) { function createStore(options) { return new SdkSpoolV2({ ...options.storeOptions, - dir: options.dir + dir: options.dir, + deadline: options.deadline }); } function held(code, sourceAcknowledged = true, referenceAcknowledged = true) { @@ -545,16 +646,17 @@ async function park(store, target, sourceId, code) { return held(code); } async function sendReference(store, target, sourceId, post, deadline) { - try { - await store.markSending(target, sourceId); - } catch (error) { - return held(safeCode(error)); - } let body; try { - body = await store.load(target, sourceId); - } catch { - return park(store, target, sourceId, "SOURCE_UNVERIFIED"); + body = await store.claimForSend(target, sourceId, deadline); + } catch (error) { + const code = error?.code; + if (code === "SOURCE_BACKOFF" || code === "SOURCE_SEND_BUDGET_EXHAUSTED") return { + outcome: "spooled", + sourceAcknowledged: true, + referenceAcknowledged: true + }; + return held(code === "SOURCE_SENDING" ? "SDK_SEND_IN_PROGRESS_OR_UNCERTAIN" : safeCode(error)); } let response; const budget = deadline === void 0 ? void 0 : { @@ -614,7 +716,7 @@ async function deliverSdkCaptureV2(target, body, post, options = {}) { sourceAcknowledged: false, referenceAcknowledged: false }; - const retained = await store.retain(target, body.eventId, body); + const handoff = await store.retainAndEnqueue(target, body.eventId, body), retained = handoff.retained; sourceId = retained.sourceId; storedBody = retained.body; if (retained.conflict) return held("SOURCE_CONFLICT", true, false); @@ -624,48 +726,30 @@ async function deliverSdkCaptureV2(target, body, post, options = {}) { referenceAcknowledged: true, sdkReplyConfirmed: true }; - } catch (error) { - return { + if (!handoff.referenceAcknowledged) return { outcome: "dropped", - code: safeCode(error), - sourceAcknowledged: false, + code: safeCode(handoff.error), + sourceAcknowledged: true, referenceAcknowledged: false }; - } - try { - await store.enqueue(target, sourceId); } catch (error) { return { outcome: "dropped", code: safeCode(error), - sourceAcknowledged: true, + sourceAcknowledged: false, referenceAcknowledged: false }; } - let pending; - try { - pending = (await store.pending(target)).find((record) => record.sourceId === sourceId); - } catch (error) { - return held(safeCode(error)); - } - if (!pending) { - try { - if ((await store.retain(target, storedBody.eventId, storedBody)).completed) return { - outcome: "duplicate", - sourceAcknowledged: true, - referenceAcknowledged: true, - sdkReplyConfirmed: true - }; - } catch {} - return held("QUEUE_STATE_UNCONFIRMED"); - } - if (pending.state !== "pending") return held(pending.state === "sending" ? "SDK_SEND_IN_PROGRESS_OR_UNCERTAIN" : "SOURCE_HELD"); - if (pending.nextAttemptAt > Date.now()) return { - outcome: "spooled", - sourceAcknowledged: true, - referenceAcknowledged: true - }; - return sendReference(store, target, sourceId, post); + const sent = await sendReference(store, target, sourceId, post, options.deadline === void 0 ? void 0 : options.deadline - 750); + if (sent.code === "SOURCE_HELD") try { + if ((await store.retain(target, storedBody.eventId, storedBody)).completed) return { + outcome: "duplicate", + sourceAcknowledged: true, + referenceAcknowledged: true, + sdkReplyConfirmed: true + }; + } catch {} + return sent; } async function drainSdkCapturesV2(target, post, options = {}) { const result = { @@ -677,7 +761,7 @@ async function drainSdkCapturesV2(target, post, options = {}) { error: void 0 }; const limit = Math.min(8, Math.max(0, options.limit ?? 8)), budget = Math.min(1500, Math.max(0, options.budgetMs ?? 1500)), start = performance.now(); - const deadline = Math.min(options.deadline ?? Infinity, start + budget); + const deadline = Math.min(options.deadline === void 0 ? Infinity : options.deadline - 750, start + budget); try { const store = createStore(options), records = await store.pending(target); for (const ref of records) { @@ -743,7 +827,10 @@ const SECRET_PATTERN_SOURCES = [ const PRIVATE_KEY_BLOCK_RE = /-----BEGIN [A-Z0-9 ]*PRIVATE KEY(?: BLOCK)?-----[\s\S]*?(?:-----END [A-Z0-9 ]*PRIVATE KEY(?: BLOCK)?-----|$)/g; const URL_CREDENTIALS_RE = /\b([a-z][a-z0-9+.-]*:\/\/)[^\s/?#@:"'<>]*:[^\s/?#@"'<>]+@/gi; function stripPrivateData(input) { - let result = redactPrivateBlocks(input); + return stripSecrets(redactPrivateBlocks(input)); +} +function stripSecrets(input) { + let result = input; result = result.replace(new RegExp(PRIVATE_KEY_BLOCK_RE.source, PRIVATE_KEY_BLOCK_RE.flags), "[REDACTED_SECRET]"); result = result.replace(new RegExp(URL_CREDENTIALS_RE.source, URL_CREDENTIALS_RE.flags), "$1[REDACTED_SECRET]@"); for (const source of SECRET_PATTERN_SOURCES) { @@ -753,12 +840,13 @@ function stripPrivateData(input) { return result; } function scrubRecord(value, options = {}) { - if (typeof value === "string") return stripPrivateData(value); + const strip = options.strip ?? stripPrivateData; + if (typeof value === "string") return strip(value); if (Array.isArray(value)) return value.map((item) => scrubRecord(item, options)); if (value !== null && typeof value === "object") { const proto = Object.getPrototypeOf(value); if (proto !== Object.prototype && proto !== null) return value; - const entries = Object.entries(value).map(([key, item]) => [options.scrubKeys ? stripPrivateData(key) : key, item]); + const entries = Object.entries(value).map(([key, item]) => [options.scrubKeys ? strip(key) : key, item]); const reserved = new Set(entries.map(([key]) => key)); const used = /* @__PURE__ */ new Set(); return Object.fromEntries(entries.map(([key, item]) => { @@ -1731,7 +1819,7 @@ function unquote(value) { const hash = value.indexOf(" #"); return hash === -1 ? value : value.slice(0, hash).trim(); } -function readEnvFileSecret() { +function readEnvFileValue(name) { let content; try { content = readFileSync(process.env["AGENTMEMORY_ENV_FILE"] || join(homedir(), ".agentmemory", ".env"), "utf-8"); @@ -1745,11 +1833,14 @@ function readEnvFileSecret() { if (trimmed.startsWith("#")) continue; const eq = trimmed.indexOf("="); if (eq === -1) continue; - if (trimmed.slice(0, eq).replace(/^export\s+/, "").trim() !== SECRET_KEY) continue; - found = usable(unquote(trimmed.slice(eq + 1).trim())); + if (trimmed.slice(0, eq).replace(/^export\s+/, "").trim() !== name) continue; + found = unquote(trimmed.slice(eq + 1).trim()); } return found; } +function readEnvFileSecret() { + return usable(readEnvFileValue(SECRET_KEY)); +} function readSecretFile(path) { try { return usable(readFileSync(path, "utf-8")); @@ -2124,9 +2215,12 @@ async function captureObservationV2(body, timeoutMs) { eventId, captureCompletion: "sdk-v1" })), { scrubKeys: true }); - const options = { dir: process.env["AGENTMEMORY_SDK_CAPTURE_V2_DIR"] }; - const result = await deliverSdkCaptureV2(REST_URL, payload, (b) => postRawV2(b, timeoutMs), options); - const now = performance.now(), deadline = Math.min(started + timeoutMs, now + 250), remaining = Math.floor(deadline - now); + const deadline = started + timeoutMs, options = { + dir: process.env["AGENTMEMORY_SDK_CAPTURE_V2_DIR"], + deadline + }; + const result = await deliverSdkCaptureV2(REST_URL, payload, (b, budget) => postRawV2(b, budget?.remainingMs ?? timeoutMs, budget?.deadline), options); + const now = performance.now(), remaining = Math.floor(Math.min(deadline - 750 - now, 250)); if (body.hookType !== "stop" && (result.outcome === "delivered" || result.outcome === "duplicate") && remaining >= 100) { const backlog = await drainSdkCapturesV2(REST_URL, (b, budget) => postRawV2(b, budget?.remainingMs ?? remaining, budget?.deadline ?? deadline), { ...options, diff --git a/plugin/scripts/codex-compat/_capture.mjs b/plugin/scripts/codex-compat/_capture.mjs index 8ecb006c0e..88443c9783 100644 --- a/plugin/scripts/codex-compat/_capture.mjs +++ b/plugin/scripts/codex-compat/_capture.mjs @@ -1,6 +1,6 @@ import { performance } from "node:perf_hooks"; -import { chmodSync, closeSync, constants, existsSync, fstatSync, fsyncSync, lstatSync, mkdirSync, openSync, readFileSync, readdirSync, renameSync, statSync, unlinkSync, utimesSync, writeFileSync, writeSync } from "node:fs"; import { createHash, randomUUID } from "node:crypto"; +import { chmodSync, closeSync, constants, existsSync, fstatSync, fsyncSync, lstatSync, mkdirSync, openSync, readFileSync, readdirSync, renameSync, statSync, unlinkSync, utimesSync, writeFileSync, writeSync } from "node:fs"; import { homedir, platform } from "node:os"; import { dirname, isAbsolute, join, parse, relative, resolve, sep } from "node:path"; import { isDeepStrictEqual } from "node:util"; @@ -11,6 +11,8 @@ const BLOCK = 4096; const INPUT_LIMIT = 8 * MiB; const ORIGINAL_LIMIT = 256 * MiB; const QUEUE_LIMIT = 5 * MiB; +const QUEUE_TRANSITION_RESERVE = 2 * BLOCK; +const ACK_TRANSITION_RESERVE = 64 * 1024 + BLOCK; const CHUNK_LIMIT = 256 * 1024; const ID = /^[a-f0-9]{64}$/; function fail(code) { @@ -140,19 +142,23 @@ function limit(value, maximum) { /** Linux-only durable source storage. The lock covers all targets and every quota decision. * Source/staging files are never reclaimed. Disk charges are conservative logical bytes, * rounded to 4 KiB and at least stat.blocks*512; they are not filesystem hard quotas. -* The queue includes a 4 KiB lease plus queue/root directory charges (12 KiB minimum). +* The queue includes a 4 KiB lease plus queue/root/workspace directory charges (16 KiB for initialized stores). * Writes additionally reserve a parent-directory block, and 256 bytes per new chunk-directory entry. +* Admission cannot consume transition workspace: existing refs may use an additional 8 KiB, +* and completed manifests an additional 68 KiB. Preserved crash temporaries count toward both. * /usr/bin/flock is required; no lock fallback, lease deletion, or source cleanup exists. */ var SdkSpoolV2 = class { dir; originals; queue; lease; + queueWorkspace; maxOriginalBytes; maxQueueBytes; maxInputBytes; chunkBytes; onPhase; + deadline; constructor(options = {}) { const path = options.dir ?? process.env.AGENTMEMORY_SDK_CAPTURE_V2_DIR ?? join(homedir(), ".local", "share", "agentmemory", "capture-spool-sdk-v2"); if (!isAbsolute(path)) fail("STORE_OPTIONS_INVALID"); @@ -160,12 +166,15 @@ var SdkSpoolV2 = class { this.originals = join(this.dir, "originals"); this.queue = join(this.dir, "queue"); this.lease = join(this.queue, "lease.lock"); + this.queueWorkspace = join(this.queue, ".transitions"); this.maxOriginalBytes = limit(options.maxOriginalBytes, ORIGINAL_LIMIT); this.maxQueueBytes = limit(options.maxQueueBytes, QUEUE_LIMIT); if (this.maxOriginalBytes < BLOCK || this.maxQueueBytes < 3 * BLOCK) fail("STORE_OPTIONS_INVALID"); this.maxInputBytes = limit(options.maxInputBytes, INPUT_LIMIT); this.chunkBytes = limit(options.chunkBytes, CHUNK_LIMIT); this.onPhase = options.onPhase; + if (options.deadline !== void 0 && (!Number.isFinite(options.deadline) || options.deadline < 0)) fail("STORE_OPTIONS_INVALID"); + this.deadline = options.deadline; } phase(name) { this.onPhase?.(name); @@ -184,10 +193,22 @@ var SdkSpoolV2 = class { queueBytes: measure(this.queue) + BLOCK }; } - budget(kind, additional) { - const usage = this.rawUsage(); - if (kind === "original" && usage.originalBytes + additional > this.maxOriginalBytes) fail("ORIGINALS_FULL"); - if (kind === "queue" && usage.queueBytes + additional > this.maxQueueBytes) fail("QUEUE_FULL"); + budget(kind, additional, transition = false) { + const usage = kind === "original" ? measure(this.originals) : measure(this.queue) + BLOCK; + const maximum = kind === "original" ? this.maxOriginalBytes : this.maxQueueBytes; + const reserve = transition ? kind === "original" ? ACK_TRANSITION_RESERVE : QUEUE_TRANSITION_RESERVE : 0; + if (usage + additional > maximum + reserve) fail(kind === "original" ? "ORIGINALS_FULL" : "QUEUE_FULL"); + } + prepareQueueWorkspace() { + try { + checkDirectory(this.queueWorkspace); + return; + } catch (error) { + if (code(error) !== "ENOENT") throw error; + } + this.budget("queue", 2 * BLOCK, true); + mkdirSync(this.queueWorkspace, { mode: 448 }); + syncDirectory$1(this.queue); } writeFile(path, bytes) { const fd = openSync(path, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 384); @@ -199,16 +220,25 @@ var SdkSpoolV2 = class { closeSync(fd); } } - atomicJson(path, value, kind, beforePublish) { + atomicJson(path, value, kind, beforePublish, transition = false) { const bytes = Buffer.from(canonical$1(value)); - if (kind) this.budget(kind, charge(bytes.length) + BLOCK); - const temp = path + "." + randomUUID() + ".tmp"; + if (kind && bytes.length > (kind === "queue" ? BLOCK : 64 * 1024)) fail("SOURCE_UNVERIFIED"); + const stagedQueue = kind === "queue" && transition; + if (stagedQueue) { + this.prepareQueueWorkspace(); + if (readdirSync(this.queueWorkspace).length) fail("QUEUE_FULL"); + } + if (kind) this.budget(kind, charge(bytes.length) + BLOCK, transition); + const temp = stagedQueue ? join(this.queueWorkspace, "pending.tmp") : path + "." + randomUUID() + ".tmp"; this.writeFile(temp, bytes); if (beforePublish) this.phase(beforePublish); + if (stagedQueue) this.phase("queue_transition_staged"); renameSync(temp, path); syncDirectory$1(join(path, "..")); + if (stagedQueue) syncDirectory$1(this.queueWorkspace); } async locked(operation) { + const operationDeadline = this.deadline; let fd; try { if (process.platform !== "linux") fail("STORE_LOCK_UNSUPPORTED"); @@ -217,8 +247,9 @@ var SdkSpoolV2 = class { const stat = fstatSync(fd); privateStat(stat, false); if (stat.size !== 0) fail("STORE_UNSAFE"); - const deadline = performance.now() + 150; + const deadline = operationDeadline ?? performance.now() + 150; while (true) { + if (operationDeadline !== void 0 && performance.now() >= deadline) fail("STORE_BUSY"); const lock = spawnSync("/usr/bin/flock", [ "--exclusive", "--nonblock", @@ -241,6 +272,8 @@ var SdkSpoolV2 = class { } fsyncSync(fd); syncDirectory$1(this.queue); + if (readdirSync(this.queue).every((name) => name === "lease.lock")) this.prepareQueueWorkspace(); + if (operationDeadline !== void 0 && performance.now() >= deadline) fail("STORE_BUSY"); return operation(); } catch (error) { if (typeof code(error) === "string" && /^(SOURCE_|ORIGINALS_|QUEUE_|STORE_)/.test(code(error))) throw error; @@ -255,13 +288,22 @@ var SdkSpoolV2 = class { conflictKey(target, eventId, body) { return digest$1(this.key(target, eventId) + "\0conflict\0" + sourceFingerprint(body)); } - source(target, sourceId) { + manifest(target, sourceId) { if (!ID.test(sourceId)) fail("SOURCE_UNVERIFIED"); try { const folder = join(this.originals, sourceId); checkDirectory(folder); const manifest = JSON.parse(regularRead(join(folder, "manifest.json"), 64 * 1024).toString()); if (!manifest || manifest.v !== 2 || manifest.sourceId !== sourceId || manifest.targetHash !== digest$1(target) || typeof manifest.eventId !== "string" || !Number.isSafeInteger(manifest.bytes) || manifest.bytes <= 0 || manifest.bytes > this.maxInputBytes || !ID.test(manifest.hash) || typeof manifest.conflict !== "boolean" || typeof manifest.completed !== "boolean" || !Array.isArray(manifest.chunks) || !manifest.chunks.length || manifest.chunks.length > this.maxInputBytes) fail("SOURCE_UNVERIFIED"); + return manifest; + } catch (error) { + if (code(error) === "STORE_UNSAFE") throw error; + fail("SOURCE_UNVERIFIED"); + } + } + source(target, sourceId) { + try { + const manifest = this.manifest(target, sourceId), folder = join(this.originals, sourceId); let bytes = 0; const chunks = manifest.chunks.map((chunk, index) => { if (chunk.name !== String(index).padStart(6, "0") + ".chunk" || !Number.isSafeInteger(chunk.bytes) || chunk.bytes <= 0 || chunk.bytes > CHUNK_LIMIT || !ID.test(chunk.hash)) fail("SOURCE_UNVERIFIED"); @@ -285,7 +327,7 @@ var SdkSpoolV2 = class { fail("SOURCE_UNVERIFIED"); } } - async retain(target, eventId, body) { + prepareSource(target, eventId, body) { let normalized; let bytes; try { @@ -296,79 +338,114 @@ var SdkSpoolV2 = class { fail("SOURCE_UNVERIFIED"); } if (bytes.length > this.maxInputBytes) fail("SOURCE_TOO_LARGE"); - return this.locked(() => { - let sourceId = this.key(target, eventId); - let conflict = false; + return { + normalized, + bytes + }; + } + retainSource(target, eventId, normalized, bytes) { + let sourceId = this.key(target, eventId); + let conflict = false; + try { + lstatSync(join(this.originals, sourceId)); + const prior = this.source(target, sourceId); + if (sameSource(prior.body, normalized)) return { + sourceId, + duplicate: true, + conflict: false, + completed: prior.manifest.completed, + body: prior.body + }; + conflict = true; + sourceId = this.conflictKey(target, eventId, normalized); try { lstatSync(join(this.originals, sourceId)); - const prior = this.source(target, sourceId); - if (sameSource(prior.body, normalized)) return { + const priorConflict = this.source(target, sourceId); + if (!priorConflict.manifest.conflict || !sameSource(priorConflict.body, normalized)) fail("SOURCE_UNVERIFIED"); + return { sourceId, duplicate: true, - conflict: false, - completed: prior.manifest.completed, - body: prior.body + conflict: true, + completed: false, + body: priorConflict.body }; - conflict = true; - sourceId = this.conflictKey(target, eventId, normalized); - try { - lstatSync(join(this.originals, sourceId)); - const priorConflict = this.source(target, sourceId); - if (!priorConflict.manifest.conflict || !sameSource(priorConflict.body, normalized)) fail("SOURCE_UNVERIFIED"); - return { - sourceId, - duplicate: true, - conflict: true, - completed: false, - body: priorConflict.body - }; - } catch (error) { - if (code(error) !== "ENOENT") throw error; - } } catch (error) { if (code(error) !== "ENOENT") throw error; } - const chunks = []; - for (let offset = 0; offset < bytes.length; offset += this.chunkBytes) chunks.push(bytes.subarray(offset, offset + this.chunkBytes)); - const manifest = { - v: 2, - sourceId, - targetHash: digest$1(target), - eventId, - bytes: bytes.length, - hash: digest$1(bytes), - chunks: chunks.map((value, index) => ({ - name: String(index).padStart(6, "0") + ".chunk", - bytes: value.length, - hash: digest$1(value) - })), - conflict, - completed: false + } catch (error) { + if (code(error) !== "ENOENT") throw error; + } + const chunks = []; + for (let offset = 0; offset < bytes.length; offset += this.chunkBytes) chunks.push(bytes.subarray(offset, offset + this.chunkBytes)); + const manifest = { + v: 2, + sourceId, + targetHash: digest$1(target), + eventId, + bytes: bytes.length, + hash: digest$1(bytes), + chunks: chunks.map((value, index) => ({ + name: String(index).padStart(6, "0") + ".chunk", + bytes: value.length, + hash: digest$1(value) + })), + conflict, + completed: false + }; + const manifestBytes = canonical$1(manifest); + if (Buffer.byteLength(manifestBytes) > 64 * 1024) fail("SOURCE_TOO_LARGE"); + this.budget("original", BLOCK + charge((chunks.length + 1) * 256) + charge(Buffer.byteLength(manifestBytes)) + chunks.reduce((sum, value) => sum + charge(value.length), 0)); + const stage = join(this.originals, ".stage-" + randomUUID()); + mkdirSync(stage, { mode: 448 }); + syncDirectory$1(this.originals); + for (let index = 0; index < chunks.length; index++) { + this.writeFile(join(stage, manifest.chunks[index].name), chunks[index]); + this.phase("chunk_fsynced"); + } + this.writeFile(join(stage, "manifest.json"), manifestBytes); + this.phase("manifest_fsynced"); + syncDirectory$1(stage); + this.phase("source_before_publish"); + renameSync(stage, join(this.originals, sourceId)); + syncDirectory$1(this.originals); + this.phase("source_published"); + const verified = this.source(target, sourceId); + return { + sourceId, + duplicate: false, + conflict, + completed: false, + body: verified.body + }; + } + async retain(target, eventId, body) { + const { normalized, bytes } = this.prepareSource(target, eventId, body); + return this.locked(() => this.retainSource(target, eventId, normalized, bytes)); + } + async retainAndEnqueue(target, eventId, body) { + const { normalized, bytes } = this.prepareSource(target, eventId, body); + return this.locked(() => { + const retained = this.retainSource(target, eventId, normalized, bytes); + if (retained.conflict) return { + retained, + referenceAcknowledged: false }; - const manifestBytes = canonical$1(manifest); - if (Buffer.byteLength(manifestBytes) > 64 * 1024) fail("SOURCE_TOO_LARGE"); - this.budget("original", BLOCK + charge((chunks.length + 1) * 256) + charge(Buffer.byteLength(manifestBytes)) + chunks.reduce((sum, value) => sum + charge(value.length), 0)); - const stage = join(this.originals, ".stage-" + randomUUID()); - mkdirSync(stage, { mode: 448 }); - syncDirectory$1(this.originals); - for (let index = 0; index < chunks.length; index++) { - this.writeFile(join(stage, manifest.chunks[index].name), chunks[index]); - this.phase("chunk_fsynced"); + if (retained.completed) return { + retained, + referenceAcknowledged: true + }; + try { + this.enqueueSource(target, retained.sourceId); + } catch (error) { + return { + retained, + referenceAcknowledged: false, + error + }; } - this.writeFile(join(stage, "manifest.json"), manifestBytes); - this.phase("manifest_fsynced"); - syncDirectory$1(stage); - this.phase("source_before_publish"); - renameSync(stage, join(this.originals, sourceId)); - syncDirectory$1(this.originals); - this.phase("source_published"); - const verified = this.source(target, sourceId); return { - sourceId, - duplicate: false, - conflict, - completed: false, - body: verified.body + retained, + referenceAcknowledged: true }; }); } @@ -394,30 +471,31 @@ var SdkSpoolV2 = class { fail("SOURCE_UNVERIFIED"); } } - writeRef(ref) { - this.atomicJson(this.refPath(ref.sourceId), ref, "queue"); + writeRef(ref, transition = true) { + this.atomicJson(this.refPath(ref.sourceId), ref, "queue", void 0, transition); this.phase("reference_published"); } + enqueueSource(target, sourceId) { + const { manifest } = this.source(target, sourceId); + if (manifest.conflict) fail("SOURCE_CONFLICT"); + if (manifest.completed) return; + try { + this.readRef(target, sourceId); + return; + } catch (error) { + if (code(error) !== "ENOENT") throw error; + } + this.writeRef({ + v: 2, + targetHash: digest$1(target), + sourceId, + state: "pending", + attempts: 0, + nextAttemptAt: 0 + }, false); + } async enqueue(target, sourceId) { - return this.locked(() => { - const { manifest } = this.source(target, sourceId); - if (manifest.conflict) fail("SOURCE_CONFLICT"); - if (manifest.completed) return; - try { - this.readRef(target, sourceId); - return; - } catch (error) { - if (code(error) !== "ENOENT") throw error; - } - this.writeRef({ - v: 2, - targetHash: digest$1(target), - sourceId, - state: "pending", - attempts: 0, - nextAttemptAt: 0 - }); - }); + return this.locked(() => this.enqueueSource(target, sourceId)); } async pending(target) { return this.locked(() => { @@ -432,7 +510,7 @@ var SdkSpoolV2 = class { this.writeRef(ref); } try { - if (this.source(target, ref.sourceId).manifest.completed) continue; + if (this.manifest(target, ref.sourceId).completed) continue; } catch { ref.state = "held"; } @@ -446,19 +524,39 @@ var SdkSpoolV2 = class { })); }); } - async markSending(target, sourceId) { + async markSending(target, sourceId, expectedHeldSource) { + await this.claimSource(target, sourceId, expectedHeldSource); + } + async claimForSend(target, sourceId, sendDeadline) { + return this.claimSource(target, sourceId, void 0, sendDeadline); + } + async claimSource(target, sourceId, expectedHeldSource, sendDeadline) { + if (sendDeadline !== void 0 && !Number.isFinite(sendDeadline)) fail("STORE_OPTIONS_INVALID"); return this.locked(() => { const source = this.source(target, sourceId); if (source.manifest.conflict || source.manifest.completed) fail("SOURCE_HELD"); const ref = this.readRef(target, sourceId); - if (ref.state !== "pending") fail("SOURCE_HELD"); - if (ref.nextAttemptAt > Date.now()) fail("SOURCE_BACKOFF"); + if (expectedHeldSource) { + if (ref.state !== "held" || ref.owner || !isDeepStrictEqual(source.body, expectedHeldSource)) fail("SOURCE_HELD"); + } else { + if (ref.state === "sending") { + if (alive(ref.owner)) fail("SOURCE_SENDING"); + ref.state = "held"; + ref.reason = "UNKNOWN_SEND"; + delete ref.owner; + this.writeRef(ref); + } + if (ref.state !== "pending") fail("SOURCE_HELD"); + if (ref.nextAttemptAt > Date.now()) fail("SOURCE_BACKOFF"); + } const owner = ownerIdentity(); if (!owner) fail("STORE_BUSY"); + if (sendDeadline !== void 0 && performance.now() >= sendDeadline) fail("SOURCE_SEND_BUDGET_EXHAUSTED"); ref.state = "sending"; ref.owner = owner; this.writeRef(ref); this.phase("send_intent_fsynced"); + return source.body; }); } async acknowledge(target, sourceId) { @@ -466,9 +564,10 @@ var SdkSpoolV2 = class { const source = this.source(target, sourceId); if (source.manifest.conflict) fail("SOURCE_CONFLICT"); if (!source.manifest.completed) { - if (this.readRef(target, sourceId).state !== "sending") fail("SOURCE_HELD"); + const ref = this.readRef(target, sourceId); + if (ref.state !== "sending" || !isDeepStrictEqual(ref.owner, ownerIdentity())) fail("SOURCE_HELD"); source.manifest.completed = true; - this.atomicJson(join(this.originals, sourceId, "manifest.json"), source.manifest, "original", "ack_before_publish"); + this.atomicJson(join(this.originals, sourceId, "manifest.json"), source.manifest, "original", "ack_before_publish", true); this.phase("ack_fsynced"); } try { @@ -482,7 +581,7 @@ var SdkSpoolV2 = class { async retry(target, sourceId, _reason) { return this.locked(() => { const ref = this.readRef(target, sourceId); - if (ref.state !== "sending" || !ref.owner || !alive(ref.owner)) fail("SOURCE_HELD"); + if (ref.state !== "sending" || !isDeepStrictEqual(ref.owner, ownerIdentity())) fail("SOURCE_HELD"); ref.attempts = Math.min(1e6, ref.attempts + 1); ref.nextAttemptAt = Date.now() + Math.min(3e5, 1e3 * 2 ** Math.min(9, ref.attempts - 1)); ref.state = "pending"; @@ -494,6 +593,7 @@ var SdkSpoolV2 = class { async hold(target, sourceId, _reason) { return this.locked(() => { const ref = this.readRef(target, sourceId); + if (ref.state !== "sending" || !isDeepStrictEqual(ref.owner, ownerIdentity())) fail("SOURCE_HELD"); ref.state = "held"; ref.reason = "HELD"; delete ref.owner; @@ -525,7 +625,8 @@ function safeCode(error) { function createStore(options) { return new SdkSpoolV2({ ...options.storeOptions, - dir: options.dir + dir: options.dir, + deadline: options.deadline }); } function held(code, sourceAcknowledged = true, referenceAcknowledged = true) { @@ -545,16 +646,17 @@ async function park(store, target, sourceId, code) { return held(code); } async function sendReference(store, target, sourceId, post, deadline) { - try { - await store.markSending(target, sourceId); - } catch (error) { - return held(safeCode(error)); - } let body; try { - body = await store.load(target, sourceId); - } catch { - return park(store, target, sourceId, "SOURCE_UNVERIFIED"); + body = await store.claimForSend(target, sourceId, deadline); + } catch (error) { + const code = error?.code; + if (code === "SOURCE_BACKOFF" || code === "SOURCE_SEND_BUDGET_EXHAUSTED") return { + outcome: "spooled", + sourceAcknowledged: true, + referenceAcknowledged: true + }; + return held(code === "SOURCE_SENDING" ? "SDK_SEND_IN_PROGRESS_OR_UNCERTAIN" : safeCode(error)); } let response; const budget = deadline === void 0 ? void 0 : { @@ -614,7 +716,7 @@ async function deliverSdkCaptureV2(target, body, post, options = {}) { sourceAcknowledged: false, referenceAcknowledged: false }; - const retained = await store.retain(target, body.eventId, body); + const handoff = await store.retainAndEnqueue(target, body.eventId, body), retained = handoff.retained; sourceId = retained.sourceId; storedBody = retained.body; if (retained.conflict) return held("SOURCE_CONFLICT", true, false); @@ -624,48 +726,30 @@ async function deliverSdkCaptureV2(target, body, post, options = {}) { referenceAcknowledged: true, sdkReplyConfirmed: true }; - } catch (error) { - return { + if (!handoff.referenceAcknowledged) return { outcome: "dropped", - code: safeCode(error), - sourceAcknowledged: false, + code: safeCode(handoff.error), + sourceAcknowledged: true, referenceAcknowledged: false }; - } - try { - await store.enqueue(target, sourceId); } catch (error) { return { outcome: "dropped", code: safeCode(error), - sourceAcknowledged: true, + sourceAcknowledged: false, referenceAcknowledged: false }; } - let pending; - try { - pending = (await store.pending(target)).find((record) => record.sourceId === sourceId); - } catch (error) { - return held(safeCode(error)); - } - if (!pending) { - try { - if ((await store.retain(target, storedBody.eventId, storedBody)).completed) return { - outcome: "duplicate", - sourceAcknowledged: true, - referenceAcknowledged: true, - sdkReplyConfirmed: true - }; - } catch {} - return held("QUEUE_STATE_UNCONFIRMED"); - } - if (pending.state !== "pending") return held(pending.state === "sending" ? "SDK_SEND_IN_PROGRESS_OR_UNCERTAIN" : "SOURCE_HELD"); - if (pending.nextAttemptAt > Date.now()) return { - outcome: "spooled", - sourceAcknowledged: true, - referenceAcknowledged: true - }; - return sendReference(store, target, sourceId, post); + const sent = await sendReference(store, target, sourceId, post, options.deadline === void 0 ? void 0 : options.deadline - 750); + if (sent.code === "SOURCE_HELD") try { + if ((await store.retain(target, storedBody.eventId, storedBody)).completed) return { + outcome: "duplicate", + sourceAcknowledged: true, + referenceAcknowledged: true, + sdkReplyConfirmed: true + }; + } catch {} + return sent; } async function drainSdkCapturesV2(target, post, options = {}) { const result = { @@ -677,7 +761,7 @@ async function drainSdkCapturesV2(target, post, options = {}) { error: void 0 }; const limit = Math.min(8, Math.max(0, options.limit ?? 8)), budget = Math.min(1500, Math.max(0, options.budgetMs ?? 1500)), start = performance.now(); - const deadline = Math.min(options.deadline ?? Infinity, start + budget); + const deadline = Math.min(options.deadline === void 0 ? Infinity : options.deadline - 750, start + budget); try { const store = createStore(options), records = await store.pending(target); for (const ref of records) { @@ -743,7 +827,10 @@ const SECRET_PATTERN_SOURCES = [ const PRIVATE_KEY_BLOCK_RE = /-----BEGIN [A-Z0-9 ]*PRIVATE KEY(?: BLOCK)?-----[\s\S]*?(?:-----END [A-Z0-9 ]*PRIVATE KEY(?: BLOCK)?-----|$)/g; const URL_CREDENTIALS_RE = /\b([a-z][a-z0-9+.-]*:\/\/)[^\s/?#@:"'<>]*:[^\s/?#@"'<>]+@/gi; function stripPrivateData(input) { - let result = redactPrivateBlocks(input); + return stripSecrets(redactPrivateBlocks(input)); +} +function stripSecrets(input) { + let result = input; result = result.replace(new RegExp(PRIVATE_KEY_BLOCK_RE.source, PRIVATE_KEY_BLOCK_RE.flags), "[REDACTED_SECRET]"); result = result.replace(new RegExp(URL_CREDENTIALS_RE.source, URL_CREDENTIALS_RE.flags), "$1[REDACTED_SECRET]@"); for (const source of SECRET_PATTERN_SOURCES) { @@ -753,12 +840,13 @@ function stripPrivateData(input) { return result; } function scrubRecord(value, options = {}) { - if (typeof value === "string") return stripPrivateData(value); + const strip = options.strip ?? stripPrivateData; + if (typeof value === "string") return strip(value); if (Array.isArray(value)) return value.map((item) => scrubRecord(item, options)); if (value !== null && typeof value === "object") { const proto = Object.getPrototypeOf(value); if (proto !== Object.prototype && proto !== null) return value; - const entries = Object.entries(value).map(([key, item]) => [options.scrubKeys ? stripPrivateData(key) : key, item]); + const entries = Object.entries(value).map(([key, item]) => [options.scrubKeys ? strip(key) : key, item]); const reserved = new Set(entries.map(([key]) => key)); const used = /* @__PURE__ */ new Set(); return Object.fromEntries(entries.map(([key, item]) => { @@ -1731,7 +1819,7 @@ function unquote(value) { const hash = value.indexOf(" #"); return hash === -1 ? value : value.slice(0, hash).trim(); } -function readEnvFileSecret() { +function readEnvFileValue(name) { let content; try { content = readFileSync(process.env["AGENTMEMORY_ENV_FILE"] || join(homedir(), ".agentmemory", ".env"), "utf-8"); @@ -1745,11 +1833,14 @@ function readEnvFileSecret() { if (trimmed.startsWith("#")) continue; const eq = trimmed.indexOf("="); if (eq === -1) continue; - if (trimmed.slice(0, eq).replace(/^export\s+/, "").trim() !== SECRET_KEY) continue; - found = usable(unquote(trimmed.slice(eq + 1).trim())); + if (trimmed.slice(0, eq).replace(/^export\s+/, "").trim() !== name) continue; + found = unquote(trimmed.slice(eq + 1).trim()); } return found; } +function readEnvFileSecret() { + return usable(readEnvFileValue(SECRET_KEY)); +} function readSecretFile(path) { try { return usable(readFileSync(path, "utf-8")); @@ -2124,9 +2215,12 @@ async function captureObservationV2(body, timeoutMs) { eventId, captureCompletion: "sdk-v1" })), { scrubKeys: true }); - const options = { dir: process.env["AGENTMEMORY_SDK_CAPTURE_V2_DIR"] }; - const result = await deliverSdkCaptureV2(REST_URL, payload, (b) => postRawV2(b, timeoutMs), options); - const now = performance.now(), deadline = Math.min(started + timeoutMs, now + 250), remaining = Math.floor(deadline - now); + const deadline = started + timeoutMs, options = { + dir: process.env["AGENTMEMORY_SDK_CAPTURE_V2_DIR"], + deadline + }; + const result = await deliverSdkCaptureV2(REST_URL, payload, (b, budget) => postRawV2(b, budget?.remainingMs ?? timeoutMs, budget?.deadline), options); + const now = performance.now(), remaining = Math.floor(Math.min(deadline - 750 - now, 250)); if (body.hookType !== "stop" && (result.outcome === "delivered" || result.outcome === "duplicate") && remaining >= 100) { const backlog = await drainSdkCapturesV2(REST_URL, (b, budget) => postRawV2(b, budget?.remainingMs ?? remaining, budget?.deadline ?? deadline), { ...options, diff --git a/plugin/scripts/codex-compat/manifest.json b/plugin/scripts/codex-compat/manifest.json index d0cb6f8204..849fe33165 100644 --- a/plugin/scripts/codex-compat/manifest.json +++ b/plugin/scripts/codex-compat/manifest.json @@ -14,7 +14,7 @@ "subagent-start.mjs": "24a02be53008721379cbf115884a7739baa2884f8d5f3e12a967b9ae8b0c2869", "subagent-stop.mjs": "8fc6929af24afd799cbaec3a3ca210971c9f51f139f6666fe0025fdfdc62dba8", "provenance.json": "4d768394ad40a29be9d2721fccffd0a054ec97bcc6b868305a27e755eb9b1759", - "_capture.mjs": "d48e2e7d12d6790a0851127bfef80b35e0c0cae8b7664c6897b0965b38bd9838" + "_capture.mjs": "77c06db3bd86d2eb5595561b04060fcb9ab98c74dd2d0275704a85c835924e09" }, "deploymentMapping": { "plugin/scripts/long-session-hook-client.mjs": "plugin/scripts/codex-compat/long-session-hook-client.mjs", diff --git a/plugin/scripts/cursor/run-detached.mjs b/plugin/scripts/cursor/run-detached.mjs index 25515f55e7..0acb8da80e 100755 --- a/plugin/scripts/cursor/run-detached.mjs +++ b/plugin/scripts/cursor/run-detached.mjs @@ -716,6 +716,10 @@ function delegateHook(hookKey, data, options = {}) { return 0; } const { project, payload } = enrichPayload(data); + if (hookKey === "subagentStop") { + payload.agent_type ||= data.agentType || data.agentDisplayName || data.agentName || data.subagent_type; + payload.last_assistant_message ??= data.response ?? data.summary; + } const child = spawnSync(process.execPath, [scriptPath], { input: JSON.stringify(payload), env: { diff --git a/plugin/scripts/cursor/run-hook.mjs b/plugin/scripts/cursor/run-hook.mjs index cad1c0492b..92c72dd00c 100755 --- a/plugin/scripts/cursor/run-hook.mjs +++ b/plugin/scripts/cursor/run-hook.mjs @@ -658,6 +658,10 @@ function delegateHook(hookKey, data, options = {}) { return 0; } const { project, payload } = enrichPayload(data); + if (hookKey === "subagentStop") { + payload.agent_type ||= data.agentType || data.agentDisplayName || data.agentName || data.subagent_type; + payload.last_assistant_message ??= data.response ?? data.summary; + } const child = spawnSync(process.execPath, [scriptPath], { input: JSON.stringify(payload), env: { diff --git a/plugin/scripts/plugin-bridge.mjs b/plugin/scripts/plugin-bridge.mjs index de03059905..788aa540fb 100644 --- a/plugin/scripts/plugin-bridge.mjs +++ b/plugin/scripts/plugin-bridge.mjs @@ -3,7 +3,7 @@ import { readFileSync, realpathSync } from "node:fs"; import { homedir } from "node:os"; import { join } from "node:path"; //#region src/version.ts -const VERSION = "0.9.32-daniel.55"; +const VERSION = "0.9.33-daniel.73"; //#endregion //#region src/secret-store.ts const SECRET_KEY = "AGENTMEMORY_SECRET"; @@ -29,7 +29,7 @@ function unquote(value) { const hash = value.indexOf(" #"); return hash === -1 ? value : value.slice(0, hash).trim(); } -function readEnvFileSecret() { +function readEnvFileValue(name) { let content; try { content = readFileSync(process.env["AGENTMEMORY_ENV_FILE"] || join(homedir(), ".agentmemory", ".env"), "utf-8"); @@ -43,11 +43,14 @@ function readEnvFileSecret() { if (trimmed.startsWith("#")) continue; const eq = trimmed.indexOf("="); if (eq === -1) continue; - if (trimmed.slice(0, eq).replace(/^export\s+/, "").trim() !== SECRET_KEY) continue; - found = usable(unquote(trimmed.slice(eq + 1).trim())); + if (trimmed.slice(0, eq).replace(/^export\s+/, "").trim() !== name) continue; + found = unquote(trimmed.slice(eq + 1).trim()); } return found; } +function readEnvFileSecret() { + return usable(readEnvFileValue(SECRET_KEY)); +} function readSecretFile(path) { try { return usable(readFileSync(path, "utf-8")); @@ -115,6 +118,14 @@ function resolveSettingSecret(url, env = process.env) { } //#endregion //#region src/mcp/rest-proxy.ts +const DEFAULT_CALL_TIMEOUT_MS = 19e4; +const MAX_TIMER_MS = 2147483647; +function callTimeoutMs() { + const name = "AGENTMEMORY_CALL_TIMEOUT_MS"; + const raw = (resolveEnvOrEmpty(name) || readEnvFileValue(name)).trim(); + const n = /^\d+$/.test(raw) ? Number(raw) : 0; + return n >= 1 && n <= MAX_TIMER_MS ? n : DEFAULT_CALL_TIMEOUT_MS; +} function resolveEnvOrEmpty(name) { return hookSetting(name) ?? ""; } @@ -343,7 +354,7 @@ function createPluginBridge() { }, body: body === void 0 ? void 0 : JSON.stringify(body), redirect: "error", - signal: AbortSignal.timeout(15e3) + signal: AbortSignal.timeout(callTimeoutMs()) }); } catch { throw new Error(`Agent Memory daemon is unreachable or timed out. ${SETUP} No fallback store was used; check state before retrying a write.`); diff --git a/plugin/scripts/post-tool-failure.mjs b/plugin/scripts/post-tool-failure.mjs index 85cf906e5c..3beb4070ba 100755 --- a/plugin/scripts/post-tool-failure.mjs +++ b/plugin/scripts/post-tool-failure.mjs @@ -3,6 +3,86 @@ import { execSync } from "node:child_process"; import { realpathSync } from "node:fs"; import { basename, dirname, resolve } from "node:path"; import { captureObservation, isDrainChild, runDrainChild, withEventId } from "./_capture.mjs"; +//#region src/functions/privacy.ts +const PRIVATE_TAG_TOKEN_RE = /)[^>]*>|<\/private\s*>/gi; +function redactPrivateBlocks(input) { + const tokens = new RegExp(PRIVATE_TAG_TOKEN_RE.source, PRIVATE_TAG_TOKEN_RE.flags); + let out = ""; + let cursor = 0; + let depth = 0; + for (let match = tokens.exec(input); match; match = tokens.exec(input)) { + const closing = match[0].startsWith(" 0) return `${out}[REDACTED]`; + return out + input.slice(cursor); +} +const SECRET_PATTERN_SOURCES = [ + /(?:api[_-]?key|secret|token|password|credential|auth)[\s]*[=:]\s*["']?[A-Za-z0-9_\-/.+]{20,}["']?/gi, + /Bearer\s+[A-Za-z0-9._\-+/=]{20,}/gi, + /(?]*:[^\s/?#@"'<>]+@/gi; +function stripPrivateData(input) { + return stripSecrets(redactPrivateBlocks(input)); +} +function stripSecrets(input) { + let result = input; + result = result.replace(new RegExp(PRIVATE_KEY_BLOCK_RE.source, PRIVATE_KEY_BLOCK_RE.flags), "[REDACTED_SECRET]"); + result = result.replace(new RegExp(URL_CREDENTIALS_RE.source, URL_CREDENTIALS_RE.flags), "$1[REDACTED_SECRET]@"); + for (const source of SECRET_PATTERN_SOURCES) { + const pattern = new RegExp(source.source, source.flags); + result = result.replace(pattern, "[REDACTED_SECRET]"); + } + return result; +} +function scrubRecord(value, options = {}) { + const strip = options.strip ?? stripPrivateData; + if (typeof value === "string") return strip(value); + if (Array.isArray(value)) return value.map((item) => scrubRecord(item, options)); + if (value !== null && typeof value === "object") { + const proto = Object.getPrototypeOf(value); + if (proto !== Object.prototype && proto !== null) return value; + const entries = Object.entries(value).map(([key, item]) => [options.scrubKeys ? strip(key) : key, item]); + const reserved = new Set(entries.map(([key]) => key)); + const used = /* @__PURE__ */ new Set(); + return Object.fromEntries(entries.map(([key, item]) => { + let unique = key; + if (used.has(unique)) { + let suffix = 2; + do + unique = `${key}_${suffix++}`; + while (reserved.has(unique) || used.has(unique)); + } + used.add(unique); + return [unique, scrubRecord(item, options)]; + })); + } + return value; +} +//#endregion //#region src/hooks/_capture-filter.ts const DEFAULT_DENY_PATTERNS = [ "memory_*", @@ -43,11 +123,56 @@ function shouldCaptureTool(toolName) { if (allow) return matchesAny(toolName, allow); return !matchesAny(toolName, [...DEFAULT_DENY_PATTERNS, ...parseEnvList(process.env["AGENTMEMORY_CAPTURE_DENY"]) ?? []]); } -function truncateText(value, max) { +function cut(value, max) { if (value.length <= max) return value; const end = /[\uD800-\uDBFF]/.test(value[max - 1] ?? "") ? max - 1 : max; return value.slice(0, end); } +function truncateText(value, max) { + if (value.length <= max) return value; + return cut(stripPrivateData(value), max); +} +const JSON_WHITESPACE_ESCAPES = { + n: "\n", + r: "\r", + t: " " +}; +const SECRET_KEY_SUFFIXES = [ + "apikey", + "secret", + "token", + "password", + "passwd", + "credential", + "credentials" +]; +const SECRET_KEY_WORDS = new Set(["auth", "authorization"]); +function isSecretKeyName(key) { + if (!/^[\w.-]{1,64}$/.test(key)) return false; + const words = key.replace(/([a-z])([A-Z])/g, "$1_$2").replace(/([A-Za-z])(\d)/g, "$1_$2").toLowerCase().split(/[_.\-\d]+/).filter(Boolean); + return words.some((word, i) => SECRET_KEY_WORDS.has(word) || SECRET_KEY_SUFFIXES.some((suffix) => word.endsWith(suffix)) || word === "api" && words[i + 1] === "key"); +} +function redactSecretKeys(value) { + if (Array.isArray(value)) return value.map(redactSecretKeys); + if (value === null || typeof value !== "object") return value; + const proto = Object.getPrototypeOf(value); + if (proto !== Object.prototype && proto !== null) return value; + return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, typeof item === "string" && item && isSecretKeyName(key) ? "[REDACTED_SECRET]" : redactSecretKeys(item)])); +} +function scrubSecrets(value) { + return scrubRecord(redactSecretKeys(value), { + scrubKeys: true, + strip: stripSecrets + }); +} +function redactSerialized(serialized) { + return stripPrivateData(serialized.replace(/\\([\\nrt])/g, (escape, c) => JSON_WHITESPACE_ESCAPES[c] ?? escape)).replace(/[\n\r\t]/g, (c) => JSON.stringify(c).slice(1, -1)); +} +function truncateJson(value, max) { + const str = JSON.stringify(value); + if (str.length <= max) return str; + return cut(redactSerialized(JSON.stringify(scrubSecrets(value))), max); +} //#endregion //#region src/hooks/_project.ts function resolveProject(cwd) { @@ -121,8 +246,8 @@ async function main() { timestamp: (/* @__PURE__ */ new Date()).toISOString(), data: { tool_name: toolName, - tool_input: typeof toolInput === "string" ? truncateText(toolInput, 4e3) : truncateText(JSON.stringify(toolInput ?? ""), 4e3), - error: typeof error === "string" ? truncateText(error, 4e3) : truncateText(JSON.stringify(error ?? ""), 4e3) + tool_input: typeof toolInput === "string" ? truncateText(toolInput, 4e3) : truncateJson(toolInput ?? "", 4e3), + error: typeof error === "string" ? truncateText(error, 4e3) : truncateJson(error ?? "", 4e3) } }, data), OBSERVE_TIMEOUT_MS); setTimeout(() => process.exit(0), EXIT_CAP_MS).unref(); diff --git a/plugin/scripts/post-tool-use.mjs b/plugin/scripts/post-tool-use.mjs index 4f77d76bdb..35ef0642d1 100755 --- a/plugin/scripts/post-tool-use.mjs +++ b/plugin/scripts/post-tool-use.mjs @@ -3,6 +3,86 @@ import { execSync } from "node:child_process"; import { realpathSync } from "node:fs"; import { basename, dirname, resolve } from "node:path"; import { captureObservationForHook, isDrainChild, runDrainChild, withEventId } from "./_capture.mjs"; +//#region src/functions/privacy.ts +const PRIVATE_TAG_TOKEN_RE = /)[^>]*>|<\/private\s*>/gi; +function redactPrivateBlocks(input) { + const tokens = new RegExp(PRIVATE_TAG_TOKEN_RE.source, PRIVATE_TAG_TOKEN_RE.flags); + let out = ""; + let cursor = 0; + let depth = 0; + for (let match = tokens.exec(input); match; match = tokens.exec(input)) { + const closing = match[0].startsWith(" 0) return `${out}[REDACTED]`; + return out + input.slice(cursor); +} +const SECRET_PATTERN_SOURCES = [ + /(?:api[_-]?key|secret|token|password|credential|auth)[\s]*[=:]\s*["']?[A-Za-z0-9_\-/.+]{20,}["']?/gi, + /Bearer\s+[A-Za-z0-9._\-+/=]{20,}/gi, + /(?]*:[^\s/?#@"'<>]+@/gi; +function stripPrivateData(input) { + return stripSecrets(redactPrivateBlocks(input)); +} +function stripSecrets(input) { + let result = input; + result = result.replace(new RegExp(PRIVATE_KEY_BLOCK_RE.source, PRIVATE_KEY_BLOCK_RE.flags), "[REDACTED_SECRET]"); + result = result.replace(new RegExp(URL_CREDENTIALS_RE.source, URL_CREDENTIALS_RE.flags), "$1[REDACTED_SECRET]@"); + for (const source of SECRET_PATTERN_SOURCES) { + const pattern = new RegExp(source.source, source.flags); + result = result.replace(pattern, "[REDACTED_SECRET]"); + } + return result; +} +function scrubRecord(value, options = {}) { + const strip = options.strip ?? stripPrivateData; + if (typeof value === "string") return strip(value); + if (Array.isArray(value)) return value.map((item) => scrubRecord(item, options)); + if (value !== null && typeof value === "object") { + const proto = Object.getPrototypeOf(value); + if (proto !== Object.prototype && proto !== null) return value; + const entries = Object.entries(value).map(([key, item]) => [options.scrubKeys ? strip(key) : key, item]); + const reserved = new Set(entries.map(([key]) => key)); + const used = /* @__PURE__ */ new Set(); + return Object.fromEntries(entries.map(([key, item]) => { + let unique = key; + if (used.has(unique)) { + let suffix = 2; + do + unique = `${key}_${suffix++}`; + while (reserved.has(unique) || used.has(unique)); + } + used.add(unique); + return [unique, scrubRecord(item, options)]; + })); + } + return value; +} +//#endregion //#region src/hooks/_capture-filter.ts const DEFAULT_DENY_PATTERNS = [ "memory_*", @@ -49,22 +129,56 @@ function captureOutputMax() { const parsed = Number.parseInt(raw, 10); return Number.isFinite(parsed) && parsed > 0 ? parsed : 8e3; } -function truncateText(value, max) { +function cut(value, max) { if (value.length <= max) return value; const end = /[\uD800-\uDBFF]/.test(value[max - 1] ?? "") ? max - 1 : max; return value.slice(0, end); } +const JSON_WHITESPACE_ESCAPES = { + n: "\n", + r: "\r", + t: " " +}; +const SECRET_KEY_SUFFIXES = [ + "apikey", + "secret", + "token", + "password", + "passwd", + "credential", + "credentials" +]; +const SECRET_KEY_WORDS = new Set(["auth", "authorization"]); +function isSecretKeyName(key) { + if (!/^[\w.-]{1,64}$/.test(key)) return false; + const words = key.replace(/([a-z])([A-Z])/g, "$1_$2").replace(/([A-Za-z])(\d)/g, "$1_$2").toLowerCase().split(/[_.\-\d]+/).filter(Boolean); + return words.some((word, i) => SECRET_KEY_WORDS.has(word) || SECRET_KEY_SUFFIXES.some((suffix) => word.endsWith(suffix)) || word === "api" && words[i + 1] === "key"); +} +function redactSecretKeys(value) { + if (Array.isArray(value)) return value.map(redactSecretKeys); + if (value === null || typeof value !== "object") return value; + const proto = Object.getPrototypeOf(value); + if (proto !== Object.prototype && proto !== null) return value; + return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, typeof item === "string" && item && isSecretKeyName(key) ? "[REDACTED_SECRET]" : redactSecretKeys(item)])); +} +function scrubSecrets(value) { + return scrubRecord(redactSecretKeys(value), { + scrubKeys: true, + strip: stripSecrets + }); +} +function redactSerialized(serialized) { + return stripPrivateData(serialized.replace(/\\([\\nrt])/g, (escape, c) => JSON_WHITESPACE_ESCAPES[c] ?? escape)).replace(/[\n\r\t]/g, (c) => JSON.stringify(c).slice(1, -1)); +} +function truncateWithSuffix(value, max, suffix) { + const safe = stripPrivateData(value); + if (safe.length <= max) return safe; + return max <= suffix.length ? suffix.slice(0, max) : cut(safe, max - suffix.length) + suffix; +} function truncateCaptureOutput(value, max) { - if (typeof value === "string" && value.length > max) { - const suffix = "\n[...truncated]"; - return max <= 15 ? suffix.slice(0, max) : truncateText(value, max - 15) + suffix; - } + if (typeof value === "string" && value.length > max) return truncateWithSuffix(value, max, "\n[...truncated]"); if (typeof value === "object" && value !== null) { - const str = JSON.stringify(value); - if (str.length > max) { - const suffix = "...[truncated]"; - return max <= 14 ? suffix.slice(0, max) : truncateText(str, max - 14) + suffix; - } + if (JSON.stringify(value).length > max) return truncateWithSuffix(redactSerialized(JSON.stringify(scrubSecrets(value))), max, "...[truncated]"); return value; } return value; diff --git a/plugin/scripts/pre-compact.mjs b/plugin/scripts/pre-compact.mjs index 56172e63f7..489ab1d830 100755 --- a/plugin/scripts/pre-compact.mjs +++ b/plugin/scripts/pre-compact.mjs @@ -36,7 +36,7 @@ function unquote(value) { const hash = value.indexOf(" #"); return hash === -1 ? value : value.slice(0, hash).trim(); } -function readEnvFileSecret() { +function readEnvFileValue(name) { let content; try { content = readFileSync(process.env["AGENTMEMORY_ENV_FILE"] || join(homedir(), ".agentmemory", ".env"), "utf-8"); @@ -50,11 +50,14 @@ function readEnvFileSecret() { if (trimmed.startsWith("#")) continue; const eq = trimmed.indexOf("="); if (eq === -1) continue; - if (trimmed.slice(0, eq).replace(/^export\s+/, "").trim() !== SECRET_KEY) continue; - found = usable(unquote(trimmed.slice(eq + 1).trim())); + if (trimmed.slice(0, eq).replace(/^export\s+/, "").trim() !== name) continue; + found = unquote(trimmed.slice(eq + 1).trim()); } return found; } +function readEnvFileSecret() { + return usable(readEnvFileValue(SECRET_KEY)); +} function readSecretFile(path) { try { return usable(readFileSync(path, "utf-8")); diff --git a/plugin/scripts/pre-tool-use.mjs b/plugin/scripts/pre-tool-use.mjs index 4a339b8aec..f23c02e450 100755 --- a/plugin/scripts/pre-tool-use.mjs +++ b/plugin/scripts/pre-tool-use.mjs @@ -1,7 +1,8 @@ #!/usr/bin/env node -import { readFileSync } from "node:fs"; +import { readFileSync, realpathSync } from "node:fs"; import { homedir } from "node:os"; -import { join } from "node:path"; +import { basename, dirname, join, resolve } from "node:path"; +import { execSync } from "node:child_process"; //#region src/secret-store.ts const SECRET_KEY = "AGENTMEMORY_SECRET"; function agentmemoryHomeDir() { @@ -26,7 +27,7 @@ function unquote(value) { const hash = value.indexOf(" #"); return hash === -1 ? value : value.slice(0, hash).trim(); } -function readEnvFileSecret() { +function readEnvFileValue(name) { let content; try { content = readFileSync(process.env["AGENTMEMORY_ENV_FILE"] || join(homedir(), ".agentmemory", ".env"), "utf-8"); @@ -40,11 +41,14 @@ function readEnvFileSecret() { if (trimmed.startsWith("#")) continue; const eq = trimmed.indexOf("="); if (eq === -1) continue; - if (trimmed.slice(0, eq).replace(/^export\s+/, "").trim() !== SECRET_KEY) continue; - found = usable(unquote(trimmed.slice(eq + 1).trim())); + if (trimmed.slice(0, eq).replace(/^export\s+/, "").trim() !== name) continue; + found = unquote(trimmed.slice(eq + 1).trim()); } return found; } +function readEnvFileSecret() { + return usable(readEnvFileValue(SECRET_KEY)); +} function readSecretFile(path) { try { return usable(readFileSync(path, "utf-8")); @@ -143,6 +147,43 @@ function hookAuthHeaders(endpoint) { return h; } //#endregion +//#region src/hooks/_project.ts +function resolveProject(cwd) { + const explicit = process.env["AGENTMEMORY_PROJECT_NAME"]; + if (explicit && explicit.trim()) return explicit.trim(); + const dir = cwd && cwd.trim() ? cwd : process.cwd(); + try { + const [top, gitDir, commonDir] = execSync("git rev-parse --show-toplevel --git-dir --git-common-dir", { + cwd: dir, + stdio: [ + "ignore", + "pipe", + "ignore" + ], + timeout: 500 + }).toString().trim().split(/\r?\n/); + if (top) return basename(repositoryRoot(dir, top, gitDir, commonDir)); + } catch {} + return basename(dir); +} +function repositoryRoot(dir, top, gitDir, commonDir) { + if (!gitDir || !commonDir) return top; + const physical = realpathSync(dir); + const common = resolve(physical, commonDir); + if (resolve(physical, gitDir) === common || basename(common) !== ".git") return top; + return dirname(common); +} +function hookCwd(data) { + if (!data || typeof data !== "object") return void 0; + if (typeof data.cwd === "string" && data.cwd.trim()) return data.cwd; + const roots = data.workspace_roots; + if (Array.isArray(roots)) { + for (const root of roots) if (typeof root === "string" && root.trim()) return root; + } + const projectDir = process.env["DEVIN_PROJECT_DIR"] || process.env["CLAUDE_PROJECT_DIR"]; + if (projectDir && projectDir.trim()) return projectDir; +} +//#endregion //#region src/hooks/pre-tool-use.ts function isSdkChildContext(payload) { if (process.env["AGENTMEMORY_SDK_CHILD"] === "1") return true; @@ -152,9 +193,26 @@ function isSdkChildContext(payload) { const INJECT_CONTEXT = process.env["AGENTMEMORY_INJECT_CONTEXT"] === "true"; const ENDPOINT = resolveHookEndpoint(); const REST_URL = ENDPOINT.url; +const DEFAULT_ENRICH_TIMEOUT_MS = 2e3; function authHeaders() { return hookAuthHeaders(ENDPOINT); } +const MAX_TIMER_MS = 2147483647; +function enrichTimeoutMs() { + const raw = hookSetting("AGENTMEMORY_ENRICH_TIMEOUT_MS")?.trim() ?? ""; + const n = /^\d+$/.test(raw) ? Number(raw) : 0; + return n >= 1 && n <= MAX_TIMER_MS ? n : DEFAULT_ENRICH_TIMEOUT_MS; +} +function isPlainTextHost() { + return Boolean(process.env["FACTORY_PROJECT_DIR"] || process.env["DROID_PLUGIN_ROOT"]); +} +function contextPayload(data, context) { + if (data.hook_event_name !== "PreToolUse" || isPlainTextHost()) return context; + return JSON.stringify({ hookSpecificOutput: { + hookEventName: "PreToolUse", + additionalContext: context + } }); +} async function main() { if (!INJECT_CONTEXT) return; if (reportBlockedTransport(ENDPOINT)) return; @@ -168,6 +226,7 @@ async function main() { } if (!data || typeof data !== "object") return; if (isSdkChildContext(data)) return; + if (typeof data.cursor_version === "string") return; const toolName = typeof data.tool_name === "string" ? data.tool_name : typeof data.toolName === "string" ? data.toolName : void 0; if (!toolName) return; const normalizedToolName = toolName.toLowerCase(); @@ -201,7 +260,8 @@ async function main() { } const rawSessionId = data.session_id || data.sessionId || data.conversation_id; const sessionId = typeof rawSessionId === "string" && rawSessionId.length > 0 ? rawSessionId : "unknown"; - const project = typeof data.project === "string" && data.project.trim().length > 0 ? data.project.trim() : void 0; + const project = ((typeof data.project === "string" && data.project.trim().length > 0 ? data.project.trim() : void 0) ?? resolveProject(hookCwd(data) || process.cwd())).trim(); + if (!project) return; try { const res = await fetch(`${REST_URL}/agentmemory/enrich`, { method: "POST", @@ -212,13 +272,13 @@ async function main() { files, terms, toolName, - ...project !== void 0 && { project } + project }), - signal: AbortSignal.timeout(2e3) + signal: AbortSignal.timeout(enrichTimeoutMs()) }); if (res.ok) { const result = await res.json(); - if (result.context) process.stdout.write(result.context); + if (result.context) process.stdout.write(contextPayload(data, result.context)); } } catch {} } diff --git a/plugin/scripts/session-end.mjs b/plugin/scripts/session-end.mjs index d64c0b2797..d146fecf14 100755 --- a/plugin/scripts/session-end.mjs +++ b/plugin/scripts/session-end.mjs @@ -1,14 +1,74 @@ #!/usr/bin/env node +import { execSync, spawn } from "node:child_process"; import { readFileSync, realpathSync } from "node:fs"; -import { execSync } from "node:child_process"; import { basename, dirname, resolve } from "node:path"; import { REST_URL, authHeaders, captureObservation, isDrainChild, runDrainChild, transportBlocked, withEventId } from "./_capture.mjs"; +//#region src/functions/privacy.ts +const PRIVATE_TAG_TOKEN_RE = /)[^>]*>|<\/private\s*>/gi; +function redactPrivateBlocks(input) { + const tokens = new RegExp(PRIVATE_TAG_TOKEN_RE.source, PRIVATE_TAG_TOKEN_RE.flags); + let out = ""; + let cursor = 0; + let depth = 0; + for (let match = tokens.exec(input); match; match = tokens.exec(input)) { + const closing = match[0].startsWith(" 0) return `${out}[REDACTED]`; + return out + input.slice(cursor); +} +const SECRET_PATTERN_SOURCES = [ + /(?:api[_-]?key|secret|token|password|credential|auth)[\s]*[=:]\s*["']?[A-Za-z0-9_\-/.+]{20,}["']?/gi, + /Bearer\s+[A-Za-z0-9._\-+/=]{20,}/gi, + /(?]*:[^\s/?#@"'<>]+@/gi; +function stripPrivateData(input) { + return stripSecrets(redactPrivateBlocks(input)); +} +function stripSecrets(input) { + let result = input; + result = result.replace(new RegExp(PRIVATE_KEY_BLOCK_RE.source, PRIVATE_KEY_BLOCK_RE.flags), "[REDACTED_SECRET]"); + result = result.replace(new RegExp(URL_CREDENTIALS_RE.source, URL_CREDENTIALS_RE.flags), "$1[REDACTED_SECRET]@"); + for (const source of SECRET_PATTERN_SOURCES) { + const pattern = new RegExp(source.source, source.flags); + result = result.replace(pattern, "[REDACTED_SECRET]"); + } + return result; +} +//#endregion //#region src/hooks/_capture-filter.ts -function truncateText(value, max) { +function cut(value, max) { if (value.length <= max) return value; const end = /[\uD800-\uDBFF]/.test(value[max - 1] ?? "") ? max - 1 : max; return value.slice(0, end); } +function truncateText(value, max) { + if (value.length <= max) return value; + return cut(stripPrivateData(value), max); +} //#endregion //#region src/hooks/_project.ts function resolveProject(cwd) { @@ -110,6 +170,37 @@ function extractTranscriptPrompts(data) { } return prompts; } +const WORKER_ENV = "AGENTMEMORY_SESSION_END_WORKER"; +function handOff(input) { + if (process.env[WORKER_ENV] === "1" || !process.argv[1]) return Promise.resolve(false); + return new Promise((resolve) => { + try { + const child = spawn(process.execPath, [process.argv[1]], { + detached: true, + stdio: [ + "pipe", + "ignore", + "ignore" + ], + windowsHide: true, + env: { + ...process.env, + [WORKER_ENV]: "1" + } + }); + child.on("error", () => resolve(false)); + child.stdin.on("error", () => resolve(false)); + child.once("spawn", () => { + child.stdin.end(input, () => { + child.unref(); + resolve(true); + }); + }); + } catch { + resolve(false); + } + }); +} async function main() { if (isDrainChild()) return runDrainChild(); let input = ""; @@ -122,6 +213,7 @@ async function main() { } if (!data || typeof data !== "object") return; if (isSdkChildContext(data)) return; + if (data.hook_event_name === "SessionEnd" && await handOff(input)) return; const sessionId = data.session_id || data.sessionId || data.conversation_id || "unknown"; const transcriptPrompts = extractTranscriptPrompts(data); if (transcriptPrompts.length > 0) { diff --git a/plugin/scripts/session-start.mjs b/plugin/scripts/session-start.mjs index 73c5038b6c..045957aa23 100755 --- a/plugin/scripts/session-start.mjs +++ b/plugin/scripts/session-start.mjs @@ -27,7 +27,7 @@ function unquote(value) { const hash = value.indexOf(" #"); return hash === -1 ? value : value.slice(0, hash).trim(); } -function readEnvFileSecret() { +function readEnvFileValue(name) { let content; try { content = readFileSync(process.env["AGENTMEMORY_ENV_FILE"] || join(homedir(), ".agentmemory", ".env"), "utf-8"); @@ -41,11 +41,14 @@ function readEnvFileSecret() { if (trimmed.startsWith("#")) continue; const eq = trimmed.indexOf("="); if (eq === -1) continue; - if (trimmed.slice(0, eq).replace(/^export\s+/, "").trim() !== SECRET_KEY) continue; - found = usable(unquote(trimmed.slice(eq + 1).trim())); + if (trimmed.slice(0, eq).replace(/^export\s+/, "").trim() !== name) continue; + found = unquote(trimmed.slice(eq + 1).trim()); } return found; } +function readEnvFileSecret() { + return usable(readEnvFileValue(SECRET_KEY)); +} function readSecretFile(path) { try { return usable(readFileSync(path, "utf-8")); diff --git a/plugin/scripts/subagent-stop.mjs b/plugin/scripts/subagent-stop.mjs index a01bff6cd0..66c330136d 100755 --- a/plugin/scripts/subagent-stop.mjs +++ b/plugin/scripts/subagent-stop.mjs @@ -3,12 +3,72 @@ import { execSync } from "node:child_process"; import { realpathSync } from "node:fs"; import { basename, dirname, resolve } from "node:path"; import { captureObservation, isDrainChild, runDrainChild, stripPrivateData, withEventId } from "./_capture.mjs"; +//#region src/functions/privacy.ts +const PRIVATE_TAG_TOKEN_RE = /)[^>]*>|<\/private\s*>/gi; +function redactPrivateBlocks(input) { + const tokens = new RegExp(PRIVATE_TAG_TOKEN_RE.source, PRIVATE_TAG_TOKEN_RE.flags); + let out = ""; + let cursor = 0; + let depth = 0; + for (let match = tokens.exec(input); match; match = tokens.exec(input)) { + const closing = match[0].startsWith(" 0) return `${out}[REDACTED]`; + return out + input.slice(cursor); +} +const SECRET_PATTERN_SOURCES = [ + /(?:api[_-]?key|secret|token|password|credential|auth)[\s]*[=:]\s*["']?[A-Za-z0-9_\-/.+]{20,}["']?/gi, + /Bearer\s+[A-Za-z0-9._\-+/=]{20,}/gi, + /(?]*:[^\s/?#@"'<>]+@/gi; +function stripPrivateData$1(input) { + return stripSecrets(redactPrivateBlocks(input)); +} +function stripSecrets(input) { + let result = input; + result = result.replace(new RegExp(PRIVATE_KEY_BLOCK_RE.source, PRIVATE_KEY_BLOCK_RE.flags), "[REDACTED_SECRET]"); + result = result.replace(new RegExp(URL_CREDENTIALS_RE.source, URL_CREDENTIALS_RE.flags), "$1[REDACTED_SECRET]@"); + for (const source of SECRET_PATTERN_SOURCES) { + const pattern = new RegExp(source.source, source.flags); + result = result.replace(pattern, "[REDACTED_SECRET]"); + } + return result; +} +//#endregion //#region src/hooks/_capture-filter.ts -function truncateText(value, max) { +function cut(value, max) { if (value.length <= max) return value; const end = /[\uD800-\uDBFF]/.test(value[max - 1] ?? "") ? max - 1 : max; return value.slice(0, end); } +function truncateText(value, max) { + if (value.length <= max) return value; + return cut(stripPrivateData$1(value), max); +} //#endregion //#region src/hooks/_project.ts function resolveProject(cwd) { diff --git a/plugin/scripts/task-completed.mjs b/plugin/scripts/task-completed.mjs index d92bb7240e..63bd03d239 100755 --- a/plugin/scripts/task-completed.mjs +++ b/plugin/scripts/task-completed.mjs @@ -3,12 +3,72 @@ import { execSync } from "node:child_process"; import { realpathSync } from "node:fs"; import { basename, dirname, resolve } from "node:path"; import { captureObservation, isDrainChild, runDrainChild, withEventId } from "./_capture.mjs"; +//#region src/functions/privacy.ts +const PRIVATE_TAG_TOKEN_RE = /)[^>]*>|<\/private\s*>/gi; +function redactPrivateBlocks(input) { + const tokens = new RegExp(PRIVATE_TAG_TOKEN_RE.source, PRIVATE_TAG_TOKEN_RE.flags); + let out = ""; + let cursor = 0; + let depth = 0; + for (let match = tokens.exec(input); match; match = tokens.exec(input)) { + const closing = match[0].startsWith(" 0) return `${out}[REDACTED]`; + return out + input.slice(cursor); +} +const SECRET_PATTERN_SOURCES = [ + /(?:api[_-]?key|secret|token|password|credential|auth)[\s]*[=:]\s*["']?[A-Za-z0-9_\-/.+]{20,}["']?/gi, + /Bearer\s+[A-Za-z0-9._\-+/=]{20,}/gi, + /(?]*:[^\s/?#@"'<>]+@/gi; +function stripPrivateData(input) { + return stripSecrets(redactPrivateBlocks(input)); +} +function stripSecrets(input) { + let result = input; + result = result.replace(new RegExp(PRIVATE_KEY_BLOCK_RE.source, PRIVATE_KEY_BLOCK_RE.flags), "[REDACTED_SECRET]"); + result = result.replace(new RegExp(URL_CREDENTIALS_RE.source, URL_CREDENTIALS_RE.flags), "$1[REDACTED_SECRET]@"); + for (const source of SECRET_PATTERN_SOURCES) { + const pattern = new RegExp(source.source, source.flags); + result = result.replace(pattern, "[REDACTED_SECRET]"); + } + return result; +} +//#endregion //#region src/hooks/_capture-filter.ts -function truncateText(value, max) { +function cut(value, max) { if (value.length <= max) return value; const end = /[\uD800-\uDBFF]/.test(value[max - 1] ?? "") ? max - 1 : max; return value.slice(0, end); } +function truncateText(value, max) { + if (value.length <= max) return value; + return cut(stripPrivateData(value), max); +} //#endregion //#region src/hooks/_project.ts function resolveProject(cwd) { diff --git a/plugin/skills/agentmemory-config/REFERENCE.md b/plugin/skills/agentmemory-config/REFERENCE.md index 425dcfa83f..cc92582160 100644 --- a/plugin/skills/agentmemory-config/REFERENCE.md +++ b/plugin/skills/agentmemory-config/REFERENCE.md @@ -3,7 +3,7 @@ Generated by scanning `src/` for `AGENTMEMORY_*` usage. Do not edit the block below by hand; run `npm run skills:gen` after adding or removing a variable. Internal markers ending in two underscores are excluded. -Configuration is read from the environment and from `~/.agentmemory/.env` (no `export` prefix). 88 recognized variables: +Configuration is read from the environment and from `~/.agentmemory/.env` (no `export` prefix). 95 recognized variables: - `AGENTMEMORY_AGENT_SCOPE` - `AGENTMEMORY_ALLOW_AGENT_SDK` @@ -12,6 +12,7 @@ Configuration is read from the environment and from `~/.agentmemory/.env` (no `e - `AGENTMEMORY_AUDIT_RETENTION_MONTHS` - `AGENTMEMORY_AUTO_CAPTURE_RECOVERY` - `AGENTMEMORY_AUTO_COMPRESS` +- `AGENTMEMORY_CALL_TIMEOUT_MS` - `AGENTMEMORY_CAPTURE_ALLOW` - `AGENTMEMORY_CAPTURE_DEAD_MAX` - `AGENTMEMORY_CAPTURE_DEDUP_HOURS` @@ -38,6 +39,7 @@ Configuration is read from the environment and from `~/.agentmemory/.env` (no `e - `AGENTMEMORY_DROP_STALE_INDEX` - `AGENTMEMORY_ENGINE_MANAGES_WORKER` - `AGENTMEMORY_ENGINE_READY_TIMEOUT_MS` +- `AGENTMEMORY_ENRICH_TIMEOUT_MS` - `AGENTMEMORY_ENV_FILE` - `AGENTMEMORY_EXPORT_ROOT` - `AGENTMEMORY_FEEDBACK_DIR` @@ -48,6 +50,7 @@ Configuration is read from the environment and from `~/.agentmemory/.env` (no `e - `AGENTMEMORY_GRAPH_COMPACT_ON_BOOT` - `AGENTMEMORY_GRAPH_WEIGHT` - `AGENTMEMORY_HOME` +- `AGENTMEMORY_HOOK_INJECT_CONTEXT` - `AGENTMEMORY_III_CONFIG` - `AGENTMEMORY_III_VERSION` - `AGENTMEMORY_IMAGE_EMBEDDINGS` @@ -58,6 +61,7 @@ Configuration is read from the environment and from `~/.agentmemory/.env` (no `e - `AGENTMEMORY_LLM_NOTHINK` - `AGENTMEMORY_LLM_PROVIDER` - `AGENTMEMORY_LLM_TIMEOUT_MS` +- `AGENTMEMORY_LOCAL_DISPATCH` - `AGENTMEMORY_MCP_BLOCK` - `AGENTMEMORY_METRICS_PORT` - `AGENTMEMORY_NATIVE_III_BIN` @@ -74,6 +78,7 @@ Configuration is read from the environment and from `~/.agentmemory/.env` (no `e - `AGENTMEMORY_SDK_CAPTURE_V2_DIR` - `AGENTMEMORY_SDK_CHILD` - `AGENTMEMORY_SECRET` +- `AGENTMEMORY_SESSION_END_WORKER` - `AGENTMEMORY_SESSION_ID` - `AGENTMEMORY_SESSION_SWEEP_ENABLED` - `AGENTMEMORY_SESSION_SWEEP_STALE_HOURS` @@ -81,6 +86,8 @@ Configuration is read from the environment and from `~/.agentmemory/.env` (no `e - `AGENTMEMORY_STATE_BACKEND` - `AGENTMEMORY_STATE_SAVE_INTERVAL_MS` - `AGENTMEMORY_STATE_STORE_PATH` +- `AGENTMEMORY_STATE_WORKER_ARTIFACT` +- `AGENTMEMORY_SUMMARY_TIMEOUT_MS` - `AGENTMEMORY_SUPPRESS_COST_WARNING` - `AGENTMEMORY_TOOLS` - `AGENTMEMORY_UPDATE_VIEWER` diff --git a/plugin/skills/agentmemory-mcp-tools/REFERENCE.md b/plugin/skills/agentmemory-mcp-tools/REFERENCE.md index f5fae804f2..ce7efad7dd 100644 --- a/plugin/skills/agentmemory-mcp-tools/REFERENCE.md +++ b/plugin/skills/agentmemory-mcp-tools/REFERENCE.md @@ -7,7 +7,7 @@ agentmemory exposes 55 MCP tools. 8 are in the lean core set (`--tools core` or | Tool | Core | Parameters | Purpose | | --- | --- | --- | --- | -| `memory_action_create` | | `title`*: string, `description`: string, `priority`: number, `project`: string, `tags`: string, `parentId`: string, `requires`: string | Create an actionable work item with typed dependencies. Actions track what agents need to do and how work items relate to each other. | +| `memory_action_create` | | `title`*: string, `description`: string, `priority`: number, `project`: string, `tags`: string, `parentId`: string, `requires`: string, `agentId`: string | Create an actionable work item with typed dependencies. Actions track what agents need to do and how work items relate to each other. | | `memory_action_update` | | `actionId`*: string, `status`: string, `result`: string, `priority`: number | Update an action's status, priority, or details. Set status to 'done' to complete it and unblock dependent actions. | | `memory_audit` | | `operation`: string, `limit`: number | View the audit trail of memory operations. | | `memory_checkpoint` | | `operation`*: string, `name`: string, `checkpointId`: string, `status`: string, `type`: string, `linkedActionIds`: string | Create or resolve an external checkpoint (CI result, approval, deploy status) that gates action progress. | diff --git a/plugin/skills/agentmemory-rest-api/REFERENCE.md b/plugin/skills/agentmemory-rest-api/REFERENCE.md index 10b5139a44..e66420190b 100644 --- a/plugin/skills/agentmemory-rest-api/REFERENCE.md +++ b/plugin/skills/agentmemory-rest-api/REFERENCE.md @@ -5,7 +5,7 @@ Generated from `src/triggers/api.ts`. Do not edit the block below by hand; run ` The REST API is the primary surface. All paths are under `http://localhost:3111` (override with `--port`). Send `Authorization: Bearer `: the value of `AGENTMEMORY_SECRET`, or the one generated into `~/.agentmemory/secret` on first start when it is unset. Writes must use `Content-Type: application/json`. -160 registered endpoints: +161 registered endpoints: | Method | Path | | --- | --- | @@ -24,6 +24,7 @@ The REST API is the primary surface. All paths are under `http://localhost:3111` | POST | `/agentmemory/capture/drain` | | GET | `/agentmemory/capture/index-evidence` | | GET | `/agentmemory/capture/index-membership` | +| POST | `/agentmemory/capture/inspect-uncertain` | | GET | `/agentmemory/capture/jobs` | | POST | `/agentmemory/capture/jobs` | | POST | `/agentmemory/capture/jobs/stop` | diff --git a/scripts/audit-server-sdk-bundle.mjs b/scripts/audit-server-sdk-bundle.mjs index 074ed463f3..a74a6795aa 100644 --- a/scripts/audit-server-sdk-bundle.mjs +++ b/scripts/audit-server-sdk-bundle.mjs @@ -3,7 +3,7 @@ import {join,relative,resolve} from 'node:path'; import {fileURLToPath} from 'node:url'; import {createHash} from 'node:crypto'; import ts from 'typescript'; -const versions={'iii-sdk':'0.25.0','@iii-dev/helpers':'0.25.0',yaml:'2.8.1',json5:'2.2.3'}; +const versions={'iii-sdk':'0.25.1','@iii-dev/helpers':'0.25.1',yaml:'2.8.1',json5:'2.2.3'}; export async function auditServerSdkBundle(root){ const files={},externalSpecifiers={};let totalBytes=0; for(const [name,version]of Object.entries(versions)){const installed=JSON.parse(await readFile(join(root,'node_modules',name,'package.json'),'utf8'));if(installed.version!==version)throw Error('SDK_SERVER_VERSION_MISMATCH');} diff --git a/scripts/docs-sync.state.json b/scripts/docs-sync.state.json index c8f54bd353..4aaa9fb0b7 100644 --- a/scripts/docs-sync.state.json +++ b/scripts/docs-sync.state.json @@ -1,5 +1,5 @@ { - "version": "0.9.32-daniel.55", + "version": "0.9.33-daniel.73", "facts": { "mcpTools": [ "55" @@ -11,7 +11,7 @@ "7" ], "restEndpoints": [ - "160" + "161" ], "skills": [ "17" @@ -20,20 +20,20 @@ "12" ], "tests": [ - "6300+" + "7000+" ], "functions": [ - "353" + "355" ], "sourceFiles": [ - "280" + "287" ], "loc": [ - "63000" + "65000" ], "kvScopes": [ - "70" + "71" ] }, - "engineVersion": "0.25.0" + "engineVersion": "0.25.1" } diff --git a/scripts/engine-hashes.mjs b/scripts/engine-hashes.mjs index a18ce81137..70f0682865 100644 --- a/scripts/engine-hashes.mjs +++ b/scripts/engine-hashes.mjs @@ -47,6 +47,10 @@ if (!table.test(source)) { console.error("III_RELEASE_SHA256 table not found in src/cli/engine-install.ts"); process.exit(1); } -const block = `export const III_RELEASE_SHA256: ChecksumTable = {\n "${version}": {\n${lines.join("\n")}\n },\n};\n`; +const kept = [...source.match(table)[0].matchAll(/\n "([^"]+)": \{\n[\s\S]*?\n \},/g)] + .filter((entry) => entry[1] !== version) + .map((entry) => entry[0]) + .join(""); +const block = `export const III_RELEASE_SHA256: ChecksumTable = {\n "${version}": {\n${lines.join("\n")}\n },${kept}\n};\n`; writeFileSync(installFile, source.replace(table, block)); console.log(`wrote hashes for iii v${version} to src/cli/engine-install.ts`); diff --git a/scripts/migration/native-v3-to-v4.mjs b/scripts/migration/native-v3-to-v4.mjs new file mode 100644 index 0000000000..ffbd430a50 --- /dev/null +++ b/scripts/migration/native-v3-to-v4.mjs @@ -0,0 +1,25 @@ +#!/usr/bin/env node +// Explicit two-phase, no production defaults. Store bytes are never restored or reset. +import {readFileSync} from 'node:fs'; +import {join,resolve} from 'node:path'; +import {registerWorker} from 'iii-sdk'; +import {createEngineEpochEvidence,waitForEngineEpoch} from '../../src/ops/engine-epoch.mjs'; +import {prepareNativeMigration,saveNativeMigrationProof,finalizeNativeMigration} from '../../src/ops/native-lease-migration.mjs'; +const [mode,...args]=process.argv.slice(2),values={}; +for(let i=0;i{const s=sdk.ws?._socket;return s?{remoteAddress:s.remoteAddress,remotePort:s.remotePort,localAddress:s.localAddress,localPort:s.localPort}:null;}}); + await waitForEngineEpoch(evidence); + const proof=prepareNativeMigration({storePath,runtimeDir,expectedExecutable:resolve(values['native-executable']),engineEvidence:evidence}); + saveNativeMigrationProof(proof,proofPath); + console.log(JSON.stringify({prepared:true,sourceSha256:proof.sourceSha256,requiresOfflineFinalization:true})); + }finally{await sdk.shutdown();} +}else if(mode==='finalize'){ + const result=finalizeNativeMigration({storePath,proof:JSON.parse(readFileSync(proofPath,'utf8'))}); + console.log(JSON.stringify({...result,finalized:true,markerRewritten:false,storeRestored:false})); +}else throw Error('Usage: prepare --store PATH --proof NEW_FILE --runtime PATH --native-executable FILE --engine-url URL; finalize --store PATH --proof FILE'); diff --git a/scripts/personal-release/kit-runtime.py b/scripts/personal-release/kit-runtime.py index f288f8a5a1..ed975dbd9b 100644 --- a/scripts/personal-release/kit-runtime.py +++ b/scripts/personal-release/kit-runtime.py @@ -353,6 +353,8 @@ def dropin_bytes(self): native = ' --native-compat' if self.sdk_native_compat else '' native_stop = native + ' --data-dir ' + str(self.paths['data']) if self.sdk_native_compat else '' native_env = ('Environment="AGENTMEMORY_NATIVE_III_BIN=' + str(self.paths['engine']) + '"\nEnvironment="AGENTMEMORY_III_CONFIG=' + str(self.paths['data'] / 'iii-config.yaml') + '"\n') if self.sdk_native_compat else '' + if self.sdk_native_compat: + native_env += 'RestartPreventExitStatus=78\n' provider_env = 'Environment=\"AGENTMEMORY_LLM_PROVIDER=openai\"\n' if self.service_provider_selection else '' return ('[Service]\nExecStart=\nExecStart=/usr/bin/node ' + str(self.version / 'dist/cli.mjs') + ' --data-dir ' + str(self.paths['data']) + native + '\nExecStop=\nExecStop=/usr/bin/node ' + str(self.version / 'dist/cli.mjs') + ' stop' + native_stop + '\nExecStartPre=\nExecStartPre=/usr/bin/python3 -I -B ' + str(self.public / 'kit-runtime.py') + ' --guard\nEnvironment="AGENTMEMORY_FEEDBACK_DIR=' + str(self.paths['feedback']) + '"\nEnvironment="AGENTMEMORY_RECEIPT_DIR=' + str(self.receipt.parent) + '"\n' + native_env + provider_env).encode() diff --git a/scripts/personal-release/test_dashboard_kit.py b/scripts/personal-release/test_dashboard_kit.py index e0d23e22ff..048d04718c 100644 --- a/scripts/personal-release/test_dashboard_kit.py +++ b/scripts/personal-release/test_dashboard_kit.py @@ -1,3 +1,4 @@ +import hashlib import importlib.util import json import os @@ -144,6 +145,14 @@ def api(self, path, body=None): return {'control': {'pausedRequested': True, 'status': 'paused' if self.system.started else 'pausing', 'activeOperations': 0 if self.system.started else 3, 'error': None}} return {'sessions': [], 'flags': [], 'scheduleEnabled': False} + def test_only_native_units_suppress_restarts_for_unresolved_lifecycle(self): + self.assertNotIn(b'RestartPreventExitStatus=', self.tx.dropin_bytes()) + self.tx.sdk_native_compat = True + dropin = self.tx.dropin_bytes() + self.assertEqual([line for line in dropin.splitlines() if line.startswith(b'RestartPreventExitStatus=')], + [b'RestartPreventExitStatus=78']) + self.assertEqual(self.tx.post_files()[str(self.paths['dropin'])], hashlib.sha256(dropin).hexdigest()) + def test_reviewed_snapshot_enrolls_without_apply_and_entire_transaction_preserves_data(self): self.assertEqual(self.calls[0][4], '--enroll') self.assertFalse(any('systemctl' in item or item == '--apply' for item in self.calls[0])) diff --git a/scripts/plugins/package-codex.mjs b/scripts/plugins/package-codex.mjs index 5c327dfdc5..da86af56f7 100644 --- a/scripts/plugins/package-codex.mjs +++ b/scripts/plugins/package-codex.mjs @@ -63,7 +63,7 @@ for (const [flavor, { mcp, hooks }] of Object.entries(FLAVORS)) { cpSync(join(plugin, ".mcp.codex.json"), join(destination, ".mcp.json")); cpSync(join(plugin, "scripts/plugin-bridge.mjs"), join(destination, "scripts/plugin-bridge.mjs")); cpSync(join(root, "docs/plugins/codex-local.md"), join(destination, "README.md")); - if (hooks === "sdk-v2") writeFileSync(join(destination, "README.md"), `# Agent Memory native SDK capture v2 (prepared policy)\n\nThis is the full six-hook local package. Global hooks are not installed or changed. SessionStart, PreToolUse and PreCompact preserve their context/session behavior. Only UserPromptSubmit, PostToolUse and Stop receive the explicit --agentmemory-sdk-capture-v2 flag; Stop retains its nonfinal session end. The selected MCP contract is unchanged.\n\nUbuntu/Linux requires the already installed /usr/bin/flock; no fallback or new privileged helper is installed. The approved default storage is HOME/.local/share/agentmemory/capture-spool-sdk-v2. A development-only isolated-directory override is available as AGENTMEMORY_SDK_CAPTURE_V2_DIR and must be reviewed before host activation.\n\nThe privacy-scrubbed canonical envelope limit is 8 MiB, chunk limit 256 KiB, originals budget 256 MiB and reference queue budget 5 MiB. Directory/file allocation and partial/conflict data count conservatively; these are not a filesystem hard quota. There is no automatic deletion or expiry of originals. Old spools and unknown history are not read, moved, replayed or deleted. Full quota or preservation failure returns an explicit failure; this cannot promise acceptance of unlimited future input.\n\nComplete source publication and hash readback precede its reference and every POST to the same endpoint; C v2 POSTs refuse redirects. Only a matching SDK completion ACK is recorded; it is not a disk-flush receipt. Known-not-sent or explicit admission rejection may retry via the reference. Uncertain sending is held, never silently marked complete. Ordinary later prompt/tool invocations may opportunistically drain at most one eligible reference with a monotonic POST-admission/network-wait budget capped at 250ms and the remaining invocation budget. Source readback and JSON serialization consume that budget; an expired budget starts no POST and keeps a known-not-sent reference for backoff. Required synchronous state fsync and the final consistent report may finish after the deadline, so this is not a hard wall-clock guarantee for disk I/O. There is no perpetual scheduler or host retry guarantee. Stop does not perform this extra drain.\n\nLarge/image normal observe retransmission can use the verified original. The existing guarded restore API refuses images and payloads over256 KiB; this package does not bypass that restore guard. Other manual restore requires separate review.\n\nPackaging does not install, enable or trust this policy. Review exact six hook definitions, the new storage policy and the actual endpoint/environment before activation.\n`); + if (hooks === "sdk-v2") writeFileSync(join(destination, "README.md"), `# Agent Memory native SDK capture v2 (prepared policy)\n\nThis is the full six-hook local package. Global hooks are not installed or changed. SessionStart, PreToolUse and PreCompact preserve their context/session behavior. Only UserPromptSubmit, PostToolUse and Stop receive the explicit --agentmemory-sdk-capture-v2 flag; Stop retains its nonfinal session end. The selected MCP contract is unchanged.\n\nUbuntu/Linux requires the already installed /usr/bin/flock; no fallback or new privileged helper is installed. The approved default storage is HOME/.local/share/agentmemory/capture-spool-sdk-v2. A development-only isolated-directory override is available as AGENTMEMORY_SDK_CAPTURE_V2_DIR and must be reviewed before host activation.\n\nThe privacy-scrubbed canonical envelope limit is 8 MiB, chunk limit 256 KiB, originals admission budget 256 MiB and reference queue admission budget 5 MiB. Existing reference transitions have an additional bounded 8 KiB workspace; completed-manifest replacement has 68 KiB. New source/reference admission cannot consume this workspace, and preserved crash temporary files still count. Directory/file allocation and partial/conflict data count conservatively; these are not a filesystem hard quota. There is no automatic deletion or expiry of originals. Old spools and unknown history are not read, moved, replayed or deleted. Full quota or preservation failure returns an explicit failure; this cannot promise acceptance of unlimited future input.\n\nComplete source publication and hash readback precede its reference and every POST to the same endpoint; C v2 POSTs refuse redirects. Only a matching SDK completion ACK is recorded; it is not a disk-flush receipt. Known-not-sent or explicit admission rejection may retry via the reference. Uncertain sending is held, never silently marked complete. Ordinary later prompt/tool invocations may opportunistically drain at most one eligible reference with a monotonic POST-admission/network-wait budget capped at 250ms and the remaining invocation budget. Source readback and JSON serialization consume that budget; an expired budget starts no POST and keeps a known-not-sent reference for backoff. Required synchronous state fsync and the final consistent report may finish after the deadline, so this is not a hard wall-clock guarantee for disk I/O. There is no perpetual scheduler or host retry guarantee. Stop does not perform this extra drain.\n\nLarge/image normal observe retransmission can use the verified original. The existing guarded restore API refuses images and payloads over256 KiB; this package does not bypass that restore guard. Other manual restore requires separate review.\n\nPackaging does not install, enable or trust this policy. Review exact six hook definitions, the new storage policy and the actual endpoint/environment before activation.\n`); if (hooks === "pre-tool") writeFileSync(join(destination, "README.md"), `# Agent Memory companion for existing global hooks\n\nThis package preserves MCP, skills and the existing PreToolUse matcher and command. It excludes SessionStart, UserPromptSubmit, PostToolUse, PreCompact and Stop to avoid a second registration alongside separately configured global hooks.\n\nThe existing personal global compatibility artifact has 9 event registrations and 13 files including shared helpers and metadata. Those are different units. This package does not install or trust global hooks, change their event IDs or retained outbox, or verify which host dispatched an event. Select this package only after reviewing the host's active hooks. Existing PreToolUse enrichment data and destination rules remain unchanged.\n`); } else { delete manifest.mcpServers; diff --git a/scripts/release-gate/graph-index.mjs b/scripts/release-gate/graph-index.mjs index 33b579f43a..47793288b5 100644 --- a/scripts/release-gate/graph-index.mjs +++ b/scripts/release-gate/graph-index.mjs @@ -124,7 +124,7 @@ async function profile(name, instance = 0, sharedHome) { "containers:", " state:", " worker: 'package://api.workers.iii.dev/state'", - " version: '0.22.20'", + " version: '0.22.22'", " config_name: state", " config_override:", " adapter:", @@ -135,7 +135,7 @@ async function profile(name, instance = 0, sharedHome) { " save_interval_ms: 2000", " http:", " worker: 'package://api.workers.iii.dev/http'", - " version: '0.22.7'", + " version: '0.22.8'", " config_name: http", " config_override:", " host: 127.0.0.1", @@ -186,7 +186,7 @@ async function relay(p) { let summary = p.traceSummary.get(p.phase); if (!summary) p.traceSummary.set(p.phase, summary = { graphOperations: 0, maxGraphResponseBytes: 0, maxReadLatencyMs: 0, canonicalLists: [], snapshotOperations: 0 }); summary.graphOperations++; - if (record.operation === "state::list" && canonicalScopes.some((scope) => record.scope.startsWith(scope))) summary.canonicalLists.push(record.scope); + if ((record.operation === "state::list" || record.operation === "state::list_keys") && canonicalScopes.some((scope) => record.scope.startsWith(scope))) summary.canonicalLists.push(record.scope); if (record.scope.startsWith(scopes.snapshot)) summary.snapshotOperations++; inFlight.set(message.invocation_id, { record, summary, started: performance.now() }); // A durable sealed descriptor exists before the first manifest read. Hold only this request to make the crash deterministic. @@ -452,7 +452,19 @@ try { assert(remembered.memory?.id); assert.equal((await rest(primary, "/mesh/receive", { graphNodes: [node("node-0", "OnyxRestored", { updatedAt: "2026-10-17T00:00:00.000Z", sourceObservationIds: [remembered.memory.id] })] })).success, true); const graphRecall = () => rest(primary, "/search", { query: "OnyxRestored", project: "fixture-primary", agentId: "fixture-agent", targetLayer: "memory", limit: 5, format: "full" }); - assert((await graphRecall()).results.some((row) => row.observation?.id === remembered.memory.id), "Graph-only query must find captured memory through current provenance"); + const graphRecalls = async (message) => { + let lastError; + for (let attempt = 0; attempt < 10; attempt++) { + if (attempt > 0) await new Promise((done) => setTimeout(done, 1000)); + try { + if ((await graphRecall()).results.some((row) => row.observation?.id === remembered.memory.id)) return; + } catch (error) { + lastError = error; + } + } + assert.fail(lastError ? `${message}; last request error: ${lastError.message}` : message); + }; + await graphRecalls("Graph-only query must find captured memory through current provenance"); assert.equal((await rest(primary, "/search", { query: "OnyxRestored", project: "different-project", targetLayer: "memory", limit: 5 })).results.length, 0); assert((await rest(primary, "/search", { query: "Indigo durable capture", project: "fixture-primary", agentId: "fixture-agent", limit: 5 })).results.length > 0); note("canonical-writer-reconciliation", { importRename: true, meshRename: true, staleExcluded: true, snapshotOverlay: true, capturedMemory: true, graphOnlyMemoryRecall: true }); @@ -467,7 +479,7 @@ try { await measuredReads(primary, "cold-engine-query", [["graph-query", async () => assert((await find(primary, "OnyxRestored")).some((row) => row.id === "node-0"))]]); await measuredReads(primary, "repeated-engine-query", [["graph-query", async () => assert((await find(primary, "OnyxRestored")).some((row) => row.id === "node-0"))]]); primary.phase = "completed-restart-recall"; - assert((await graphRecall()).results.some((row) => row.observation?.id === remembered.memory.id), "Graph-only memory recall must survive a restart"); + await graphRecalls("Graph-only memory recall must survive a restart"); assert((await rest(primary, "/search", { query: "Indigo durable capture", project: "fixture-primary", agentId: "fixture-agent", limit: 5 })).results.length > 0); assert.equal((await rest(primary, "/search", { query: "Indigo durable capture", project: "different-project", limit: 5 })).results.length, 0); note("completed-restart-recall", ordinaryReads(primary, primary.phase)); @@ -493,7 +505,7 @@ try { assert.notEqual(rebuilt.buildId, indexed.buildId); assert.equal(rebuilt.nodes, nodeCount); await assertEdgeKey(primary, "node-0", "node-2", "uses", ["edge-0"]); - assert((await graphRecall()).results.some((row) => row.observation?.id === remembered.memory.id)); + await graphRecalls("Graph-only memory recall must survive an index rebuild"); note("explicit-index-rebuild", { canonicalMemoryPreserved: true, exactEdgeKeyRestored: true }); const secondary = await profile("secondary", 1, primary.home); secondary.phase = "named-instance-startup-status-search"; diff --git a/scripts/release-gate/large-store.mjs b/scripts/release-gate/large-store.mjs new file mode 100644 index 0000000000..eaab957052 --- /dev/null +++ b/scripts/release-gate/large-store.mjs @@ -0,0 +1,486 @@ +#!/usr/bin/env node +import assert from "node:assert/strict"; +import { spawn, execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { appendFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, symlinkSync, writeFileSync } from "node:fs"; +import net from "node:net"; +import { arch, platform, tmpdir } from "node:os"; +import { delimiter, dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { registerWorker } from "iii-sdk"; +import { assertSupportedPlatform } from "./command.mjs"; + +assertSupportedPlatform(); +const repo = resolve(dirname(fileURLToPath(import.meta.url)), "../.."); +const options = {}; +const booleanFlags = new Set(["--small", "--assert", "--help"]); +const valueFlags = new Set(["--tarball", "--engine", "--out", "--dependencies", "--sizes", "--sessions", "--state-version", "--http-version", "--migrate-timeout-ms", "--probe-timeout-ms"]); +for (let i = 2; i < process.argv.length; i++) { + const key = process.argv[i]; + if (booleanFlags.has(key)) { options[key.slice(2)] = true; continue; } + assert(valueFlags.has(key), `Unknown argument ${key}`); + const value = process.argv[++i]; + assert(value && !value.startsWith("--"), `Missing value for ${key}`); + options[key.slice(2).replaceAll("-", "_")] = value; +} +if (options.help) { + console.log("Usage: node scripts/release-gate/large-store.mjs --tarball file.tgz [--engine bin] [--out dir] [--dependencies dir] [--small] [--sizes semantic=30,nodes=70,edges=140,insights=500] [--sessions N] [--state-version v] [--http-version v] [--migrate-timeout-ms N] [--probe-timeout-ms N] [--assert]"); + process.exit(0); +} +assert(options.tarball, "Pass --tarball with the built npm package to measure"); +options.tarball = resolve(options.tarball); +if (options.out) options.out = resolve(options.out); +if (options.dependencies) options.dependencies = resolve(options.dependencies); +if (options.engine) options.engine = resolve(options.engine); + +const scopeOrder = ["semantic", "nodes", "edges", "insights"]; +const scopeNames = { + semantic: "mem:semantic", + nodes: "mem:graph:nodes", + edges: "mem:graph:edges", + insights: "mem:insights", + sessions: "mem:sessions", +}; +const defaults = options.small + ? { semantic: 18, nodes: 18, edges: 20, insights: 20, sessions: 300 } + : { semantic: 30, nodes: 70, edges: 140, insights: 500, sessions: 3000 }; +const sizesMb = { semantic: defaults.semantic, nodes: defaults.nodes, edges: defaults.edges, insights: defaults.insights }; +if (options.sizes) { + for (const pair of options.sizes.split(",")) { + const [name, raw] = pair.split("=").map((part) => part.trim()); + assert(scopeOrder.includes(name), `--sizes key must be one of ${scopeOrder.join(",")}: ${name}`); + const mb = Number(raw); + assert(Number.isFinite(mb) && mb >= 0, `--sizes ${name} must be a non-negative number of MiB`); + sizesMb[name] = mb; + } +} +const sessionCount = options.sessions !== undefined ? Number(options.sessions) : defaults.sessions; +assert(Number.isSafeInteger(sessionCount) && sessionCount >= 0, "--sessions must be a non-negative integer"); +const migrateTimeoutMs = options.migrate_timeout_ms !== undefined ? Number(options.migrate_timeout_ms) : 300_000; +const probeTimeoutMs = options.probe_timeout_ms !== undefined ? Number(options.probe_timeout_ms) : 300_000; +assert(Number.isFinite(migrateTimeoutMs) && migrateTimeoutMs > 0, "--migrate-timeout-ms must be positive"); +assert(Number.isFinite(probeTimeoutMs) && probeTimeoutMs > 0, "--probe-timeout-ms must be positive"); + +const composeSource = readFileSync(join(repo, "src/cli/compose-project.ts"), "utf8"); +const workerVersions = { + state: options.state_version ?? composeSource.match(/state:\s*"([^"]+)"/)?.[1], + http: options.http_version ?? composeSource.match(/http:\s*"([^"]+)"/)?.[1], +}; +assert(workerVersions.state && workerVersions.http, "Could not resolve worker versions; pass --state-version and --http-version"); + +const out = options.out ?? mkdtempSync(join(tmpdir(), "agentmemory-scale-")); +mkdirSync(out, { recursive: true }); +const root = mkdtempSync(join(out, "run-")); +console.log(`Artifacts: ${root}`); +const prefix = join(root, "prefix"), modules = join(prefix, "node_modules"), pkg = join(modules, "@agentmemory/agentmemory"); +mkdirSync(pkg, { recursive: true }); +const children = new Set(); +const probes = []; +const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); +const free = (port) => new Promise((resolve) => { + const server = net.createServer(); server.once("error", () => resolve(false)); + server.listen(port, "127.0.0.1", () => server.close(() => resolve(true))); +}); +async function poll(label, check, timeout = 30_000) { + const end = Date.now() + timeout; + let error; + do { + try { const value = await check(); if (value) return value; } catch (cause) { error = cause; } + await sleep(250); + } while (Date.now() < end); + throw new Error(`${label} timed out: ${error?.message ?? "not ready"}`); +} +function kill(child, signal = "SIGKILL") { + if (!child || child.exitCode !== null || child.signalCode !== null) return; + try { process.kill(-child.pid, signal); } catch { try { child.kill(signal); } catch {} } +} +process.on("exit", () => { for (const child of children) kill(child); }); +for (const signal of ["SIGINT", "SIGTERM", "SIGHUP"]) { + process.on(signal, () => { for (const child of children) kill(child); process.exit(130); }); +} + +let manifest, engine, cli, fatal; +const p = { name: "large-store", sockets: new Set() }; +function environment() { + return { + HOME: p.home, USERPROFILE: p.home, PATH: [dirname(process.execPath), "/usr/bin", "/bin", "/usr/sbin", "/sbin"].join(delimiter), TMPDIR: root, + CI: "1", NO_COLOR: "1", LANG: "C.UTF-8", OTEL_ENABLED: "false", III_TELEMETRY_ENABLED: "false", + AGENTMEMORY_DATA_DIR: p.data, AGENTMEMORY_RUNTIME_DIR: p.runtime, + III_REST_PORT: String(p.port), III_VIEWER_PORT: String(p.port + 2), + III_ENGINE_PORT: String(p.enginePort), III_ENGINE_URL: `ws://127.0.0.1:${p.enginePort}`, III_NAMESPACE: `agentmemory-${p.port}`, AGENTMEMORY_METRICS_PORT: String(p.port + 6353), + LLM_PROVIDER: "none", EMBEDDING_PROVIDER: "none", AGENTMEMORY_GRAPH_WEIGHT: "0.3", + AGENTMEMORY_GRAPH_COMPACT_ON_BOOT: "false", AGENTMEMORY_AUTO_COMPRESS: "false", AGENTMEMORY_SESSION_SWEEP_ENABLED: "false", AGENTMEMORY_FINALIZE_IDLE_MS: "0", + GRAPH_EXTRACTION_ENABLED: "false", CONSOLIDATION_ENABLED: "false", AGENTMEMORY_REFLECT: "false", + }; +} +function launch(executable, args, name) { + const child = spawn(executable, args, { env: environment(), cwd: p.dir, detached: true, stdio: ["ignore", "pipe", "pipe"] }); + children.add(child); child.text = ""; + const log = (chunk) => { child.text = (child.text + chunk).slice(-1_048_576); appendFileSync(join(p.dir, `${name}.log`), chunk); }; + child.stdout.on("data", log); child.stderr.on("data", log); child.on("error", (error) => log(String(error))); + child.done = new Promise((resolve) => child.on("close", (code, signal) => { children.delete(child); resolve({ code, signal }); })); + return child; +} +async function install() { + if (options.dependencies) { + execFileSync("tar", ["-xzf", options.tarball, "--strip-components=1", "-C", pkg]); + for (const entry of readdirSync(options.dependencies, { withFileTypes: true })) { + if (entry.name.startsWith(".")) continue; + if (entry.name.startsWith("@")) { + mkdirSync(join(modules, entry.name), { recursive: true }); + for (const name of readdirSync(join(options.dependencies, entry.name))) { + const target = join(modules, entry.name, name); + if (!existsSync(target)) symlinkSync(join(options.dependencies, entry.name, name), target); + } + } else symlinkSync(join(options.dependencies, entry.name), join(modules, entry.name)); + } + } else { + writeFileSync(join(prefix, "package.json"), '{"name":"large-store-gate","private":true}\n'); + const installHome = join(root, "install-home"); mkdirSync(installHome); + execFileSync("npm", ["install", "--ignore-scripts", "--omit=optional", "--no-audit", "--no-fund", options.tarball], { + cwd: prefix, stdio: "inherit", timeout: 300_000, + env: { HOME: installHome, USERPROFILE: installHome, PATH: [dirname(process.execPath), "/usr/bin", "/bin"].join(delimiter), npm_config_cache: join(root, "npm-cache") }, + }); + } + manifest = JSON.parse(readFileSync(join(pkg, "package.json"), "utf8")); + cli = join(pkg, "dist/cli.mjs"); +} +async function ensureEngine() { + const pinned = manifest.dependencies["iii-sdk"]; + const candidate = options.engine ?? process.env.AGENTMEMORY_GATE_ENGINE; + if (candidate) { + const version = execFileSync(candidate, ["--version"], { encoding: "utf8" }); + assert(version.includes(pinned), `Engine version mismatch: ${version} (expected ${pinned})`); + return candidate; + } + const targets = { + darwin: { arm64: "aarch64-apple-darwin", x64: "x86_64-apple-darwin" }, + linux: { x64: "x86_64-unknown-linux-gnu", arm64: "aarch64-unknown-linux-gnu", arm: "armv7-unknown-linux-gnueabihf" }, + }; + const target = targets[platform()]?.[arch()]; + assert(target, `No iii engine asset mapping for ${platform()}-${arch()}`); + const asset = `iii-${target}.tar.gz`; + const base = `https://github.com/iii-hq/iii/releases/download/iii/v${pinned}`; + const dir = join(root, "engine"); mkdirSync(dir, { recursive: true }); + const archive = join(dir, asset); + const checksumResponse = await fetch(`${base}/${asset.replace(/\.tar\.gz$/, "")}.sha256`); + assert(checksumResponse.ok, `Engine checksum download: HTTP ${checksumResponse.status}`); + const expected = (await checksumResponse.text()).trim().split(/\s+/)[0]; + const response = await fetch(`${base}/${asset}`); + assert(response.ok, `Engine download: HTTP ${response.status}`); + writeFileSync(archive, Buffer.from(await response.arrayBuffer())); + const actual = createHash("sha256").update(readFileSync(archive)).digest("hex"); + assert.equal(actual, expected, `Engine archive checksum mismatch for ${asset}`); + execFileSync("tar", ["-xzf", archive, "-C", dir]); + const bin = join(dir, "iii"); + assert(existsSync(bin), `Expected engine binary at ${bin}`); + const version = execFileSync(bin, ["--version"], { encoding: "utf8" }); + assert(version.includes(pinned), `Engine version mismatch: ${version} (expected ${pinned})`); + return bin; +} +function writeWorkerCompose() { + const quote = (value) => `'${String(value).replaceAll("'", "''")}'`; + const composeText = [ + `namespace: agentmemory-${p.port}`, + "engine:", + ` url: ws://127.0.0.1:${p.enginePort}`, + " workers:", + " configuration:", + " adapter:", + " name: fs", + " config:", + ` directory: ${quote(join(p.data, "compose-config"))}`, + " iii-worker-manager:", + " host: 127.0.0.1", + ` port: ${p.enginePort}`, + "containers:", + " state:", + " worker: 'package://api.workers.iii.dev/state'", + ` version: '${workerVersions.state}'`, + " config_name: state", + " config_override:", + " adapter:", + " name: kv", + " config:", + ` file_path: ${quote(join(p.data, "state_store.db"))}`, + " store_method: file_based", + " save_interval_ms: 2000", + " http:", + " worker: 'package://api.workers.iii.dev/http'", + ` version: '${workerVersions.http}'`, + " config_name: http", + " config_override:", + " host: 127.0.0.1", + ` port: ${p.port}`, + " default_timeout: 180000", + "", + ].join("\n"); + writeFileSync(join(p.dir, "worker-compose.yaml"), composeText); +} +async function startEngine() { + p.engine = launch(engine, ["compose", "--namespace", `agentmemory-${p.port}`, "--up", "--file", join(p.dir, "worker-compose.yaml")], "compose"); + await poll("Compose engine", async () => (await fetch(`http://127.0.0.1:${p.port}/`, { signal: AbortSignal.timeout(500) })).status > 0, 120_000); + p.client = registerWorker(`ws://127.0.0.1:${p.enginePort}`, { workerName: "large-store-fixture", namespace: `agentmemory-${p.port}`, invocationTimeoutMs: 60_000, enableMetricsReporting: false, otel: { enabled: false } }); + await poll("Compose state worker", async () => { + await p.client.trigger({ function_id: "state::get", payload: { scope: scopeNames.sessions, key: "_probe" } }); + return true; + }, 120_000); +} +async function startWorker() { + p.worker = launch(process.execPath, [cli, "--no-engine", "--port", String(p.port), "--data-dir", p.data], "worker"); + await poll("packaged CLI", async () => { + if (p.worker.exitCode !== null) throw new Error(`CLI exited: ${p.worker.text}`); + if (!p.worker.text.includes("REST API")) return false; + const res = await fetch(`http://127.0.0.1:${p.port}/agentmemory/livez`, { signal: AbortSignal.timeout(2000) }); + if (!res.ok) return false; + const value = await res.json(); + return value.viewerPort || value.viewerSkipped; + }, 300_000); + p.secret = readFileSync(join(p.runtime, "secret"), "utf8").trim(); +} +const state = (id, payload) => p.client.trigger({ function_id: id, payload }); + +const timestamp = "2026-10-14T00:00:00.000Z"; +const RECORD_TARGET_BYTES = 1024 * 1024; +const pad = (record, size, assign) => { + const skeleton = JSON.stringify(assign(record, "")).length; + return assign(record, "x".repeat(Math.max(1, RECORD_TARGET_BYTES - skeleton))); +}; +const semanticRecord = (i) => pad({ id: `sem-${i}`, fact: "", confidence: 0.9, sourceSessionIds: ["sess-seed"], sourceMemoryIds: [], accessCount: 0, lastAccessedAt: timestamp, strength: 5, createdAt: timestamp, updatedAt: timestamp }, RECORD_TARGET_BYTES, (r, fill) => (r.fact = `Seeded semantic fact ${i}. ${fill}`, r)); +const nodeRecord = (i) => pad({ id: `node-${i}`, name: `SeedNode${i}`, type: "concept", properties: {}, sourceObservationIds: ["mem_seed_source"], createdAt: timestamp }, RECORD_TARGET_BYTES, (r, fill) => (r.properties = { payload: fill }, r)); +const edgeRecord = (i, nodeTotal) => pad({ id: `edge-${i}`, type: "related_to", sourceNodeId: `node-${i % Math.max(1, nodeTotal)}`, targetNodeId: `node-${(i + 1) % Math.max(1, nodeTotal)}`, weight: 0.8, sourceObservationIds: ["mem_seed_source"], createdAt: timestamp, isLatest: true }, RECORD_TARGET_BYTES, (r, fill) => (r.context = { reasoning: fill }, r)); +const insightRecord = (i) => pad({ id: `insight-${i}`, title: `Seeded insight ${i}`, content: "", confidence: 0.85, reinforcements: 1, sourceConceptCluster: ["seed"], sourceMemoryIds: [], sourceLessonIds: [], sourceCrystalIds: [], evidenceVersion: 1, project: "fixture-large", tags: ["seed"], createdAt: timestamp, updatedAt: timestamp, decayRate: 0 }, RECORD_TARGET_BYTES, (r, fill) => (r.content = `Seeded insight content ${i}. ${fill}`, r)); +const sessionRecord = (i) => ({ id: `sess-${i}`, project: "fixture-large", cwd: "/fixture/large", startedAt: timestamp, updatedAt: timestamp, endedAt: timestamp, status: "completed", observationCount: 0, agentId: "fixture-agent", firstPrompt: `Seeded session ${i}` }); + +async function seedScope(scope, recordCount, make) { + let seeded = 0; + const concurrency = 8; + while (seeded < recordCount) { + const batch = Math.min(concurrency, recordCount - seeded); + await Promise.all(Array.from({ length: batch }, (_, offset) => { + const record = make(seeded + offset); + return state("state::set", { scope, key: record.id, value: record }); + })); + seeded += batch; + if (seeded % 64 < concurrency) console.log(`seeded ${scope}: ${seeded}/${recordCount}`); + } + return seeded; +} +async function seed() { + const seeded = {}; + const perScope = { + semantic: { count: Math.round(sizesMb.semantic), make: semanticRecord }, + nodes: { count: Math.round(sizesMb.nodes), make: nodeRecord }, + edges: { count: Math.round(sizesMb.edges), make: (i) => edgeRecord(i, Math.max(1, Math.round(sizesMb.nodes))) }, + insights: { count: Math.round(sizesMb.insights), make: insightRecord }, + }; + for (const name of scopeOrder) { + const { count, make } = perScope[name]; + const start = performance.now(); + seeded[name] = { records: await seedScope(scopeNames[name], count, make), targetMiB: sizesMb[name], seedMs: Math.round(performance.now() - start) }; + } + const start = performance.now(); + seeded.sessions = { records: await seedScope(scopeNames.sessions, sessionCount, sessionRecord), seedMs: Math.round(performance.now() - start) }; + return seeded; +} +async function response(path, body) { + return fetch(`http://127.0.0.1:${p.port}/agentmemory${path}`, { + method: body === undefined ? "GET" : "POST", + headers: { "content-type": "application/json", authorization: `Bearer ${p.secret}` }, + ...(body === undefined ? {} : { body: JSON.stringify(body) }), + signal: AbortSignal.timeout(probeTimeoutMs), + }); +} +async function probe(name, path, body, { allowRefusal = false } = {}) { + const started = performance.now(); + const result = { name, method: body === undefined ? "GET" : "POST", path }; + try { + const res = await response(path, body); + result.status = res.status; + let bytes = 0; + const captured = []; + const reader = res.body?.getReader(); + if (reader) { + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + bytes += value.byteLength; + if (captured.reduce((sum, chunk) => sum + chunk.byteLength, 0) < 4096) captured.push(value); + } + } + result.bytes = bytes; + const text = Buffer.concat(captured.map((chunk) => Buffer.from(chunk))).toString("utf8").slice(0, 4096); + try { + const json = JSON.parse(text); + result.json = json; + if (json && typeof json === "object" && json.oversized === true) result.refused = true; + const detail = json && typeof json === "object" ? json.error ?? json.message : undefined; + if (detail) result.error = String(typeof detail === "string" ? detail : JSON.stringify(detail)).slice(0, 500); + else if (json && typeof json === "object" && json.success === false) result.error = "success:false"; + } catch {} + if (!result.error && res.status >= 400) result.error = `HTTP ${res.status}`; + } catch (cause) { + result.error = String(cause?.message ?? cause).slice(0, 500); + } + result.latencyMs = Math.round(performance.now() - started); + result.ok = (result.status >= 200 && result.status < 300 && !result.error) || (allowRefusal && result.status === 413 && result.refused === true); + delete result.json; + probes.push(result); + console.log(JSON.stringify(result)); + return result; +} +async function agentmemoryWorkers() { + try { + const listed = await state("engine::workers::list", {}); + return (listed?.workers ?? []).filter((worker) => (worker.name ?? "").includes("agentmemory")).map((worker) => ({ id: worker.id, name: worker.name, status: worker.status })); + } catch (cause) { + return [{ error: String(cause?.message ?? cause) }]; + } +} +function scanLogs() { + const findings = { payload_too_large: 0, WS_ERR_UNSUPPORTED_MESSAGE_LENGTH: 0, files: [] }; + const candidates = [join(p.dir, "compose.log"), join(p.dir, "worker.log")]; + const walk = (dir) => { + for (const entry of readdirSync(dir, { withFileTypes: true })) { + const full = join(dir, entry.name); + if (entry.isDirectory()) walk(full); + else if (entry.isFile() && entry.name.endsWith(".log")) candidates.push(full); + } + }; + try { walk(join(p.dir, ".iii")); } catch {} + try { walk(join(p.data, "compose-state")); } catch {} + try { walk(join(p.data, "compose", "logs")); } catch {} + for (const file of new Set(candidates)) { + let text; + try { text = readFileSync(file, "utf8"); } catch { continue; } + const hits = { + payload_too_large: (text.match(/payload_too_large/g) ?? []).length, + WS_ERR_UNSUPPORTED_MESSAGE_LENGTH: (text.match(/WS_ERR_UNSUPPORTED_MESSAGE_LENGTH/g) ?? []).length, + }; + if (hits.payload_too_large || hits.WS_ERR_UNSUPPORTED_MESSAGE_LENGTH) findings.files.push({ file, ...hits }); + findings.payload_too_large += hits.payload_too_large; + findings.WS_ERR_UNSUPPORTED_MESSAGE_LENGTH += hits.WS_ERR_UNSUPPORTED_MESSAGE_LENGTH; + } + return findings; +} + +let seeded = {}, workersBefore = [], workersAfter = [], workerObserved = false, workerReregistered = false, logFindings = {}, migrateFinal; +try { + await install(); + engine = await ensureEngine(); + const dir = join(root, "instance"), home = join(dir, "home"), data = join(dir, "data"), runtime = join(home, ".agentmemory"); + Object.assign(p, { dir, home, data, runtime }); + for (const path of [runtime, data]) mkdirSync(path, { recursive: true }); + for (let i = 0; i < 100 && p.port === undefined; i++) { + const candidate = 12_000 + Math.floor(Math.random() * 7_000); + const block = [0, 1, 2, 3, 6353, 46023].map((n) => candidate + n); + if ((await Promise.all(block.map(free))).every(Boolean)) p.port = candidate; + } + assert(p.port, "No free port block"); + p.enginePort = p.port + 46023; + writeWorkerCompose(); + await startEngine(); + const seedStart = performance.now(); + seeded = await seed(); + console.log(JSON.stringify({ name: "seeded", scopes: seeded, totalMs: Math.round(performance.now() - seedStart) })); + await startWorker(); + workersBefore = await poll("agentmemory worker registration", async () => { + const listed = await agentmemoryWorkers(); + return listed.some((worker) => worker.id) ? listed : false; + }, 60_000); + + await probe("semantic", "/semantic?limit=50"); + await probe("insights", "/insights?limit=50"); + await probe("sessions", "/sessions"); + await probe("counts", "/counts"); + + const migrate = await probe("graph-index-migrate", "/graph/index/migrate", { allowLegacyScan: true }); + const migrateDeadline = Date.now() + migrateTimeoutMs; + let terminal; + if (migrate.ok) { + do { + await sleep(3000); + const res = await response("/graph/index"); + terminal = res.ok ? await res.json() : { state: `http-${res.status}` }; + } while (!terminal.ready && !terminal.error && !["paused", "unavailable"].includes(terminal.state) && Date.now() < migrateDeadline); + } + migrateFinal = { + name: "graph-index-final", + accepted: migrate.ok, + state: terminal?.state ?? "not-started", + ready: Boolean(terminal?.ready), + processed: terminal?.processed, + total: terminal?.total, + error: terminal?.error ? String(terminal.error).slice(0, 500) : terminal === undefined ? migrate.error : undefined, + elapsedMs: migrate.latencyMs, + }; + probes.push(migrateFinal); + console.log(JSON.stringify(migrateFinal)); + + await probe("export", "/export", undefined, { allowRefusal: true }); + { + const started = performance.now(); + let cursor, pages = 0, totalBytes = 0, lastStatus = 0, meshError; + do { + const res = await response(`/mesh/export?limit=500${cursor ? `&cursor=${encodeURIComponent(cursor)}` : ""}`); + lastStatus = res.status; + const text = await res.text(); + totalBytes += text.length; + pages++; + let json; + try { json = JSON.parse(text); } catch {} + cursor = json?.nextCursor; + if (!res.ok) { meshError = `HTTP ${res.status}: ${text.slice(0, 300)}`; break; } + if (pages > 200) { meshError = "pagination did not terminate"; break; } + } while (cursor); + const result = { name: "mesh-export", method: "GET", path: "/mesh/export (paged)", status: lastStatus, bytes: totalBytes, pages, latencyMs: Math.round(performance.now() - started), ...(meshError ? { error: meshError } : {}) }; + result.ok = lastStatus >= 200 && lastStatus < 300 && !meshError; + probes.push(result); + console.log(JSON.stringify(result)); + } + + workersAfter = await agentmemoryWorkers(); + const ids = (list) => list.filter((worker) => worker.id).map((worker) => worker.id); + const before = ids(workersBefore), after = ids(workersAfter); + workerObserved = before.length > 0 && after.length > 0 && !workersAfter.some((worker) => worker.error); + workerReregistered = !workerObserved || before.length !== after.length || before.some((id) => !after.includes(id)) || after.some((id) => !before.includes(id)); + logFindings = scanLogs(); +} catch (error) { + fatal = error.stack ?? String(error); + console.error(fatal); +} finally { + if (p.worker) { kill(p.worker); await Promise.race([p.worker.done, sleep(3000)]); } + if (p.client) { try { await p.client.shutdown(); } catch {} } + for (const socket of p.sockets) socket.terminate(); + if (p.engine) { kill(p.engine); await Promise.race([p.engine.done, sleep(3000)]); } + for (const child of [...children]) kill(child); +} + +const probeFailed = probes.filter((entry) => !entry.ok && entry.name !== "graph-index-final"); +const migrateOk = migrateFinal ? migrateFinal.accepted === true && migrateFinal.ready === true : false; +const summary = { + ok: !fatal, + root, + tarball: options.tarball, + engine, + engineVersion: manifest?.dependencies?.["iii-sdk"], + workerVersions, + sizes: seeded, + sessions: sessionCount, + probes, + failedProbes: probeFailed.map((entry) => entry.name), + migrate: migrateFinal, + stability: { workersBefore, workersAfter, workerObserved, workerReregistered }, + logFindings, + fatal, +}; +writeFileSync(join(root, "results.json"), JSON.stringify(summary, null, 2) + "\n"); +console.log("\nprobe status ms MiB error"); +for (const entry of probes) { + const mib = entry.bytes !== undefined ? (entry.bytes / (1024 * 1024)).toFixed(1) : "-"; + console.log(`${entry.name.padEnd(24)} ${String(entry.status ?? entry.state ?? "-").padEnd(8)} ${String(entry.latencyMs ?? entry.elapsedMs ?? "-").padEnd(9)} ${mib.padEnd(8)} ${entry.error ?? ""}`); +} +console.log(`workerObserved: ${workerObserved} workerReregistered: ${workerReregistered} payload_too_large hits: ${logFindings.payload_too_large ?? 0} WS_ERR hits: ${logFindings.WS_ERR_UNSUPPORTED_MESSAGE_LENGTH ?? 0}`); +console.log(`Results: ${join(root, "results.json")}`); +const assertFailed = options.assert && (fatal || probeFailed.length > 0 || (migrateFinal && !migrateOk) || workerReregistered); +process.exit(fatal ? 1 : assertFailed ? 1 : 0); diff --git a/scripts/release-gate/reflection.mjs b/scripts/release-gate/reflection.mjs index e0035693ca..64dcc68077 100644 --- a/scripts/release-gate/reflection.mjs +++ b/scripts/release-gate/reflection.mjs @@ -11,6 +11,7 @@ import { fileURLToPath } from "node:url"; const repo = resolve(dirname(fileURLToPath(import.meta.url)), "../.."); const options = { tarball: null, out: null, skipBuild: false, keep: false }; +const SAFE_PAYLOAD_BYTES = 60 * 1024 * 1024; for (let i = 2; i < process.argv.length; i++) { const arg = process.argv[i]; if (arg === "--tarball" || arg === "--out") { @@ -158,6 +159,8 @@ async function startModel() { if (modelMode === "missing") content = 'An unsupported checkpoint assertion.'; else if (modelMode === "malformed") content = 'No closing tag'; else if (modelMode === "empty") content = ""; + else if (modelMode === "oversized") content = "x".repeat(65537); + else if (modelMode === "excessive-citations") content = `${modelContent}`; else { const fact = sourceLabel(prompt, "FACT_ALPHA", "F"); const lesson = sourceLabel(prompt, "LESSON_SELECTED", "L"); @@ -214,15 +217,19 @@ async function install() { const binDir = join(runtime, "bin"); mkdirSync(binDir); copyFileSync(engine, join(binDir, "iii")); chmodSync(join(binDir, "iii"), 0o755); } -async function rest(path, body, { auth = true } = {}) { +async function rest(path, body, { auth = true, expectedStatus = 200 } = {}) { const headers = { "content-type": "application/json" }; if (auth) headers.authorization = `Bearer ${secret}`; const response = await fetch(`http://127.0.0.1:${port}/agentmemory${path}`, { method: body === undefined ? "GET" : "POST", headers, body: body === undefined ? undefined : JSON.stringify(body), signal: AbortSignal.timeout(30_000) }); const text = await response.text(); let json; try { json = JSON.parse(text); } catch { json = text; } - if (path !== "/livez") wire.push({ path, status: response.status, request: body, response: json }); - assert.equal(response.status, 200, `${path}: ${response.status} ${text.slice(0, 600)}`); + if (path !== "/livez") wire.push({ + path, status: response.status, + request: JSON.stringify(body ?? "").length > 65536 ? { bytes: Buffer.byteLength(JSON.stringify(body)) } : body, + response: text.length > 65536 ? { bytes: Buffer.byteLength(text), insightIds: json.insights?.map((item) => item.id) } : json, + }); + assert.equal(response.status, expectedStatus, `${path}: ${response.status} ${text.slice(0, 600)}`); return json; } async function startDaemon() { @@ -396,6 +403,72 @@ async function main() { const throughBridge = await bridge.call("memory_insight_list", { project: "reflection-a" }); assert.deepEqual(throughBridge.insights, before); }); + await scenario("bounded-evidence-and-reinforcement", async () => { + const imported = fixture(); + const template = imported.semanticMemories[0]; + imported.semanticMemories.unshift({ ...template, id: "fact-oversized", fact: `checkpoint ${"界".repeat(3000)}` }); + for (let i = 0; i < 50; i++) imported.semanticMemories.push({ ...template, id: `zz-extra-${i}`, fact: `EXTRA_${i} checkpoint rotation evidence` }); + assert.equal((await rest("/import", { exportData: imported, strategy: "replace" })).success, true); + modelMode = "valid"; + const first = await rest("/reflect", { project: "reflection-a", maxClusters: 1 }); + assert.equal(first.newInsights, 1); + const request = modelRequests.at(-1); + assert(Buffer.byteLength(request.system + request.prompt) <= 65536); + assert(!request.prompt.includes("界")); + assert.equal([...request.prompt.matchAll(/"source":"F\d+"/g)].length, 20); + verifyEvidence((await rest("/insights")).insights[0], "a"); + modelMode = "excessive-citations"; + const rejected = await rest("/reflect", { project: "reflection-a", maxClusters: 1 }); + assert.equal(rejected.rejectedInsights, 1); assert.equal(rejected.reinforced, 0); + modelMode = "valid"; + assert.equal((await rest("/reflect", { project: "reflection-a", maxClusters: 1 })).reinforced, 1); + modelMode = "oversized"; + const oversized = await rest("/reflect", { project: "reflection-a", maxClusters: 1 }); + assert.equal(oversized.failedClusters, 1); assert.equal(oversized.success, false); + }); + await scenario("count-validation-on-rest-and-mcp", async () => { + await rest("/reflect", { maxClusters: -1 }, { expectedStatus: 400 }); + await rest("/insights?limit=-1", undefined, { expectedStatus: 400 }); + await rest("/insights/search", { query: "checkpoint", limit: 1.5 }, { expectedStatus: 400 }); + await rest("/mcp/call", { name: "memory_insight_list", arguments: { limit: -1 } }, { expectedStatus: 400 }); + const rows = Array.from({ length: 510 }, (_, i) => ({ + id: `count-${i}`, title: "Checkpoint", content: "checkpoint count fixture", confidence: 0.8, + reinforcements: 0, sourceConceptCluster: [], sourceMemoryIds: [], sourceLessonIds: [], sourceCrystalIds: [], + project: "count-proof", tags: [], createdAt: new Date().toISOString(), updatedAt: new Date().toISOString(), decayRate: 0.05, + })); + await rest("/import", { exportData: { ...fixture(), insights: rows }, strategy: "replace" }); + assert.equal((await rest("/insights?project=count-proof&limit=30000")).insights.length, 500); + assert.equal((await directTool("memory_insight_list", { project: "count-proof", limit: 30000 })).insights.length, 500); + }); + await scenario("large-legacy-results-refuse-without-disconnecting-worker", async () => { + const base = (await rest("/insights?project=count-proof&limit=1")).insights[0]; + const largeRows = Math.ceil(SAFE_PAYLOAD_BYTES / (950 * 1024)) + 4; + for (let i = 0; i < largeRows; i++) { + const row = { ...base, id: `large-${i}`, project: "large-proof", content: `checkpoint ${"x".repeat(950 * 1024)}` }; + await rest("/import", { exportData: { ...fixture(), insights: [row] }, strategy: "merge" }); + } + const result = await rest(`/insights?project=large-proof&limit=${largeRows}`, undefined, { expectedStatus: 413 }); + assert.equal(result.oversized, true); + await rest("/insights/search", { query: "checkpoint", project: "large-proof", limit: 500 }, { expectedStatus: 413 }); + await rest("/mcp/call", { name: "memory_insight_list", arguments: { project: "large-proof", limit: largeRows } }, { expectedStatus: 413 }); + assert.equal((await rest("/insights?project=count-proof&limit=1")).insights.length, 1); + assert.equal((await rest("/insights?project=large-proof&limit=1")).insights.length, 1); + }); + await scenario("mcp-escaping-budget-with-small-inner-response", async () => { + const base = (await rest("/insights?project=count-proof&limit=1")).insights[0]; + const escapedRows = 48; + for (let i = 0; i < escapedRows; i++) { + const row = { ...base, id: `escaped-${i}`, project: "escaped-proof", content: '"'.repeat(400 * 1024) }; + await rest("/import", { exportData: { ...fixture(), insights: [row] }, strategy: "merge" }); + } + const raw = await rest("/insights?project=escaped-proof"); + assert.equal(raw.insights.length, escapedRows); + assert(Buffer.byteLength(JSON.stringify(raw)) < SAFE_PAYLOAD_BYTES); + const response = await rest("/mcp/call", { name: "memory_insight_list", arguments: { project: "escaped-proof" } }, { expectedStatus: 413 }); + assert.equal(response.isError, true); + assert(Buffer.byteLength(JSON.stringify(response)) < 2048); + assert.equal((await rest("/insights?project=count-proof&limit=1")).insights.length, 1); + }); await scenario("installed-package-unchanged", async () => assert.equal(packageDigest(), packageBefore)); } diff --git a/scripts/release-gate/run.mjs b/scripts/release-gate/run.mjs index a49e2e1727..59b2c3d314 100644 --- a/scripts/release-gate/run.mjs +++ b/scripts/release-gate/run.mjs @@ -629,8 +629,8 @@ class Instance { childProcessStarted: false, required: true, configurationRss: "No separate PID/RSS exists; configuration is loaded in-process inside iii-engine. Its RSS is the iii-engine entry in startupRss.", - pinSupported: "No per-dependency exact version selector is supported for engine-kind graph nodes in this Compose runtime. The active implementation is the engine-bundled configuration worker in the pinned iii 0.25.0, not the independent configuration@latest registry binary.", - explanation: "The published state/http manifests declare configuration: latest. The registry resolver returns dependency graph nodes in the same response; iii-compose::expand_graph omits nodes with kind=engine because they are supplied by the engine. Their dependency is satisfied by engine.workers.configuration (fs adapter), which Compose includes in engine-config.yaml. It is required by workers that use typed configuration, but is not downloaded/started as a child by Compose. Public registry info for configuration@latest currently reports 0.24.5; that is separate from the built-in configuration implementation in iii 0.25.0. It has no separate child RSS. No exact per-package dependency pin is available in this runtime; changing the engine pin is the control over the bundled implementation.", + pinSupported: `No per-dependency exact version selector is supported for engine-kind graph nodes in this Compose runtime. The active implementation is the engine-bundled configuration worker in the pinned iii ${gate.info.enginePin}, not the independent configuration@latest registry binary.`, + explanation: `The published state/http manifests declare configuration: latest. The registry resolver returns dependency graph nodes in the same response; iii-compose::expand_graph omits nodes with kind=engine because they are supplied by the engine. Their dependency is satisfied by engine.workers.configuration (fs adapter), which Compose includes in engine-config.yaml. It is required by workers that use typed configuration, but is not downloaded/started as a child by Compose. Public registry info for configuration@latest currently reports 0.24.5; that is separate from the built-in configuration implementation in iii ${gate.info.enginePin}. It has no separate child RSS. No exact per-package dependency pin is available in this runtime; changing the engine pin is the control over the bundled implementation.`, }; writeFileSync(join(OUT, `configuration-${this.name}.json`), JSON.stringify(metadata, null, 2) + "\n"); return metadata; diff --git a/scripts/release-gate/startup.mjs b/scripts/release-gate/startup.mjs index 156af0fc73..e9bfe0f0c4 100644 --- a/scripts/release-gate/startup.mjs +++ b/scripts/release-gate/startup.mjs @@ -9,25 +9,65 @@ import net from "node:net"; import { arch, homedir, platform, tmpdir } from "node:os"; import { dirname, join, resolve, delimiter } from "node:path"; import { fileURLToPath, pathToFileURL } from "node:url"; +import { runCommand } from "./command.mjs"; const repo = resolve(dirname(fileURLToPath(import.meta.url)), "../.."); -const options = { tarball: null, out: null, only: null, keep: false, delayMs: 48_000, skipBuild: false, workerBinDir: process.env.AGENTMEMORY_WORKER_BIN_DIR ?? null }; +const options = { tarball: null, out: null, only: null, keep: false, delayMs: 48_000, skipBuild: false, workerBinDir: process.env.AGENTMEMORY_WORKER_BIN_DIR ?? null, repeat: null }; for (let i = 2; i < process.argv.length; i++) { const arg = process.argv[i]; if (arg === "--tarball") options.tarball = resolve(process.argv[++i]); else if (arg === "--out") options.out = resolve(process.argv[++i]); else if (arg === "--only") options.only = new Set(process.argv[++i].split(",")); else if (arg === "--delay-ms") options.delayMs = Number(process.argv[++i]); + else if (arg === "--repeat") options.repeat = Number(process.argv[++i]); else if (arg === "--keep") options.keep = true; else if (arg === "--skip-build") options.skipBuild = true; else if (arg === "--worker-bin-dir") options.workerBinDir = resolve(process.argv[++i]); else if (arg === "--worker-bin-dir") options.workerBinDir = resolve(process.argv[++i]); else if (arg === "--help") { - console.log("Usage: node scripts/release-gate/startup.mjs [--tarball file.tgz] [--out dir] [--only names] [--delay-ms 48000] [--skip-build] [--worker-bin-dir directory] [--keep]"); + console.log("Usage: node scripts/release-gate/startup.mjs [--tarball file.tgz] [--out dir] [--only names] [--delay-ms 48000] [--skip-build] [--worker-bin-dir directory] [--keep] [--repeat N]"); process.exit(0); } else throw new Error(`Unknown argument ${arg}`); } assert(Number.isSafeInteger(options.delayMs) && options.delayMs > 0 && options.delayMs <= 300_000, "--delay-ms must be 1..300000"); +assert(options.repeat === null || (Number.isSafeInteger(options.repeat) && options.repeat > 0), "--repeat must be a positive integer"); +if (options.repeat) { + const self = fileURLToPath(import.meta.url); + const childArgs = process.argv.slice(2).flatMap((arg, index, argv) => + arg === "--repeat" ? [] : argv[index - 1] === "--repeat" ? [] : arg === "--out" ? [] : argv[index - 1] === "--out" ? [] : [arg]); + const repeatOut = options.out ?? mkdtempSync(join(tmpdir(), "agentmemory-startup-repeat-")); + mkdirSync(repeatOut, { recursive: true }); + console.log(`Repeat runs: ${repeatOut}`); + const runs = []; + const active = new Set(); + let interrupted = null; + const forward = (signal) => { + const escalate = interrupted !== null; + interrupted = signal; + for (const pid of active) { + try { process.kill(-pid, escalate ? "SIGKILL" : "SIGTERM"); } catch {} + } + }; + process.on("SIGINT", () => forward("SIGINT")); + process.on("SIGTERM", () => forward("SIGTERM")); + process.on("SIGHUP", () => forward("SIGHUP")); + for (let run = 1; run <= options.repeat && !interrupted; run++) { + const runOut = join(repeatOut, `run-${run}`); + console.log(`\n=== run ${run}/${options.repeat}: ${runOut} ===`); + const result = await runCommand(process.execPath, [self, ...childArgs, "--out", runOut], { env: process.env, cwd: repo, timeoutMs: 3_600_000, children: active }); + process.stdout.write(result.output); + let summary; + try { summary = JSON.parse(readFileSync(join(runOut, "summary.json"), "utf8")); } catch {} + const passed = summary?.scenarios?.filter((s) => s.status === "pass").length ?? 0; + const total = summary?.scenarios?.length ?? 0; + const ok = result.code === 0 && Boolean(summary?.ok); + runs.push({ run, ok, code: result.code, passed, total, out: runOut, timedOut: result.timedOut }); + } + console.log("\nrun result passed out"); + for (const r of runs) console.log(`${String(r.run).padEnd(5)} ${(r.ok ? "pass" : "fail").padEnd(7)} ${`${r.passed}/${r.total}`.padEnd(7)} ${r.out}`); + writeFileSync(join(repeatOut, "repeat-summary.json"), JSON.stringify({ repeat: options.repeat, runs, interrupted }, null, 2) + "\n"); + process.exit(interrupted ? 130 : runs.every((r) => r.ok) ? 0 : 1); +} const root = mkdtempSync(join(tmpdir(), "am-startup-")); const out = options.out ?? mkdtempSync(join(tmpdir(), "agentmemory-startup-gate-")); mkdirSync(out, { recursive: true }); @@ -189,7 +229,7 @@ class Instance { "containers:", " state:", ` worker: ${quote(localWorkers ? `path://${join(this.data, "local-workers/state")}` : "package://api.workers.iii.dev/state")}`, - ...(!localWorkers ? [" version: '0.22.20'"] : []), + ...(!localWorkers ? [" version: '0.22.22'"] : []), " config_name: state", ...(localWorkers ? [" scripts:", ` run: ${quote(join(localWorkers, "state"))}`] : []), " config_override:", @@ -201,7 +241,7 @@ class Instance { ` file_path: ${quote(join(this.data, "state_store.db"))}`, " http:", ` worker: ${quote(localWorkers ? `path://${join(this.data, "local-workers/http")}` : "package://api.workers.iii.dev/http")}`, - ...(!localWorkers ? [" version: '0.22.7'"] : []), + ...(!localWorkers ? [" version: '0.22.8'"] : []), " config_name: http", ...(localWorkers ? [" scripts:", ` run: ${quote(join(localWorkers, "http"))}`] : []), " config_override:", @@ -392,10 +432,13 @@ async function main() { d.exitCode = result.code; d.ownedComposeStopped = true; }); await scenario("cancel-external-startup", async (i, d) => { - await i.startEngine(i.config({ delayMs: 10_000, earlyPid: true })); + await i.startEngine(i.config({ delayMs: 120_000, earlyPid: true })); await poll("delayed Compose worker process", () => i.wrapperStarts()[0]?.pid, 10_000); - i.start(["--no-engine"], { AGENTMEMORY_READY_TIMEOUT_MS: "300000" }); - await sleep(1000); + i.start(["--no-engine"], { AGENTMEMORY_READY_TIMEOUT_MS: "300000", AGENTMEMORY_VERBOSE: "1" }); + await poll("CLI worker readiness wait", () => { + if (i.cli.result) throw Object.assign(new Error(`CLI exited before its readiness wait ${JSON.stringify(i.cli.result)}: ${i.cli.text.slice(-800)}`), { fatal: true }); + return i.cli.text.includes("Skipping engine check (--no-engine)"); + }, 30_000); i.cli.kill("SIGTERM"); const result = await exited(i.cli, 6_000); assert.equal(result.code, 143, `Cancellation result ${JSON.stringify(result)}: ${i.cli.text.slice(-800)}`); assert(alive(i.composePid), "Cancellation stopped an external Compose project"); assert(alive(i.wrapperStarts()[0].pid), "Cancellation killed the Compose-owned delayed worker"); diff --git a/scripts/release-gate/vector-recovery.mjs b/scripts/release-gate/vector-recovery.mjs index 40691181ee..49dd5af199 100644 --- a/scripts/release-gate/vector-recovery.mjs +++ b/scripts/release-gate/vector-recovery.mjs @@ -103,7 +103,7 @@ const config = [ "containers:", " state:", " worker: 'package://api.workers.iii.dev/state'", - " version: '0.22.20'", + " version: '0.22.22'", " config_name: state", " config_override:", " adapter:", @@ -114,7 +114,7 @@ const config = [ " save_interval_ms: 2000", " http:", " worker: 'package://api.workers.iii.dev/http'", - " version: '0.22.7'", + " version: '0.22.8'", " config_name: http", " config_override:", " host: 127.0.0.1", diff --git a/scripts/release-gate/windows-smoke.mjs b/scripts/release-gate/windows-smoke.mjs index 6aae869ba6..c89b16dafb 100644 --- a/scripts/release-gate/windows-smoke.mjs +++ b/scripts/release-gate/windows-smoke.mjs @@ -159,11 +159,16 @@ function leftoverProcesses() { const ps = spawnSync("powershell.exe", [ "-NoProfile", "-Command", [ - `$byPath = Get-CimInstance Win32_Process | Where-Object { $_.ProcessId -ne $PID -and $_.CommandLine -like '*${root.replace(/'/g, "''")}*' } | ForEach-Object { "$($_.ProcessId) $($_.Name)" }`, + "$ErrorActionPreference = 'Stop'", + `$root = '${root.replace(/'/g, "''")}'`, + `$byPath = Get-CimInstance Win32_Process | Where-Object { $_.ProcessId -ne $PID -and ($_.CommandLine -like "*$root*" -or $_.ExecutablePath -like "*$root*") } | ForEach-Object { "$($_.ProcessId) $($_.Name)" }`, `$byPort = Get-NetTCPConnection -State Listen -LocalPort 3911,3913,49950 -ErrorAction SilentlyContinue | ForEach-Object { "$($_.OwningProcess) port:$($_.LocalPort)" }`, `@($byPath) + @($byPort) | Where-Object { $_ }`, ].join("; "), ], { encoding: "utf8" }); + if (ps.error || ps.status !== 0) { + throw new Error(`leftover process query failed (status ${ps.status}): ${ps.error?.message ?? (ps.stderr ?? "").slice(-400)}`); + } return (ps.stdout ?? "").split(/\r?\n/).map((l) => l.trim()).filter(Boolean); } @@ -280,6 +285,16 @@ await step("stop cleans up", async () => { assert.equal(leftovers.length, 0, `leftover processes: ${leftovers.join(",")}`); return { leftovers: leftovers.length }; }); +await step("demo stops its Compose runtime", () => { + const demo = cli("demo", "--serve", "--port", "3911"); + const output = demo.stdout + demo.stderr; + writeFileSync(join(out, "demo.txt"), output); + assert.equal(demo.status, 0, `demo failed: ${output.slice(-1200)}`); + assert(output.includes("demo complete"), "demo did not finish seeding and searching"); + const leftovers = leftoverProcesses(); + assert.equal(leftovers.length, 0, `demo left processes: ${leftovers.join(",")}`); + return { leftovers: leftovers.length }; +}); await step("restart reaches livez", async () => startWorker()); await step("final stop", async () => { const stop = cli("stop", "--port", "3911"); diff --git a/scripts/state-worker/build.py b/scripts/state-worker/build.py new file mode 100644 index 0000000000..7b71a06dd8 --- /dev/null +++ b/scripts/state-worker/build.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +"""Build local State artifact from pinned clean git objects, never working-tree files.""" +import argparse, hashlib, json, os, pathlib, subprocess, tarfile + +def sha(path): return hashlib.sha256(path.read_bytes()).hexdigest() +def run(args, **kwargs): return subprocess.run(args, check=True, **kwargs) +def output(args, **kwargs): return subprocess.check_output(args, text=True, **kwargs).strip() + +def main(): + parser=argparse.ArgumentParser() + parser.add_argument('--workers-source', required=True, type=pathlib.Path) + parser.add_argument('--output', required=True, type=pathlib.Path) + parser.add_argument('--work-dir', required=True, type=pathlib.Path) + parser.add_argument('--cargo', default='cargo') + parser.add_argument('--jobs', type=int, default=2) + parser.add_argument('--resume', action='store_true', help='Reuse a marked build workspace and Cargo cache') + parser.add_argument('--online', action='store_true', help='Allow Cargo registry downloads; default uses cached dependencies') + args=parser.parse_args() + if not 1 <= args.jobs <= 32: parser.error('jobs must be 1..32') + repo=pathlib.Path(__file__).resolve().parents[2] + pin=json.loads((repo/'deploy/state-worker/pin.json').read_text()); patch=repo/'deploy/state-worker/compat.patch' + if sha(patch)!=pin['patchSha256']: raise RuntimeError('State patch digest mismatch') + source=args.workers_source.resolve(); work=args.work_dir.resolve(); dest=args.output.resolve() + marker=work/'.state-build-workspace.json' + if dest.exists(): raise RuntimeError('Output directory must not exist') + if work.exists() and (not args.resume or not marker.exists() or json.loads(marker.read_text()).get('upstreamCommit')!=pin['upstreamCommit']): raise RuntimeError('Work directory must be absent or a matching --resume build workspace') + if output(['git','rev-parse',pin['upstreamCommit']+'^{commit}'],cwd=source)!=pin['upstreamCommit']: raise RuntimeError('Upstream commit mismatch') + work.mkdir(parents=True,exist_ok=True); dest.mkdir(parents=True) + marker.write_text(json.dumps({'upstreamCommit':pin['upstreamCommit']})) + archive=work/'upstream.tar' + run(['git','archive','--format=tar','--output='+str(archive),pin['upstreamCommit'],'state','crates','LICENSE','package.json','pnpm-workspace.yaml','pnpm-lock.yaml','packages'],cwd=source) + import shutil + tree=work/'source' + if tree.exists(): shutil.rmtree(tree) + tree.mkdir() + with tarfile.open(archive) as f: f.extractall(tree, filter='data') + run(['git','apply','--check',str(patch)],cwd=tree) + run(['git','apply',str(patch)],cwd=tree) + env=dict(os.environ,CARGO_BUILD_JOBS=str(args.jobs),CARGO_TARGET_DIR=str(work/'target'),SOURCE_DATE_EPOCH=output(['git','show','-s','--format=%ct',pin['upstreamCommit']],cwd=source)) + env['RUSTFLAGS']='--remap-path-prefix='+str(work)+'=/usr/src/agentmemory-state' + pnpm=['pnpm','--filter','@iii-workers/state-ui...','install','--frozen-lockfile'] + if not args.online: pnpm.append('--offline') + run(pnpm,cwd=tree) + run(['pnpm','--filter','@iii-workers/state-ui','build'],cwd=tree) + env['SKIP_UI_BUILD']='1' + command=[args.cargo,'build','--release','--locked','--bin','state'] + if not args.online: command.append('--offline') + run(command,cwd=tree/'state',env=env) + binary=work/'target/release/state'; target=dest/'state'; target.write_bytes(binary.read_bytes());target.chmod(0o755) + import shutil + shutil.copyfile(patch,dest/'compat.patch');shutil.copyfile(tree/'LICENSE',dest/'LICENSE') + manifest={**pin,'profile':'release','binary':'state','binarySha256':sha(target),'sourceArchiveSha256':sha(archive),'cargoLockSha256':sha(tree/'state/Cargo.lock'),'rustc':output(['rustc','-Vv']),'cargo':output([args.cargo,'-V']),'pnpm':output(['pnpm','--version']),'node':output(['node','--version']),'sourceDateEpoch':env['SOURCE_DATE_EPOCH'],'target':output(['rustc','-vV']).split('host: ')[1].splitlines()[0]} + (dest/'manifest.json').write_text(json.dumps(manifest,indent=2)+'\n') + archive_path=dest.with_name(pin['artifactVersion']+'-'+manifest['target']+'.tar.gz') + if archive_path.exists(): raise RuntimeError('Artifact archive already exists') + import gzip + with archive_path.open('wb') as raw, gzip.GzipFile(filename='',mode='wb',fileobj=raw,mtime=0) as compressed, tarfile.open(fileobj=compressed,mode='w') as tar: + for path in sorted(dest.iterdir()): + info=tar.gettarinfo(str(path),arcname='state-worker/'+path.name);info.uid=info.gid=0;info.uname=info.gname='';info.mtime=int(env['SOURCE_DATE_EPOCH']) + with path.open('rb') as handle: tar.addfile(info,handle) + print(json.dumps({'archive':str(archive_path),'archiveSha256':sha(archive_path),'artifact':str(dest),'sha256':manifest['binarySha256'],'version':pin['artifactVersion']})) +if __name__=='__main__': main() diff --git a/scripts/state-worker/large-overwrite-proof.mjs b/scripts/state-worker/large-overwrite-proof.mjs new file mode 100644 index 0000000000..e01218ef36 --- /dev/null +++ b/scripts/state-worker/large-overwrite-proof.mjs @@ -0,0 +1,25 @@ +// Isolated release smoke. Explicit binaries/modules only; never contacts production. +import {mkdtempSync,mkdirSync,writeFileSync,readFileSync} from 'node:fs';import {tmpdir} from 'node:os';import {join,resolve} from 'node:path';import {pathToFileURL} from 'node:url';import {spawn} from 'node:child_process';import {createServer} from 'node:net';import {createHash} from 'node:crypto';import assert from 'node:assert/strict'; +const [engineBinary,stateBinary,sdkModule,outputDirectory]=process.argv.slice(2);if(!engineBinary||!stateBinary||!sdkModule)throw Error('Usage: node --max-old-space-size=1536 large-overwrite-proof.mjs ENGINE STATE SDK_MODULE [OUTPUT]'); +const {registerWorker}=await import(pathToFileURL(resolve(sdkModule)).href); +const dir=outputDirectory?resolve(outputDirectory):mkdtempSync(join(tmpdir(),'state-large-proof-'));mkdirSync(dir,{recursive:true});mkdirSync(join(dir,'home'),{recursive:true}); +const server=createServer();await new Promise(r=>server.listen(0,'127.0.0.1',r));const port=server.address().port;await new Promise(r=>server.close(r)); +writeFileSync(join(dir,'engine.yaml'),`workers:\n - name: configuration\n config:\n adapter:\n name: fs\n config:\n directory: ${JSON.stringify(join(dir,'config'))}\n - name: iii-worker-manager\n config:\n host: 127.0.0.1\n port: ${port}\n`); +writeFileSync(join(dir,'state.yaml'),`adapter:\n name: kv\n config:\n store_method: file_based\n file_path: ${JSON.stringify(join(dir,'store'))}\n save_interval_ms: 3600000\n`); +let engine,state,sdk,workerPeakKiB=0,enginePeakKiB=0;const processes=[]; +const env={...process.env,HOME:join(dir,'home'),III_TELEMETRY_ENABLED:'false',OTEL_ENABLED:'false'}; +const start=(binary,args,log)=>{const child=spawn(resolve(binary),args,{env,stdio:['ignore','pipe','pipe']});processes.push(child);for(const stream of [child.stdout,child.stderr])stream.on('data',data=>{import('node:fs').then(fs=>fs.appendFileSync(join(dir,log),data));});return child;}; +const stop=async child=>{if(child&&child.exitCode===null&&child.signalCode===null){const ended=new Promise(r=>child.once('exit',r));child.kill('SIGTERM');const timer=setTimeout(()=>child.kill('SIGKILL'),10000);await ended;clearTimeout(timer);}}; +const sample=setInterval(()=>{for(const [child,kind]of [[state,'state'],[engine,'engine']]){try{const match=/^VmHWM:\s+(\d+)/m.exec(readFileSync(`/proc/${child.pid}/status`,'utf8'));const n=Number(match?.[1]??0);if(kind==='state')workerPeakKiB=Math.max(workerPeakKiB,n);else enginePeakKiB=Math.max(enginePeakKiB,n);}catch{}}},100); +const connect=async()=>{engine=start(engineBinary,['--no-update-check','-c',join(dir,'engine.yaml')],'engine.log');await new Promise(r=>setTimeout(r,1000));state=start(stateBinary,['--url',`ws://127.0.0.1:${port}`,'--config',join(dir,'state.yaml')],'state.log');sdk=registerWorker(`ws://127.0.0.1:${port}`,{workerName:'large-release-proof',invocationTimeoutMs:60000,enableMetricsReporting:false,otel:{enabled:false}});for(let i=0;i<60;i++){try{return await sdk.trigger({function_id:'state::boot_identity',payload:{}});}catch{await new Promise(r=>setTimeout(r,100));}}throw Error('isolated State did not become ready');}; +const key={scope:'synthetic:large-overwrite',key:'record'},size=34*1024*1024; +const value=c=>({sourceRaw:c.repeat(size),offsets:{offset:size},retry:{attempts:2},holds:['storage_error'],unknown:true});const hash=v=>createHash('sha256').update(v.sourceRaw).digest('hex'); +try{ + const identity=await connect();const call=(name,payload)=>sdk.trigger({function_id:`state::${identity.instanceId}::${name}`,payload}); + let expected; + for(const char of ['A','B']){const data=value(char);assert(Buffer.byteLength(JSON.stringify(data))>=33556506);expected=hash(data);assert.equal(await call('set_ack',{...key,value:data}),true);const got=await call('get',key);assert.equal(got.sourceRaw.length,size);assert.equal(hash(got),expected);assert.deepEqual(got.offsets,data.offsets);console.log(JSON.stringify({stage:'overwrite',character:char,jsonBytes:Buffer.byteLength(JSON.stringify(data)),ack:true,digest:expected}));} + const receipt=await call('seal_flush',{});assert.deepEqual(receipt,{version:1,instanceId:identity.instanceId,storePath:identity.storePath,sealed:true,synced:true}); + await sdk.shutdown();sdk=null;await stop(state);await stop(engine); + const next=await connect();assert.notEqual(next.instanceId,identity.instanceId);const got=await sdk.trigger({function_id:`state::${next.instanceId}::get`,payload:key});assert.equal(got.sourceRaw.length,size);assert.equal(hash(got),expected);assert.equal(got.retry.attempts,2);assert.deepEqual(got.holds,['storage_error']);assert.equal(got.unknown,true); + console.log(JSON.stringify({stage:'restart',digest:expected,preserved:true,sealedFsync:true,workerPeakKiB,enginePeakKiB,nodePeakKiB:process.resourceUsage().maxRSS,output:dir})); +}finally{clearInterval(sample);if(sdk)await sdk.shutdown();for(const child of processes.reverse())await stop(child);} diff --git a/src/capture/auto-recovery.ts b/src/capture/auto-recovery.ts index 4769e43da2..36a1087c61 100644 --- a/src/capture/auto-recovery.ts +++ b/src/capture/auto-recovery.ts @@ -1,6 +1,6 @@ import {readerEvidence,validReaderEvidence} from '../ops/read-progress.mjs'; import {recordAudit} from "../functions/audit.js"; -import type {CaptureRecoveryCheckpoint,CaptureRecoveryFailure} from "../types.js"; +import type {CaptureRecoveryCheckpoint,CaptureRecoveryFailure,CaptureReviewHistory} from "../types.js"; import {createHash,randomUUID} from 'node:crypto'; import {isDeepStrictEqual} from 'node:util'; import type {StateKV} from '../state/kv.js'; @@ -8,6 +8,7 @@ import {withKeyedLock} from '../state/keyed-mutex.js'; import {KV} from '../state/schema.js'; import {captureEventScope} from './event-record.js'; import {capturePayloadDigest,captureSourceDigest} from './receipt-digest.js'; +import {committedSdkResolution} from './sdk-resolution-record.js'; const SCOPE=KV.captureRecovery; const SAFE_REASONS=new Set(['PUBLICATION_PENDING', 'PUBLICATION_WAIT_EXHAUSTED', 'ADMISSION_HELD', 'CHECK_PENDING', 'COMPLETION_PROOF_PENDING', 'VERIFIED_DURABLE', 'VERIFICATION_BUDGET_EXHAUSTED', 'VERIFICATION_PENDING', 'DELETION_RECORDED', 'IDENTITY_CONFLICT', 'SOURCE_MISSING', 'SOURCE_UNSUPPORTED', 'SOURCE_MISMATCH', 'GENERATION_CHANGED', 'INTENT_ACTIVE_OR_UNSUPPORTED', 'PUBLICATION_UNVERIFIED', 'INDEX_RESTORATION_REQUIRED', 'MANUAL_OR_ACTIVE_RETRY', 'REQUEST_REQUIRES_MANUAL_RESTORE', 'RECEIPT_CONFLICT', 'MANUAL_OR_UNKNOWN_COMPLETION', 'STATE_CHANGED', 'COMPLETION_UNVERIFIED', 'ADMISSION_CHANGED', 'FAILURE_UNCLASSIFIED']); @@ -29,7 +30,7 @@ function failureEvidence(error:any,operation:string):CaptureRecoveryFailure { function validRow(v:any):v is Row{return v?.version===1&&v.kind==='item'&&/^cap_[a-f0-9]{40}$/.test(v.key)&&typeof v.id==='string'&&['sessionId','observationId','eventId'].every(k=>typeof v[k]==='string'&&v[k])&&/^[a-f0-9]{64}$/.test(v.sourceDigest)&&/^[a-f0-9]{64}$/.test(v.projection)&&['checking','verifying','resolved','blocked'].includes(v.phase)&&Number.isSafeInteger(v.verificationAttempts)&&v.verificationAttempts>=0&&v.verificationAttempts<=MAX_PROOFS&&Number.isFinite(v.nextAttemptAt)&&v.nextAttemptAt>=0&&(v.lastVerificationFailure===undefined||validFailure(v.lastVerificationFailure));} export function createCaptureAutoRecovery(kv:StateKV,options:CaptureAutoRecoveryOptions){ const now=options.now??Date.now;let busy=false,halted=false,enabled=options.enabled,revision=0,initialized:Promise|undefined;let durableEnabled:boolean|null=null,configurationStatus='unloaded'; - let view={enabled:options.enabled,supported:false,phase:'idle',resolved:0,verifying:0,blocked:0,reasonCounts:{}as Record,breakerUntil:0,lastCheckedAt:null as string|null,lastErrorCode:null as string|null,verificationFailureCounts:{}as Record}; + let view={enabled:options.enabled,supported:false,phase:'idle',resolved:0,sdkReviewed:0,verifying:0,blocked:0,reasonCounts:{}as Record,breakerUntil:0,lastCheckedAt:null as string|null,lastErrorCode:null as string|null,verificationFailureCounts:{}as Record}; const blockReason=()=>{ if(kv.durability?.completion==='sdk')return 'SDK_COMPLETION_ONLY'; if(halted)return 'RECOVERY_STORAGE_OR_STATE_UNAVAILABLE'; @@ -67,9 +68,27 @@ export function createCaptureAutoRecovery(kv:StateKV,options:CaptureAutoRecovery const all=await kv.list(SCOPE);if(all.length>5001||all.some(r=>r?.kind==='item'&&!validRow(r)))throw fail('RECOVERY_STATE_UNSUPPORTED'); const rows=all.filter(validRow),control=all.find(r=>r?.kind==='control') as Control|undefined; if(control&&!validControl(control))throw fail('RECOVERY_STATE_UNSUPPORTED'); - const counts:Record={};for(const r of rows)if(r.phase!=='resolved'){const reason=SAFE_REASONS.has(r.reason)?r.reason:'RECOVERY_STATE_UNSUPPORTED';counts[reason]=(counts[reason]??0)+1;} - const verificationFailureCounts:Record={};for(const r of rows)if(r.phase!=='resolved'&&r.lastVerificationFailure){const f=r.lastVerificationFailure,k=f.operation+':'+f.stage+':'+f.code;verificationFailureCounts[k]=(verificationFailureCounts[k]??0)+1;} - view={...view,verificationFailureCounts,supported:kv.durability?.completion!=='sdk'&&kv.recoveryMutationSafe&&!!kv.recoveryWrite&&kv.backend==='file'&&!!kv.durability&&options.writerOwned(),resolved:rows.filter(r=>r.phase==='resolved').length,verifying:rows.filter(r=>['checking','verifying'].includes(r.phase)).length,blocked:rows.filter(r=>r.phase==='blocked').length,reasonCounts:counts,breakerUntil:control?.openUntil??0,lastCheckedAt:new Date(now()).toISOString()}; + let unresolved=rows.filter(row=>row.phase!=='resolved'); + const reviewed=new Set(); + if(kv.durability?.completion==='sdk'){ + // Bound extra SDK reads. Older checkpoints stay in storage as historical evidence. + for(const row of unresolved.slice(0,16)){ + try{ + const [history,inbox]=await Promise.all([kv.get(KV.captureReview,row.key),kv.get>(KV.captureInbox,row.key)]); + if(history?.key===row.key&&history.eventId===row.eventId&&history.sessionId===row.sessionId&&history.observationId===row.observationId&& + history.sdkResolution?.sourceDigest===row.sourceDigest&&inbox?.sdkRestoreState!=='pending'&&committedSdkResolution(history,inbox))reviewed.add(row.key); + }catch{/* Leave the item in the warning if its SDK record cannot be confirmed. */} + } + unresolved=unresolved.filter(row=>!reviewed.has(row.key)); + } + const counts:Record={}; + const verificationFailureCounts:Record={}; + for(const row of unresolved){ + const reason=SAFE_REASONS.has(row.reason)?row.reason:'RECOVERY_STATE_UNSUPPORTED'; + counts[reason]=(counts[reason]??0)+1; + if(row.lastVerificationFailure){const f=row.lastVerificationFailure,key=f.operation+':'+f.stage+':'+f.code;verificationFailureCounts[key]=(verificationFailureCounts[key]??0)+1;} + } + view={...view,verificationFailureCounts,supported:kv.durability?.completion!=='sdk'&&kv.recoveryMutationSafe&&!!kv.recoveryWrite&&kv.backend==='file'&&!!kv.durability&&options.writerOwned(),resolved:rows.filter(row=>row.phase==='resolved').length,sdkReviewed:reviewed.size,verifying:unresolved.filter(row=>row.phase!=='blocked').length,blocked:unresolved.filter(row=>row.phase==='blocked').length,reasonCounts:counts,breakerUntil:control?.openUntil??0,lastCheckedAt:new Date(now()).toISOString()}; return {rows,control:control??{version:1,kind:'control',failures:0,openUntil:0}as Control}; } function keys(row:Row){return [{scope:KV.captureReview,key:row.key},{scope:KV.captureInbox,key:row.key},{scope:KV.observations(row.sessionId),key:row.observationId},{scope:'mem:embedding-retry:v1',key:row.observationId},{scope:'mem:embedding-committed:v1',key:row.observationId},{scope:captureEventScope(row.key),key:row.key}];} @@ -139,6 +158,7 @@ export function createCaptureAutoRecovery(kv:StateKV,options:CaptureAutoRecovery } return { snapshot,initialize,setEnabled, + async refreshSdkReviews(){if(kv.durability?.completion==='sdk'){await initialize();await refresh();}}, async tick(){ if(busy)return;busy=true; try{await initialize();if(!enabled)return;await withKeyedLock('capture-auto-recovery',async()=>{ diff --git a/src/capture/inbox-metadata.ts b/src/capture/inbox-metadata.ts new file mode 100644 index 0000000000..9b093089d4 --- /dev/null +++ b/src/capture/inbox-metadata.ts @@ -0,0 +1,54 @@ +import { isDeepStrictEqual } from 'node:util'; +import type { InboxRecord } from '../functions/capture.js'; +import type { CaptureReviewHistory } from '../types.js'; +import type { StateKV } from '../state/kv.js'; +import { KV } from '../state/schema.js'; +import { committedSdkResolution } from './sdk-resolution-record.js'; + +export type CaptureInboxMetadata = Omit & { + preview: string; + reviewPreview: string; +}; + +export function projectCaptureInbox(value: unknown): CaptureInboxMetadata { + const record = value as InboxRecord; + if (!record) return record; + const { payload, ...metadata } = record; + const data = payload?.data && typeof payload.data === 'object' ? payload.data as Record : {}; + const label = typeof data.tool_name === 'string' ? data.tool_name : payload?.hookType; + let text = ''; + try { text = JSON.stringify(data.tool_input ?? data.prompt ?? payload?.data ?? ''); } catch {} + const reviewLabel = typeof data.tool_name === 'string' ? data.tool_name : record.hookType; + return { + ...metadata, + preview: payload ? `${label}: ${text}`.slice(0, 160) : '', + reviewPreview: (reviewLabel + ': ' + JSON.stringify(data.tool_input ?? data.prompt ?? data.message ?? '')).slice(0, 500), + }; +} + +export function listCaptureInboxMetadata(kv: StateKV): Promise { + return kv.list(KV.captureInbox, { project: projectCaptureInbox }); +} + +export async function listLocalCaptureResolutionKeys(kv: StateKV): Promise | undefined> { + if (!kv.graphFileList?.accepts(KV.captureReview)) return undefined; + const rows = await kv.list(KV.captureReview, { project: (value, key) => { + if (!value || typeof value !== 'object' || Array.isArray(value) || typeof key !== 'string' || !key || + (value as CaptureReviewHistory).key !== key) throw Error('capture_review_key_unverified'); + return { key, hasResolution: (value as CaptureReviewHistory).sdkResolution !== undefined }; + } }); + return new Set(rows.filter(row => row.hasResolution).map(row => row.key)); +} + +export async function committedInboxMetadata( + kv: StateKV, + history: CaptureReviewHistory | null | undefined, + metadata: CaptureInboxMetadata, +): Promise { + if (history?.sdkResolution === undefined) return false; + const current = await kv.get(KV.captureInbox, metadata.key); + if (!current) return false; + const { payload, ...currentMetadata } = current; + const { preview, reviewPreview, ...expectedMetadata } = metadata; + return isDeepStrictEqual(currentMetadata, expectedMetadata) && committedSdkResolution(history, current); +} diff --git a/src/capture/retry-jobs.ts b/src/capture/retry-jobs.ts index 44699b63a9..0a81617bff 100644 --- a/src/capture/retry-jobs.ts +++ b/src/capture/retry-jobs.ts @@ -1,8 +1,9 @@ import { randomUUID } from 'node:crypto'; import { KV } from '../state/schema.js'; import type { StateKV } from '../state/kv.js'; -import { withKeyedLock } from '../state/keyed-mutex.js'; import type { InboxRecord } from '../functions/capture.js'; +import { withKeyedLock } from '../state/keyed-mutex.js'; +import { listCaptureInboxMetadata } from './inbox-metadata.js'; import { captureEventScope } from './event-record.js'; import type { CaptureReviewHistory, CaptureRetryJob as Job, CaptureRetryJobItem as Item, CaptureRetryJobItemState as ItemState } from '../types.js'; @@ -44,7 +45,7 @@ export function createCaptureRetryJobs(kv: StateKV, retry: (key: string, force: const running=await latest();if(running&&active(running)){ const now=new Date().toISOString();await save({id:input.requestId,aliasOf:running.id,all:input.all===true,...(input.eventId?{eventId:input.eventId}:{}),...(input.key?{key:input.key}:{}),createdAt:now,updatedAt:now});return view(running)!; } - const records=(await kv.list(KV.captureInbox)).filter(r=>r.status==='dead'&&(input.all||r.eventId===input.eventId&&(!input.key||r.key===input.key))); + const records=(await listCaptureInboxMetadata(kv)).filter(r=>r.status==='dead'&&(input.all||r.eventId===input.eventId&&(!input.key||r.key===input.key))); if(records.length>10000)throw Error('CAPTURE_JOB_TOO_LARGE'); const now=new Date().toISOString();const job:Job={id:input.requestId,all:input.all===true,...(input.eventId?{eventId:input.eventId}:{}),...(input.key?{key:input.key}:{}),status:records.length?'running':'completed',stopRequested:false,createdAt:now,updatedAt:now, items:records.sort((a,b)=>a.acceptedAt.localeCompare(b.acceptedAt)||a.key.localeCompare(b.key)).map(r=>({key:r.key,eventId:r.eventId,requestId:randomUUID(),status:'queued'}))}; diff --git a/src/capture/review.ts b/src/capture/review.ts index f54d7a5463..247b266e63 100644 --- a/src/capture/review.ts +++ b/src/capture/review.ts @@ -2,7 +2,8 @@ import type { CaptureReviewHistory } from '../types.js'; import type { InboxRecord } from '../functions/capture.js'; import type { StateKV } from '../state/kv.js'; import { KV } from '../state/schema.js'; -import{validSdkResolution,committedSdkResolution}from'./sdk-resolution-record.js'; +import{validSdkResolution}from'./sdk-resolution-record.js'; +import { listCaptureInboxMetadata, committedInboxMetadata } from './inbox-metadata.js'; export function captureFailureCode(error: string): string { if (error.includes('Session observation limit reached')) return 'SESSION_OBSERVATION_LIMIT'; @@ -54,18 +55,17 @@ export function reviewMetadata(history: CaptureReviewHistory) { export async function captureReviewPage(kv: StateKV, input: {status?:string;eventId?:string;key?:string;cursor?:string;limit?:number;historyOffset?:number}) { const limit=Math.min(Math.max(1,Number(input.limit)||25),50); - const inbox=await kv.list(KV.captureInbox); + const inbox=await listCaptureInboxMetadata(kv); const histories=await kv.list(KV.captureReview); const byKey=new Map(histories.map(h=>[h.key,h])); const rows=new Map(); for (const h of histories) rows.set(h.key,{key:h.key,eventId:h.eventId,sessionId:h.sessionId,project:h.project,hookType:h.hookType, observationId:h.observationId,status:h.status,attempts:h.totalAttempts,acceptedAt:h.acceptedAt,updatedAt:h.updatedAt,review:reviewMetadata(h),preview:'복구 기록 · 원문은 저장된 관측에서 확인'}); for (const rec of inbox) { - const h=byKey.get(rec.key);const d=rec.payload?.data as Record|undefined; - const text=typeof d?.tool_name==='string' ? d.tool_name : rec.hookType; + const h=byKey.get(rec.key); rows.set(rec.key,{key:rec.key,eventId:rec.eventId,sessionId:rec.sessionId,project:rec.project,hookType:rec.hookType, - observationId:rec.observationId,status:committedSdkResolution(h,rec)?'completed':rec.status,attempts:rec.attempts,acceptedAt:rec.acceptedAt,updatedAt:rec.updatedAt, - errorCode:rec.lastError ? captureFailureCode(rec.lastError) : null,preview:text, + observationId:rec.observationId,status:await committedInboxMetadata(kv,h,rec)?'completed':rec.status,attempts:rec.attempts,acceptedAt:rec.acceptedAt,updatedAt:rec.updatedAt, + errorCode:rec.lastError ? captureFailureCode(rec.lastError) : null,preview:rec.reviewPreview, review:h ? reviewMetadata(h) : {status:rec.status,totalAttempts:rec.attempts,manualRetries:0,lastFailureCode:rec.lastError?captureFailureCode(rec.lastError):null,recoveredAt:null,lastRequest:null}}); } let filtered=[...rows.values()].filter(r=>(!input.status||r.status===input.status)&&(!input.eventId||r.eventId===input.eventId)&&(!input.key||r.key===input.key)); @@ -78,7 +78,6 @@ export async function captureReviewPage(kv: StateKV, input: {status?:string;even filtered=filtered.filter(r=>r.acceptedAt>cursor[0]||r.acceptedAt===cursor[0]&&r.key>cursor[1]); } const items=filtered.slice(0,limit);const last=items.at(-1); - for(const item of items){const rec=inbox.find(r=>r.key===item.key);if(rec){const d=rec.payload?.data as Record|undefined;item.preview=(item.preview+': '+JSON.stringify(d?.tool_input??d?.prompt??d?.message??'')).slice(0,500);}} if(input.eventId)for(const item of items){const h=byKey.get(item.key);const offset=Number.isSafeInteger(input.historyOffset)&&input.historyOffset!>=0 ? input.historyOffset! : 0; item.failures=h?.failures.slice(offset,offset+50)||[];item.nextHistoryOffset=h&&h.failures.length>offset+50 ? offset+50 : null;} return {total,items,nextCursor:filtered.length>limit&&last?Buffer.from(JSON.stringify([last.acceptedAt,last.key])).toString('base64url'):null}; diff --git a/src/capture/sdk-delivery-v2.ts b/src/capture/sdk-delivery-v2.ts index 5fc3096aa2..513fe0e126 100644 --- a/src/capture/sdk-delivery-v2.ts +++ b/src/capture/sdk-delivery-v2.ts @@ -1,21 +1,26 @@ import {performance} from 'node:perf_hooks'; +import {createHash} from 'node:crypto'; import {SdkSpoolV2,type SdkSpoolV2Options} from './sdk-spool-v2.js'; export {SdkSpoolV2} from './sdk-spool-v2.js'; export const SDK_CAPTURE_V2_FLAG='--agentmemory-sdk-capture-v2'; export type V2PostResult={status:number;body:unknown;notSent?:boolean}; export type V2PostBudget={deadline:number;remainingMs:number}; export type V2Post=(body:Record,budget?:V2PostBudget)=>Promise; -export type V2Options={dir?:string;storeOptions?:Omit}; +export type V2Options={dir?:string;deadline?:number;storeOptions?:Omit}; +export const SDK_FINALIZATION_RESERVE_MS=750; export type V2DeliveryResult={outcome:'delivered'|'duplicate'|'spooled'|'held'|'dropped';code?:string;sourceAcknowledged:boolean;referenceAcknowledged:boolean;sdkReplyConfirmed?:boolean}; const STORE_CODES=new Set(['SOURCE_TOO_LARGE','ORIGINALS_FULL','QUEUE_FULL','SOURCE_CONFLICT','SOURCE_UNVERIFIED','SOURCE_HELD','SOURCE_BACKOFF','STORE_BUSY','STORE_IO','STORE_OPTIONS_INVALID','STORE_UNSAFE','STORE_LOCK_UNSUPPORTED']); function safeCode(error:unknown){const code=(error as {code?:unknown})?.code;return typeof code==='string'&&STORE_CODES.has(code)?code:'STORE_OR_DELIVERY_UNCONFIRMED';} -function createStore(options:V2Options){return new SdkSpoolV2({...options.storeOptions,dir:options.dir});} +function createStore(options:V2Options){return new SdkSpoolV2({...options.storeOptions,dir:options.dir,deadline:options.deadline});} function held(code:string,sourceAcknowledged=true,referenceAcknowledged=true):V2DeliveryResult{return {outcome:'held',code,sourceAcknowledged,referenceAcknowledged};} async function park(store:SdkSpoolV2,target:string,sourceId:string,code:string){try{await store.hold(target,sourceId,code);}catch{return held('QUEUE_STATE_UNCONFIRMED');}return held(code);} async function sendReference(store:SdkSpoolV2,target:string,sourceId:string,post:V2Post,deadline?:number):Promise{ - try{await store.markSending(target,sourceId);}catch(error){return held(safeCode(error));} let body:Record; - try{body=await store.load(target,sourceId);}catch{return park(store,target,sourceId,'SOURCE_UNVERIFIED');} + try{body=await store.claimForSend(target,sourceId,deadline);}catch(error){ + const code=(error as {code?:unknown})?.code; + if(code==='SOURCE_BACKOFF'||code==='SOURCE_SEND_BUDGET_EXHAUSTED')return {outcome:'spooled',sourceAcknowledged:true,referenceAcknowledged:true}; + return held(code==='SOURCE_SENDING'?'SDK_SEND_IN_PROGRESS_OR_UNCERTAIN':safeCode(error)); + } let response:V2PostResult; const budget=deadline===undefined?undefined:{deadline,remainingMs:Math.max(0,Math.floor(deadline-performance.now()))}; try{response=budget&&budget.remainingMs<=0?{status:0,body:null,notSent:true}:await post(body,budget);}catch{return park(store,target,sourceId,'SDK_SEND_UNCERTAIN');} @@ -37,25 +42,21 @@ export async function deliverSdkCaptureV2(target:string,body:Recordrecord.sourceId===sourceId);}catch(error){return held(safeCode(error));} - if(!pending){ + const sent=await sendReference(store,target,sourceId,post,options.deadline===undefined?undefined:options.deadline-SDK_FINALIZATION_RESERVE_MS); + if(sent.code==='SOURCE_HELD'){ try{if((await store.retain(target,storedBody.eventId as string,storedBody)).completed)return {outcome:'duplicate',sourceAcknowledged:true,referenceAcknowledged:true,sdkReplyConfirmed:true};}catch{} - return held('QUEUE_STATE_UNCONFIRMED'); } - if(pending.state!=='pending')return held(pending.state==='sending'?'SDK_SEND_IN_PROGRESS_OR_UNCERTAIN':'SOURCE_HELD'); - if(pending.nextAttemptAt>Date.now())return {outcome:'spooled',sourceAcknowledged:true,referenceAcknowledged:true}; - return sendReference(store,target,sourceId,post); + return sent; } export async function drainSdkCapturesV2(target:string,post:V2Post,options:V2Options&{limit?:number;budgetMs?:number;deadline?:number}={}){ const result={attempted:0,delivered:0,duplicates:0,held:0,remaining:0,error:undefined as string|undefined}; const limit=Math.min(8,Math.max(0,options.limit??8)),budget=Math.min(1500,Math.max(0,options.budgetMs??1500)),start=performance.now(); - const deadline=Math.min(options.deadline??Infinity,start+budget); + const deadline=Math.min(options.deadline===undefined?Infinity:options.deadline-SDK_FINALIZATION_RESERVE_MS,start+budget); try{ const store=createStore(options),records=await store.pending(target); for(const ref of records){ @@ -80,6 +81,47 @@ function hasImage(value:unknown):boolean{ } return false; } +// Operator-only. Callers must durably record a one-shot intent before invoking this function. +// A lost response stays held and requires inspection, never automatic re-submission. +export async function restoreSelectedSdkCaptureV2(target:string,sourceId:string,post:V2Post,options:V2Options={}){ + const store=createStore(options); + let body:Record,observationId:string; + try{body=await store.load(target,sourceId);} + catch(error){return {sourceRetained:false,completion:'sdk' as const,durable:false,completed:false,status:'held' as const,code:safeCode(error)};} + const retained={sourceRetained:true,completion:'sdk' as const,durable:false}; + try{ + if(body.captureCompletion!=='sdk-v1'||Object.keys(body).some(key=>!['eventId','captureCompletion','hookType','sessionId','project','cwd','timestamp','data'].includes(key))|| + Buffer.byteLength(JSON.stringify(body))>256*1024||hasImage(body.data))return {...retained,completed:false,status:'held' as const,code:'SDK_SOURCE_FORMAT_UNSUPPORTED'}; + if(!['eventId','hookType','sessionId','project','cwd','timestamp'].every(key=>typeof body[key]==='string'&&(body[key] as string).length>0)|| + !/^[A-Za-z0-9_.:-]{8,128}$/.test(body.eventId as string))return {...retained,completed:false,status:'held' as const,code:'SDK_SOURCE_IDENTITY_INVALID'}; + const timestamp=Date.parse(body.timestamp as string); + if(!Number.isFinite(timestamp)||timestamp=Date.UTC(2100,0,1))return {...retained,completed:false,status:'held' as const,code:'SDK_SOURCE_IDENTITY_INVALID'}; + const key=createHash('sha256').update(body.project+'\0'+body.sessionId+'\0'+body.eventId).digest('hex'); + observationId='obs_'+timestamp.toString(36)+'_'+key.slice(0,12); + const existing=await store.retain(target,body.eventId as string,body); + if(existing.sourceId!==sourceId||existing.conflict)return {...retained,completed:false,status:'held' as const,code:'SOURCE_CONFLICT'}; + if(existing.completed)return {...retained,completed:true,status:'duplicate' as const,sdkReplyConfirmed:true}; + await store.markSending(target,sourceId,body); + }catch(error){return {...retained,completed:false,status:'held' as const,code:safeCode(error)};} + let response:V2PostResult; + try{response=await post(body);} + catch{ + try{await store.hold(target,sourceId,'SDK_RESTORE_OUTCOME_UNCERTAIN');}catch{} + return {...retained,completed:false,status:'held' as const,code:'SDK_RESTORE_OUTCOME_UNCERTAIN'}; + } + const value=response.body&&typeof response.body==='object'?response.body as Record:{}; + const confirmed=(response.status===200||response.status===201)&&value.eventId===body.eventId&&value.observationId===observationId&& + value.sourceRestore==='sdk-v1'&&value.completion==='sdk'&&value.completed===true&&value.durable===false&& + (value.status==='restored'||value.status==='duplicate'); + if(!confirmed){ + try{await store.hold(target,sourceId,'SDK_RESTORE_OUTCOME_UNCERTAIN');}catch{} + return {...retained,completed:false,status:'held' as const,code:'SDK_RESTORE_REPLY_UNVERIFIED'}; + } + try{await store.acknowledge(target,sourceId);} + catch{return {...retained,completed:false,status:'held' as const,code:'SDK_ACK_STATE_UNCONFIRMED',sdkReplyConfirmed:true};} + return {...retained,completed:true,status:value.status as 'restored'|'duplicate',sdkReplyConfirmed:true}; +} + export async function restoreSdkCaptureV2(target:string,sourceId:string,_post:V2Post,options:V2Options={}){ try{ const body=await createStore(options).load(target,sourceId); diff --git a/src/capture/sdk-reconciliation.ts b/src/capture/sdk-reconciliation.ts index 4406b1e994..b129037577 100644 --- a/src/capture/sdk-reconciliation.ts +++ b/src/capture/sdk-reconciliation.ts @@ -25,7 +25,7 @@ export function sdkReconciliationInvocation(input:unknown):Selection{ return body; } -export function createSdkCaptureReconciliation(kv:StateKV,options:{admitted:()=>boolean;indexEvidence:(observation:CompressedObservation)=>{keyword:{present:boolean;sessionId?:string};vectorSessionId?:string};audit:(selection:{key:string;requestId:string})=>Promise;now?:()=>number}){ +export function createSdkCaptureReconciliation(kv:StateKV,options:{admitted:()=>boolean;verifyCaptureVectorPending?:(id:string,sessionId:string,pending:unknown)=>Promise;indexEvidence:(observation:CompressedObservation)=>{keyword:{present:boolean;sessionId?:string};vectorSessionId?:string};audit:(selection:{key:string;requestId:string})=>Promise;now?:()=>number}){ const now=()=>new Date((options.now??Date.now)()).toISOString(); function admitted(){ if(kv.durability?.completion!=='sdk')throw fail('SDK_RECONCILIATION_UNSUPPORTED'); @@ -50,7 +50,11 @@ export function createSdkCaptureReconciliation(kv:StateKV,options:{admitted:()=> if(done?.state==='deleted')throw fail('SDK_RECONCILIATION_DELETED'); if(done&&(done.key!==rec.key||done.eventId!==rec.eventId||done.sessionId!==rec.sessionId||done.observationId!==rec.observationId||done.state!==undefined&&done.state!=='completed'))throw fail('SDK_RECONCILIATION_RECEIPT_CONFLICT'); if(!sdkObservationMatches(obs,rec.payload,rec.key,rec.observationId,rec.eventId))throw fail('SDK_RECONCILIATION_SOURCE_CONFLICT'); - if(retry!=null||pending!=null||clear!=null)throw fail('SDK_RECONCILIATION_INDEX_PENDING'); + if(retry!=null||clear!=null)throw fail('SDK_RECONCILIATION_INDEX_PENDING'); + const pendingVerified=options.verifyCaptureVectorPending + ? await options.verifyCaptureVectorPending(rec.observationId,rec.sessionId,pending) : pending==null; + if(!pendingVerified)throw fail('SDK_RECONCILIATION_INDEX_PENDING'); + admitted(); if(typeof obs!.title!=='string'||typeof obs!.narrative!=='string')throw fail('SDK_RECONCILIATION_SOURCE_CONFLICT'); const observedIndex=options.indexEvidence(obs!); const indexEvidence:CaptureSdkIndexEvidence={...observedIndex,vectorSessionId:observedIndex.vectorSessionId!,keyword:{...observedIndex.keyword,applicability:observationIndexEligible(obs!)?'required':obs!.title===''?(obs!.narrative===''?'excluded_empty_title_and_narrative':'excluded_empty_title'):'excluded_empty_narrative'}}; diff --git a/src/capture/sdk-spool-v2.ts b/src/capture/sdk-spool-v2.ts index e3882c5d4e..f01e3e2f9e 100644 --- a/src/capture/sdk-spool-v2.ts +++ b/src/capture/sdk-spool-v2.ts @@ -12,12 +12,15 @@ const BLOCK = 4096; const INPUT_LIMIT = 8 * MiB; const ORIGINAL_LIMIT = 256 * MiB; const QUEUE_LIMIT = 5 * MiB; +const QUEUE_TRANSITION_RESERVE = 2 * BLOCK; +const ACK_TRANSITION_RESERVE = 64 * 1024 + BLOCK; const CHUNK_LIMIT = 256 * 1024; const ID = /^[a-f0-9]{64}$/; type Owner = { pid: number; start: string; boot: string }; type Ref = { v: 2; targetHash: string; sourceId: string; state: 'pending' | 'sending' | 'held'; attempts: number; nextAttemptAt: number; owner?: Owner; reason?: 'RETRY' | 'HELD' | 'UNKNOWN_SEND' }; type Manifest = { v: 2; sourceId: string; targetHash: string; eventId: string; bytes: number; hash: string; chunks: Array<{ name: string; bytes: number; hash: string }>; conflict: boolean; completed: boolean }; -export type SdkSpoolV2Options = { dir?: string; maxOriginalBytes?: number; maxQueueBytes?: number; maxInputBytes?: number; chunkBytes?: number; onPhase?: (phase: string) => void }; +// Optional deadline uses performance.now(); omitted callers retain the 150ms lock wait. +export type SdkSpoolV2Options = { dir?: string; deadline?: number; maxOriginalBytes?: number; maxQueueBytes?: number; maxInputBytes?: number; chunkBytes?: number; onPhase?: (phase: string) => void }; export type RetainedSdkSource = { sourceId: string; duplicate: boolean; conflict: boolean; completed?: boolean; body: Record }; function fail(code: string): never { throw Object.assign(new Error(code), { code }); } @@ -118,29 +121,35 @@ function limit(value: number | undefined, maximum: number): number { /** Linux-only durable source storage. The lock covers all targets and every quota decision. * Source/staging files are never reclaimed. Disk charges are conservative logical bytes, * rounded to 4 KiB and at least stat.blocks*512; they are not filesystem hard quotas. - * The queue includes a 4 KiB lease plus queue/root directory charges (12 KiB minimum). + * The queue includes a 4 KiB lease plus queue/root/workspace directory charges (16 KiB for initialized stores). * Writes additionally reserve a parent-directory block, and 256 bytes per new chunk-directory entry. + * Admission cannot consume transition workspace: existing refs may use an additional 8 KiB, + * and completed manifests an additional 68 KiB. Preserved crash temporaries count toward both. * /usr/bin/flock is required; no lock fallback, lease deletion, or source cleanup exists. */ export class SdkSpoolV2 { readonly dir: string; private readonly originals: string; private readonly queue: string; private readonly lease: string; + private readonly queueWorkspace: string; private readonly maxOriginalBytes: number; private readonly maxQueueBytes: number; private readonly maxInputBytes: number; private readonly chunkBytes: number; private readonly onPhase?: (phase: string) => void; + private readonly deadline?: number; constructor(options: SdkSpoolV2Options = {}) { const path = options.dir ?? process.env.AGENTMEMORY_SDK_CAPTURE_V2_DIR ?? join(homedir(), '.local', 'share', 'agentmemory', 'capture-spool-sdk-v2'); if (!isAbsolute(path)) fail('STORE_OPTIONS_INVALID'); - this.dir = resolve(path); this.originals = join(this.dir, 'originals'); this.queue = join(this.dir, 'queue'); this.lease = join(this.queue, 'lease.lock'); + this.dir = resolve(path); this.originals = join(this.dir, 'originals'); this.queue = join(this.dir, 'queue'); this.lease = join(this.queue, 'lease.lock'); this.queueWorkspace = join(this.queue, '.transitions'); this.maxOriginalBytes = limit(options.maxOriginalBytes, ORIGINAL_LIMIT); this.maxQueueBytes = limit(options.maxQueueBytes, QUEUE_LIMIT); if (this.maxOriginalBytes < BLOCK || this.maxQueueBytes < 3 * BLOCK) fail('STORE_OPTIONS_INVALID'); this.maxInputBytes = limit(options.maxInputBytes, INPUT_LIMIT); this.chunkBytes = limit(options.chunkBytes, CHUNK_LIMIT); this.onPhase = options.onPhase; + if (options.deadline !== undefined && (!Number.isFinite(options.deadline) || options.deadline < 0)) fail('STORE_OPTIONS_INVALID'); + this.deadline = options.deadline; } private phase(name: string): void { this.onPhase?.(name); } private init(): void { @@ -154,24 +163,43 @@ export class SdkSpoolV2 { private rawUsage(): { originalBytes: number; queueBytes: number } { return { originalBytes: measure(this.originals), queueBytes: measure(this.queue) + BLOCK }; } - private budget(kind: 'original' | 'queue', additional: number): void { - const usage = this.rawUsage(); - if (kind === 'original' && usage.originalBytes + additional > this.maxOriginalBytes) fail('ORIGINALS_FULL'); - if (kind === 'queue' && usage.queueBytes + additional > this.maxQueueBytes) fail('QUEUE_FULL'); + private budget(kind: 'original' | 'queue', additional: number, transition = false): void { + const usage = kind === 'original' ? measure(this.originals) : measure(this.queue) + BLOCK; + const maximum = kind === 'original' ? this.maxOriginalBytes : this.maxQueueBytes; + const reserve = transition ? kind === 'original' ? ACK_TRANSITION_RESERVE : QUEUE_TRANSITION_RESERVE : 0; + if (usage + additional > maximum + reserve) fail(kind === 'original' ? 'ORIGINALS_FULL' : 'QUEUE_FULL'); + } + private prepareQueueWorkspace(): void { + try { checkDirectory(this.queueWorkspace); return; } + catch (error) { if (code(error) !== 'ENOENT') throw error; } + // Bootstrap includes both the new directory and possible queue-parent growth. + // Legacy saturated layouts may safely refuse; never reclaim their files. + this.budget('queue', 2 * BLOCK, true); + mkdirSync(this.queueWorkspace, { mode: 0o700 }); syncDirectory(this.queue); } private writeFile(path: string, bytes: Buffer | string): void { const fd = openSync(path, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600); try { privateStat(fstatSync(fd), false); writeFileSync(fd, bytes); fsyncSync(fd); } finally { closeSync(fd); } } - private atomicJson(path: string, value: unknown, kind?: 'original' | 'queue', beforePublish?: string): void { + private atomicJson(path: string, value: unknown, kind?: 'original' | 'queue', beforePublish?: string, transition = false): void { const bytes = Buffer.from(canonical(value)); - if (kind) this.budget(kind, charge(bytes.length) + BLOCK); - const temp = path + '.' + randomUUID() + '.tmp'; + if (kind && bytes.length > (kind === 'queue' ? BLOCK : 64 * 1024)) fail('SOURCE_UNVERIFIED'); + const stagedQueue = kind === 'queue' && transition; + if (stagedQueue) { + this.prepareQueueWorkspace(); + // A prior crash remains evidence. No overwrite, reuse, or automatic cleanup. + if (readdirSync(this.queueWorkspace).length) fail('QUEUE_FULL'); + } + if (kind) this.budget(kind, charge(bytes.length) + BLOCK, transition); + const temp = stagedQueue ? join(this.queueWorkspace, 'pending.tmp') : path + '.' + randomUUID() + '.tmp'; this.writeFile(temp, bytes); if (beforePublish) this.phase(beforePublish); + if (stagedQueue) this.phase('queue_transition_staged'); renameSync(temp, path); syncDirectory(join(path, '..')); + if (stagedQueue) syncDirectory(this.queueWorkspace); } private async locked(operation: () => T): Promise { + const operationDeadline = this.deadline; let fd: number | undefined; try { if (process.platform !== 'linux') fail('STORE_LOCK_UNSUPPORTED'); @@ -180,8 +208,9 @@ export class SdkSpoolV2 { const stat = fstatSync(fd); privateStat(stat, false); if (stat.size !== 0) fail('STORE_UNSAFE'); - const deadline = performance.now() + 150; + const deadline = operationDeadline ?? performance.now() + 150; while (true) { + if (operationDeadline !== undefined && performance.now() >= deadline) fail('STORE_BUSY'); // flock inherits this open file description; its exit does not release our fd's lock. const lock = spawnSync('/usr/bin/flock', ['--exclusive', '--nonblock', '--conflict-exit-code', '73', '3'], { stdio: ['ignore', 'ignore', 'ignore', fd], timeout: 150 }); if (lock.error || lock.signal || lock.status !== 0 && lock.status !== 73) fail('STORE_LOCK_UNSUPPORTED'); @@ -190,6 +219,9 @@ export class SdkSpoolV2 { await new Promise(resolveWait => setTimeout(resolveWait, 5)); } fsyncSync(fd); syncDirectory(this.queue); + // Prepare empty stores before admission, so workspace is charged from day one. + if (readdirSync(this.queue).every(name => name === 'lease.lock')) this.prepareQueueWorkspace(); + if (operationDeadline !== undefined && performance.now() >= deadline) fail('STORE_BUSY'); return operation(); } catch (error) { if (typeof code(error) === 'string' && /^(SOURCE_|ORIGINALS_|QUEUE_|STORE_)/.test(code(error)!)) throw error; @@ -201,12 +233,18 @@ export class SdkSpoolV2 { } private key(target: string, eventId: string): string { return digest(digest(target) + '\0' + eventId); } private conflictKey(target: string, eventId: string, body: Record): string { return digest(this.key(target, eventId) + '\0conflict\0' + sourceFingerprint(body)); } - private source(target: string, sourceId: string): { manifest: Manifest; body: Record } { + private manifest(target: string, sourceId: string): Manifest { if (!ID.test(sourceId)) fail('SOURCE_UNVERIFIED'); try { const folder = join(this.originals, sourceId); checkDirectory(folder); const manifest = JSON.parse(regularRead(join(folder, 'manifest.json'), 64 * 1024).toString()) as Manifest; if (!manifest || manifest.v !== 2 || manifest.sourceId !== sourceId || manifest.targetHash !== digest(target) || typeof manifest.eventId !== 'string' || !Number.isSafeInteger(manifest.bytes) || manifest.bytes <= 0 || manifest.bytes > this.maxInputBytes || !ID.test(manifest.hash) || typeof manifest.conflict !== 'boolean' || typeof manifest.completed !== 'boolean' || !Array.isArray(manifest.chunks) || !manifest.chunks.length || manifest.chunks.length > this.maxInputBytes) fail('SOURCE_UNVERIFIED'); + return manifest; + } catch (error) { if (code(error) === 'STORE_UNSAFE') throw error; fail('SOURCE_UNVERIFIED'); } + } + private source(target: string, sourceId: string): { manifest: Manifest; body: Record } { + try { + const manifest = this.manifest(target, sourceId), folder = join(this.originals, sourceId); let bytes = 0; const chunks = manifest.chunks.map((chunk, index) => { if (chunk.name !== String(index).padStart(6, '0') + '.chunk' || !Number.isSafeInteger(chunk.bytes) || chunk.bytes <= 0 || chunk.bytes > CHUNK_LIMIT || !ID.test(chunk.hash)) fail('SOURCE_UNVERIFIED'); @@ -225,7 +263,7 @@ export class SdkSpoolV2 { return { manifest, body }; } catch (error) { if (code(error) === 'STORE_UNSAFE') throw error; fail('SOURCE_UNVERIFIED'); } } - async retain(target: string, eventId: string, body: Record): Promise { + private prepareSource(target: string, eventId: string, body: Record): { normalized: Record; bytes: Buffer } { let normalized: Record; let bytes: Buffer; try { normalized = JSON.parse(JSON.stringify(body)); @@ -233,34 +271,51 @@ export class SdkSpoolV2 { bytes = Buffer.from(canonical(normalized)); } catch { fail('SOURCE_UNVERIFIED'); } if (bytes.length > this.maxInputBytes) fail('SOURCE_TOO_LARGE'); - return this.locked(() => { - let sourceId = this.key(target, eventId); let conflict = false; + return { normalized, bytes }; + } + private retainSource(target: string, eventId: string, normalized: Record, bytes: Buffer): RetainedSdkSource { + let sourceId = this.key(target, eventId); let conflict = false; + try { + lstatSync(join(this.originals, sourceId)); + const prior = this.source(target, sourceId); + if (sameSource(prior.body, normalized)) return { sourceId, duplicate: true, conflict: false, completed: prior.manifest.completed, body: prior.body }; + conflict = true; sourceId = this.conflictKey(target, eventId, normalized); try { lstatSync(join(this.originals, sourceId)); - const prior = this.source(target, sourceId); - if (sameSource(prior.body, normalized)) return { sourceId, duplicate: true, conflict: false, completed: prior.manifest.completed, body: prior.body }; - conflict = true; sourceId = this.conflictKey(target, eventId, normalized); - try { - lstatSync(join(this.originals, sourceId)); - const priorConflict = this.source(target, sourceId); - if (!priorConflict.manifest.conflict || !sameSource(priorConflict.body, normalized)) fail('SOURCE_UNVERIFIED'); - return { sourceId, duplicate: true, conflict: true, completed: false, body: priorConflict.body }; - } catch (error) { if (code(error) !== 'ENOENT') throw error; } + const priorConflict = this.source(target, sourceId); + if (!priorConflict.manifest.conflict || !sameSource(priorConflict.body, normalized)) fail('SOURCE_UNVERIFIED'); + return { sourceId, duplicate: true, conflict: true, completed: false, body: priorConflict.body }; } catch (error) { if (code(error) !== 'ENOENT') throw error; } - const chunks: Buffer[] = []; - for (let offset = 0; offset < bytes.length; offset += this.chunkBytes) chunks.push(bytes.subarray(offset, offset + this.chunkBytes)); - const manifest: Manifest = { v: 2, sourceId, targetHash: digest(target), eventId, bytes: bytes.length, hash: digest(bytes), chunks: chunks.map((value, index) => ({ name: String(index).padStart(6, '0') + '.chunk', bytes: value.length, hash: digest(value) })), conflict, completed: false }; - const manifestBytes = canonical(manifest); - if (Buffer.byteLength(manifestBytes) > 64 * 1024) fail('SOURCE_TOO_LARGE'); - this.budget('original', BLOCK + charge((chunks.length + 1) * 256) + charge(Buffer.byteLength(manifestBytes)) + chunks.reduce((sum, value) => sum + charge(value.length), 0)); - const stage = join(this.originals, '.stage-' + randomUUID()); - mkdirSync(stage, { mode: 0o700 }); syncDirectory(this.originals); - for (let index = 0; index < chunks.length; index++) { this.writeFile(join(stage, manifest.chunks[index].name), chunks[index]); this.phase('chunk_fsynced'); } - this.writeFile(join(stage, 'manifest.json'), manifestBytes); this.phase('manifest_fsynced'); - syncDirectory(stage); this.phase('source_before_publish'); - renameSync(stage, join(this.originals, sourceId)); syncDirectory(this.originals); this.phase('source_published'); - const verified = this.source(target, sourceId); - return { sourceId, duplicate: false, conflict, completed: false, body: verified.body }; + } catch (error) { if (code(error) !== 'ENOENT') throw error; } + const chunks: Buffer[] = []; + for (let offset = 0; offset < bytes.length; offset += this.chunkBytes) chunks.push(bytes.subarray(offset, offset + this.chunkBytes)); + const manifest: Manifest = { v: 2, sourceId, targetHash: digest(target), eventId, bytes: bytes.length, hash: digest(bytes), chunks: chunks.map((value, index) => ({ name: String(index).padStart(6, '0') + '.chunk', bytes: value.length, hash: digest(value) })), conflict, completed: false }; + const manifestBytes = canonical(manifest); + if (Buffer.byteLength(manifestBytes) > 64 * 1024) fail('SOURCE_TOO_LARGE'); + this.budget('original', BLOCK + charge((chunks.length + 1) * 256) + charge(Buffer.byteLength(manifestBytes)) + chunks.reduce((sum, value) => sum + charge(value.length), 0)); + const stage = join(this.originals, '.stage-' + randomUUID()); + mkdirSync(stage, { mode: 0o700 }); syncDirectory(this.originals); + for (let index = 0; index < chunks.length; index++) { this.writeFile(join(stage, manifest.chunks[index].name), chunks[index]); this.phase('chunk_fsynced'); } + this.writeFile(join(stage, 'manifest.json'), manifestBytes); this.phase('manifest_fsynced'); + syncDirectory(stage); this.phase('source_before_publish'); + renameSync(stage, join(this.originals, sourceId)); syncDirectory(this.originals); this.phase('source_published'); + const verified = this.source(target, sourceId); + return { sourceId, duplicate: false, conflict, completed: false, body: verified.body }; + } + async retain(target: string, eventId: string, body: Record): Promise { + const { normalized, bytes } = this.prepareSource(target, eventId, body); + return this.locked(() => this.retainSource(target, eventId, normalized, bytes)); + } + async retainAndEnqueue(target: string, eventId: string, body: Record): Promise<{ retained: RetainedSdkSource; referenceAcknowledged: boolean; error?: unknown }> { + const { normalized, bytes } = this.prepareSource(target, eventId, body); + return this.locked(() => { + const retained = this.retainSource(target, eventId, normalized, bytes); + if (retained.conflict) return { retained, referenceAcknowledged: false }; + if (retained.completed) return { retained, referenceAcknowledged: true }; + // Source publication is not rolled back if queue admission or I/O fails. + try { this.enqueueSource(target, retained.sourceId); } + catch (error) { return { retained, referenceAcknowledged: false, error }; } + return { retained, referenceAcknowledged: true }; }); } async load(target: string, sourceId: string): Promise> { @@ -275,15 +330,16 @@ export class SdkSpoolV2 { return ref; } catch (error) { if (code(error) === 'ENOENT' || code(error) === 'STORE_UNSAFE') throw error; fail('SOURCE_UNVERIFIED'); } } - private writeRef(ref: Ref): void { this.atomicJson(this.refPath(ref.sourceId), ref, 'queue'); this.phase('reference_published'); } + private writeRef(ref: Ref, transition = true): void { this.atomicJson(this.refPath(ref.sourceId), ref, 'queue', undefined, transition); this.phase('reference_published'); } + private enqueueSource(target: string, sourceId: string): void { + const { manifest } = this.source(target, sourceId); + if (manifest.conflict) fail('SOURCE_CONFLICT'); + if (manifest.completed) return; + try { this.readRef(target, sourceId); return; } catch (error) { if (code(error) !== 'ENOENT') throw error; } + this.writeRef({ v: 2, targetHash: digest(target), sourceId, state: 'pending', attempts: 0, nextAttemptAt: 0 }, false); + } async enqueue(target: string, sourceId: string): Promise { - return this.locked(() => { - const { manifest } = this.source(target, sourceId); - if (manifest.conflict) fail('SOURCE_CONFLICT'); - if (manifest.completed) return; - try { this.readRef(target, sourceId); return; } catch (error) { if (code(error) !== 'ENOENT') throw error; } - this.writeRef({ v: 2, targetHash: digest(target), sourceId, state: 'pending', attempts: 0, nextAttemptAt: 0 }); - }); + return this.locked(() => this.enqueueSource(target, sourceId)); } async pending(target: string): Promise> { return this.locked(() => { @@ -293,22 +349,40 @@ export class SdkSpoolV2 { if (peek.targetHash !== digest(target)) continue; const ref = this.readRef(target, file.slice(0, -4)); if (ref.state === 'sending' && !alive(ref.owner!)) { ref.state = 'held'; ref.reason = 'UNKNOWN_SEND'; delete ref.owner; this.writeRef(ref); } - try { if (this.source(target, ref.sourceId).manifest.completed) continue; } catch { ref.state = 'held'; } + try { if (this.manifest(target, ref.sourceId).completed) continue; } catch { ref.state = 'held'; } result.push(ref); } return result.map(({ sourceId, state, attempts, nextAttemptAt }) => ({ sourceId, state, attempts, nextAttemptAt })); }); } - async markSending(target: string, sourceId: string): Promise { + async markSending(target: string, sourceId: string, expectedHeldSource?: Record): Promise { + await this.claimSource(target, sourceId, expectedHeldSource); + } + async claimForSend(target: string, sourceId: string, sendDeadline?: number): Promise> { + return this.claimSource(target, sourceId, undefined, sendDeadline); + } + private async claimSource(target: string, sourceId: string, expectedHeldSource?: Record, sendDeadline?: number): Promise> { + if (sendDeadline !== undefined && !Number.isFinite(sendDeadline)) fail('STORE_OPTIONS_INVALID'); return this.locked(() => { const source = this.source(target, sourceId); if (source.manifest.conflict || source.manifest.completed) fail('SOURCE_HELD'); const ref = this.readRef(target, sourceId); - if (ref.state !== 'pending') fail('SOURCE_HELD'); - if (ref.nextAttemptAt > Date.now()) fail('SOURCE_BACKOFF'); + if (expectedHeldSource) { + if (ref.state !== 'held' || ref.owner || !isDeepStrictEqual(source.body, expectedHeldSource)) fail('SOURCE_HELD'); + } else { + if (ref.state === 'sending') { + if (alive(ref.owner!)) fail('SOURCE_SENDING'); + ref.state = 'held'; ref.reason = 'UNKNOWN_SEND'; delete ref.owner; this.writeRef(ref); + } + if (ref.state !== 'pending') fail('SOURCE_HELD'); + if (ref.nextAttemptAt > Date.now()) fail('SOURCE_BACKOFF'); + } const owner = ownerIdentity(); if (!owner) fail('STORE_BUSY'); + // Only a ref verified pending under this lease can be reported as spooled. + if (sendDeadline !== undefined && performance.now() >= sendDeadline) fail('SOURCE_SEND_BUDGET_EXHAUSTED'); ref.state = 'sending'; ref.owner = owner; this.writeRef(ref); this.phase('send_intent_fsynced'); + return source.body; }); } async acknowledge(target: string, sourceId: string): Promise { @@ -317,9 +391,9 @@ export class SdkSpoolV2 { if (source.manifest.conflict) fail('SOURCE_CONFLICT'); if (!source.manifest.completed) { const ref = this.readRef(target, sourceId); - if (ref.state !== 'sending') fail('SOURCE_HELD'); + if (ref.state !== 'sending' || !isDeepStrictEqual(ref.owner, ownerIdentity())) fail('SOURCE_HELD'); source.manifest.completed = true; - this.atomicJson(join(this.originals, sourceId, 'manifest.json'), source.manifest, 'original', 'ack_before_publish'); + this.atomicJson(join(this.originals, sourceId, 'manifest.json'), source.manifest, 'original', 'ack_before_publish', true); this.phase('ack_fsynced'); } try { unlinkSync(this.refPath(sourceId)); syncDirectory(this.queue); } catch (error) { if (code(error) !== 'ENOENT') throw error; } @@ -328,13 +402,13 @@ export class SdkSpoolV2 { async retry(target: string, sourceId: string, _reason: string): Promise { return this.locked(() => { const ref = this.readRef(target, sourceId); - if (ref.state !== 'sending' || !ref.owner || !alive(ref.owner)) fail('SOURCE_HELD'); + if (ref.state !== 'sending' || !isDeepStrictEqual(ref.owner, ownerIdentity())) fail('SOURCE_HELD'); ref.attempts = Math.min(1_000_000, ref.attempts + 1); ref.nextAttemptAt = Date.now() + Math.min(300_000, 1000 * 2 ** Math.min(9, ref.attempts - 1)); ref.state = 'pending'; ref.reason = 'RETRY'; delete ref.owner; this.writeRef(ref); }); } async hold(target: string, sourceId: string, _reason: string): Promise { - return this.locked(() => { const ref = this.readRef(target, sourceId); ref.state = 'held'; ref.reason = 'HELD'; delete ref.owner; this.writeRef(ref); }); + return this.locked(() => { const ref = this.readRef(target, sourceId); if (ref.state !== 'sending' || !isDeepStrictEqual(ref.owner, ownerIdentity())) fail('SOURCE_HELD'); ref.state = 'held'; ref.reason = 'HELD'; delete ref.owner; this.writeRef(ref); }); } async usage(): Promise<{ originalBytes: number; queueBytes: number }> { return this.locked(() => this.rawUsage()); } } diff --git a/src/capture/uncertain-inspection.ts b/src/capture/uncertain-inspection.ts new file mode 100644 index 0000000000..69d418d6fa --- /dev/null +++ b/src/capture/uncertain-inspection.ts @@ -0,0 +1,29 @@ +import {createHash} from 'node:crypto'; +import {isDeepStrictEqual} from 'node:util'; +import type {StateKV} from '../state/kv.js'; +import type {InboxRecord} from '../functions/capture.js'; +import type {CompressedObservation} from '../types.js'; +import {KV} from '../state/schema.js'; +import {isValidEventId} from './event-id.js'; +import {isCaptureKey} from './event-record.js'; +import {capturePayloadDigest} from './receipt-digest.js'; +import {sdkObservationMatches} from './sdk-source-restore.js'; +export type UncertainInspection={key:string;eventId:string;sourceDigest:string;project:string;sessionId:string;cwd:string;hookType:string;timestamp:string}; +export function validUncertainInspection(input:unknown):input is UncertainInspection{ + if(!input||typeof input!=='object'||Array.isArray(input))return false; + const x=input as Record,keys=['key','eventId','sourceDigest','project','sessionId','cwd','hookType','timestamp']; + return Object.keys(x).length===keys.length&&Object.keys(x).every(k=>keys.includes(k))&&isCaptureKey(x.key)&&typeof x.eventId==='string'&&isValidEventId(x.eventId)&&typeof x.sourceDigest==='string'&&/^[a-f0-9]{64}$/.test(x.sourceDigest)&&['project','sessionId','cwd','hookType','timestamp'].every(k=>typeof x[k]==='string'&&(x[k] as string).length>0&&(x[k] as string).length<=2048)&&Number.isFinite(Date.parse(x.timestamp as string)); +} +export async function inspectUncertainCapture(kv:StateKV,input:UncertainInspection){ + if(!validUncertainInspection(input))throw Error('INVALID_UNCERTAIN_INSPECTION'); + const result=(outcome:string)=>({outcome,readOnly:true,diskVerified:false,synced:false,originalReplay:false,retryAuthorized:false,atomicSnapshot:false}); + const rec=structuredClone(await kv.get(KV.captureInbox,input.key)); + if(!rec)return result('source_not_found'); + const derived='cap_'+createHash('sha256').update(`${rec.project}\0${rec.sessionId}\0${rec.eventId}`).digest('hex').slice(0,40); + if(rec.eventSource!=='client'||rec.key!==input.key||derived!==input.key||rec.eventId!==input.eventId||!rec.payload||rec.project!==input.project||rec.sessionId!==input.sessionId||rec.hookType!==input.hookType||rec.payload.project!==input.project||rec.payload.sessionId!==input.sessionId||rec.payload.cwd!==input.cwd||rec.payload.hookType!==input.hookType||rec.payload.timestamp!==input.timestamp||capturePayloadDigest(rec.payload)!==input.sourceDigest)return result('source_conflict'); + const read=()=>kv.get(KV.observations(rec.sessionId),rec.observationId); + const first=structuredClone(await read()),again=structuredClone(await read()),recAgain=structuredClone(await kv.get(KV.captureInbox,input.key)); + if(!isDeepStrictEqual(rec,recAgain)||!isDeepStrictEqual(first,again))return result('changed_during_inspection'); + if(!first)return result('observation_not_found'); + return result(sdkObservationMatches(first,rec.payload,rec.key,rec.observationId,rec.eventId)?'observation_matches':'observation_conflict'); +} diff --git a/src/cli.ts b/src/cli.ts index be7ad222bb..d3c9e8d7ea 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -1,4 +1,5 @@ #!/usr/bin/env node +import {loadStateWorkerArtifact} from "./cli/state-worker-artifact.js"; import { createHash } from "node:crypto"; import {isStorageWorkCompleted} from './ops/storage-completion.mjs'; @@ -647,6 +648,7 @@ type ComposeEngineState = { composePid?: number; enginePid?: number; workerBinDir?: string; + stateExecutable?: {path:string;sha256:string}; workerFailure?: string; workerVersions?: typeof COMPOSE_WORKER_VERSIONS | { local: true }; }; @@ -1251,7 +1253,17 @@ function composeEnginePid(state: ComposeEngineState): number | undefined { return composeChildProcesses(state).find(({ args }) => args.includes(" --config ") && args.includes(state.namespace))?.pid; } -async function terminateComposeChildren(state: ComposeEngineState): Promise { +async function terminateComposeChildren(state: ComposeEngineState, captured?: Array<{pid:number;start:string}>): Promise { + if (captured) { + // The daemon may already be gone: never rediscover/adopt reparented PIDs. + for (const child of captured) { + if (!pidAlive(child.pid)) continue; + try { if (nativeProcessStart(child.pid) !== child.start) return false; } + catch { if (!pidAlive(child.pid)) continue; return false; } + if (!(await signalAndWait(child.pid, "SIGTERM", 5_000, true))) return false; + } + return true; + } if (IS_WINDOWS) return true; const groups = [...new Set(composeChildProcesses(state).map((child) => child.pgid).filter((pgid) => pgid !== state.composePid))]; let stopped = true; @@ -1274,11 +1286,14 @@ async function terminateComposeChildren(state: ComposeEngineState): Promise { + const artifactDir=process.env["AGENTMEMORY_STATE_WORKER_ARTIFACT"]?.trim(); + const artifact=artifactDir?loadStateWorkerArtifact(artifactDir):undefined; const workerBinDir = process.env["AGENTMEMORY_WORKER_BIN_DIR"]?.trim() || undefined; if (workerBinDir && (!existsSync(resolve(process.cwd(), workerBinDir)) || !["state", "http"].every((name) => existsSync(join(resolve(process.cwd(), workerBinDir), name))))) { throw new Error(`AGENTMEMORY_WORKER_BIN_DIR must contain executable state and http binaries: ${workerBinDir}`); } + if(getStateBackend()==="file"&&!artifact&&!workerBinDir)throw new Error("File-backed Compose requires the patched State artifact: set AGENTMEMORY_STATE_WORKER_ARTIFACT to the verified release artifact directory (or AGENTMEMORY_WORKER_BIN_DIR for existing local workers). Official registry State lacks the required ACK/epoch/seal contract."); const envPath = process.env["AGENTMEMORY_ENV_FILE"] || join(homedir(), ".agentmemory", ".env"); const namespace = composeNamespace(getRestPort()); const projectRoot = resolve(__dirname, ".."); @@ -1286,9 +1301,12 @@ async function startIiiBin(iiiBin: string): Promise { // Fast, actionable diagnosis before Compose attempts to bind its managed listeners. // Only reachable local TCP listeners count; failures to connect mean free/closed. await assertComposePortsFree([ports.restPort, ports.viewerPort, ports.enginePort]); + const stateExecutable = artifact ?? (workerBinDir ? {path:realpathSync(join(workerBinDir,"state")),sha256:createHash("sha256").update(readFileSync(join(workerBinDir,"state"))).digest("hex")} : undefined); const compose = startComposeDaemon({ + stateWorkerExecutable: artifact?.path, ports, dataDir: dataDirResolution.dataDir, + runtimeDir: dirname(engineStatePath()), projectRoot, nodeBin: process.execPath, homeDir: homedir(), @@ -1322,6 +1340,7 @@ async function startIiiBin(iiiBin: string): Promise { logStartOffset: compose.logStartOffset, binPath: iiiBin, composePid: compose.child.pid, + ...(stateExecutable ? {stateExecutable} : {}), ...(workerBinDir ? { workerBinDir: resolve(process.cwd(), workerBinDir) } : {}), }; writeComposeState(engineStatePath(), { @@ -1389,16 +1408,20 @@ function nativeCompatPidMatches(pid: number, state: NativeEngineState): boolean } catch { return false; } } +class UnresolvedNativeLifecycleError extends Error { + readonly exitCode = 78; +} + function assertNativeCompatStartupOwnership(): void { const previous = readEngineState(); if (previous?.kind === "native" && previous.compatVersion && previous.workerIdentity) { const pinned = previous.workerIdentity; let sameWorker = false; try { sameWorker = nativeCompatWorkerIdentityMatches(pinned, pinned.pid, nativeWorkerIdentity(pinned.pid)); } catch {} - if (sameWorker) throw new Error("Unresolved prior lifecycle has a live pinned native worker; no process was started and state was preserved."); + if (sameWorker) throw new UnresolvedNativeLifecycleError("Unresolved prior lifecycle has a live pinned native worker; no process was started and state was preserved."); } if (existsSync(engineStatePath()) || existsSync(enginePidfilePath()) || existsSync(workerPidfilePath())) { - throw new Error("Unresolved prior lifecycle evidence must be reconciled before native startup; no process was started and state was preserved."); + throw new UnresolvedNativeLifecycleError("Unresolved prior lifecycle evidence must be reconciled before native startup; no process was started and state was preserved."); } } @@ -2877,7 +2900,16 @@ async function startServerForDemo(): Promise<() => Promise> { return async () => { if (importedStartupWorker && readWorkerPidfile() === process.pid) clearWorkerPidfile(); - if (startedEngine) await cleanupFailedStartup(); + if (!startedEngine) return; + if (IS_WINDOWS && spawnedComposeDaemon) { + const state = readEngineState(); + if (state?.kind !== "compose" || state.composePid !== spawnedComposeDaemon.pid) { + throw new Error("Demo Compose lifecycle state no longer matches the daemon it started. Lifecycle state was preserved."); + } + await stopComposeRuntime(state, true); + return; + } + await cleanupFailedStartup(); }; } @@ -3302,6 +3334,43 @@ function findEnginePidsByPort(port: number): number[] { } } +async function stopComposeRuntime(state: ComposeEngineState, running: boolean): Promise { + const pid = state.composePid; + const ownedDaemonAlive = pid !== undefined && pidAlive(pid); + if (ownedDaemonAlive && !composePidMatches(pid, state)) { + throw new Error(`Recorded Compose pid ${pid} does not match the saved daemon identity; refusing to invoke Compose control or signal it. Lifecycle state was preserved.`); + } + if (running && !ownedDaemonAlive) { + throw new Error("The REST endpoint is responding, but the recorded Compose daemon is not alive. Refusing to stop or clear an engine that may be unrelated; restore the owning daemon or inspect the endpoint manually."); + } + if (running && ownedDaemonAlive) { + try { + await composeCall({ engineUrl: state.engineUrl, namespace: state.namespace, functionId: "compose::down", file: state.file, timeoutMs: composeDownTimeoutMs() }); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + if (!message.includes("ENGINE_RESTART_REQUIRED")) { + throw new Error(`compose::down failed for ${state.file}: ${message}. Lifecycle state was preserved.`); + } + p.log.warn("Compose requested an engine restart while stopping; terminating its verified owning daemon to finish shutdown."); + } + } + if (ownedDaemonAlive) { + const children = platform() === "linux" ? composeChildPids(state).filter(pidAlive).map((child) => ({pid:child,start:nativeProcessStart(child)})) : undefined; + // Revalidate after down; let Compose itself retire its engine. Killing the + // engine first makes a clean stop race with Compose's ENGINE_EXITED failure. + if (pidAlive(pid) && !composePidMatches(pid, state)) throw new Error("Compose daemon identity changed before stop; lifecycle state was preserved."); + if (!(await signalAndWait(pid, "SIGTERM", 20_000, true))) { + throw new Error(`Worker Compose daemon pid ${pid} did not stop; lifecycle state was preserved. Inspect ${state.logFile}.`); + } + if (!(await terminateComposeChildren(state, children))) { + throw new Error(`One or more child workers of Compose ${pid} did not stop; lifecycle state was preserved.`); + } + } + clearEnginePidfile(); + clearWorkerPidfile(); + clearEngineState(); +} + async function runStop(): Promise { const savedState = readEngineState(); if (savedState?.kind === "native" && Object.hasOwn(savedState, "compatVersion") && savedState.compatVersion !== NATIVE_COMPAT_VERSION) { @@ -3329,41 +3398,7 @@ async function runStop(): Promise { const force = args.includes("--force"); if (state?.kind === "compose") { - const ownedDaemonAlive = Boolean(state.composePid && pidAlive(state.composePid)); - if (ownedDaemonAlive && !composePidMatches(state.composePid!, state)) { - p.log.error(`Recorded Compose pid ${state.composePid} does not match the saved daemon identity; refusing to invoke Compose control or signal it. Lifecycle state was preserved.`); - process.exit(1); - } - if (running && !ownedDaemonAlive) { - p.log.error("The REST endpoint is responding, but the recorded Compose daemon is not alive. Refusing to stop or clear an engine that may be unrelated; restore the owning daemon or inspect the endpoint manually."); - process.exit(1); - } - if (running && ownedDaemonAlive) { - try { - await composeCall({ engineUrl: state.engineUrl, namespace: state.namespace, functionId: "compose::down", file: state.file, timeoutMs: composeDownTimeoutMs() }); - } catch (err) { - const message = err instanceof Error ? err.message : String(err); - if (!message.includes("ENGINE_RESTART_REQUIRED")) { - p.log.error(`compose::down failed for ${state.file}: ${message}. Lifecycle state was preserved.`); - process.exit(1); - } - p.log.warn("Compose requested an engine restart while stopping; terminating its verified owning daemon to finish shutdown."); - } - } - if (ownedDaemonAlive) { - const childrenStopped = await terminateComposeChildren(state); - if (!childrenStopped) { - p.log.error(`One or more child workers of Compose ${state.composePid} did not stop; lifecycle state was preserved.`); - process.exit(1); - } - if (!(await signalAndWait(state.composePid!, "SIGTERM", 20_000))) { - p.log.error(`Worker Compose daemon pid ${state.composePid} did not stop; lifecycle state was preserved. Inspect ${state.logFile}.`); - process.exit(1); - } - } - clearEnginePidfile(); - clearWorkerPidfile(); - clearEngineState(); + await stopComposeRuntime(state, running); p.outro("Stopped Worker Compose, engine, and agentmemory workers. Data was preserved."); return; } @@ -3835,7 +3870,7 @@ async function runConsole(): Promise { p.note([ `AgentMemory viewer: ${getViewerUrl()} (start agentmemory first if it is offline).`, "Use agentmemory status for runtime health and worker status.", - "Engine console migration: https://github.com/iii-hq/iii/blob/iii/v0.25.0/docs/using-iii/console.mdx", + "Engine console migration: https://github.com/iii-hq/iii/blob/iii/v0.25.1/docs/using-iii/console.mdx", ].join("\n"), "Console migration"); } @@ -3894,7 +3929,8 @@ async function unknownCommand(): Promise { const handler = commands[first] ?? (first && !first.startsWith("-") ? unknownCommand : main); handler().catch(async (err) => { p.log.error(err instanceof Error ? err.message : String(err)); - const exitCode = await runStartupCleanup(1).catch(() => 1); + const failureCode = err instanceof UnresolvedNativeLifecycleError ? err.exitCode : 1; + const exitCode = await runStartupCleanup(failureCode).catch(() => 1); if (args.includes("--native-compat") && importedStartupWorker) { process.exitCode = exitCode; return; diff --git a/src/cli/compose-control.ts b/src/cli/compose-control.ts index e113819ecc..8bb26e28cb 100644 --- a/src/cli/compose-control.ts +++ b/src/cli/compose-control.ts @@ -1,4 +1,4 @@ -import { chmodSync, closeSync, fstatSync, mkdirSync, openSync, readFileSync, readSync, readdirSync, realpathSync, writeFileSync } from "node:fs"; +import { renameSync, unlinkSync, chmodSync, closeSync, fstatSync, mkdirSync, openSync, readFileSync, readSync, readdirSync, realpathSync, writeFileSync } from "node:fs"; import { dirname } from "node:path"; import { spawn, type ChildProcess } from "node:child_process"; import { createConnection } from "node:net"; @@ -297,10 +297,16 @@ export function writeComposeState(path: string, state: { binPath: string; enginePid?: number; workerBinDir?: string; + stateExecutable?: {path:string;sha256:string}; workerFailure?: string; workerVersions: typeof COMPOSE_WORKER_VERSIONS | { local: true }; }): void { mkdirSync(dirname(path), { recursive: true, mode: 0o700 }); - writeFileSync(path, `${JSON.stringify(state)}\n`, { encoding: "utf8", mode: 0o600 }); - chmodSync(path, 0o600); + const temporary = `${path}.${process.pid}.${Date.now()}.${Math.random().toString(16).slice(2)}.tmp`; + try { + writeFileSync(temporary, `${JSON.stringify(state)}\n`, { encoding: "utf8", mode: 0o600, flag: "wx" }); + renameSync(temporary, path); + } finally { + try { unlinkSync(temporary); } catch (error) { if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; } + } } diff --git a/src/cli/compose-project.ts b/src/cli/compose-project.ts index 687284f219..e0ab9dfa42 100644 --- a/src/cli/compose-project.ts +++ b/src/cli/compose-project.ts @@ -17,8 +17,8 @@ export interface StateBackendOptions { /** Registry versions are pinned here so every generated project uses one reviewed set. */ export const COMPOSE_WORKER_VERSIONS = { - state: "0.22.20", - http: "0.22.7", + state: "0.22.22", + http: "0.22.8", } as const; export const COMPOSE_STOP_TIMEOUT_SECONDS = 20; @@ -78,7 +78,8 @@ const AGENTMEMORY_WORKER_ENV_ALLOWLIST = new Set([ "AGENTMEMORY_CAPTURE_SPOOL", "AGENTMEMORY_CAPTURE_SPOOL_DIR", "AGENTMEMORY_CAPTURE_SPOOL_MAX_BYTES", "AGENTMEMORY_CAPTURE_SPOOL_MAX_AGE_HOURS", "AGENTMEMORY_IMPORT_ROOT", "AGENTMEMORY_EXPORT_ROOT", "AGENTMEMORY_GRAPH_WEIGHT", - "SUMMARIZE_CHUNK_SIZE", "SUMMARIZE_CHUNK_CONCURRENCY", "CONSOLIDATION_DECAY_DAYS", + "SUMMARIZE_CHUNK_SIZE", "SUMMARIZE_CHUNK_CONCURRENCY", "AGENTMEMORY_SUMMARY_TIMEOUT_MS", + "CONSOLIDATION_DECAY_DAYS", "CONSOLIDATION_INTERVAL_MS", "AGENTMEMORY_CONSOLIDATION_COOLDOWN_MS", "AGENTMEMORY_FOLLOWUP_WINDOW_SECONDS", "GRAPH_EXTRACTION_BATCH_SIZE", "AGENTMEMORY_LLM_NOTHINK", "AGENTMEMORY_REFLECT", "AGENTMEMORY_IMAGE_EMBEDDINGS", "AGENTMEMORY_IMAGE_STORE_MAX_BYTES", @@ -87,6 +88,11 @@ const AGENTMEMORY_WORKER_ENV_ALLOWLIST = new Set([ "AGENTMEMORY_VECTOR_BACKFILL", "AGENTMEMORY_SLOTS", "AGENTMEMORY_PROVIDER", "AGENTMEMORY_VERBOSE", "OBSIDIAN_AUTO_EXPORT", "AUTO_FORGET_ENABLED", "AUTO_FORGET_INTERVAL_MS", "LESSON_DECAY_ENABLED", "INSIGHT_DECAY_ENABLED", "REBUILD_EMBED_BATCH_SIZE", "RERANK_ENABLED", + "AGENTMEMORY_LOCAL_DISPATCH", + "OPENAI_API_KEY_FOR_LLM", "AGENTMEMORY_OLLAMA_NATIVE_CHAT", + "SUMMARIZE_MIN_NEW_OBS", "SUMMARIZE_MAX_AGE_MS", "SUMMARIZE_BATCH_OBS", + "SUMMARIZE_INPUT_CHARS", "SUMMARIZE_RECOVERY_INTERVAL_MS", + "AGENTMEMORY_HOOK_INJECT_CONTEXT", "AGENTMEMORY_FEEDBACK_DIR", "AGENTMEMORY_RECEIPT_DIR", ]); const COMPOSE_RESERVED_ENV = new Set([ @@ -169,6 +175,9 @@ function yamlString(value: string): string { if (/[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f]/.test(value)) { throw new Error("A Compose path or configuration value contains an unsupported control character."); } + if (/[\r\n\u0080-\u009f\u2028\u2029]/.test(value)) { + return JSON.stringify(value).replace(/[\u0080-\u009f\u2028\u2029]/g, (char) => `\\u${char.charCodeAt(0).toString(16).padStart(4, "0")}`); + } return `'${value.replace(/'/g, "''")}'`; } @@ -234,6 +243,8 @@ function ensureEngineTraceStorageConfig(configDir: string, dataDir: string): voi } export interface ComposeProjectOptions { + runtimeDir?: string; + stateWorkerExecutable?: string; ports: EngineRuntimePorts; dataDir: string; projectRoot: string; @@ -269,11 +280,11 @@ export function renderWorkerCompose(options: ComposeProjectOptions): string { } const workerSource = (name: keyof typeof COMPOSE_WORKER_VERSIONS): string => - workerBinDir + workerBinDir || (name === "state" && options.stateWorkerExecutable) ? `path://${localWorkerProjectDir(paths, name)}` : `package://api.workers.iii.dev/${name}`; const workerScript = (name: keyof typeof COMPOSE_WORKER_VERSIONS): string => - workerBinDir ? (windows ? win32.join(workerBinDir, name) : join(workerBinDir, name)) : ""; + name === "state" && options.stateWorkerExecutable ? options.stateWorkerExecutable : workerBinDir ? (windows ? win32.join(workerBinDir, name) : join(workerBinDir, name)) : ""; const filteredEnvFile = options.envFile ? join(paths.directory, ".agentmemory-filtered.env") : undefined; @@ -300,10 +311,10 @@ export function renderWorkerCompose(options: ComposeProjectOptions): string { lines.push( " state:", ` worker: ${yamlString(workerSource("state"))}`, - ...(!workerBinDir ? [` version: ${yamlString(COMPOSE_WORKER_VERSIONS.state)}`] : []), + ...(!stateScript ? [` version: ${yamlString(COMPOSE_WORKER_VERSIONS.state)}`] : []), " config_name: state", ...restartPolicyLines(" ", restartEnabled), - ...(workerBinDir ? [` scripts:`, ` run: ${yamlString(windows ? windowsWorkerScript(stateScript) : shellCommand(stateScript, []))}`] : []), + ...(stateScript ? [` scripts:`, ` run: ${yamlString(windows ? windowsWorkerScript(stateScript) : shellCommand(stateScript, []))}`] : []), " config_override:", " adapter:", " name: kv", @@ -368,7 +379,7 @@ export function renderWorkerCompose(options: ComposeProjectOptions): string { ` HOME: ${yamlString(options.homeDir)}`, ` PATH: ${yamlString(windows ? windowsWorkerPath(options.nodeBin, options.workerEnvironment?.PATH ?? process.env.PATH) : `${dirname(options.nodeBin)}:/usr/local/bin:/usr/bin:/bin`)}`, ` AGENTMEMORY_DATA_DIR: ${yamlString(options.dataDir)}`, - ` AGENTMEMORY_RUNTIME_DIR: ${yamlString(options.dataDir)}`, + ` AGENTMEMORY_RUNTIME_DIR: ${yamlString(options.runtimeDir ?? options.dataDir)}`, ...(filteredEnvFile ? [` AGENTMEMORY_ENV_FILE: ${yamlString(filteredEnvFile)}`] : []), ...(options.stateBackend?.kind === "redis" && options.redisUrl ? [` AGENTMEMORY_REDIS_URL: ${yamlString(options.redisUrl)}`, " AGENTMEMORY_STATE_BACKEND: 'redis'"] : []), ` III_ENGINE_PORT: ${yamlString(String(enginePort))}`, @@ -388,11 +399,13 @@ export function writeWorkerCompose(options: ComposeProjectOptions): { file: stri mkdirSync(paths.stateDir, { recursive: true, mode: 0o700 }); ensureEngineTraceStorageConfig(paths.configDir, options.dataDir); mkdirSync(join(options.dataDir, "observability", "traces"), { recursive: true, mode: 0o700 }); - if (options.workerBinDir) { - const binDir = options.workerBinDirIsAbsolute + if (options.workerBinDir || options.stateWorkerExecutable) { + const binDir = options.workerBinDir ? (options.workerBinDirIsAbsolute ? resolve(options.workerBinDir) - : localWorkerDirectory(options.workerBinDir, options.cwd ?? process.cwd(), options.projectRoot); + : localWorkerDirectory(options.workerBinDir, options.cwd ?? process.cwd(), options.projectRoot)) : null; for (const name of Object.keys(COMPOSE_WORKER_VERSIONS) as Array) { + const binary = name === "state" && options.stateWorkerExecutable ? options.stateWorkerExecutable : binDir ? join(binDir,name) : null; + if (!binary) continue; const projectDir = localWorkerProjectDir(paths, name); mkdirSync(projectDir, { recursive: true, mode: 0o700 }); const manifest = [ @@ -400,7 +413,7 @@ export function writeWorkerCompose(options: ComposeProjectOptions): { file: stri `name: ${name}`, "language: rust", "scripts:", - ` start: ${yamlString(join(binDir, name))}`, + ` start: ${yamlString(binary)}`, "", ].join("\n"); writeFileSync(join(projectDir, "iii.worker.yaml"), manifest, { encoding: "utf8", mode: 0o600 }); diff --git a/src/cli/connect/claude-code.ts b/src/cli/connect/claude-code.ts index 1882be86cc..271468bf2e 100644 --- a/src/cli/connect/claude-code.ts +++ b/src/cli/connect/claude-code.ts @@ -1,7 +1,8 @@ -import { existsSync, mkdirSync, writeFileSync } from "node:fs"; +import { existsSync, mkdirSync, statSync, writeFileSync } from "node:fs"; import { homedir } from "node:os"; -import { join } from "node:path"; +import { isAbsolute, join, normalize, resolve } from "node:path"; import * as p from "@clack/prompts"; +import { isEnvFileValue } from "../../config.js"; import type { ConnectAdapter, ConnectOptions, ConnectResult } from "./types.js"; import { AGENTMEMORY_MCP_BLOCK, @@ -18,9 +19,18 @@ import { type HookManifest, } from "./codex-hooks.js"; -const CLAUDE_DIR = join(homedir(), ".claude"); -const CLAUDE_JSON = join(homedir(), ".claude.json"); -const CLAUDE_SETTINGS = join(CLAUDE_DIR, "settings.json"); +function claudeConfigPaths() { + const rawConfigDir = process.env["CLAUDE_CONFIG_DIR"]; + const configDir = rawConfigDir?.trim() ? rawConfigDir : undefined; + const dir = configDir ? normalize(configDir) : join(homedir(), ".claude"); + return { + rawConfigDir, + configDir, + dir, + mcp: configDir ? join(dir, ".claude.json") : join(homedir(), ".claude.json"), + settings: join(dir, "settings.json"), + }; +} type ClaudeMcpEntry = typeof AGENTMEMORY_MCP_BLOCK; type ClaudeConfig = { @@ -45,10 +55,48 @@ export const adapter: ConnectAdapter = { "→ Using MCP. Hooks are also available — see https://github.com/rohitg00/agentmemory#claude-code-one-block-paste-it.", detect(): boolean { - return existsSync(CLAUDE_DIR); + const { configDir, dir } = claudeConfigPaths(); + let detected: boolean; + try { + detected = statSync(dir, { throwIfNoEntry: false })?.isDirectory() ?? false; + } catch (err) { + const code = (err as NodeJS.ErrnoException).code; + if (code !== "EACCES" && code !== "EPERM") throw err; + p.log.warn( + `Cannot inspect Claude Code configuration directory ${JSON.stringify(dir)} (${code}). Check directory permissions and retry.`, + ); + return false; + } + if (!detected && configDir) { + p.log.warn( + `Claude Code configuration path ${JSON.stringify(configDir)} is not an existing directory. Create it or correct CLAUDE_CONFIG_DIR; check for unintended leading or trailing whitespace.`, + ); + } + return detected; }, async install(opts: ConnectOptions): Promise { + const { rawConfigDir, configDir, dir: CLAUDE_DIR, mcp: CLAUDE_JSON } = claudeConfigPaths(); + const fromEnvFile = isEnvFileValue("CLAUDE_CONFIG_DIR"); + if (configDir && fromEnvFile) { + p.log.warn( + `CLAUDE_CONFIG_DIR was loaded only from agentmemory's env file. Launch Claude Code with the same CLAUDE_CONFIG_DIR value ${JSON.stringify(configDir)} so it reads the same MCP and hook configuration.`, + ); + } else if (rawConfigDir && !configDir && !fromEnvFile) { + p.log.warn( + "CLAUDE_CONFIG_DIR contains only whitespace. AgentMemory will use the default Claude Code profile. Unset CLAUDE_CONFIG_DIR before launching Claude Code so it uses the same profile.", + ); + } + if (configDir && configDir !== configDir.trim()) { + p.log.warn( + `CLAUDE_CONFIG_DIR contains leading or trailing whitespace. Using the path exactly as provided: ${JSON.stringify(configDir)}. If this is unintended, correct CLAUDE_CONFIG_DIR and rerun this command.`, + ); + } + if (configDir && !isAbsolute(configDir)) { + p.log.warn( + `CLAUDE_CONFIG_DIR is relative and resolves to ${JSON.stringify(resolve(configDir))} from this working directory. Set the same absolute path for agentmemory and Claude Code so they use the same profile.`, + ); + } const existing = readJsonSafe(CLAUDE_JSON); const next: ClaudeConfig = existing ? { ...existing } : {}; const servers: Record = { @@ -116,6 +164,7 @@ export const adapter: ConnectAdapter = { }; function installClaudeHooks(opts: ConnectOptions): ConnectResult { + const { dir: CLAUDE_DIR, settings: CLAUDE_SETTINGS } = claudeConfigPaths(); let pluginRoot: string; try { pluginRoot = findPluginRoot(); diff --git a/src/cli/doctor-diagnostics.ts b/src/cli/doctor-diagnostics.ts index 8cc84f7845..b7bea3868f 100644 --- a/src/cli/doctor-diagnostics.ts +++ b/src/cli/doctor-diagnostics.ts @@ -16,7 +16,14 @@ // agentmemory doctor --all # apply every available fix without prompting (CI) // agentmemory doctor --dry-run # show what each fix WOULD do; execute nothing -import { detectLlmProviderKind, embeddingOptInHint, LLM_PROVIDER_KEYS, llmOptInHint, parseEnvContent } from "../config.js"; +import { + detectLlmProviderKind, + embeddingOptInHint, + isRealProviderKey, + LLM_PROVIDER_KEYS, + llmOptInHint, + parseEnvContent, +} from "../config.js"; export type DiagnosticStatus = { ok: boolean; @@ -91,17 +98,6 @@ export type DiagnosticId = (typeof DIAGNOSTIC_IDS)[number]; // Pure helpers (no I/O) — exported for direct unit testing. // --------------------------------------------------------------------------- -/** Common placeholder values shipped in .env.example. */ -const PLACEHOLDER_VALUES = new Set([ - "", - "your-key-here", - "sk-ant-...", - "sk-...", - "changeme", - "todo", - "xxx", -]); - const PROVIDER_KEY_NAMES = [ "ANTHROPIC_API_KEY", "OPENAI_API_KEY", @@ -115,25 +111,12 @@ export const parseEnvFile = parseEnvContent; /** Returns the list of provider keys that look real (non-placeholder). */ export function realProviderKeys(env: Record): string[] { - return PROVIDER_KEY_NAMES.filter((k) => { - const v = (env[k] ?? "").trim(); - if (!v) return false; - if (PLACEHOLDER_VALUES.has(v.toLowerCase())) return false; - // Reject values that are just dots/placeholders like "xxxx-xxxx". - if (/^x+$/i.test(v.replace(/[-_]/g, ""))) return false; - return true; - }); + return PROVIDER_KEY_NAMES.filter((k) => isRealProviderKey(env, k)); } /** Returns the list of provider key NAMES that exist but are placeholders. */ export function placeholderProviderKeys(env: Record): string[] { - return PROVIDER_KEY_NAMES.filter((k) => { - const v = (env[k] ?? "").trim(); - if (!v) return false; - if (PLACEHOLDER_VALUES.has(v.toLowerCase())) return true; - if (/^x+$/i.test(v.replace(/[-_]/g, ""))) return true; - return false; - }); + return PROVIDER_KEY_NAMES.filter((k) => (env[k] ?? "").trim() !== "" && !isRealProviderKey(env, k)); } /** @@ -210,7 +193,6 @@ export function buildDiagnostics(effects: DoctorEffects): Diagnostic[] { for (const [k, v] of Object.entries(effects.processEnv())) { if (v !== undefined) env[k] = v; } - for (const k of placeholderProviderKeys(env)) delete env[k]; if (detectLlmProviderKind(env) !== "llm") { const hint = llmOptInHint(env); if (hint) return { ok: false, detail: hint }; diff --git a/src/cli/engine-install.ts b/src/cli/engine-install.ts index 83ee9ff434..790258cf53 100644 --- a/src/cli/engine-install.ts +++ b/src/cli/engine-install.ts @@ -5,12 +5,25 @@ import { basename, join } from "node:path"; export const III_INSTALL_CONNECT_TIMEOUT_S = 15; export const III_INSTALL_MAX_TIME_S = 300; +export const III_INSTALL_RETRIES = 3; +export const III_INSTALL_RETRY_DELAY_S = 2; +export const III_INSTALL_RETRY_MAX_TIME_S = 120; export const III_INSTALL_SPAWN_TIMEOUT_MS = - (III_INSTALL_CONNECT_TIMEOUT_S + III_INSTALL_MAX_TIME_S + 30) * 1000; + (III_INSTALL_CONNECT_TIMEOUT_S + III_INSTALL_MAX_TIME_S + III_INSTALL_RETRY_MAX_TIME_S + 30) * 1000; +const RETRYABLE_HTTP_STATUS = new Set([408, 429, 500, 502, 503, 504]); type ChecksumTable = Readonly>>>; export const III_RELEASE_SHA256: ChecksumTable = { + "0.25.1": { + "iii-aarch64-apple-darwin.tar.gz": "f7eff803540436a4964958048abc0083e638f53b500d737819f62913c270f141", + "iii-x86_64-apple-darwin.tar.gz": "d93511a72336c19a6028a18f43819e6b3c49522589fc6303f0a0880b3f02d8ae", + "iii-x86_64-unknown-linux-gnu.tar.gz": "0f486f0fb175013fdebd482735228a47de5fd5014d375a9720cb4f958341208a", + "iii-aarch64-unknown-linux-gnu.tar.gz": "5938c6d2f25df8e301c058ef0bcfc9f858840c8b77d3eff1fa1bc83f67bd136c", + "iii-armv7-unknown-linux-gnueabihf.tar.gz": "d872197aa19312e84250ee40eb38a6d225600939bc2aaaac04a91d2671c47f14", + "iii-x86_64-pc-windows-msvc.zip": "a4648976760d76a8c7c457b36256d3381cd4a0332c288f2b434fe2b2d02d746a", + "iii-aarch64-pc-windows-msvc.zip": "f2ef00b6363cfb97d31f7356cce88ea0771b8fa0e8e5bbd561c0210337b83b64", + }, "0.25.0": { "iii-aarch64-apple-darwin.tar.gz": "5f23087a41ce1cf86f9859d2f430d1809a1e048094e69b1df0cdf1ebefc2a985", "iii-x86_64-apple-darwin.tar.gz": "e2ac4ef109ddbf459b4683171c6b5b9c8366b16dac2e1f8c4fe08b4a4e19d49a", @@ -35,7 +48,11 @@ export function sha256File(path: string): string { } function curlTimeoutFlags(): string { - return `--connect-timeout ${III_INSTALL_CONNECT_TIMEOUT_S} --max-time ${III_INSTALL_MAX_TIME_S}`; + return `--connect-timeout ${III_INSTALL_CONNECT_TIMEOUT_S} --max-time ${III_INSTALL_MAX_TIME_S} ${curlRetryFlags()}`; +} + +function curlRetryFlags(): string { + return `--retry ${III_INSTALL_RETRIES} --retry-delay ${III_INSTALL_RETRY_DELAY_S} --retry-max-time ${III_INSTALL_RETRY_MAX_TIME_S}`; } export function iiiArchivePath(binDir: string): string { @@ -59,7 +76,7 @@ export function iiiManualInstallCommand( const verify = sha256 ? ` && printf '%s %s\\n' "${sha256}" "$archive" | { sha256sum -c - 2>/dev/null || shasum -a 256 -c -; }` : ""; - return `mkdir -p "${binDir}" && archive="$(mktemp "${binDir}/.iii-download.XXXXXX")" && { curl -fL --connect-timeout ${III_INSTALL_CONNECT_TIMEOUT_S} -o "$archive" "${releaseUrl}"${verify} && tar -xzf "$archive" -C "${binDir}"; code=$?; rm -f "$archive"; (exit $code); } && chmod +x "${binPath}"`; + return `mkdir -p "${binDir}" && archive="$(mktemp "${binDir}/.iii-download.XXXXXX")" && { curl -fL --connect-timeout ${III_INSTALL_CONNECT_TIMEOUT_S} ${curlRetryFlags()} -o "$archive" "${releaseUrl}"${verify} && tar -xzf "$archive" -C "${binDir}"; code=$?; rm -f "$archive"; (exit $code); } && chmod +x "${binPath}"`; } export function describeInstallFailure(result: { @@ -207,6 +224,12 @@ export function windowsCurlDownloadCommand( String(III_INSTALL_CONNECT_TIMEOUT_S), "--max-time", String(III_INSTALL_MAX_TIME_S), + "--retry", + String(III_INSTALL_RETRIES), + "--retry-delay", + String(III_INSTALL_RETRY_DELAY_S), + "--retry-max-time", + String(III_INSTALL_RETRY_MAX_TIME_S), "-o", dest, url, @@ -244,6 +267,8 @@ export async function downloadToFile( dest: string, env: NodeJS.ProcessEnv = process.env, run: ZipCommandRunner = runInstallCommand, + request: typeof fetch = fetch, + sleep: (ms: number) => Promise = (ms) => new Promise((done) => setTimeout(done, ms)), ): Promise { if (proxyConfigured(env)) { const step = windowsCurlDownloadCommand(url, dest, env); @@ -251,12 +276,39 @@ export async function downloadToFile( if (result.status !== 0) throw new Error(describeInstallFailure(result)); return; } - const res = await fetch(url, { - redirect: "follow", - signal: AbortSignal.timeout(III_INSTALL_MAX_TIME_S * 1000), - }); - if (!res.ok) throw new Error(`download failed with HTTP ${res.status} ${res.statusText}`.trim()); - writeFileSync(dest, Buffer.from(await res.arrayBuffer())); + const deadline = Date.now() + III_INSTALL_RETRY_MAX_TIME_S * 1000; + for (let attempt = 0; ; attempt++) { + let res: Response | undefined; + let body: Buffer | undefined; + let failure: unknown; + try { + res = await request(url, { redirect: "follow", signal: AbortSignal.timeout(III_INSTALL_MAX_TIME_S * 1000) }); + if (res.ok) body = Buffer.from(await res.arrayBuffer()); + else await res.body?.cancel().catch(() => {}); + } catch (error) { + failure = error; + } + if (body) { + writeFileSync(dest, body); + return; + } + if (res && !res.ok) { + failure = new Error(`download failed with HTTP ${res.status} ${res.statusText}`.trim()); + if (!RETRYABLE_HTTP_STATUS.has(res.status)) throw failure; + } + const delayMs = retryDelayMs(res); + if (attempt >= III_INSTALL_RETRIES || Date.now() + delayMs >= deadline) throw failure; + await sleep(delayMs); + } +} + +function retryDelayMs(res: Response | undefined): number { + const fallback = III_INSTALL_RETRY_DELAY_S * 1000; + const header = res?.ok ? null : res?.headers.get("retry-after"); + if (!header) return fallback; + const seconds = Number(header); + const ms = Number.isFinite(seconds) ? seconds * 1000 : Date.parse(header) - Date.now(); + return Number.isFinite(ms) ? Math.max(ms, fallback) : fallback; } export function runInstallCommand(command: string, args: string[]): Promise { diff --git a/src/cli/state-worker-artifact.ts b/src/cli/state-worker-artifact.ts new file mode 100644 index 0000000000..cfb870986d --- /dev/null +++ b/src/cli/state-worker-artifact.ts @@ -0,0 +1,16 @@ +import {readFileSync,realpathSync,statSync} from 'node:fs'; +import {join} from 'node:path'; +import {createHash} from 'node:crypto'; +export const STATE_ARTIFACT_PIN={artifactVersion:'0.22.22-daniel.1',upstreamCommit:'a8ed4889f54cb34cb9ffdb6e44190484d90ba950',patchSha256:'504bbb8a3b6b2eda1d4714f33c89f8ad47aa3af74dc61a99ee5efec76b9f4224'}; +export function loadStateWorkerArtifact(directory:string){ + const root=realpathSync(directory),manifestPath=join(root,'manifest.json'); + if(statSync(manifestPath).size>16384)throw Error('STATE_ARTIFACT_MANIFEST_INVALID'); + const m=JSON.parse(readFileSync(manifestPath,'utf8')); + if(m.schemaVersion!==1||m.profile!=='release'||m.binary!=='state'||m.engineVersion!=='0.25.1'||Object.entries(STATE_ARTIFACT_PIN).some(([k,v])=>m[k]!==v))throw Error('STATE_ARTIFACT_PROVENANCE_MISMATCH'); + if(process.platform!=='linux'||!['x86_64-unknown-linux-gnu','aarch64-unknown-linux-gnu'].includes(m.target)||!m.target.startsWith(process.arch==='x64'?'x86_64-':process.arch==='arm64'?'aarch64-':'unsupported'))throw Error('STATE_ARTIFACT_PLATFORM_MISMATCH'); + const path=realpathSync(join(root,'state')),stat=statSync(path); + if(!stat.isFile()||stat.size>256*1024*1024||!(stat.mode&0o111))throw Error('STATE_ARTIFACT_BINARY_INVALID'); + const sha256=createHash('sha256').update(readFileSync(path)).digest('hex'); + if(sha256!==m.binarySha256)throw Error('STATE_ARTIFACT_HASH_MISMATCH'); + return {path,sha256}; +} diff --git a/src/config.ts b/src/config.ts index 53823f19c7..d0653bea67 100644 --- a/src/config.ts +++ b/src/config.ts @@ -41,6 +41,7 @@ function writeProviderNoticeOnce(text: string): void { // mutate the file between cases reset the module (clearing this via reload) or // call __resetEnvFileCache(). let envFileCache: Record | undefined; +const hydratedEnvValues = new Map(); function loadEnvFile(): Record { if (envFileCache) return envFileCache; @@ -86,6 +87,31 @@ function hasRealValue(v: string | undefined): v is string { return typeof v === "string" && v.trim().length > 0; } +const PLACEHOLDER_KEY_VALUES = new Set(["your-key-here", "changeme", "todo"]); + +function hasRealKey(value: string | undefined): boolean { + const v = value?.trim().toLowerCase(); + return ( + !!v && + !v.endsWith("...") && + !PLACEHOLDER_KEY_VALUES.has(v) && + !/^x+$/.test(v.replace(/[-_]/g, "")) + ); +} + +function hasRealOpenAIKey(env: Record): boolean { + return /^http:\/\//i.test(env["OPENAI_BASE_URL"]?.trim() ?? "") + ? !!env["OPENAI_API_KEY"]?.trim() + : hasRealKey(env["OPENAI_API_KEY"]); +} + +export function isRealProviderKey( + env: Record, + key: string, +): boolean { + return key === "OPENAI_API_KEY" ? hasRealOpenAIKey(env) : hasRealKey(env[key]); +} + // Hydrate ~/.agentmemory/.env into process.env at boot. loadEnvFile() is // otherwise only consumed via getMergedEnv(), which the many modules that // read raw process.env["X"] never call — so .env-only values were silently @@ -95,11 +121,19 @@ function hasRealValue(v: string | undefined): v is string { export function hydrateProcessEnvFromFile(): Record { const beforeHydration = { ...process.env }; for (const [k, v] of Object.entries(loadEnvFile())) { - if (process.env[k] === undefined) process.env[k] = v; + if (process.env[k] === undefined) { + process.env[k] = v; + hydratedEnvValues.set(k, v); + } } return beforeHydration; } +export function isEnvFileValue(key: string): boolean { + const hydrated = hydratedEnvValues.get(key); + return hydrated !== undefined && process.env[key] === hydrated; +} + // Warn once when a configured model is in the premium tier and likely to burn // money on background compression. Captured workload data shows ~$5/35h on // claude-sonnet-4 vs ~$0.46/35h on deepseek-v4-pro for the same compression @@ -152,7 +186,7 @@ function selectedLlmProviderValue(env: Record): stri } function llmProviderHasKey(env: Record, name: LlmProviderName): boolean { - return LLM_PROVIDER_KEYS[name].some((k) => hasRealValue(env[k])); + return LLM_PROVIDER_KEYS[name].some((k) => isRealProviderKey(env, k)); } export function envFileLabel(path: string = ENV_FILE): string { @@ -174,7 +208,7 @@ export function llmOptInHint(env: Record = getMerged if (env["AGENTMEMORY_ALLOW_AGENT_SDK"] === "true") return null; const withKey = LLM_PROVIDER_NAMES.filter((name) => llmProviderHasKey(env, name)); if (withKey.length === 0) return null; - const keys = withKey.flatMap((name) => LLM_PROVIDER_KEYS[name].filter((k) => hasRealValue(env[k]))); + const keys = withKey.flatMap((name) => LLM_PROVIDER_KEYS[name].filter((k) => isRealProviderKey(env, k))); return `Found ${keys.join(", ")}, but agentmemory sends nothing to an LLM provider until you choose one for it. To turn on LLM summaries and compression, add AGENTMEMORY_LLM_PROVIDER=${withKey[0]} to ${envFileLabel()}.`; } @@ -220,7 +254,7 @@ function detectProvider(env: Record): ProviderConfig { baseURL: env["ANTHROPIC_BASE_URL"], }; case "gemini": - if (!hasRealValue(env["GEMINI_API_KEY"])) { + if (!hasRealKey(env["GEMINI_API_KEY"])) { process.stderr.write( "[agentmemory] GOOGLE_API_KEY detected — treating as GEMINI_API_KEY. " + "Set GEMINI_API_KEY in ~/.agentmemory/.env to silence this warning.\n", @@ -317,6 +351,11 @@ export function getEnvVar(key: string): string | undefined { return getMergedEnv()[key]; } +export function getProviderKey(key: string): string | undefined { + const env = getMergedEnv(); + return isRealProviderKey(env, key) ? env[key] : undefined; +} + export function isSessionSweepEnabled(): boolean { return getMergedEnv()["AGENTMEMORY_SESSION_SWEEP_ENABLED"] !== "false"; } diff --git a/src/functions/audit.ts b/src/functions/audit.ts index f2832487b3..da4fef3c3c 100644 --- a/src/functions/audit.ts +++ b/src/functions/audit.ts @@ -4,6 +4,7 @@ import { isDeepStrictEqual } from "node:util"; import type { AuditEntry, AuditMigrationState, AuditQueryResult } from "../types.js"; import { KV, generateId } from "../state/schema.js"; import type { StateKV } from "../state/kv.js"; +import {collectAll} from "../state/kv.js"; import { logger } from "../logger.js"; import { withKeyedLock } from "../state/keyed-mutex.js"; import { getEnvVar, getAuditMigrateMaxBytes } from "../config.js"; @@ -75,7 +76,7 @@ export async function probeLegacyAuditFile(): Promise { export async function probeLegacyAuditScope(kv: StateKV): Promise { let rows: AuditEntry[]; try { - rows = await kv.list(KV.audit); + rows = await collectAll(kv, KV.audit); } catch { return { safe: false, reason: "unreadable" }; } @@ -285,7 +286,7 @@ export async function queryAudit( if (collected.length >= limit) break; if (toMs !== undefined && monthStartMs(month) > toMs) continue; if (fromMs !== undefined && monthEndMs(month) < fromMs) break; - const rows = await kv.list(KV.auditMonth(month)); + const rows = await collectAll(kv, KV.auditMonth(month)); for (const row of rows) addIfNew(row); } @@ -295,7 +296,7 @@ export async function queryAudit( if (collected.length < limit) { if (migrationState?.safeToListLegacy) { - const legacyRows = await kv.list(KV.audit); + const legacyRows = await collectAll(kv, KV.audit); for (const row of legacyRows) addIfNew(row); } } @@ -387,7 +388,7 @@ export async function startAuditMigration( if (marker?.status !== "copied") { let legacy: AuditEntry[]; try { - legacy = await kv.list(KV.audit); + legacy = await collectAll(kv, KV.audit); } catch (err) { logger.warn("audit migration could not read the legacy audit log", { error: err instanceof Error ? err.message : String(err), @@ -448,7 +449,7 @@ export async function startAuditMigration( let remaining: AuditEntry[]; try { - remaining = await kv.list(KV.audit); + remaining = await collectAll(kv, KV.audit); } catch (err) { logger.warn( "audit migration could not re-read the legacy audit log before deleting; it resumes on the next boot", @@ -539,7 +540,7 @@ export async function runAuditRetentionSweep( let droppedRows = 0; for (const month of toDrop) { - const rows = await kv.list(KV.auditMonth(month)); + const rows = await collectAll(kv, KV.auditMonth(month)); for (const row of rows.slice().reverse()) { await kv.delete(KV.auditMonth(month), row.id); droppedRows++; diff --git a/src/functions/auto-forget.ts b/src/functions/auto-forget.ts index 832ead7bda..b9d2fcb44f 100644 --- a/src/functions/auto-forget.ts +++ b/src/functions/auto-forget.ts @@ -2,7 +2,7 @@ import type { IIIClient } from "iii-sdk"; import { markCaptureEventDeleted } from "../capture/event-record.js"; import type { Memory, CompressedObservation, Session } from "../types.js"; import { KV } from "../state/schema.js"; -import { StateKV } from "../state/kv.js"; +import { StateKV, collectAll } from "../state/kv.js"; import { supersededAt } from "../state/memory-utils.js"; import { withKeyedLock } from "../state/keyed-mutex.js"; import { unindexObservationSession } from "../state/obs-index.js"; @@ -40,7 +40,7 @@ export function registerAutoForgetFunction(sdk: IIIClient, kv: StateKV): void { dryRun, }; - const memories = await kv.list(KV.memories); + const memories = await collectAll(kv, KV.memories); const deletedIds = new Set(); const facetIds = new Set(); try { diff --git a/src/functions/capture.ts b/src/functions/capture.ts index c53ce4c6cf..9e810679d5 100644 --- a/src/functions/capture.ts +++ b/src/functions/capture.ts @@ -1,9 +1,11 @@ +import { listCaptureInboxMetadata, listLocalCaptureResolutionKeys, projectCaptureInbox, committedInboxMetadata, type CaptureInboxMetadata } from '../capture/inbox-metadata.js'; +import {inspectUncertainCapture,validUncertainInspection} from "../capture/uncertain-inspection.js"; import { committedSdkResolution } from '../capture/sdk-resolution-record.js'; import {recordAudit} from "./audit.js"; import {requireSdkSourceWriter,sameSdkSource,sdkObservationMatches,sdkSourceKeys,sdkSourceFailure,assertSdkSourceClaim,assertSdkSourceLegacyProtection} from "../capture/sdk-source-restore.js"; import{createSdkCaptureReconciliation,sdkReconciliationInvocation}from'../capture/sdk-reconciliation.js'; import {createCaptureAutoRecovery,type CaptureAutoRecoveryOptions} from "../capture/auto-recovery.js"; -import {getSearchIndex,getVectorIndex,getKeywordRebuildEpoch,isKeywordRebuildInProgress} from "./search.js"; +import {getSearchIndex,getVectorIndex,getKeywordRebuildEpoch,isKeywordRebuildInProgress,verifyCaptureVectorPending} from "./search.js"; import {createCaptureIndexEvidence,captureIndexEvidenceInvocationQuery} from "../capture/index-evidence.js"; import { createOutboxMetadata, type OutboxMetadata } from "../capture/outbox-metadata.js"; import { registerCaptureReceipts } from "../capture/receipts.js"; @@ -28,7 +30,7 @@ import { } from "../capture/event-record.js"; import { captureReviewPage, recordCaptureFailure, resolveCaptureReview, reviewHistory } from "../capture/review.js"; import { createCaptureRetryJobs } from "../capture/retry-jobs.js"; -import { backgroundTask, backgroundPaused } from "../ops/background-control.mjs"; +import { backgroundTask, backgroundPaused, backgroundControl } from "../ops/background-control.mjs"; import { restoreIndexEntries,extractImage } from "./observe.js"; import { prunePromptLedgers } from "../capture/prompt-ledger.js"; import { scrubRecord,normalizeCaptureData } from "./privacy.js"; @@ -57,6 +59,7 @@ export interface InboxRecord { sdkRestoreState?:"pending"|"completed"; sdkResolutionId?:string; sdkResolutionState?:'pending'|'committed'; + deferredByStorageHold?: true; key: string; eventId: string; eventSource: "client" | "server"; @@ -83,7 +86,7 @@ export type CaptureResult = | { status: "accepted"; state: "completed" | "sdk_completed"; eventId: string; observationId: string; attempts: number } | { status: "accepted"; state: "retrying"; eventId: string; observationId: string; attempts: number; nextAttemptAt: string; error: string } | { status: "duplicate"; state: "completed" | "sdk_completed" | InboxState; eventId: string; observationId?: string; deduplicated: true } - | { status: "rejected"; eventId: string; error: string; retryable: boolean; observationId?: string }; + | { status: "rejected"; eventId: string; error: string; retryable: boolean; observationId?: string; reconciliationRequired?: true }; export interface CapturePolicy { maxAttempts: number; @@ -226,7 +229,7 @@ export interface CaptureController { sweep(): Promise<{ processed: number; recovered: number; deadLettered: number }>; prune(): Promise; drainLocalSpool(): Promise; - status(options?:{sdkProofPage?:CaptureSdkProofQuery}): Promise; + status(options?:{scanInbox?:boolean;sdkProofPage?:CaptureSdkProofQuery}): Promise; setAutomaticRecoveryEnabled(value:boolean):ReturnType["setEnabled"]>; progress(): CaptureProgress; durability(): SentMark | null; @@ -276,7 +279,7 @@ export function registerCaptureFunctions( let lastPruneAt: string | null = null; let lastBootDrain: CaptureStatus["lastBootDrain"] = null; let sweeping: Promise<{ processed: number; recovered: number; deadLettered: number }> | null = null; - let inboxSize: number | null = null; + let inboxKeys: Set | null = null; let activeInboxKeys: Set | null = null; let stopped = false; const timers: Array> = []; @@ -288,6 +291,7 @@ export function registerCaptureFunctions( } const trackMaintenance=(work:()=>Promise)=>{void trackWork(work).catch(()=>{});}; const sdkReconciliation=createSdkCaptureReconciliation(kv,{ + verifyCaptureVectorPending, admitted:()=>!stopped&&!backgroundPaused()&&(options.autoRecovery?.admitted()??true), indexEvidence:obs=>({keyword:{present:getSearchIndex().has(obs.id),sessionId:getSearchIndex().sessionOf(obs.id)},vectorSessionId:getVectorIndex()?.get(obs.id)?.sessionId}), audit:selection=>recordAudit(kv,'capture_sdk_reconciliation','api::capture-reconcile-sdk',[],{requestId:selection.requestId,completion:'sdk',stage:'verification_pending'}), @@ -299,8 +303,7 @@ export function registerCaptureFunctions( else if (activeInboxKeys && pendingSince === null) pendingSince = Date.now(); } - function refreshInbox(records: InboxRecord[]): void { - inboxSize = records.length; + function refreshInbox(records: Array): void { activeInboxKeys = new Set(records.filter((rec) => rec && rec.status !== "dead" && rec.storageCompletion !== "sdk").map((rec) => rec.key)); updatePendingSince(); } @@ -312,13 +315,22 @@ export function registerCaptureFunctions( } async function currentInboxSize(): Promise { - if (inboxSize === null) refreshInbox(await kv.list(KV.captureInbox)); - return inboxSize!; + // Only admission initializes membership, under its lock. Status/sweep snapshots + // must not overwrite it after a concurrent completion or admission. + if (inboxKeys === null) { + const records = await listCaptureInboxMetadata(kv); + inboxKeys = new Set(records.filter(rec => rec?.storageCompletion !== "sdk").map(rec => typeof rec?.key === "string" ? rec.key : {})); + refreshInbox(records); + } + return inboxKeys.size; } async function complete(rec: InboxRecord, observationId: string): Promise { if(sdkCompletion){ - await kv.set(KV.captureInbox,rec.key,{...rec,observationId,storageCompletion:'sdk',updatedAt:new Date().toISOString()}); + await withKeyedLock('capture:inbox-admission',async()=>{ + await kv.set(KV.captureInbox,rec.key,{...rec,observationId,storageCompletion:'sdk',updatedAt:new Date().toISOString()}); + inboxKeys?.delete(rec.key); // Idempotent; retained evidence is not pending capacity. + }); updateActiveInbox(rec.key,false);return; } await resolveCaptureReview(kv,rec,observationId); @@ -333,8 +345,10 @@ export function registerCaptureFunctions( attempts: rec.attempts, }; if (rec.eventSource === "client") await kv.set(eventScope(rec.key), rec.key, done); - await kv.delete(KV.captureInbox, rec.key); - if (inboxSize !== null) inboxSize = Math.max(0, inboxSize - 1); + await withKeyedLock('capture:inbox-admission',async()=>{ + await kv.delete(KV.captureInbox, rec.key); + inboxKeys?.delete(rec.key); + }); updateActiveInbox(rec.key, false); counters.completed++; lastCompletedAt = done.completedAt; @@ -352,7 +366,7 @@ export function registerCaptureFunctions( async function attempt(rec: InboxRecord): Promise { rec.cycleAttempts = (rec.cycleAttempts ?? rec.attempts) + 1; rec.attempts++; - type ObserveResult = { observationId?: string; deduplicated?: boolean; success?: boolean; error?: string }; + type ObserveResult = { observationId?: string; deduplicated?: boolean; success?: boolean; error?: string; retryable?: boolean }; let result = null as ObserveResult | null; let failure: string | null = null; try { @@ -386,6 +400,11 @@ export function registerCaptureFunctions( counters.duplicates++; return { status: "duplicate", state: "completed", eventId: rec.eventId, deduplicated: true }; } + if (!failure && result?.success === false && result.retryable === true && result.error === "vector_recovery_paused") { + rec.attempts = Math.max(0, rec.attempts - 1); + rec.cycleAttempts = Math.max(0, (rec.cycleAttempts ?? 1) - 1); + } + if (!failure && result?.success === false && result.retryable === true) failure = typeof result.error === "string" ? result.error : "observation_temporarily_unavailable"; const permanent = !failure; const message = failure ?? (result && typeof result.error === "string" ? result.error : "mem::observe returned no observation"); if (permanent || rec.cycleAttempts >= policy.maxAttempts) { @@ -396,21 +415,24 @@ export function registerCaptureFunctions( updateActiveInbox(rec.key, false); counters.deadLettered++; logger.warn("capture moved to dead letters", { eventId: rec.eventId, attempts: rec.attempts, error: message }); - return { status: "rejected", eventId: rec.eventId, error: message, retryable: false, observationId: rec.observationId }; + return { status: "rejected", eventId: rec.eventId, error: message, retryable: false, observationId: rec.observationId, ...(sdkCompletion && message === "vector_recovery_interrupted" ? {reconciliationRequired:true as const} : {}) }; } const nextAttemptAt = new Date(now.getTime() + backoffMs(rec.cycleAttempts, policy)).toISOString(); - const retrying: InboxRecord = { ...rec, status: "retrying", updatedAt: now.toISOString(), nextAttemptAt, lastError: message }; + const retrying: InboxRecord = { ...rec, status: "retrying", updatedAt: now.toISOString(), nextAttemptAt, lastError: message, ...(sdkCompletion && message === 'vector_recovery_paused' && result?.retryable === true ? {deferredByStorageHold:true as const} : {}) }; await recordCaptureFailure(kv,retrying); await kv.set(KV.captureInbox, rec.key, retrying); counters.retried++; logger.warn("capture processing failed, will retry", { eventId: rec.eventId, attempts: rec.attempts, nextAttemptAt, error: message }); + if (sdkCompletion && retrying.deferredByStorageHold) return { status: "rejected", eventId: rec.eventId, error: "capture_reconciliation_required", retryable: false, reconciliationRequired: true, observationId: rec.observationId }; return { status: "accepted", state: "retrying", eventId: rec.eventId, observationId: rec.observationId, attempts: rec.attempts, nextAttemptAt, error: message }; } - async function sdkRecordCompleted(rec:InboxRecord):Promise{ + async function sdkRecordCompleted(rec:InboxRecord | CaptureInboxMetadata,resolutionKeys?:Set):Promise{ if(rec.storageCompletion!=='sdk'||rec.sdkRestoreState==='pending'||rec.sdkResolutionState==='pending')return false; + const requiresResolution=resolutionKeys?.has(rec.key)||rec.sdkResolutionId!==undefined||rec.sdkResolutionState!==undefined; + if(resolutionKeys&&!requiresResolution)return true; const history=await kv.get(KV.captureReview,rec.key); - if(rec.sdkResolutionId!==undefined||rec.sdkResolutionState!==undefined||history?.sdkResolution!==undefined)return committedSdkResolution(history,rec); + if(requiresResolution||history?.sdkResolution!==undefined)return 'payload' in rec?committedSdkResolution(history,rec):committedInboxMetadata(kv,history,rec); return true; } @@ -418,6 +440,10 @@ export function registerCaptureFunctions( const payload = input.payload; const clientEventId = isValidEventId(input.eventId) ? input.eventId : undefined; const eventId = clientEventId ?? generateId("evs"); + const state = backgroundControl?.snapshot(); + if (sdkCompletion && state && (state.holds.includes("storage_error") || state.status === "resuming" || state.status === "initializing")) { + return { status: "rejected", eventId, error: "vector_recovery_paused", retryable: true }; + } const key = captureKey(payload.project, payload.sessionId, eventId); return withKeyedLock(`capture:${key}`, async () => { let storedObservationId: string | undefined; @@ -438,7 +464,7 @@ export function registerCaptureFunctions( const stored = await storedObservation(done); if (stored === undefined && done.observationId) throw new Error("capture_existing_observation_unavailable"); if (stored !== null) { - if (stored) await restoreIndexEntries(stored); + if (stored) await restoreIndexEntriesAdmitted(stored); counters.duplicates++; return { status: "duplicate", state: "completed", eventId, observationId: done.observationId, deduplicated: true }; } @@ -451,6 +477,11 @@ export function registerCaptureFunctions( } const queued = await kv.get(KV.captureInbox, key); if (queued) { + if (sdkCompletion && queued.lastError === "vector_recovery_interrupted") return { status: "rejected", eventId, error: "vector_recovery_interrupted", retryable: false, reconciliationRequired: true, observationId: queued.observationId }; + if (sdkCompletion && queued.deferredByStorageHold === true && queued.status === "retrying" && queued.lastError === "vector_recovery_paused") { + if (!clientEventId || queued.eventSource !== "client" || !sameSdkSource(queued.payload, payload)) return { status: "rejected", eventId, error: "capture_deferred_source_mismatch", retryable: false }; + return { status: "rejected", eventId, error: "capture_reconciliation_required", retryable: false, reconciliationRequired: true, observationId: queued.observationId }; + } counters.duplicates++; return { status: "duplicate", state: queued.status, eventId, observationId: queued.observationId, deduplicated: true }; } @@ -472,8 +503,8 @@ export function registerCaptureFunctions( }; const admitted=await withKeyedLock('capture:inbox-admission',async()=>{ if((await currentInboxSize())>=policy.inboxMax)return false; - try{await kv.set(KV.captureInbox,key,rec);}catch(error){inboxSize=null;throw error;} - if(inboxSize!==null)inboxSize++;updateActiveInbox(key,true);return true; + try{await kv.set(KV.captureInbox,key,rec);}catch(error){inboxKeys=null;throw error;} + inboxKeys?.add(key);updateActiveInbox(key,true);return true; }); if(!admitted){counters.rejected++;return {status:'rejected',eventId,error:`capture inbox is full (${policy.inboxMax} unprocessed events)`,retryable:true};} counters.accepted++; @@ -481,6 +512,14 @@ export function registerCaptureFunctions( }); } + async function restoreIndexEntriesAdmitted(observation: CompressedObservation): Promise { + if (!backgroundControl) return restoreIndexEntries(observation); + const result = await backgroundControl.runVectorWrite(() => restoreIndexEntries(observation)); + if (result && typeof result === "object" && "success" in result && result.success === false) { + throw Object.assign(new Error(result.error), { code: result.error }); + } + } + async function restoreSdkSource(input:{payload:HookPayload;eventId:string}) { const {payload,eventId}=input,key=captureKey(payload.project,payload.sessionId,eventId); const base={eventId,completion:'sdk',durable:false,sourceRestore:'sdk-v1'}; @@ -500,15 +539,14 @@ export function registerCaptureFunctions( alreadyPresent=!!obs; if(rec&&['sdkRestoreState','sdkRestoreRequestId','sdkRestoreAttempts'].some(field=>Object.hasOwn(rec,field))&&(!['pending','completed'].includes(rec.sdkRestoreState??'')||typeof rec.sdkRestoreRequestId!=='string'||!/^[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$/.test(rec.sdkRestoreRequestId)||rec.sdkRestoreAttempts===undefined))throw sdkSourceFailure('SDK_SOURCE_RESTORE_METADATA_INVALID'); if(rec?.sdkRestoreAttempts!==undefined&&(!Number.isSafeInteger(rec.sdkRestoreAttempts)||rec.sdkRestoreAttempts<0||rec.sdkRestoreAttempts>3))throw sdkSourceFailure('SDK_SOURCE_BUDGET_INVALID'); - if(!rec&&(await currentInboxSize())>=policy.inboxMax)throw sdkSourceFailure('SDK_SOURCE_INBOX_FULL'); if(!obs&&(rec?.sdkRestoreAttempts??0)>=3)throw sdkSourceFailure('SDK_SOURCE_RESTORE_LIMIT'); const now=new Date().toISOString(); source={...(rec??{key,eventId,eventSource:'client',sessionId:payload.sessionId,project:payload.project,hookType:payload.hookType,observationId:id,status:'pending',attempts:0,acceptedAt:now,payload:{...payload,data:scrubRecord(payload.data)}}),storageCompletion:'sdk',updatedAt:now,sdkRestoreRequestId:requestId,sdkRestoreState:'pending',sdkRestoreAttempts:(rec?.sdkRestoreAttempts??0)+(obs?0:1)}; - try{await kv.set(KV.captureInbox,key,source);}catch(error){if(!rec)inboxSize=null;throw error;} - if(!rec&&inboxSize!==null)inboxSize++; + try{await kv.set(KV.captureInbox,key,source);}catch(error){if(!rec)inboxKeys=null;throw error;} + // SDK restore preserves a source record, not an unprocessed capture slot. })); if(!alreadyPresent){const outcome:any=await sdk.trigger({function_id:'mem::observe',payload:{...payload,eventId,captureKey:key,observationId:id,sdkSourceRestore:{key,requestId}}});alreadyPresent=outcome?.deduplicated===true;} - else await restoreIndexEntries((await kv.get(KV.observations(payload.sessionId),id))!); + else await restoreIndexEntriesAdmitted((await kv.get(KV.observations(payload.sessionId),id))!); await recordAudit(kv,'capture_sdk_source_restore','mem::capture-restore-sdk',[key],{completion:'sdk',stage:'verify_pending'}); await kv.recoveryWrite!.exclusive(keys,async()=>{ await assertSdkSourceClaim(kv,{...payload,eventId,captureKey:key,observationId:id},{key,requestId});if(stopped)throw sdkSourceFailure('SDK_SOURCE_STOPPED'); @@ -529,13 +567,15 @@ export function registerCaptureFunctions( return withKeyedLock(`capture:${key}`, async () => { if (stopped) return "stopped"; if(sdkCompletion)return "waiting"; - await currentInboxSize(); const stored=await kv.get(KV.captureInbox,key); if (!stored) return "gone"; const rec={...stored}; + if(rec.lastError==='vector_recovery_interrupted')return 'waiting'; if(rec.eventSource==='client'&&(await kv.get(eventScope(key),key))?.state==='deleted'){ - await kv.delete(KV.captureInbox,key); - if(inboxSize!==null)inboxSize=Math.max(0,inboxSize-1); + await withKeyedLock('capture:inbox-admission',async()=>{ + await kv.delete(KV.captureInbox,key); + inboxKeys?.delete(key); + }); updateActiveInbox(key, false); counters.duplicates++;return 'gone'; } @@ -584,7 +624,7 @@ export function registerCaptureFunctions( async function runSweep(): Promise<{ processed: number; recovered: number; deadLettered: number }> { if (stopped || sdkCompletion) return { processed: 0, recovered: 0, deadLettered: 0 }; - const records = await kv.list(KV.captureInbox); + const records = await listCaptureInboxMetadata(kv); refreshInbox(records); let processed = 0; let recovered = 0; @@ -637,7 +677,7 @@ export function registerCaptureFunctions( } // Unprocessed originals and their review history have no age/count expiry. if (!stopped) removed += await prunePromptLedgers(kv, cutoff, policy.eventsMax); - inboxSize = null; + inboxKeys = null; activeInboxKeys = null; counters.pruned += removed; lastPruneAt = new Date().toISOString(); @@ -702,32 +742,41 @@ export function registerCaptureFunctions( } let inboxStatusReading=false; - async function status(options:{sdkProofPage?:CaptureSdkProofQuery}={}): Promise { + async function status(options:{scanInbox?:boolean;sdkProofPage?:CaptureSdkProofQuery}={}): Promise { const page=options.sdkProofPage; if(page&&(!Number.isSafeInteger(page.offset)||page.offset<0||!Number.isSafeInteger(page.limit)||page.limit<1||page.limit>32||page.generation!==undefined&&!/^[0-9a-f]{64}$/.test(page.generation)||page.offset>0&&page.generation===undefined))throw Object.assign(Error('INVALID_CAPTURE_PROOF_PAGE'),{code:'INVALID_CAPTURE_PROOF_PAGE'}); await automaticRecovery.initialize().catch(()=>{}); let inbox: CaptureStatus["inbox"] = null; - if(!inboxStatusReading){ + if(options.scanInbox!==false&&!inboxStatusReading){ inboxStatusReading=true; try { - const records = (await kv.list(KV.captureInbox)).filter(Boolean); + const proofSources = sdkCompletion && page ? (await kv.list(KV.captureInbox, { project: value => { + if (!value) return null; + return { + metadata: projectCaptureInbox(value), + digest: createHash('sha256').update(JSON.stringify(value)).digest('hex'), + }; + } })).filter(row => row !== null) : undefined; + const records = proofSources?.map(row => row.metadata) ?? (await listCaptureInboxMetadata(kv)).filter(Boolean); refreshInbox(records); let sdkCompleted=0; let sdkProofPage:NonNullable['sdkProofPage']; let sdkRetained:number|undefined; let selected=records; - if(sdkCompletion&&page){ - const ordered=[...records].sort((a,b)=>a.key.localeCompare(b.key)); - const hash=createHash('sha256');for(const record of ordered)hash.update(JSON.stringify(record)+'\n'); + if(proofSources&&page){ + const ordered=proofSources.sort((a,b)=>a.metadata.key.localeCompare(b.metadata.key)); + const hash=createHash('sha256'); + for(const row of ordered)hash.update(row.digest); const generation=hash.digest('hex'); if(page.generation!==undefined&&page.generation!==generation)throw Object.assign(Error('CAPTURE_PROOF_GENERATION_CHANGED'),{code:'CAPTURE_PROOF_GENERATION_CHANGED'}); - const archived=ordered.filter(r=>r.storageCompletion==='sdk');sdkRetained=archived.length; + const archived=ordered.filter(r=>r.metadata.storageCompletion==='sdk');sdkRetained=archived.length; if(page.offset>archived.length)throw Object.assign(Error('INVALID_CAPTURE_PROOF_PAGE'),{code:'INVALID_CAPTURE_PROOF_PAGE'}); - selected=archived.slice(page.offset,page.offset+page.limit); + selected=archived.slice(page.offset,page.offset+page.limit).map(row=>row.metadata); sdkProofPage={offset:page.offset,checked:selected.length,completed:0,total:archived.length,nextOffset:page.offset+selected.lengthsdkRecordCompleted(r))); + const checked=await Promise.allSettled(selected.slice(at,at+4).map(r=>sdkRecordCompleted(r,resolutionKeys))); if(checked.some(r=>r.status==='rejected'))throw Error('CAPTURE_STATUS_HISTORY_UNAVAILABLE'); sdkCompleted+=checked.filter(r=>r.status==='fulfilled'&&r.value).length; } @@ -763,13 +812,29 @@ export function registerCaptureFunctions( return { automaticRecovery:automaticRecovery.snapshot(),policy, inbox, legacyOutbox:await inspectOutbox(), progress:progress(), sinceStart: { ...counters }, lastSweepAt, lastPruneAt, lastBootDrain, spool }; } + sdk.registerFunction('mem::capture-inspect-uncertain', async (input: unknown) => { + if (!input || typeof input !== 'object' || Array.isArray(input)) throw new Error('INVALID_UNCERTAIN_INSPECTION'); + const { _caller_worker_id, ...selection } = input as Record; + if ((_caller_worker_id !== undefined && typeof _caller_worker_id !== 'string') || !validUncertainInspection(selection)) throw new Error('INVALID_UNCERTAIN_INSPECTION'); + return inspectUncertainCapture(kv, selection); + }); sdk.registerFunction("mem::capture-restore-sdk",(input:{payload:HookPayload;eventId:string})=>trackWork(()=>backgroundTask(restoreSdkSource)(input))); sdk.registerFunction('mem::capture-reconcile-sdk',input=>{ const selection=sdkReconciliationInvocation(input); return trackWork(()=>backgroundTask(async()=>{ try{ - const result=await sdkReconciliation.reconcile(selection); - if(result.status==='reconciled'){counters.recovered++;updateActiveInbox(selection.key,false);} + let result:Awaited>; + try{result=await sdkReconciliation.reconcile(selection);} + finally{ + // Reconcile releases its capture:key lock before taking admission; never + // invert capture/restore's key -> admission order. A partial SDK source + // remains evidence, not a claim of completed reconciliation. + await withKeyedLock('capture:inbox-admission',async()=>{ + try{if((await kv.get(KV.captureInbox,selection.key))?.storageCompletion==='sdk')inboxKeys?.delete(selection.key);} + catch{inboxKeys=null;} + }); + } + if(result.status==='reconciled'){counters.recovered++;updateActiveInbox(selection.key,false);trackMaintenance(()=>automaticRecovery.refreshSdkReviews());} return result; }catch(error){const code=(error as {code?:unknown})?.code;return{completed:false,completion:'sdk',verified:false,synced:false,code:typeof code==='string'&&/^SDK_RECONCILIATION_[A-Z_]+$/.test(code)?code:'SDK_RECONCILIATION_UNAVAILABLE'};} })()); @@ -784,7 +849,7 @@ export function registerCaptureFunctions( sdk.registerFunction("mem::capture-retry", async (input: { eventId?: string; key?: string; all?: boolean; requestId?: string } = {}) => { if(input.requestId && (!input.eventId || !/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/.test(input.requestId))) throw new Error("INVALID_RETRY_REQUEST"); - const records = (await kv.list(KV.captureInbox)).filter(Boolean); + const records = (await listCaptureInboxMetadata(kv)).filter(Boolean); refreshInbox(records); if(input.eventId && input.requestId && !records.some(r=>input.key?r.key===input.key:r.eventId===input.eventId)){ const h=(await kv.list(KV.captureReview)).find(h=>(input.key?h.key===input.key:h.eventId===input.eventId)&&h.eventId===input.eventId&&h.requests[input.requestId!]); @@ -809,15 +874,12 @@ export function registerCaptureFunctions( sdk.registerFunction("mem::capture-list", async (input: { status?: InboxState | "completed"; limit?: number; review?: boolean; eventId?: string; key?: string; cursor?: string; historyOffset?: number } = {}) => { if(input.review) return captureReviewPage(kv,input); const limit = Math.min(Math.max(1, Number(input.limit) || 50), 500); - const records = (await kv.list(KV.captureInbox)) + const records = (await listCaptureInboxMetadata(kv)) .filter((r) => r && (!input.status || r.status === input.status)) .sort((a, b) => b.updatedAt.localeCompare(a.updatedAt)); return { total: records.length, - items: records.slice(0, limit).map(({ payload, ...rest }) => ({ - ...rest, - preview: previewPayload(payload), - })), + items: records.slice(0, limit).map(({ reviewPreview, ...rest }) => rest), }; }); @@ -878,14 +940,3 @@ function spoolPayload(body: Record): HookPayload | null { if (!hookType || !sessionId || !project || !cwd || !timestamp) return null; return { hookType: hookType as HookPayload["hookType"], sessionId, project, cwd, timestamp, data: body["data"] }; } - -function previewPayload(payload: HookPayload | undefined): string { - if (!payload) return ""; - const d = payload.data && typeof payload.data === "object" ? (payload.data as Record) : {}; - const label = typeof d["tool_name"] === "string" ? d["tool_name"] : payload.hookType; - let text = ""; - try { - text = JSON.stringify(d["tool_input"] ?? d["prompt"] ?? payload.data ?? ""); - } catch {} - return `${label}: ${text}`.slice(0, 160); -} diff --git a/src/functions/cascade.ts b/src/functions/cascade.ts index da92265944..8b01bb607c 100644 --- a/src/functions/cascade.ts +++ b/src/functions/cascade.ts @@ -3,11 +3,11 @@ import { setImmediate as yieldToEventLoop } from "node:timers/promises"; import { getGraphGeneration, pinGraphState } from "../state/graph-generation.js"; import type { IIIClient } from "iii-sdk"; import type { StateKV } from "../state/kv.js"; +import { collectAll } from "../state/kv.js"; import { KV } from "../state/schema.js"; import { createGraphIndexReader, mutateIndexedGraphRecord } from "../state/graph-retrieval-index.js"; import type { Memory, GraphCascadeProgress } from "../types.js"; import { safeAudit } from "./audit.js"; - export function registerCascadeFunction(sdk: IIIClient, state: StateKV): void { sdk.registerFunction("mem::cascade-update", async (data: { supersededMemoryId: string }) => { @@ -116,7 +116,7 @@ export function registerCascadeFunction(sdk: IIIClient, state: StateKV): void { (superseded.concepts ?? []).map((c) => c.toLowerCase()), ); if (supersededConcepts.size >= 2) { - const allMemories = await kv.list(KV.memories); + const allMemories = await collectAll(kv, KV.memories); for (const mem of allMemories) { if (mem.id === data.supersededMemoryId) continue; if (!mem.isLatest) continue; diff --git a/src/functions/claude-bridge.ts b/src/functions/claude-bridge.ts index c67f6db6a1..e50b2463d6 100644 --- a/src/functions/claude-bridge.ts +++ b/src/functions/claude-bridge.ts @@ -8,10 +8,10 @@ import { dirname, join } from "node:path"; import type { Memory, ClaudeBridgeConfig } from "../types.js"; import { KV } from "../state/schema.js"; import type { StateKV } from "../state/kv.js"; +import { collectAll } from "../state/kv.js"; import { recordAudit } from "./audit.js"; import { logger } from "../logger.js"; import { resolveProject } from "../hooks/_project.js"; - function hasIdenticalContent(path: string, content: string): boolean { let fd: number | undefined; try { @@ -170,7 +170,7 @@ export function registerClaudeBridgeFunction( try { const project = resolveProject(config.projectPath); - const memories = await kv.list(KV.memories); + const memories = await collectAll(kv, KV.memories); const latestMemories = memories.filter( (m) => m.isLatest && (m.project === project || m.project === config.projectPath), ); diff --git a/src/functions/consolidate.ts b/src/functions/consolidate.ts index 8d41e6296f..5f778af1dd 100644 --- a/src/functions/consolidate.ts +++ b/src/functions/consolidate.ts @@ -5,10 +5,12 @@ import type { Session, MemoryProvider, } from "../types.js"; -import { KV, generateId } from "../state/schema.js"; -import { StateKV } from "../state/kv.js"; +import { KV, fingerprintId, generateId } from "../state/schema.js"; +import { StateKV, collectAll } from "../state/kv.js"; +import { safeAudit } from "./audit.js"; +import { withKeyedLocks } from "../state/keyed-mutex.js"; +import { stableStringify } from "../capture/event-id.js"; import { supersededAt } from "../state/memory-utils.js"; -import { recordAudit } from "./audit.js"; const CONSOLIDATION_SYSTEM = `You are a memory consolidation engine. Given a set of related observations from coding sessions, synthesize them into a single long-term memory. @@ -63,6 +65,22 @@ function parseMemoryXml( }; } +function evidence(o: CompressedObservation) { + return { id: o.id, type: o.type, title: o.title, narrative: o.narrative, files: o.files, importance: o.importance, concepts: o.concepts }; +} + +function consolidationKey(sources: Array): string { + const rows = sources + .map((o) => ({ sid: o.sid, ...evidence(o) })) + .sort((a, b) => (a.sid === b.sid ? (a.id < b.id ? -1 : a.id > b.id ? 1 : 0) : a.sid < b.sid ? -1 : 1)); + return fingerprintId("cons", stableStringify(rows)); +} + +function sameIds(left: string[] | undefined, right: string[]): boolean { + const ids = new Set(left); + return ids.size === right.length && right.every((id) => ids.has(id)); +} + export function registerConsolidateFunction( sdk: IIIClient, kv: StateKV, @@ -112,10 +130,6 @@ export function registerConsolidateFunction( } let consolidated = 0; - const existingMemories = await kv.list(KV.memories); - const existingTitles = new Set( - existingMemories.map((m) => m.title.toLowerCase()), - ); const MAX_LLM_CALLS = 10; let llmCallCount = 0; @@ -139,97 +153,138 @@ export function registerConsolidateFunction( ) .join("\n\n"); + let timer: ReturnType | undefined; try { + llmCallCount++; const response = await Promise.race([ provider.compress( CONSOLIDATION_SYSTEM, `Concept: "${concept}"\n\nObservations:\n${prompt}`, ), - new Promise((_, reject) => - setTimeout(() => reject(new Error("compress timeout")), 30_000), - ), + new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error("compress timeout")), 30_000); + }), ]); - llmCallCount++; + clearTimeout(timer); const parsed = parseMemoryXml(response, sessionIds); if (!parsed) continue; const now = new Date().toISOString(); const obsIds = [...new Set(top.map((o) => o.id))]; + const sourceKey = consolidationKey(top); const scopedProject = typeof data.project === "string" && data.project.trim().length > 0 ? data.project.trim() : undefined; - // A scoped consolidation run must only evolve memories that belong - // to the same project. Without this guard, two projects that happen - // to consolidate observations into an identically-titled memory would - // cause one project's memory to silently evolve the other's — the - // exact class of cross-project corruption this fix is designed to - // prevent. An unscoped run (no data.project, background cron path) - // preserves the pre-existing behavior and may evolve any memory. - const existingMatch = existingMemories.find( - (m) => - m.title.toLowerCase() === parsed.title.toLowerCase() && - (!scopedProject || !m.project || m.project === scopedProject), - ); - - if (existingMatch) { - existingMatch.isLatest = false; - existingMatch.updatedAt = supersededAt(existingMatch.updatedAt, now); - await kv.set(KV.memories, existingMatch.id, existingMatch); - await recordAudit(kv, "evolve", "mem::consolidate", [existingMatch.id], { - action: "mark_non_latest", - concept, - }); + const publicationLocks = ["mem:remember", ...sessionIds.map((sid) => `obs:${sid}`)]; + let adoptedId: string | undefined; + const published = await withKeyedLocks(publicationLocks, async () => { + for (const sid of sessionIds) { + const current = await kv.get(KV.sessions, sid); + const original = filtered.find((session) => session.id === sid)!; + if (!current || current.project !== original.project || current.agentId !== original.agentId) return false; + } + for (const { sid, ...original } of top) { + const current = await kv.get(KV.observations(sid), original.id); + if (!current || stableStringify(evidence(current)) !== stableStringify(evidence(original))) return false; + } + const memories = await collectAll(kv, KV.memories); + const candidates = memories.filter((memory) => + memory.isLatest !== false && + memory.agentId === undefined && + (!scopedProject || memory.project === scopedProject), + ); + const children = new Map(); + for (const memory of candidates) { + if (!memory.parentId) continue; + const siblings = children.get(memory.parentId) ?? []; + siblings.push(memory); + children.set(memory.parentId, siblings); + } + for (const parent of candidates) { + const successors = children.get(parent.id); + if (successors?.length !== 1) continue; + const child = successors[0]; + if (child.title.toLowerCase() !== parent.title.toLowerCase() || + child.version !== (parent.version || 1) + 1 || + !child.supersedes?.includes(parent.id)) continue; + const updatedAt = supersededAt(parent.updatedAt, new Date().toISOString()); + await kv.set(KV.memories, parent.id, { ...parent, isLatest: false, updatedAt }); + parent.isLatest = false; + parent.updatedAt = updatedAt; + await safeAudit(kv, "evolve", "mem::consolidate", [parent.id], { + action: "complete_publication", newId: child.id, + }); + } + const heads = candidates.filter((memory) => memory.isLatest !== false); + if (heads.some((memory) => memory.consolidationKeys?.[concept] === sourceKey)) return false; + const legacy = heads.find((memory) => + !memory.consolidationKeys && + sameIds(memory.sourceObservationIds, obsIds) && + sameIds(memory.sessionIds, sessionIds), + ); + if (legacy) { + await kv.set(KV.memories, legacy.id, { ...legacy, consolidationKeys: { [concept]: sourceKey } }); + adoptedId = legacy.id; + return false; + } - const evolved: Memory = { - id: generateId("mem"), - createdAt: now, - updatedAt: now, - ...parsed, - version: (existingMatch.version || 1) + 1, - parentId: existingMatch.id, - supersedes: [ - existingMatch.id, - ...(existingMatch.supersedes || []), - ], - sourceObservationIds: obsIds, - isLatest: true, - ...(scopedProject !== undefined && { project: scopedProject }), - }; - await kv.set(KV.memories, evolved.id, evolved); - await recordAudit(kv, "evolve", "mem::consolidate", [evolved.id], { - action: "evolve_memory", - oldId: existingMatch.id, - newId: evolved.id, - concept, - }); - existingTitles.add(evolved.title.toLowerCase()); - consolidated++; - } else { + const sameTitle = heads.filter((memory) => + memory.title.toLowerCase() === parsed.title.toLowerCase(), + ); + if (sameTitle.length > 1) return false; + const previous = sameTitle[0]; const memory: Memory = { id: generateId("mem"), createdAt: now, updatedAt: now, ...parsed, sourceObservationIds: obsIds, - version: 1, + consolidationKeys: { ...previous?.consolidationKeys, [concept]: sourceKey }, + version: previous ? (previous.version || 1) + 1 : 1, isLatest: true, + ...(previous && { + recurrence: (previous.recurrence ?? 1) + + (obsIds.some((id) => !(previous.sourceObservationIds ?? []).includes(id)) ? 1 : 0), + parentId: previous.id, + supersedes: [previous.id, ...(previous.supersedes || [])], + }), ...(scopedProject !== undefined && { project: scopedProject }), }; - await kv.set(KV.memories, memory.id, memory); - await recordAudit(kv, "remember", "mem::consolidate", [memory.id], { - action: "create_memory", + + try { + await kv.set(KV.memories, memory.id, memory); + } catch (error) { + if (!(await kv.get(KV.memories, memory.id))) throw error; + } + if (previous) { + try { + await kv.set(KV.memories, previous.id, { ...previous, isLatest: false, updatedAt: supersededAt(previous.updatedAt, now) }); + } catch (error) { + await kv.set(KV.memories, previous.id, previous); + await kv.delete(KV.memories, memory.id); + throw error; + } + } + await safeAudit(kv, previous ? "evolve" : "remember", "mem::consolidate", [memory.id], { + action: previous ? "evolve_memory" : "create_memory", + ...(previous && { oldId: previous.id, newId: memory.id }), concept, }); - existingTitles.add(memory.title.toLowerCase()); - consolidated++; + return true; + }); + if (adoptedId) { + await safeAudit(kv, "evolve", "mem::consolidate", [adoptedId], { action: "adopt_legacy", concept }); } + if (published) consolidated++; } catch (err) { logger.warn("Consolidation failed for concept", { concept, error: err instanceof Error ? err.message : String(err), }); + } finally { + clearTimeout(timer); } } diff --git a/src/functions/consolidation-pipeline.ts b/src/functions/consolidation-pipeline.ts index 792b586a71..ef177aad52 100644 --- a/src/functions/consolidation-pipeline.ts +++ b/src/functions/consolidation-pipeline.ts @@ -9,6 +9,7 @@ import type { } from "../types.js"; import { KV, generateId } from "../state/schema.js"; import type { StateKV } from "../state/kv.js"; +import { collectAll } from "../state/kv.js"; import { SEMANTIC_MERGE_SYSTEM, buildSemanticMergePrompt, @@ -16,7 +17,7 @@ import { buildProceduralExtractionPrompt, } from "../prompts/consolidation.js"; import { recordAudit } from "./audit.js"; -import { CONSOLIDATION_LAST_RUN_KEY } from "./consolidation-status.js"; +import { CONSOLIDATION_LAST_RUN_KEY, PROCEDURAL_MIN_SESSIONS_PER_PATTERN, patternFrequency } from "./consolidation-status.js"; import { getConsolidationDecayDays, isConsolidationEnabled } from "../config.js"; import { logger } from "../logger.js"; import { isReflectEnabled } from "./slots.js"; @@ -175,7 +176,7 @@ export function registerConsolidationPipelineFunction( if (tier === "all" || tier === "semantic") { const {total:totalSummaries,recent:recentSummaries} = await kv.getSummaryMergeInputs(); - const existingSemantic = await kv.list(KV.semantic); + const existingSemantic = await collectAll(kv, KV.semantic); if (totalSummaries >= 5) { @@ -265,14 +266,14 @@ export function registerConsolidationPipelineFunction( } if (tier === "all" || tier === "procedural") { - const memories = await kv.list(KV.memories); + const memories = await collectAll(kv, KV.memories); const patterns = memories .filter((m) => m.isLatest && m.type === "pattern") .map((m) => ({ content: m.content, - frequency: m.sessionIds.length || 1, + frequency: patternFrequency(m), })) - .filter((p) => p.frequency >= 2) + .filter((p) => p.frequency >= PROCEDURAL_MIN_SESSIONS_PER_PATTERN) .sort((a, b) => a.content.localeCompare(b.content) || a.frequency - b.frequency); if (patterns.length >= 2) { @@ -299,9 +300,7 @@ export function registerConsolidationPipelineFunction( let match; let newProcs = 0; const now = new Date().toISOString(); - const existingProcs = await kv.list( - KV.procedural, - ); + const existingProcs = await collectAll(kv, KV.procedural); while ((match = procRegex.exec(response)) !== null) { const name = match[1]; @@ -359,8 +358,8 @@ export function registerConsolidationPipelineFunction( } if (tier === "all" || tier === "decay") { - const semantic = await kv.list(KV.semantic); - const procedural = await kv.list(KV.procedural); + const semantic = await collectAll(kv, KV.semantic); + const procedural = await collectAll(kv, KV.procedural); const semanticResult = await decayAndWriteChanged( kv, diff --git a/src/functions/consolidation-status.ts b/src/functions/consolidation-status.ts index 3b31082c62..902fbb1b39 100644 --- a/src/functions/consolidation-status.ts +++ b/src/functions/consolidation-status.ts @@ -4,6 +4,12 @@ export const PROCEDURAL_MIN_PATTERNS = 2; export const PROCEDURAL_MIN_SESSIONS_PER_PATTERN = 2; export const CONSOLIDATION_COUNTS_REUSE_MS = 30_000; +export function patternFrequency(memory: { sessionIds?: string[]; recurrence?: number }): number { + const sessions = new Set(memory.sessionIds ?? []).size; + const saves = Number.isSafeInteger(memory.recurrence) && (memory.recurrence as number) > 0 ? (memory.recurrence as number) : 1; + return Math.max(sessions, saves); +} + export type TierState = "off" | "waiting" | "ready" | "ran" | "error" | "retry_pending"; export interface ConsolidationRunRecord { @@ -124,7 +130,7 @@ export function describeConsolidation(input: ConsolidationStatusInput): Consolid detail: `Extracted from each finished session with a summary and 3+ observations. ` + `The cross-session pass also needs ${PROCEDURAL_MIN_PATTERNS} pattern memories seen in ` + - `${PROCEDURAL_MIN_SESSIONS_PER_PATTERN}+ sessions (you have ${input.recurringPatterns}).`, + `${PROCEDURAL_MIN_SESSIONS_PER_PATTERN}+ sessions or saved ${PROCEDURAL_MIN_SESSIONS_PER_PATTERN}+ times (you have ${input.recurringPatterns}).`, } : { state: "ready", detail: "Ready: runs when the next session ends." })) }; diff --git a/src/functions/diagnostics.ts b/src/functions/diagnostics.ts index e88c8660b0..3d2013b0ed 100644 --- a/src/functions/diagnostics.ts +++ b/src/functions/diagnostics.ts @@ -3,6 +3,8 @@ import type { StateKV } from "../state/kv.js"; import { KV } from "../state/schema.js"; import { withKeyedLock } from "../state/keyed-mutex.js"; import { recordAudit } from "./audit.js"; +import { findOrphanFacets, isFacetStillOrphan } from "./facets.js"; +import { collectAll } from "../state/kv.js"; import type { Action, ActionEdge, @@ -38,6 +40,7 @@ const ALL_CATEGORIES = [ "crystals", "insights", "mesh", + "facets", ]; const TWENTY_FOUR_HOURS_MS = 24 * 60 * 60 * 1000; @@ -319,7 +322,7 @@ export function registerDiagnosticsFunction(sdk: IIIClient, kv: StateKV): void { } if (categories.includes("memories")) { - const memories = await kv.list(KV.memories); + const memories = await collectAll(kv, KV.memories); const memoryIds = new Set(memories.map((m) => m.id)); const supersededBy = new Map(); let memoryIssues = 0; @@ -469,7 +472,7 @@ export function registerDiagnosticsFunction(sdk: IIIClient, kv: StateKV): void { } if (categories.includes("semantic")) { - const semantic = await kv.list(KV.semantic); + const semantic = await collectAll(kv, KV.semantic); let semanticIssues = 0; for (const s of semantic) { if ( @@ -499,7 +502,7 @@ export function registerDiagnosticsFunction(sdk: IIIClient, kv: StateKV): void { } if (categories.includes("procedural")) { - const procedural = await kv.list(KV.procedural); + const procedural = await collectAll(kv, KV.procedural); let proceduralIssues = 0; for (const p of procedural) { if (!Array.isArray(p.steps) || p.steps.length === 0) { @@ -551,7 +554,7 @@ export function registerDiagnosticsFunction(sdk: IIIClient, kv: StateKV): void { } if (categories.includes("insights")) { - const insights = await kv.list(KV.insights); + const insights = await collectAll(kv, KV.insights); let insightIssues = 0; for (const i of insights) { if ( @@ -622,6 +625,27 @@ export function registerDiagnosticsFunction(sdk: IIIClient, kv: StateKV): void { } } + if (categories.includes("facets")) { + const orphans = await findOrphanFacets(kv); + checks.push( + orphans.length > 0 + ? { + name: "orphan-facets", + category: "facets", + status: "warn", + message: `${orphans.length} facets point at deleted targets`, + fixable: true, + } + : { + name: "facets-ok", + category: "facets", + status: "pass", + message: "Every facet points at a live target", + fixable: false, + }, + ); + } + const summary = { pass: checks.filter((c) => c.status === "pass").length, warn: checks.filter((c) => c.status === "warn").length, @@ -1011,7 +1035,7 @@ export function registerDiagnosticsFunction(sdk: IIIClient, kv: StateKV): void { } if (categories.includes("memories")) { - const memories = await kv.list(KV.memories); + const memories = await collectAll(kv, KV.memories); const supersededBy = new Map(); for (const memory of memories) { @@ -1059,6 +1083,30 @@ export function registerDiagnosticsFunction(sdk: IIIClient, kv: StateKV): void { } } + if (categories.includes("facets")) { + for (const facet of await findOrphanFacets(kv)) { + const label = `facet ${facet.dimension}:${facet.value} on deleted ${facet.targetType} ${facet.targetId}`; + if (dryRun) { + details.push(`[dry-run] Would remove ${label}`); + fixed++; + continue; + } + if (!(await isFacetStillOrphan(kv, facet))) { + skipped++; + continue; + } + await kv.delete(KV.facets, facet.id); + await recordAudit(kv, "heal", "mem::heal", [facet.id], { + entityType: "facet", + reason: "orphan-facet", + targetId: facet.targetId, + action: "delete", + }); + details.push(`Removed ${label}`); + fixed++; + } + } + return { success: true, fixed, skipped, details }; }, ); diff --git a/src/functions/enrich.ts b/src/functions/enrich.ts index 05f62acb9d..4b84f666b0 100644 --- a/src/functions/enrich.ts +++ b/src/functions/enrich.ts @@ -1,7 +1,7 @@ import type { IIIClient } from "iii-sdk"; import type { Memory } from "../types.js"; import { KV } from "../state/schema.js"; -import { StateKV } from "../state/kv.js"; +import { StateKV, collectAll } from "../state/kv.js"; import { logger } from "../logger.js"; import { escapeXml } from "../prompts/xml.js"; @@ -56,8 +56,7 @@ export function registerEnrichFunction(sdk: IIIClient, kv: StateKV): void { .catch(() => ({ results: [] })) : Promise.resolve({ results: [] }); - const bugMemoriesPromise = kv - .list(KV.memories) + const bugMemoriesPromise = collectAll(kv, KV.memories) .then((memories) => memories .filter( diff --git a/src/functions/evict.ts b/src/functions/evict.ts index b3d61566b0..4def6f45d9 100644 --- a/src/functions/evict.ts +++ b/src/functions/evict.ts @@ -7,7 +7,7 @@ import type { Memory, } from "../types.js"; import { KV } from "../state/schema.js"; -import { StateKV } from "../state/kv.js"; +import { StateKV, collectAll } from "../state/kv.js"; import { withKeyedLock } from "../state/keyed-mutex.js"; import { removeSessionFromProjectIndex } from "../state/session-index.js"; import { unindexObservationSession } from "../state/obs-index.js"; @@ -328,7 +328,7 @@ export function registerEvictFunction(sdk: IIIClient, kv: StateKV): void { } } - const memories = await kv.list(KV.memories).catch(() => []); + const memories = await collectAll(kv, KV.memories).catch(() => []); const evictedMemIds = new Set(); const deletedMemIds = new Set(); try { @@ -354,6 +354,9 @@ export function registerEvictFunction(sdk: IIIClient, kv: StateKV): void { }); continue; } + getSearchIndex().remove(mem.id); + vectorIndexRemove(mem.id); + indexCleaned = true; if (mem.imageRef) { await decrementImageRef(kv, sdk, mem.imageRef); } @@ -390,6 +393,9 @@ export function registerEvictFunction(sdk: IIIClient, kv: StateKV): void { }); continue; } + getSearchIndex().remove(mem.id); + vectorIndexRemove(mem.id); + indexCleaned = true; if (mem.imageRef) { await decrementImageRef(kv, sdk, mem.imageRef); } @@ -404,10 +410,10 @@ export function registerEvictFunction(sdk: IIIClient, kv: StateKV): void { } } } finally { - await deleteFacetsForTargets(kv, deletedMemIds); + try { await deleteFacetsForTargets(kv, deletedMemIds); } + finally { if (indexCleaned) await flushIndexSave(); } } - if (indexCleaned) await flushIndexSave(); logger.info("Eviction complete", { stats }); return stats; }, diff --git a/src/functions/export-import.ts b/src/functions/export-import.ts index b7248988d7..42d1f9c2e5 100644 --- a/src/functions/export-import.ts +++ b/src/functions/export-import.ts @@ -28,7 +28,7 @@ import { importOrigin } from "../types.js"; import { normalizeAccessLog } from "./access-tracker.js"; import { KV } from "../state/schema.js"; import { checkPayloadFrameSize, oversizedPayloadError, payloadByteLength, SAFE_PAYLOAD_BYTES } from "../state/frame-guard.js"; -import { StateKV } from "../state/kv.js"; +import { StateKV, collectAll } from "../state/kv.js"; import { addSessionToProjectIndex, removeSessionFromProjectIndex, @@ -172,8 +172,6 @@ export function registerExportImportFunction(sdk: IIIClient, state: StateKV): vo stoppedAt: collection, }; }; - const optional = async (scope: string): Promise => - kv.list(scope); const allSessions = await kv.list(KV.sessions); const sessionWindow = maxSessions !== undefined @@ -182,7 +180,7 @@ export function registerExportImportFunction(sdk: IIIClient, state: StateKV): vo const paginatedSessions = isSelected("sessions") ? sessionWindow : []; if (!budget.fits(paginatedSessions)) return refuse("sessions"); - const memories = sliceCollection("memories", await kv.list(KV.memories)); + const memories = sliceCollection("memories", await collectAll(kv, KV.memories)); if (!budget.fits(memories)) return refuse("memories"); let summaries: SessionSummary[]; @@ -202,9 +200,7 @@ export function registerExportImportFunction(sdk: IIIClient, state: StateKV): vo const observations: Record = {}; for (const session of paginatedSessions) { - const obs = await optional( - KV.observations(session.id), - ); + const obs = await collectAll(kv, KV.observations(session.id)); if (obs.length === 0) continue; if (!budget.fits(obs, session.id)) return refuse("observations"); observations[session.id] = obs; @@ -231,7 +227,7 @@ export function registerExportImportFunction(sdk: IIIClient, state: StateKV): vo }; const collections: Partial = {}; for (const [field, scope] of EXPORT_COLLECTIONS) { - const rows = await optional(scope); + const rows = await collectAll(kv, scope); const items = field === "accessLogs" && partial ? accessLogsFor(rows as AccessLogExport[]) @@ -319,7 +315,7 @@ export function registerExportImportFunction(sdk: IIIClient, state: StateKV): vo const strategy = data.strategy || "merge"; const importData = data.exportData; - const supportedVersions = new Set(["0.3.0", "0.4.0", "0.5.0", "0.6.0", "0.6.1", "0.7.0", "0.7.2", "0.7.3", "0.7.4", "0.7.5", "0.7.6", "0.7.7", "0.7.9", "0.8.0", "0.8.1", "0.8.2", "0.8.3", "0.8.4", "0.8.5", "0.8.6", "0.8.7", "0.8.8", "0.8.9", "0.8.10", "0.8.11", "0.8.12", "0.8.13", "0.9.0", "0.9.1", "0.9.2", "0.9.3", "0.9.4", "0.9.5", "0.9.6", "0.9.7", "0.9.8", "0.9.9", "0.9.10", "0.9.11", "0.9.12", "0.9.13", "0.9.14", "0.9.15", "0.9.16", "0.9.17", "0.9.18", "0.9.19", "0.9.20", "0.9.21", "0.9.22", "0.9.23", "0.9.24", "0.9.25", "0.9.26", "0.9.27", "0.9.28", "0.9.29", "0.9.30", "0.9.31", "0.9.31-daniel.1", "0.9.31-daniel.33", "0.9.31-daniel.34", "0.9.31-daniel.35", "0.9.31-daniel.36", "0.9.31-daniel.37", "0.9.31-daniel.38", "0.9.31-daniel.39", "0.9.31-daniel.40", "0.9.31-daniel.41", "0.9.31-daniel.42", "0.9.31-daniel.43", "0.9.31-daniel.44", "0.9.31-daniel.45", "0.9.31-daniel.46", "0.9.31-daniel.47", "0.9.31-daniel.48", "0.9.31-daniel.49", "0.9.31-daniel.50", "0.9.31-daniel.51", "0.9.31-daniel.52", "0.9.31-daniel.53", "0.9.31-daniel.54", "0.9.32-daniel.55", "0.9.30-daniel.1", "0.9.30-daniel.10", "0.9.30-daniel.11", "0.9.30-daniel.15", "0.9.30-daniel.16", "0.9.30-daniel.18", "0.9.30-daniel.19", "0.9.30-daniel.2", "0.9.30-daniel.20", "0.9.30-daniel.21", "0.9.30-daniel.22", "0.9.30-daniel.23", "0.9.30-daniel.24", "0.9.30-daniel.25", "0.9.30-daniel.26", "0.9.30-daniel.27", "0.9.30-daniel.28", "0.9.30-daniel.3", "0.9.30-daniel.4", "0.9.30-daniel.5", "0.9.30-daniel.6", "0.9.30-daniel.7", "0.9.30-daniel.8", "0.9.30-daniel.9", "0.9.30-daniel.29", "0.9.30-daniel.30", "0.9.30-daniel.31", "0.9.30-daniel.32"]); + const supportedVersions = new Set(["0.3.0", "0.4.0", "0.5.0", "0.6.0", "0.6.1", "0.7.0", "0.7.2", "0.7.3", "0.7.4", "0.7.5", "0.7.6", "0.7.7", "0.7.9", "0.8.0", "0.8.1", "0.8.2", "0.8.3", "0.8.4", "0.8.5", "0.8.6", "0.8.7", "0.8.8", "0.8.9", "0.8.10", "0.8.11", "0.8.12", "0.8.13", "0.9.0", "0.9.1", "0.9.2", "0.9.3", "0.9.4", "0.9.5", "0.9.6", "0.9.7", "0.9.8", "0.9.9", "0.9.10", "0.9.11", "0.9.12", "0.9.13", "0.9.14", "0.9.15", "0.9.16", "0.9.17", "0.9.18", "0.9.19", "0.9.20", "0.9.21", "0.9.22", "0.9.23", "0.9.24", "0.9.25", "0.9.26", "0.9.27", "0.9.28", "0.9.29", "0.9.30", "0.9.31", "0.9.31-daniel.1", "0.9.31-daniel.33", "0.9.31-daniel.34", "0.9.31-daniel.35", "0.9.31-daniel.36", "0.9.31-daniel.37", "0.9.31-daniel.38", "0.9.31-daniel.39", "0.9.31-daniel.40", "0.9.31-daniel.41", "0.9.31-daniel.42", "0.9.31-daniel.43", "0.9.31-daniel.44", "0.9.31-daniel.45", "0.9.31-daniel.46", "0.9.31-daniel.47", "0.9.31-daniel.48", "0.9.31-daniel.49", "0.9.31-daniel.50", "0.9.31-daniel.51", "0.9.31-daniel.52", "0.9.31-daniel.53", "0.9.31-daniel.54", "0.9.32-daniel.55", "0.9.32-daniel.57", "0.9.32-daniel.58", "0.9.32-daniel.59", "0.9.32-daniel.60", "0.9.32-daniel.61", "0.9.32-daniel.62", "0.9.32-daniel.63", "0.9.32-daniel.64", "0.9.32-daniel.65", "0.9.32-daniel.66", "0.9.32-daniel.67", "0.9.32-daniel.68", "0.9.33-daniel.69", "0.9.33-daniel.70", "0.9.33-daniel.71", "0.9.33-daniel.72", "0.9.33-daniel.73", "0.9.30-daniel.1", "0.9.30-daniel.10", "0.9.30-daniel.11", "0.9.30-daniel.15", "0.9.30-daniel.16", "0.9.30-daniel.18", "0.9.30-daniel.19", "0.9.30-daniel.2", "0.9.30-daniel.20", "0.9.30-daniel.21", "0.9.30-daniel.22", "0.9.30-daniel.23", "0.9.30-daniel.24", "0.9.30-daniel.25", "0.9.30-daniel.26", "0.9.30-daniel.27", "0.9.30-daniel.28", "0.9.30-daniel.3", "0.9.30-daniel.4", "0.9.30-daniel.5", "0.9.30-daniel.6", "0.9.30-daniel.7", "0.9.30-daniel.8", "0.9.30-daniel.9", "0.9.30-daniel.29", "0.9.30-daniel.30", "0.9.30-daniel.31", "0.9.30-daniel.32"]); if (!supportedVersions.has(importData.version)) { return { success: false, @@ -483,11 +479,36 @@ export function registerExportImportFunction(sdk: IIIClient, state: StateKV): vo if (strategy === "replace") { const existing = replaceSessions; + const obsBySession: Array<{ session: Session; obs: CompressedObservation[] }> = []; + for (let i = 0; i < existing.length; i += IMPORT_CHUNK_SIZE) { + const chunk = existing.slice(i, i + IMPORT_CHUNK_SIZE); + obsBySession.push(...(await Promise.all(chunk.map(async (session) => ({ + session, + obs: await collectAll(kv, KV.observations(session.id)), + }))))); + } + const memories = await collectAll(kv, KV.memories); + const summaries = replaceSummaries; + const actions = await kv.list(KV.actions); + const actionEdges = await kv.list(KV.actionEdges); + const routines = await kv.list(KV.routines); + const signals = await kv.list(KV.signals); + const checkpoints = await kv.list(KV.checkpoints); + const sentinels = await kv.list(KV.sentinels); + const sketches = await kv.list(KV.sketches); + const crystals = await kv.list(KV.crystals); + const facets = await kv.list(KV.facets); + const lessons = await kv.list(KV.lessons); + const insights = await collectAll(kv, KV.insights); + const semanticRows = await collectAll<{ id: string }>(kv, KV.semantic); + const proceduralRows = await collectAll<{ id: string }>(kv, KV.procedural); + const profiles = await kv.list(KV.profiles); + const accessLogs = await kv.list(KV.accessLog); // Collect observation deletes across all sessions, then run them in // one bounded pass: a runChunked nested inside a runChunked callback // multiplies in-flight deletes to chunk-size squared. const obsDeletes: Array<{ sessionId: string; obsId: string; captureKey?: string }> = []; - await runChunked(existing, async (session) => { + await runChunked(obsBySession, async ({ session, obs }) => { await kv.delete(KV.sessions, session.id); await kv.delete(KV.summaryRepairs, session.id); await removeSessionFromProjectIndex( @@ -495,9 +516,6 @@ export function registerExportImportFunction(sdk: IIIClient, state: StateKV): vo session.project, session.id, ).catch(() => {}); - const obs = await kv - .list(KV.observations(session.id)) - .catch(() => []); for (const o of obs) { obsDeletes.push({ sessionId: session.id, obsId: o.id, captureKey: o.captureKey }); } @@ -507,72 +525,55 @@ export function registerExportImportFunction(sdk: IIIClient, state: StateKV): vo await kv.delete(KV.observations(d.sessionId), d.obsId); await unindexObservationSession(kv, d.obsId).catch(() => {}); }); - await runChunked(await kv.list(KV.memories), (m) => + await runChunked(memories, (m) => kv.delete(KV.memories, m.id), ); - await runChunked( - replaceSummaries, - (s) => kv.delete(KV.summaries, s.sessionId), - ); - await runChunked(await kv.list(KV.actions).catch(() => []), (a) => + await runChunked(summaries, (s) => kv.delete(KV.summaries, s.sessionId)); + await runChunked(actions, (a) => kv.delete(KV.actions, a.id), ); - await runChunked( - await kv.list(KV.actionEdges).catch(() => []), - (e) => kv.delete(KV.actionEdges, e.id), + await runChunked(actionEdges, (e) => + kv.delete(KV.actionEdges, e.id), ); - await runChunked( - await kv.list(KV.routines).catch(() => []), - (r) => kv.delete(KV.routines, r.id), + await runChunked(routines, (r) => + kv.delete(KV.routines, r.id), ); - await runChunked( - await kv.list(KV.signals).catch(() => []), - (s) => kv.delete(KV.signals, s.id), + await runChunked(signals, (s) => + kv.delete(KV.signals, s.id), ); - await runChunked( - await kv.list(KV.checkpoints).catch(() => []), - (c) => kv.delete(KV.checkpoints, c.id), + await runChunked(checkpoints, (c) => + kv.delete(KV.checkpoints, c.id), ); - await runChunked( - await kv.list(KV.sentinels).catch(() => []), - (s) => kv.delete(KV.sentinels, s.id), + await runChunked(sentinels, (s) => + kv.delete(KV.sentinels, s.id), ); - await runChunked( - await kv.list(KV.sketches).catch(() => []), - (s) => kv.delete(KV.sketches, s.id), + await runChunked(sketches, (s) => + kv.delete(KV.sketches, s.id), ); - await runChunked( - await kv.list(KV.crystals).catch(() => []), - (c) => kv.delete(KV.crystals, c.id), + await runChunked(crystals, (c) => + kv.delete(KV.crystals, c.id), ); - await runChunked( - await kv.list(KV.facets).catch(() => []), - (f) => kv.delete(KV.facets, f.id), + await runChunked(facets, (f) => + kv.delete(KV.facets, f.id), ); - await runChunked( - await kv.list(KV.lessons).catch(() => []), - (l) => kv.delete(KV.lessons, l.id), + await runChunked(lessons, (l) => + kv.delete(KV.lessons, l.id), ); resetLessonIndex(); - await runChunked( - await kv.list(KV.insights).catch(() => []), - (i) => kv.delete(KV.insights, i.id), + await runChunked(insights, (i) => + kv.delete(KV.insights, i.id), ); - await runChunked( - await kv.list<{ id: string }>(KV.semantic).catch(() => []), - (s) => kv.delete(KV.semantic, s.id), + await runChunked(semanticRows, (s) => + kv.delete(KV.semantic, s.id), ); - await runChunked( - await kv.list<{ id: string }>(KV.procedural).catch(() => []), - (p) => kv.delete(KV.procedural, p.id), + await runChunked(proceduralRows, (p) => + kv.delete(KV.procedural, p.id), ); - await runChunked( - await kv.list(KV.profiles).catch(() => []), - (profile) => kv.delete(KV.profiles, profile.project), + await runChunked(profiles, (profile) => + kv.delete(KV.profiles, profile.project), ); - await runChunked( - await kv.list(KV.accessLog).catch(() => []), - (a) => kv.delete(KV.accessLog, a.memoryId), + await runChunked(accessLogs, (a) => + kv.delete(KV.accessLog, a.memoryId), ); } diff --git a/src/functions/facets.ts b/src/functions/facets.ts index 943ecbc017..65fa4e0d25 100644 --- a/src/functions/facets.ts +++ b/src/functions/facets.ts @@ -1,5 +1,6 @@ +import {isDeepStrictEqual} from "node:util"; import type { IIIClient } from "iii-sdk"; -import type { StateKV } from "../state/kv.js"; +import { collectAll, type StateKV } from "../state/kv.js"; import { lookupObservationSession } from "../state/obs-index.js"; import { KV, generateId } from "../state/schema.js"; import type { Facet, Lesson } from "../types.js"; @@ -10,9 +11,13 @@ interface TargetLiveness { record: boolean; } -async function targetLiveness(kv: StateKV, id: string): Promise { +function checksObservation(id: string, targetType?: string): boolean { + return targetType === "observation" || id.startsWith("obs_"); +} + +async function targetLiveness(kv: StateKV, id: string, observationTarget = checksObservation(id)): Promise { let observation = false; - if (id.startsWith("obs_")) { + if (observationTarget) { const sessionId = await lookupObservationSession(kv, id); observation = !!sessionId && !!(await kv.get(KV.observations(sessionId), id)); } @@ -48,7 +53,7 @@ export async function deleteFacetsForTargets( [...ids].map(async (id) => { for (let attempt = 0; attempt < 2; attempt++) { try { - live.set(id, await targetLiveness(kv, id)); + live.set(id, await targetLiveness(kv, id, targetTypes.has("observation") || checksObservation(id))); return; } catch { if (attempt === 1) { @@ -61,7 +66,7 @@ export async function deleteFacetsForTargets( } }), ); - const all = await kv.list(KV.facets); + const all = await collectAll(kv, KV.facets); const matches = all.filter( (f) => ids.has(f.targetId) && @@ -77,6 +82,70 @@ export async function deleteFacetsForTargets( } } +export async function findOrphanFacets(kv: StateKV): Promise { + const facets = await collectAll(kv, KV.facets); + const actionIds = facets.some((f) => f.targetType === "action") + ? new Set((await collectAll<{ id: string }>(kv, KV.actions)).map((a) => a.id)) + : new Set(); + const live = new Map(); + let observationIds: Set | null | undefined; + const orphans: Facet[] = []; + for (const facet of facets) { + if (facet.targetType === "action") { + if (!actionIds.has(facet.targetId)) orphans.push(facet); + continue; + } + const observationTarget = checksObservation(facet.targetId, facet.targetType); + const cacheKey = `${observationTarget ? "o" : "r"}:${facet.targetId}`; + if (!live.has(cacheKey)) { + live.set(cacheKey, await targetLiveness(kv, facet.targetId, observationTarget).catch(() => null)); + } + const liveness = live.get(cacheKey); + if (!liveness || facetTargetLive(facet, liveness)) continue; + if (observationTarget) { + if (observationIds === undefined) observationIds = await listObservationIds(kv); + if (!observationIds || observationIds.has(facet.targetId)) continue; + } + orphans.push(facet); + } + return orphans; +} + +const OBSERVATION_SCAN_BATCH = 16; +const UNSESSIONED_OBSERVATION_SCOPE = "migrated"; + +async function listObservationIds(kv: StateKV): Promise | null> { + try { + const sessions = await collectAll<{ id: string }>(kv, KV.sessions); + const scopes = [...new Set([...sessions.map((session) => session.id), UNSESSIONED_OBSERVATION_SCOPE])]; + const ids = new Set(); + for (let start = 0; start < scopes.length; start += OBSERVATION_SCAN_BATCH) { + const batch = scopes.slice(start, start + OBSERVATION_SCAN_BATCH); + const lists = await Promise.all(batch.map((scope) => collectAll<{ id: string }>(kv, KV.observations(scope)))); + for (const list of lists) for (const obs of list) ids.add(obs.id); + } + return ids; + } catch { + return null; + } +} + +export async function isFacetStillOrphan(kv: StateKV, facet: Facet): Promise { + const current = await kv.get(KV.facets, facet.id); + if (!current || !isDeepStrictEqual(current, facet)) return false; + if (facet.targetType === "action") return !(await kv.get(KV.actions, facet.targetId)); + const observationTarget = checksObservation(facet.targetId, facet.targetType); + const liveness = await targetLiveness(kv, facet.targetId, observationTarget).catch(() => null); + if (!liveness || facetTargetLive(facet, liveness)) return false; + if (observationTarget) { + // Recheck the fallback scan too: an observation may reappear without its + // session index between diagnosis and repair. Unknown liveness is not deletion proof. + const observationIds = await listObservationIds(kv); + if (!observationIds || observationIds.has(facet.targetId)) return false; + } + return true; +} + export function registerFacetsFunction(sdk: IIIClient, kv: StateKV): void { sdk.registerFunction("mem::facet-tag", async (data: { @@ -116,7 +185,7 @@ export function registerFacetsFunction(sdk: IIIClient, kv: StateKV): void { const dimension = data.dimension.trim(); const value = data.value.trim(); - const existing = await kv.list(KV.facets); + const existing = await collectAll(kv, KV.facets); const duplicate = existing.find( (f) => f.targetId === data.targetId && @@ -154,7 +223,7 @@ export function registerFacetsFunction(sdk: IIIClient, kv: StateKV): void { return { success: false, error: "dimension is required" }; } - const all = await kv.list(KV.facets); + const all = await collectAll(kv, KV.facets); const matches = all.filter((f) => { if (f.targetId !== data.targetId || f.dimension !== data.dimension) { return false; @@ -190,7 +259,7 @@ export function registerFacetsFunction(sdk: IIIClient, kv: StateKV): void { }; } - const all = await kv.list(KV.facets); + const all = await collectAll(kv, KV.facets); const filtered = data.targetType ? all.filter((f) => f.targetType === data.targetType) : all; @@ -245,7 +314,7 @@ export function registerFacetsFunction(sdk: IIIClient, kv: StateKV): void { return { success: false, error: "targetId is required" }; } - const all = await kv.list(KV.facets); + const all = await collectAll(kv, KV.facets); const targetFacets = all.filter((f) => f.targetId === data.targetId); const dimMap = new Map(); @@ -269,7 +338,7 @@ export function registerFacetsFunction(sdk: IIIClient, kv: StateKV): void { sdk.registerFunction("mem::facet-stats", async (data: { targetType?: string }) => { - const all = await kv.list(KV.facets); + const all = await collectAll(kv, KV.facets); const filtered = data.targetType ? all.filter((f) => f.targetType === data.targetType) : all; @@ -298,7 +367,7 @@ export function registerFacetsFunction(sdk: IIIClient, kv: StateKV): void { sdk.registerFunction("mem::facet-dimensions", async () => { - const all = await kv.list(KV.facets); + const all = await collectAll(kv, KV.facets); const counts = new Map(); for (const f of all) { diff --git a/src/functions/governance.ts b/src/functions/governance.ts index 1c48d655ac..41ac200c62 100644 --- a/src/functions/governance.ts +++ b/src/functions/governance.ts @@ -2,12 +2,12 @@ import type { IIIClient } from "iii-sdk"; import type { Memory, GovernanceFilter, AuditEntry } from "../types.js"; import { KV } from "../state/schema.js"; import type { StateKV } from "../state/kv.js"; +import { collectAll } from "../state/kv.js"; import { recordAudit, safeAudit, queryAudit } from "./audit.js"; import { deleteAccessLog } from "./access-tracker.js"; import { deleteFacetsForTargets } from "./facets.js"; import { getSearchIndex, vectorIndexRemove, flushIndexSave } from "./search.js"; import { logger } from "../logger.js"; - const BULK_FILTER_KEYS = ["type", "dateFrom", "dateTo", "project", "qualityBelow"]; const BULK_CONTROL_KEYS = ["dryRun", "reason"]; @@ -134,7 +134,7 @@ export function registerGovernanceFunction(sdk: IIIClient, kv: StateKV): void { }; } - const memories = await kv.list(KV.memories); + const memories = await collectAll(kv, KV.memories); let candidates = memories; if (data.type && data.type.length > 0) { diff --git a/src/functions/graph.ts b/src/functions/graph.ts index 7f175e00a6..e7ea89148a 100644 --- a/src/functions/graph.ts +++ b/src/functions/graph.ts @@ -14,6 +14,7 @@ import type { } from "../types.js"; import { KV, generateId } from "../state/schema.js"; import type { StateKV } from "../state/kv.js"; +import { collectAll } from "../state/kv.js"; import { withKeyedLock } from "../state/keyed-mutex.js"; import { GRAPH_EXTRACTION_SYSTEM, @@ -23,7 +24,6 @@ import { isGraphExtractionEnabled } from "../config.js"; import { recordAudit, safeAudit } from "./audit.js"; import { logger } from "../logger.js"; import { scrubRecord } from "./privacy.js"; - const DEFAULT_GRAPH_QUERY_LIMIT = 500; const SNAPSHOT_TOP_NODES = DEFAULT_GRAPH_QUERY_LIMIT; @@ -699,7 +699,7 @@ export async function compactGraphProvenance( } if (scope === undefined || scope === "history") { const h = await trimScope( - async () => (await kv.list(KV.graphEdgeHistory)).map((r) => r?.id), + async () => (await collectAll(kv, KV.graphEdgeHistory)).map((r) => r?.id), KV.graphEdgeHistory, ); result.historyScanned = h.scanned; @@ -895,8 +895,8 @@ export function registerGraphFunction( const outcome = await withKeyedLock("graph:persist", async () => { const [nodes, edges] = await withTimeout( Promise.all([ - kv.list(KV.graphNodes), - kv.list(KV.graphEdges), + collectAll(kv, KV.graphNodes), + collectAll(kv, KV.graphEdges), ]), LIVE_ENUMERATION_BUDGET_MS, "graph-snapshot-rebuild enumeration", diff --git a/src/functions/mesh.ts b/src/functions/mesh.ts index 794a004da2..002a8017b7 100644 --- a/src/functions/mesh.ts +++ b/src/functions/mesh.ts @@ -2,6 +2,7 @@ import { pinGraphState } from "../state/graph-generation.js"; import { persistGraphRecords } from "../state/graph-ingestion.js"; import type { IIIClient } from "iii-sdk"; import type { StateKV } from "../state/kv.js"; +import { collectAll } from "../state/kv.js"; import { KV, generateId } from "../state/schema.js"; import { withKeyedLock } from "../state/keyed-mutex.js"; import { recordAudit } from "./audit.js"; @@ -56,6 +57,8 @@ async function isAllowedUrl(urlStr: string): Promise { } } +const MESH_PULL_PAGE_LIMIT = 500; + const DEFAULT_SHARED_SCOPES = [ "memories", "actions", @@ -233,6 +236,7 @@ export function registerMeshFunction( continue; } + const syncStartedAt = new Date().toISOString(); if (direction === "push" || direction === "both") { const pushData = await collectSyncData(kv, scopes, peer.lastSyncAt, peer.syncFilter); try { @@ -259,24 +263,45 @@ export function registerMeshFunction( if (direction === "pull" || direction === "both") { try { - const response = await fetch( - `${peer.url}/agentmemory/mesh/export?since=${peer.lastSyncAt || ""}`, - { - headers: { - Authorization: `Bearer ${meshAuthToken}`, + let cursor: string | undefined; + let pulled = 0; + let failed = false; + let pages = 0; + for (;;) { + const response = await fetch( + `${peer.url}/agentmemory/mesh/export?since=${peer.lastSyncAt || ""}&limit=${MESH_PULL_PAGE_LIMIT}${peer.syncFilter?.project ? `&project=${encodeURIComponent(peer.syncFilter.project)}` : ""}${cursor ? `&cursor=${encodeURIComponent(cursor)}` : ""}`, + { + headers: { + Authorization: `Bearer ${meshAuthToken}`, + }, + signal: AbortSignal.timeout(30000), + redirect: "error", }, - signal: AbortSignal.timeout(30000), - redirect: "error", - }, - ); - if (response.ok) { - const pullData = (await response.json()) as { - memories?: Memory[]; - actions?: Action[]; - }; - result.pulled = await applySyncData(kv, pullData, scopes); - } else { - result.errors.push(`pull failed: HTTP ${response.status}`); + ); + if (!response.ok) { + result.errors.push(`pull failed: HTTP ${response.status}`); + failed = true; + break; + } + const pullData = (await response.json()) as MeshSyncPayload & { nextCursor?: string }; + pulled += await applySyncData(kv, pullData, scopes); + pages++; + const next = typeof pullData.nextCursor === "string" ? pullData.nextCursor : undefined; + if (next && next === cursor) { + result.errors.push("pull failed: cursor did not advance"); + failed = true; + break; + } + cursor = next; + if (!cursor) break; + if (pages >= 10000) { + result.errors.push("pull failed: page limit reached"); + failed = true; + break; + } + } + if (!failed) { + result.pulled = pulled; } } catch (err) { result.errors.push(`pull failed: ${String(err)}`); @@ -285,7 +310,7 @@ export function registerMeshFunction( peer.status = result.errors.length > 0 ? "error" : "connected"; if (result.errors.length === 0) { - peer.lastSyncAt = new Date().toISOString(); + peer.lastSyncAt = syncStartedAt; } } catch (err) { peer.status = "disconnected"; @@ -389,7 +414,10 @@ async function collectSyncData( const sinceTime = Number.isNaN(parsed) ? 0 : parsed; if (scopes.includes("memories")) { - const all = await kv.list(KV.memories); + let all = await collectAll(kv, KV.memories); + if (syncFilter?.project) { + all = all.filter((m) => m.project === syncFilter.project); + } result.memories = deltaFilter(all, sinceTime, "updatedAt"); } @@ -404,29 +432,29 @@ async function collectSyncData( const projectScoped = !!syncFilter?.project; if (scopes.includes("semantic") && !projectScoped) { - const all = await kv.list(KV.semantic); + const all = await collectAll(kv, KV.semantic); result.semantic = deltaFilter(all, sinceTime, "updatedAt"); } if (scopes.includes("procedural") && !projectScoped) { - const all = await kv.list(KV.procedural); + const all = await collectAll(kv, KV.procedural); result.procedural = deltaFilter(all, sinceTime, "updatedAt"); } if (scopes.includes("relations") && !projectScoped) { - const all = await kv.list(KV.relations); + const all = await collectAll(kv, KV.relations); result.relations = deltaFilter(all, sinceTime, "createdAt"); } if (scopes.includes("graph:nodes") && !projectScoped) { - const all = await kv.list(KV.graphNodes); + const all = await collectAll(kv, KV.graphNodes); result.graphNodes = all.filter( (n) => new Date(graphNodeTs(n)).getTime() > sinceTime, ); } if (scopes.includes("graph:edges") && !projectScoped) { - const all = await kv.list(KV.graphEdges); + const all = await collectAll(kv, KV.graphEdges); result.graphEdges = deltaFilter(all, sinceTime, "createdAt"); } diff --git a/src/functions/migrate-vector-index.ts b/src/functions/migrate-vector-index.ts index 3139e96e0d..e2d1b81a7b 100644 --- a/src/functions/migrate-vector-index.ts +++ b/src/functions/migrate-vector-index.ts @@ -2,6 +2,7 @@ import type { EmbeddingProvider, CompressedObservation, Memory } from "../types. import { VectorIndex } from "../state/vector-index.js"; import { KV } from "../state/schema.js"; import type { StateKV } from "../state/kv.js"; +import { collectAll } from "../state/kv.js"; import { logger } from "../logger.js"; import { observationEmbedText } from "../state/memory-utils.js"; @@ -52,7 +53,7 @@ export async function migrateVectorIndex( // size of the batch we were about to embed), not a flat +1. let textMems: Memory[] = []; try { - const memories = await kv.list(KV.memories); + const memories = await collectAll(kv, KV.memories); textMems = memories.filter( (m) => m.isLatest !== false && m.title && m.content && m.content.trim() !== "", ); diff --git a/src/functions/migrate.ts b/src/functions/migrate.ts index 2a2564f02c..3a3e17e61e 100644 --- a/src/functions/migrate.ts +++ b/src/functions/migrate.ts @@ -2,7 +2,7 @@ import type { IIIClient } from "iii-sdk"; import { resolve } from "node:path"; import { homedir } from "node:os"; import { KV, generateId } from "../state/schema.js"; -import { StateKV } from "../state/kv.js"; +import { StateKV, collectAll } from "../state/kv.js"; import { addSessionToProjectIndex } from "../state/session-index.js"; import { withKeyedLock } from "../state/keyed-mutex.js"; import type { @@ -22,7 +22,7 @@ export async function inferMemoryProjects( kv: StateKV, dryRun = false, ): Promise<{ updated: number; skipped: number; ambiguous: number }> { - const memories = await kv.list(KV.memories); + const memories = await collectAll(kv, KV.memories); const sessionCache = new Map(); const loadSession = async (sid: string): Promise => { diff --git a/src/functions/observe.ts b/src/functions/observe.ts index a506961011..247bfc9ac8 100644 --- a/src/functions/observe.ts +++ b/src/functions/observe.ts @@ -189,7 +189,7 @@ export function registerObserveFunction( const livePrompt = submittedPrompt && !backfillPrompt ? submittedPrompt : undefined; const promptRef = durable || typeof payload.eventId !== "string" ? obsId : payload.eventId; - const deferCertification = kv.backend !== "file" || (typeof kv.observationReader?.mutationRevision === "function" && typeof kv.observationReader?.pendingMutations === "function"); + const deferCertification = kv.backend !== "file" || typeof kv.durability?.watchKeys === "function"; let certify: () => Promise = async () => {}; const result = await withKeyedLock(`obs:${payload.sessionId}`, async () => { // A timed-out capture can already be stored; replay must not scan the session again. @@ -395,7 +395,6 @@ export function registerObserveFunction( synthetic = (await kv.get(KV.observations(payload.sessionId), obsId))!; getSearchIndex().add(synthetic); scheduleIndexSave(); - certify = await prepareObservationEmbedding(synthetic.id, synthetic.sessionId, deferCertification); emitLiveEvent({ id: `compressed-${obsId}`, type: "compressed_observation", data: { type: "compressed", observation: withoutObservationSource(synthetic), sessionId: payload.sessionId } }); @@ -405,6 +404,7 @@ export function registerObserveFunction( hook: payload.hookType, compress: autoCompress ? "llm" : "synthetic", }); + certify = await prepareObservationEmbedding(synthetic.id, synthetic.sessionId, deferCertification); return { observationId: obsId }; }); await certify(); diff --git a/src/functions/obsidian-export.ts b/src/functions/obsidian-export.ts index 467eadb420..12f81f0bf7 100644 --- a/src/functions/obsidian-export.ts +++ b/src/functions/obsidian-export.ts @@ -11,6 +11,8 @@ import type { } from "../types.js"; import { recordAudit } from "./audit.js"; import { confinePath, mkdirConfined, writeConfinedFile } from "./path-guard.js"; +import { collectAll } from "../state/kv.js"; + const DEFAULT_EXPORT_ROOT = join(homedir(), ".agentmemory"); function getExportRoot(): string { @@ -286,7 +288,7 @@ export function registerObsidianExportFunction( const sessionMoc: string[] = []; const [memories, lessons, crystals, sessions] = await Promise.all([ - exportTypes.has("memories") ? kv.list(KV.memories) : Promise.resolve([] as Memory[]), + exportTypes.has("memories") ? collectAll(kv, KV.memories) : Promise.resolve([] as Memory[]), exportTypes.has("lessons") ? kv.list(KV.lessons) : Promise.resolve([] as Lesson[]), exportTypes.has("crystals") ? kv.list(KV.crystals) : Promise.resolve([] as Crystal[]), exportTypes.has("sessions") ? kv.list(KV.sessions) : Promise.resolve([] as Session[]), diff --git a/src/functions/privacy.ts b/src/functions/privacy.ts index ad4d3aaaaa..002475fcff 100644 --- a/src/functions/privacy.ts +++ b/src/functions/privacy.ts @@ -51,7 +51,11 @@ const URL_CREDENTIALS_RE = /\b([a-z][a-z0-9+.-]*:\/\/)[^\s/?#@:"'<>]*:[^\s/?#@"'<>]+@/gi; export function stripPrivateData(input: string): string { - let result = redactPrivateBlocks(input); + return stripSecrets(redactPrivateBlocks(input)); +} + +export function stripSecrets(input: string): string { + let result = input; result = result.replace(new RegExp(PRIVATE_KEY_BLOCK_RE.source, PRIVATE_KEY_BLOCK_RE.flags), "[REDACTED_SECRET]"); result = result.replace(new RegExp(URL_CREDENTIALS_RE.source, URL_CREDENTIALS_RE.flags), "$1[REDACTED_SECRET]@"); for (const source of SECRET_PATTERN_SOURCES) { @@ -61,14 +65,18 @@ export function stripPrivateData(input: string): string { return result; } -export function scrubRecord(value: T, options: { scrubKeys?: boolean } = {}): T { - if (typeof value === "string") return stripPrivateData(value) as T; +export function scrubRecord( + value: T, + options: { scrubKeys?: boolean; strip?: (input: string) => string } = {}, +): T { + const strip = options.strip ?? stripPrivateData; + if (typeof value === "string") return strip(value) as T; if (Array.isArray(value)) return value.map((item) => scrubRecord(item, options)) as T; if (value !== null && typeof value === "object") { const proto = Object.getPrototypeOf(value); if (proto !== Object.prototype && proto !== null) return value; const entries = Object.entries(value as Record).map(([key, item]) => [ - options.scrubKeys ? stripPrivateData(key) : key, + options.scrubKeys ? strip(key) : key, item, ] as const); const reserved = new Set(entries.map(([key]) => key)); diff --git a/src/functions/profile.ts b/src/functions/profile.ts index 127b6940b6..45c321b759 100644 --- a/src/functions/profile.ts +++ b/src/functions/profile.ts @@ -6,7 +6,7 @@ import type { ProjectProfile, } from "../types.js"; import { KV } from "../state/schema.js"; -import { StateKV } from "../state/kv.js"; +import { StateKV, collectAll } from "../state/kv.js"; import { logger } from "../logger.js"; export function registerProfileFunction(sdk: IIIClient, kv: StateKV): void { @@ -92,8 +92,7 @@ export function registerProfileFunction(sdk: IIIClient, kv: StateKV): void { } } - const memories = await kv - .list(KV.memories) + const memories = await collectAll(kv, KV.memories) .catch(() => [] as Memory[]); for (const memory of memories) { if (memory.project !== project || memory.isLatest === false) continue; diff --git a/src/functions/reflect.ts b/src/functions/reflect.ts index 5effb6ed51..2a406ef4ff 100644 --- a/src/functions/reflect.ts +++ b/src/functions/reflect.ts @@ -3,6 +3,7 @@ import { createGraphIndexReader } from "../state/graph-retrieval-index.js"; import { lookupObservationSession } from "../state/obs-index.js"; import type { IIIClient } from "iii-sdk"; import type { StateKV } from "../state/kv.js"; +import { collectAll } from "../state/kv.js"; import { KV, fingerprintId } from "../state/schema.js"; import type { Insight, @@ -19,6 +20,45 @@ import type { import { recordAudit } from "./audit.js"; import { REFLECT_SYSTEM, buildReflectPrompt } from "../prompts/reflect.js"; import { withKeyedLock } from "../state/keyed-mutex.js"; +import { checkPayloadFrameSize, payloadByteLength } from "../state/frame-guard.js"; + +export const INSIGHT_MAX_RESULTS = 500; +export const INSIGHT_MAX_SOURCES = 20; +export const REFLECT_MAX_RECORD_BYTES = 4 * 1024; +export const REFLECT_MAX_TYPE_BYTES = 16 * 1024; +export const REFLECT_MAX_CONCEPTS = 32; +export const REFLECT_MAX_CONCEPT_BYTES = 128; +export const REFLECT_MAX_RESPONSE_BYTES = 64 * 1024; + +function positiveLimit(value: unknown, fallback: number, maximum: number): number | null { + if (value === undefined) return fallback; + if (typeof value !== "number" || !Number.isSafeInteger(value) || value < 1) return null; + return Math.min(value, maximum); +} + +function validConfidence(value: unknown): boolean { + return value === undefined || (typeof value === "number" && Number.isFinite(value) && value >= 0 && value <= 1); +} + +function invalidOption(error: string): { success: false; invalid: true; error: string } { + return { success: false, invalid: true, error }; +} + +function selectEvidence(items: T[], record: (item: T) => unknown): T[] { + const selected: T[] = []; + const seen = new Set(); + let bytes = 0; + for (const item of items) { + if (selected.length >= INSIGHT_MAX_SOURCES) break; + if (seen.has(item.id)) continue; + const size = payloadByteLength([item.id, record(item)]); + if (size > REFLECT_MAX_RECORD_BYTES || bytes + size > REFLECT_MAX_TYPE_BYTES) continue; + selected.push(item); + seen.add(item.id); + bytes += size; + } + return selected; +} interface ConceptCluster { concepts: string[]; @@ -40,10 +80,14 @@ function parseInsight(attributes: string, content: string, cluster: ConceptClust } const confidenceText = values.get("confidence")?.trim(); const confidence = Number(confidenceText); - const title = values.get("title")?.trim(); + let title = values.get("title")?.trim(); content = content.trim(); if (!confidenceText || !Number.isFinite(confidence) || confidence < 0 || confidence > 1 || !title || !content || /<\/?insight\b/.test(content)) return null; + title = decodeXmlText(title); + content = decodeXmlText(content); + if (Buffer.byteLength(title, "utf8") > 256 || Buffer.byteLength(content, "utf8") > REFLECT_MAX_RECORD_BYTES) return null; + const sourceMemoryIds = new Set(); const sourceLessonIds = new Set(); const sourceCrystalIds = new Set(); @@ -58,9 +102,10 @@ function parseInsight(attributes: string, content: string, cluster: ConceptClust if (typeof id !== "string" || !id) return null; selected.add(id); } - if (sourceMemoryIds.size + sourceLessonIds.size + sourceCrystalIds.size < 2) return null; + const sourceCount = sourceMemoryIds.size + sourceLessonIds.size + sourceCrystalIds.size; + if (sourceCount < 2 || sourceCount > INSIGHT_MAX_SOURCES) return null; return { - confidence, title: decodeXmlText(title), content: decodeXmlText(content), + confidence, title, content, sourceMemoryIds: [...sourceMemoryIds], sourceLessonIds: [...sourceLessonIds], sourceCrystalIds: [...sourceCrystalIds], @@ -308,23 +353,26 @@ export function registerReflectFunctions( const kv = state; sdk.registerFunction("mem::reflect", async (data: { maxClusters?: number; project?: string }) => { - if (operationalState.featureLifecycle) return operationalState.featureLifecycle.reflect(data ?? {}); + const maxClusters = positiveLimit(data?.maxClusters, 10, 20); + if (maxClusters === null) return invalidOption("maxClusters must be a positive safe integer"); + if (operationalState.featureLifecycle) return operationalState.featureLifecycle.reflect({ ...data, maxClusters }); const kv = state; - const maxClusters = Math.min(data?.maxClusters ?? 10, 20); const maxInsightsPerCluster = 5; const maxTotal = 50; const [graph, semanticMemories, lessons, crystals] = await Promise.all([ reflectionGraph(kv, data?.project), - kv.list(KV.semantic).catch(() => []), - kv.list(KV.lessons).catch(() => []), - kv.list(KV.crystals).catch(() => []), + collectAll(kv, KV.semantic).catch(() => []), + collectAll(kv, KV.lessons).catch(() => []), + collectAll(kv, KV.crystals).catch(() => []), ]); let activeLessons = lessons.filter((l) => !l.deleted); + let scopedCrystals = crystals; if (data?.project) { activeLessons = activeLessons.filter((l) => l.project === data.project); + scopedCrystals = crystals.filter((c) => c.project === data.project); } let conceptClusters = buildGraphClusters( @@ -351,30 +399,33 @@ export function registerReflectFunctions( let failedClusters = 0; const acceptedIds = new Set(); - for (const conceptNames of conceptClusters) { + for (const candidates of conceptClusters) { if (totalInsights >= maxTotal) break; + const conceptNames = [...new Set(candidates)] + .filter((name) => payloadByteLength(name) <= REFLECT_MAX_CONCEPT_BYTES) + .slice(0, REFLECT_MAX_CONCEPTS); const conceptSet = new Set(conceptNames.map((c) => c.toLowerCase())); - const clusterFacts = semanticMemories.filter((s) => { + const clusterFacts = selectEvidence(semanticMemories.filter((s) => { const factTerms = s.fact.toLowerCase().split(/\s+/); return factTerms.some((t) => conceptSet.has(t)); - }); + }), (f) => ({ source: `F${INSIGHT_MAX_SOURCES}`, fact: f.fact, confidence: f.confidence })); - const clusterLessons = activeLessons.filter((l) => + const clusterLessons = selectEvidence(activeLessons.filter((l) => l.tags.some((t) => conceptSet.has(t.toLowerCase())) || conceptNames.some((c) => l.content.toLowerCase().includes(c.toLowerCase()), ), - ); + ), (l) => ({ source: `L${INSIGHT_MAX_SOURCES}`, content: l.content, confidence: l.confidence })); - const clusterCrystals = crystals.filter((c) => + const clusterCrystals = selectEvidence(scopedCrystals.filter((c) => (c.lessons || []).some((l) => conceptNames.some((cn) => l.toLowerCase().includes(cn.toLowerCase()), ), ), - ); + ), (c) => ({ source: `C${INSIGHT_MAX_SOURCES}`, narrative: c.narrative })); const totalItems = clusterFacts.length + clusterLessons.length + clusterCrystals.length; @@ -402,6 +453,10 @@ export function registerReflectFunctions( try { const prompt = buildReflectPrompt(cluster); const response = await provider.summarize(REFLECT_SYSTEM, prompt); + if (Buffer.byteLength(response, "utf8") > REFLECT_MAX_RESPONSE_BYTES) { + failedClusters++; + continue; + } const matches = [...response.matchAll(/]*)>([\s\S]*?)<\/insight>/g)]; const incomplete = (response.match(/ { - const limit = data?.limit ?? 50; + const limit = positiveLimit(data?.limit, 50, INSIGHT_MAX_RESULTS); + if (limit === null) return invalidOption("limit must be a positive safe integer"); + if (!validConfidence(data?.minConfidence)) return invalidOption("minConfidence must be a finite number between 0 and 1"); const minConfidence = data?.minConfidence ?? 0; - let items = await kv.list(KV.insights); + let items = await collectAll(kv, KV.insights); items = items.filter( (i) => !i.deleted && i.confidence >= minConfidence, @@ -524,7 +584,8 @@ export function registerReflectFunctions( items.sort((a, b) => b.confidence - a.confidence); - return { success: true, insights: items.slice(0, limit) }; + const result = { success: true, insights: items.slice(0, limit) }; + return checkPayloadFrameSize(result, "reduce limit or filter by project") ?? result; }, ); @@ -535,15 +596,16 @@ export function registerReflectFunctions( minConfidence?: number; limit?: number; }) => { - if (!data?.query?.trim()) { - return { success: false, error: "query is required" }; + if (typeof data?.query !== "string" || !data.query.trim()) { + return invalidOption("query is required"); } - + const limit = positiveLimit(data.limit, 10, INSIGHT_MAX_RESULTS); + if (limit === null) return invalidOption("limit must be a positive safe integer"); + if (!validConfidence(data.minConfidence)) return invalidOption("minConfidence must be a finite number between 0 and 1"); const query = data.query.toLowerCase(); const minConfidence = data.minConfidence ?? 0.1; - const limit = data.limit ?? 10; - let items = await kv.list(KV.insights); + let items = await collectAll(kv, KV.insights); items = items.filter( (i) => !i.deleted && i.confidence >= minConfidence, ); @@ -582,19 +644,20 @@ export function registerReflectFunctions( }); } catch {} - return { + const result = { success: true, insights: scored.slice(0, limit).map((s) => ({ ...s.insight, score: Math.round(s.score * 1000) / 1000, })), }; + return checkPayloadFrameSize(result, "reduce limit or narrow the search") ?? result; }, ); sdk.registerFunction("mem::insight-decay-sweep", async () => { - const items = await kv.list(KV.insights); + const items = await collectAll(kv, KV.insights); let decayed = 0; let softDeleted = 0; const now = Date.now(); diff --git a/src/functions/relations.ts b/src/functions/relations.ts index d384052cf6..4a52c5ea8b 100644 --- a/src/functions/relations.ts +++ b/src/functions/relations.ts @@ -9,7 +9,7 @@ import { vectorIndexRemove, } from "./search.js"; import { KV, generateId } from "../state/schema.js"; -import { StateKV } from "../state/kv.js"; +import { StateKV, collectAll } from "../state/kv.js"; import { withKeyedLock } from "../state/keyed-mutex.js"; import { safeAudit } from "./audit.js"; import { recordAccessBatch } from "./access-tracker.js"; @@ -285,8 +285,7 @@ export function registerRelationsFunction(sdk: IIIClient, kv: StateKV): void { ? Math.max(0, Math.min(1, rawMinConf)) : 0; - const allRelations = await kv - .list(KV.relations) + const allRelations = await collectAll(kv, KV.relations) .catch(() => []); const visited = new Set(); diff --git a/src/functions/remember.ts b/src/functions/remember.ts index d0e20a7843..587c931d8e 100644 --- a/src/functions/remember.ts +++ b/src/functions/remember.ts @@ -2,7 +2,7 @@ import { TriggerAction, type IIIClient } from "iii-sdk"; import { markCaptureEventDeleted } from "../capture/event-record.js"; import type { Memory, Session } from "../types.js"; import { KV, generateId, jaccardSimilarity } from "../state/schema.js"; -import { StateKV } from "../state/kv.js"; +import { StateKV, collectAll } from "../state/kv.js"; import { withKeyedLock } from "../state/keyed-mutex.js"; import { removeSessionFromProjectIndex } from "../state/session-index.js"; import { unindexObservationSession } from "../state/obs-index.js"; @@ -108,7 +108,7 @@ export function registerRememberFunction(sdk: IIIClient, kv: StateKV): void { ); candidateMemories = loaded.filter((m): m is Memory => m !== null); } else { - candidateMemories = await kv.list(KV.memories); + candidateMemories = await collectAll(kv, KV.memories); } } catch (err) { // Candidate generation is an optimization; a failure here must @@ -116,7 +116,7 @@ export function registerRememberFunction(sdk: IIIClient, kv: StateKV): void { logger.warn("supersession candidate lookup failed, using full scan", { error: err instanceof Error ? err.message : JSON.stringify(err), }); - candidateMemories = await kv.list(KV.memories); + candidateMemories = await collectAll(kv, KV.memories); } let supersededId: string | undefined; let supersededVersion = 1; @@ -166,6 +166,7 @@ export function registerRememberFunction(sdk: IIIClient, kv: StateKV): void { sessionIds: [], strength: 7, version: supersededId ? supersededVersion + 1 : 1, + ...(supersededMemory ? { recurrence: (supersededMemory.recurrence ?? 1) + 1 } : {}), parentId: supersededId, supersedes: supersededId ? [supersededId] : [], sourceObservationIds: (data.sourceObservationIds || []).filter( @@ -338,12 +339,15 @@ export function registerRememberFunction(sdk: IIIClient, kv: StateKV): void { if (data.memoryId) { const memoryId = data.memoryId; await attempt(memoryId, async () => { - const mem = await kv.get(KV.memories, memoryId); + const mem = await withKeyedLock("mem:remember", async () => { + const current = await kv.get(KV.memories, memoryId); + if (current) await kv.delete(KV.memories, memoryId); + return current; + }); if (!mem) { await deleteFacetsForTargets(kv, [memoryId]); return false; } - await kv.delete(KV.memories, memoryId); deletedMemoryIds.push(memoryId); getSearchIndex().remove(memoryId); vectorIndexRemove(memoryId); diff --git a/src/functions/retention.ts b/src/functions/retention.ts index 74feb1d9e5..97c05db910 100644 --- a/src/functions/retention.ts +++ b/src/functions/retention.ts @@ -7,6 +7,7 @@ import type { } from "../types.js"; import { KV } from "../state/schema.js"; import type { StateKV } from "../state/kv.js"; +import { collectAll } from "../state/kv.js"; import type { AccessLog } from "./access-tracker.js"; import { emptyAccessLog, @@ -18,7 +19,6 @@ import { deleteFacetsForTargets } from "./facets.js"; import { getSearchIndex, vectorIndexRemove, flushIndexSave } from "./search.js"; import { logger } from "../logger.js"; import { getAuditRetentionMonths } from "../config.js"; - const DEFAULT_DECAY: DecayConfig = { lambda: 0.01, sigma: 0.3, @@ -135,8 +135,8 @@ export function registerRetentionFunctions( const { config } = resolved; const [memories, semanticMems, allLogs] = await Promise.all([ - kv.list(KV.memories), - kv.list(KV.semantic), + collectAll(kv, KV.memories), + collectAll(kv, KV.semantic), kv.list(KV.accessLog).catch(() => [] as unknown[]), ]); const logsById = new Map(); diff --git a/src/functions/search.ts b/src/functions/search.ts index b305e37ff8..bec51700f0 100644 --- a/src/functions/search.ts +++ b/src/functions/search.ts @@ -5,7 +5,7 @@ import type { HybridSearchRequest } from "../state/hybrid-search.js"; import type { IndexPersistenceStatus } from "../state/index-persistence.js"; import type { CompressedObservation, Memory, SearchResult, Session } from '../types.js' import { KV } from '../state/schema.js' -import { StateKV } from '../state/kv.js' +import { StateKV, collectAll } from '../state/kv.js' import { getSearchResultLayer, isSearchLayer, matchesSearchLayer, type SearchLayer, type SearchResultLayer } from "../state/search-layer.js"; import { SearchIndex, type SearchIndexRevision } from '../state/search-index.js' import { VectorIndex } from '../state/vector-index.js' @@ -189,6 +189,7 @@ type IndexPersistenceHook = { scheduleSave: () => void; save: () => Promise; status?: () => IndexPersistenceStatus; + verifyCaptureVectorPending?: (id: string, sessionId: string, pending: unknown) => Promise; }; let indexPersistence: IndexPersistenceHook | null = null; @@ -201,6 +202,10 @@ export function getIndexPersistenceStatus(): IndexPersistenceStatus | null { return indexPersistence?.status?.() ?? null; } +export async function verifyCaptureVectorPending(id: string, sessionId: string, pending: unknown): Promise { + return indexPersistence?.verifyCaptureVectorPending?.(id, sessionId, pending) ?? pending == null; +} + export function scheduleIndexSave(): void { indexPersistence?.scheduleSave(); } @@ -516,7 +521,7 @@ export async function rebuildIndex(kv: StateKV): Promise { let memories: Memory[] = [] let memoriesLoaded = false try { - memories = await kv.list(KV.memories) + memories = await collectAll(kv, KV.memories) memoriesLoaded = true } catch (err) { logger.warn('rebuildIndex: failed to load memories', { @@ -584,7 +589,7 @@ async function listWithRetry(kv: StateKV, scope: string, label: string, deadl while (attempts < LIST_RETRY_ATTEMPTS) { attempts++ try { - return await kv.list(scope) + return await collectAll(kv, scope) } catch (err) { lastErr = err if (!isStateNotReadyError(err)) break @@ -601,7 +606,7 @@ async function listWithRetry(kv: StateKV, scope: string, label: string, deadl return null } -type KeywordRebuildResult = { documents: number; vectorJobs: VectorBackfillJob[]; fullBackfillPending: number } +type KeywordRebuildResult = { documents: number; vectorJobs: VectorBackfillJob[]; fullBackfillPending: number; historicalVectorGaps: number } export async function rebuildKeywordIndex( kv: StateKV, @@ -635,6 +640,7 @@ async function runKeywordRebuild( const backfillCap = backfillActive ? getVectorBackfillMax() : 0 const vectorJobs: VectorBackfillJob[] = [] let fullBackfillPending = 0 + let historicalVectorGaps = 0 const consider = ( id: string, sessionId: string, @@ -643,7 +649,7 @@ async function runKeywordRebuild( kind: "memory" | "observation", ): void => { if (!backfillEligible || vi?.has(id)) return - if (!Number.isNaN(cutoff) && !(Date.parse(timestamp ?? '') > cutoff)) return + if (!Number.isNaN(cutoff) && !(Date.parse(timestamp ?? '') > cutoff)) { historicalVectorGaps++; return } pendingVectorBackfillIds?.add(id) if (backfillGated || (wholeStoreBackfill && vectorJobs.length >= backfillCap)) { fullBackfillPending++ @@ -674,7 +680,7 @@ async function runKeywordRebuild( if (sessions === null) { bm25RebuildIncomplete = true if (memoriesLoaded) memoryIndexReady = true - return { documents, vectorJobs, fullBackfillPending } + return { documents, vectorJobs, fullBackfillPending, historicalVectorGaps } } const failedSessions: string[] = [] for (let batch = 0; batch < sessions.length; batch += 10) { @@ -701,7 +707,7 @@ async function runKeywordRebuild( logger.warn('rebuildKeywordIndex: failed to load observations for sessions', { failedSessions }) } if (memoriesLoaded) memoryIndexReady = true - return { documents, vectorJobs, fullBackfillPending } + return { documents, vectorJobs, fullBackfillPending, historicalVectorGaps } } const BACKFILL_SAVE_EVERY_BATCHES = 10 diff --git a/src/functions/skill-extract.ts b/src/functions/skill-extract.ts index fd9dcd26af..ad2977a87a 100644 --- a/src/functions/skill-extract.ts +++ b/src/functions/skill-extract.ts @@ -7,7 +7,7 @@ import type { MemoryProvider, } from "../types.js"; import { KV, generateId, fingerprintId } from "../state/schema.js"; -import { StateKV } from "../state/kv.js"; +import { StateKV, collectAll } from "../state/kv.js"; import { recordAudit } from "./audit.js"; import { logger } from "../logger.js"; @@ -243,7 +243,7 @@ export function registerSkillExtractFunctions( sdk.registerFunction("mem::skill-list", async (data: { limit?: number }) => { const limit = data?.limit ?? 50; - const skills = await kv.list(KV.procedural); + const skills = await collectAll(kv, KV.procedural); const sorted = skills.sort((a, b) => b.strength - a.strength); return { success: true, @@ -263,7 +263,7 @@ export function registerSkillExtractFunctions( const query = data.query.toLowerCase(); const terms = query.split(/\s+/).filter((t) => t.length > 2); - const skills = await kv.list(KV.procedural); + const skills = await collectAll(kv, KV.procedural); const scored = skills .map((skill) => { diff --git a/src/functions/snapshot.ts b/src/functions/snapshot.ts index e17bfcbf9a..f628585685 100644 --- a/src/functions/snapshot.ts +++ b/src/functions/snapshot.ts @@ -19,6 +19,7 @@ import type { } from "../types.js"; import { KV, generateId } from "../state/schema.js"; import type { StateKV } from "../state/kv.js"; +import { collectAll } from "../state/kv.js"; import type { SearchIndexRevision } from "../state/search-index.js"; import { addSessionToProjectIndex } from "../state/session-index.js"; import { withKeyedLock } from "../state/keyed-mutex.js"; @@ -28,7 +29,6 @@ import { persistGraphRecords, validateGraphRecords } from "../state/graph-ingest import { flushIndexSave, getSearchIndex, indexRecords, vectorIndexRemove } from "./search.js"; import { VERSION } from "../version.js"; import { logger } from "../logger.js"; - const COMMIT_HASH_RE = /^[0-9a-f]{7,40}$/i; const execFileAsync = promisify(execFile); @@ -65,11 +65,11 @@ export function registerSnapshotFunction( phase = "read_sessions"; const sessions = await kv.list(KV.sessions); phase = "read_memories"; - const memories = await kv.list(KV.memories); + const memories = await collectAll(kv, KV.memories); phase = "read_graph_nodes"; - const graphNodes = await kv.list(KV.graphNodes); + const graphNodes = await collectAll(kv, KV.graphNodes); phase = "read_graph_edges"; - const graphEdges = await kv.list(KV.graphEdges); + const graphEdges = await collectAll(kv, KV.graphEdges); phase = "read_access"; const accessLogs = await kv.list(KV.accessLog); diff --git a/src/functions/status.ts b/src/functions/status.ts index ee16db6359..671817f388 100644 --- a/src/functions/status.ts +++ b/src/functions/status.ts @@ -484,8 +484,8 @@ export function evaluateStatus(input: StatusInputs): StatusReport { problems.push({ level: "info", code: "capture-check-unavailable", - message: "The capture inbox did not answer in time, so capture was not checked.", - fix: "The check runs again on the next status request. If it keeps timing out, the state store is overloaded: check the server log.", + message: "The capture inbox was not checked or did not answer in time.", + fix: "Use the explicit capture status request to inspect the inbox. If that request times out, check the server log.", }); } problems.push(...captureProblems(input.capture ?? null, input.now, input.ports.rest)); @@ -721,10 +721,10 @@ function indexPersistenceRows(report: StatusReport): string { "Vector storage", escapeHtml(`${plural(persistence.buckets, "bucket")}, ${plural(persistence.pendingChanges, "unsaved change")}`), ); - if (persistence.pendingLog !== undefined) { + if (persistence.pendingLog !== undefined || persistence.pendingLogKnown === false) { rows += row( "Pending vector log", - escapeHtml(plural(persistence.pendingLog, "vector change")) + + (persistence.pendingLogKnown === false || persistence.pendingLog === undefined ? "not checked" : escapeHtml(plural(persistence.pendingLog, "vector change"))) + `

${escapeHtml(PENDING_LOG_EXPLAINER)}

`, ); } diff --git a/src/functions/summarize.ts b/src/functions/summarize.ts index 0be0d86bad..067bc4e941 100644 --- a/src/functions/summarize.ts +++ b/src/functions/summarize.ts @@ -25,7 +25,7 @@ import { logger } from "../logger.js"; import { scrubRecord } from "./privacy.js"; import { registerSummaryRecoveryFunction } from "./summary-recovery.js"; import { withKeyedLock } from "../state/keyed-mutex.js"; -import { summaryFingerprint, summaryObservations } from "./summary-state.js"; +import { getSummaryTimeoutMs, summaryFingerprint, summaryObservations } from "./summary-state.js"; import { registerLongSessionSummarizer } from "../ops/long-session.mjs"; type SummaryRequest = { sessionId: string; force?: boolean } | undefined; @@ -86,7 +86,7 @@ function requiresLongSession(session: Session, observations: CompressedObservati // LLM call. Default ≈ 50k input tokens per chunk at ~110 tok/obs — fits // comfortably in 128k-window models. Override via SUMMARIZE_CHUNK_SIZE. const CHUNK_SIZE_DEFAULT = 400; -export const SUMMARY_TIMEOUT_MS = 120_000; +export { SUMMARY_TIMEOUT_MS } from "./summary-state.js"; // Concurrent chunk calls share the summary deadline. Increase this for // large sessions only when the configured provider supports the load. const CHUNK_CONCURRENCY_DEFAULT = 6; @@ -400,13 +400,14 @@ export function registerSummarizeFunction( }; } + const timeoutMs = getSummaryTimeoutMs(); const boundedProvider: MemoryProvider = { name: provider.name, compress: (system, prompt) => provider.compress(system, prompt), summarize: async (system, prompt) => { if (longSessionHandle.snapshot().stopped) throw new Error("shutting_down"); if (prompt.length > summaryInputLimit()) throw new Error("summary_input_budget_exceeded"); - const remaining = SUMMARY_TIMEOUT_MS - (Date.now() - startMs); + const remaining = timeoutMs - (Date.now() - startMs); if (remaining <= 0) throw new Error("summary_timeout"); let timer: ReturnType | undefined; try { diff --git a/src/functions/summary-recovery.ts b/src/functions/summary-recovery.ts index b23c7220b1..81c2551443 100644 --- a/src/functions/summary-recovery.ts +++ b/src/functions/summary-recovery.ts @@ -6,7 +6,7 @@ import { withKeyedLock } from "../state/keyed-mutex.js"; import { logger } from "../logger.js"; import { compressionInFlight } from "./compress.js"; import { compressionRaw, needsCompression, type StoredObservation } from "./compression-state.js"; -import { summaryCovers, summaryObservations } from "./summary-state.js"; +import { getSummaryTriggerTimeoutMs, summaryCovers, summaryObservations } from "./summary-state.js"; export const SUMMARY_RETRY_MS = 30_000; export const SUMMARY_RETRY_MAX_MS = 60 * 60_000; @@ -55,7 +55,7 @@ export function registerSummaryRecoveryFunction(sdk: IIIClient, kv: StateKV, has const remaining = observations.filter(needsCompression); if (remaining.length === 0) { const result = await sdk.trigger<{ sessionId: string }, { success?: boolean; incomplete?: boolean }>({ - function_id: "mem::summarize", payload: { sessionId }, + function_id: "mem::summarize", payload: { sessionId }, timeoutMs: getSummaryTriggerTimeoutMs(), }); if (!result?.success || result.incomplete) lastError = "summary_failed"; } else if (remaining.every((row) => !compressionRaw(row))) { diff --git a/src/functions/summary-state.ts b/src/functions/summary-state.ts index 048c847d66..71eb5a3331 100644 --- a/src/functions/summary-state.ts +++ b/src/functions/summary-state.ts @@ -4,6 +4,21 @@ import type { StateKV } from "../state/kv.js"; import { KV } from "../state/schema.js"; import { isKeyedLockHeld, withKeyedLock } from "../state/keyed-mutex.js"; +export const SUMMARY_TIMEOUT_MS = 120_000; +const SUMMARY_TRIGGER_HEADROOM_MS = 60_000; + +const MAX_TIMER_MS = 2_147_483_647; + +export function getSummaryTimeoutMs(): number { + const raw = process.env.AGENTMEMORY_SUMMARY_TIMEOUT_MS?.trim() ?? ""; + const n = /^\d+$/.test(raw) ? Number(raw) : 0; + return n >= 1 && n <= MAX_TIMER_MS - SUMMARY_TRIGGER_HEADROOM_MS ? n : SUMMARY_TIMEOUT_MS; +} + +export function getSummaryTriggerTimeoutMs(): number { + return getSummaryTimeoutMs() + SUMMARY_TRIGGER_HEADROOM_MS; +} + export function summaryObservations(observations: CompressedObservation[]): CompressedObservation[] { return observations.filter((row) => row.title).sort((a, b) => (a.compressedAt ?? a.timestamp).localeCompare(b.compressedAt ?? b.timestamp) || a.id.localeCompare(b.id)); diff --git a/src/functions/working-memory.ts b/src/functions/working-memory.ts index 53e55a57d6..3e2a8fca80 100644 --- a/src/functions/working-memory.ts +++ b/src/functions/working-memory.ts @@ -1,7 +1,7 @@ import type { IIIClient } from "iii-sdk"; import type { Memory, CompressedObservation, ContextBlock } from "../types.js"; import { KV, generateId } from "../state/schema.js"; -import { StateKV } from "../state/kv.js"; +import { StateKV, collectAll } from "../state/kv.js"; import { recordAudit } from "./audit.js"; import { recordAccessBatch } from "./access-tracker.js"; import { logger } from "../logger.js"; @@ -132,7 +132,7 @@ export function registerWorkingMemoryFunctions( const archivalLines: string[] = []; - const memories = await kv.list(KV.memories); + const memories = await collectAll(kv, KV.memories); const active = memories .filter((m) => m.isLatest !== false) .sort((a, b) => { diff --git a/src/health/monitor.ts b/src/health/monitor.ts index c0cd3c2a02..ed372f4a64 100644 --- a/src/health/monitor.ts +++ b/src/health/monitor.ts @@ -1,3 +1,4 @@ +import { availableParallelism } from "node:os"; import { getCpuCapacityCores } from "../ops/metrics-cpu-capacity.mjs"; import { getHeapStatistics } from "node:v8"; import type { IIIClient } from "iii-sdk"; @@ -7,6 +8,15 @@ import { KV } from "../state/schema.js"; import { evaluateHealth } from "./thresholds.js"; import { getLiveEventsStats } from "../viewer/live-events.js"; +export function cpuCoreCount(): number { + const count = availableParallelism(); + return count > 0 ? count : 1; +} + +export function normalizeCpuPercent(singleCorePercent: number, coreCount = cpuCoreCount()): number { + return singleCorePercent / (coreCount > 0 ? coreCount : 1); +} + export function registerHealthMonitor( sdk: IIIClient, kv: StateKV, @@ -31,8 +41,10 @@ export function registerHealthMonitor( const elapsedMs = now - prevCpuTime; const userDelta = currentCpu.user - prevCpuUsage.user; const systemDelta = currentCpu.system - prevCpuUsage.system; - const cpuPercent = + const singleCoreCpuPercent = elapsedMs > 0 ? ((userDelta + systemDelta) / 1000 / elapsedMs) * 100 : 0; + const coreCount = cpuCoreCount(); + const cpuPercent = normalizeCpuPercent(singleCoreCpuPercent, coreCount); prevCpuUsage = currentCpu; prevCpuTime = now; @@ -84,8 +96,9 @@ export function registerHealthMonitor( userMicros: currentCpu.user, systemMicros: currentCpu.system, percent: Math.round(cpuPercent * 100) / 100, + cores: coreCount, capacityCores, - capacityPercent: capacityCores ? Math.round(cpuPercent / capacityCores * 100) / 100 : null, + capacityPercent: capacityCores ? Math.round(singleCoreCpuPercent / capacityCores * 100) / 100 : null, }, eventLoopLagMs, uptimeSeconds: uptime, diff --git a/src/hooks/_capture-filter.ts b/src/hooks/_capture-filter.ts index 762f52420e..29c3df7fa1 100644 --- a/src/hooks/_capture-filter.ts +++ b/src/hooks/_capture-filter.ts @@ -1,3 +1,5 @@ +import { scrubRecord, stripPrivateData, stripSecrets } from "../functions/privacy.js"; + const DEFAULT_DENY_PATTERNS = [ "memory_*", "toolsearch", @@ -60,27 +62,79 @@ export function captureOutputMax(): number { return Number.isFinite(parsed) && parsed > 0 ? parsed : 8000; } -export function truncateText(value: string, max: number): string { +function cut(value: string, max: number): string { if (value.length <= max) return value; const end = /[\uD800-\uDBFF]/.test(value[max - 1] ?? "") ? max - 1 : max; return value.slice(0, end); } +export function truncateText(value: string, max: number): string { + if (value.length <= max) return value; + return cut(stripPrivateData(value), max); +} + +const JSON_WHITESPACE_ESCAPES: Record = { n: "\n", r: "\r", t: "\t" }; + +const SECRET_KEY_SUFFIXES = ["apikey", "secret", "token", "password", "passwd", "credential", "credentials"]; +const SECRET_KEY_WORDS = new Set(["auth", "authorization"]); + +function isSecretKeyName(key: string): boolean { + if (!/^[\w.-]{1,64}$/.test(key)) return false; + const words = key + .replace(/([a-z])([A-Z])/g, "$1_$2") + .replace(/([A-Za-z])(\d)/g, "$1_$2") + .toLowerCase() + .split(/[_.\-\d]+/) + .filter(Boolean); + return words.some( + (word, i) => + SECRET_KEY_WORDS.has(word) || + SECRET_KEY_SUFFIXES.some((suffix) => word.endsWith(suffix)) || + (word === "api" && words[i + 1] === "key"), + ); +} + +function redactSecretKeys(value: unknown): unknown { + if (Array.isArray(value)) return value.map(redactSecretKeys); + if (value === null || typeof value !== "object") return value; + const proto = Object.getPrototypeOf(value); + if (proto !== Object.prototype && proto !== null) return value; + return Object.fromEntries( + Object.entries(value as Record).map(([key, item]) => [ + key, + typeof item === "string" && item && isSecretKeyName(key) ? "[REDACTED_SECRET]" : redactSecretKeys(item), + ]), + ); +} + +function scrubSecrets(value: unknown): unknown { + return scrubRecord(redactSecretKeys(value), { scrubKeys: true, strip: stripSecrets }); +} + +function redactSerialized(serialized: string): string { + const unescaped = serialized.replace(/\\([\\nrt])/g, (escape, c: string) => JSON_WHITESPACE_ESCAPES[c] ?? escape); + return stripPrivateData(unescaped).replace(/[\n\r\t]/g, (c) => JSON.stringify(c).slice(1, -1)); +} + +export function truncateJson(value: unknown, max: number): string { + const str = JSON.stringify(value); + if (str.length <= max) return str; + return cut(redactSerialized(JSON.stringify(scrubSecrets(value))), max); +} + +function truncateWithSuffix(value: string, max: number, suffix: string): string { + const safe = stripPrivateData(value); + if (safe.length <= max) return safe; + return max <= suffix.length ? suffix.slice(0, max) : cut(safe, max - suffix.length) + suffix; +} + export function truncateCaptureOutput(value: unknown, max: number): unknown { if (typeof value === "string" && value.length > max) { - const suffix = "\n[...truncated]"; - return max <= suffix.length - ? suffix.slice(0, max) - : truncateText(value, max - suffix.length) + suffix; + return truncateWithSuffix(value, max, "\n[...truncated]"); } if (typeof value === "object" && value !== null) { const str = JSON.stringify(value); - if (str.length > max) { - const suffix = "...[truncated]"; - return max <= suffix.length - ? suffix.slice(0, max) - : truncateText(str, max - suffix.length) + suffix; - } + if (str.length > max) return truncateWithSuffix(redactSerialized(JSON.stringify(scrubSecrets(value))), max, "...[truncated]"); return value; } return value; diff --git a/src/hooks/_capture.ts b/src/hooks/_capture.ts index 02c1b11fb4..ecd255edc8 100644 --- a/src/hooks/_capture.ts +++ b/src/hooks/_capture.ts @@ -1,4 +1,4 @@ -import {deliverSdkCaptureV2,drainSdkCapturesV2,SDK_CAPTURE_V2_FLAG,type V2DeliveryResult,type V2PostResult} from "../capture/sdk-delivery-v2.js"; +import {deliverSdkCaptureV2,drainSdkCapturesV2,SDK_CAPTURE_V2_FLAG,SDK_FINALIZATION_RESERVE_MS,type V2DeliveryResult,type V2PostResult} from "../capture/sdk-delivery-v2.js"; import {scrubRecord} from "../functions/privacy.js"; import {performance} from "node:perf_hooks"; import {sendCapture,restoreSdkOriginal} from "../capture/delivery.js"; @@ -246,9 +246,9 @@ async function captureObservationV2(body:ObserveBody,timeoutMs:number):Promise; - const options={dir:process.env["AGENTMEMORY_SDK_CAPTURE_V2_DIR"]}; - const result=await deliverSdkCaptureV2(REST_URL,payload,b=>postRawV2(b,timeoutMs),options); - const now=performance.now(),deadline=Math.min(started+timeoutMs,now+250),remaining=Math.floor(deadline-now); + const deadline=started+timeoutMs,options={dir:process.env["AGENTMEMORY_SDK_CAPTURE_V2_DIR"],deadline}; + const result=await deliverSdkCaptureV2(REST_URL,payload,(b,budget)=>postRawV2(b,budget?.remainingMs??timeoutMs,budget?.deadline),options); + const now=performance.now(),remaining=Math.floor(Math.min(deadline-SDK_FINALIZATION_RESERVE_MS-now,250)); if(body.hookType!=="stop"&&(result.outcome==="delivered"||result.outcome==="duplicate")&&remaining>=100){ const backlog=await drainSdkCapturesV2(REST_URL,(b,budget)=>postRawV2(b,budget?.remainingMs??remaining,budget?.deadline??deadline),{...options,limit:1,budgetMs:remaining,deadline}); if(backlog.held||backlog.error){try{writeSync(2,JSON.stringify({code:"CAPTURE_BACKLOG_HELD",held:backlog.held,error:backlog.error})+"\n");}catch{}} diff --git a/src/hooks/cursor/delegate.ts b/src/hooks/cursor/delegate.ts index c07020d6c5..8cb6ea5a28 100644 --- a/src/hooks/cursor/delegate.ts +++ b/src/hooks/cursor/delegate.ts @@ -85,6 +85,10 @@ export function delegateHook( } const { project, payload } = enrichPayload(data); + if (hookKey === "subagentStop") { + payload.agent_type ||= data.agentType || data.agentDisplayName || data.agentName || data.subagent_type; + payload.last_assistant_message ??= data.response ?? data.summary; + } const child = spawnSync(process.execPath, [scriptPath], { input: JSON.stringify(payload), diff --git a/src/hooks/post-tool-failure.ts b/src/hooks/post-tool-failure.ts index 372da6b1a7..9dbe015dcb 100644 --- a/src/hooks/post-tool-failure.ts +++ b/src/hooks/post-tool-failure.ts @@ -1,5 +1,5 @@ #!/usr/bin/env node -import { shouldCaptureTool, truncateText } from "./_capture-filter.js"; +import { shouldCaptureTool, truncateJson, truncateText } from "./_capture-filter.js"; import { resolveProject, hookCwd } from "./_project.js"; import { captureObservation, isDrainChild, runDrainChild, withEventId } from "./_capture.js"; @@ -52,11 +52,11 @@ async function main() { tool_input: typeof toolInput === "string" ? truncateText(toolInput, 4000) - : truncateText(JSON.stringify(toolInput ?? ""), 4000), + : truncateJson(toolInput ?? "", 4000), error: typeof error === "string" ? truncateText(error, 4000) - : truncateText(JSON.stringify(error ?? ""), 4000), + : truncateJson(error ?? "", 4000), }, }, data, diff --git a/src/hooks/pre-tool-use.ts b/src/hooks/pre-tool-use.ts index 7eb25234cb..10fdb9072a 100644 --- a/src/hooks/pre-tool-use.ts +++ b/src/hooks/pre-tool-use.ts @@ -1,5 +1,6 @@ #!/usr/bin/env node -import { hookAuthHeaders, reportBlockedTransport, resolveHookEndpoint } from "./_endpoint.js"; +import { hookAuthHeaders, hookSetting, reportBlockedTransport, resolveHookEndpoint } from "./_endpoint.js"; +import { resolveProject, hookCwd } from "./_project.js"; function isSdkChildContext(payload: unknown): boolean { if (process.env["AGENTMEMORY_SDK_CHILD"] === "1") return true; @@ -12,10 +13,31 @@ const INJECT_CONTEXT = process.env["AGENTMEMORY_INJECT_CONTEXT"] === "true"; const ENDPOINT = resolveHookEndpoint(); const REST_URL = ENDPOINT.url; +const DEFAULT_ENRICH_TIMEOUT_MS = 2000; + function authHeaders(): Record { return hookAuthHeaders(ENDPOINT); } +const MAX_TIMER_MS = 2_147_483_647; + +function enrichTimeoutMs(): number { + const raw = hookSetting("AGENTMEMORY_ENRICH_TIMEOUT_MS")?.trim() ?? ""; + const n = /^\d+$/.test(raw) ? Number(raw) : 0; + return n >= 1 && n <= MAX_TIMER_MS ? n : DEFAULT_ENRICH_TIMEOUT_MS; +} + +function isPlainTextHost(): boolean { + return Boolean(process.env["FACTORY_PROJECT_DIR"] || process.env["DROID_PLUGIN_ROOT"]); +} + +function contextPayload(data: Record, context: string): string { + if (data.hook_event_name !== "PreToolUse" || isPlainTextHost()) return context; + return JSON.stringify({ + hookSpecificOutput: { hookEventName: "PreToolUse", additionalContext: context }, + }); +} + async function main() { // Default off: exit immediately so we don't even open stdin. This keeps // Claude Code's tool-call hot path as cheap as possible. @@ -36,6 +58,7 @@ async function main() { if (!data || typeof data !== "object") return; if (isSdkChildContext(data)) return; + if (typeof data.cursor_version === "string") return; const toolName = typeof data.tool_name === "string" @@ -80,10 +103,12 @@ async function main() { typeof rawSessionId === "string" && rawSessionId.length > 0 ? rawSessionId : "unknown"; - const project = + const explicitProject = typeof data.project === "string" && data.project.trim().length > 0 ? data.project.trim() : undefined; + const project = (explicitProject ?? resolveProject(hookCwd(data) || process.cwd())).trim(); + if (!project) return; try { const res = await fetch(`${REST_URL}/agentmemory/enrich`, { @@ -95,15 +120,15 @@ async function main() { files, terms, toolName, - ...(project !== undefined && { project }), + project, }), - signal: AbortSignal.timeout(2000), + signal: AbortSignal.timeout(enrichTimeoutMs()), }); if (res.ok) { const result = (await res.json()) as { context?: string }; if (result.context) { - process.stdout.write(result.context); + process.stdout.write(contextPayload(data, result.context)); } } } catch { diff --git a/src/hooks/session-end.ts b/src/hooks/session-end.ts index d5e682f91e..27632e1533 100644 --- a/src/hooks/session-end.ts +++ b/src/hooks/session-end.ts @@ -1,4 +1,5 @@ #!/usr/bin/env node +import { spawn } from "node:child_process"; import { readFileSync } from "node:fs"; import { truncateText } from "./_capture-filter.js"; import { resolveProject, hookCwd } from "./_project.js"; @@ -91,6 +92,32 @@ function extractTranscriptPrompts(data: Record): TranscriptProm return prompts; } +const WORKER_ENV = "AGENTMEMORY_SESSION_END_WORKER"; + +function handOff(input: string): Promise { + if (process.env[WORKER_ENV] === "1" || !process.argv[1]) return Promise.resolve(false); + return new Promise((resolve) => { + try { + const child = spawn(process.execPath, [process.argv[1]], { + detached: true, + stdio: ["pipe", "ignore", "ignore"], + windowsHide: true, + env: { ...process.env, [WORKER_ENV]: "1" }, + }); + child.on("error", () => resolve(false)); + child.stdin.on("error", () => resolve(false)); + child.once("spawn", () => { + child.stdin.end(input, () => { + child.unref(); + resolve(true); + }); + }); + } catch { + resolve(false); + } + }); +} + async function main() { if (isDrainChild()) return runDrainChild(); let input = ""; @@ -107,6 +134,7 @@ async function main() { if (!data || typeof data !== "object") return; if (isSdkChildContext(data)) return; + if (data.hook_event_name === "SessionEnd" && (await handOff(input))) return; const sessionId = ((data.session_id || data.sessionId || data.conversation_id) as string) || "unknown"; diff --git a/src/index.ts b/src/index.ts index 7cd9e67ee2..bc8dbad0e8 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,6 +1,10 @@ +import { withRegistrationDefaults } from "./runtime/registration.js"; +import {releaseSupersededVectorJobs,retryRetainedVectorRevisions,type RetainedVectorRevision} from "./state/vector-revision-release.js"; import {isStorageWorkCompleted} from './ops/storage-completion.mjs'; -import {createEngineEpochEvidence} from "./ops/engine-epoch.mjs"; -import {acquireFileWriterLease,markFileWriterClean,ownsFileWriterLease} from "./ops/file-writer-lease.mjs"; +import {stateSealDiskProof,type StateSealReceipt} from "./ops/state-seal-shutdown.mjs"; +import {createStateBoundEvidence,pinStateClient,type StateBootIdentity} from "./ops/state-epoch.mjs"; +import {createEngineEpochEvidence,readEnginePid,waitForEngineEpoch} from "./ops/engine-epoch.mjs"; +import {acquireFileWriterLease,markFileWriterClean,markFileWriterSealedClean,ownsFileWriterLease} from "./ops/file-writer-lease.mjs"; import { createBackgroundControl, installBackgroundControl, controlBackgroundFunctions, registerBackgroundControlApi, backgroundTask } from "./ops/background-control.mjs"; import { createMemoryPressureGuard } from "./ops/memory-pressure.mjs"; import { serializeLongSessionProvider } from "./ops/long-session.mjs"; @@ -84,7 +88,6 @@ import { setHybridRanker, setPendingVectorBackfillCount, setVectorBackfillState, - getPendingVectorBackfillCount, } from "./functions/search.js"; import { registerContextFunction } from "./functions/context.js"; import { registerSessionIndexMaintenanceFunction } from "./functions/session-index-maintenance.js"; @@ -222,6 +225,8 @@ function resolveWorkerSecret(): string { let fatalShutdown: (() => Promise) | undefined; +const unreleasedSupersededRevisions: RetainedVectorRevision[] = []; + async function main() { // Fold ~/.agentmemory/.env into process.env before anything reads config // or raw process.env. Only-if-unset, so real process.env still wins. @@ -262,7 +267,7 @@ async function main() { `REST API: http://localhost:${config.restPort}/agentmemory/*`, ); - const sdk = withWriteScrubbing(registerWorker(config.engineUrl, { + const sdk = withWriteScrubbing(withRegistrationDefaults(registerWorker(config.engineUrl, { workerName: "agentmemory", invocationTimeoutMs: 180000, otel: { @@ -282,7 +287,7 @@ async function main() { language: "node", framework: "iii-sdk", }, - })); + }))); let stateBackend: "file" | "redis" = "file"; @@ -290,6 +295,7 @@ async function main() { stateBackend = getStateBackend(); } catch {} const stateReadiness = await waitForStateReady(sdk, { + requireSetAck: stateBackend === "file", onWaiting: (error) => bootWarn(`Waiting for the state worker to register state::get and state::list (${error})`), }); @@ -301,10 +307,25 @@ async function main() { } if (!process.env["AGENTMEMORY_STATE_STORE_PATH"] && process.env["AGENTMEMORY_DATA_DIR"]) process.env["AGENTMEMORY_STATE_STORE_PATH"] = join(process.env["AGENTMEMORY_DATA_DIR"], "state_store.db"); const writerStorePath=process.env["AGENTMEMORY_STATE_STORE_PATH"]; - const engineEvidence=createEngineEpochEvidence({engineUrl:config.engineUrl,enginePidPath:runtimeMetadataPath("iii.pid"),peer:()=>{const socket=(sdk as unknown as {ws?:{_socket?:{remoteAddress?:string;remotePort?:number;localAddress?:string;localPort?:number}}}).ws?._socket;return socket?{remoteAddress:socket.remoteAddress,remotePort:socket.remotePort,localAddress:socket.localAddress,localPort:socket.localPort}:null;}}); - if(stateBackend==="file"&&writerStorePath&&process.platform==="linux"){const until=Date.now()+3000;while(!engineEvidence.capture()&&Date.now()setTimeout(resolve,25));} - const writerLease=stateBackend==="file"&&writerStorePath&&process.platform==="linux"?acquireFileWriterLease(writerStorePath,{evidence:engineEvidence}):undefined; - const kv = new StateKV(sdk, { backend: stateBackend,writerLease }); + let engineEvidence=createEngineEpochEvidence({engineUrl:config.engineUrl,enginePidPath:runtimeMetadataPath("iii.pid"),engineStatePath:runtimeMetadataPath("engine-state.json"),peer:()=>{const socket=(sdk as unknown as {ws?:{_socket?:{remoteAddress?:string;remotePort?:number;localAddress?:string;localPort?:number}}}).ws?._socket;return socket?{remoteAddress:socket.remoteAddress,remotePort:socket.remotePort,localAddress:socket.localAddress,localPort:socket.localPort}:null;}}); + if(stateBackend==="file"&&writerStorePath&&process.platform==="linux")await waitForEngineEpoch(engineEvidence); + let stateClient=sdk;let stateBinding:StateBootIdentity|undefined; + if(stateBackend==="file"&&writerStorePath&&process.platform==="linux"){ + let metadata;try{metadata=JSON.parse(readFileSync(runtimeMetadataPath("engine-state.json"),"utf8"));}catch(error){if((error as NodeJS.ErrnoException).code!=="ENOENT")throw error;} + if(metadata?.kind==="compose"){ + const identity=await sdk.trigger({function_id:"state::boot_identity",payload:{}}); + const prefix=`state::${identity.instanceId}::`; + const aliasIdentity=await sdk.trigger({function_id:prefix+"boot_identity",payload:{}}); + if(JSON.stringify(identity)!==JSON.stringify(aliasIdentity))throw Error("STATE_WRITER_IDENTITY_CHANGED"); + engineEvidence=createStateBoundEvidence({engineEvidence,identity,expectedExecutable:metadata.stateExecutable,storePath:writerStorePath}); + stateBinding=identity; + const boundEpoch=engineEvidence.capture(); + if(!boundEpoch)throw Error("STATE_WRITER_IDENTITY_UNVERIFIED"); + stateClient=pinStateClient(sdk,prefix,()=>engineEvidence.current(boundEpoch),()=>engineEvidence.diagnostic?.()??null); + } + } + const writerLease=stateBackend==="file"&&writerStorePath&&process.platform==="linux"?acquireFileWriterLease(writerStorePath,{evidence:engineEvidence,requireVerifiedEpoch:true}):undefined; + const kv = new StateKV(stateClient, { backend: stateBackend,writerLease, compactSummaryWrites: stateBackend === 'file' }); writeWorkerPidfile(); const secret = resolveWorkerSecret(); const ingress = createShutdownIngress(); @@ -327,7 +348,7 @@ async function main() { receiptDir: dirname(runtimeMetadataPath("shutdown-receipt.json")), engineAlive: () => { try { - const pid = Number(readFileSync(runtimeMetadataPath("iii.pid"), "utf8").trim()); + const pid = readEnginePid({enginePidPath:runtimeMetadataPath("iii.pid"),engineStatePath:runtimeMetadataPath("engine-state.json"),engineUrl:config.engineUrl}); if (!Number.isSafeInteger(pid) || pid < 1) return null; process.kill(pid, 0); return true; } catch (error) { return (error as NodeJS.ErrnoException).code === "ESRCH" ? false : null; } @@ -352,11 +373,21 @@ async function main() { progress({ stage: "freeze" }); kv.durability.freeze(); progress({ stage: kv.durability.completion==='sdk'?"final_sdk_completion":"final_disk_fence" }); disk = await kv.durability.wait(kv); }, progress); + // A failed release keeps ownership and prevents a clean shutdown receipt. + retryRetainedVectorRevisions(unreleasedSupersededRevisions); + if(stateBinding){ + progress({stage:"final_disk_fence"}); + const seal=await stateClient.trigger({function_id:"state::seal_flush",payload:{}}); + disk=stateSealDiskProof(seal,stateBinding,kv.durability!.snapshot().mutationFailures); + } return disk; }, }); resources.viewer?.closeAllConnections(); if (resources.viewer) await settleWithin(new Promise(resolve => resources.viewer!.close(() => resolve())), SHUTDOWN_FLUSH_TIMEOUT_MS).catch(() => false); + if(writerLease&&stateBinding&&outcome.verified&&!bootFailed&&!kv.mutationOutcomeUncertain){ + try{markFileWriterSealedClean(writerLease,(outcome.durability as {stateSeal?:StateSealReceipt})?.stateSeal);}catch(error){bootWarn("State disk seal verified but clean lease publication failed: "+String(error));} + } const sdkDrained=await settleWithin(sdk.shutdown(), SHUTDOWN_FLUSH_TIMEOUT_MS).catch(() => false); if(writerLease&&kv.recoveryMutationSafe&&outcome.verified&&!bootFailed&&sdkDrained&&!kv.mutationOutcomeUncertain){try{markFileWriterClean(writerLease);}catch{/* The dirty marker remains conservative on failure. */}} if (outcome.completed) clearWorkerPidfile(); @@ -371,6 +402,8 @@ async function main() { operationalState.backgroundControl = control; controlBackgroundFunctions(sdk); } + let indexForExplicitResume: IndexPersistence | undefined; + let scheduleBackfillAfterResume: (() => Promise) | undefined; const pressure = persistentBackground ? lifecycle.track(createMemoryPressureGuard({ control })) : null; pressure?.install(sdk, (req) => checkAuth(req, secret)); operationalState.memoryPressure = pressure; @@ -565,7 +598,12 @@ async function main() { markKeywordRebuildPending(); registerApiTriggers(sdk, kv, secret, metricsStore, provider); registerDashboardUpdateApi(sdk, secret, createDashboardUpdates({requestRoot: runtimeMetadataPath("update-requests"), version: VERSION})); - if (persistentBackground) registerBackgroundControlApi(sdk, secret, control); + if (persistentBackground) registerBackgroundControlApi(sdk, secret, control, async allowReconciliation => { + await bootGate; + if (bootFailed || !indexForExplicitResume || !scheduleBackfillAfterResume) return false; + if (!await indexForExplicitResume.resumeStorageRecovery(embeddingProvider?.dimensions ?? 0, allowReconciliation)) return false; + return scheduleBackfillAfterResume(); + }); if (getEnvVar("AGENTMEMORY_FEEDBACK_DIR")) { const feedback = lifecycle.track(registerFeedbackRuntime(sdk, secret, { root: getEnvVar("AGENTMEMORY_FEEDBACK_DIR"), @@ -585,7 +623,8 @@ async function main() { registerMcpEndpoints(sdk, kv, secret); const healthMonitor = resources.health = registerHealthMonitor(sdk, kv); - const indexPersistence = resources.index = new IndexPersistence(kv, vectorIndex, { requireRecoveryPlan: true }); + const indexPersistence = resources.index = new IndexPersistence(kv, vectorIndex, { requireRecoveryPlan: true, isStorageHeld: () => persistentBackground && (control.snapshot().holds.includes('storage_error') || kv.mutationOutcomeUncertain), storageGeneration: () => persistentBackground ? control.generation() : 0 }); + indexForExplicitResume = indexPersistence; setIndexPersistence(indexPersistence); const loaded = await indexPersistence.load().catch((err) => { @@ -669,6 +708,7 @@ async function main() { operationalState.embeddingRecovery = lifecycle.track(registerEmbeddingRecovery({ kv, KV, index: vectorIndex, provider: embeddingProvider, readiness, withKeyedLock, logger, schedule: false, persistIndex: (publication?: VectorPublication) => indexPersistence.saveVerified(publication), + verifyRetainedPublication: publication => indexPersistence.verifyRetainedVectorPublication(publication), })); await operationalState.embeddingRecovery.initialize(); } @@ -681,9 +721,23 @@ async function main() { const auditMigration = trackBoot(startAuditMigration(kv).catch(() => {})); - const keywordStart = Date.now(); - try { - const keyword = await rebuildKeywordIndex(kv, vectorBackfillSince); + const rebuildAndScheduleVectorBackfill = async (vectorBackfillSince: string | null | undefined): Promise => { + const keywordStart = Date.now(); + try { + // This fork can retry on resume without a process restart. Do not rebuild + // over a superseded watch whose release failed during an earlier attempt. + retryRetainedVectorRevisions(unreleasedSupersededRevisions); + let keyword = await rebuildKeywordIndex(kv, vectorBackfillSince); + let supersededReleaseFailed = false; + if (typeof vectorBackfillSince === "string" && !indexPersistence.isRecoveryPaused() && keyword.historicalVectorGaps > 0) { + supersededReleaseFailed = !releaseSupersededVectorJobs(keyword.vectorJobs,unreleasedSupersededRevisions); + if(supersededReleaseFailed)bootWarn("Failed to release superseded vector revisions; recovery remains paused."); + keyword.vectorJobs = []; + if (!supersededReleaseFailed) { + vectorBackfillSince = await indexPersistence.prepareBackfill(loaded, true); + if (vectorBackfillSince === null) keyword = await rebuildKeywordIndex(kv, vectorBackfillSince); + } + } bootLog( `Rebuilt BM25 index from stored content (${keyword.documents} docs in ${Date.now() - keywordStart} ms)`, ); @@ -693,12 +747,18 @@ async function main() { const activeIds = new Set(keyword.vectorJobs.map((job) => job.id)); const hasVector = (id: string) => Boolean(vectorIndex?.has(id)); const countRemaining = () => bm25Index.countMissingDocuments(hasVector, vectorBackfillSince, activeIds); - setPendingVectorBackfillCount(indexPersistence.isRecoveryPaused() ? countRemaining() : pending); - if (indexPersistence.isRecoveryPaused()) { + setPendingVectorBackfillCount(indexPersistence.isRecoveryPaused() || supersededReleaseFailed ? countRemaining() : pending); + if (indexPersistence.isRecoveryPaused() || supersededReleaseFailed) { setVectorBackfillState("paused"); } else if (vectorBackfillSince !== undefined) { setVectorBackfillState(keyword.fullBackfillPending > 0 && !optedIn ? "waiting-for-opt-in" : backlogScanComplete ? "idle" : "paused"); } + if (!backlogScanComplete || supersededReleaseFailed) { + setVectorBackfillState("paused"); + activeIds.clear(); + for (const job of keyword.vectorJobs) job.revision?.release(); + return false; + } if (keyword.fullBackfillPending > 0) { bootLog(optedIn ? `Vector backfill has ${keyword.fullBackfillPending} additional documents beyond this start's limit; restart to resume.` @@ -741,7 +801,13 @@ async function main() { } catch (err) { setVectorBackfillState("paused"); console.warn(`[agentmemory] Failed to rebuild the BM25 index:`, err); + return false; } + return !indexPersistence.isRecoveryPaused(); + + }; + scheduleBackfillAfterResume = () => rebuildAndScheduleVectorBackfill(indexPersistence.recoveryCutoff()); + await rebuildAndScheduleVectorBackfill(vectorBackfillSince); // Ready / Endpoints lines are emitted via `bootLog` so they're // buffered in quiet mode and printed verbatim under --verbose. The @@ -751,7 +817,7 @@ async function main() { `Ready. ${embeddingProvider ? "Triple-stream (BM25+Vector+Graph)" : "BM25+Graph"} search active.`, ); bootLog( - `REST API: 160 endpoints at http://localhost:${config.restPort}/agentmemory/*`, + `REST API: 161 endpoints at http://localhost:${config.restPort}/agentmemory/*`, ); bootLog( `MCP surface (opt-in via \`npx @agentmemory/mcp\`): ${getAllTools().length} tools · 6 resources · 3 prompts`, diff --git a/src/mcp/in-memory-kv.ts b/src/mcp/in-memory-kv.ts index e234896477..122cd8728b 100644 --- a/src/mcp/in-memory-kv.ts +++ b/src/mcp/in-memory-kv.ts @@ -37,9 +37,10 @@ export class InMemoryKV { this.store.get(scope)?.delete(key); } - async list(scope: string): Promise { + async list(scope: string, options?: { project?: (value: unknown, key?: string) => T }): Promise { const entries = this.store.get(scope); - return entries ? (Array.from(entries.values()) as T[]) : []; + const values = entries ? Array.from(entries.values()) : []; + return options?.project ? Array.from(entries ?? [], ([key, value]) => options.project!(value, key)) : values as T[]; } persist(): boolean { diff --git a/src/mcp/plugin-bridge.ts b/src/mcp/plugin-bridge.ts index 1ec1c56f36..593e6f9e2a 100644 --- a/src/mcp/plugin-bridge.ts +++ b/src/mcp/plugin-bridge.ts @@ -2,7 +2,7 @@ import { pathToFileURL } from "node:url"; import { realpathSync } from "node:fs"; import { VERSION } from "../version.js"; import { resolveSettingSecret } from "../hooks/_endpoint.js"; -import { resolveEnvOrEmpty } from "./rest-proxy.js"; +import { callTimeoutMs, resolveEnvOrEmpty } from "./rest-proxy.js"; import { createStdioTransport, JsonRpcError, type RequestHandler } from "./transport.js"; const PROTOCOLS = ["2025-11-25", "2025-06-18", "2025-03-26", "2024-11-05"]; @@ -37,7 +37,7 @@ export function createPluginBridge(): RequestHandler { }, body: body === undefined ? undefined : JSON.stringify(body), redirect: "error", - signal: AbortSignal.timeout(15_000), + signal: AbortSignal.timeout(callTimeoutMs()), }); } catch { throw new Error(`Agent Memory daemon is unreachable or timed out. ${SETUP} No fallback store was used; check state before retrying a write.`); diff --git a/src/mcp/rest-proxy.ts b/src/mcp/rest-proxy.ts index 9fcf752778..b2b343afb0 100644 --- a/src/mcp/rest-proxy.ts +++ b/src/mcp/rest-proxy.ts @@ -1,7 +1,9 @@ import { hookSetting, resolveSettingSecret } from "../hooks/_endpoint.js"; +import { readEnvFileValue } from "../secret-store.js"; const DEFAULT_URL = "http://localhost:3111"; const DEFAULT_HEALTH_PROBE_TIMEOUT_MS = 2_000; -const CALL_TIMEOUT_MS = 15_000; +const DEFAULT_CALL_TIMEOUT_MS = 190_000; +const MAX_TIMER_MS = 2_147_483_647; const LOCAL_MODE_TTL_MS = 30_000; function probeTimeoutMs(): number { @@ -11,6 +13,13 @@ function probeTimeoutMs(): number { return Number.isFinite(n) && n > 0 ? Math.floor(n) : DEFAULT_HEALTH_PROBE_TIMEOUT_MS; } +export function callTimeoutMs(): number { + const name = "AGENTMEMORY_CALL_TIMEOUT_MS"; + const raw = (resolveEnvOrEmpty(name) || readEnvFileValue(name)).trim(); + const n = /^\d+$/.test(raw) ? Number(raw) : 0; + return n >= 1 && n <= MAX_TIMER_MS ? n : DEFAULT_CALL_TIMEOUT_MS; +} + export function isProxyRequired(): boolean { const raw = process.env["AGENTMEMORY_FORCE_PROXY"]; return raw === "1" || raw === "true"; @@ -164,7 +173,7 @@ export async function resolveHandle(): Promise { ...authHeader(url), ...(init?.headers as Record | undefined), }, - signal: AbortSignal.timeout(CALL_TIMEOUT_MS), + signal: AbortSignal.timeout(callTimeoutMs()), redirect: "error", }); if (!res.ok) { diff --git a/src/mcp/server.ts b/src/mcp/server.ts index 3449736353..8a67a8228d 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -2,17 +2,18 @@ import { parseSlotOwner } from "../functions/slots.js"; import type { IIIClient } from "iii-sdk"; import type { HttpRequest } from "@iii-dev/helpers/http"; import type { StateKV } from "../state/kv.js"; +import { collectAll } from "../state/kv.js"; import { KV } from "../state/schema.js"; import type { SessionSummary, Memory, Session } from "../types.js"; import { getAllTools, getVisibleTools } from "./tools-registry.js"; import { parseSearchExpansionIds } from "./search-arguments.js"; import { parseSessionArguments } from "./session-arguments.js"; import { isSearchLayer } from "../state/search-layer.js"; +import { insightHttpResponse } from "../state/insight-response.js"; import { checkAuth, slotsDisabledResponse } from "../triggers/api.js"; import { isSlotsEnabled } from "../functions/slots.js"; import { LESSON_SOURCE_IDS_MAX, normalizeLessonSourceIds } from "../functions/lessons.js"; import { getAgentId, isAgentScopeIsolated } from "../config.js"; - type McpResponse = { status_code: number; headers?: Record; @@ -720,6 +721,13 @@ export function registerMcpEndpoints( body: { error: "title is required" }, }; } + if (args.agentId !== undefined && typeof args.agentId !== "string") { + return { status_code: 400, body: { error: "agentId must be a string" } }; + } + const creatorId = typeof args.agentId === "string" ? args.agentId.trim() : ""; + if (creatorId.length > 128) { + return { status_code: 400, body: { error: "agentId must be at most 128 characters" } }; + } const edges: Array<{ type: string; targetActionId: string }> = []; if (typeof args.requires === "string" && args.requires.trim()) { for (const id of args.requires.split(",").map((s: string) => s.trim()).filter(Boolean)) { @@ -737,6 +745,7 @@ export function registerMcpEndpoints( tags, parentId: args.parentId, edges: edges.length > 0 ? edges : undefined, + createdBy: creatorId || getAgentId(), } }); return { status_code: 200, @@ -1197,7 +1206,7 @@ export function registerMcpEndpoints( project: args.project, maxClusters: args.maxClusters, } }); - return { status_code: 200, body: { content: [{ type: "text", text: JSON.stringify(reflectResult, null, 2) }] } }; + return insightHttpResponse(reflectResult, true); } case "memory_insight_list": { @@ -1206,7 +1215,7 @@ export function registerMcpEndpoints( minConfidence: args.minConfidence, limit: args.limit, } }); - return { status_code: 200, body: { content: [{ type: "text", text: JSON.stringify(insightListResult, null, 2) }] } }; + return insightHttpResponse(insightListResult, true); } case "memory_obsidian_export": { @@ -1427,7 +1436,7 @@ export function registerMcpEndpoints( try { if (uri === "agentmemory://status") { const sessions = await kv.list(KV.sessions); - const memories = await kv.list(KV.memories); + const memories = await collectAll(kv, KV.memories); const healthData = await kv.list(KV.health).catch(() => []); return { status_code: 200, @@ -1512,7 +1521,7 @@ export function registerMcpEndpoints( } if (uri === "agentmemory://memories/latest") { - const memories = await kv.list(KV.memories); + const memories = await collectAll(kv, KV.memories); const latest = memories .filter((m) => m.isLatest) .sort( @@ -1728,7 +1737,7 @@ export function registerMcpEndpoints( payload: { query: taskDesc, limit: 10 }, }) .catch(() => ({ results: [] })); - const memories = await kv.list(KV.memories); + const memories = await collectAll(kv, KV.memories); const isolated = isAgentScopeIsolated(); const activeAgentId = isolated ? getAgentId() : undefined; const relevant = diff --git a/src/mcp/tools-registry.ts b/src/mcp/tools-registry.ts index fcc68815ef..cee831f5f0 100644 --- a/src/mcp/tools-registry.ts +++ b/src/mcp/tools-registry.ts @@ -517,6 +517,7 @@ export const V050_TOOLS: McpToolDef[] = [ description: "Comma-separated action IDs that must complete before this", }, + agentId: { type: "string", description: "Agent creating the action" }, }, required: ["title"], }, diff --git a/src/ops/background-control.d.mts b/src/ops/background-control.d.mts index 3dda066908..530f7e6702 100644 --- a/src/ops/background-control.d.mts +++ b/src/ops/background-control.d.mts @@ -5,18 +5,20 @@ export interface BackgroundControl { init(store:StateKV,finish?:()=>Promise):Promise; snapshot():{formatVersion:2;holds:string[];recovery:{attempts:number;nextAttemptAt:number;maxAttempts:number};status:string;pausedRequested:boolean;activeOperations:number;error:string|null;diagnostics:unknown;transition:unknown}; isPaused():boolean; + generation():number; run(fn:()=>Promise):Promise; + runVectorWrite(fn:()=>Promise):Promise; infer(fn:()=>Promise):Promise; - change(action:"pause"|"resume",options?:{source?:"manual"|"deployment"|"memory";canResume?:()=>Promise}):Promise>; + change(action:"pause"|"resume",options?:{source?:"manual"|"deployment"|"memory";canResume?:()=>Promise;beforeResume?:()=>Promise}):Promise>; commitCleanShutdown():Promise<{success:boolean;paused:boolean}>; settled():Promise; } export const MEMORY_RECOVERY_MAX_ATTEMPTS:number; export let backgroundControl:BackgroundControl|null; export function createBackgroundControl(options?:{clock?:()=>number}):BackgroundControl; -export function installBackgroundControl(control:BackgroundControl):void; +export function installBackgroundControl(control:BackgroundControl,onExplicitResume?:()=>Promise):void; export function backgroundPaused():boolean; export function backgroundTaskunknown>(fn:T):T; export function controlBackgroundFunctions(sdk:IIIClient):void; export function backgroundViewerRequestAllowed(req:HttpRequest):boolean; -export function registerBackgroundControlApi(sdk:IIIClient,secret:string,control:BackgroundControl):void; +export function registerBackgroundControlApi(sdk:IIIClient,secret:string,control:BackgroundControl,onExplicitResume?:(allowReconciliation:boolean)=>Promise):void; diff --git a/src/ops/background-control.mjs b/src/ops/background-control.mjs index d03015228e..92b27a0cb4 100644 --- a/src/ops/background-control.mjs +++ b/src/ops/background-control.mjs @@ -14,16 +14,16 @@ export const backgroundTask=fn=>(...args)=>backgroundControl?backgroundControl.r export function createBackgroundControl({clock=Date.now}={}){ const context=new AsyncLocalStorage(),active=new Set(); let kv,drain=async()=>{},status='initializing',requested=true,error=null,diagnostics=null,completion=null,changes=Promise.resolve(); - let transition=null,holds=new Set(),revision=0,recovery={attempts:0,nextAttemptAt:0}; + let transition=null,holds=new Set(),revision=0,storageRevision=0,transitionsPending=0,recovery={attempts:0,nextAttemptAt:0}; const record=(paused,resumePending,remaining=holds)=>({version:2,paused,resumePending,holds:[...remaining].sort(),recovery:{...recovery}}); const writeRecord=(paused,resumePending,remaining)=>kv.set(NAMESPACE,'manual-pause',record(paused,resumePending,remaining)); - const snapshot=()=>({formatVersion:2,holds:[...holds].sort(),recovery:{...recovery,maxAttempts:MEMORY_RECOVERY_MAX_ATTEMPTS},status,pausedRequested:requested,activeOperations:active.size,error,diagnostics, - transition:transition?{...transition,elapsedMs:Math.max(0,(transition.completedAt??clock())-transition.startedAt),stageElapsedMs:Math.max(0,(transition.completedAt??clock())-transition.stageStartedAt)}:null}); - function begin(action){transition={action,stage:'request',startedAt:clock(),stageStartedAt:clock()};error=null;diagnostics=null;} + const snapshot=()=>({formatVersion:2,holds:[...holds].sort(),recovery:{...recovery,maxAttempts:MEMORY_RECOVERY_MAX_ATTEMPTS},status:holds.has('storage_error')?'error':status,pausedRequested:requested,activeOperations:active.size,error:holds.has('storage_error')?'background_storage_unavailable':error,diagnostics:holds.has('storage_error')?(diagnostics??{errorCode:'BACKGROUND_STORAGE_HOLD_RETAINED'}):diagnostics, + transition:transition?{...transition,pending:transitionsPending>0||!!completion,elapsedMs:Math.max(0,(transition.completedAt??clock())-transition.startedAt),stageElapsedMs:Math.max(0,(transition.completedAt??clock())-transition.stageStartedAt)}:null}); + function begin(action){transition={action,stage:'request',startedAt:clock(),stageStartedAt:clock()};if(!holds.has('storage_error')){error=null;diagnostics=null;}} function stage(value){transition.stage=value;transition.stageStartedAt=clock();if(value==='complete')transition.completedAt=clock();} const safeCode=value=>typeof value==='string'&&/^[A-Za-z0-9_.:-]{1,80}$/.test(value)?value:null; function fail(e){ - holds.add('storage_error');requested=true;status='error';error='background_storage_unavailable';if(transition)transition.completedAt=clock(); + revision++;storageRevision++;holds.add('storage_error');requested=true;status='error';error='background_storage_unavailable';if(transition)transition.completedAt=clock(); const details=e?.diagnostics??{}; diagnostics={errorCode:safeCode(e?.code)??'BACKGROUND_STORAGE_UNAVAILABLE',causeCode:safeCode(details.errorCode),stage:safeCode(details.stage)??transition?.stage??'request'}; for(const name of ['category','phase'])if(safeCode(details[name]))diagnostics[name]=details[name]; @@ -53,7 +53,7 @@ export function createBackgroundControl({clock=Date.now}={}){ await drain(); await persist(); await writeRecord(true,false);await confirmPauseRecord(); - status='paused';error=null;diagnostics=null;stage('complete'); + status='paused';if(!holds.has('storage_error')){error=null;diagnostics=null;}stage('complete'); })().catch(async e=>{fail(e);try{await writeRecord(true,true);await confirmPauseRecord(false);}catch{}}).finally(()=>{completion=null;}); } return { @@ -71,19 +71,31 @@ export function createBackgroundControl({clock=Date.now}={}){ await writeRecord(false,true);await confirmPauseRecord(false); requested=false;status='running'; }else{requested=true;status='paused';} - },snapshot,isPaused:()=>requested, + },snapshot,isPaused:()=>requested,generation:()=>storageRevision, run(fn){return requested?Promise.resolve(deferred()):track(fn);}, + runVectorWrite(fn){ + const blocked=()=>holds.has('storage_error')||status==='resuming'||status==='initializing'; + const denied=()=>({success:false,error:'vector_recovery_paused',retryable:true}); + if(blocked())return Promise.resolve(denied()); + const admitted=storageRevision; + return track(async()=>{ + if(blocked())return denied(); + try{const result=await fn();if(storageRevision!==admitted||holds.has('storage_error'))return{success:false,error:'vector_recovery_interrupted',partial:true,retryable:false,result};return result;} + catch(error){if(storageRevision!==admitted||holds.has('storage_error')||error?.code==='VECTOR_RECOVERY_PAUSED')return{success:false,error:'vector_recovery_interrupted',partial:true,retryable:false};throw error;} + }); + }, infer(fn){ // An admitted task finishes its current unit, including any subsequent writes. if(context.getStore()?.active)return Promise.resolve().then(fn); if(requested)return Promise.reject(Object.assign(Error('background_paused'),{providerUnavailable:true,code:'BACKGROUND_PAUSED'})); return track(fn); }, - change(action,{source='manual',canResume}={}){ + change(action,{source='manual',canResume,beforeResume}={}){ if(!['manual','deployment','memory'].includes(source)||!['pause','resume'].includes(action))return Promise.reject(Error('invalid_background_action')); // Close admission at request time, including while another transition awaits disk. const requestRevision=revision,added=action==='pause'&&!holds.has(source); if(action==='pause'){holds.add(source);requested=true;revision++;} + transitionsPending++; const operation=changes.then(async()=>{ if(action==='pause'){ if((status==='paused'||status==='pausing')&&!added)return snapshot(); @@ -93,19 +105,33 @@ export function createBackgroundControl({clock=Date.now}={}){ catch(e){fail(e);throw Error(error);} }else{ if(revision!==requestRevision)return snapshot(); - if(status==='pausing'||completion)throw Error('background_draining'); + if(status==='pausing'||completion||(beforeResume&&active.size))throw Error('background_draining'); const automatic=source==='memory'; - if(automatic&&(status!=='paused'||holds.size!==1||!holds.has('memory')||recovery.attempts>=MEMORY_RECOVERY_MAX_ATTEMPTS||clock()=MEMORY_RECOVERY_MAX_ATTEMPTS||clock(){});return operation; + const tracked=operation.finally(()=>{transitionsPending--;});changes=tracked.catch(()=>{});return tracked; },commitCleanShutdown(){ // The caller must first stop ingress/producers and verify the full disk fence. const operation=changes.then(async()=>{ @@ -155,15 +194,22 @@ export function createBackgroundControl({clock=Date.now}={}){ } const INFERENCE_FUNCTIONS=new Set(['mem::session-finalize','mem::summary-recover','mem::summary-repair','mem::compression-retry','mem::summarize','mem::compress','mem::graph-extract','mem::consolidate','mem::consolidate-pipeline','mem::reflect','mem::slot-reflect','mem::flow-compress','mem::crystallize','mem::skill-extract','mem::enrich-window','mem::temporal-graph-extract','mem::compress-file','mem::expand-query','mem::snapshot-create']); +const FOREGROUND_VECTOR_WRITES=new Set(['mem::import','mem::replay::import-jsonl','mem::observe','mem::remember','mem::forget','mem::evolve','mem::evict','mem::snapshot-restore','mem::retention-evict','mem::auto-forget','mem::governance-delete','mem::governance-bulk']); export function controlBackgroundFunctions(sdk){ const register=sdk.registerFunction.bind(sdk); - sdk.registerFunction=(id,handler,...rest)=>register(id,INFERENCE_FUNCTIONS.has(id)?backgroundTask(handler):handler,...rest); + sdk.registerFunction=(id,handler,...rest)=>{ + if(FOREGROUND_VECTOR_WRITES.has(id)){ + const guarded=(...args)=>backgroundControl?backgroundControl.runVectorWrite(()=>handler(...args)):handler(...args); + return register(id,guarded,...rest); + } + return register(id,INFERENCE_FUNCTIONS.has(id)?backgroundTask(handler):handler,...rest); + }; } export function backgroundViewerRequestAllowed(req){ return req.headers?.['x-agentmemory-background']==='1'&&/^application\/json(?:;|$)/i.test(req.headers?.['content-type']||'')&&!['cross-site','same-site'].includes(req.headers?.['sec-fetch-site']); } -export function registerBackgroundControlApi(sdk,secret,control){ +export function registerBackgroundControlApi(sdk,secret,control,onExplicitResume){ function auth(req){ if(!secret)return{status_code:503,body:{error:'background_control_requires_authentication'}}; const a=Buffer.from(req.headers?.authorization||req.headers?.Authorization||''),b=Buffer.from('Bearer '+secret); @@ -177,8 +223,13 @@ export function registerBackgroundControlApi(sdk,secret,control){ if(!backgroundViewerRequestAllowed(req))return{status_code:403,body:{error:'background_control_marker_required'}}; const body=req.body; if(!body||Array.isArray(body)||Object.keys(body).some(k=>!['action','reason'].includes(k))||body.reason!==undefined&&body.reason!=='deployment'||!['pause','resume'].includes(body.action))return{status_code:400,body:{error:'invalid_background_action'}}; - try{return{status_code:202,body:{success:true,control:await control.change(body.action,{source:body.reason??'manual'})}};} - catch(e){return{status_code:e.message==='background_draining'?409:503,body:{success:false,error:e.message==='background_draining'?'background_draining':'background_storage_unavailable',control:control.snapshot()}};} + try{ + const source=body.reason??'manual'; + const beforeResume=body.action==='resume'&&onExplicitResume?()=>onExplicitResume(source==='manual'):undefined; + const snapshot=await control.change(body.action,{source,beforeResume}); + return{status_code:202,body:{success:true,control:snapshot,...(body.action==='resume'&&snapshot.status==='running'&&onExplicitResume?{vectorRecovery:'replay_complete_backfill_pending'}:{})}}; + } + catch(e){const conflict=['background_draining','background_storage_recovery_requires_full_resume'].includes(e?.message);return{status_code:conflict?409:503,body:{success:false,error:e?.code==='VECTOR_RECOVERY_UNAVAILABLE'?'vector_recovery_unavailable':conflict?e.message:'background_storage_unavailable',control:control.snapshot()}};} }); sdk.registerTrigger({type:'http',function_id:id,config:{api_path:'/agentmemory/background/control',http_method:method,middleware_function_ids:['middleware::api-auth']}}); } diff --git a/src/ops/embedding-recovery.d.mts b/src/ops/embedding-recovery.d.mts index edce6ca9cc..b313e62804 100644 --- a/src/ops/embedding-recovery.d.mts +++ b/src/ops/embedding-recovery.d.mts @@ -27,4 +27,5 @@ export function registerEmbeddingRecovery(options: { logger?: Pick; clock?: () => number; schedule?: boolean; onIndexChanged?: () => void; persistIndex?: (publication?: VectorPublication) => Promise; + verifyRetainedPublication?: (publication: VectorPublication) => Promise; }): EmbeddingRecovery; diff --git a/src/ops/embedding-recovery.mjs b/src/ops/embedding-recovery.mjs index 93b48dda7f..2e350198af 100644 --- a/src/ops/embedding-recovery.mjs +++ b/src/ops/embedding-recovery.mjs @@ -4,10 +4,10 @@ import {networkErrorCode} from './provider-readiness.mjs'; export const EMBEDDING_RETRY_NAMESPACE='mem:embedding-retry:v1'; export const EMBEDDING_COMMITTED_NAMESPACE='mem:embedding-committed:v1'; const digest=text=>createHash('sha256').update(text).digest('hex'); -export function registerEmbeddingRecovery({kv,KV,index,provider,readiness,withKeyedLock=async(k,fn)=>fn(),logger,clock=Date.now,schedule=true,onIndexChanged,persistIndex}) { +export function registerEmbeddingRecovery({kv,KV,index,provider,readiness,withKeyedLock=async(k,fn)=>fn(),logger,clock=Date.now,schedule=true,onIndexChanged,persistIndex,verifyRetainedPublication}) { const state={pendingCount:0,completedCount:0,counterEpoch:randomUUID(),initialized:false,lastError:null,lastSuccessAt:null}; - let pumping=false,reconciled=false,stopped=false,interval=null,initialization=null,committedRows=null,committedOffset=0; - const active=new Set(),publications=new Map(),accepted=new Map(),pendingIds=new Set(); + let pumping=false,reconciled=false,stopped=false,started=false,interval=null,initialization=null,committedRows=null,committedOffset=0; + const active=new Set(),sourceGuards=new Set(),publications=new Map(),accepted=new Map(),pendingIds=new Set(); // Historical proof observed by this worker, never a fresh disk-verification cache. const observedPublications=new Map(); let publicationObservations=0,publicationEvidenceEvicted=0,publicationEvidenceUnavailable=0,publicationEvidenceAvailable=true; @@ -65,16 +65,18 @@ export function registerEmbeddingRecovery({kv,KV,index,provider,readiness,withKe })); } function prepareEnqueue(item){ - if(kv.backend==='file'&&(typeof kv.observationReader?.mutationRevision!=='function'||typeof kv.observationReader?.pendingMutations!=='function'))return Promise.reject(Object.assign(Error('embedding_source_guard_unavailable'),{code:'EMBEDDING_SOURCE_GUARD_UNAVAILABLE'})); + if(kv.backend==='file'&&typeof kv.durability?.watchKeys!=='function')return Promise.reject(Object.assign(Error('embedding_source_guard_unavailable'),{code:'EMBEDDING_SOURCE_GUARD_UNAVAILABLE'})); if(stopped)return Promise.reject(Object.assign(Error('embedding_recovery_stopped'),{code:'EMBEDDING_RECOVERY_STOPPED'})); return track(withKeyedLock(intentKey(item.id),async()=>{ await initialize(); - const scope=sourceScope(item),revision=()=>kv.observationReader?.mutationRevision?.(scope); - const sourceRevision=revision(); + if(stopped)throw Object.assign(Error('embedding_recovery_stopped'),{code:'EMBEDDING_RECOVERY_STOPPED'}); + const scope=sourceScope(item),guard=kv.backend==='file'?kv.durability.watchKeys([{scope,key:item.id}]):null; + const release=()=>{guard?.release();sourceGuards.delete(release);};sourceGuards.add(release); + try{ const sourceChanged=()=>Object.assign(Error('embedding_source_changed'),{code:'EMBEDDING_SOURCE_CHANGED'}); async function checkSource(){ const row=await kv.get(scope,item.id); - if(kv.observationReader?.pendingMutations?.(scope)||!row||row.id!==item.id||item.kind!=='memory'&&row.sessionId!==item.sessionId||revision()!==sourceRevision)throw sourceChanged(); + if(guard&&!guard.unchanged()||!row||row.id!==item.id||item.kind!=='memory'&&row.sessionId!==item.sessionId)throw sourceChanged(); } await checkSource(); const old=await kv.get(EMBEDDING_RETRY_NAMESPACE,item.id); @@ -94,11 +96,14 @@ export function registerEmbeddingRecovery({kv,KV,index,provider,readiness,withKe } let certification; return()=>certification??=track((async()=>{ + try{ await withKeyedLock(intentKey(item.id),check); await certifyIntent(item); await withKeyedLock(intentKey(item.id),check); return !old; + }finally{release();} })()); + }catch(error){release();throw error;} })); } function enqueueBatch(items){ @@ -157,7 +162,8 @@ export function registerEmbeddingRecovery({kv,KV,index,provider,readiness,withKe const canonical=await current(item); if(canonical!==null&&digest(canonical)!==fingerprint){await retainUnlocked(item,{code:'EMBEDDING_PROJECTION_CHANGED'},false);return;} if(canonical===null){await discardUnlocked(item);return;} - await kv.set(EMBEDDING_COMMITTED_NAMESPACE,item.id,{id:item.id,sessionId:item.sessionId,kind:item.kind,fingerprint,committedAt:new Date(clock()).toISOString()}); + await kv.set(EMBEDDING_COMMITTED_NAMESPACE,item.id,{id:item.id,sessionId:item.sessionId,kind:item.kind,fingerprint,committedAt:new Date(clock()).toISOString(), + ...(queued.retainedRecovery?{retainedRecovery:queued.retainedRecovery}:{})}); await fence([{scope:EMBEDDING_COMMITTED_NAMESPACE,key:item.id}]); await kv.delete(EMBEDDING_RETRY_NAMESPACE,item.id); const latest=await current(item); @@ -188,16 +194,22 @@ export function registerEmbeddingRecovery({kv,KV,index,provider,readiness,withKe if(!inMemory&&readiness&&!await readiness.ensureReady()){await retain(item,{code:'PROVIDER_UNAVAILABLE'},false);return false;} const generation=randomUUID(); // Durable intent precedes inference and mutation; restart always reads canonical text. + let previousIntent; await withKeyedLock(intentKey(item.id),async()=>{ forgetPublication(item.id); const queued=await kv.get(EMBEDDING_RETRY_NAMESPACE,item.id); + if(verifyRetainedPublication&&queued)previousIntent=Object.fromEntries( + ['id','sessionId','kind','attempts','retryAt','generation','fingerprint','phase','errorCode'].flatMap(key=>{ + const value=queued[key]; + return typeof value==='string'?[[key,value.slice(0,256)]]:Number.isSafeInteger(value)&&value>=0?[[key,value]]:[]; + })); await kv.set(EMBEDDING_RETRY_NAMESPACE,item.id,{...queued,id:item.id,sessionId:item.sessionId,kind:item.kind,attempts:queued?.attempts??0,retryAt:queued?.retryAt??0,generation,fingerprint,phase:inMemory?'embedded':'queued'}); pending(item); if(certify)await certifyIntent(item); }); - return {item,text,generation,fingerprint,inMemory}; + return {item,text,generation,fingerprint,inMemory,previousIntent}; } - async function executePrepared({item,text,generation,fingerprint,inMemory}) { + async function executePrepared({item,text,generation,fingerprint,inMemory,previousIntent}) { if(stopped||backgroundPaused())return false; const queued=await kv.get(EMBEDDING_RETRY_NAMESPACE,item.id); if(queued?.generation!==generation||queued.fingerprint!==fingerprint)return false; @@ -209,12 +221,29 @@ export function registerEmbeddingRecovery({kv,KV,index,provider,readiness,withKe const vector=inMemory?null:await provider.embed(text); if(!inMemory&&(!vector||vector.length!==provider.dimensions))throw Object.assign(Error('embedding_dimension_mismatch'),{code:'EMBEDDING_DIMENSION_MISMATCH'}); const ready=await withKeyedLock(intentKey(item.id),async()=>{ - const owned=await kv.get(EMBEDDING_RETRY_NAMESPACE,item.id); + let owned=await kv.get(EMBEDDING_RETRY_NAMESPACE,item.id); if(owned?.generation!==generation||owned.fingerprint!==fingerprint)return false; const canonical=await current(item); if(canonical===null){await discardUnlocked(item);return false;} if(digest(canonical)!==fingerprint){await retainUnlocked(item,{code:'EMBEDDING_PROJECTION_CHANGED'},false);return false;} - if(vector){index.add(item.id,item.sessionId,vector);accepted.set(item.id,fingerprint);onIndexChanged?.();} + if(vector){ + const reused=verifyRetainedPublication && await verifyRetainedPublication({id:item.id,entry:{sessionId:item.sessionId,embedding:vector}}); + if(!reused){index.add(item.id,item.sessionId,vector);onIndexChanged?.();} + else { + if(stopped||backgroundPaused())return false; + owned=await kv.get(EMBEDDING_RETRY_NAMESPACE,item.id); + if(owned?.generation!==generation||owned.fingerprint!==fingerprint)return false; + const currentText=await current(item); + if(currentText===null||digest(currentText)!==fingerprint)return false; + if(!owned.retainedRecovery){ + const committed=await kv.get(EMBEDDING_COMMITTED_NAMESPACE,item.id); + const previousCommit=committed?Object.fromEntries( + ['id','sessionId','kind','fingerprint','committedAt'].filter(key=>typeof committed[key]==='string').map(key=>[key,committed[key].slice(0,256)])):null; + owned={...owned,retainedRecovery:{version:1,completion:'sdk',verified:false,synced:false,previousIntent:previousIntent??null,previousCommit}}; + } + } + accepted.set(item.id,fingerprint); + } await kv.set(EMBEDDING_RETRY_NAMESPACE,item.id,{...owned,phase:'embedded'}); return true; }); @@ -250,12 +279,26 @@ export function registerEmbeddingRecovery({kv,KV,index,provider,readiness,withKe if(reconciled){committedRows=null;committedOffset=0;} } async function runPump(){ - if(stopped||backgroundPaused()||pumping)return;pumping=true; + if(stopped||backgroundPaused()||pumping)return false;pumping=true; + const roundPublications=new Set(),selected=[]; + const beforeCompleted=state.completedCount;let failed=false,more=false; try{ if(!reconciled)await reconcile(50); const rows=await kv.list(EMBEDDING_RETRY_NAMESPACE); + const byId=new Map(rows.map(item=>[item.id,item])); + for(const item of rows)if(!pendingIds.has(item.id))pending(item); + // Keep a process-local round robin independent of engine enumeration. + // New intents join the end; a failed attempt must not monopolize the head. + for(const id of pendingIds){ + const item=byId.get(id); + if(item&&!(item.retryAt>clock())&&!publications.has(id))selected.push(item); + if(selected.length===5)break; + } + for(const item of selected){pendingIds.delete(item.id);pendingIds.add(item.id);} + const selectedIds=new Set(selected.map(item=>item.id)); + more=rows.some(item=>!selectedIds.has(item.id)&&!(item.retryAt>clock())&&!publications.has(item.id)); const groups=new Map(); - for(const item of rows.filter(item=>!(item.retryAt>clock())&&!publications.has(item.id)).slice(0,5)){ + for(const item of selected){ const scope=sourceScope(item);if(!groups.has(scope))groups.set(scope,[]);groups.get(scope).push(item); } for(const items of groups.values()){ @@ -270,7 +313,7 @@ export function registerEmbeddingRecovery({kv,KV,index,provider,readiness,withKe try{ const queued=await kv.get(EMBEDDING_RETRY_NAMESPACE,item.id);if(!queued||queued.retryAt>clock())continue; const result=await prepare(queued,await current(queued),false);if(result)prepared.push(result); - }catch(error){state.lastError=networkErrorCode(error);} + }catch(error){failed=true;state.lastError=networkErrorCode(error);} } if(!prepared.length)return; const selections=prepared.map(({item})=>[{scope:sourceScope(item),key:item.id},{scope:EMBEDDING_RETRY_NAMESPACE,key:item.id}]); @@ -285,23 +328,45 @@ export function registerEmbeddingRecovery({kv,KV,index,provider,readiness,withKe if(stopped||backgroundPaused())break; const result=results?.[offset]; if(result?.status!=='fulfilled'){ + failed=true; await withKeyedLock(intentKey(entry.item.id),async()=>{ if((await kv.get(EMBEDDING_RETRY_NAMESPACE,entry.item.id))?.generation===entry.generation)await retainUnlocked(entry.item,result?.reason,true); });continue; } - try{await executePrepared(entry);}catch(error){state.lastError=networkErrorCode(error);} + try{ + if(!await executePrepared(entry))failed=true; + const publication=publications.get(entry.item.id);if(publication)roundPublications.add(publication); + }catch(error){failed=true;state.lastError=networkErrorCode(error);} if(readiness?.snapshot?.().status==='waiting')break; } }); if(readiness?.snapshot?.().status==='waiting')break; } - }catch(error){state.lastError=networkErrorCode(error);logger?.warn('Embedding recovery scan deferred',{errorCode:state.lastError});} - finally{pumping=false;} + }catch(error){failed=true;state.lastError=networkErrorCode(error);logger?.warn('Embedding recovery scan deferred',{errorCode:state.lastError});} + finally{ + // A fast provider must not grow an unbounded queue of unpublished vectors. + // Release the model locks first; publication retains its existing fences. + const outcomes=await Promise.allSettled([...roundPublications]); + if(outcomes.some(result=>result.status==='rejected'))failed=true; + pumping=false; + } + return !failed&&!stopped&&!backgroundPaused()&&readiness?.snapshot?.().status!=='waiting'&& + more&&state.completedCount>beforeCompleted&&selected.every(item=>!pendingIds.has(item.id)); } - function pump(){if(stopped)return Promise.resolve();return track(initialize().then(()=>backgroundTask(runPump)()));} - function stop(){stopped=true;if(interval){clearInterval(interval);interval=null;}} + function pumpWork(){if(stopped)return Promise.resolve(false);return track(initialize().then(()=>backgroundTask(runPump)()));} + function pump(){return pumpWork().then(()=>{});} + function stop(){stopped=true;for(const release of sourceGuards)release();if(interval){clearTimeout(interval);interval=null;}} async function drain(){while(active.size)await Promise.allSettled([...active]);} - function start(){if(stopped||interval)return;void track(initialize()).catch(error=>{state.lastError=networkErrorCode(error);});interval=setInterval(()=>void pump().catch(error=>{state.lastError=networkErrorCode(error);logger?.warn('Embedding recovery initialization deferred',{errorCode:state.lastError});}),15000);interval.unref();} + function scheduleNext(delay){ + if(stopped)return; + interval=setTimeout(()=>{ + interval=null; + void pumpWork().then(fast=>scheduleNext(fast===true?1000:15000),error=>{ + state.lastError=networkErrorCode(error);logger?.warn('Embedding recovery initialization deferred',{errorCode:state.lastError});scheduleNext(15000); + }); + },delay);interval.unref(); + } + function start(){if(stopped||started)return;started=true;void track(initialize()).catch(error=>{state.lastError=networkErrorCode(error);});scheduleNext(15000);} if(schedule)start(); return {add,enqueue,prepareEnqueue,enqueueBatch,pump,start,initialize:()=>track(initialize()),reconcile:()=>track(reconcile()),stop,drain, observedPublication:id=>{try{return publicationEvidenceAvailable?structuredClone(observedPublications.get(id)??null):null;}catch{evidenceFailure();return null;}}, diff --git a/src/ops/engine-epoch.d.mts b/src/ops/engine-epoch.d.mts index 8908b29db4..e82b709731 100644 --- a/src/ops/engine-epoch.d.mts +++ b/src/ops/engine-epoch.d.mts @@ -1,4 +1,7 @@ export interface ProcessEpoch {pid:number;start:string} -export interface EngineEpoch {host:string;bootId:string;endpoint:string;pidNamespace:string;netNamespace:string;workerUid:number;worker:ProcessEpoch;engine:ProcessEpoch} -export interface EngineEpochEvidence {bootIdentity():string|null;capture():EngineEpoch|null;current(expected:EngineEpoch):boolean;gone(previous:ProcessEpoch):boolean} -export function createEngineEpochEvidence(options:{engineUrl:string;enginePidPath:string;peer:()=>{remoteAddress?:string;remotePort?:number;localAddress?:string;localPort?:number}|null;procRoot?:string;machineIdPath?:string;workerPid?:number;workerUid?:number;pidAlive?:(pid:number)=>boolean}):EngineEpochEvidence; +export interface EngineEpoch {host:string;bootId:string;endpoint:string;pidNamespace:string;netNamespace:string;workerUid:number;worker:ProcessEpoch;engine:ProcessEpoch;state?:ProcessEpoch&{instanceId:string;store:string;executableSha256:string}} +export interface EngineEpochEvidence {diagnostic?():string|null;bootIdentity():string|null;capture():EngineEpoch|null;current(expected:EngineEpoch):boolean;gone(previous:ProcessEpoch):boolean} +export function createEngineEpochEvidence(options:{engineUrl:string;enginePidPath:string;engineStatePath?:string;peer:()=>{remoteAddress?:string;remotePort?:number;localAddress?:string;localPort?:number}|null;procRoot?:string;machineIdPath?:string;workerPid?:number;workerUid?:number;pidAlive?:(pid:number)=>boolean}):EngineEpochEvidence; + +export function readEnginePid(options:{enginePidPath:string;engineStatePath?:string;engineUrl:string}):number; +export function waitForEngineEpoch(evidence:Pick,options?:{timeoutMs?:number;pollMs?:number}):Promise; diff --git a/src/ops/engine-epoch.mjs b/src/ops/engine-epoch.mjs index 7a80800ad5..77e33cf1da 100644 --- a/src/ops/engine-epoch.mjs +++ b/src/ops/engine-epoch.mjs @@ -1,8 +1,26 @@ +import {readOwnedTcpRows} from './proc-tcp-rows.mjs'; import {readFileSync,readlinkSync,readdirSync} from 'node:fs'; import {join,basename} from 'node:path'; import {createHash} from 'node:crypto'; const bootPattern=/^[a-f0-9-]{36}$/; -export function createEngineEpochEvidence({engineUrl,enginePidPath,peer,procRoot='/proc',machineIdPath='/etc/machine-id',workerPid=process.pid,workerUid=process.getuid?.()??null,pidAlive=pid=>{try{process.kill(pid,0);return true;}catch(e){return e?.code!=='ESRCH';}}}){ +// Metadata supplies only a candidate PID. capture() still verifies process and socket ownership. +export function readEnginePid({enginePidPath,engineStatePath,engineUrl}){ + let state,metadataExists=false; + if(engineStatePath){ + try{state=JSON.parse(readFileSync(engineStatePath,'utf8'));metadataExists=true;} + catch(error){if(error?.code!=='ENOENT')throw error;} + } + if(metadataExists&&(!state||Array.isArray(state)||!['compose','native'].includes(state.kind)))throw Error('ENGINE_METADATA_UNSUPPORTED'); + let pid; + if(state?.kind==='compose'){ + if(new URL(state.engineUrl).href!==new URL(engineUrl).href)throw Error('ENGINE_ENDPOINT_MISMATCH'); + pid=state.enginePid; + }else{pid=Number(readFileSync(enginePidPath,'utf8').trim());} + if(!Number.isSafeInteger(pid)||pid<1)throw Error('ENGINE_PID_UNAVAILABLE'); + return pid; +} +export function createEngineEpochEvidence({engineUrl,enginePidPath,engineStatePath,peer,procRoot='/proc',machineIdPath='/etc/machine-id',workerPid=process.pid,workerUid=process.getuid?.()??null,pidAlive=pid=>{try{process.kill(pid,0);return true;}catch(e){return e?.code!=='ESRCH';}}}){ + let lastFailure=null;const reject=code=>{lastFailure=code;return null;}; function identity(pid){ if(!Number.isSafeInteger(pid)||pid<1)throw Error('PROCESS_ID_UNSUPPORTED'); const raw=readFileSync(join(procRoot,String(pid),'stat'),'utf8'),at=raw.lastIndexOf(') '),fields=raw.slice(at+2).trim().split(/\s+/); @@ -11,25 +29,40 @@ export function createEngineEpochEvidence({engineUrl,enginePidPath,peer,procRoot } const same=(a,b)=>!!a&&!!b&&a.pid===b.pid&&a.start===b.start; const addresses=address=>address==='127.0.0.1'||address==='::ffff:127.0.0.1'?['0100007F','0000000000000000FFFF00000100007F']:address==='::1'?['00000000000000000000000001000000']:[]; - function capture(){try{ - const url=new URL(engineUrl);if(!['ws:','wss:'].includes(url.protocol)||!['127.0.0.1','localhost','[::1]'].includes(url.hostname)||url.username||url.password) return null; - const socket=peer?.();if(!socket||!addresses(socket.remoteAddress).length||!addresses(socket.localAddress).length||socket.remotePort!==Number(url.port||(url.protocol==='wss:'?443:80))||!Number.isSafeInteger(socket.localPort)||socket.localPort<1)return null; - const worker=identity(workerPid),pid=Number(readFileSync(enginePidPath,'utf8').trim()),engine=identity(pid); - const exe=readlinkSync(join(procRoot,String(pid),'exe'));if(basename(exe.replace(/ \(deleted\)$/,''))!=='iii')return null; - if(readlinkSync(join(procRoot,String(pid),'ns/net'))!==readlinkSync(join(procRoot,String(workerPid),'ns/net')))return null; - const fds=readdirSync(join(procRoot,String(pid),'fd'));if(fds.length>4096)return null; + function capture(){lastFailure=null;try{ + const url=new URL(engineUrl);if(!['ws:','wss:'].includes(url.protocol)||!['127.0.0.1','localhost','[::1]'].includes(url.hostname)||url.username||url.password) return reject('ENGINE_ENDPOINT_UNSUPPORTED'); + const socket=peer?.();if(!socket||!addresses(socket.remoteAddress).length||!addresses(socket.localAddress).length||socket.remotePort!==Number(url.port||(url.protocol==='wss:'?443:80))||!Number.isSafeInteger(socket.localPort)||socket.localPort<1)return reject('ENGINE_SDK_SOCKET_UNAVAILABLE'); + const worker=identity(workerPid),pid=readEnginePid({enginePidPath,engineStatePath,engineUrl}),engine=identity(pid); + const exe=readlinkSync(join(procRoot,String(pid),'exe'));if(basename(exe.replace(/ \(deleted\)$/,''))!=='iii')return reject('ENGINE_EXECUTABLE_UNVERIFIED'); + if(readlinkSync(join(procRoot,String(pid),'ns/net'))!==readlinkSync(join(procRoot,String(workerPid),'ns/net')))return reject('ENGINE_NETWORK_NAMESPACE_CHANGED'); + const fds=readdirSync(join(procRoot,String(pid),'fd'));if(fds.length>4096)return reject('ENGINE_FD_LIMIT'); const owned=new Set();for(const fd of fds){try{const link=readlinkSync(join(procRoot,String(pid),'fd',fd)),m=/^socket:\[(\d+)\]$/.exec(link);if(m)owned.add(m[1]);}catch{}} const entries=[];for(const file of ['tcp','tcp6']){ - const raw=readFileSync(join(procRoot,String(pid),'net',file),'utf8');if(raw.length>1024*1024)return null; - for(const line of raw.split('\n')){const v=line.trim().split(/\s+/),local=v[1]?.split(':'),remote=v[2]?.split(':');if(local?.length===2&&remote?.length===2)entries.push({local:local[0].toUpperCase(),port:parseInt(local[1],16),remote:remote[0].toUpperCase(),remotePort:parseInt(remote[1],16),state:v[3],inode:v[9]});} + for(const row of readOwnedTcpRows(join(procRoot,String(pid),'net',file),owned,row=>parseInt(row.local?.split(':')[1],16)===socket.remotePort&&(row.state==='0A'||parseInt(row.remote?.split(':')[1],16)===socket.localPort))){ + const local=row.local.split(':'),remote=row.remote.split(':'); + entries.push({local:local[0],port:parseInt(local[1],16),remote:remote[0],remotePort:parseInt(remote[1],16),state:row.state,inode:row.inode}); + } } const localAddresses=addresses(socket.remoteAddress),remoteAddresses=addresses(socket.localAddress); const connections=entries.filter(e=>e.state==='01'&&e.port===socket.remotePort&&e.remotePort===socket.localPort&&localAddresses.includes(e.local)&&remoteAddresses.includes(e.remote)); const listeners=entries.filter(e=>e.state==='0A'&&e.port===socket.remotePort&&(localAddresses.includes(e.local)||/^0{8}$|^0{32}$/.test(e.local))); - if(connections.length!==1||listeners.length!==1||!owned.has(connections[0].inode)||!owned.has(listeners[0].inode))return null; - const latest=peer?.();if(!latest||['remoteAddress','remotePort','localAddress','localPort'].some(k=>latest[k]!==socket[k])||!same(engine,identity(pid))||!same(worker,identity(workerPid)))return null; - const bootId=readFileSync(join(procRoot,'sys/kernel/random/boot_id'),'utf8').trim(),machine=readFileSync(machineIdPath,'utf8').trim();if(!bootPattern.test(bootId)||!machine)return null; - const pidNamespace=readlinkSync(join(procRoot,String(workerPid),'ns/pid')),netNamespace=readlinkSync(join(procRoot,String(workerPid),'ns/net'));if(!Number.isSafeInteger(workerUid)||workerUid<0)return null;return {host:createHash('sha256').update(machine).digest('hex'),bootId,endpoint:url.origin,pidNamespace,netNamespace,workerUid,worker,engine}; - }catch{return null;}} - return {bootIdentity(){try{const value=readFileSync(join(procRoot,'sys/kernel/random/boot_id'),'utf8').trim();return bootPattern.test(value)?value:null;}catch{return null;}},capture,current(expected){const value=capture();return !!value&&!!expected&&value.host===expected.host&&value.bootId===expected.bootId&&value.endpoint===expected.endpoint&&value.pidNamespace===expected.pidNamespace&&value.netNamespace===expected.netNamespace&&value.workerUid===expected.workerUid&&same(value.engine,expected.engine)&&same(value.worker,expected.worker);},gone(previous){try{if(!previous||!/^\d+$/.test(previous.start))return false;return !same(previous,identity(previous.pid));}catch(error){return (error?.code==='ENOENT'||error?.code==='ESRCH')&&!pidAlive(previous?.pid);}}}; + if(connections.length!==1)return reject('ENGINE_PEER_CONNECTION_UNVERIFIED'); + if(listeners.length!==1)return reject('ENGINE_LISTENER_UNVERIFIED'); + if(!owned.has(connections[0].inode)||!owned.has(listeners[0].inode))return reject('ENGINE_SOCKET_OWNER_UNVERIFIED'); + const latest=peer?.();if(!latest||['remoteAddress','remotePort','localAddress','localPort'].some(k=>latest[k]!==socket[k])||!same(engine,identity(pid))||!same(worker,identity(workerPid)))return reject('ENGINE_PEER_OR_PROCESS_CHANGED'); + const bootId=readFileSync(join(procRoot,'sys/kernel/random/boot_id'),'utf8').trim(),machine=readFileSync(machineIdPath,'utf8').trim();if(!bootPattern.test(bootId)||!machine)return reject('ENGINE_BOOT_ID_UNVERIFIED'); + const pidNamespace=readlinkSync(join(procRoot,String(workerPid),'ns/pid')),netNamespace=readlinkSync(join(procRoot,String(workerPid),'ns/net'));if(!Number.isSafeInteger(workerUid)||workerUid<0)return reject('ENGINE_WORKER_UID_UNVERIFIED');return {host:createHash('sha256').update(machine).digest('hex'),bootId,endpoint:url.origin,pidNamespace,netNamespace,workerUid,worker,engine}; + }catch(error){return reject(['ENOENT','ESRCH','EACCES','EPERM'].includes(error?.code)?'ENGINE_PROC_'+error.code:/^(ENGINE_|PROCESS_|PROC_TCP_)[A-Z_]+$/.test(error?.message??'')?error.message:'ENGINE_CAPTURE_FAILED');}} + return {diagnostic:()=>lastFailure,bootIdentity(){try{const value=readFileSync(join(procRoot,'sys/kernel/random/boot_id'),'utf8').trim();return bootPattern.test(value)?value:null;}catch{return null;}},capture,current(expected){const value=capture();return !!value&&!!expected&&value.host===expected.host&&value.bootId===expected.bootId&&value.endpoint===expected.endpoint&&value.pidNamespace===expected.pidNamespace&&value.netNamespace===expected.netNamespace&&value.workerUid===expected.workerUid&&same(value.engine,expected.engine)&&same(value.worker,expected.worker);},gone(previous){try{if(!previous||!/^\d+$/.test(previous.start))return false;return !same(previous,identity(previous.pid));}catch(error){return (error?.code==='ENOENT'||error?.code==='ESRCH')&&!pidAlive(previous?.pid);}}}; +} + +// Never acquire/write a lease marker until delayed CLI metadata is verifiable. +export async function waitForEngineEpoch(evidence,{timeoutMs=30000,pollMs=25}={}){ + const deadline=Date.now()+timeoutMs; + do{ + const epoch=evidence.capture(); + if(epoch)return epoch; + if(Date.now()>=deadline)throw Object.assign(Error('ENGINE_EPOCH_UNVERIFIED'),{code:'ENGINE_EPOCH_UNVERIFIED'}); + await new Promise(resolve=>setTimeout(resolve,Math.min(pollMs,Math.max(1,deadline-Date.now())))); + }while(true); } diff --git a/src/ops/file-writer-lease.d.mts b/src/ops/file-writer-lease.d.mts index 73c4ee3d96..617d3ef81d 100644 --- a/src/ops/file-writer-lease.d.mts +++ b/src/ops/file-writer-lease.d.mts @@ -1,8 +1,10 @@ import type {EngineEpochEvidence} from './engine-epoch.mjs'; export interface FileWriterLease {close():void} -export function acquireFileWriterLease(storePath:string,options?:{evidence?:EngineEpochEvidence}):FileWriterLease; +export function acquireFileWriterLease(storePath:string,options?:{evidence?:EngineEpochEvidence;requireVerifiedEpoch?:boolean}):FileWriterLease; export function ownsFileWriterLease(lease:unknown,storePath:string):boolean; export function fileWriterReadCurrent(lease:FileWriterLease):boolean; export function fileWriterRecoveryAdmitted(lease:FileWriterLease):boolean; export function fileWriterRecoveryReason(lease:FileWriterLease):string|null; export function markFileWriterClean(lease:FileWriterLease):void; + +export function markFileWriterSealedClean(lease:FileWriterLease,receipt:unknown):void; diff --git a/src/ops/file-writer-lease.mjs b/src/ops/file-writer-lease.mjs index d9752c7194..b035b46508 100644 --- a/src/ops/file-writer-lease.mjs +++ b/src/ops/file-writer-lease.mjs @@ -1,3 +1,4 @@ +import {nativeMigrationForMarker} from './native-lease-migration.mjs'; import {createHash} from 'node:crypto'; import {openSync,closeSync,realpathSync,fstatSync,lstatSync,constants,readFileSync,writeSync,ftruncateSync,fsyncSync} from 'node:fs'; import {join} from 'node:path'; @@ -5,36 +6,40 @@ import {spawnSync} from 'node:child_process'; const owners=new WeakMap(); const failure=code=>Object.assign(Error(code),{code}); const sameProcess=(a,b)=>!!a&&!!b&&a.pid===b.pid&&a.start===b.start; -const validEpoch=e=>e&&/^[a-f0-9]{64}$/.test(e.host)&&/^[a-f0-9-]{36}$/.test(e.bootId)&&typeof e.endpoint==='string'&&typeof e.pidNamespace==='string'&&typeof e.netNamespace==='string'&&Number.isSafeInteger(e.workerUid)&&e.workerUid>=0&&[e.worker,e.engine].every(p=>Number.isSafeInteger(p?.pid)&&p.pid>0&&/^\d+$/.test(p.start)); -function writeMarker(entry,clean){ftruncateSync(entry.fd,0);writeSync(entry.fd,JSON.stringify({version:3,store:entry.store,epoch:entry.epoch,clean,blockedUntilBoot:entry.blockedUntilBoot,unverifiedWriter:entry.unverifiedWriter,uncertaintyBootId:entry.uncertaintyBootId}),0,'utf8');fsyncSync(entry.fd);} +const validState=s=>s&&Number.isSafeInteger(s.pid)&&s.pid>0&&/^\d+$/.test(s.start)&&/^[a-f0-9-]{36}$/.test(s.instanceId)&&/^[a-f0-9]{64}$/.test(s.store)&&/^[a-f0-9]{64}$/.test(s.executableSha256); +const validEpoch=e=>e&&/^[a-f0-9]{64}$/.test(e.host)&&/^[a-f0-9-]{36}$/.test(e.bootId)&&typeof e.endpoint==='string'&&typeof e.pidNamespace==='string'&&typeof e.netNamespace==='string'&&Number.isSafeInteger(e.workerUid)&&e.workerUid>=0&&(!e.state||validState(e.state))&&[e.worker,e.engine].every(p=>Number.isSafeInteger(p?.pid)&&p.pid>0&&/^\d+$/.test(p.start)); +function writeMarker(entry,clean){ftruncateSync(entry.fd,0);writeSync(entry.fd,JSON.stringify({version:entry.epoch?.state?4:3,store:entry.store,epoch:entry.epoch,clean,blockedUntilBoot:entry.blockedUntilBoot,unverifiedWriter:entry.unverifiedWriter,uncertaintyBootId:entry.uncertaintyBootId,...(entry.migration?{migration:entry.migration}:{})}),0,'utf8');fsyncSync(entry.fd);entry.markerOwned=true;} export function acquireFileWriterLease(storePath,options={}){ if(process.platform!=='linux')throw failure('FILE_WRITER_UNSUPPORTED'); + if(options.requireVerifiedEpoch&&!validEpoch(options.evidence?.capture()))throw failure('ENGINE_EPOCH_UNVERIFIED'); const directory=realpathSync(storePath),store=createHash('sha256').update(directory).digest('hex'),path=join(directory,'.agentmemory-writer.lock'); const fd=openSync(path,constants.O_CREAT|constants.O_RDWR|constants.O_NOFOLLOW,0o600); try{ if(!fstatSync(fd).isFile())throw failure('FILE_WRITER_UNSUPPORTED'); const result=spawnSync('/usr/bin/flock',['--exclusive','--nonblock','3'],{stdio:['ignore','ignore','ignore',fd],timeout:3000}); if(result.error||result.status!==0)throw failure(result.status===1?'FILE_WRITER_BUSY':'FILE_WRITER_UNSUPPORTED'); - let prior=null,unsupported=false;const size=fstatSync(fd).size; - if(size){try{if(size>4096)throw Error();prior=JSON.parse(readFileSync(fd,'utf8'));unsupported=prior?.version!==3||typeof prior.clean!=='boolean'||prior.blockedUntilBoot!==undefined&&typeof prior.blockedUntilBoot!=='boolean'||!/^[a-f0-9]{64}$/.test(prior.store??'')||!validEpoch(prior.epoch);}catch{unsupported=true;}} - const epoch=options.evidence?.capture()??null;let admitted=false,reason='WRITER_EPOCH_UNVERIFIED',blockedUntilBoot=false,unverifiedWriter=false,uncertaintyBootId=null; + let prior=null,original='',unsupported=false;const size=fstatSync(fd).size; + if(size){try{if(size>4096)throw Error();original=readFileSync(fd,'utf8');prior=JSON.parse(original);unsupported=prior?.version!==(prior?.epoch?.state?4:3)||typeof prior.clean!=='boolean'||prior.blockedUntilBoot!==undefined&&typeof prior.blockedUntilBoot!=='boolean'||!/^[a-f0-9]{64}$/.test(prior.store??'')||!validEpoch(prior.epoch)||!!prior.epoch?.state&&prior.epoch.state.store!==prior.store;}catch{unsupported=true;}} + const epoch=options.evidence?.capture()??null;if(options.requireVerifiedEpoch&&!validEpoch(epoch))throw failure('ENGINE_EPOCH_UNVERIFIED');const migration=epoch?.state&&prior?.version===3?nativeMigrationForMarker(directory,original,epoch,options.evidence):prior?.migration??null;if(options.requireVerifiedEpoch&&prior?.version===4&&!epoch?.state)throw failure('WRITER_NATIVE_DOWNGRADE_UNSUPPORTED');let admitted=false,reason='WRITER_EPOCH_UNVERIFIED',blockedUntilBoot=false,unverifiedWriter=false,uncertaintyBootId=null; if(!validEpoch(epoch))reason='NATIVE_ENGINE_UNVERIFIED'; - else if(unsupported)reason='WRITER_PRIOR_UNSUPPORTED'; + else if(unsupported||(prior&&Boolean(prior.epoch?.state)!==Boolean(epoch?.state)&&!(prior.version===3&&!!epoch.state&&migration?.kind==='verified-native-v3')))reason='WRITER_PRIOR_UNSUPPORTED'; + else if(epoch.state&&epoch.state.store!==store)reason='WRITER_PRIOR_FOREIGN'; else if(prior&&(prior.store!==store||prior.epoch.host!==epoch.host||prior.epoch.endpoint!==epoch.endpoint||prior.epoch.pidNamespace!==epoch.pidNamespace||prior.epoch.netNamespace!==epoch.netNamespace||prior.epoch.workerUid!==epoch.workerUid))reason='WRITER_PRIOR_FOREIGN'; else if(prior){ const old=prior.epoch; if(prior.unverifiedWriter===true&&(!/^[a-f0-9-]{36}$/.test(prior.uncertaintyBootId??'')||prior.uncertaintyBootId===epoch.bootId)){blockedUntilBoot=true;unverifiedWriter=true;uncertaintyBootId=prior.uncertaintyBootId??null;reason='WRITER_NATIVE_PROBE_UNCERTAIN';} else if(old.bootId!==epoch.bootId||prior.unverifiedWriter===true)admitted=true; else { - const workerGone=options.evidence.gone(old.worker),engineChanged=!sameProcess(old.engine,epoch.engine),engineGone=engineChanged&&options.evidence.gone(old.engine); - blockedUntilBoot=prior.blockedUntilBoot===true||!workerGone||engineChanged&&!engineGone; + const workerGone=options.evidence.gone(old.worker),stateChanged=!!old.state&&(!sameProcess(old.state,epoch.state)||old.state.instanceId!==epoch.state?.instanceId),stateGone=!stateChanged||options.evidence.gone(old.state),engineChanged=!sameProcess(old.engine,epoch.engine)||stateChanged,engineGone=engineChanged&&stateGone&&(sameProcess(old.engine,epoch.engine)||options.evidence.gone(old.engine)); + const externalWriterUnclean=!!old.state&&prior.clean!==true&&!options.evidence.gone(old.state); + blockedUntilBoot=prior.blockedUntilBoot===true||!workerGone||engineChanged&&!engineGone||externalWriterUnclean; admitted=!blockedUntilBoot&&workerGone&&(prior.clean===true&&!engineChanged||engineChanged&&engineGone); if(blockedUntilBoot)reason='WRITER_ENGINE_OVERLAP'; } } - const entry={fd,directory,path,store,epoch,evidence:options.evidence,admitted,reason,blockedUntilBoot,unverifiedWriter,uncertaintyBootId}; + const entry={fd,directory,path,store,epoch,migration,evidence:options.evidence,admitted,reason,blockedUntilBoot,unverifiedWriter,uncertaintyBootId}; // Foreign/unsupported evidence is retained; a local restart cannot erase it. - if(!size||validEpoch(epoch)&&!unsupported&&reason!=='WRITER_PRIOR_FOREIGN'){ + if(!size||validEpoch(epoch)&&!unsupported&&reason!=='WRITER_PRIOR_UNSUPPORTED'&&reason!=='WRITER_PRIOR_FOREIGN'){ if(validEpoch(epoch))writeMarker(entry,false); else {writeSync(fd,JSON.stringify({version:3,unsupported:true}),0,'utf8');fsyncSync(fd);} } @@ -53,3 +58,9 @@ export function fileWriterReadCurrent(lease){const e=owners.get(lease);return !! export function fileWriterRecoveryAdmitted(lease){const e=owners.get(lease);return !!e?.admitted&&ownsFileWriterLease(lease,e.directory)&&e.evidence.current(e.epoch);} export function fileWriterRecoveryReason(lease){const e=owners.get(lease);return !e?'FILE_WRITER_UNSUPPORTED':fileWriterRecoveryAdmitted(lease)?null:e.admitted?'NATIVE_ENGINE_CHANGED':e.reason;} export function markFileWriterClean(lease){const e=owners.get(lease);if(!e||!ownsFileWriterLease(lease,e.directory))throw failure('FILE_WRITER_LOST');if(!fileWriterRecoveryAdmitted(lease))throw failure('WRITER_EPOCH_UNVERIFIED');writeMarker(e,true);} + +export function markFileWriterSealedClean(lease,receipt){ + const e=owners.get(lease); + if(!e?.markerOwned||!e.epoch?.state||e.blockedUntilBoot||e.unverifiedWriter||!ownsFileWriterLease(lease,e.directory)||!e.evidence.current(e.epoch)||receipt?.version!==1||receipt.sealed!==true||receipt.synced!==true||receipt.instanceId!==e.epoch.state.instanceId||typeof receipt.storePath!=='string'||realpathSync(receipt.storePath)!==e.directory)throw failure('STATE_SEAL_UNVERIFIED'); + writeMarker(e,true); +} diff --git a/src/ops/graph-file-list.d.mts b/src/ops/graph-file-list.d.mts index f781d8497f..fe9e8a9c31 100644 --- a/src/ops/graph-file-list.d.mts +++ b/src/ops/graph-file-list.d.mts @@ -2,5 +2,5 @@ export function createGraphFileList(options:{storePath:string}):{ accepts(scope:string):boolean; record(scope:string,key:string,value:unknown):void; invalidate(scope:string):void; - list(scope:string):Promise; + list(scope:string,options?:{project?:(value:unknown,key?:string)=>T}):Promise; }; diff --git a/src/ops/graph-file-list.mjs b/src/ops/graph-file-list.mjs index d38e2aefc5..24693bad45 100644 --- a/src/ops/graph-file-list.mjs +++ b/src/ops/graph-file-list.mjs @@ -4,28 +4,43 @@ import {readNamespaceEntries} from './namespace-reader.mjs'; export function createGraphFileList({storePath}) { const pending=new Map(),uncertain=new Set(); - const accepts=scope=>/^mem:graph:(?:nodes|edges)(?::generation:[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})?$/.test(scope); + // Capture originals and review history can exceed one SDK frame. + const captureScope=scope=>scope==='mem:capture:inbox'||scope==='mem:capture:review'; + const accepts=scope=>captureScope(scope)||/^mem:graph:(?:nodes|edges)(?::generation:[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})?$/.test(scope); function record(scope,key,value) { if(!accepts(scope)||value===undefined)return; if(!pending.has(scope))pending.set(scope,new Map()); pending.get(scope).set(key,structuredClone(value)); } function invalidate(scope){if(accepts(scope))uncertain.add(scope);} - async function list(scope) { + async function list(scope,{project}={}) { if(!accepts(scope))throw Error('invalid_graph_scope'); if(uncertain.has(scope))throw Error('graph_mutation_uncertain'); - const rows=new Map(); - // shortcut: one full local scan per call; replace with native graph queries when iii provides them. + const rows=new Map(),matched=new Map(); + // Projection discards each disk row's payload before retaining the result. + // Pending SDK acknowledgements retain full values until a matching disk checkpoint. for(let attempt=0;;attempt++) { - rows.clear(); - try {await readNamespaceEntries(join(storePath,encodeURIComponent(scope)+'.bin'),{onEntry:(key,value)=>rows.set(key,value)});break;} + rows.clear();matched.clear(); + try { + await readNamespaceEntries(join(storePath,encodeURIComponent(scope)+'.bin'),{onEntry:(key,value)=>{ + const acknowledged=pending.get(scope)?.get(key); + if(acknowledged!==undefined&&isDeepStrictEqual(value,acknowledged))matched.set(key,acknowledged); + rows.set(key,project?project(value,key):value); + }}); + break; + } catch(error){if(error.code==='ENOENT')break;if(error.code==='state_namespace_changed'&&attempt<1)continue;throw error;} } if(uncertain.has(scope))throw Error('graph_mutation_uncertain'); const overlay=pending.get(scope); for(const [key,value] of overlay||[]) { - if(value===null){if(!rows.has(key))overlay.delete(key);rows.delete(key);} - else {if(isDeepStrictEqual(rows.get(key),value))overlay.delete(key);rows.set(key,value);} + // Absence cannot identify the deletion's checkpoint generation. Capture + // tombstones stay until a newer acknowledgement for that key or restart. + if(value===null){if(!captureScope(scope)&&!rows.has(key))overlay.delete(key);rows.delete(key);} + else { + if(matched.get(key)===value)overlay.delete(key); + rows.set(key,project?project(structuredClone(value),key):captureScope(scope)?structuredClone(value):value); + } } if(overlay&&!overlay.size)pending.delete(scope); return [...rows.values()]; diff --git a/src/ops/memory-pressure.mjs b/src/ops/memory-pressure.mjs index 7c5fe4b5b8..6af2f1ba49 100644 --- a/src/ops/memory-pressure.mjs +++ b/src/ops/memory-pressure.mjs @@ -16,7 +16,7 @@ export function createMemoryPressureGuard({control,readSample=readHostMemory,sch async function readFresh(){ const sequence=++sampleSequence,before=clock(),m=await readSample(),now=clock(); if(sequence!==sampleSequence||!m||!Number.isFinite(m.total)||m.total<=0||!Number.isFinite(m.available)||m.available<0||m.available>m.total||!Number.isFinite(now)||now10000)throw Error('memory_sample_unavailable'); - const reserve=Math.max(2*GiB,m.total*.20),critical=Math.max(GiB/2,m.total*.05),recoveryBytes=Math.max(3*GiB,m.total*.25); + const reserve=Math.max(2*GiB,m.total*.18),critical=Math.max(GiB/2,m.total*.05),recoveryBytes=Math.max(3*GiB,m.total*.22); state={...state,level:m.availablerecoveryBytes}; } @@ -36,7 +36,9 @@ export function createMemoryPressureGuard({control,readSample=readHostMemory,sch if(pendingRecovery){state.recoveryStatus='recovering';return;} const c=control.snapshot(); if(!c.holds?.includes('memory')){state.recoveryStatus='observing';return;} - if(c.holds.length!==1||c.status==='error'){state.recoveryStatus='held';return;} + const memoryWithStorage=c.holds.length===2&&c.holds.includes('storage_error'); + const protectedMemoryRelease=c.holds.includes('storage_error')&&(c.holds.includes('manual')||c.holds.includes('deployment')); + if(!memoryWithStorage&&!protectedMemoryRelease&&(c.holds.length!==1||c.status==='error')){state.recoveryStatus='held';return;} if(c.recovery.attempts>=c.recovery.maxAttempts){state.recoveryStatus='attempt_limit';return;} if(nowboolean;publish?:(path:string,bytes:string)=>void}): {path:string;sourceSha256:string;preparedSourceSha256:string;reconciledCleanShutdown:boolean}; +export function nativeMigrationForMarker(store:string,original:string,epoch:EngineEpoch,evidence:EngineEpochEvidence):NativeMigrationProvenance|null; + +export function resolveNativeStoreConfig(options:{workers:unknown;store:string;exeSha256:string;procRoot:string;pid:number}):{path:string;sha256:string;cwd:string}|undefined; diff --git a/src/ops/native-lease-migration.mjs b/src/ops/native-lease-migration.mjs new file mode 100644 index 0000000000..7d17ff5473 --- /dev/null +++ b/src/ops/native-lease-migration.mjs @@ -0,0 +1,99 @@ +import {readFileSync,readlinkSync,realpathSync,openSync,closeSync,writeFileSync,fsyncSync,mkdirSync,unlinkSync,constants,existsSync} from 'node:fs'; +import {join,dirname,isAbsolute,resolve} from 'node:path'; +import {createHash} from 'node:crypto'; +import {spawnSync} from 'node:child_process'; +import {parse} from 'yaml'; +import {isDeepStrictEqual} from 'node:util'; +const hash=x=>createHash('sha256').update(x).digest('hex'); +const fail=code=>{throw Object.assign(Error(code),{code});}; +const same=(a,b)=>a?.pid===b?.pid&&a?.start===b?.start; +function identity(pid,procRoot){const raw=readFileSync(join(procRoot,String(pid),'stat'),'utf8');const at=raw.lastIndexOf(') ');if(at<0||Number(raw.split(' ')[0])!==pid)fail('MIGRATION_PROCESS_INVALID');return {pid,start:raw.slice(at+2).trim().split(/\s+/)[19]};} +function gone(process,procRoot,pidAlive){try{return !same(identity(process.pid,procRoot),process);}catch(error){return error.code==='ENOENT'&&!pidAlive(process.pid);}} +const alive=pid=>{try{process.kill(pid,0);return true;}catch(e){return e.code!=='ESRCH';}}; +function durableExclusive(path,bytes){ + const fd=openSync(path,constants.O_WRONLY|constants.O_CREAT|constants.O_EXCL|constants.O_NOFOLLOW,0o600); + try{writeFileSync(fd,bytes);fsyncSync(fd);}finally{closeSync(fd);} +} +function syncDir(path){const fd=openSync(path,'r');try{fsyncSync(fd);}finally{closeSync(fd);}} +function lock(path){const fd=openSync(path,constants.O_RDWR|constants.O_CREAT|constants.O_NOFOLLOW,0o600);const r=spawnSync('/usr/bin/flock',['--exclusive','--nonblock','3'],{stdio:['ignore','ignore','ignore',fd],timeout:3000});if(r.error||r.status!==0){closeSync(fd);fail('MIGRATION_STORE_BUSY');}return fd;} +// This exact reviewed native build resolves name-only iii-state via +// ./config/iii-state.yaml value.adapter before constructing its adapter. +const PERSISTED_NATIVE_SHA='a44eba0ca91a2805ea43355e30aec482a688eab33ac84801aec65f6636142654'; +export function resolveNativeStoreConfig({workers,store,exeSha256,procRoot,pid}){ + const state=Array.isArray(workers)?workers.filter(x=>x?.name==='iii-state'):[]; + if(state.length!==1)fail('MIGRATION_STORE_CONFIG_MISMATCH'); + let adapter=state[0]?.config?.adapter,persistence; + if(!adapter){ + if(exeSha256!==PERSISTED_NATIVE_SHA||Object.keys(state[0]).join(',')!=='name'||workers.some(x=>x?.name==='configuration'))fail('MIGRATION_STORE_CONFIG_MISMATCH'); + const cwd=realpathSync(join(procRoot,String(pid),'cwd')),path=realpathSync(join(cwd,'config','iii-state.yaml')),bytes=readFileSync(path),entry=parse(bytes.toString()); + if(entry?.id!=='iii-state'||!entry.value||Object.keys(entry.value).some(k=>k!=='adapter')||bytes.includes(Buffer.from('${')))fail('MIGRATION_PERSISTED_CONFIG_UNSUPPORTED'); + adapter=entry.value.adapter; + persistence={path,sha256:hash(bytes),cwd}; + } + if(adapter?.name!=='kv'||adapter?.config?.store_method!=='file_based'||typeof adapter.config.file_path!=='string'||realpathSync(persistence?resolve(persistence.cwd,adapter.config.file_path):adapter.config.file_path)!==store)fail('MIGRATION_STORE_CONFIG_MISMATCH'); + return persistence; +} +export function prepareNativeMigration({storePath,runtimeDir,expectedExecutable,engineEvidence,procRoot='/proc'}){ + const store=realpathSync(storePath),markerPath=join(store,'.agentmemory-writer.lock'),original=readFileSync(markerPath,'utf8'),marker=JSON.parse(original); + const metadata=JSON.parse(readFileSync(join(runtimeDir,'engine-state.json'),'utf8')); + if(marker.version!==3||marker.epoch?.state||marker.store!==hash(store)||marker.unverifiedWriter||marker.blockedUntilBoot)fail('MIGRATION_LEGACY_EVIDENCE_UNSUPPORTED'); + if(metadata.kind!=='native'||(metadata.compatVersion!==undefined&&metadata.compatVersion!=='0.22.1')||!metadata.binPath||!isAbsolute(metadata.configPath))fail('MIGRATION_NOT_VERIFIED_NATIVE'); + const observed=engineEvidence.capture(); + if(!observed||!same(observed.engine,marker.epoch.engine)||observed.host!==marker.epoch.host||observed.bootId!==marker.epoch.bootId||observed.endpoint!==marker.epoch.endpoint||(metadata.processStart!==undefined&&metadata.processStart!==marker.epoch.engine.start))fail('MIGRATION_NATIVE_EPOCH_MISMATCH'); + const pid=marker.epoch.engine.pid,exe=realpathSync(expectedExecutable),processExe=realpathSync(join(procRoot,String(pid),'exe')); + if(realpathSync(metadata.binPath)!==exe||processExe!==exe)fail('MIGRATION_EXECUTABLE_MISMATCH'); + const argv=readFileSync(join(procRoot,String(pid),'cmdline'),'utf8').split('\0').filter(Boolean),at=argv.indexOf('--config'); + if(at<0||argv[at+1]!==metadata.configPath)fail('MIGRATION_CONFIG_MISMATCH'); + const configBytes=readFileSync(metadata.configPath),config=parse(configBytes.toString()),workers=config?.workers; + const executableSha256=hash(readFileSync(exe)); + const persistence=resolveNativeStoreConfig({workers,store,exeSha256:executableSha256,procRoot,pid}); + if(persistence&&(hash(readFileSync(persistence.path))!==persistence.sha256||realpathSync(join(procRoot,String(pid),'cwd'))!==persistence.cwd))fail('MIGRATION_CAPTURE_CHANGED'); + if(!same(identity(marker.epoch.worker.pid,procRoot),marker.epoch.worker)||!same(identity(pid,procRoot),marker.epoch.engine)||readFileSync(markerPath,'utf8')!==original||!engineEvidence.current(observed))fail('MIGRATION_CAPTURE_CHANGED'); + return {version:1,kind:'verified-native-v3',store,sourceMarker:original,sourceSha256:hash(original),epoch:marker.epoch,native:{executable:exe,executableSha256,...(persistence?{persistence}:{}),configPath:metadata.configPath,configSha256:hash(configBytes),argvSha256:hash(JSON.stringify(argv))}}; +} +export function saveNativeMigrationProof(proof,path){durableExclusive(path,JSON.stringify(proof)+'\n');syncDir(dirname(path));} +// A normal stopped native writer may publish only its clean bit after prepare. +// Preserve both byte strings; never rewrite the original proof or live marker. +function cleanShutdownTransition(original,current){ + try{ + const before=JSON.parse(original),after=JSON.parse(current); + return before.version===3&&!before.epoch?.state&&before.clean===false&&after.clean===true&& + !before.unverifiedWriter&&!before.blockedUntilBoot&&!before.uncertaintyBootId&& + isDeepStrictEqual({...before,clean:true},after); + }catch{return false;} +} +export function finalizeNativeMigration({storePath,proof,procRoot='/proc',pidAlive=alive,publish=durableExclusive}){ + const store=realpathSync(storePath); + if(proof?.version!==1||proof.kind!=='verified-native-v3'||proof.store!==store||proof.sourceSha256!==hash(proof.sourceMarker)||JSON.stringify(JSON.parse(proof.sourceMarker).epoch)!==JSON.stringify(proof.epoch))fail('MIGRATION_PROOF_INVALID'); + if(proof.native.persistence&&hash(readFileSync(proof.native.persistence.path))!==proof.native.persistence.sha256)fail('MIGRATION_PERSISTED_CONFIG_CHANGED'); + if(hash(readFileSync(proof.native.executable))!==proof.native.executableSha256||hash(readFileSync(proof.native.configPath))!==proof.native.configSha256)fail('MIGRATION_ARTIFACT_CHANGED'); + const appLock=lock(join(store,'.agentmemory-writer.lock'));let stateLock; + try{ + stateLock=lock(join(store,'.agentmemory-state.lock')); + const finalMarker=readFileSync(join(store,'.agentmemory-writer.lock'),'utf8'); + const reconciled=finalMarker!==proof.sourceMarker; + if(reconciled&&!cleanShutdownTransition(proof.sourceMarker,finalMarker))fail('MIGRATION_MARKER_CHANGED'); + if(!gone(proof.epoch.worker,procRoot,pidAlive)||!gone(proof.epoch.engine,procRoot,pidAlive))fail('MIGRATION_PRIOR_PROCESS_ALIVE'); + const dir=join(store,'.agentmemory-migrations');mkdirSync(dir,{recursive:true,mode:0o700}); + const sourceSha256=hash(finalMarker),path=join(dir,sourceSha256+'.json'),seal=path+'.complete'; + if(existsSync(path)||existsSync(seal))fail('MIGRATION_RECEIPT_EXISTS'); + const receipt={version:1,kind:'native-v3-to-state-v4',proof,sourceSha256,...(reconciled?{finalMarker,transition:'native-clean-shutdown'}:{}),finalizedAt:new Date().toISOString()}; + const bytes=JSON.stringify(receipt)+'\n'; + publish(path,bytes);syncDir(dir); + try{publish(seal,hash(bytes)+'\n');syncDir(dir);}catch(error){try{unlinkSync(seal);}catch{}throw error;} + return {path,sourceSha256,preparedSourceSha256:proof.sourceSha256,reconciledCleanShutdown:reconciled}; + }finally{if(stateLock!==undefined)closeSync(stateLock);closeSync(appLock);} +} +export function nativeMigrationForMarker(store,original,epoch,evidence){ + try{ + const sourceSha256=hash(original),path=join(store,'.agentmemory-migrations',sourceSha256+'.json'),bytes=readFileSync(path,'utf8'); + if(readFileSync(path+'.complete','utf8').trim()!==hash(bytes))return null; + const receipt=JSON.parse(bytes),proof=receipt.proof,prior=JSON.parse(original); + if(receipt.version!==1||receipt.kind!=='native-v3-to-state-v4'||receipt.sourceSha256!==sourceSha256||proof?.version!==1||proof.kind!=='verified-native-v3'||proof.sourceSha256!==hash(proof.sourceMarker)||proof.store!==realpathSync(store)||prior.version!==3||prior.epoch?.state||prior.unverifiedWriter||prior.blockedUntilBoot||JSON.stringify(proof.epoch)!==JSON.stringify(prior.epoch))return null; + const exact=proof.sourceMarker===original&&proof.sourceSha256===sourceSha256; + const reconciled=receipt.transition==='native-clean-shutdown'&&receipt.finalMarker===original&&cleanShutdownTransition(proof.sourceMarker,original); + if(!exact&&!reconciled)return null; + if(proof.epoch.host!==epoch.host||prior.store!==epoch.state?.store||!evidence.gone(proof.epoch.worker)||!evidence.gone(proof.epoch.engine))return null; + return {version:1,sourceSha256,receiptSha256:hash(bytes),kind:'verified-native-v3'}; + }catch{return null;} +} diff --git a/src/ops/proc-tcp-rows.d.mts b/src/ops/proc-tcp-rows.d.mts new file mode 100644 index 0000000000..1cfec8be8e --- /dev/null +++ b/src/ops/proc-tcp-rows.d.mts @@ -0,0 +1,2 @@ +export interface ProcTcpRow {local:string;remote:string;state:string;inode:string} +export function readOwnedTcpRows(path:string,owned:ReadonlySet,includeRow?:(row:ProcTcpRow)=>boolean):ProcTcpRow[]; diff --git a/src/ops/proc-tcp-rows.mjs b/src/ops/proc-tcp-rows.mjs new file mode 100644 index 0000000000..6207d952ec --- /dev/null +++ b/src/ops/proc-tcp-rows.mjs @@ -0,0 +1,28 @@ +import {openSync,readSync,closeSync} from 'node:fs'; +// /proc//net/tcp is the entire network namespace, not that PID's sockets. +// Stream it with bounded scratch space; retain only rows for explicitly owned +// socket inodes plus caller-selected endpoint conflicts. Host TIME_WAIT churn +// must not invalidate unrelated identities. +export function readOwnedTcpRows(path,owned,includeRow=()=>false){ + const fd=openSync(path,'r'),chunk=Buffer.allocUnsafe(64*1024),rows=[]; + let pending=''; + const line=value=>{ + if(value.length>4096)throw Object.assign(Error('PROC_TCP_LINE_LIMIT'),{code:'PROC_TCP_LINE_LIMIT'}); + const p=value.trim().split(/\s+/); + if(!['01','0A'].includes(p[3]))return; + const row={local:p[1]?.toUpperCase(),remote:p[2]?.toUpperCase(),state:p[3],inode:p[9]}; + if(!owned.has(p[9])&&!includeRow(row))return; + if(!/^[A-Fa-f0-9]{8}(?:[A-Fa-f0-9]{24})?:[A-Fa-f0-9]{4}$/.test(p[1]??'')||!/^[A-Fa-f0-9]{8}(?:[A-Fa-f0-9]{24})?:[A-Fa-f0-9]{4}$/.test(p[2]??''))throw Object.assign(Error('PROC_TCP_OWNED_ROW_INVALID'),{code:'PROC_TCP_OWNED_ROW_INVALID'}); + rows.push(row); + if(rows.length>owned.size*2+2)throw Object.assign(Error('PROC_TCP_OWNED_ROW_LIMIT'),{code:'PROC_TCP_OWNED_ROW_LIMIT'}); + }; + try{ + let count; + while((count=readSync(fd,chunk,0,chunk.length,null))>0){ + const text=chunk.toString('ascii',0,count);let start=0,next; + while((next=text.indexOf('\n',start))!==-1){line(pending+text.slice(start,next));pending='';start=next+1;} + pending+=text.slice(start);if(pending.length>4096)throw Object.assign(Error('PROC_TCP_LINE_LIMIT'),{code:'PROC_TCP_LINE_LIMIT'}); + } + if(pending)line(pending);return rows; + }finally{closeSync(fd);} +} diff --git a/src/ops/state-dispatch-error.d.mts b/src/ops/state-dispatch-error.d.mts new file mode 100644 index 0000000000..0f3047c00b --- /dev/null +++ b/src/ops/state-dispatch-error.d.mts @@ -0,0 +1,3 @@ +export function stateDispatchRejected(diagnostic:string):Error & {code:string;phase:string;diagnostic:string}; +export function isStateDispatchRejected(error:unknown):boolean; +export function localMutationRejected(code:'FILE_WRITER_LOST'|'WRITER_EPOCH_UNVERIFIED'):Error; diff --git a/src/ops/state-dispatch-error.mjs b/src/ops/state-dispatch-error.mjs new file mode 100644 index 0000000000..b885c810ea --- /dev/null +++ b/src/ops/state-dispatch-error.mjs @@ -0,0 +1,13 @@ +// Only locally created errors prove that State RPC dispatch never began. +// Remote error codes/properties cannot manufacture this process-local evidence. +const undispatched=new WeakSet(); +export function stateDispatchRejected(diagnostic){ + const error=Object.assign(Error('STATE_WRITER_IDENTITY_CHANGED:before_dispatch:'+diagnostic),{code:'STATE_WRITER_IDENTITY_CHANGED',phase:'before_dispatch',diagnostic}); + undispatched.add(error);return error; +} +export function isStateDispatchRejected(error){return error!==null&&typeof error==='object'&&undispatched.has(error);} + +export function localMutationRejected(code){ + if(!['FILE_WRITER_LOST','WRITER_EPOCH_UNVERIFIED'].includes(code))throw Error('invalid_local_mutation_rejection'); + const error=Object.assign(Error(code),{code,phase:'before_dispatch',diagnostic:code});undispatched.add(error);return error; +} diff --git a/src/ops/state-durability.d.mts b/src/ops/state-durability.d.mts index 3840a29edd..630721adfa 100644 --- a/src/ops/state-durability.d.mts +++ b/src/ops/state-durability.d.mts @@ -7,6 +7,7 @@ export function createStateDurabilityFence(options: { storePath: string; timeout mutate(scope: string, key: string, fn: () => Promise): Promise; keys(): DirtyKey[]; keysFor(keys: DirtyKey[]): DirtyKey[]; + watchKeys(keys: DirtyKey[]): { unchanged(): boolean; release(): void }; forgetKeys(keys: DirtyKey[]): void; stableKeys(options?: { minAgeMs?: number; limit?: number }): DirtyKey[]; wait(kv: Reader): Promise; @@ -15,5 +16,5 @@ export function createStateDurabilityFence(options: { storePath: string; timeout flushStable(kv: Reader, options?: { minAgeMs?: number; limit?: number }): Promise; freeze(): void; drain(): Promise; - snapshot(): { completion: 'disk'|'sdk'; sdkCompletion: unknown; frozen: boolean; activeMutations: number; touchedScopes: number; mutationFailures: number; pruning: boolean; lastVerification: unknown; measurements: unknown; readSharing: unknown }; + snapshot(): { completion: 'disk'|'sdk'; sdkCompletion: unknown; frozen: boolean; activeMutations: number; touchedScopes: number; mutationFailures: number; firstMutationFailure: {at:number;code:string;phase:string;diagnostic:string|null}|null; latestMutationFailure: {at:number;code:string;phase:string;diagnostic:string|null}|null; pruning: boolean; lastVerification: unknown; measurements: unknown; readSharing: unknown }; }; diff --git a/src/ops/state-durability.mjs b/src/ops/state-durability.mjs index c73b806d56..4c9ce83fa7 100644 --- a/src/ops/state-durability.mjs +++ b/src/ops/state-durability.mjs @@ -1,3 +1,4 @@ +import {isStateDispatchRejected} from './state-dispatch-error.mjs'; import {open,stat} from 'node:fs/promises'; import {AsyncLocalStorage} from 'node:async_hooks'; import {isAbsolute,join} from 'node:path'; @@ -27,9 +28,9 @@ export function createStateDurabilityFence({storePath,timeoutMs=15000,pollInterv if(typeof storePath!=='string'||!isAbsolute(storePath))throw Error('absolute_state_store_path_required'); if(!Number.isSafeInteger(timeoutMs)||timeoutMs<1||!Number.isSafeInteger(pollIntervalMs)||pollIntervalMs<1)throw Error('state_durability_interval_invalid'); if(!['sdk','disk'].includes(completion))throw Error('state_completion_invalid'); - let sdkCompleted=0,sdkFailed=0; + let sdkCompleted=0,sdkFailed=0,firstMutationFailure=null,latestMutationFailure=null; let frozen=false,mutationFailures=0,revision=0,pruning=false,lastVerification=null,pruneSequence=0; - const active=new Set(),touched=new Map(),pruneAttempts=new Map(),revisionObservers=new Map(); + const active=new Set(),mutationKeys=new WeakMap(),touched=new Map(),pruneAttempts=new Map(),revisionObservers=new Map(); const certificates=new Set(),certificateContext=new AsyncLocalStorage(); const measurements=createDurabilityMeasurements(); const readCoordinator=createReadCoordinator({clock}); @@ -43,6 +44,15 @@ export function createStateDurabilityFence({storePath,timeoutMs=15000,pollInterv } return()=>{for(const item of items){const scope=revisionObservers.get(item.scope),keys=scope?.get(item.key);keys?.delete(item);if(!keys?.size)scope?.delete(item.key);if(!scope?.size)revisionObservers.delete(item.scope);}}; } + function watchKeys(selected){ + if(!Array.isArray(selected)||!selected.length||selected.length>200||selected.some(item=>typeof item?.scope!=='string'||typeof item?.key!=='string'))throw Error('state_mutation_selection_invalid'); + const items=selected.map(({scope,key})=>({scope,key,revision:touched.get(scope)?.get(key)?.revision})); + const release=observeRevisions(items); + const pending=[...active].some(promise=>{const pair=mutationKeys.get(promise);return items.some(item=>pair?.scope===item.scope&&pair?.key===item.key);}); + let released=false; + // Observers survive dirty-key pruning; no permanent per-record history is retained. + return {unchanged:()=>!released&&!pending&&items.every(item=>item.observedRevision===item.revision),release(){if(!released){released=true;release();}}}; + } function keys(){return [...touched].flatMap(([scope,items])=>[...items].map(([key,stamp])=>({scope,key,...stamp})));} function keysFor(selected){return selected.map(({scope,key})=>({scope,key,...touched.get(scope)?.get(key)}));} function forgetKeys(selected){for(const {scope,key,revision} of selected){const map=touched.get(scope);if(revision===undefined||map?.get(key)?.revision!==revision)continue;map.delete(key);if(!map.size){touched.delete(scope);pruneAttempts.delete(scope);}}} @@ -52,8 +62,8 @@ export function createStateDurabilityFence({storePath,timeoutMs=15000,pollInterv } function mutate(scope,key,fn){ if(frozen)return Promise.reject(Object.assign(Error('state_mutations_frozen'),{code:'STATE_FROZEN'})); - remember(scope,key);const promise=Promise.resolve().then(fn);active.add(promise); - promise.then(()=>{remember(scope,key);active.delete(promise);},()=>{remember(scope,key);active.delete(promise);mutationFailures++;});return promise; + remember(scope,key);const promise=Promise.resolve().then(fn);mutationKeys.set(promise,{scope,key});active.add(promise); + promise.then(()=>{remember(scope,key);active.delete(promise);},error=>{remember(scope,key);active.delete(promise);if(!isStateDispatchRejected(error)){mutationFailures++;const codes=['STATE_WRITER_IDENTITY_CHANGED','FILE_WRITER_LOST','WRITER_EPOCH_UNVERIFIED','STATE_SET_ACK_UNVERIFIED','STATE_FROZEN'];const reasons=['STATE_ENGINE_SOCKET_OWNER_UNVERIFIED','STATE_ENGINE_CONNECTION_MISSING','STATE_ENGINE_CONNECTION_LIMIT','STATE_EPOCH_CHANGED','ENGINE_EPOCH_CHANGED','STATE_PROCESS_START_CHANGED','STATE_PROCESS_CHANGED_DURING_CAPTURE','STATE_BOOT_CHANGED','STATE_STORE_PATH_CHANGED','STATE_UID_CHANGED','STATE_EXPECTED_EPOCH_MISSING','STATE_EXECUTABLE_CHANGED','STATE_NAMESPACE_CHANGED'];const detail={at:clock(),code:codes.includes(error?.code)?error.code:'MUTATION_FAILED',phase:error?.phase==='after_dispatch'?'after_dispatch':'unknown',diagnostic:reasons.includes(error?.diagnostic)?error.diagnostic:null};firstMutationFailure??=detail;latestMutationFailure=detail;}});return promise; } async function drain(){ if(certificateContext.getStore()?.active)throw Object.assign(Error('state_read_coordination_reentrant'),{code:'STATE_READ_COORDINATION_REENTRANT'}); @@ -243,5 +253,5 @@ export function createStateDurabilityFence({storePath,timeoutMs=15000,pollInterv catch(error){return {success:false,deferred:true,errorCode:typeof error.code==='string'?error.code:'STATE_DURABILITY_UNAVAILABLE',diagnostics:error.diagnostics??null,retained:true};} finally{pruning=false;} } - return {completion,withBulkRead:(work,options)=>readCoordinator.run('bulk',work,options),mutate,keys,keysFor,forgetKeys,stableKeys,flushStable,freeze(){frozen=true;},drain,wait,waitForKeys,waitForKeyGroups,snapshot:()=>({completion,sdkCompletion:completion==='sdk'?{completed:sdkCompleted,failed:sdkFailed}:null,frozen,activeMutations:active.size,touchedScopes:touched.size,mutationFailures,pruning,lastVerification:structuredClone(lastVerification),measurements:completion==='disk'?measurements.snapshot():null,readSharing:readSharing.snapshot(),readCoordination:readCoordinator.snapshot()})}; + return {completion,withBulkRead:(work,options)=>readCoordinator.run('bulk',work,options),mutate,keys,keysFor,watchKeys,forgetKeys,stableKeys,flushStable,freeze(){frozen=true;},drain,wait,waitForKeys,waitForKeyGroups,snapshot:()=>({completion,sdkCompletion:completion==='sdk'?{completed:sdkCompleted,failed:sdkFailed}:null,frozen,activeMutations:active.size,touchedScopes:touched.size,mutationFailures,firstMutationFailure:structuredClone(firstMutationFailure),latestMutationFailure:structuredClone(latestMutationFailure),pruning,lastVerification:structuredClone(lastVerification),measurements:completion==='disk'?measurements.snapshot():null,readSharing:readSharing.snapshot(),readCoordination:readCoordinator.snapshot()})}; } diff --git a/src/ops/state-epoch.d.mts b/src/ops/state-epoch.d.mts new file mode 100644 index 0000000000..30cc45e43a --- /dev/null +++ b/src/ops/state-epoch.d.mts @@ -0,0 +1,5 @@ +import type {IIIClient} from 'iii-sdk'; +import type {EngineEpochEvidence} from './engine-epoch.mjs'; +export interface StateBootIdentity {version:1;pid:number;start:string;instanceId:string;bootId:string;adapter:string;storePath:string|null} +export function createStateBoundEvidence(options:{engineEvidence:EngineEpochEvidence;identity:StateBootIdentity;expectedExecutable:{path:string;sha256:string};storePath:string;procRoot?:string}):EngineEpochEvidence; +export function pinStateClient(sdk:IIIClient,prefix:string,current?:()=>boolean,diagnostic?:()=>string|null):IIIClient; diff --git a/src/ops/state-epoch.mjs b/src/ops/state-epoch.mjs new file mode 100644 index 0000000000..7813e209f5 --- /dev/null +++ b/src/ops/state-epoch.mjs @@ -0,0 +1,64 @@ +import {readOwnedTcpRows} from './proc-tcp-rows.mjs'; +import {stateDispatchRejected} from './state-dispatch-error.mjs'; +import {readFileSync,readlinkSync,readdirSync,realpathSync,statSync} from 'node:fs'; +import {join} from 'node:path'; +import {createHash} from 'node:crypto'; +const fail=()=>{throw Error('STATE_WRITER_IDENTITY_UNVERIFIED');}; +const uuid=/^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/; +const digest=x=>createHash('sha256').update(x).digest('hex'); +const signature=s=>[s.dev,s.ino,s.size,s.ctimeMs,s.mtimeMs].join(':'); +export function createStateBoundEvidence({engineEvidence,identity,expectedExecutable,storePath,procRoot='/proc'}){ + let lastFailure=null;const reject=code=>{lastFailure=code;return false;}; + const binding=structuredClone(identity); + if(binding?.version!==1||!Number.isSafeInteger(binding.pid)||binding.pid<1||!/^\d+$/.test(binding.start)||!uuid.test(binding.instanceId)||!uuid.test(binding.bootId)||binding.adapter!=='kv'||typeof binding.storePath!=='string')fail(); + const directory=realpathSync(storePath); + if(realpathSync(binding.storePath)!==directory||!expectedExecutable||!/^([a-f0-9]{64})$/.test(expectedExecutable.sha256))fail(); + const exe=realpathSync(expectedExecutable.path),exeStat=signature(statSync(exe)); + if(digest(readFileSync(exe))!==expectedExecutable.sha256)fail(); + function processStart(pid){ + const raw=readFileSync(join(procRoot,String(pid),'stat'),'utf8'),at=raw.lastIndexOf(') '); + if(at<0||Number(raw.slice(0,raw.indexOf(' ')))!==pid)fail(); + return raw.slice(at+2).trim().split(/\s+/)[19]; + } + function owned(pid){const set=new Set();for(const fd of readdirSync(join(procRoot,String(pid),'fd'))){try{const m=/^socket:\[(\d+)\]$/.exec(readlinkSync(join(procRoot,String(pid),'fd',fd)));if(m)set.add(m[1]);}catch{}}return set;} + function stateCurrent(epoch){ + if(binding.bootId!==epoch.bootId)return reject('STATE_BOOT_CHANGED'); + if(processStart(binding.pid)!==binding.start)return reject('STATE_PROCESS_START_CHANGED'); + if(realpathSync(binding.storePath)!==directory)return reject('STATE_STORE_PATH_CHANGED'); + const base=join(procRoot,String(binding.pid)); + if(realpathSync(join(base,'exe'))!==exe||signature(statSync(join(base,'exe')))!==exeStat||signature(statSync(exe))!==exeStat)return reject('STATE_EXECUTABLE_CHANGED'); + if(readlinkSync(join(base,'ns/net'))!==epoch.netNamespace||readlinkSync(join(base,'ns/pid'))!==epoch.pidNamespace)return reject('STATE_NAMESPACE_CHANGED'); + const uids=/^Uid:\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)/m.exec(readFileSync(join(base,'status'),'utf8')); + if(!uids||uids.slice(1).some(x=>Number(x)!==epoch.workerUid))return reject('STATE_UID_CHANGED'); + const sockets=owned(binding.pid),engineSockets=owned(epoch.engine.pid),port=Number(new URL(epoch.endpoint).port); + const entries=[],relevant=new Set([...sockets,...engineSockets]); + for(const file of ['tcp','tcp6'])for(const row of readOwnedTcpRows(join(base,'net',file),relevant))if(row.state==='01')entries.push(row); + const loopback=address=>/^(0100007F|0000000000000000FFFF00000100007F|00000000000000000000000001000000):/i.test(address); + const connections=entries.filter(e=>sockets.has(e.inode)&&loopback(e.local)&&loopback(e.remote)&&parseInt(e.remote.split(':')[1],16)===port); + if(connections.length<1)return reject('STATE_ENGINE_CONNECTION_MISSING'); + if(connections.length>16)return reject('STATE_ENGINE_CONNECTION_LIMIT'); + if(connections.some(connection=>entries.filter(e=>e.local===connection.remote&&e.remote===connection.local&&engineSockets.has(e.inode)).length!==1))return reject('STATE_ENGINE_SOCKET_OWNER_UNVERIFIED'); + return processStart(binding.pid)===binding.start||reject('STATE_PROCESS_CHANGED_DURING_CAPTURE'); + } + const state={pid:binding.pid,start:binding.start,instanceId:binding.instanceId,store:digest(directory),executableSha256:expectedExecutable.sha256}; + function capture(){lastFailure=null;try{const epoch=engineEvidence.capture();if(!epoch){lastFailure=engineEvidence.diagnostic?.()??'ENGINE_EPOCH_UNVERIFIED';return null;}for(let attempt=0;attempt<3;attempt++){ + if(stateCurrent(epoch)){lastFailure=null;return {...epoch,state:{...state}};} + if(!['STATE_ENGINE_SOCKET_OWNER_UNVERIFIED','STATE_ENGINE_CONNECTION_MISSING'].includes(lastFailure))return null; + if(!engineEvidence.current(epoch)){lastFailure=engineEvidence.diagnostic?.()??'ENGINE_EPOCH_CHANGED';return null;} + }return null;}catch(error){lastFailure=['ENOENT','ESRCH','EACCES','EPERM'].includes(error?.code)?'STATE_PROC_'+error.code:/^PROC_TCP_[A-Z_]+$/.test(error?.code??'')?error.code:'STATE_CAPTURE_FAILED';return null;}} + const initial=capture();if(!initial)fail(); + return {...engineEvidence,capture,diagnostic:()=>lastFailure,current(expected){const now=capture();if(!now)return false;if(!expected?.state)return reject('STATE_EXPECTED_EPOCH_MISSING');if(!engineEvidence.current(expected))return reject(engineEvidence.diagnostic?.()??'ENGINE_EPOCH_CHANGED');if(JSON.stringify(now.state)!==JSON.stringify(expected.state))return reject('STATE_EPOCH_CHANGED');return true;}}; +} +export function pinStateClient(sdk,prefix,current=()=>true,diagnostic=()=>'STATE_EPOCH_NOT_CURRENT'){ + if(!/^state::[a-f0-9-]{36}::$/.test(prefix))fail(); + return new Proxy(sdk,{get(target,key){ + if(key==='trigger')return async request=>{ + const stateCall=request.function_id.startsWith('state::'); + if(stateCall&&!current())throw stateDispatchRejected(diagnostic()??'STATE_EPOCH_NOT_CURRENT'); + const result=await target.trigger({...request,function_id:stateCall?prefix+request.function_id.slice(7):request.function_id}); + if(stateCall&&!current()){const reason=diagnostic()??'STATE_EPOCH_NOT_CURRENT';throw Object.assign(Error('STATE_WRITER_IDENTITY_CHANGED:after_dispatch:'+reason),{code:'STATE_WRITER_IDENTITY_CHANGED',phase:'after_dispatch',diagnostic:reason});} + return result; + }; + const value=Reflect.get(target,key);return typeof value==='function'?value.bind(target):value; + }}); +} diff --git a/src/ops/state-seal-shutdown.d.mts b/src/ops/state-seal-shutdown.d.mts new file mode 100644 index 0000000000..4e4bd9ccad --- /dev/null +++ b/src/ops/state-seal-shutdown.d.mts @@ -0,0 +1,2 @@ +export interface StateSealReceipt {version:1;instanceId:string;storePath:string;sealed:true;synced:true} +export function stateSealDiskProof(receipt:unknown,identity:{instanceId:string;storePath:string|null},mutationFailures:number):{completion:'disk';completed:true;synced:true;mutationFailures:0;keys:number;scopes:number;waitMs:number;stateSeal:StateSealReceipt}; diff --git a/src/ops/state-seal-shutdown.mjs b/src/ops/state-seal-shutdown.mjs new file mode 100644 index 0000000000..f6af9465c0 --- /dev/null +++ b/src/ops/state-seal-shutdown.mjs @@ -0,0 +1,5 @@ +import {realpathSync} from 'node:fs'; +export function stateSealDiskProof(receipt,identity,mutationFailures){ + if(receipt?.version!==1||receipt.sealed!==true||receipt.synced!==true||receipt.instanceId!==identity?.instanceId||typeof receipt.storePath!=='string'||typeof identity.storePath!=='string'||realpathSync(receipt.storePath)!==realpathSync(identity.storePath)||mutationFailures!==0)throw Error('STATE_SEAL_UNVERIFIED'); + return {completion:'disk',completed:true,synced:true,mutationFailures:0,keys:0,scopes:0,waitMs:0,stateSeal:receipt}; +} diff --git a/src/ops/vector-checkpoint-proof.d.mts b/src/ops/vector-checkpoint-proof.d.mts new file mode 100644 index 0000000000..874f66fe26 --- /dev/null +++ b/src/ops/vector-checkpoint-proof.d.mts @@ -0,0 +1,8 @@ +type Evidence = {storePath:string; pendingRows:readonly unknown[]; admitted:()=>boolean}; +export function verifyVectorCheckpointFiles(options: Evidence & { + waitForCaptureMeta?:boolean; meta:unknown; buckets:readonly {bucket:number; ids:readonly string[]}[]; +}):Promise<()=>Promise>; +export function verifyRemainingVectorPending(options: Evidence):Promise; +export function verifyVectorRetentionReceipts(options: { + storePath:string; receipts:Readonly>; admitted:()=>boolean; +}):Promise; diff --git a/src/ops/vector-checkpoint-proof.mjs b/src/ops/vector-checkpoint-proof.mjs new file mode 100644 index 0000000000..a60ef99578 --- /dev/null +++ b/src/ops/vector-checkpoint-proof.mjs @@ -0,0 +1,197 @@ +import {createHash} from 'node:crypto'; +import {constants} from 'node:fs'; +import {open, lstat, realpath} from 'node:fs/promises'; +import {isAbsolute, join} from 'node:path'; +import {isDeepStrictEqual} from 'node:util'; + +const META = 'mem:index:bm25', PENDING = 'mem:index:vec-pending'; +const MiB = 1024 * 1024; +const READ_FLAGS = constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK; +function requireProof(ok) { + if (!ok) throw Object.assign(new Error('VECTOR_CHECKPOINT_UNVERIFIED'), {code:'VECTOR_CHECKPOINT_UNVERIFIED'}); +} +const signature = s => [s.dev,s.ino,s.size,s.mtimeNs,s.ctimeNs].join(':'); + +// Native iii 0.22.1 stores KeyStorage(String) as a little-endian rkyv 0.8 archive. +// Accept only its canonical encoding. A valid JSON prefix alone cannot certify an archive. +function namespaceRows(bytes) { + requireProof(bytes.length >= 8 && bytes.length % 4 === 0); + const root = bytes.length - 8; + let length; + if (root === 0) { + length = bytes.indexOf(0xff); + if (length < 0) length = 8; + requireProof(bytes.subarray(length).every(b=>b===0xff)); + } else { + const size = bytes.readUInt32LE(root); + length = (size & 63) + Math.floor(size / 256) * 64; + requireProof((size & 192) === 128 && bytes.readInt32LE(root+4) === -root && + length > 8 && Math.ceil(length/4)*4 === root && bytes.subarray(length,root).every(b=>b===0)); + } + const json = new TextDecoder('utf-8',{fatal:true,ignoreBOM:true}).decode(bytes.subarray(0,length)); + const rows = JSON.parse(json); + // This narrow repair refuses duplicate keys, rounded numbers and noncanonical serializations. + requireProof(rows !== null && typeof rows === 'object' && !Array.isArray(rows) && JSON.stringify(rows) === json); + return rows; +} + +function pendingMap(rows) { + requireProof(Array.isArray(rows) && rows.length <= 16); + const byId = new Map(), sequences = new Set(); + for (const row of rows) { + requireProof(row && Object.keys(row).every(k=>['id','s','e','q','k'].includes(k)) && + typeof row.id === 'string' && row.id.length > 0 && row.id.length <= 1024 && row.id !== '~clear' && + typeof row.s === 'string' && row.s.length > 0 && row.s.length <= 1024 && + Number.isSafeInteger(row.q) && row.q > 0 && !sequences.has(row.q) && !byId.has(row.id) && + (row.k === undefined || row.k === 'memory' || row.k === 'observation') && + typeof row.e === 'string' && row.e.length > 0 && row.e.length <= 256*1024); + const embedding = Buffer.from(row.e,'base64'); + requireProof(embedding.length > 0 && embedding.length % 4 === 0 && embedding.toString('base64') === row.e); + for (let offset=0; offset0 && performance.now()=8 && size<=file.limit && total<=64*MiB); + const data=Buffer.allocUnsafe(size); + for(let offset=0;offset0 && performance.now() {await current();await verifyNames(storePath,true);await verifyNames(storePath);await current();}; +} + +export async function verifyVectorCheckpointFiles({storePath,pendingRows,meta,buckets,admitted,waitForCaptureMeta=false}) { + const byId=pendingMap(pendingRows), selected=new Map(); + requireProof(byId.size>0 && meta?.v===3 && Number.isSafeInteger(meta.bucketCount) && meta.bucketCount>0 && + Number.isSafeInteger(meta.count) && meta.count>=byId.size && Array.isArray(buckets) && buckets.length>0 && buckets.length<=byId.size); + const expectedMeta=structuredClone(meta); + const laggingTimestamp = Symbol('laggingCheckpointTimestamp'); + const deadline = performance.now() + 1500; + // Check metadata first. Retrying a small, older timestamp must never reread buckets. + const files=[{scope:META,limit:4*MiB,verify(rows,size){ + const current = rows['vectors:meta']; + if (isDeepStrictEqual(current,expectedMeta)) return; + if (waitForCaptureMeta && size <= 64*1024 && current && + typeof current.savedAt === 'string' && typeof expectedMeta.savedAt === 'string' && + Date.parse(current.savedAt) < Date.parse(expectedMeta.savedAt) && + isDeepStrictEqual({...current,savedAt:expectedMeta.savedAt},expectedMeta)) throw laggingTimestamp; + requireProof(false); + }}, pendingFile(byId)]; + const seenBuckets=new Set(); + for (const {bucket,ids} of buckets) { + requireProof(Number.isSafeInteger(bucket)&&bucket>=0&&bucket0&&ids.length<=byId.size); + seenBuckets.add(bucket); + for(const id of ids){requireProof(byId.has(id)&&!selected.has(id));selected.set(id,bucket);} + files.push({scope:META+':vec:'+String(bucket).padStart(4,'0'), limit:16*MiB, verify(rows) { + for (const [key,value] of Object.entries(rows)) { + if (!byId.has(key)&&!byId.has(value?.id))continue; + const row=byId.get(key); + requireProof(row && selected.get(key)===bucket && isDeepStrictEqual(value,{id:key,s:row.s,e:row.e})); + } + for(const [id,location] of selected)if(location===bucket)requireProof(Object.hasOwn(rows,id)); + }}); + } + requireProof(selected.size===byId.size); + for (;;) { + try { return await prove(storePath,admitted,files); } + catch (error) { + if (error !== laggingTimestamp) throw error; + requireProof(admitted() === true && performance.now() < deadline); + await new Promise(resolve=>setTimeout(resolve,Math.min(125,deadline-performance.now()))); + } + } +} + +export async function verifyRemainingVectorPending({storePath,pendingRows,admitted}) { + const verify=await prove(storePath,admitted,[pendingFile(pendingMap(pendingRows))]); + await verify(); +} + +export async function verifyVectorRetentionReceipts({storePath,receipts,admitted}) { + requireProof(receipts && typeof receipts === 'object' && + (Object.getPrototypeOf(receipts) === Object.prototype || Object.getPrototypeOf(receipts) === null)); + const entries = Object.entries(receipts); + requireProof(entries.length <= 16 && entries.length === Reflect.ownKeys(receipts).length); + const ids = new Set(), sequences = new Set(); + const expected = {}; + for (const [key, row] of entries) { + requireProof(key.length === 64 && /^[a-f0-9]+$/.test(key) && + row && typeof row === 'object' && Reflect.ownKeys(row).length === 5 && Object.keys(row).length === 5 && + ['v','id','q','bucket','digest'].every(field => Object.hasOwn(row,field)) && + row.v === 1 && row.digest === key && typeof row.id === 'string' && + row.id.length > 0 && row.id.length <= 512 && row.id !== '~clear' && !ids.has(row.id) && + Number.isSafeInteger(row.q) && row.q > 0 && !sequences.has(row.q) && + Number.isSafeInteger(row.bucket) && row.bucket >= 0); + ids.add(row.id); + sequences.add(row.q); + expected[key] = {...row}; + } + const recheck = await prove(storePath,admitted,[{ + scope:'mem:index:vec-retained:v1', limit:128*1024, allowMissing:entries.length === 0, + verify(rows) { requireProof(isDeepStrictEqual(rows,expected)); } + }]); + await recheck(); +} diff --git a/src/prompts/reflect.ts b/src/prompts/reflect.ts index 57675938d7..244690caa1 100644 --- a/src/prompts/reflect.ts +++ b/src/prompts/reflect.ts @@ -1,3 +1,5 @@ +export const REFLECT_MAX_PROMPT_BYTES = 64 * 1024; + export const REFLECT_SYSTEM = `You are a higher-order reasoning engine. Given a cluster of related concepts, facts, lessons, and action outcomes, synthesize cross-cutting insights that span multiple individual memories. Output format (XML): @@ -14,7 +16,7 @@ Rules: - Content should be the actual observation (1-3 sentences) - Prefer actionable insights over abstract summaries - Skip insights that merely restate a single source item -- Cite at least two distinct supplied source records in the sources attribute +- Cite between two and twenty distinct supplied source records in the sources attribute - Use only the supplied F, L, and C labels; never invent labels or cite uncited context - Cite only records used to form this particular insight, not the entire cluster - Return when the evidence does not support an insight @@ -56,5 +58,9 @@ export function buildReflectPrompt(cluster: { ); } - return `Synthesize higher-order insights from this cluster of related memories:\n\n${sections.join("\n")}`; + const prompt = `Synthesize higher-order insights from this cluster of related memories:\n\n${sections.join("\n")}`; + if (Buffer.byteLength(REFLECT_SYSTEM, "utf8") + Buffer.byteLength(prompt, "utf8") > REFLECT_MAX_PROMPT_BYTES) { + throw new Error("Reflection prompt exceeds its byte budget"); + } + return prompt; } diff --git a/src/providers/circuit-breaker.ts b/src/providers/circuit-breaker.ts index bcb27e416f..0bce44dab7 100644 --- a/src/providers/circuit-breaker.ts +++ b/src/providers/circuit-breaker.ts @@ -45,12 +45,11 @@ export class CircuitBreaker { } recordSuccess(): void { - if (this.state === "half-open") { - this.state = "closed"; - this.failures = 0; - this.lastFailureAt = null; - this.openedAt = null; - } + if (this.state === "open") return; + this.state = "closed"; + this.failures = 0; + this.lastFailureAt = null; + this.openedAt = null; } recordFailure(): void { diff --git a/src/providers/embedding/clip.ts b/src/providers/embedding/clip.ts index 4ee102bce2..984c4995a1 100644 --- a/src/providers/embedding/clip.ts +++ b/src/providers/embedding/clip.ts @@ -7,13 +7,14 @@ type ClipPipeline = ( input: string[] | RawImage | RawImage[], options?: { pooling?: string; normalize?: boolean }, ) => Promise<{ tolist: () => number[][]; data: Float32Array }>; +type ClipTextEncoder = (texts: string[]) => Promise; const DEFAULT_MODEL = "Xenova/clip-vit-base-patch32"; export class ClipEmbeddingProvider implements EmbeddingProvider { readonly name = "clip"; readonly dimensions = 512; - private textExtractor: ClipPipeline | null = null; + private textExtractor: ClipTextEncoder | null = null; private imageExtractor: ClipPipeline | null = null; private readonly modelId: string; @@ -27,9 +28,8 @@ export class ClipEmbeddingProvider implements EmbeddingProvider { } async embedBatch(texts: string[]): Promise { - const extractor = await this.getTextExtractor(); - const output = await extractor(texts, { pooling: "mean", normalize: true }); - return output.tolist().map((v) => new Float32Array(v)); + const encode = await this.getTextExtractor(); + return encode(texts); } async embedImage(src: string): Promise { @@ -41,10 +41,18 @@ export class ClipEmbeddingProvider implements EmbeddingProvider { return normalize(vec); } - private async getTextExtractor(): Promise { + private async getTextExtractor(): Promise { if (this.textExtractor) return this.textExtractor; const t = await loadTransformers(); - this.textExtractor = (await t.pipeline("feature-extraction", this.modelId, { dtype: "q8" })) as ClipPipeline; + const tokenizer = await t.AutoTokenizer.from_pretrained(this.modelId); + const model = await t.CLIPTextModelWithProjection.from_pretrained(this.modelId, { + dtype: "q8", + }); + this.textExtractor = async (texts: string[]) => { + const inputs = tokenizer(texts, { padding: true, truncation: true }); + const { text_embeds } = await model(inputs); + return text_embeds.tolist().map((v: number[]) => normalize(new Float32Array(v))); + }; return this.textExtractor; } diff --git a/src/providers/index.ts b/src/providers/index.ts index 4245e03ad1..a2c71ccbd0 100644 --- a/src/providers/index.ts +++ b/src/providers/index.ts @@ -11,12 +11,12 @@ import { OpenAIProvider } from "./openai.js"; import { OpenRouterProvider } from "./openrouter.js"; import { ResilientProvider } from "./resilient.js"; import { FallbackChainProvider } from "./fallback-chain.js"; -import { getEnvVar } from "../config.js"; +import { getEnvVar, getProviderKey } from "../config.js"; export { createEmbeddingProvider, createImageEmbeddingProvider } from "./embedding/index.js"; function requireEnvVar(key: string): string { - const value = getEnvVar(key); + const value = getProviderKey(key); if (!value) { throw new Error( `Missing required environment variable: ${key}. Set it in ~/.agentmemory/.env or as an environment variable.`, @@ -95,7 +95,7 @@ function createBaseProvider(config: ProviderConfig): MemoryProvider { ); case "gemini": { const geminiKey = - getEnvVar("GEMINI_API_KEY") || getEnvVar("GOOGLE_API_KEY"); + getProviderKey("GEMINI_API_KEY") || getProviderKey("GOOGLE_API_KEY"); if (!geminiKey) { throw new Error( "GEMINI_API_KEY (or GOOGLE_API_KEY) is required for the gemini provider", @@ -116,7 +116,7 @@ function createBaseProvider(config: ProviderConfig): MemoryProvider { "https://openrouter.ai/api/v1/chat/completions", ); case "openai": { - const openaiKey = getEnvVar("OPENAI_API_KEY"); + const openaiKey = getProviderKey("OPENAI_API_KEY"); if (!openaiKey) { throw new Error( "OPENAI_API_KEY is required for the openai provider", diff --git a/src/providers/resilient.ts b/src/providers/resilient.ts index 95ece40c96..f82e578d72 100644 --- a/src/providers/resilient.ts +++ b/src/providers/resilient.ts @@ -1,6 +1,14 @@ import type { MemoryProvider, CircuitBreakerState } from "../types.js"; import { CircuitBreaker } from "./circuit-breaker.js"; +const FILTER_CODE = /content_filter|ResponsibleAIPolicyViolation/; +const REJECTED_STATUS = /^[^{]*\(400\)/; + +export function isPayloadRejection(err: unknown): boolean { + const message = err instanceof Error ? err.message : String(err); + return REJECTED_STATUS.test(message) && FILTER_CODE.test(message); +} + export class ResilientProvider implements MemoryProvider { private breaker = new CircuitBreaker(); name: string; @@ -18,7 +26,9 @@ export class ResilientProvider implements MemoryProvider { this.breaker.recordSuccess(); return result; } catch (err) { - this.breaker.recordFailure(); + if (!isPayloadRejection(err)) { + this.breaker.recordFailure(); + } throw err; } } diff --git a/src/runtime/registration.ts b/src/runtime/registration.ts new file mode 100644 index 0000000000..9b6cbbca1e --- /dev/null +++ b/src/runtime/registration.ts @@ -0,0 +1,19 @@ +type RegisterFunction = (id: string, handler: unknown, ...rest: unknown[]) => unknown; + +export function withRegistrationDefaults(sdk: T): T { + return new Proxy(sdk, { + get(target, prop) { + const value = Reflect.get(target, prop, target); + if (typeof value !== "function") return value; + if (prop !== "registerFunction" || process.env.AGENTMEMORY_LOCAL_DISPATCH !== "false") { + return value.bind(target); + } + const register = value as RegisterFunction; + return (id: string, handler: unknown, ...rest: unknown[]) => { + const [options, ...extra] = rest; + const merged = { ...((options ?? {}) as object), localDispatch: false }; + return register.call(target, id, handler, merged, ...extra); + }; + }, + }); +} diff --git a/src/secret-store.ts b/src/secret-store.ts index 12dc2d9df2..915bb0b0c7 100644 --- a/src/secret-store.ts +++ b/src/secret-store.ts @@ -41,7 +41,7 @@ function unquote(value: string): string { return hash === -1 ? value : value.slice(0, hash).trim(); } -export function readEnvFileSecret(): string { +export function readEnvFileValue(name: string): string { let content: unknown; try { content = readFileSync(process.env["AGENTMEMORY_ENV_FILE"] || join(homedir(), ".agentmemory", ".env"), "utf-8"); @@ -56,12 +56,16 @@ export function readEnvFileSecret(): string { const eq = trimmed.indexOf("="); if (eq === -1) continue; const key = trimmed.slice(0, eq).replace(/^export\s+/, "").trim(); - if (key !== SECRET_KEY) continue; - found = usable(unquote(trimmed.slice(eq + 1).trim())); + if (key !== name) continue; + found = unquote(trimmed.slice(eq + 1).trim()); } return found; } +export function readEnvFileSecret(): string { + return usable(readEnvFileValue(SECRET_KEY)); +} + function readSecretFile(path: string): string { try { return usable(readFileSync(path, "utf-8")); diff --git a/src/state/frame-guard.ts b/src/state/frame-guard.ts index e8b5875f06..9b85fadf73 100644 --- a/src/state/frame-guard.ts +++ b/src/state/frame-guard.ts @@ -1,8 +1,6 @@ -// Retain the existing conservative application payload budget. It was chosen -// under the older 16 MiB framing assumption; it is not the current SDK's -// 64 MiB message limit. Refusals must not tear down the shared SDK connection. -const FRAME_LIMIT_BYTES = 16 * 1024 * 1024; -export const SAFE_PAYLOAD_BYTES = 15 * 1024 * 1024; +// SDK 0.25.1 permits 64 MiB frames; leave room for invocation envelopes. +const FRAME_LIMIT_BYTES = 64 * 1024 * 1024; +export const SAFE_PAYLOAD_BYTES = 60 * 1024 * 1024; export type OversizedPayload = { success: false; diff --git a/src/state/graph-generation.ts b/src/state/graph-generation.ts index cd0e172d86..84acf6018d 100644 --- a/src/state/graph-generation.ts +++ b/src/state/graph-generation.ts @@ -86,13 +86,27 @@ export async function pinGraphState(kv: StateKV): Promise { } function graphView(kv: StateKV, id: string | null): StateKV { - const methods = new Set(["get", "set", "update", "delete", "list"]); + const methods = new Set(["get", "set", "update", "delete", "list", "listKeys", "getMany", "iterate", "listPage"]); const view = new Proxy(kv, { get(target, property, receiver) { const value = Reflect.get(target, property, receiver); - if (typeof property !== "string" || !methods.has(property)) return value; + if (typeof property !== "string" || typeof value !== "function" || !methods.has(property)) return value; + const ctx = { kv: target, generation: id }; + if (property === "iterate") { + return (scope: string, ...args: unknown[]) => { + const inner = pinned.run(ctx, () => + (value as (s: string, ...a: unknown[]) => AsyncGenerator).call(target, physicalScope(scope, id), ...args), + ); + return { + [Symbol.asyncIterator]() { return this; }, + next: (...a: unknown[]) => pinned.run(ctx, () => inner.next(...(a as []))), + return: (v?: unknown) => pinned.run(ctx, () => inner.return!(v)), + throw: (e?: unknown) => pinned.run(ctx, () => inner.throw!(e)), + }; + }; + } return (scope: string, ...args: unknown[]) => pinned.run( - { kv: target, generation: id }, + ctx, () => value.call(target, physicalScope(scope, id), ...args), ); }, @@ -105,7 +119,7 @@ function graphView(kv: StateKV, id: string | null): StateKV { export async function resetGraphGeneration(kv: StateKV, snapshot: unknown): Promise { const id = randomUUID(); await kv.set(physicalScope(KV.graphSnapshot, id), "current", snapshot); - await initializeGraphIndex(graphView(kv, id), { status: "ready" }); + await initializeGraphIndex(graphView(kv, id), { status: "ready", schema: 3 }); await kv.set(CONTROL_SCOPE, CONTROL_KEY, id); } @@ -114,7 +128,7 @@ export async function replaceGraphGeneration(kv: StateKV, initialize: (fresh: const { target, id: previous } = views.get(pinnedState)!; const id = randomUUID(); const fresh = graphView(target, id); - await initializeGraphIndex(fresh, { status: "ready" }); + await initializeGraphIndex(fresh, { status: "ready", schema: 3 }); const result = await initialize(fresh); // A delayed import cannot replace a reset that happened after it was pinned. if (await generation(target) === previous) { diff --git a/src/state/graph-index-migration.ts b/src/state/graph-index-migration.ts index 9e77bfec98..2bd150a6f9 100644 --- a/src/state/graph-index-migration.ts +++ b/src/state/graph-index-migration.ts @@ -206,11 +206,13 @@ export class GraphIndexMigrationController { let job = readJob(await state.get(KV.graphIndexMigration, GRAPH_INDEX_MIGRATION_KEY)); if (job?.generation !== generation) job = null; const unfinishedCheckpoint = job?.buildId === control?.buildId && (job?.state === "paused" || Boolean(job?.error)); - if (control?.status === "ready" && !control.pending && !control.batchPending && !unfinishedCheckpoint) return; + const upgradeLayout = control?.schema === 2 && control.status === "ready" && + !control.pending && !control.batchPending && !unfinishedCheckpoint && !options.resume && options.allowLegacyScan; + if (control?.status === "ready" && !control.pending && !control.batchPending && !unfinishedCheckpoint && !upgradeLayout) return; if (options.resume && !job) throw new GraphIndexMigrationError("no-job", "No graph index migration exists. Run graph-index migrate --allow-legacy-scan first."); if (options.resume && job?.sealed && !control) throw new GraphIndexMigrationError("invalid-state", "The sealed build control is missing. Start a new build with graph-index migrate --allow-legacy-scan."); if (options.resume && control?.status === "paused") throw new GraphIndexMigrationError("invalid-state", "The index requires a new build. Run graph-index migrate --allow-legacy-scan after fixing the reported data."); - if (!job || replaceCorruptControl || (!control && job.sealed) || control?.status === "paused" || (control && control.buildId !== job.buildId) || + if (!job || replaceCorruptControl || upgradeLayout || (!control && job.sealed) || control?.status === "paused" || (control && control.buildId !== job.buildId) || (!options.resume && options.allowLegacyScan && control?.batchPending)) { if (options.resume) throw new GraphIndexMigrationError("invalid-state", "The graph index build changed. Start an explicit migration again."); if (!options.allowLegacyScan) throw new GraphIndexMigrationError("consent-required", "Initial discovery requires allowLegacyScan: true; complete legacy graph scopes are read during maintenance."); @@ -218,14 +220,14 @@ export class GraphIndexMigrationController { job = { schema: 1, jobId: randomUUID(), buildId: randomUUID(), generation, state: "discovering", sealed: false, phase: "nodes", page: 0, offset: 0, processed: 0, totals: { nodes: 0, edges: 0, history: 0 }, pages: { nodes: 0, edges: 0, history: 0 }, startedAt: now, updatedAt: now }; await state.set(KV.graphIndexMigration, GRAPH_INDEX_MIGRATION_KEY, job); - await initializeGraphIndex(state, { buildId: job.buildId, status: "building" }); + await initializeGraphIndex(state, { buildId: job.buildId, status: "building", schema: 3 }); } else { if (!job.sealed && !options.allowLegacyScan) throw new GraphIndexMigrationError("consent-required", "Discovery did not finish. Resume with --allow-legacy-scan to authorize another complete legacy scan."); if (!control) { // A previous control write may have been lost after other index writes succeeded. job = { ...job, buildId: randomUUID() }; await this.save(state, job); - await initializeGraphIndex(state, { buildId: job.buildId, status: "building" }); + await initializeGraphIndex(state, { buildId: job.buildId, status: "building", schema: 3 }); } job = { ...job, state: job.sealed ? "building" : "discovering", error: undefined, updatedAt: new Date().toISOString() }; await state.set(KV.graphIndexMigration, GRAPH_INDEX_MIGRATION_KEY, job); @@ -283,7 +285,7 @@ export class GraphIndexMigrationController { job.phase = phase; await this.assertActive(state, job); await this.save(state, job); - const rows = await state.list<{ id?: unknown }>(SCOPES[phase]); + const keys = await state.listKeys(SCOPES[phase]); await this.assertActive(state, job); let ids: string[] = []; const flush = async () => { @@ -292,12 +294,12 @@ export class GraphIndexMigrationController { await state.set(KV.graphIndexManifest, graphIndexManifestKey(job.jobId, phase, page.page), page); job.pages[phase]++; job.totals[phase] += ids.length; ids = []; }; - for (const row of rows) { - if (!row || !validId(row.id)) throw new GraphIndexMigrationError("invalid-state", "A legacy graph record has an invalid or oversized ID; repair it before migrating."); - if (ids.length >= GRAPH_INDEX_MANIFEST_MAX_IDS || Buffer.byteLength(JSON.stringify(ids)) + Buffer.byteLength(JSON.stringify(row.id)) + 256 > GRAPH_INDEX_MANIFEST_MAX_BYTES) { + for (const id of keys) { + if (!validId(id)) throw new GraphIndexMigrationError("invalid-state", "A legacy graph record has an invalid or oversized ID; repair it before migrating."); + if (ids.length >= GRAPH_INDEX_MANIFEST_MAX_IDS || Buffer.byteLength(JSON.stringify(ids)) + Buffer.byteLength(JSON.stringify(id)) + 256 > GRAPH_INDEX_MANIFEST_MAX_BYTES) { await flush(); await this.assertActive(state, job); await yieldToEventLoop(); } - ids.push(row.id); + ids.push(id); } await flush(); } diff --git a/src/state/graph-retrieval-index.ts b/src/state/graph-retrieval-index.ts index feb71d66b7..29f8e9b164 100644 --- a/src/state/graph-retrieval-index.ts +++ b/src/state/graph-retrieval-index.ts @@ -43,9 +43,15 @@ const recordKey = (buildId: string, kind: GraphIndexRecordKind, id: string) => ` const rootKey = (buildId: string, posting: string) => `${buildId}:root:${posting}`; const entryKey = (buildId: string, posting: string, id: string) => `${buildId}:entry:${posting}:${id}`; const countKey = (kind: GraphIndexRecordKind) => kind === "node" ? "nodes" : kind === "edge" ? "edges" : "historyEdges"; +function indexDataScope(control: GraphRetrievalIndexControl, key: string): string { + if (control.schema === 2) return KV.graphRetrievalData; + // Each key ends in a SHA-256 digest. Two hex digits make 256 independent + // files per build, including for a single very large linked posting. + return `${KV.graphRetrievalData}:build:${control.buildId}:${key.slice(-64, -62)}`; +} function validateControl(value: GraphRetrievalIndexControl): void { - if (bytes(value) > MAX_CONTROL_BYTES || value.schema !== 2 || typeof value.buildId !== "string" || !/^[a-zA-Z0-9_-]{1,128}$/.test(value.buildId) || + if (bytes(value) > MAX_CONTROL_BYTES || (value.schema !== 2 && value.schema !== 3) || typeof value.buildId !== "string" || !/^[a-zA-Z0-9_-]{1,128}$/.test(value.buildId) || !["building", "ready", "paused"].includes(value.status) || ![value.revision, value.nodes, value.edges, value.historyEdges, value.incompleteRecords, value.otherTypes].every(finiteCount) || !value.nodesByType || !value.edgesByType || @@ -63,10 +69,10 @@ export async function getGraphIndexStatus(state: StateKV): Promise { +export async function initializeGraphIndex(state: StateKV, options: { status: "building" | "ready"; buildId?: string; schema?: 2 | 3 }): Promise { const kv = await pinGraphState(state); return lock(kv, async () => { - const value: GraphRetrievalIndexControl = { schema: 2, buildId: options.buildId ?? randomUUID(), status: options.status, + const value: GraphRetrievalIndexControl = { schema: options.schema ?? 2, buildId: options.buildId ?? randomUUID(), status: options.status, revision: 0, nodes: 0, edges: 0, historyEdges: 0, incompleteRecords: 0, nodesByType: {}, edgesByType: {}, otherTypes: 0, updatedAt: new Date().toISOString() }; validateControl(value); const previous = await kv.get(KV.graphRetrievalControl, CONTROL_KEY); @@ -172,7 +178,7 @@ function validateRecord(value: GraphRetrievalIndexRecord): void { async function planRecord(kv: StateKV, control: GraphRetrievalIndexControl, kind: GraphIndexRecordKind, id: string, input: unknown): Promise { const writes = new Map(); - const read = async (key: string): Promise => writes.has(key) ? writes.get(key) as T | null : kv.get(KV.graphRetrievalData, key); + const read = async (key: string): Promise => writes.has(key) ? writes.get(key) as T | null : kv.get(indexDataScope(control, key), key); const put = (key: string, value: unknown) => { if (bytes(value) > GRAPH_INDEX_MAX_RECORD_BYTES) throw new GraphIndexError("Graph retrieval index write exceeds record budget"); writes.set(key, value); }; const key = recordKey(control.buildId, kind, id); const before = await read(key); @@ -243,16 +249,18 @@ async function planRecord(kv: StateKV, control: GraphRetrievalIndexControl, kind async function applyJournal(kv: StateKV, control: GraphRetrievalIndexControl, journal: IndexJournal): Promise { if (journal.schema !== 1 || journal.buildId !== control.buildId || journal.revision !== control.revision || !Array.isArray(journal.writes) || journal.writes.length > 2048 || bytes(journal) > MAX_JOURNAL_BYTES || journal.next.buildId !== control.buildId || - journal.next.revision !== control.revision + 1 || journal.next.pending) throw new GraphIndexError("Invalid graph retrieval recovery journal"); + journal.next.schema !== control.schema || journal.next.revision !== control.revision + 1 || journal.next.pending) throw new GraphIndexError("Invalid graph retrieval recovery journal"); validateControl(journal.next); for (const write of journal.writes) { - if (!write || typeof write.key !== "string" || !write.key.startsWith(`${control.buildId}:`) || write.key.length > 512 || bytes(write.value) > GRAPH_INDEX_MAX_RECORD_BYTES) { + if (!write || typeof write.key !== "string" || !write.key.startsWith(`${control.buildId}:`) || write.key.length > 512 || + (control.schema === 3 && !/:[a-f0-9]{64}$/.test(write.key)) || bytes(write.value) > GRAPH_INDEX_MAX_RECORD_BYTES) { throw new GraphIndexError("Invalid graph retrieval recovery write"); } } for (const write of journal.writes) { - if (write.value === null) await kv.delete(KV.graphRetrievalData, write.key); - else await kv.set(KV.graphRetrievalData, write.key, write.value); + const scope = indexDataScope(control, write.key); + if (write.value === null) await kv.delete(scope, write.key); + else await kv.set(scope, write.key, write.value); } await kv.set(KV.graphRetrievalControl, CONTROL_KEY, journal.next); } @@ -325,6 +333,10 @@ export async function finishGraphIndexBuild(state: StateKV, buildId: string): Pr return lock(kv, async () => { const current = await getGraphIndexStatus(kv); if (!current || current.buildId !== buildId || current.status === "paused" || current.pending || current.batchPending) throw new GraphIndexError("Graph index build is incomplete or superseded"); + if (current.schema === 3) { + const reader = new GraphIndexReader(kv, await getGraphGeneration(kv), current, { maxReads: 16 }); + for (const kind of ["nodes", "edges", "historyEdges"] as const) await reader.posting(kind, "", { limit: 1 }); + } const next = { ...current, status: "ready" as const, revision: current.revision + 1, updatedAt: new Date().toISOString() }; await kv.set(KV.graphRetrievalControl, CONTROL_KEY, next); return next; @@ -429,7 +441,7 @@ export class GraphIndexReader { else this.active++; try { if (this.options.signal?.aborted || this.exhausted) { this.exhausted = true; return null; } - const value = await this.state.get(scope, key); + const value = await this.state.get(scope === KV.graphRetrievalData ? indexDataScope(this.status, key) : scope, key); const size = bytes(value); this.readBytes += size; if (this.readBytes > this.maxBytes || this.options.signal?.aborted) { this.exhausted = true; return null; } diff --git a/src/state/hybrid-search.ts b/src/state/hybrid-search.ts index caeb8f03b8..2c7eb4e2ad 100644 --- a/src/state/hybrid-search.ts +++ b/src/state/hybrid-search.ts @@ -23,6 +23,7 @@ import { logger } from "../logger.js"; const RRF_K = 60; const GRAPH_BUDGET_MS = 500; +const MIN_FUSION_POOL = 40; export interface HybridSearchRequest { embedding?: { query: string; value: Promise }; @@ -103,7 +104,8 @@ export class HybridSearch { request?: HybridSearchRequest, ): Promise { const retrievalLimit = this.rerankEnabled ? Math.max(limit, 20) : limit; - const bm25Results = this.bm25.search(query, retrievalLimit * 2, targetLayer); + const poolLimit = Math.max(retrievalLimit * 2, MIN_FUSION_POOL); + const bm25Results = this.bm25.search(query, poolLimit, targetLayer); let vectorResults: Array<{ obsId: string; @@ -128,7 +130,7 @@ export class HybridSearch { } else { queryEmbedding = await this.embeddingProvider.embed(query); } - vectorResults = this.vector.search(queryEmbedding, retrievalLimit * 2, targetLayer, (id) => this.bm25.layerOf(id)); + vectorResults = this.vector.search(queryEmbedding, poolLimit, targetLayer, (id) => this.bm25.layerOf(id)); } catch { // fall through to BM25-only } @@ -255,7 +257,7 @@ export class HybridSearch { if (request) request.candidateCount = combined.length; const retrievalDepth = Math.max(limit, 20); - const diversified = this.diversifyBySession(combined, retrievalDepth); + const diversified = this.diversifyBySession(combined, combined.length); const enriched = await this.enrichResults(diversified, retrievalDepth); return applyReranking ? this.rerankResults(query, enriched, limit) : enriched.slice(0, limit); @@ -327,6 +329,11 @@ export class HybridSearch { const sessionCounts = new Map(); for (const r of results) { + if (r.obsId.startsWith("mem_")) { + selected.push(r); + if (selected.length >= limit) break; + continue; + } const count = sessionCounts.get(r.sessionId) || 0; if (count >= maxPerSession) continue; selected.push(r); @@ -335,9 +342,10 @@ export class HybridSearch { } if (selected.length < limit) { + const picked = new Set(selected.map((s) => s.obsId)); for (const r of results) { if (selected.length >= limit) break; - if (!selected.some(s => s.obsId === r.obsId)) { + if (!picked.has(r.obsId)) { selected.push(r); } } @@ -358,7 +366,27 @@ export class HybridSearch { }>, limit: number, ): Promise { - const sliced = results.slice(0, limit); + const enriched: HybridSearchResult[] = []; + let next = 0; + while (next < results.length && enriched.length < limit) { + const batch = results.slice(next, next + limit); + next += batch.length; + enriched.push(...(await this.enrichBatch(batch))); + } + return enriched.slice(0, limit); + } + + private async enrichBatch( + sliced: Array<{ + obsId: string; + sessionId: string; + bm25Score: number; + vectorScore: number; + graphScore: number; + combinedScore: number; + graphContext?: string; + }>, + ): Promise { const observations = await Promise.all( sliced.map(async (r) => { const obs = await this.kv diff --git a/src/state/index-persistence.ts b/src/state/index-persistence.ts index 566883872c..af4352a252 100644 --- a/src/state/index-persistence.ts +++ b/src/state/index-persistence.ts @@ -1,3 +1,4 @@ +import {recoveryUpserts, type RecoveryUpsert, type RecoverySelection} from './vector-checkpoint-reconciliation.js'; import { VectorIndex, base64ToFloat32, float32ToBase64, type VectorEntry } from "./vector-index.js"; import type { StateKV } from "./kv.js"; import { KV } from "./schema.js"; @@ -86,6 +87,8 @@ type IndexPersistenceOptions = { bucketSize?: number; now?: () => number; requireRecoveryPlan?: boolean; + isStorageHeld?: () => boolean; + storageGeneration?: () => number; }; export interface IndexLegStatus { @@ -100,6 +103,55 @@ export interface VectorCountShortfall { loaded: number; } +const RETAINED_PROOF_ERROR_CODES = [ + 'VECTOR_CHECKPOINT_UNVERIFIED', 'FILE_WRITER_EPOCH_UNVERIFIED', + 'vector_recovery_writer_unverified', 'vector_recovery_selection_invalid', 'vector_recovery_epoch_changed', + 'vector_recovery_pending_changed', 'vector_recovery_meta_changed', 'vector_recovery_checkpoint_changed', + 'vector_recovery_checkpoint_incomplete', 'vector_recovery_checkpoint_mismatch', + 'vector_retention_store_unverified', 'vector_retention_receipts_invalid', 'vector_retention_original_missing', + 'vector_retention_original_changed', 'vector_retention_manual_verification_required', + 'vector_retention_disk_sync_unconfirmed', 'vector_retention_receipts_changed', + 'ENOENT', 'EACCES', 'EPERM', 'EIO', 'ELOOP', 'UNCLASSIFIED_PROOF_ERROR', +] as const; +type RetainedProofErrorCode = typeof RETAINED_PROOF_ERROR_CODES[number]; +type RetainedProofReason = 'verified' | 'retention_not_ready' | 'recovery_not_admitted' | 'pending_row_mismatch' | + 'vector_unavailable' | 'checkpoint_meta_unverified' | 'checkpoint_selection_rejected' | 'physical_proof_rejected' | + 'postproof_not_admitted' | 'sdk_required' | 'publication_invalid' | 'publication_mismatch' | 'pending_read_rejected' | + 'capture_pending_rejected'; + +export interface RetainedProofDiagnostic { + id: string; + operation: 'capture_pending' | 'retained_publication'; + reason: RetainedProofReason; + freshBytesMatch: boolean | null; + loadedBytesMatch: boolean | null; + freshSessionMatch: boolean | null; + loadedSessionMatch: boolean | null; + retentionReady: boolean; + stopped: boolean; + storageHeld: boolean; + storageGenerationChanged: boolean; + storageRecovering: boolean; + storageBlocked: boolean; + recoveryPaused: boolean; + pendingLog: number; + logChains: number; + dirty: number; + revisionChanged: boolean | null; + physicalProofErrorCode: RetainedProofErrorCode | null; +} + +function retainedProofErrorCode(error: unknown): RetainedProofErrorCode { + try { + const value = error as {code?: unknown; message?: unknown} | null; + for (const candidate of [value?.code, value?.message]) { + if (typeof candidate === 'string' && RETAINED_PROOF_ERROR_CODES.includes(candidate as RetainedProofErrorCode)) + return candidate as RetainedProofErrorCode; + } + } catch { /* An error accessor is untrusted metadata too. */ } + return 'UNCLASSIFIED_PROOF_ERROR'; +} + export interface IndexPersistenceStatus { saveIntervalMs: number; firstCheckpointPending?: boolean; @@ -110,8 +162,11 @@ export interface IndexPersistenceStatus { vector: IndexLegStatus | null; vectorCountShortfall: VectorCountShortfall | null; pendingLog?: number; + pendingLogKnown?: boolean; + retainedCheckpointRows?: number; pendingLogAcknowledged?: number; pendingLogError?: string | null; + retainedProofLastCheck?: RetainedProofDiagnostic | null; } export type VectorLoadState = "buckets" | "migrated" | "none" | "unavailable"; @@ -121,6 +176,7 @@ export interface VectorLoadResult { state: VectorLoadState; savedAt: string | null; expectedCount?: number; + checkpointMeta?: VectorMeta; backfillSince?: string | null; } @@ -179,13 +235,19 @@ function isPersistedVector(row: unknown): row is PersistedVector { return typeof candidate.id === "string" && typeof candidate.s === "string" && typeof candidate.e === "string"; } -async function inBatches(items: T[], size: number, run: (item: T) => Promise): Promise { +async function inBatches( + items: T[], + size: number, + run: (item: T) => Promise, + stopOnFailure = false, +): Promise { const failures: unknown[] = []; for (let offset = 0; offset < items.length; offset += size) { const results = await Promise.allSettled(items.slice(offset, offset + size).map(run)); for (const result of results) { if (result.status === "rejected") failures.push(result.reason); } + if (stopOnFailure && failures.length > 0) break; } return failures; } @@ -219,6 +281,12 @@ export class IndexPersistence { private readonly saveIntervalMs: number; private readonly bucketSize: number; private readonly now: () => number; + private readonly isStorageHeld: () => boolean; + private readonly storageGeneration: () => number; + private bootStorageGeneration: number; + private storageBlocked = false; + private storageResume: Promise | null = null; + private storageRecovering = false; private bucketOfId: Map = new Map(); private bucketCounts: Map = new Map(); // Retain the original address until the deletion is certified on disk. @@ -227,11 +295,15 @@ export class IndexPersistence { private hasOpenBucket = false; private vectorCountShortfall: VectorCountShortfall | null = null; private seq = 0; + private retainedRows = new Map(); + private retainedProofLastCheck: RetainedProofDiagnostic | null = null; + private retentionReady = true; private pendingLog: Map = new Map(); private acknowledgedLog: Map = new Map(); private logChains: Map> = new Map(); private logSuppressed = false; private pendingLogUnknown = false; + private pendingLogInspected = false; private pendingLogError: string | null = null; private earlySaveQueued = false; private backfillQueue: Promise = Promise.resolve(); @@ -248,7 +320,20 @@ export class IndexPersistence { options: IndexPersistenceOptions = {}, ) { this.recoveryReady = !options.requireRecoveryPlan || !vector; + this.pendingLogInspected = !options.requireRecoveryPlan || !vector; + this.isStorageHeld = options.isStorageHeld ?? (() => false); + this.storageGeneration = options.storageGeneration ?? (() => 0); + this.bootStorageGeneration = this.storageGeneration(); this.vector?.setChangeListener((id, entry) => this.logChange(id, entry)); + this.retentionReady = !this.hasRetentionStore() || + (!options.requireRecoveryPlan && this.kv.canUseUnloadedDiskIndex?.() === true); + this.vector?.setMutationGuard(id => { + if (!this.retentionReady || (id === null ? this.retainedRows.size > 0 : this.retainedRows.has(id)) || + (!this.logSuppressed && (this.isStorageHeld() || this.storageBlocked || this.storageRecovering || + this.storageGeneration() !== this.bootStorageGeneration))) { + throw Object.assign(new Error('VECTOR_RECOVERY_PAUSED'), {code: 'VECTOR_RECOVERY_PAUSED'}); + } + }); this.now = options.now ?? Date.now; const interval = options.saveIntervalMs; this.saveIntervalMs = @@ -320,7 +405,8 @@ export class IndexPersistence { const validMeta = meta === null ? expected === null && bucket === undefined : meta.v === 3 && Number.isInteger(meta.count) && meta.count >= (expected ? 1 : 0) && Number.isInteger(meta.bucketCount) && meta.bucketCount >= 0 && (bucket === undefined || bucket < meta.bucketCount); - if (!isDeepStrictEqual(row, expected) || pending !== null || clear !== null || !validMeta) throw unverified(); + if (!isDeepStrictEqual(row, expected) || clear !== null || !validMeta || + ((pending !== null || this.retainedRows.has(id)) && !await this.verifyRetainedVectorPublication(publication))) throw unverified(); }; await Promise.all([this.logChains.get(id), this.logChains.get(PENDING_CLEAR_KEY)]); await check(); @@ -339,26 +425,57 @@ export class IndexPersistence { pendingChanges: this.vector?.pendingChanges ?? 0, vector: this.vector ? { ...this.leg } : null, vectorCountShortfall: this.vectorCountShortfall, - pendingLog: this.vector ? this.pendingLog.size : 0, - pendingLogAcknowledged: this.vector ? this.acknowledgedLog.size : 0, + pendingLog: this.pendingLogInspected && !this.pendingLogUnknown ? this.pendingLog.size + this.retainedRows.size : undefined, + retainedCheckpointRows: this.retainedRows.size, + pendingLogKnown: this.pendingLogInspected && !this.pendingLogUnknown, + pendingLogAcknowledged: this.pendingLogInspected && !this.pendingLogUnknown ? this.acknowledgedLog.size : undefined, pendingLogError: this.pendingLogError, + retainedProofLastCheck: this.retainedProofLastCheck ? {...this.retainedProofLastCheck} : null, }; } async replayPendingLog(expectedDimensions = 0): Promise { const result: PendingReplayResult = { entries: 0, added: 0, removed: 0, cleared: false, skipped: 0 }; const vector = this.vector; - if (!vector || this.recoveryError) return result; + if (!vector || !this.retentionReady || this.blockIfStorageHeld() || this.recoveryError) return result; let rows: unknown[]; try { rows = await this.kv.list(KV.vectorPendingLog); } catch (err) { this.pendingLogUnknown = true; this.pendingLogError = errorMessage(err); + this.pauseRecovery(new Error("Pending vector log read failed")); logger.warn("index persistence: could not read the pending vector log", { message: this.pendingLogError }); return result; } + if (this.blockIfStorageHeld()) return result; + this.pendingLogInspected = true; + if (!Array.isArray(rows) || rows.some(row => !isPendingVectorRow(row))) { + this.pendingLogUnknown = true; + this.pauseRecovery(new Error('Pending vector log contains invalid records')); + return result; + } + if (this.retainedRows.size) { + if (rows.length !== this.retainedRows.size || + ![...this.retainedRows.values()].every(original => rows.some(row => isDeepStrictEqual(row, original)))) { + this.storageBlocked = true; + this.pendingLogUnknown = true; + this.pauseRecovery(new Error('Retained vector log changed')); + } + return result; + } const valid = (Array.isArray(rows) ? rows : []).filter(isPendingVectorRow).sort((a, b) => a.q - b.q); + for (const row of valid) { + if (row.c === 1 || row.t === 1) continue; + try { + const decoded = base64ToFloat32(row.e!); + if (Buffer.from(row.e!, 'base64').toString('base64') !== row.e || !decoded.length || (expectedDimensions > 0 && decoded.length !== expectedDimensions) || decoded.some(value => !Number.isFinite(value))) throw new Error('invalid vector'); + } catch { + this.pendingLogUnknown = true; + this.pauseRecovery(new Error('Pending vector log contains invalid embeddings')); + return result; + } + } this.logSuppressed = true; try { for (const row of valid) { @@ -416,8 +533,9 @@ export class IndexPersistence { markBackfillSince(since: string | null): Promise { this.backfillRevision++; return this.queueBackfillChange(async () => { - if (!this.vector || (since !== null && !Number.isFinite(Date.parse(since)))) return; + if (!this.vector || this.blockIfStorageHeld() || (since !== null && !Number.isFinite(Date.parse(since)))) return; const current = await this.readBackfillMarker(); + if (this.blockIfStorageHeld()) throw new Error('Storage hold prevents vector recovery'); const next = earliestBackfillSince(current, this.backfillSince, since)!; if (current !== next) await this.kv.set(KV.bm25Index, BACKFILL_MARKER_KEY, { v: 2, since: next }); this.backfillSince = next; @@ -443,8 +561,204 @@ export class IndexPersistence { logger.warn(this.recoveryError); } + private blockIfStorageHeld(): boolean { + if (!this.isStorageHeld() && this.storageGeneration() === this.bootStorageGeneration) return false; + this.storageBlocked = true; + if (!this.recoveryError) this.pauseRecovery(new Error('Storage hold prevents vector recovery')); + return true; + } + + recoveryCutoff(): string | null | undefined { return this.backfillSince; } + + resumeStorageRecovery(expectedDimensions = 0, manualReconcile = false): Promise { + if (this.storageResume) return this.storageResume; + this.storageRecovering = true; + const work = this.resumeStorageRecoveryOnce(expectedDimensions,manualReconcile).finally(() => { this.storageRecovering = false; this.storageResume = null; }); + this.storageResume = work; + return work; + } + + private hasRetentionStore(): boolean { + return this.kv.backend === 'file' && typeof this.kv.readVectorRetention === 'function'; + } + + private checkpointSelection(loaded: VectorLoadResult, rows: RecoveryUpsert[]): RecoverySelection { + if (rows.length && (!loaded.vector || !loaded.checkpointMeta || loaded.vector.size !== loaded.expectedCount)) { + throw Error('vector_recovery_checkpoint_incomplete'); + } + const buckets = new Map(); + for (const row of rows) { + const entry = loaded.vector?.get(row.id), bucket = this.bucketOfId.get(row.id); + if (bucket === undefined || !entry || entry.sessionId !== row.s || float32ToBase64(entry.embedding) !== row.e) + throw Error('vector_recovery_checkpoint_mismatch'); + const ids = buckets.get(bucket) ?? []; + ids.push(row.id); + buckets.set(bucket, ids); + } + return {pendingRows: rows, meta: loaded.checkpointMeta ?? null, buckets: [...buckets].map(([bucket, ids]) => ({bucket, ids}))}; + } + + /** Revalidate existing WAL evidence for one explicitly selected capture, without creating receipts. */ + async verifyCaptureVectorPending(id: string, sessionId: string, pending: unknown): Promise { + if (!this.retentionReady || this.storageRecovering || this.storageBlocked || this.isRecoveryPaused()) { + this.recordRetainedProof(id, sessionId, 'capture_pending', this.retentionReady ? 'recovery_not_admitted' : 'retention_not_ready'); + return false; + } + const row = this.retainedRows.get(id); + if (!row) return pending == null; + if (!this.vector || row.s !== sessionId || !isDeepStrictEqual(row, pending)) { + this.recordRetainedProof(id, sessionId, 'capture_pending', this.vector ? 'pending_row_mismatch' : 'vector_unavailable'); + return false; + } + const revision = this.kv.vectorCheckpointRevision; + const finish = (result: boolean, reason: RetainedProofReason, error: RetainedProofErrorCode | null = null) => { + this.recordRetainedProof(id, sessionId, 'capture_pending', reason, undefined, revision, error); + return result; + }; + const current = () => !this.stopped && !this.isStorageHeld() && this.storageGeneration() === this.bootStorageGeneration && + !this.storageBlocked && !this.storageRecovering && !this.isRecoveryPaused() && + !this.pendingLog.size && !this.logChains.size && !this.vector?.pendingChanges; + let stage: RetainedProofReason = 'checkpoint_meta_unverified'; + try { + if (!current()) return finish(false, 'recovery_not_admitted'); + const meta = await this.kv.get(KV.bm25Index, VECTOR_META_KEY); + if (meta?.v !== 3) return finish(false, 'checkpoint_meta_unverified'); + const rows = [...this.retainedRows.values()]; + stage = 'checkpoint_selection_rejected'; + const selected = this.checkpointSelection({state: 'buckets', vector: this.vector, savedAt: meta.savedAt, + checkpointMeta: meta, expectedCount: meta.count}, rows); + stage = 'physical_proof_rejected'; + await this.kv.reconcileVectorCheckpoint(selected, current, revision, false, true); + const result = current(); + return finish(result, result ? 'verified' : 'postproof_not_admitted'); + } catch (error) { return finish(false, stage, retainedProofErrorCode(error)); } + } + + async verifyRetainedVectorPublication(publication: VectorPublication): Promise { + const retained = this.retainedRows.get(publication.id), entry = publication.entry; + if (this.kv.durability?.completion !== 'sdk' || !retained || !entry || !(entry.embedding instanceof Float32Array)) { + this.recordRetainedProof(publication.id, undefined, 'retained_publication', + this.kv.durability?.completion !== 'sdk' ? 'sdk_required' : 'publication_invalid'); + return false; + } + const finish = (result: boolean, reason: RetainedProofReason, nested?: RetainedProofDiagnostic | null) => { + this.recordRetainedProof(publication.id, entry.sessionId, 'retained_publication', reason, entry, undefined, + nested?.physicalProofErrorCode ?? null, nested?.revisionChanged ?? null); + return result; + }; + const matches = () => { + const loaded = this.vector?.get(publication.id); + return entry.sessionId === retained.s && float32ToBase64(entry.embedding) === retained.e && + loaded?.sessionId === retained.s && float32ToBase64(loaded.embedding) === retained.e; + }; + let stage: RetainedProofReason = 'publication_mismatch'; + try { + if (!matches()) return finish(false, 'publication_mismatch'); + stage = 'pending_read_rejected'; + const pending = await this.kv.get(KV.vectorPendingLog, publication.id); + stage = 'capture_pending_rejected'; + const previousDiagnostic = this.retainedProofLastCheck; + const verified = await this.verifyCaptureVectorPending(publication.id, entry.sessionId, pending); + const nested = this.retainedProofLastCheck !== previousDiagnostic && this.retainedProofLastCheck?.id === publication.id && this.retainedProofLastCheck.operation === 'capture_pending' + ? this.retainedProofLastCheck : null; + if (!verified) return finish(false, nested?.reason ?? 'capture_pending_rejected', nested); + stage = 'publication_mismatch'; + const result = matches(); + return finish(result, result ? 'verified' : 'publication_mismatch', nested); + } catch { return finish(false, stage); } + } + + private recordRetainedProof(id: string, sessionId: string | undefined, operation: RetainedProofDiagnostic['operation'], + reason: RetainedProofReason, fresh?: VectorEntry, revision?: number, physicalProofErrorCode: RetainedProofErrorCode | null = null, + revisionChanged: boolean | null = null): void { + const row = this.retainedRows.get(id); + if (!row) return; + // Diagnostic reads must never alter a verification decision or retain provider vector bytes. + try { + const loaded = this.vector?.get(id); + this.retainedProofLastCheck = { + id, operation, reason, + freshBytesMatch: fresh?.embedding instanceof Float32Array ? float32ToBase64(fresh.embedding) === row.e : null, + loadedBytesMatch: loaded?.embedding instanceof Float32Array ? float32ToBase64(loaded.embedding) === row.e : null, + freshSessionMatch: sessionId === undefined ? null : sessionId === row.s, + loadedSessionMatch: loaded ? loaded.sessionId === row.s : null, + retentionReady: this.retentionReady, stopped: this.stopped, storageHeld: this.isStorageHeld(), + storageGenerationChanged: this.storageGeneration() !== this.bootStorageGeneration, + storageRecovering: this.storageRecovering, storageBlocked: this.storageBlocked, recoveryPaused: this.isRecoveryPaused(), + pendingLog: this.pendingLog.size, logChains: this.logChains.size, dirty: this.vector?.pendingChanges ?? 0, + revisionChanged: revision === undefined ? revisionChanged : this.kv.vectorCheckpointRevision !== revision, physicalProofErrorCode, + }; + } catch { this.retainedProofLastCheck = null; } + } + + private rememberRetained(rows: RecoveryUpsert[]): void { + this.retainedRows = new Map(rows.map(row => [row.id, row])); + for (const row of rows) this.seq = Math.max(this.seq, row.q); + this.retentionReady = true; + } + + private async requireEmptySdkPendingLog(): Promise { + const rows = await this.kv.list(KV.vectorPendingLog); + if (!Array.isArray(rows) || rows.length > 0 || this.pendingLog.size > 0 || this.logChains.size > 0) { + throw new Error('Pending vector records require durable reconciliation before SDK recovery'); + } + } + + private async resumeStorageRecoveryOnce(expectedDimensions: number, manualReconcile:boolean): Promise { + if (this.stopped || this.isStorageHeld()) return false; + const generation = this.storageGeneration(); + const sdkRecovery = this.kv.durability?.completion === 'sdk' && + (this.storageBlocked || generation !== this.bootStorageGeneration); + if (!sdkRecovery && !this.vector) return true; + if (!sdkRecovery && !this.storageBlocked) return !this.isRecoveryPaused(); + try { + let selected:RecoveryUpsert[] | undefined; + const checkpointRevision=this.kv.vectorCheckpointRevision; + const current=()=>!this.stopped && !this.isStorageHeld() && generation===this.storageGeneration() && + this.pendingLog.size===0 && this.logChains.size===0 && this.storageRecovering; + if (sdkRecovery && manualReconcile) { + if(!current())throw Error('vector_recovery_busy'); + selected=recoveryUpserts(await this.kv.list(KV.vectorPendingLog),expectedDimensions); + } else if (sdkRecovery) await this.requireEmptySdkPendingLog(); + if (this.isStorageHeld() || generation !== this.storageGeneration()) throw new Error('Storage hold changed during vector recovery'); + this.bootStorageGeneration = generation; + this.storageBlocked = false; + this.recoveryError = null; + if (!this.vector) { + if(selected?.length)throw Error('vector_recovery_index_unavailable'); + if(selected)await this.kv.reconcileVectorCheckpoint({pendingRows:[],buckets:[],meta:null},current,checkpointRevision,true); + return true; + } + const loaded = await this.load(Boolean(selected?.length), selected !== undefined); + if (this.blockIfStorageHeld() || !loaded || loaded.state === 'unavailable') throw new Error('Persisted vector checkpoint is unavailable'); + if (loaded.vector && expectedDimensions > 0 && loaded.vector.validateDimensions(expectedDimensions).mismatches.length) throw new Error('Persisted vector dimensions mismatch'); + // SDK checkpoint reads may span minutes. Recheck before the first backfill write. + if (sdkRecovery) { + if (selected) { + await this.kv.reconcileVectorCheckpoint(this.checkpointSelection(loaded, selected), current, checkpointRevision, true); + if (!current()) throw Error('vector_recovery_epoch_changed'); + this.rememberRetained(selected); + } else await this.requireEmptySdkPendingLog(); + this.pendingLogInspected = true; + this.pendingLogUnknown = false; + } + await this.prepareBackfill(loaded); + if (this.blockIfStorageHeld() || this.isRecoveryPaused()) { this.storageBlocked = true; return false; } + if (loaded.vector && loaded.vector.size > 0) this.vector.restoreFrom(loaded.vector); + if (!sdkRecovery) await this.replayPendingLog(expectedDimensions); + if (this.blockIfStorageHeld() || this.isRecoveryPaused()) { this.storageBlocked = true; return false; } + this.pendingLogError = null; + this.leg.lastError = null; this.leg.lastErrorAt = null; + return true; + } catch (error) { + this.storageBlocked = true; + if (sdkRecovery || !this.recoveryError) this.pauseRecovery(error); + return false; + } + } + async prepareBackfill(loaded: VectorLoadResult | null, forceFull = false): Promise { - if (!this.vector || this.recoveryError) return undefined; + if (!this.vector || this.blockIfStorageHeld() || this.recoveryError) return undefined; if (!loaded || loaded.state === "unavailable") { this.pauseRecovery(new Error("Persisted vector checkpoint is unavailable")); return undefined; @@ -457,7 +771,9 @@ export class IndexPersistence { throw new Error("Invalid persisted vector recovery cutoff"); } await this.markBackfillSince(earliestBackfillSince(since, fallback)!); + if (this.blockIfStorageHeld() || this.recoveryError) return undefined; const marker = await this.readBackfillMarker(); + if (this.blockIfStorageHeld() || this.recoveryError) return undefined; this.recoveryReady = true; this.scheduleSave(); return marker; @@ -533,7 +849,7 @@ export class IndexPersistence { } private logChange(id: string | null, entry: VectorEntry | null): void { - if (this.logSuppressed || !this.recoveryReady || this.recoveryError) return; + if (this.logSuppressed || this.blockIfStorageHeld() || !this.recoveryReady || this.recoveryError) return; const key = id ?? PENDING_CLEAR_KEY; const q = this.nextSeq(); let row: PendingVectorRow; @@ -553,6 +869,7 @@ export class IndexPersistence { this.pendingLog.set(key, q); this.acknowledgedLog.delete(key); this.enqueueLog(key, async () => { + if (this.blockIfStorageHeld()) throw new Error('Storage hold prevents vector log write'); await this.kv.set(KV.vectorPendingLog, key, row); if (this.pendingLog.get(key) === q) this.acknowledgedLog.set(key, q); }).then( @@ -593,7 +910,9 @@ export class IndexPersistence { private deletePendingKey(key: string, q: number | undefined): Promise { return this.enqueueLog(key, async () => { + if (this.retainedRows.has(key) || (key === PENDING_CLEAR_KEY && this.retainedRows.size)) throw Error('vector_retention_original_protected'); if (this.pendingLog.get(key) !== q) return; + if (this.blockIfStorageHeld()) throw new Error('Storage hold prevents vector log deletion'); await this.kv.delete(KV.vectorPendingLog, key); if (this.pendingLog.get(key) === q) { this.pendingLog.delete(key); @@ -620,8 +939,20 @@ export class IndexPersistence { this.clearTimer(); } - async load(): Promise { + async load(strictCheckpoint = false, operatorWillVerify = false): Promise { + const unavailable: VectorLoadResult = {vector: null, state: 'unavailable', savedAt: null}; + if (this.blockIfStorageHeld()) return { vector: null, state: 'unavailable', savedAt: null }; + const revision = this.kv.vectorCheckpointRevision, generation = this.storageGeneration(); + let hasReceipts = false; + if (this.hasRetentionStore()) { + this.retentionReady = false; + try { hasReceipts = (await this.kv.readVectorRetention()).length > 0; } + catch (error) { this.storageBlocked = true; this.pauseRecovery(error); return unavailable; } + this.retentionReady = !hasReceipts && !operatorWillVerify; + strictCheckpoint ||= hasReceipts; + } if (!this.vector) { + if (hasReceipts) { this.storageBlocked = true; this.pauseRecovery(Error('vector_retention_index_unavailable')); return unavailable; } await this.finishLegacyCleanup(); return { vector: null, state: "none", savedAt: null }; } @@ -634,8 +965,11 @@ export class IndexPersistence { return { vector: null, state: "unavailable", savedAt: null }; } + if (this.blockIfStorageHeld()) return { vector: null, state: 'unavailable', savedAt: null }; + if (meta && meta.v === 3 && Number.isInteger(meta.bucketCount) && meta.bucketCount >= 0) { - const loaded = await this.loadBuckets(meta.bucketCount); + const loaded = await this.loadBuckets(meta.bucketCount,strictCheckpoint); + if (this.blockIfStorageHeld()) return { vector: null, state: 'unavailable', savedAt: null }; if (!loaded) return { vector: null, state: "unavailable", savedAt: null }; this.hasVectorCheckpoint = loaded.size > 0; this.checkpointBackfillSince = meta.backfillSince; @@ -651,8 +985,25 @@ export class IndexPersistence { ...this.vectorCountShortfall, }); } - return { vector: loaded, state: replacementVerified ? "buckets" : "unavailable", savedAt: meta.savedAt ?? null, expectedCount, backfillSince: meta.backfillSince }; + const result: VectorLoadResult = { vector: loaded, state: replacementVerified ? "buckets" : "unavailable", savedAt: meta.savedAt ?? null, + expectedCount, ...(strictCheckpoint ? {checkpointMeta:meta} : {}), backfillSince: meta.backfillSince }; + if (hasReceipts && !operatorWillVerify) { + try { + if (!replacementVerified) throw Error('vector_retention_checkpoint_unavailable'); + const raw = await this.kv.list(KV.vectorPendingLog); + const first = raw?.[0]?.id; + const dimensions = typeof first === 'string' ? loaded.get(first)?.embedding.length ?? 0 : 0; + const rows = recoveryUpserts(raw, dimensions); + const current = () => !this.stopped && !this.isStorageHeld() && this.storageGeneration() === generation && + !this.pendingLog.size && !this.logChains.size; + await this.kv.reconcileVectorCheckpoint(this.checkpointSelection(result, rows), current, revision, false); + if (!current()) throw Error('vector_retention_epoch_changed'); + this.rememberRetained(rows); + } catch (error) { this.storageBlocked = true; this.pauseRecovery(error); return unavailable; } + } + return result; } + if (hasReceipts) { this.storageBlocked = true; this.pauseRecovery(Error('vector_retention_checkpoint_missing')); return unavailable; } return this.migrateLegacy(); } @@ -725,7 +1076,7 @@ export class IndexPersistence { private async runSave(): Promise { const vector = this.vector; - if (!vector || this.isRecoveryPaused()) return; + if (!vector || !this.retentionReady || this.blockIfStorageHeld() || this.isRecoveryPaused()) return; if (!vector.pendingChanges && !this.needsRecoveryCheckpoint()) return; const hasVectorChanges = vector.pendingChanges > 0; const epoch = this.dirtyEpoch; @@ -737,6 +1088,7 @@ export class IndexPersistence { const coveredSeq = this.seq; try { await this.writeChanges(vector); + if (this.blockIfStorageHeld()) return; await this.clearCoveredPendingLog(coveredSeq).catch((err) => { this.pendingLogError = errorMessage(err); this.logFailure(err); @@ -815,26 +1167,33 @@ export class IndexPersistence { if (!this.hasVectorCheckpoint) this.firstVectorSaveAttempted = false; } - private async loadBuckets(bucketCount: number): Promise { + private async loadBuckets(bucketCount: number, strictCheckpoint = false): Promise { const loaded = new VectorIndex(); const bucketOfId = new Map(); const bucketCounts = new Map(); const bucketIds = Array.from({ length: bucketCount }, (_, bucket) => bucket); - const failures = await inBatches(bucketIds, LOAD_CONCURRENCY, async (bucket) => { + // SDK bucket lists share the worker connection. Stop issuing reads once the checkpoint is unavailable. + const sdkReads = this.kv.durability?.completion === "sdk"; + const failures = await inBatches(bucketIds, sdkReads ? 1 : LOAD_CONCURRENCY, async (bucket) => { const rows = await this.kv.list(vectorBucketScope(bucket)); let count = 0; for (const row of rows) { - if (!isPersistedVector(row)) continue; + if (!isPersistedVector(row)) { + if(strictCheckpoint)throw Error('vector_recovery_bucket_invalid'); + continue; + } + if(strictCheckpoint && bucketOfId.has(row.id))throw Error('vector_recovery_duplicate_id'); try { loaded.loadPersisted(row.id, row.s, base64ToFloat32(row.e)); } catch { + if(strictCheckpoint)throw Error('vector_recovery_bucket_invalid'); continue; } bucketOfId.set(row.id, bucket); count++; } if (count > 0) bucketCounts.set(bucket, count); - }); + }, sdkReads); if (failures.length > 0) { logger.warn("index persistence: vector bucket read failed", { failed: failures.length, diff --git a/src/state/insight-response.ts b/src/state/insight-response.ts new file mode 100644 index 0000000000..afdce08137 --- /dev/null +++ b/src/state/insight-response.ts @@ -0,0 +1,23 @@ +import { checkPayloadFrameSize } from "./frame-guard.js"; + +export function insightHttpResponse(result: unknown, mcp = false): { status_code: number; body: unknown } { + const flags = result as { invalid?: boolean; oversized?: boolean; success?: boolean } | null; + let status = 200; + if (flags?.invalid) status = 400; + if (flags?.oversized) status = 413; + + function response(body: unknown, status_code: number) { + if (!mcp) return { status_code, body }; + return { + status_code, + body: { + content: [{ type: "text", text: JSON.stringify(body, null, 2) }], + ...(status_code !== 200 || flags?.success === false ? { isError: true } : {}), + }, + }; + } + + const envelope = response(result, status); + const oversized = checkPayloadFrameSize(envelope, "reduce limit or filter by project"); + return oversized ? response(oversized, 413) : envelope; +} diff --git a/src/state/kv.ts b/src/state/kv.ts index ca42624744..bc794db4c9 100644 --- a/src/state/kv.ts +++ b/src/state/kv.ts @@ -1,6 +1,12 @@ -import {createRecordRecoveryGate} from './recovery-write-gate.js' +import {isStateDispatchRejected,localMutationRejected} from '../ops/state-dispatch-error.mjs' +import { lstatSync, realpathSync } from 'node:fs' +import { join } from 'node:path' +import {bucketScope, reconcileCheckpoint, readRetention, receiptFor, RETAINED_SCOPE, type RetentionReceipt, type RecoverySelection} from './vector-checkpoint-reconciliation.js' +import {KV} from './schema.js' +import {createRecordRecoveryGate, createRecoveryWriteGate} from './recovery-write-gate.js' import {ownsFileWriterLease,fileWriterReadCurrent,fileWriterRecoveryAdmitted,fileWriterRecoveryReason,type FileWriterLease} from '../ops/file-writer-lease.mjs' import { createCaptureReceiptPage } from '../capture/receipt-page.js' +import {decodeCursor,encodeCursor} from './list-query.js' import type { IIIClient } from 'iii-sdk' import { withKeyedLock } from './keyed-mutex.js' import { getEnvVar } from '../config.js' @@ -20,6 +26,7 @@ export interface StateKVOptions { writerLease?: FileWriterLease backend?: StateBackend durability?: ReturnType + compactSummaryWrites?: boolean } type UpdateOp = { type: string; path: string; value?: unknown } @@ -117,10 +124,22 @@ export function orderLikeInsertion(values: T[]): T[] { .map((entry) => entry.value) } +export async function collectAll(kv: StateKV, scope: string): Promise { + if (typeof kv.iterate !== 'function') return kv.list(scope) + const out: T[] = [] + for await (const page of kv.iterate(scope)) out.push(...page) + if (kv.backend === 'redis') return orderLikeInsertion(out) + return out +} + export class StateKV { private readonly summaryNamespaceLock = `summary-namespace:${++summaryLockSequence}` private writerLease?:FileWriterLease private writerStorePath?:string + private checkpointStorePath?:string + private readonly vectorWrites = createRecoveryWriteGate() + private vectorRevision = 0 + get vectorCheckpointRevision() { return this.vectorRevision } private uncertainMutation=false get recoveryBlockReason(){return this.uncertainMutation?'WRITER_MUTATION_UNCERTAIN':this.writerLease?fileWriterRecoveryReason(this.writerLease):'FILE_WRITER_UNSUPPORTED'} get mutationOutcomeUncertain(){return this.uncertainMutation} @@ -133,13 +152,16 @@ export class StateKV { readonly captureReceiptPage: ReturnType | null readonly auditReader: ReturnType | null private readonly summaryCount: ReturnType | null + private readonly compactSummaryWrites: boolean constructor( private sdk: IIIClient, options: StateKVOptions = {}, ) { + this.compactSummaryWrites = options.compactSummaryWrites === true this.backend = options.backend ?? 'file' const storePath = this.backend === 'file' ? getEnvVar('AGENTMEMORY_STATE_STORE_PATH') : undefined + this.checkpointStorePath = storePath if (this.backend === 'file' && storePath && ownsFileWriterLease(options.writerLease,storePath)) {this.recoveryWrite=createRecordRecoveryGate();this.writerLease=options.writerLease;this.writerStorePath=storePath;} this.durability = this.backend === 'file' ? options.durability ?? (storePath ? createStateDurabilityFence({ storePath, completion: 'sdk' }) : null) @@ -152,12 +174,57 @@ export class StateKV { } private guardedMutation(scope: string, key: string, fn: () => Promise): Promise { - const operation=()=>{if(this.writerLease&&!ownsFileWriterLease(this.writerLease,this.writerStorePath!))return Promise.reject(Error('FILE_WRITER_LOST'));if(this.recoveryWrite?.inExclusive()&&!this.recoveryMutationSafe)return Promise.reject(Error('WRITER_EPOCH_UNVERIFIED'));return fn()} - const serialized=async()=>{try{return await operation()}catch(error){this.markMutationUncertain(scope);throw error}} + const operation=()=>{if(this.writerLease&&!ownsFileWriterLease(this.writerLease,this.writerStorePath!))return Promise.reject(localMutationRejected('FILE_WRITER_LOST'));if(this.recoveryWrite?.inExclusive()&&!this.recoveryMutationSafe)return Promise.reject(localMutationRejected('WRITER_EPOCH_UNVERIFIED'));return fn()} + const serialized=async()=>{try{return await operation()}catch(error){if(!isStateDispatchRejected(error))this.markMutationUncertain(scope);throw error}} // Keep queued mutations visible to the durability/shutdown fence. The gate // belongs inside the tracked operation, rather than before performMutation. - const perform=()=>{if(this.writerLease&&!ownsFileWriterLease(this.writerLease,this.writerStorePath!))return Promise.reject(Error('FILE_WRITER_LOST'));return this.performMutation(scope,key,()=>scope==='mem:summaries'?withKeyedLock(this.summaryNamespaceLock,serialized):operation())} - return this.recoveryWrite ? this.recoveryWrite.read(scope,key,perform) : perform() + const perform=()=>{if(this.writerLease&&!ownsFileWriterLease(this.writerLease,this.writerStorePath!))return Promise.reject(localMutationRejected('FILE_WRITER_LOST'));return this.performMutation(scope,key,()=>scope==='mem:summaries'?withKeyedLock(this.summaryNamespaceLock,serialized):operation())} + if (!this.recoveryWrite) return perform() + const vectorNamespace = scope===KV.vectorPendingLog || scope===RETAINED_SCOPE || scope.startsWith(KV.bm25Index+':vec:') || + (scope===KV.bm25Index && key==='vectors:meta') + if (!vectorNamespace) return this.recoveryWrite.read(scope,key,perform) + return this.recoveryWrite.read(scope,key,()=>this.vectorWrites.read(async()=>{ + try { return await perform() } finally { this.vectorRevision++ } + })) + } + + canUseUnloadedDiskIndex(): boolean { + // Standalone disk-fenced callers may start with an empty index. Once a receipt + // exists, only the normal load/revalidation path can admit its retained WAL. + const store = this.checkpointStorePath + if (this.durability?.completion !== 'disk' || !store) return false + try { + return realpathSync(store) === store && lstatSync(store).isDirectory() && + lstatSync(join(store, encodeURIComponent(RETAINED_SCOPE) + '.bin'), {throwIfNoEntry: false}) === undefined + } catch { return false } + } + + async readVectorRetention():Promise { + if (this.backend !== 'file') return [] + if (!this.checkpointStorePath) throw Error('vector_retention_store_unverified') + return readRetention(this,this.checkpointStorePath,()=>!this.uncertainMutation && + (!this.writerLease || ownsFileWriterLease(this.writerLease,this.checkpointStorePath!))) + } + + async reconcileVectorCheckpoint(selection:RecoverySelection,current:()=>boolean,expectedRevision:number,authorize:boolean,waitForCaptureMeta=false):Promise{ + const gate=this.recoveryWrite, storePath=this.writerStorePath + if (waitForCaptureMeta && authorize) throw Error('vector_retention_capture_is_read_only') + if(!gate || !storePath || !this.recoveryMutationSafe || (authorize && this.durability?.completion!=='sdk'))throw Error('vector_recovery_writer_unverified') + const keys:Array<{scope:string;key:string}>=[{scope:KV.vectorPendingLog,key:'~clear'},{scope:KV.bm25Index,key:'vectors:meta'}, + ...selection.pendingRows.map(({id})=>({scope:KV.vectorPendingLog,key:id}))] + for(const {bucket,ids} of selection.buckets)for(const id of ids){ + const row=selection.pendingRows.find(item=>item.id===id) + if(!row)throw Error('vector_recovery_selection_invalid') + keys.push({scope:bucketScope(bucket),key:id},{scope:RETAINED_SCOPE,key:receiptFor(row,bucket).digest}) + } + const verify=()=>this.vectorWrites.exclusive(async()=>{ + if (this.vectorRevision!==expectedRevision) throw Error('vector_recovery_checkpoint_changed') + const admitted=()=>current() && (!authorize || gate.inExclusive()) && this.recoveryMutationSafe + const existing=await readRetention(this,storePath,admitted) + return reconcileCheckpoint(this,storePath,selection,existing,authorize,admitted,waitForCaptureMeta) + }) + // Read-only rechecks can run within capture's existing record gate. They never acquire another record lock. + return authorize ? gate.exclusive(keys,verify) : verify() } private markMutationUncertain(scope:string):void { @@ -170,11 +237,11 @@ export class StateKV { const finish = this.observationReader?.beginMutation?.(scope) try { return (this.durability ? this.durability.mutate(scope, key, fn) : fn()).catch(error => { - this.markMutationUncertain(scope) + if(!isStateDispatchRejected(error))this.markMutationUncertain(scope) throw error }).finally(() => finish?.()) } catch (error) { - this.markMutationUncertain(scope) + if(!isStateDispatchRejected(error))this.markMutationUncertain(scope) finish?.() throw error } @@ -211,7 +278,7 @@ export class StateKV { if(scope==='mem:summaries'&&this.summaryCount)return withKeyedLock(`summary-state:${key}`,async()=>{ const projection=knownValue&&!knownValue.deleted?this.summaryCount!.prepare(knownValue.value,{wire:false}):undefined const result=await operation(scope) - if(knownValue)this.summaryCount!.record(key,knownValue.deleted!==true,projection,result&&typeof result==='object'&&Object.hasOwn(result,'old_value')&&Object.hasOwn(result,'new_value')?Reflect.get(result,'new_value'):undefined) + if(knownValue)this.summaryCount!.record(key,knownValue.deleted!==true,projection,this.compactSummaryWrites?knownValue.value:result&&typeof result==='object'&&Object.hasOwn(result,'old_value')&&Object.hasOwn(result,'new_value')?Reflect.get(result,'new_value'):undefined) else if(result&&typeof result==='object'&&Object.hasOwn(result,'new_value'))this.summaryCount!.record(key,true,this.summaryCount!.prepare(Reflect.get(result,'new_value'))) else this.summaryCount!.invalidate() return result @@ -237,6 +304,18 @@ export class StateKV { } async set(scope: string, key: string, value: T): Promise { + if (scope === KV.summaries && this.compactSummaryWrites) { + const sent = JSON.parse(JSON.stringify(value)) as T + await this.mutate(scope, key, async (resolvedScope) => { + const acknowledged = await this.sdk.trigger({ + function_id: 'state::set_ack', + payload: { scope: resolvedScope, key, value: sent }, + }) + if (acknowledged !== true) throw Object.assign(Error('state_set_ack_unverified'), { code: 'STATE_SET_ACK_UNVERIFIED' }) + return { new_value: sent } + }, { value: sent }) + return sent + } return this.mutate(scope, key, (resolvedScope) => this.sdk.trigger<{ scope: string; key: string; value: T }, T>({ function_id: 'state::set', @@ -316,11 +395,14 @@ export class StateKV { }) } - async list(scope: string, options?: { observationConsistency?: 'opened-generation' }): Promise { + async list(scope: string, options?: { observationConsistency?: 'opened-generation'; project?: (value: unknown, key?: string) => T }): Promise { scope = await resolveGraphScope(this, scope) await ensureGraphConsistency(this, scope) - if (this.graphFileList?.accepts(scope)) return this.graphFileList.list(scope) - if (scope.startsWith('mem:obs:') && this.observationReader) return this.observationReader.list(scope, { consistency: options?.observationConsistency }) + if (this.graphFileList?.accepts(scope)) return this.graphFileList.list(scope, options) + if (scope.startsWith('mem:obs:') && this.observationReader) { + const rows = await this.observationReader.list(scope, { consistency: options?.observationConsistency }) + return options?.project ? rows.map(value => options.project!(value)) : rows + } const read = () => this.sdk.trigger<{ scope: string }, T[]>({ function_id: 'state::list', payload: { scope }, @@ -338,8 +420,147 @@ export class StateKV { return rows }) : await read() - if (this.backend !== 'redis' || !Array.isArray(values)) return values - return orderLikeInsertion(values) + const ordered = this.backend === 'redis' && Array.isArray(values) ? orderLikeInsertion(values) : values + return options?.project && Array.isArray(ordered) ? ordered.map(value => options.project!(value)) : ordered + } + + async listKeys(scope: string): Promise { + scope = await resolveGraphScope(this, scope) + await ensureGraphConsistency(this, scope) + return this.rawListKeys(scope) + } + + private async rawListKeys(scope: string): Promise { + const result = await this.sdk.trigger<{ scope: string }, unknown>({ + function_id: 'state::list_keys', + payload: { scope }, + }) + const keys = Array.isArray(result) ? result : (result as {keys?:unknown})?.keys + if(!Array.isArray(keys)||!keys.every(key=>typeof key==='string'))throw Error('state::list_keys returned an unexpected shape') + return keys + } + + async getMany( + scope: string, + keys: string[], + opts: { concurrency?: number } = {}, + ): Promise> { + scope = await resolveGraphScope(this, scope) + await ensureGraphConsistency(this, scope) + const concurrency = Math.min(64, Math.max(1, Math.floor(opts.concurrency ?? 16))) + return this.rawGetMany(scope, keys, concurrency) + } + + private async rawGetMany( + scope: string, + keys: string[], + concurrency: number, + ): Promise> { + const out: Array = new Array(keys.length).fill(null) + for (let start = 0; start < keys.length; start += concurrency) { + const slice = keys.slice(start, start + concurrency) + const values = await Promise.all( + slice.map((key) => + this.sdk.trigger<{ scope: string; key: string }, T | null>({ + function_id: 'state::get', + payload: { scope, key }, + }), + ), + ) + for (let i = 0; i < slice.length; i++) out[start + i] = values[i] ?? null + } + return out + } + + private async withSummaryRead(scope: string, read: () => Promise): Promise { + if (scope !== KV.summaries || !this.summaryCount) return read() + return withKeyedLock(this.summaryNamespaceLock, async () => { + const check = () => { + if (this.uncertainMutation) throw Error('summary_list_mutation_uncertain') + if (this.writerLease && !fileWriterReadCurrent(this.writerLease)) throw Error('FILE_WRITER_EPOCH_UNVERIFIED') + } + check() + const result = await read() + check() + return result + }) + } + + async *iterate( + scope: string, + opts: { pageSize?: number } = {}, + ): AsyncGenerator { + const pageSize = Math.min(1000, Math.max(1, Math.floor(opts.pageSize ?? 200))) + scope = await resolveGraphScope(this, scope) + await ensureGraphConsistency(this, scope) + // Keep custom opened-generation readers. + if(this.graphFileList?.accepts(scope)||(scope.startsWith('mem:obs:')&&this.observationReader)){ + const rows=await this.list(scope); + for(let start=0;start { + const keys = await this.rawListKeys(scope) + const rows: T[] = [] + for (let start = 0; start < keys.length; start += pageSize) { + const page = await this.rawGetMany(scope, keys.slice(start, start + pageSize), 16) + rows.push(...page.filter((value): value is T => value !== null && value !== undefined)) + } + return rows + }) + for (let start = 0; start < rows.length; start += pageSize) yield rows.slice(start, start + pageSize) + return + } + const keys = await this.rawListKeys(scope) + for (let start = 0; start < keys.length; start += pageSize) { + const values = await this.rawGetMany(scope, keys.slice(start, start + pageSize), 16) + let page = values.filter((value): value is T => value !== null && value !== undefined) + if (this.backend === 'redis') page = orderLikeInsertion(page) + if (page.length) yield page + } + } + + async listPage( + scope: string, + opts: { limit: number; cursor?: string; order?: 'desc' | 'asc' }, + ): Promise<{ items: T[]; nextCursor?: string; total: number }> { + const limit = Math.max(1, Math.floor(opts.limit)) + const direction = opts.order === 'asc' ? 1 : -1 + scope = await resolveGraphScope(this, scope) + await ensureGraphConsistency(this, scope) + return this.withSummaryRead(scope, async () => { + const keys = await this.rawListKeys(scope) + const ranked = keys.map((key) => ({ key, at: generatedIdTime(key) })) + const rank = (entry: { key: string; at: number | null }) => + entry.at === null ? Number.NEGATIVE_INFINITY : entry.at + const compare = (a: { key: string; at: number | null }, b: { key: string; at: number | null }) => { + if (rank(a) !== rank(b)) return (rank(a) - rank(b)) * direction + return (a.key < b.key ? -1 : a.key > b.key ? 1 : 0) * direction + } + ranked.sort(compare) + const position = decodeCursor(opts.cursor) + let start = 0 + if (position?.key) { + const at = generatedIdTime(position.key) + start = ranked.findIndex((entry) => compare(entry, { key: position.key!, at }) > 0) + if (start < 0) start = ranked.length + } + const pageKeys = ranked.slice(start, start + limit).map((entry) => entry.key) + const values = await this.rawGetMany(scope, pageKeys, 16) + const items = values.filter((value): value is T => value !== null && value !== undefined) + const hasMore = start + pageKeys.length < ranked.length + return { + items, + total: ranked.length, + ...(hasMore && pageKeys.length + ? { nextCursor: encodeCursor({ key: pageKeys[pageKeys.length - 1]! }) } + : {}), + } + }) } async listGroups(): Promise { diff --git a/src/state/list-query.ts b/src/state/list-query.ts index 0f01596339..6c55f1d863 100644 --- a/src/state/list-query.ts +++ b/src/state/list-query.ts @@ -19,6 +19,43 @@ function firstParam(params: QueryParams, name: string): string | undefined { return trimmed.length > 0 ? trimmed : undefined; } +const TIME_PARAMS = ["startedAfter", "startedBefore", "endedAfter", "endedBefore"] as const; +export type TimeParam = (typeof TIME_PARAMS)[number]; + +const ISO_TIMESTAMP = /^\d{4}-\d{2}-\d{2}(?:[T ]\d{2}:\d{2}(?::\d{2}(?:\.\d+)?)?(?:Z|[+-]\d{2}:?\d{2}| \d{2}:?\d{2})?)?$/; + +function parseIso(raw: string): number { + if (!ISO_TIMESTAMP.test(raw)) return Number.NaN; + const [year, month, day] = raw.slice(0, 10).split("-").map(Number); + if (new Date(Date.UTC(year, month - 1, day)).getUTCDate() !== day) return Number.NaN; + const hasTime = raw.length > 10; + const dated = raw.replace(/^(\d{4}-\d{2}-\d{2}) /, "$1T"); + const hasZone = /(?:Z|[+-]\d{2}:?\d{2}| \d{2}:?\d{2})$/.test(dated) && hasTime; + let normalized = dated.replace(/ (\d{2}:?\d{2})$/, "+$1"); + if (hasTime && !hasZone) normalized += "Z"; + return Date.parse(normalized); +} + +export function parseTimeBounds( + params: QueryParams, +): { bounds: Partial> } | { error: string } { + const bounds: Partial> = {}; + for (const name of TIME_PARAMS) { + if (params?.[name] === undefined) continue; + const ms = parseIso(firstParam(params, name) ?? ""); + if (Number.isNaN(ms)) return { error: `${name} must be an ISO 8601 timestamp` }; + bounds[name] = ms; + } + return { bounds }; +} + +export function inTimeRange(value: string | undefined, after?: number, before?: number): boolean { + if (after === undefined && before === undefined) return true; + const ms = value ? Date.parse(value) : Number.NaN; + if (Number.isNaN(ms)) return false; + return (after === undefined || ms >= after) && (before === undefined || ms < before); +} + export function parseListQuery(params: QueryParams): ListQuery { const rawLimit = firstParam(params, "limit"); const parsedLimit = rawLimit === undefined ? Number.NaN : Number(rawLimit); diff --git a/src/state/schema.ts b/src/state/schema.ts index 33076b78d9..2a7d4ee3f4 100644 --- a/src/state/schema.ts +++ b/src/state/schema.ts @@ -12,6 +12,7 @@ export const KV = { health: "mem:health", embeddings: (obsId: string) => `mem:emb:${obsId}`, bm25Index: "mem:index:bm25", + vectorRetentionReceipts: "mem:index:vec-retained:v1", vectorPendingLog: "mem:index:vec-pending", embeddingRetry: "mem:embedding-retry:v1", embeddingCommitted: "mem:embedding-committed:v1", diff --git a/src/state/state-readiness.ts b/src/state/state-readiness.ts index c281a5ea42..32aa9ced4a 100644 --- a/src/state/state-readiness.ts +++ b/src/state/state-readiness.ts @@ -16,6 +16,7 @@ export type StateReadiness = { type TriggerClient = Pick; type WaitOptions = { + requireSetAck?: boolean; scope?: string; timeoutMs?: number; intervalMs?: number; @@ -37,7 +38,7 @@ function isRetryableProbeError(error: unknown): boolean { ) { return true; } - return /Function state::(?:get|list) not found|timed out|timeout|ETIMEDOUT/i.test( + return /Function state::(?:get|list|list_keys) not found|timed out|timeout|ETIMEDOUT/i.test( errorMessage(error), ); } @@ -72,6 +73,15 @@ export async function waitForStateReady( payload: { scope, key: READINESS_PROBE_KEY }, timeoutMs: probeTimeoutMs(), }); + await sdk.trigger({function_id:"state::list_keys",payload:{scope},timeoutMs:probeTimeoutMs()}); + if (options.requireSetAck) { + const info = await sdk.trigger<{ function_id: string }, { function_id?: string } | null>({ + function_id: "engine::functions::info", + payload: { function_id: "state::set_ack" }, + timeoutMs: probeTimeoutMs(), + }); + if (info?.function_id !== "state::set_ack") throw new Error("STATE_SET_ACK_REQUIRED"); + } return { ready: true, waitedMs: Math.round(now() - startedAt), attempts }; } catch (error) { lastError = errorMessage(error); diff --git a/src/state/vector-checkpoint-reconciliation.ts b/src/state/vector-checkpoint-reconciliation.ts new file mode 100644 index 0000000000..d6fc411b01 --- /dev/null +++ b/src/state/vector-checkpoint-reconciliation.ts @@ -0,0 +1,120 @@ +import { createHash } from 'node:crypto'; +import { isDeepStrictEqual } from 'node:util'; +import { createStateDurabilityFence } from '../ops/state-durability.mjs'; +import { verifyVectorCheckpointFiles, verifyRemainingVectorPending, verifyVectorRetentionReceipts } from '../ops/vector-checkpoint-proof.mjs'; +import type { VectorRetentionReceipt } from '../types.js'; +import type { StateKV } from './kv.js'; +import { KV } from './schema.js'; + +export type RetentionReceipt = VectorRetentionReceipt; +export type RecoveryUpsert = { id: string; q: number; s: string; e: string; k?: 'memory' | 'observation' }; +export type RecoverySelection = { + pendingRows: RecoveryUpsert[]; + meta: Record | null; + buckets: Array<{ bucket: number; ids: string[] }>; +}; + +export function recoveryUpserts(values: unknown, dimensions: number): RecoveryUpsert[] { + if (!Array.isArray(values) || values.length > 16) throw Error('vector_recovery_selection_invalid'); + const ids = new Set(), sequences = new Set(); + for (const row of values) { + if (!row || typeof row !== 'object' || Object.keys(row).some(key => !['id', 'q', 's', 'e', 'k'].includes(key)) || + !Number.isSafeInteger(row.q) || row.q <= 0 || row.q === Number.MAX_SAFE_INTEGER || sequences.has(row.q) || + typeof row.id !== 'string' || !row.id || row.id === '~clear' || row.id.length > 512 || ids.has(row.id) || + typeof row.s !== 'string' || !row.s || row.s.length > 512 || + (row.k !== undefined && row.k !== (row.id.startsWith('mem_') ? 'memory' : 'observation')) || + !Number.isSafeInteger(dimensions) || dimensions < 1 || dimensions > 8192 || + typeof row.e !== 'string' || row.e.length !== 4 * Math.ceil(dimensions * 4 / 3)) + throw Error('vector_recovery_selection_invalid'); + const bytes = Buffer.from(row.e, 'base64'); + if (bytes.length !== dimensions * 4 || bytes.toString('base64') !== row.e) throw Error('vector_recovery_embedding_invalid'); + for (let offset = 0; offset < bytes.length; offset += 4) { + if (!Number.isFinite(bytes.readFloatLE(offset))) throw Error('vector_recovery_embedding_invalid'); + } + ids.add(row.id); + sequences.add(row.q); + } + return [...values].sort((a, b) => a.id < b.id ? -1 : a.id > b.id ? 1 : 0); +} + +export const bucketScope = (bucket: number) => KV.bm25Index + ':vec:' + String(bucket).padStart(4, '0'); +export const RETAINED_SCOPE = KV.vectorRetentionReceipts; + +export function receiptFor(row: RecoveryUpsert, bucket: number): RetentionReceipt { + const digest = createHash('sha256').update(JSON.stringify([row.id, row.q, row.s, row.e, row.k ?? null, bucket])).digest('hex'); + return { v: 1, id: row.id, q: row.q, bucket, digest }; +} + +export async function readRetention(kv: StateKV, storePath: string, admitted: () => boolean): Promise { + const values = await kv.list(RETAINED_SCOPE); + if (!Array.isArray(values) || values.length > 16 || values.some(row => !row || typeof row.digest !== 'string')) + throw Error('vector_retention_receipts_invalid'); + const receipts = Object.fromEntries(values.map(row => [row.digest, row])); + if (Object.keys(receipts).length !== values.length) throw Error('vector_retention_receipts_invalid'); + await verifyVectorRetentionReceipts({ storePath, receipts, admitted }); + return values; +} + +export async function reconcileCheckpoint( + kv: StateKV, storePath: string, selection: RecoverySelection, existing: RetentionReceipt[], + authorize: boolean, admitted: () => boolean, waitForCaptureMeta = false, +): Promise { + const { pendingRows, buckets, meta } = selection; + function check() { + if (!admitted()) throw Error('vector_recovery_epoch_changed'); + } + async function confirmPending() { + check(); + const current = await kv.list(KV.vectorPendingLog); + if (!Array.isArray(current) || current.length !== pendingRows.length || + !pendingRows.every(row => current.some(value => isDeepStrictEqual(value, row)))) throw Error('vector_recovery_pending_changed'); + check(); + } + + await confirmPending(); + if (!pendingRows.length) { + if (existing.length) throw Error('vector_retention_original_missing'); + await verifyRemainingVectorPending({ storePath, pendingRows, admitted }); + return []; + } + if (!meta || !isDeepStrictEqual(await kv.get(KV.bm25Index, 'vectors:meta'), meta)) throw Error('vector_recovery_meta_changed'); + const expected: RetentionReceipt[] = []; + for (const {bucket, ids} of buckets) { + for (const id of ids) { + check(); + const row = pendingRows.find(item => item.id === id); + if (!row || !isDeepStrictEqual(await kv.get(KV.vectorPendingLog, id), row) || + !isDeepStrictEqual(await kv.get(bucketScope(bucket), id), {id, s: row.s, e: row.e})) + throw Error('vector_recovery_checkpoint_changed'); + expected.push(receiptFor(row, bucket)); + } + } + if (existing.some(stamp => !expected.some(wanted => isDeepStrictEqual(stamp, wanted)))) + throw Error('vector_retention_original_changed'); + const missing = expected.filter(wanted => !existing.some(stamp => isDeepStrictEqual(stamp, wanted))); + if (missing.length && !authorize) throw Error('vector_retention_manual_verification_required'); + const recheck = await verifyVectorCheckpointFiles({storePath, pendingRows, meta, buckets, admitted, waitForCaptureMeta}); + await recheck(); + + // iii has no conditional delete. Preserve the entire WAL. Immutable receipt keys make + // an interrupted batch retryable without replaying or rewriting even one vector. + for (const stamp of missing) { + check(); + await kv.set(RETAINED_SCOPE, stamp.digest, stamp); + } + if (missing.length) { + const disk = createStateDurabilityFence({storePath, completion: 'disk'}); + const keys = missing.map(stamp => ({scope: RETAINED_SCOPE, key: stamp.digest})); + const result = await disk.waitForKeys(kv, keys, {expectedValues: missing}); + if (result.synced !== true || result.keys !== keys.length || result.scopes !== 1 || result.mutationFailures !== 0) + throw Error('vector_retention_disk_sync_unconfirmed'); + } + const saved = await readRetention(kv, storePath, admitted); + if (saved.length !== expected.length || !expected.every(stamp => saved.some(actual => isDeepStrictEqual(stamp, actual)))) + throw Error('vector_retention_receipts_changed'); + await confirmPending(); + if (waitForCaptureMeta && !isDeepStrictEqual(await kv.get(KV.bm25Index, 'vectors:meta'), meta)) + throw Error('vector_recovery_meta_changed'); + await recheck(); + return saved; +} diff --git a/src/state/vector-index.ts b/src/state/vector-index.ts index b31459655e..ff3e671bcb 100644 --- a/src/state/vector-index.ts +++ b/src/state/vector-index.ts @@ -37,12 +37,16 @@ export class VectorIndex { private vectors: Map = new Map(); private changes: Map = new Map(); private listener: VectorChangeListener | null = null; + private mutationGuard: ((id: string | null) => void) | null = null; + + setMutationGuard(guard: ((id: string | null) => void) | null): void { this.mutationGuard = guard; } setChangeListener(listener: VectorChangeListener | null): void { this.listener = listener; } add(obsId: string, sessionId: string, embedding: Float32Array): void { + this.mutationGuard?.(obsId); const entry = { embedding, sessionId }; this.vectors.set(obsId, entry); this.changes.set(obsId, true); @@ -50,6 +54,7 @@ export class VectorIndex { } remove(obsId: string): void { + this.mutationGuard?.(obsId); if (this.vectors.delete(obsId)) { this.changes.set(obsId, false); this.listener?.(obsId, null); @@ -89,10 +94,12 @@ export class VectorIndex { } markRemoved(obsId: string): void { + this.mutationGuard?.(obsId); if (!this.vectors.has(obsId)) this.changes.set(obsId, false); } markAllChanged(): void { + this.mutationGuard?.(null); for (const obsId of this.vectors.keys()) this.changes.set(obsId, true); } @@ -157,6 +164,7 @@ export class VectorIndex { } clear(): void { + this.mutationGuard?.(null); const hadVectors = this.vectors.size > 0; for (const obsId of this.vectors.keys()) this.changes.set(obsId, false); this.vectors.clear(); diff --git a/src/state/vector-revision-release.ts b/src/state/vector-revision-release.ts new file mode 100644 index 0000000000..6652ed3834 --- /dev/null +++ b/src/state/vector-revision-release.ts @@ -0,0 +1,15 @@ +export type RetainedVectorRevision = {id:string;revision:{release():void}}; +// Never lose ownership when a superseded watch cannot be released. The caller +// must not promote/rebuild over a partially released selection. +export function releaseSupersededVectorJobs(jobs: Array<{id:string;revision?:{release():void}}>, retained: RetainedVectorRevision[]): boolean { + let complete = true; + for(const job of jobs){ + if(!job.revision)continue; + try{job.revision.release();} + catch{complete=false;retained.push({id:job.id,revision:job.revision});} + } + return complete; +} +export function retryRetainedVectorRevisions(retained:RetainedVectorRevision[]):void { + while(retained.length){retained[0]!.revision.release();retained.shift();} +} diff --git a/src/triggers/api.ts b/src/triggers/api.ts index ac34df4765..e537655cf6 100644 --- a/src/triggers/api.ts +++ b/src/triggers/api.ts @@ -1,3 +1,5 @@ +import { insightHttpResponse } from "../state/insight-response.js"; +import {validUncertainInspection} from "../capture/uncertain-inspection.js"; import {liveResourceSample} from '../health/live-resource.js'; import {createSessionEndDiagnostics} from "../ops/session-end-diagnostics.js"; import {finishLongSession} from "../ops/long-session.mjs"; @@ -6,20 +8,21 @@ import { InvocationError, TriggerAction, type IIIClient } from "iii-sdk"; import { DEDUP_WINDOW_MS, getDedupSkippedCount } from "../functions/dedup.js"; import type { HttpRequest } from "@iii-dev/helpers/http"; import { randomBytes } from "node:crypto"; +import { isDeepStrictEqual } from "node:util"; import type { Session, CompressedObservation, HookPayload, CommitLink, SessionSummary, HealthSnapshot, AuditQueryResult, AuditMigrationState, Lesson } from "../types.js"; import { AUDIT_MIGRATION_STATE_KEY } from "../functions/audit.js"; import { withKeyedLock } from "../state/keyed-mutex.js"; import { KV } from "../state/schema.js"; import { isSearchLayer, type SearchLayer } from "../state/search-layer.js"; -import { checkPayloadFrameSize } from "../state/frame-guard.js"; -import { StateKV } from "../state/kv.js"; +import { checkPayloadFrameSize, oversizedPayloadError, payloadByteLength, SAFE_PAYLOAD_BYTES } from "../state/frame-guard.js"; +import { StateKV, collectAll } from "../state/kv.js"; import { pinGraphState } from "../state/graph-generation.js"; import { addSessionToProjectIndex } from "../state/session-index.js"; import { getLatestHealth } from "../health/monitor.js"; import type { MetricsStore } from "../eval/metrics-store.js"; import type { ResilientProvider } from "../providers/resilient.js"; import { III_PINNED_VERSION, VERSION } from "../version.js"; -import { CONSOLIDATION_COUNTS_REUSE_MS, CONSOLIDATION_LAST_RUN_KEY, PROCEDURAL_MIN_SESSIONS_PER_PATTERN, describeConsolidation, type ConsolidationRunRecord, type ConsolidationStatus } from "../functions/consolidation-status.js"; +import { CONSOLIDATION_COUNTS_REUSE_MS, CONSOLIDATION_LAST_RUN_KEY, PROCEDURAL_MIN_SESSIONS_PER_PATTERN, describeConsolidation, patternFrequency, type ConsolidationRunRecord, type ConsolidationStatus } from "../functions/consolidation-status.js"; import { UNINDEXED_SCAN_REUSE_MS, evaluateStatus, prefersHtml, renderStatusHtml, singleFlight, type GraphStatsInput, type StatusReport } from "../functions/status.js"; import { countUnindexedObservations, @@ -41,6 +44,8 @@ import { pageAfterCursor, pageByOffset, parseListQuery, + parseTimeBounds, + inTimeRange, sortByKeyDesc, } from "../state/list-query.js"; import { checkRequestGuard, configuredAllowedOrigins, timingSafeCompare } from "../auth.js"; @@ -67,7 +72,7 @@ import { isValidEventId } from "../capture/event-id.js"; import { getCaptureController, type CaptureResult } from "../functions/capture.js"; import { withoutObservationSource } from "../functions/observation-source.js"; import { clearSessionFinalize, scheduleSessionFinalize } from "../functions/session-finalize.js"; -import { requestSummaryRefresh } from "../functions/summary-state.js"; +import { requestSummaryRefresh, summaryCovers, summaryObservations } from "../functions/summary-state.js"; import { registerCompressionApi } from "./compression.js"; import { getFinalizeIdleMs, @@ -99,6 +104,85 @@ function parseOptionalInt(raw: unknown): number | undefined { return Number.isFinite(n) ? n : undefined; } +function createSessionSummaryCoverage(kv: StateKV) { + let busy = false; + return async (sessions: Session[]): Promise> => { + const reader = kv.observationReader; + const durability = kv.durability; + if (!reader || !durability || busy) return new Map(); + const candidates = sessions.filter(session => !("summaryProgress" in session) && + Number.isSafeInteger(session.observationCount) && session.observationCount > 0 && + session.observationCount <= 16).slice(0, 8); + if (!candidates.length) return new Map(); + + const controller = new AbortController(); + const { signal } = controller; + const expiresAt = performance.now() + 1250; + const releases: Array<() => void> = []; + busy = true; + const work = (async () => { + const verified: Array<{ session: Session; fresh: () => boolean }> = []; + for (const session of candidates) { + if (signal.aborted) break; + try { + const guard = durability.watchKeys([ + { scope: KV.sessions, key: session.id }, { scope: KV.summaries, key: session.id }, + ]); + releases.push(() => guard.release()); + if (!guard.unchanged()) continue; + const current = await kv.get(KV.sessions, session.id); + signal.throwIfAborted(); + if (!isDeepStrictEqual(current, session)) continue; + const summary = await kv.get(KV.summaries, session.id); + signal.throwIfAborted(); + if (summary?.sessionId !== session.id || !summary.coveredFingerprint || + summary.observationCount !== session.observationCount) continue; + + const scope = KV.observations(session.id); + const mutation = reader.mutationRevision(scope); + const fresh = () => !signal.aborted && performance.now() < expiresAt && guard.unchanged() && + mutation === reader.mutationRevision(scope) && !reader.pendingMutations(scope); + if (!fresh()) continue; + const revision = await reader.revision(scope); + signal.throwIfAborted(); + const view = await reader.work(scope, { kind: "summary", signal }); + if (view.rows.length !== session.observationCount) continue; + const rows: CompressedObservation[] = []; + let bytes = 0; + for (const descriptor of view.rows) { + signal.throwIfAborted(); + const row = await view.load(descriptor); + if (!row || row.sessionId !== session.id || typeof row.title !== "string") break; + bytes += Buffer.byteLength(JSON.stringify(row)); + if (bytes > 2 * 1024 * 1024) break; + rows.push(row); + } + if (rows.length !== view.rows.length) continue; + const observations = summaryObservations(rows); + if (observations.length !== rows.length || !summaryCovers(session, observations, summary)) continue; + if (revision !== await reader.revision(scope) || !fresh()) continue; + view.validate(); + verified.push({ session, fresh: () => { view.validate(); return fresh(); } }); + } catch { /* A missing or unstable source leaves coverage unknown. */ } + } + const coverage = new Map(); + for (const { session, fresh } of verified) { + try { if (fresh()) coverage.set(session.id, session.observationCount); } catch { /* Writer changed. */ } + } + return coverage; + })().finally(() => { busy = false; }); + let timer: ReturnType | undefined; + const deadline = new Promise>(resolve => { + timer = setTimeout(() => { controller.abort(); resolve(new Map()); }, 1250); + }); + try { return await Promise.race([work, deadline]); } + finally { + clearTimeout(timer); + for (const release of releases) release(); + } + }; +} + let restOriginPorts: () => Array = () => []; export function setRestOriginPorts(resolve: () => Array): void { @@ -399,8 +483,12 @@ export interface StatusReporterDeps { provider?: ResilientProvider | { circuitState?: unknown }; } +function isVectorRecoveryPaused(result: unknown): boolean { + return typeof result === "object" && result !== null && "error" in result && (result.error === "vector_recovery_paused" || result.error === "vector_recovery_interrupted"); +} + export function createStatusReporter(sdk: IIIClient, kv: StateKV, deps: StatusReporterDeps) { - return async function statusReport(options: { health?: HealthSnapshot | null; scanMaxAgeMs?: number; scanUnindexed?: boolean; functionMetrics?:ReturnType } = {}): Promise { + return async function statusReport(options: { health?: HealthSnapshot | null; scanMaxAgeMs?: number; scanUnindexed?: boolean; scanCaptureInbox?: boolean; functionMetrics?:ReturnType } = {}): Promise { const idx = getSearchIndex(); const keywordRebuildRunning = isKeywordRebuildInProgress(); const scan = unindexedScanFor(kv); @@ -420,7 +508,7 @@ export function createStatusReporter(sdk: IIIClient, kv: StateKV, deps: StatusRe STATUS_CHECK_TIMEOUT_MS, ), probeStateStore(kv), - valueWithin(getCaptureController()?.status({sdkProofPage:{offset:0,limit:32}}) ?? Promise.resolve(null), STATUS_CHECK_TIMEOUT_MS), + valueWithin(getCaptureController()?.status({scanInbox:options.scanCaptureInbox,sdkProofPage:{offset:0,limit:32}}) ?? Promise.resolve(null), STATUS_CHECK_TIMEOUT_MS), valueWithin(getGraphIndexMigrationStatus(kv), STATUS_CHECK_TIMEOUT_MS), ]); const observationsIndexed = [...idx.observationCountsBySession().values()].reduce((a, n) => a + n, 0); @@ -491,10 +579,10 @@ export function createConsolidationStatusReader(kv: StateKV) { const counts = singleFlight(async () => { const [summaries, memories, semantic, procedural, relations] = await Promise.all([ kv.countSummaries(), - kv.list(KV.memories).catch(() => []), - kv.list(KV.semantic).catch(() => []), - kv.list(KV.procedural).catch(() => []), - kv.list(KV.relations).catch(() => []), + collectAll(kv, KV.memories).catch(() => []), + collectAll(kv, KV.semantic).catch(() => []), + collectAll(kv, KV.procedural).catch(() => []), + collectAll(kv, KV.relations).catch(() => []), ]); return { summaries, @@ -502,7 +590,7 @@ export function createConsolidationStatusReader(kv: StateKV) { (m) => m.isLatest && m.type === "pattern" && - (m.sessionIds?.length ?? 0) >= PROCEDURAL_MIN_SESSIONS_PER_PATTERN, + patternFrequency(m) >= PROCEDURAL_MIN_SESSIONS_PER_PATTERN, ).length, semanticFacts: semantic.length, procedures: procedural.length, @@ -540,6 +628,47 @@ function statusViewerUrl(req: HttpRequest, viewerPort: number | null): string { return "/agentmemory/viewer#health"; } +const RECORD_PAGE_BYTES = SAFE_PAYLOAD_BYTES - 1024 * 1024; + +async function listRecordPage( + kv: StateKV, + scope: string, + params: HttpRequest["query_params"], +): Promise<{ items: T[]; nextCursor?: string }> { + const query = parseListQuery(params); + const limit = query.limit ?? LIST_PAGE_MAX; + let items: T[]; + let more: boolean; + let storageCursor: string | undefined; + if (typeof kv.listPage === "function") { + const page = await kv.listPage(scope, { limit, cursor: query.cursor }); + items = page.items; + more = page.nextCursor !== undefined; + storageCursor = page.nextCursor; + } else { + const sorted = sortByKeyDesc(await collectAll(kv, scope), (item) => item.id, (item) => item.id); + const page = pageAfterCursor(sorted, (item) => item.id, (item) => item.id, query.cursor, limit); + items = page.page; + more = page.nextCursor !== null; + } + let bytes = 0; + let kept = 0; + while (kept < items.length) { + const size = payloadByteLength(items[kept]) + 1; + if (kept > 0 && bytes + size > RECORD_PAGE_BYTES) break; + bytes += size; + kept++; + } + if (kept < items.length) { + items = items.slice(0, kept); + more = true; + storageCursor = undefined; + } + const last = items[items.length - 1]; + const nextCursor = storageCursor ?? (more && last ? encodeCursor({ key: last.id, id: last.id }) : undefined); + return { items, ...(nextCursor ? { nextCursor } : {}) }; +} + export function registerApiTriggers( sdk: IIIClient, kv: StateKV, @@ -635,7 +764,8 @@ export function registerApiTriggers( const ops = await operationalHealth(); const hostLevel=ops.memoryPressure?.level; const backgroundStatus=ops.backgroundControl?.status; - const status = hostLevel==="critical"?"critical":hostLevel==="high"&&health?.status!=="critical"?"degraded":(backgroundStatus==="error"||ops.captureProgress?.stalled)&&health?.status!=="critical"?"degraded":health?.status||"healthy"; + const captureUnknown=ops.captureProgress?.pending===null; + const status = hostLevel==="critical"?"critical":hostLevel==="high"&&health?.status!=="critical"?"degraded":(backgroundStatus==="error"||ops.captureProgress?.stalled||captureUnknown)&&health?.status!=="critical"?"degraded":health?.status||"healthy"; const statusCode = status === "critical" ? 503 : 200; return { @@ -823,6 +953,14 @@ export function registerApiTriggers( catch{return{status_code:503,headers,body:{completed:false,code:'SDK_RECONCILIATION_UNAVAILABLE'}};} }); sdk.registerTrigger({type:'http',function_id:'api::capture-reconcile-sdk',config:{api_path:'/agentmemory/capture/reconcile-sdk',http_method:'POST'}}); + sdk.registerFunction('api::capture-inspect-uncertain',async(req:HttpRequest):Promise=>{ + const headers={'Cache-Control':'no-store'};if(!secret)return{status_code:503,headers,body:{error:'UNCERTAIN_INSPECTION_AUTH_REQUIRED'}};const denied=checkAuth(req,secret);if(denied)return{...denied,headers}; + const q=req.body;if(!validUncertainInspection(q))return{status_code:400,headers,body:{error:'INVALID_UNCERTAIN_INSPECTION'}}; + try{const value:any=await sdk.trigger({function_id:'mem::capture-inspect-uncertain',payload:q});return{status_code:['source_conflict','observation_conflict','changed_during_inspection'].includes(value?.outcome)?409:200,headers,body:value};} + catch{return{status_code:503,headers,body:{error:'UNCERTAIN_INSPECTION_UNAVAILABLE'}};} + }); + sdk.registerTrigger({type:'http',function_id:'api::capture-inspect-uncertain',config:{api_path:'/agentmemory/capture/inspect-uncertain',http_method:'POST'}}); + sdk.registerFunction('api::capture-reconcile-sdk-status',async(req:HttpRequest):Promise=>{ const headers={'Cache-Control':'no-store'},denied=checkAuth(req,secret);if(denied)return{...denied,headers}; const q=req.query_params??{}; @@ -1226,7 +1364,7 @@ export function registerApiTriggers( function_id: "mem::replay::import-jsonl", payload, }); - return { status_code: 202, body: result }; + return { status_code: isVectorRecoveryPaused(result) ? 503 : 202, body: result }; }, ); sdk.registerTrigger({ @@ -1571,6 +1709,7 @@ export function registerApiTriggers( }, }); + const sessionSummaryCoverage = createSessionSummaryCoverage(kv); sdk.registerFunction("api::sessions", async (req: HttpRequest): Promise => { const authErr = checkAuth(req, secret); @@ -1595,6 +1734,9 @@ export function registerApiTriggers( : explicitAgentId ?? (isAgentScopeIsolated() ? getAgentId() : undefined); const listQuery = parseListQuery(req.query_params); + const time = parseTimeBounds(req.query_params); + if ("error" in time) return { status_code: 400, body: { error: time.error } }; + const { startedAfter, startedBefore, endedAfter, endedBefore } = time.bounds; let filtered = filterAgentId ? validSessions.filter((s) => s.agentId === filterAgentId) : validSessions; @@ -1608,6 +1750,11 @@ export function registerApiTriggers( : undefined; if (listQuery.project) filtered = filtered.filter((s) => s.project === listQuery.project); if (listQuery.status) filtered = filtered.filter((s) => s.status === listQuery.status); + filtered = filtered.filter( + (s) => + inTimeRange(s.startedAt, startedAfter, startedBefore) && + inTimeRange(s.endedAt, endedAfter, endedBefore), + ); if (listQuery.q) { filtered = filtered.filter((s) => matchesText(listQuery.q, s.id, s.project, s.cwd, s.firstPrompt, s.agentId), @@ -1640,9 +1787,17 @@ export function registerApiTriggers( ); summaries.push(...results); } + const metadataOnly = req.query_params?.["includeSummary"] === "false"; + const coverage = metadataOnly && req.query_params?.["includeSummaryCoverage"] === "true" + ? await sessionSummaryCoverage(filtered) : undefined; const withSummary = filtered.map((s, i) => { - if(req.query_params?.["includeSummary"]==="false") {const {summary:_,...metadata}=s;return metadata;} - return summaries[i] ? { ...s, summary: summaries[i] } : s; + const { summaryCoveredCount: _storedCount, ...session } = s as Session & { summaryCoveredCount?: unknown }; + if (metadataOnly) { + const { summary: _, ...metadata } = session; + const count = coverage?.get(s.id); + return count === undefined ? metadata : { ...metadata, summaryCoveredCount: count }; + } + return summaries[i] ? { ...session, summary: summaries[i] } : session; }); return { status_code: 200, @@ -1867,7 +2022,7 @@ export function registerApiTriggers( : {}), }, }); - return { status_code: 201, body: result }; + return { status_code: isVectorRecoveryPaused(result) ? 503 : 201, body: result }; }, ); sdk.registerTrigger({ @@ -1893,7 +2048,7 @@ export function registerApiTriggers( }; } const result = await sdk.trigger({ function_id: "mem::forget", payload: req.body }); - return { status_code: 200, body: result }; + return { status_code: isVectorRecoveryPaused(result) ? 503 : 200, body: result }; }, ); sdk.registerTrigger({ @@ -1989,7 +2144,7 @@ export function registerApiTriggers( const dryRun = req.query_params?.["dryRun"] === "true" || req.body?.dryRun === true; const result = await sdk.trigger({ function_id: "mem::evict", payload: { dryRun } }); - return { status_code: 200, body: result }; + return { status_code: isVectorRecoveryPaused(result) ? 503 : 200, body: result }; }, ); sdk.registerTrigger({ @@ -2220,7 +2375,7 @@ export function registerApiTriggers( return { status_code: 400, body: { error: "exportData is required" } }; } const result = await sdk.trigger({ function_id: "mem::import", payload: req.body }); - return { status_code: 200, body: result }; + return { status_code: isVectorRecoveryPaused(result) ? 503 : 200, body: result }; }, ); sdk.registerTrigger({ @@ -2272,6 +2427,7 @@ export function registerApiTriggers( function_id: "mem::evolve", payload: req.body, }); + if (isVectorRecoveryPaused(result)) return { status_code: 503, body: result }; if (result && result.success === false) { const status = result.code === "not_found" ? 404 : result.code === "not_latest" ? 409 : 400; @@ -2293,7 +2449,7 @@ export function registerApiTriggers( const dryRun = req.query_params?.["dryRun"] === "true" || req.body?.dryRun === true; const result = await sdk.trigger({ function_id: "mem::auto-forget", payload: { dryRun } }); - return { status_code: 200, body: result }; + return { status_code: isVectorRecoveryPaused(result) ? 503 : 200, body: result }; }, ); sdk.registerTrigger({ @@ -2775,7 +2931,7 @@ export function registerApiTriggers( }; } const result = await sdk.trigger({ function_id: "mem::governance-delete", payload: req.body }); - return { status_code: 200, body: result }; + return { status_code: isVectorRecoveryPaused(result) ? 503 : 200, body: result }; }, ); sdk.registerTrigger({ @@ -2801,7 +2957,7 @@ export function registerApiTriggers( const authErr = checkAuth(req, secret); if (authErr) return authErr; const result = await sdk.trigger({ function_id: "mem::governance-bulk", payload: req.body || {} }); - return { status_code: 200, body: result }; + return { status_code: isVectorRecoveryPaused(result) ? 503 : 200, body: result }; }, ); sdk.registerTrigger({ @@ -2867,7 +3023,7 @@ export function registerApiTriggers( } try { const result = await sdk.trigger({ function_id: "mem::snapshot-restore", payload: req.body }); - return { status_code: 200, body: result }; + return { status_code: isVectorRecoveryPaused(result) ? 503 : 200, body: result }; } catch { return { status_code: 404, body: { error: "Snapshots not enabled" } }; } @@ -2883,7 +3039,7 @@ export function registerApiTriggers( async (req: HttpRequest): Promise => { const authErr = checkAuth(req, secret); if (authErr) return authErr; - const memories = await kv.list(KV.memories); + const memories = await collectAll(kv, KV.memories); const latest = req.query_params?.["latest"] === "true"; // agentId filter. Request param wins, env AGENT_ID (when // scope=isolated) is the fallback. Shared mode keeps the tag but @@ -3008,18 +3164,18 @@ export function registerApiTriggers( } } - return { - status_code: 200, - body: { - memories: sliced, - total: filtered.length, - offset, - limit: limit ?? null, - nextCursor, - ...(searchMode ? { search: { query: listQuery.q, mode: searchMode } } : {}), - ...(facets ? { facets } : {}), - }, + const body = { + memories: sliced, + total: filtered.length, + offset, + limit: limit ?? null, + nextCursor, + ...(searchMode ? { search: { query: listQuery.q, mode: searchMode } } : {}), + ...(facets ? { facets } : {}), }; + const oversized = checkPayloadFrameSize(body, "use ?limit and ?cursor to page"); + if (oversized) return { status_code: 413, body: oversized }; + return { status_code: 200, body }; }, ); sdk.registerTrigger({ @@ -3053,8 +3209,11 @@ export function registerApiTriggers( async (req: HttpRequest): Promise => { const authErr = checkAuth(req, secret); if (authErr) return authErr; - const semantic = await kv.list(KV.semantic); - return { status_code: 200, body: { semantic } }; + const page = await listRecordPage(kv, KV.semantic, req.query_params); + const body = { semantic: page.items, ...(page.nextCursor ? { nextCursor: page.nextCursor } : {}) }; + const oversized = checkPayloadFrameSize(body, "use ?limit and ?cursor to page"); + if (oversized) return { status_code: 413, body: oversized }; + return { status_code: 200, body }; }, ); sdk.registerTrigger({ @@ -3067,8 +3226,11 @@ export function registerApiTriggers( async (req: HttpRequest): Promise => { const authErr = checkAuth(req, secret); if (authErr) return authErr; - const procedural = await kv.list(KV.procedural); - return { status_code: 200, body: { procedural } }; + const page = await listRecordPage(kv, KV.procedural, req.query_params); + const body = { procedural: page.items, ...(page.nextCursor ? { nextCursor: page.nextCursor } : {}) }; + const oversized = checkPayloadFrameSize(body, "use ?limit and ?cursor to page"); + if (oversized) return { status_code: 413, body: oversized }; + return { status_code: 200, body }; }, ); sdk.registerTrigger({ @@ -3096,8 +3258,11 @@ export function registerApiTriggers( async (req: HttpRequest): Promise => { const authErr = checkAuth(req, secret); if (authErr) return authErr; - const relations = await kv.list(KV.relations); - return { status_code: 200, body: { relations } }; + const relations = await collectAll(kv, KV.relations); + const body = { relations }; + const oversized = checkPayloadFrameSize(body, "export relations with /mesh/export, which pages"); + if (oversized) return { status_code: 413, body: oversized }; + return { status_code: 200, body }; }, ); sdk.registerTrigger({ @@ -3837,42 +4002,122 @@ export function registerApiTriggers( } } const project = req.query_params?.["project"] as string | undefined; + const rawLimit = Number(req.query_params?.["limit"]); + const pageLimit = Number.isFinite(rawLimit) && rawLimit >= 1 ? Math.min(Math.floor(rawLimit), 5000) : 500; + const cursorRaw = req.query_params?.["cursor"] as string | undefined; + const paged = req.query_params?.["limit"] !== undefined || cursorRaw !== undefined; const sinceTime = since ? new Date(since).getTime() : 0; const df = (items: T[], field: "updatedAt" | "createdAt") => items.filter((i) => new Date((i as Record)[field] as string).getTime() > sinceTime); - let memories = await kv.list(KV.memories); + let memories = await collectAll(kv, KV.memories); let actions = await kv.list(KV.actions); if (project) { memories = memories.filter((m) => m.project === project); actions = actions.filter((a) => a.project === project); } - const body: Record = { - memories: df(memories, "updatedAt"), - actions: df(actions, "updatedAt"), - }; + const collections: Array<[string, unknown[]]> = [ + ["memories", df(memories, "updatedAt")], + ["actions", df(actions, "updatedAt")], + ]; if (!project) { const graphState = await pinGraphState(kv); - const semantic = await kv.list(KV.semantic); - const procedural = await kv.list(KV.procedural); - const relations = await kv.list(KV.relations); - const graphNodes = await graphState.list(KV.graphNodes); - const graphEdges = await graphState.list(KV.graphEdges); - body.semantic = df(semantic, "updatedAt"); - body.procedural = df(procedural, "updatedAt"); - body.relations = df(relations, "createdAt"); - body.graphNodes = graphNodes.filter( - (n) => new Date(n.updatedAt || n.createdAt).getTime() > sinceTime, + const semantic = await collectAll(kv, KV.semantic); + const procedural = await collectAll(kv, KV.procedural); + const relations = await collectAll(kv, KV.relations); + const graphNodes = await collectAll(graphState, KV.graphNodes); + const graphEdges = await collectAll(graphState, KV.graphEdges); + collections.push( + ["semantic", df(semantic, "updatedAt")], + ["procedural", df(procedural, "updatedAt")], + ["relations", df(relations, "createdAt")], + ["graphNodes", graphNodes.filter( + (n) => new Date(n.updatedAt || n.createdAt).getTime() > sinceTime, + )], + ["graphEdges", df(graphEdges, "createdAt")], ); - body.graphEdges = df(graphEdges, "createdAt"); } - // Fail an over-frame export with 413 instead of dropping the worker. - const oversized = checkPayloadFrameSize( - body, - "use ?since to fetch only changes after a timestamp, or ?project to scope the export", - ); - if (oversized) { - return { status_code: 413, body: oversized }; + const rowId = (row: unknown) => { + const r = row as { id?: unknown; sourceId?: unknown; targetId?: unknown; type?: unknown }; + if (r.id !== undefined && r.id !== null) return String(r.id); + if (r.sourceId !== undefined) return `${String(r.sourceId)}:${String(r.targetId)}:${String(r.type)}`; + return ""; + }; + for (const [, rows] of collections) { + rows.sort((a, b) => (rowId(a) < rowId(b) ? -1 : rowId(a) > rowId(b) ? 1 : 0)); } + let cursorPos: { c: number; after?: string } | null = null; + if (cursorRaw) { + try { + const parsed = JSON.parse(Buffer.from(cursorRaw, "base64url").toString("utf8")); + if ( + parsed && typeof parsed === "object" && + Number.isInteger(parsed.c) && parsed.c >= 0 && parsed.c < collections.length && + (parsed.after === undefined || typeof parsed.after === "string") + ) { + cursorPos = { c: parsed.c, ...(parsed.after !== undefined ? { after: parsed.after } : {}) }; + } + } catch {} + if (!cursorPos) return { status_code: 400, body: { error: "invalid_cursor" } }; + } + for (const [name, rows] of collections) { + const unexportable = rows.some((row) => payloadByteLength(row) > SAFE_PAYLOAD_BYTES); + if (unexportable) { + return { status_code: 413, body: oversizedPayloadError(SAFE_PAYLOAD_BYTES + 1, `use ?since, ?project, or ?collections to exclude the oversized ${name} record`) }; + } + } + if (!paged) { + const full = Object.fromEntries(collections); + const oversized = checkPayloadFrameSize( + full, + "use ?limit and ?cursor to page, ?since to fetch only changes after a timestamp, or ?project to scope the export", + ); + if (oversized) return { status_code: 413, body: oversized }; + return { status_code: 200, body: full }; + } + let limit = pageLimit; + let body: Record = {}; + let nextCursor: string | undefined; + for (;;) { + body = Object.fromEntries(collections.map(([name]) => [name, []])); + nextCursor = undefined; + let remaining = limit; + let ci = cursorPos ? cursorPos.c : 0; + let after = cursorPos?.after; + for (; ci < collections.length && remaining > 0; ci++) { + const [name, rows] = collections[ci]!; + let start = 0; + if (after !== undefined) { + const resumeAfter = after; + start = rows.findIndex((row) => rowId(row) > resumeAfter); + if (start < 0) { after = undefined; continue; } + } + const take = Math.min(remaining, rows.length - start); + const slice = take > 0 ? rows.slice(start, start + take) : []; + body[name] = slice; + remaining -= take; + after = undefined; + if (start + take < rows.length) { + nextCursor = Buffer.from(JSON.stringify({ c: ci, after: rowId(slice[slice.length - 1]) })).toString("base64url"); + break; + } + } + if (!nextCursor && remaining === 0) { + for (let c = ci; c < collections.length; c++) { + if (collections[c]![1].length > 0) { + nextCursor = Buffer.from(JSON.stringify({ c })).toString("base64url"); + break; + } + } + } + const oversized = checkPayloadFrameSize( + body, + "use ?since to fetch only changes after a timestamp, ?cursor to page, or ?project to scope the export", + ); + if (!oversized) break; + if (limit <= 1) return { status_code: 413, body: oversized }; + limit = Math.max(1, Math.floor(limit / 2)); + } + if (nextCursor) body.nextCursor = nextCursor; return { status_code: 200, body }; }, ); @@ -4349,9 +4594,9 @@ export function registerApiTriggers( const body = (req.body as Record) || {}; const result = await sdk.trigger({ function_id: "mem::reflect", payload: { project: typeof body.project === "string" ? body.project : undefined, - maxClusters: typeof body.maxClusters === "number" ? body.maxClusters : undefined, + maxClusters: body.maxClusters, } }); - return { status_code: 200, body: result }; + return insightHttpResponse(result); }); sdk.registerTrigger({ type: "http", function_id: "api::reflect", config: { api_path: "/agentmemory/reflect", http_method: "POST" } }); @@ -4378,7 +4623,7 @@ export function registerApiTriggers( minConfidence, limit, } }); - return { status_code: 200, body: result }; + return insightHttpResponse(result); }); sdk.registerTrigger({ type: "http", function_id: "api::insight-list", config: { api_path: "/agentmemory/insights", http_method: "GET" } }); @@ -4390,10 +4635,10 @@ export function registerApiTriggers( const result = await sdk.trigger({ function_id: "mem::insight-search", payload: { query: body.query, project: typeof body.project === "string" ? body.project : undefined, - minConfidence: typeof body.minConfidence === "number" ? body.minConfidence : undefined, - limit: typeof body.limit === "number" ? body.limit : undefined, + minConfidence: body.minConfidence, + limit: body.limit, } }); - return { status_code: 200, body: result }; + return insightHttpResponse(result); }); sdk.registerTrigger({ type: "http", function_id: "api::insight-search", config: { api_path: "/agentmemory/insights/search", http_method: "POST" } }); } diff --git a/src/triggers/events.ts b/src/triggers/events.ts index 3de5286654..2571e687fc 100644 --- a/src/triggers/events.ts +++ b/src/triggers/events.ts @@ -16,6 +16,7 @@ import { isConsolidationEnabled, } from "../config.js"; import { logger } from "../logger.js"; +import { getSummaryTriggerTimeoutMs } from "../functions/summary-state.js"; import { noteMemoryChange, noteSessionChange } from "../state/viewer-counts.js"; // Global marker recording when corpus consolidation last ran, used to debounce @@ -121,7 +122,7 @@ export function registerEventTriggers(sdk: IIIClient, kv: StateKV): void { if(found)await operationalState.featureLifecycle.enqueue(data.sessionId); return {success:true,queued:found}; } - const summary = await sdk.trigger({ function_id: "mem::summarize", payload: data }); + const summary = await sdk.trigger({ function_id: "mem::summarize", payload: data, timeoutMs: getSummaryTriggerTimeoutMs() }); if (summary?.unchanged === true) return summary; const fireVoid = (function_id: string, payload: unknown) => sdk diff --git a/src/triggers/viewer-streams.ts b/src/triggers/viewer-streams.ts index 44419338f4..5084558cbc 100644 --- a/src/triggers/viewer-streams.ts +++ b/src/triggers/viewer-streams.ts @@ -96,7 +96,7 @@ export function registerViewerStreamTriggers( const [functionMetrics, counts, report] = await Promise.all([ metricsRead, getViewerCounts(kv), - statusReport({ health: snapshot, scanMaxAgeMs, scanUnindexed, functionMetrics:metricsRead }).catch(() => null), + statusReport({ health: snapshot, scanMaxAgeMs, scanUnindexed, scanCaptureInbox:false, functionMetrics:metricsRead }).catch(() => null), ]); return { status: snapshot?.status ?? "healthy", diff --git a/src/types.ts b/src/types.ts index 4b881ad6d4..f1ec9d3a88 100644 --- a/src/types.ts +++ b/src/types.ts @@ -150,10 +150,12 @@ export interface Memory { sessionIds: string[]; strength: number; version: number; + recurrence?: number; parentId?: string; supersedes?: string[]; relatedIds?: string[]; sourceObservationIds?: string[]; + consolidationKeys?: Record; isLatest: boolean; forgetAfter?: string; imageRef?: string; @@ -305,7 +307,7 @@ export interface HealthSnapshot { rss: number; external: number; }; - cpu: { userMicros: number; systemMicros: number; percent: number; capacityCores?: number|null; capacityPercent?: number|null }; + cpu: { userMicros: number; systemMicros: number; percent: number; cores?: number; capacityCores?: number|null; capacityPercent?: number|null }; eventLoopLagMs: number; uptimeSeconds: number; kvConnectivity?: { status: string; latencyMs?: number; error?: string }; @@ -419,7 +421,7 @@ export interface ExportCollectionPagination { } export interface ExportData { - version: "0.3.0" | "0.4.0" | "0.5.0" | "0.6.0" | "0.6.1" | "0.7.0" | "0.7.2" | "0.7.3" | "0.7.4" | "0.7.5" | "0.7.6" | "0.7.7" | "0.7.9" | "0.8.0" | "0.8.1" | "0.8.2" | "0.8.3" | "0.8.4" | "0.8.5" | "0.8.6" | "0.8.7" | "0.8.8" | "0.8.9" | "0.8.10" | "0.8.11" | "0.8.12" | "0.8.13" | "0.9.0" | "0.9.1" | "0.9.2" | "0.9.3" | "0.9.4" | "0.9.5" | "0.9.6" | "0.9.7" | "0.9.8" | "0.9.9" | "0.9.10" | "0.9.11" | "0.9.12" | "0.9.13" | "0.9.14" | "0.9.15" | "0.9.16" | "0.9.17" | "0.9.18" | "0.9.19" | "0.9.20" | "0.9.21" | "0.9.22" | "0.9.23" | "0.9.24" | "0.9.25" | "0.9.26" | "0.9.27" | "0.9.28" | "0.9.29" | "0.9.30" | "0.9.30-daniel.1" | "0.9.30-daniel.2" | "0.9.30-daniel.3" | "0.9.30-daniel.4" | "0.9.30-daniel.5" | "0.9.30-daniel.6" | "0.9.30-daniel.7" | "0.9.30-daniel.8" | "0.9.30-daniel.9" | "0.9.30-daniel.10" | "0.9.30-daniel.11" | "0.9.30-daniel.15" | "0.9.30-daniel.16" | "0.9.30-daniel.18" | "0.9.30-daniel.19" | "0.9.30-daniel.20" | "0.9.30-daniel.21" | "0.9.30-daniel.22" | "0.9.30-daniel.23" | "0.9.30-daniel.24" | "0.9.30-daniel.25" | "0.9.30-daniel.26" | "0.9.30-daniel.27" | "0.9.30-daniel.28" | "0.9.31" | "0.9.31-daniel.1" | "0.9.31-daniel.33" | "0.9.31-daniel.34" | "0.9.31-daniel.35" | "0.9.31-daniel.36" | "0.9.31-daniel.37" | "0.9.31-daniel.38" | "0.9.31-daniel.39" | "0.9.31-daniel.40" | "0.9.31-daniel.41" | "0.9.31-daniel.42" | "0.9.31-daniel.43" | "0.9.31-daniel.44" | "0.9.31-daniel.45" | "0.9.31-daniel.46" | "0.9.31-daniel.47" | "0.9.31-daniel.48" | "0.9.31-daniel.49" | "0.9.31-daniel.50" | "0.9.31-daniel.51" | "0.9.31-daniel.52" | "0.9.31-daniel.53" | "0.9.31-daniel.54" | "0.9.32-daniel.55" | "0.9.30-daniel.29" | "0.9.30-daniel.30" | "0.9.30-daniel.31" | "0.9.30-daniel.32"; + version: "0.3.0" | "0.4.0" | "0.5.0" | "0.6.0" | "0.6.1" | "0.7.0" | "0.7.2" | "0.7.3" | "0.7.4" | "0.7.5" | "0.7.6" | "0.7.7" | "0.7.9" | "0.8.0" | "0.8.1" | "0.8.2" | "0.8.3" | "0.8.4" | "0.8.5" | "0.8.6" | "0.8.7" | "0.8.8" | "0.8.9" | "0.8.10" | "0.8.11" | "0.8.12" | "0.8.13" | "0.9.0" | "0.9.1" | "0.9.2" | "0.9.3" | "0.9.4" | "0.9.5" | "0.9.6" | "0.9.7" | "0.9.8" | "0.9.9" | "0.9.10" | "0.9.11" | "0.9.12" | "0.9.13" | "0.9.14" | "0.9.15" | "0.9.16" | "0.9.17" | "0.9.18" | "0.9.19" | "0.9.20" | "0.9.21" | "0.9.22" | "0.9.23" | "0.9.24" | "0.9.25" | "0.9.26" | "0.9.27" | "0.9.28" | "0.9.29" | "0.9.30" | "0.9.30-daniel.1" | "0.9.30-daniel.2" | "0.9.30-daniel.3" | "0.9.30-daniel.4" | "0.9.30-daniel.5" | "0.9.30-daniel.6" | "0.9.30-daniel.7" | "0.9.30-daniel.8" | "0.9.30-daniel.9" | "0.9.30-daniel.10" | "0.9.30-daniel.11" | "0.9.30-daniel.15" | "0.9.30-daniel.16" | "0.9.30-daniel.18" | "0.9.30-daniel.19" | "0.9.30-daniel.20" | "0.9.30-daniel.21" | "0.9.30-daniel.22" | "0.9.30-daniel.23" | "0.9.30-daniel.24" | "0.9.30-daniel.25" | "0.9.30-daniel.26" | "0.9.30-daniel.27" | "0.9.30-daniel.28" | "0.9.31" | "0.9.31-daniel.1" | "0.9.31-daniel.33" | "0.9.31-daniel.34" | "0.9.31-daniel.35" | "0.9.31-daniel.36" | "0.9.31-daniel.37" | "0.9.31-daniel.38" | "0.9.31-daniel.39" | "0.9.31-daniel.40" | "0.9.31-daniel.41" | "0.9.31-daniel.42" | "0.9.31-daniel.43" | "0.9.31-daniel.44" | "0.9.31-daniel.45" | "0.9.31-daniel.46" | "0.9.31-daniel.47" | "0.9.31-daniel.48" | "0.9.31-daniel.49" | "0.9.31-daniel.50" | "0.9.31-daniel.51" | "0.9.31-daniel.52" | "0.9.31-daniel.53" | "0.9.31-daniel.54" | "0.9.32-daniel.55" | "0.9.32-daniel.57" | "0.9.32-daniel.58" | "0.9.32-daniel.59" | "0.9.32-daniel.60" | "0.9.32-daniel.61" | "0.9.32-daniel.62" | "0.9.32-daniel.63" | "0.9.32-daniel.64" | "0.9.32-daniel.65" | "0.9.32-daniel.66" | "0.9.32-daniel.67" | "0.9.32-daniel.68" | "0.9.33-daniel.69" | "0.9.33-daniel.70" | "0.9.33-daniel.71" | "0.9.33-daniel.72" | "0.9.33-daniel.73" | "0.9.30-daniel.29" | "0.9.30-daniel.30" | "0.9.30-daniel.31" | "0.9.30-daniel.32"; exportedAt: string; sessions: Session[]; observations: Record; @@ -568,7 +570,7 @@ export interface GraphIndexMembership { } export interface GraphRetrievalIndexControl { - schema: 2; + schema: 2 | 3; buildId: string; status: "building" | "ready" | "paused"; revision: number; @@ -665,6 +667,7 @@ export interface ProceduralMemory { frequency: number; sourceSessionIds: string[]; sourceObservationIds?: string[]; + consolidationKeys?: Record; tags?: string[]; concepts?: string[]; strength: number; @@ -1132,6 +1135,7 @@ export interface EmbeddingIntent { fingerprint?: string; phase?: "queued" | "embedded"; errorCode?: string; + retainedRecovery?: EmbeddingCommit['retainedRecovery']; } export interface EmbeddingCommit { id: string; @@ -1139,6 +1143,11 @@ export interface EmbeddingCommit { kind: EmbeddingIntent["kind"]; fingerprint: string; committedAt: string; + retainedRecovery?: { + version: 1; completion: 'sdk'; verified: false; synced: false; + previousIntent: Omit | null; + previousCommit: Omit | null; + }; } export interface CaptureSdkIndexEvidence { @@ -1170,3 +1179,6 @@ export type CaptureRetryJob = { id: string; all: boolean; eventId?: string; key? export interface CaptureRecoveryFailure {operation:string;stage:string;code:string;elapsedMs?:number;reader?:{readerPhase:string;readerFailurePhase?:string;readerTimings:Record;bytesRead?:number;processCpuMs?:number;eventLoopActiveMs?:number;eventLoopIdleMs?:number;complete?:boolean}} export interface CaptureRecoveryCheckpoint {version:1;kind:"item";key:string;id:string;sessionId:string;observationId:string;eventId:string;sourceDigest:string;projection:string;phase:"checking"|"verifying"|"resolved"|"blocked";reason:string;createdAt:string;updatedAt:string;verificationAttempts:number;nextAttemptAt:number;lastVerificationFailure?:CaptureRecoveryFailure} + +/** An immutable record of a disk-verified upsert whose WAL entry is retained. */ +export interface VectorRetentionReceipt { v:1; id:string; q:number; bucket:number; digest:string } diff --git a/src/version.ts b/src/version.ts index 9752b85c28..dee9c44668 100644 --- a/src/version.ts +++ b/src/version.ts @@ -1,2 +1,2 @@ -export const VERSION = "0.9.32-daniel.55"; -export const III_PINNED_VERSION = "0.25.0"; +export const VERSION = "0.9.33-daniel.73"; +export const III_PINNED_VERSION = "0.25.1"; diff --git a/src/viewer/capture-review-viewer.js b/src/viewer/capture-review-viewer.js index 77515ef4ae..7528931df3 100644 --- a/src/viewer/capture-review-viewer.js +++ b/src/viewer/capture-review-viewer.js @@ -87,7 +87,7 @@ async function load(cursor){try{ var page=await api('?review=true&status='+select.value+'&limit=25'+(cursor?'&cursor='+encodeURIComponent(cursor):''));rows=page.items||[];nextCursor=page.nextCursor||null; var cap=page.capture?.inbox;deadCount=cap?cap.dead:select.value==='dead'?page.total:deadCount; - summary.textContent='선택 상태 '+page.total+'건'+(cap?' · 수집 격리 '+cap.dead+'건 · 자동 재시도 '+cap.retrying+'건':'');var auto=page.capture?.automaticRecovery;automaticState=auto||null;automaticDescription.textContent=auto?('검증 자동 복구 '+(auto.enabled?'켜짐':'꺼짐')+' · '+(auto.admitted?'조건 충족':automaticReasons[auto.blockedReason]||(auto.blockedReason?'보류':'사용 가능 · 비활성'))+(auto.configurationStatus==='pending'?' · 설정 저장 중':auto.configurationStatus==='unknown'?' · 설정 저장 결과 미확정':'')+' · 원본이 일치하고 이미 게시된 항목만 복구합니다.'):'이 서버는 검증 자동 복구 제어를 제공하지 않습니다.';if(auto)summary.textContent+=' · 검증 복구 '+auto.resolved+'건 / 확인 중 '+auto.verifying+'건 / 점검 필요 '+auto.blocked+'건'+(auto.lastErrorCode?' · 자동 변경 보류':'')+(auto.breakerUntil>Date.now()?' · 반복 실패로 대기':'');render(); + summary.textContent='선택 상태 '+page.total+'건'+(cap?' · 수집 격리 '+cap.dead+'건 · 자동 재시도 '+cap.retrying+'건':'');var auto=page.capture?.automaticRecovery;automaticState=auto||null;automaticDescription.textContent=auto?('검증 자동 복구 '+(auto.enabled?'켜짐':'꺼짐')+' · '+(auto.admitted?'조건 충족':automaticReasons[auto.blockedReason]||(auto.blockedReason?'보류':'사용 가능 · 비활성'))+(auto.configurationStatus==='pending'?' · 설정 저장 중':auto.configurationStatus==='unknown'?' · 설정 저장 결과 미확정':'')+' · 원본이 일치하고 이미 게시된 항목만 복구합니다.'):'이 서버는 검증 자동 복구 제어를 제공하지 않습니다.';if(auto)summary.textContent+=' · 검증 복구 '+auto.resolved+'건 / 확인 중 '+auto.verifying+'건 / 점검 필요 '+auto.blocked+'건'+(auto.sdkReviewed>0?' · 기존 SDK 완료 기록 '+auto.sdkReviewed+'건':'')+(auto.lastErrorCode?' · 자동 변경 보류':'')+(auto.breakerUntil>Date.now()?' · 반복 실패로 대기':'');render(); }catch{announce('수집 목록 조회 실패 · 비어 있다고 판정하지 않습니다.');}finally{controls();}} async function toggleAutomaticRecovery(){ if(busy||!automaticState)return;var desired=!automaticWantsOff();busy=true;controls(); diff --git a/src/viewer/index.html b/src/viewer/index.html index 7836f82b61..87008bcb53 100644 --- a/src/viewer/index.html +++ b/src/viewer/index.html @@ -1742,8 +1742,8 @@

agentmemory

lesson: { path: 'lessons', key: 'lessons' }, action: { path: 'actions', key: 'actions', allPages: true }, crystal: { path: 'crystals', key: 'crystals' }, - semantic: { path: 'semantic', key: 'facts', fallbackKey: 'semantic' }, - procedural: { path: 'procedural', key: 'procedures', fallbackKey: 'procedural' }, + semantic: { path: 'semantic', key: 'facts', fallbackKey: 'semantic', allPages: true }, + procedural: { path: 'procedural', key: 'procedures', fallbackKey: 'procedural', allPages: true }, audit: { path: 'audit?limit=100', key: 'entries' } }; var bootGateOpen = null; @@ -2516,6 +2516,15 @@

agentmemory

return records; } + function summaryStageProgress(session) { + if (session.summaryProgress) return session.summaryProgress; + var count = session.summaryCoveredCount; + if (Number.isSafeInteger(count) && count > 0 && count === session.observationCount) { + return {status:'current',pendingCount:0,processedCount:count}; + } + return null; + } + function backgroundWorkSnapshot(sessions, health, available) { var totals = {summary:0,compress:0,graph:0,embedding:null,features:null}, knownTotals = {summary:0,compress:0,graph:0,embedding:null,features:null}; var unknownStageCounts = {summary:0,compress:0,graph:0}, unknownSessionsCount = 0, knownCounts = {summary:0,compress:0,graph:0}, rows = [], quarantined = 0; @@ -2525,7 +2534,7 @@

agentmemory

return empty ? 0 : null; } (sessions || []).forEach(function(s) { - var enrichment = s.enrichmentProgress || {}, progress = [s.summaryProgress,enrichment.compress,enrichment.graph]; + var enrichment = s.enrichmentProgress || {}, progress = [summaryStageProgress(s),enrichment.compress,enrichment.graph]; var empty = s.observationCount === 0 || s.observations === 0; var counts = {summary:pending(progress[0],empty),compress:pending(progress[1],empty),graph:pending(progress[2],empty)}; Object.keys(counts).forEach(function(k) { @@ -2646,7 +2655,7 @@

agentmemory

[['summary','요약'],['compress','압축'],['graph','그래프']].forEach(function(pair) { var counters = Object.create(null), statuses = [], complete = available, blocked = 0; (sessions || []).forEach(function(s) { - var p = pair[0] === 'summary' ? s.summaryProgress : (s.enrichmentProgress || {})[pair[0]]; + var p = pair[0] === 'summary' ? summaryStageProgress(s) : (s.enrichmentProgress || {})[pair[0]]; if (p && p.enabled === false || s.observationCount === 0 || s.observations === 0) return; if (!p || !Number.isSafeInteger(p.processedCount) || p.processedCount < 0 || typeof s.id !== 'string') { complete = false; return; } counters[s.id] = p.processedCount; statuses.push(p.status); @@ -2739,6 +2748,7 @@

agentmemory

if (!['running','pausing','resuming','paused','error'].includes(control.status)) html += '

백그라운드 실행 상태 미확인 · 현황을 다시 조회하세요.

'; if (work.quarantined) html += '

요약·후처리 격리 '+work.quarantined+'건 · 사용자 점검이 필요하며 자동 재처리하지 않습니다.

'; var inbox = ((store.report || {}).capture || {}).inbox; + if (!inbox) html += '

수집 inbox를 자동 조회하지 않아 격리·재시도·마지막 오류·SDK 복원 및 수집 정체 상태를 확인하지 못했습니다. 수집 격리·복구 이력의 수동 조회도 오래 걸리거나 실패할 수 있습니다.

'; if (inbox && inbox.sdkRestorePending) html += ''; if (inbox && inbox.sdkProofPage && (inbox.sdkCompleted === undefined || inbox.sdkCompleted !== inbox.sdkProofPage.total)) html += '

SDK 원본 보존 '+fmtNum(inbox.sdkRetained)+'건 · 현재 페이지 '+fmtNum(inbox.sdkProofPage.checked ? inbox.sdkProofPage.offset+1 : 0)+'–'+fmtNum(inbox.sdkProofPage.offset+inbox.sdkProofPage.checked)+' · 검증 '+fmtNum(inbox.sdkProofPage.checked)+'건 / '+fmtNum(inbox.sdkProofPage.total)+'건 · 완료 '+fmtNum(inbox.sdkProofPage.completed)+'건 · 전체 완료 미확정

'; if (inbox && inbox.sdkCompleted) html += '

SDK 저장 완료 · 원본 보존 '+fmtNum(inbox.sdkCompleted)+'건. 디스크 확정 영수증이 없어 자동 제거·재처리하지 않습니다.

'; @@ -2787,7 +2797,7 @@

agentmemory

operations.loading = true; renderOperations(); operations.request = (async function() { try { - var results = await Promise.all([apiGet('health'),apiGet('sessions?includeSummary=false'),apiGet('background/control')]); + var results = await Promise.all([apiGet('health'),apiGet('sessions?includeSummary=false&includeSummaryCoverage=true'),apiGet('background/control')]); if (generation !== operations.generation) return; var h = results[0], sessions = results[1], control = results[2]; operations.error = h ? '' : 'Health 조회 실패 · 이전 상태는 최신 상태가 아닐 수 있습니다.'; @@ -7814,8 +7824,8 @@

agentmemory

html += hrow('Keyword rebuild', ri.bm25Incomplete ? 'incomplete' : 'complete', null, 'Rebuilt from stored data at every start, so it is never saved to disk.'); if (p) { html += hrow('Vector save', esc(legText(p.vector)), null, 'Saved at most every ' + esc(durationText(p.saveIntervalMs || 600000)) + ' (AGENTMEMORY_INDEX_SAVE_INTERVAL_MS). ' + esc(fmtNum(p.buckets) + ' bucket' + (p.buckets === 1 ? '' : 's') + ', ' + fmtNum(p.pendingChanges) + ' unsaved change' + (p.pendingChanges === 1 ? '' : 's') + (p.saving ? ', saving now' : '') + '.')); - if (typeof p.pendingLog === 'number') { - html += hrow('Pending vector log', esc(fmtNum(p.pendingLog) + ' vector change' + (p.pendingLog === 1 ? '' : 's')), null, + if (typeof p.pendingLog === 'number' || p.pendingLogKnown === false) { + html += hrow('Pending vector log', p.pendingLogKnown === false ? 'not checked' : esc(fmtNum(p.pendingLog) + ' vector change' + (p.pendingLog === 1 ? '' : 's')), null, p.pendingLogError ? '' + esc('Log writes failing: ' + p.pendingLogError) + '' : 'New and removed vectors are written here right away and replayed after a crash, so nothing is embedded twice. Each save empties it.'); } @@ -7901,7 +7911,8 @@

agentmemory

var drainedAt = (cap.spool || []).map(function(s) { return s.stats && s.stats.lastDrainAt; }).filter(Boolean).sort().pop(); var dropped = (cap.spool || []).reduce(function(n, s) { return n + ((s.stats && s.stats.dropped) || 0); }, 0); html += hrow('Inbox', inbox ? esc(fmtNum(inbox.pending) + ' pending · ' + fmtNum(inbox.retrying) + ' retrying · ' + fmtNum(inbox.dead) + ' dead' + (inbox.sdkProofPage ? ' · '+fmtNum(inbox.sdkRetained)+' SDK 원본 보존 · '+(typeof inbox.sdkCompleted === 'number' ? fmtNum(inbox.sdkCompleted)+' 완료 증거 확인' : '전체 완료 미확정') : inbox.sdkCompleted ? ' · '+fmtNum(inbox.sdkCompleted)+' SDK 완료 원본 보존' : '')) : 'not checked', 'captureInbox', - inbox && inbox.dead ? '' + esc(inbox.lastError || 'Could not be stored.') + ' · 작업 관리에서 수집 격리·복구 이력을 확인하세요.' : + !inbox ? 'Inbox status is unknown; automatic capture checks are unavailable.' : + inbox.dead ? '' + esc(inbox.lastError || 'Could not be stored.') + ' · 작업 관리에서 수집 격리·복구 이력을 확인하세요.' : inbox && inbox.retrying ? '' + esc('Retried every ' + durationText(cap.policy.retryIntervalMs) + ' with backoff, up to ' + cap.policy.maxAttempts + ' attempts' + (inbox.lastError ? '. Last error: ' + inbox.lastError : '.')) + '' : inbox && inbox.sdkProofPage ? '현재 페이지의 SDK 응답 증거 기준입니다. 페이지 간 원자성과 디스크 flush는 증명하지 않습니다.' : 'Empty means every accepted observation is stored.'); html += hrow('Since start', esc(fmtNum(since.accepted || 0) + ' accepted · ' + fmtNum(since.completed || 0) + ' stored · ' + fmtNum(since.duplicates || 0) + ' duplicates skipped · ' + fmtNum(since.recovered || 0) + ' recovered by retry · ' + fmtNum(since.deadLettered || 0) + ' dead'), null, @@ -7995,7 +8006,7 @@

agentmemory

'
1 · captured
Observations
Hooks record every tool call, edit and command.
' + fmtNum(c.observations) + '
' + '
2 · session end
Summaries
The LLM writes one summary when a session ends.
' + fmtNum(cs.summaries) + '
' + '
3 · merge
Semantic facts ' + help('semantic') + '
Merged from the newest 20 summaries once at least ' + (t.semanticMinSummaries || 5) + ' exist.
' + fmtNum(c.semantic) + '
' + - '
4 · repeat
Procedures ' + help('procedural') + '
From pattern memories seen in ' + (t.proceduralMinSessionsPerPattern || 2) + '+ sessions, once ' + (t.proceduralMinPatterns || 2) + ' exist.
' + fmtNum(c.procedural) + '
' + + '
4 · repeat
Procedures ' + help('procedural') + '
From pattern memories seen in ' + (t.proceduralMinSessionsPerPattern || 2) + '+ sessions or saved that many times, once ' + (t.proceduralMinPatterns || 2) + ' exist.
' + fmtNum(c.procedural) + '
' + '
5 · forget
Decay ' + help('decay') + '
Facts and procedures unused for ' + (sch.decayDays || 30) + ' days lose 10% strength per period (never below 0.1).
' + ''; var why = consolidationWhyEmpty(cs, c); diff --git a/src/ws.d.ts b/src/ws.d.ts index a3d842ed32..420d5bd5dc 100644 --- a/src/ws.d.ts +++ b/src/ws.d.ts @@ -3,6 +3,7 @@ declare module "ws" { import type { IncomingMessage } from "node:http"; import type { Duplex } from "node:stream"; export default class WebSocket extends EventEmitter { + constructor(address:string|URL,options?:{handshakeTimeout?:number}); static readonly OPEN:number; readonly readyState:number; readonly bufferedAmount:number; diff --git a/test/api-list-pagination.test.ts b/test/api-list-pagination.test.ts index 70bfeef334..fad871a391 100644 --- a/test/api-list-pagination.test.ts +++ b/test/api-list-pagination.test.ts @@ -198,6 +198,24 @@ describe("paginated and filtered list endpoints", () => { expect((search.body.sessions as Array<{ id: string }>).map((s) => s.id)).toEqual(["s2"]); }); + it("applies session time bounds before paging and rejects invalid ones", async () => { + const sessions: Record = {}; + for (let i = 0; i < 5; i++) { + sessions[`s${i}`] = { id: `s${i}`, startedAt: `2026-09-0${i + 1}T00:00:00Z` }; + } + const { handlers } = setup({ [KV.sessions]: sessions }); + const bounds = { startedAfter: "2026-09-02", startedBefore: "2026-09-05", limit: "2" }; + + const page = await call(handlers, "api::sessions", bounds); + expect((page.body.sessions as Array<{ id: string }>).map((s) => s.id)).toEqual(["s3", "s2"]); + const next = await call(handlers, "api::sessions", { ...bounds, cursor: page.body.nextCursor as string }); + expect((next.body.sessions as Array<{ id: string }>).map((s) => s.id)).toEqual(["s1"]); + expect(next.body.nextCursor).toBeNull(); + + expect((await call(handlers, "api::sessions", { startedAfter: "2026-02-30" })).status_code).toBe(400); + expect((await call(handlers, "api::sessions", { endedBefore: "" })).status_code).toBe(400); + }); + it("filters and pages observations inside a session", async () => { const observations: Record = {}; for (let i = 0; i < 5; i++) { diff --git a/test/api-record-list-paging.test.ts b/test/api-record-list-paging.test.ts new file mode 100644 index 0000000000..4ebfe75956 --- /dev/null +++ b/test/api-record-list-paging.test.ts @@ -0,0 +1,142 @@ +import { describe, it, expect, vi } from "vitest"; + +vi.mock("../src/logger.js", () => ({ + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, +})); + +import { registerApiTriggers } from "../src/triggers/api.js"; +import { KV } from "../src/state/schema.js"; +import { StateKV } from "../src/state/kv.js"; +import { SAFE_PAYLOAD_BYTES } from "../src/state/frame-guard.js"; + +const SECRET = "record-list-secret"; + +type StatePayload = { scope: string; key?: string; value?: unknown }; +type Row = { id: string }; +type ListResponse = { status_code: number; body: Record & { nextCursor?: string } }; +type Handler = (input: unknown) => unknown; + +function rig() { + const store = new Map>(); + const fns = new Map(); + const listed: string[] = []; + const sdk = { + registerFunction: (id: string | { id: string }, h: Handler) => fns.set(typeof id === "string" ? id : id.id, h), + registerTrigger: () => {}, + trigger: async ({ function_id: id, payload }: { function_id: string; payload: StatePayload }): Promise => { + if (!id.startsWith("state::")) return fns.get(id)?.(payload); + const rows = store.get(payload.scope) ?? new Map(); + store.set(payload.scope, rows); + if (id === "state::get") return rows.get(payload.key!) ?? null; + if (id === "state::set") { rows.set(payload.key!, payload.value); return payload.value; } + if (id === "state::list_keys") return { keys: [...rows.keys()] }; + if (id === "state::list") { listed.push(payload.scope); return [...rows.values()]; } + throw new Error(`unexpected state call ${id}`); + }, + }; + const kv = new StateKV(sdk as never); + registerApiTriggers(sdk as never, kv, SECRET); + const get = (fn: string, query: Record = {}) => + fns.get(fn)!({ headers: { authorization: `Bearer ${SECRET}` }, query_params: query }) as Promise; + const rowsOf = (res: ListResponse, key: string) => res.body[key] as Row[]; + return { kv, get, listed, rowsOf }; +} + +const id = (prefix: string, i: number) => `${prefix}_${(1_800_000_000_000 + i).toString(36)}_${i}`; + +function fact(i: number, size = 8) { + return { id: id("sem", i), fact: "f".repeat(size), confidence: 0.5, sourceSessionIds: [], accessCount: 0, createdAt: "", updatedAt: "" }; +} + +function procedure(i: number) { + return { id: id("proc", i), name: `p${i}`, steps: [], triggerCondition: "", frequency: 1, sourceSessionIds: [], createdAt: "", updatedAt: "" }; +} + +async function walk(r: ReturnType, fn: string, key: string, limit: string) { + const ids: string[] = []; + let cursor: string | undefined; + let pages = 0; + do { + const res = await r.get(fn, { limit, ...(cursor ? { cursor } : {}) }); + expect(res.status_code).toBe(200); + ids.push(...r.rowsOf(res, key).map((row) => row.id)); + cursor = res.body.nextCursor; + pages++; + } while (cursor && pages < 100); + return { ids, pages }; +} + +describe("semantic and procedural list paging", () => { + it("returns every record in one response without nextCursor when the scope fits", async () => { + const r = rig(); + for (let i = 0; i < 3; i++) await r.kv.set(KV.semantic, fact(i).id, fact(i)); + const res = await r.get("api::semantic-list"); + expect(res.status_code).toBe(200); + expect(r.rowsOf(res, "semantic")).toHaveLength(3); + expect(res.body).not.toHaveProperty("nextCursor"); + expect(r.listed).toEqual([]); + }); + + it("honors limit and walks every semantic record once through nextCursor", async () => { + const r = rig(); + for (let i = 0; i < 7; i++) await r.kv.set(KV.semantic, fact(i).id, fact(i)); + const first = await r.get("api::semantic-list", { limit: "3" }); + expect(r.rowsOf(first, "semantic")).toHaveLength(3); + expect(typeof first.body.nextCursor).toBe("string"); + const { ids, pages } = await walk(r, "api::semantic-list", "semantic", "3"); + expect(pages).toBe(3); + expect(new Set(ids).size).toBe(7); + expect(r.listed).toEqual([]); + }); + + it("pages procedural records the same way", async () => { + const r = rig(); + for (let i = 0; i < 5; i++) await r.kv.set(KV.procedural, procedure(i).id, procedure(i)); + const { ids, pages } = await walk(r, "api::procedural-list", "procedural", "2"); + expect(pages).toBe(3); + expect(new Set(ids).size).toBe(5); + }); + + it("caps limit at the list page maximum", async () => { + const r = rig(); + for (let i = 0; i < 502; i++) await r.kv.set(KV.semantic, fact(i).id, fact(i)); + const res = await r.get("api::semantic-list", { limit: "100000" }); + expect(r.rowsOf(res, "semantic")).toHaveLength(500); + expect(typeof res.body.nextCursor).toBe("string"); + }); + + it("ends a page early when the records would exceed the transport budget", async () => { + const r = rig(); + const size = Math.ceil(SAFE_PAYLOAD_BYTES / 3); + for (let i = 0; i < 4; i++) await r.kv.set(KV.semantic, fact(i).id, fact(i, size)); + const first = await r.get("api::semantic-list", { limit: "50" }); + expect(r.rowsOf(first, "semantic").length).toBeLessThan(4); + expect(JSON.stringify(first.body).length).toBeLessThan(SAFE_PAYLOAD_BYTES); + expect(typeof first.body.nextCursor).toBe("string"); + const { ids } = await walk(r, "api::semantic-list", "semantic", "50"); + expect(new Set(ids).size).toBe(4); + }); + + it("keeps paging when every record selected for a page was deleted before its read", async () => { + const r = rig(); + for (let i = 0; i < 4; i++) await r.kv.set(KV.semantic, fact(i).id, fact(i)); + await r.kv.set(KV.semantic, fact(3).id, null); + await r.kv.set(KV.semantic, fact(2).id, null); + const first = await r.get("api::semantic-list", { limit: "2" }); + expect(r.rowsOf(first, "semantic")).toHaveLength(0); + expect(typeof first.body.nextCursor).toBe("string"); + const { ids } = await walk(r, "api::semantic-list", "semantic", "2"); + expect(ids.sort()).toEqual([fact(0).id, fact(1).id].sort()); + }); + + it("returns 413 when one record and its continuation cursor exceed the transport budget", async () => { + const r = rig(); + const longId = "z".repeat(4096); + const base = { ...fact(0, 0), id: longId }; + const fill = SAFE_PAYLOAD_BYTES - JSON.stringify({ semantic: [base] }).length - 64; + await r.kv.set(KV.semantic, longId, { ...base, fact: "f".repeat(fill) }); + await r.kv.set(KV.semantic, "a1", { ...fact(1), id: "a1" }); + const res = await r.get("api::semantic-list", { limit: "50" }); + expect(res.status_code).toBe(413); + }); +}); diff --git a/test/api-unbounded-list-oversize.test.ts b/test/api-unbounded-list-oversize.test.ts new file mode 100644 index 0000000000..ccda244f48 --- /dev/null +++ b/test/api-unbounded-list-oversize.test.ts @@ -0,0 +1,98 @@ +import { describe, it, expect, vi } from "vitest"; + +vi.mock("../src/logger.js", () => ({ + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, +})); + +import { registerApiTriggers } from "../src/triggers/api.js"; +import { KV } from "../src/state/schema.js"; +import { StateKV } from "../src/state/kv.js"; +import { SAFE_PAYLOAD_BYTES } from "../src/state/frame-guard.js"; + +const AUTH = ["unbounded", "list", "test"].join("-"); + +type StatePayload = { scope: string; key?: string; value?: unknown }; +type Response = { status_code: number; body: Record }; +type Handler = (input: unknown) => unknown; + +function rig() { + const store = new Map>(); + const fns = new Map(); + const sdk = { + registerFunction: (id: string | { id: string }, h: Handler) => fns.set(typeof id === "string" ? id : id.id, h), + registerTrigger: () => {}, + trigger: async ({ function_id: id, payload }: { function_id: string; payload: StatePayload }): Promise => { + if (!id.startsWith("state::")) return fns.get(id)?.(payload); + const rows = store.get(payload.scope) ?? new Map(); + store.set(payload.scope, rows); + if (id === "state::get") return rows.get(payload.key!) ?? null; + if (id === "state::set") { rows.set(payload.key!, payload.value); return payload.value; } + if (id === "state::list_keys") return { keys: [...rows.keys()] }; + if (id === "state::list") return [...rows.values()]; + throw new Error(`unexpected state call ${id}`); + }, + }; + const kv = new StateKV(sdk as never); + registerApiTriggers(sdk as never, kv, AUTH); + const get = (fn: string, query: Record = {}) => + fns.get(fn)!({ headers: { authorization: `Bearer ${AUTH}` }, query_params: query }) as Promise; + return { kv, get }; +} + +const ROWS = 4; +const rowSize = () => Math.ceil(SAFE_PAYLOAD_BYTES / (ROWS - 1)); + +function memory(id: string, content: string, day: number) { + const at = `2026-11-0${day}T00:00:00.000Z`; + return { + id, type: "fact", title: id, content, concepts: [], files: [], sessionIds: [], + strength: 1, version: 1, isLatest: true, createdAt: at, updatedAt: at, + }; +} + +async function seedMemories(r: ReturnType) { + for (let i = 0; i < ROWS; i++) { + await r.kv.set(KV.memories, `mem_${i}`, memory(`mem_${i}`, "c".repeat(rowSize()), i + 1)); + } +} + +describe("unbounded list endpoints over the transport budget", () => { + it("GET /memories without limit returns 413 with a paging hint instead of an oversized frame", async () => { + const r = rig(); + await seedMemories(r); + const res = await r.get("api::memories"); + expect(res.status_code).toBe(413); + expect(res.body.oversized).toBe(true); + expect(String(res.body.error)).toMatch(/\?limit and \?cursor/); + }, 30_000); + + it("GET /memories with limit still pages the same oversized scope", async () => { + const r = rig(); + await seedMemories(r); + const res = await r.get("api::memories", { limit: "1" }); + expect(res.status_code).toBe(200); + expect((res.body.memories as unknown[]).length).toBe(1); + expect(res.body.total).toBe(ROWS); + expect(typeof res.body.nextCursor).toBe("string"); + }, 30_000); + + it("GET /memories without limit still returns every memory when the scope fits", async () => { + const r = rig(); + await r.kv.set(KV.memories, "mem_small", memory("mem_small", "c", 1)); + const res = await r.get("api::memories"); + expect(res.status_code).toBe(200); + expect((res.body.memories as unknown[]).length).toBe(1); + }); + + it("GET /relations returns 413 instead of an oversized frame", async () => { + const r = rig(); + for (let i = 0; i < ROWS; i++) { + await r.kv.set(KV.relations, `rel_${i}`, { + sourceId: `a${i}`, targetId: `b${i}`, type: "related", createdAt: "", note: "n".repeat(rowSize()), + }); + } + const res = await r.get("api::relations-list"); + expect(res.status_code).toBe(413); + expect(res.body.oversized).toBe(true); + }, 30_000); +}); diff --git a/test/capture-auto-recovery.test.ts b/test/capture-auto-recovery.test.ts index d1571b621b..349d0ebe2a 100644 --- a/test/capture-auto-recovery.test.ts +++ b/test/capture-auto-recovery.test.ts @@ -17,7 +17,7 @@ async function setup(){ const fingerprint=createHash('sha256').update(obs.title+' '+obs.narrative).digest('hex'); state.set(KV.captureInbox+'|'+key,inbox);state.set(KV.captureReview+'|'+key,history);state.set(KV.observations(sessionId)+'|'+observationId,obs);state.set('mem:embedding-committed:v1|'+observationId,{kind:'observation',id:observationId,sessionId,fingerprint}); const durability={mutate:(_s:string,_k:string,fn:()=>unknown)=>fn(),waitForKeys:async(_kv:any,keys:any[],opts:any)=>{calls++;if(onProof&&keys.some(x=>x.scope===KV.observations(sessionId))){const fn=onProof;onProof=undefined;await fn();}if(failProof&&keys.some(x=>x.scope===captureEventScope(key))&&state.has(captureEventScope(key)+'|'+key))throw Error('state_durability_timeout');if(opts?.expectedValues)for(let i=0;itrue,admitted:()=>admitted,indexReady:()=>true,now:()=>clock}; return {kv,state,key,lease,evidence,storePath:d,sdk,durability,sessionId,observationId,eventId,obs,history,options,advance:()=>clock+=3600000,failProof:(v:boolean)=>failProof=v,onProof:(fn:()=>Promise)=>onProof=fn,admit:(v:boolean)=>admitted=v,calls:()=>calls}; } diff --git a/test/capture-durable.test.ts b/test/capture-durable.test.ts index a8954e4f32..6eb0badacd 100644 --- a/test/capture-durable.test.ts +++ b/test/capture-durable.test.ts @@ -517,3 +517,8 @@ describe("acceptance durability across a restart", () => { expect((await controller.status()).spool[0]).toMatchObject({ records: 0, retained: 1 }); }); }); +it('retains capture for retry when observation is temporarily blocked by vector recovery',async()=>{const kv=mockKV();const {sdk,capture,controller,dedup}=await boot(kv);try{sdk.fns.set('mem::observe',async()=>({success:false,error:'vector_recovery_paused',retryable:true}));const result=await capture(payload('synthetic-hold'),'evc_000000003333');expect(result).toMatchObject({status:'accepted',state:'retrying'});expect(inbox(kv)).toHaveLength(1);expect(inbox(kv)[0]).toMatchObject({status:'retrying'});expect(observations(kv)).toHaveLength(0)}finally{controller.stop();dedup.stop()}}); +it('does not exhaust capture attempts while vector recovery admission remains held',async()=>{vi.stubEnv('AGENTMEMORY_CAPTURE_MAX_ATTEMPTS','1');const kv=mockKV();const {sdk,capture,controller,dedup}=await boot(kv);try{sdk.fns.set('mem::observe',async()=>({success:false,error:'vector_recovery_paused',retryable:true}));const result=await capture(payload('synthetic-held-budget'),'evc_000000003334');expect(result).toMatchObject({status:'accepted',state:'retrying',attempts:0});expect(inbox(kv)[0]).toMatchObject({status:'retrying',attempts:0})}finally{controller.stop();dedup.stop()}}); +it('retains delayed captures across repeated holds and processes them after release',async()=>{vi.stubEnv('AGENTMEMORY_CAPTURE_MAX_ATTEMPTS','1');vi.stubEnv('AGENTMEMORY_CAPTURE_RETRY_INTERVAL_MS','1000');const kv=mockKV();const {sdk,capture,controller,dedup}=await boot(kv),observe=sdk.fns.get('mem::observe')!;try{vi.useFakeTimers({now:Date.now(),toFake:['Date']});sdk.fns.set('mem::observe',async()=>({success:false,error:'vector_recovery_paused',retryable:true}));await capture(payload('synthetic-long-hold'),'evc_000000003335');for(let i=0;i<3;i++){vi.setSystemTime(Date.now()+60000);await controller.sweep();expect(inbox(kv)[0]).toMatchObject({status:'retrying',attempts:0})}sdk.fns.set('mem::observe',observe);vi.setSystemTime(Date.now()+60000);await controller.sweep();expect(inbox(kv)).toHaveLength(0);expect(observations(kv)).toHaveLength(1)}finally{controller.stop();dedup.stop();vi.useRealTimers();vi.unstubAllEnvs()}}); +it('quarantines an interrupted observation and prevents blind manual retry',async()=>{const kv=mockKV();const {sdk,capture,controller,dedup}=await boot(kv);try{const observe=vi.fn(async()=>({success:false,error:'vector_recovery_interrupted',partial:true,retryable:false}));sdk.fns.set('mem::observe',observe);const first=await capture(payload('synthetic-interrupted'),'evc_000000003336');expect(first).toMatchObject({status:'rejected',retryable:false});expect(inbox(kv)[0]).toMatchObject({status:'dead',lastError:'vector_recovery_interrupted'});const again=await sdk.trigger('mem::capture-retry',{all:true}) as any;expect(again).toMatchObject({matched:1,waiting:1,recovered:0});expect(observe).toHaveBeenCalledTimes(1);expect(inbox(kv)).toHaveLength(1)}finally{controller.stop();dedup.stop()}}); +it('does not mutate the index for a duplicate capture while vector recovery is held',async()=>{const kv=mockKV();const {capture,controller,dedup}=await boot(kv);const {createBackgroundControl,installBackgroundControl}=await import('../src/ops/background-control.mjs');const first=await capture(payload('synthetic-duplicate-held'),'evc_000000003337');expect(first.status).toBe('accepted');const state={version:2,paused:false,resumePending:true,holds:[],recovery:{attempts:0,nextAttemptAt:0}};const control=createBackgroundControl();await control.init({get:async()=>state,set:async()=>{},durability:{wait:async()=>({synced:true,mutationFailures:0}),waitForKeys:async()=>{}}} as any);installBackgroundControl(control);try{await expect(capture(payload('synthetic-duplicate-held'),'evc_000000003337')).rejects.toThrow(/vector_recovery_paused/);expect(observations(kv)).toHaveLength(1)}finally{controller.stop();dedup.stop()}}); diff --git a/test/capture-filter.test.ts b/test/capture-filter.test.ts index 0d576103b7..a75f5292e6 100644 --- a/test/capture-filter.test.ts +++ b/test/capture-filter.test.ts @@ -5,6 +5,7 @@ import { preCompactBudget, shouldCaptureTool, truncateCaptureOutput, + truncateJson, truncateText, } from "../src/hooks/_capture-filter.js"; @@ -116,6 +117,104 @@ describe("truncateText", () => { }); }); +describe("truncation redacts before cutting", () => { + const token = "ghp_" + "A".repeat(40); + + it("truncateText leaves text within the limit unchanged", () => { + expect(truncateText(`short ${token}`, 100)).toBe(`short ${token}`); + }); + + it("truncateText does not keep the start of a token the cut lands in", () => { + const out = truncateText("x".repeat(10) + " " + token + " tail", 20); + expect(out).not.toContain("ghp_"); + expect(out.length).toBeLessThanOrEqual(20); + }); + + it("truncateCaptureOutput redacts strings and JSON before cutting", () => { + const str = truncateCaptureOutput("x".repeat(10) + " " + token + "y".repeat(100), 40) as string; + expect(str).not.toContain("ghp_"); + expect(str.length).toBeLessThanOrEqual(40); + const json = truncateCaptureOutput({ out: "x".repeat(10) + " " + token + "y".repeat(100) }, 40) as string; + expect(json).not.toContain("ghp_"); + expect(json.length).toBeLessThanOrEqual(40); + }); +}); + +describe("truncateCaptureOutput redacts structured values before serializing", () => { + it("redacts values under secret-like keys in both helpers", () => { + const value = { password: "A".repeat(100), nested: { apiKey: "K".repeat(40) }, note: "kept" }; + for (const out of [truncateJson(value, 90), truncateCaptureOutput(value, 90) as string]) { + expect(out).not.toContain("AAAA"); + expect(out).not.toContain("KKKK"); + expect(out).toContain("[REDACTED_SECRET]"); + } + }); + + it("keeps values whose key only contains a secret word inside another word", () => { + const value = { tokenization: "words", secretary: "office", access_token: "T".repeat(40), accesstoken: "U".repeat(40), apiKey2: "V".repeat(40), pad: "y".repeat(200) }; + const out = truncateJson(value, 200); + expect(out).toContain("words"); + expect(out).toContain("office"); + expect(out).not.toContain("TTTT"); + expect(out).not.toContain("UUUU"); + expect(out).not.toContain("VVVV"); + }); + + it("leaves a short value with a secret-like key unchanged", () => { + const value = { password: "short" }; + expect(truncateJson(value, 100)).toBe(JSON.stringify(value)); + expect(truncateCaptureOutput(value, 100)).toBe(value); + }); + + it("removes a private block whose tag spans a line break", () => { + const out = truncateCaptureOutput({ log: "hidden-value" + "y".repeat(200) }, 60) as string; + expect(out).not.toContain("hidden"); + }); + + it("removes a private block that opens in one field and closes in the next", () => { + const value = { a: "start ", b: "hidden-value end", pad: "y".repeat(200) }; + expect(truncateCaptureOutput(value, 80) as string).not.toContain("hidden"); + expect(truncateJson(value, 80)).not.toContain("hidden"); + }); + + it("redacts a credential in a key that a line break separates from its scheme", () => { + const key = "Bearer\n" + "A".repeat(32); + const value = { [key]: "x", pad: "y".repeat(200) }; + expect(truncateJson(value, 100)).not.toContain("A".repeat(32)); + expect(truncateCaptureOutput(value, 100) as string).not.toContain("A".repeat(32)); + }); + + it("redacts a quoted credential whose quote JSON escapes", () => { + const value = { cmd: 'secret="' + "B".repeat(32) + '"', pad: "y".repeat(200) }; + expect(truncateJson(value, 100)).not.toContain("B".repeat(20)); + expect(truncateCaptureOutput(value, 100) as string).not.toContain("B".repeat(20)); + }); + + it("redacts a credential that a vertical tab separates from its scheme", () => { + const value = { header: "Bearer\u000b" + "C".repeat(32), pad: "y".repeat(200) }; + expect(truncateJson(value, 100)).not.toContain("C".repeat(20)); + }); + + it("keeps every field when two keys redact to the same text", () => { + const value = { ["token=" + "A".repeat(24)]: "first", ["token=" + "B".repeat(24)]: "second", pad: "y".repeat(200) }; + const out = truncateJson(value, 150); + expect(out).toContain("first"); + expect(out).toContain("second"); + }); + + it("keeps escaped backslashes and line breaks in the cut JSON", () => { + const out = truncateJson({ a: "one\ntwo\\nthree", pad: "y".repeat(200) }, 60); + expect(out.startsWith('{"a":"one\\ntwo\\\\nthree"')).toBe(true); + }); + + it("truncateJson leaves short JSON unchanged and redacts long JSON before cutting", () => { + expect(truncateJson({ a: 1 }, 100)).toBe('{"a":1}'); + const out = truncateJson({ log: "hidden-value" + "y".repeat(200) }, 60); + expect(out).not.toContain("hidden"); + expect(out.length).toBeLessThanOrEqual(60); + }); +}); + describe("truncateCaptureOutput", () => { it("leaves short strings alone", () => { expect(truncateCaptureOutput("hello", 20)).toBe("hello"); diff --git a/test/capture-inbox-capacity.test.ts b/test/capture-inbox-capacity.test.ts new file mode 100644 index 0000000000..85f13107a6 --- /dev/null +++ b/test/capture-inbox-capacity.test.ts @@ -0,0 +1,89 @@ +import {afterEach,expect,it,vi} from 'vitest'; +import {mockKV,mockSdk} from './helpers/mocks.js'; +import {registerCaptureFunctions} from '../src/functions/capture.js'; +import {KV} from '../src/state/schema.js'; +import {createStateDurabilityFence} from '../src/ops/state-durability.mjs'; +vi.mock('../src/logger.js',()=>({logger:{info:vi.fn(),warn:vi.fn(),error:vi.fn()}})); +const stops:(()=>void)[]=[]; +afterEach(()=>{stops.splice(0).forEach(f=>f());vi.unstubAllEnvs();}); +function setup(kv=mockKV(),sdkOnly=true){ + vi.stubEnv('AGENTMEMORY_CAPTURE_INBOX_MAX','1'); + if(sdkOnly)Object.assign(kv,{durability:createStateDurabilityFence({storePath:'/synthetic-capacity',completion:'sdk'})}); + const sdk=mockSdk({looseTrigger:true});sdk.fns.set('mem::observe',async(p:any)=>({observationId:p.observationId})); + const control=registerCaptureFunctions(sdk as never,kv as never);stops.push(()=>control.stop()); + const capture=(n:number)=>sdk.trigger('mem::capture',{eventId:`capacity_event_${n}`,payload:{hookType:'post_tool_use',sessionId:'capacity',project:'synthetic',cwd:'/synthetic',timestamp:'2026-11-14T00:00:00Z',data:{tool_name:'Read',tool_output:`synthetic ${n}`}}}) as Promise; + return {kv,sdk,control,capture}; +} +it('SDK completion releases pending capacity without deleting retained originals; warm and cold cache agree',async()=>{ + const f=setup();expect(await f.capture(1)).toMatchObject({status:'accepted',state:'sdk_completed'}); + const original=await f.kv.list(KV.captureInbox);expect(original).toHaveLength(1); + expect(await f.capture(2)).toMatchObject({status:'accepted',state:'sdk_completed'}); + f.control.stop();const restarted=setup(f.kv); + expect(await restarted.capture(3)).toMatchObject({status:'accepted',state:'sdk_completed'}); + const all=await f.kv.list(KV.captureInbox);expect(all).toHaveLength(3);expect(all.find(x=>x.key===(original[0] as any).key)).toEqual(original[0]); +}); +it('retained SDK records do not hide a genuinely full pending queue',async()=>{ + const f=setup();await f.capture(1); + f.sdk.fns.set('mem::observe',async()=>{throw Error('synthetic offline');}); + expect(await f.capture(2)).toMatchObject({status:'accepted',state:'retrying'}); + expect(await f.capture(3)).toMatchObject({status:'rejected',retryable:true}); + expect(await f.kv.list(KV.captureInbox)).toHaveLength(2); +}); +for(const status of ['dead','pending','retrying'])it(`unknown/non-SDK ${status} retains original admission protection`,async()=>{ + const kv=mockKV();await kv.set(KV.captureInbox,'legacy',{key:'legacy',status,acceptedAt:'2026-11-14T00:00:00Z',updatedAt:'2026-11-14T00:00:00Z',payload:{untouched:true}}); + const before=await kv.get(KV.captureInbox,'legacy');const f=setup(kv); + expect(await f.capture(1)).toMatchObject({status:'rejected',retryable:true});expect(await kv.get(KV.captureInbox,'legacy')).toEqual(before); +}); +it('partial SDK proof and unresolved review stay untouched and are not pending-capacity slots',async()=>{ + const kv=mockKV();const archived={key:'retained',status:'retrying',storageCompletion:'sdk',sdkRestoreState:'pending',sdkResolutionState:'pending',lastError:'unverified',acceptedAt:'2026-11-14T00:00:00Z',updatedAt:'2026-11-14T00:00:00Z',payload:{unknownOriginal:true}}; + await kv.set(KV.captureInbox,'retained',archived);await kv.set(KV.captureReview,'retained',{status:'unknown',failures:[{code:'UNCONFIRMED'}]}); + const f=setup(kv);await f.control.status({sdkProofPage:{offset:0,limit:1}}); + expect(await f.capture(1)).toMatchObject({status:'accepted'});expect(await kv.get(KV.captureInbox,'retained')).toEqual(archived);expect(await kv.get(KV.captureReview,'retained')).toEqual({status:'unknown',failures:[{code:'UNCONFIRMED'}]}); +}); +it('repeated SDK delivery and status refresh do not consume or double-release pending slots',async()=>{ + const f=setup();await f.capture(1);await f.control.status({scanInbox:true});await f.capture(1);await f.control.status({scanInbox:true}); + f.sdk.fns.set('mem::observe',async()=>{throw Error('synthetic offline');}); + expect(await f.capture(2)).toMatchObject({status:'accepted',state:'retrying'}); + expect(await f.capture(3)).toMatchObject({status:'rejected'});expect(await f.kv.list(KV.captureInbox)).toHaveLength(2); +}); +it('deleting an SDK-retained tombstone in disk mode does not release another pending slot',async()=>{ + const {captureEventScope}=await import('../src/capture/event-record.js');const kv=mockKV(); + const archived={key:'archived',eventSource:'client',status:'pending',storageCompletion:'sdk',acceptedAt:'2026-11-14T00:00:00Z'}; + await kv.set(KV.captureInbox,'archived',archived);await kv.set(captureEventScope('archived'),'archived',{state:'deleted'}); + await kv.set(KV.captureInbox,'pending',{key:'pending',status:'dead',acceptedAt:'2026-11-14T00:00:00Z'}); + const f=setup(kv,false);await f.control.sweep();expect(await kv.get(KV.captureInbox,'archived')).toBeNull(); + expect(await f.capture(1)).toMatchObject({status:'rejected'});await f.control.sweep();expect(await f.capture(2)).toMatchObject({status:'rejected'}); +}); +it('disk completion and deleted-event retry restore exactly one available slot',async()=>{ + const {captureEventScope}=await import('../src/capture/event-record.js');const f=setup(mockKV(),false); + expect(await f.capture(1)).toMatchObject({state:'completed'});expect(await f.capture(2)).toMatchObject({state:'completed'}); + f.sdk.fns.set('mem::observe',async()=>{throw Error('synthetic offline');});await f.capture(3); + const [rec]=await f.kv.list(KV.captureInbox);await f.kv.set(captureEventScope(rec.key),rec.key,{state:'deleted'});await f.kv.set(KV.captureInbox,rec.key,{...rec,nextAttemptAt:'2020-01-01T00:00:00Z'}); + await f.control.sweep();expect(await f.capture(4)).toMatchObject({status:'accepted',state:'retrying'});expect(await f.capture(5)).toMatchObject({status:'rejected'}); +}); +it('does not rescan retained metadata after each SDK completion',async()=>{ + const kv=mockKV(),list=vi.spyOn(kv,'list');const f=setup(kv); + for(let n=1;n<=12;n++)expect(await f.capture(n)).toMatchObject({status:'accepted',state:'sdk_completed'}); + expect(list.mock.calls.filter(([scope])=>scope===KV.captureInbox)).toHaveLength(1); + expect(kv.store.get(KV.captureInbox)?.size).toBe(12); +}); +it('a stale empty status snapshot cannot erase a concurrent pending admission',async()=>{ + const f=setup();await f.capture(1);const original=f.kv.list.bind(f.kv); + let unblock!:()=>void,seen!:()=>void;const entered=new Promise(r=>seen=r),barrier=new Promise(r=>unblock=r);let intercepted=false; + f.kv.list=async (...args:any[])=>{const rows=await (original as any)(...args);if(args[0]===KV.captureInbox&&!intercepted){intercepted=true;seen();await barrier;}return rows;}; + const status=f.control.status();await entered; + f.sdk.fns.set('mem::observe',async()=>{throw Error('synthetic offline');}); + expect(await f.capture(2)).toMatchObject({status:'accepted',state:'retrying'});unblock();await status; + expect(await f.capture(3)).toMatchObject({status:'rejected'}); +}); +it('a stale pending status snapshot cannot resurrect a completed slot or overwrite a newer key',async()=>{ + const f=setup();let finishFirst!:(v:any)=>void,firstEntered!:()=>void;const firstSeen=new Promise(r=>firstEntered=r); + f.sdk.fns.set('mem::observe',async(p:any)=>{firstEntered();return new Promise(r=>finishFirst=()=>r({observationId:p.observationId}));}); + const first=f.capture(1);await firstSeen;const original=f.kv.list.bind(f.kv);let unblock!:()=>void,seen!:()=>void;const entered=new Promise(r=>seen=r),barrier=new Promise(r=>unblock=r);let intercepted=false; + f.kv.list=async (...args:any[])=>{const rows=await (original as any)(...args);if(args[0]===KV.captureInbox&&!intercepted){intercepted=true;seen();await barrier;}return rows;}; + const status=f.control.status();await entered;finishFirst(null);await first; + let finishSecond!:()=>void,secondEntered!:()=>void;const secondSeen=new Promise(r=>secondEntered=r); + f.sdk.fns.set('mem::observe',async(p:any)=>{secondEntered();return new Promise(r=>finishSecond=()=>r({observationId:p.observationId}));}); + const second=f.capture(2);await secondSeen;unblock();await status;finishSecond();await second; + f.sdk.fns.set('mem::observe',async(p:any)=>({observationId:p.observationId}));expect(await f.capture(3)).toMatchObject({status:'accepted'});expect(f.kv.store.get(KV.captureInbox)?.size).toBe(3); +}); diff --git a/test/capture-inbox-file-list.test.ts b/test/capture-inbox-file-list.test.ts new file mode 100644 index 0000000000..081078dddd --- /dev/null +++ b/test/capture-inbox-file-list.test.ts @@ -0,0 +1,354 @@ +import { afterEach, expect, it, vi } from 'vitest'; +import { createHash } from 'node:crypto'; +import { createReadStream } from 'node:fs'; +import { mkdtemp, open, readFile, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { StateKV, applyUpdateOps } from '../src/state/kv.js'; +import { KV } from '../src/state/schema.js'; +import { registerCaptureFunctions } from '../src/functions/capture.js'; +import { listCaptureInboxMetadata, projectCaptureInbox } from '../src/capture/inbox-metadata.js'; +import { mockSdk } from './helpers/mocks.js'; +import { capturePayloadDigest } from '../src/capture/receipt-digest.js'; +import { captureFailureHistoryDigest } from '../src/capture/sdk-resolution-record.js'; +import { createStateDurabilityFence } from '../src/ops/state-durability.mjs'; + +vi.mock('../src/logger.js', () => ({ logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() } })); +const directories: string[] = []; +const controllers: Array> = []; +afterEach(async () => { + for (const controller of controllers.splice(0)) controller.stop(); + vi.unstubAllEnvs(); + await Promise.all(directories.splice(0).map(path => rm(path, { recursive: true, force: true }))); +}); +async function fixture() { + const dir = await mkdtemp(join(tmpdir(), 'am-capture-list-')); + directories.push(dir); + vi.stubEnv('AGENTMEMORY_STATE_STORE_PATH', dir); + return join(dir, encodeURIComponent(KV.captureInbox) + '.bin'); +} +function stateSdk(rows: Record = {}) { + const sdk = mockSdk(); + const store = new Map>([[KV.captureInbox, new Map(Object.entries(rows))]]); + const list = vi.fn(async ({ scope }: { scope: string }) => { + if (scope === KV.captureInbox) throw Error('SDK_LIST_EXCEEDS_64_MIB'); + return [...(store.get(scope)?.values() ?? [])]; + }); + sdk.registerFunction('state::list', list as never); + sdk.registerFunction('state::get', async (input: any) => store.get(input.scope)?.get(input.key) ?? null); + sdk.registerFunction('state::set', async (input: any) => { + if (!store.has(input.scope)) store.set(input.scope, new Map()); + store.get(input.scope)!.set(input.key, input.value); + return input.value; + }); + sdk.registerFunction('state::update', async (input: any) => { + const before = store.get(input.scope)?.get(input.key); + const after = applyUpdateOps(before, input.ops); + store.get(input.scope)?.set(input.key, after); + return { old_value: before, new_value: after, errors: [] }; + }); + sdk.registerFunction('state::delete', async (input: any) => { + const before = store.get(input.scope)?.get(input.key); + store.get(input.scope)?.delete(input.key); + return before ?? null; + }); + return { sdk, list, store }; +} +async function digest(path: string) { + const hash = createHash('sha256'); + for await (const block of createReadStream(path)) hash.update(block); + return hash.digest('hex'); +} + +it('preserves capture admission and status with an intact inbox beyond a 64 MiB SDK list', async () => { + const path = await fixture(); + const { sdk, list, store } = stateSdk(); + const largeText = 'x'.repeat(512 * 1024); + const handle = await open(path, 'w'); + let sdkBytes = 2; + try { + await handle.write('{'); + for (let i = 0; i < 128; i++) { + const key = 'retained-' + i; + const row = { + key, eventId: key, sessionId: 'old-session', project: 'fixture', + status: 'dead', attempts: 1, updatedAt: '2026-11-01T00:00:00.000Z', + payload: { data: { synthetic: largeText } }, + }; + store.get(KV.captureInbox)!.set(key, row); + const value = JSON.stringify(row); + sdkBytes += Buffer.byteLength(value) + Number(i > 0); + await handle.write((i ? ',' : '') + JSON.stringify(key) + ':' + value); + } + await handle.write('}'); + await handle.write(Buffer.from([0, 0, 0, 0, 255, 255, 255, 255])); + } finally { + await handle.close(); + } + expect(sdkBytes).toBeGreaterThan(64 * 1024 * 1024); + const original = await digest(path); + const kv = new StateKV(sdk as never); + const localList = vi.spyOn(kv.graphFileList!, 'list'); + sdk.registerFunction('mem::observe', async (payload: any) => ({ observationId: payload.observationId })); + sdk.registerFunction('fixture::health', async () => ({ connected: true })); + const controller = registerCaptureFunctions(sdk as never, kv); + controllers.push(controller); + const result = await sdk.trigger('mem::capture', { + eventId: 'evc_000000000099', + payload: { hookType: 'post_tool_use', sessionId: 'new-session', project: 'fixture', + cwd: '/synthetic', timestamp: '2026-11-07T00:00:00.000Z', data: { tool_name: 'Bash', tool_output: 'synthetic' } }, + }); + expect(result).toMatchObject({ status: 'accepted', state: 'sdk_completed' }); + const report = await controller.status(); + expect(report.inbox).toMatchObject({ dead: 128, pending: 0, sdkCompleted: 1 }); + expect(await sdk.trigger('fixture::health', {})).toEqual({ connected: true }); + expect(list.mock.calls.some(([input]) => input.scope === KV.captureInbox)).toBe(false); + expect(localList.mock.calls.every(([, options]) => typeof options?.project === 'function')).toBe(true); + expect(await digest(path)).toBe(original); +}, 30_000); + +it('overlays acknowledged inbox set/update/delete before disk catches up', async () => { + const path = await fixture(); + const rows = { original: { key: 'original', attempts: 0 }, removed: { key: 'removed' } }; + await writeFile(path, JSON.stringify(rows)); + const { sdk, list } = stateSdk(rows); + const kv = new StateKV(sdk as never); + await kv.update(KV.captureInbox, 'original', [{ type: 'set', path: 'attempts', value: 2 }]); + await Promise.all([ + kv.set(KV.captureInbox, 'first', { key: 'first' }), + kv.set(KV.captureInbox, 'second', { key: 'second' }), + ]); + await kv.delete(KV.captureInbox, 'removed'); + const expected = [{ key: 'original', attempts: 2 }, { key: 'first' }, { key: 'second' }]; + expect(await kv.list(KV.captureInbox)).toEqual(expected); + await writeFile(path, JSON.stringify(Object.fromEntries(expected.map(row => [row.key, row])))); + expect(await kv.list(KV.captureInbox)).toEqual(expected); + expect(list).not.toHaveBeenCalled(); +}); + +it('keeps an in-flight SDK mutation out of the overlay until acknowledged', async () => { + await fixture(); + const { sdk } = stateSdk(); + let acknowledge!: (value: unknown) => void; + let entered!: () => void; + const started = new Promise(resolve => { entered = resolve; }); + sdk.registerFunction('state::set', async () => { + entered(); + return new Promise(resolve => { acknowledge = resolve; }); + }); + const kv = new StateKV(sdk as never); + const write = kv.set(KV.captureInbox, 'new', { key: 'new' }); + await started; + expect(await kv.list(KV.captureInbox)).toEqual([]); + acknowledge({ key: 'new' }); + await write; + expect(await kv.list(KV.captureInbox)).toEqual([{ key: 'new' }]); +}); + +it('does not let a capture retry mutate the acknowledged overlay before another SDK write', async () => { + await fixture(); + const { sdk } = stateSdk(); + const kv = new StateKV(sdk as never); + await kv.set(KV.captureInbox, 'new', { key: 'new', attempts: 1, payload: { data: { text: 'original' } } }); + const first = await kv.list(KV.captureInbox); + first[0].attempts++; + first[0].payload.data.text = 'locally changed'; + expect(await kv.list(KV.captureInbox)).toEqual([ + { key: 'new', attempts: 1, payload: { data: { text: 'original' } } }, + ]); +}); + +it('retains an inbox deletion ACK when an absent checkpoint cannot prove the deletion generation', async () => { + const path = await fixture(); + await writeFile(path, '{}'); + const { sdk } = stateSdk(); + const kv = new StateKV(sdk as never); + const old = { key: 'old', attempts: 1 }; + await kv.set(KV.captureInbox, old.key, old); + await kv.delete(KV.captureInbox, old.key); + expect(await kv.list(KV.captureInbox)).toEqual([]); + // An absent key alone does not identify which SDK mutation was checkpointed. + await writeFile(path, JSON.stringify({ old })); + expect(await kv.list(KV.captureInbox)).toEqual([]); + await kv.set(KV.captureInbox, old.key, { ...old, attempts: 2 }); + expect(await kv.list(KV.captureInbox)).toEqual([{ key: 'old', attempts: 2 }]); +}); + +it('projects each inbox entry and retires the ACK overlay only after the matching checkpoint', async () => { + const path = await fixture(); + await writeFile(path, '{"old":{"key":"old","attempts":0,"payload":{"data":"large original"}}}'); + const { sdk } = stateSdk(); + const kv = new StateKV(sdk as never); + const project = (value: any) => ({ key: value.key, attempts: value.attempts }); + await kv.set(KV.captureInbox, 'old', { key: 'old', attempts: 1, payload: { data: 'uncheckpointed' } }); + expect(await kv.list(KV.captureInbox, { project })).toEqual([{ key: 'old', attempts: 1 }]); + expect(await kv.list(KV.captureInbox)).toEqual([ + { key: 'old', attempts: 1, payload: { data: 'uncheckpointed' } }, + ]); + await writeFile(path, '{"old":{"key":"old","attempts":1,"payload":{"data":"uncheckpointed"}}}'); + await kv.list(KV.captureInbox, { project }); + await writeFile(path, '{"old":{"key":"old","attempts":2,"payload":{"data":"checkpoint advanced"}}}'); + expect(await kv.list(KV.captureInbox, { project })).toEqual([{ key: 'old', attempts: 2 }]); +}); + +it('retains the exact bounded list and review previews without payloads', async () => { + const row = { key: 'preview', hookType: 'user_prompt_submit', + payload: { hookType: 'post_tool_use', data: { tool_name: 'Bash', tool_input: 'x'.repeat(800) } } }; + const metadata = projectCaptureInbox(row); + expect(metadata).not.toHaveProperty('payload'); + expect(metadata.preview).toBe(('Bash: ' + JSON.stringify(row.payload.data.tool_input)).slice(0, 160)); + expect(metadata.reviewPreview).toBe(('Bash: ' + JSON.stringify(row.payload.data.tool_input)).slice(0, 500)); + const message = projectCaptureInbox({ key: 'message', hookType: 'stop', payload: { hookType: 'stop', data: { message: 'done' } } }); + expect(message.preview).toBe('stop: {"message":"done"}'); + expect(message.reviewPreview).toBe('stop: "done"'); +}); + +it('projects a recent ACK independently of callers and projector mutation', async () => { + await fixture(); + const { sdk } = stateSdk(); + const kv = new StateKV(sdk as never); + await kv.set(KV.captureInbox, 'new', { key: 'new', attempts: 1, payload: { data: 'original' } }); + const rows = await listCaptureInboxMetadata(kv); + rows[0].attempts = 9; + await kv.list(KV.captureInbox, { project: (row: any) => { row.payload.data = 'changed'; return row.key; } }); + expect(await kv.list(KV.captureInbox)).toEqual([{ key: 'new', attempts: 1, payload: { data: 'original' } }]); +}); + +it('refuses corrupt or uncertain inbox state without an SDK list fallback', async () => { + const path = await fixture(); + const { sdk, list } = stateSdk(); + const kv = new StateKV(sdk as never); + await writeFile(path, '{"old":broken}'); + await expect(kv.list(KV.captureInbox)).rejects.toThrow(/state_namespace/); + await writeFile(path, '{"old":{"key":"old"}}'); + sdk.registerFunction('state::set', async () => { throw Error('SDK outcome unknown'); }); + await expect(kv.set(KV.captureInbox, 'new', { key: 'new' })).rejects.toThrow('SDK outcome unknown'); + await expect(kv.list(KV.captureInbox)).rejects.toThrow('graph_mutation_uncertain'); + expect(list).not.toHaveBeenCalled(); + expect(await readFile(path, 'utf8')).toBe('{"old":{"key":"old"}}'); +}); + +it.each(['redis', 'no-local-path'] as const)('preserves SDK fallback for %s', async mode => { + const { sdk, list } = stateSdk(); + if (mode === 'redis') await fixture(); + else vi.stubEnv('AGENTMEMORY_STATE_STORE_PATH', ''); + list.mockImplementation(async () => [{ key: 'legacy' }]); + const kv = new StateKV(sdk as never, mode === 'redis' ? { backend: 'redis' } : {}); + expect(await kv.list(KV.captureInbox)).toEqual([{ key: 'legacy' }]); + expect(await kv.list(KV.captureInbox, { project: (row: any) => row.key })).toEqual(['legacy']); + expect(list).toHaveBeenCalledWith({ scope: KV.captureInbox }); +}); + +it('keeps unrelated capture namespaces on their SDK path', async () => { + await fixture(); + const { sdk, list } = stateSdk(); + const kv = new StateKV(sdk as never); + for (const scope of [KV.captureReview + ':other', KV.captureInbox + ':other', 'mem:capture:events:ab']) { + expect(await kv.list(scope)).toEqual([]); + expect(list).toHaveBeenCalledWith({ scope }); + } +}); + +function completedRow(key: string) { + return { key, eventId: key, sessionId: 'synthetic-session', project: 'synthetic', hookType: 'post_tool_use', + observationId: 'obs-' + key, status: 'pending', storageCompletion: 'sdk', attempts: 1, + acceptedAt: '2026-11-01T00:00:00.000Z', updatedAt: '2026-11-01T00:00:00.000Z', + payload: { hookType: 'post_tool_use', sessionId: 'synthetic-session', project: 'synthetic', + cwd: '/synthetic', timestamp: '2026-11-01T00:00:00.000Z', data: { synthetic: 'original' } } }; +} + +it('counts 5074 ordinary SDK completions with one keyed local review projection and no per-row history GET', async () => { + const path = await fixture(); + const rows = Object.fromEntries(Array.from({ length: 5074 }, (_, i) => { const row = completedRow('sdk-' + i); return [row.key, row]; })); + await writeFile(path, JSON.stringify(rows)); + const { sdk, list } = stateSdk(rows); + const kv = new StateKV(sdk as never); + const get = vi.spyOn(kv, 'get'); + const localList = vi.spyOn(kv.graphFileList!, 'list'); + const controller = registerCaptureFunctions(sdk as never, kv);controllers.push(controller); + expect((await controller.status()).inbox).toMatchObject({ sdkCompleted: 5074, pending: 0 }); + expect(get.mock.calls.filter(([scope]) => scope === KV.captureReview)).toHaveLength(0); + expect(localList.mock.calls.filter(([scope]) => scope === KV.captureReview)).toHaveLength(1); + expect(list.mock.calls.some(([input]) => input.scope === KV.captureReview)).toBe(false); +}); + +it('passes actual review storage keys through disk and acknowledged projections and retains deletion tombstones', async () => { + const path = await fixture(); + const reviewPath = join(dirname(path), encodeURIComponent(KV.captureReview) + '.bin'); + await writeFile(reviewPath, '{"disk-key":{"key":"value-alias"}}'); + const { sdk, list } = stateSdk();const kv = new StateKV(sdk as never); + const project = (value: any, key?: string) => ({ key, embeddedKey: value.key }); + expect(await kv.list(KV.captureReview, { project })).toEqual([{ key: 'disk-key', embeddedKey: 'value-alias' }]); + await kv.set(KV.captureReview, 'ack-key', { key: 'ack-alias' }); + expect(await kv.list(KV.captureReview, { project })).toContainEqual({ key: 'ack-key', embeddedKey: 'ack-alias' }); + await kv.delete(KV.captureReview, 'ack-key'); + await writeFile(reviewPath, '{}');expect(await kv.list(KV.captureReview)).toEqual([]); + await writeFile(reviewPath, '{"ack-key":{"key":"ack-alias"}}'); + expect(await kv.list(KV.captureReview)).toEqual([]); + expect(list.mock.calls.some(([input]) => input.scope === KV.captureReview)).toBe(false); +}); + +it.each([null, [], 'malformed', {}, { key: 'wrong-key' }].map(value => [value]))('does not count SDK completion from malformed or miskeyed local history %j', async history => { + const path = await fixture();const row = completedRow('selected'); + await writeFile(path, JSON.stringify({ selected: row })); + await writeFile(join(dirname(path), encodeURIComponent(KV.captureReview) + '.bin'), JSON.stringify({ selected: history })); + const { sdk } = stateSdk({ selected: row });const kv = new StateKV(sdk as never); + const controller = registerCaptureFunctions(sdk as never, kv);controllers.push(controller); + expect((await controller.status()).inbox).toBeNull(); +}); + +it('rechecks full canonical resolution and original payload only for locally marked history', async () => { + const path = await fixture();const requestId = '00000000-0000-4000-8000-000000000001'; + const row = { ...completedRow('resolved'), sdkResolutionId: requestId, sdkResolutionState: 'committed' }; + const history: any = { key: row.key, eventId: row.eventId, sessionId: row.sessionId, project: row.project, + hookType: row.hookType, observationId: row.observationId, status: 'completed', totalAttempts: 1, + cycleAttempts: 1, manualRetries: 0, failures: [], requests: {}, acceptedAt: row.acceptedAt, updatedAt: row.updatedAt }; + history.sdkResolution = { version: 1, completion: 'sdk', phase: 'committed', requestId, + sourceDigest: capturePayloadDigest(row.payload), initialProjection: 'a'.repeat(64), + priorFailureDigest: captureFailureHistoryDigest(history), priorInboxState: 'dead', + startedAt: row.acceptedAt, completedAt: row.updatedAt, verifiedProjection: 'b'.repeat(64) }; + await writeFile(path, JSON.stringify({ resolved: row })); + await writeFile(join(dirname(path), encodeURIComponent(KV.captureReview) + '.bin'), JSON.stringify({ resolved: history })); + const { sdk, store } = stateSdk({ resolved: row });store.set(KV.captureReview, new Map([[row.key, history]])); + const kv = new StateKV(sdk as never);const get = vi.spyOn(kv, 'get'); + const controller = registerCaptureFunctions(sdk as never, kv);controllers.push(controller); + expect((await controller.status()).inbox).toMatchObject({ sdkCompleted: 1 }); + expect(get.mock.calls.filter(([scope]) => scope === KV.captureReview)).toHaveLength(1); + expect(get.mock.calls.filter(([scope]) => scope === KV.captureInbox)).toHaveLength(1); + store.get(KV.captureInbox)!.set(row.key, { ...row, payload: { ...row.payload, data: { synthetic: 'changed' } } }); + expect((await controller.status()).inbox).toMatchObject({ sdkCompleted: 0 }); +}); + +it('does not turn a projected resolution that disappears from the SDK into an ordinary completion', async () => { + const path = await fixture();const row = completedRow('selected'); + await writeFile(path, JSON.stringify({ selected: row })); + await writeFile(join(dirname(path), encodeURIComponent(KV.captureReview) + '.bin'), '{"selected":{"key":"selected","sdkResolution":null}}'); + const { sdk } = stateSdk({ selected: row });const kv = new StateKV(sdk as never); + const controller = registerCaptureFunctions(sdk as never, kv);controllers.push(controller); + expect((await controller.status()).inbox).toMatchObject({ sdkCompleted: 0 }); +}); + +it('sees an acknowledged review resolution before its file checkpoint and refuses an uncertain review mutation', async () => { + const path = await fixture();const row = completedRow('selected'); + await writeFile(path, JSON.stringify({ selected: row })); + const { sdk } = stateSdk({ selected: row });const kv = new StateKV(sdk as never); + const controller = registerCaptureFunctions(sdk as never, kv);controllers.push(controller); + expect((await controller.status()).inbox).toMatchObject({ sdkCompleted: 1 }); + await kv.set(KV.captureReview, row.key, { key: row.key, sdkResolution: null }); + expect((await controller.status()).inbox).toMatchObject({ sdkCompleted: 0 }); + sdk.registerFunction('state::set', async () => { throw Error('synthetic unknown SDK mutation'); }); + await expect(kv.set(KV.captureReview, row.key, { key: row.key })).rejects.toThrow('synthetic unknown SDK mutation'); + expect((await controller.status()).inbox).toBeNull(); +}); + +it('keeps per-row history GET and unknown physical projection keys without a local reader', async () => { + vi.stubEnv('AGENTMEMORY_STATE_STORE_PATH', ''); + const row = completedRow('legacy');const { sdk, list } = stateSdk({ legacy: row }); + list.mockImplementation(async () => [row]); + const kv = new StateKV(sdk as never, { durability: createStateDurabilityFence({ storePath: '/synthetic-only', completion: 'sdk' }) }); + const get = vi.spyOn(kv, 'get'); + const controller = registerCaptureFunctions(sdk as never, kv);controllers.push(controller); + expect((await controller.status()).inbox).toMatchObject({ sdkCompleted: 1 }); + expect(get.mock.calls.filter(([scope]) => scope === KV.captureReview)).toHaveLength(1); + expect(await kv.list(KV.captureInbox, { project: (_value, key) => key })).toEqual([undefined]); +}); diff --git a/test/capture-receipt-digest.test.ts b/test/capture-receipt-digest.test.ts index 7424f7c60c..db634232a6 100644 --- a/test/capture-receipt-digest.test.ts +++ b/test/capture-receipt-digest.test.ts @@ -9,10 +9,10 @@ it('matches the full sanitized Unicode source regardless of object field order a expect(capturePayloadDigest(body)).not.toBe(captureSourceDigest(body.hookType,body.timestamp,{a:stored.a,z:stored.z.slice(0,16000)})); }); it('accounts for sourceRaw duplication near the existing transport ceiling without truncating it',()=>{ - const tool='x'.repeat(7_034_880),raw={raw:{tool_output:tool},toolOutput:tool}; - expect(payloadByteLength({sourceRaw:raw})).toBeGreaterThan(14_000_000); + const sourceLength=Math.floor(SAFE_PAYLOAD_BYTES*.45),tool='x'.repeat(sourceLength),raw={raw:{tool_output:tool},toolOutput:tool}; + expect(payloadByteLength({sourceRaw:raw})).toBeGreaterThan(SAFE_PAYLOAD_BYTES*.89); expect(payloadByteLength({sourceRaw:raw})).toBeLessThan(SAFE_PAYLOAD_BYTES); const unsupported={raw:{tool_output:tool,tool_response:tool},toolOutput:tool}; expect(payloadByteLength({sourceRaw:unsupported})).toBeGreaterThan(SAFE_PAYLOAD_BYTES); - expect(raw.raw.tool_output.length).toBe(7_034_880); + expect(raw.raw.tool_output.length).toBe(sourceLength); }); diff --git a/test/capture-sdk-reconciliation-api.test.ts b/test/capture-sdk-reconciliation-api.test.ts index f2e0bfdfa6..ea96497aad 100644 --- a/test/capture-sdk-reconciliation-api.test.ts +++ b/test/capture-sdk-reconciliation-api.test.ts @@ -21,6 +21,44 @@ it('reads explicit recorded outcome without claiming disk proof or current sourc it('exposes current index membership in one body-free readonly function',async()=>{const f=await fixture();const result=await f.sdk.fns.get('api::capture-index-membership')!({headers:f.headers,query_params:{ids:id}})as any;expect(result).toMatchObject({status_code:200,body:{sourceVerified:false,diskVerified:false,items:[{id,keywordPresent:true,vectorPresent:true}]}});expect(JSON.stringify(result)).not.toContain('private synthetic original');expect(f.writes).not.toHaveBeenCalled()}); it('stops admission after capture shutdown and drains existing tracked work',async()=>{const f=await fixture();f.control.stop();expect((await f.call()).body.completed).toBe(false);await f.control.drain();expect(f.writes).not.toHaveBeenCalled()}); +it('does not read inbox for automatic viewer status and keeps result unknown',async()=>{ + const f=await fixture();const list=vi.spyOn(f.kv,'list'); + const {createStatusReporter}=await import('../src/triggers/api.js'); + const report=createStatusReporter(f.sdk as never,f.kv as never,{}); + const result=await report({health:null,scanUnindexed:false,scanCaptureInbox:false} as never); + expect(list.mock.calls.filter(([scope])=>scope===KV.captureInbox)).toHaveLength(0); + expect(result.capture?.inbox).toBeNull(); + expect(result.problems.some(p=>p.code==='capture-check-unavailable')).toBe(true); +}); +it('retains the explicit authenticated capture inbox check',async()=>{ + const f=await fixture();const list=vi.spyOn(f.kv,'list'); + const status=await f.control.status();expect(status.inbox).not.toBeNull(); + expect(list.mock.calls.filter(([scope])=>scope===KV.captureInbox)).toHaveLength(1); +}); +it('skips inbox reads in viewer snapshot and health events',async()=>{ + const f=await fixture();Object.assign(f.kv,{countSummaries:async()=>0}); + const {registerViewerStreamTriggers}=await import('../src/triggers/viewer-streams.js'); + registerViewerStreamTriggers(f.sdk as never,f.kv as never,{}); + const list=vi.spyOn(f.kv,'list'); + const snapshot=await f.sdk.fns.get('api::viewer-snapshot')!({headers:{}}) as any; + expect(snapshot.status_code).toBe(200);expect(snapshot.body.report.capture.inbox).toBeNull(); + list.mockClear(); + const healthEvent=await f.sdk.fns.get('event::viewer::health-changed')!({key:'latest',event_type:'state:updated',new_value:{status:'healthy',alerts:[]}}) as any; + expect(healthEvent).toMatchObject({emitted:true}); + expect(list.mock.calls.filter(([scope])=>scope===KV.captureInbox)).toHaveLength(0); +}); +it('does not report healthy when capture pending work is unknown',async()=>{ + const f=await fixture();const {operationalState}=await import('../src/ops/context.js'); + const prior=operationalState.captureProgress; + try{ + operationalState.captureProgress={snapshot:()=>({pending:null,stalled:false})} as never; + await f.kv.set(KV.health,'latest',{status:'healthy',alerts:[]}); + const response=await f.sdk.fns.get('api::health')!({}) as any; + expect(response.body.status).toBe('degraded'); + expect(response.body.captureProgress.pending).toBeNull(); + }finally{operationalState.captureProgress=prior} +}); + it('holds drain until an in-flight reconciliation exits and stops before further writes after shutdown',async()=>{ const f=await fixture();let entered!:()=>void,release!:()=>void;const reached=new Promise(r=>entered=r),blocked=new Promise(r=>release=r);const get=f.kv.get.bind(f.kv);let intercepted=false; f.kv.get=async(s,k)=>{if(!intercepted&&s===KV.captureInbox&&k===key){intercepted=true;entered();await blocked;}return get(s,k)}; @@ -53,6 +91,18 @@ it('bounds SDK archive status history reads to four while preserving validated c expect(peak).toBeLessThanOrEqual(4); }); +it('real viewer health update omits the inbox scan',async()=>{const f=await fixture();Object.assign(f.kv,{countSummaries:async()=>0});const {registerViewerStreamTriggers}=await import('../src/triggers/viewer-streams.js');registerViewerStreamTriggers(f.sdk as never,f.kv as never,{});const list=vi.spyOn(f.kv,'list');const event=await f.sdk.fns.get('event::viewer::health-changed')!({key:'latest',event_type:'state:updated',new_value:{status:'healthy',alerts:[]}}) as any;expect(event).toMatchObject({emitted:true});expect(list.mock.calls.filter(([scope])=>scope===KV.captureInbox)).toHaveLength(0)}); +it('returns a retryable service error when remember is refused by the vector hold',async()=>{const f=await fixture();f.sdk.fns.set('mem::remember',async()=>({success:false,error:'vector_recovery_paused',retryable:true}));const out=await f.sdk.fns.get('api::remember')!({headers:{authorization:'Bearer synthetic-secret'},body:{content:'synthetic'}}) as any;expect(out).toMatchObject({status_code:503,body:{success:false,error:'vector_recovery_paused',retryable:true}})}); +it.each([['forget','mem::forget',{memoryId:'synthetic'}],['evict','mem::evict',{}],['evolve','mem::evolve',{memoryId:'synthetic',newContent:'synthetic'}],['governance-delete','mem::governance-delete',{memoryIds:['synthetic']}],['governance-bulk','mem::governance-bulk',{}]])('returns a retryable 503 for held %s',async(api,fn,body)=>{const f=await fixture();f.sdk.fns.set(fn,async()=>({success:false,error:'vector_recovery_paused',retryable:true}));const out=await f.sdk.fns.get('api::'+api)!({headers:{authorization:'Bearer synthetic-secret'},body,query_params:{}}) as any;expect(out).toMatchObject({status_code:503,body:{success:false,error:'vector_recovery_paused',retryable:true}})}); +it.each([['auto-forget','mem::auto-forget',{}],['snapshot-restore','mem::snapshot-restore',{commitHash:'synthetic'}]])('returns retryable 503 for held %s',async(api,fn,body)=>{const f=await fixture();f.sdk.fns.set(fn,async()=>({success:false,error:'vector_recovery_paused',retryable:true}));const out=await f.sdk.fns.get('api::'+api)!({headers:{authorization:'Bearer synthetic-secret'},body,query_params:{}}) as any;expect(out).toMatchObject({status_code:503,body:{success:false,error:'vector_recovery_paused',retryable:true}})}); +it.each([['import','mem::import',{exportData:{}}],['replay::import','mem::replay::import-jsonl',{}]])('returns retryable 503 for held %s',async(api,fn,body)=>{const f=await fixture();f.sdk.fns.set(fn,async()=>({success:false,error:'vector_recovery_paused',retryable:true}));const out=await f.sdk.fns.get('api::'+api)!({headers:{authorization:'Bearer synthetic-secret'},body,query_params:{}}) as any;expect(out).toMatchObject({status_code:503,body:{success:false,error:'vector_recovery_paused',retryable:true}})}); +it('inspects an uncertain item by original identifiers and content without writes or replay',async()=>{const f=await fixture();const inspect=f.sdk.fns.get('api::capture-inspect-uncertain');expect(inspect).toBeTypeOf('function');const out=await inspect!({headers:f.headers,body:{key,eventId,sourceDigest:capturePayloadDigest(payload),project:payload.project,sessionId:payload.sessionId,cwd:payload.cwd,hookType:payload.hookType,timestamp:payload.timestamp}}) as any;expect(out).toMatchObject({status_code:200,body:{outcome:'observation_matches',readOnly:true,diskVerified:false,originalReplay:false,retryAuthorized:false}});expect(f.writes).not.toHaveBeenCalled()}); +it('does not infer retry is safe when an uncertain observation is absent or source differs',async()=>{const f=await fixture();await f.kv.delete(KV.observations(payload.sessionId),id);const inspect=f.sdk.fns.get('api::capture-inspect-uncertain');expect(inspect).toBeTypeOf('function');const absent=await inspect!({headers:f.headers,body:{key,eventId,sourceDigest:capturePayloadDigest(payload),project:payload.project,sessionId:payload.sessionId,cwd:payload.cwd,hookType:payload.hookType,timestamp:payload.timestamp}}) as any;expect(absent).toMatchObject({status_code:200,body:{outcome:'observation_not_found',retryAuthorized:false,diskVerified:false}});const mismatch=await inspect!({headers:f.headers,body:{key,eventId,sourceDigest:'0'.repeat(64),project:payload.project,sessionId:payload.sessionId,cwd:payload.cwd,hookType:payload.hookType,timestamp:payload.timestamp}}) as any;expect(mismatch.status_code).toBe(409);expect(f.writes).not.toHaveBeenCalled()}); +it('allows the SDK caller metadata without accepting extra inspection fields',async()=>{const f=await fixture();const direct=f.sdk.fns.get('mem::capture-inspect-uncertain')!;expect(await direct({key,eventId,sourceDigest:capturePayloadDigest(payload),project:payload.project,sessionId:payload.sessionId,cwd:payload.cwd,hookType:payload.hookType,timestamp:payload.timestamp,_caller_worker_id:'synthetic-worker'})).toMatchObject({outcome:'observation_matches',readOnly:true});await expect(direct({key,eventId,sourceDigest:capturePayloadDigest(payload),project:payload.project,sessionId:payload.sessionId,cwd:payload.cwd,hookType:payload.hookType,timestamp:payload.timestamp,extra:true})).rejects.toThrow();expect(f.writes).not.toHaveBeenCalled()}); + +it('requires a configured secret and exact original metadata for uncertain inspection',async()=>{const f=await fixture(),body={key,eventId,sourceDigest:capturePayloadDigest(payload),project:payload.project,sessionId:payload.sessionId,cwd:payload.cwd,hookType:payload.hookType,timestamp:payload.timestamp};const handler=f.sdk.fns.get('api::capture-inspect-uncertain')!;expect((await handler({headers:{},body}) as any).status_code).toBe(401);expect((await handler({headers:f.headers,body:{...body,cwd:'/different'}}) as any).status_code).toBe(409);registerApiTriggers(f.sdk as never,f.kv as never,'');const unconfigured=await f.sdk.fns.get('api::capture-inspect-uncertain')!({headers:{},body}) as any;expect(unconfigured.status_code).toBe(503);expect(f.writes).not.toHaveBeenCalled()}); +it('refuses changing observations and reports read failures without mutation',async()=>{const f=await fixture(),body={key,eventId,sourceDigest:capturePayloadDigest(payload),project:payload.project,sessionId:payload.sessionId,cwd:payload.cwd,hookType:payload.hookType,timestamp:payload.timestamp},handler=f.sdk.fns.get('api::capture-inspect-uncertain')!,get=f.kv.get;let reads=0;f.kv.get=async(scope:string,k:string)=>{const v=await get(scope,k);if(scope===KV.observations(payload.sessionId)&&++reads===2)return {...v as object,title:'changed'} as T;return v};expect((await handler({headers:f.headers,body}) as any)).toMatchObject({status_code:409,body:{outcome:'changed_during_inspection',retryAuthorized:false}});f.kv.get=async()=>{throw Error('synthetic read unavailable')};expect((await handler({headers:f.headers,body}) as any)).toMatchObject({status_code:503,body:{error:'UNCERTAIN_INSPECTION_UNAVAILABLE'}});expect(f.writes).not.toHaveBeenCalled()}); + it('keeps timed-out or overlapping inbox calculations single while returning unavailable rather than stale counts',async()=>{ const f=await fixture();await f.call(); const list=f.kv.list.bind(f.kv);let release!:()=>void,entered!:()=>void,inboxReads=0; @@ -125,9 +175,32 @@ it('uses the bounded proof page in the authenticated capture HTTP path and rejec it('keeps next offsets usable after more than ten thousand retained SDK originals',async()=>{ const f=await fixture();await f.call();const rec=await f.kv.get(KV.captureInbox,key),{sdkResolutionId,sdkResolutionState,...ordinary}=rec; const records=Array.from({length:10033},(_,i)=>({...ordinary,key:'cap_'+i.toString(16).padStart(40,'0')})),list=f.kv.list.bind(f.kv); - f.kv.list=async scope=>scope===KV.captureInbox?records:list(scope); + f.kv.list=async (scope,options)=>scope===KV.captureInbox?options?.project?records.map(row=>options.project!(row,row.key)):records:list(scope,options); const first=await f.control.status({sdkProofPage:{offset:0,limit:32}} as never),generation=first.inbox!.sdkProofPage!.generation; const near=await f.control.status({sdkProofPage:{offset:9984,limit:32,generation}} as never);expect(near.inbox?.sdkProofPage?.nextOffset).toBe(10016); const last=await f.control.status({sdkProofPage:{offset:10016,limit:32,generation}} as never);expect(last.inbox?.sdkProofPage).toMatchObject({checked:17,nextOffset:null}); const response=await f.sdk.fns.get('api::capture')!({headers:f.headers,query_params:{proofOffset:'10016',proofGeneration:generation}}) as any;expect(response.status_code).toBe(200);expect(response.body.capture.inbox.sdkProofPage.checked).toBe(17); }); +it('reconciliation releases only its own pending membership without rescanning preserved sources',async()=>{ + vi.stubEnv('AGENTMEMORY_CAPTURE_INBOX_MAX','1'); + try{ + const f=await fixture(),list=vi.spyOn(f.kv,'list'); + const capture=()=>f.sdk.trigger('mem::capture',{payload:{...payload,data:{tool_output:'new synthetic'}},eventId:'synthetic_capacity_after_review'}); + expect(await capture()).toMatchObject({status:'rejected'}); + expect((await f.call()).body).toMatchObject({completed:true,verified:false}); + f.sdk.fns.set('mem::observe',async(p:any)=>({observationId:p.observationId})); + expect(await capture()).toMatchObject({status:'accepted',state:'sdk_completed'}); + expect(list.mock.calls.filter(([scope])=>scope===KV.captureInbox)).toHaveLength(1); + expect(await f.kv.get(KV.captureInbox,key)).toMatchObject({storageCompletion:'sdk',payload}); + expect(await f.kv.get(KV.captureReview,key)).toMatchObject({failures:[{code:'CAPTURE_STORE_TIMEOUT'}]}); + }finally{vi.unstubAllEnvs();} +}); +it('reconciliation waiting for a capture key never holds the admission lock in reverse order',async()=>{ + const {withKeyedLock}=await import('../src/state/keyed-mutex.js');const f=await fixture(); + let release!:()=>void,entered!:()=>void;const held=new Promise(r=>entered=r),gate=new Promise(r=>release=r); + const owner=withKeyedLock('capture:'+key,async()=>{entered();await gate;});await held; + const reconciliation=f.call();await new Promise(r=>setTimeout(r,20)); + const admission=withKeyedLock('capture:inbox-admission',async()=>true); + try{expect(await Promise.race([admission,new Promise(r=>setTimeout(()=>r(false),100))])).toBe(true);} + finally{release();await owner;await reconciliation;await admission;} +}); diff --git a/test/capture-sdk-reconciliation.test.ts b/test/capture-sdk-reconciliation.test.ts index 70a198c7b3..5bbe23e6d6 100644 --- a/test/capture-sdk-reconciliation.test.ts +++ b/test/capture-sdk-reconciliation.test.ts @@ -1,7 +1,10 @@ import{it,expect}from'vitest';import{createHash}from'node:crypto'; -import{mockKV}from'./helpers/mocks.js';import{KV}from'../src/state/schema.js';import{createRecordRecoveryGate}from'../src/state/recovery-write-gate.js';import{captureEventScope}from'../src/capture/event-record.js';import{createSdkCaptureReconciliation}from'../src/capture/sdk-reconciliation.js'; +import{mockKV,mockSdk}from'./helpers/mocks.js';import{KV}from'../src/state/schema.js';import{createRecordRecoveryGate}from'../src/state/recovery-write-gate.js';import{captureEventScope}from'../src/capture/event-record.js';import{createSdkCaptureReconciliation}from'../src/capture/sdk-reconciliation.js'; import{capturePayloadDigest}from'../src/capture/receipt-digest.js'; import{assertSdkSourceLegacyProtection}from'../src/capture/sdk-source-restore.js'; +import{committedInboxMetadata,projectCaptureInbox}from'../src/capture/inbox-metadata.js'; +import{captureReviewPage}from'../src/capture/review.js'; +import{registerCaptureFunctions}from'../src/functions/capture.js'; const req='00000000-0000-4000-8000-000000000001',otherReq='00000000-0000-4000-8000-000000000002'; const payload={sessionId:'synthetic',project:'synthetic',cwd:'/synthetic',hookType:'post_tool_use',timestamp:'2026-10-29T00:00:00Z',data:{tool_output:'original synthetic source'}}; const eventId='synthetic_reconcile_001',key='cap_'+createHash('sha256').update(payload.project+'\0'+payload.sessionId+'\0'+eventId).digest('hex').slice(0,40),id='obs_synthetic_reconcile'; @@ -23,7 +26,7 @@ async function fixture(saved?:Map>){ } writes=0; const auditEntries:any[]=[]; - const make=()=>createSdkCaptureReconciliation(kv as never,{admitted:()=>admitted,indexEvidence:(obs:any)=>({keyword:{present:indexed,sessionId:indexed?owner:undefined},vectorSessionId:vectorOwner}),audit:async e=>{auditEntries.push(e)},now:()=>Date.parse('2026-10-29T01:00:00Z')}); + const make=(verifyCaptureVectorPending?:(id:string,sessionId:string,pending:unknown)=>Promise)=>createSdkCaptureReconciliation(kv as never,{admitted:()=>admitted,verifyCaptureVectorPending,indexEvidence:(obs:any)=>({keyword:{present:indexed,sessionId:indexed?owner:undefined},vectorSessionId:vectorOwner}),audit:async e=>{auditEntries.push(e)},now:()=>Date.parse('2026-10-29T01:00:00Z')}); return{kv,gate,get,set,helper:make(),make,writeCount:()=>writes,pause:()=>admitted=false,unindex:()=>indexed=false,keywordOwner:(v:string|undefined)=>owner=v,vectorOwner:(v:string|undefined)=>vectorOwner=v,fail:(n:number,stored=false)=>{failAt=n;after=stored},resetWrites:()=>writes=0,call(){return this.helper.reconcile({key,requestId:req,sourceDigest})}}; } it('reconciles an actually stored capture with SDK completion and preserves every original and failure/budget field',async()=>{ @@ -35,6 +38,26 @@ it('reconciles an actually stored capture with SDK completion and preserves ever expect(await f.kv.get(KV.observations(payload.sessionId),id)).toEqual(obs);expect(await f.kv.get(KV.captureRecovery,'control')).toEqual(control);expect(await f.kv.get(captureEventScope(key),key)).toBeNull();expect(f.writeCount()).toBe(4); }); it('same or later request on an already reconciled key verifies without replay or new writes',async()=>{const f=await fixture();await f.call();const writes=f.writeCount();expect(await f.call()).toMatchObject({status:'already_reconciled',completed:true});expect(await f.helper.reconcile({key,requestId:otherReq,sourceDigest})).toMatchObject({status:'already_reconciled',requestId:req});expect(f.writeCount()).toBe(writes)}); +it.each([1,2])('preserves v%s completion through projected metadata while checking the current original',async version=>{ + const f=await fixture();await f.call();const h=await f.kv.get(KV.captureReview,key); + if(version===1){h.sdkResolution.version=1;delete h.sdkResolution.indexEvidence;delete h.sdkResolution.priorInboxDigest;delete h.sdkResolution.priorReviewState;await f.kv.set(KV.captureReview,key,h)} + const rec=await f.kv.get(KV.captureInbox,key),metadata=projectCaptureInbox(rec); + const controller=registerCaptureFunctions(mockSdk() as never,f.kv as never); + try{ + expect(metadata).not.toHaveProperty('payload'); + expect(await committedInboxMetadata(f.kv as never,h,metadata)).toBe(true); + expect((await controller.status()).inbox?.sdkCompleted).toBe(1); + expect((await captureReviewPage(f.kv as never,{})).items[0].status).toBe('completed'); + const changed=structuredClone(rec);changed.payload.data.tool_output='changed original';await f.kv.set(KV.captureInbox,key,changed); + expect(await committedInboxMetadata(f.kv as never,h,metadata)).toBe(false); + expect((await controller.status()).inbox?.sdkCompleted).toBe(0); + expect((await captureReviewPage(f.kv as never,{})).items[0].status).toBe('pending'); + await f.kv.set(KV.captureInbox,key,{...rec,updatedAt:'2026-11-07T00:00:00Z'}); + expect(await committedInboxMetadata(f.kv as never,h,metadata)).toBe(false); + await f.kv.delete(KV.captureInbox,key); + expect(await committedInboxMetadata(f.kv as never,h,metadata)).toBe(false); + }finally{controller.stop()} +}); it.each(['source','identity','index','deleted','queued','writer','paused','manual'])('refuses unsafe %s before any write',async mode=>{ const f=await fixture();if(mode==='source'){const obs=await f.kv.get(KV.observations(payload.sessionId),id);obs.sourceRaw.raw={tool_output:'different'};await f.kv.set(KV.observations(payload.sessionId),id,obs)} if(mode==='identity'){const obs=await f.kv.get(KV.observations(payload.sessionId),id);obs.captureKey='cap_'+'b'.repeat(40);await f.kv.set(KV.observations(payload.sessionId),id,obs)} @@ -91,3 +114,14 @@ it('whole-review source restore mutation retains completed v2 protection',async( await f.kv.set(KV.captureInbox,key,{...rec,sdkRestoreRequestId:otherReq,sdkRestoreState:'pending',sdkRestoreAttempts:0}); await expect(assertSdkSourceLegacyProtection(f.kv as never,payload as never,key,id,eventId)).resolves.toBeUndefined(); }); + +it('requires current physical proof on each phase for a retained capture and keeps the original WAL',async()=>{ + const f=await fixture(),row={id,s:payload.sessionId,q:3,e:'AACAPw=='}; + await f.set(KV.vectorPendingLog,id,row); + await expect(f.call()).rejects.toThrow('SDK_RECONCILIATION_INDEX_PENDING'); + let checks=0; + const repaired=f.make(async(obs,session,actual)=>{checks++;return obs===id && session===payload.sessionId && JSON.stringify(actual)===JSON.stringify(row);}); + expect(await repaired.reconcile({key,requestId:req,sourceDigest})).toMatchObject({status:'reconciled',verified:false,synced:false,originalReplay:false}); + expect(checks).toBeGreaterThanOrEqual(5);expect(await f.kv.get(KV.vectorPendingLog,id)).toEqual(row); + const stale=f.make(async()=>false);await expect(stale.reconcile({key,requestId:req,sourceDigest})).rejects.toThrow('SDK_RECONCILIATION_INDEX_PENDING'); +}); diff --git a/test/capture-sdk-resolution-restore.test.ts b/test/capture-sdk-resolution-restore.test.ts index 092f2aa65f..3270112a60 100644 --- a/test/capture-sdk-resolution-restore.test.ts +++ b/test/capture-sdk-resolution-restore.test.ts @@ -45,3 +45,17 @@ for(const missing of [false,true])it(`whole-review actual v2 restore consumer mi expect(await f.kv.list(KV.observations(body.sessionId))).toEqual(observationsBeforeRepeat); expect(observationsBeforeRepeat).toHaveLength(1); }); + +it('SDK source restore preserves originals without consuming pending capacity or bypassing pending admission',async()=>{ + vi.stubEnv('AGENTMEMORY_CAPTURE_INBOX_MAX','1'); + try{ + const f=fixture();const pending={key:'synthetic-pending-capacity',status:'dead',acceptedAt:body.timestamp,updatedAt:body.timestamp,payload:{original:true}}; + await f.kv.set(KV.captureInbox,pending.key,pending); + expect((await f.call()).body).toMatchObject({completed:true}); + const source=await f.kv.get(KV.captureInbox,f.key);expect(source).toMatchObject({storageCompletion:'sdk',sdkRestoreState:'completed'}); + expect((await f.call()).body).toMatchObject({completed:true,status:'duplicate'}); + const capture=await f.sdk.trigger('mem::capture',{payload:{...body,eventId:'synthetic_new_capacity'},eventId:'synthetic_new_capacity'}); + expect(capture).toMatchObject({status:'rejected',retryable:true});expect(await f.kv.get(KV.captureInbox,pending.key)).toEqual(pending); + expect(await f.kv.list(KV.captureInbox)).toHaveLength(2); + }finally{vi.unstubAllEnvs();} +}); diff --git a/test/circuit-breaker.test.ts b/test/circuit-breaker.test.ts index be7489d9b7..52cd6d0e76 100644 --- a/test/circuit-breaker.test.ts +++ b/test/circuit-breaker.test.ts @@ -127,4 +127,65 @@ describe("CircuitBreaker", () => { expect(cb.getState().state).toBe("closed"); expect(cb.getState().failures).toBe(0); }); + + it("a success in closed state clears the accumulated failure count", () => { + const cb = new CircuitBreaker(); + cb.recordFailure(); + cb.recordFailure(); + expect(cb.getState().failures).toBe(2); + cb.recordSuccess(); + expect(cb.getState().failures).toBe(0); + expect(cb.getState().lastFailureAt).toBeNull(); + }); + + it("stays closed across 100 calls at a 5% failure rate", () => { + const cb = new CircuitBreaker(); + for (let i = 0; i < 100; i++) { + if (i % 20 === 0) cb.recordFailure(); + else cb.recordSuccess(); + vi.advanceTimersByTime(1_000); + expect(cb.getState().state).toBe("closed"); + } + expect(cb.isAllowed).toBe(true); + }); + + it("stays closed when failures are interleaved with successes inside the window", () => { + const cb = new CircuitBreaker(); + for (let i = 0; i < 5; i++) { + cb.recordFailure(); + cb.recordSuccess(); + } + expect(cb.getState().state).toBe("closed"); + expect(cb.getState().failures).toBe(0); + }); + + it("keeps openedAt when a success arrives while open", () => { + const cb = new CircuitBreaker(); + cb.recordFailure(); + cb.recordFailure(); + cb.recordFailure(); + expect(cb.getState().state).toBe("open"); + const openedAt = cb.getState().openedAt; + + cb.recordSuccess(); + + expect(cb.getState().state).toBe("open"); + expect(cb.getState().openedAt).toBe(openedAt); + expect(cb.isAllowed).toBe(false); + + vi.advanceTimersByTime(30_000); + expect(cb.isAllowed).toBe(true); + expect(cb.getState().state).toBe("half-open"); + }); + + it("still opens after consecutive failures with no success between them", () => { + const cb = new CircuitBreaker(); + cb.recordSuccess(); + for (let i = 0; i < 3; i++) { + cb.recordFailure(); + vi.advanceTimersByTime(1_000); + } + expect(cb.getState().state).toBe("open"); + expect(cb.isAllowed).toBe(false); + }); }); diff --git a/test/claude-code-with-hooks.test.ts b/test/claude-code-with-hooks.test.ts index e3d35af552..c79acb3cd1 100644 --- a/test/claude-code-with-hooks.test.ts +++ b/test/claude-code-with-hooks.test.ts @@ -1,5 +1,6 @@ import { afterEach, beforeEach, describe, it, expect, vi } from "vitest"; -import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { spawnSync } from "node:child_process"; +import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; import { @@ -41,6 +42,13 @@ describe("buildMergedHooks against plugin/hooks/hooks.json (Claude Code)", () => ).toBe(true); }); + it("gives SessionEnd more time than Claude Code's 1.5 s default so the transcript back-fill and exit timer can finish", () => { + const merged = buildMergedHooks(null, PLUGIN_ROOT, "hooks.json"); + const handler = merged.hooks["SessionEnd"]![0]!.hooks[0]!; + expect(handler.args![0]).toContain("session-end.mjs"); + expect(handler.timeout).toBeGreaterThanOrEqual(10); + }); + it("appends to existing user hooks without dropping them", () => { const existing: HookManifest = { hooks: { @@ -109,6 +117,7 @@ describe("connect claude-code --with-hooks", () => { home = mkdtempSync(join(tmpdir(), "am-claude-hooks-")); vi.stubEnv("HOME", home); vi.stubEnv("USERPROFILE", home); + vi.stubEnv("CLAUDE_CONFIG_DIR", undefined); vi.resetModules(); }); @@ -138,4 +147,452 @@ describe("connect claude-code --with-hooks", () => { ...buildMergedHooks(null, PLUGIN_ROOT, "hooks.json").hooks["Stop"]!, ]); }); + + it("detects Claude Code in CLAUDE_CONFIG_DIR without a default directory", async () => { + const configDir = join(home, "custom Claude config"); + mkdirSync(configDir); + vi.stubEnv("CLAUDE_CONFIG_DIR", configDir); + + const { adapter } = await import("../src/cli/connect/claude-code.js"); + expect(adapter.detect()).toBe(true); + expect(existsSync(join(home, ".claude"))).toBe(false); + }); + + it.skipIf(process.platform === "win32")("preserves a literal trailing space instead of changing a different profile", async () => { + const configDir = join(home, "custom Claude config "); + const trimmedDir = configDir.trim(); + mkdirSync(configDir); + mkdirSync(trimmedDir); + writeFileSync(join(trimmedDir, ".claude.json"), '{"keep":"trimmed MCP"}'); + writeFileSync(join(trimmedDir, "settings.json"), '{"keep":"trimmed settings"}'); + vi.stubEnv("CLAUDE_CONFIG_DIR", configDir); + + const { adapter } = await import("../src/cli/connect/claude-code.js"); + expect(adapter.detect()).toBe(true); + expect(await adapter.install({ dryRun: false, force: false, withHooks: true })).toMatchObject({ + kind: "installed", + mutatedPath: join(configDir, ".claude.json"), + }); + expect(JSON.parse(readFileSync(join(configDir, "settings.json"), "utf-8")).hooks.SessionStart).toBeDefined(); + expect(readFileSync(join(trimmedDir, ".claude.json"), "utf-8")).toBe('{"keep":"trimmed MCP"}'); + expect(readFileSync(join(trimmedDir, "settings.json"), "utf-8")).toBe('{"keep":"trimmed settings"}'); + expect(existsSync(join(home, ".claude.json"))).toBe(false); + expect(existsSync(join(home, ".claude"))).toBe(false); + }); + + it("skips a custom profile directory that does not exist", async () => { + const configDir = join(home, "missing Claude config"); + vi.stubEnv("CLAUDE_CONFIG_DIR", configDir); + + const { resolveAdapter, runAdapter } = await import("../src/cli/connect/index.js"); + expect(await runAdapter(resolveAdapter("claude-code")!, { + dryRun: false, + force: false, + withHooks: true, + guidelines: false, + })).toEqual({ kind: "skipped", reason: "not-detected" }); + expect(existsSync(configDir)).toBe(false); + expect(existsSync(join(home, ".claude.json"))).toBe(false); + expect(existsSync(join(home, ".claude"))).toBe(false); + }); + + it.each(["custom", "default"])("skips a %s profile path that is a file", async (profile) => { + const configDir = join(home, profile === "custom" ? "Claude config" : ".claude"); + writeFileSync(configDir, "keep this file"); + if (profile === "custom") vi.stubEnv("CLAUDE_CONFIG_DIR", configDir); + + const { resolveAdapter, runAdapter } = await import("../src/cli/connect/index.js"); + expect(await runAdapter(resolveAdapter("claude-code")!, { + dryRun: false, + force: false, + withHooks: true, + guidelines: false, + })).toEqual({ kind: "skipped", reason: "not-detected" }); + expect(readFileSync(configDir, "utf-8")).toBe("keep this file"); + expect(existsSync(join(home, ".claude.json"))).toBe(false); + }); + + it("uses a profile override set after the adapter was imported for detection, MCP and hooks", async () => { + const { adapter } = await import("../src/cli/connect/claude-code.js"); + const configDir = join(home, "late Claude config"); + mkdirSync(configDir); + vi.stubEnv("CLAUDE_CONFIG_DIR", configDir); + + expect(adapter.detect()).toBe(true); + expect(await adapter.install({ dryRun: false, force: false, withHooks: true })).toMatchObject({ + kind: "installed", + mutatedPath: join(configDir, ".claude.json"), + }); + expect(JSON.parse(readFileSync(join(configDir, "settings.json"), "utf-8")).hooks.SessionStart).toBeDefined(); + expect(existsSync(join(home, ".claude.json"))).toBe(false); + expect(existsSync(join(home, ".claude"))).toBe(false); + }); + + describe.skipIf(process.platform === "win32")("CLI env-file hydration", () => { + it.each([ + { source: "default .env", customEnvFile: false, shellOverride: false, matchingShell: false, padding: "none" }, + { source: "AGENTMEMORY_ENV_FILE", customEnvFile: true, shellOverride: false, matchingShell: false, padding: "none" }, + { source: "shell overriding the env file", customEnvFile: true, shellOverride: true, matchingShell: false, padding: "none" }, + { source: "matching shell and env-file values", customEnvFile: true, shellOverride: true, matchingShell: true, padding: "none" }, + { source: "a literal env-file value with surrounding spaces", customEnvFile: true, shellOverride: false, matchingShell: false, padding: "both" }, + { source: "a literal shell value with leading spaces", customEnvFile: true, shellOverride: true, matchingShell: false, padding: "leading" }, + { source: "a literal shell value with trailing spaces", customEnvFile: true, shellOverride: true, matchingShell: false, padding: "trailing" }, + ])("installs MCP and hooks using $source", ({ customEnvFile, shellOverride, matchingShell, padding }) => { + const fileProfile = join(home, "env-file Claude config"); + const shellProfile = join(home, "shell Claude config"); + const withPadding = (value: string) => + `${padding === "leading" || padding === "both" ? " " : ""}${value}${padding === "trailing" || padding === "both" ? " " : ""}`; + const fileValue = shellOverride ? fileProfile : withPadding(fileProfile); + const selectedValue = shellOverride && !matchingShell ? withPadding(shellProfile) : fileValue; + const selected = resolve(home, selectedValue); + mkdirSync(fileProfile); + mkdirSync(shellProfile); + mkdirSync(selected, { recursive: true }); + mkdirSync(join(home, ".agentmemory")); + const envFile = customEnvFile + ? join(home, "custom.env") + : join(home, ".agentmemory", ".env"); + writeFileSync(envFile, `CLAUDE_CONFIG_DIR="${fileValue}"\n`); + + const result = spawnSync(process.execPath, [ + resolve(__dirname, "..", "dist", "cli.mjs"), + "connect", "claude-code", "--with-hooks", "--no-guidelines", + ], { + cwd: home, + env: { + ...process.env, + AGENTMEMORY_ENV_FILE: customEnvFile ? envFile : undefined, + CLAUDE_CONFIG_DIR: shellOverride ? selectedValue : undefined, + CI: "1", + NO_COLOR: "1", + }, + encoding: "utf-8", + timeout: 10000, + }); + + expect(result.status, result.stderr).toBe(0); + const output = result.stdout + result.stderr; + if (shellOverride) { + expect(output).not.toContain("CLAUDE_CONFIG_DIR was loaded only"); + } else { + expect(output).toContain("CLAUDE_CONFIG_DIR was loaded only from agentmemory's env file."); + expect(output).toContain(`Launch Claude Code with the same CLAUDE_CONFIG_DIR value ${JSON.stringify(selectedValue)}`); + expect(output.match(/CLAUDE_CONFIG_DIR was loaded only/g)).toHaveLength(1); + expect(output.indexOf("CLAUDE_CONFIG_DIR was loaded only")).toBeLessThan(output.indexOf("wired into")); + } + if (padding === "none" || padding === "trailing") { + expect(output).not.toContain("CLAUDE_CONFIG_DIR is relative"); + } + if (padding === "none") { + expect(output).not.toContain("CLAUDE_CONFIG_DIR contains leading or trailing whitespace"); + } else { + expect(output).toContain(`Using the path exactly as provided: ${JSON.stringify(selectedValue)}`); + expect(output).toContain("If this is unintended, correct CLAUDE_CONFIG_DIR and rerun this command."); + expect(output.match(/CLAUDE_CONFIG_DIR contains leading or trailing whitespace/g)).toHaveLength(1); + expect(output.indexOf("CLAUDE_CONFIG_DIR contains leading or trailing whitespace")).toBeLessThan(output.indexOf("wired into")); + } + expect(result.stdout).toContain(`wired into ${join(selectedValue, ".claude.json")}`); + expect(JSON.parse(readFileSync(join(selected, ".claude.json"), "utf-8")).mcpServers.agentmemory).toBeDefined(); + expect(JSON.parse(readFileSync(join(selected, "settings.json"), "utf-8")).hooks.SessionStart).toBeDefined(); + for (const profile of [fileProfile, shellProfile]) { + if (profile !== selected) { + expect(existsSync(join(profile, ".claude.json"))).toBe(false); + expect(existsSync(join(profile, "settings.json"))).toBe(false); + } + } + expect(existsSync(join(home, ".claude.json"))).toBe(false); + expect(existsSync(join(home, ".claude"))).toBe(false); + }, 15000); + + it.each(["env file", "shell"])("keeps legacy paths and warns only for a shell whitespace-only value from the %s", (source) => { + const fileProfile = join(home, "env-file Claude config"); + mkdirSync(fileProfile); + mkdirSync(join(home, ".claude")); + mkdirSync(join(home, ".agentmemory")); + writeFileSync(join(home, ".agentmemory", ".env"), + `CLAUDE_CONFIG_DIR="${source === "env file" ? " \t " : fileProfile}"\n`); + + const result = spawnSync(process.execPath, [ + resolve(__dirname, "..", "dist", "cli.mjs"), + "connect", "claude-code", "--with-hooks", "--no-guidelines", + ], { + cwd: home, + env: { + ...process.env, + AGENTMEMORY_ENV_FILE: undefined, + CLAUDE_CONFIG_DIR: source === "shell" ? " \t " : undefined, + CI: "1", + NO_COLOR: "1", + }, + encoding: "utf-8", + timeout: 10000, + }); + + expect(result.status, result.stderr).toBe(0); + const output = result.stdout + result.stderr; + expect(output).not.toContain("CLAUDE_CONFIG_DIR was loaded only"); + expect(output).not.toContain("CLAUDE_CONFIG_DIR contains leading or trailing whitespace"); + if (source === "shell") { + expect(output).toContain("CLAUDE_CONFIG_DIR contains only whitespace."); + expect(output).toContain("Unset CLAUDE_CONFIG_DIR before launching Claude Code"); + expect(output.match(/CLAUDE_CONFIG_DIR contains only whitespace/g)).toHaveLength(1); + } else { + expect(output).not.toContain("CLAUDE_CONFIG_DIR contains only whitespace."); + } + expect(result.stdout).toContain(`wired into ${join(home, ".claude.json")}`); + expect(JSON.parse(readFileSync(join(home, ".claude.json"), "utf-8")).mcpServers.agentmemory).toBeDefined(); + expect(JSON.parse(readFileSync(join(home, ".claude", "settings.json"), "utf-8")).hooks.SessionStart).toBeDefined(); + expect(existsSync(join(fileProfile, ".claude.json"))).toBe(false); + expect(existsSync(join(fileProfile, "settings.json"))).toBe(false); + }, 15000); + + it.each(["leading", "trailing"])("skips a missing literal profile with %s whitespace even when the trimmed directory exists", (padding) => { + const trimmedDir = join(home, "existing Claude config"); + const configDir = padding === "leading" ? ` ${trimmedDir}` : `${trimmedDir} `; + mkdirSync(trimmedDir); + + const result = spawnSync(process.execPath, [ + resolve(__dirname, "..", "dist", "cli.mjs"), + "connect", "claude-code", "--with-hooks", "--no-guidelines", + ], { + cwd: home, + env: { + ...process.env, + AGENTMEMORY_ENV_FILE: undefined, + CLAUDE_CONFIG_DIR: configDir, + CI: "1", + NO_COLOR: "1", + }, + encoding: "utf-8", + timeout: 10000, + }); + + expect(result.status, result.stderr).toBe(0); + const output = result.stdout + result.stderr; + expect(output).toContain(JSON.stringify(configDir)); + expect(output).toContain("is not an existing directory"); + expect(output).toContain("leading or trailing whitespace"); + expect(output).toContain("not detected on this machine (skipping)"); + expect(output).not.toContain("wired into"); + expect(existsSync(resolve(home, configDir))).toBe(false); + expect(existsSync(join(trimmedDir, ".claude.json"))).toBe(false); + expect(existsSync(join(trimmedDir, "settings.json"))).toBe(false); + }, 15000); + + it("warns before installing into a profile relative to the CLI working directory", () => { + const configDir = "relative Claude config"; + const selected = join(home, configDir); + mkdirSync(selected); + + const result = spawnSync(process.execPath, [ + resolve(__dirname, "..", "dist", "cli.mjs"), + "connect", "claude-code", "--with-hooks", "--no-guidelines", + ], { + cwd: home, + env: { + ...process.env, + AGENTMEMORY_ENV_FILE: undefined, + CLAUDE_CONFIG_DIR: configDir, + CI: "1", + NO_COLOR: "1", + }, + encoding: "utf-8", + timeout: 10000, + }); + + expect(result.status, result.stderr).toBe(0); + const output = result.stdout + result.stderr; + expect(output).toContain(`CLAUDE_CONFIG_DIR is relative and resolves to ${JSON.stringify(realpathSync(selected))}`); + expect(output).toContain("Set the same absolute path for agentmemory and Claude Code"); + expect(output.match(/CLAUDE_CONFIG_DIR is relative/g)).toHaveLength(1); + expect(output.indexOf("CLAUDE_CONFIG_DIR is relative")).toBeLessThan(output.indexOf("wired into")); + expect(JSON.parse(readFileSync(join(selected, ".claude.json"), "utf-8")).mcpServers.agentmemory).toBeDefined(); + expect(JSON.parse(readFileSync(join(selected, "settings.json"), "utf-8")).hooks.SessionStart).toBeDefined(); + expect(existsSync(join(home, ".claude.json"))).toBe(false); + expect(existsSync(join(home, ".claude"))).toBe(false); + }, 15000); + + it.each(["existing", "missing"])("uses the %s normalized profile for a symlink followed by ..", (profile) => { + const alias = join(home, "alias"); + const target = join(home, "target"); + const normalizedProfile = join(alias, "profile"); + const targetProfile = join(target, "profile"); + mkdirSync(alias); + mkdirSync(join(target, "child"), { recursive: true }); + symlinkSync(join(target, "child"), join(alias, "link"), "dir"); + const configDir = `${alias}/link/../profile`; + const existingProfile = profile === "existing" ? normalizedProfile : targetProfile; + mkdirSync(existingProfile); + mkdirSync(join(home, ".claude")); + const original = '{"keep":"original configuration"}'; + for (const path of [ + join(existingProfile, ".claude.json"), + join(existingProfile, "settings.json"), + join(home, ".claude.json"), + join(home, ".claude", "settings.json"), + ]) { + writeFileSync(path, original); + } + + const result = spawnSync(process.execPath, [ + resolve(__dirname, "..", "dist", "cli.mjs"), + "connect", "claude-code", "--with-hooks", "--no-guidelines", + ], { + cwd: home, + env: { + ...process.env, + AGENTMEMORY_ENV_FILE: undefined, + CLAUDE_CONFIG_DIR: configDir, + CI: "1", + NO_COLOR: "1", + }, + encoding: "utf-8", + timeout: 10000, + }); + + expect(result.status, result.stderr).toBe(0); + const output = result.stdout + result.stderr; + if (profile === "existing") { + expect(output).toContain(`wired into ${join(normalizedProfile, ".claude.json")}`); + const config = JSON.parse(readFileSync(join(normalizedProfile, ".claude.json"), "utf-8")); + const settings = JSON.parse(readFileSync(join(normalizedProfile, "settings.json"), "utf-8")); + expect(config.mcpServers.agentmemory).toBeDefined(); + expect(settings.hooks.SessionStart).toBeDefined(); + expect(config.keep).toBe("original configuration"); + expect(settings.keep).toBe("original configuration"); + expect(existsSync(targetProfile)).toBe(false); + } else { + expect(output).toContain("not detected on this machine (skipping)"); + expect(output).not.toContain("wired into"); + expect(existsSync(normalizedProfile)).toBe(false); + expect(readFileSync(join(targetProfile, ".claude.json"), "utf-8")).toBe(original); + expect(readFileSync(join(targetProfile, "settings.json"), "utf-8")).toBe(original); + } + expect(readFileSync(join(home, ".claude.json"), "utf-8")).toBe(original); + expect(readFileSync(join(home, ".claude", "settings.json"), "utf-8")).toBe(original); + }, 15000); + + it.skipIf(process.getuid?.() === 0)("warns and skips an inaccessible profile without changing either profile", () => { + const parent = join(home, "inaccessible"); + const configDir = join(parent, "Claude config"); + const configPath = join(configDir, ".claude.json"); + const settingsPath = join(configDir, "settings.json"); + const defaultConfig = join(home, ".claude.json"); + const defaultSettings = join(home, ".claude", "settings.json"); + mkdirSync(configDir, { recursive: true }); + mkdirSync(join(home, ".claude")); + const original = '{"keep":"original configuration"}'; + for (const path of [configPath, settingsPath, defaultConfig, defaultSettings]) { + writeFileSync(path, original); + } + + chmodSync(parent, 0o000); + try { + const result = spawnSync(process.execPath, [ + resolve(__dirname, "..", "dist", "cli.mjs"), + "connect", "claude-code", "--with-hooks", "--no-guidelines", + ], { + cwd: home, + env: { + ...process.env, + AGENTMEMORY_ENV_FILE: undefined, + CLAUDE_CONFIG_DIR: configDir, + CI: "1", + NO_COLOR: "1", + }, + encoding: "utf-8", + timeout: 10000, + }); + + expect(result.status, result.stderr).toBe(0); + const output = result.stdout + result.stderr; + expect(output).toContain(`Cannot inspect Claude Code configuration directory ${JSON.stringify(configDir)} (EACCES)`); + expect(output).toContain("Check directory permissions and retry."); + expect(output).toContain("not detected on this machine (skipping)"); + expect(output).not.toContain("wired into"); + } finally { + chmodSync(parent, 0o700); + } + for (const path of [configPath, settingsPath, defaultConfig, defaultSettings]) { + expect(readFileSync(path, "utf-8")).toBe(original); + } + }, 15000); + }); + + it("merges MCP and hooks in CLAUDE_CONFIG_DIR and leaves default files untouched", async () => { + const configDir = join(home, "custom Claude config"); + const configPath = join(configDir, ".claude.json"); + const settingsPath = join(configDir, "settings.json"); + const existingServer = { command: "existing-server", args: [] }; + const userHook = { hooks: [{ type: "command", command: "echo user-hook" }] }; + mkdirSync(configDir); + mkdirSync(join(home, ".claude")); + writeFileSync(configPath, JSON.stringify({ mcpServers: { existing: existingServer }, theme: "dark" })); + writeFileSync(settingsPath, JSON.stringify({ env: { KEEP: "yes" }, hooks: { SessionStart: [userHook] } })); + writeFileSync(join(home, ".claude.json"), '{"keep":"default MCP"}'); + writeFileSync(join(home, ".claude", "settings.json"), '{"keep":"default settings"}'); + vi.stubEnv("CLAUDE_CONFIG_DIR", configDir); + + const { adapter } = await import("../src/cli/connect/claude-code.js"); + const options = { dryRun: false, force: false, withHooks: true }; + const result = await adapter.install(options); + + expect(result).toMatchObject({ kind: "installed", mutatedPath: configPath }); + const config = JSON.parse(readFileSync(configPath, "utf-8")); + expect(config.theme).toBe("dark"); + expect(config.mcpServers.existing).toEqual(existingServer); + expect(config.mcpServers.agentmemory.args).toContain("@agentmemory/mcp"); + const settings = JSON.parse(readFileSync(settingsPath, "utf-8")); + expect(settings.env).toEqual({ KEEP: "yes" }); + expect(settings.hooks.SessionStart).toEqual([ + userHook, + ...buildMergedHooks(null, PLUGIN_ROOT, "hooks.json").hooks.SessionStart!, + ]); + expect(readFileSync(join(home, ".claude.json"), "utf-8")).toBe('{"keep":"default MCP"}'); + expect(readFileSync(join(home, ".claude", "settings.json"), "utf-8")).toBe('{"keep":"default settings"}'); + + const installedSettings = readFileSync(settingsPath, "utf-8"); + expect((await adapter.install(options)).kind).toBe("already-wired"); + expect(readFileSync(settingsPath, "utf-8")).toBe(installedSettings); + }); + + it("reports the custom MCP path on a dry run without writing files", async () => { + const configDir = join(home, "custom Claude config"); + mkdirSync(configDir); + vi.stubEnv("CLAUDE_CONFIG_DIR", configDir); + + const { adapter } = await import("../src/cli/connect/claude-code.js"); + const result = await adapter.install({ dryRun: true, force: false, withHooks: true }); + + expect(result).toMatchObject({ kind: "installed", mutatedPath: join(configDir, ".claude.json") }); + expect(existsSync(join(configDir, ".claude.json"))).toBe(false); + expect(existsSync(join(configDir, "settings.json"))).toBe(false); + expect(existsSync(join(home, ".claude.json"))).toBe(false); + }); + + it("puts MCP inside an explicitly configured default .claude directory", async () => { + const configDir = join(home, ".claude"); + mkdirSync(configDir); + vi.stubEnv("CLAUDE_CONFIG_DIR", configDir); + + const { adapter } = await import("../src/cli/connect/claude-code.js"); + const result = await adapter.install({ dryRun: false, force: false }); + + expect(result).toMatchObject({ kind: "installed", mutatedPath: join(configDir, ".claude.json") }); + expect(JSON.parse(readFileSync(join(configDir, ".claude.json"), "utf-8")).mcpServers.agentmemory).toBeDefined(); + expect(existsSync(join(home, ".claude.json"))).toBe(false); + }); + + it.each([undefined, "", " \t "])("keeps legacy paths when CLAUDE_CONFIG_DIR is %s", async (override) => { + vi.stubEnv("CLAUDE_CONFIG_DIR", override); + mkdirSync(join(home, ".claude")); + + const { adapter } = await import("../src/cli/connect/claude-code.js"); + expect(adapter.detect()).toBe(true); + const result = await adapter.install({ dryRun: false, force: false, withHooks: true }); + + expect(result).toMatchObject({ kind: "installed", mutatedPath: join(home, ".claude.json") }); + expect(JSON.parse(readFileSync(join(home, ".claude.json"), "utf-8")).mcpServers.agentmemory).toBeDefined(); + expect(JSON.parse(readFileSync(join(home, ".claude", "settings.json"), "utf-8")).hooks.SessionStart).toBeDefined(); + }); }); diff --git a/test/cli-compose-env.test.ts b/test/cli-compose-env.test.ts index edf904a131..85c72bd5b7 100644 --- a/test/cli-compose-env.test.ts +++ b/test/cli-compose-env.test.ts @@ -28,6 +28,9 @@ describe.skipIf(process.platform === "win32")("CLI Compose environment", () => { `if [ "$1" = "--version" ]; then printf '%s\\n' 'iii ${III_PINNED_VERSION}'; exit 0; fi`, "exit 1", "", ].join("\n"), { mode: 0o755 }); + // This fixture stops at the fake engine; select local worker placeholders + // so the file-mode artifact preflight does not hide env-file generation. + for (const worker of ["state", "http"]) writeFileSync(join(binDir, worker), "#!/bin/sh\nexit 1\n", { mode: 0o755 }); writeFileSync(join(home, ".agentmemory", ".env"), "CUSTOM_SETTING=wrong-profile\n"); writeFileSync(customEnv, "CUSTOM_SETTING=selected-profile\nIII_NAMESPACE=wrong-namespace\n"); const result = await promisify(execFile)(process.execPath, ["--import", "tsx", "src/cli.ts"], { @@ -37,6 +40,7 @@ describe.skipIf(process.platform === "win32")("CLI Compose environment", () => { PATH: [binDir, dirname(process.execPath), "/usr/bin", "/bin"].join(delimiter), CI: "1", NO_COLOR: "1", AGENTMEMORY_DATA_DIR: dataDir, + AGENTMEMORY_WORKER_BIN_DIR: binDir, AGENTMEMORY_ENV_FILE: customEnv, AGENTMEMORY_ENGINE_READY_TIMEOUT_MS: "1000", AGENTMEMORY_READY_TIMEOUT_MS: "1000", diff --git a/test/cli-connect.test.ts b/test/cli-connect.test.ts index e303fe1ab6..71548625b1 100644 --- a/test/cli-connect.test.ts +++ b/test/cli-connect.test.ts @@ -100,6 +100,7 @@ describe("agentmemory connect — claude-code adapter (mock filesystem)", () => originalUserprofile = process.env["USERPROFILE"]; process.env["HOME"] = tmpHome; process.env["USERPROFILE"] = tmpHome; + vi.stubEnv("CLAUDE_CONFIG_DIR", undefined); vi.resetModules(); }); @@ -109,6 +110,7 @@ describe("agentmemory connect — claude-code adapter (mock filesystem)", () => if (originalUserprofile !== undefined) process.env["USERPROFILE"] = originalUserprofile; else delete process.env["USERPROFILE"]; + vi.unstubAllEnvs(); rmSync(tmpHome, { recursive: true, force: true }); vi.resetModules(); }); diff --git a/test/cli-doctor-fixes.test.ts b/test/cli-doctor-fixes.test.ts index 9e55490d54..3418f68ebf 100644 --- a/test/cli-doctor-fixes.test.ts +++ b/test/cli-doctor-fixes.test.ts @@ -266,7 +266,7 @@ describe("doctor v2 diagnostic catalog", () => { ).toEqual({ ok: true, detail: "agent-sdk fallback, no API key" }); }); - it("treats a placeholder exported in the environment as missing, unlike the daemon", async () => { + it("treats a placeholder exported in the environment as missing, like the daemon", async () => { expect( ( await statusFor( @@ -276,6 +276,21 @@ describe("doctor v2 diagnostic catalog", () => { ).ok, ).toBe(false); }); + + it("names a placeholder OpenAI key behind a plain-http base URL, like the daemon", async () => { + expect( + await statusFor( + { AGENTMEMORY_LLM_PROVIDER: "openai", OPENAI_API_KEY: "xxx" }, + { OPENAI_BASE_URL: "http://localhost:11434/v1" }, + ), + ).toEqual({ ok: true, detail: "found: OPENAI_API_KEY" }); + expect( + await statusFor( + { AGENTMEMORY_LLM_PROVIDER: "openai", OPENAI_API_KEY: "xxx", OPENAI_BASE_URL: "http://localhost:11434/v1" }, + {}, + ), + ).toEqual({ ok: true, detail: "found: OPENAI_API_KEY" }); + }); }); it("engine-version-mismatch fails when iii reports the wrong version", async () => { diff --git a/test/cli-native-compat.test.ts b/test/cli-native-compat.test.ts index 442972753d..424832e187 100644 --- a/test/cli-native-compat.test.ts +++ b/test/cli-native-compat.test.ts @@ -224,7 +224,12 @@ describe("native startup prior ownership", () => { for(const [saved,pidfile] of [[JSON.stringify(prior),undefined],[JSON.stringify(prior),"99999999"],["{malformed",undefined],[JSON.stringify({kind:"unknown"}),undefined],[JSON.stringify({...prior,workerIdentity:undefined}),undefined]]as const){ fs.writeFileSync(join(runtime,"engine-state.json"),saved);if(pidfile)fs.writeFileSync(join(runtime,"worker.pid"),pidfile);else fs.rmSync(join(runtime,"worker.pid"),{force:true}); const result=spawnSync(process.execPath,["--import","tsx",resolve("src/cli.ts"),"--native-compat","--port",String(port)],{env,cwd:process.cwd(),encoding:"utf8",timeout:15000}); - expect(result.error).toBeUndefined();expect(result.status).toBe(1);expect(result.stdout+result.stderr).toContain("prior lifecycle");expect(fs.existsSync(binLog)).toBe(false);expect(fs.existsSync(marker)).toBe(false);expect(fs.readFileSync(join(runtime,"engine-state.json"),"utf8")).toBe(saved); + expect(result.error).toBeUndefined(); + expect(result.stdout + result.stderr).toContain("prior lifecycle"); + expect(result.status).toBe(78); + expect(fs.existsSync(binLog)).toBe(false); + expect(fs.existsSync(marker)).toBe(false); + expect(fs.readFileSync(join(runtime, "engine-state.json"), "utf8")).toBe(saved); } for(const saved of [undefined,"{malformed",JSON.stringify({kind:"unknown"})]){ if(saved)fs.writeFileSync(join(runtime,"engine-state.json"),saved);else fs.rmSync(join(runtime,"engine-state.json"),{force:true});fs.writeFileSync(join(runtime,"worker.pid"),String(child.pid)); diff --git a/test/clip-embedding-provider.test.ts b/test/clip-embedding-provider.test.ts index b605c1ef70..cfa35481b6 100644 --- a/test/clip-embedding-provider.test.ts +++ b/test/clip-embedding-provider.test.ts @@ -20,41 +20,67 @@ describe("ClipEmbeddingProvider (package unavailable)", () => { describe("ClipEmbeddingProvider (with loaded pipeline)", () => { function mockSuccessModule() { - const textExtractor = vi.fn(async (texts: string[]) => ({ - tolist: () => texts.map(() => [0.1, 0.2]), + const tokenized: unknown[] = []; + const textModel = vi.fn(async (inputs: { input_ids: number[][] }) => ({ + text_embeds: { + tolist: () => inputs.input_ids.map(() => Array.from({ length: 512 }, (_, i) => (i % 7) + 1)), + }, })); + const tokenizer = vi.fn((texts: string[]) => { + const out = { input_ids: texts.map(() => [1, 2, 3]) }; + tokenized.push(out); + return out; + }); + const fromPretrainedText = vi.fn(async () => textModel); + const fromPretrainedTokenizer = vi.fn(async () => tokenizer); + const imageVec = Array.from({ length: 512 }, (_, i) => (i % 5) + 1); const imageExtractor = vi.fn(async () => ({ - tolist: () => [[0.3, 0.4]], - data: new Float32Array([0.3, 0.4]), + tolist: () => [imageVec], + data: new Float32Array(imageVec), })); const fromBlob = vi.fn(async () => ({})); const pipeline = vi.fn((task: string) => { - if (task === "feature-extraction") return Promise.resolve(textExtractor); if (task === "image-feature-extraction") return Promise.resolve(imageExtractor); return Promise.reject(new Error(`unmocked task: ${task}`)); }); vi.doMock("@huggingface/transformers", () => ({ pipeline, + AutoTokenizer: { from_pretrained: fromPretrainedTokenizer }, + CLIPTextModelWithProjection: { from_pretrained: fromPretrainedText }, RawImage: { fromBlob }, })); vi.resetModules(); - return { pipeline, textExtractor, imageExtractor, fromBlob }; + return { + pipeline, + textModel, + tokenizer, + tokenized, + fromPretrainedText, + fromPretrainedTokenizer, + imageExtractor, + fromBlob, + }; } - it("loads text pipeline with dtype: q8 and returns mapped Float32Array", async () => { - const { pipeline } = mockSuccessModule(); + it("loads the CLIP text tower with dtype: q8 and returns a normalized Float32Array", async () => { + const { pipeline, fromPretrainedText, fromPretrainedTokenizer } = mockSuccessModule(); const { ClipEmbeddingProvider: Fresh } = await import( "../src/providers/embedding/clip.js" ); const vec = await new Fresh().embed("hello"); - expect(pipeline).toHaveBeenCalledWith( + expect(fromPretrainedTokenizer).toHaveBeenCalledWith("Xenova/clip-vit-base-patch32"); + expect(fromPretrainedText).toHaveBeenCalledWith("Xenova/clip-vit-base-patch32", { + dtype: "q8", + }); + expect(pipeline).not.toHaveBeenCalledWith( "feature-extraction", - "Xenova/clip-vit-base-patch32", - { dtype: "q8" }, + expect.anything(), + expect.anything(), ); expect(vec).toBeInstanceOf(Float32Array); - expect(vec).toEqual(new Float32Array([0.1, 0.2])); + const norm = Math.sqrt(vec.reduce((s, v) => s + v * v, 0)); + expect(norm).toBeCloseTo(1, 6); }); it("embedBatch returns one Float32Array per input", async () => { @@ -85,16 +111,110 @@ describe("ClipEmbeddingProvider (with loaded pipeline)", () => { }); it("accepts custom model ID via constructor", async () => { - const { pipeline } = mockSuccessModule(); + const { fromPretrainedText, fromPretrainedTokenizer } = mockSuccessModule(); const { ClipEmbeddingProvider: Fresh } = await import( "../src/providers/embedding/clip.js" ); await new Fresh("Xenova/clip-vit-large-patch14").embed("hello"); + expect(fromPretrainedTokenizer).toHaveBeenCalledWith("Xenova/clip-vit-large-patch14"); + expect(fromPretrainedText).toHaveBeenCalledWith("Xenova/clip-vit-large-patch14", { + dtype: "q8", + }); + }); + + it("caches the text tower across calls instead of reloading it", async () => { + const { fromPretrainedText, fromPretrainedTokenizer } = mockSuccessModule(); + const { ClipEmbeddingProvider: Fresh } = await import( + "../src/providers/embedding/clip.js" + ); + const provider = new Fresh(); + await provider.embed("hello"); + await provider.embed("world"); + await provider.embedBatch(["a", "b"]); + + expect(fromPretrainedTokenizer).toHaveBeenCalledTimes(1); + expect(fromPretrainedText).toHaveBeenCalledTimes(1); + }); + + it("tokenizes with padding so a batch of uneven texts stays one tensor", async () => { + const { tokenizer, tokenized, textModel } = mockSuccessModule(); + const { ClipEmbeddingProvider: Fresh } = await import( + "../src/providers/embedding/clip.js" + ); + await new Fresh().embedBatch(["hi", "a much longer caption"]); + + expect(tokenizer).toHaveBeenCalledWith( + ["hi", "a much longer caption"], + expect.objectContaining({ padding: true, truncation: true }), + ); + expect(tokenized).toHaveLength(1); + expect(textModel).toHaveBeenCalledWith(tokenized[0]); + }); + + it("keeps text and image vectors in the same space via the projection head", async () => { + const { fromPretrainedText, pipeline } = mockSuccessModule(); + const { ClipEmbeddingProvider: Fresh } = await import( + "../src/providers/embedding/clip.js" + ); + const provider = new Fresh(); + const text = await provider.embed("a red square"); + const image = await provider.embedImage("data:image/png;base64,AAAA"); + + expect(fromPretrainedText).toHaveBeenCalled(); expect(pipeline).toHaveBeenCalledWith( - "feature-extraction", - "Xenova/clip-vit-large-patch14", - { dtype: "q8" }, + "image-feature-extraction", + expect.anything(), + expect.anything(), + ); + expect(text).toHaveLength(512); + expect(image).toHaveLength(512); + const norm = (v: Float32Array) => Math.sqrt(v.reduce((s, x) => s + x * x, 0)); + expect(norm(text)).toBeCloseTo(1, 6); + expect(norm(image)).toBeCloseTo(1, 6); + }); + + it("propagates a non-missing-module load failure untouched", async () => { + const boom = Object.assign(new Error("wasm backend unavailable"), { + code: "ERR_DLOPEN_FAILED", + }); + vi.doMock("@huggingface/transformers", () => ({ + get pipeline() { + throw boom; + }, + get AutoTokenizer() { + throw boom; + }, + get CLIPTextModelWithProjection() { + throw boom; + }, + RawImage: { fromBlob: vi.fn() }, + })); + vi.resetModules(); + const { ClipEmbeddingProvider: Fresh } = await import( + "../src/providers/embedding/clip.js" ); + + await expect(new Fresh().embed("hello")).rejects.toThrow( + "wasm backend unavailable", + ); + }); + + it("rethrows a failed module import instead of the install hint", async () => { + const boom = Object.assign(new Error("wasm backend unavailable"), { + code: "ERR_DLOPEN_FAILED", + }); + vi.doMock("@huggingface/transformers", () => { + throw boom; + }); + vi.resetModules(); + const { ClipEmbeddingProvider: Fresh } = await import( + "../src/providers/embedding/clip.js" + ); + + const err = await new Fresh().embed("hello").catch((e: unknown) => e); + expect(err).toBeInstanceOf(Error); + expect((err as Error).message).not.toContain("Install @huggingface/transformers"); + expect(err === boom || (err as Error).cause === boom).toBe(true); }); }); diff --git a/test/codex-compat-sdk-inventory.test.ts b/test/codex-compat-sdk-inventory.test.ts index 4d937b449b..61a66ee877 100644 --- a/test/codex-compat-sdk-inventory.test.ts +++ b/test/codex-compat-sdk-inventory.test.ts @@ -11,7 +11,7 @@ async function fixture(){ cpSync(resolve('integrations/codex-compat'),join(root,'integrations/codex-compat'),{recursive:true}); for(const dir of ['plugin/scripts','dist/hooks']){mkdirSync(join(root,dir),{recursive:true});writeFileSync(join(root,dir,'_capture.mjs'),'function handoffObservation(){}\nexport { handoffObservation };\n');} for(const name of ['cli.mjs','index.mjs'])writeFileSync(join(root,'dist',name),'export {};\n'); - for(const [name,version]of Object.entries({'iii-sdk':'0.25.0','@iii-dev/helpers':'0.25.0',yaml:'2.8.1',json5:'2.2.3'})){ + for(const [name,version]of Object.entries({'iii-sdk':'0.25.1','@iii-dev/helpers':'0.25.1',yaml:'2.8.1',json5:'2.2.3'})){ mkdirSync(join(root,'node_modules',name),{recursive:true});writeFileSync(join(root,'node_modules',name,'package.json'),JSON.stringify({version})); } writeFileSync(join(root,'dist/server-sdk-bundle.json'),JSON.stringify(await auditServerSdkBundle(root))); diff --git a/test/compose-c1-integration.test.ts b/test/compose-c1-integration.test.ts new file mode 100644 index 0000000000..0cfd24e619 --- /dev/null +++ b/test/compose-c1-integration.test.ts @@ -0,0 +1,95 @@ +import { spawn, spawnSync, type ChildProcess } from "node:child_process"; +import { mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { createServer } from "node:net"; +import { createRequire } from "node:module"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { pathToFileURL } from "node:url"; +import { setTimeout as sleep } from "node:timers/promises"; +import { describe, expect, it } from "vitest"; +import { composeNamespace, writeWorkerCompose } from "../src/cli/compose-project.js"; +import { III_PINNED_VERSION } from "../src/version.js"; + +const engine = process.env.AGENTMEMORY_TEST_III; +const version = engine ? spawnSync(engine, ["--version"], { encoding: "utf8", timeout: 3000 }) : null; +const available = version?.status === 0 && version.stdout.trim() === III_PINNED_VERSION; +const skipped = !available || process.platform === "win32" || process.env.AGENTMEMORY_SKIP_LIVE_TESTS === "1"; +if (process.env.AGENTMEMORY_REQUIRE_LIVE_TESTS === "1" && skipped) { + throw new Error(`Required Compose C1 test needs iii ${III_PINNED_VERSION} and live tests enabled on macOS or Linux`); +} + +async function freePort(): Promise { + const server = createServer(); + await new Promise((accept, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", accept); + }); + const port = (server.address() as { port: number }).port; + await new Promise((accept) => server.close(() => accept())); + return port; +} + +async function stop(child: ChildProcess): Promise { + if (child.exitCode !== null || child.signalCode !== null) return; + const exited = new Promise((accept) => child.once("exit", () => accept())); + child.kill("SIGINT"); + const result = await Promise.race([exited.then(() => true), sleep(5000).then(() => false)]); + if (!result && child.pid) { + try { process.kill(-child.pid, "SIGKILL"); } catch {} + await exited; + } +} + +describe.skipIf(skipped)("pinned Compose C1 environment round trip", () => { + it("delivers all C1 characters and line breaks unchanged to a real worker", async () => { + const root = mkdtempSync(join(tmpdir(), "agentmemory-compose-c1-")); + let child: ChildProcess | undefined; + try { + const pkg = join(root, "worker"); + const snapshot = join(root, "received.json"); + mkdirSync(join(pkg, "dist"), { recursive: true }); + const value = `first\n${String.fromCharCode(...Array.from({ length: 32 }, (_, i) => 0x80 + i))}\u2028\u2029\r\nquote'\"\\\tend\n`; + const sdk = pathToFileURL(createRequire(import.meta.url).resolve("iii-sdk")).href; + writeFileSync(join(pkg, "dist/index.mjs"), ` + import sdkModule from ${JSON.stringify(sdk)}; + import { renameSync, writeFileSync } from "node:fs"; + const { registerWorker } = sdkModule; + const worker = registerWorker(process.env.III_URL, { workerName: "agentmemory" }); + worker.registerFunction("test::env", async () => process.env.OPENAI_BASE_URL); + writeFileSync(${JSON.stringify(`${snapshot}.tmp`)}, JSON.stringify(process.env.OPENAI_BASE_URL)); + renameSync(${JSON.stringify(`${snapshot}.tmp`)}, ${JSON.stringify(snapshot)}); + `); + const ports = { restPort: await freePort(), viewerPort: await freePort(), enginePort: await freePort() }; + const project = writeWorkerCompose({ + ports, projectRoot: pkg, dataDir: join(root, "data"), homeDir: join(root, "home"), + nodeBin: process.execPath, workerEnvironment: { OPENAI_BASE_URL: value }, + }); + const header = project.text.slice(0, project.text.indexOf("containers:\n")); + const app = project.text.slice(project.text.indexOf(" agentmemory:\n")) + .replace(" start_after: [state, http]", " start_after: []"); + writeFileSync(project.file, `${header}containers:\n${app}`); + child = spawn(engine!, ["compose", "--namespace", composeNamespace(ports.restPort), "--up", "--file", project.file], { + env: { PATH: process.env.PATH, HOME: join(root, "home"), USERPROFILE: join(root, "home"), + III_COMPOSE_STATE_DIR: project.stateDir, III_TELEMETRY_ENABLED: "false", III_BUNDLE_WORKERS_DISABLED: "1", RUST_LOG: "error" }, + detached: true, stdio: ["ignore", "ignore", "pipe"], + }); + let logs = ""; + child.stderr?.on("data", (chunk) => { logs = `${logs}${chunk}`.slice(-16000); }); + const deadline = Date.now() + 20000; + while (Date.now() < deadline) { + try { + expect(JSON.parse(readFileSync(snapshot, "utf8"))).toBe(value); + return; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + if (child.exitCode !== null || child.signalCode !== null) throw new Error(`Compose exited before the worker received its value: ${logs}`); + await sleep(50); + } + throw new Error(`Worker environment snapshot timed out: ${logs}`); + } finally { + if (child) await stop(child); + rmSync(root, { recursive: true, force: true }); + } + }, 30000); +}); diff --git a/test/compose-control.test.ts b/test/compose-control.test.ts index 63dda5450d..e85944786e 100644 --- a/test/compose-control.test.ts +++ b/test/compose-control.test.ts @@ -1,11 +1,11 @@ -import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { statSync, openSync, closeSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; const temp = mkdtempSync(join(tmpdir(), "compose-state-error-")); afterEach(() => rmSync(temp, { recursive: true, force: true })); -import { composeCall, composeEnvironment, composeRegistryFailureMessage, composeStateRegistryFailure, composeWorkerFailureSummary, ownedComposeProject, readComposeLogFromOffset, startComposeDaemon, withQuietSdkDebug, type ComposeStatus } from "../src/cli/compose-control.js"; +import { writeComposeState, composeCall, composeEnvironment, composeRegistryFailureMessage, composeStateRegistryFailure, composeWorkerFailureSummary, ownedComposeProject, readComposeLogFromOffset, startComposeDaemon, withQuietSdkDebug, type ComposeStatus } from "../src/cli/compose-control.js"; describe("Compose control plane", () => { it("does not leak arbitrary machine credentials to the Compose daemon", () => { @@ -164,3 +164,14 @@ describe("Compose control client output", () => { expect(console.debug).toBe(original); }); }); + +it("atomically replaces Compose metadata without truncating existing readers", () => { + mkdirSync(temp,{recursive:true});const path=join(temp,'engine-state.json'); + const state:any={kind:'compose',enginePid:123,engineUrl:'ws://127.0.0.1:1',workerVersions:{local:true}}; + writeComposeState(path,state);const fd=openSync(path,'r'); + try {writeComposeState(path,{...state,enginePid:456}); + expect(JSON.parse(readFileSync(fd,'utf8')).enginePid).toBe(123); + expect(JSON.parse(readFileSync(path,'utf8')).enginePid).toBe(456); + expect(statSync(path).mode & 0o777).toBe(0o600); + } finally {closeSync(fd);} +}); diff --git a/test/compose-project.test.ts b/test/compose-project.test.ts index 621a246667..7b562169e7 100644 --- a/test/compose-project.test.ts +++ b/test/compose-project.test.ts @@ -24,6 +24,30 @@ const options = (overrides: Record = {}) => ({ }); describe("Worker Compose project", () => { + it.each(Array.from({ length: 32 }, (_, offset) => 0x80 + offset))( + "escapes C1 character U+%s in single-line and multiline scalars", (code) => { + for (const value of [`first${String.fromCharCode(code)}second`, `first\n${String.fromCharCode(code)}second`]) { + const text = renderWorkerCompose(options({ workerEnvironment: { OPENAI_BASE_URL: value } })); + expect(text).not.toMatch(/[\u0080-\u009f]/); + expect(parse(text).containers.agentmemory.environment.OPENAI_BASE_URL).toBe(value); + } + }, + ); + it.each(["first\u2028second", "first\u2029second", "first\n\u2028\u2029second"])( + "escapes Unicode line and paragraph separators: %j", (value) => { + const text = renderWorkerCompose(options({ workerEnvironment: { OPENAI_BASE_URL: value } })); + expect(text).not.toMatch(/[\u2028\u2029]/); + expect(text).toContain(`OPENAI_BASE_URL: ${JSON.stringify(value).replace(/[\u2028\u2029]/g, (char) => `\\u${char.charCodeAt(0).toString(16)}`)}`); + expect(parse(text).containers.agentmemory.environment.OPENAI_BASE_URL).toBe(value); + }, + ); + it.each(["first\nsecond", "first\rsecond", "first\r\nsecond", "\nleading\n\ntrailing\n", "quote'\"\\\nnext\tline"])( + "preserves multiline configuration values through YAML parsing: %j", (value) => { + const text = renderWorkerCompose(options({ workerEnvironment: { OPENAI_BASE_URL: value } })); + expect(parse(text).containers.agentmemory.environment.OPENAI_BASE_URL).toBe(value); + }, + ); + it("pins registry workers and configures the isolated engine, state and HTTP", () => { const text = renderWorkerCompose(options()); expect(text).toContain("namespace: agentmemory-3511"); @@ -120,6 +144,37 @@ describe("Worker Compose project", () => { expect(app.env_file).toHaveLength(1); }); + it("preserves native summary and service settings through env-file and explicit inheritance", () => { + const settings = { + OPENAI_API_KEY_FOR_LLM: "synthetic-unused-alias", + SUMMARIZE_MIN_NEW_OBS: "17", SUMMARIZE_MAX_AGE_MS: "60000", + AGENTMEMORY_OLLAMA_NATIVE_CHAT: "true", SUMMARIZE_BATCH_OBS: "3", + SUMMARIZE_INPUT_CHARS: "2400", SUMMARIZE_RECOVERY_INTERVAL_MS: "12345", + AGENTMEMORY_HOOK_INJECT_CONTEXT: "false", + AGENTMEMORY_FEEDBACK_DIR: join(root, "synthetic-feedback"), + AGENTMEMORY_RECEIPT_DIR: join(root, "synthetic-receipts"), + }; + mkdirSync(root, { recursive: true }); + const envPath = join(root, "native.env"); + writeFileSync(envPath, Object.entries(settings).map(([key, value]) => `${key}=${value}`).join("\n")); + const fileOnly = writeWorkerCompose(options({ envFile: envPath })); + const app = parse(fileOnly.text).containers.agentmemory; + const preserved = readFileSync(app.env_file[0], "utf8"); + for (const [key, value] of Object.entries(settings)) expect(preserved).toContain(`${key}=${value}`); + const explicit = parse(renderWorkerCompose(options({ envFile: envPath, workerEnvironment: { + ...settings, SUMMARIZE_INPUT_CHARS: "2600", STATE_STORE_PATH: "/synthetic/wrong-store", AGENTMEMORY_STATE_STORE_PATH: "/synthetic/wrong-store", + AGENTMEMORY_DATA_DIR: "/synthetic/wrong-data", AGENTMEMORY_RUNTIME_DIR: "/synthetic/wrong-runtime", + UNRELATED_SECRET: "synthetic-not-forwarded", + } }))); + expect(explicit.containers.agentmemory.environment).toMatchObject({ ...settings, SUMMARIZE_INPUT_CHARS: "2600" }); + expect(explicit.containers.agentmemory.environment).not.toHaveProperty("STATE_STORE_PATH"); + expect(explicit.containers.agentmemory.environment).not.toHaveProperty("AGENTMEMORY_STATE_STORE_PATH"); + expect(explicit.containers.agentmemory.environment).not.toHaveProperty("UNRELATED_SECRET"); + expect(explicit.containers.agentmemory.environment.AGENTMEMORY_DATA_DIR).toBe(join(root, "data")); + expect(explicit.containers.agentmemory.environment.AGENTMEMORY_RUNTIME_DIR).toBe(join(root, "data")); + expect(explicit.containers.state.config_override.adapter.config.file_path).toBe(join(root, "data", "state_store.db")); + }); + it("uses the forwarded tools setting for the effective MCP tool list", async () => { const text = renderWorkerCompose(options({ workerEnvironment: { AGENTMEMORY_TOOLS: "core" } })); const app = (parse(text) as { containers: Record }> }).containers.agentmemory!; @@ -165,6 +220,35 @@ describe("Worker Compose project", () => { expect(text.slice(0, text.indexOf("containers:")).concat(text.slice(text.indexOf(" state:"), text.indexOf(" agentmemory:")))).not.toContain("fake-gate-key"); }); + it("forwards every provider selector and gate wherever provider keys are forwarded", () => { + const selectors = { + AGENTMEMORY_LLM_PROVIDER: "anthropic", + EMBEDDING_PROVIDER: "openai", + AGENTMEMORY_ALLOW_AGENT_SDK: "true", + AGENTMEMORY_INJECT_CONTEXT: "true", + ANTHROPIC_MODEL: "model-a", + ANTHROPIC_BASE_URL: "http://127.0.0.1:4556", + OPENAI_MODEL: "model-b", + OPENAI_BASE_URL: "http://127.0.0.1:4555", + GEMINI_MODEL: "model-c", + OPENROUTER_MODEL: "model-d", + MINIMAX_MODEL: "model-e", + MINIMAX_BASE_URL: "http://127.0.0.1:4557", + }; + const native = parse(renderWorkerCompose(options({ workerEnvironment: selectors }))); + expect(native.containers.agentmemory.environment).toMatchObject(selectors); + const files = { + "deploy/runtime/worker-compose.yaml": parse(readFileSync(new URL("../deploy/runtime/worker-compose.yaml", import.meta.url), "utf8")).containers.agentmemory.environment, + "docker-compose.yml": parse(readFileSync(new URL("../docker-compose.yml", import.meta.url), "utf8")).services.agentmemory.environment, + "deploy/coolify/docker-compose.yml": parse(readFileSync(new URL("../deploy/coolify/docker-compose.yml", import.meta.url), "utf8")).services.agentmemory.environment, + }; + for (const [file, environment] of Object.entries(files)) { + for (const key of Object.keys(selectors)) { + expect(environment[key], `${file} forwards ${key}`).toBe("${" + key + ":-}"); + } + } + }); + it("preserves memory policy, provider and durability overrides in native and Docker workers", async () => { const settings = { FALLBACK_PROVIDERS: "openai,gemini", @@ -327,3 +411,10 @@ describe("Worker Compose project", () => { expect(statSync(written.file).mode & 0o777).toBe(0o600); }); }); + +it("preserves the CLI metadata directory when data is stored elsewhere", () => { + const runtimeDir = join(root, "home", ".agentmemory"); + const config = parse(renderWorkerCompose(options({ runtimeDir }))); + expect(config.containers.agentmemory.environment.AGENTMEMORY_RUNTIME_DIR).toBe(runtimeDir); + expect(config.containers.agentmemory.environment.AGENTMEMORY_DATA_DIR).toBe(join(root, "data")); +}); diff --git a/test/consolidate-project-scope.test.ts b/test/consolidate-project-scope.test.ts index ccb5d133d9..c4f8e08c76 100644 --- a/test/consolidate-project-scope.test.ts +++ b/test/consolidate-project-scope.test.ts @@ -5,7 +5,7 @@ vi.mock("../src/logger.js", () => ({ })); vi.mock("../src/functions/audit.js", () => ({ - recordAudit: vi.fn(), + safeAudit: vi.fn(), })); import { registerConsolidateFunction } from "../src/functions/consolidate.js"; @@ -270,3 +270,28 @@ describe("mem::consolidate — cross-project existingMatch guard", () => { expect(memories[0].project).toBeUndefined(); }); }); + +describe("mem::consolidate recurrence", () => { + async function evolve(sourceObservationIds: string[]) { + const sdk = makeMockSdk(); + const kv = makeMockKV(); + const existing = { ...makeExistingMemory("mem_old", "synthesized memory title", "api"), recurrence: 2, sourceObservationIds }; + await kv.set(KV.memories, existing.id, existing); + const session = makeSession("sess_api", "api"); + await kv.set(KV.sessions, session.id, session); + for (let i = 0; i < 3; i++) { + await kv.set(KV.observations(session.id), `obs_${i}`, makeObs(`obs_${i}`, session.id, "auth")); + } + registerConsolidateFunction(sdk as never, kv as never, makeProvider("synthesized memory title") as never); + await sdk.trigger("mem::consolidate", { project: "api", minObservations: 1 }); + return (await kv.list(KV.memories)).find((m) => m.isLatest)!; + } + + it("does not raise recurrence when a run evolves a memory from the same observations", async () => { + expect((await evolve(["obs_0", "obs_1", "obs_2"])).recurrence).toBe(2); + }); + + it("raises recurrence when the evolving run brings observations the memory has not seen", async () => { + expect((await evolve(["obs_0"])).recurrence).toBe(3); + }); +}); diff --git a/test/consolidate-publication.test.ts b/test/consolidate-publication.test.ts new file mode 100644 index 0000000000..ab0ae2fda3 --- /dev/null +++ b/test/consolidate-publication.test.ts @@ -0,0 +1,453 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { registerConsolidateFunction } from "../src/functions/consolidate.js"; +import { KV } from "../src/state/schema.js"; +import { withKeyedLock } from "../src/state/keyed-mutex.js"; +import { safeAudit } from "../src/functions/audit.js"; +import type { CompressedObservation, Memory, MemoryProvider, Session } from "../src/types.js"; + +vi.mock("../src/logger.js", () => ({ logger: { info: vi.fn(), warn: vi.fn() } })); +vi.mock("../src/functions/audit.js", () => ({ safeAudit: vi.fn() })); + +function fixture() { + const rows = new Map>(); + const kv = { + get: async (scope: string, id: string): Promise => + structuredClone(rows.get(scope)?.get(id) as T ?? null), + list: async (scope: string): Promise => + structuredClone([...rows.get(scope)?.values() ?? []] as T[]), + set: vi.fn(async (scope: string, id: string, value: T): Promise => { + if (!rows.has(scope)) rows.set(scope, new Map()); + rows.get(scope)!.set(id, structuredClone(value)); + return value; + }), + delete: vi.fn(async (scope: string, id: string) => { rows.get(scope)?.delete(id); }), + }; + const functions = new Map Promise>(); + const sdk = { registerFunction: (id: string, handler: (data: unknown) => Promise) => functions.set(id, handler) }; + const compress = vi.fn(async (_system: string, _prompt: string) => xml("Shared title")); + registerConsolidateFunction(sdk as never, kv as never, { compress } as unknown as MemoryProvider); + const run = (project = "app") => functions.get("mem::consolidate")!({ project, minObservations: 1 }); + const memories = () => kv.list(KV.memories); + return { kv, compress, run, memories }; +} + +function xml(title: string) { + return `fact${title}Independent contentauth8`; +} + +async function seed(f: ReturnType, sessionId = "session-a", concept = "auth", project = "app") { + const session: Session = { id: sessionId, project, cwd: "/synthetic", startedAt: "2026-01-01T00:00:00Z", status: "completed", observationCount: 3 }; + await f.kv.set(KV.sessions, sessionId, session); + for (let i = 0; i < 3; i++) { + const id = `${sessionId}-${concept}-${i}`; + const observation: CompressedObservation = { + id, sessionId, timestamp: session.startedAt, type: "decision", title: `${concept} ${i}`, + narrative: `Evidence ${i}`, facts: [], concepts: [concept], files: [], importance: 8, + }; + await f.kv.set(KV.observations(sessionId), id, observation); + } +} + +function memory(id: string, extra: Partial = {}): Memory { + return { + id, createdAt: "2026-01-01T00:00:00Z", updatedAt: "2026-01-01T00:00:00Z", type: "fact", + title: "Shared title", content: "Previous content", concepts: [], files: [], sessionIds: [], + strength: 7, version: 1, isLatest: true, project: "app", ...extra, + }; +} + +afterEach(() => vi.useRealTimers()); + +describe("consolidation publication", () => { + it("publishes overlapping concurrent runs once even when model titles differ", async () => { + const f = fixture(); + await seed(f); + let release!: () => void; + const barrier = new Promise((resolve) => { release = resolve; }); + let started = 0; + f.compress.mockImplementation(async () => { + const title = `Title ${++started}`; + if (started === 2) release(); + await barrier; + return xml(title); + }); + const results = await Promise.all([f.run(), f.run()]); + expect(results.map((r) => (r as { consolidated: number }).consolidated).sort()).toEqual([0, 1]); + expect(await f.memories()).toHaveLength(1); + expect(f.compress).toHaveBeenCalledTimes(2); + }); + + it("uses the current latest version for multiple groups within one run", async () => { + const f = fixture(); + await seed(f, "session-a", "auth"); + await seed(f, "session-a", "tokens"); + await f.kv.set(KV.memories, "old", memory("old", { isLatest: false })); + await f.kv.set(KV.memories, "current", memory("current", { version: 2, parentId: "old", supersedes: ["old"] })); + expect(await f.run()).toMatchObject({ consolidated: 2 }); + const rows = await f.memories(); + const latest = rows.filter((m) => m.isLatest !== false); + expect(latest).toHaveLength(1); + expect(latest[0].version).toBe(4); + expect(latest[0].supersedes).toHaveLength(3); + expect(latest[0].supersedes).toContain("current"); + }); + + it.each(["delete", "edit", "move-project", "move-agent"])("discards model output after source changes: %s", async (change) => { + const f = fixture(); + await seed(f); + f.compress.mockImplementation(async () => { + if (change === "delete") { + await f.kv.delete(KV.observations("session-a"), "session-a-auth-0"); + } else if (change === "edit") { + const row = await f.kv.get(KV.observations("session-a"), "session-a-auth-0"); + await f.kv.set(KV.observations("session-a"), row!.id, { ...row, narrative: "Changed evidence" }); + } else { + const session = await f.kv.get(KV.sessions, "session-a"); + await f.kv.set(KV.sessions, "session-a", { ...session, + ...(change === "move-project" ? { project: "other" } : { agentId: "other-agent" }), + }); + } + return xml("Stale model result"); + }); + expect(await f.run()).toMatchObject({ consolidated: 0 }); + expect(await f.memories()).toHaveLength(0); + }); + + it.each(["session-a", "session-b"])("holds source locks through publication before a writer changes %s", async (sessionId) => { + const f = fixture(); + await seed(f, "session-b"); + await seed(f, "session-a"); + let entered!: () => void; + let release!: () => void; + const publicationEntered = new Promise((resolve) => { entered = resolve; }); + const publicationBarrier = new Promise((resolve) => { release = resolve; }); + const list = f.kv.list; + f.kv.list = async (scope: string): Promise => { + if (scope === KV.memories) { + entered(); + await publicationBarrier; + } + return list(scope); + }; + const publication = f.run(); + await publicationEntered; + let writerStarted = false; + const writer = withKeyedLock(`obs:${sessionId}`, async () => { + writerStarted = true; + await f.kv.delete(KV.observations(sessionId), `${sessionId}-auth-0`); + }); + try { + await Promise.resolve(); + expect(writerStarted).toBe(false); + } finally { + release(); + await Promise.all([publication, writer]); + } + expect(await publication).toMatchObject({ consolidated: 1 }); + expect(writerStarted).toBe(true); + expect(await f.memories()).toHaveLength(1); + expect(await f.kv.get(KV.observations(sessionId), `${sessionId}-auth-0`)).toBeNull(); + }); + + it("does not deduplicate equal observation IDs from different sessions", async () => { + const f = fixture(); + await seed(f); + await f.kv.set(KV.memories, "elsewhere", memory("elsewhere", { + sessionIds: ["another-session"], sourceObservationIds: ["session-a-auth-0", "session-a-auth-1", "session-a-auth-2"], + })); + expect(await f.run()).toMatchObject({ consolidated: 1 }); + expect((await f.memories()).filter((m) => m.isLatest !== false)).toHaveLength(1); + }); + + it("does not deduplicate another project's matching provenance", async () => { + const f = fixture(); + await seed(f); + await f.kv.set(KV.memories, "elsewhere", memory("elsewhere", { + project: "other", sessionIds: ["session-a"], sourceObservationIds: ["session-a-auth-0", "session-a-auth-1", "session-a-auth-2"], + })); + expect(await f.run()).toMatchObject({ consolidated: 1 }); + expect((await f.memories()).filter((m) => m.isLatest !== false)).toHaveLength(2); + }); + + it("preserves the previous latest memory when successor creation fails", async () => { + const f = fixture(); + await seed(f); + const previous = memory("previous"); + await f.kv.set(KV.memories, previous.id, previous); + const write = f.kv.set.getMockImplementation()!; + f.kv.set.mockImplementation(async (scope, id, value) => { + if (scope === KV.memories && id !== previous.id) throw new Error("storage unavailable"); + return write(scope, id, value); + }); + expect(await f.run()).toMatchObject({ consolidated: 0 }); + expect(await f.memories()).toEqual([previous]); + }); + + it("confirms a committed successor after its write acknowledgement fails", async () => { + const f = fixture(); + await seed(f); + await f.kv.set(KV.memories, "previous", memory("previous")); + const write = f.kv.set.getMockImplementation()!; + f.kv.set.mockImplementation(async (scope, id, value) => { + await write(scope, id, value); + if (scope === KV.memories && id !== "previous") throw new Error("lost acknowledgement"); + return value; + }); + expect(await f.run()).toMatchObject({ consolidated: 1 }); + const rows = await f.memories(); + expect(rows.filter((m) => m.isLatest !== false)).toHaveLength(1); + expect(rows.find((m) => m.id === "previous")?.isLatest).toBe(false); + }); + + it("repairs an interrupted publication before evolving changed evidence", async () => { + const f = fixture(); + await seed(f); + await f.kv.set(KV.memories, "previous", memory("previous")); + await f.kv.set(KV.memories, "committed", memory("committed", { + version: 2, parentId: "previous", supersedes: ["previous"], + sourceObservationIds: ["older-source"], sessionIds: ["session-a"], + })); + expect(await f.run()).toMatchObject({ consolidated: 1 }); + const rows = await f.memories(); + const latest = rows.filter((m) => m.isLatest !== false); + expect(latest).toHaveLength(1); + expect(latest[0]).toMatchObject({ version: 3, parentId: "committed" }); + }); + + it("does not repair ambiguous siblings or unrelated parent references", async () => { + const f = fixture(); + await seed(f); + for (const row of [memory("parent"), + memory("child-one", { version: 2, parentId: "parent", supersedes: ["parent"] }), + memory("child-two", { version: 2, parentId: "parent", supersedes: ["parent"] }), + memory("other", { title: "Other title" }), + memory("unrelated", { title: "Another title", version: 2, parentId: "other", supersedes: ["other"] }), + ]) await f.kv.set(KV.memories, row.id, row); + f.compress.mockResolvedValue(xml("New independent title")); + await f.run(); + expect((await f.memories()).filter((m) => m.isLatest !== false)).toHaveLength(6); + }); + + it.each([false, true])("rolls back a failed predecessor demotion (write committed: %s)", async (committed) => { + const f = fixture(); + await seed(f); + const previous = memory("previous"); + await f.kv.set(KV.memories, previous.id, previous); + const write = f.kv.set.getMockImplementation()!; + f.kv.set.mockImplementation(async (scope, id, value) => { + if (scope === KV.memories && id === previous.id && (value as Memory).isLatest === false) { + if (committed) await write(scope, id, value); + throw new Error("acknowledgement failed"); + } + return write(scope, id, value); + }); + expect(await f.run()).toMatchObject({ consolidated: 0 }); + expect(await f.memories()).toEqual([previous]); + }); + + it("keeps the successor if restoring the predecessor also fails", async () => { + const f = fixture(); + await seed(f); + await f.kv.set(KV.memories, "previous", memory("previous")); + const write = f.kv.set.getMockImplementation()!; + f.kv.set.mockImplementation(async (scope, id, value) => { + if (scope === KV.memories && id === "previous") { + if ((value as Memory).isLatest === false) await write(scope, id, value); + throw new Error("storage unavailable"); + } + return write(scope, id, value); + }); + await f.run(); + const latest = (await f.memories()).filter((m) => m.isLatest !== false); + expect(latest).toHaveLength(1); + expect(latest[0].parentId).toBe("previous"); + expect(f.kv.delete).not.toHaveBeenCalled(); + }); + + it("limits failed provider attempts and clears every timeout", async () => { + vi.useFakeTimers(); + const f = fixture(); + for (let i = 0; i < 15; i++) await seed(f, "session-a", `concept-${i}`); + f.compress.mockRejectedValue(new Error("provider unavailable")); + expect(await f.run()).toMatchObject({ consolidated: 0 }); + expect(f.compress).toHaveBeenCalledTimes(10); + expect(vi.getTimerCount()).toBe(0); + }); + + it("clears the timeout after a valid or malformed response", async () => { + vi.useFakeTimers(); + const f = fixture(); + await seed(f); + await f.run(); + expect(vi.getTimerCount()).toBe(0); + f.compress.mockResolvedValue("not XML"); + await f.run(); + expect(vi.getTimerCount()).toBe(0); + }); + + it("publishes a distinct insight for each concept that shares the same observations", async () => { + const f = fixture(); + const session: Session = { id: "session-a", project: "app", cwd: "/synthetic", startedAt: "2026-01-01T00:00:00Z", status: "completed", observationCount: 3 }; + await f.kv.set(KV.sessions, session.id, session); + for (let i = 0; i < 3; i++) { + const id = `shared-${i}`; + await f.kv.set(KV.observations(session.id), id, { + id, sessionId: session.id, timestamp: session.startedAt, type: "decision", title: `shared ${i}`, + narrative: `Evidence ${i}`, facts: [], concepts: ["auth", "tokens"], files: [], importance: 8, + } satisfies CompressedObservation); + } + f.compress.mockImplementation(async (_system: string, prompt: string) => + xml(prompt.startsWith('Concept: "auth"') ? "Auth insight" : "Token insight")); + expect(await f.run()).toMatchObject({ consolidated: 2 }); + const titles = (await f.memories()).filter((m) => m.isLatest !== false).map((m) => m.title).sort(); + expect(titles).toEqual(["Auth insight", "Token insight"]); + }); + + it("publishes a new version when an observation is recompressed in place", async () => { + const f = fixture(); + await seed(f); + expect(await f.run()).toMatchObject({ consolidated: 1 }); + const row = await f.kv.get(KV.observations("session-a"), "session-a-auth-0"); + await f.kv.set(KV.observations("session-a"), row!.id, { ...row!, narrative: "Recompressed evidence" }); + expect(await f.run()).toMatchObject({ consolidated: 1 }); + const latest = (await f.memories()).filter((m) => m.isLatest !== false); + expect(latest).toHaveLength(1); + expect(latest[0].version).toBe(2); + }); + + it("skips a rerun when the concept and its source contents are unchanged", async () => { + const f = fixture(); + await seed(f); + expect(await f.run()).toMatchObject({ consolidated: 1 }); + expect(await f.run()).toMatchObject({ consolidated: 0 }); + expect(await f.memories()).toHaveLength(1); + }); + + it("recognises a memory published before consolidation keys existed", async () => { + const f = fixture(); + await seed(f); + await f.kv.set(KV.memories, "legacy", memory("legacy", { + sourceObservationIds: ["session-a-auth-0", "session-a-auth-1", "session-a-auth-2"], + sessionIds: ["session-a"], + })); + expect(await f.run()).toMatchObject({ consolidated: 0 }); + expect(await f.memories()).toHaveLength(1); + }); + + it("keeps both concepts settled when they produce the same title", async () => { + const f = fixture(); + const session: Session = { id: "session-a", project: "app", cwd: "/synthetic", startedAt: "2026-01-01T00:00:00Z", status: "completed", observationCount: 3 }; + await f.kv.set(KV.sessions, session.id, session); + for (let i = 0; i < 3; i++) { + await f.kv.set(KV.observations(session.id), `shared-${i}`, { + id: `shared-${i}`, sessionId: session.id, timestamp: session.startedAt, type: "decision", title: `shared ${i}`, + narrative: `Evidence ${i}`, facts: [], concepts: ["auth", "tokens"], files: [], importance: 8, + } satisfies CompressedObservation); + } + expect(await f.run()).toMatchObject({ consolidated: 2 }); + const count = (await f.memories()).length; + expect(await f.run()).toMatchObject({ consolidated: 0 }); + expect(await f.memories()).toHaveLength(count); + }); + + it("ignores compression bookkeeping when deciding whether evidence changed", async () => { + const f = fixture(); + await seed(f); + expect(await f.run()).toMatchObject({ consolidated: 1 }); + const row = await f.kv.get(KV.observations("session-a"), "session-a-auth-0"); + await f.kv.set(KV.observations("session-a"), row!.id, { ...row!, compressedAt: "2026-02-01T00:00:00Z", compression: { revision: "r2", attempts: 2 } } as CompressedObservation); + expect(await f.run()).toMatchObject({ consolidated: 0 }); + }); + + it("does not extend a lineage that already has two latest heads with the same title", async () => { + const f = fixture(); + await seed(f); + await f.kv.set(KV.memories, "a", memory("a")); + await f.kv.set(KV.memories, "b", memory("b")); + expect(await f.run()).toMatchObject({ consolidated: 0 }); + expect((await f.memories()).filter((m) => m.isLatest !== false).map((m) => m.id).sort()).toEqual(["a", "b"]); + }); + + it("leaves an agent-owned memory with the same title as the latest version", async () => { + const f = fixture(); + await seed(f); + await f.kv.set(KV.memories, "owned", memory("owned", { agentId: "agent-1" })); + expect(await f.run()).toMatchObject({ consolidated: 1 }); + const owned = await f.kv.get(KV.memories, "owned"); + expect(owned!.isLatest).toBe(true); + const created = (await f.memories()).find((m) => m.id !== "owned")!; + expect(created.parentId).toBeUndefined(); + }); + + it("does not demote a global memory during a project-scoped run", async () => { + const f = fixture(); + await seed(f); + await f.kv.set(KV.memories, "global", memory("global", { project: undefined })); + expect(await f.run()).toMatchObject({ consolidated: 1 }); + expect((await f.kv.get(KV.memories, "global"))!.isLatest).toBe(true); + }); + + it("publishes when only compression bookkeeping changes while the model runs", async () => { + const f = fixture(); + await seed(f); + f.compress.mockImplementation(async () => { + const row = await f.kv.get(KV.observations("session-a"), "session-a-auth-0"); + await f.kv.set(KV.observations("session-a"), row!.id, { ...row!, compression: { revision: "retry", attempts: 2 } } as CompressedObservation); + return xml("Shared title"); + }); + expect(await f.run()).toMatchObject({ consolidated: 1 }); + }); + + it("adopts a legacy memory once and publishes when its evidence later changes", async () => { + const f = fixture(); + await seed(f); + await f.kv.set(KV.memories, "legacy", memory("legacy", { + sourceObservationIds: ["session-a-auth-0", "session-a-auth-1", "session-a-auth-2"], + sessionIds: ["session-a"], + })); + expect(await f.run()).toMatchObject({ consolidated: 0 }); + expect((await f.kv.get(KV.memories, "legacy"))!.consolidationKeys?.auth).toBeTruthy(); + expect(safeAudit).toHaveBeenCalledWith(f.kv, "evolve", "mem::consolidate", ["legacy"], { action: "adopt_legacy", concept: "auth" }); + const row = await f.kv.get(KV.observations("session-a"), "session-a-auth-0"); + await f.kv.set(KV.observations("session-a"), row!.id, { ...row!, narrative: "Recompressed evidence" }); + expect(await f.run()).toMatchObject({ consolidated: 1 }); + }); + + it("records the legacy adoption after releasing the publication locks", async () => { + const f = fixture(); + await seed(f); + await f.kv.set(KV.memories, "legacy", memory("legacy", { + sourceObservationIds: ["session-a-auth-0", "session-a-auth-1", "session-a-auth-2"], + sessionIds: ["session-a"], + })); + let lockFree = false; + vi.mocked(safeAudit).mockImplementationOnce(async () => { + lockFree = await Promise.race([ + withKeyedLock("mem:remember", async () => true), + new Promise((resolve) => setTimeout(() => resolve(false), 200)), + ]); + }); + await f.run(); + expect(lockFree).toBe(true); + }); +}); + + +describe("custom retained evidence through upstream publication", () => { + it("retains original source payload, cursor, retry history and predecessor extension fields", async () => { + const f = fixture(); + await seed(f); + const original = await f.kv.get(KV.observations("session-a"), "session-a-auth-0"); + const retained = { ...original!, sourceRaw: "synthetic original source", summaryCursor: 42, retryHistory: [{ code: "prior_timeout" }], customUnknown: { retained: true } }; + await f.kv.set(KV.observations("session-a"), retained.id, retained); + const previous = { ...memory("previous"), customUnknown: { retained: true } }; + await f.kv.set(KV.memories, previous.id, previous); + await f.run(); + expect(await f.kv.get(KV.observations("session-a"), retained.id)).toEqual(retained); + expect(await f.kv.get(KV.memories, previous.id)).toMatchObject({ + content: previous.content, customUnknown: previous.customUnknown, isLatest: false, + }); + const current = (await f.memories()).find((row) => row.isLatest); + expect(current).toMatchObject({ parentId: previous.id, supersedes: [previous.id], version: 2 }); + }); +}); diff --git a/test/consolidation-pipeline.test.ts b/test/consolidation-pipeline.test.ts index b442237b37..12ec1eaabb 100644 --- a/test/consolidation-pipeline.test.ts +++ b/test/consolidation-pipeline.test.ts @@ -299,6 +299,76 @@ describe("Consolidation Pipeline", () => { expect(await kv.list("mem:procedural")).toEqual([original]); }); + it("counts re-saved pattern memories with no session attribution as recurring", async () => { + const provider = { + name: "test", + compress: vi.fn(), + summarize: vi.fn().mockResolvedValue( + `Save it`, + ), + }; + registerConsolidationPipelineFunction(sdk as never, kv as never, provider as never); + + for (let i = 0; i < 2; i++) { + await kv.set("mem:memories", `mem_${i}`, { ...makePattern(i), sessionIds: [], version: 2, recurrence: 2 }); + } + + const result = (await sdk.trigger("mem::consolidate-pipeline", { + tier: "procedural", + })) as { success: boolean; results: Record }; + + expect(provider.summarize).toHaveBeenCalledOnce(); + expect(result.results.procedural).toMatchObject({ newProcedures: 1 }); + }); + + it("does not treat edited pattern memories as recurring", async () => { + const provider = { name: "test", compress: vi.fn(), summarize: vi.fn() }; + registerConsolidationPipelineFunction(sdk as never, kv as never, provider as never); + + for (let i = 0; i < 3; i++) { + await kv.set("mem:memories", `mem_${i}`, { ...makePattern(i), sessionIds: [], version: 3 }); + } + + const result = (await sdk.trigger("mem::consolidate-pipeline", { + tier: "procedural", + })) as { success: boolean; results: Record }; + + expect(provider.summarize).not.toHaveBeenCalled(); + expect(result.results.procedural).toMatchObject({ skipped: true }); + }); + + it("still skips single-save pattern memories with no session attribution", async () => { + const provider = { name: "test", compress: vi.fn(), summarize: vi.fn() }; + registerConsolidationPipelineFunction(sdk as never, kv as never, provider as never); + + for (let i = 0; i < 3; i++) { + await kv.set("mem:memories", `mem_${i}`, { ...makePattern(i), sessionIds: [] }); + } + + const result = (await sdk.trigger("mem::consolidate-pipeline", { + tier: "procedural", + })) as { success: boolean; results: Record }; + + expect(provider.summarize).not.toHaveBeenCalled(); + expect(result.results.procedural).toMatchObject({ skipped: true }); + }); + + it("does not count duplicate session ids twice", async () => { + const provider = { name: "test", compress: vi.fn(), summarize: vi.fn() }; + registerConsolidationPipelineFunction(sdk as never, kv as never, provider as never); + + for (let i = 0; i < 3; i++) { + await kv.set("mem:memories", `mem_${i}`, { ...makePattern(i), sessionIds: ["ses_1", "ses_1"] }); + } + + const result = (await sdk.trigger("mem::consolidate-pipeline", { + tier: "procedural", + })) as { success: boolean; results: Record }; + + expect(provider.summarize).not.toHaveBeenCalled(); + expect(result.results.procedural).toMatchObject({ skipped: true }); + }); + it("consolidation records an audit entry", async () => { const provider = { name: "test", diff --git a/test/cursor-adapter.test.ts b/test/cursor-adapter.test.ts index f8c7517f16..c1b2145465 100644 --- a/test/cursor-adapter.test.ts +++ b/test/cursor-adapter.test.ts @@ -239,6 +239,137 @@ describe("built shims against a local server", () => { rmSync(workspace, { recursive: true, force: true }); }, 30000); + it.each([ + { + name: "Cursor summary and subagent type", + fields: { subagent_type: "explore", summary: "Found authenticateUser in auth.ts" }, + expected: { agent_type: "explore", last_message: "Found authenticateUser in auth.ts" }, + }, + { + name: "canonical fields when both forms are present", + fields: { + subagent_type: "explore", + summary: "Cursor summary", + agent_type: "Explore", + last_assistant_message: "Canonical reply", + }, + expected: { agent_type: "Explore", last_message: "Canonical reply" }, + }, + { + name: "existing response and agentType aliases", + fields: { + subagent_type: "explore", + summary: "Cursor summary", + agentType: "Explore", + response: "Existing reply", + }, + expected: { agent_type: "Explore", last_message: "Existing reply" }, + }, + { + name: "an existing response alongside a non-string summary", + fields: { subagent_type: "explore", summary: { text: "Preview" }, response: "Existing reply" }, + expected: { agent_type: "explore", last_message: "Existing reply" }, + }, + { + name: "an explicitly empty response alias", + fields: { subagent_type: "explore", summary: "Cursor summary", response: "" }, + expected: { agent_type: "explore", last_message: "" }, + }, + { + name: "the agentDisplayName fallback after empty type fields", + fields: { subagent_type: "explore", agent_type: "", agentType: "", agentDisplayName: "Explorer" }, + expected: { agent_type: "Explorer", last_message: "" }, + }, + { + name: "the agentName type fallback", + fields: { subagent_type: "explore", agentName: "named-agent" }, + expected: { agent_type: "named-agent", last_message: "" }, + }, + { + name: "an explicitly empty canonical reply", + fields: { subagent_type: "explore", summary: "Cursor summary", last_assistant_message: "" }, + expected: { agent_type: "explore", last_message: "" }, + }, + { + name: "a missing summary", + fields: { subagent_type: "explore" }, + expected: { agent_type: "explore", last_message: "" }, + }, + { + name: "a non-string summary", + fields: { subagent_type: "explore", summary: { text: "Not a string" } }, + expected: { agent_type: "explore", last_message: "" }, + }, + ])("subagentStop preserves $name through the built adapter", async ({ fields, expected }) => { + const workspace = mkdtempSync(join(tmpdir(), "am-cursor-subagent-")).replace(/\\/g, "/"); + const sessionId = uniqueId("shim-subagent"); + try { + const { code, stderr } = await runShim( + "run-hook.mjs", + ["subagentStop"], + { conversation_id: sessionId, workspace_roots: [workspace], status: "completed", ...fields }, + { + ...hookEnv(), + HOME: join(workspace, "home"), + USERPROFILE: join(workspace, "home"), + AGENTMEMORY_CAPTURE_SPOOL_DIR: join(workspace, "spool"), + }, + ); + expect(code).toBe(0); + expect(stderr).toBe(""); + const hits = server.captures.filter((c) => + (c.body as { sessionId?: string })?.sessionId === sessionId, + ); + expect(hits).toHaveLength(1); + expect(hits[0].url).toBe("/agentmemory/observe"); + expect(hits[0].body).toMatchObject({ + hookType: "subagent_stop", + sessionId, + project: basename(workspace), + cwd: workspace, + data: expected, + }); + } finally { + rmSync(workspace, { recursive: true, force: true }); + } + }, 15000); + + it("subagentStop redacts the Cursor summary before truncating it", async () => { + const workspace = mkdtempSync(join(tmpdir(), "am-cursor-subagent-")).replace(/\\/g, "/"); + const sessionId = uniqueId("shim-private-summary"); + try { + const { code } = await runShim( + "run-hook.mjs", + ["subagentStop"], + { + conversation_id: sessionId, + workspace_roots: [workspace], + subagent_type: "explore", + summary: "A".repeat(3980) + "fixture-private-value" + "x".repeat(100) + "" + "B".repeat(100), + }, + { + ...hookEnv(), + HOME: join(workspace, "home"), + USERPROFILE: join(workspace, "home"), + AGENTMEMORY_CAPTURE_SPOOL_DIR: join(workspace, "spool"), + }, + ); + expect(code).toBe(0); + const hit = server.captures.find((c) => + (c.body as { sessionId?: string })?.sessionId === sessionId, + ); + expect(hit).toBeDefined(); + const body = hit!.body as { data: { agent_type: string; last_message: string } }; + expect(body.data.agent_type).toBe("explore"); + expect(body.data.last_message).toHaveLength(4000); + expect(body.data.last_message.startsWith("A".repeat(3980))).toBe(true); + expect(body.data.last_message).not.toContain("fixture-private-value"); + expect(body.data.last_message).not.toContain(""); + } finally { + rmSync(workspace, { recursive: true, force: true }); + } + }, 15000); + it("run-detached delivers a large payload through its background worker", async () => { const workspace = mkdtempSync(join(tmpdir(), "am-cursor-ws-")); const sessionId = uniqueId("shim-stop"); diff --git a/test/devtools-port.test.ts b/test/devtools-port.test.ts new file mode 100644 index 0000000000..dda14078e1 --- /dev/null +++ b/test/devtools-port.test.ts @@ -0,0 +1,13 @@ +import {it,expect} from 'vitest'; +import {waitForDevToolsPort} from './helpers/devtools-port.js'; +const endpoint='42093\n/devtools/browser/11111111-1111-4111-8111-111111111111'; +it('waits through absent, empty, partial port and incomplete endpoint publication',async()=>{ + let now=0,reads=0;const rows=[null,'','42','42093','42093\n/devtools/browser/',endpoint]; + const port=await waitForDevToolsPort('unused',100,()=>true,{clock:()=>now,pause:async()=>{now++;},read:()=>{const row=rows[reads++];if(row===null)throw Object.assign(Error('absent'),{code:'ENOENT'});return row!;}}); + expect(port).toBe(42093);expect(reads).toBe(6);expect(now).toBe(5); +}); +it.each(['0','65536','junk'])('rejects invalid port %s until bounded deadline',async port=>{ + let now=0;await expect(waitForDevToolsPort('unused',3,()=>true,{clock:()=>now,pause:async()=>{now++;},read:()=>endpoint.replace('42093',port)})).rejects.toThrow('deadline');expect(now).toBe(3); +}); +it('fails on child exit without accepting stale endpoint contents',async()=>{await expect(waitForDevToolsPort('unused',100,()=>false,{clock:()=>0,read:()=>endpoint})).rejects.toThrow('exited');}); +it('propagates read permissions failure rather than retrying it',async()=>{await expect(waitForDevToolsPort('unused',100,()=>true,{clock:()=>0,read:()=>{throw Object.assign(Error('denied'),{code:'EACCES'});}})).rejects.toThrow('denied');}); diff --git a/test/diagnostics.test.ts b/test/diagnostics.test.ts index b2d1c0b345..419c14becd 100644 --- a/test/diagnostics.test.ts +++ b/test/diagnostics.test.ts @@ -5,6 +5,7 @@ vi.mock("../src/logger.js", () => ({ })); import { registerDiagnosticsFunction } from "../src/functions/diagnostics.js"; +import { isFacetStillOrphan } from "../src/functions/facets.js"; import type { Action, ActionEdge, @@ -195,11 +196,12 @@ describe("Diagnostics Functions", () => { }; expect(result.success).toBe(true); - // 15 = 8 original (actions, leases, sentinels, sketches, signals, + // 16 = 8 original (actions, leases, sentinels, sketches, signals, // sessions, memories, mesh) + 6 added in #lesson-visibility // (lessons, summaries, semantic, procedural, crystals, insights) + - // 1 added in #memory-project-scope (memory-project-coverage). - expect(result.summary.pass).toBe(15); + // 1 added in #memory-project-scope (memory-project-coverage) + + // facets-ok. + expect(result.summary.pass).toBe(16); expect(result.summary.warn).toBe(0); expect(result.summary.fail).toBe(0); expect(result.summary.fixable).toBe(0); @@ -493,6 +495,89 @@ describe("Diagnostics Functions", () => { }); }); + describe("orphan facets", () => { + const facet = (id: string, targetId: string, targetType: "memory" | "action" | "observation") => ({ + id, targetId, targetType, dimension: "tool", value: "x", createdAt: new Date().toISOString(), + }); + + beforeEach(async () => { + const memory = { id: "mem_live", title: "live", content: "c", type: "fact", concepts: [], files: [], sessionIds: [], strength: 5, version: 1, isLatest: true, createdAt: "", updatedAt: "" }; + await kv.set(KV.memories, memory.id, memory); + const action = makeAction({ id: "act_live" } as Partial); + await kv.set(KV.actions, action.id, action); + for (const f of [ + facet("fct_1", "mem_live", "memory"), + facet("fct_2", "mem_gone", "memory"), + facet("fct_3", action.id, "action"), + facet("fct_4", "act_gone", "action"), + facet("fct_5", "obs_unindexed", "observation"), + facet("fct_6", "obs_gone", "observation"), + ]) await kv.set(KV.facets, f.id, f); + await kv.set(KV.sessions, "ses_1", { id: "ses_1" }); + await kv.set(KV.observations("ses_1"), "obs_unindexed", { id: "obs_unindexed", sessionId: "ses_1" }); + }); + + it("does not delete a facet retargeted after the orphan scan", async()=>{ + const stale=facet("fct_changed","mem_missing","memory"); + await kv.set(KV.facets,stale.id,{...stale,targetId:"mem_live"}); + expect(await isFacetStillOrphan(kv as never,stale)).toBe(false); + }); + it("rejects a replaced facet with a different creation timestamp", async()=>{ + const stored=(await kv.get>(KV.facets,"fct_2"))!; + const stale={...stored,createdAt:new Date(Date.parse(stored.createdAt)-1).toISOString()}; + expect(await isFacetStillOrphan(kv as never,stale)).toBe(false); + expect(await isFacetStillOrphan(kv as never,stored)).toBe(true); + }); + it("rechecks unindexed observations restored into a live session before delete", async()=>{ + const stale=facet("fct_restored","restored-no-prefix","observation"); + await kv.set(KV.facets,stale.id,stale); + await kv.set(KV.observations("ses_1"),stale.targetId,{id:stale.targetId,sessionId:"ses_1"}); + expect(await isFacetStillOrphan(kv as never,stale)).toBe(false); + }); + it("diagnose reports facets whose target was deleted", async () => { + const result = (await sdk.trigger("mem::diagnose", { categories: ["facets"] })) as { checks: DiagnosticCheck[] }; + expect(result.checks).toEqual([expect.objectContaining({ name: "orphan-facets", status: "warn", fixable: true })]); + }); + + it("heal dry run reports orphan facets without removing them", async () => { + const result = (await sdk.trigger("mem::heal", { categories: ["facets"], dryRun: true })) as { fixed: number }; + expect(result.fixed).toBe(3); + expect((await kv.list(KV.facets)).length).toBe(6); + }); + + it("heal removes only facets whose target was deleted", async () => { + const result = (await sdk.trigger("mem::heal", { categories: ["facets"] })) as { fixed: number }; + expect(result.fixed).toBe(3); + expect((await kv.list<{ id: string }>(KV.facets)).map((f) => f.id).sort()).toEqual(["fct_1", "fct_3", "fct_5"]); + const after = (await sdk.trigger("mem::diagnose", { categories: ["facets"] })) as { checks: DiagnosticCheck[] }; + expect(after.checks[0]!.name).toBe("facets-ok"); + }); + + it("heal keeps a facet on a migrated observation that has no session record", async () => { + await kv.set(KV.facets, "fct_7", facet("fct_7", "obs_migrated", "observation")); + await kv.set(KV.observations("migrated"), "obs_migrated", { id: "obs_migrated", sessionId: "migrated" }); + await sdk.trigger("mem::heal", { categories: ["facets"] }); + expect((await kv.list<{ id: string }>(KV.facets)).map((f) => f.id)).toContain("fct_7"); + expect(await isFacetStillOrphan(kv as never, (await kv.get>(KV.facets,"fct_7"))!)).toBe(false); + }); + + it("heal keeps a facet on a migrated observation whose id has no obs_ prefix", async () => { + await kv.set(KV.facets, "fct_8", facet("fct_8", "mig_abc", "observation")); + await kv.set(KV.observations("migrated"), "mig_abc", { id: "mig_abc", sessionId: "migrated" }); + await sdk.trigger("mem::heal", { categories: ["facets"] }); + expect((await kv.list<{ id: string }>(KV.facets)).map((f) => f.id)).toContain("fct_8"); + expect(await isFacetStillOrphan(kv as never, (await kv.get>(KV.facets,"fct_8"))!)).toBe(false); + }); + + it("heal keeps a facet whose target comes back before the delete", async () => { + const orphan = (await kv.get>(KV.facets, "fct_2"))!; + expect(orphan).not.toBeNull(); + expect(await isFacetStillOrphan(kv as never, orphan)).toBe(true); + await kv.set(KV.memories, "mem_gone", { id: "mem_gone" }); + expect(await isFacetStillOrphan(kv as never, orphan)).toBe(false); + }); + }); + describe("mem::heal", () => { it("unblocks stuck blocked action", async () => { const dep = makeAction({ status: "done" }); diff --git a/test/docker-port-config.test.ts b/test/docker-port-config.test.ts index 7c69e96237..bfd1dba008 100644 --- a/test/docker-port-config.test.ts +++ b/test/docker-port-config.test.ts @@ -50,8 +50,8 @@ describe("Docker Worker Compose runtime", () => { it("starts the pinned runtime through the shared entrypoint with telemetry disabled", () => { expect(compose).toContain('III_TELEMETRY_ENABLED: "false"'); - expect(entrypoint).toContain("/usr/local/bin/iii compose"); - expect(entrypoint).toContain("--namespace agentmemory --up --file"); + expect(entrypoint).toContain("node /opt/agentmemory/deploy-runtime/state-bootstrap.mjs"); + expect(readFileSync(join(root,"deploy/runtime/state-bootstrap.mjs"),"utf8")).toContain("['compose','--namespace',config.namespace,'--up','--file',file]"); expect(workerCompose).toContain("configuration:"); expect(workerCompose).toContain('save_interval_ms: "${AGENTMEMORY_STATE_SAVE_INTERVAL_MS:-2000}"'); expect(entrypoint).toContain('AGENTMEMORY_STATE_SAVE_INTERVAL_MS="${AGENTMEMORY_STATE_SAVE_INTERVAL_MS:-2000}"'); diff --git a/test/embedding-scheduling.test.ts b/test/embedding-scheduling.test.ts new file mode 100644 index 0000000000..08b7f8e016 --- /dev/null +++ b/test/embedding-scheduling.test.ts @@ -0,0 +1,117 @@ +import {afterEach,expect,it,vi} from 'vitest'; +import {mockKV} from './helpers/mocks.js'; +import {KV} from '../src/state/schema.js'; +import {VectorIndex} from '../src/state/vector-index.js'; +import {withKeyedLock} from '../src/state/keyed-mutex.js'; +import {installBackgroundControl} from '../src/ops/background-control.mjs'; +import {serializeLongSessionProvider} from '../src/ops/long-session.mjs'; +import {registerEmbeddingRecovery,EMBEDDING_RETRY_NAMESPACE as RETRY} from '../src/ops/embedding-recovery.mjs'; + +const workers:ReturnType[]=[]; +afterEach(async()=>{installBackgroundControl(null);for(const worker of workers.splice(0)){worker.stop();await worker.drain();}vi.useRealTimers();}); +async function fixture(count=12){ + vi.useFakeTimers();vi.setSystemTime(100000); + const kv=mockKV(),index=new VectorIndex();let active=0,peak=0; + const embed=vi.fn(async()=>{peak=Math.max(peak,++active);await Promise.resolve();active--;return new Float32Array([1,0]);}); + const persistIndex=vi.fn(async()=>{}); + const options={kv:kv as never,KV,index,provider:{dimensions:2,embed},withKeyedLock,schedule:false,persistIndex}; + const worker=registerEmbeddingRecovery(options);workers.push(worker); + for(let n=0;npeak}; +} +it('continues a published backlog after a one-second yield with at most five serial inferences per round',async()=>{ + const f=await fixture();f.worker.start();await vi.advanceTimersByTimeAsync(15000); + expect(f.worker.snapshot().completedCount).toBe(5);expect(f.peak()).toBe(1); + await vi.advanceTimersByTimeAsync(999);expect(f.embed).toHaveBeenCalledTimes(5); + await vi.advanceTimersByTimeAsync(1);expect(f.worker.snapshot().completedCount).toBe(10); + await vi.advanceTimersByTimeAsync(1000);expect(f.worker.snapshot().completedCount).toBe(12); + expect(await f.kv.list(RETRY)).toHaveLength(0);expect(f.peak()).toBe(1); +}); +it('rotates past unchanged failing head items without changing their attempts or source',async()=>{ + const f=await fixture(6),get=f.kv.get; + f.kv.get=async(scope,id)=>{if(scope===KV.observations('s')&&Number(id)<5)throw Error('synthetic read failure');return get(scope,id);}; + f.worker.start();await vi.advanceTimersByTimeAsync(15000);expect(f.embed).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(14999);expect(f.embed).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(1);expect(f.index.has('5')).toBe(true); + for(let n=0;n<5;n++){expect(await get(RETRY,String(n))).toMatchObject({attempts:0,retryAt:0});expect(await get(KV.observations('s'),String(n))).toMatchObject({title:'synthetic '+n});} +}); +it('does not start another round while any selected publication is unresolved',async()=>{ + const f=await fixture(11);let release!:()=>void;const held=new Promise(r=>release=r); + f.persistIndex.mockImplementation(async()=>held);f.worker.start(); + try{await vi.advanceTimersByTimeAsync(15000);expect(f.embed).toHaveBeenCalledTimes(5);expect(f.worker.snapshot().completedCount).toBe(0); + await vi.advanceTimersByTimeAsync(45000);expect(f.embed).toHaveBeenCalledTimes(5); + }finally{release();await f.worker.drain();} + await vi.advanceTimersByTimeAsync(999);expect(f.embed).toHaveBeenCalledTimes(5); + await vi.advanceTimersByTimeAsync(1);expect(f.embed).toHaveBeenCalledTimes(10); +}); +it('uses the idle interval after a failed publication even when other entries complete',async()=>{ + const f=await fixture();let failed=false; + f.persistIndex.mockImplementation(async()=>{if(!failed){failed=true;throw Error('synthetic disk failure');}}); + f.worker.start();await vi.advanceTimersByTimeAsync(15000);expect(f.worker.snapshot().completedCount).toBe(4); + await vi.advanceTimersByTimeAsync(14999);expect(f.embed).toHaveBeenCalledTimes(5); + await vi.advanceTimersByTimeAsync(1);expect(f.embed).toHaveBeenCalledTimes(10); + expect(await f.kv.get(RETRY,'0')).toMatchObject({attempts:1,retryAt:145000}); +}); +it('keeps provider backoff and does not poll a future-only queue at the fast rate',async()=>{ + const f=await fixture(1);f.embed.mockRejectedValueOnce(Object.assign(Error('synthetic provider failure'),{code:'ECONNRESET'})); + const list=vi.spyOn(f.kv,'list');f.worker.start();await vi.advanceTimersByTimeAsync(15000); + expect(await f.kv.get(RETRY,'0')).toMatchObject({attempts:1,retryAt:145000});const scans=list.mock.calls.length; + await vi.advanceTimersByTimeAsync(14999);expect(list.mock.calls.length).toBe(scans); + await vi.advanceTimersByTimeAsync(1);expect(f.embed).toHaveBeenCalledTimes(1); + await vi.advanceTimersByTimeAsync(15000);expect(f.embed).toHaveBeenCalledTimes(2);expect(f.index.has('0')).toBe(true); +}); +it('does not bypass provider waiting or create multiple schedules through repeated start',async()=>{ + const f=await fixture();f.worker.stop();let ready=false; + const worker=registerEmbeddingRecovery({...f.options,readiness:{ensureReady:async()=>ready,snapshot:()=>({status:ready?'ready':'waiting'})}as never});workers.push(worker); + worker.start();worker.start();await vi.advanceTimersByTimeAsync(15000);expect(f.embed).not.toHaveBeenCalled(); + ready=true;await vi.advanceTimersByTimeAsync(14999);expect(f.embed).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(1);expect(f.embed).toHaveBeenCalledTimes(5); + await vi.advanceTimersByTimeAsync(1000);expect(f.embed).toHaveBeenCalledTimes(10); +}); +it.each(['pause','stop'])('closes fast admission on %s while preserving remaining intents',async mode=>{ + const f=await fixture();let paused=false;installBackgroundControl({isPaused:()=>paused,run:async(fn:()=>unknown)=>paused?{deferred:true}:fn()}as never); + f.worker.start();await vi.advanceTimersByTimeAsync(15000);expect(f.embed).toHaveBeenCalledTimes(5); + if(mode==='pause')paused=true;else f.worker.stop(); + await vi.advanceTimersByTimeAsync(16000);expect(f.embed).toHaveBeenCalledTimes(5);expect(await f.kv.list(RETRY)).toHaveLength(7); + if(mode==='pause'){paused=false;await vi.advanceTimersByTimeAsync(15000);expect(f.embed.mock.calls.length).toBeGreaterThan(5);} +}); +it('lets an already-waiting chat run between individual embeddings',async()=>{ + const f=await fixture(5);f.worker.stop();const order:string[]=[];let release!:()=>void,entered!:()=>void; + const held=new Promise(r=>release=r),started=new Promise(r=>entered=r);let active=0,peak=0,n=0; + const provider=serializeLongSessionProvider({dimensions:2,async embed(){const id=n++;order.push('embed'+id);peak=Math.max(peak,++active);if(id===0){entered();await held;}active--;return new Float32Array([1,0]);}},withKeyedLock); + const chat=serializeLongSessionProvider({async summarize(){order.push('chat');peak=Math.max(peak,++active);active--;return 'synthetic';}},withKeyedLock); + const worker=registerEmbeddingRecovery({...f.options,provider});workers.push(worker); + try{const pump=worker.pump();await started;const chatting=chat.summarize();release();await Promise.all([pump,chatting]);await worker.drain();expect(order).toEqual(['embed0','chat','embed1','embed2','embed3','embed4']);expect(peak).toBe(1);}finally{release();} +}); +it('keeps old unattempted work ahead of new arrivals despite reversed engine enumeration',async()=>{ + const f=await fixture(6),get=f.kv.get,list=f.kv.list; + f.kv.get=async(scope,id)=>{if(scope===KV.observations('s')&&Number(id)<5)throw Error('synthetic read failure');return get(scope,id);}; + await f.worker.pump();expect(f.embed).not.toHaveBeenCalled(); + for(let n=6;n<12;n++){const id=String(n);await f.kv.set(KV.observations('s'),id,{id,sessionId:'s',title:'new '+id});await f.worker.enqueue({id,sessionId:'s',kind:'synthetic'});} + f.kv.list=async scope=>(await list(scope)).reverse(); + await f.worker.pump();expect(f.index.has('5')).toBe(true);expect(f.embed).toHaveBeenCalledTimes(1); + expect(await f.kv.list(RETRY)).toHaveLength(11); +}); +it('stops scheduling while drain still waits for already admitted publications',async()=>{ + const f=await fixture();let release!:()=>void;const held=new Promise(r=>release=r);f.persistIndex.mockImplementation(async()=>held); + f.worker.start(); + try{await vi.advanceTimersByTimeAsync(15000);f.worker.start();f.worker.stop();let drained=false;const draining=f.worker.drain().then(()=>{drained=true;}); + await vi.advanceTimersByTimeAsync(30000);expect(f.embed).toHaveBeenCalledTimes(5);expect(drained).toBe(false); + release();await draining;expect(f.worker.snapshot().completedCount).toBe(5); + await vi.advanceTimersByTimeAsync(30000);expect(f.embed).toHaveBeenCalledTimes(5);expect(await f.kv.list(RETRY)).toHaveLength(7); + }finally{release();await f.worker.drain();} +}); +it('does not schedule a fast round when readiness changes during publication',async()=>{ + const f=await fixture();f.worker.stop();let ready=true; + const worker=registerEmbeddingRecovery({...f.options,readiness:{ensureReady:async()=>ready,snapshot:()=>({status:ready?'ready':'waiting'})}as never});workers.push(worker); + f.persistIndex.mockImplementation(async()=>{ready=false;});worker.start();await vi.advanceTimersByTimeAsync(15000); + const calls=f.embed.mock.calls.length;expect(calls).toBeGreaterThan(0);ready=true; + await vi.advanceTimersByTimeAsync(14999);expect(f.embed).toHaveBeenCalledTimes(calls); + await vi.advanceTimersByTimeAsync(1);expect(f.embed.mock.calls.length).toBeGreaterThan(calls); +}); +it('returns to idle polling once the final publication finishes',async()=>{ + const f=await fixture(1),list=vi.spyOn(f.kv,'list');f.worker.start();await vi.advanceTimersByTimeAsync(15000); + expect(f.worker.snapshot().completedCount).toBe(1);const scans=list.mock.calls.length; + await vi.advanceTimersByTimeAsync(14999);expect(list.mock.calls.length).toBe(scans); + await vi.advanceTimersByTimeAsync(1);expect(list.mock.calls.length).toBe(scans+1);expect(f.embed).toHaveBeenCalledOnce(); +}); diff --git a/test/embedding-scoped-publication.test.ts b/test/embedding-scoped-publication.test.ts index 05eec11b05..7fb80a3f40 100644 --- a/test/embedding-scoped-publication.test.ts +++ b/test/embedding-scoped-publication.test.ts @@ -12,7 +12,7 @@ import { registerEmbeddingRecovery, EMBEDDING_RETRY_NAMESPACE as RETRY, EMBEDDIN vi.mock('../src/logger.js', () => ({ logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() } })); const fixtures: ReturnType[] = []; -function fixture() { +function fixture(timeoutMs = 2000) { const directory = mkdtempSync(join(tmpdir(), 'scoped-vector-publication-')); const kv = mockKV(), index = new VectorIndex(); const blocked = new Set(); @@ -23,7 +23,9 @@ function fixture() { const set = kv.set, remove = kv.delete; kv.set = async (scope, key, value) => { const result = await set(scope, key, value); flush(scope); changed?.(scope, key); return result; }; kv.delete = async (scope, key) => { await remove(scope, key); flush(scope); changed?.(scope, key); }; - Object.assign(kv, { backend: 'file', durability: createStateDurabilityFence({ storePath: directory, timeoutMs: 80, pollIntervalMs: 5 }) }); + // These tests certify real async filesystem I/O, not 80ms CI throughput. + // Injected mismatch tests still require the fence to expire without publishing. + Object.assign(kv, { backend: 'file', durability: createStateDurabilityFence({ storePath: directory, timeoutMs, pollIntervalMs: 5 }) }); const persistence = new IndexPersistence(kv as never, index); persistence.stop(); let clock = 0; const provider = { dimensions: 2, embed: vi.fn(async () => new Float32Array([1, 0])) }; @@ -116,6 +118,8 @@ it('keeps retry until the committed fingerprint is on disk and retries publicati const f=fixture();await f.kv.set(KV.observations('s'),item.id,{id:item.id,title:'canonical'});f.blocked.add(COMMITTED); await f.recovery.add(item);await f.recovery.drain();expect(await f.kv.get(RETRY,item.id)).toBeTruthy(); f.blocked.clear();f.advance();await f.recovery.pump();await f.recovery.drain(); + expect(await f.kv.get(RETRY,item.id)).toBeNull(); + expect(await f.kv.get(COMMITTED,item.id)).toBeTruthy(); expect(f.disk(COMMITTED)[item.id]).toBeTruthy();expect(f.disk(RETRY)[item.id]).toBeUndefined();expect(f.provider.embed).toHaveBeenCalledOnce(); }); @@ -181,3 +185,19 @@ it('bounds observed publication history and reports evicted identities as unknow expect(JSON.stringify(worker.snapshot())).not.toContain('proof_'); }finally{worker.stop();await worker.drain();} }); + + +it('retains retry without a committed file when an earlier vector read exceeds its finite budget', async () => { + const f = fixture(80); + await f.kv.set(KV.observations('s'), item.id, { id: item.id, title: 'canonical' }); + const get = f.kv.get; + f.kv.get = async (scope, key) => { + if (scope === vectorBucketScope(0)) await new Promise(resolve => setTimeout(resolve, 120)); + return get(scope, key); + }; + await f.recovery.add(item); await f.recovery.drain(); + expect(await get(RETRY, item.id)).toMatchObject({ attempts: 1, errorCode: 'VECTOR_PUBLICATION_UNVERIFIED' }); + expect(await get(COMMITTED, item.id)).toBeNull(); + expect(() => f.disk(COMMITTED)).toThrow(/ENOENT/); + expect(f.provider.embed).toHaveBeenCalledOnce(); +}); diff --git a/test/embedding-selected-source.test.ts b/test/embedding-selected-source.test.ts new file mode 100644 index 0000000000..8092b87bc1 --- /dev/null +++ b/test/embedding-selected-source.test.ts @@ -0,0 +1,72 @@ +import {afterEach,expect,it,vi} from 'vitest'; +import {mkdtemp,rm} from 'node:fs/promises'; +import {tmpdir} from 'node:os'; +import {join} from 'node:path'; +import {StateKV} from '../src/state/kv.js'; +import {KV} from '../src/state/schema.js'; +import {VectorIndex} from '../src/state/vector-index.js'; +import {withKeyedLock} from '../src/state/keyed-mutex.js'; +import {registerEmbeddingRecovery,EMBEDDING_RETRY_NAMESPACE as RETRY} from '../src/ops/embedding-recovery.mjs'; + +const cleanups:Array<()=>Promise>=[]; +afterEach(async()=>{vi.unstubAllEnvs();for(const cleanup of cleanups.splice(0))await cleanup();vi.restoreAllMocks();}); +async function fixture(){ + const dir=await mkdtemp(join(tmpdir(),'am-selected-source-'));vi.stubEnv('AGENTMEMORY_STATE_STORE_PATH',dir); + const rows=new Map>();let heldKey:string|undefined,started!:()=>void,resolve!:()=>void,reject!:(error:Error)=>void; + const entered=new Promise(yes=>started=yes),gate=new Promise((yes,no)=>{resolve=yes;reject=no;});void gate.catch(()=>{}); + const sdk={trigger:async({function_id,payload:p}:any)=>{ + const table=rows.get(p.scope); + if(function_id==='state::get')return structuredClone(table?.get(p.key)??null); + if(function_id==='state::list')return structuredClone([...table?.values()??[]]); + if(function_id==='state::set'){ + if(!rows.has(p.scope))rows.set(p.scope,new Map());rows.get(p.scope)!.set(p.key,structuredClone(p.value)); + if(p.key===heldKey){started();await gate;}return p.value; + } + if(function_id==='state::delete')return table?.delete(p.key)??false; + throw Error('unexpected synthetic SDK call'); + }}; + const kv=new StateKV(sdk as never),provider={name:'synthetic',dimensions:2,embed:vi.fn(async()=>new Float32Array([1,0]))}; + const worker=registerEmbeddingRecovery({kv,KV,index:new VectorIndex(),provider,withKeyedLock,schedule:false,persistIndex:async()=>{}}); + cleanups.push(async()=>{resolve();worker.stop();await worker.drain();await kv.durability!.drain();await rm(dir,{recursive:true,force:true});}); + const item={id:'selected',sessionId:'shared-session',kind:'synthetic' as const},scope=KV.observations(item.sessionId),row={id:item.id,sessionId:item.sessionId,title:'selected original',sourceRaw:{raw:'synthetic original'}}; + await kv.set(scope,item.id,row); + return{kv,worker,provider,item,scope,row,entered,resolve,reject,hold:(key:string)=>{heldKey=key;}}; +} + +it.each([false,true])('certifies a selected source with another same-session applied SDK write starting beforePrepare=%s',async beforePrepare=>{ + const f=await fixture();f.hold('other'); + let other:Promise|undefined;if(beforePrepare){other=f.kv.set(f.scope,'other',{id:'other',sessionId:f.item.sessionId});await f.entered;} + const certify=await f.worker.prepareEnqueue(f.item); + if(!other){other=f.kv.set(f.scope,'other',{id:'other',sessionId:f.item.sessionId});await f.entered;} + let completed=false;const pending=certify().then(value=>{completed=true;return value;});void pending.catch(()=>{}); + await new Promise(resolve=>setTimeout(resolve,10));expect(completed).toBe(false); + f.resolve();await other;await expect(pending).resolves.toBe(true);expect(await f.kv.get(f.scope,f.item.id)).toEqual(f.row); + expect(await f.kv.get(RETRY,f.item.id)).toMatchObject({id:f.item.id});expect(f.provider.embed).not.toHaveBeenCalled(); +}); + +it('refuses same-key ABA even when the final SDK row equals the original',async()=>{ + const f=await fixture(),certify=await f.worker.prepareEnqueue(f.item); + await f.kv.set(f.scope,f.item.id,{...f.row,title:'replacement'});await f.kv.delete(f.scope,f.item.id);await f.kv.set(f.scope,f.item.id,f.row); + await expect(certify()).rejects.toMatchObject({code:'EMBEDDING_SOURCE_CHANGED'});expect(await f.kv.get(f.scope,f.item.id)).toEqual(f.row); +}); + +it('refuses an unrelated uncertain SDK outcome without confusing it with target mutation',async()=>{ + const f=await fixture(),certify=await f.worker.prepareEnqueue(f.item);f.hold('other'); + const other=f.kv.set(f.scope,'other',{id:'other',sessionId:f.item.sessionId});void other.catch(()=>{});await f.entered; + const pending=certify();void pending.catch(()=>{});await new Promise(resolve=>setTimeout(resolve,10));f.reject(Error('synthetic lost response')); + await expect(other).rejects.toThrow('synthetic lost response');await expect(pending).rejects.toMatchObject({code:'STATE_DURABILITY_TIMEOUT',diagnostics:{errorCode:'SDK_MUTATION_UNCERTAIN'}}); + expect(f.kv.mutationOutcomeUncertain).toBe(true);expect(await f.kv.get(f.scope,f.item.id)).toEqual(f.row);expect(await f.kv.get(RETRY,f.item.id)).not.toBeNull(); +}); + +it.each(['success','prepareFailure','fenceFailure','stopped'] as const)('releases the selected mutation watch after %s',async outcome=>{ + const f=await fixture(),fence=f.kv.durability!,original=fence.watchKeys;let live=0; + vi.spyOn(fence,'watchKeys').mockImplementation(keys=>{const watch=original(keys);live++;let released=false;return{unchanged:watch.unchanged,release(){if(!released){released=true;live--;}watch.release();}};}); + if(outcome==='prepareFailure'){await f.kv.delete(f.scope,f.item.id);await expect(f.worker.prepareEnqueue(f.item)).rejects.toMatchObject({code:'EMBEDDING_SOURCE_CHANGED'});} + else{ + const certify=await f.worker.prepareEnqueue(f.item);expect(live).toBe(1); + if(outcome==='stopped'){f.worker.stop();expect(live).toBe(0);await expect(certify()).rejects.toMatchObject({code:'EMBEDDING_RECOVERY_STOPPED'});} + else if(outcome==='fenceFailure'){vi.spyOn(fence,'waitForKeys').mockRejectedValue(Object.assign(Error('synthetic missing fence'),{code:'EMBEDDING_FENCE_UNAVAILABLE'}));await expect(certify()).rejects.toMatchObject({code:'EMBEDDING_FENCE_UNAVAILABLE'});} + else await expect(certify()).resolves.toBe(true); + } + expect(live).toBe(0); +}); diff --git a/test/engine-config.test.ts b/test/engine-config.test.ts index c1dd680ba5..f70aa418eb 100644 --- a/test/engine-config.test.ts +++ b/test/engine-config.test.ts @@ -3,7 +3,7 @@ import { COMPOSE_WORKER_VERSIONS, renderWorkerCompose } from "../src/cli/compose describe("native Worker Compose runtime configuration", () => { it("pins state/http workers at the reviewed registry versions", () => { - expect(COMPOSE_WORKER_VERSIONS).toEqual({ state: "0.22.20", http: "0.22.7" }); + expect(COMPOSE_WORKER_VERSIONS).toEqual({ state: "0.22.22", http: "0.22.8" }); }); it("renders instance-specific worker ports and persistent state storage", () => { @@ -16,9 +16,9 @@ describe("native Worker Compose runtime configuration", () => { linux: false, }); expect(rendered).toContain("worker: 'package://api.workers.iii.dev/state'"); - expect(rendered).toContain("version: '0.22.20'"); + expect(rendered).toContain("version: '0.22.22'"); expect(rendered).toContain("worker: 'package://api.workers.iii.dev/http'"); - expect(rendered).toContain("version: '0.22.7'"); + expect(rendered).toContain("version: '0.22.8'"); expect(rendered).toContain("port: 3211"); expect(rendered).toContain("port: 49234"); expect(rendered).toContain("file_path: '/tmp/agentmemory/state_store.db'"); diff --git a/test/engine-epoch.test.ts b/test/engine-epoch.test.ts index 9624846859..7e1549b962 100644 --- a/test/engine-epoch.test.ts +++ b/test/engine-epoch.test.ts @@ -31,3 +31,35 @@ it.each(['foreign-peer','unowned-socket','duplicate-listen','namespace','missing it('does not treat hidden-but-live processes as terminated',async()=>{const f=await fixture();f.setLiveness(true);expect(f.evidence.gone({pid:201,start:'30'})).toBe(false);}); it('supports the actual IPv6 loopback socket tuple without DNS assumptions',async()=>{const f=await fixture(),address='00000000000000000000000001000000';f.setPeer({remoteAddress:'::1',remotePort:1000,localAddress:'::1',localPort:2000});f.write(join(f.proc,'200/net/tcp'),'');f.write(join(f.proc,'200/net/tcp6'),f.listen.replaceAll('0100007F',address).replace('00000000:0000','00000000000000000000000000000000:0000')+'\n'+f.connected.replaceAll('0100007F',address));expect(f.evidence.capture()).not.toBeNull();}); it('refuses a socket peer that changes while evidence is being captured',async()=>{const f=await fixture();let calls=0;f.setPeer(()=>({remoteAddress:'127.0.0.1',remotePort:1000,localAddress:'127.0.0.1',localPort:++calls===1?2000:2001}));expect(f.evidence.capture()).toBeNull();}); + +it('uses matching Compose metadata instead of a stale native PID',async()=>{ + const f=await fixture();const {readEnginePid}=await import('../src/ops/engine-epoch.mjs'); + f.write(join(f.d,'engine-state.json'),JSON.stringify({kind:'compose',engineUrl:'ws://localhost:1000',enginePid:200})); + const opts={enginePidPath:join(f.d,'iii.pid'),engineStatePath:join(f.d,'engine-state.json'),engineUrl:'ws://localhost:1000'}; + f.write(opts.enginePidPath,'999');expect(readEnginePid(opts)).toBe(200); + for(const state of [{},null,[],{kind:'unknown'},{kind:'compose',engineUrl:opts.engineUrl},{kind:'compose',engineUrl:'ws://localhost:2000',enginePid:200},{kind:'compose',engineUrl:opts.engineUrl,enginePid:0}]){ + f.write(opts.engineStatePath,JSON.stringify(state));expect(()=>readEnginePid(opts)).toThrow(); + } + f.write(opts.engineStatePath,'{');expect(()=>readEnginePid(opts)).toThrow(); + rmSync(opts.engineStatePath);expect(readEnginePid(opts)).toBe(999); +}); +it('validates Compose PID against the actual SDK socket',async()=>{ + const f=await fixture();const {createEngineEpochEvidence}=await import('../src/ops/engine-epoch.mjs'); + rmSync(join(f.d,'iii.pid'));f.write(join(f.d,'engine-state.json'),JSON.stringify({kind:'compose',engineUrl:'ws://localhost:1000',enginePid:200})); + const evidence=createEngineEpochEvidence({engineUrl:'ws://localhost:1000',enginePidPath:join(f.d,'iii.pid'),engineStatePath:join(f.d,'engine-state.json'),peer:()=>({remoteAddress:'127.0.0.1',remotePort:1000,localAddress:'127.0.0.1',localPort:2000}),procRoot:f.proc,machineIdPath:join(f.d,'machine-id'),workerPid:100,workerUid:1000}); + expect(evidence.capture()?.engine.pid).toBe(200);rmSync(join(f.proc,'200/fd/4'));expect(evidence.capture()).toBeNull(); +}); +it('waits for delayed PID publication before admitting a writer',async()=>{ + const {waitForEngineEpoch}=await import('../src/ops/engine-epoch.mjs'); + let calls=0;const epoch={engine:{pid:200,start:'20'}}; + expect(await waitForEngineEpoch({capture:()=>++calls<3?null:epoch} as any,{timeoutMs:100,pollMs:1})).toBe(epoch); + expect(calls).toBe(3); +}); +it('fails startup on missing engine evidence rather than returning an unsupported writer epoch',async()=>{ + const {waitForEngineEpoch}=await import('../src/ops/engine-epoch.mjs'); + await expect(waitForEngineEpoch({capture:()=>null} as any,{timeoutMs:5,pollMs:1})).rejects.toThrow('ENGINE_EPOCH_UNVERIFIED'); +}); +it('streams large unrelated TIME_WAIT tables without rejecting the exact owned connection',async()=>{ + const f=await fixture();f.write(join(f.proc,'200/net/tcp'),('0: 0100007F:0AAA 0100007F:0BBB 06 0 0 0 1000 0 0\n').repeat(26000)+f.listen+'\n'+f.connected);expect(f.evidence.capture()?.engine.pid).toBe(200); + f.write(join(f.proc,'200/net/tcp'),('0: 0100007F:0AAA 0100007F:0BBB 06 0 0 0 1000 0 0\n').repeat(26000)+f.listen+'\n'+f.connected.replace('501','999'));expect(f.evidence.capture()).toBeNull(); +}); diff --git a/test/engine-install.test.ts b/test/engine-install.test.ts index d7a065f5a2..bbee8c1c53 100644 --- a/test/engine-install.test.ts +++ b/test/engine-install.test.ts @@ -6,6 +6,9 @@ import { afterEach, describe, expect, it } from "vitest"; import { III_INSTALL_CONNECT_TIMEOUT_S, III_INSTALL_MAX_TIME_S, + III_INSTALL_RETRIES, + III_INSTALL_RETRY_DELAY_S, + III_INSTALL_RETRY_MAX_TIME_S, III_RELEASE_SHA256, describeInstallFailure, expectedIiiSha256, @@ -111,6 +114,14 @@ describe("iii engine auto-installer", () => { } }); + it("keeps the previous engine release auto-installable for a version override", () => { + expect(expectedIiiSha256("0.25.0", "iii-x86_64-unknown-linux-gnu.tar.gz")).toMatch(/^[0-9a-f]{64}$/); + expect(expectedIiiSha256("0.25.0", "iii-x86_64-pc-windows-msvc.zip")).toMatch(/^[0-9a-f]{64}$/); + expect(expectedIiiSha256("0.25.0", "iii-x86_64-unknown-linux-gnu.tar.gz")).not.toBe( + expectedIiiSha256(III_PINNED_VERSION, "iii-x86_64-unknown-linux-gnu.tar.gz"), + ); + }); + it("leaves no partial binary or archive behind when the download times out", () => { const dir = sandbox(); const bin = fakeCurl( @@ -451,6 +462,61 @@ describe("iii engine Windows zip installer", () => { await expect(downloadToFile("https://example.invalid/iii.zip", "C:\\bin\\a.zip", { HTTPS_PROXY: "http://proxy:3128" }, () => ({ status: 22, stderr: "curl: (22) 407" }))).rejects.toThrow("407"); }); + it("retries transient download failures before giving up", async () => { + expect(iiiDownloadShellCommand()).toContain(`--retry ${III_INSTALL_RETRIES} --retry-delay ${III_INSTALL_RETRY_DELAY_S}`); + expect(iiiManualInstallCommand("https://x/iii.tar.gz", "/b", "/b/iii", null)).toContain(`--retry ${III_INSTALL_RETRIES}`); + const curlCalls: string[][] = []; + await downloadToFile("https://example.invalid/iii.zip", "C:\\bin\\a.zip", { HTTPS_PROXY: "http://proxy:3128" }, (_command, args) => { + curlCalls.push(args); + return { status: 0 }; + }); + expect(curlCalls[0]).toEqual(expect.arrayContaining(["--retry", String(III_INSTALL_RETRIES), "--retry-delay", String(III_INSTALL_RETRY_DELAY_S)])); + + const dir = mkdtempSync(join(tmpdir(), "am-retry-")); + const dest = join(dir, "iii.zip"); + const statuses = [504, 503, 200]; + const sleeps: number[] = []; + const request = (async () => { + const status = statuses.shift()!; + return new Response(status === 200 ? "zip-bytes" : "busy", { status }); + }) as typeof fetch; + await downloadToFile("https://example.invalid/iii.zip", dest, {}, undefined, request, async (ms) => { sleeps.push(ms); }); + expect(readFileSync(dest, "utf8")).toBe("zip-bytes"); + expect(sleeps).toEqual([III_INSTALL_RETRY_DELAY_S * 1000, III_INSTALL_RETRY_DELAY_S * 1000]); + + let attempts = 0; + const alwaysBusy = (async () => { attempts++; return new Response("busy", { status: 504 }); }) as typeof fetch; + await expect(downloadToFile("https://example.invalid/iii.zip", dest, {}, undefined, alwaysBusy, async () => {})).rejects.toThrow("HTTP 504"); + expect(attempts).toBe(III_INSTALL_RETRIES + 1); + + attempts = 0; + const missing = (async () => { attempts++; return new Response("missing", { status: 404 }); }) as typeof fetch; + await expect(downloadToFile("https://example.invalid/iii.zip", dest, {}, undefined, missing, async () => {})).rejects.toThrow("HTTP 404"); + expect(attempts).toBe(1); + + const limited = [new Response("slow down", { status: 429, headers: { "retry-after": "30" } }), new Response("zip-bytes", { status: 200 })]; + const waits: number[] = []; + await downloadToFile("https://example.invalid/iii.zip", dest, {}, undefined, (async () => limited.shift()!) as typeof fetch, async (ms) => { waits.push(ms); }); + expect(waits).toEqual([30_000]); + + attempts = 0; + const tooLong = (async () => { attempts++; return new Response("later", { status: 503, headers: { "retry-after": String(III_INSTALL_RETRY_MAX_TIME_S + 1) } }); }) as typeof fetch; + await expect(downloadToFile("https://example.invalid/iii.zip", dest, {}, undefined, tooLong, async () => {})).rejects.toThrow("HTTP 503"); + expect(attempts).toBe(1); + + attempts = 0; + const dropped = (async () => { + attempts++; + if (attempts > 1) return new Response("zip-bytes", { status: 200 }); + const stream = new ReadableStream({ start(controller) { controller.error(new Error("socket hang up")); } }); + return new Response(stream, { status: 200 }); + }) as typeof fetch; + writeFileSync(dest, ""); + await downloadToFile("https://example.invalid/iii.zip", dest, {}, undefined, dropped, async () => {}); + expect(attempts).toBe(2); + expect(readFileSync(dest, "utf8")).toBe("zip-bytes"); + }); + it("pins a SHA-256 for both Windows assets of the pinned release", () => { expect(expectedIiiSha256(III_PINNED_VERSION, "iii-x86_64-pc-windows-msvc.zip")).toMatch(/^[0-9a-f]{64}$/); expect(expectedIiiSha256(III_PINNED_VERSION, "iii-aarch64-pc-windows-msvc.zip")).toMatch(/^[0-9a-f]{64}$/); diff --git a/test/engine-launch-hardening.test.ts b/test/engine-launch-hardening.test.ts index fd4e56e4ff..207ffa15be 100644 --- a/test/engine-launch-hardening.test.ts +++ b/test/engine-launch-hardening.test.ts @@ -30,8 +30,8 @@ describe("Worker Compose runtime configuration", () => { homeDir: "/home/u", linux: false, }); - expect(rendered).toContain("version: '0.22.20'"); - expect(rendered).toContain("version: '0.22.7'"); + expect(rendered).toContain("version: '0.22.22'"); + expect(rendered).toContain("version: '0.22.8'"); expect(rendered).toContain("save_interval_ms: 2000"); expect(rendered).toContain("file_path: '/var/lib/agentmemory/state_store.db'"); expect(rendered).not.toContain("iii-config.yaml"); diff --git a/test/events-consolidation.test.ts b/test/events-consolidation.test.ts index 0c082ae2bb..b4c337d4c9 100644 --- a/test/events-consolidation.test.ts +++ b/test/events-consolidation.test.ts @@ -114,6 +114,23 @@ describe("event::session::stopped consolidation fan-out", () => { }); }); + it("waits on mem::summarize for longer than the configured summary budget", async () => { + const previous = process.env.AGENTMEMORY_SUMMARY_TIMEOUT_MS; + process.env.AGENTMEMORY_SUMMARY_TIMEOUT_MS = "600000"; + try { + const { sdk, handlers, trigger } = mockSdk(); + registerEventTriggers(sdk as never, mockKV() as never); + await handlers.get("event::session::stopped")!({ sessionId: "ses_1" }); + const summarizeCall = trigger.mock.calls.find( + (c) => (c[0] as { function_id: string }).function_id === "mem::summarize", + ); + expect((summarizeCall![0] as { timeoutMs?: number }).timeoutMs).toBeGreaterThan(600_000); + } finally { + if (previous === undefined) delete process.env.AGENTMEMORY_SUMMARY_TIMEOUT_MS; + else process.env.AGENTMEMORY_SUMMARY_TIMEOUT_MS = previous; + } + }); + it("skips consolidate-pipeline and auto-crystallize when consolidation disabled but still summarizes", async () => { vi.mocked(isConsolidationEnabled).mockReturnValue(false); const { sdk, handlers, trigger } = mockSdk(); diff --git a/test/evict.test.ts b/test/evict.test.ts index 3dddba5aa9..f519761af3 100644 --- a/test/evict.test.ts +++ b/test/evict.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, it, vi } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; import type { CompressedObservation, RawObservation, @@ -13,6 +13,13 @@ import { indexObservationSession, lookupObservationSession, } from "../src/state/obs-index.js"; +import { + getSearchIndex, + setIndexPersistence, + setVectorIndex, +} from "../src/functions/search.js"; +import { VectorIndex } from "../src/state/vector-index.js"; +import { memoryToObservation } from "../src/state/memory-utils.js"; vi.mock("../src/logger.js", () => ({ logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, @@ -486,4 +493,85 @@ describe("mem::evict memory cleanup", () => { expect(result.expiredMemories).toBe(1); expect(await kv.list(KV.facets)).toHaveLength(1); }); + + describe("search-index cleanup", () => { + afterEach(() => { + setIndexPersistence(null); + setVectorIndex(null); + getSearchIndex().remove("mem_expired"); + getSearchIndex().remove("mem_stale"); + }); + + it("removes an expired memory from the BM25 index and flushes", async () => { + const kv = mockKV(new Map()); + const { sdk } = mockSdk(); + const persistence = { scheduleSave: vi.fn(), save: vi.fn(async () => {}) }; + setIndexPersistence(persistence); + const vectors = new VectorIndex(); + setVectorIndex(vectors); + + const memory = makeEvictMemory({ + id: "mem_expired", + forgetAfter: daysAgo(1), + }); + await kv.set(KV.memories, memory.id, memory); + getSearchIndex().add(memoryToObservation(memory)); + vectors.add("mem_expired", "memory", new Float32Array([1, 0, 0])); + expect(getSearchIndex().has("mem_expired")).toBe(true); + + registerEvictStubs(sdk, kv); + await sdk.trigger({ function_id: "mem::evict", payload: {} }); + + expect(getSearchIndex().has("mem_expired")).toBe(false); + expect(vectors.has("mem_expired")).toBe(false); + expect(persistence.save).toHaveBeenCalled(); + }); + + it("removes an old non-latest memory from the BM25 index", async () => { + const kv = mockKV(new Map()); + const { sdk } = mockSdk(); + const vectors = new VectorIndex(); + setVectorIndex(vectors); + + const memory = makeEvictMemory({ + id: "mem_stale", + createdAt: daysAgo(120), + updatedAt: daysAgo(120), + isLatest: false, + }); + await kv.set(KV.memories, memory.id, memory); + getSearchIndex().add(memoryToObservation(memory)); + vectors.add("mem_stale", "memory", new Float32Array([0, 1, 0])); + + registerEvictStubs(sdk, kv); + await sdk.trigger({ function_id: "mem::evict", payload: {} }); + + expect(getSearchIndex().has("mem_stale")).toBe(false); + expect(vectors.has("mem_stale")).toBe(false); + }); + + it("does not touch the index in dryRun", async () => { + const kv = mockKV(new Map()); + const { sdk } = mockSdk(); + const persistence = { scheduleSave: vi.fn(), save: vi.fn(async () => {}) }; + setIndexPersistence(persistence); + const vectors = new VectorIndex(); + setVectorIndex(vectors); + + const memory = makeEvictMemory({ + id: "mem_expired", + forgetAfter: daysAgo(1), + }); + await kv.set(KV.memories, memory.id, memory); + getSearchIndex().add(memoryToObservation(memory)); + vectors.add("mem_expired", "memory", new Float32Array([1, 0, 0])); + + registerEvictStubs(sdk, kv); + await sdk.trigger({ function_id: "mem::evict", payload: { dryRun: true } }); + + expect(getSearchIndex().has("mem_expired")).toBe(true); + expect(vectors.has("mem_expired")).toBe(true); + expect(persistence.save).not.toHaveBeenCalled(); + }); + }); }); diff --git a/test/export-incremental-budget.test.ts b/test/export-incremental-budget.test.ts index 95e599a893..51b3ca43be 100644 --- a/test/export-incremental-budget.test.ts +++ b/test/export-incremental-budget.test.ts @@ -1,5 +1,10 @@ import { describe, it, expect, vi } from "vitest"; +vi.mock("../src/state/frame-guard.js", async (importOriginal) => ({ + ...await importOriginal(), + SAFE_PAYLOAD_BYTES: 1024 * 1024, +})); + vi.mock("../src/logger.js", () => ({ logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, })); @@ -13,7 +18,8 @@ import { KV } from "../src/state/schema.js"; import { SAFE_PAYLOAD_BYTES, payloadByteLength } from "../src/state/frame-guard.js"; const BEARER = "export-budget-bearer"; -const MIB = 1024 * 1024; +// Scale algorithm fixtures; the actual 64/60 MiB boundary is tested separately. +const BUDGET_UNIT = SAFE_PAYLOAD_BYTES / 15; function countingKV() { const store = new Map>(); @@ -87,7 +93,7 @@ describe("mem::export stops assembling once the frame budget is exceeded", () => const kv = countingKV(); const sdk = mockSdk(); registerExportImportFunction(sdk as never, kv as never); - seedSessions(kv, 10, 6 * MIB); + seedSessions(kv, 10, 6 * BUDGET_UNIT); kv.seed(KV.graphNodes, "n_1", { id: "n_1" }); const result = (await sdk.trigger({ function_id: "mem::export", payload: {} })) as { @@ -102,7 +108,7 @@ describe("mem::export stops assembling once the frame budget is exceeded", () => expect(obsLists).toHaveLength(3); expect(kv.listCalls).not.toContain(KV.graphNodes); expect(result.bytes).toBeGreaterThan(SAFE_PAYLOAD_BYTES); - expect(result.bytes).toBeLessThan(19 * MIB); + expect(result.bytes).toBeLessThan(19 * BUDGET_UNIT); }); it("never lists later collections once the graph exceeds the limit", async () => { @@ -110,7 +116,7 @@ describe("mem::export stops assembling once the frame budget is exceeded", () => const sdk = mockSdk(); registerExportImportFunction(sdk as never, kv as never); for (let i = 0; i < 20; i++) { - kv.seed(KV.graphNodes, `n_${i}`, { id: `n_${i}`, name: "g".repeat(MIB) }); + kv.seed(KV.graphNodes, `n_${i}`, { id: `n_${i}`, name: "g".repeat(BUDGET_UNIT) }); } const result = (await sdk.trigger({ function_id: "mem::export", payload: {} })) as { @@ -126,7 +132,7 @@ describe("mem::export stops assembling once the frame budget is exceeded", () => const kv = countingKV(); const sdk = mockSdk(); registerExportImportFunction(sdk as never, kv as never); - seedSessions(kv, 10, 6 * MIB); + seedSessions(kv, 10, 6 * BUDGET_UNIT); const result = (await sdk.trigger({ function_id: "mem::export", @@ -147,7 +153,7 @@ describe("mem::export stops assembling once the frame budget is exceeded", () => const sdk = mockSdk(); registerExportImportFunction(sdk as never, kv as never); registerApiTriggers(sdk as never, kv as never, BEARER); - seedSessions(kv, 10, 6 * MIB); + seedSessions(kv, 10, 6 * BUDGET_UNIT); const resp = (await sdk.fns.get("api::export")!({ headers: { authorization: `Bearer ${BEARER}` }, @@ -170,7 +176,7 @@ describe("mem::export pages a collection that alone exceeds the frame limit", () const ids: string[] = []; for (let i = 0; i < 24; i++) { ids.push(`ses_${i}`); - kv.seed(KV.summaries, `ses_${i}`, { sessionId: `ses_${i}`, narrative: "s".repeat(MIB) }); + kv.seed(KV.summaries, `ses_${i}`, { sessionId: `ses_${i}`, narrative: "s".repeat(BUDGET_UNIT) }); } type Page = { status_code: number; diff --git a/test/file-writer-lease.test.ts b/test/file-writer-lease.test.ts index dfef507113..7b6bcf285d 100644 --- a/test/file-writer-lease.test.ts +++ b/test/file-writer-lease.test.ts @@ -19,7 +19,7 @@ describe('native file writer ownership',()=>{ it('rejects another process and releases only when the owning process exits',async()=>{ const m=await load(),d=dir();expect(typeof m.acquireFileWriterLease).toBe('function'); const ready=join(d,'ready'); - const child=spawn('/usr/bin/node',['--input-type=module','-e',`import {acquireFileWriterLease} from ${JSON.stringify('file://'+modulePath)};import {writeFileSync} from 'node:fs';const lease=acquireFileWriterLease(${JSON.stringify(d)});writeFileSync(${JSON.stringify(ready)},'ready');setInterval(()=>{},1000);`],{stdio:'ignore',env:{PATH:'/usr/bin:/bin',HOME:d}}); + const child=spawn(process.execPath,['--input-type=module','-e',`import {acquireFileWriterLease} from ${JSON.stringify('file://'+modulePath)};import {writeFileSync} from 'node:fs';const lease=acquireFileWriterLease(${JSON.stringify(d)});writeFileSync(${JSON.stringify(ready)},'ready');setInterval(()=>{},1000);`],{stdio:'ignore',env:{PATH:'/usr/bin:/bin',HOME:d}}); try{ const until=Date.now()+1500;while(!existsSync(ready)&&child.exitCode===null&&Date.now()setTimeout(r,10)); expect(existsSync(ready)).toBe(true); @@ -51,3 +51,68 @@ it('refuses a marker copied from another local store without rewriting it',async it('invalidates old clean admission before an unverified worker can write',async()=>{const m=await load(),d=dir(),{epoch,storeHash,evidence}=engineFixture(d);const prior={version:3,store:storeHash,epoch,clean:true};writeFileSync(join(d,'.agentmemory-writer.lock'),JSON.stringify(prior));let lease=m.acquireFileWriterLease(d,{evidence:{...evidence,capture:()=>null}});expect(m.fileWriterRecoveryAdmitted(lease)).toBe(false);lease.close();const saved=JSON.parse((await import('node:fs')).readFileSync(join(d,'.agentmemory-writer.lock'),'utf8'));expect(saved).toMatchObject({...prior,blockedUntilBoot:true,unverifiedWriter:true});lease=m.acquireFileWriterLease(d,{evidence});expect(m.fileWriterRecoveryAdmitted(lease)).toBe(false);expect(()=>m.markFileWriterClean(lease)).toThrow('WRITER_EPOCH_UNVERIFIED');lease.close();}); it('does not erase a current-boot unknown writer using a prior-boot marker',async()=>{const m=await load(),d=dir(),{epoch,storeHash,evidence}=engineFixture(d),old={...epoch,bootId:'22222222-2222-2222-2222-222222222222'};writeFileSync(join(d,'.agentmemory-writer.lock'),JSON.stringify({version:3,store:storeHash,epoch:old,clean:true}));let lease=m.acquireFileWriterLease(d,{evidence:{...evidence,capture:()=>null}});lease.close();lease=m.acquireFileWriterLease(d,{evidence});expect(m.fileWriterRecoveryAdmitted(lease)).toBe(false);expect(m.fileWriterRecoveryReason(lease)).toBe('WRITER_NATIVE_PROBE_UNCERTAIN');lease.close();const later={...epoch,bootId:'33333333-3333-3333-3333-333333333333'};lease=m.acquireFileWriterLease(d,{evidence:{...evidence,capture:()=>later,current:()=>true}});expect(m.fileWriterRecoveryAdmitted(lease)).toBe(true);lease.close();}); +it('strict startup leaves existing evidence untouched and creates no marker without a verified epoch',async()=>{ + const m=await load(),d=dir(),path=join(d,'.agentmemory-writer.lock'); + const options={requireVerifiedEpoch:true,evidence:{capture:()=>null}}; + expect(()=>m.acquireFileWriterLease(d,options)).toThrow('ENGINE_EPOCH_UNVERIFIED'); + expect(existsSync(path)).toBe(false); + writeFileSync(path,'prior-evidence'); + expect(()=>m.acquireFileWriterLease(d,options)).toThrow('ENGINE_EPOCH_UNVERIFIED'); + expect((await import('node:fs')).readFileSync(path,'utf8')).toBe('prior-evidence'); +}); +it('does not promote engine-only v3 evidence into a State-bound writer',async()=>{ + const m=await load(),d=dir(),f=engineFixture(d),path=join(d,'.agentmemory-writer.lock'); + const original=(await import('node:fs')).readFileSync(path,'utf8'); + const state={pid:300,start:'30',instanceId:'22222222-2222-4222-8222-222222222222',store:f.storeHash,executableSha256:'a'.repeat(64)}; + const epoch={...f.epoch,state};const evidence={...f.evidence,capture:()=>epoch,current:()=>true}; + const lease=m.acquireFileWriterLease(d,{evidence});try{ + expect(m.fileWriterRecoveryAdmitted(lease)).toBe(false);expect(m.fileWriterRecoveryReason(lease)).toBe('WRITER_PRIOR_UNSUPPORTED'); + expect((await import('node:fs')).readFileSync(path,'utf8')).toBe(original); + }finally{lease.close();} +}); +it('State-only replacement invalidates v4 evidence and refuses a surviving prior State',async()=>{ + const m=await load(),d=dir(),f=engineFixture(d),path=join(d,'.agentmemory-writer.lock'); + const oldState={pid:300,start:'30',instanceId:'22222222-2222-4222-8222-222222222222',store:f.storeHash,executableSha256:'a'.repeat(64)}; + const oldEpoch={...f.epoch,worker:{pid:101,start:'11'},state:oldState}; + writeFileSync(path,JSON.stringify({version:4,store:f.storeHash,epoch:oldEpoch,clean:true})); + const epoch={...f.epoch,state:{...oldState,pid:301,start:'31',instanceId:'33333333-3333-4333-8333-333333333333'}}; + const evidence={...f.evidence,capture:()=>epoch,current:()=>true,gone:(p:any)=>p.pid!==300}; + const lease=m.acquireFileWriterLease(d,{evidence});try{expect(m.fileWriterRecoveryAdmitted(lease)).toBe(false);expect(m.fileWriterRecoveryReason(lease)).toBe('WRITER_ENGINE_OVERLAP');}finally{lease.close();} +}); +it('engine-only restart cannot admit dirty evidence while old external State survives',async()=>{ + const m=await load(),d=dir(),f=engineFixture(d),path=join(d,'.agentmemory-writer.lock'); + const state={pid:300,start:'30',instanceId:'22222222-2222-4222-8222-222222222222',store:f.storeHash,executableSha256:'a'.repeat(64)}; + writeFileSync(path,JSON.stringify({version:4,store:f.storeHash,epoch:{...f.epoch,worker:{pid:101,start:'11'},engine:{pid:201,start:'21'},state},clean:false})); + const epoch={...f.epoch,state},evidence={...f.evidence,capture:()=>epoch,current:()=>true,gone:(p:any)=>p.pid!==300}; + const lease=m.acquireFileWriterLease(d,{evidence});try{expect(m.fileWriterRecoveryAdmitted(lease)).toBe(false);}finally{lease.close();} +}); +it('preserves v4 evidence whose bound State store contradicts its marker store',async()=>{ + const m=await load(),d=dir(),f=engineFixture(d),path=join(d,'.agentmemory-writer.lock'); + const state={pid:300,start:'30',instanceId:'22222222-2222-4222-8222-222222222222',store:f.storeHash,executableSha256:'a'.repeat(64)}; + const original=JSON.stringify({version:4,store:f.storeHash,epoch:{...f.epoch,state:{...state,store:'b'.repeat(64)}},clean:true});writeFileSync(path,original); + const evidence={...f.evidence,capture:()=>({...f.epoch,state}),current:()=>true}; + const lease=m.acquireFileWriterLease(d,{evidence});try{expect(m.fileWriterRecoveryAdmitted(lease)).toBe(false);expect((await import('node:fs')).readFileSync(path,'utf8')).toBe(original);}finally{lease.close();} +}); +it('does not use upgrade provenance to permit an inverse native downgrade',async()=>{ + const m=await load(),d=dir(),f=engineFixture(d),path=join(d,'.agentmemory-writer.lock'); + const state={pid:300,start:'30',instanceId:'22222222-2222-4222-8222-222222222222',store:f.storeHash,executableSha256:'a'.repeat(64)}; + const original=JSON.stringify({version:4,store:f.storeHash,epoch:{...f.epoch,state},clean:true,migration:{kind:'verified-native-v3',sourceSha256:'c'.repeat(64)}});writeFileSync(path,original); + const lease=m.acquireFileWriterLease(d,{evidence:f.evidence});try{expect(m.fileWriterRecoveryAdmitted(lease)).toBe(false);expect((await import('node:fs')).readFileSync(path,'utf8')).toBe(original);}finally{lease.close();} +}); +it('publishes a fresh v4 clean marker only from matching State seal, never overwrites unsupported prior evidence',async()=>{ + const m=await load(),d=dir(),f=engineFixture(d),path=join(d,'.agentmemory-writer.lock'); + const state={pid:300,start:'30',instanceId:'22222222-2222-4222-8222-222222222222',store:f.storeHash,executableSha256:'a'.repeat(64)},epoch={...f.epoch,state},evidence={...f.evidence,capture:()=>epoch,current:()=>true}; + const receipt={version:1,instanceId:state.instanceId,storePath:d,sealed:true,synced:true}; + const original=(await import('node:fs')).readFileSync(path,'utf8');let lease=m.acquireFileWriterLease(d,{evidence}); + expect(()=>m.markFileWriterSealedClean(lease,receipt)).toThrow('STATE_SEAL_UNVERIFIED');lease.close(); + expect((await import('node:fs')).readFileSync(path,'utf8')).toBe(original); + rmSync(path);lease=m.acquireFileWriterLease(d,{evidence}); + try{expect(()=>m.markFileWriterSealedClean(lease,{...receipt,instanceId:'wrong'})).toThrow();m.markFileWriterSealedClean(lease,receipt);expect(JSON.parse((await import('node:fs')).readFileSync(path,'utf8')).clean).toBe(true);}finally{lease.close();} +}); +it('strict candidate native startup rejects a v4 marker without rewriting it',async()=>{ + const m=await load(),d=dir(),f=engineFixture(d),path=join(d,'.agentmemory-writer.lock'); + const state={pid:300,start:'30',instanceId:'22222222-2222-4222-8222-222222222222',store:f.storeHash,executableSha256:'a'.repeat(64)}; + const original=JSON.stringify({version:4,store:f.storeHash,epoch:{...f.epoch,state},clean:true});writeFileSync(path,original); + expect(()=>m.acquireFileWriterLease(d,{evidence:f.evidence,requireVerifiedEpoch:true})).toThrow('WRITER_NATIVE_DOWNGRADE_UNSUPPORTED'); + expect((await import('node:fs')).readFileSync(path,'utf8')).toBe(original); +}); diff --git a/test/fixtures/summary-sdk-boundary.mts b/test/fixtures/summary-sdk-boundary.mts index 480ae20f29..a423bbe84e 100644 --- a/test/fixtures/summary-sdk-boundary.mts +++ b/test/fixtures/summary-sdk-boundary.mts @@ -5,7 +5,7 @@ import {performance} from 'node:perf_hooks'; import assert from 'node:assert/strict'; const root=resolve(process.env.AM_SYNTHETIC_ROOT??'');let rootVerified=false,state:string|undefined,WebSocketServer:any,registerWorker:any,StateKV:any,registerConsolidationPipelineFunction:any,registerEvictFunction:any; const require=createRequire(new URL('../../node_modules/iii-sdk/dist/index.mjs',import.meta.url));const net=require('node:net'); -const clients:any[]=[],servers:any[]=[],ports:number[]=[],active=new Set>();const result:any={sdk:'0.25.0',engine:'synthetic WebSocket protocol server, no installed/operating engine'}; +const clients:any[]=[],servers:any[]=[],ports:number[]=[],active=new Set>();const result:any={sdk:'0.25.1',engine:'synthetic WebSocket protocol server, no installed/operating engine'}; const started=performance.now(),events:any[]=[];let maxRss=0,lastSample=0,stopRequested=false;process.stdin.on('data',()=>{stopRequested=true;}); function budget(stage:string){ const u=process.resourceUsage(),now=performance.now(),rss=process.memoryUsage().rss;maxRss=Math.max(maxRss,rss); @@ -22,6 +22,7 @@ async function server(mode:string){ let value:any=null;const key=p.scope+'/'+p.key; if(x.function_id==='state::get')value=data.get(key)??null; else if(x.function_id==='state::set'){value={old_value:data.get(key)??null,new_value:p.value};data.set(key,p.value);} + else if(x.function_id==='state::list_keys')value={keys:[...data.keys()].filter(k=>k.startsWith(p.scope+'/')).map(k=>k.slice(p.scope.length+1))}; else if(x.function_id==='state::list')value=p.scope==='mem:sessions'?[{id:'synthetic_s24',project:'synthetic',cwd:'/synthetic',startedAt:'2020-01-01T00:00:00Z',status:'completed',observationCount:0}]:[]; else if(x.function_id==='state::delete'){value={old_value:data.get(key)??null};data.delete(key);} else{socket.send(JSON.stringify({type:'invocationresult',invocation_id:x.invocation_id,error:{code:'unexpected_synthetic_function',message:'fixture refused unsupported function'}}));return;} diff --git a/test/foreground-vector-hold.test.ts b/test/foreground-vector-hold.test.ts new file mode 100644 index 0000000000..8c85de17f4 --- /dev/null +++ b/test/foreground-vector-hold.test.ts @@ -0,0 +1,25 @@ +import {it,expect,vi} from 'vitest'; +import {mockKV,mockSdk} from './helpers/mocks.js'; +import {registerRememberFunction} from '../src/functions/remember.js'; +import {registerApiTriggers} from '../src/triggers/api.js'; +import {createBackgroundControl,installBackgroundControl,controlBackgroundFunctions} from '../src/ops/background-control.mjs'; +it('rejects held remember at the real API boundary before changing memory',async()=>{ + const state={version:2,paused:false,resumePending:true,holds:[],recovery:{attempts:0,nextAttemptAt:0}}; + const control=createBackgroundControl();await control.init({get:async()=>state,set:async()=>{},durability:{wait:async()=>({synced:true,mutationFailures:0}),waitForKeys:async()=>{}}} as any);installBackgroundControl(control); + const kv=mockKV(),writes=vi.spyOn(kv,'set'),sdk=mockSdk({looseTrigger:true});controlBackgroundFunctions(sdk as any);registerRememberFunction(sdk as any,kv as any);registerApiTriggers(sdk as any,kv as any,'fixture'); + const response=await sdk.fns.get('api::remember')!({headers:{authorization:'Bearer fixture'},body:{content:'synthetic'}}) as any; + expect(response).toMatchObject({status_code:503,body:{success:false,error:'vector_recovery_paused',retryable:true}});expect(writes).not.toHaveBeenCalled(); +}); +it('retains a hook capture for retry when observation admission is held',async()=>{ + const state={version:2,paused:false,resumePending:true,holds:[],recovery:{attempts:0,nextAttemptAt:0}};const control=createBackgroundControl();await control.init({get:async()=>state,set:async()=>{},durability:{wait:async()=>({synced:true,mutationFailures:0}),waitForKeys:async()=>{}}} as any);installBackgroundControl(control); + const kv=mockKV(),sdk=mockSdk({looseTrigger:true}),observe=vi.fn(async()=>({observationId:'should-not-exist'}));controlBackgroundFunctions(sdk as any);sdk.registerFunction('mem::observe',observe);const {registerCaptureFunctions}=await import('../src/functions/capture.js');const capture=registerCaptureFunctions(sdk as any,kv as any); + try{const result=await sdk.trigger('mem::capture',{eventId:'evt_synthetic_00001',payload:{hookType:'post_tool_use',sessionId:'synthetic',project:'synthetic',cwd:'/synthetic',timestamp:'2026-01-01T00:00:00Z',data:{tool_output:'synthetic'}}}) as any;expect(result).toMatchObject({status:'accepted',state:'retrying'});expect(observe).not.toHaveBeenCalled();expect((kv.store.get('mem:capture:inbox')?.size??0)).toBe(1)}finally{capture.stop()} +}); +it('rejects SDK-mode capture before accepting it while storage recovery is held',async()=>{const state={version:2,paused:false,resumePending:true,holds:[],recovery:{attempts:0,nextAttemptAt:0}};const control=createBackgroundControl();await control.init({get:async()=>state,set:async()=>{},durability:{wait:async()=>({synced:true,mutationFailures:0}),waitForKeys:async()=>{}}} as any);installBackgroundControl(control);const kv=mockKV();Object.assign(kv,{backend:'file',durability:{completion:'sdk'}});const writes=vi.spyOn(kv,'set'),sdk=mockSdk({looseTrigger:true});controlBackgroundFunctions(sdk as any);const {registerCaptureFunctions}=await import('../src/functions/capture.js');const capture=registerCaptureFunctions(sdk as any,kv as any);try{const out=await sdk.trigger('mem::capture',{eventId:'evt_synthetic_00002',payload:{hookType:'post_tool_use',sessionId:'synthetic',project:'synthetic',cwd:'/synthetic',timestamp:'2026-01-01T00:00:00Z',data:{tool_output:'synthetic'}}}) as any;expect(out).toMatchObject({status:'rejected',retryable:true,error:'vector_recovery_paused'});expect(writes).not.toHaveBeenCalled()}finally{capture.stop()}}); +it('parks the same SDK capture after hold starts during inbox admission without re-execution',async()=>{ + let saved:any={paused:false};const store={get:async()=>saved,set:async(_s:string,_k:string,v:any)=>{saved=v},durability:{wait:async()=>({synced:true,mutationFailures:0}),waitForKeys:async()=>{}}};const control=createBackgroundControl();await control.init(store as any);installBackgroundControl(control); + const kv=mockKV();Object.assign(kv,{backend:'file',durability:{completion:'sdk'}});let enter!:()=>void,release!:()=>void;const arrived=new Promise(r=>enter=r),wait=new Promise(r=>release=r),get=kv.get;let blocked=false;kv.get=async(scope:string,key:string)=>{if(scope==='mem:capture:inbox'&&!blocked){blocked=true;enter();await wait}return get(scope,key)}; + const sdk=mockSdk({looseTrigger:true});controlBackgroundFunctions(sdk as any);const observe=vi.fn(async(input:any)=>{await kv.set('mem:obs:'+input.sessionId,input.observationId,{id:input.observationId,sessionId:input.sessionId});return{observationId:input.observationId}});sdk.registerFunction('mem::observe',observe);const {registerCaptureFunctions}=await import('../src/functions/capture.js');const capture=registerCaptureFunctions(sdk as any,kv as any);const body={eventId:'evt_synthetic_race01',payload:{hookType:'post_tool_use',sessionId:'synthetic',project:'synthetic',cwd:'/synthetic',timestamp:'2026-01-01T00:00:00Z',data:{tool_output:'synthetic'}}}; + try{const first=sdk.trigger('mem::capture',body) as Promise;await arrived;const set=store.set;store.set=async()=>{throw Error('synthetic storage failure')};await expect(control.change('pause')).rejects.toThrow();store.set=set;release();expect(await first).toMatchObject({status:'rejected',eventId:body.eventId,retryable:false,error:'capture_reconciliation_required',reconciliationRequired:true});expect(observe).not.toHaveBeenCalled();const row=[...(kv.store.get('mem:capture:inbox')?.values()??[])][0] as any;expect(row).toMatchObject({eventId:body.eventId,deferredByStorageHold:true});await control.change('resume');const mismatch=await sdk.trigger('mem::capture',{...body,payload:{...body.payload,data:{tool_output:'different'}}}) as any;expect(mismatch).toMatchObject({status:'rejected',retryable:false,error:'capture_deferred_source_mismatch'});expect(observe).not.toHaveBeenCalled();const second=await sdk.trigger('mem::capture',body) as any;expect(second).toMatchObject({status:'rejected',retryable:false,error:'capture_reconciliation_required',reconciliationRequired:true});expect(observe).not.toHaveBeenCalled()}finally{capture.stop()} +}); +it('parks an interrupted SDK observation and never blindly re-executes it',async()=>{const state={paused:false};const control=createBackgroundControl();await control.init({get:async()=>state,set:async()=>{},durability:{wait:async()=>({synced:true,mutationFailures:0}),waitForKeys:async()=>{}}} as any);installBackgroundControl(control);const kv=mockKV();Object.assign(kv,{backend:'file',durability:{completion:'sdk'}});const sdk=mockSdk({looseTrigger:true}),observe=vi.fn(async()=>({success:false,error:'vector_recovery_interrupted',partial:true,retryable:false}));sdk.registerFunction('mem::observe',observe);const {registerCaptureFunctions}=await import('../src/functions/capture.js');const capture=registerCaptureFunctions(sdk as any,kv as any);const body={eventId:'evt_synthetic_unc01',payload:{hookType:'post_tool_use',sessionId:'synthetic',project:'synthetic',cwd:'/synthetic',timestamp:'2026-01-01T00:00:00Z',data:{tool_output:'synthetic'}}};try{expect(await sdk.trigger('mem::capture',body)).toMatchObject({status:'rejected',retryable:false,error:'vector_recovery_interrupted',reconciliationRequired:true});expect(await sdk.trigger('mem::capture',body)).toMatchObject({status:'rejected',retryable:false,error:'vector_recovery_interrupted',reconciliationRequired:true});expect(observe).toHaveBeenCalledTimes(1);expect(kv.store.get('mem:capture:inbox')?.size).toBe(1)}finally{capture.stop()}}); diff --git a/test/frame-guard.test.ts b/test/frame-guard.test.ts index fbb73461c1..d715adda29 100644 --- a/test/frame-guard.test.ts +++ b/test/frame-guard.test.ts @@ -19,7 +19,7 @@ import type { Session } from "../src/types.js"; // that would drop the worker is never shipped. describe("frame-guard", () => { - it("keeps the safe cap under the 16 MiB frame limit with headroom", () => { + it("keeps the safe cap under the 64 MiB frame limit with headroom", () => { expect(SAFE_PAYLOAD_BYTES).toBeLessThan(FRAME_LIMIT_BYTES_FOR_TEST); expect(FRAME_LIMIT_BYTES_FOR_TEST - SAFE_PAYLOAD_BYTES).toBeGreaterThanOrEqual( 1024 * 1024, diff --git a/test/graph-generation-surfaces.test.ts b/test/graph-generation-surfaces.test.ts index e35390e438..8f85cab306 100644 --- a/test/graph-generation-surfaces.test.ts +++ b/test/graph-generation-surfaces.test.ts @@ -27,6 +27,14 @@ function rig() { store.get(scope)!.set(key, value); return value; } + if (function_id === "state::list_keys" || function_id === "state::get") { + if (resetOnNodeRead && (scope === KV.graphNodes || scope.startsWith(`${KV.graphNodes}:generation:`))) { + resetOnNodeRead = false; + await resetGraphGeneration(kv, emptySnapshot); + } + if (function_id === "state::list_keys") return { keys: [...(store.get(scope)?.keys() ?? [])] }; + return store.get(scope)?.get(payload.key) ?? null; + } if (function_id === "state::list") { const rows = [...(store.get(scope)?.values() ?? [])]; if (resetOnNodeRead && (scope === KV.graphNodes || scope.startsWith(`${KV.graphNodes}:generation:`))) { diff --git a/test/graph-generation.test.ts b/test/graph-generation.test.ts index 70918dcd1c..68675368f4 100644 --- a/test/graph-generation.test.ts +++ b/test/graph-generation.test.ts @@ -55,6 +55,7 @@ async function setup(provider = { name: "noop", compress: vi.fn(async () => xml) const rows = store.get(payload.scope) ?? new Map(); if (id === "state::get") return structuredClone(rows.get(payload.key) ?? null); if (id === "state::list") return structuredClone([...rows.values()]); + if (id === "state::list_keys") return { keys: [...rows.keys()] }; if (id === "state::delete") { rows.delete(payload.key); return; } if (id === "state::set") { rows.set(payload.key, structuredClone(payload.value)); diff --git a/test/graph-index-held-boot.test.ts b/test/graph-index-held-boot.test.ts index d2567bbbfd..d64e552054 100644 --- a/test/graph-index-held-boot.test.ts +++ b/test/graph-index-held-boot.test.ts @@ -20,6 +20,7 @@ function fixture() { await intercept?.(fn, p); const scope = rows.get(p.scope) ?? new Map(); if (fn === "state::get") return structuredClone(scope.get(p.key) ?? null); + if (fn === "state::list_keys") return {keys:[...scope.keys()]}; if (fn === "state::list") return structuredClone([...scope.values()]); if (fn === "state::set") { scope.set(p.key, structuredClone(p.value)); rows.set(p.scope, scope); await after?.(fn, p); return p.value; } if (fn === "state::delete") { scope.delete(p.key); await after?.(fn, p); return; } @@ -33,7 +34,7 @@ function fixture() { const f = fixture(); f.rows.set(KV.graphNodes, new Map([["a", node("a")], ["b", node("b")]])); const controller = new GraphIndexMigrationController(f.kv), entered = deferred(), release = deferred(); f.intercept(async (fn, p) => { - if (fn === "state::set" && p.scope === KV.graphRetrievalData) { entered.resolve(); await release.promise; } + if (fn === "state::set" && p.scope.startsWith(`${KV.graphRetrievalData}:build:`)) { entered.resolve(); await release.promise; } }); await controller.start({ allowLegacyScan: true }); await entered.promise; const stopped = controller.stop(); release.resolve(); await stopped; f.intercept(); diff --git a/test/graph-index-migration.test.ts b/test/graph-index-migration.test.ts index b1d90901da..46a96d0e1d 100644 --- a/test/graph-index-migration.test.ts +++ b/test/graph-index-migration.test.ts @@ -7,7 +7,8 @@ import { GraphIndexMigrationController, getGraphIndexMigrationStatus, graphIndexManifestKey, type GraphIndexManifestPage, type GraphIndexMigration, } from "../src/state/graph-index-migration.js"; -import { createGraphIndexReader, getGraphIndexStatus } from "../src/state/graph-retrieval-index.js"; +import { createGraphIndexReader, getGraphIndexStatus, initializeGraphIndex } from "../src/state/graph-retrieval-index.js"; +import { queryGraphIndex } from "../src/functions/graph-index-query.js"; import type { GraphNode } from "../src/types.js"; const now = "2026-10-14T00:00:00.000Z"; @@ -27,7 +28,7 @@ function setup() { await intercept?.(fn, payload); const scope = rows.get(payload.scope) ?? new Map(); if (fn === "state::get") return structuredClone(scope.get(payload.key) ?? null); - if (fn === "state::list") return structuredClone([...scope.values()]); + if (fn === "state::list_keys") return {keys: [...scope.keys()]}; if (fn === "state::set") { scope.set(payload.key, structuredClone(payload.value)); rows.set(payload.scope, scope); return payload.value; } if (fn === "state::delete") { scope.delete(payload.key); return; } throw new Error(`Unexpected ${fn}`); @@ -37,14 +38,51 @@ function setup() { } describe("explicit graph index migration", () => { + it("replaces a ready legacy index with isolated shards without changing its data or canonical rows", async () => { + const { kv, controller, rows, intercept } = setup(); + const old = await initializeGraphIndex(kv, { status: "ready" }); + for (const id of ["a", "b", "c", "d"]) await kv.set(KV.graphNodes, id, node(id)); + await kv.set(KV.graphEdges, "ab", { + id: "ab", sourceNodeId: "a", targetNodeId: "b", type: "related_to", weight: 1, + sourceObservationIds: ["observation"], createdAt: now, + }); + const protectedScopes = [KV.graphNodes, KV.graphEdges, KV.graphRetrievalData]; + const before = protectedScopes.map(scope => structuredClone(rows.get(scope))); + expect((await controller.start()).buildId).toBe(old.buildId); + let shardWrites = 0; + intercept((fn, payload) => { + if (fn === "state::set" && payload.scope.startsWith(`${KV.graphRetrievalData}:build:`) && ++shardWrites === 3) { + throw new Error("interrupted shard publication"); + } + }); + await controller.start({ allowLegacyScan: true }); + await controller.waitForIdle(); + expect(await controller.status()).toMatchObject({ state: "paused", ready: false }); + const interrupted = await getGraphIndexStatus(kv); + expect(interrupted).toMatchObject({ schema: 3, pending: { kind: "node" } }); + intercept(); + await controller.start({ resume: true }); + await controller.waitForIdle(); + const index = await getGraphIndexStatus(kv); + expect(index).toMatchObject({ schema: 3, status: "ready", nodes: 4, edges: 1 }); + expect(index!.buildId).toBe(interrupted!.buildId); + expect(index!.buildId).not.toBe(old.buildId); + expect(protectedScopes.map(scope => rows.get(scope))).toEqual(before); + expect([...rows.keys()].filter(scope => scope.startsWith(`${KV.graphRetrievalData}:build:${index!.buildId}:`)).length).toBeGreaterThan(1); + const result = await queryGraphIndex(kv, { limit: 500 }); + expect(result.nodes.map(row => row.id).sort()).toEqual(["a", "b", "c", "d"]); + expect(result.edges.map(row => row.id)).toEqual(["ab"]); + expect((await controller.start({ allowLegacyScan: true })).buildId).toBe(index!.buildId); + }); + it("keeps legacy status and boot read-only without a snapshot or graph listing", async () => { const { kv, controller, calls, rows } = setup(); rows.set(KV.graphNodes, new Map([["old", node("old")]])); expect(await controller.status()).toMatchObject({ state: "needs-migration", ready: false, coverage: "unknown" }); await controller.resumeOnBoot(); - expect(calls.some((call) => call.fn === "state::list" || call.fn === "state::set" || call.scope === KV.graphSnapshot)).toBe(false); + expect(calls.some((call) => ["state::list", "state::list_keys"].includes(call.fn) || call.fn === "state::set" || call.scope === KV.graphSnapshot)).toBe(false); await expect(controller.start()).rejects.toThrow("allowLegacyScan"); - expect(calls.some((call) => call.fn === "state::list" || call.fn === "state::set")).toBe(false); + expect(calls.some((call) => ["state::list", "state::list_keys"].includes(call.fn) || call.fn === "state::set")).toBe(false); expect(await getGraphIndexMigrationStatus(kv)).toHaveProperty("ready", false); }); @@ -53,7 +91,7 @@ describe("explicit graph index migration", () => { rows.set(KV.graphNodes, new Map([["old", node("old")]])); const entered = deferred(); const release = deferred(); intercept(async (fn, payload) => { - if (fn === "state::list" && payload.scope === KV.graphNodes) { entered.resolve(); await release.promise; } + if (fn === "state::list_keys" && payload.scope === KV.graphNodes) { entered.resolve(); await release.promise; } }); const first = await controller.start({ allowLegacyScan: true }); await entered.promise; @@ -61,11 +99,11 @@ describe("explicit graph index migration", () => { expect((await controller.start({ allowLegacyScan: true })).jobId).toBe(first.jobId); expect((await controller.start({ resume: true })).jobId).toBe(first.jobId); const intent = calls.findIndex((call) => call.fn === "state::set" && call.scope === KV.graphIndexMigration); - const scan = calls.findIndex((call) => call.fn === "state::list" && call.scope === KV.graphNodes); + const scan = calls.findIndex((call) => call.fn === "state::list_keys" && call.scope === KV.graphNodes); expect(intent).toBeGreaterThanOrEqual(0); expect(intent).toBeLessThan(scan); release.resolve(); await controller.waitForIdle(); expect(await controller.status()).toMatchObject({ ready: true, processed: 1, total: 1, coverage: "complete" }); - expect(calls.filter((call) => call.fn === "state::list").map((call) => call.scope)).toEqual([KV.graphNodes, KV.graphEdges, KV.graphEdgeHistory]); + expect(calls.filter((call) => call.fn === "state::list_keys").map((call) => call.scope)).toEqual([KV.graphNodes, KV.graphEdges, KV.graphEdgeHistory]); }); it("indexes empty scopes and history without changing the canonical generation or rows", async () => { @@ -154,14 +192,14 @@ describe("explicit graph index migration", () => { it("requires renewed consent after an interrupted discovery and never scans at boot", async () => { const { controller, kv, calls, rows, intercept, sdk } = setup(); rows.set(KV.graphNodes, new Map([["old", node("old")]])); - intercept((fn, payload) => { if (fn === "state::list" && payload.scope === KV.graphEdges) throw new Error("offline"); }); + intercept((fn, payload) => { if (fn === "state::list_keys" && payload.scope === KV.graphEdges) throw new Error("offline"); }); await controller.start({ allowLegacyScan: true }); await controller.waitForIdle(); expect(await controller.status()).toMatchObject({ state: "paused", sealed: false }); intercept(); calls.length = 0; const restarted = new GraphIndexMigrationController(new StateKV(sdk as never)); await restarted.resumeOnBoot(); await expect(restarted.start({ resume: true })).rejects.toThrow("authorize another complete legacy scan"); - expect(calls.some((call) => call.fn === "state::list")).toBe(false); + expect(calls.some((call) => ["state::list", "state::list_keys"].includes(call.fn))).toBe(false); await restarted.start({ resume: true, allowLegacyScan: true }); await restarted.waitForIdle(); expect(await restarted.status()).toHaveProperty("ready", true); expect(await kv.get(KV.graphControl, "generation")).toBeNull(); @@ -179,7 +217,7 @@ describe("explicit graph index migration", () => { const restarted = new GraphIndexMigrationController(new StateKV(sdk as never)); await restarted.resumeOnBoot(); await restarted.waitForIdle(); expect(await restarted.status()).toMatchObject({ ready: true, nodes: 1, processed: 2 }); - expect(calls.some((call) => call.fn === "state::list" || call.scope === KV.graphSnapshot)).toBe(false); + expect(calls.some((call) => ["state::list", "state::list_keys"].includes(call.fn) || call.scope === KV.graphSnapshot)).toBe(false); }); it("keeps current writer updates, deletes and late inserts while discovery is running", async () => { @@ -187,7 +225,7 @@ describe("explicit graph index migration", () => { rows.set(KV.graphNodes, new Map([["a", node("a")], ["b", node("b")]])); const entered = deferred(); const release = deferred(); intercept(async (fn, payload) => { - if (fn === "state::list" && payload.scope === KV.graphEdgeHistory) { entered.resolve(); await release.promise; } + if (fn === "state::list_keys" && payload.scope === KV.graphEdgeHistory) { entered.resolve(); await release.promise; } }); await controller.start({ allowLegacyScan: true }); await entered.promise; await kv.set(KV.graphNodes, "a", { ...node("a"), name: "current" }); @@ -211,7 +249,7 @@ describe("explicit graph index migration", () => { const restarted = new GraphIndexMigrationController(new StateKV(sdk as never)); await restarted.resumeOnBoot(); await expect(restarted.start({ resume: true })).rejects.toThrow("authorize another complete legacy scan"); - expect(calls.some((call) => call.fn === "state::list")).toBe(false); + expect(calls.some((call) => ["state::list", "state::list_keys"].includes(call.fn))).toBe(false); await restarted.start({ resume: true, allowLegacyScan: true }); await restarted.waitForIdle(); expect(await restarted.status()).toMatchObject({ ready: true, nodes: 1 }); }); @@ -253,7 +291,7 @@ describe("explicit graph index migration", () => { const { controller, kv, rows, intercept } = setup(); rows.set(KV.graphNodes, new Map([["old", node("old")]])); const entered = deferred(); const release = deferred(); - intercept(async (fn, payload) => { if (fn === "state::list" && payload.scope === KV.graphNodes) { entered.resolve(); await release.promise; } }); + intercept(async (fn, payload) => { if (fn === "state::list_keys" && payload.scope === KV.graphNodes) { entered.resolve(); await release.promise; } }); await controller.start({ allowLegacyScan: true }); await entered.promise; await resetGraphGeneration(kv, { version: 1, topNodes: [], topEdges: [], topDegrees: {}, stats: { totalNodes: 0, totalEdges: 0, nodesByType: {}, edgesByType: {} }, updatedAt: now, dirty: false }); release.resolve(); await controller.waitForIdle(); @@ -276,7 +314,7 @@ describe("explicit graph index migration", () => { const restarted = new GraphIndexMigrationController(new StateKV(sdk as never)); await restarted.resumeOnBoot(); await restarted.waitForIdle(); expect(await restarted.status()).toMatchObject({ state: "paused", ready: false, error: expect.stringContaining("explicit") }); - expect(calls.some((call) => call.fn === "state::list" || call.scope === KV.graphSnapshot)).toBe(false); + expect(calls.some((call) => ["state::list", "state::list_keys"].includes(call.fn) || call.scope === KV.graphSnapshot)).toBe(false); await restarted.start({ resume: true }); await restarted.waitForIdle(); expect(await restarted.status()).toMatchObject({ ready: false, action: expect.stringContaining("migrate --allow-legacy-scan"), error: expect.stringContaining("new build") }); await restarted.start({ allowLegacyScan: true }); await restarted.waitForIdle(); @@ -288,7 +326,7 @@ describe("explicit graph index migration", () => { const { controller, calls, intercept } = setup(); intercept((fn, payload) => { if (fn === "state::set" && payload.scope === KV.graphIndexMigration) throw new Error("disk failure"); }); await expect(controller.start({ allowLegacyScan: true })).rejects.toThrow("disk failure"); - expect(calls.some((call) => call.fn === "state::list")).toBe(false); + expect(calls.some((call) => ["state::list", "state::list_keys"].includes(call.fn))).toBe(false); }); it("never reuses orphaned journals when a sealed build loses its control", async () => { @@ -304,7 +342,7 @@ describe("explicit graph index migration", () => { await expect(restarted.resumeOnBoot()).rejects.toThrow("sealed build control is missing"); await expect(restarted.start({ resume: true, allowLegacyScan: true })).rejects.toThrow("sealed build control is missing"); await expect(restarted.start()).rejects.toThrow("allowLegacyScan"); - expect(calls.some((call) => call.fn === "state::set" || call.fn === "state::list")).toBe(false); + expect(calls.some((call) => call.fn === "state::set" || ["state::list", "state::list_keys"].includes(call.fn))).toBe(false); await restarted.start({ allowLegacyScan: true }); await restarted.waitForIdle(); const ready = await restarted.status(); expect(ready).toMatchObject({ ready: true, nodes: 2, processed: 2, coverage: "complete" }); @@ -336,7 +374,7 @@ describe("explicit graph index migration", () => { error: "Graph index migration stopped after a storage failure. Check the server and resume when storage is available.", }); await expect(controller.start({ allowLegacyScan: true })).rejects.toThrow("storage offline"); - expect(calls.some((call) => call.fn === "state::set" || call.fn === "state::list")).toBe(false); + expect(calls.some((call) => call.fn === "state::set" || ["state::list", "state::list_keys"].includes(call.fn))).toBe(false); intercept(); await controller.start({ allowLegacyScan: true }); await controller.waitForIdle(); expect(await controller.status()).toMatchObject({ ready: true, nodes: 1 }); @@ -348,7 +386,7 @@ describe("explicit graph index migration", () => { rows.set(KV.graphIndexMigration, new Map([[GRAPH_INDEX_MIGRATION_KEY, { schema: 999 }]])); expect(await controller.status()).toMatchObject({ state: "unavailable", ready: false }); await expect(controller.start({ allowLegacyScan: true })).rejects.toThrow("state is invalid"); - expect(calls.some((call) => call.fn === "state::list")).toBe(false); + expect(calls.some((call) => ["state::list", "state::list_keys"].includes(call.fn))).toBe(false); rows.delete(KV.graphIndexMigration); const id = "x".repeat(1025); rows.set(KV.graphNodes, new Map([[id, node(id)]])); await controller.start({ allowLegacyScan: true }); await controller.waitForIdle(); diff --git a/test/graph-ingestion-import-mesh.test.ts b/test/graph-ingestion-import-mesh.test.ts index cec1d93179..a2070c6eaa 100644 --- a/test/graph-ingestion-import-mesh.test.ts +++ b/test/graph-ingestion-import-mesh.test.ts @@ -48,6 +48,7 @@ async function rig(rotated = true) { const rows = store.get(payload.scope) ?? new Map(); if (id === "state::get") return structuredClone(rows.get(payload.key) ?? null); if (id === "state::list") return structuredClone([...rows.values()]); + if (id === "state::list_keys") return { keys: [...rows.keys()] }; const before = structuredClone(rows.get(payload.key) ?? null); if (id === "state::delete") rows.delete(payload.key); else if (id === "state::set") rows.set(payload.key, structuredClone(payload.value)); diff --git a/test/graph-ingestion-snapshot.test.ts b/test/graph-ingestion-snapshot.test.ts index 7ad72a8826..c0ce5ab6e1 100644 --- a/test/graph-ingestion-snapshot.test.ts +++ b/test/graph-ingestion-snapshot.test.ts @@ -37,6 +37,7 @@ async function setup() { const rows = store.get(payload.scope) ?? new Map(); if (id === "state::get") return structuredClone(rows.get(payload.key) ?? null); if (id === "state::list") return structuredClone([...rows.values()]); + if (id === "state::list_keys") return { keys: [...rows.keys()] }; if (id === "state::delete") { rows.delete(payload.key); return; } if (id === "state::set") { rows.set(payload.key, structuredClone(payload.value)); diff --git a/test/graph-ingestion.test.ts b/test/graph-ingestion.test.ts index 73ea454a1a..9fcde2fdc2 100644 --- a/test/graph-ingestion.test.ts +++ b/test/graph-ingestion.test.ts @@ -561,7 +561,7 @@ describe("raw graph ingestion reconciliation", () => { await persistGraphRecords(kv, [node("before")], [], { mode: "merge" }); let sentOversized = false; intercept((id, p) => { - if (id === "state::set" && Buffer.byteLength(JSON.stringify(p), "utf8") > 16 * 1024 * 1024) { + if (id === "state::set" && Buffer.byteLength(JSON.stringify(p), "utf8") > 64 * 1024 * 1024) { sentOversized = true; throw new Error("Engine transport disconnected"); } diff --git a/test/graph-query.test.ts b/test/graph-query.test.ts index 6b482c7d34..71e29e8164 100644 --- a/test/graph-query.test.ts +++ b/test/graph-query.test.ts @@ -59,6 +59,10 @@ async function rig() { await afterList?.(payload.scope); return result; } + if (id === "state::list_keys") { + lists.push(payload.scope); + return { keys: [...rows.keys()] }; + } if (id === "state::list") { lists.push(payload.scope); const result = structuredClone([...rows.values()]); diff --git a/test/health-monitor-cpu.test.ts b/test/health-monitor-cpu.test.ts new file mode 100644 index 0000000000..3a9aa05241 --- /dev/null +++ b/test/health-monitor-cpu.test.ts @@ -0,0 +1,107 @@ +import { describe, expect, it, vi } from "vitest"; +import { availableParallelism } from "node:os"; +import { cpuCoreCount, normalizeCpuPercent, registerHealthMonitor } from "../src/health/monitor.js"; +import { KV } from "../src/state/schema.js"; +import type { HealthSnapshot } from "../src/types.js"; + +vi.mock("../src/ops/metrics-cpu-capacity.mjs", () => ({ getCpuCapacityCores: async () => 4 })); + +const coreCount = vi.hoisted(() => ({ value: 8 })); + +vi.mock("node:os", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + availableParallelism: () => coreCount.value, + }; +}); + +describe("normalizeCpuPercent", () => { + it("divides single-core percent by the core count", () => { + expect(normalizeCpuPercent(244, 16)).toBeCloseTo(15.25, 2); + }); + + it("clamps a zero or bogus core count to 1 rather than dividing by zero", () => { + expect(normalizeCpuPercent(50, 0)).toBe(50); + }); + + it("uses the available parallelism by default", () => { + expect(normalizeCpuPercent(availableParallelism() * 100)).toBeCloseTo(100, 2); + }); +}); + +describe("cpuCoreCount", () => { + it("reports the available parallelism", () => { + expect(cpuCoreCount()).toBe(8); + }); + + it("clamps a zero parallelism to 1", () => { + coreCount.value = 0; + try { + expect(cpuCoreCount()).toBe(1); + } finally { + coreCount.value = 8; + } + }); +}); + +function mockKV() { + const store = new Map>(); + return { + get: async (scope: string, key: string): Promise => { + return (store.get(scope)?.get(key) as T) ?? null; + }, + set: async (scope: string, key: string, data: T): Promise => { + if (!store.has(scope)) store.set(scope, new Map()); + store.get(scope)!.set(key, data); + return data; + }, + }; +} + +function mockSdk() { + return { + trigger: async () => ({ workers: [] }), + }; +} + +async function waitForSnapshot( + kv: ReturnType, +): Promise { + for (let i = 0; i < 50; i++) { + const snapshot = await kv.get(KV.health, "latest"); + if (snapshot) return snapshot; + await new Promise((resolve) => setImmediate(resolve)); + } + throw new Error("collectHealth did not persist a snapshot in time"); +} + +describe("registerHealthMonitor producer wiring", () => { + it("persists a machine-relative cpu.percent with the divisor it used", async () => { + const cpuUsageSpy = vi + .spyOn(process, "cpuUsage") + .mockReturnValueOnce({ user: 0, system: 0 } as NodeJS.CpuUsage) + .mockReturnValueOnce({ user: 5_000_000, system: 0 } as NodeJS.CpuUsage); + const dateNowSpy = vi + .spyOn(Date, "now") + .mockReturnValueOnce(1_000_000) + .mockReturnValueOnce(1_001_000); + + const kv = mockKV(); + const sdk = mockSdk(); + const { stop } = registerHealthMonitor(sdk as never, kv as never); + + try { + const snapshot = await waitForSnapshot(kv); + + expect(snapshot.cpu.cores).toBe(8); + expect(snapshot.cpu.percent).toBeCloseTo(62.5, 2); + expect(snapshot.cpu.capacityCores).toBe(4); + expect(snapshot.cpu.capacityPercent).toBeCloseTo(125, 2); + } finally { + stop(); + cpuUsageSpy.mockRestore(); + dateNowSpy.mockRestore(); + } + }); +}); diff --git a/test/health-thresholds.test.ts b/test/health-thresholds.test.ts index e41708aca0..9e26ca4118 100644 --- a/test/health-thresholds.test.ts +++ b/test/health-thresholds.test.ts @@ -111,3 +111,17 @@ describe("evaluateHealth memory severity", () => { expect(strict.status).toBe("healthy"); }); }); + +describe("evaluateHealth cpu scale", () => { + it("treats cpu.percent as machine-relative", () => { + const s = snap({ cpu: { userMicros: 0, systemMicros: 0, percent: 15.25, cores: 16 } }); + expect(evaluateHealth(s).status).toBe("healthy"); + }); + + it("still goes critical on genuine machine-wide saturation", () => { + const s = snap({ cpu: { userMicros: 0, systemMicros: 0, percent: 95, cores: 16 } }); + const { status, alerts } = evaluateHealth(s); + expect(status).toBe("critical"); + expect(alerts.find((a) => a.startsWith("cpu_critical_"))).toBeDefined(); + }); +}); diff --git a/test/helpers/devtools-port.ts b/test/helpers/devtools-port.ts new file mode 100644 index 0000000000..08523d9b11 --- /dev/null +++ b/test/helpers/devtools-port.ts @@ -0,0 +1,16 @@ +import {readFileSync} from 'node:fs'; + +export async function waitForDevToolsPort(path:string,deadline:number,alive:()=>boolean,options:{read?:()=>string;clock?:()=>number;pause?:()=>Promise}={}):Promise{ + const read=options.read??(()=>readFileSync(path,'utf8')); + const clock=options.clock??Date.now; + const pause=options.pause??(()=>new Promise(resolve=>setTimeout(resolve,25))); + while(clock()(); if (id === "state::get") return structuredClone(rows.get(payload.key) ?? null); if (id === "state::list") return structuredClone([...rows.values()]); + if (id === "state::list_keys") return { keys: [...rows.keys()] }; if (id === "state::delete") { rows.delete(payload.key); return; } if (id === "state::set") { rows.set(payload.key, structuredClone(payload.value)); diff --git a/test/helpers/mocks.ts b/test/helpers/mocks.ts index 7b3fd335c9..cb936c674d 100644 --- a/test/helpers/mocks.ts +++ b/test/helpers/mocks.ts @@ -33,9 +33,10 @@ export function mockKV() { delete: async (scope: string, key: string): Promise => { store.get(scope)?.delete(key); }, - list: async (scope: string): Promise => { + list: async (scope: string, options?: { project?: (value: unknown, key?: string) => T }): Promise => { const entries = store.get(scope); - return entries ? (Array.from(entries.values()) as T[]) : []; + const values = entries ? Array.from(entries.values()) : []; + return options?.project ? Array.from(entries ?? [], ([key, value]) => options.project!(value, key)) : values as T[]; }, }; } diff --git a/test/helpers/sdk-deadline-worker.mjs b/test/helpers/sdk-deadline-worker.mjs new file mode 100644 index 0000000000..ffda5f4d3f --- /dev/null +++ b/test/helpers/sdk-deadline-worker.mjs @@ -0,0 +1,21 @@ +import {openSync,closeSync} from 'node:fs'; +import {join} from 'node:path'; +import {spawnSync} from 'node:child_process'; +import {pathToFileURL} from 'node:url'; +import {performance} from 'node:perf_hooks'; +if(!process.send||process.env.FIXTURE_DIR!==join(process.env.HOME,'spool'))throw Error('FIXTURE_ISOLATION'); +if(process.env.MODE==='holder'){ + const fd=openSync(join(process.env.FIXTURE_DIR,'queue/lease.lock'),'r+'); + const lock=spawnSync('/usr/bin/flock',['--exclusive','--nonblock','3'],{stdio:['ignore','ignore','ignore',fd]}); + if(lock.status!==0)throw Error('FIXTURE_LOCK_FAILED'); + process.send({ready:true}); + process.once('message',message=>setTimeout(()=>{closeSync(fd);process.disconnect();},message.holdMs)); +}else{ + const {deliverSdkCaptureV2}=await import(pathToFileURL(process.env.FIXTURE_MODULE)); + process.send({ready:true}); + process.once('message',async()=>{ + const started=performance.now(),phases=[];let posts=0;process.send({started:true}); + const result=await deliverSdkCaptureV2(process.env.FIXTURE_TARGET,{eventId:'multiprocess-'+process.env.FIXTURE_INDEX},async body=>{posts++;return {status:201,body:{eventId:body.eventId,completion:'sdk',completed:true,durable:false}};},{dir:process.env.FIXTURE_DIR,deadline:started+3000,storeOptions:{onPhase:phase=>phases.push({phase,elapsedMs:performance.now()-started})}}); + process.send({result:{...result,posts,elapsedMs:performance.now()-started,phases}});process.disconnect(); + }); +} diff --git a/test/helpers/sdk-native-deadline-preload.mjs b/test/helpers/sdk-native-deadline-preload.mjs new file mode 100644 index 0000000000..b6f54d2be0 --- /dev/null +++ b/test/helpers/sdk-native-deadline-preload.mjs @@ -0,0 +1,40 @@ +// Fixture-only filesystem latency and mock fetch; the native hook source is unchanged by this helper. +import fs from 'node:fs'; +import { syncBuiltinESMExports } from 'node:module'; +import { performance } from 'node:perf_hooks'; +const dir = process.env.AGENTMEMORY_SDK_CAPTURE_V2_DIR; +const recordPath = process.env.FIXTURE_TRACE; +if (!dir?.startsWith(process.env.HOME + '/') || !recordPath?.startsWith(process.env.HOME + '/') || process.env.AGENTMEMORY_URL !== 'http://127.0.0.1:43111') throw Error('FIXTURE_ISOLATION'); +const trace = (event, fields = {}) => fs.appendFileSync(recordPath, JSON.stringify({ event, atMs: Math.round(performance.now()), ...fields }) + '\n', { mode: 0o600 }); +const actualTimer = globalThis.setTimeout; +let capArmed = false, delayDone = false; +globalThis.setTimeout = function(fn, ms, ...args) { + if (ms === 3500) { capArmed = true; trace('native_cap_armed', { milliseconds: ms }); } + return actualTimer(fn, ms, ...args); +}; +const actualStat = fs.lstatSync; +fs.lstatSync = function(path, ...args) { + if (capArmed && !delayDone && String(path).startsWith(dir)) { + delayDone = true; + const ms = Number(process.env.FIXTURE_LOCAL_DELAY_MS); + trace('local_delay', { milliseconds: ms }); + if (ms) Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms); + } + return actualStat.call(this, path, ...args); +}; +syncBuiltinESMExports(); +globalThis.fetch = async function(url, options) { + if (url !== 'http://127.0.0.1:43111/agentmemory/observe' || options?.method !== 'POST') throw Error('FIXTURE_NETWORK_REFUSED'); + const body = JSON.parse(options.body); + trace('mock_post_started'); + await new Promise((resolve, reject) => { + let timer; + const abort = () => { clearTimeout(timer); trace('mock_post_aborted'); reject(options.signal.reason); }; + if (options.signal.aborted) return abort(); + options.signal.addEventListener('abort', abort, { once: true }); + timer = actualTimer(() => { options.signal.removeEventListener('abort', abort); resolve(); }, Number(process.env.FIXTURE_REPLY_DELAY_MS)); + }); + trace('mock_ack_returned'); + return { status: 201, json: async () => ({ eventId: body.eventId, completion: 'sdk', completed: true, durable: false }) }; +}; +process.once('exit', code => trace('process_exit', { code })); diff --git a/test/helpers/storage-unit-adapter.ts b/test/helpers/storage-unit-adapter.ts index 1fc1b5740c..8789e95800 100644 --- a/test/helpers/storage-unit-adapter.ts +++ b/test/helpers/storage-unit-adapter.ts @@ -12,7 +12,7 @@ export function createStorageUnitAdapter(physical:StateKV,logicalScopes:string[] const fence=new Proxy(rawFence,{get(target,property){ if(property==='waitForKeys')return(_kv:unknown,rows:any[],...options:any[])=>target.waitForKeys(physical,selections(rows),...options); if(property==='waitForKeyGroups')return(_kv:unknown,groups:any[][],...options:any[])=>target.waitForKeyGroups(physical,groups.map(selections),...options); - if(property==='keysFor')return(rows:any[])=>target.keysFor(selections(rows)); + if(property==='keysFor'||property==='watchKeys')return(rows:any[])=>(target[property] as any)(selections(rows)); const value=Reflect.get(target,property);return typeof value==='function'?value.bind(target):value; }}); const reader=physical.observationReader; diff --git a/test/hook-capture-admission.test.ts b/test/hook-capture-admission.test.ts index 8219ea4c71..ec3506b06e 100644 --- a/test/hook-capture-admission.test.ts +++ b/test/hook-capture-admission.test.ts @@ -11,7 +11,7 @@ async function runHook(name:string,body:Record,extra:Record{if(process.env.SYNTHETIC_FETCH_STALL==='1'){setTimeout(()=>{},10000);return new Promise(()=>{})}const b=JSON.parse(options.body||'{}');paths.push(new URL(url).pathname);return new Response(JSON.stringify({eventId:b.eventId,status:'accepted',completion:'sdk',completed:true,durable:false}),{status:201})};process.on('exit',()=>writeFileSync(process.env.SYNTHETIC_FETCH_LOG,JSON.stringify(paths)));`); const env={PATH:'/usr/bin:/bin',HOME:root,TMPDIR:root,AGENTMEMORY_URL:'http://127.0.0.1:63000',AGENTMEMORY_SECRET:'synthetic-only',AGENTMEMORY_CAPTURE_SPOOL_DIR:join(root,'spool'),SYNTHETIC_FETCH_LOG:log,...extra}; - const child=spawn('/usr/bin/node',['--max-old-space-size=192','--import',preload,resolve('dist/hooks/'+name+'.mjs')],{cwd:root,env,stdio:['pipe','pipe','pipe']}); + const child=spawn(process.execPath,['--max-old-space-size=192','--import',preload,resolve('dist/hooks/'+name+'.mjs')],{cwd:root,env,stdio:['pipe','pipe','pipe']}); let stdout='',stderr='';child.stdout.on('data',d=>stdout+=d);child.stderr.on('data',d=>stderr+=d);child.stdin.on('error',()=>{});child.stdin.end(JSON.stringify(body)); const result=await new Promise<{code:number|null,signal:NodeJS.Signals|null}>((resolve,reject)=>{let uncertain:ReturnType|undefined;const deadline=setTimeout(()=>{child.kill('SIGTERM');uncertain=setTimeout(()=>{retainRoot=true;child.unref();child.stdout.destroy();child.stderr.destroy();reject(new Error('synthetic child unreaped; inspect retained '+root));},2000);},8000);child.once('error',e=>{clearTimeout(deadline);if(uncertain)clearTimeout(uncertain);reject(e)});child.once('close',(code,signal)=>{clearTimeout(deadline);if(uncertain)clearTimeout(uncertain);resolve({code,signal})});}); return {...result,stdout,stderr,paths:existsSync(log)?JSON.parse(readFileSync(log,'utf8')) as string[]:[],startupError:existsSync(log)?'':stderr}; diff --git a/test/hook-truncation.test.ts b/test/hook-truncation.test.ts index b14f3ce11e..c794bbce11 100644 --- a/test/hook-truncation.test.ts +++ b/test/hook-truncation.test.ts @@ -159,4 +159,24 @@ describe("hook truncation at a surrogate pair", () => { expect(whole).toContain(SMILE); expect(count(whole, "x")).toBe(kept - 2); }); + + it.each(sites.map((site) => [site.name, site] as const))("%s redacts a secret before cutting through it", async (_name, site) => { + const kept = count(await sent(site, "x".repeat(LONG)), "x"); + const token = await sent(site, "x".repeat(kept - 12) + " ghp_" + "A".repeat(40) + " " + "x".repeat(LONG)); + expect(token).not.toContain("ghp_"); + expect(token).not.toContain("AAAA"); + const bearer = await sent(site, "x".repeat(kept - 12) + " Bearer " + "B".repeat(40) + " " + "x".repeat(LONG)); + expect(bearer).not.toContain("BBBB"); + const block = await sent(site, "x".repeat(kept - 12) + " hidden-value " + "x".repeat(LONG)); + expect(block).not.toContain("hidden"); + }); + it.each(["tool_input", "error"] as const)("redacts structured failure %s before truncation", async (field) => { + const site: Site = { + name: "structured failure secret", script: "post-tool-failure.mjs", + payload: () => ({ ...base, tool_name: "Bash", tool_input: "ls", error: "failed", [field]: { databasePassword: "synthetic-private-value", padding: "x".repeat(LONG) } }), + field: (data) => data[field], + }; + expect(await sent(site, "unused")).not.toContain("synthetic-private-value"); + }); + }); diff --git a/test/hybrid-search.test.ts b/test/hybrid-search.test.ts index fde7aa2deb..2467c37359 100644 --- a/test/hybrid-search.test.ts +++ b/test/hybrid-search.test.ts @@ -3,7 +3,8 @@ import { VectorIndex } from "../src/state/vector-index.js"; import { HybridSearch } from "../src/state/hybrid-search.js"; import { KV } from "../src/state/schema.js"; import { SearchIndex } from "../src/state/search-index.js"; -import type { CompressedObservation, EmbeddingProvider } from "../src/types.js"; +import type { CompressedObservation, EmbeddingProvider, Memory } from "../src/types.js"; +import { memoryToObservation } from "../src/state/memory-utils.js"; function makeObs( overrides: Partial = {}, @@ -54,6 +55,63 @@ describe("HybridSearch", () => { kv = mockKV(); }); + it("does not cap saved memories at three by their shared synthetic session", async () => { + for (let i = 1; i <= 6; i++) { + const memory: Memory = { + id: `mem_${i}`, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + type: "fact", + title: `quasar policy ${i}`, + content: `quasar quasar quasar policy rule ${i}`, + concepts: [], + files: [], + sessionIds: [], + strength: 7, + version: 1, + isLatest: true, + }; + await kv.set(KV.memories, memory.id, memory); + bm25.add(memoryToObservation(memory), "memory"); + } + for (let i = 1; i <= 5; i++) { + const obs = makeObs({ id: `obs_${i}`, sessionId: `ses_${i}`, title: `quasar note ${i}`, narrative: `quasar mention ${i}`, facts: [], concepts: [], files: [] }); + bm25.add(obs); + await kv.set(KV.observations(obs.sessionId), obs.id, obs); + } + const hybrid = new HybridSearch(bm25, null, null, kv as never, 0.4, 0.6, 0, false); + const results = await hybrid.search("quasar policy", 6); + expect(results.map((r) => r.observation.id).filter((id) => id.startsWith("mem_"))).toHaveLength(6); + }); + + it("ranks a record first when it scores well in both keyword and vector search, even for a one-result search", async () => { + const keywordOnly = makeObs({ id: "obs_keyword", sessionId: "ses_k", title: "zephyr zephyr zephyr", narrative: "zephyr zephyr zephyr", facts: [], concepts: [], files: [] }); + const keywordSecond = makeObs({ id: "obs_keyword2", sessionId: "ses_k2", title: "zephyr zephyr notes", narrative: "zephyr zephyr notes", facts: [], concepts: [], files: [] }); + const both = makeObs({ id: "obs_both", sessionId: "ses_b", title: "zephyr notes notes", narrative: "zephyr notes notes", facts: [], concepts: [], files: [] }); + const vectorOnly = makeObs({ id: "obs_vector", sessionId: "ses_v", title: "unrelated", narrative: "unrelated", facts: [], concepts: [], files: [] }); + for (const obs of [keywordOnly, keywordSecond, both]) bm25.add(obs); + for (const obs of [keywordOnly, keywordSecond, both, vectorOnly]) await kv.set(KV.observations(obs.sessionId), obs.id, obs); + const vector = new VectorIndex(); + vector.add(vectorOnly.id, vectorOnly.sessionId, new Float32Array([1, 0])); + vector.add(both.id, both.sessionId, new Float32Array([0.9, 0.1])); + const provider = { name: "test", dimensions: 2, embed: vi.fn(async () => new Float32Array([1, 0])), embedBatch: vi.fn() }; + const hybrid = new HybridSearch(bm25, vector, provider, kv as never, 0.4, 0.6, 0, false); + const results = await hybrid.search("zephyr", 1); + expect(results.map((r) => r.observation.id)).toEqual(["obs_both"]); + }); + + it("returns a valid lower-ranked record when stale index entries fill the first enrichment pass", async () => { + for (let i = 0; i < 25; i++) { + bm25.add(makeObs({ id: `obs_stale_${i}`, sessionId: `ses_stale_${i}`, title: "nebula nebula nebula", narrative: "nebula nebula nebula", facts: [], concepts: [], files: [] })); + } + const valid = makeObs({ id: "obs_valid", sessionId: "ses_valid", title: "nebula notes notes", narrative: "nebula notes notes", facts: [], concepts: [], files: [] }); + bm25.add(valid); + await kv.set(KV.observations(valid.sessionId), valid.id, valid); + const hybrid = new HybridSearch(bm25, null, null, kv as never, 0.4, 0.6, 0, false); + const results = await hybrid.search("nebula", 5); + expect(results.map((r) => r.observation.id)).toEqual(["obs_valid"]); + }); + it("reuses query embeddings only within a recall request", async () => { const obs = makeObs(); bm25.add(obs); diff --git a/test/index-persistence.test.ts b/test/index-persistence.test.ts index db6e8e8d1f..1302943a3a 100644 --- a/test/index-persistence.test.ts +++ b/test/index-persistence.test.ts @@ -387,6 +387,54 @@ describe("IndexPersistence bucketed vector storage", () => { expect(loaded).toEqual({ vector: null, state: "unavailable", savedAt: null }); }); + it.each([["sdk", 1], ["disk", 8]] as const)( + "limits overlapping %s checkpoint bucket reads to %i", + async (completion, expectedPeak) => { + const bucketCount = 12; + await kv.set(INDEX_SCOPE, META_KEY, { v: 3, bucketCount, count: 1 }); + const entry = { id: "v_last", s: "session", e: Buffer.from(vec([1, 2, 3]).buffer).toString("base64") }; + await kv.set(vectorBucketScope(bucketCount - 1), entry.id, entry); + let active = 0, peak = 0; + const store = { + ...kv, + durability: { completion }, + async list(scope: string) { + peak = Math.max(peak, ++active); + try { + return await kv.list(scope); + } finally { + active--; + } + }, + }; + + const loaded = await new IndexPersistence(store as never, new VectorIndex()).load(); + expect(peak).toBe(expectedPeak); + expect(loaded.state).toBe("buckets"); + expect(loaded.vector?.get(entry.id)?.embedding).toEqual(vec([1, 2, 3])); + }, + ); + + it("stops SDK checkpoint reads on the first failure without publishing a partial index", async () => { + await kv.set(INDEX_SCOPE, META_KEY, { v: 3, bucketCount: 12, count: 0 }); + kv.ops.length = 0; + const visited: string[] = []; + const store = { + ...kv, + durability: { completion: "sdk" }, + async list(scope: string) { + visited.push(scope); + if (scope === vectorBucketScope(2)) throw new Error("SDK read timed out"); + return kv.list(scope); + }, + }; + + const loaded = await new IndexPersistence(store as never, new VectorIndex()).load(); + expect(loaded).toEqual({ vector: null, state: "unavailable", savedAt: null }); + expect(visited).toEqual([0, 1, 2].map(vectorBucketScope)); + expect(kv.ops.every(({ op }) => op === "list")).toBe(true); + }); + it("treats a missing store as empty, including adapters that return undefined", async () => { const undefinedKv = { ...kv, get: async () => undefined }; const loaded = await new IndexPersistence(undefinedKv as never, new VectorIndex(), { bucketSize: 16 }).load(); diff --git a/test/kv-paged.test.ts b/test/kv-paged.test.ts new file mode 100644 index 0000000000..85714c7713 --- /dev/null +++ b/test/kv-paged.test.ts @@ -0,0 +1,218 @@ +import { describe, it, expect, vi } from "vitest"; +import { StateKV } from "../src/state/kv.js"; +import { KV } from "../src/state/schema.js"; +import { pinGraphState } from "../src/state/graph-generation.js"; +import { encodeCursor } from "../src/state/list-query.js"; + +type Call = { function_id: string; payload: Record }; + +function fakeSdk(store: Map>, opts: { getMany?: boolean; slow?: number } = {}) { + const calls: Call[] = []; + let inFlight = 0; + let maxInFlight = 0; + const sdk = { + calls, + get maxInFlight() { return maxInFlight; }, + trigger: async (req: Call) => { + calls.push(req); + const p = req.payload as { scope: string; key?: string; keys?: string[]; value?: unknown }; + const scope = store.get(p.scope) ?? new Map(); + if (req.function_id === "engine::functions::list") { + return { functions: opts.getMany ? [{ id: "state::get_many" }, { id: "state::get" }] : ["state::get", "state::set", "state::list_keys"] }; + } + if (req.function_id === "state::list_keys") return { keys: [...scope.keys()] }; + if (req.function_id === "state::get_many") { + if (!opts.getMany) throw new Error("unknown function state::get_many"); + return { values: (p.keys ?? []).map((k) => scope.get(k) ?? null) }; + } + if (req.function_id === "state::get") { + inFlight++; + maxInFlight = Math.max(maxInFlight, inFlight); + try { + if (opts.slow) await new Promise((r) => setTimeout(r, 0)); + return scope.get(p.key!) ?? null; + } finally { inFlight--; } + } + if (req.function_id === "state::list") return [...scope.values()]; + if (req.function_id === "state::set") { scope.set(p.key!, p.value); store.set(p.scope, scope); return p.value; } + if (req.function_id === "state::delete") { scope.delete(p.key!); return; } + throw new Error(`unexpected ${req.function_id}`); + }, + }; + return sdk; +} + +const seed = (store: Map>, scope: string, entries: Record) => { + store.set(scope, new Map(Object.entries(entries))); +}; +const id = (time36: string, suffix: string) => `mem_x_kwghexc${time36}_${suffix}`; + +describe("StateKV paged reads", () => { + it("listKeys accepts both {keys} and bare-array responses", async () => { + const store = new Map(); + seed(store, KV.insights, { a: { id: "a" }, b: { id: "b" } }); + const kv = new StateKV(fakeSdk(store) as never); + expect(await kv.listKeys(KV.insights)).toEqual(["a", "b"]); + const bare = fakeSdk(store); + const raw = bare.trigger; + bare.trigger = async (req: Call) => req.function_id === "state::list_keys" ? [...store.get(KV.insights)!.keys()] : raw(req); + expect(await new StateKV(bare as never).listKeys(KV.insights)).toEqual(["a", "b"]); + }); + + it("getMany preserves input key order and reports null holes", async () => { + const store = new Map(); + seed(store, KV.semantic, { a: { id: "a", v: 1 }, c: { id: "c", v: 3 } }); + const kv = new StateKV(fakeSdk(store) as never); + expect(await kv.getMany(KV.semantic, ["c", "missing", "a"])).toEqual([{ id: "c", v: 3 }, null, { id: "a", v: 1 }]); + }); + + it("getMany honours the concurrency bound", async () => { + const store = new Map(); + seed(store, KV.semantic, Object.fromEntries([...Array(60)].map((_, i) => [`k${i}`, { id: `k${i}` }]))); + const sdk = fakeSdk(store, { slow: 1 }); + const kv = new StateKV(sdk as never); + await kv.getMany(KV.semantic, [...Array(60)].map((_, i) => `k${i}`), { concurrency: 8 }); + expect(sdk.maxInFlight).toBeLessThanOrEqual(8); + expect(sdk.maxInFlight).toBeGreaterThan(1); + }); + + it("iterate yields non-null values page by page", async () => { + const store = new Map(); + seed(store, KV.graphNodes, Object.fromEntries([...Array(5)].map((_, i) => [`n${i}`, { id: `n${i}` }]))); + store.get(KV.graphNodes)!.set("dead", undefined as never); + store.get(KV.graphNodes)!.delete("dead"); + const sdk = fakeSdk(store); + sdk.trigger = async (req: Call) => { + if (req.function_id === "state::list_keys") return { keys: [...store.get(KV.graphNodes)!.keys(), "dead"] }; + if (req.function_id === "state::get") return store.get((req.payload as { scope: string; key: string }).scope)?.get((req.payload as { scope: string; key: string }).key) ?? null; + if (req.function_id === "engine::functions::list") return []; + throw new Error("unexpected"); + }; + const kv = new StateKV(sdk as never); + const pages: unknown[][] = []; + for await (const page of kv.iterate(KV.graphNodes, { pageSize: 2 })) pages.push(page); + expect(pages.map((p) => p.length)).toEqual([2, 2, 1]); + expect(pages.flat()).toHaveLength(5); + }); + + it("listPage pages desc by generated id time with an opaque cursor", async () => { + const store = new Map(); + const entries = { + [id("1", "a")]: { id: id("1", "a") }, + [id("5", "b")]: { id: id("5", "b") }, + [id("3", "c")]: { id: id("3", "c") }, + plain: { id: "plain" }, + }; + seed(store, KV.memories, entries); + const kv = new StateKV(fakeSdk(store) as never); + const first = await kv.listPage<{ id: string }>(KV.memories, { limit: 2 }); + expect(first.total).toBe(4); + expect(first.items.map((i) => i.id)).toEqual([id("5", "b"), id("3", "c")]); + expect(first.nextCursor).toBeTruthy(); + const second = await kv.listPage<{ id: string }>(KV.memories, { limit: 2, cursor: first.nextCursor }); + expect(second.items.map((i) => i.id)).toEqual([id("1", "a"), "plain"]); + expect(second.nextCursor).toBeUndefined(); + }); + + it("listPage asc order and deleted cursor keys keep paging", async () => { + const store = new Map(); + const a = id("1", "a"), b = id("3", "b"), c = id("5", "c"); + seed(store, KV.memories, { [a]: { id: a }, [b]: { id: b }, [c]: { id: c } }); + const kv = new StateKV(fakeSdk(store) as never); + const page = await kv.listPage<{ id: string }>(KV.memories, { limit: 3, order: "asc", cursor: encodeCursor({ key: id("4", "gone") }) }); + expect(page.items.map((i) => i.id)).toEqual([c]); + }); + + it("applies graph scope resolution through the pinned view", async () => { + const store = new Map(); + seed(store, "mem:graph:control", { generation: "11111111-2222-3333-4444-555555555555" }); + seed(store, `${KV.graphNodes}:generation:11111111-2222-3333-4444-555555555555`, { n1: { id: "n1" } }); + const kv = new StateKV(fakeSdk(store) as never); + const pinned = await pinGraphState(kv); + const seen: unknown[] = []; + for await (const page of pinned.iterate(KV.graphNodes)) seen.push(...page); + expect(seen).toEqual([{ id: "n1" }]); + }); + + it("redis backend orders each iterate page like insertion", async () => { + const store = new Map(); + seed(store, KV.insights, { b: { id: "b", createdAt: "2026-01-02T00:00:00Z" }, a: { id: "a", createdAt: "2026-01-01T00:00:00Z" } }); + const kv = new StateKV(fakeSdk(store) as never, { backend: "redis" }); + const pages: { id: string }[][] = []; + for await (const page of kv.iterate<{ id: string }>(KV.insights, { pageSize: 10 })) pages.push(page); + expect(pages[0]!.map((i) => i.id)).toEqual(["a", "b"]); + }); +}); + +describe("collectAll redis ordering", () => { + it("orders the whole result like list() even across page boundaries", async () => { + const store = new Map(); + const entries: Record = {}; + for (let i = 0; i < 250; i++) { + const key = `k${i}`; + entries[key] = { id: key, createdAt: new Date(1700000000000 + i * 1000).toISOString() }; + } + seed(store, KV.insights, entries); + const kv = new StateKV(fakeSdk(store) as never, { backend: "redis" }); + const listed = await kv.list<{ id: string }>(KV.insights); + const { collectAll } = await import("../src/state/kv.js"); + const collected = await collectAll<{ id: string }>(kv, KV.insights); + expect(collected.map((r) => r.id)).toEqual(listed.map((r) => r.id)); + }); +}); + +describe("paging custom protection compatibility", () => { + it("fails closed on malformed list_keys rather than reporting empty state", async () => { + for (const result of [null, {}, {keys: ["valid", 42]}, [42]]) { + const kv = new StateKV({ trigger: async () => result } as never); + await expect(kv.listKeys("synthetic")).rejects.toThrow("unexpected shape"); + } + }); + + it("preserves mixed retained source fields across bounded fetches", async () => { + const store = new Map(); + const row = {id: "retained", sourceRaw: {toolOutput: "PRIVATE_FIXTURE_SOURCE"}, offsets: {next: 42}, holds: ["manual"], unknown: {keep: true}}; + seed(store, "synthetic", {retained: row}); + const sdk = fakeSdk(store); + const kv = new StateKV(sdk as never); + const {collectAll} = await import("../src/state/kv.js"); + expect(await collectAll(kv, "synthetic")).toEqual([row]); + expect(sdk.calls.every(call => !/set|update|delete/.test(call.function_id))).toBe(true); + }); + + it("retains observation opened-generation paths", async () => { + for (const [scope, property] of [["mem:obs:synthetic", "observationReader"]]) { + const kv = new StateKV({trigger: async () => { throw Error("must not bypass guarded list") }} as never); + Object.defineProperty(kv, property, {value: {}}); + const list = vi.spyOn(kv, "list").mockRejectedValue(Error("protected_generation_changed")); + const {collectAll} = await import("../src/state/kv.js"); + await expect(collectAll(kv, scope)).rejects.toThrow("protected_generation_changed"); + expect(list).toHaveBeenCalledWith(scope); + } + }); +}); + + it("pages summaries without full list while retaining uncertain-mutation admission", async () => { + const store = new Map(); seed(store, KV.summaries, {one: {id: "one", sourceRaw: {keep: true}}}); + const sdk = fakeSdk(store); const kv = new StateKV(sdk as never); + Object.defineProperty(kv, "summaryCount", {value: {sdkListBytes: () => {throw Error("no full-list budget")}}}); + const {collectAll} = await import("../src/state/kv.js"); + expect(await collectAll(kv, KV.summaries)).toEqual([{id:"one",sourceRaw:{keep:true}}]); + expect((await kv.listPage(KV.summaries,{limit:2})).items).toHaveLength(1); + expect(sdk.calls.some(call=>call.function_id==='state::list')).toBe(false); + Object.defineProperty(kv,"uncertainMutation",{value:true}); + await expect(collectAll(kv, KV.summaries)).rejects.toThrow("summary_list_mutation_uncertain"); + await expect(kv.listPage(KV.summaries,{limit:2})).rejects.toThrow("summary_list_mutation_uncertain"); + }); + +it("does not return a summary page when mutation certainty changes during reads", async () => { + let kv: StateKV; + const sdk = {trigger: async ({function_id}: {function_id:string}) => { + if (function_id === "state::list_keys") return {keys:["one"]}; + if (function_id === "state::get") {Object.defineProperty(kv,"uncertainMutation",{value:true});return {id:"one"};} + throw Error("unexpected call"); + }}; + kv = new StateKV(sdk as never); + Object.defineProperty(kv,"summaryCount",{value:{}}); + await expect(kv.listPage(KV.summaries,{limit:1})).rejects.toThrow("summary_list_mutation_uncertain"); +}); diff --git a/test/list-query.test.ts b/test/list-query.test.ts index a409b4b19c..e29d41ead3 100644 --- a/test/list-query.test.ts +++ b/test/list-query.test.ts @@ -6,7 +6,9 @@ import { matchesText, pageAfterCursor, pageByOffset, + inTimeRange, parseListQuery, + parseTimeBounds, sortByKeyDesc, } from "../src/state/list-query.js"; @@ -81,3 +83,38 @@ describe("list query helpers", () => { expect(matchesText(undefined, "anything")).toBe(true); }); }); + +describe("time range filters", () => { + it("parses ISO bounds and rejects invalid ones", () => { + const ok = parseTimeBounds({ startedAfter: "2026-10-09T00:00:00Z", endedBefore: " 2026-10-10 " }); + expect(ok).toEqual({ bounds: { startedAfter: Date.parse("2026-10-09T00:00:00Z"), endedBefore: Date.parse("2026-10-10") } }); + expect(parseTimeBounds(undefined)).toEqual({ bounds: {} }); + for (const bad of ["5", "2026", "Oct 5", "0", "2026-13-45", "2026-10-09T25:00:00Z", "2026-02-30", "2027-02-29", "", " "]) { + expect(parseTimeBounds({ startedAfter: bad })).toEqual({ error: "startedAfter must be an ISO 8601 timestamp" }); + } + expect(parseTimeBounds({ endedBefore: "" })).toEqual({ error: "endedBefore must be an ISO 8601 timestamp" }); + expect(parseTimeBounds({ startedAfter: "2028-02-29" })).toEqual({ bounds: { startedAfter: Date.parse("2028-02-29") } }); + expect(parseTimeBounds({ startedAfter: "yesterday" })).toEqual({ error: "startedAfter must be an ISO 8601 timestamp" }); + }); + + it("reads zone-less values as UTC and accepts an offset whose plus was decoded to a space", () => { + const utc = Date.parse("2026-10-09T12:00:00Z"); + expect(parseTimeBounds({ startedAfter: "2026-10-09T12:00" })).toEqual({ bounds: { startedAfter: utc } }); + expect(parseTimeBounds({ startedAfter: "2026-10-09" })).toEqual({ bounds: { startedAfter: Date.parse("2026-10-09T00:00:00Z") } }); + const plus = parseTimeBounds({ startedAfter: "2026-10-09T20:00:00+08:00" }); + expect(plus).toEqual({ bounds: { startedAfter: utc } }); + expect(parseTimeBounds({ startedAfter: "2026-10-09T20:00:00 08:00" })).toEqual(plus); + expect(parseTimeBounds({ startedAfter: "2026-10-09 12:00" })).toEqual({ bounds: { startedAfter: utc } }); + expect(parseTimeBounds({ startedAfter: "2026-10-09 20:00:00 08:00" })).toEqual(plus); + }); + + it("treats after as inclusive, before as exclusive, and drops rows without a time", () => { + const after = Date.parse("2026-10-09T00:00:00Z"); + const before = Date.parse("2026-10-10T00:00:00Z"); + expect(inTimeRange("2026-10-09T00:00:00Z", after, before)).toBe(true); + expect(inTimeRange("2026-10-10T00:00:00Z", after, before)).toBe(false); + expect(inTimeRange("2026-10-08T23:59:59Z", after, before)).toBe(false); + expect(inTimeRange(undefined, after)).toBe(false); + expect(inTimeRange(undefined)).toBe(true); + }); +}); diff --git a/test/manual-vector-checkpoint-recovery.test.ts b/test/manual-vector-checkpoint-recovery.test.ts new file mode 100644 index 0000000000..ee1696127f --- /dev/null +++ b/test/manual-vector-checkpoint-recovery.test.ts @@ -0,0 +1,102 @@ +import {describe,expect,it,vi} from 'vitest'; +import {IndexPersistence, vectorBucketScope} from '../src/state/index-persistence.js'; +import {receiptFor} from '../src/state/vector-checkpoint-reconciliation.js'; +import {VectorIndex} from '../src/state/vector-index.js'; + +const pending='mem:index:vec-pending',index='mem:index:bm25'; +function fixture() { + const vec=Buffer.from(new Float32Array([1,2,3]).buffer).toString('base64'); + const rows = ['obs_a','obs_b','obs_c'].map((id,i)=>({id,s:'s',q:i+1,e:vec,k:'observation'})); + const scopes = new Map>([ + [pending,new Map(rows.map(r=>[r.id,r]))], + [index,new Map([['vectors:meta',{v:3,count:3,bucketCount:1,savedAt:'2026-01-01T00:00:00.000Z'}]])], + [vectorBucketScope(0),new Map(rows.map(({id,s,e})=>[id,{id,s,e}]))], + ]); + let held=true; + const kv = { + backend:'file', + readVectorRetention: async()=>[...scopes.get('receipts')?.values()??[]], + delete: async(scope:string,key:string)=>scopes.get(scope)?.delete(key), + durability:{completion:'sdk',waitForKeys:async()=>({completed:true,synced:false})}, + get:async(scope:string,key:string)=>scopes.get(scope)?.get(key)??null, + set:async(scope:string,key:string,v:unknown)=>{if(!scopes.has(scope))scopes.set(scope,new Map());scopes.get(scope)!.set(key,v);}, + list:async(scope:string)=>[...scopes.get(scope)?.values()??[]], + reconcileVectorCheckpoint:vi.fn(async(data:{pendingRows:typeof rows},current:()=>boolean,_rev:number,authorize:boolean)=>{ + if(!current())throw Error('stale'); + const desired=data.pendingRows.map(row=>receiptFor(row as never,0)); + const stored=scopes.get('receipts')??new Map(); + if ([...stored].some(([key])=>!desired.some(row=>row.digest===key))) throw Error('original changed'); + if (!authorize && desired.some(row=>!stored.has(row.digest))) throw Error('new WAL'); + scopes.set('receipts',new Map(desired.map(row=>[row.digest,row]))); + return desired; + }), + }; + const v=new VectorIndex(); + const p=new IndexPersistence(kv as never,v,{requireRecoveryPlan:true,isStorageHeld:()=>held}); + const release=async(explicit:boolean)=>{await p.load();held=false;return p.resumeStorageRecovery(3,explicit);}; + return {scopes,rows,kv,v,p,release}; +} + +describe('operator-requested vector checkpoint recovery',()=>{ + it('certifies three exact rows before release and does not replay them',async()=>{ + const f=fixture();try{ + expect(await f.release(true)).toBe(true); + expect(f.kv.reconcileVectorCheckpoint).toHaveBeenCalledTimes(1); + expect(f.kv.reconcileVectorCheckpoint.mock.calls[0][0]).toMatchObject({pendingRows:f.rows,buckets:[{bucket:0,ids:f.rows.map(r=>r.id)}]}); + expect(f.v.size).toBe(3);expect([...f.scopes.get(pending)!.values()]).toEqual(f.rows); + expect(f.p.status()).toMatchObject({pendingLog:3,pendingLogAcknowledged:0,retainedCheckpointRows:3,pendingChanges:0}); + expect(()=>f.v.add('obs_a','s',new Float32Array([3,2,1]))).toThrow(/PAUSED/); + expect(()=>f.v.remove('obs_b')).toThrow(/PAUSED/);expect(()=>f.v.clear()).toThrow(/PAUSED/);expect(()=>f.v.markAllChanged()).toThrow(/PAUSED/); + const added=new Float32Array([4,5,6]); + f.v.add('new','s',added);await f.p.save(); + expect(f.scopes.get(vectorBucketScope(0))!.get('new')).toMatchObject({id:'new'}); + expect([...f.scopes.get(pending)!.values()]).toEqual(f.rows); + expect(f.p.status()).toMatchObject({pendingChanges:0,pendingLog:3}); + expect(await f.p.verifyCaptureVectorPending('obs_a','s',f.rows[0])).toBe(true); + expect(await f.p.verifyCaptureVectorPending('obs_a','s',null)).toBe(false); + expect(await f.p.verifyCaptureVectorPending('obs_a','other',f.rows[0])).toBe(false); + expect(await f.p.verifyCaptureVectorPending('obs_a','s',{...f.rows[0],q:9})).toBe(false); + expect(await f.p.verifyCaptureVectorPending('new','s',null)).toBe(true); + expect(await f.p.verifyCaptureVectorPending('new','s',f.rows[0])).toBe(false); + expect(f.kv.reconcileVectorCheckpoint.mock.calls.at(-1)?.[3]).toBe(false); + const next=new VectorIndex(),p2=new IndexPersistence(f.kv as never,next,{requireRecoveryPlan:true}); + try{ + const loaded=await p2.load();expect(loaded.state).toBe('buckets');await p2.prepareBackfill(loaded); + next.restoreFrom(loaded.vector!); + expect(await p2.replayPendingLog(3)).toMatchObject({entries:0,added:0}); + expect(next.size).toBe(4);expect(next.pendingChanges).toBe(0); + expect(()=>next.markRemoved('obs_a')).toThrow(/PAUSED/); + await p2.save();expect([...f.scopes.get(pending)!.values()]).toEqual(f.rows); + expect(f.kv.reconcileVectorCheckpoint.mock.calls.at(-1)?.[3]).toBe(false); + }finally{p2.stop();} + }finally{f.p.stop();} + }); + it.each(['updated q','mismatched bytes','additional row','missing row','clear'])('fails closed on subsequent boot with %s',async mode=>{ + const f=fixture();const v2=new VectorIndex(),p2=new IndexPersistence(f.kv as never,v2,{requireRecoveryPlan:true}); + try{ + expect(await f.release(true)).toBe(true);const log=f.scopes.get(pending)!; + if(mode==='updated q')log.set('obs_a',{...f.rows[0],q:12}); + if(mode==='mismatched bytes')log.set('obs_a',{...f.rows[0],e:Buffer.from(new Float32Array([8,2,3]).buffer).toString('base64')}); + if(mode==='additional row')log.set('extra',{...f.rows[0],id:'extra',q:18}); + if(mode==='missing row')log.delete('obs_a'); + if(mode==='clear')log.set('~clear',{q:19,c:1}); + const before=new Map(log); + expect((await p2.load()).state).toBe('unavailable');expect(p2.isRecoveryPaused()).toBe(true); + await p2.replayPendingLog(3);await p2.save();expect(log).toEqual(before); + expect(()=>v2.add('obs_a','s',new Float32Array(3))).toThrow(/PAUSED/); + }finally{f.p.stop();p2.stop();} + }); + it('requires an explicit request even when all checkpoints match',async()=>{ + const f=fixture();try{expect(await f.release(false)).toBe(false);expect(f.kv.reconcileVectorCheckpoint).not.toHaveBeenCalled();expect(f.scopes.get(pending)!.size).toBe(3);}finally{f.p.stop();} + }); + it.each(['clear','tombstone','duplicate ID in two buckets','checkpoint shortfall','mismatched vector'])('retains the originals on %s',async reason=>{ + const f=fixture(); + if(reason==='clear')f.scopes.get(pending)!.set('~clear',{q:20,c:1}); + if(reason==='tombstone')f.scopes.get(pending)!.set('obs_a',{q:1,id:'obs_a',t:1}); + if(reason==='duplicate ID in two buckets'){f.scopes.get(index)!.get('vectors:meta').bucketCount=2;f.scopes.set(vectorBucketScope(1),new Map(f.scopes.get(vectorBucketScope(0))));} + if(reason==='checkpoint shortfall')f.scopes.get(index)!.get('vectors:meta').count=42; + if(reason==='mismatched vector')f.scopes.get(vectorBucketScope(0))!.get('obs_a').e=Buffer.from(new Float32Array([9,2,3]).buffer).toString('base64'); + const before=new Map(f.scopes.get(pending)); + try{expect(await f.release(true)).toBe(false);expect(f.kv.reconcileVectorCheckpoint).not.toHaveBeenCalled();expect(f.scopes.get(pending)).toEqual(before);}finally{f.p.stop();} + }); +}); diff --git a/test/mcp-action-create-tool.test.ts b/test/mcp-action-create-tool.test.ts new file mode 100644 index 0000000000..a79ff32077 --- /dev/null +++ b/test/mcp-action-create-tool.test.ts @@ -0,0 +1,106 @@ +import { describe, it, expect, beforeEach, vi } from "vitest"; + +vi.mock("../src/logger.js", () => ({ + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, +})); + +import { registerMcpEndpoints } from "../src/mcp/server.js"; + +function mockKV() { + const store = new Map>(); + return { + get: async (scope: string, key: string): Promise => + (store.get(scope)?.get(key) as T) ?? null, + set: async (scope: string, key: string, data: T): Promise => { + if (!store.has(scope)) store.set(scope, new Map()); + store.get(scope)!.set(key, data); + return data; + }, + delete: async (scope: string, key: string): Promise => { + store.get(scope)?.delete(key); + }, + list: async (scope: string): Promise => { + const entries = store.get(scope); + return entries ? (Array.from(entries.values()) as T[]) : []; + }, + }; +} + +function mockSdk() { + const functions = new Map(); + const overrides = new Map(); + return { + registerFunction: (idOrOpts: string | { id: string }, handler: Function) => { + functions.set( + typeof idOrOpts === "string" ? idOrOpts : idOrOpts.id, + handler, + ); + }, + registerTrigger: () => {}, + trigger: async ( + idOrInput: string | { function_id: string; payload: unknown }, + data?: unknown, + ) => { + const id = + typeof idOrInput === "string" ? idOrInput : idOrInput.function_id; + const payload = typeof idOrInput === "string" ? data : idOrInput.payload; + if (overrides.has(id)) return overrides.get(id)!(payload); + const fn = functions.get(id); + if (!fn) throw new Error(`No function: ${id}`); + return fn(payload); + }, + overrideTrigger: (id: string, handler: Function) => { + overrides.set(id, handler); + }, + getFunction: (id: string) => functions.get(id), + }; +} + +function callTool(sdk: ReturnType, args: unknown) { + const fn = sdk.getFunction("mcp::tools::call")!; + return fn({ + body: { name: "memory_action_create", arguments: args }, + headers: {}, + query_params: {}, + }) as Promise<{ status_code: number; body: unknown }>; +} + +describe("memory_action_create MCP tool", () => { + let sdk: ReturnType; + let seen: Record | undefined; + + beforeEach(() => { + sdk = mockSdk(); + registerMcpEndpoints(sdk as never, mockKV() as never); + seen = undefined; + sdk.overrideTrigger("mem::action-create", (payload: Record) => { + seen = payload; + return { success: true }; + }); + }); + + it("records the calling agent as the action creator", async () => { + await callTool(sdk, { title: "Ship it", agentId: "agent-a" }); + expect(seen?.createdBy).toBe("agent-a"); + }); + + it("rejects a non-string agentId without creating an action", async () => { + const res = await callTool(sdk, { title: "Ship it", agentId: 42 }); + expect(res.status_code).toBe(400); + expect(seen).toBeUndefined(); + }); + + it("rejects an agentId longer than 128 characters instead of shortening it", async () => { + const res = await callTool(sdk, { title: "Ship it", agentId: "a".repeat(129) }); + expect(res.status_code).toBe(400); + expect(seen).toBeUndefined(); + await callTool(sdk, { title: "Ship it", agentId: "a".repeat(128) }); + expect(seen?.createdBy).toBe("a".repeat(128)); + }); + + it("lists agentId in the tool schema", async () => { + const { getAllTools } = await import("../src/mcp/tools-registry.js"); + const tool = getAllTools().find((t) => t.name === "memory_action_create")!; + expect(Object.keys(tool.inputSchema.properties as object)).toContain("agentId"); + }); +}); diff --git a/test/mcp-resources.test.ts b/test/mcp-resources.test.ts index 5eab1bdddf..dd8e89532b 100644 --- a/test/mcp-resources.test.ts +++ b/test/mcp-resources.test.ts @@ -364,6 +364,7 @@ describe("MCP graph/stats resource over live graph state", () => { if (!id.startsWith("state::")) return handlers.get(id)!(payload); const rows = store.get(payload.scope) ?? new Map(); if (id === "state::get") return structuredClone(rows.get(payload.key) ?? null); + if (id === "state::list_keys") { listed.push(payload.scope); return {keys:[...rows.keys()]}; } if (id === "state::list") { listed.push(payload.scope); return structuredClone([...rows.values()]); diff --git a/test/mcp-standalone-proxy.test.ts b/test/mcp-standalone-proxy.test.ts index 6101e9bda0..dd7b27bb39 100644 --- a/test/mcp-standalone-proxy.test.ts +++ b/test/mcp-standalone-proxy.test.ts @@ -2,6 +2,9 @@ import { describe, expect, it, beforeEach, afterEach, vi } from "vitest"; import { handleToolCall, handleToolsList } from "../src/mcp/standalone.js"; import { resetHandleForTests } from "../src/mcp/rest-proxy.js"; import { InMemoryKV } from "../src/mcp/in-memory-kv.js"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; type FetchMock = ReturnType; @@ -274,10 +277,10 @@ describe("@agentmemory/mcp standalone — server proxy", () => { let probeCount = 0; const refusal = { success: false, - error: "Response is 20.0 MiB, over the ~15 MiB engine transport frame limit; narrow the range", + error: "Response is 70.0 MiB, over the ~60 MiB engine transport message limit; narrow the range", oversized: true, - bytes: 20 * 1024 * 1024, - limitBytes: 15 * 1024 * 1024, + bytes: 70 * 1024 * 1024, + limitBytes: 60 * 1024 * 1024, }; installFetch((url) => { if (url.endsWith("/agentmemory/livez")) { @@ -516,6 +519,41 @@ describe("@agentmemory/mcp standalone — server proxy", () => { delete process.env["AGENTMEMORY_TOOLS"]; }); + it("gives proxied calls more than the worker invocation budget and honours AGENTMEMORY_CALL_TIMEOUT_MS", async () => { + installFetch((url) => { + if (url.endsWith("/agentmemory/livez")) return new Response("ok", { status: 200 }); + return new Response(JSON.stringify({ sessions: [] }), { status: 200 }); + }); + const saved = { call: process.env["AGENTMEMORY_CALL_TIMEOUT_MS"], file: process.env["AGENTMEMORY_ENV_FILE"] }; + const dir = mkdtempSync(join(tmpdir(), "am-call-timeout-")); + const envFile = join(dir, "agentmemory.env"); + delete process.env["AGENTMEMORY_CALL_TIMEOUT_MS"]; + process.env["AGENTMEMORY_ENV_FILE"] = envFile; + const timeout = vi.spyOn(AbortSignal, "timeout"); + const lastTimeout = async () => { + await handleToolCall("memory_sessions", { limit: 1 }); + return timeout.mock.lastCall?.[0]; + }; + try { + expect(await lastTimeout()).toBe(190_000); + writeFileSync(envFile, "AGENTMEMORY_CALL_TIMEOUT_MS=600000\n"); + expect(await lastTimeout()).toBe(600_000); + process.env["AGENTMEMORY_CALL_TIMEOUT_MS"] = "900000"; + expect(await lastTimeout()).toBe(900_000); + for (const bad of ["bogus", "0.5", "1e6", "600s", "2592000000"]) { + process.env["AGENTMEMORY_CALL_TIMEOUT_MS"] = bad; + expect(await lastTimeout()).toBe(190_000); + } + } finally { + timeout.mockRestore(); + rmSync(dir, { recursive: true, force: true }); + if (saved.call === undefined) delete process.env["AGENTMEMORY_CALL_TIMEOUT_MS"]; + else process.env["AGENTMEMORY_CALL_TIMEOUT_MS"] = saved.call; + if (saved.file === undefined) delete process.env["AGENTMEMORY_ENV_FILE"]; + else process.env["AGENTMEMORY_ENV_FILE"] = saved.file; + } + }); + it("AGENTMEMORY_PROBE_TIMEOUT_MS overrides the default probe timeout", async () => { process.env["AGENTMEMORY_PROBE_TIMEOUT_MS"] = "50"; let probeStarted = 0; diff --git a/test/mcp-standalone.test.ts b/test/mcp-standalone.test.ts index 5cf120563e..00cb276661 100644 --- a/test/mcp-standalone.test.ts +++ b/test/mcp-standalone.test.ts @@ -4,7 +4,8 @@ vi.mock("../src/logger.js", () => ({ logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, })); -vi.mock("node:fs", () => ({ +vi.mock("node:fs", async () => ({ + constants: (await vi.importActual("node:fs")).constants, existsSync: vi.fn().mockReturnValue(false), readFileSync: vi.fn(), writeFileSync: vi.fn(), diff --git a/test/mesh-export-project-scope.test.ts b/test/mesh-export-project-scope.test.ts index 7ff72c7e17..c177fdae23 100644 --- a/test/mesh-export-project-scope.test.ts +++ b/test/mesh-export-project-scope.test.ts @@ -63,11 +63,12 @@ function memory(id: string, project: string, content = "x"): Memory { async function meshExport( sdk: ReturnType, project?: string, + extra?: Record, ): Promise<{ status_code: number; body: Record }> { const handler = sdk._fns.get("api::mesh-export")!; return handler({ headers: { authorization: `Bearer ${SECRET}` }, - query_params: project ? { project } : {}, + query_params: { ...(project ? { project } : {}), ...extra }, }); } @@ -127,3 +128,127 @@ describe("api::mesh-export project scoping", () => { expect((unscoped.body as { oversized?: boolean }).oversized).toBe(true); }); }); + +describe("api::mesh-export cursor paging", () => { + it("does not skip surviving rows when a row is deleted between pages", async () => { + const kv = mockKV(); + for (const id of ["m-1", "m-2", "m-3", "m-4", "m-5"]) { + await kv.set(KV.memories, id, memory(id, "alpha")); + } + const sdk = mockSdk(); + registerApiTriggers(sdk as never, kv as never, SECRET); + + const seen: string[] = []; + let cursor: string | undefined; + for (;;) { + const res = await meshExport(sdk, "alpha", { limit: "2", ...(cursor ? { cursor } : {}) }); + expect(res.status_code).toBe(200); + seen.push(...(res.body.memories as Memory[]).map((m) => m.id)); + cursor = res.body.nextCursor as string | undefined; + if (seen.length === 2) kv._store.get(KV.memories)!.delete("m-1"); + if (!cursor) break; + } + expect(seen.sort()).toEqual(["m-1", "m-2", "m-3", "m-4", "m-5"]); + expect(new Set(seen).size).toBe(5); + }); + + it("rejects an invalid cursor with 400", async () => { + const kv = mockKV(); + await kv.set(KV.memories, "m-1", memory("m-1", "alpha")); + const sdk = mockSdk(); + registerApiTriggers(sdk as never, kv as never, SECRET); + + const bad = await meshExport(sdk, "alpha", { cursor: "not-a-cursor" }); + expect(bad.status_code).toBe(400); + expect((bad.body as { error?: string }).error).toBe("invalid_cursor"); + + const outOfRange = await meshExport(sdk, "alpha", { + cursor: Buffer.from(JSON.stringify({ c: 99, after: "m-1" })).toString("base64url"), + }); + expect(outOfRange.status_code).toBe(400); + expect((outOfRange.body as { error?: string }).error).toBe("invalid_cursor"); + }); +}); + +describe("api::mesh-export page boundary", () => { + it("emits a cursor when the limit lands exactly on a collection end", async () => { + const kv = mockKV(); + for (let i = 0; i < 5; i++) await kv.set(KV.memories, `m-${i}`, memory(`m-${i}`, "alpha")); + await kv.set(KV.actions, "a-1", { id: "a-1", project: "alpha", updatedAt: "2026-08-01T00:00:00Z" } as never); + const sdk = mockSdk(); + registerApiTriggers(sdk as never, kv as never, SECRET); + + const page1 = await meshExport(sdk, "alpha", { limit: "5" }); + expect(page1.status_code).toBe(200); + expect((page1.body.memories as Memory[]).length).toBe(5); + expect(page1.body.nextCursor).toBeTruthy(); + const page2 = await meshExport(sdk, "alpha", { limit: "5", cursor: page1.body.nextCursor as string }); + expect(page2.status_code).toBe(200); + expect(((page2.body.actions as Array<{ id: string }>).map((a) => a.id))).toEqual(["a-1"]); + expect(page2.body.nextCursor).toBeUndefined(); + }); +}); + +describe("api::mesh-export cursor keys", () => { + async function walkExport(sdk: ReturnType, key: string) { + const rows: Array> = []; + let cursor: string | undefined; + for (let pages = 0; pages < 50; pages++) { + const res = await meshExport(sdk, undefined, { limit: "1", ...(cursor ? { cursor } : {}) }); + expect(res.status_code).toBe(200); + rows.push(...((res.body[key] as Array>) ?? [])); + cursor = res.body.nextCursor as string | undefined; + if (!cursor) break; + } + return rows; + } + + it("pages relations that have no id without skipping the rest of the collection", async () => { + const kv = mockKV(); + for (const [s, t] of [["a", "b"], ["b", "c"], ["c", "d"]]) { + await kv.set(KV.relations, `${s}:${t}:related`, { type: "related", sourceId: s, targetId: t, createdAt: "2026-08-01T00:00:00Z" }); + } + const sdk = mockSdk(); + registerApiTriggers(sdk as never, kv as never, SECRET); + const rows = await walkExport(sdk, "relations"); + expect(rows.map((r) => r.sourceId).sort()).toEqual(["a", "b", "c"]); + }); + + it("accepts the cursor it emits after a record id longer than 512 characters", async () => { + const kv = mockKV(); + const longId = `m-${"x".repeat(600)}`; + await kv.set(KV.memories, longId, memory(longId, "alpha")); + await kv.set(KV.memories, "z-1", memory("z-1", "alpha")); + const sdk = mockSdk(); + registerApiTriggers(sdk as never, kv as never, SECRET); + const rows = await walkExport(sdk, "memories"); + expect(rows.map((r) => r.id)).toEqual([longId, "z-1"]); + }); +}); + +describe("api::mesh-export single-request callers", () => { + it("returns every row without nextCursor when the caller sends neither limit nor cursor", async () => { + const kv = mockKV(); + for (let i = 0; i < 600; i++) await kv.set(KV.memories, `m-${String(i).padStart(4, "0")}`, memory(`m-${String(i).padStart(4, "0")}`, "alpha")); + const sdk = mockSdk(); + registerApiTriggers(sdk as never, kv as never, SECRET); + const res = await meshExport(sdk, "alpha"); + expect(res.status_code).toBe(200); + expect((res.body.memories as Memory[]).length).toBe(600); + expect(res.body.nextCursor).toBeUndefined(); + }); + + it("returns 413 instead of a partial page when the single response exceeds the frame limit", async () => { + const kv = mockKV(); + const third = Math.ceil(SAFE_PAYLOAD_BYTES / 3); + for (let i = 0; i < 4; i++) await kv.set(KV.memories, `m-${i}`, memory(`m-${i}`, "alpha", "y".repeat(third))); + const sdk = mockSdk(); + registerApiTriggers(sdk as never, kv as never, SECRET); + const res = await meshExport(sdk, "alpha"); + expect(res.status_code).toBe(413); + expect((res.body as { oversized?: boolean }).oversized).toBe(true); + const paged = await meshExport(sdk, "alpha", { limit: "500" }); + expect(paged.status_code).toBe(200); + expect(paged.body.nextCursor).toBeTruthy(); + }, 30_000); +}); diff --git a/test/mesh.test.ts b/test/mesh.test.ts index 5c5ecfd629..9bd15417cf 100644 --- a/test/mesh.test.ts +++ b/test/mesh.test.ts @@ -5,6 +5,7 @@ vi.mock("../src/logger.js", () => ({ })); import { registerMeshFunction } from "../src/functions/mesh.js"; +import { KV } from "../src/state/schema.js"; import type { MeshPeer, Memory, @@ -233,6 +234,179 @@ describe("Mesh Functions", () => { vi.unstubAllGlobals(); }); + + it("aborts when the peer repeats the same cursor and leaves lastSyncAt untouched", async () => { + const authedSdk = mockSdk(); + const authedKv = mockKV(); + registerMeshFunction(authedSdk as never, authedKv as never, "mesh-secret"); + + vi.stubGlobal("fetch", vi.fn(async () => + new Response(JSON.stringify({ memories: [], nextCursor: "cA" }), { + status: 200, + headers: { "Content-Type": "application/json" }, + }), + )); + + const regResult = (await authedSdk.trigger("mem::mesh-register", { + url: "https://peer-loop.example.com", + name: "peer-loop", + })) as { success: boolean; peer: MeshPeer }; + await authedKv.set(KV.mesh, regResult.peer.id, { ...regResult.peer, lastSyncAt: "2026-01-01T00:00:00.000Z" }); + + const result = (await authedSdk.trigger("mem::mesh-sync", { + peerId: regResult.peer.id, + direction: "pull", + })) as { success: boolean; results: Array<{ errors: string[] }> }; + + expect(result.results[0].errors.some((e) => e.includes("cursor did not advance"))).toBe(true); + const peer = await authedKv.get(KV.mesh, regResult.peer.id); + expect(peer!.lastSyncAt).toBe("2026-01-01T00:00:00.000Z"); + + vi.unstubAllGlobals(); + }); + + it("does not advance lastSyncAt when the push fails but the pull succeeds", async () => { + const authedSdk = mockSdk(); + const authedKv = mockKV(); + registerMeshFunction(authedSdk as never, authedKv as never, "mesh-secret"); + + vi.stubGlobal("fetch", vi.fn(async (input: unknown) => { + const url = String(input); + if (url.includes("/mesh/receive")) { + return new Response("boom", { status: 500 }); + } + return new Response(JSON.stringify({ memories: [] }), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); + })); + + const regResult = (await authedSdk.trigger("mem::mesh-register", { + url: "https://peer-pushfail.example.com", + name: "peer-pushfail", + })) as { success: boolean; peer: MeshPeer }; + await authedKv.set(KV.mesh, regResult.peer.id, { ...regResult.peer, lastSyncAt: "2026-01-01T00:00:00.000Z" }); + + const result = (await authedSdk.trigger("mem::mesh-sync", { + peerId: regResult.peer.id, + direction: "both", + })) as { success: boolean; results: Array<{ errors: string[]; pulled?: number }> }; + + expect(result.results[0].errors.some((e) => e.includes("push failed"))).toBe(true); + const peer = await authedKv.get(KV.mesh, regResult.peer.id); + expect(peer!.lastSyncAt).toBe("2026-01-01T00:00:00.000Z"); + + vi.unstubAllGlobals(); + }); + + it("scopes the push and every pull page to the peer project filter", async () => { + const authedSdk = mockSdk(); + const authedKv = mockKV(); + registerMeshFunction(authedSdk as never, authedKv as never, "mesh-secret"); + const base = { type: "pattern", title: "t", content: "c", createdAt: "2026-08-01T00:00:00Z", updatedAt: "2026-08-01T00:00:00Z", concepts: [], files: [], sessionIds: [], strength: 5, version: 1, isLatest: true }; + await authedKv.set(KV.memories, "m-alpha", { ...base, id: "m-alpha", project: "alpha" }); + await authedKv.set(KV.memories, "m-beta", { ...base, id: "m-beta", project: "beta" }); + + const pushed: Array<{ id: string }> = []; + const pullUrls: string[] = []; + vi.stubGlobal("fetch", vi.fn(async (input: unknown, init?: { body?: string }) => { + const url = String(input); + if (url.endsWith("/mesh/receive")) { + pushed.push(...((JSON.parse(init!.body!) as { memories?: Array<{ id: string }> }).memories ?? [])); + return new Response(JSON.stringify({ accepted: 0 }), { status: 200, headers: { "Content-Type": "application/json" } }); + } + pullUrls.push(url); + const body = pullUrls.length === 1 ? { memories: [], nextCursor: "c1" } : { memories: [] }; + return new Response(JSON.stringify(body), { status: 200, headers: { "Content-Type": "application/json" } }); + })); + + const regResult = (await authedSdk.trigger("mem::mesh-register", { + url: "https://peer-scoped.example.com", + name: "peer-scoped", + syncFilter: { project: "alpha" }, + })) as { success: boolean; peer: MeshPeer }; + + const result = (await authedSdk.trigger("mem::mesh-sync", { + peerId: regResult.peer.id, + direction: "both", + })) as { success: boolean; results: Array<{ errors: string[] }> }; + + expect(result.results[0].errors).toEqual([]); + expect(pushed.map((m) => m.id)).toEqual(["m-alpha"]); + expect(pullUrls).toHaveLength(2); + for (const url of pullUrls) { + expect(new URL(url).searchParams.get("project")).toBe("alpha"); + expect(new URL(url).searchParams.get("limit")).toBe("500"); + } + + vi.unstubAllGlobals(); + }); + + it("stamps push-only success with the sync start time", async () => { + const authedSdk = mockSdk(); + const authedKv = mockKV(); + registerMeshFunction(authedSdk as never, authedKv as never, "mesh-secret"); + + const start = new Date("2026-03-02T12:00:00.000Z"); + vi.useFakeTimers({ now: start }); + vi.stubGlobal("fetch", vi.fn(async () => { + vi.setSystemTime(new Date("2026-03-02T12:04:00.000Z")); + return new Response(JSON.stringify({ accepted: 0 }), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); + })); + + const regResult = (await authedSdk.trigger("mem::mesh-register", { + url: "https://peer-push.example.com", + name: "peer-push", + })) as { success: boolean; peer: MeshPeer }; + + const result = (await authedSdk.trigger("mem::mesh-sync", { + peerId: regResult.peer.id, + direction: "push", + })) as { success: boolean; results: Array<{ errors: string[] }> }; + + expect(result.results[0].errors).toEqual([]); + const peer = await authedKv.get(KV.mesh, regResult.peer.id); + expect(peer!.lastSyncAt).toBe("2026-03-02T12:00:00.000Z"); + + vi.unstubAllGlobals(); + vi.useRealTimers(); + }); + + it("stamps lastSyncAt with the sync start time after a successful pull", async () => { + const authedSdk = mockSdk(); + const authedKv = mockKV(); + registerMeshFunction(authedSdk as never, authedKv as never, "mesh-secret"); + + const start = new Date("2026-03-01T12:00:00.000Z"); + vi.useFakeTimers({ now: start }); + vi.stubGlobal("fetch", vi.fn(async () => { + vi.setSystemTime(new Date("2026-03-01T12:05:00.000Z")); + return new Response(JSON.stringify({ memories: [] }), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); + })); + + const regResult = (await authedSdk.trigger("mem::mesh-register", { + url: "https://peer-time.example.com", + name: "peer-time", + })) as { success: boolean; peer: MeshPeer }; + + const result = (await authedSdk.trigger("mem::mesh-sync", { + peerId: regResult.peer.id, + direction: "pull", + })) as { success: boolean; results: Array<{ errors: string[] }> }; + + expect(result.results[0].errors).toEqual([]); + const peer = await authedKv.get(KV.mesh, regResult.peer.id); + expect(peer!.lastSyncAt).toBe("2026-03-01T12:00:00.000Z"); + + vi.unstubAllGlobals(); + vi.useRealTimers(); + }); }); describe("mesh-receive", () => { diff --git a/test/native-lease-migration.test.ts b/test/native-lease-migration.test.ts new file mode 100644 index 0000000000..b2adbd8732 --- /dev/null +++ b/test/native-lease-migration.test.ts @@ -0,0 +1,119 @@ +import {it,expect,afterEach} from 'vitest'; +import {mkdtempSync,mkdirSync,writeFileSync,readFileSync,symlinkSync,rmSync,existsSync} from 'node:fs'; +import {join} from 'node:path';import {tmpdir} from 'node:os';import {createHash} from 'node:crypto'; +import {prepareNativeMigration,finalizeNativeMigration,nativeMigrationForMarker,resolveNativeStoreConfig} from '../src/ops/native-lease-migration.mjs'; +const dirs:string[]=[];afterEach(()=>dirs.splice(0).forEach(d=>rmSync(d,{recursive:true,force:true}))); +function fixture(){ + const d=mkdtempSync(join(tmpdir(),'native-migration-'));dirs.push(d);const store=join(d,'store'),runtime=join(d,'runtime'),proc=join(d,'proc'),exe=join(d,'iii'),config=join(d,'native.yaml');mkdirSync(store);mkdirSync(runtime);writeFileSync(exe,'known-native-executable'); + const hash=(s:string)=>createHash('sha256').update(s).digest('hex'); + const epoch={host:'a'.repeat(64),bootId:'11111111-1111-1111-1111-111111111111',endpoint:'ws://localhost:1000',pidNamespace:'pid:[1]',netNamespace:'net:[1]',workerUid:1000,worker:{pid:100,start:'10'},engine:{pid:200,start:'20'}}; + const marker={version:3,store:hash(store),epoch,clean:false};const original=JSON.stringify(marker);writeFileSync(join(store,'.agentmemory-writer.lock'),original); + writeFileSync(join(store,'synthetic.json'),JSON.stringify({raw:'source',offset:42,holds:['manual'],unknown:true})); + for(const p of [epoch.worker,epoch.engine]){mkdirSync(join(proc,String(p.pid)),{recursive:true});writeFileSync(join(proc,String(p.pid),'stat'),p.pid+' (native) '+['S',...Array(18).fill('0'),p.start].join(' '));} + symlinkSync(exe,join(proc,'200/exe'));writeFileSync(join(proc,'200/cmdline'),exe+'\0--config\0'+config+'\0'); + writeFileSync(config,JSON.stringify({workers:[{name:'iii-state',config:{adapter:{name:'kv',config:{store_method:'file_based',file_path:store}}}}]})); + writeFileSync(join(runtime,'engine-state.json'),JSON.stringify({kind:'native',compatVersion:'0.22.1',processStart:'20',binPath:exe,configPath:config})); + const evidence:any={capture:()=>epoch,current:()=>true,gone:()=>true}; + const options={storePath:store,runtimeDir:runtime,expectedExecutable:exe,engineEvidence:evidence,procRoot:proc}; + const stop=()=>{rmSync(join(proc,'100'),{recursive:true});rmSync(join(proc,'200'),{recursive:true})}; + return {d,store,runtime,proc,exe,config,epoch,original,evidence,options,stop}; +} +it('preserves original bytes and store, issues receipt offline, validates exact v3 migration',()=>{ + const f=fixture(),proof=prepareNativeMigration(f.options); + expect(()=>finalizeNativeMigration({storePath:f.store,proof,procRoot:f.proc,pidAlive:()=>false})).toThrow('MIGRATION_PRIOR_PROCESS_ALIVE'); + f.stop();const result=finalizeNativeMigration({storePath:f.store,proof,procRoot:f.proc,pidAlive:()=>false}); + expect(readFileSync(join(f.store,'.agentmemory-writer.lock'),'utf8')).toBe(f.original); + expect(JSON.parse(readFileSync(join(f.store,'synthetic.json'),'utf8'))).toEqual({raw:'source',offset:42,holds:['manual'],unknown:true}); + const epoch={...f.epoch,state:{store:JSON.parse(f.original).store}}; + expect(nativeMigrationForMarker(f.store,f.original,epoch,f.evidence)?.sourceSha256).toBe(result.sourceSha256); + expect(nativeMigrationForMarker(f.store,f.original+' ',epoch,f.evidence)).toBeNull(); + expect(()=>finalizeNativeMigration({storePath:f.store,proof,procRoot:f.proc,pidAlive:()=>false})).toThrow('MIGRATION_RECEIPT_EXISTS'); +}); +it.each(['config','store','start','executable'])('rejects mismatched live native %s proof',kind=>{ + const f=fixture(); + if(kind==='config')writeFileSync(join(f.proc,'200/cmdline'),f.exe+'\0--config\0/wrong\0'); + if(kind==='store')writeFileSync(f.config,JSON.stringify({workers:[{name:'iii-state',config:{adapter:{name:'kv',config:{store_method:'file_based',file_path:f.d}}}}]})); + if(kind==='start')writeFileSync(join(f.proc,'200/stat'),'200 (native) '+['S',...Array(18).fill('0'),'21'].join(' ')); + if(kind==='executable'){const other=join(f.d,'other');writeFileSync(other,'other');f.options.expectedExecutable=other;} + expect(()=>prepareNativeMigration(f.options)).toThrow(); +}); +it('rejects changed executable/config after capture and interrupted receipt publication',()=>{ + const f=fixture(),proof=prepareNativeMigration(f.options);f.stop();writeFileSync(f.exe,'changed'); + expect(()=>finalizeNativeMigration({storePath:f.store,proof,procRoot:f.proc,pidAlive:()=>false})).toThrow('MIGRATION_ARTIFACT_CHANGED'); + writeFileSync(f.exe,'known-native-executable'); + expect(()=>finalizeNativeMigration({storePath:f.store,proof,procRoot:f.proc,pidAlive:()=>false,publish:(path:string,bytes:string)=>{writeFileSync(path,bytes);throw Error('injected_sync_failure')}})).toThrow('injected_sync_failure'); + expect(nativeMigrationForMarker(f.store,f.original,{...f.epoch,state:{store:JSON.parse(f.original).store}},f.evidence)).toBeNull(); + expect(readFileSync(join(f.store,'.agentmemory-writer.lock'),'utf8')).toBe(f.original); +}); +it('first state-bound acquisition records migration provenance and never rewinds source data',async()=>{ + const f=fixture(),proof=prepareNativeMigration(f.options);f.stop();finalizeNativeMigration({storePath:f.store,proof,procRoot:f.proc,pidAlive:()=>false}); + const {acquireFileWriterLease,fileWriterRecoveryAdmitted}=await import('../src/ops/file-writer-lease.mjs'); + const epoch={...f.epoch,worker:{pid:101,start:'11'},engine:{pid:201,start:'21'},state:{pid:301,start:'31',instanceId:'22222222-2222-4222-8222-222222222222',store:JSON.parse(f.original).store,executableSha256:'a'.repeat(64)}}; + const lease=acquireFileWriterLease(f.store,{evidence:{...f.evidence,capture:()=>epoch}}); + try{ + expect(fileWriterRecoveryAdmitted(lease)).toBe(true); + const marker=JSON.parse(readFileSync(join(f.store,'.agentmemory-writer.lock'),'utf8')); + expect(marker.version).toBe(4);expect(marker.migration.sourceSha256).toBe(proof.sourceSha256); + expect(readFileSync(join(f.store,'.agentmemory-migrations',proof.sourceSha256+'.json'),'utf8')).toContain(proof.sourceSha256); + expect(JSON.parse(readFileSync(join(f.store,'synthetic.json'),'utf8')).holds).toEqual(['manual']); + }finally{lease.close();} +}); +it('derives process start from verified marker/proc for older native metadata without newer optional fields',()=>{ + const f=fixture();writeFileSync(join(f.runtime,'engine-state.json'),JSON.stringify({kind:'native',binPath:f.exe,configPath:f.config})); + expect(prepareNativeMigration(f.options).epoch.engine.start).toBe('20'); +}); +it('reconciles only a normal offline clean transition while preserving original proof and both markers',async()=>{ + const f=fixture(),proof=prepareNativeMigration(f.options),saved=JSON.stringify(proof); + const final=JSON.stringify({...JSON.parse(f.original),clean:true});writeFileSync(join(f.store,'.agentmemory-writer.lock'),final); + expect(()=>finalizeNativeMigration({storePath:f.store,proof,procRoot:f.proc,pidAlive:()=>false})).toThrow('MIGRATION_PRIOR_PROCESS_ALIVE'); + f.stop();const result=finalizeNativeMigration({storePath:f.store,proof,procRoot:f.proc,pidAlive:()=>false}); + expect(result.reconciledCleanShutdown).toBe(true);expect(result.preparedSourceSha256).toBe(proof.sourceSha256);expect(JSON.stringify(proof)).toBe(saved); + const receipt=JSON.parse(readFileSync(result.path,'utf8'));expect(receipt.proof.sourceMarker).toBe(f.original);expect(receipt.finalMarker).toBe(final); + expect(readFileSync(join(f.store,'.agentmemory-writer.lock'),'utf8')).toBe(final); + const epoch={...f.epoch,state:{store:JSON.parse(final).store}}; + expect(nativeMigrationForMarker(f.store,final,epoch,f.evidence)?.sourceSha256).toBe(result.sourceSha256); + expect(nativeMigrationForMarker(f.store,f.original,epoch,f.evidence)).toBeNull(); + expect(()=>finalizeNativeMigration({storePath:f.store,proof,procRoot:f.proc,pidAlive:()=>false})).toThrow('MIGRATION_RECEIPT_EXISTS'); + const {acquireFileWriterLease,fileWriterRecoveryAdmitted}=await import('../src/ops/file-writer-lease.mjs'); + const next={...epoch,worker:{pid:101,start:'11'},engine:{pid:201,start:'21'},state:{...epoch.state,pid:301,start:'31',instanceId:'22222222-2222-4222-8222-222222222222',executableSha256:'a'.repeat(64)}}; + const lease=acquireFileWriterLease(f.store,{evidence:{...f.evidence,capture:()=>next}}); + try{expect(fileWriterRecoveryAdmitted(lease)).toBe(true);expect(JSON.parse(readFileSync(join(f.store,'.agentmemory-writer.lock'),'utf8')).migration.sourceSha256).toBe(result.sourceSha256);}finally{lease.close();} + + expect(JSON.parse(readFileSync(join(f.store,'synthetic.json'),'utf8'))).toEqual({raw:'source',offset:42,holds:['manual'],unknown:true}); +}); +it.each(['epoch','store','version','blocked','uncertain','extra','clean-reversal'])('rejects offline %s changes rather than refreshing proof',kind=>{ + const f=fixture(),proof=prepareNativeMigration(f.options);f.stop();const changed:any={...JSON.parse(f.original),clean:true}; + if(kind==='epoch')changed.epoch={...changed.epoch,engine:{pid:200,start:'21'}}; + if(kind==='store')changed.store='different';if(kind==='version')changed.version=4; + if(kind==='blocked')changed.blockedUntilBoot=true;if(kind==='uncertain')changed.uncertaintyBootId='other'; + if(kind==='extra')changed.unexpected='evidence';if(kind==='clean-reversal'){proof.sourceMarker=JSON.stringify(changed);proof.sourceSha256=createHash('sha256').update(proof.sourceMarker).digest('hex');changed.clean=false;} + writeFileSync(join(f.store,'.agentmemory-writer.lock'),JSON.stringify(changed)); + expect(()=>finalizeNativeMigration({storePath:f.store,proof,procRoot:f.proc,pidAlive:()=>false})).toThrow('MIGRATION_MARKER_CHANGED'); +}); + +it('does not admit an interrupted clean-transition receipt publication',()=>{ + const f=fixture(),proof=prepareNativeMigration(f.options);f.stop();const final=JSON.stringify({...JSON.parse(f.original),clean:true});writeFileSync(join(f.store,'.agentmemory-writer.lock'),final); + expect(()=>finalizeNativeMigration({storePath:f.store,proof,procRoot:f.proc,pidAlive:()=>false,publish:(path:string,bytes:string)=>{writeFileSync(path,bytes);throw Error('interrupted')}})).toThrow('interrupted'); + expect(nativeMigrationForMarker(f.store,final,{...f.epoch,state:{store:JSON.parse(final).store}},f.evidence)).toBeNull(); + expect(readFileSync(join(f.store,'.agentmemory-writer.lock'),'utf8')).toBe(final); +}); + +it('resolves reviewed native name-only state through persisted cwd config and pins its digest',()=>{ + const f=fixture();symlinkSync(f.d,join(f.proc,'200/cwd'));mkdirSync(join(f.d,'config')); + const path=join(f.d,'config/iii-state.yaml'),body={id:'iii-state',value:{adapter:{name:'kv',config:{store_method:'file_based',file_path:f.store}}}};writeFileSync(path,JSON.stringify(body)); + const args={workers:[{name:'iii-state'}],store:f.store,exeSha256:'a44eba0ca91a2805ea43355e30aec482a688eab33ac84801aec65f6636142654',procRoot:f.proc,pid:200}; + const persistence=resolveNativeStoreConfig(args);expect(persistence).toMatchObject({path,cwd:f.d,sha256:createHash('sha256').update(JSON.stringify(body)).digest('hex')}); + expect(()=>resolveNativeStoreConfig({...args,exeSha256:'unknown'})).toThrow('MIGRATION_STORE_CONFIG_MISMATCH'); + expect(()=>resolveNativeStoreConfig({...args,workers:[{name:'iii-state'},{name:'configuration'}]})).toThrow('MIGRATION_STORE_CONFIG_MISMATCH'); + const proof=prepareNativeMigration(f.options);proof.native.persistence=persistence;f.stop();writeFileSync(path,JSON.stringify({...body,value:{adapter:{name:'redis'}}})); + expect(()=>finalizeNativeMigration({storePath:f.store,proof,procRoot:f.proc,pidAlive:()=>false})).toThrow('MIGRATION_PERSISTED_CONFIG_CHANGED'); +}); +it.each(['store','placeholder','id','unknown-field'])('rejects unproven persisted native %s config',kind=>{ + const f=fixture();symlinkSync(f.d,join(f.proc,'200/cwd'));mkdirSync(join(f.d,'config')); + const body:any={id:'iii-state',value:{adapter:{name:'kv',config:{store_method:'file_based',file_path:f.store}}}}; + if(kind==='store')body.value.adapter.config.file_path=f.d; + if(kind==='placeholder')body.value.adapter.config.file_path='${STORE}'; + if(kind==='id')body.id='other';if(kind==='unknown-field')body.value.unknown=true; + writeFileSync(join(f.d,'config/iii-state.yaml'),JSON.stringify(body)); + expect(()=>resolveNativeStoreConfig({workers:[{name:'iii-state'}],store:f.store,exeSha256:'a44eba0ca91a2805ea43355e30aec482a688eab33ac84801aec65f6636142654',procRoot:f.proc,pid:200})).toThrow(); +}); diff --git a/test/native-sdk-v2.test.ts b/test/native-sdk-v2.test.ts index 2707c711e6..33c00e5bc1 100644 --- a/test/native-sdk-v2.test.ts +++ b/test/native-sdk-v2.test.ts @@ -14,7 +14,7 @@ async function run(script:string,input:Record,extra:Record{const b=JSON.parse(options.body||'{}');calls.push({redirect:options.redirect,path:new URL(url).pathname,imageBytes:typeof b.data?.image_data==='string'?Buffer.byteLength(b.data.image_data):0,imageHash:typeof b.data?.image_data==='string'?createHash('sha256').update(b.data.image_data).digest('hex'):null,final:b.final});return new Response(JSON.stringify({eventId:b.eventId,status:'accepted',completion:'sdk',completed:true,durable:false}),{status:201})};process.on('exit',()=>writeFileSync(process.env.SYNTHETIC_FETCH_LOG,JSON.stringify(calls)));`); const env={PATH:'/usr/bin:/bin',HOME:root,TMPDIR:root,AGENTMEMORY_URL:target,AGENTMEMORY_SECRET:'synthetic-only',AGENTMEMORY_SDK_CAPTURE_V2_DIR:join(root,'v2'),AGENTMEMORY_CAPTURE_SPOOL_DIR:join(root,'v1'),SYNTHETIC_FETCH_LOG:log,...extra}; - const child=spawn('/usr/bin/node',['--max-old-space-size=256','--import',preload,...(sourceOnly?['--import',resolve('node_modules/tsx/dist/loader.mjs'),resolve('src/hooks/'+script+'.ts')]:[resolve('dist/hooks/'+script+'.mjs')]),'--agentmemory-sdk-capture-v2'],{cwd:root,env,stdio:['pipe','pipe','pipe']}); + const child=spawn(process.execPath,['--max-old-space-size=256','--import',preload,...(sourceOnly?['--import',resolve('node_modules/tsx/dist/loader.mjs'),resolve('src/hooks/'+script+'.ts')]:[resolve('dist/hooks/'+script+'.mjs')]),'--agentmemory-sdk-capture-v2'],{cwd:root,env,stdio:['pipe','pipe','pipe']}); let stdout='',stderr='';child.stdout.on('data',b=>stdout+=b);child.stderr.on('data',b=>stderr+=b);child.stdin.on('error',()=>{});child.stdin.end(JSON.stringify(input)); const result=await new Promise<{code:number|null,signal:NodeJS.Signals|null}>((resolve,reject)=>{let uncertain:ReturnType|undefined;const deadline=setTimeout(()=>{child.kill('SIGTERM');uncertain=setTimeout(()=>{retainRoot=true;child.unref();child.stdout.destroy();child.stderr.destroy();reject(Error('synthetic child unreaped at '+root))},2000)},10000);child.once('error',e=>{clearTimeout(deadline);if(uncertain)clearTimeout(uncertain);reject(e)});child.once('close',(code,signal)=>{clearTimeout(deadline);if(uncertain)clearTimeout(uncertain);resolve({code,signal})})}); expect(existsSync(log),stderr).toBe(true);return {...result,stdout,stderr,calls:JSON.parse(readFileSync(log,'utf8')) as Array}; diff --git a/test/observation-certification-lock.test.ts b/test/observation-certification-lock.test.ts index d322c45ddf..ce807ac5e7 100644 --- a/test/observation-certification-lock.test.ts +++ b/test/observation-certification-lock.test.ts @@ -8,14 +8,16 @@ import {registerObserveFunction} from '../src/functions/observe.js'; import {withKeyedLock} from '../src/state/keyed-mutex.js'; import {finishLongSession} from '../src/ops/long-session.mjs'; import {registerEmbeddingRecovery,EMBEDDING_RETRY_NAMESPACE as RETRY} from '../src/ops/embedding-recovery.mjs'; +import {createStateDurabilityFence} from '../src/ops/state-durability.mjs'; vi.mock('../src/logger.js',()=>({logger:{info:vi.fn(),warn:vi.fn(),error:vi.fn()}})); const workers:ReturnType[]=[]; afterEach(async()=>{for(const w of workers.splice(0)){w.stop();await w.drain();}search.setEmbeddingRecovery(null);search.setVectorIndex(null);search.setEmbeddingProvider(null);}); function fixture(){ const kv=mockKV(),index=new VectorIndex(),provider={name:'synthetic',dimensions:2,embed:vi.fn(async()=>new Float32Array([1,0]))}; let release!:()=>void,entered=false;const gate=new Promise(resolve=>release=resolve); - let mutation=0;for(const method of ['set','delete','update'] as const){const original=kv[method];(kv as any)[method]=async(scope:string,...args:any[])=>{if(scope.startsWith('mem:obs:'))mutation++;return(original as any)(scope,...args);};} - Object.assign(kv,{observationReader:{mutationRevision:()=>mutation,pendingMutations:()=>0,captureSummary:async(scope:string,excludeId:string)=>{const rows:any[]=await kv.list(scope);const entries=rows.filter(r=>r.id!==excludeId).map(observationBudgetEntry);return{count:rows.length,summaryCount:entries.length,summaryBytes:entries.reduce((n,r)=>n+r.summaryBytes,0),sourceBytes:entries.reduce((n,r)=>n+r.sourceBytes,0)};}},backend:'file',durability:{waitForKeys:async()=>{entered=true;await gate;return{synced:true};}}}); + const mutationFence=createStateDurabilityFence({storePath:'/tmp/synthetic-certification',completion:'sdk'}); + for(const method of ['set','delete','update'] as const){const original=kv[method];(kv as any)[method]=async(scope:string,key:string,...args:any[])=>mutationFence.mutate(scope,key,()=>(original as any)(scope,key,...args));} + Object.assign(kv,{observationReader:{captureSummary:async(scope:string,excludeId:string)=>{const rows:any[]=await kv.list(scope);const entries=rows.filter(r=>r.id!==excludeId).map(observationBudgetEntry);return{count:rows.length,summaryCount:entries.length,summaryBytes:entries.reduce((n,r)=>n+r.summaryBytes,0),sourceBytes:entries.reduce((n,r)=>n+r.sourceBytes,0)};}},backend:'file',durability:{...mutationFence,waitForKeys:async()=>{entered=true;await gate;return{synced:true};}}}); const recovery=registerEmbeddingRecovery({kv:kv as never,KV,index,provider,withKeyedLock,schedule:false,persistIndex:async()=>{}});workers.push(recovery); search.setVectorIndex(index);search.setEmbeddingProvider(provider);search.setEmbeddingRecovery(recovery); const sdk=mockSdk({looseTrigger:true});registerObserveFunction(sdk as never,kv as never); @@ -159,7 +161,31 @@ for(const beginBeforePrepare of [false,true])it(`invalidates a certificate when }finally{worker.stop();await worker.drain();vi.unstubAllEnvs();await rm(dir,{recursive:true,force:true});} }); it('refuses unsafe deferred file certification without a mutation guard',async()=>{ - const f=fixture();delete(f.kv as any).observationReader; + const f=fixture();delete(f.kv as any).durability.watchKeys; const item={id:f.payload.observationId,sessionId:f.payload.sessionId,kind:'synthetic' as const};await f.kv.set(KV.observations(item.sessionId),item.id,{id:item.id,sessionId:item.sessionId,title:'canonical',facts:[],concepts:[],files:[]}); await expect(f.recovery.prepareEnqueue(item)).rejects.toMatchObject({code:'EMBEDDING_SOURCE_GUARD_UNAVAILABLE'});f.release(); }); + +it('certifies its unchanged observation while another observation in the same session is written',async()=>{ + const f=fixture(),item={id:f.payload.observationId,sessionId:f.payload.sessionId,kind:'synthetic' as const}; + await f.kv.set(KV.observations(item.sessionId),item.id,{id:item.id,sessionId:item.sessionId,title:'selected'}); + const certify=await f.recovery.prepareEnqueue(item),pending=certify();void pending.catch(()=>{});await vi.waitFor(()=>expect(f.entered()).toBe(true)); + await f.kv.set(KV.observations(item.sessionId),'obs_other',{id:'obs_other',sessionId:item.sessionId,title:'unrelated'});f.release(); + await expect(pending).resolves.toBe(true);expect(await f.kv.get(RETRY,item.id)).toMatchObject({id:item.id}); + expect(f.provider.embed).not.toHaveBeenCalled(); +}); + +it('preserves the source and inbox without leaking a watch on live-event failure, then retries',async()=>{ + const f=fixture(),fence=(f.kv as any).durability,watchKeys=fence.watchKeys;let live=0; + const watched=vi.spyOn(fence,'watchKeys').mockImplementation((keys:any)=>{const watch=watchKeys(keys);live++;let released=false;return{unchanged:watch.unchanged,release(){if(!released){released=true;live--;}watch.release();}};}); + const {liveEvents}=await import('../src/viewer/live-events.js'),original=liveEvents.emit; + const emit=vi.spyOn(liveEvents,'emit').mockImplementation(function(event){if(event.type==='compressed_observation')throw Error('synthetic live writer failed');return original.call(liveEvents,event);}); + const {registerCaptureFunctions}=await import('../src/functions/capture.js'),sdk=mockSdk({looseTrigger:true});registerObserveFunction(sdk as never,f.kv as never);const capture=registerCaptureFunctions(sdk as never,f.kv as never); + f.release(); + try{ + const failed:any=await sdk.trigger('mem::capture',{payload:f.payload,eventId:f.payload.eventId});expect(failed.state).toBe('retrying'); + expect(await f.kv.list(KV.captureInbox)).toHaveLength(1);expect(await f.kv.list(KV.observations(f.payload.sessionId))).toHaveLength(1);expect(live).toBe(0); + emit.mockRestore();expect(await sdk.trigger('mem::observe',{...f.payload,observationId:failed.observationId})).toMatchObject({existing:true,observationId:failed.observationId}); + expect(await f.kv.get(RETRY,failed.observationId)).not.toBeNull();expect(await f.kv.list(KV.observations(f.payload.sessionId))).toHaveLength(1);expect(live).toBe(0); + }finally{emit.mockRestore();watched.mockRestore();capture.stop();await capture.drain();} +}); diff --git a/test/observation-source.test.ts b/test/observation-source.test.ts index b8e2b674c8..9cd5db0a69 100644 --- a/test/observation-source.test.ts +++ b/test/observation-source.test.ts @@ -451,7 +451,7 @@ describe("retained observation source", () => { it("refuses a replay result that expands beyond the transport frame", async () => { const r = rig(); - const template = buildSyntheticCompression(raw({ toolName: "tool".repeat(10000) })); + const template = { ...buildSyntheticCompression(raw()), source: undefined, title: "t".repeat(Math.ceil(SAFE_PAYLOAD_BYTES / 500) + 100) }; for (let i = 0; i < 500; i++) { const row = { ...template, id: `large-label-${i}` }; await r.kv.set(KV.observations(row.sessionId), row.id, row); diff --git a/test/operations-background-control.test.ts b/test/operations-background-control.test.ts index 1f8cbd061f..f323b56e5e 100644 --- a/test/operations-background-control.test.ts +++ b/test/operations-background-control.test.ts @@ -11,6 +11,14 @@ function fixture(saved: unknown = null) { } describe("background transition status and safe diagnostics", () => { + it('changes storage generation on storage failure but not a successful manual pause',async()=>{ + const f=fixture({paused:false});const control=createBackgroundControl();await control.init(f.kv); + const initial=control.generation();await control.change('pause');await control.settled(); + expect(control.generation()).toBe(initial); + f.kv.durability.wait=async()=>{throw Error('synthetic_storage_failure')}; + await expect(control.change('resume')).rejects.toThrow(); + expect(control.generation()).toBeGreaterThan(initial); + }); it("keeps cold-start admission closed until the crash guard reaches disk", async () => { const f = fixture({ paused: false }), disk = gate(); f.kv.durability.waitForKeys = async () => { await disk.promise; }; @@ -148,3 +156,25 @@ it('fails closed when the full-dirty wait API is unavailable',async()=>{ const f=fixture({paused:true});delete (f.kv.durability as any).wait;const control=createBackgroundControl();await control.init(f.kv); await expect(control.change('resume')).rejects.toThrow('background_storage_unavailable');expect(control.snapshot()).toMatchObject({status:'error',pausedRequested:true}); }); + +describe('explicit resume vector recovery handoff',()=>{ + it('runs vector recovery after a successful explicit release',async()=>{ + const f=fixture({version:2,paused:false,resumePending:true,holds:[],recovery:{attempts:0,nextAttemptAt:0}}),control=createBackgroundControl();await control.init(f.kv);const handlers=new Map();const resume=vi.fn(async()=>true); + registerBackgroundControlApi({registerFunction:(id:string,fn:any)=>handlers.set(id,fn),registerTrigger:()=>{}} as any,'fixture',control,resume); + const out=await handlers.get('api::background-control-set')({headers:{authorization:'Bearer fixture','x-agentmemory-background':'1','content-type':'application/json'},body:{action:'resume'}}); + expect(resume).toHaveBeenCalledOnce();expect(out).toMatchObject({status_code:202,body:{success:true,vectorRecovery:'replay_complete_backfill_pending',control:{status:'running'}}}); + }); + it('keeps admission held when vector recovery fails',async()=>{ + const f=fixture({version:2,paused:false,resumePending:true,holds:[],recovery:{attempts:0,nextAttemptAt:0}}),control=createBackgroundControl();await control.init(f.kv);const handlers=new Map(); + registerBackgroundControlApi({registerFunction:(id:string,fn:any)=>handlers.set(id,fn),registerTrigger:()=>{}} as any,'fixture',control,async()=>false); + const out=await handlers.get('api::background-control-set')({headers:{authorization:'Bearer fixture','x-agentmemory-background':'1','content-type':'application/json'},body:{action:'resume'}}); + expect(out).toMatchObject({status_code:503,body:{success:false,error:'vector_recovery_unavailable',control:{status:'error',pausedRequested:true,holds:['storage_error']}}}); + }); +}); + +it('keeps background admission closed until explicit vector recovery completes',async()=>{const f=fixture({version:2,paused:false,resumePending:true,holds:[],recovery:{attempts:0,nextAttemptAt:0}}),control=createBackgroundControl();await control.init(f.kv);const handlers=new Map(),pending=gate(),entered=gate();registerBackgroundControlApi({registerFunction:(id:string,fn:any)=>handlers.set(id,fn),registerTrigger:()=>{}} as any,'fixture',control,async()=>{entered.resolve();await pending.promise;return true});const request=handlers.get('api::background-control-set')({headers:{authorization:'Bearer fixture','x-agentmemory-background':'1','content-type':'application/json'},body:{action:'resume'}});await entered.promise;expect(control.snapshot()).toMatchObject({status:'resuming',pausedRequested:true});expect(await control.run(()=>42)).toMatchObject({deferred:true});pending.resolve();expect(await request).toMatchObject({status_code:202,body:{success:true,control:{status:'running'}}})}); +it('retains a new pause requested during vector resume',async()=>{const f=fixture({version:2,paused:false,resumePending:true,holds:[],recovery:{attempts:0,nextAttemptAt:0}}),control=createBackgroundControl();await control.init(f.kv);const handlers=new Map(),pending=gate(),entered=gate();registerBackgroundControlApi({registerFunction:(id:string,fn:any)=>handlers.set(id,fn),registerTrigger:()=>{}} as any,'fixture',control,async()=>{entered.resolve();await pending.promise;return true});const request=handlers.get('api::background-control-set')({headers:{authorization:'Bearer fixture','x-agentmemory-background':'1','content-type':'application/json'},body:{action:'resume'}});await entered.promise;const pause=control.change('pause');pending.resolve();const response=await request;await pause;expect(response.status_code).toBe(202);expect(control.snapshot()).toMatchObject({status:'paused',pausedRequested:true,holds:['manual'],error:null})}); +it('rejects new foreground vector-writing work before its handler while storage is held',async()=>{const {installBackgroundControl,controlBackgroundFunctions}=await import('../src/ops/background-control.mjs');const f=fixture({version:2,paused:false,resumePending:true,holds:[],recovery:{attempts:0,nextAttemptAt:0}}),control=createBackgroundControl();await control.init(f.kv);installBackgroundControl(control);const handlers=new Map(),run=vi.fn(async()=>({success:true}));const sdk={registerFunction:(id:string,fn:any)=>handlers.set(id,fn)};controlBackgroundFunctions(sdk as any);sdk.registerFunction('mem::remember',run);const result=await handlers.get('mem::remember')({content:'synthetic'});expect(result).toMatchObject({success:false,error:'vector_recovery_paused',retryable:true});expect(run).not.toHaveBeenCalled()}); +it('reports a potentially partial vector-writing request if storage fails while it runs',async()=>{const {installBackgroundControl,controlBackgroundFunctions}=await import('../src/ops/background-control.mjs');const f=fixture({paused:false}),control=createBackgroundControl();await control.init(f.kv);installBackgroundControl(control);const handlers=new Map(),entered=gate(),release=gate();const sdk={registerFunction:(id:string,fn:any)=>handlers.set(id,fn)};controlBackgroundFunctions(sdk as any);sdk.registerFunction('mem::remember',async()=>{entered.resolve();await release.promise;return {success:true,memory:{id:'synthetic'}}});const running=handlers.get('mem::remember')({content:'synthetic'});await entered.promise;f.kv.set=async()=>{throw Error('synthetic failure')};await expect(control.change('pause')).rejects.toThrow();release.resolve();expect(await running).toMatchObject({success:false,error:'vector_recovery_interrupted',partial:true,retryable:false})}); +it('refuses vector resume while an earlier foreground vector write remains active',async()=>{const {installBackgroundControl,controlBackgroundFunctions}=await import('../src/ops/background-control.mjs');const f=fixture({paused:false}),control=createBackgroundControl();await control.init(f.kv);installBackgroundControl(control);const handlers=new Map(),entered=gate(),release=gate(),resume=vi.fn(async()=>true);const sdk={registerFunction:(id:string,fn:any)=>handlers.set(id,fn),registerTrigger:()=>{}};controlBackgroundFunctions(sdk as any);sdk.registerFunction('mem::remember',async()=>{entered.resolve();await release.promise;return{success:true}});registerBackgroundControlApi(sdk as any,'fixture',control,resume);const work=handlers.get('mem::remember')({content:'synthetic'});await entered.promise;const set=f.kv.set;f.kv.set=async()=>{throw Error('storage failed')};await expect(control.change('pause')).rejects.toThrow();f.kv.set=set;const response=await handlers.get('api::background-control-set')({headers:{authorization:'Bearer fixture','x-agentmemory-background':'1','content-type':'application/json'},body:{action:'resume'}});expect(response).toMatchObject({status_code:409,body:{error:'background_draining',control:{pausedRequested:true}}});expect(resume).not.toHaveBeenCalled();release.resolve();await work;expect(control.snapshot().holds).toContain('storage_error')}); +it.each(['mem::import','mem::replay::import-jsonl'])('rejects %s before import during storage recovery',async(id)=>{const {installBackgroundControl,controlBackgroundFunctions}=await import('../src/ops/background-control.mjs');const f=fixture({version:2,paused:false,resumePending:true,holds:[],recovery:{attempts:0,nextAttemptAt:0}}),control=createBackgroundControl();await control.init(f.kv);installBackgroundControl(control);const handlers=new Map(),run=vi.fn(async()=>({success:true}));const sdk={registerFunction:(name:string,fn:any)=>handlers.set(name,fn)};controlBackgroundFunctions(sdk as any);sdk.registerFunction(id,run);expect(await handlers.get(id)({})).toMatchObject({success:false,error:'vector_recovery_paused',retryable:true});expect(run).not.toHaveBeenCalled()}); diff --git a/test/operations-protection-recovery.test.ts b/test/operations-protection-recovery.test.ts index a2e5457e34..2f4945583d 100644 --- a/test/operations-protection-recovery.test.ts +++ b/test/operations-protection-recovery.test.ts @@ -15,7 +15,7 @@ it('persists distinct overlapping holds and never automatically releases manual }); it('recovers only after strictly greater recovery threshold and sustained fresh samples',async()=>{ const f=fixture();await f.control.init(f.kv);await f.guard.refresh();await f.control.settled();expect(f.control.snapshot().holds).toEqual(['memory']); - f.memory(4*GiB);for(let i=0;i<30;i++){f.time(f.clock()+5000);await f.guard.refresh();}expect(f.control.isPaused()).toBe(true); + f.memory(16*GiB*.22);for(let i=0;i<30;i++){f.time(f.clock()+5000);await f.guard.refresh();}expect(f.control.isPaused()).toBe(true); await f.normal(115000);expect(f.control.isPaused()).toBe(true);f.time(f.clock()+5000);await f.guard.refresh();await f.guard.drain();expect(f.control.isPaused()).toBe(false); expect(f.record()).toMatchObject({version:2,paused:false,resumePending:true,recovery:{attempts:1}}); }); @@ -42,14 +42,90 @@ it('rechecks fresh pressure during activation and preserves memory hold if high }); it('persists storage failures as a hold and requires manual recovery',async()=>{ const f=fixture();await f.control.init(f.kv);f.kv.durability.wait=async()=>{throw Error('synthetic');};await f.guard.refresh();await f.control.settled();expect(f.control.snapshot().holds).toContain('storage_error'); - f.kv.durability.wait=async()=>({synced:true,mutationFailures:0});await f.normal();expect(f.control.isPaused()).toBe(true);await f.control.change('resume');expect(f.control.snapshot().holds).toEqual(['memory']); + f.kv.durability.wait=async()=>({synced:true,mutationFailures:0});await f.normal(); + expect(f.control.snapshot()).toMatchObject({status:'error',pausedRequested:true,holds:['storage_error']}); + let recoveries=0;await f.control.change('resume',{beforeResume:async()=>{recoveries++;return true;}}); + expect(recoveries).toBe(1);expect(f.control.snapshot().holds).toEqual([]); +}); + +it.each([false,true])('releases only recovered memory after held boot with resumePending=%s',async resumePending=>{ + const holds=['memory','storage_error']; + const f=fixture({version:2,paused:true,resumePending,holds,recovery:{attempts:0,nextAttemptAt:0}}); + await f.control.init(f.kv);await f.normal(115000);expect(f.control.snapshot().holds).toEqual(holds); + f.time(f.clock()+5000);await f.guard.refresh();await f.guard.drain(); + expect(f.control.snapshot()).toMatchObject({status:'error',pausedRequested:true,holds:['storage_error']}); + expect(await f.control.run(async()=>42)).toMatchObject({deferred:true}); + expect(await f.control.runVectorWrite(async()=>42)).toMatchObject({success:false}); + // An unclean restart must recheck memory even after this worker observed recovery. + const restarted=fixture(f.record());await restarted.control.init(restarted.kv); + expect(restarted.control.snapshot()).toMatchObject({status:'error',holds}); +}); + +it.each(['pressure','stale','pause','failed_fence'])('retains both holds on %s during compound memory recovery',async risk=>{ + const holds=['memory','storage_error']; + const f=fixture({version:2,paused:true,resumePending:true,holds,recovery:{attempts:0,nextAttemptAt:0}}); + await f.control.init(f.kv);await f.normal(115000); + let waits=0; + f.kv.durability.wait=async()=>{ + if(++waits===1){ + if(risk==='pressure')f.memory(GiB); + if(risk==='stale')f.time(f.clock()+10001); + if(risk==='pause')void f.control.change('pause',{source:'manual'}); + if(risk==='failed_fence')throw Error('synthetic storage failure'); + } + return {synced:true,mutationFailures:0}; + }; + f.time(f.clock()+5000);await f.guard.refresh();await f.guard.drain();await f.control.settled(); + expect(waits).toBeGreaterThan(0); + const expected=risk==='pause'?['manual',...holds]:holds; + expect(f.control.snapshot()).toMatchObject({status:'error',holds:expected,pausedRequested:true}); + expect(f.record().holds).toEqual(expected); +}); + +it('reports an unconfirmed rollback and keeps both holds when compound recovery loses storage',async()=>{ + const holds=['memory','storage_error']; + const f=fixture({version:2,paused:true,resumePending:true,holds,recovery:{attempts:0,nextAttemptAt:0}}); + await f.control.init(f.kv);await f.normal(115000); + f.kv.durability.wait=async()=>{f.fail(true);throw Error('fence unavailable');}; + f.time(f.clock()+5000);await f.guard.refresh();await f.guard.drain(); + expect(f.control.snapshot()).toMatchObject({holds,pausedRequested:true,diagnostics:{rollbackVerified:false}}); + expect(f.record()).toMatchObject({holds,paused:true,resumePending:true}); +}); + +it('requires a new manual resume after compound memory recovery completes',async()=>{ + const holds=['memory','storage_error']; + const f=fixture({version:2,paused:true,resumePending:true,holds,recovery:{attempts:0,nextAttemptAt:0}}); + await f.control.init(f.kv); + let release!:()=>void; + const fence=new Promise(r=>{release=r;}); + f.kv.durability.wait=async()=>{await fence;return {synced:true,mutationFailures:0};}; + const partial=f.control.change('resume',{source:'memory',canResume:async()=>true}); + await new Promise(r=>setImmediate(r)); + let vectorAttempts=0; + const queued=f.control.change('resume',{beforeResume:async()=>{vectorAttempts++;return true;}}); + release();await partial;await queued; + expect(f.control.snapshot()).toMatchObject({holds:['storage_error'],pausedRequested:true}); + expect(vectorAttempts).toBe(0); }); it('bounds automatic attempts durably and honors persisted backoff',async()=>{ const f=fixture({version:2,paused:true,resumePending:false,holds:['memory'],recovery:{attempts:3,nextAttemptAt:0}});await f.control.init(f.kv);await f.normal();expect(f.control.isPaused()).toBe(true); const g=fixture({version:2,paused:true,resumePending:false,holds:['memory'],recovery:{attempts:1,nextAttemptAt:999999}});await g.control.init(g.kv);await g.normal();expect(g.control.isPaused()).toBe(true);expect(g.record().recovery.attempts).toBe(1); }); it('uses exact pressure boundaries and never recovers with unavailable samples',async()=>{ - const f=fixture();await f.control.init(f.kv);f.memory(16*GiB*.2);await f.guard.refresh();expect(f.guard.snapshot().level).toBe('normal');f.memory(16*GiB*.05);await f.guard.refresh();expect(f.guard.snapshot().level).toBe('high');await f.control.settled();f.memory(NaN);await f.normal(0);f.memory(NaN);f.time(f.clock()+120000);await f.guard.refresh();expect(f.control.isPaused()).toBe(true); + const f=fixture();await f.control.init(f.kv);f.memory(16*GiB*.18);await f.guard.refresh();expect(f.guard.snapshot().level).toBe('normal');f.memory(16*GiB*.05);await f.guard.refresh();expect(f.guard.snapshot().level).toBe('high');await f.control.settled();f.memory(NaN);await f.normal(0);f.memory(NaN);f.time(f.clock()+120000);await f.guard.refresh();expect(f.control.isPaused()).toBe(true); +}); + +it('uses the approved 18% reserve and recovers only above 22% for two minutes',async()=>{ + const f=fixture();await f.control.init(f.kv); + f.memory(16*GiB*.18-1);await f.guard.refresh();await f.guard.drain();await f.control.settled(); + expect(f.guard.snapshot()).toMatchObject({level:'high',reserveBytes:16*GiB*.18,recoveryBytes:16*GiB*.22,criticalBytes:16*GiB*.05}); + f.memory(16*GiB*.18);f.time(f.clock()+5000);await f.guard.refresh();expect(f.guard.snapshot().level).toBe('normal'); + f.memory(16*GiB*.22);for(let i=0;i<30;i++){f.time(f.clock()+5000);await f.guard.refresh();await f.guard.drain();} + expect(f.control.isPaused()).toBe(true); + f.memory(16*GiB*.22+1);for(let i=0;i<24;i++){f.time(f.clock()+5000);await f.guard.refresh();await f.guard.drain();} + expect(f.control.isPaused()).toBe(true); + f.time(f.clock()+5000);await f.guard.refresh();await f.guard.drain();expect(f.control.isPaused()).toBe(false); + await f.guard.stop(); }); it('retains cumulative attempts through repeated pressure and clean restart until manual acknowledgment',async()=>{ const f=fixture();await f.control.init(f.kv); @@ -92,6 +168,24 @@ it('honors future backoff even when stable samples survive a clean restart and a const f=fixture({version:2,paused:true,resumePending:false,holds:['memory'],recovery:{attempts:1,nextAttemptAt:130001}});await f.control.init(f.kv);await f.normal();expect(f.control.isPaused()).toBe(true);f.time(130001);await f.guard.refresh();await f.guard.drain();expect(f.control.isPaused()).toBe(false);expect(f.control.snapshot().recovery.attempts).toBe(2); }); +it.each(['manual', 'deployment'] as const)('limits SDK vector reconciliation to a manual resume (%s)', async source => { + const f = fixture({version:2, paused:true, resumePending:false, holds:[source], recovery:{attempts:0,nextAttemptAt:0}}); + await f.control.init(f.kv); + const calls: boolean[] = []; + const handlers = new Map(); + const sdk = {registerFunction:(id:string, fn:any) => handlers.set(id, fn), registerTrigger:() => {}}; + registerBackgroundControlApi(sdk as any, 'fixture', f.control, async allowReconciliation => { + calls.push(allowReconciliation); + return true; + }); + const result = await handlers.get('api::background-control-set')({ + headers:{authorization:'Bearer fixture','x-agentmemory-background':'1','content-type':'application/json'}, + body:source === 'manual' ? {action:'resume'} : {action:'resume', reason:'deployment'} + }); + expect(result.status_code).toBe(202); + expect(calls).toEqual([source === 'manual']); +}); + it('keeps action-only HTTP pauses manual and permits only an explicit deployment reason',async()=>{ const f=fixture(),handlers=new Map();await f.control.init(f.kv);registerBackgroundControlApi({registerFunction:(id:string,fn:any)=>handlers.set(id,fn),registerTrigger:()=>{}} as any,'fixture',f.control); const post=(body:any)=>handlers.get('api::background-control-set')({headers:{authorization:'Bearer fixture','x-agentmemory-background':'1','content-type':'application/json'},body}); @@ -158,3 +252,49 @@ it('does not replace a newer critical sample with a delayed recovery sample',asy release();await guard.drain();await f.control.settled();expect(guard.snapshot().level).toBe('critical');expect(f.control.isPaused()).toBe(true); }finally{release();await guard.stop();await f.control.settled();} }); + +it('releases only the recovered memory hold from an interrupted protected storage pause',async()=>{ + const f=fixture({version:2,paused:true,resumePending:true,holds:['manual','memory','storage_error'],recovery:{attempts:0,nextAttemptAt:0}}); + await f.control.init(f.kv);await f.normal(115000); + expect(f.control.snapshot().holds).toEqual(['manual','memory','storage_error']); + f.time(f.clock()+5000);await f.guard.refresh();await f.guard.drain();await f.control.settled(); + expect(f.control.snapshot()).toMatchObject({holds:['manual','storage_error'],pausedRequested:true,activeOperations:0}); + expect(f.record()).toMatchObject({paused:true,resumePending:false,holds:['manual','storage_error']}); + expect(await f.control.run(async()=>42)).toMatchObject({deferred:true}); + let verified=false;await f.control.change('resume',{beforeResume:async()=>{verified=true;return true;}}); + expect(verified).toBe(true);expect(f.control.snapshot()).toMatchObject({status:'running',holds:[],pausedRequested:false}); +}); +it('requires a fresh stable-memory check before releasing one hold from a protected storage pause',async()=>{ + const f=fixture({version:2,paused:true,resumePending:true,holds:['manual','memory','storage_error'],recovery:{attempts:0,nextAttemptAt:0}}); + await f.control.init(f.kv);const before=structuredClone(f.record());let checked=false; + await f.control.change('resume',{source:'memory',canResume:async()=>{checked=true;return false;}}); + expect(checked).toBe(true);expect(f.record()).toEqual(before);expect(f.control.snapshot().holds).toEqual(['manual','memory','storage_error']); +}); +it('retains all protection holds on failed partial storage confirmation and bounds automatic retries',async()=>{ + const f=fixture({version:2,paused:true,resumePending:true,holds:['manual','memory','storage_error'],recovery:{attempts:2,nextAttemptAt:0}}); + await f.control.init(f.kv);const original=structuredClone(f.record());f.fail(true); + await expect(f.control.change('resume',{source:'memory',canResume:async()=>true})).rejects.toThrow('background_storage_unavailable'); + expect(f.control.snapshot()).toMatchObject({holds:['manual','memory','storage_error'],pausedRequested:true,recovery:{attempts:3}}); + expect(f.record()).toEqual(original);f.fail(false);f.time(1000000000);let retried=false; + await f.control.change('resume',{source:'memory',canResume:async()=>{retried=true;return true;}}); + expect(retried).toBe(false);expect(f.control.snapshot().holds).toEqual(['manual','memory','storage_error']); +}); +it('rechecks stable memory after the storage fence before releasing a protected memory hold',async()=>{ + const f=fixture({version:2,paused:true,resumePending:true,holds:['manual','memory','storage_error'],recovery:{attempts:0,nextAttemptAt:0}}); + await f.control.init(f.kv);let entered!:()=>void,release!:()=>void,stable=true; + const waiting=new Promise(r=>entered=r),held=new Promise(r=>release=r); + f.kv.durability.wait=async()=>{entered();await held;return{synced:true,mutationFailures:0};}; + const recovering=f.control.change('resume',{source:'memory',canResume:async()=>stable});await waiting;stable=false;release();await recovering; + expect(f.control.snapshot().holds).toEqual(['manual','memory','storage_error']);expect(f.record().holds).toEqual(['manual','memory','storage_error']); + expect(f.control.isPaused()).toBe(true); +}); +it('requires a new manual resume after the protected memory release completes',async()=>{ + const f=fixture({version:2,paused:true,resumePending:true,holds:['manual','memory','storage_error'],recovery:{attempts:0,nextAttemptAt:0}}); + await f.control.init(f.kv);let entered!:()=>void,release!:()=>void; + const waiting=new Promise(r=>entered=r),held=new Promise(r=>release=r); + f.kv.durability.wait=async()=>{entered();await held;return{synced:true,mutationFailures:0};}; + const recovering=f.control.change('resume',{source:'memory',canResume:async()=>true});await waiting;let callbacks=0; + const queued=f.control.change('resume',{beforeResume:async()=>{callbacks++;return true;}});release();await recovering;await queued; + expect(callbacks).toBe(0);expect(f.control.snapshot()).toMatchObject({holds:['manual','storage_error'],pausedRequested:true}); + await f.control.change('resume',{beforeResume:async()=>{callbacks++;return true;}});expect(callbacks).toBe(1);expect(f.control.isPaused()).toBe(false); +}); diff --git a/test/operations-throughput.test.ts b/test/operations-throughput.test.ts index 71b3c85d4f..9194297405 100644 --- a/test/operations-throughput.test.ts +++ b/test/operations-throughput.test.ts @@ -4,7 +4,7 @@ import {describe,it,expect} from 'vitest'; const html=readFileSync('src/viewer/index.html','utf8'); function setup(){ const ctx:any={operations:{progressSamples:{}},autoRefresh:{seconds:30},Number,Date}; - for(const name of ['progressDuration','operationsStages','recordOperationsProgress','stageProgress']){ + for(const name of ['progressDuration','summaryStageProgress','operationsStages','recordOperationsProgress','stageProgress']){ const start=html.indexOf('function '+name+'(');if(start<0)continue; let depth=0;for(let n=html.indexOf('{',start);n { expect(authorization()).toBeUndefined(); }); }); + +describe("plugin bridge call timeout", () => { + it("uses the shared MCP call timeout instead of a fixed 15 s", async () => { + vi.stubEnv("AGENTMEMORY_CALL_TIMEOUT_MS", ""); + vi.stubEnv("CLAUDE_PLUGIN_OPTION_AGENTMEMORY_CALL_TIMEOUT_MS", ""); + vi.stubEnv("AGENTMEMORY_ENV_FILE", ""); + const timeout = vi.spyOn(AbortSignal, "timeout"); + try { + await createPluginBridge()("tools/list", {}); + expect(timeout).toHaveBeenLastCalledWith(190_000); + vi.stubEnv("AGENTMEMORY_CALL_TIMEOUT_MS", "600000"); + await createPluginBridge()("tools/list", {}); + expect(timeout).toHaveBeenLastCalledWith(600_000); + } finally { + timeout.mockRestore(); + } + }); +}); diff --git a/test/port07-feature-wiring.test.ts b/test/port07-feature-wiring.test.ts index eab20613be..c3cea49cf9 100644 --- a/test/port07-feature-wiring.test.ts +++ b/test/port07-feature-wiring.test.ts @@ -1,4 +1,4 @@ import {it,expect,vi,afterEach} from 'vitest';import {readFileSync} from 'node:fs';import {transpileModule,ScriptTarget,ModuleKind} from 'typescript'; import {registerReflectFunctions} from '../src/functions/reflect.js';import {operationalState} from '../src/ops/context.js'; afterEach(()=>{delete operationalState.featureLifecycle;}); -it('registers public reflection once and delegates to the tracked conservative lifecycle',async()=>{const source=readFileSync('src/index.ts','utf8');const start=source.indexOf(' const features = lifecycle.track('),end=source.indexOf(' const bm25Index',start);const features={slotReflect:vi.fn(),reflect:vi.fn(async()=>({success:true}))};const functions=new Map();const sdk={registerFunction:(id:string,handler:unknown)=>{if(functions.has(id))throw Error('duplicate function id');functions.set(id,handler);}};registerReflectFunctions(sdk as never,{} as never,{} as never);const deps={lifecycle:{track:(x:unknown)=>x},createFeatureLifecycle:()=>features,registerEnrichmentWorker:()=>({}),sdk,kv:{},KV:{},withKeyedLock:()=>{},provider:{},operationalState,isSlotsEnabled:()=>true,isReflectEnabled:()=>true,logger:{},isGraphExtractionEnabled:()=>true,isAutoCompressEnabled:()=>true,metricsStore:{},rawFromObservationSource:()=>{}};const code=transpileModule(source.slice(start,end),{compilerOptions:{target:ScriptTarget.ES2022,module:ModuleKind.ES2022}}).outputText;new Function('deps',`const {${Object.keys(deps).join(',')}}=deps;${code}`)(deps);expect(functions.get('mem::slot-reflect')).toBe(features.slotReflect);await functions.get('mem::reflect')({project:'fixture'});expect(features.reflect).toHaveBeenCalledWith({project:'fixture'});}); +it('registers public reflection once and delegates to the tracked conservative lifecycle',async()=>{const source=readFileSync('src/index.ts','utf8');const start=source.indexOf(' const features = lifecycle.track('),end=source.indexOf(' const bm25Index',start);const features={slotReflect:vi.fn(),reflect:vi.fn(async()=>({success:true}))};const functions=new Map();const sdk={registerFunction:(id:string,handler:unknown)=>{if(functions.has(id))throw Error('duplicate function id');functions.set(id,handler);}};registerReflectFunctions(sdk as never,{} as never,{} as never);const deps={lifecycle:{track:(x:unknown)=>x},createFeatureLifecycle:()=>features,registerEnrichmentWorker:()=>({}),sdk,kv:{},KV:{},withKeyedLock:()=>{},provider:{},operationalState,isSlotsEnabled:()=>true,isReflectEnabled:()=>true,logger:{},isGraphExtractionEnabled:()=>true,isAutoCompressEnabled:()=>true,metricsStore:{},rawFromObservationSource:()=>{}};const code=transpileModule(source.slice(start,end),{compilerOptions:{target:ScriptTarget.ES2022,module:ModuleKind.ES2022}}).outputText;new Function('deps',`const {${Object.keys(deps).join(',')}}=deps;${code}`)(deps);expect(functions.get('mem::slot-reflect')).toBe(features.slotReflect);await functions.get('mem::reflect')({project:'fixture'});expect(features.reflect).toHaveBeenCalledWith({project:'fixture',maxClusters:10});}); diff --git a/test/pre-tool-use-output.test.ts b/test/pre-tool-use-output.test.ts new file mode 100644 index 0000000000..bdc8f3c59b --- /dev/null +++ b/test/pre-tool-use-output.test.ts @@ -0,0 +1,123 @@ +import { describe, expect, it } from "vitest"; +import { join, resolve } from "node:path"; +import { createServer } from "node:http"; +import { spawn } from "node:child_process"; + +const script = join(resolve(__dirname, ".."), "plugin", "scripts", "pre-tool-use.mjs"); +const CONTEXT = 'remembered "quoted"\nline'; + +const HOST_ENV_KEYS = [ + "AGENTMEMORY_INJECT_CONTEXT", + "AGENTMEMORY_ENRICH_TIMEOUT_MS", + "CLAUDE_PLUGIN_OPTION_AGENTMEMORY_ENRICH_TIMEOUT_MS", + "FACTORY_PROJECT_DIR", + "DROID_PLUGIN_ROOT", +]; + +async function runPreToolUse( + payload: Record, + env: Record = {}, + delayMs = 0, +): Promise<{ stdout: string }> { + const server = createServer((req, res) => { + req.resume(); + req.on("end", () => { + setTimeout(() => { + res.writeHead(200, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ context: CONTEXT })); + }, delayMs); + }); + }); + await new Promise((r) => server.listen(0, "127.0.0.1", r)); + const address = server.address(); + if (!address || typeof address === "string") { + server.close(); + throw new Error("test server did not bind to a TCP port"); + } + const baseEnv: Record = { ...process.env }; + for (const k of HOST_ENV_KEYS) delete baseEnv[k]; + try { + const child = spawn(process.execPath, [script], { + env: { + ...baseEnv, + AGENTMEMORY_URL: `http://127.0.0.1:${address.port}`, + AGENTMEMORY_SECRET: "", + AGENTMEMORY_INJECT_CONTEXT: "true", + ...env, + }, + stdio: ["pipe", "pipe", "ignore"], + }); + let stdout = ""; + child.stdout.on("data", (chunk) => { + stdout += chunk; + }); + child.stdin.end(JSON.stringify(payload)); + await new Promise((resolveExit, reject) => { + const timeout = setTimeout(() => { + child.kill(); + reject(new Error("pre-tool-use timed out")); + }, 10_000); + child.on("error", reject); + child.on("close", () => { + clearTimeout(timeout); + resolveExit(); + }); + }); + return { stdout }; + } finally { + server.closeAllConnections(); + await new Promise((r) => server.close(() => r())); + } +} + +const claudePayload = { + session_id: "ses-1", + hook_event_name: "PreToolUse", + tool_name: "Read", + tool_input: { file_path: "src/foo.ts" }, +}; + +describe("pre-tool-use context output", () => { + it("wraps context in the PreToolUse hook envelope so Claude Code shows it to the model", async () => { + const { stdout } = await runPreToolUse(claudePayload); + expect(JSON.parse(stdout)).toEqual({ + hookSpecificOutput: { hookEventName: "PreToolUse", additionalContext: CONTEXT }, + }); + }); + + it("keeps plain text for payloads without a PreToolUse event name", async () => { + const { hook_event_name: _ignored, ...payload } = claudePayload; + const { stdout } = await runPreToolUse(payload); + expect(stdout).toBe(CONTEXT); + }); + + it("keeps plain text on plain-text hosts", async () => { + const { stdout } = await runPreToolUse(claudePayload, { FACTORY_PROJECT_DIR: "/repo" }); + expect(stdout).toBe(CONTEXT); + }); + + it("drops a reply slower than the default 2 s budget", async () => { + const { stdout } = await runPreToolUse(claudePayload, {}, 2500); + expect(stdout).toBe(""); + }); + + it("AGENTMEMORY_ENRICH_TIMEOUT_MS gives a slow enrich reply time to arrive", async () => { + const { stdout } = await runPreToolUse(claudePayload, { AGENTMEMORY_ENRICH_TIMEOUT_MS: "6000" }, 2500); + expect(JSON.parse(stdout).hookSpecificOutput.additionalContext).toBe(CONTEXT); + }); + + it.each(["0.5", "3000000000"])("falls back to the default instead of aborting at once for %s", async (value) => { + const { stdout } = await runPreToolUse(claudePayload, { AGENTMEMORY_ENRICH_TIMEOUT_MS: value }, 100); + expect(JSON.parse(stdout).hookSpecificOutput.additionalContext).toBe(CONTEXT); + }); + + it.each(["1e6", "6000ms"])("keeps the 2 s default for the malformed value %s", async (value) => { + const { stdout } = await runPreToolUse(claudePayload, { AGENTMEMORY_ENRICH_TIMEOUT_MS: value }, 2500); + expect(stdout).toBe(""); + }); + + it("writes nothing for Cursor, whose preToolUse response has no context field", async () => { + const { stdout } = await runPreToolUse({ ...claudePayload, hook_event_name: "preToolUse", cursor_version: "1.7.0" }); + expect(stdout).toBe(""); + }); +}); diff --git a/test/pre-tool-use-project-scope.test.ts b/test/pre-tool-use-project-scope.test.ts new file mode 100644 index 0000000000..e1801d2e76 --- /dev/null +++ b/test/pre-tool-use-project-scope.test.ts @@ -0,0 +1,150 @@ +import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import { execFileSync, spawn } from "node:child_process"; +import { createServer, type Server } from "node:http"; +import { mkdtempSync, mkdirSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +let server: Server; +let port: number; +let posts: Array<{ path: string; body: Record }> = []; + +const REPO_NAME = "amem-pretool-fixture"; + +function runHook( + payload: Record, + env: Record = {}, +): Promise { + return new Promise((resolve) => { + const child = spawn("node", ["plugin/scripts/pre-tool-use.mjs"], { + env: { + ...process.env, + AGENTMEMORY_URL: `http://127.0.0.1:${port}`, + AGENTMEMORY_INJECT_CONTEXT: "true", + AGENTMEMORY_PROJECT_NAME: "", + ...env, + }, + }); + child.on("exit", (code) => resolve(code ?? 1)); + child.stdin.write(JSON.stringify(payload)); + child.stdin.end(); + }); +} + +describe("pre-tool-use enrich is project-scoped", () => { + let tmpRoot: string; + let repoDir: string; + let nestedDir: string; + + beforeAll(async () => { + tmpRoot = mkdtempSync(join(tmpdir(), "am-pretool-")); + repoDir = join(tmpRoot, REPO_NAME); + nestedDir = join(repoDir, "src", "deep"); + mkdirSync(nestedDir, { recursive: true }); + execFileSync("git", ["init", "--quiet"], { cwd: repoDir, stdio: "ignore" }); + + server = createServer((req, res) => { + let body = ""; + req.on("data", (c) => (body += c)); + req.on("end", () => { + try { + posts.push({ path: req.url ?? "", body: JSON.parse(body || "{}") }); + } catch { + posts.push({ path: req.url ?? "", body: {} }); + } + res.writeHead(200, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ context: "" })); + }); + }); + await new Promise((r) => server.listen(0, "127.0.0.1", r)); + port = (server.address() as { port: number }).port; + }); + + afterAll(() => { + server.close(); + rmSync(tmpRoot, { recursive: true, force: true }); + }); + + it("derives project from the payload cwd", async () => { + posts = []; + await runHook({ + session_id: "s1", + cwd: repoDir, + tool_name: "Read", + tool_input: { file_path: join(repoDir, "src", "thing.ts") }, + }); + + expect(posts.length).toBe(1); + expect(posts[0]!.path).toContain("/agentmemory/enrich"); + expect(posts[0]!.body.project).toBe(REPO_NAME); + }); + + it("resolves to the git toplevel from a nested cwd", async () => { + posts = []; + await runHook({ + session_id: "s2", + cwd: nestedDir, + tool_name: "Edit", + tool_input: { file_path: join(nestedDir, "x.ts") }, + }); + + expect(posts.length).toBe(1); + expect(posts[0]!.body.project).toBe(REPO_NAME); + }); + + it("never sends an unscoped enrich request", async () => { + posts = []; + await runHook({ + session_id: "s3", + cwd: repoDir, + tool_name: "Grep", + tool_input: { pattern: "handleError", path: repoDir }, + }); + + expect(posts.length).toBe(1); + const project = posts[0]!.body.project; + expect(project).toBeTruthy(); + expect(typeof project).toBe("string"); + }); + + it("an explicit payload project still wins (hosts that do supply one)", async () => { + posts = []; + await runHook({ + session_id: "s4", + cwd: repoDir, + project: "explicit-project", + tool_name: "Read", + tool_input: { file_path: join(repoDir, "y.ts") }, + }); + + expect(posts.length).toBe(1); + expect(posts[0]!.body.project).toBe("explicit-project"); + }); + + it("skips enrich when the resolved project is empty", async () => { + posts = []; + await runHook({ + session_id: "s6", + cwd: "/", + tool_name: "Read", + tool_input: { file_path: "/etc/hosts" }, + }); + + expect(posts.length).toBe(0); + }); + + it("stays a no-op when AGENTMEMORY_INJECT_CONTEXT is not true", async () => { + posts = []; + await runHook( + { + session_id: "s5", + cwd: repoDir, + tool_name: "Read", + tool_input: { file_path: join(repoDir, "z.ts") }, + }, + { AGENTMEMORY_INJECT_CONTEXT: "false" }, + ); + + expect(posts.length).toBe(0); + }); +}); diff --git a/test/proc-tcp-rows.test.ts b/test/proc-tcp-rows.test.ts new file mode 100644 index 0000000000..774bfcdead --- /dev/null +++ b/test/proc-tcp-rows.test.ts @@ -0,0 +1,8 @@ +import {it,expect,afterEach} from 'vitest';import {mkdtempSync,writeFileSync,rmSync} from 'node:fs';import {tmpdir} from 'node:os';import {join} from 'node:path';import {readOwnedTcpRows} from '../src/ops/proc-tcp-rows.mjs'; +const dirs:string[]=[];afterEach(()=>dirs.splice(0).forEach(d=>rmSync(d,{recursive:true,force:true}))); +function file(text:string){const d=mkdtempSync(join(tmpdir(),'proc-tcp-'));dirs.push(d);const p=join(d,'tcp');writeFileSync(p,text);return p;} +const row='0: 0100007F:03E8 0100007F:07D0 01 0 0 0 1000 0 501'; +it('reads owned rows across chunk boundaries and final unterminated line',()=>{const p=file(('unrelated\n').repeat(8000)+row+'\n'+('other\n').repeat(12000)+row);expect(readOwnedTcpRows(p,new Set(['501']))).toHaveLength(2);}); +it('bounds retained owned rows rather than total host table',()=>{expect(()=>readOwnedTcpRows(file((row+'\n').repeat(5)),new Set(['501']))).toThrow('PROC_TCP_OWNED_ROW_LIMIT');}); +it('rejects malformed oversized lines with bounded scratch space',()=>{expect(()=>readOwnedTcpRows(file('x'.repeat(70000)),new Set(['501']))).toThrow('PROC_TCP_LINE_LIMIT');}); +it('ignores unrelated established sockets but preserves duplicate owned rows for caller refusal',()=>{const p=file(row+'\n'+row.replace('501','999')+'\n'+row);expect(readOwnedTcpRows(p,new Set(['501']))).toHaveLength(2);}); diff --git a/test/provider-placeholder-keys.test.ts b/test/provider-placeholder-keys.test.ts new file mode 100644 index 0000000000..1066fd0a24 --- /dev/null +++ b/test/provider-placeholder-keys.test.ts @@ -0,0 +1,261 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { mkdtempSync, readFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +const fakeHome = mkdtempSync(join(tmpdir(), "agentmemory-placeholder-")); + +vi.mock("node:os", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, homedir: () => fakeHome }; +}); + +const PROVIDER_ENV = /(API_KEY|_KEY$|_TOKEN$|PROVIDER|_BASE_URL$|^AGENTMEMORY_ALLOW_AGENT_SDK$|^OLLAMA)/; + +describe("placeholder provider keys", () => { + const saved: Record = {}; + + beforeEach(() => { + vi.resetModules(); + vi.spyOn(process.stderr, "write").mockImplementation(() => true); + for (const key of Object.keys(process.env)) { + if (PROVIDER_ENV.test(key)) { + saved[key] = process.env[key]; + delete process.env[key]; + } + } + }); + + afterEach(() => { + for (const key of Object.keys(process.env)) { + if (PROVIDER_ENV.test(key)) delete process.env[key]; + } + for (const [key, value] of Object.entries(saved)) { + if (value !== undefined) process.env[key] = value; + } + vi.restoreAllMocks(); + vi.unstubAllGlobals(); + }); + + it.each([ + ["anthropic", "ANTHROPIC_API_KEY", "sk-ant-..."], + ["openai", "OPENAI_API_KEY", "sk-..."], + ["openrouter", "OPENROUTER_API_KEY", "changeme"], + ["minimax", "MINIMAX_API_KEY", "your-key-here"], + ["gemini", "GEMINI_API_KEY", "xxxx-xxxx"], + ["gemini", "GOOGLE_API_KEY", "your-key-here"], + ["openrouter", "OPENROUTER_API_KEY", "CHANGEME"], + ["anthropic", "ANTHROPIC_API_KEY", "todo"], + ["gemini", "GEMINI_API_KEY", "XXXX-XXXX"], + ["openrouter", "OPENROUTER_API_KEY", "sk-or-v1-..."], + ["anthropic", "ANTHROPIC_API_KEY", " "], + ["anthropic", "ANTHROPIC_API_KEY", "sk-ant-... "], + ])("runs zero-LLM when %s is chosen but %s=%s", async (provider, key, value) => { + process.env["AGENTMEMORY_LLM_PROVIDER"] = provider; + process.env[key] = value; + const { loadConfig, detectLlmProviderKind, llmOptInHint } = await import("../src/config.js"); + + expect(loadConfig().provider.provider).toBe("noop"); + expect(detectLlmProviderKind()).toBe("noop"); + expect(llmOptInHint()).toContain(`AGENTMEMORY_LLM_PROVIDER=${provider} is set, but`); + }); + + it("does not offer a provider whose only key is a placeholder", async () => { + process.env["ANTHROPIC_API_KEY"] = "sk-ant-..."; + process.env["GEMINI_API_KEY"] = "your-key-here"; + process.env["GOOGLE_API_KEY"] = "AIzaRealKey123"; + const { llmOptInHint } = await import("../src/config.js"); + + const hint = llmOptInHint(); + expect(hint).toContain("Found GOOGLE_API_KEY,"); + expect(hint).toContain("AGENTMEMORY_LLM_PROVIDER=gemini"); + expect(hint).not.toContain("ANTHROPIC_API_KEY"); + expect(hint).not.toContain("GEMINI_API_KEY"); + + delete process.env["GOOGLE_API_KEY"]; + expect(llmOptInHint()).toBeNull(); + }); + + it("rejects every sample value in .env.example", async () => { + const { isRealProviderKey } = await import("../src/config.js"); + const example = readFileSync(join(import.meta.dirname, "..", ".env.example"), "utf-8"); + const samples = [...example.matchAll(/^#\s*([A-Z_]+_API_KEY)=(\S+)/gm)]; + + expect(samples.length).toBeGreaterThan(0); + for (const [, key, value] of samples) { + expect(isRealProviderKey({ [key]: value }, key), `${key}=${value}`).toBe(false); + } + }); + + it("keeps a dummy OpenAI key for a plain-http OPENAI_BASE_URL", async () => { + process.env["AGENTMEMORY_LLM_PROVIDER"] = "openai"; + process.env["OPENAI_API_KEY"] = "xxx"; + process.env["OPENAI_BASE_URL"] = "http://localhost:11434/v1"; + const fetchMock = vi.fn(async () => + new Response(JSON.stringify({ choices: [{ message: { content: "ok" } }] }), { + status: 200, + headers: { "Content-Type": "application/json" }, + }), + ); + vi.stubGlobal("fetch", fetchMock); + const { loadConfig, detectLlmProviderKind } = await import("../src/config.js"); + const { createProvider } = await import("../src/providers/index.js"); + const { realProviderKeys, placeholderProviderKeys } = await import( + "../src/cli/doctor-diagnostics.js" + ); + + const config = loadConfig(); + expect(config.provider.provider).toBe("openai"); + expect(detectLlmProviderKind()).toBe("llm"); + await createProvider(config.provider).compress("system", "user"); + const [url, init] = fetchMock.mock.calls[0] as unknown as [string, RequestInit]; + expect(String(url)).toContain("localhost:11434"); + expect((init.headers as Record)["Authorization"]).toBe("Bearer xxx"); + + const env = { OPENAI_API_KEY: "xxx", OPENAI_BASE_URL: "http://localhost:11434/v1" }; + expect(realProviderKeys(env)).toEqual(["OPENAI_API_KEY"]); + expect(placeholderProviderKeys(env)).toEqual([]); + }); + + it.each([undefined, "", " "])("does not count OPENAI_API_KEY=%j behind a plain-http OPENAI_BASE_URL", async (key) => { + process.env["AGENTMEMORY_LLM_PROVIDER"] = "openai"; + process.env["OPENAI_BASE_URL"] = "http://localhost:11434/v1"; + if (key !== undefined) process.env["OPENAI_API_KEY"] = key; + const { loadConfig, detectLlmProviderKind, llmOptInHint } = await import("../src/config.js"); + const { realProviderKeys } = await import("../src/cli/doctor-diagnostics.js"); + + expect(loadConfig().provider.provider).toBe("noop"); + expect(detectLlmProviderKind()).toBe("noop"); + expect(llmOptInHint()).toContain("OPENAI_API_KEY is not"); + expect(realProviderKeys({ OPENAI_API_KEY: key ?? "", OPENAI_BASE_URL: "http://localhost:11434/v1" })).toEqual([]); + }); + + it("keeps a dummy OpenAI key for a plain-http server on another host", async () => { + process.env["AGENTMEMORY_LLM_PROVIDER"] = "openai"; + process.env["OPENAI_API_KEY"] = "xxx"; + process.env["OPENAI_BASE_URL"] = "http://vllm.lan:8000/v1"; + const { loadConfig, detectLlmProviderKind } = await import("../src/config.js"); + const { realProviderKeys } = await import("../src/cli/doctor-diagnostics.js"); + + expect(loadConfig().provider.provider).toBe("openai"); + expect(detectLlmProviderKind()).toBe("llm"); + expect( + realProviderKeys({ OPENAI_API_KEY: "xxx", OPENAI_BASE_URL: "http://vllm.lan:8000/v1" }), + ).toEqual(["OPENAI_API_KEY"]); + }); + + it("keeps a sample OpenAI key for a plain-http OPENAI_BASE_URL", async () => { + process.env["AGENTMEMORY_LLM_PROVIDER"] = "openai"; + process.env["OPENAI_API_KEY"] = "sk-..."; + process.env["OPENAI_BASE_URL"] = "http://localhost:11434/v1"; + const { loadConfig, detectLlmProviderKind } = await import("../src/config.js"); + const { realProviderKeys } = await import("../src/cli/doctor-diagnostics.js"); + + expect(loadConfig().provider.provider).toBe("openai"); + expect(detectLlmProviderKind()).toBe("llm"); + expect( + realProviderKeys({ OPENAI_API_KEY: "sk-...", OPENAI_BASE_URL: "http://localhost:11434/v1" }), + ).toEqual(["OPENAI_API_KEY"]); + }); + + it("does not keep a placeholder OpenAI key for an https OPENAI_BASE_URL", async () => { + process.env["AGENTMEMORY_LLM_PROVIDER"] = "openai"; + process.env["OPENAI_API_KEY"] = "sk-..."; + process.env["OPENAI_BASE_URL"] = "https://api.openai.com"; + const { loadConfig, detectLlmProviderKind } = await import("../src/config.js"); + const { placeholderProviderKeys } = await import("../src/cli/doctor-diagnostics.js"); + + expect(loadConfig().provider.provider).toBe("noop"); + expect(detectLlmProviderKind()).toBe("noop"); + expect( + placeholderProviderKeys({ OPENAI_API_KEY: "sk-...", OPENAI_BASE_URL: "https://api.openai.com" }), + ).toEqual(["OPENAI_API_KEY"]); + }); + + it("keeps OPENAI_BASE_URL from excusing other placeholder keys", async () => { + process.env["AGENTMEMORY_LLM_PROVIDER"] = "anthropic"; + process.env["ANTHROPIC_API_KEY"] = "sk-ant-..."; + process.env["OPENAI_BASE_URL"] = "http://localhost:11434/v1"; + const { loadConfig, detectLlmProviderKind } = await import("../src/config.js"); + const { placeholderProviderKeys } = await import("../src/cli/doctor-diagnostics.js"); + + expect(loadConfig().provider.provider).toBe("noop"); + expect(detectLlmProviderKind()).toBe("noop"); + expect( + placeholderProviderKeys({ ANTHROPIC_API_KEY: "sk-ant-...", OPENAI_BASE_URL: "http://localhost:11434/v1" }), + ).toEqual(["ANTHROPIC_API_KEY"]); + }); + + it.each([ + ["anthropic", "ANTHROPIC_API_KEY", "sk-ant-..."], + ["minimax", "MINIMAX_API_KEY", "changeme"], + ["openrouter", "OPENROUTER_API_KEY", "sk-or-..."], + ["openai", "OPENAI_API_KEY", "sk-..."], + ] as const)("does not build the %s provider from a placeholder %s", async (provider, key, value) => { + process.env[key] = value; + const { createProvider } = await import("../src/providers/index.js"); + + expect(() => createProvider({ provider, model: "m", maxTokens: 1 })).toThrow(key); + }); + + it.each([ + ["anthropic", "ANTHROPIC_API_KEY", "sk-ant-api03-abc123"], + ["openai", "OPENAI_API_KEY", "xai-abc123def"], + ] as const)("still uses the chosen %s provider with a real %s", async (provider, key, value) => { + process.env["AGENTMEMORY_LLM_PROVIDER"] = provider; + process.env[key] = value; + const { loadConfig } = await import("../src/config.js"); + + expect(loadConfig().provider.provider).toBe(provider); + }); + + it("sends GEMINI_API_KEY without a warning when both Gemini keys are real", async () => { + process.env["AGENTMEMORY_LLM_PROVIDER"] = "gemini"; + process.env["GEMINI_API_KEY"] = "AIzaGeminiKey456"; + process.env["GOOGLE_API_KEY"] = "AIzaRealKey123"; + const fetchMock = vi.fn(async () => + new Response(JSON.stringify({ choices: [{ message: { content: "ok" } }] }), { + status: 200, + headers: { "Content-Type": "application/json" }, + }), + ); + vi.stubGlobal("fetch", fetchMock); + const { loadConfig } = await import("../src/config.js"); + const { createProvider } = await import("../src/providers/index.js"); + + const config = loadConfig(); + expect(config.provider.provider).toBe("gemini"); + await createProvider(config.provider).compress("system", "user"); + + const init = fetchMock.mock.calls[0][1] as RequestInit; + expect((init.headers as Record)["Authorization"]).toBe("Bearer AIzaGeminiKey456"); + expect(process.stderr.write).not.toHaveBeenCalledWith( + expect.stringContaining("GOOGLE_API_KEY detected"), + ); + }); + + it("sends GOOGLE_API_KEY when GEMINI_API_KEY is a placeholder", async () => { + process.env["AGENTMEMORY_LLM_PROVIDER"] = "gemini"; + process.env["GEMINI_API_KEY"] = "your-key-here"; + process.env["GOOGLE_API_KEY"] = "AIzaRealKey123"; + const fetchMock = vi.fn(async () => + new Response(JSON.stringify({ choices: [{ message: { content: "ok" } }] }), { + status: 200, + headers: { "Content-Type": "application/json" }, + }), + ); + vi.stubGlobal("fetch", fetchMock); + const { loadConfig } = await import("../src/config.js"); + const { createProvider } = await import("../src/providers/index.js"); + + const config = loadConfig(); + expect(config.provider.provider).toBe("gemini"); + await createProvider(config.provider).compress("system", "user"); + + const init = fetchMock.mock.calls[0][1] as RequestInit; + expect((init.headers as Record)["Authorization"]).toBe("Bearer AIzaRealKey123"); + expect(process.stderr.write).toHaveBeenCalledWith( + expect.stringContaining("GOOGLE_API_KEY detected"), + ); + }); +}); diff --git a/test/redis-state-backend.test.ts b/test/redis-state-backend.test.ts index 4e83bcabd7..f334de7fcd 100644 --- a/test/redis-state-backend.test.ts +++ b/test/redis-state-backend.test.ts @@ -75,6 +75,10 @@ function fakeRedisEngine() { events.push({ type: "deleted", scope: p.scope, key: p.key, old, next: null }); return old; } + case "state::list_keys": { + const h = hashes.get(`state:${p.scope}`); + return { keys: h ? shuffled([...h.keys()]) : [] }; + } case "state::list": { const h = hashes.get(`state:${p.scope}`); return h ? shuffled([...h.values()].map((raw) => JSON.parse(raw))) : []; diff --git a/test/reflection-bounds.test.ts b/test/reflection-bounds.test.ts new file mode 100644 index 0000000000..1aab3a9a7d --- /dev/null +++ b/test/reflection-bounds.test.ts @@ -0,0 +1,279 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +vi.mock("../src/logger.js", () => ({ logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() } })); + +import { registerReflectFunctions, INSIGHT_MAX_RESULTS, INSIGHT_MAX_SOURCES, REFLECT_MAX_CONCEPTS, REFLECT_MAX_RESPONSE_BYTES } from "../src/functions/reflect.js"; +import { REFLECT_MAX_PROMPT_BYTES, REFLECT_SYSTEM, buildReflectPrompt } from "../src/prompts/reflect.js"; +import { registerApiTriggers } from "../src/triggers/api.js"; +import { registerMcpEndpoints } from "../src/mcp/server.js"; +import { SAFE_PAYLOAD_BYTES, payloadByteLength } from "../src/state/frame-guard.js"; +import { KV } from "../src/state/schema.js"; +import { initializeGraphIndex, reconcileGraphIndexRecord } from "../src/state/graph-retrieval-index.js"; +import { indexObservationSession } from "../src/state/obs-index.js"; + +const now = "2026-10-13T00:00:00Z"; +const headers = { authorization: "Bearer reflection-test-secret" }; +const xml = (sources = "F1,F2", content = "Validate security boundaries.") => + `${content}`; + +type Row = Record & { id: string; sourceObservationIds?: string[]; tags?: string[] }; +type Result = Record; + +function fixture() { + const store = new Map>(); + const kv = { + get: vi.fn(async (scope: string, id: string) => store.get(scope)?.get(id) ?? null), + list: vi.fn(async (scope: string) => [...(store.get(scope)?.values() ?? [])]), + set: async (scope: string, id: string, value: Row): Promise => { + if (!store.has(scope)) store.set(scope, new Map()); + store.get(scope)!.set(id, value); + if (scope === KV.graphNodes || scope === KV.graphEdges) { + for (const obsId of value.sourceObservationIds ?? []) { + await indexObservationSession(kv as never, obsId, "session"); + await kv.set(KV.observations("session"), obsId, { id: obsId, sessionId: "session", title: "source" }); + } + await reconcileGraphIndexRecord(kv as never, scope === KV.graphNodes ? "node" : "edge", id, value as never); + } + return value; + }, + delete: async (scope: string, id: string) => { store.get(scope)?.delete(id); }, + }; + const fns = new Map(); + const sdk = { + registerFunction: (id: string | { id: string }, fn: Function) => { fns.set(typeof id === "string" ? id : id.id, fn); }, + registerTrigger: () => {}, + trigger: async ({ function_id, payload }: { function_id: string; payload?: unknown }): Promise => fns.get(function_id)!(payload), + }; + const provider = { name: "test", compress: vi.fn(), summarize: vi.fn().mockResolvedValue(xml()) }; + registerReflectFunctions(sdk as never, kv as never, provider as never); + registerApiTriggers(sdk as never, kv as never, "reflection-test-secret"); + registerMcpEndpoints(sdk as never, kv as never, "reflection-test-secret"); + const call = (function_id: string, payload: unknown = {}) => sdk.trigger({ function_id, payload }); + const mcp = (name: string, args: unknown) => call("mcp::tools::call", { headers, body: { name, arguments: args } }); + const fact = (id: string, text = `security validation evidence ${id}`) => kv.set(KV.semantic, id, { id, fact: text, confidence: 0.8 }); + const insight = (id: string, content = "security insight", extra = {}) => kv.set(KV.insights, id, { + id, title: "Security", content, confidence: 0.8, reinforcements: 0, sourceConceptCluster: [], + sourceMemoryIds: [], sourceLessonIds: [], sourceCrystalIds: [], tags: [], createdAt: now, updatedAt: now, decayRate: 0.05, ...extra, + }); + const node = (id: string, name = id) => + kv.set(KV.graphNodes, id, { id, name, type: "concept", properties: {}, sourceObservationIds: ["obs-graph"], createdAt: now }); + const edge = (id: string, sourceNodeId: string, targetNodeId: string) => + kv.set(KV.graphEdges, id, { id, type: "related_to", sourceNodeId, targetNodeId, weight: 1, sourceObservationIds: ["obs-graph"], createdAt: now }); + async function seed(count = 3) { + await initializeGraphIndex(kv as never, { status: "ready" }); + for (const name of ["security", "validation"]) await node(name); + await edge("edge", "security", "validation"); + for (let i = 0; i < count; i++) await fact(`fact-${i}`); + } + return { kv, call, mcp, provider, fact, insight, seed, node, edge }; +} + +describe("reflection budgets", () => { + let f: ReturnType; + beforeEach(() => { f = fixture(); }); + + it("skips oversized evidence without relabeling it as a different source, and retains mixed evidence", async () => { + await f.seed(0); + await f.fact("huge", `security ${'界"\\\n'.repeat(3000)}`); + await f.fact("usable-a"); + await f.fact("usable-b"); + await f.kv.set(KV.lessons, "lesson", { id: "lesson", content: "security lesson", confidence: 0.8, tags: ["security"] }); + await f.kv.set(KV.crystals, "crystal", { id: "crystal", narrative: "security outcome", lessons: ["security"] }); + f.provider.summarize.mockResolvedValue(xml("F1,L1,C1")); + expect(await f.call("mem::reflect")).toMatchObject({ newInsights: 1 }); + const [result] = await f.kv.list(KV.insights); + expect(result).toMatchObject({ sourceMemoryIds: ["usable-a"], sourceLessonIds: ["lesson"], sourceCrystalIds: ["crystal"] }); + const prompt = f.provider.summarize.mock.calls[0][1]; + expect(prompt).not.toContain("界"); + expect(prompt).toContain('"source":"F1","fact":"security validation evidence usable-a"'); + expect(Buffer.byteLength(REFLECT_SYSTEM + prompt)).toBeLessThanOrEqual(REFLECT_MAX_PROMPT_BYTES); + }); + + it("keeps crystals from other projects out of a project-scoped reflection", async () => { + await f.seed(3); + await f.kv.set(KV.lessons, "lesson", { id: "lesson", content: "security lesson", confidence: 0.8, tags: ["security"], project: "a" }); + await f.kv.set(KV.crystals, "own", { id: "own", narrative: "security outcome own", lessons: ["security"], project: "a" }); + await f.kv.set(KV.crystals, "foreign", { id: "foreign", narrative: "security outcome foreign", lessons: ["security"], project: "b" }); + await f.call("mem::reflect", { project: "a" }); + const prompts = f.provider.summarize.mock.calls.map((call) => String(call[1])); + expect(prompts.length).toBeGreaterThan(0); + expect(prompts.join("\n")).toContain("security outcome own"); + expect(prompts.join("\n")).not.toContain("security outcome foreign"); + }); + + it("bounds all evidence types by serialized UTF-8 bytes while keeping later small records", async () => { + await f.seed(0); + for (let i = 0; i < 50; i++) { + const text = `security ${'界"\\\n'.repeat(i < 10 ? 350 : 1)} ${i}`; + await f.fact(`fact-${i}`, text); + await f.kv.set(KV.lessons, `lesson-${i}`, { id: `lesson-${i}`, content: text, confidence: 0.8, tags: ["security"] }); + await f.kv.set(KV.crystals, `crystal-${i}`, { id: `crystal-${i}`, narrative: text, lessons: ["security"] }); + } + f.provider.summarize.mockResolvedValue(xml("F1,L1,C1")); + expect(await f.call("mem::reflect")).toMatchObject({ newInsights: 1 }); + const prompt = f.provider.summarize.mock.calls[0][1]; + expect(Buffer.byteLength(REFLECT_SYSTEM + prompt)).toBeLessThanOrEqual(REFLECT_MAX_PROMPT_BYTES); + for (const type of ["F", "L", "C"]) { + const labels = [...prompt.matchAll(new RegExp(`"source":"${type}(\\d+)"`, "g"))]; + expect(labels.length).toBeGreaterThan(4); + expect(labels.length).toBeLessThanOrEqual(INSIGHT_MAX_SOURCES); + expect(labels.map((m) => Number(m[1]))).toEqual(labels.map((_, i) => i + 1)); + } + }); + + it("keeps at most twenty distinct citations and rejects discarded labels on reinforcement", async () => { + await f.seed(50); + f.provider.summarize.mockResolvedValue(xml(Array.from({ length: 20 }, (_, i) => `F${i + 1}`).join(","))); + expect(await f.call("mem::reflect")).toMatchObject({ newInsights: 1 }); + const [old] = await f.kv.list(KV.insights); + expect(old.sourceMemoryIds).toHaveLength(20); + await f.kv.set(KV.insights, old.id, { ...old, sourceMemoryIds: Array(1000).fill("legacy"), tags: Array(1000).fill("legacy"), sourceConceptCluster: Array(1000).fill("legacy") }); + f.provider.summarize.mockResolvedValue(xml("F1,F2,F1")); + expect(await f.call("mem::reflect")).toMatchObject({ reinforced: 1 }); + const refreshed = (await f.kv.get(KV.insights, old.id))!; + expect(refreshed.sourceMemoryIds).toEqual(["fact-0", "fact-1"]); + expect([...refreshed.tags!].sort()).toEqual(["security", "validation"]); + expect(refreshed.sourceConceptCluster).toEqual(refreshed.tags); + f.provider.summarize.mockResolvedValue(xml("F1,F21")); + expect(await f.call("mem::reflect")).toMatchObject({ reinforced: 0, rejectedInsights: 1 }); + expect(await f.kv.get(KV.insights, old.id)).toEqual(refreshed); + }); + + it("rejects more than twenty valid citations across evidence types before evidence reads", async () => { + await f.seed(20); + await f.kv.set(KV.lessons, "lesson", { id: "lesson", content: "security lesson", confidence: 0.8, tags: ["security"] }); + f.provider.summarize.mockResolvedValue(xml(`${Array.from({ length: 20 }, (_, i) => `F${i + 1}`).join(",")},L1`)); + expect(await f.call("mem::reflect")).toMatchObject({ newInsights: 0, rejectedInsights: 1 }); + expect(f.kv.get.mock.calls.filter(([scope]) => scope === KV.semantic || scope === KV.lessons)).toHaveLength(0); + }); + + it("retains the original full text for mutation checks after evidence selection", async () => { + await f.seed(0); + await f.fact("oversized", `security ${"x".repeat(5000)}`); + for (const id of ["a", "b", "c"]) await f.fact(id, `security original ${id} "\\\n界`); + f.provider.summarize.mockImplementation(async () => { + await f.fact("a", "security replacement"); + return xml(); + }); + expect(await f.call("mem::reflect")).toMatchObject({ newInsights: 0, rejectedInsights: 1 }); + }); + + it("bounds stored concepts without truncating names or following oversized model output", async () => { + await f.seed(); + for (let i = 0; i < 60; i++) { + const id = `node-${i}`; + await f.node(id, i === 0 ? "界".repeat(1000) : id); + await f.edge(id, "security", id); + } + expect(await f.call("mem::reflect")).toMatchObject({ newInsights: 1 }); + const [insight] = await f.kv.list(KV.insights); + expect(insight.tags).toHaveLength(REFLECT_MAX_CONCEPTS); + expect(insight.tags).not.toContain("界"); + f.provider.summarize.mockResolvedValue(xml("F1,F2", "x".repeat(REFLECT_MAX_RESPONSE_BYTES))); + expect(await f.call("mem::reflect")).toMatchObject({ newInsights: 0, reinforced: 0, failedClusters: 1 }); + }); + + it("rejects excessive generated title/content and independently guards the complete prompt", async () => { + await f.seed(); + f.provider.summarize.mockResolvedValue(xml("F1,F2", "界".repeat(2000))); + expect(await f.call("mem::reflect")).toMatchObject({ newInsights: 0, rejectedInsights: 1 }); + f.provider.summarize.mockResolvedValue(xml().replace("Boundary checks", "界".repeat(100))); + expect(await f.call("mem::reflect")).toMatchObject({ newInsights: 0, rejectedInsights: 1 }); + expect(() => buildReflectPrompt({ concepts: [], facts: [{ fact: "界".repeat(REFLECT_MAX_PROMPT_BYTES), confidence: 1 }], lessons: [], crystalNarratives: [] })).toThrow("byte budget"); + }); +}); + +const invalidNumbers = [0, -1, 1.5, NaN, Infinity, -Infinity, "2", null, true, Number.MAX_SAFE_INTEGER + 1]; + +describe("insight boundaries", () => { + it.each(invalidNumbers)("rejects invalid count %s on all inner functions before reading", async (limit) => { + const f = fixture(); + for (const functionId of ["mem::insight-list", "mem::insight-search", "mem::reflect"]) { + expect(await f.call(functionId, { limit, maxClusters: limit, query: "security" })).toMatchObject({ success: false, invalid: true }); + } + expect(f.kv.list).not.toHaveBeenCalled(); + expect(f.provider.summarize).not.toHaveBeenCalled(); + }); + + it.each([-1, 1.1, NaN, Infinity, "0.5", null])("rejects invalid confidence %s", async (minConfidence) => { + const f = fixture(); + for (const id of ["mem::insight-list", "mem::insight-search"]) { + expect(await f.call(id, { minConfidence, query: "security" })).toMatchObject({ invalid: true }); + } + expect(f.kv.list).not.toHaveBeenCalled(); + }); + + it("caps large limits and preserves defaults, filtering, and both boundary confidence values", async () => { + const f = fixture(); + for (let i = 0; i < 510; i++) await f.insight(`ins-${i}`, "security insight", { project: "a", confidence: 1 }); + await f.insight("foreign", "security insight", { project: "b" }); + expect((await f.call("mem::insight-list")).insights).toHaveLength(50); + expect((await f.call("mem::insight-search", { query: "security" })).insights).toHaveLength(10); + for (const id of ["mem::insight-list", "mem::insight-search"]) { + const result = await f.call(id, { query: "security", limit: 30000, project: "a", minConfidence: 1 }); + expect(result.insights).toHaveLength(INSIGHT_MAX_RESULTS); + expect((result.insights as Row[]).every((i) => i.project === "a")).toBe(true); + expect((await f.call(id, { query: "security", minConfidence: 0, limit: 1 })).insights).toHaveLength(1); + } + }); + + it("refuses oversized legacy records inside the function before either transport wrapper", async () => { + const f = fixture(); + await f.insight("huge", `security ${"界".repeat(Math.ceil(SAFE_PAYLOAD_BYTES / 3))}`); + for (const id of ["mem::insight-list", "mem::insight-search"]) { + const result = await f.call(id, { query: "security", limit: 1 }); + expect(result).toMatchObject({ success: false, oversized: true }); + expect(payloadByteLength(result)).toBeLessThan(1024); + } + for (const [id, request] of [["api::insight-list", { query_params: { limit: "1" } }], ["api::insight-search", { body: { query: "security", limit: 1 } }]] as const) { + const result = await f.call(id, { headers, ...request }); + expect(result).toMatchObject({ status_code: 413, body: { oversized: true } }); + } + expect(await f.mcp("memory_insight_list", { limit: 1 })).toMatchObject({ status_code: 413, body: { isError: true } }); + }); + + it("protects the final MCP envelope when escaping expands a valid inner result", async () => { + const f = fixture(); + await f.insight("escaped", '"'.repeat(Math.floor(SAFE_PAYLOAD_BYTES / 3))); + const inner = await f.call("mem::insight-list", { limit: 1 }); + expect(inner.success).toBe(true); + expect(payloadByteLength(inner)).toBeLessThan(SAFE_PAYLOAD_BYTES); + const rest = await f.call("api::insight-list", { headers, query_params: { limit: "1" } }); + expect(rest.status_code).toBe(200); + const mcp = await f.mcp("memory_insight_list", { limit: 1 }); + expect(mcp).toMatchObject({ status_code: 413, body: { isError: true } }); + expect(payloadByteLength(mcp)).toBeLessThan(2048); + }, 60_000); + + it("preserves authorization and returns small validation errors through REST and MCP", async () => { + const f = fixture(); + expect(await f.call("api::insight-list", { headers: {}, query_params: {} })).toMatchObject({ status_code: 401 }); + expect(await f.call("api::reflect", { headers, body: { maxClusters: "wrong" } })).toMatchObject({ status_code: 400 }); + expect(await f.call("api::insight-search", { headers, body: { query: "security", limit: "wrong" } })).toMatchObject({ status_code: 400 }); + expect(await f.call("api::insight-list", { headers, query_params: { minConfidence: "2" } })).toMatchObject({ status_code: 400 }); + expect(await f.mcp("memory_insight_list", { limit: -1 })).toMatchObject({ status_code: 400, body: { isError: true } }); + expect(await f.mcp("memory_reflect", { maxClusters: 1.5 })).toMatchObject({ status_code: 400 }); + expect(await f.call("mem::insight-search", { query: 42 })).toMatchObject({ invalid: true }); + }); +}); + +describe("custom reflection lifecycle compatibility", () => { + it("validates bounds before preserving the conservative lifecycle dispatch", async () => { + const { operationalState } = await import("../src/ops/context.js"); + const previous = operationalState.featureLifecycle; + const calls: unknown[] = []; + operationalState.featureLifecycle = { reflect: async (data: unknown) => { + calls.push(data); + return { success: true, reflectionMode: "evidence-linked-extractive" }; + } } as never; + try { + const f = fixture(); + expect(await f.call("mem::reflect", { project: "owned", maxClusters: 0 })).toMatchObject({ invalid: true }); + expect(calls).toHaveLength(0); + expect(await f.call("mem::reflect", { project: "owned", maxClusters: 500 })).toMatchObject({ reflectionMode: "evidence-linked-extractive" }); + expect(calls).toEqual([{ project: "owned", maxClusters: 20 }]); + } finally { + operationalState.featureLifecycle = previous; + } + }); +}); diff --git a/test/registration-defaults.test.ts b/test/registration-defaults.test.ts new file mode 100644 index 0000000000..7dedd00b78 --- /dev/null +++ b/test/registration-defaults.test.ts @@ -0,0 +1,68 @@ +import { describe, it, expect, afterEach, beforeEach } from "vitest"; +import { withRegistrationDefaults } from "../src/runtime/registration.js"; + +function fakeSdk() { + const calls: Array<{ id: string; handler: unknown; options: unknown }> = []; + return { + calls, + counter: 7, + registerFunction(id: string, handler: unknown, options?: unknown) { + calls.push({ id, handler, options }); + return id; + }, + trigger(input: { function_id: string }) { + return Promise.resolve({ echoed: input.function_id, counter: this.counter }); + }, + }; +} + +const originalLocalDispatch = process.env.AGENTMEMORY_LOCAL_DISPATCH; + +afterEach(() => { + if (originalLocalDispatch === undefined) delete process.env.AGENTMEMORY_LOCAL_DISPATCH; + else process.env.AGENTMEMORY_LOCAL_DISPATCH = originalLocalDispatch; +}); + +describe("withRegistrationDefaults", () => { + beforeEach(() => { + delete process.env.AGENTMEMORY_LOCAL_DISPATCH; + }); + + it("passes registerFunction options through unchanged when the env is unset", () => { + const sdk = fakeSdk(); + const wrapped = withRegistrationDefaults(sdk); + const handler = () => "ok"; + wrapped.registerFunction("mem::a", handler); + wrapped.registerFunction("mem::b", handler, { observe: { onStart() {} } }); + expect(sdk.calls[0].options).toBeUndefined(); + expect(sdk.calls[1].options).toEqual({ observe: { onStart: expect.any(Function) } }); + expect(sdk.calls[1].options).not.toHaveProperty("localDispatch"); + }); + + it("merges localDispatch: false when AGENTMEMORY_LOCAL_DISPATCH is false", () => { + process.env.AGENTMEMORY_LOCAL_DISPATCH = "false"; + const sdk = fakeSdk(); + const wrapped = withRegistrationDefaults(sdk); + wrapped.registerFunction("mem::a", () => {}, { observe: { tag: "t" }, invocationTimeoutMs: 5000 }); + expect(sdk.calls[0].options).toEqual({ observe: { tag: "t" }, invocationTimeoutMs: 5000, localDispatch: false }); + wrapped.registerFunction("mem::b", () => {}); + expect(sdk.calls[1].options).toEqual({ localDispatch: false }); + }); + + it("honors an explicit localDispatch override request only through the env flag", () => { + process.env.AGENTMEMORY_LOCAL_DISPATCH = "false"; + const sdk = fakeSdk(); + const wrapped = withRegistrationDefaults(sdk); + wrapped.registerFunction("mem::a", () => {}, { localDispatch: true }); + expect(sdk.calls[0].options).toEqual({ localDispatch: false }); + }); + + it("keeps non-registerFunction methods bound to the target", async () => { + const sdk = fakeSdk(); + const wrapped = withRegistrationDefaults(sdk); + await expect(wrapped.trigger({ function_id: "x::y" })).resolves.toEqual({ echoed: "x::y", counter: 7 }); + process.env.AGENTMEMORY_LOCAL_DISPATCH = "false"; + await expect(wrapped.trigger({ function_id: "x::z" })).resolves.toEqual({ echoed: "x::z", counter: 7 }); + expect(wrapped.counter).toBe(7); + }); +}); diff --git a/test/remember-lock-scope.test.ts b/test/remember-lock-scope.test.ts index 0d2f5c4926..5ad6b3eaa3 100644 --- a/test/remember-lock-scope.test.ts +++ b/test/remember-lock-scope.test.ts @@ -16,6 +16,7 @@ import { import { logger } from "../src/logger.js"; import { VectorIndex } from "../src/state/vector-index.js"; import { KV } from "../src/state/schema.js"; +import { withKeyedLock } from "../src/state/keyed-mutex.js"; import type { EmbeddingProvider, Memory } from "../src/types.js"; function mockKV() { @@ -256,4 +257,20 @@ describe("mem::remember lock scope", () => { expect.anything(), ); }); + + it("waits for the mem:remember lock before deleting a memory", async () => { + const sdk = mockSdk(); + const kv = mockKV(); + registerRememberFunction(sdk as never, kv as never); + await kv.set(KV.memories, "held", { id: "held", title: "t", content: "c", isLatest: true } as unknown as Memory); + let release!: () => void; + const held = withKeyedLock("mem:remember", () => new Promise((resolve) => { release = resolve; })); + const forget = sdk.trigger({ function_id: "mem::forget", payload: { memoryId: "held" } }); + await delay(20); + expect(await kv.get(KV.memories, "held")).not.toBeNull(); + release(); + await held; + await forget; + expect(await kv.get(KV.memories, "held")).toBeNull(); + }); }); diff --git a/test/remember-supersede-recall.test.ts b/test/remember-supersede-recall.test.ts index 4db26fc309..fdb30e2558 100644 --- a/test/remember-supersede-recall.test.ts +++ b/test/remember-supersede-recall.test.ts @@ -40,6 +40,17 @@ describe("mem::remember supersession and recall hygiene", () => { expect(idx.has(second.memory.id)).toBe(true); }); + it("counts each superseding save as a recurrence", async () => { + const { sdk } = await setup(); + const content = "always run the migration dry run before applying schema changes"; + const first = (await sdk.trigger("mem::remember", { content, type: "pattern" })) as { memory: { recurrence?: number } }; + const second = (await sdk.trigger("mem::remember", { content: `${content} now`, type: "pattern" })) as { memory: { recurrence?: number } }; + const third = (await sdk.trigger("mem::remember", { content: `${content} now ok`, type: "pattern" })) as { memory: { recurrence?: number } }; + expect(first.memory.recurrence).toBeUndefined(); + expect(second.memory.recurrence).toBe(2); + expect(third.memory.recurrence).toBe(3); + }); + it("reports a close-but-below-threshold match as similarTo without superseding", async () => { const { sdk } = await setup(); const first = (await sdk.trigger("mem::remember", { diff --git a/test/resilient-content-filter.test.ts b/test/resilient-content-filter.test.ts new file mode 100644 index 0000000000..0c6026a18c --- /dev/null +++ b/test/resilient-content-filter.test.ts @@ -0,0 +1,106 @@ +import { describe, expect, it } from "vitest"; +import { ResilientProvider, isPayloadRejection } from "../src/providers/resilient.js"; +import type { MemoryProvider } from "../src/types.js"; + +const CONTENT_FILTER_ERROR = new Error( + 'OpenAI API error (400): {"error":{"message":"The response was filtered due to ' + + 'the prompt triggering Azure OpenAI\'s content management policy.",' + + '"code":"content_filter","status":400,"innererror":{"code":"ResponsibleAIPolicyViolation",' + + '"content_filter_result":{"jailbreak":{"detected":true,"filtered":true}}}}}', +); + +const GATEWAY_ECHO_ERROR = new Error( + 'OpenAI API error (502): {"error":{"message":"upstream returned an error: ' + + '{\\"code\\":\\"content_filter\\",\\"innererror\\":' + + '{\\"code\\":\\"ResponsibleAIPolicyViolation\\"}}","code":"bad_gateway"}}', +); + +function providerThatFails(err: Error, okAfter = Infinity): MemoryProvider { + let calls = 0; + return { + name: "stub", + async compress() { + calls += 1; + if (calls > okAfter) return ""; + throw err; + }, + async summarize() { + return ""; + }, + } as MemoryProvider; +} + +describe("isPayloadRejection", () => { + it("recognises Azure content filter rejections", () => { + expect(isPayloadRejection(CONTENT_FILTER_ERROR)).toBe(true); + expect( + isPayloadRejection( + new Error('OpenAI API error (400): {"error":{"code":"ResponsibleAIPolicyViolation"}}'), + ), + ).toBe(true); + expect( + isPayloadRejection( + new Error('openrouter API error (400): {"error":{"code":"content_filter"}}'), + ), + ).toBe(true); + }); + + it("does not treat the filter code alone as a rejection", () => { + expect(isPayloadRejection(new Error("ResponsibleAIPolicyViolation"))).toBe(false); + }); + + it("treats genuine provider trouble as a failure", () => { + expect(isPayloadRejection(new Error("OpenAI API error (503): upstream down"))).toBe(false); + expect(isPayloadRejection(new Error("request timed out after 60000ms"))).toBe(false); + expect(isPayloadRejection("socket hang up")).toBe(false); + }); + + it("does not excuse a gateway failure that merely quotes the filter body", () => { + expect(isPayloadRejection(GATEWAY_ECHO_ERROR)).toBe(false); + }); +}); + +describe("ResilientProvider — content filter vs circuit breaker", () => { + it("keeps the breaker closed across repeated filter rejections", async () => { + const provider = new ResilientProvider(providerThatFails(CONTENT_FILTER_ERROR)); + + for (let i = 0; i < 5; i++) { + await expect(provider.compress("sys", "bad")).rejects.toThrow(/content_filter/); + } + + expect(provider.circuitState.state).toBe("closed"); + }); + + it("lets an unrelated call through after filtered ones", async () => { + const provider = new ResilientProvider(providerThatFails(CONTENT_FILTER_ERROR, 3)); + + for (let i = 0; i < 3; i++) { + await expect(provider.compress("sys", "bad")).rejects.toThrow(/content_filter/); + } + + await expect(provider.compress("sys", "harmless")).resolves.toBe(""); + }); + + it("opens the breaker when a gateway failure quotes the filter body", async () => { + const provider = new ResilientProvider(providerThatFails(GATEWAY_ECHO_ERROR)); + + for (let i = 0; i < 3; i++) { + await expect(provider.compress("sys", "x")).rejects.toThrow(/502/); + } + + expect(provider.circuitState.state).toBe("open"); + }); + + it("still opens the breaker for real provider failures", async () => { + const provider = new ResilientProvider( + providerThatFails(new Error("OpenAI API error (503): upstream down")), + ); + + for (let i = 0; i < 3; i++) { + await expect(provider.compress("sys", "x")).rejects.toThrow(/503/); + } + + expect(provider.circuitState.state).toBe("open"); + await expect(provider.compress("sys", "x")).rejects.toThrow("circuit_breaker_open"); + }); +}); diff --git a/test/retained-embedding-publication.test.ts b/test/retained-embedding-publication.test.ts new file mode 100644 index 0000000000..78011e582a --- /dev/null +++ b/test/retained-embedding-publication.test.ts @@ -0,0 +1,250 @@ +import {afterEach,expect,it,vi} from 'vitest'; +import {createHash} from 'node:crypto'; +import {mkdtempSync,readFileSync,rmSync,writeFileSync} from 'node:fs'; +import {join} from 'node:path'; +import {tmpdir} from 'node:os'; +import {StateKV} from '../src/state/kv.js'; +import {KV} from '../src/state/schema.js'; +import {IndexPersistence,vectorBucketScope} from '../src/state/index-persistence.js'; +import {VectorIndex,float32ToBase64} from '../src/state/vector-index.js'; +import {acquireFileWriterLease} from '../src/ops/file-writer-lease.mjs'; +import {registerEmbeddingRecovery} from '../src/ops/embedding-recovery.mjs'; +import {withKeyedLock} from '../src/state/keyed-mutex.js'; +import {createBackgroundControl,installBackgroundControl} from '../src/ops/background-control.mjs'; +import {engineFixture} from './helpers/engine-evidence.js'; + +const cleanups:Array<()=>Promise>=[]; +afterEach(async()=>{for(const close of cleanups.splice(0))await close();installBackgroundControl(null as never);vi.unstubAllEnvs();vi.restoreAllMocks();}); +function archive(value:unknown){ + const bytes=Buffer.from(JSON.stringify(value)); + if(bytes.length<=8){const inline=Buffer.alloc(8,0xff);bytes.copy(inline);return inline;} + const root=Math.ceil(bytes.length/4)*4,out=Buffer.alloc(root+8); + bytes.copy(out);out.writeUInt32LE((bytes.length&63)+128+Math.floor(bytes.length/64)*256,root);out.writeInt32LE(-root,root+4);return out; +} +async function fixture(){ + const store=mkdtempSync(join(tmpdir(),'am-retained-embedding-'));vi.stubEnv('AGENTMEMORY_STATE_STORE_PATH',store); + const {evidence}=engineFixture(store),lease=acquireFileWriterLease(store,{evidence}); + const data=new Map>(),disk=(scope:string)=>join(store,encodeURIComponent(scope)+'.bin'); + const write=(scope:string)=>writeFileSync(disk(scope),archive(Object.fromEntries(data.get(scope)??[]))); + const embedding=new Float32Array([1,0]),rows=[1,2,3,4].map(q=>({id:'obs_retained_'+q,s:'session',q,e:float32ToBase64(embedding),k:'observation'})); + const oldCommit={id:rows[3].id,sessionId:'session',kind:'synthetic',fingerprint:'0'.repeat(64),committedAt:'2025-01-01T00:00:00.000Z'}; + data.set(KV.bm25Index,new Map([['vectors:meta',{v:3,count:4,bucketCount:1,savedAt:'2026-01-01T00:00:00.000Z'}]])); + data.set(vectorBucketScope(0),new Map(rows.map(({id,s,e})=>[id,{id,s,e}]))); + data.set(KV.vectorPendingLog,new Map(rows.map(row=>[row.id,row]))); + data.set(KV.observations('session'),new Map(rows.map(({id})=>[id,{id,sessionId:'session',title:'canonical '+id,narrative:'source text',sourceRaw:{original:'preserved synthetic'}}]))); + const fingerprint=(id:string)=>createHash('sha256').update('canonical '+id+' source text').digest('hex'); + const retries=rows.map(({id})=>({id,sessionId:'session',kind:'observation',attempts:3,retryAt:0,generation:'previous-'+id,fingerprint:fingerprint(id),phase:'queued',errorCode:'VECTOR_RECOVERY_PAUSED'})); + data.set(KV.embeddingRetry,new Map(retries.map(row=>[row.id,row]))); + data.set(KV.embeddingCommitted,new Map([[oldCommit.id,oldCommit]])); + for(const scope of data.keys())write(scope); + const actions:Array<{scope:string;key:string;op:string}>=[]; + const sdk={trigger:async({function_id,payload:{scope,key,value}}:any)=>{ + const table=data.get(scope)??new Map(); + if(function_id==='state::get')return structuredClone(table.get(key)??null); + if(function_id==='state::list')return structuredClone([...table.values()]); + if(function_id==='state::set'){data.set(scope,table);table.set(key,structuredClone(value));write(scope);actions.push({scope,key,op:'set'});return value;} + if(function_id==='state::delete'){table.delete(key);write(scope);actions.push({scope,key,op:'delete'});return;} + throw Error('unexpected synthetic SDK call'); + }}; + const kv=new StateKV(sdk as never,{writerLease:lease}),index=new VectorIndex();let held=true,epoch=0; + const persistence=new IndexPersistence(kv,index,{requireRecoveryPlan:true,isStorageHeld:()=>held,storageGeneration:()=>epoch}); + await persistence.load();held=false;expect(await persistence.resumeStorageRecovery(2,true)).toBe(true); + const provider={name:'synthetic',dimensions:2,embed:vi.fn(async(_text:string)=>new Float32Array(embedding)),embedBatch:async(texts:string[])=>texts.map(()=>new Float32Array(embedding))}; + const recovery=registerEmbeddingRecovery({kv,KV,index,provider,withKeyedLock,schedule:false, + verifyRetainedPublication:publication=>persistence.verifyRetainedVectorPublication(publication), + persistIndex:publication=>persistence.saveVerified(publication)}); + cleanups.push(async()=>{recovery.stop();await recovery.drain();persistence.stop();lease.close();await kv.durability?.drain();rmSync(store,{recursive:true,force:true});}); + const wal=readFileSync(disk(KV.vectorPendingLog)),buckets=readFileSync(disk(vectorBucketScope(0))); + actions.length=0; + return{kv,index,persistence,recovery,provider,rows,retries,oldCommit,data,write,disk,actions,wal,buckets, + item:(n=0)=>({id:rows[n].id,sessionId:'session',kind:'observation' as const}), + changeEpoch:()=>{epoch++;},hold:()=>{held=true;}, + assertOriginals:()=>{expect(readFileSync(disk(KV.vectorPendingLog))).toEqual(wal);expect(readFileSync(disk(vectorBucketScope(0)))).toEqual(buckets);}, + }; +} + +it('publishes all four freshly reembedded exact retained vectors without mutating WAL or vectors',async()=>{ + const f=await fixture();await f.recovery.pump();await f.recovery.drain(); + expect(f.provider.embed).toHaveBeenCalledTimes(4);expect(f.recovery.snapshot()).toMatchObject({pendingCount:0,completedCount:4,lastError:null,publicationEvidence:{completion:'sdk'}}); + for(const [n,row] of f.rows.entries()){ + expect(f.provider.embed).toHaveBeenCalledWith('canonical '+row.id+' source text'); + expect(await f.kv.get(KV.embeddingRetry,row.id)).toBeNull(); + const marker:any=await f.kv.get(KV.embeddingCommitted,row.id); + expect(marker.fingerprint).toBe(f.retries[n].fingerprint); + expect(marker.retainedRecovery).toMatchObject({version:1,completion:'sdk',verified:false,synced:false,previousIntent:f.retries[n]}); + expect(marker.retainedRecovery.previousCommit).toEqual(n===3?f.oldCommit:null); + expect(f.recovery.observedPublication(row.id)).toMatchObject({completion:'sdk',durable:false}); + } + expect(f.actions.filter(a=>a.scope===KV.vectorPendingLog||a.scope.startsWith(KV.bm25Index+':vec:')||a.scope===KV.vectorRetentionReceipts)).toEqual([]); + f.assertOriginals();expect(f.index.pendingChanges).toBe(0); + expect(f.persistence.status()).toMatchObject({pendingLog:4,retainedCheckpointRows:4,pendingLogAcknowledged:0}); + expect(()=>f.index.add(f.rows[0].id,'session',new Float32Array([1,0]))).toThrow(/VECTOR_RECOVERY_PAUSED/); + expect(()=>f.index.remove(f.rows[0].id)).toThrow(/VECTOR_RECOVERY_PAUSED/);expect(()=>f.index.clear()).toThrow(/VECTOR_RECOVERY_PAUSED/); +}); + +it.each(['different bytes','negative zero','wrong owner','epoch changed','missing receipt','stale receipt','changed checkpoint'])('retains an exact original on %s',async mode=>{ + const f=await fixture(); + if(mode==='different bytes')f.provider.embed.mockResolvedValue(new Float32Array([0,1])); + if(mode==='negative zero')f.provider.embed.mockResolvedValue(new Float32Array([1,-0])); + if(mode==='wrong owner')f.provider.embed.mockImplementation(async()=>{f.index.get(f.rows[0].id)!.sessionId='other';return new Float32Array([1,0]);}); + if(mode==='epoch changed')f.provider.embed.mockImplementation(async()=>{f.changeEpoch();return new Float32Array([1,0]);}); + if(mode==='missing receipt'||mode==='stale receipt'){ + const table=f.data.get(KV.vectorRetentionReceipts)!; + if(mode==='missing receipt')table.clear();else{const first=table.keys().next().value!;table.get(first).q++;} + f.write(KV.vectorRetentionReceipts); + } + if(mode==='changed checkpoint'){f.data.get(vectorBucketScope(0))!.get(f.rows[0].id).e=float32ToBase64(new Float32Array([0,1]));f.write(vectorBucketScope(0));} + await f.recovery.add(f.item());await f.recovery.drain(); + expect(await f.kv.get(KV.embeddingRetry,f.rows[0].id)).not.toBeNull();expect(await f.kv.get(KV.embeddingCommitted,f.rows[0].id)).toBeNull(); + expect(f.recovery.snapshot()).toMatchObject({completedCount:0});expect(f.recovery.observedPublication(f.rows[0].id)).toBeNull(); + expect(readFileSync(f.disk(KV.vectorPendingLog))).toEqual(f.wal); +}); + +it.each(['source','intent'])('rejects a changed %s during retained physical verification',async mode=>{ + const f=await fixture(),verify=f.persistence.verifyCaptureVectorPending.bind(f.persistence); + vi.spyOn(f.persistence,'verifyCaptureVectorPending').mockImplementation(async(...args)=>{ + const valid=await verify(...args); + if(mode==='source')await f.kv.set(KV.observations('session'),f.rows[0].id,{id:f.rows[0].id,sessionId:'session',title:'changed',narrative:'source text'}); + else await f.kv.set(KV.embeddingRetry,f.rows[0].id,{...await f.kv.get(KV.embeddingRetry,f.rows[0].id),generation:'new-owner'}); + return valid; + }); + await f.recovery.add(f.item());await f.recovery.drain(); + expect(await f.kv.get(KV.embeddingRetry,f.rows[0].id)).not.toBeNull();expect(await f.kv.get(KV.embeddingCommitted,f.rows[0].id)).toBeNull();f.assertOriginals(); +}); + +it.each(['receipt','WAL'])('rechecks %s evidence during publication instead of trusting earlier reuse',async evidence=>{ + const f=await fixture(),save=f.persistence.saveVerified.bind(f.persistence); + vi.spyOn(f.persistence,'saveVerified').mockImplementation(async publication=>{ + if(evidence==='receipt'){f.data.get(KV.vectorRetentionReceipts)!.clear();f.write(KV.vectorRetentionReceipts);} + else{f.data.get(KV.vectorPendingLog)!.delete(f.rows[0].id);f.write(KV.vectorPendingLog);} + return save(publication); + }); + await f.recovery.add(f.item());await f.recovery.drain(); + expect(await f.kv.get(KV.embeddingRetry,f.rows[0].id)).not.toBeNull();expect(await f.kv.get(KV.embeddingCommitted,f.rows[0].id)).toBeNull(); + if(evidence==='receipt')f.assertOriginals(); + else expect(readFileSync(f.disk(vectorBucketScope(0)))).toEqual(f.buckets); +}); + +it('does not infer or reuse retained vectors while background admission is paused',async()=>{ + const f=await fixture(),control=createBackgroundControl();await control.init(f.kv);installBackgroundControl(control);await control.change('pause');await control.settled(); + await f.recovery.add(f.item());await f.recovery.pump();await f.recovery.drain(); + expect(f.provider.embed).not.toHaveBeenCalled();expect(await f.kv.get(KV.embeddingRetry,f.rows[0].id)).not.toBeNull();f.assertOriginals(); +}); + +it('keeps completed SDK metadata and retained WAL valid after a new persistence worker loads them',async()=>{ + const f=await fixture();await f.recovery.pump();await f.recovery.drain(); + const nextIndex=new VectorIndex(),nextPersistence=new IndexPersistence(f.kv,nextIndex,{requireRecoveryPlan:true}); + try { + const loaded=await nextPersistence.load();expect(loaded.state).toBe('buckets');await nextPersistence.prepareBackfill(loaded);nextIndex.restoreFrom(loaded.vector!); + expect(await nextPersistence.replayPendingLog(2)).toMatchObject({entries:0,added:0}); + expect(nextIndex.size).toBe(4);expect(nextPersistence.status()).toMatchObject({pendingLog:4,retainedCheckpointRows:4,pendingChanges:0}); + for(const row of f.rows)expect(await nextPersistence.verifyRetainedVectorPublication({id:row.id,entry:nextIndex.get(row.id)!})).toBe(true); + expect(()=>nextIndex.remove(f.rows[0].id)).toThrow(/VECTOR_RECOVERY_PAUSED/); + f.assertOriginals(); + }finally{nextPersistence.stop();} +}); + +it('records only one defensive-copy retained proof diagnostic and ignores unknown IDs',async()=>{ + const f=await fixture();expect(f.persistence.status().retainedProofLastCheck).toBeNull(); + expect(await f.persistence.verifyRetainedVectorPublication({id:'unknown-untrusted-id',entry:null})).toBe(false); + expect(f.persistence.status().retainedProofLastCheck).toBeNull(); + for(const row of f.rows)expect(await f.persistence.verifyRetainedVectorPublication({id:row.id,entry:f.index.get(row.id)!})).toBe(true); + const status=f.persistence.status();expect(status.retainedProofLastCheck).toMatchObject({id:f.rows[3].id,operation:'retained_publication',reason:'verified', + freshBytesMatch:true,loadedBytesMatch:true,freshSessionMatch:true,loadedSessionMatch:true,retentionReady:true, + stopped:false,storageHeld:false,storageGenerationChanged:false,storageRecovering:false,storageBlocked:false,recoveryPaused:false, + pendingLog:0,logChains:0,dirty:0,revisionChanged:false,physicalProofErrorCode:null}); + status.retainedProofLastCheck!.reason='publication_invalid';expect(f.persistence.status().retainedProofLastCheck!.reason).toBe('verified'); + expect(JSON.stringify(status)).not.toContain(f.rows[0].e);expect(JSON.stringify(status)).not.toContain('source text'); + expect(f.actions).toEqual([]);f.assertOriginals(); +}); + +it.each(['different bytes','negative zero','fresh owner','loaded owner','loaded bytes'])('diagnoses %s without relaxing exact publication rejection',async mode=>{ + const f=await fixture(),row=f.rows[0],entry={...f.index.get(row.id)!,embedding:new Float32Array([1,0])}; + if(mode==='different bytes')entry.embedding=new Float32Array([0,1]); + if(mode==='negative zero')entry.embedding=new Float32Array([1,-0]); + if(mode==='fresh owner')entry.sessionId='other'; + if(mode==='loaded owner')f.index.get(row.id)!.sessionId='other'; + if(mode==='loaded bytes')f.index.get(row.id)!.embedding[0]=0; + expect(await f.persistence.verifyRetainedVectorPublication({id:row.id,entry})).toBe(false); + expect(f.persistence.status().retainedProofLastCheck).toMatchObject({id:row.id,reason:'publication_mismatch',physicalProofErrorCode:null, + freshBytesMatch:mode!=='different bytes'&&mode!=='negative zero',loadedBytesMatch:mode!=='loaded bytes', + freshSessionMatch:mode!=='fresh owner',loadedSessionMatch:mode!=='loaded owner'}); + expect(f.actions).toEqual([]);f.assertOriginals(); + expect(()=>f.index.remove(row.id)).toThrow(/VECTOR_RECOVERY_PAUSED/); +}); + +it.each(['known','unknown'])('retains only allowlisted physical proof error codes (%s)',async mode=>{ + const f=await fixture(),row=f.rows[0]; + vi.spyOn(f.kv,'reconcileVectorCheckpoint').mockRejectedValue(Object.assign(new Error('synthetic private path and source text'),{code:mode==='known'?'VECTOR_CHECKPOINT_UNVERIFIED':'PRIVATE_UNKNOWN_SECRET'})); + expect(await f.persistence.verifyRetainedVectorPublication({id:row.id,entry:f.index.get(row.id)!})).toBe(false); + expect(f.persistence.status().retainedProofLastCheck).toMatchObject({reason:'physical_proof_rejected',freshBytesMatch:true,loadedBytesMatch:true, + physicalProofErrorCode:mode==='known'?'VECTOR_CHECKPOINT_UNVERIFIED':'UNCLASSIFIED_PROOF_ERROR'}); + expect(JSON.stringify(f.persistence.status().retainedProofLastCheck)).not.toMatch(/private|source text|PRIVATE_UNKNOWN_SECRET/); + expect(f.actions).toEqual([]);f.assertOriginals(); +}); + +it.each(['stopped','held','generation','dirty','log chain','pending log','not ready','recovering','blocked','paused','pending mismatch'])('reports %s while preserving capture pending false',async mode=>{ + const f=await fixture(),row=f.rows[0],p=f.persistence as any; + if(mode==='stopped')p.stopped=true; + if(mode==='held')f.hold(); + if(mode==='generation')f.changeEpoch(); + if(mode==='dirty')vi.spyOn(f.index,'pendingChanges','get').mockReturnValue(1); + if(mode==='log chain')p.logChains.set('synthetic',Promise.resolve()); + if(mode==='pending log')p.pendingLog.set('synthetic',1); + if(mode==='not ready')p.retentionReady=false; + if(mode==='recovering')p.storageRecovering=true; + if(mode==='blocked')p.storageBlocked=true; + if(mode==='paused')p.recoveryReady=false; + const pending=mode==='pending mismatch'?{...row,q:99}:row; + expect(await f.persistence.verifyCaptureVectorPending(row.id,'session',pending)).toBe(false); + const last=f.persistence.status().retainedProofLastCheck!; + expect(last).toMatchObject({id:row.id,operation:'capture_pending',freshBytesMatch:null,physicalProofErrorCode:null}); + if(mode==='stopped')expect(last.stopped).toBe(true); + if(mode==='held')expect(last.storageHeld).toBe(true); + if(mode==='generation')expect(last.storageGenerationChanged).toBe(true); + if(mode==='dirty')expect(last.dirty).toBe(1); + if(mode==='log chain')expect(last.logChains).toBe(1); + if(mode==='pending log')expect(last.pendingLog).toBe(1); + if(mode==='not ready')expect(last.retentionReady).toBe(false); + if(mode==='recovering')expect(last.storageRecovering).toBe(true); + if(mode==='blocked')expect(last.storageBlocked).toBe(true); + if(mode==='paused')expect(last.recoveryPaused).toBe(true); + if(mode==='pending mismatch')expect(last.reason).toBe('pending_row_mismatch'); + expect(f.actions).toEqual([]);f.assertOriginals(); +}); + +it('reports revision changes and preserves both successful and rejected capture decisions',async()=>{ + const f=await fixture(),row=f.rows[0]; + expect(await f.persistence.verifyCaptureVectorPending(row.id,'session',row)).toBe(true); + expect(f.persistence.status().retainedProofLastCheck).toMatchObject({reason:'verified',freshBytesMatch:null,loadedBytesMatch:true,revisionChanged:false}); + vi.spyOn(f.kv,'reconcileVectorCheckpoint').mockImplementation(async()=>{Object.defineProperty(f.kv,'vectorCheckpointRevision',{value:999});throw Error('vector_recovery_checkpoint_changed');}); + expect(await f.persistence.verifyCaptureVectorPending(row.id,'session',row)).toBe(false); + expect(f.persistence.status().retainedProofLastCheck).toMatchObject({reason:'physical_proof_rejected',revisionChanged:true,physicalProofErrorCode:'vector_recovery_checkpoint_changed'}); + expect(f.actions).toEqual([]);f.assertOriginals(); +}); + +it('never lets hostile physical error properties escape the original false result',async()=>{ + const f=await fixture(),row=f.rows[0]; + vi.spyOn(f.kv,'reconcileVectorCheckpoint').mockRejectedValue(Object.defineProperty({},'code',{get(){throw Error('synthetic private getter');}})); + expect(await f.persistence.verifyCaptureVectorPending(row.id,'session',row)).toBe(false); + expect(f.persistence.status().retainedProofLastCheck).toMatchObject({reason:'physical_proof_rejected',physicalProofErrorCode:'UNCLASSIFIED_PROOF_ERROR'}); + f.assertOriginals(); +}); + +it('does not label a rejected publication with an earlier verified capture diagnostic',async()=>{ + const f=await fixture(),row=f.rows[0]; + expect(await f.persistence.verifyCaptureVectorPending(row.id,'session',row)).toBe(true); + vi.spyOn(f.persistence,'verifyCaptureVectorPending').mockResolvedValue(false); + expect(await f.persistence.verifyRetainedVectorPublication({id:row.id,entry:f.index.get(row.id)!})).toBe(false); + expect(f.persistence.status().retainedProofLastCheck).toMatchObject({reason:'capture_pending_rejected'}); + f.assertOriginals(); +}); + +it('clears an earlier diagnostic if local diagnostic collection becomes unavailable',async()=>{ + const f=await fixture(),row=f.rows[0]; + expect(await f.persistence.verifyCaptureVectorPending(row.id,'session',row)).toBe(true); + (f.persistence as any).isStorageHeld=()=>{throw Error('synthetic unavailable local observation');}; + expect(await f.persistence.verifyCaptureVectorPending(row.id,'session',row)).toBe(false); + expect(f.persistence.status().retainedProofLastCheck).toBeNull(); + f.assertOriginals(); +}); diff --git a/test/sdk-capture-deadline.test.ts b/test/sdk-capture-deadline.test.ts new file mode 100644 index 0000000000..98cf075f71 --- /dev/null +++ b/test/sdk-capture-deadline.test.ts @@ -0,0 +1,154 @@ +import {afterEach,expect,it,vi} from 'vitest'; +import {createHash} from 'node:crypto'; +import {existsSync,mkdirSync,mkdtempSync,readFileSync,readdirSync,rmSync,writeFileSync} from 'node:fs'; +import {join,resolve} from 'node:path'; +import {tmpdir} from 'node:os'; +import {performance} from 'node:perf_hooks'; +import {fork} from 'node:child_process'; +import {SdkSpoolV2,deliverSdkCaptureV2,drainSdkCapturesV2} from '../src/capture/sdk-delivery-v2.js'; +const target='http://127.0.0.1:43111',roots:string[]=[]; +const hash=(v:string|Buffer)=>createHash('sha256').update(v).digest('hex'); +const id=(event:string)=>hash(hash(target)+'\0'+event); +const ack=(body:Record)=>({status:201,body:{eventId:body.eventId,completion:'sdk',completed:true,durable:false}}); +const delay=(ms:number)=>Atomics.wait(new Int32Array(new SharedArrayBuffer(4)),0,0,ms); +function root(){const value=mkdtempSync(join(tmpdir(),'sdk-deadline-'));roots.push(value);return value;} +function ref(dir:string,event:string){return JSON.parse(readFileSync(join(dir,'queue',id(event)+'.ref'),'utf8'));} +function manifest(dir:string,event:string){return JSON.parse(readFileSync(join(dir,'originals',id(event),'manifest.json'),'utf8'));} +afterEach(()=>{vi.restoreAllMocks();for(const value of roots.splice(0))rmSync(value,{recursive:true,force:true});}); + +it('publishes retained source and reference under one lease, without calling separate enqueue',async()=>{ + const dir=root();const enqueue=vi.spyOn(SdkSpoolV2.prototype,'enqueue').mockRejectedValue(Object.assign(Error('STORE_BUSY'),{code:'STORE_BUSY'})); + const result=await deliverSdkCaptureV2(target,{eventId:'composite'},async body=>ack(body),{dir}); + expect(result.outcome).toBe('delivered');expect(enqueue).not.toHaveBeenCalled();expect(manifest(dir,'composite').completed).toBe(true); +}); +it('retains original acknowledgment if reference quota rejects composite handoff',async()=>{ + const dir=root();let posts=0; + const result=await deliverSdkCaptureV2(target,{eventId:'full'},async body=>{posts++;return ack(body);},{dir,storeOptions:{maxQueueBytes:3*4096}}); + expect(result).toMatchObject({outcome:'dropped',code:'QUEUE_FULL',sourceAcknowledged:true,referenceAcknowledged:false});expect(posts).toBe(0); + expect(manifest(dir,'full').completed).toBe(false);expect(existsSync(join(dir,'queue',id('full')+'.ref'))).toBe(false); +}); +it('preserves raw published before a simulated crash and does not pretend a reference exists',async()=>{ + const dir=root();let posts=0; + const result=await deliverSdkCaptureV2(target,{eventId:'crash'},async body=>{posts++;return ack(body);},{dir,storeOptions:{onPhase:phase=>{if(phase==='source_published')throw Error('synthetic crash');}}}); + expect(result.outcome).toBe('dropped');expect(posts).toBe(0);expect(manifest(dir,'crash').completed).toBe(false); + expect(existsSync(join(dir,'queue',id('crash')+'.ref'))).toBe(false); +}); +it('reports retained source conservatively when reference publication reports an I/O fault',async()=>{ + const dir=root(),post=vi.fn(async(body:Record)=>ack(body)); + const result=await deliverSdkCaptureV2(target,{eventId:'queue-fault'},post,{dir,storeOptions:{onPhase:phase=>{if(phase==='reference_published')throw Object.assign(Error('synthetic I/O'),{code:'STORE_IO'});}}}); + expect(result).toMatchObject({outcome:'dropped',code:'STORE_IO',sourceAcknowledged:true,referenceAcknowledged:false});expect(post).not.toHaveBeenCalled();expect(manifest(dir,'queue-fault').completed).toBe(false); +}); +it('leaves pending without sending or attempts when local work consumes the transport budget',async()=>{ + const dir=root();let posts=0; + const result=await deliverSdkCaptureV2(target,{eventId:'budget'},async body=>{posts++;return ack(body);},{dir,deadline:performance.now()+900,storeOptions:{onPhase:phase=>{if(phase==='source_published')delay(200);}}}); + expect(result.outcome).toBe('spooled');expect(posts).toBe(0);expect(ref(dir,'budget')).toMatchObject({state:'pending',attempts:0,nextAttemptAt:0});expect(manifest(dir,'budget').completed).toBe(false); +}); +it('passes the same absolute transport deadline with finalization headroom to direct POST',async()=>{ + const dir=root(),deadline=performance.now()+2000;let budget:any; + const result=await deliverSdkCaptureV2(target,{eventId:'deadline'},async(body,value)=>{budget=value;return ack(body);},{dir,deadline}); + expect(result.outcome).toBe('delivered');expect(budget.deadline).toBe(deadline-750);expect(budget.remainingMs).toBeGreaterThan(0);expect(budget.remainingMs).toBeLessThanOrEqual(1250); +}); +it('never retries an uncertain POST but retains explicit not-sent retry semantics with a deadline',async()=>{ + const dir=root(),post=vi.fn(async()=>{throw Error('synthetic timeout after POST');}); + expect(await deliverSdkCaptureV2(target,{eventId:'unknown'},post,{dir,deadline:performance.now()+2000})).toMatchObject({outcome:'held',code:'SDK_SEND_UNCERTAIN'}); + expect(ref(dir,'unknown')).toMatchObject({state:'held',reason:'HELD',attempts:0}); + expect((await deliverSdkCaptureV2(target,{eventId:'unknown'},post,{dir,deadline:performance.now()+2000})).outcome).toBe('held');expect(post).toHaveBeenCalledTimes(1); + expect((await deliverSdkCaptureV2(target,{eventId:'unknown'},post,{dir,deadline:performance.now()+500})).outcome).toBe('held');expect(post).toHaveBeenCalledTimes(1); + expect((await deliverSdkCaptureV2(target,{eventId:'not-sent'},async()=>({status:0,body:null,notSent:true}),{dir,deadline:performance.now()+2000})).outcome).toBe('spooled'); + expect(ref(dir,'not-sent')).toMatchObject({state:'pending',reason:'RETRY',attempts:1}); +}); +it('does not call an existing sending reference pending when its transport budget has expired',async()=>{ + const dir=root();let entered!:()=>void,release!:()=>void,posts=0; + const started=new Promise(r=>entered=r),wait=new Promise(r=>release=r); + const first=deliverSdkCaptureV2(target,{eventId:'in-flight'},async body=>{posts++;entered();await wait;return ack(body);},{dir,deadline:performance.now()+3000});await started; + try{ + expect(await deliverSdkCaptureV2(target,{eventId:'in-flight'},async body=>{posts++;return ack(body);},{dir,deadline:performance.now()+500})).toMatchObject({outcome:'held',code:'SDK_SEND_IN_PROGRESS_OR_UNCERTAIN'}); + expect(ref(dir,'in-flight').state).toBe('sending');expect(posts).toBe(1); + }finally{release();await first;} +}); +it('does not convert a failed claim lock into evidence that a source is pending',async()=>{ + const dir=root();vi.spyOn(SdkSpoolV2.prototype,'claimForSend').mockRejectedValue(Object.assign(Error('STORE_BUSY'),{code:'STORE_BUSY'})); + const post=vi.fn(async(body:Record)=>ack(body)); + expect(await deliverSdkCaptureV2(target,{eventId:'claim-unknown'},post,{dir,deadline:performance.now()+500})).toMatchObject({outcome:'held',code:'STORE_BUSY'});expect(post).not.toHaveBeenCalled(); +}); +it('keeps SDK confirmation separate if local ACK publication fails',async()=>{ + const dir=root();const result=await deliverSdkCaptureV2(target,{eventId:'ack-fail'},async body=>ack(body),{dir,deadline:performance.now()+2000,storeOptions:{onPhase:(phase:string)=>{if(phase==='ack_before_publish')throw Error('synthetic failure');}}}); + expect(result).toMatchObject({outcome:'held',code:'SDK_ACK_STATE_UNCONFIRMED',sdkReplyConfirmed:true});expect(ref(dir,'ack-fail').state).toBe('sending');expect(manifest(dir,'ack-fail').completed).toBe(false); +}); +it('rejects non-finite public claim deadlines without creating a sending intent',async()=>{ + const dir=root(),store=new SdkSpoolV2({dir}),source=await store.retain(target,'invalid-clock',{eventId:'invalid-clock'});await store.enqueue(target,source.sourceId); + for(const deadline of [NaN,Infinity,-Infinity])await expect(store.claimForSend(target,source.sourceId,deadline)).rejects.toMatchObject({code:'STORE_OPTIONS_INVALID'}); + await expect(store.claimForSend(target,source.sourceId,-1)).rejects.toMatchObject({code:'SOURCE_SEND_BUDGET_EXHAUSTED'}); + expect(ref(dir,'invalid-clock')).toMatchObject({state:'pending',attempts:0}); +}); +it('bounded drain preserves a short shared deadline and the legacy unspecified deadline behavior',async()=>{ + const dir=root(),store=new SdkSpoolV2({dir}),source=await store.retain(target,'drain-budget',{eventId:'drain-budget'});await store.enqueue(target,source.sourceId); + let posts=0; + const short=await drainSdkCapturesV2(target,async body=>{posts++;return ack(body);},{dir,deadline:performance.now()+500,budgetMs:250}); + expect(short).toMatchObject({attempted:0,remaining:1});expect(posts).toBe(0);expect(ref(dir,'drain-budget').state).toBe('pending'); + const result=await drainSdkCapturesV2(target,async body=>{posts++;return ack(body);},{dir,budgetMs:250}); + expect(result).toMatchObject({attempted:1,delivered:1,remaining:0});expect(posts).toBe(1); +}); + +for(const [processes,holdMs] of [[2,350],[6,350],[2,3200]])it(`${processes} real processes preserve deadline outcomes after ${holdMs}ms contention against 6000 retained originals`,async()=>{ + const home=root(),dir=join(home,'spool'),store=new SdkSpoolV2({dir});await store.usage(); + // Synthetic completed trees exercise the actual recursive quota scan. Nothing is read from the host spool. + for(let index=0;index<6000;index++){ + const folder=join(dir,'originals',hash('seed-'+index));mkdirSync(folder,{mode:0o700}); + writeFileSync(join(folder,'manifest.json'),'synthetic quota-only fixture',{mode:0o600});writeFileSync(join(folder,'000000.chunk'),'synthetic',{mode:0o600}); + } + const children:any[]=[]; + function child(mode:string,index=0){ + const worker=fork(resolve('test/helpers/sdk-deadline-worker.mjs'),[],{execArgv:['--import',resolve('node_modules/tsx/dist/loader.mjs')],env:{PATH:'/usr/bin:/bin',HOME:home,USERPROFILE:home,TMPDIR:home,MODE:mode,FIXTURE_DIR:dir,FIXTURE_TARGET:target,FIXTURE_INDEX:String(index),FIXTURE_MODULE:process.env.SDK_DEADLINE_FIXTURE_MODULE??resolve('src/capture/sdk-delivery-v2.ts')},stdio:['ignore','pipe','pipe','ipc']}); + children.push(worker);let result:any; + const started=new Promise(done=>worker.on('message',(m:any)=>{if(m.started)done();})); + const ready=new Promise((done,reject)=>{worker.on('message',(m:any)=>{if(m.ready)done();if(m.result)result=m.result;});worker.once('error',reject);}); + const exited=new Promise((done,reject)=>{worker.once('exit',code=>code===0?done(result):reject(Error('FIXTURE_CHILD_FAILED')));worker.once('error',reject);}); + return {worker,ready,started,exited}; + } + try{ + const holder=child('holder');await holder.ready; + const senders=Array.from({length:processes},(_,i)=>child('sender',i));await Promise.all(senders.map(x=>x.ready)); + for(const sender of senders)sender.worker.send('go'); + await Promise.all(senders.map(x=>x.started));holder.worker.send({holdMs}); + const results=await Promise.all(senders.map(x=>x.exited));await holder.exited; + console.log(JSON.stringify({syntheticConcurrentCapture:true,processes,holdMs,results})); + // A finite deadline guarantees conservative state, not CI filesystem throughput. + // The explicit overrun case makes STORE_BUSY reproducible without CPU stress. + let published=0; + for(const [index,result] of results.entries()){ + const event='multiprocess-'+index,sourceExists=existsSync(join(dir,'originals',id(event),'manifest.json')),refExists=existsSync(join(dir,'queue',id(event)+'.ref')); + expect(result.elapsedMs).toBeGreaterThan(150); + if(sourceExists)published++; + expect(result.sourceAcknowledged).toBe(sourceExists);expect(result.referenceAcknowledged).toBe(refExists||result.outcome==='delivered'); + if(result.outcome==='delivered'){ + expect(result).toMatchObject({sourceAcknowledged:true,referenceAcknowledged:true,sdkReplyConfirmed:true,posts:1}); + expect(manifest(dir,event).completed).toBe(true);expect(refExists).toBe(false); + }else{ + expect(result.posts).toBe(0);expect(result.sdkReplyConfirmed).not.toBe(true); + if(result.outcome==='spooled')expect(result).toMatchObject({sourceAcknowledged:true,referenceAcknowledged:true}); + else{expect(['held','dropped']).toContain(result.outcome);expect(result.code).toBe('STORE_BUSY');} + if(sourceExists)expect(manifest(dir,event).completed).toBe(false); + if(refExists)expect(ref(dir,event)).toMatchObject({state:'pending',attempts:0,nextAttemptAt:0}); + expect(result.elapsedMs).toBeGreaterThanOrEqual(result.outcome==='spooled'?2250:3000); + } + } + expect(readdirSync(join(dir,'originals'))).toHaveLength(6000+published); + if(holdMs>3000)for(const result of results)expect(result).toMatchObject({outcome:'dropped',code:'STORE_BUSY',posts:0,sourceAcknowledged:false,referenceAcknowledged:false}); + // Only the branches above proved conclusively not-sent may retry. Contention is + // now gone; each gets its own original 3s bound, never a larger runtime limit. + // Unknown dispatch/sending or ACK failures fail above instead of being replayed. + for(const [index,result] of results.entries()){ + if(result.outcome==='delivered')continue; + const retry=child('sender',index);await retry.ready;retry.worker.send('go'); + const resolved=await retry.exited; + expect(resolved).toMatchObject({outcome:'delivered',sourceAcknowledged:true,referenceAcknowledged:true,sdkReplyConfirmed:true,posts:1}); + expect(result.posts+resolved.posts).toBe(1); + } + expect(readdirSync(join(dir,'originals'))).toHaveLength(6000+processes); + for(let i=0;i[scope,[...rows]]))).toBe(before);} finally{persistence.stop();} }); +it('reports matching SDK resolutions as historical reviews and keeps unverifiable items in the warning',async()=>{ + const {kv}=setup(); + const {createCaptureAutoRecovery}=await import('../src/capture/auto-recovery.js'); + const {capturePayloadDigest}=await import('../src/capture/receipt-digest.js'); + const {captureFailureHistoryDigest}=await import('../src/capture/sdk-resolution-record.js'); + const sourceDigest=capturePayloadDigest(payload); + const projection='a'.repeat(64); + const at='2026-11-10T00:00:00.000Z'; + const requestId='00000000-0000-4000-8000-000000000001'; + for(const [n,phase] of ['blocked','checking','blocked','checking'].entries()){ + const key='cap_'+String(n).repeat(40), eventId='synthetic-event-'+n, observationId='synthetic-obs-'+n; + const identity={key,eventId,observationId,sessionId:payload.sessionId,project:payload.project,hookType:payload.hookType}; + const row={...identity,id:'historical-'+n,version:1,kind:'item',phase,reason:phase==='blocked'?'VERIFICATION_BUDGET_EXHAUSTED':'VERIFICATION_PENDING', + sourceDigest:n===2?'b'.repeat(64):sourceDigest,projection,verificationAttempts:phase==='blocked'?3:2,nextAttemptAt:0}; + const history={...identity,status:'completed',failures:[],requests:{},totalAttempts:3,cycleAttempts:3,manualRetries:0,acceptedAt:at}; + const sdkResolution={version:1,phase:'committed',completion:'sdk',requestId,sourceDigest, + initialProjection:projection,verifiedProjection:projection,priorFailureDigest:captureFailureHistoryDigest(history as never), + priorInboxState:'dead',startedAt:at,completedAt:at}; + await kv.set(KV.captureRecovery,key,row); + await kv.set(KV.captureReview,key,{...history,sdkResolution}); + await kv.set(KV.captureInbox,key,{...identity,status:'pending',storageCompletion:'sdk',payload, + sdkResolutionId:requestId,sdkResolutionState:n===3?'pending':'committed'}); + } + const before=JSON.stringify([...kv.store].map(([scope,rows])=>[scope,[...rows]])); + const auto=createCaptureAutoRecovery(kv as never,{enabled:false,writerOwned:()=>true,admitted:()=>true,indexReady:()=>true}); + await auto.initialize(); + expect(auto.snapshot()).toMatchObject({blocked:1,verifying:1,resolved:0,sdkReviewed:2,blockedReason:'SDK_COMPLETION_ONLY'}); + expect(JSON.stringify([...kv.store].map(([scope,rows])=>[scope,[...rows]]))).toBe(before); +}); diff --git a/test/sdk-delivery-v2.test.ts b/test/sdk-delivery-v2.test.ts index ed3b8962ad..23498b6b1c 100644 --- a/test/sdk-delivery-v2.test.ts +++ b/test/sdk-delivery-v2.test.ts @@ -16,3 +16,4 @@ it('treats mismatched event or disk-like ACKs as held and preserves original',as it('preserves prior source and makes no POST for a conflicting payload',async()=>{const m=await api();await m.deliverSdkCaptureV2(target,body(),async(b:any)=>ack(b),{dir});let calls=0;const changed={...body(),data:{image_data:'different'}};const r=await m.deliverSdkCaptureV2(target,changed,async()=>{calls++;return ack(changed)},{dir});expect(r).toMatchObject({outcome:'held',code:'SOURCE_CONFLICT'});expect(calls).toBe(0);}); it('does not POST if reference quota is full even though source retention succeeded',async()=>{const m=await api();let calls=0;await new m.SdkSpoolV2({dir}).usage();const orphan=join(dir,'queue','synthetic-partial.tmp');const fd=openSync(orphan,'wx',0o600);try{ftruncateSync(fd,5*1024*1024)}finally{closeSync(fd)}const r=await m.deliverSdkCaptureV2(target,body(),async(b:any)=>{calls++;return ack(b)},{dir});expect(r).toMatchObject({outcome:'dropped',sourceAcknowledged:true,code:'QUEUE_FULL'});expect(calls).toBe(0);expect((await new m.SdkSpoolV2({dir}).usage()).originalBytes).toBeGreaterThan(300*1024);}); it('refuses large/image guarded restore without POST or source deletion',async()=>{const m=await api();let calls=0;await m.deliverSdkCaptureV2(target,body(),async()=>({status:0,body:null,notSent:true}),{dir});const store=new m.SdkSpoolV2({dir}),id=(await store.pending(target))[0].sourceId;const before=await store.usage();const r=await m.restoreSdkCaptureV2(target,id,async()=>{calls++;return ack(body())},{dir});expect(r).toMatchObject({completed:false,status:'held',code:'SDK_SOURCE_FORMAT_UNSUPPORTED'});expect(calls).toBe(0);expect(await store.usage()).toEqual(before);}); +it('parks the original and does not auto-retry an explicit uncertain capture outcome',async()=>{const m=await api();const b=body('synthetic-uncertain',8),post=vi.fn(async()=>({status:422,body:{eventId:b.eventId,status:'rejected',retryable:false,error:'capture_reconciliation_required',reconciliationRequired:true}}));const r=await m.deliverSdkCaptureV2(target,b,post,{dir});expect(r).toMatchObject({outcome:'held',sourceAcknowledged:true,code:'SDK_ACK_UNVERIFIED'});const store=new m.SdkSpoolV2({dir}),pending=await store.pending(target);expect(pending).toHaveLength(1);expect(pending[0].state).toBe('held');expect(await store.load(target,pending[0].sourceId)).toEqual(b);expect((await m.deliverSdkCaptureV2(target,b,post,{dir})).outcome).toBe('held');expect(post).toHaveBeenCalledTimes(1)}); diff --git a/test/sdk-explicit-recovery-v2.test.ts b/test/sdk-explicit-recovery-v2.test.ts new file mode 100644 index 0000000000..43169304b4 --- /dev/null +++ b/test/sdk-explicit-recovery-v2.test.ts @@ -0,0 +1,101 @@ +import {afterEach,beforeEach,expect,it,vi} from 'vitest'; +import {mkdtempSync,readFileSync,rmSync,writeFileSync} from 'node:fs'; +import {join} from 'node:path'; +import {tmpdir} from 'node:os'; +import {createHash} from 'node:crypto'; +import * as delivery from '../src/capture/sdk-delivery-v2.js'; + +const target='http://127.0.0.1:63000'; +let dir:string; +beforeEach(()=>{dir=mkdtempSync(join(tmpdir(),'sdk-explicit-recovery-'));}); +afterEach(()=>{vi.restoreAllMocks();rmSync(dir,{recursive:true,force:true});}); +const payload=(eventId='selected-held-0001')=>({eventId,hookType:'post_tool_use',sessionId:'synthetic-session',project:'synthetic',cwd:'/synthetic',timestamp:'2026-11-02T00:00:00Z',captureCompletion:'sdk-v1',data:{text:'preserved fixture'}}); +function restored(body:ReturnType){ + const key=createHash('sha256').update(body.project+'\0'+body.sessionId+'\0'+body.eventId).digest('hex'); + return {status:201,body:{eventId:body.eventId,completion:'sdk',durable:false,sourceRestore:'sdk-v1',status:'restored',completed:true, + observationId:'obs_'+Date.parse(body.timestamp).toString(36)+'_'+key.slice(0,12)}}; +} +async function held(eventId='selected-held-0001'){ + const body=payload(eventId),store=new delivery.SdkSpoolV2({dir}); + await delivery.deliverSdkCaptureV2(target,body,async()=>({status:422,body:{}}),{dir}); + const sourceId=(await store.pending(target)).find(row=>row.sourceId===createHash('sha256').update(createHash('sha256').update(target).digest('hex')+'\0'+eventId).digest('hex'))!.sourceId; + return {store,sourceId,body}; +} +function reference(sourceId:string){return JSON.parse(readFileSync(join(dir,'queue',sourceId+'.ref'),'utf8'));} +function manifest(sourceId:string){return JSON.parse(readFileSync(join(dir,'originals',sourceId,'manifest.json'),'utf8'));} + +it('explicit restore uses the exact retained source once, requires matched SDK identity, and preserves source bytes',async()=>{ + const {store,sourceId,body}=await held();const original=manifest(sourceId); + const post=vi.fn(async(sent:any)=>{expect(sent).toEqual(body);expect(reference(sourceId)).toMatchObject({state:'sending',owner:{pid:process.pid}});return restored(body);}); + expect(await delivery.restoreSelectedSdkCaptureV2(target,sourceId,post,{dir})).toMatchObject({completed:true,status:'restored',sourceRetained:true,sdkReplyConfirmed:true,durable:false}); + expect(post).toHaveBeenCalledTimes(1);expect(manifest(sourceId)).toEqual({...original,completed:true}); + expect(await store.load(target,sourceId)).toEqual(body);expect(await store.pending(target)).toEqual([]); + expect(await delivery.restoreSelectedSdkCaptureV2(target,sourceId,post,{dir})).toMatchObject({completed:true,status:'duplicate',sourceRetained:true}); + expect(post).toHaveBeenCalledTimes(1); +}); + +it('an uncertain response remains held and cannot be retried or acknowledged by an ordinary hook',async()=>{ + const {store,sourceId,body}=await held();const post=vi.fn(async()=>{throw Error('synthetic timeout');}); + expect(await delivery.restoreSelectedSdkCaptureV2(target,sourceId,post,{dir})).toMatchObject({completed:false,status:'held'}); + expect(reference(sourceId)).toMatchObject({state:'held'}); + expect(manifest(sourceId).completed).toBe(false); + expect(post).toHaveBeenCalledTimes(1); + await expect(store.acknowledge(target,sourceId)).rejects.toMatchObject({code:'SOURCE_HELD'}); + await expect(store.retry(target,sourceId,'retry')).rejects.toMatchObject({code:'SOURCE_HELD'}); + await expect(store.hold(target,sourceId,'hold')).rejects.toMatchObject({code:'SOURCE_HELD'}); + expect(await delivery.deliverSdkCaptureV2(target,body,post,{dir})).toMatchObject({outcome:'held'}); + expect(post).toHaveBeenCalledTimes(1); +}); + +it('rejects a mismatching restoration response and leaves the original unacknowledged',async()=>{ + const {sourceId,body}=await held(); + const response=restored(body);response.body.observationId='obs_wrong_000000000000'; + expect(await delivery.restoreSelectedSdkCaptureV2(target,sourceId,async()=>response,{dir})).toMatchObject({completed:false,status:'held'}); + expect(manifest(sourceId).completed).toBe(false); +}); + +it('excludes a concurrent manual restore and hook for the same original',async()=>{ + const {sourceId,body}=await held();let release!:(reply:any)=>void;let entered!:()=>void; + const started=new Promise(resolve=>{entered=resolve;}); + const post=vi.fn(()=>new Promise(resolve=>{release=resolve;entered();})); + const first=delivery.restoreSelectedSdkCaptureV2(target,sourceId,post,{dir});await started; + expect(await delivery.restoreSelectedSdkCaptureV2(target,sourceId,post,{dir})).toMatchObject({completed:false,status:'held'}); + expect(await delivery.deliverSdkCaptureV2(target,body,post,{dir})).toMatchObject({outcome:'held'}); + release(restored(body));expect(await first).toMatchObject({completed:true});expect(post).toHaveBeenCalledTimes(1); +}); + +it('a crash after a durable source ACK allows local duplicate recognition without another POST',async()=>{ + const {sourceId,body,store}=await held();const post=vi.fn(async()=>restored(body)); + const first=await delivery.restoreSelectedSdkCaptureV2(target,sourceId,post,{dir,storeOptions:{onPhase(name){if(name==='ack_fsynced')throw Error('crash');}}}); + expect(first).toMatchObject({completed:false,sdkReplyConfirmed:true});expect(manifest(sourceId).completed).toBe(true); + expect(await delivery.restoreSelectedSdkCaptureV2(target,sourceId,post,{dir})).toMatchObject({completed:true,status:'duplicate'}); + expect(post).toHaveBeenCalledTimes(1);expect(await store.load(target,sourceId)).toEqual(body); +}); + +it('selected normal retry sends only the designated pending original and never drains a held neighbour',async()=>{ + const neighbour=await held('other-held-0002');const body=payload('selected-pending-0003'); + await delivery.deliverSdkCaptureV2(target,body,async()=>({status:0,body:null,notSent:true}),{dir}); + const pending=(await neighbour.store.pending(target)).find(row=>row.state==='pending')!; + vi.spyOn(Date,'now').mockReturnValue(pending.nextAttemptAt+1); + const post=vi.fn(async(sent:any)=>{expect(sent).toEqual(body);return {status:200,body:{eventId:body.eventId,completed:true,completion:'sdk',durable:false,status:'accepted'}};}); + expect(await delivery.deliverSdkCaptureV2(target,await neighbour.store.load(target,pending.sourceId),post,{dir})).toMatchObject({outcome:'delivered'}); + expect(post).toHaveBeenCalledTimes(1);expect(manifest(neighbour.sourceId).completed).toBe(false); + expect(await neighbour.store.pending(target)).toMatchObject([{sourceId:neighbour.sourceId,state:'held'}]); +}); + +it('the old restore stub never transmits, and a changed held payload cannot be claimed',async()=>{ + const {store,sourceId,body}=await held();const post=vi.fn(async()=>restored(body)); + expect(await delivery.restoreSdkCaptureV2(target,sourceId,post,{dir})).toMatchObject({code:'SDK_SOURCE_RESTORE_REQUIRES_REVIEW'}); + expect(post).not.toHaveBeenCalled(); + await expect(store.markSending(target,sourceId,{...body,data:{text:'changed'}})).rejects.toMatchObject({code:'SOURCE_HELD'}); + expect(reference(sourceId).state).toBe('held'); +}); +it('a stale owner cannot acknowledge, retry or hold the current sender reference',async()=>{ + const {store,sourceId,body}=await held();await store.markSending(target,sourceId,body); + const ref=reference(sourceId);ref.owner.start='mismatched-generation'; + writeFileSync(join(dir,'queue',sourceId+'.ref'),JSON.stringify(ref)); + for(const operation of [()=>store.acknowledge(target,sourceId),()=>store.retry(target,sourceId,'retry'),()=>store.hold(target,sourceId,'hold')]){ + await expect(operation()).rejects.toMatchObject({code:'SOURCE_HELD'}); + expect(reference(sourceId)).toEqual(ref);expect(manifest(sourceId).completed).toBe(false); + } +}); diff --git a/test/sdk-native-deadline.test.ts b/test/sdk-native-deadline.test.ts new file mode 100644 index 0000000000..35615756b0 --- /dev/null +++ b/test/sdk-native-deadline.test.ts @@ -0,0 +1,32 @@ +import {afterEach,expect,it} from 'vitest'; +import {mkdtempSync,readFileSync,readdirSync,rmSync} from 'node:fs'; +import {join,resolve} from 'node:path'; +import {tmpdir} from 'node:os'; +import {spawn} from 'node:child_process'; +import {isolatedHookEnv} from './helpers/isolated-hook-env.js'; +const roots:string[]=[]; +afterEach(()=>{for(const dir of roots.splice(0))rmSync(dir,{recursive:true,force:true});}); +async function run(localMs:number,replyMs:number){ + const root=mkdtempSync(join(tmpdir(),'native-sdk-deadline-'));roots.push(root);const dir=join(root,'spool-v2'),trace=join(root,'trace.jsonl'); + const env=isolatedHookEnv(root,{AGENTMEMORY_URL:'http://127.0.0.1:43111',AGENTMEMORY_SECRET:'synthetic-only',AGENTMEMORY_PROJECT_NAME:'synthetic',AGENTMEMORY_SDK_CAPTURE_V2_DIR:dir,TMPDIR:root,TMP:root,TEMP:root,FIXTURE_TRACE:trace,FIXTURE_LOCAL_DELAY_MS:String(localMs),FIXTURE_REPLY_DELAY_MS:String(replyMs)}); + const entry=process.env.SDK_NATIVE_FIXTURE_ENTRY??resolve('src/hooks/post-tool-use.ts'); + const child=spawn(process.execPath,['--import',resolve('test/helpers/sdk-native-deadline-preload.mjs'),'--import',resolve('node_modules/tsx/dist/loader.mjs'),entry,'--agentmemory-sdk-capture-v2'],{cwd:root,env,stdio:['pipe','pipe','pipe']}); + let stderr='',stdout='';child.stderr.on('data',v=>stderr+=v);child.stdout.on('data',v=>stdout+=v); + const watchdog=setTimeout(()=>child.kill(),8000); + child.stdin.end(JSON.stringify({session_id:'synthetic',tool_name:'fixture_tool',tool_input:{synthetic:true},tool_response:'synthetic',cwd:root})); + const exit=await new Promise((done,reject)=>{child.once('exit',done);child.once('error',reject);});clearTimeout(watchdog); + const ids=readdirSync(join(dir,'originals'));expect(ids).toHaveLength(1); + const manifest=JSON.parse(readFileSync(join(dir,'originals',ids[0],'manifest.json'),'utf8')); + const refs=readdirSync(join(dir,'queue')).filter(v=>v.endsWith('.ref')).map(v=>JSON.parse(readFileSync(join(dir,'queue',v),'utf8'))); + return {exit,stderr,stdout,manifest,refs,trace:readFileSync(trace,'utf8').trim().split('\n').map(v=>JSON.parse(v))}; +} +for(const [localMs,replyMs] of [[0,20],[900,1000]])it(`native shared deadline completes ACK after local ${localMs}ms and reply ${replyMs}ms`,async()=>{ + const r=await run(localMs,replyMs);expect(r.exit).toBe(0);expect(r.stderr).toBe('');expect(r.stdout).toBe('');expect(r.manifest.completed).toBe(true);expect(r.refs).toEqual([]);expect(r.trace.filter(v=>v.event==='mock_ack_returned')).toHaveLength(1); +},10000); +it('native shared deadline aborts a late POST and durably parks before the hard exit cap',async()=>{ + const r=await run(1000,2800);expect(r.exit).toBe(1);expect(r.stdout).toBe('');expect(r.stderr).toContain('SDK_SEND_UNCERTAIN');expect(r.stderr).not.toContain('CAPTURE_LOCAL_HANDOFF_UNCONFIRMED'); + expect(r.manifest.completed).toBe(false);expect(r.refs).toHaveLength(1);expect(r.refs[0]).toMatchObject({state:'held',reason:'HELD',attempts:0});expect(r.refs[0].owner).toBeUndefined(); + expect(r.trace.filter(v=>v.event==='mock_ack_returned')).toHaveLength(0);expect(r.trace.filter(v=>v.event==='mock_post_aborted')).toHaveLength(1); + const start=r.trace.find(v=>v.event==='native_cap_armed').atMs,end=r.trace.find(v=>v.event==='process_exit').atMs; + expect(end-start).toBeLessThan(3500);console.log(JSON.stringify({syntheticNativeDeadline:true,localDelayMs:1000,mockReplyMs:2800,state:r.refs[0].state,attempts:r.refs[0].attempts,elapsedAfterCapArmedMs:end-start,networkCalls:0})); +},10000); diff --git a/test/sdk-server-bundle.test.ts b/test/sdk-server-bundle.test.ts index 7db717be3f..badb193a76 100644 --- a/test/sdk-server-bundle.test.ts +++ b/test/sdk-server-bundle.test.ts @@ -10,7 +10,7 @@ import config from '../tsdown.config.js'; it('ships the changed server SDK/parsers without resolving them from an old operating dependency tree',async()=>{ const root=await mkdtemp(join(tmpdir(),'am-sdk-closure-')); try{ - await writeFile(join(root,'package.json'),JSON.stringify({type:'module',dependencies:{'iii-sdk':'0.25.0','@iii-dev/helpers':'0.25.0',yaml:'^2.8.1',json5:'^2.2.3'}})); + await writeFile(join(root,'package.json'),JSON.stringify({type:'module',dependencies:{'iii-sdk':'0.25.1','@iii-dev/helpers':'0.25.1',yaml:'^2.8.1',json5:'^2.2.3'}})); await symlink(await realpath(resolve('node_modules')),join(root,'node_modules'),'dir'); await writeFile(join(root,'entry.ts'),`import {registerWorker} from 'iii-sdk';import {http} from '@iii-dev/helpers/http';import {parse} from 'yaml';import JSON5 from 'json5';export const factory=registerWorker;export const helper=http;export const value=JSON5.parse('{ok:true}').ok&&parse('ok: true').ok;`); const settings=Array.isArray(config)?config[0]:config; diff --git a/test/sdk-server-closure-audit.test.ts b/test/sdk-server-closure-audit.test.ts index 6303f3e0ed..09725c5cdb 100644 --- a/test/sdk-server-closure-audit.test.ts +++ b/test/sdk-server-closure-audit.test.ts @@ -5,7 +5,7 @@ import {auditServerSdkBundle} from '../scripts/audit-server-sdk-bundle.mjs'; let root:string; beforeEach(async()=>{ root=await mkdtemp(join(tmpdir(),'am-closure-audit-'));await mkdir(join(root,'dist')); - for(const [name,version]of Object.entries({'iii-sdk':'0.25.0','@iii-dev/helpers':'0.25.0',yaml:'2.8.1',json5:'2.2.3'})){ + for(const [name,version]of Object.entries({'iii-sdk':'0.25.1','@iii-dev/helpers':'0.25.1',yaml:'2.8.1',json5:'2.2.3'})){ await mkdir(join(root,'node_modules',name),{recursive:true});await writeFile(join(root,'node_modules',name,'package.json'),JSON.stringify({version})); } await writeFile(join(root,'dist/cli.mjs'),"import fs from 'node:fs';export const cli=true;");await writeFile(join(root,'dist/index.mjs'),"export const worker=true;"); diff --git a/test/sdk-source-restore.test.ts b/test/sdk-source-restore.test.ts index 1bfb751035..f67d52b344 100644 --- a/test/sdk-source-restore.test.ts +++ b/test/sdk-source-restore.test.ts @@ -65,10 +65,18 @@ it.each([-1,'2',4,1.5])('holds malformed restoration metadata without resetting const f=fixture();await f.sdk.trigger('mem::capture',{payload:body,eventId:body.eventId});const rec=await f.kv.get(KV.captureInbox,f.key);await f.kv.set(KV.captureInbox,f.key,{...rec,sdkRestoreAttempts:attempts});await f.kv.delete(KV.observations(body.sessionId),f.id); expect((await f.call()).body.completed).toBe(false);expect((await f.kv.get(KV.captureInbox,f.key)).sdkRestoreAttempts).toBe(attempts); }); -it('shares inbox admission with ordinary capture and never bypasses capacity',async()=>{ +it('retains concurrent SDK restores outside capacity while bounding genuine ordinary pending capture',async()=>{ vi.stubEnv('AGENTMEMORY_CAPTURE_INBOX_MAX','1');try{ - const f=fixture();const values=await Promise.all([f.call(),f.call({...body,eventId:'synthetic_restore_002'})]);expect(values.map(r=>r.status_code).sort()).toEqual([201,409]);expect(await f.kv.list(KV.captureInbox)).toHaveLength(1); - const third=await f.sdk.trigger('mem::capture',{payload:body,eventId:'synthetic_restore_003'})as any;expect(third).toMatchObject({status:'rejected',retryable:true});expect(await f.kv.list(KV.captureInbox)).toHaveLength(1); + const f=fixture();const values=await Promise.all([f.call(),f.call({...body,eventId:'synthetic_restore_002'})]); + expect(values.map(r=>r.status_code)).toEqual([201,201]); + const retained=await f.kv.list(KV.captureInbox);expect(retained).toHaveLength(2); + for(const row of retained)expect(row).toMatchObject({storageCompletion:'sdk',sdkRestoreState:'completed',payload:{data:body.data}}); + f.sdk.fns.set('mem::observe',async()=>{throw Error('synthetic ordinary pending');}); + const pending=await f.sdk.trigger('mem::capture',{payload:body,eventId:'synthetic_restore_003'}); + expect(pending).toMatchObject({status:'accepted',state:'retrying'}); + const blocked=await f.sdk.trigger('mem::capture',{payload:body,eventId:'synthetic_restore_004'}); + expect(blocked).toMatchObject({status:'rejected',retryable:true});expect(await f.kv.list(KV.captureInbox)).toHaveLength(3); + for(const row of retained)expect(await f.kv.get(KV.captureInbox,row.key)).toEqual(row); }finally{vi.unstubAllEnvs();} }); it('records a body-free attempt audit without calling it a durable completion receipt',async()=>{ diff --git a/test/sdk-spool-progress.test.ts b/test/sdk-spool-progress.test.ts new file mode 100644 index 0000000000..fd99611904 --- /dev/null +++ b/test/sdk-spool-progress.test.ts @@ -0,0 +1,172 @@ +import {afterEach,expect,it,vi} from 'vitest'; +import {createHash} from 'node:crypto'; +import {chmodSync,mkdirSync,mkdtempSync,readFileSync,lstatSync,readdirSync,symlinkSync,rmSync,truncateSync,writeFileSync} from 'node:fs'; +import {join} from 'node:path'; +import {tmpdir} from 'node:os'; +import {performance} from 'node:perf_hooks'; +import {SdkSpoolV2} from '../src/capture/sdk-spool-v2.js'; +import {deliverSdkCaptureV2,drainSdkCapturesV2} from '../src/capture/sdk-delivery-v2.js'; + +const BLOCK=4096,MiB=1024*1024,target='https://synthetic.invalid'; +const roots:string[]=[]; +const hash=(v:string|Buffer)=>createHash('sha256').update(v).digest('hex'); +const canonical=(v:any):string=>Array.isArray(v)?'['+v.map(canonical).join(',')+']':v&&typeof v==='object'?'{'+Object.keys(v).sort().map(k=>JSON.stringify(k)+':'+canonical(v[k])).join(',')+'}':JSON.stringify(v); +const ack=(body:Record)=>({status:201,body:{eventId:body.eventId,status:'accepted',completion:'sdk',completed:true,durable:false}}); +afterEach(()=>{vi.restoreAllMocks();for(const root of roots.splice(0))rmSync(root,{recursive:true,force:true});}); +function root(){const path=mkdtempSync(join(tmpdir(),'sdk-progress-'));roots.push(path);return path;} +async function fixture(refs=1240,completed=2328,textBytes=32*1024){ + const dir=root(),store=new SdkSpoolV2({dir});await store.usage(); + const selected:Array<{sourceId:string;body:Record}>=[]; + for(let index=0;index=refs}),{mode:0o600}); + if(index{ + // Keep reference count portable; exact byte saturation below still tests the same cap. + // ext4 directory blocks can make 1240 refs alone exceed 5MiB on some layouts. + const {dir,store,selected}=await fixture(1200,2368); + const filler=join(dir,'queue','legacy-preserved.tmp');writeFileSync(filler,'',{mode:0o600}); + fill(filler,5*MiB-(await store.usage()).queueBytes+BLOCK); + const before=await store.usage();expect(before.queueBytes).toBeLessThanOrEqual(5*MiB);expect(5*MiB-before.queueBytes).toBeLessThan(2*BLOCK); + const source=selected[0],chunk=join(dir,'originals',source.sourceId,'000000.chunk'),digest=hash(readFileSync(chunk)); + const extra=await store.retain(target,'new-after-full',{eventId:'new-after-full',data:'synthetic'}); + await expect(store.enqueue(target,extra.sourceId)).rejects.toMatchObject({code:'QUEUE_FULL'}); + await store.markSending(target,source.sourceId);await store.retry(target,source.sourceId,'known-not-sent'); + vi.spyOn(Date,'now').mockReturnValue(Date.now()+10000); + await store.markSending(target,source.sourceId);await store.hold(target,source.sourceId,'unknown'); + await expect(store.markSending(target,source.sourceId)).rejects.toMatchObject({code:'SOURCE_HELD'}); + await store.markSending(target,source.sourceId,source.body);await store.acknowledge(target,source.sourceId); + expect(hash(readFileSync(chunk))).toBe(digest);expect(readdirSync(join(dir,'originals'))).toHaveLength(3569); + expect(JSON.parse(readFileSync(join(dir,'originals',source.sourceId,'manifest.json'),'utf8')).completed).toBe(true); + await store.markSending(target,selected[1].sourceId);await store.acknowledge(target,selected[1].sourceId); + await expect(store.enqueue(target,extra.sourceId)).resolves.toBeUndefined(); + expect((await store.usage()).queueBytes).toBeLessThanOrEqual(5*MiB+2*BLOCK); +},30000); + +it('ACKs at the original admission limit while retaining chunks and refusing new originals',async()=>{ + const dir=root(),store=new SdkSpoolV2({dir}),body={eventId:'original-limit',data:'synthetic'}; + const source=await store.retain(target,body.eventId,body);await store.enqueue(target,source.sourceId);await store.markSending(target,source.sourceId); + const before=await store.usage(),limited=new SdkSpoolV2({dir,maxOriginalBytes:before.originalBytes}); + await expect(limited.retain(target,'new',{eventId:'new'})).rejects.toMatchObject({code:'ORIGINALS_FULL'}); + await limited.acknowledge(target,source.sourceId); + expect(await limited.load(target,source.sourceId)).toEqual(body);expect((await limited.usage()).originalBytes).toBeLessThanOrEqual(before.originalBytes+17*BLOCK); +}); + +it('counts preserved crash files against transition workspace and refuses overcommit',async()=>{ + const {dir,store,selected}=await fixture(1,0,256),before=await store.usage(); + fill(join(dir,'queue','preserved-crash.tmp'),BLOCK); + await expect(new SdkSpoolV2({dir,maxQueueBytes:before.queueBytes}).markSending(target,selected[0].sourceId)).rejects.toMatchObject({code:'QUEUE_FULL'}); + expect((await store.pending(target))[0].state).toBe('pending');expect(readFileSync(join(dir,'queue','preserved-crash.tmp'))).toHaveLength(BLOCK); + await store.markSending(target,selected[0].sourceId); + fill(join(dir,'originals','preserved-crash.tmp'),17*BLOCK); + await expect(new SdkSpoolV2({dir,maxOriginalBytes:before.originalBytes}).acknowledge(target,selected[0].sourceId)).rejects.toMatchObject({code:'ORIGINALS_FULL'}); + expect((await store.pending(target))[0].state).toBe('sending');expect(await store.load(target,selected[0].sourceId)).toEqual(selected[0].body); +}); + +it('delivers more than 1200 retained refs without deleting originals or replaying completed entries',async()=>{ + const {dir,store,selected}=await fixture(1201,0,256);let calls=0;const start=performance.now(); + for(const source of selected){const result=await deliverSdkCaptureV2(target,source.body,async body=>{calls++;return ack(body);},{dir});expect(result.outcome).toBe('delivered');} + expect(calls).toBe(1201);expect(await store.pending(target)).toEqual([]);expect(readdirSync(join(dir,'originals'))).toHaveLength(1201); + for(const source of selected){const manifest=JSON.parse(readFileSync(join(dir,'originals',source.sourceId,'manifest.json'),'utf8'));expect(manifest.completed).toBe(true);expect(hash(readFileSync(join(dir,'originals',source.sourceId,'000000.chunk')))).toBe(manifest.hash);} + const completed=await deliverSdkCaptureV2(target,selected[0].body,async body=>{calls++;return ack(body);},{dir});expect(completed.outcome).toBe('duplicate');expect(calls).toBe(1201); + console.log(JSON.stringify({syntheticSpoolDrain:true,delivered:calls,originalsPreserved:1201,elapsedMs:Math.round(performance.now()-start)})); +},120000); + +it('one sender owns delivery while concurrent attempts and unknown replies cannot replay',async()=>{ + const dir=root(),body={eventId:'concurrent',data:'synthetic'};let calls=0,release!:()=>void,started!:()=>void; + const entered=new Promise(resolve=>started=resolve),wait=new Promise(resolve=>release=resolve); + const first=deliverSdkCaptureV2(target,body,async()=>{calls++;started();await wait;throw Error('uncertain');},{dir});await entered; + const second=await deliverSdkCaptureV2(target,body,async value=>{calls++;return ack(value);},{dir});expect(second).toMatchObject({outcome:'held',code:'SDK_SEND_IN_PROGRESS_OR_UNCERTAIN'}); + release();expect(await first).toMatchObject({outcome:'held',code:'SDK_SEND_UNCERTAIN'}); + const third=await deliverSdkCaptureV2(target,body,async value=>{calls++;return ack(value);},{dir});expect(third).toMatchObject({outcome:'held',code:'SOURCE_HELD'});expect(calls).toBe(1); +}); + +it('a malformed unrelated reference cannot prevent selected delivery',async()=>{ + const {dir,selected}=await fixture(2,0,256),unrelated=join(dir,'queue',selected[1].sourceId+'.ref');writeFileSync(unrelated,'{malformed'); + let calls=0;expect(await deliverSdkCaptureV2(target,selected[0].body,async body=>{calls++;return ack(body);},{dir})).toMatchObject({outcome:'delivered'}); + expect(calls).toBe(1);expect(readFileSync(unrelated,'utf8')).toBe('{malformed'); +}); + +it('lists metadata and verifies only selected source bytes for direct delivery and bounded drain',async()=>{ + const {dir,store,selected}=await fixture(1200,2368);let verified=0; + const options={dir,storeOptions:{onPhase:(phase:string)=>{if(phase==='source_verified')verified++;}}}; + const start=performance.now(),records=await new SdkSpoolV2({...options.storeOptions,dir}).pending(target),listingMs=performance.now()-start; + expect(records).toHaveLength(1200);expect(verified).toBe(0); + const result=await deliverSdkCaptureV2(target,selected[0].body,async body=>ack(body),options); + expect(result.outcome).toBe('delivered');expect(verified).toBeLessThanOrEqual(5); + verified=0;const drained=await drainSdkCapturesV2(target,async body=>ack(body),{...options,limit:1,budgetMs:1500}); + expect(drained).toMatchObject({attempted:1,delivered:1,remaining:1198});expect(verified).toBeLessThanOrEqual(3); + console.log(JSON.stringify({syntheticSpoolProgress:true,refs:1200,originals:3568,listingMs:Math.round(listingMs),selectedSourceReads:verified})); + expect((await store.usage()).originalBytes).toBeGreaterThan(100*MiB); +},30000); + +it('queue transitions do not inspect unrelated original trees and corrupt selected bytes never POST',async()=>{ + const {dir,store,selected}=await fixture(2,1),unrelated=join(dir,'originals','unrelated'); + mkdirSync(unrelated,{mode:0o700});writeFileSync(join(unrelated,'private'),'',{mode:0o600});chmodSync(join(unrelated,'private'),0o644); + await store.markSending(target,selected[0].sourceId);await store.hold(target,selected[0].sourceId,'unknown'); + const chunk=join(dir,'originals',selected[1].sourceId,'000000.chunk');writeFileSync(chunk,'corrupt');let posts=0; + const result=await deliverSdkCaptureV2(target,selected[1].body,async body=>{posts++;return ack(body);},{dir}); + expect(result.outcome).toBe('dropped');expect(posts).toBe(0);expect(readFileSync(chunk,'utf8')).toBe('corrupt'); +}); + + +it('preserves crashed transition workspace and refuses replacement without consuming another slot',async()=>{ + const {dir,store,selected}=await fixture(1,0,256); + const slot=join(dir,'queue','.transitions','pending.tmp');writeFileSync(slot,'synthetic crash evidence',{mode:0o600}); + const before=await store.usage(); + await expect(store.markSending(target,selected[0].sourceId)).rejects.toMatchObject({code:'QUEUE_FULL'}); + expect(readFileSync(slot,'utf8')).toBe('synthetic crash evidence'); + expect(await store.usage()).toEqual(before);expect((await store.pending(target))[0].state).toBe('pending'); +}); + +it('refuses legacy workspace bootstrap over its reserved peak without deleting preserved pressure',async()=>{ + const {dir,store,selected}=await fixture(1,0,256); + rmSync(join(dir,'queue','.transitions'),{recursive:true}); + const before=await store.usage(),pressure=join(dir,'queue','legacy-crash.tmp'); + fill(pressure,5*MiB-before.queueBytes+BLOCK); + const usage=await store.usage(); + await expect(store.markSending(target,selected[0].sourceId)).rejects.toMatchObject({code:'QUEUE_FULL'}); + expect(await store.usage()).toEqual(usage); + expect(readdirSync(join(dir,'queue'))).not.toContain('.transitions'); + expect((await store.pending(target))[0].state).toBe('pending'); +}); + + +it('keeps actual staged replacement peak inside the unchanged 8KiB reserve',async()=>{ + const {dir,store,selected}=await fixture(1,0,256); + const before=await store.usage();let peaks=0; + const measure=(path:string):number=>{const stat=lstatSync(path);return Math.max(BLOCK,Math.ceil(stat.size/BLOCK)*BLOCK,stat.blocks*512)+(stat.isDirectory()?readdirSync(path).reduce((sum,name)=>sum+measure(join(path,name)),0):0);}; + const constrained=new SdkSpoolV2({dir,maxQueueBytes:before.queueBytes,onPhase:phase=>{ + if(phase==='queue_transition_staged'){peaks++;expect(measure(join(dir,'queue'))+BLOCK).toBeLessThanOrEqual(before.queueBytes+2*BLOCK);} + }}); + await constrained.markSending(target,selected[0].sourceId);await constrained.hold(target,selected[0].sourceId,'unknown'); + expect(peaks).toBe(2);expect(readdirSync(join(dir,'queue','.transitions'))).toEqual([]); + expect(await store.load(target,selected[0].sourceId)).toEqual(selected[0].body); +}); + +it('rejects a symlinked transition workspace without writing outside the spool',async()=>{ + const {dir,store,selected}=await fixture(1,0,256),outside=root(); + rmSync(join(dir,'queue','.transitions'),{recursive:true});symlinkSync(outside,join(dir,'queue','.transitions')); + await expect(store.markSending(target,selected[0].sourceId)).rejects.toMatchObject({code:'STORE_UNSAFE'}); + expect(readdirSync(outside)).toEqual([]); +}); + + +it('preserves a legacy 1240-ref layout that is actually over the byte quota',async()=>{ + const {dir,store,selected}=await fixture(1240,0,256); + rmSync(join(dir,'queue','.transitions'),{recursive:true}); + const usage=await store.usage(); + if(usage.queueBytes<=5*MiB)fill(join(dir,'queue','preserved-pressure.tmp'),5*MiB-usage.queueBytes+BLOCK); + const before=await store.usage();expect(before.queueBytes).toBeGreaterThan(5*MiB); + await expect(store.markSending(target,selected[0].sourceId)).rejects.toMatchObject({code:'QUEUE_FULL'}); + expect(await store.usage()).toEqual(before);expect(await store.pending(target)).toHaveLength(1240); + expect(readdirSync(join(dir,'originals'))).toHaveLength(1240); + expect(await store.load(target,selected[0].sourceId)).toEqual(selected[0].body); +}); diff --git a/test/sdk-spool-v2.test.ts b/test/sdk-spool-v2.test.ts index 61ca0bdc56..499d804fa2 100644 --- a/test/sdk-spool-v2.test.ts +++ b/test/sdk-spool-v2.test.ts @@ -119,11 +119,12 @@ describe('SdkSpoolV2 durable originals and references',()=>{ expect(results.filter(value=>value.status==='error').every(value=>value.code==='ORIGINALS_FULL')).toBe(true); expect((await store().usage()).originalBytes).toBeLessThanOrEqual(20480); }); - it('enforces queue quota on sending-state atomic replacement',async()=>{ + it('reserves bounded sending-state workspace beyond the queue admission quota',async()=>{ const s=store();const r=await s.retain(target,'event-1',body());await s.enqueue(target,r.sourceId); const before=(await s.usage()).queueBytes;const constrained=store({maxQueueBytes:before}); - await expect(constrained.markSending(target,r.sourceId)).rejects.toMatchObject({code:'QUEUE_FULL'}); - expect((await s.pending(target))[0].state).toBe('pending'); + await expect(constrained.markSending(target,r.sourceId)).resolves.toBeUndefined(); + expect((await s.pending(target))[0].state).toBe('sending'); + expect((await s.usage()).queueBytes).toBeLessThanOrEqual(before+8192); }); it('fails explicitly when a conflict original cannot fit',async()=>{ const s=store();const r=await s.retain(target,'event-1',body());const before=(await s.usage()).originalBytes; @@ -147,7 +148,7 @@ describe('SdkSpoolV2 durable originals and references',()=>{ it('uses one bounded lease file and no growing ticket bookkeeping',async()=>{ const s=store();for(let index=0;index<10;index++)await s.usage(); - expect(readdirSync(join(dir,'queue'))).toEqual(['lease.lock']); + expect(readdirSync(join(dir,'queue'))).toEqual(['.transitions','lease.lock']); expect(statSync(join(dir,'queue','lease.lock')).size).toBe(0); }); it('kernel releases crashed writer lease while preserving its partial original',async()=>{ @@ -157,7 +158,7 @@ describe('SdkSpoolV2 durable originals and references',()=>{ expect(output.trim()).toBe('synthetic-crash'); const s=store();const r=await s.retain(target,'event-1',body());expect(await s.load(target,r.sourceId)).toEqual(body()); expect(readdirSync(join(dir,'originals')).some(name=>name.startsWith('.stage-'))).toBe(true); - expect(readdirSync(join(dir,'queue'))).toEqual(['lease.lock']); + expect(readdirSync(join(dir,'queue'))).toEqual(['.transitions','lease.lock']); }); it('rejects impossible fixture budgets below fixed directory and lease charges',()=>{ diff --git a/test/sdk-v2-transport-safety.test.ts b/test/sdk-v2-transport-safety.test.ts index 9158b0f0aa..aadf92a4fa 100644 --- a/test/sdk-v2-transport-safety.test.ts +++ b/test/sdk-v2-transport-safety.test.ts @@ -40,8 +40,8 @@ it('does not POST when send-intent fsync consumes the monotonic drain budget; ke it('passes the original absolute deadline and fresh remaining budget after source readback',async()=>{ await queued();let clock=1000,received:unknown; vi.spyOn(performance,'now').mockImplementation(()=>clock); - const load=SdkSpoolV2.prototype.load; - vi.spyOn(SdkSpoolV2.prototype,'load').mockImplementation(async function(this:SdkSpoolV2,t,id){const value=await load.call(this,t,id);clock+=30;return value;}); + const claim=SdkSpoolV2.prototype.claimForSend; + vi.spyOn(SdkSpoolV2.prototype,'claimForSend').mockImplementation(async function(this:SdkSpoolV2,t,id,deadline){const value=await claim.call(this,t,id,deadline);clock+=30;return value;}); const result=await drainSdkCapturesV2(target,async(b,...args:unknown[])=>{received=args[0];return ack(b);},{dir:join(root,'v2'),limit:1,budgetMs:250,storeOptions:{onPhase:phase=>{if(phase==='send_intent_fsynced')clock+=70;}}}); expect(received).toEqual({deadline:1250,remainingMs:150}); expect(result).toMatchObject({attempted:1,delivered:1,held:0,remaining:0}); @@ -56,7 +56,7 @@ async function listen(handler:(req:IncomingMessage,res:ServerResponse)=>void){ async function readBody(req:IncomingMessage){const chunks:Buffer[]=[];for await(const chunk of req)chunks.push(Buffer.from(chunk));return JSON.parse(Buffer.concat(chunks).toString()) as Record;} async function runNative(url:string){ const image='data:image/png;base64,'+'A'.repeat(300*1024); - const child=spawn('/usr/bin/node',['--max-old-space-size=256','--import',resolve('node_modules/tsx/dist/loader.mjs'),resolve('src/hooks/post-tool-use.ts'),'--agentmemory-sdk-capture-v2'],{ + const child=spawn(process.execPath,['--max-old-space-size=256','--import',resolve('node_modules/tsx/dist/loader.mjs'),resolve('src/hooks/post-tool-use.ts'),'--agentmemory-sdk-capture-v2'],{ cwd:root,env:{PATH:'/usr/bin:/bin',HOME:root,TMPDIR:root,AGENTMEMORY_URL:url,AGENTMEMORY_SECRET:'synthetic-only',AGENTMEMORY_SDK_CAPTURE_V2_DIR:join(root,'v2'),AGENTMEMORY_CAPTURE_SPOOL_DIR:join(root,'v1')},stdio:['pipe','pipe','pipe'], }); let stdout='',stderr='';child.stdout.on('data',b=>stdout+=b);child.stderr.on('data',b=>stderr+=b);child.stdin.on('error',()=>{}); diff --git a/test/sealed-recovery-procedure.test.ts b/test/sealed-recovery-procedure.test.ts new file mode 100644 index 0000000000..5949c2dffe --- /dev/null +++ b/test/sealed-recovery-procedure.test.ts @@ -0,0 +1,11 @@ +import {it,expect,vi} from 'vitest';import {createHash} from 'node:crypto'; +import {sealCurrentForRecovery} from '../deploy/recovery/seal-current.mjs'; +const identity={instanceId:'synthetic-state',storePath:'/synthetic/store'}; +const markerBytes=JSON.stringify({version:4,clean:false,epoch:{state:{instanceId:identity.instanceId}}}); +function fixture(){const h={backgroundControl:{status:'error',pausedRequested:true,activeOperations:0,holds:['deployment','storage_error']},storageDurability:{activeMutations:0,mutationFailures:1,lastVerification:{errorCode:'SDK_MUTATION_UNCERTAIN'}}};return{backgroundRecord:vi.fn(async()=>({version:2,paused:true,holds:['deployment','storage_error'],resumePending:true,recovery:{attempts:2,nextAttemptAt:42}})),health:vi.fn(async()=>h),current:()=>true,seal:vi.fn(async()=>({version:1,...identity,sealed:true,synced:true})),identity,markerBytes,expectedMarkerSha256:createHash('sha256').update(markerBytes).digest('hex'),persist:vi.fn(async()=>{}),h};} +it('preserves actual failed-count/holds and never claims clean after exact State seal',async()=>{const f=fixture();const p=await sealCurrentForRecovery(f);expect(p).toMatchObject({verifiedClean:false,semanticOutcomeUncertain:true,before:{storage:{mutationFailures:1}},after:{storage:{mutationFailures:1},background:{holds:['deployment','storage_error']}}});expect(f.seal).toHaveBeenCalledOnce();expect(f.persist).toHaveBeenCalledWith(p);}); +it.each(['activeMutation','activeOperation','unpaused','noDeployment','changedMarker','changedEpoch'])('refuses %s before irreversible seal',async(kind)=>{const f=fixture();if(kind==='activeMutation')f.h.storageDurability.activeMutations=1;if(kind==='activeOperation')f.h.backgroundControl.activeOperations=1;if(kind==='unpaused')f.h.backgroundControl.pausedRequested=false;if(kind==='noDeployment')f.h.backgroundControl.holds=['storage_error'];if(kind==='changedMarker')f.expectedMarkerSha256='0'.repeat(64);if(kind==='changedEpoch')f.current=()=>false;await expect(sealCurrentForRecovery(f)).rejects.toThrow();expect(f.seal).not.toHaveBeenCalled();expect(f.persist).not.toHaveBeenCalled();}); +it.each(['epochAfter','wrongInstance','unsynced','lostHistory','persistFailure'])('does not certify completion for %s',async(kind)=>{const f=fixture();if(kind==='epochAfter'){let n=0;f.current=()=>++n===1;}if(kind==='wrongInstance')f.seal=vi.fn(async()=>({version:1,...identity,instanceId:'other',sealed:true,synced:true}));if(kind==='unsynced')f.seal=vi.fn(async()=>({version:1,...identity,sealed:true,synced:false}));if(kind==='lostHistory'){let n=0;f.health=vi.fn(async()=>++n===1?f.h:{...f.h,storageDurability:{...f.h.storageDurability,mutationFailures:0}});}if(kind==='persistFailure')f.persist=vi.fn(async()=>{throw Error('fsync failed');});await expect(sealCurrentForRecovery(f)).rejects.toThrow();}); + +it('refuses absent persisted deployment hold before seal',async()=>{const f=fixture();f.backgroundRecord=vi.fn(async()=>({version:2,paused:false,holds:[],resumePending:true,recovery:{attempts:2,nextAttemptAt:42}}));await expect(sealCurrentForRecovery(f)).rejects.toThrow('RECOVERY_PERSISTED_DEPLOYMENT_HOLD_REQUIRED');expect(f.seal).not.toHaveBeenCalled();}); +it('retains exact persisted recovery counters and refuses changes after seal',async()=>{const f=fixture();let n=0;f.backgroundRecord=vi.fn(async()=>({version:2,paused:true,holds:['deployment','storage_error'],resumePending:true,recovery:{attempts:++n,nextAttemptAt:42}}));await expect(sealCurrentForRecovery(f)).rejects.toThrow('RECOVERY_PERSISTED_HOLD_CHANGED');expect(f.persist).not.toHaveBeenCalled();}); diff --git a/test/search-project-isolation.test.ts b/test/search-project-isolation.test.ts index 99069dd7eb..83a4d9b413 100644 --- a/test/search-project-isolation.test.ts +++ b/test/search-project-isolation.test.ts @@ -198,7 +198,7 @@ describe.each(["keyword", "hybrid", "hybrid-fallback"] as const)( }); if (ranking === "hybrid") { - it("expands a real hybrid window shortened by a missing record", async () => { + it("fills a real hybrid window shortened by a missing record", async () => { for (let i = 0; i < 100; i++) { addMemory({ id: `mem_foreign_${i.toString().padStart(3, "0")}`, project: "project-b" }); } @@ -210,7 +210,7 @@ describe.each(["keyword", "hybrid", "hybrid-fallback"] as const)( const ranker = vi.fn(hybrid.search.bind(hybrid)); setHybridRanker(ranker); expect(await recall({ limit: 1 })).toEqual([matching.id]); - expect(ranker.mock.calls.map((call) => call[1])).toEqual([100, 200]); + expect(ranker.mock.calls.map((call) => call[1])).toEqual([100]); }); } diff --git a/test/session-end-transcript.test.ts b/test/session-end-transcript.test.ts index c9f452e8b0..4ed260733a 100644 --- a/test/session-end-transcript.test.ts +++ b/test/session-end-transcript.test.ts @@ -8,14 +8,16 @@ import { join } from "node:path"; let server: Server; let port: number; const posts: Array<{ path: string; body: Record }> = []; +let observeDelayMs = 0; function runHook( payload: Record, script = "plugin/scripts/session-end.mjs", + extraEnv: Record = {}, ): Promise { return new Promise((resolve) => { const child = spawn("node", [script], { - env: { ...process.env, AGENTMEMORY_URL: `http://127.0.0.1:${port}` }, + env: { ...process.env, AGENTMEMORY_URL: `http://127.0.0.1:${port}`, ...extraEnv }, }); child.on("exit", (code) => resolve(code ?? 1)); child.stdin.write(JSON.stringify(payload)); @@ -37,8 +39,10 @@ describe("session-end transcript prompt backfill", () => { } catch { posts.push({ path: req.url ?? "", body: {} }); } - res.writeHead(200, { "Content-Type": "application/json" }); - res.end("{}"); + setTimeout(() => { + res.writeHead(200, { "Content-Type": "application/json" }); + res.end("{}"); + }, (req.url ?? "").includes("/observe") ? observeDelayMs : 0); }); }); await new Promise((r) => server.listen(0, "127.0.0.1", r)); @@ -107,6 +111,40 @@ describe("session-end transcript prompt backfill", () => { } }); + it("hands the work to a detached worker so the hook exits inside the 1.5 s SessionEnd budget", async () => { + posts.length = 0; + const transcript = join(dir, "handoff.jsonl"); + writeFileSync(transcript, JSON.stringify({ type: "user", message: { role: "user", content: "handoff prompt" } }) + "\n"); + observeDelayMs = 2500; + const started = Date.now(); + const code = await runHook({ session_id: "ses_handoff", hook_event_name: "SessionEnd", transcript_path: transcript }); + expect(code).toBe(0); + expect(Date.now() - started).toBeLessThan(1500); + const deadline = Date.now() + 10_000; + while (!posts.some((p) => p.path.includes("/session/end")) && Date.now() < deadline) { + await new Promise((r) => setTimeout(r, 50)); + } + observeDelayMs = 0; + const prompts = posts.filter((p) => p.path.includes("/observe")).map((p) => (p.body.data as { prompt: string }).prompt); + expect(prompts).toEqual(["handoff prompt"]); + const end = posts.find((p) => p.path.includes("/session/end")); + expect(end?.body).toMatchObject({ sessionId: "ses_handoff", final: true }); + }, 20_000); + + it("runs the Claude Code SessionEnd work inline inside the detached worker", async () => { + posts.length = 0; + const transcript = join(dir, "worker.jsonl"); + writeFileSync(transcript, JSON.stringify({ type: "user", message: { role: "user", content: "worker prompt" } }) + "\n"); + const code = await runHook( + { session_id: "ses_worker", hook_event_name: "SessionEnd", transcript_path: transcript }, + undefined, + { AGENTMEMORY_SESSION_END_WORKER: "1" }, + ); + expect(code).toBe(0); + expect(posts.filter((p) => p.path.includes("/observe")).map((p) => (p.body.data as { prompt: string }).prompt)).toEqual(["worker prompt"]); + expect(posts.some((p) => p.path.includes("/session/end"))).toBe(true); + }); + it("posts prompts from a Claude Code transcript, skipping tool results", async () => { posts.length = 0; const transcript = join(dir, "t-cc.jsonl"); diff --git a/test/snapshot.test.ts b/test/snapshot.test.ts index 96ce49b61d..973ca96ef4 100644 --- a/test/snapshot.test.ts +++ b/test/snapshot.test.ts @@ -17,7 +17,8 @@ vi.mock("node:fs/promises", async () => ({ rename: vi.fn(async () => {}), unlink: vi.fn(async () => {}), })); -vi.mock("node:fs", () => ({ +vi.mock("node:fs", async () => ({ + constants: (await vi.importActual("node:fs")).constants, existsSync: vi.fn().mockReturnValue(true), readFileSync: vi.fn(() => snapshotContent()), mkdirSync: vi.fn(), diff --git a/test/state-bootstrap.test.ts b/test/state-bootstrap.test.ts new file mode 100644 index 0000000000..aff542a20e --- /dev/null +++ b/test/state-bootstrap.test.ts @@ -0,0 +1,4 @@ +import {it,expect} from 'vitest'; +import {mkdtempSync,mkdirSync,writeFileSync,symlinkSync,rmSync} from 'node:fs';import {join} from 'node:path';import {tmpdir} from 'node:os'; +import {selectEnginePid} from '../deploy/runtime/state-bootstrap.mjs'; +it('selects only tracked descendants owning the engine listener, rejects ambiguity',()=>{const dir=mkdtempSync(join(tmpdir(),'bootstrap-'));try{const exe=join(dir,'iii');writeFileSync(exe,'fixture');const write=(p:string,s:string)=>{mkdirSync(join(p,'..'),{recursive:true});writeFileSync(p,s)};for(const pid of [10,11]){write(join(dir,`${pid}/task/${pid}/children`),pid===10?'11':'');mkdirSync(join(dir,`${pid}/fd`));symlinkSync(exe,join(dir,`${pid}/exe`));write(join(dir,`${pid}/net/tcp`),'0: 0100007F:C350 00000000:0000 0A 0 0 0 1000 0 600');write(join(dir,`${pid}/net/tcp6`),'');}symlinkSync('socket:[600]',join(dir,'11/fd/3'));expect(selectEnginePid(10,exe,50000,dir)).toBe(11);expect(selectEnginePid(10,exe,50001,dir)).toBeNull();symlinkSync('socket:[600]',join(dir,'10/fd/3'));expect(()=>selectEnginePid(10,exe,50000,dir)).toThrow('AMBIGUOUS');}finally{rmSync(dir,{recursive:true,force:true});}}); diff --git a/test/state-dispatch-boundary.test.ts b/test/state-dispatch-boundary.test.ts new file mode 100644 index 0000000000..7b7d770147 --- /dev/null +++ b/test/state-dispatch-boundary.test.ts @@ -0,0 +1,22 @@ +import {it,expect,vi,afterEach} from 'vitest'; +import {pinStateClient} from '../src/ops/state-epoch.mjs'; +import {StateKV} from '../src/state/kv.js'; +import {createStateDurabilityFence} from '../src/ops/state-durability.mjs'; +afterEach(()=>vi.unstubAllEnvs()); +const prefix='state::22222222-2222-4222-8222-222222222222::'; +it('pre-dispatch rejection does not latch mutation uncertainty or a failed mutation',async()=>{ + vi.stubEnv('AGENTMEMORY_STATE_STORE_PATH','');let current=false;const trigger=vi.fn(async()=>true);const sdk=pinStateClient({trigger} as any,prefix,()=>current); + const durability=createStateDurabilityFence({storePath:'/synthetic/unused',completion:'sdk'});const kv=new StateKV(sdk,{backend:'file',durability}); + await expect(kv.set('fixture','one',{})).rejects.toThrow('STATE_WRITER_IDENTITY_CHANGED');expect(trigger).not.toHaveBeenCalled();expect(kv.mutationOutcomeUncertain).toBe(false);expect(durability.snapshot().mutationFailures).toBe(0); + current=true;await kv.set('fixture','two',{});expect(kv.mutationOutcomeUncertain).toBe(false);expect(trigger).toHaveBeenCalledOnce(); +}); +it.each(['post','transport','forged'])('preserves uncertain latch for %s dispatch failure',async(mode)=>{ + vi.stubEnv('AGENTMEMORY_STATE_STORE_PATH','');let current=true;const sdk=pinStateClient({trigger:async()=>{if(mode==='post'){current=false;return true;}throw Object.assign(Error('STATE_WRITER_IDENTITY_CHANGED'),{dispatched:false,phase:'before_dispatch'});}} as any,prefix,()=>current); + const durability=createStateDurabilityFence({storePath:'/synthetic/unused',completion:'sdk'});const kv=new StateKV(sdk,{backend:'file',durability});await expect(kv.set('fixture','one',{})).rejects.toThrow();expect(kv.mutationOutcomeUncertain).toBe(true);expect(durability.snapshot().mutationFailures).toBe(1); +}); +it('does not clear an existing uncertain outcome when a later precheck rejects',async()=>{ + vi.stubEnv('AGENTMEMORY_STATE_STORE_PATH','');let current=true;const sdk=pinStateClient({trigger:async()=>{current=false;return true;}} as any,prefix,()=>current); + const durability=createStateDurabilityFence({storePath:'/synthetic/unused',completion:'sdk'});const kv=new StateKV(sdk,{backend:'file',durability}); + await expect(kv.set('fixture','first',{})).rejects.toThrow();await expect(kv.set('fixture','second',{})).rejects.toThrow(); + expect(kv.mutationOutcomeUncertain).toBe(true);expect(durability.snapshot().mutationFailures).toBe(1); +}); diff --git a/test/state-durability-key-watch.test.ts b/test/state-durability-key-watch.test.ts new file mode 100644 index 0000000000..5e977fc857 --- /dev/null +++ b/test/state-durability-key-watch.test.ts @@ -0,0 +1,27 @@ +import {expect,it} from 'vitest'; +import {createStateDurabilityFence} from '../src/ops/state-durability.mjs'; + +const selected=[{scope:'mem:obs:synthetic',key:'selected'}]; +const fixture=()=>createStateDurabilityFence({storePath:'/tmp/synthetic-key-watch',completion:'sdk'}); + +it('ignores unrelated keys and dirty-key pruning while observing the selected key',async()=>{ + const fence=fixture();await fence.mutate(selected[0].scope,selected[0].key,async()=>{}); + const watch=fence.watchKeys(selected); + await fence.mutate(selected[0].scope,'other',async()=>{});fence.forgetKeys(fence.keysFor(selected)); + expect(watch.unchanged()).toBe(true);watch.release();expect(watch.unchanged()).toBe(false);watch.release(); +}); +it('rejects selected mutations that begin before or after watch registration including ABA',async()=>{ + for(const startBefore of [false,true]){ + const fence=fixture();let release!:()=>void;const gate=new Promise(resolve=>release=resolve); + const before=startBefore?fence.mutate(selected[0].scope,selected[0].key,()=>gate):null; + const watch=fence.watchKeys(selected),pending=before??fence.mutate(selected[0].scope,selected[0].key,()=>gate); + expect(watch.unchanged()).toBe(false);release();await pending; + await fence.mutate(selected[0].scope,selected[0].key,async()=>{});expect(watch.unchanged()).toBe(false);watch.release(); + } +}); +it('ignores an unrelated pending key without hiding its uncertain SDK outcome',async()=>{ + const fence=fixture();let reject!:(error:Error)=>void; + const pending=fence.mutate(selected[0].scope,'other',()=>new Promise((_,no)=>reject=no));void pending.catch(()=>{}); + const watch=fence.watchKeys(selected);expect(watch.unchanged()).toBe(true);await Promise.resolve();reject(Error('lost response'));await expect(pending).rejects.toThrow('lost response'); + expect(watch.unchanged()).toBe(true);await expect(fence.waitForKeys({get:async()=>({id:'selected'})},selected)).rejects.toMatchObject({code:'SDK_MUTATION_UNCERTAIN'});watch.release(); +}); diff --git a/test/state-epoch.test.ts b/test/state-epoch.test.ts new file mode 100644 index 0000000000..107883b073 --- /dev/null +++ b/test/state-epoch.test.ts @@ -0,0 +1,78 @@ +import {it,expect,afterEach,vi} from 'vitest'; +import {mkdtempSync,mkdirSync,writeFileSync,symlinkSync,rmSync} from 'node:fs'; +import {tmpdir} from 'node:os';import {join} from 'node:path';import {createHash} from 'node:crypto'; +const race=vi.hoisted(()=>({afterRead:null as null|((path:string)=>void)})); +vi.mock('node:fs',async importOriginal=>{const fs=await importOriginal();return {...fs,readdirSync:(path:any,...args:any[])=>{const result=fs.readdirSync(path,...args);race.afterRead?.(String(path));return result;}};}); +afterEach(()=>{race.afterRead=null;}); +const dirs:string[]=[];afterEach(()=>dirs.splice(0).forEach(d=>rmSync(d,{recursive:true,force:true}))); +async function fixture(){ + const d=mkdtempSync(join(tmpdir(),'state-epoch-'));dirs.push(d);const proc=join(d,'proc'),store=join(d,'store'),exe=join(d,'state'); + mkdirSync(store);writeFileSync(exe,'trusted-binary'); + const epoch:any={bootId:'11111111-1111-1111-1111-111111111111',engine:{pid:200,start:'20'},worker:{pid:100,start:'10'},workerUid:1000,netNamespace:'net:[1]',pidNamespace:'pid:[1]',endpoint:'ws://127.0.0.1:1000'}; + const write=(p:string,v:string)=>{mkdirSync(join(p,'..'),{recursive:true});writeFileSync(p,v)}; + const stat=(pid:number,start:string)=>pid+' (state) '+['S',...Array(18).fill('0'),start].join(' '); + for(const pid of [200,300]){write(join(proc,pid+'/stat'),stat(pid,pid===200?'20':'30'));write(join(proc,pid+'/status'),'Uid:\t1000 1000 1000 1000');mkdirSync(join(proc,pid+'/ns'));symlinkSync('net:[1]',join(proc,pid+'/ns/net'));symlinkSync('pid:[1]',join(proc,pid+'/ns/pid'));mkdirSync(join(proc,pid+'/fd'));mkdirSync(join(proc,pid+'/net'))} + symlinkSync(exe,join(proc,'300/exe'));symlinkSync('socket:[600]',join(proc,'300/fd/3'));symlinkSync('socket:[500]',join(proc,'200/fd/3')); + write(join(proc,'300/net/tcp'),'0: 0100007F:07D0 0100007F:03E8 01 0 0 0 1000 0 600\n0: 0100007F:03E8 0100007F:07D0 01 0 0 0 1000 0 500');write(join(proc,'300/net/tcp6'),''); + const identity={version:1,pid:300,start:'30',instanceId:'22222222-2222-4222-8222-222222222222',bootId:epoch.bootId,adapter:'kv',storePath:store}; + const engineEvidence:any={capture:()=>epoch,current:()=>true,gone:()=>true,bootIdentity:()=>epoch.bootId}; + const expectedExecutable={path:exe,sha256:createHash('sha256').update('trusted-binary').digest('hex')}; + const m=await import('../src/ops/state-epoch.mjs'); + return {d,proc,store,exe,identity,epoch,write,stat,engineEvidence,expectedExecutable,m}; +} +it('binds immutable State identity, trusted executable and engine-owned socket pair',async()=>{ + const f=await fixture();const evidence=f.m.createStateBoundEvidence({...f,storePath:f.store,procRoot:f.proc}); + const e=evidence.capture();expect(e?.state.instanceId).toBe(f.identity.instanceId);expect(evidence.current(e!)).toBe(true); + f.write(join(f.proc,'300/stat'),f.stat(300,'31'));expect(evidence.current(e!)).toBe(false); +}); +it.each(['hash','store','socket','boot','uid'])('rejects State %s mismatch',async(kind)=>{ + const f=await fixture(); + if(kind==='hash')f.expectedExecutable.sha256='f'.repeat(64); + if(kind==='store')f.identity.storePath=f.d; + if(kind==='socket')rmSync(join(f.proc,'200/fd/3')); + if(kind==='boot')f.identity.bootId='33333333-3333-4333-8333-333333333333'; + if(kind==='uid')f.write(join(f.proc,'300/status'),'Uid:\t0 0 0 0'); + expect(()=>f.m.createStateBoundEvidence({...f,storePath:f.store,procRoot:f.proc})).toThrow(); +}); +it('pins state RPCs to one immutable instance and preserves non-state SDK calls',async()=>{ + const {pinStateClient}=await import('../src/ops/state-epoch.mjs');const seen:any[]=[]; + const sdk:any={trigger:(req:any)=>{seen.push(req);return Promise.resolve(true)}}; + const client=pinStateClient(sdk,'state::22222222-2222-4222-8222-222222222222::'); + await client.trigger({function_id:'state::set_ack',payload:{value:1}}); + await client.trigger({function_id:'engine::workers::list',payload:{}}); + expect(seen.map(r=>r.function_id)).toEqual(['state::22222222-2222-4222-8222-222222222222::set_ack','engine::workers::list']); +}); +it('rejects completion if binding changes during an awaited pinned RPC',async()=>{ + const {pinStateClient}=await import('../src/ops/state-epoch.mjs');let current=true; + const client=pinStateClient({trigger:async()=>{current=false;return true}} as any,'state::22222222-2222-4222-8222-222222222222::',()=>current); + await expect(client.trigger({function_id:'state::set_ack',payload:{}})).rejects.toThrow('STATE_WRITER_IDENTITY_CHANGED'); +}); +it('reports the failing State identity component without weakening checks',async()=>{ + const f=await fixture();const evidence=f.m.createStateBoundEvidence({...f,storePath:f.store,procRoot:f.proc});const epoch=evidence.capture()!; + rmSync(join(f.proc,'200/fd/3'));expect(evidence.current(epoch)).toBe(false);expect(evidence.diagnostic?.()).toBe('STATE_ENGINE_SOCKET_OWNER_UNVERIFIED'); +}); +it('attaches phase and reason to identity rejection',async()=>{ + const {pinStateClient}=await import('../src/ops/state-epoch.mjs'); + const client=pinStateClient({trigger:async()=>true} as any,'state::22222222-2222-4222-8222-222222222222::',()=>false,()=> 'STATE_UID_CHANGED'); + await expect(client.trigger({function_id:'state::set',payload:{}})).rejects.toMatchObject({phase:'before_dispatch',diagnostic:'STATE_UID_CHANGED'}); +}); +it('ignores over 1MiB of unrelated host TCP rows while verifying owned sockets',async()=>{ + const f=await fixture();const table=join(f.proc,'300/net/tcp');const prior=(await import('node:fs')).readFileSync(table,'utf8'); + f.write(table,('0: 0100007F:0AAA 0100007F:0BBB 06 0 0 0 1000 0 0\n').repeat(26000)+prior); + const evidence=f.m.createStateBoundEvidence({...f,storePath:f.store,procRoot:f.proc});expect(evidence.capture()?.state.instanceId).toBe(f.identity.instanceId); +}); + +it('reobserves an accepted socket missing from the earlier engine FD snapshot',async()=>{ + const f=await fixture();rmSync(join(f.proc,'200/fd/3'));let reads=0; + race.afterRead=path=>{if(path===join(f.proc,'200/fd')&&++reads===1)symlinkSync('socket:[500]',join(path,'3'));}; + const evidence=f.m.createStateBoundEvidence({...f,storePath:f.store,procRoot:f.proc});expect(evidence.capture()?.state.instanceId).toBe(f.identity.instanceId);expect(reads).toBeGreaterThan(1); +}); +it('does not retry through process replacement during socket reobservation',async()=>{ + const f=await fixture();rmSync(join(f.proc,'200/fd/3'));let reads=0; + race.afterRead=path=>{if(path===join(f.proc,'200/fd')&&++reads===1){symlinkSync('socket:[500]',join(path,'3'));f.write(join(f.proc,'300/stat'),f.stat(300,'31'));}}; + expect(()=>f.m.createStateBoundEvidence({...f,storePath:f.store,procRoot:f.proc})).toThrow();expect(reads).toBe(1); +}); +it('bounds socket reobservation and still rejects persistent foreign ownership',async()=>{ + const f=await fixture();rmSync(join(f.proc,'200/fd/3'));let reads=0;race.afterRead=path=>{if(path===join(f.proc,'200/fd'))reads++;}; + expect(()=>f.m.createStateBoundEvidence({...f,storePath:f.store,procRoot:f.proc})).toThrow();expect(reads).toBeLessThanOrEqual(3); +}); diff --git a/test/state-local-admission.test.ts b/test/state-local-admission.test.ts new file mode 100644 index 0000000000..ac843cace2 --- /dev/null +++ b/test/state-local-admission.test.ts @@ -0,0 +1,10 @@ +import {it,expect,vi,afterEach} from 'vitest'; +const gates=vi.hoisted(()=>({owns:vi.fn(()=>true),admitted:true})); +vi.mock('../src/ops/file-writer-lease.mjs',async importOriginal=>({...await importOriginal(),ownsFileWriterLease:gates.owns,fileWriterRecoveryAdmitted:()=>gates.admitted})); +import {StateKV} from '../src/state/kv.js'; +import {createStateDurabilityFence} from '../src/ops/state-durability.mjs'; +afterEach(()=>{vi.unstubAllEnvs();gates.owns.mockReset().mockReturnValue(true);gates.admitted=true;}); +function setup(){vi.stubEnv('AGENTMEMORY_STATE_STORE_PATH','/synthetic/unused');const trigger=vi.fn(async()=>true);const durability=createStateDurabilityFence({storePath:'/synthetic/unused',completion:'sdk'});const kv=new StateKV({trigger} as any,{backend:'file',durability,writerLease:{} as any});return{kv,trigger,durability};} +it('lease lost inside tracked admission never dispatches or latches uncertainty',async()=>{const {kv,trigger,durability}=setup();gates.owns.mockReturnValueOnce(true).mockReturnValue(false);await expect(kv.set('fixture','one',{})).rejects.toThrow('FILE_WRITER_LOST');expect(trigger).not.toHaveBeenCalled();expect(kv.mutationOutcomeUncertain).toBe(false);expect(durability.snapshot().mutationFailures).toBe(0);}); +it('exclusive recovery admission failure never dispatches or latches uncertainty',async()=>{const {kv,trigger,durability}=setup();gates.admitted=false;await expect(kv.recoveryWrite!.exclusive([{scope:'fixture',key:'one'}],()=>kv.set('fixture','one',{}))).rejects.toThrow('WRITER_EPOCH_UNVERIFIED');expect(trigger).not.toHaveBeenCalled();expect(kv.mutationOutcomeUncertain).toBe(false);expect(durability.snapshot().mutationFailures).toBe(0);}); +it('bounded first/latest diagnostic excludes payload, key, scope and arbitrary error strings',async()=>{const d=createStateDurabilityFence({storePath:'/synthetic/unused',completion:'sdk'});await expect(d.mutate('private-scope','private-key',()=>Promise.reject(Object.assign(Error('private-content'),{code:'STATE_WRITER_IDENTITY_CHANGED',phase:'after_dispatch',diagnostic:'STATE_ENGINE_SOCKET_OWNER_UNVERIFIED'})))).rejects.toThrow();await expect(d.mutate('private-scope','private-key',()=>Promise.reject(Object.assign(Error('private-content'),{code:'SECRET_VALUE',phase:'SECRET_VALUE',diagnostic:'SECRET_VALUE'})))).rejects.toThrow();const s=d.snapshot() as any;expect(s.mutationFailures).toBe(2);expect(s.firstMutationFailure).toMatchObject({code:'STATE_WRITER_IDENTITY_CHANGED',phase:'after_dispatch',diagnostic:'STATE_ENGINE_SOCKET_OWNER_UNVERIFIED'});expect(s.latestMutationFailure).toMatchObject({code:'MUTATION_FAILED',phase:'unknown',diagnostic:null});expect(JSON.stringify([s.firstMutationFailure,s.latestMutationFailure])).not.toMatch(/private|SECRET/);}); diff --git a/test/state-readiness.test.ts b/test/state-readiness.test.ts index 038c95332d..910e51f31a 100644 --- a/test/state-readiness.test.ts +++ b/test/state-readiness.test.ts @@ -22,6 +22,30 @@ function asClient(trigger: unknown): Pick { } describe("waitForStateReady", () => { + it("waits for state::list_keys to register before reporting ready", async () => { + let clock = 0; + let keyProbes = 0; + const trigger = vi.fn(async (call: TriggerCall) => { + if (call.function_id === "state::list_keys") { + keyProbes += 1; + if (keyProbes === 1) throw missingFunction("state::list_keys"); + } + return null; + }); + + const result = await waitForStateReady(asClient(trigger), { + timeoutMs: 1000, + intervalMs: 100, + now: () => clock, + sleep: async (ms) => { + clock += ms; + }, + }); + + expect(result).toEqual({ ready: true, waitedMs: 100, attempts: 2 }); + expect(keyProbes).toBe(2); + }); + it("returns immediately when the state functions are registered", async () => { const trigger = vi.fn(async () => null); @@ -30,7 +54,7 @@ describe("waitForStateReady", () => { expect(result).toEqual({ ready: true, waitedMs: 0, attempts: 1 }); expect( trigger.mock.calls.map(([call]) => (call as TriggerCall).function_id), - ).toEqual(["state::list", "state::get"]); + ).toEqual(["state::list", "state::get", "state::list_keys"]); expect(trigger).toHaveBeenNthCalledWith(1, { function_id: "state::list", payload: { scope: KV.config }, @@ -62,7 +86,7 @@ describe("waitForStateReady", () => { }); expect(result).toEqual({ ready: true, waitedMs: 100, attempts: 2 }); - expect(trigger).toHaveBeenCalledTimes(3); + expect(trigger).toHaveBeenCalledTimes(4); }); it("returns not-ready when every probe times out", async () => { @@ -111,7 +135,7 @@ describe("waitForStateReady", () => { expect(result).toEqual({ ready: true, waitedMs: 200, attempts: 3 }); expect(sleeps).toEqual([100, 100]); - expect(trigger).toHaveBeenCalledTimes(4); + expect(trigger).toHaveBeenCalledTimes(5); expect(trigger).toHaveBeenNthCalledWith( 3, expect.objectContaining({ @@ -241,3 +265,16 @@ describe("assertStateReady", () => { ).not.toThrow(); }); }); + +describe('compact state acknowledgement readiness',()=>{ + it.each([undefined,{function_id:'state::set'},null])('refuses an old/malformed engine before enabling compact writes: %j',async detail=>{ + const trigger=vi.fn(async({function_id}:any)=>function_id==='engine::functions::info'?detail:null); + const result=await waitForStateReady(asClient(trigger),{requireSetAck:true}); + expect(result.ready).toBe(false);expect(result.error).toBe('STATE_SET_ACK_REQUIRED'); + }); + it('checks registered support without writing a probe value',async()=>{ + const trigger=vi.fn(async({function_id}:any)=>function_id==='engine::functions::info'?{function_id:'state::set_ack'}:null); + expect((await waitForStateReady(asClient(trigger),{requireSetAck:true})).ready).toBe(true); + expect(trigger.mock.calls.map(([r])=>r.function_id)).toEqual(['state::list','state::get','state::list_keys','engine::functions::info']); + }); +}); diff --git a/test/state-seal-shutdown.test.ts b/test/state-seal-shutdown.test.ts new file mode 100644 index 0000000000..964a496512 --- /dev/null +++ b/test/state-seal-shutdown.test.ts @@ -0,0 +1,14 @@ +import {it,expect} from 'vitest'; +it('does not promote SDK completion into a disk-clean proof',async()=>{ + const {stateSealDiskProof}=await import('../src/ops/state-seal-shutdown.mjs'); + expect(()=>stateSealDiskProof({completed:true,synced:false},{instanceId:'x',storePath:'/tmp/test'},0)).toThrow(); +}); +it.each([{sealed:false,synced:true},{sealed:true,synced:false},{sealed:true,synced:true,instanceId:'wrong'},{sealed:true,synced:true,storePath:'/wrong'}])('rejects invalid seal %j',async(change)=>{ + const {stateSealDiskProof}=await import('../src/ops/state-seal-shutdown.mjs'),identity={instanceId:'test-instance',storePath:'/tmp'}; + expect(()=>stateSealDiskProof({version:1,...identity,sealed:true,synced:true,...change},identity,0)).toThrow(); +}); +it('accepts only matching immutable sealed fsync identity and zero app failures',async()=>{ + const {stateSealDiskProof}=await import('../src/ops/state-seal-shutdown.mjs'),identity={instanceId:'test-instance',storePath:'/tmp'},receipt={version:1,...identity,sealed:true,synced:true}; + expect(stateSealDiskProof(receipt,identity,0)).toMatchObject({completion:'disk',completed:true,synced:true,mutationFailures:0}); + expect(()=>stateSealDiskProof(receipt,identity,1)).toThrow(); +}); diff --git a/test/state-worker-artifact.test.ts b/test/state-worker-artifact.test.ts new file mode 100644 index 0000000000..f6b83e85aa --- /dev/null +++ b/test/state-worker-artifact.test.ts @@ -0,0 +1,10 @@ +import {afterEach,it,expect} from 'vitest'; +import {mkdtempSync,writeFileSync,readFileSync,rmSync} from 'node:fs'; +import {tmpdir} from 'node:os';import {join} from 'node:path';import {createHash} from 'node:crypto'; +import {loadStateWorkerArtifact,STATE_ARTIFACT_PIN} from '../src/cli/state-worker-artifact.js'; +import {writeWorkerCompose} from '../src/cli/compose-project.js';import {parse} from 'yaml'; +const dirs:string[]=[];afterEach(()=>dirs.splice(0).forEach(d=>rmSync(d,{recursive:true,force:true}))); +function fixture(){const dir=mkdtempSync(join(tmpdir(),'state-artifact-'));dirs.push(dir);const binary='synthetic fixture';writeFileSync(join(dir,'state'),binary,{mode:0o755});const m={schemaVersion:1,...STATE_ARTIFACT_PIN,profile:'release',binary:'state',engineVersion:'0.25.1',binarySha256:createHash('sha256').update(binary).digest('hex'),target:process.arch==='arm64'?'aarch64-unknown-linux-gnu':'x86_64-unknown-linux-gnu'};writeFileSync(join(dir,'manifest.json'),JSON.stringify(m));return{dir,m};} +it('loads verified artifact and rejects modified binary/manifest',()=>{const{dir,m}=fixture();expect(loadStateWorkerArtifact(dir).sha256).toBe(m.binarySha256);writeFileSync(join(dir,'state'),'changed');expect(()=>loadStateWorkerArtifact(dir)).toThrow('HASH_MISMATCH');m.upstreamCommit='bad';writeFileSync(join(dir,'manifest.json'),JSON.stringify(m));expect(()=>loadStateWorkerArtifact(dir)).toThrow('PROVENANCE_MISMATCH');}); +it('checked-in source pin matches runtime verifier',()=>{const pin=JSON.parse(readFileSync(new URL('../deploy/state-worker/pin.json',import.meta.url),'utf8'));for(const[k,v]of Object.entries(STATE_ARTIFACT_PIN))expect(pin[k]).toBe(v);}); +it('local State artifact keeps HTTP registry source and creates only State local manifest',()=>{const{dir}=fixture();const r=writeWorkerCompose({ports:{restPort:3511,viewerPort:3513,enginePort:49534},dataDir:join(dir,'data'),projectRoot:dir,nodeBin:process.execPath,homeDir:dir,stateWorkerExecutable:join(dir,'state')});const parsed=parse(r.text);expect(parsed.containers.state.worker).toMatch(/^path:/);expect(parsed.containers.state.version).toBeUndefined();expect(parsed.containers.state.scripts.run).toContain(join(dir,'state'));expect(parsed.containers.http.worker).toBe('package://api.workers.iii.dev/http');}); diff --git a/test/status.test.ts b/test/status.test.ts index 8ded5c9e49..2f2f7b0e34 100644 --- a/test/status.test.ts +++ b/test/status.test.ts @@ -531,7 +531,7 @@ describe("status wiring", () => { it("streams reports with cached-only health ticks and explicit snapshot scans", () => { const streams = readFileSync("src/triggers/viewer-streams.ts", "utf-8"); - expect(streams).toMatch(/statusReport\(\{ health: snapshot, scanMaxAgeMs, scanUnindexed, functionMetrics:metricsRead \}\)/); + expect(streams).toMatch(/statusReport\(\{ health: snapshot, scanMaxAgeMs, scanUnindexed, scanCaptureInbox:false, functionMetrics:metricsRead \}\)/); expect(streams).toContain("healthPayload(payload.new_value ?? null, HEALTH_TICK_SCAN_MAX_AGE_MS, false)"); expect(streams).toContain("healthPayload(health ?? null, UNINDEXED_SCAN_REUSE_MS)"); }); @@ -548,7 +548,7 @@ describe("status wiring", () => { it("reports the active state backend from config, not a hardcoded value", () => { expect(api).toMatch(/stateBackend: kv.backend === "redis" \? "redis" : "file"/); - expect(index).toMatch(/new StateKV\(sdk, \{ backend: stateBackend,writerLease \}\)/); + expect(index).toMatch(/new StateKV\(stateClient,\s*\{\s*backend:\s*stateBackend,/); }); it("the viewer has a Health tab that reads the status report", () => { diff --git a/test/storage-hold-transitions-regression.test.ts b/test/storage-hold-transitions-regression.test.ts new file mode 100644 index 0000000000..561e07f36f --- /dev/null +++ b/test/storage-hold-transitions-regression.test.ts @@ -0,0 +1,7 @@ +import {it,expect,vi} from 'vitest'; +import {createBackgroundControl} from '../src/ops/background-control.mjs'; +function fixture(holds:string[]){let saved:any={version:2,paused:true,resumePending:false,holds,recovery:{attempts:0,nextAttemptAt:0}};return {get:async()=>saved,set:async(_s:string,_k:string,v:any)=>{saved=v},durability:{wait:async()=>({synced:true,mutationFailures:0}),waitForKeys:async()=>{}}};} +it('reports a persisted storage error after boot and a subsequent pause',async()=>{const c=createBackgroundControl();await c.init(fixture(['storage_error']) as any);expect(c.snapshot()).toMatchObject({status:'error',error:'background_storage_unavailable',holds:['storage_error']});await c.change('pause',{source:'deployment'});await c.settled();expect(c.snapshot()).toMatchObject({status:'error',error:'background_storage_unavailable'});}); +it('does not release a storage hold as part of a partial resume without vector recovery',async()=>{const c=createBackgroundControl();await c.init(fixture(['memory','storage_error']) as any);const recovery=vi.fn(async()=>false);await expect(c.change('resume',{source:'manual',beforeResume:recovery})).rejects.toThrow();expect(recovery).not.toHaveBeenCalled();expect(c.snapshot().holds).toContain('storage_error');}); +it('reports a failed resume as pending until rollback confirmation completes',async()=>{let calls=0,release!:()=>void,entered!:()=>void;const arrived=new Promise(r=>entered=r),wait=new Promise(r=>release=r);const s=fixture(['storage_error']);s.durability.wait=async()=>({synced:false,mutationFailures:0});s.durability.waitForKeys=async()=>{if(++calls===2){entered();await wait;}};const c=createBackgroundControl();await c.init(s as any);const change=c.change('resume',{beforeResume:async()=>true});const caught=change.catch(()=>{});await arrived;expect(c.snapshot().transition).toMatchObject({pending:true});release();await caught;expect(c.snapshot().transition).toMatchObject({pending:false});}); +it('explains the other hold when refusing a partial storage recovery via API',async()=>{const c=createBackgroundControl();await c.init(fixture(['memory','storage_error']) as any);const {registerBackgroundControlApi}=await import('../src/ops/background-control.mjs');const handlers=new Map();registerBackgroundControlApi({registerFunction:(id:string,f:any)=>handlers.set(id,f),registerTrigger:()=>{}} as any,'fixture',c,async()=>true);const out=await handlers.get('api::background-control-set')({headers:{authorization:'Bearer fixture','x-agentmemory-background':'1','content-type':'application/json'},body:{action:'resume'}});expect(out).toMatchObject({status_code:409,body:{error:'background_storage_recovery_requires_full_resume',control:{holds:['memory','storage_error']}}});}); diff --git a/test/summary-coverage-api.test.ts b/test/summary-coverage-api.test.ts new file mode 100644 index 0000000000..9ef2dd0f70 --- /dev/null +++ b/test/summary-coverage-api.test.ts @@ -0,0 +1,120 @@ +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, expect, it, vi } from "vitest"; +import { summaryFingerprint } from "../src/functions/summary-state.js"; +import { createObservationReader } from "../src/ops/observation-pages.mjs"; +import { createStateDurabilityFence } from "../src/ops/state-durability.mjs"; +import { KV } from "../src/state/schema.js"; +import { registerApiTriggers } from "../src/triggers/api.js"; +import { mockKV, mockSdk } from "./helpers/mocks.js"; + +const roots: string[] = []; +const query = { includeSummary: "false", includeSummaryCoverage: "true", agentId: "*" }; +afterEach(async () => { + vi.useRealTimers(); + await Promise.all(roots.splice(0).map(path => rm(path, { recursive: true, force: true }))); +}); + +async function fixture() { + const root = await mkdtemp(join(tmpdir(), "am-summary-coverage-")); + roots.push(root); + const kv = mockKV(); + const id = "covered-legacy"; + const scope = KV.observations(id); + const session = { id, project: "synthetic", cwd: "/fixture", startedAt: "2026-01-01T00:00:00.000Z", + status: "completed" as const, observationCount: 1 }; + const row = { id: "obs_fixture", sessionId: id, timestamp: session.startedAt, + title: "Canonical title", narrative: "Canonical narrative", sourceRaw: { raw: "PRIVATE_FIXTURE_SOURCE" } }; + const summary = { sessionId: id, project: session.project, createdAt: session.startedAt, + title: "PRIVATE_FIXTURE_SUMMARY", observationCount: 1, coveredFingerprint: summaryFingerprint(session, [row as never]) }; + await kv.set(KV.sessions, id, session); + await kv.set(KV.summaries, id, summary); + await kv.set(scope, row.id, row); + await writeFile(join(root, encodeURIComponent(scope) + ".bin"), JSON.stringify({ [row.id]: row })); + const durability = createStateDurabilityFence({ storePath: root, completion: "sdk" }); + const reader = createObservationReader({ storePath: root, kv: { ...kv, durability } as never }); + const work = vi.spyOn(reader, "work"); + const get = vi.spyOn(kv, "get"); + const set = vi.spyOn(kv, "set"); + const update = vi.spyOn(kv, "update"); + const del = vi.spyOn(kv, "delete"); + const sdk = mockSdk({ looseTrigger: true }); + registerApiTriggers(sdk as never, Object.assign(kv, { observationReader: reader, durability }) as never, "fixture-secret"); + const call = (query_params = query) => sdk.fns.get("api::sessions")!({ headers: { authorization: "Bearer fixture-secret" }, query_params }) as Promise; + return { call, kv, work, get, set, update, del, session, summary, row, scope, reader, durability }; +} + +it("returns a read-only count from full canonical rows only when metadata coverage was requested", async () => { + const f = await fixture(); + expect((await f.call({ ...query, includeSummaryCoverage: "false" })).body.sessions).toEqual([f.session]); + expect(f.work).not.toHaveBeenCalled(); + expect(f.get).not.toHaveBeenCalled(); + const result = await f.call(); + expect(result.body.sessions).toEqual([{ ...f.session, summaryCoveredCount: 1 }]); + expect(f.work).toHaveBeenCalledWith(f.scope, { kind: "summary", signal: expect.any(AbortSignal) }); + expect(JSON.stringify(result)).not.toMatch(/PRIVATE_FIXTURE|summaryProgress/); + expect(f.set).not.toHaveBeenCalled(); + expect(f.update).not.toHaveBeenCalled(); + expect(f.del).not.toHaveBeenCalled(); +}); + +it("leaves coverage unknown when the legacy fingerprint is stale or explicit progress already exists", async () => { + const f = await fixture(); + await f.kv.set(KV.summaries, f.session.id, { ...f.summary, coveredFingerprint: "0".repeat(64) }); + expect((await f.call()).body.sessions[0]).not.toHaveProperty("summaryCoveredCount"); + await f.kv.set(KV.summaries, f.session.id, f.summary); + const summaryProgress = { status: "retrying", pendingCount: 1, processedCount: 0 }; + await f.kv.set(KV.sessions, f.session.id, { ...f.session, summaryProgress }); + f.work.mockClear(); + expect((await f.call()).body.sessions[0]).toEqual({ ...f.session, summaryProgress }); + expect(f.work).not.toHaveBeenCalled(); +}); + +it("strips a persisted coverage claim and requires fresh proof for each response", async () => { + const f = await fixture(); + await f.kv.set(KV.sessions, f.session.id, { ...f.session, summaryCoveredCount: 1 }); + await f.kv.set(KV.summaries, f.session.id, { ...f.summary, coveredFingerprint: "stale" }); + expect((await f.call()).body.sessions[0]).toEqual(f.session); + expect((await f.call({ ...query, includeSummaryCoverage: "false" })).body.sessions[0]).toEqual(f.session); + expect((await f.call({ ...query, includeSummary: "true" })).body.sessions[0]).not.toHaveProperty("summaryCoveredCount"); + await f.kv.set(KV.summaries, f.session.id, f.summary); + expect((await f.call()).body.sessions[0]).toEqual({ ...f.session, summaryCoveredCount: 1 }); +}); + +it.each(["observation", "summary"])("withholds matching coverage when a %s mutation races the proof", async (kind) => { + const f = await fixture(); + const work = f.reader.work.bind(f.reader); + f.work.mockImplementationOnce(async (scope, options) => { + const view = await work(scope, options); + const load = view.load.bind(view); + return { ...view, load: async row => { + const value = await load(row); + if (kind === "observation") { + f.reader.record(scope, "another-observation", { ...f.row, id: "another-observation" }); + } else { + await f.durability.mutate(KV.summaries, f.session.id, async () => {}); + } + return value; + } }; + }); + expect((await f.call()).body.sessions[0]).not.toHaveProperty("summaryCoveredCount"); +}); + +it("bounds observation work and returns metadata if a read stalls", async () => { + const f = await fixture(); + await f.kv.set(KV.sessions, f.session.id, { ...f.session, observationCount: 200 }); + expect((await f.call()).body.sessions[0]).not.toHaveProperty("summaryCoveredCount"); + expect(f.work).not.toHaveBeenCalled(); + await f.kv.set(KV.sessions, f.session.id, f.session); + vi.useFakeTimers(); + let finish!: (value: null) => void; + f.get.mockImplementationOnce(() => new Promise(resolve => { finish = resolve; })); + const response = f.call(); + await vi.advanceTimersByTimeAsync(2000); + expect((await response).body.sessions).toEqual([f.session]); + const reads = f.get.mock.calls.length; + expect((await f.call()).body.sessions).toEqual([f.session]); + expect(f.get.mock.calls).toHaveLength(reads); + finish(null); +}); diff --git a/test/summary-sdk-boundary.test.ts b/test/summary-sdk-boundary.test.ts index 3096ec6d1c..038e7e4be2 100644 --- a/test/summary-sdk-boundary.test.ts +++ b/test/summary-sdk-boundary.test.ts @@ -5,7 +5,9 @@ import {readFileSync,appendFileSync} from 'node:fs'; import {join,resolve} from 'node:path'; it('keeps real SDK transport alive for active projections over a namespace above its message cap',async()=>{ const available=()=>Number(/^MemAvailable:\s+(\d+)/m.exec(readFileSync('/proc/meminfo','utf8'))?.[1]??0)*1024; - expect(available(),'isolated fixture requires12GiB available before starting').toBeGreaterThanOrEqual(12*1024**3); + // The child has a 512 MiB V8 heap and an independent 1 GiB RSS cap. + // Reserve its full RSS budget plus 2 GiB host headroom, not a fixed 12 GiB. + expect(available(),'isolated fixture requires 3 GiB available before starting').toBeGreaterThanOrEqual(3*1024**3); const root=await mkdtemp('/tmp/am-summary-sdk-');for(const d of ['tmp','config','data','cache'])await mkdir(join(root,d)); await writeFile(join(root,'stdout.log'),'');await writeFile(join(root,'stderr.log'),''); const env={PATH:'/usr/local/bin:/usr/bin:/bin',HOME:root,TMPDIR:join(root,'tmp'),XDG_CONFIG_HOME:join(root,'config'),XDG_DATA_HOME:join(root,'data'),XDG_CACHE_HOME:join(root,'cache'),LANG:'C.UTF-8',TZ:'UTC',OTEL_ENABLED:'false',CONSOLIDATION_ENABLED:'true',AGENTMEMORY_STATE_STORE_PATH:join(root,'state'),AM_SYNTHETIC_ROOT:root}; @@ -13,7 +15,7 @@ it('keeps real SDK transport alive for active projections over a namespace above const child=spawn(process.execPath,['--max-old-space-size=512','--import',resolve('node_modules/tsx/dist/loader.mjs'),resolve('test/fixtures/summary-sdk-boundary.mts')],{cwd:root,env,stdio:['pipe','pipe','pipe']});child.stdin.on('error',()=>{}); const stop=()=>{if(!stopRequested){stopRequested=true;child.stdin.write('stop\n');}}; child.stdout.on('data',b=>{if(stdout.length+b.length>2*1024*1024)stop();else{stdout+=b.toString();appendFileSync(join(root,'stdout.log'),b);}});child.stderr.on('data',b=>{if(stderr.length+b.length>2*1024*1024)stop();else{stderr+=b.toString();appendFileSync(join(root,'stderr.log'),b);}}); - const monitor=setInterval(()=>{if(available()<8*1024**3)stop();},250); + const monitor=setInterval(()=>{if(available()<1024**3)stop();},250); const deadline=setTimeout(stop,120000); try{ const code=await new Promise((resolve,reject)=>{child.once('close',resolve);child.once('error',reject);}); diff --git a/test/summary-sdk-budget.test.ts b/test/summary-sdk-budget.test.ts index a327db4691..0ad262efca 100644 --- a/test/summary-sdk-budget.test.ts +++ b/test/summary-sdk-budget.test.ts @@ -2,6 +2,9 @@ import {test,expect,afterEach,vi} from 'vitest'; import {mkdtemp,writeFile,rm} from 'node:fs/promises'; import {join} from 'node:path'; import {tmpdir} from 'node:os'; +// Small budget exercises admission without allocating several production-size frames. +vi.mock('../src/state/frame-guard.js',async(importOriginal)=>({...await importOriginal(),SAFE_PAYLOAD_BYTES:1024*1024})); +import {SAFE_PAYLOAD_BYTES} from '../src/state/frame-guard.js'; import {StateKV} from '../src/state/kv.js'; import {createBackgroundControl} from '../src/ops/background-control.mjs'; const dirs:string[]=[]; @@ -19,14 +22,14 @@ async function fixture(rows:Record={}){ return {kv:new StateKV({trigger} as never),map,calls,seed:(scope:string,key:string,value:any)=>scopeMap(scope).set(key,value),peek:(scope:string,key:string)=>scopeMap(scope).get(key),setHook:(fn:any)=>{hook=fn}}; } test('oversized summary collection is refused before the shared SDK read, without poisoning mutation state',async()=>{ - const f=await fixture({big:{sessionId:'big',segments:['S'.repeat(16*1024*1024)]}}); + const f=await fixture({big:{sessionId:'big',segments:['S'.repeat(SAFE_PAYLOAD_BYTES+1024)]}}); const refused=await f.kv.list('mem:summaries').then(()=>({code:'unexpected_success'}),(e:any)=>({code:e.code})); expect(refused.code).toBe('SUMMARY_COLLECTION_TOO_LARGE'); expect(f.calls).toEqual([]);expect(f.kv.mutationOutcomeUncertain).toBe(false); expect(f.kv.durability!.snapshot().mutationFailures).toBe(0); }); test('pending replacement sizes replace old disk sizes instead of being double counted',async()=>{ - const f=await fixture({big:{sessionId:'big',segments:['S'.repeat(16*1024*1024)]}}); + const f=await fixture({big:{sessionId:'big',segments:['S'.repeat(SAFE_PAYLOAD_BYTES+1024)]}}); await f.kv.set('mem:summaries','big',{sessionId:'big',title:'small'}); expect(await f.kv.list('mem:summaries')).toEqual([{sessionId:'big',title:'small'}]); }); @@ -65,7 +68,7 @@ test('queued list sees the uncertainty latch before the namespace gate releases' const controlScope='mem:background-control:v1'; const recoveryRecord=()=>({version:2,paused:false,resumePending:true,holds:[],recovery:{attempts:0,nextAttemptAt:0}}); test('guarded bulk-read refusal still permits the supported current-epoch recovery without rewriting old evidence',async()=>{ - const f=await fixture({big:{sessionId:'big',segments:['S'.repeat(16*1024*1024)]}}); + const f=await fixture({big:{sessionId:'big',segments:['S'.repeat(SAFE_PAYLOAD_BYTES+1024)]}}); const historical={status:'unknown',completed:false,verified:false,applicationFlushVerified:false}; f.seed('synthetic:history','old',structuredClone(historical));f.seed(controlScope,'manual-pause',recoveryRecord()); const refused=await f.kv.list('mem:summaries').then(()=>({code:'unexpected_success'}),(e:any)=>({code:e.code})); @@ -99,7 +102,7 @@ test.each(['metadata','recent'])('the %s snapshot waits for an already active su }); test('wire admission uses the actual SDK acknowledged value instead of a smaller caller-side guess',async()=>{ - const f=await fixture();const stored={sessionId:'a',segments:['S'.repeat(16*1024*1024)]}; + const f=await fixture();const stored={sessionId:'a',segments:['S'.repeat(SAFE_PAYLOAD_BYTES+1024)]}; f.setHook(async(req:any)=>{if(req.function_id==='state::set'){f.map.set('a',stored);return {handled:true,value:{old_value:null,new_value:stored}};}return null}); await f.kv.set('mem:summaries','a',{sessionId:'a',title:'small caller snapshot'}); const refused=await f.kv.list('mem:summaries').then(()=>({code:'unexpected_success'}),(e:any)=>({code:e.code})); diff --git a/test/summary-set-ack.test.ts b/test/summary-set-ack.test.ts new file mode 100644 index 0000000000..f494791041 --- /dev/null +++ b/test/summary-set-ack.test.ts @@ -0,0 +1,48 @@ +import {afterEach,expect,it,vi} from 'vitest'; +import {mkdtemp,writeFile,rm} from 'node:fs/promises'; +import {tmpdir} from 'node:os'; +import {join} from 'node:path'; +import {StateKV} from '../src/state/kv.js'; +const dirs:string[]=[]; +afterEach(async()=>{vi.unstubAllEnvs();await Promise.all(dirs.splice(0).map(p=>rm(p,{recursive:true,force:true})));}); +async function fixture(trigger:ReturnType,compactSummaryWrites=true){ + const root=await mkdtemp(join(tmpdir(),'am-summary-ack-'));dirs.push(root); + vi.stubEnv('AGENTMEMORY_STATE_STORE_PATH',root); + await writeFile(join(root,'mem%3Asummaries.bin'),'{}'); + return new StateKV({trigger}as never,{compactSummaryWrites}); +} +it('sends a pinned JSON snapshot through compact ack and retains the stored summary projection',async()=>{ + let entered!:()=>void,release!:()=>void; + const waiting=new Promise(r=>entered=r),held=new Promise(r=>release=r); + const trigger=vi.fn(async(req:any)=>{entered();await held;return true;}); + const kv=await fixture(trigger),value={sessionId:'a',title:'sent',optional:undefined}; + const saving=kv.set('mem:summaries','a',value);await waiting;value.title='changed';release(); + expect(await saving).toEqual({sessionId:'a',title:'sent'}); + expect(trigger).toHaveBeenCalledWith({function_id:'state::set_ack',payload:{scope:'mem:summaries',key:'a',value:{sessionId:'a',title:'sent'}}}); + expect(await kv.getSummaryMetadata()).toMatchObject([{sessionId:'a',title:'sent'}]); + expect(kv.mutationOutcomeUncertain).toBe(false); +}); +it.each([false,null,{}, {new_value:'unexpected'}])('rejects a malformed acknowledgement without certifying a write: %j',async ack=>{ + const kv=await fixture(vi.fn(async()=>ack)); + await expect(kv.set('mem:summaries','a',{sessionId:'a'})).rejects.toMatchObject({code:'STATE_SET_ACK_UNVERIFIED'}); + expect(kv.mutationOutcomeUncertain).toBe(true); + expect(kv.durability!.snapshot().mutationFailures).toBe(1); +}); +it('propagates failed acknowledgement and retains the storage uncertainty latch',async()=>{ + const kv=await fixture(vi.fn(async()=>{throw Object.assign(Error('adapter failed'),{code:'SET_ERROR'});})); + await expect(kv.set('mem:summaries','a',{sessionId:'a'})).rejects.toMatchObject({code:'SET_ERROR'}); + expect(kv.mutationOutcomeUncertain).toBe(true); +}); +it('keeps ordinary namespaces and compatibility-mode summary writes on the existing API',async()=>{ + const trigger=vi.fn(async(req:any)=>({old_value:null,new_value:req.payload.value})); + const kv=await fixture(trigger);await kv.set('mem:other','a',{kept:true}); + const legacy=await fixture(trigger,false);await legacy.set('mem:summaries','a',{sessionId:'a'}); + expect(trigger.mock.calls.map(([r])=>r.function_id)).toEqual(['state::set','state::set']); +}); +it('keeps byte-budgeted summary listing available after compact acknowledgement',async()=>{ + const value={sessionId:'a',title:'kept'}; + const trigger=vi.fn(async(req:any)=>req.function_id==='state::set_ack'?true:[value]); + const kv=await fixture(trigger);await kv.set('mem:summaries','a',value); + expect(await kv.list('mem:summaries')).toEqual([value]); + expect(trigger.mock.calls.map(([req])=>req.function_id)).toEqual(['state::set_ack','state::list']); +}); diff --git a/test/unbounded-list-guard.test.ts b/test/unbounded-list-guard.test.ts new file mode 100644 index 0000000000..8a738bd810 --- /dev/null +++ b/test/unbounded-list-guard.test.ts @@ -0,0 +1,36 @@ +import { describe, it, expect } from "vitest"; +import { readdirSync, readFileSync, statSync } from "node:fs"; +import { join } from "node:path"; + +const SRC = join(__dirname, "../src"); +const UNBOUNDED = "(insights|semantic|procedural|graphNodes|graphEdges|graphEdgeHistory|memories|audit|relations|auditMonth)"; +const callRe = new RegExp( + "\\w+\\s*\\.list(?:<[^>]*>)?\\(\\s*KV\\." + UNBOUNDED + "(\\b|\\()", +); + +const ALLOWLIST: Record = { +}; + +function* walk(dir: string): Generator { + for (const entry of readdirSync(dir)) { + const full = join(dir, entry); + if (statSync(full).isDirectory()) yield* walk(full); + else if (entry.endsWith(".ts")) yield full; + } +} + +describe("unbounded scope reads stay paged", () => { + it("no .list() call targets a scope that can exceed the transport limit", () => { + const hits: string[] = []; + for (const file of walk(SRC)) { + const text = readFileSync(file, "utf8"); + const rel = file.slice(SRC.length + 1); + for (const match of text.matchAll(new RegExp(callRe.source, "g"))) { + const line = text.slice(0, match.index).split("\n").length; + const key = `${rel}:${line}`; + if (!ALLOWLIST[key]) hits.push(`${key} -> ${match[0].replace(/\s+/g, " ")}`); + } + } + expect(hits, `unbounded .list() calls found:\n${hits.join("\n")}`).toEqual([]); + }); +}); diff --git a/test/upstream33-cli-summary.test.ts b/test/upstream33-cli-summary.test.ts new file mode 100644 index 0000000000..a44617ab77 --- /dev/null +++ b/test/upstream33-cli-summary.test.ts @@ -0,0 +1,43 @@ +import {afterEach, expect, it, vi} from 'vitest'; +import {readFileSync} from 'node:fs'; +import ts from 'typescript'; +import {getSummaryTimeoutMs,getSummaryTriggerTimeoutMs} from '../src/functions/summary-state.js'; +import {registerSummarizeFunction} from '../src/functions/summarize.js'; +import {KV} from '../src/state/schema.js'; +afterEach(()=>{vi.unstubAllEnvs();vi.useRealTimers();}); +it('validates the official configurable summary deadline and trigger headroom',()=>{ + for(const bad of ['nope','600s','1e6','0.5','0','2592000000']){vi.stubEnv('AGENTMEMORY_SUMMARY_TIMEOUT_MS',bad);expect(getSummaryTimeoutMs()).toBe(120000);} + vi.stubEnv('AGENTMEMORY_SUMMARY_TIMEOUT_MS','600000');expect(getSummaryTimeoutMs()).toBe(600000);expect(getSummaryTriggerTimeoutMs()).toBe(660000); +}); +it.each([['120000',false],['600000',true]])('small XML path honors %s ms deadline',async(timeout,success)=>{ + vi.stubEnv('AGENTMEMORY_SUMMARY_TIMEOUT_MS',timeout);vi.useFakeTimers(); + const handlers=new Map();const rows=new Map();const obs={id:'o',sessionId:'s',title:'edit',narrative:'synthetic',timestamp:'2026-01-01',compressedAt:'2026-01-01',facts:[],concepts:[],files:[],importance:5}; + rows.set(`${KV.sessions}:s`,{id:'s',status:'completed',project:'fixture',startedAt:'2026-01-01',observationCount:1}); + const kv={get:async(s:string,k:string)=>rows.get(`${s}:${k}`)??null,list:async(s:string)=>s===KV.observations('s')?[obs]:[],set:async(s:string,k:string,v:any)=>rows.set(`${s}:${k}`,v),update:async()=>{},delete:async()=>{}}; + const provider={name:'fixture',compress:vi.fn(),summarize:async()=>{await new Promise(r=>setTimeout(r,150000));return 'syntheticSynthetic coding work completed safely.';}}; + const worker=registerSummarizeFunction({registerFunction:(id:string,fn:any)=>handlers.set(id,fn),trigger:async()=>({})} as never,kv as never,provider as never,undefined,{schedule:false}); + try{const pending=handlers.get('mem::summarize')({sessionId:'s'});await vi.advanceTimersByTimeAsync(160000);expect(await pending).toMatchObject({success});}finally{await worker.stop();} +}); +function stopFixture(overrides:Record={}){ + const source=readFileSync(new URL('../src/cli.ts',import.meta.url),'utf8');const block=source.slice(source.indexOf('async function stopComposeRuntime('),source.indexOf('async function runStop()')); + const deps:any={platform:()=> 'linux',composeChildPids:()=>[],nativeProcessStart:()=> '1',pidAlive:()=>true,composePidMatches:()=>true,composeCall:vi.fn(async()=>{}),composeDownTimeoutMs:()=>1000,terminateComposeChildren:vi.fn(async()=>true),signalAndWait:vi.fn(async()=>true),clearEnginePidfile:vi.fn(),clearWorkerPidfile:vi.fn(),clearEngineState:vi.fn(),p:{log:{warn:vi.fn()}},...overrides}; + const fn=new Function(...Object.keys(deps),ts.transpileModule(block,{compilerOptions:{target:ts.ScriptTarget.ES2022}}).outputText+';return stopComposeRuntime;')(...Object.values(deps));return{fn,deps}; +} +it('Compose demo uses verified shared stop while native cleanup stays separate',()=>{ + const s=readFileSync(new URL('../src/cli.ts',import.meta.url),'utf8');const demo=s.slice(s.indexOf('async function startServerForDemo'),s.indexOf('async function runDemo()'));expect(demo).toContain('IS_WINDOWS && spawnedComposeDaemon');expect(demo).toContain('state.composePid !== spawnedComposeDaemon.pid');expect(demo).toContain('await stopComposeRuntime(state, true)');expect(demo).toContain('await cleanupFailedStartup()'); +}); +it('Compose stop refuses mismatched owner without clearing or signaling',async()=>{const{fn,deps}=stopFixture({composePidMatches:()=>false});await expect(fn({composePid:42},true)).rejects.toThrow(/does not match/);expect(deps.composeCall).not.toHaveBeenCalled();expect(deps.signalAndWait).not.toHaveBeenCalled();expect(deps.clearEngineState).not.toHaveBeenCalled();}); +it('Compose stop preserves metadata on failed worker stop',async()=>{const{fn,deps}=stopFixture({terminateComposeChildren:async()=>false});await expect(fn({composePid:42},true)).rejects.toThrow(/did not stop/);expect(deps.clearEngineState).not.toHaveBeenCalled();}); +it('Compose stop clears metadata only after workers and owner stop',async()=>{const{fn,deps}=stopFixture();await fn({composePid:42},true);expect(deps.composeCall).toHaveBeenCalledOnce();expect(deps.signalAndWait).toHaveBeenCalledWith(42,'SIGTERM',20000,true);expect(deps.clearEngineState).toHaveBeenCalledOnce();}); +it('lets Compose stop its engine before any child cleanup',async()=>{ + const order:string[]=[];const{fn}=stopFixture({composeCall:async()=>{order.push('down');},signalAndWait:async()=>{order.push('daemon');return true;},terminateComposeChildren:async()=>{order.push('children');return true;}}); + await fn({composePid:42},true);expect(order).toEqual(['down','daemon','children']); +}); +it('keeps metadata and skips child cleanup when daemon refuses graceful stop',async()=>{ + const{fn,deps}=stopFixture({signalAndWait:async()=>false});await expect(fn({composePid:42},true)).rejects.toThrow(/daemon.*did not stop/);expect(deps.terminateComposeChildren).not.toHaveBeenCalled();expect(deps.clearEngineState).not.toHaveBeenCalled(); +}); +it('captured Linux child cleanup refuses reused PID and never escalates',async()=>{ + const source=readFileSync(new URL('../src/cli.ts',import.meta.url),'utf8');const block=source.slice(source.indexOf('async function terminateComposeChildren('),source.indexOf('async function startIiiBin('));const signal=vi.fn(async()=>true); + const load=(start:string)=>new Function('pidAlive','nativeProcessStart','signalAndWait',ts.transpileModule(block,{compilerOptions:{target:ts.ScriptTarget.ES2022}}).outputText+';return terminateComposeChildren;')(()=>true,()=>start,signal); + expect(await load('new')({},[{pid:12,start:'old'}])).toBe(false);expect(signal).not.toHaveBeenCalled();expect(await load('same')({},[{pid:12,start:'same'}])).toBe(true);expect(signal).toHaveBeenCalledWith(12,'SIGTERM',5000,true); +}); diff --git a/test/vector-backfill-recovery.test.ts b/test/vector-backfill-recovery.test.ts index 10b00928dd..01065e9608 100644 --- a/test/vector-backfill-recovery.test.ts +++ b/test/vector-backfill-recovery.test.ts @@ -1,7 +1,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { IndexPersistence } from "../src/state/index-persistence.js"; import { VectorIndex } from "../src/state/vector-index.js"; -import { backfillVectorBacklog, getPendingVectorBackfillCount, getSearchIndex, indexRecords, rebuildKeywordIndex, setEmbeddingProvider, setIndexPersistence, setVectorIndex, vectorIndexRemove } from "../src/functions/search.js"; +import { backfillVectorBacklog, getPendingVectorBackfillCount, getSearchIndex, indexRecords, isBm25RebuildIncomplete, rebuildKeywordIndex, setEmbeddingProvider, setIndexPersistence, setVectorIndex, vectorIndexRemove } from "../src/functions/search.js"; import type { CompressedObservation } from "../src/types.js"; import { KV } from "../src/state/schema.js"; @@ -48,6 +48,135 @@ afterEach(() => { }); describe("vector recovery across checkpoints", () => { + it("counts historical gaps by eligible ID, including undated latest memories and observations", async () => { + const kv = storage(); await seed(kv, 0); + const first = await boot(kv); + const cutoff = "2026-01-01T00:00:00Z"; + const rows = [ + { id: "equal", timestamp: cutoff }, + { id: "undated" }, + { id: "invalid", timestamp: "invalid" }, + { id: "newer", timestamp: "2099-01-01T00:00:00Z" }, + { id: "no_title", title: "" }, + { id: "no_narrative", narrative: "" }, + ]; + for (const row of rows) await kv.set(KV.observations("session"), row.id, { sessionId: "session", title: "fixture", narrative: "record", type: "discovery", facts: [], concepts: [], files: [], ...row }); + for (const row of [ + { id: "latest", createdAt: cutoff }, { id: "undated_memory" }, { id: "invalid_memory", createdAt: "invalid" }, + { id: "superseded", isLatest: false }, { id: "no_title_memory", title: "" }, { id: "no_content", content: "" }, + ]) await kv.set(KV.memories, row.id, { title: "fixture", content: "record", type: "discovery", facts: [], concepts: [], files: [], ...row }); + for (let i = 0; i < 10; i++) first.vector.add(`stale_${i}`, "session", new Float32Array([1, 1, 1])); + let scan = await rebuildKeywordIndex(kv as never, cutoff); + expect(scan.historicalVectorGaps).toBe(6); + expect(scan.documents).toBe(7); + expect(scan.vectorJobs.map((job) => job.id)).toEqual(["newer"]); + for (const job of scan.vectorJobs) job.revision?.release(); + for (const id of ["equal", "undated", "invalid", "latest", "undated_memory", "invalid_memory"]) first.vector.add(id, "session", new Float32Array([1, 1, 1])); + scan = await rebuildKeywordIndex(kv as never, cutoff); + expect(scan.historicalVectorGaps).toBe(0); expect(scan.vectorJobs.map((job) => job.id)).toEqual(["newer"]); + for (const job of scan.vectorJobs) job.revision?.release(); + first.vector.add("newer", "session", new Float32Array([1, 1, 1])); + expect(await rebuildKeywordIndex(kv as never, cutoff)).toMatchObject({ historicalVectorGaps: 0, vectorJobs: [] }); + first.vector.remove("equal"); + setEmbeddingProvider(null); + expect(await rebuildKeywordIndex(kv as never, cutoff)).toMatchObject({ historicalVectorGaps: 0, vectorJobs: [] }); + setEmbeddingProvider(first.embedding); setVectorIndex(null); + expect(await rebuildKeywordIndex(kv as never, cutoff)).toMatchObject({ historicalVectorGaps: 0, vectorJobs: [] }); + }); + + it("pauses historical gap promotion without executing the superseded incremental jobs when marker writes fail", async () => { + const kv = storage(); await seed(kv); + const original = await boot(kv); + original.vector.add("obs_0", "session", new Float32Array([1, 1, 1])); + await original.persistence.save(); original.persistence.stop(); + await kv.set(KV.observations("session"), "newer", { id: "newer", sessionId: "session", title: "fixture", narrative: "record", timestamp: "2099-01-01T00:00:00Z", type: "discovery", facts: [], concepts: [], files: [] }); + let fail = false; + const faulty = { ...kv, set: async (scope: string, key: string, value: T) => { + if (fail && key === "vectors:backfill") throw new Error("promotion unavailable"); + return kv.set(scope, key, value); + } }; + const first = await boot(faulty); + const since = await first.persistence.prepareBackfill(first.loaded); + const scan = await rebuildKeywordIndex(faulty as never, since); + expect(scan.historicalVectorGaps).toBe(2); expect(scan.vectorJobs.map((job) => job.id)).toEqual(["newer"]); + for (const job of scan.vectorJobs) job.revision?.release(); + fail = true; + expect(await first.persistence.prepareBackfill(first.loaded, true)).toBeUndefined(); + expect(first.persistence.isRecoveryPaused()).toBe(true); + expect(await rebuildKeywordIndex(faulty as never, undefined)).toMatchObject({ historicalVectorGaps: 0, vectorJobs: [] }); + expect(first.embedding.embedBatch).not.toHaveBeenCalled(); + }); + + it("retains known historical gaps and recovery intent when keyword scans are incomplete", async () => { + const kv = storage(); await seed(kv); + const faulty = { ...kv, list: async (scope: string): Promise => { + if (scope === KV.memories) throw new Error("scan unavailable"); + return kv.list(scope); + } }; + const first = await boot(faulty); + const scan = await rebuildKeywordIndex(faulty as never, "2026-01-01T00:00:00Z"); + expect(scan.historicalVectorGaps).toBe(3); expect(isBm25RebuildIncomplete()).toBe(true); + const since = await first.persistence.prepareBackfill(first.loaded, true); + const replacement = await rebuildKeywordIndex(faulty as never, since); + expect(replacement.fullBackfillPending).toBe(3); expect(replacement.vectorJobs).toHaveLength(0); + expect(isBm25RebuildIncomplete()).toBe(true); + expect(await first.persistence.readBackfillMarker()).toBeNull(); + expect(first.embedding.embedBatch).not.toHaveBeenCalled(); + }); + + it("already selects whole-store recovery when the recorded checkpoint count exceeds loadable vectors", async () => { + const kv = storage(); await seed(kv); + const first = await boot(kv); + first.vector.add("obs_0", "session", new Float32Array([1, 1, 1])); + await first.persistence.save(); first.persistence.stop(); + const meta = (await kv.get>(KV.bm25Index, "vectors:meta"))!; + delete meta.backfillSince; + await kv.set(KV.bm25Index, "vectors:meta", { ...meta, count: 2 }); + await kv.delete(KV.bm25Index, "vectors:backfill"); + const restarted = await boot(kv); + expect(restarted.loaded.expectedCount).toBe(2); expect(restarted.vector.size).toBe(1); + expect(await restarted.persistence.prepareBackfill(restarted.loaded)).toBeNull(); + }); + + it("promotes matching-count partial checkpoints before opt-in and repairs historical gaps across capped restarts", async () => { + const kv = storage(); await seed(kv); + const original = await boot(kv); + original.vector.add("obs_0", "session", new Float32Array([1, 1, 1])); + await original.persistence.save(); original.persistence.stop(); + const meta = (await kv.get>(KV.bm25Index, "vectors:meta"))!; + delete meta.backfillSince; + await kv.set(KV.bm25Index, "vectors:meta", meta); + await kv.delete(KV.bm25Index, "vectors:backfill"); + const first = await boot(kv); + expect(first.loaded.expectedCount).toBe(1); expect(first.vector.size).toBe(1); + let since = await first.persistence.prepareBackfill(first.loaded); + expect(typeof since).toBe("string"); + let scan = await rebuildKeywordIndex(kv as never, since); + expect(scan).toMatchObject({ historicalVectorGaps: 2, vectorJobs: [], fullBackfillPending: 0 }); + for (const job of scan.vectorJobs) job.revision?.release(); + since = await first.persistence.prepareBackfill(first.loaded, true); + expect(since).toBeNull(); expect(await first.persistence.readBackfillMarker()).toBeNull(); + scan = await rebuildKeywordIndex(kv as never, since); + expect(scan.vectorJobs).toHaveLength(0); expect(scan.fullBackfillPending).toBe(2); + expect(getSearchIndex().countMissingDocuments((id) => first.vector.has(id), since)).toBe(2); + expect(first.embedding.embedBatch).not.toHaveBeenCalled(); + vi.stubEnv("AGENTMEMORY_VECTOR_BACKFILL", "all"); + vi.stubEnv("AGENTMEMORY_VECTOR_BACKFILL_MAX", "1"); + scan = await rebuildKeywordIndex(kv as never, since); + expect(scan.vectorJobs.map((job) => job.id)).toEqual(["obs_1"]); + expect(await backfillVectorBacklog(scan.vectorJobs, { pauseMs: 0, remainingAfter: scan.fullBackfillPending })).toMatchObject({ added: 1, remaining: 1, complete: false }); + for (const job of scan.vectorJobs) job.revision?.release(); + await first.persistence.save(); first.persistence.stop(); + const restarted = await boot(kv); + since = await restarted.persistence.prepareBackfill(restarted.loaded); + expect(since).toBeNull(); + scan = await rebuildKeywordIndex(kv as never, since); + expect(scan.vectorJobs.map((job) => job.id)).toEqual(["obs_2"]); + expect(await backfillVectorBacklog(scan.vectorJobs, { pauseMs: 0 })).toMatchObject({ added: 1, complete: true }); + expect(await restarted.persistence.completeBackfill(() => getSearchIndex().countMissingDocuments((id) => restarted.vector.has(id), since) === 0)).toBe(true); + expect(restarted.vector.size).toBe(3); + }); + it("retains a whole-store backlog beyond the cap and resumes old rows after a checkpoint and restart", async () => { vi.stubEnv("AGENTMEMORY_VECTOR_BACKFILL", "all"); vi.stubEnv("AGENTMEMORY_VECTOR_BACKFILL_MAX", "2"); diff --git a/test/vector-checkpoint-proof.test.ts b/test/vector-checkpoint-proof.test.ts new file mode 100644 index 0000000000..b77ce75b0f --- /dev/null +++ b/test/vector-checkpoint-proof.test.ts @@ -0,0 +1,154 @@ +import {afterEach, beforeEach, expect, it} from "vitest"; +import {mkdtemp, rename, rm, symlink, writeFile} from "node:fs/promises"; +import {tmpdir} from "node:os"; +import {join} from "node:path"; +import {verifyVectorCheckpointFiles, verifyRemainingVectorPending, verifyVectorRetentionReceipts} from "../src/ops/vector-checkpoint-proof.mjs"; + +const pending = "mem:index:vec-pending", metaScope = "mem:index:bm25", bucket = metaScope + ":vec:0000"; +const row = {id:"v1",q:1,s:"session",e:"AACAPw=="}; +const vector = {id:row.id,s:row.s,e:row.e}; +const metadata = {v:3,bucketCount:1,count:1,savedAt:"2026-11-07T00:00:00Z"}; +let storePath:string; +const file = (scope:string) => join(storePath,encodeURIComponent(scope)+".bin"); + +// rkyv 0.8 LE32 ArchivedString: short JSON is inline, long strings use an aligned length/offset. +function archive(text:string){ + const bytes=Buffer.from(text); + if(bytes.length<=8){const result=Buffer.alloc(8,0xff);bytes.copy(result);return result;} + const position=Math.ceil(bytes.length/4)*4,result=Buffer.alloc(position+8); + bytes.copy(result); + result.writeUInt32LE((bytes.length & 63) + 128 + Math.floor(bytes.length/64)*256,position); + result.writeInt32LE(-position,position+4); + return result; +} +const put = (scope:string, value:unknown) => writeFile(file(scope),archive(JSON.stringify(value))); +const input = () => ({storePath,pendingRows:[{...row}],meta:{...metadata},buckets:[{bucket:0,ids:["v1"]}],admitted:()=>true}); +beforeEach(async()=>{ + storePath=await mkdtemp(join(tmpdir(),"am-vector-proof-")); + await put(pending,{v1:row});await put(metaScope,{"vectors:meta":metadata});await put(bucket,{v1:vector}); +}); +afterEach(async()=>{await rm(storePath,{recursive:true,force:true});}); + +it("matches physical keys and SDK values under one admitted generation",async()=>{ + const check=await verifyVectorCheckpointFiles(input()); + await expect(check()).resolves.toBeUndefined(); + await expect(verifyRemainingVectorPending({...input(),pendingRows:[]})).rejects.toThrow(); + await rm(file(pending)); + await expect(verifyRemainingVectorPending({...input(),pendingRows:[]})).resolves.toBeUndefined(); + await put(pending,{}); + await expect(verifyRemainingVectorPending({...input(),pendingRows:[]})).resolves.toBeUndefined(); +}); +it.each(["alias","extra","trailer","duplicate","changed_meta","symlink","unadmitted"])("refuses %s",async kind=>{ + const opts=input(); + if(kind==="alias")await put(bucket,{other:vector}); + if(kind==="extra")await put(pending,{v1:row,stale:{...row,id:"stale",q:2}}); + if(kind==="trailer"){const buf=archive(JSON.stringify({v1:row}));buf.writeInt32LE(0,buf.length-4);await writeFile(file(pending),buf);} + if(kind==="duplicate")await writeFile(file(pending),archive('{"v1":'+JSON.stringify(row)+',"v1":'+JSON.stringify(row)+'}')); + if(kind==="changed_meta")await put(metaScope,{"vectors:meta":{...metadata,count:2}}); + if(kind==="symlink"){await rename(file(bucket),file("elsewhere"));await symlink(file("elsewhere"),file(bucket));} + if(kind==="unadmitted")opts.admitted=()=>false; + await expect(verifyVectorCheckpointFiles(opts)).rejects.toThrow(); +}); +it("rejects same-byte inode replacement or a lost writer before mutation",async()=>{ + let admitted=true; + const check=await verifyVectorCheckpointFiles({...input(),admitted:()=>admitted}); + admitted=false;await expect(check()).rejects.toThrow(); + admitted=true;await rename(file(bucket),file("old"));await put(bucket,{v1:vector}); + await expect(check()).rejects.toThrow(); +}); + +it("rejects an extra physical copy of a selected ID even under another key or selected bucket",async()=>{ + const second={...row,id:"v2",q:2},meta={...metadata,bucketCount:2,count:2}; + await put(pending,{v1:row,v2:second});await put(metaScope,{"vectors:meta":meta}); + await put(metaScope+":vec:0001",{v2:{...vector,id:"v2"},other:vector}); + await expect(verifyVectorCheckpointFiles({...input(),meta,pendingRows:[row,second], + buckets:[{bucket:0,ids:["v1"]},{bucket:1,ids:["v2"]}]})).rejects.toThrow(); +}); + +it("never treats a missing namespace as evidence for remaining work",async()=>{ + await rm(file(pending)); + await expect(verifyRemainingVectorPending(input())).rejects.toThrow(); + await expect(verifyVectorCheckpointFiles(input())).rejects.toThrow(); + await symlink(file("missing"),file(pending)); + await expect(verifyRemainingVectorPending({...input(),pendingRows:[]})).rejects.toThrow(); +}); + +it("rejects ambiguous sequence numbers and invalid float data before proof",async()=>{ + await expect(verifyRemainingVectorPending({...input(),pendingRows:[row,{...row,id:"v2"}]})).rejects.toThrow(); + const bad={...row,e:Buffer.from(new Float32Array([NaN]).buffer).toString("base64")}; + await put(pending,{v1:bad}); + await expect(verifyRemainingVectorPending({...input(),pendingRows:[bad]})).rejects.toThrow(); +}); + +const retained = "mem:index:vec-retained:v1"; +const digest = "ab".repeat(32), otherDigest = "cd".repeat(32); +const stamp = {v:1,id:"v1",q:1,bucket:0,digest}; +const retention = (receipts:Record, admitted=()=>true) => + verifyVectorRetentionReceipts({storePath,receipts,admitted}); + +it("proves exactly all receipt keys and accepts an absent namespace only for zero receipts",async()=>{ + await expect(retention({})).resolves.toBeUndefined(); + await expect(retention({[digest]:stamp})).rejects.toThrow(); + await put(retained,{[digest]:stamp}); + await expect(retention({[digest]:stamp})).resolves.toBeUndefined(); + await expect(retention({})).rejects.toThrow(); + const other={...stamp,digest:otherDigest,id:"v2",q:2}; + await put(retained,{[otherDigest]:other,[digest]:stamp}); + await expect(retention({[digest]:stamp,[otherDigest]:other})).resolves.toBeUndefined(); + await expect(retention({[digest]:stamp})).rejects.toThrow(); +}); + +it.each(["digest","key","version","extra","missing","empty_id","clear_id","id_length","sequence","bucket", + "duplicate_id","duplicate_q","too_many"])("rejects invalid retention receipt %s even on matching disk",async kind=>{ + const row:Record={...stamp}; + const receipts:Record={[digest]:row}; + if(kind==="digest")row.digest=otherDigest; + if(kind==="key"){delete receipts[digest];row.digest="X".repeat(64);receipts[String(row.digest)]=row;} + if(kind==="version")row.v=2; + if(kind==="extra")row.unsupported=true; + if(kind==="missing")delete row.bucket; + if(kind==="empty_id")row.id=""; + if(kind==="clear_id")row.id="~clear"; + if(kind==="id_length")row.id="a".repeat(513); + if(kind==="sequence")row.q=0; + if(kind==="bucket")row.bucket=-1; + if(kind==="duplicate_id")receipts[otherDigest]={...stamp,digest:otherDigest,q:2}; + if(kind==="duplicate_q")receipts[otherDigest]={...stamp,digest:otherDigest,id:"v2"}; + if(kind==="too_many")for(let i=1;i<=16;i++){ + const d=i.toString(16).padStart(64,"0");receipts[d]={...stamp,id:"v"+(i+1),q:i+1,digest:d}; + } + await put(retained,receipts); + await expect(retention(receipts)).rejects.toThrow(); +}); + +it("refuses duplicate physical receipt keys, a symlink and writer loss during recheck", async()=>{ + const receipts={[digest]:stamp}; + const kv=JSON.stringify(digest)+":"+JSON.stringify(stamp); + await writeFile(file(retained),archive("{"+kv+","+kv+"}")); + await expect(retention(receipts)).rejects.toThrow(); + await rename(file(retained),file("old-receipts")); + await symlink(file("old-receipts"),file(retained)); + await expect(retention(receipts)).rejects.toThrow(); + await rm(file(retained)); + await put(retained,receipts); + let checks=0; + await expect(retention(receipts,()=>++checks<4)).rejects.toThrow(); +}); + +it('waits for a previous savedAt to flush, then still checks the complete selected bucket',async()=>{ + const stale={...metadata,savedAt:'2026-11-06T23:59:59Z'}; + await put(metaScope,{'vectors:meta':stale});await put(bucket,{v1:{...vector,e:'AAAAAA=='}}); + const flushed=new Promise((resolve,reject)=>setTimeout(()=>{put(metaScope,{'vectors:meta':metadata}).then(resolve,reject);},175)); + await expect(verifyVectorCheckpointFiles({...input(),waitForCaptureMeta:true})).rejects.toThrow('VECTOR_CHECKPOINT_UNVERIFIED'); + await flushed; + await put(metaScope,{'vectors:meta':stale});await put(bucket,{v1:vector}); + const next=new Promise((resolve,reject)=>setTimeout(()=>{put(metaScope,{'vectors:meta':metadata}).then(resolve,reject);},175)); + const recheck=await verifyVectorCheckpointFiles({...input(),waitForCaptureMeta:true}); + await next;await recheck(); +}); +it.each(['persistent lag','revoked epoch'])('keeps the original rejection on %s during capture proof',async mode=>{ + await put(metaScope,{'vectors:meta':{...metadata,savedAt:'2026-11-06T23:59:59Z'}}); + let active=true;const timer=mode==='revoked epoch'?setTimeout(()=>{active=false},100):undefined; + try{await expect(verifyVectorCheckpointFiles({...input(),admitted:()=>active,waitForCaptureMeta:true})).rejects.toThrow('VECTOR_CHECKPOINT_UNVERIFIED');} + finally{clearTimeout(timer);} +}); diff --git a/test/vector-checkpoint-transaction.test.ts b/test/vector-checkpoint-transaction.test.ts new file mode 100644 index 0000000000..e6d94e6607 --- /dev/null +++ b/test/vector-checkpoint-transaction.test.ts @@ -0,0 +1,135 @@ +import {afterEach,expect,it,vi} from 'vitest'; +import {mkdtempSync,writeFileSync,rmSync,readFileSync} from 'node:fs'; +import {join} from 'node:path'; +import {tmpdir} from 'node:os'; +import {IndexPersistence} from '../src/state/index-persistence.js'; +import {VectorIndex} from '../src/state/vector-index.js'; +import {createStateDurabilityFence} from '../src/ops/state-durability.mjs'; +import {StateKV} from '../src/state/kv.js'; +import {acquireFileWriterLease} from '../src/ops/file-writer-lease.mjs'; +import {engineFixture} from './helpers/engine-evidence.js'; + +const pending='mem:index:vec-pending', index='mem:index:bm25', bucket=index+':vec:0000'; +const dirs:string[]=[]; +afterEach(()=>{vi.unstubAllEnvs();for(const d of dirs.splice(0))rmSync(d,{recursive:true,force:true});}); +function archive(value:unknown){ + const bytes=Buffer.from(JSON.stringify(value)); + if(bytes.length<=8){const inline=Buffer.alloc(8,0xff);bytes.copy(inline);return inline;} + const root=Math.ceil(bytes.length/4)*4,out=Buffer.alloc(root+8); + bytes.copy(out);out.writeUInt32LE((bytes.length&63)+128+Math.floor(bytes.length/64)*256,root);out.writeInt32LE(-root,root+4); + return out; +} +function fixture(){ + const store=mkdtempSync(join(tmpdir(),'am-vector-proof-'));dirs.push(store);vi.stubEnv('AGENTMEMORY_STATE_STORE_PATH',store); + const {evidence}=engineFixture(store);const lease=acquireFileWriterLease(store,{evidence}); + const data=new Map>(),path=(scope:string)=>join(store,encodeURIComponent(scope)+'.bin'); + const write=(scope:string)=>writeFileSync(path(scope),archive(Object.fromEntries(data.get(scope)!))); + const rows=[1,2,3].map(q=>({id:'vec_'+q,s:'session',e:'AACAPw==',q})); + const meta={v:3,count:3,bucketCount:1,savedAt:'2026-01-01T00:00:00.000Z'}; + data.set(index,new Map([['vectors:meta',meta]]));data.set(bucket,new Map(rows.map(({id,s,e})=>[id,{id,s,e}]))); + data.set(pending,new Map(rows.map(row=>[row.id,row]))); + for(const scope of data.keys())write(scope); + const receipts='mem:index:vec-retained:v1'; + const actions:Array<{scope:string,key:string,op:string}>=[];let onSet=async(_key:string)=>{}; + const sdk={trigger:async({function_id,payload:{scope,key,value}}:any)=>{ + const scoped=data.get(scope)??new Map();data.set(scope,scoped); + if(function_id==='state::list')return [...scoped.values()]; + if(function_id==='state::get')return scoped.get(key)??null; + if(function_id==='state::delete')throw Error('unexpected destructive SDK call'); + if(function_id==='state::set'){await onSet(key);actions.push({scope,key,op:'set'});scoped.set(key,value);write(scope);return value;} + throw Error('unexpected SDK call'); + }}; + const kv=new StateKV(sdk as never,{writerLease:lease}); + const selection=(rs=rows)=>({pendingRows:rs,meta,buckets:[{bucket:0,ids:rs.map(row=>row.id)}]}); + const prove=(authorize=true)=>kv.reconcileVectorCheckpoint(selection(),()=>true,kv.vectorCheckpointRevision,authorize); + return {kv,sdk,store,lease,rows,disk:path,actions,data,selection,receipts,prove,write,onSet:(fn:typeof onSet)=>{onSet=fn;}}; +} +it('certifies three original rows without modifying their buckets or WAL, including retry or restart',async()=>{ + const x=fixture(),initial=readFileSync(x.disk(bucket)),wal=readFileSync(x.disk(pending)); + try{ + const result=await x.prove();expect(result).toHaveLength(3); + expect(x.actions).toEqual(result.map(s=>({scope:x.receipts,key:s.digest,op:'set'}))); + expect(await x.prove()).toEqual(result);expect(await x.prove(false)).toEqual(result); + expect(x.actions).toHaveLength(3); + expect(readFileSync(x.disk(pending))).toEqual(wal); + expect(readFileSync(x.disk(bucket))).toEqual(initial); + }finally{x.lease.close();} +}); +it.each(['changed disk checkpoint','lost authorization','no manual authority'])('performs no writes on %s',async mode=>{ + const x=fixture();try{ + if(mode==='changed disk checkpoint')writeFileSync(x.disk(bucket),archive({})); + await expect(x.kv.reconcileVectorCheckpoint(x.selection(),()=>mode!=='lost authorization',x.kv.vectorCheckpointRevision,mode!=='no manual authority')).rejects.toThrow(); + expect(x.actions).toHaveLength(0); + }finally{x.lease.close();} +}); +it('retains all source records and blocks same-epoch retry on failed receipt creation',async()=>{ + const x=fixture(),wal=readFileSync(x.disk(pending));try{ + let n=0;x.onSet(async()=>{if(++n===2)throw Error('disconnected');}); + await expect(x.prove()).rejects.toThrow();expect(x.actions).toHaveLength(1); + await expect(x.prove()).rejects.toThrow(/unverified/); + expect(readFileSync(x.disk(pending))).toEqual(wal); + }finally{x.lease.close();} +}); +it('refuses changed remote bytes even after a receipt write without deleting the changed original',async()=>{ + const x=fixture();try{ + const changed={...x.rows[0],q:17};x.onSet(async()=>{x.data.get(pending)!.set(changed.id,changed);x.write(pending);}); + await expect(x.prove()).rejects.toThrow(); + expect(x.data.get(pending)!.get(changed.id)).toEqual(changed); + expect(x.actions.every(a=>a.scope===x.receipts)).toBe(true); + await expect(x.prove()).rejects.toThrow(); + }finally{x.lease.close();} +}); +it('serializes a new local WAL key until all originals have been reverified',async()=>{ + const x=fixture();let release!:()=>void,signal!:()=>void; + const reached=new Promise(r=>signal=r);const wait=new Promise(r=>release=r); + try{ + x.onSet(async()=>{signal();await wait;}); + const repair=x.prove();await reached; + const newRow={...x.rows[0],id:'different',q:5}; + const write=x.kv.set(pending,newRow.id,newRow); + await new Promise(r=>setTimeout(r,10)); + expect(x.data.get(pending)!.has(newRow.id)).toBe(false);release();await repair;await write; + expect([...x.data.get(pending)!.keys()]).toEqual([...x.rows.map(r=>r.id),'different']); + }finally{release?.();x.lease.close();} +}); +it('rejects a checkpoint changed while the SDK was discovering its bucket map',async()=>{ + const x=fixture();try{ + const revision=x.kv.vectorCheckpointRevision; + await x.kv.set(bucket,'new-after-load',{id:'new-after-load',s:'s',e:'AACAPw=='}); + await expect(x.kv.reconcileVectorCheckpoint(x.selection(),()=>true,revision,true)).rejects.toThrow(/checkpoint_changed/); + expect(x.actions.some(a=>a.scope===x.receipts)).toBe(false); + }finally{x.lease.close();} +}); + + +it('keeps the standalone disk entry point while requiring a boot check for existing or invalid retention receipts',async()=>{ + const x=fixture();try{ + const disk=new StateKV(x.sdk as never,{durability:createStateDurabilityFence({storePath:x.store,completion:'disk'})}); + const plain=new VectorIndex(),oldCaller=new IndexPersistence(disk,plain); + expect(()=>plain.add('new','s',new Float32Array([1]))).not.toThrow(); + await oldCaller.save();oldCaller.stop(); + for(const bad of [false,true]){ + writeFileSync(x.disk(x.receipts),bad?Buffer.from('invalid'):archive({})); + const target=new VectorIndex(),fresh=new IndexPersistence(disk,target); + try{expect(()=>target.add('new','s',new Float32Array([1]))).toThrow(/VECTOR_RECOVERY_PAUSED/); + expect(target.pendingChanges).toBe(0);}finally{fresh.stop();} + } + }finally{x.lease.close();} +}); + +it('revalidates the current SDK metadata after a retained capture waited for disk, without writing',async()=>{ + const x=fixture();try{ + await x.prove(); + const selected=x.selection(), older={...selected.meta,savedAt:'2025-12-31T00:00:00.000Z'}; + const wal=readFileSync(x.disk(pending)),writes=x.actions.length; + writeFileSync(x.disk(index),archive({'vectors:meta':older})); + const repair=x.kv.reconcileVectorCheckpoint(selected,()=>true,x.kv.vectorCheckpointRevision,false,true); + const timer=setTimeout(()=>{ + writeFileSync(x.disk(index),archive({'vectors:meta':selected.meta})); + // A second publisher advanced SDK state after selection; the old disk value cannot certify that change. + x.data.get(index)!.set('vectors:meta',{...selected.meta,savedAt:'2026-01-02T00:00:00.000Z'}); + },175); + try{await expect(repair).rejects.toThrow(/meta_changed/);}finally{clearTimeout(timer);} + expect(x.actions).toHaveLength(writes);expect(readFileSync(x.disk(pending))).toEqual(wal); + }finally{x.lease.close();} +}); diff --git a/test/vector-pending-log.test.ts b/test/vector-pending-log.test.ts index 8f39300d07..a8a3d4e125 100644 --- a/test/vector-pending-log.test.ts +++ b/test/vector-pending-log.test.ts @@ -1,6 +1,7 @@ import { describe, it, expect, afterEach, vi } from "vitest"; import { IndexPersistence } from "../src/state/index-persistence.js"; import { VectorIndex } from "../src/state/vector-index.js"; +import { createStateDurabilityFence } from "../src/ops/state-durability.mjs"; import { backfillVectorBacklog, getPendingVectorBackfillCount, @@ -69,6 +70,79 @@ describe("pending vector log", () => { setIndexPersistence(null); }); + it('keeps pending records and refuses vector recovery during storage-held startup', async () => { + const kv=mockKV();const original={q:1,id:'obs_held',s:'ses_held',e:Buffer.from(v(1).buffer).toString('base64')}; + await kv.set(PENDING_SCOPE,'obs_held',original);const before=structuredClone(original); + const vector=new VectorIndex();const persistence=new IndexPersistence(kv as never,vector,{requireRecoveryPlan:true,isStorageHeld:()=>true} as never); + try{ + const loaded=await persistence.load();await persistence.prepareBackfill(loaded);await persistence.replayPendingLog(3);await persistence.save(); + expect(pendingRows(kv).get('obs_held')).toEqual(before);expect(vector.size).toBe(0);expect(persistence.isRecoveryPaused()).toBe(true); + expect((persistence.status() as {pendingLogKnown?:boolean}).pendingLogKnown).toBe(false); + expect(persistence.status().pendingLog).toBeUndefined(); + expect(persistence.status().pendingLogAcknowledged).toBeUndefined(); + expect(kv.ops.filter(x=>x.op==='delete'&&x.scope===PENDING_SCOPE)).toEqual([]); + }finally{persistence.stop()} + }); + + it('retains every pending original and refuses replay when a pending row is malformed',async()=>{ + const kv=mockKV(),invalid={q:'malformed',c:1},valid={q:1,id:'obs_valid',s:'s',e:Buffer.from(v(1).buffer).toString('base64')}; + await kv.set(PENDING_SCOPE,'~clear',invalid);await kv.set(PENDING_SCOPE,'obs_valid',valid); + const vector=new VectorIndex(),p=new IndexPersistence(kv as never,vector,{requireRecoveryPlan:true}); + try{await p.prepareBackfill(await p.load());await p.replayPendingLog(3);await p.save();expect(pendingRows(kv).get('~clear')).toEqual(invalid);expect(pendingRows(kv).get('obs_valid')).toEqual(valid);expect(vector.has('obs_valid')).toBe(false);expect(p.isRecoveryPaused()).toBe(true);expect(p.status().pendingLogKnown).toBe(false)}finally{p.stop()} + }); + it('retains all pending originals when an embedding is undecodable',async()=>{ + const kv=mockKV(),invalid={q:1,id:'obs_bad',s:'s',e:'%%%'},valid={q:2,id:'obs_ok',s:'s',e:Buffer.from(v(1).buffer).toString('base64')}; + await kv.set(PENDING_SCOPE,'obs_bad',invalid);await kv.set(PENDING_SCOPE,'obs_ok',valid); + const vector=new VectorIndex(),p=new IndexPersistence(kv as never,vector,{requireRecoveryPlan:true}); + try{await p.prepareBackfill(await p.load());await p.replayPendingLog(3);await p.save();expect(pendingRows(kv).get('obs_bad')).toEqual(invalid);expect(pendingRows(kv).get('obs_ok')).toEqual(valid);expect(vector.size).toBe(0);expect(p.isRecoveryPaused()).toBe(true);expect(p.status().pendingLogKnown).toBe(false)}finally{p.stop()} + }); + + it('retains healthy startup replay when no storage hold is present', async () => { + const kv=mockKV();const original={q:1,id:'obs_healthy',s:'ses_healthy',e:Buffer.from(v(1).buffer).toString('base64')}; + await kv.set(PENDING_SCOPE,'obs_healthy',original);const vector=new VectorIndex(); + const persistence=new IndexPersistence(kv as never,vector,{requireRecoveryPlan:true,isStorageHeld:()=>false} as never); + try{const loaded=await persistence.load();await persistence.prepareBackfill(loaded);const replay=await persistence.replayPendingLog(3);expect(replay.added).toBe(1);expect(vector.has('obs_healthy')).toBe(true);expect(persistence.isRecoveryPaused()).toBe(false)}finally{persistence.stop()} + }); + + it('does not apply replay read when a storage hold appears before the read returns', async () => { + const kv=mockKV();const original={q:1,id:'obs_race',s:'ses_race',e:Buffer.from(v(1).buffer).toString('base64')};await kv.set(PENDING_SCOPE,'obs_race',original); + let held=false,release!:()=>void,entered!:()=>void;const wait=new Promise(r=>release=r),started=new Promise(r=>entered=r),base=kv.list; + kv.list=async (scope:string)=>{if(scope===PENDING_SCOPE){entered();await wait}return base(scope)}; + const vector=new VectorIndex(),persistence=new IndexPersistence(kv as never,vector,{requireRecoveryPlan:true,isStorageHeld:()=>held} as never); + try{await persistence.prepareBackfill(await persistence.load());const replay=persistence.replayPendingLog(3);await started;held=true;release();await replay;await persistence.save();expect(vector.has('obs_race')).toBe(false);expect(pendingRows(kv).get('obs_race')).toEqual(original);expect(persistence.isRecoveryPaused()).toBe(true)}finally{persistence.stop()} + }); + + it('retains pending entries if a storage hold arrives during a checkpoint write', async () => { + const kv=mockKV(),original={q:1,id:'obs_during_save',s:'s',e:Buffer.from(v(1).buffer).toString('base64')};await kv.set(PENDING_SCOPE,'obs_during_save',original); + let held=false,release!:()=>void,entered!:()=>void;const wait=new Promise(r=>release=r),started=new Promise(r=>entered=r),base=kv.set; + kv.set=async (scope:string,key:string,data:T)=>{if(scope===INDEX_SCOPE&&key==='vectors:meta'){entered();await wait}return base(scope,key,data)}; + const vector=new VectorIndex(),p=new IndexPersistence(kv as never,vector,{requireRecoveryPlan:true,isStorageHeld:()=>held} as never); + try{await p.prepareBackfill(await p.load());await p.replayPendingLog(3);const saving=p.save();await started;held=true;release();await saving;expect(pendingRows(kv).get('obs_during_save')).toEqual(original);expect(p.isRecoveryPaused()).toBe(true)}finally{p.stop()} + }); + + it('characterizes unresolved delete already sent when storage hold appears', async () => { + const kv=mockKV(),original={q:1,id:'obs_inflight',s:'s',e:Buffer.from(v(1).buffer).toString('base64')};await kv.set(PENDING_SCOPE,'obs_inflight',original); + let held=false,release!:()=>void,entered!:()=>void;const wait=new Promise(r=>release=r),started=new Promise(r=>entered=r),base=kv.delete; + kv.delete=async(scope:string,key:string)=>{if(scope===PENDING_SCOPE&&key==='obs_inflight'){entered();await wait}await base(scope,key)}; + const vector=new VectorIndex(),p=new IndexPersistence(kv as never,vector,{requireRecoveryPlan:true,isStorageHeld:()=>held} as never); + try{await p.prepareBackfill(await p.load());await p.replayPendingLog(3);const saving=p.save();await started;held=true;release();await saving;expect(pendingRows(kv).has('obs_inflight')).toBe(false)}finally{p.stop()} + }); + + it('characterizes a transient storage hold missed between asynchronous checks', async () => { + const kv=mockKV(),original={q:1,id:'obs_aba',s:'s',e:Buffer.from(v(1).buffer).toString('base64')};await kv.set(PENDING_SCOPE,'obs_aba',original); + let held=false,release!:()=>void,entered!:()=>void;const wait=new Promise(r=>release=r),started=new Promise(r=>entered=r),base=kv.list; + kv.list=async(scope:string)=>{if(scope===PENDING_SCOPE){entered();await wait}return base(scope)}; + const vector=new VectorIndex(),p=new IndexPersistence(kv as never,vector,{requireRecoveryPlan:true,isStorageHeld:()=>held} as never); + try{await p.prepareBackfill(await p.load());const replay=p.replayPendingLog(3);await started;held=true;held=false;release();await replay;expect(vector.has('obs_aba')).toBe(true)}finally{p.stop()} + }); + it('rejects replay when the admitted hold generation changes during a read', async () => { + const kv=mockKV(),original={q:1,id:'obs_epoch',s:'s',e:Buffer.from(v(1).buffer).toString('base64')};await kv.set(PENDING_SCOPE,'obs_epoch',original); + let held=false,generation=0,release!:()=>void,entered!:()=>void;const wait=new Promise(r=>release=r),started=new Promise(r=>entered=r),base=kv.list; + kv.list=async(scope:string)=>{if(scope===PENDING_SCOPE){entered();await wait}return base(scope)}; + const vector=new VectorIndex(),p=new IndexPersistence(kv as never,vector,{requireRecoveryPlan:true,isStorageHeld:()=>held,storageGeneration:()=>generation} as never); + try{await p.prepareBackfill(await p.load());const replay=p.replayPendingLog(3);await started;held=true;generation++;held=false;release();await replay;expect(vector.has('obs_epoch')).toBe(false);expect(pendingRows(kv).get('obs_epoch')).toEqual(original);expect(p.isRecoveryPaused()).toBe(true)}finally{p.stop()} + }); + it("writes a compact entry for every added vector and a tombstone for every removal", async () => { const kv = mockKV(); const vector = new VectorIndex(); @@ -233,7 +307,7 @@ describe("pending vector log", () => { expect([...third.vector.entries()].map(([id]) => id).sort()).toEqual(["obs_2", "obs_3", "obs_4"]); }); - it("skips logged vectors whose dimension does not match the active provider", async () => { + it("preserves the pending log when any vector dimension does not match", async () => { const kv = mockKV(); const first = await boot(kv); first.vector.add("obs_ok", "ses_1", v(1)); @@ -244,7 +318,10 @@ describe("pending vector log", () => { const persistence = new IndexPersistence(kv as never, vector); await persistence.load(); const replay = await persistence.replayPendingLog(3); - expect(replay).toMatchObject({ added: 1, skipped: 1 }); + expect(replay).toMatchObject({ added: 0 }); + expect(persistence.isRecoveryPaused()).toBe(true); + expect(pendingRows(kv).size).toBe(2); + expect(vector.has("obs_ok")).toBe(false); expect(vector.has("obs_wide")).toBe(false); }); @@ -449,3 +526,92 @@ describe("durable vector backlog", () => { } }); }); + +describe("SDK-only release after a storage hold", () => { + function fixture() { + const kv = Object.assign(mockKV(), { + durability: createStateDurabilityFence({ storePath: "/unused-synthetic-sdk-store", completion: "sdk" }), + }); + let held = true; + const vector = new VectorIndex(); + const persistence = new IndexPersistence(kv as never, vector, { + requireRecoveryPlan: true, isStorageHeld: () => held, saveIntervalMs: 600_000, + }); + return { kv, vector, persistence, release: () => { held = false; } }; + } + + it("retains three original pending records before any checkpoint writes or replay", async () => { + const { kv, vector, persistence, release } = fixture(); + for (const [q, id] of ["first", "second", "third"].entries()) { + await kv.set(PENDING_SCOPE, id, { q: q + 1, id, s: "test", e: Buffer.from(v(q).buffer).toString("base64") }); + } + const original = new Map(pendingRows(kv)); + try { + await persistence.load(); + release(); + kv.ops.length = 0; + const resumed = await persistence.resumeStorageRecovery(3); + await persistence.save(); + expect(pendingRows(kv)).toEqual(original); + expect(resumed).toBe(false); + expect(persistence.status().vector?.lastError).toContain("durable reconciliation"); + expect(vector.size).toBe(0); + expect(kv.ops.filter(op => op.op !== "list")).toEqual([]); + expect(persistence.isRecoveryPaused()).toBe(true); + } finally { persistence.stop(); } + }); + + it.each(["failed read", "invalid response", "new row after first read"])("keeps the SDK release held for %s", async mode => { + const { kv, vector, persistence, release } = fixture(); + const list = kv.list; + let reads = 0; + kv.list = async scope => { + if (scope === PENDING_SCOPE) { + reads++; + if (mode === "failed read") throw new Error("synthetic timeout"); + if (mode === "invalid response") return null as never; + if (reads === 2) return [{ q: 1, id: "later", t: 1 }] as never; + } + return list(scope); + }; + try { + await persistence.load(); + release(); + expect(await persistence.resumeStorageRecovery(3)).toBe(false); + expect(persistence.isRecoveryPaused()).toBe(true); + expect(vector.size).toBe(0); + expect(kv.ops.some(op => op.op === "delete")).toBe(false); + expect(kv.ops.some(op => op.op === "set")).toBe(false); + } finally { persistence.stop(); } + }); + + it("allows SDK release when the pending namespace is empty", async () => { + const { persistence, release } = fixture(); + try { + await persistence.load(); + release(); + expect(await persistence.resumeStorageRecovery(3)).toBe(true); + expect(persistence.isRecoveryPaused()).toBe(false); + } finally { persistence.stop(); } + }); +}); + +describe('explicit storage recovery release',()=>{ + it('retries held startup recovery in the same process only after release',async()=>{ + const kv=mockKV(),row={q:1,id:'obs_resume',s:'s',e:Buffer.from(v(1).buffer).toString('base64')};await kv.set(PENDING_SCOPE,'obs_resume',row); + let held=true,generation=0;const vector=new VectorIndex();const p=new IndexPersistence(kv as never,vector,{requireRecoveryPlan:true,isStorageHeld:()=>held,storageGeneration:()=>generation} as never); + try{await p.load();expect(await (p as any).resumeStorageRecovery(3)).toBe(false);expect(pendingRows(kv).get('obs_resume')).toEqual(row);held=false;generation++;expect(await (p as any).resumeStorageRecovery(3)).toBe(true);expect(vector.has('obs_resume')).toBe(true);expect(p.isRecoveryPaused()).toBe(false)}finally{p.stop()} + }); + it('rejects vector mutation before changing memory while storage is held',async()=>{ + const kv=mockKV(),vector=new VectorIndex();let held=true;const p=new IndexPersistence(kv as never,vector,{requireRecoveryPlan:true,isStorageHeld:()=>held} as never); + try{expect(()=>vector.add('new','s',v(1))).toThrow(/VECTOR_RECOVERY_PAUSED/);expect(vector.has('new')).toBe(false);expect(pendingRows(kv).size).toBe(0)}finally{p.stop()} + }); + it('keeps recovery paused and originals intact when release reload fails',async()=>{ + const kv=mockKV(),row={q:1,id:'obs_fail',s:'s',e:Buffer.from(v(1).buffer).toString('base64')};await kv.set(PENDING_SCOPE,'obs_fail',row);let held=true;const vector=new VectorIndex();const p=new IndexPersistence(kv as never,vector,{requireRecoveryPlan:true,isStorageHeld:()=>held} as never); + try{await p.load();held=false;kv.get=async()=>{throw Error('synthetic read failure')};expect(await (p as any).resumeStorageRecovery(3)).toBe(false);expect(p.isRecoveryPaused()).toBe(true);expect(pendingRows(kv).get('obs_fail')).toEqual(row);expect(vector.size).toBe(0)}finally{p.stop()} + }); +}); + +it('rejects foreground vector changes while a released storage checkpoint is loading',async()=>{const kv=mockKV();let held=true,release!:()=>void,entered!:()=>void;const wait=new Promise(r=>release=r),started=new Promise(r=>entered=r),get=kv.get;const vector=new VectorIndex(),p=new IndexPersistence(kv as never,vector,{requireRecoveryPlan:true,isStorageHeld:()=>held} as never);try{await p.load();held=false;kv.get=async(scope:string,key:string)=>{if(scope===INDEX_SCOPE&&key==='vectors:meta'){entered();await wait}return get(scope,key)};const resume=(p as any).resumeStorageRecovery(3);await started;expect(()=>vector.add('during','s',v(1))).toThrow(/VECTOR_RECOVERY_PAUSED/);expect(vector.has('during')).toBe(false);release();expect(await resume).toBe(true)}finally{p.stop()}}); +it('does not claim successful vector recovery when pending-log read fails',async()=>{const kv=mockKV();let held=true;const list=kv.list;const vector=new VectorIndex(),p=new IndexPersistence(kv as never,vector,{requireRecoveryPlan:true,isStorageHeld:()=>held} as never);try{await p.load();held=false;kv.list=async(scope:string)=>{if(scope===PENDING_SCOPE)throw Error('synthetic pending read failure');return list(scope)};expect(await (p as any).resumeStorageRecovery(3)).toBe(false);expect(p.isRecoveryPaused()).toBe(true)}finally{p.stop()}}); +it('allows an explicit retry after a transient pending-log read failure',async()=>{const kv=mockKV(),row={q:1,id:'retry','s':'s',e:Buffer.from(v(1).buffer).toString('base64')};await kv.set(PENDING_SCOPE,'retry',row);let held=true;const list=kv.list;const vector=new VectorIndex(),p=new IndexPersistence(kv as never,vector,{requireRecoveryPlan:true,isStorageHeld:()=>held} as never);try{await p.load();held=false;kv.list=async(scope:string)=>{if(scope===PENDING_SCOPE)throw Error('temporary');return list(scope)};expect(await (p as any).resumeStorageRecovery(3)).toBe(false);kv.list=list;expect(await (p as any).resumeStorageRecovery(3)).toBe(true);expect(vector.has('retry')).toBe(true);expect(p.status().pendingLogError).toBeNull()}finally{p.stop()}}); diff --git a/test/vector-revision-release.test.ts b/test/vector-revision-release.test.ts new file mode 100644 index 0000000000..aa810ac751 --- /dev/null +++ b/test/vector-revision-release.test.ts @@ -0,0 +1,10 @@ +import {it,expect,vi} from 'vitest'; +import {releaseSupersededVectorJobs,retryRetainedVectorRevisions,type RetainedVectorRevision} from '../src/state/vector-revision-release.js'; +it('retains failed releases and permits no successful promotion claim until complete',()=>{ + const retained:RetainedVectorRevision[]=[];let fail=true;const first={release:vi.fn(()=>{if(fail)throw Error('release failed');})};const second={release:vi.fn()}; + expect(releaseSupersededVectorJobs([{id:'a',revision:first},{id:'b',revision:second}],retained)).toBe(false); + expect(second.release).toHaveBeenCalledOnce();expect(retained).toEqual([{id:'a',revision:first}]); + expect(()=>retryRetainedVectorRevisions(retained)).toThrow('release failed');expect(retained).toHaveLength(1); + fail=false;retryRetainedVectorRevisions(retained);expect(retained).toHaveLength(0); +}); +it('successful release retains no stale job and handles jobs without watches',()=>{const retained:RetainedVectorRevision[]=[];expect(releaseSupersededVectorJobs([{id:'a'},{id:'b',revision:{release(){}}}],retained)).toBe(true);expect(retained).toEqual([]);}); diff --git a/test/viewer-metrics-browser.test.ts b/test/viewer-metrics-browser.test.ts index 31461cd7c1..12f4dfe1d7 100644 --- a/test/viewer-metrics-browser.test.ts +++ b/test/viewer-metrics-browser.test.ts @@ -1,8 +1,11 @@ -import { spawnSync } from 'node:child_process'; -import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { spawn, spawnSync } from 'node:child_process'; +import { mkdirSync, mkdtempSync, readFileSync, writeFileSync } from 'node:fs'; +import { rm } from 'node:fs/promises'; import { join, resolve } from 'node:path'; import { tmpdir } from 'node:os'; import { describe, expect, it } from 'vitest'; +import WebSocket from 'ws'; +import {waitForDevToolsPort} from './helpers/devtools-port.js'; const source=readFileSync(process.env.METRICS_UI_SOURCE || 'src/viewer/index.html','utf8'); const evidence=process.env.METRICS_UI_EVIDENCE; @@ -15,20 +18,62 @@ var fixtureHealth={status:'healthy',health:{connectionState:'connected'},functio var fixtureReport={status:'ok',headline:'Synthetic metrics fixture',checkedAt:new Date().toISOString(),problems:[],service:{version:'synthetic',ports:{}},provider:{llm:'openai',embeddings:'none'},functions:[fixtureMetric],flags:[],capture:{inbox:{pending:2,retrying:3,dead:4},sinceStart:{},spool:[],policy:{retryIntervalMs:30000,maxAttempts:5}}}; window.fetch=async(url,opts={})=>{requests.push({path:String(url),method:opts.method||'GET'});var path=String(url),payload={};if(path.includes('/viewer/snapshot'))payload={version:'synthetic',health:fixtureHealth.health,report:fixtureReport,functionMetrics:[fixtureMetric],counts:{sessions:1,observations:1},flags:[]};else if(path.includes('/health'))payload=fixtureHealth;else if(path.includes('/background/control'))payload={success:true,control:{status:'running',holds:[],activeOperations:0}};else if(path.includes('/sessions'))payload={sessions:[{id:'synthetic',project:'Synthetic fixture',observationCount:1,summaryProgress:{status:'blocked',pendingCount:1,blockedObservations:{one:{status:'blocked',diagnostics:{category:'output_truncated'}}}},enrichmentProgress:{compress:{status:'current',pendingCount:0},graph:{status:'current',pendingCount:0}}}]};return new Response(JSON.stringify(payload),{status:200,headers:{'Content-Type':'application/json'}});}; window.WebSocket=class{static OPEN=1;static CONNECTING=0;constructor(){this.readyState=1;setTimeout(()=>this.onopen&&this.onopen(),0);}send(){}close(){this.readyState=3;}}; -setTimeout(async()=>{var proof={};try{await loadSnapshot();await loadOperations();setAutoRefresh(false,30);operations.metrics={'mem::summary':{totalCalls:200,failureCount:190,observedAt:'2026-10-22T00:05:00.000Z'}};operations.metricPrevious='${scenario}'==='missing'?{}:{'mem::summary':{totalCalls:'${scenario}'==='reset'?250:195,failureCount:'${scenario}'==='reset'?195:190,observedAt:'2026-10-22T00:01:00.000Z'}};switchTab('${view}');await showTab('${view}');renderTab('${view}');var root=document.getElementById('view-${view}');root.querySelectorAll('details').forEach(e=>e.open=true);var fn=root.querySelector('.metric-fn'),table=fn.closest('table'),row=fn.closest('tr'),cells=Array.from(row.cells);proof.functionWidth=fn.getBoundingClientRect().width;proof.text=root.innerText;proof.metrics=table.innerText;proof.aggregate=Array.from(root.querySelectorAll('.stat-card')).find(e=>e.textContent.includes('누적 실행 시도'))?.innerText;proof.historicalCell=cells[2].innerText;proof.historicalColor=cells[2].style.color;proof.red=getComputedStyle(document.documentElement).getPropertyValue('--red').trim();proof.rateColor=cells.find(e=>e.textContent.trim()==='95%')?.style.color;proof.recentCell=cells.find(e=>e.textContent.includes('실패')&&e.textContent.includes('시도'))?.innerText;proof.attention=document.getElementById('health-attention')?.innerText;proof.errors=fixtureErrors;proof.getOnly=requests.every(r=>r.method==='GET');proof.overflow=root.scrollWidth>root.clientWidth;fn.closest('.card').scrollIntoView({block:'start'});}catch(e){proof.error=String(e.stack||e);}document.getElementById('metrics-proof').textContent=JSON.stringify(proof);},120); +addEventListener('DOMContentLoaded',async()=>{var proof={};try{await loadSnapshot();await loadOperations();setAutoRefresh(false,30);operations.metrics={'mem::summary':{totalCalls:200,failureCount:190,observedAt:'2026-10-22T00:05:00.000Z'}};operations.metricPrevious='${scenario}'==='missing'?{}:{'mem::summary':{totalCalls:'${scenario}'==='reset'?250:195,failureCount:'${scenario}'==='reset'?195:190,observedAt:'2026-10-22T00:01:00.000Z'}};switchTab('${view}');await showTab('${view}');renderTab('${view}');var root=document.getElementById('view-${view}');root.querySelectorAll('details').forEach(e=>e.open=true);var fn=root.querySelector('.metric-fn'),table=fn.closest('table'),row=fn.closest('tr'),cells=Array.from(row.cells);proof.functionWidth=fn.getBoundingClientRect().width;proof.text=root.innerText;proof.metrics=table.innerText;proof.aggregate=Array.from(root.querySelectorAll('.stat-card')).find(e=>e.textContent.includes('누적 실행 시도'))?.innerText;proof.historicalCell=cells[2].innerText;proof.historicalColor=cells[2].style.color;proof.red=getComputedStyle(document.documentElement).getPropertyValue('--red').trim();proof.rateColor=cells.find(e=>e.textContent.trim()==='95%')?.style.color;proof.recentCell=cells.find(e=>e.textContent.includes('실패')&&e.textContent.includes('시도'))?.innerText;proof.attention=document.getElementById('health-attention')?.innerText;proof.errors=fixtureErrors;proof.getOnly=requests.every(r=>r.method==='GET');proof.overflow=root.scrollWidth>root.clientWidth;fn.closest('.card').scrollIntoView({block:'start'});}catch(e){proof.error=String(e.stack||e);}document.getElementById('metrics-proof').textContent=JSON.stringify(proof);},{once:true}); `; +async function browserProof(dir:string,path:string,width:number,screenshot:string){ + const chrome=spawn('google-chrome',['--headless','--no-sandbox','--disable-gpu','--disable-background-networking','--no-first-run','--remote-debugging-port=0','--remote-debugging-address=127.0.0.1','--user-data-dir='+join(dir,'profile'),'about:blank'],{stdio:['ignore','ignore','pipe']}); + const closed=new Promise(resolve=>chrome.once('close',()=>resolve())); + const waitClosed=(ms:number)=>new Promise(resolve=>{const timer=setTimeout(()=>resolve(false),ms);void closed.then(()=>{clearTimeout(timer);resolve(true);});}); + let stderr='',socket:WebSocket|undefined; + chrome.stderr.on('data',chunk=>{stderr=(stderr+chunk).slice(-2000);}); + const deadline=Date.now()+20000; + const pause=()=>new Promise(resolve=>setTimeout(resolve,25)); + try{ + const portFile=join(dir,'profile','DevToolsActivePort'); + const port=await waitForDevToolsPort(portFile,deadline,()=>chrome.exitCode===null&&chrome.signalCode===null).catch(error=>{throw Error(String(error)+'; '+stderr);}); + const target=await fetch(`http://127.0.0.1:${port}/json/new?about:blank`,{method:'PUT',signal:AbortSignal.timeout(5000)}).then(r=>r.json()); + socket=new WebSocket(target.webSocketDebuggerUrl,{handshakeTimeout:5000}); + await new Promise((resolve,reject)=>{const timer=setTimeout(()=>reject(Error('CDP connection timeout')),5000);socket!.once('open',()=>{clearTimeout(timer);resolve();});socket!.once('error',()=>{clearTimeout(timer);reject(Error('CDP connection failed'));});}); + let seq=0;const pending=new Mapvoid;reject:(error:Error)=>void}>(); + socket.on('close',()=>{for(const request of pending.values())request.reject(Error('CDP connection closed'));pending.clear();}); + socket.on('error',()=>{for(const request of pending.values())request.reject(Error('CDP connection error'));pending.clear();}); + socket.on('message',data=>{const message=JSON.parse(String(data)),request=pending.get(message.id);if(!request)return;pending.delete(message.id);message.error?request.reject(Error(JSON.stringify(message.error))):request.resolve(message.result);}); + const send=(method:string,params:Record={})=>new Promise((resolve,reject)=>{const id=++seq;const timer=setTimeout(()=>{pending.delete(id);reject(Error('CDP command timed out: '+method));},5000);pending.set(id,{resolve:value=>{clearTimeout(timer);resolve(value);},reject:error=>{clearTimeout(timer);reject(error);}});socket!.send(JSON.stringify({id,method,params}));}); + const evaluate=async(expression:string)=>{const result=await send('Runtime.evaluate',{expression,returnByValue:true});if(result.exceptionDetails)throw Error(JSON.stringify(result.exceptionDetails));return result.result.value;}; + await send('Page.enable'); + await send('Emulation.setDeviceMetricsOverride',{width,height:1000,deviceScaleFactor:1,mobile:false}); + const navigation=await send('Page.navigate',{url:'file://'+path});expect(navigation.errorText).toBeUndefined(); + let text=''; + while(Date.now(){ - it.each(cases.flatMap(s=>[1280,390].flatMap(w=>['health','dashboard'].map(v=>[s,w,v]))))('%s at %i/%s',(scenario,width,view)=>{ + it.each(cases.flatMap(s=>[1280,390].flatMap(w=>['health','dashboard'].map(v=>[s,w,v]))))('%s at %i/%s',async(scenario,width,view)=>{ const dir=mkdtempSync(join(tmpdir(),'am-metrics-browser-')); + let retainProfile=false; try{ if(evidence)mkdirSync(evidence,{recursive:true}); const name=`${scenario}-${width}-${view}`,path=join(dir,'fixture.html'); writeFileSync(path,source.replace('',''+fixture(String(scenario),String(view))).replace('','')); - const run=spawnSync('google-chrome',['--headless','--no-sandbox','--disable-gpu','--disable-background-networking','--no-first-run','--user-data-dir='+join(dir,'profile'),'--window-size='+width+',1000','--virtual-time-budget=1100','--screenshot='+(evidence?join(resolve(evidence),name+'.png'):join(dir,'preview.png')),'--dump-dom','file://'+path],{encoding:'utf8',timeout:25000,maxBuffer:3000000}); - expect(run.status,run.stderr.slice(-500)).toBe(0); - const match=run.stdout.match(/
]*>([\s\S]*?)<\/pre>/);expect(match).toBeTruthy();
-      const proof=JSON.parse(match![1].replaceAll('"','"').replaceAll('&','&').replaceAll('<','<').replaceAll('>','>'));
+      const proof=await browserProof(dir,path,Number(width),evidence?join(resolve(evidence),name+'.png'):join(dir,'preview.png'));
       if(evidence)writeFileSync(join(evidence,name+'.json'),JSON.stringify(proof,null,2));
       expect(proof.error).toBeUndefined();expect(proof.functionWidth).toBeGreaterThanOrEqual(120);expect(proof.errors).toEqual([]);expect(proof.getOnly).toBe(true);expect(proof.overflow).toBe(false);expect(proof.historicalCell).toBe('190');expect(proof.historicalColor).not.toBe('var(--red)');expect(proof.rateColor).not.toBe('var(--red)');expect(proof.text).toContain('집계 시작 미확인');expect(proof.metrics).toContain('실행 시도');
       if(scenario==='off'&&view==='dashboard'){expect(proof.aggregate).toContain('200');expect(proof.aggregate).toContain('190');}
@@ -39,6 +84,6 @@ describe.skipIf(spawnSync('google-chrome',['--version'],{encoding:'utf8'}).statu
       if(scenario==='reset')expect(proof.metrics).toContain('집계 기준 변경');
       if(scenario==='invalid')expect(proof.metrics).toContain('서버 관측 집계 확인 불가');
       if(view==='health'){expect(proof.attention).toContain('격리 1건');expect(proof.attention).toContain('격리 4건');expect(proof.attention).toContain('재시도 3건');expect(proof.text).toContain('현재 대기·재시도·격리');expect(proof.text).not.toContain('임베딩: 최근 실패');}
-    }finally{rmSync(dir,{recursive:true,force:true});}
+    }catch(error){retainProfile=(error as {retainProfile?:boolean})?.retainProfile===true;throw error;}finally{if(!retainProfile)await rm(dir,{recursive:true,force:true,maxRetries:10,retryDelay:100});}
   },30000);
 });
diff --git a/test/viewer-operations-browser.test.ts b/test/viewer-operations-browser.test.ts
index 5ee3f295bb..300fa2f6c3 100644
--- a/test/viewer-operations-browser.test.ts
+++ b/test/viewer-operations-browser.test.ts
@@ -95,6 +95,10 @@ describe.skipIf(spawnSync('google-chrome',['--version'],{encoding:'utf8',timeout
       await send('Page.enable');
       await send('Page.bringToFront');
       while(!(await evaluate("document.readyState==='complete' && typeof loadOperations==='function'")) && Date.now()setTimeout(resolve,25));
+      const unknownCapture=await evaluate("captureCardHtml({capture:{inbox:null,spool:[],sinceStart:{},policy:{}}})");
+      expect(unknownCapture).toContain('not checked');
+      expect(unknownCapture).toContain('Inbox status is unknown');
+      expect(unknownCapture).not.toContain('Empty means every accepted observation is stored.');
       await evaluate("(async()=>{await loadSnapshot();await loadOperations();switchTab('health');await showTab('health');setAutoRefresh(false,30);document.querySelector('#operations-details>summary').focus();})()");
       expect(await evaluate("document.activeElement===document.querySelector('#operations-details>summary')")).toBe(true);
       await send('Input.dispatchKeyEvent',{type:'keyDown',key:'Enter',code:'Enter',text:'\r',windowsVirtualKeyCode:13});
diff --git a/test/viewer-operations.test.ts b/test/viewer-operations.test.ts
index 632f333910..8fa940aa41 100644
--- a/test/viewer-operations.test.ts
+++ b/test/viewer-operations.test.ts
@@ -20,7 +20,7 @@ function context(names: string[], extra: Record = {}) {
   return ctx;
 }
 function fixtures(extra: Record = {}) {
-  return context(["esc", "fmtNum", "humanizeHealthFlag", "quarantineReason", "quarantineRecords", "backgroundWorkSnapshot", "memoryPressureStatus", "snapshotStatus", "storageDurabilityHtml", "inspectionErrorText", "inspectionRequest", "readinessStatusHtml", "controlDiagnosticHtml", "operationsStages", "recordOperationsProgress", "stageProgress", "progressDuration", "operationsProgressHtml", "operationsPanelHtml"], {
+  return context(["esc", "fmtNum", "humanizeHealthFlag", "quarantineReason", "quarantineRecords", "summaryStageProgress", "backgroundWorkSnapshot", "memoryPressureStatus", "snapshotStatus", "storageDurabilityHtml", "inspectionErrorText", "inspectionRequest", "readinessStatusHtml", "controlDiagnosticHtml", "operationsStages", "recordOperationsProgress", "stageProgress", "progressDuration", "operationsProgressHtml", "operationsPanelHtml"], {
     operations: { health: {}, sessions: [], control: null, job: null, available: false, loading: false, error: '', controlError: '', jobError: '', checking: false, submitting: false, controlBusy: false, detailsOpen: false },
     store: {}, conn: { state: 'offline' }, autoRefresh: { enabled: true, seconds: 30 }, ...extra,
   });
@@ -72,6 +72,23 @@ describe("custom operations panel", () => {
     expect(c.backgroundWorkSnapshot([{ id:'x', status:'completed', observationCount:10 }], {}, true).totals.summary).toBeNull();
     expect(c.backgroundWorkSnapshot([], {}, false).totals.graph).toBeNull();
   });
+  it("uses confirmed legacy coverage in both summary counts and stage progress", () => {
+    const c = fixtures();
+    const session = { id: 'legacy', status: 'completed', observationCount: 1, summaryCoveredCount: 1,
+      enrichmentProgress: { compress: { enabled: false }, graph: { enabled: false } } };
+    const work = c.backgroundWorkSnapshot([session], {}, true);
+    expect(work.totals.summary).toBe(0);
+    expect(work.unknownStageCounts.summary).toBe(0);
+    expect(work.rows).toEqual([]);
+    expect(c.operationsStages(work, [session], {}, true).find((stage:any) => stage.key === 'summary'))
+      .toMatchObject({ pending: 0, completed: 1, counters: { legacy: 1 } });
+    const mismatch = { ...session, observationCount: 2 };
+    const unknown = c.backgroundWorkSnapshot([mismatch], {}, true);
+    expect(unknown.totals.summary).toBeNull();
+    expect(c.operationsStages(unknown, [mismatch], {}, true).find((stage:any) => stage.key === 'summary').completed).toBeNull();
+    const retry = { ...session, summaryProgress: { status: 'retrying', pendingCount: 1, processedCount: 0 } };
+    expect(c.backgroundWorkSnapshot([retry], {}, true).totals.summary).toBe(1);
+  });
   it("shows the confirmed minimum and the unknown session/stage scope separately",()=> {
     const c=fixtures();c.operations.available=true;
     c.operations.sessions=[{id:'known',summaryProgress:{pendingCount:3,status:'running'},enrichmentProgress:{compress:{pendingCount:4,status:'waiting_provider'},graph:{pendingCount:5,status:'queued'}}},{id:'unknown',observationCount:10}];
@@ -212,7 +229,7 @@ describe("manual operations commands", () => {
     const c=fixtures({apiGet,renderOperations:vi.fn(),operations:{health:{},sessions:[],control:null,job,available:false,generation:0,progressSamples:{}}});
     runInNewContext(source('loadOperations'),c);
     await c.loadOperations();
-    expect(apiGet.mock.calls.map(call=>call[0])).toEqual(['health','sessions?includeSummary=false','background/control']);
+    expect(apiGet.mock.calls.map(call=>call[0])).toEqual(['health','sessions?includeSummary=false&includeSummaryCoverage=true','background/control']);
     expect(c.operations.job).toEqual(job);
     expect(viewer).not.toMatch(/action === ['"](?:submit-codex-inspection|copy-inspection-request)['"]/);
   });
diff --git a/test/windows-smoke-process-query.test.ts b/test/windows-smoke-process-query.test.ts
new file mode 100644
index 0000000000..9fd8c80140
--- /dev/null
+++ b/test/windows-smoke-process-query.test.ts
@@ -0,0 +1,24 @@
+import { readFileSync } from "node:fs";
+import { describe, expect, it, vi } from "vitest";
+
+const source = readFileSync(new URL("../scripts/release-gate/windows-smoke.mjs", import.meta.url), "utf8");
+const start = source.indexOf("function leftoverProcesses()");
+const end = source.indexOf("\n}\n", start) + 2;
+const createQuery = new Function("spawnSync", "root", `const isWindows = true; ${source.slice(start, end)}; return leftoverProcesses;`);
+
+describe("Windows release-gate process cleanup evidence", () => {
+  it("does not treat denied or failed PowerShell inspection as zero remaining processes", () => {
+    for (const result of [{ status: 1, stderr: "Access denied", stdout: "" }, { status: null, error: new Error("spawn failed") }]) {
+      expect(() => createQuery(() => result, "C:\\synthetic gate")()).toThrow("leftover process query failed");
+    }
+  });
+
+  it("checks executable paths as well as command lines and parses the observed owners", () => {
+    const spawn = vi.fn((_command: string, _args: string[]) => ({ status: 0, stdout: " 42 state.exe\r\n84 port:3911\r\n" }));
+    expect(createQuery(spawn, "C:\\synthetic'gate")()).toEqual(["42 state.exe", "84 port:3911"]);
+    const script = spawn.mock.calls[0][1][2];
+    expect(script).toContain("$ErrorActionPreference = 'Stop'");
+    expect(script).toContain("$_.ExecutablePath");
+    expect(script).toContain("synthetic''gate");
+  });
+});
diff --git a/test/worker-shutdown-wiring.test.ts b/test/worker-shutdown-wiring.test.ts
index 41a645312d..284ce48914 100644
--- a/test/worker-shutdown-wiring.test.ts
+++ b/test/worker-shutdown-wiring.test.ts
@@ -1,3 +1,5 @@
+import { retryRetainedVectorRevisions } from '../src/state/vector-revision-release.js';
+import {stateSealDiskProof} from '../src/ops/state-seal-shutdown.mjs';
 import {isStorageWorkCompleted} from '../src/ops/storage-completion.mjs';
 import { afterEach, expect, it, vi } from 'vitest';
 import { readFileSync } from 'node:fs';
@@ -13,15 +15,17 @@ import { mockSdk } from './helpers/mocks.js';
 const dirs:string[]=[];
 afterEach(async()=>{await Promise.all(dirs.splice(0).map(dir=>rm(dir,{recursive:true,force:true})));});
 function gate(){let release!:()=>void;const promise=new Promise(resolve=>{release=resolve;});return {promise,release};}
-async function fixture({configured=true,indexFailed=false,bootFailed=false,writerSafe=false,mutationUncertain=false,sdkFails=false,sdkOnly=false}={}) {
+async function fixture({configured=true,indexFailed=false,bootFailed=false,writerSafe=false,mutationUncertain=false,sdkFails=false,sdkOnly=false,stateBound=false,sealFails=false,sealMismatch=false,revisionReleaseFails=false}={}) {
  const dir=await mkdtemp(join(tmpdir(),'am-worker-stop-'));dirs.push(dir);
  const events:string[]=[];const sdk={...mockSdk(),shutdown:async()=>{events.push('sdk');if(sdkFails)throw Error('fixture disconnect failure');}};
- const kv={recoveryMutationSafe:writerSafe,mutationOutcomeUncertain:mutationUncertain,durability:configured?{completion:sdkOnly?"sdk":"disk",drain:async()=>{events.push('drain');},freeze:()=>{events.push('freeze');},wait:async()=>{events.push('disk');return sdkOnly?{completion:"sdk",completed:true,synced:false,mutationFailures:0}:{synced:true,mutationFailures:0};}}:null};
+ const kv={recoveryMutationSafe:writerSafe,mutationOutcomeUncertain:mutationUncertain,durability:configured?{completion:sdkOnly?"sdk":"disk",snapshot:()=>({mutationFailures:0}),drain:async()=>{events.push('drain');},freeze:()=>{events.push('freeze');},wait:async()=>{events.push('disk');return sdkOnly?{completion:"sdk",completed:true,synced:false,mutationFailures:0}:{synced:true,mutationFailures:0};}}:null};
  const source=readFileSync('src/index.ts','utf8');
  const start=source.indexOf('  const ingress = createShutdownIngress();');
  const end=source.indexOf('  try {\n  if (persistentBackground)',start);
  const body=source.slice(start,end);
- const deps={isStorageWorkCompleted,writerLease:writerSafe?{}:undefined,markFileWriterClean:()=>events.push('writer-clean'),createBackgroundControl:()=>({settled:async()=>{},commitCleanShutdown:async()=>{events.push("restart-guard");}}),installBackgroundControl:()=>{},operationalState:{},createShutdownIngress,createBackgroundLifecycle,verifiedShutdown,closeLongSessionInferenceAdmission:()=>events.push('close-model'),checkAuth:()=>null,kv,sdk,secret:'fixture-secret',dirname,runtimeMetadataPath:(name:string)=>join(dir,name),readFileSync,settleWithin,SHUTDOWN_FLUSH_TIMEOUT_MS,
+ const stateBinding=stateBound?{instanceId:'fixture-state',storePath:dir}:undefined;
+ const stateClient={trigger:async(request:{function_id:string})=>{expect(request.function_id).toBe('state::seal_flush');events.push('seal');if(sealFails)throw Error('fixture seal failure');return {version:1,instanceId:sealMismatch?'other-state':'fixture-state',storePath:dir,sealed:true,synced:true};}};
+ const deps={retryRetainedVectorRevisions,unreleasedSupersededRevisions:revisionReleaseFails?[{id:"fixture-vector",revision:{release:()=>{throw Error("fixture revision release failed");}}}]:[],stateBinding,stateClient,stateSealDiskProof,markFileWriterSealedClean:()=>events.push('writer-sealed-clean'),bootWarn:()=>events.push('warn'),config:{engineUrl:'ws://127.0.0.1:1000'},readEnginePid:()=>123,isStorageWorkCompleted,writerLease:writerSafe||stateBound?{}:undefined,markFileWriterClean:()=>events.push('writer-clean'),createBackgroundControl:()=>({settled:async()=>{},commitCleanShutdown:async()=>{events.push("restart-guard");}}),installBackgroundControl:()=>{},operationalState:{},createShutdownIngress,createBackgroundLifecycle,verifiedShutdown,closeLongSessionInferenceAdmission:()=>events.push('close-model'),checkAuth:()=>null,kv,sdk,secret:'fixture-secret',dirname,runtimeMetadataPath:(name:string)=>join(dir,name),readFileSync,settleWithin,SHUTDOWN_FLUSH_TIMEOUT_MS,
  clearWorkerPidfile:()=>events.push('clear-pid'),process:{on:vi.fn(),kill:()=>{},exit:(code:number)=>{events.push('exit:'+code);throw Error('exit:'+code);}}};
  const js=transpileModule(body,{compilerOptions:{target:ScriptTarget.ES2022,module:ModuleKind.ES2022}}).outputText;
  const load=new Function('deps',`const {${Object.keys(deps).join(',')}}=deps;let fatalShutdown;${js};return {shutdown,bootDone,resources,lifecycle,trackBoot,failBoot:()=>{bootFailed=true;}};`);
@@ -61,3 +65,25 @@ it('completes SDK-only shutdown without granting a clean durable writer marker',
  expect(f.events).toContain('clear-pid');expect(f.events).not.toContain('writer-clean');
  expect(JSON.parse(await readFile(join(f.dir,'shutdown-receipt.json'),'utf8'))).toMatchObject({completed:true,verified:false,durability:{synced:false,completion:'sdk'}});
 });
+
+it('seals the verified State after freeze and certifies clean before SDK disconnect',async()=>{
+ const f=await fixture({stateBound:true,sdkOnly:true,sdkFails:true});f.bootDone();
+ await expect(f.shutdown()).rejects.toThrow('exit:0');
+ expect(f.events.indexOf('seal')).toBeGreaterThan(f.events.indexOf('freeze'));
+ expect(f.events.indexOf('writer-sealed-clean')).toBeGreaterThan(f.events.indexOf('seal'));
+ expect(f.events.indexOf('writer-sealed-clean')).toBeLessThan(f.events.indexOf('sdk'));
+ expect(f.events).not.toContain('writer-clean');
+ expect(JSON.parse(await readFile(join(f.dir,'shutdown-receipt.json'),'utf8'))).toMatchObject({completed:true,verified:true,durability:{completion:'disk',synced:true,stateSeal:{instanceId:'fixture-state',sealed:true}}});
+});
+it.each([{sealFails:true},{sealMismatch:true}])('retains unclean state and PID when State seal fails or mismatches: %j',async(options)=>{
+ const f=await fixture({stateBound:true,sdkOnly:true,...options});f.bootDone();
+ await expect(f.shutdown()).rejects.toThrow('exit:1');
+ expect(f.events).not.toContain('writer-sealed-clean');expect(f.events).not.toContain('writer-clean');expect(f.events).not.toContain('clear-pid');
+ expect(JSON.parse(await readFile(join(f.dir,'shutdown-receipt.json'),'utf8'))).toMatchObject({completed:false,verified:false});
+});
+
+it('retains dirty state when a superseded vector revision cannot be released',async()=>{
+ const f=await fixture({stateBound:true,sdkOnly:true,revisionReleaseFails:true});f.bootDone();
+ await expect(f.shutdown()).rejects.toThrow('exit:1');
+ expect(f.events).not.toContain('seal');expect(f.events).not.toContain('writer-sealed-clean');expect(f.events).not.toContain('clear-pid');
+});
diff --git a/tsdown.config.ts b/tsdown.config.ts
index 22daba6fee..094e8f63e5 100644
--- a/tsdown.config.ts
+++ b/tsdown.config.ts
@@ -48,7 +48,7 @@ const shared = {
   // optionalDependency in package.json so users can install it only when
   // they enable local embeddings / CLIP / reranker.
   deps: {
-    alwaysBundle: [/^iii-sdk(?:\/|$)/, /^@iii-dev\/helpers(?:\/|$)/, /^yaml(?:\/|$)/, /^json5(?:\/|$)/],
+    alwaysBundle: [/^@opentelemetry\/(?:.*)/, /^iii-sdk(?:\/|$)/, /^@iii-dev\/helpers(?:\/|$)/, /^yaml(?:\/|$)/, /^json5(?:\/|$)/],
     neverBundle: [
       "@huggingface/transformers",
       "@anthropic-ai/claude-agent-sdk",
@@ -79,6 +79,20 @@ export default defineConfig([
     clean: false,
     sourcemap: false,
   },
+  {
+    entry: ["scripts/migration/native-v3-to-v4.mjs"],
+    outDir: "dist",
+    ...shared,
+    clean: false,
+    sourcemap: false,
+  },
+  {
+    entry: ["src/capture/sdk-delivery-v2.ts"],
+    outDir: "dist",
+    ...shared,
+    clean: false,
+    sourcemap: false,
+  },
   {
     entry: ["src/mcp/standalone.ts"],
     outDir: "dist",
diff --git a/website/app/security/page.tsx b/website/app/security/page.tsx
index 8094a6d28e..d3c4d074d4 100644
--- a/website/app/security/page.tsx
+++ b/website/app/security/page.tsx
@@ -127,7 +127,7 @@ export default function SecurityPage() {
         

Supply chain

  • - The iii engine is pinned to one version (0.25.0) and runs workers under Worker Compose; registry workers state@0.22.20 and http@0.22.7 provide state and REST. The installer checks the downloaded archive + The iii engine is pinned to one version (0.25.1) and runs workers under Worker Compose; registry workers state@0.22.20 and http@0.22.7 provide state and REST. The installer checks the downloaded archive against a SHA-256 recorded in the package before it extracts it.