diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000000..b96cb9a6c9 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,10 @@ +node_modules +.git +dist +eval +benchmark +test +.claude +.serena +docs +*.log diff --git a/.env.example b/.env.example index 2143098484..973877d1b1 100644 --- a/.env.example +++ b/.env.example @@ -200,3 +200,11 @@ # ----------------------------------------------------------------------------- # OBSIDIAN_AUTO_EXPORT=true # Auto-export memories to an Obsidian vault on every consolidation + +# Health memory pressure: V8, cgroup v2 hard/high limits, and optional RSS budget. +# Committed heap size and the former 512 MiB RSS floor do not define pressure. +# AGENTMEMORY_HEALTH_MEM_WARN_PCT=80 # Require 0 < warn < critical <= 100 +# AGENTMEMORY_HEALTH_MEM_CRITICAL_PCT=95 +# AGENTMEMORY_HEALTH_MEM_RSS_BUDGET_MB=512 # Optional process RSS budget; MB means MiB +# AGENTMEMORY_HEALTH_MEM_HOLD_SAMPLES=2 # Positive integer; both entry and recovery, every 30 seconds +# Missing samples break streaks and retain existing critical latches until valid recovery or restart. diff --git a/deploy/README.md b/deploy/README.md index 91aa199e28..a84f653aaa 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -1,15 +1,10 @@ # One-click deploy templates Stand up agentmemory on managed infrastructure without rolling your own -Docker host. Each template ships a self-contained Dockerfile that pulls -`@agentmemory/agentmemory` from npm at build time and copies the iii -engine binary in from the official `iiidev/iii` image — no pre-built -agentmemory image required. Storage mounts at `/data`; an HMAC secret -is generated by the first-boot entrypoint and persisted to the volume. -The entrypoint overwrites the npm-bundled iii config with a -deploy-tuned one that binds `0.0.0.0` and uses absolute `/data` paths, -then drops privileges from `root` to `node` via `gosu` before -exec'ing the agentmemory CLI. +Docker host. Each template ships a self-contained Dockerfile. Fly, Render, and Coolify +install `@agentmemory/agentmemory` from npm at build time. Railway builds it +from this repo's source instead (see `deploy/railway/README.md`). All four +copy the iii engine binary in from the `iiidev/iii` image. | Platform | Pitch | Cost floor | |----------|-------|------------| @@ -95,6 +90,7 @@ agentmemory worker reg : 2.0 s healthcheck passes : ~9-10 s ``` -Every template's health-check `grace_period` (or compose +Railway's `healthcheckTimeout` is 60 s (the BM25 startup backfill needs it). +Every other template's health-check `grace_period` (or compose `start_period`) is set to 30 s for a 3x safety margin. Tune lower once you've measured your own platform's image-pull characteristics. diff --git a/deploy/railway/Dockerfile b/deploy/railway/Dockerfile index e094699881..eb884dc24c 100644 --- a/deploy/railway/Dockerfile +++ b/deploy/railway/Dockerfile @@ -2,9 +2,29 @@ ARG III_VERSION=0.11.2 FROM iiidev/iii:${III_VERSION} AS iii-image +# Build the package from this repo rather than pulling it from the registry, so +# fixes that live on a fork actually reach the deployed container. `npm pack` +# produces the same tarball shape the registry would serve, which keeps the +# install layout below identical to the previous registry install. +FROM node:22-slim AS builder +WORKDIR /build +# No lockfile: .gitignore:23 excludes it by repo policy, so it is absent from +# any git-based build context and `npm ci` cannot run. This trades build +# reproducibility away, which is the policy's cost, not a choice made here. +COPY package.json ./ +# node:22-slim ships npm 10.9.x, whose arborist fails this dependency tree with +# "Cannot read properties of null (reading 'edgesOut')" when resolving without a +# lockfile. Reproduced on a clean git-only context; npm 11 resolves it cleanly. +RUN npm install -g npm@11.19.0 --no-fund --no-audit \ + && npm install --no-fund --no-audit +COPY . . +# npm pack does not create --pack-destination, so make it first. +RUN mkdir -p /out \ + && npm run build \ + && npm pack --pack-destination /out + FROM node:22-slim -ARG AGENTMEMORY_VERSION=0.9.29 ARG III_VERSION=0.11.2 ARG III_SDK_VERSION=0.11.2 @@ -14,21 +34,44 @@ RUN apt-get update \ COPY --from=iii-image /app/iii /usr/local/bin/iii -# Install agentmemory into a dedicated prefix so the local package.json's -# `overrides` field pins iii-sdk down to match the engine (agentmemory's -# caret range `^0.11.2` otherwise resolves to 0.11.6, the version that -# requires the new sandbox-everything worker model the agentmemory CLI -# is not refactored for yet). `npm install -g` ignores overrides, hence -# the local prefix. +# Install into a dedicated prefix. The path below is load-bearing: entrypoint.sh +# writes the iii worker config to +# /opt/agentmemory/node_modules/@agentmemory/agentmemory/dist/iii-config.yaml +# under `set -eu`, so moving this prefix kills the container at boot rather than +# at build time. Installing the packed tarball keeps npm placing the package at +# node_modules//, which reproduces that exact path. WORKDIR /opt/agentmemory -RUN printf '{"name":"agentmemory-deploy","version":"1.0.0","private":true,"overrides":{"iii-sdk":"%s"}}\n' "${III_SDK_VERSION}" > package.json \ - && npm install "@agentmemory/agentmemory@${AGENTMEMORY_VERSION}" --omit=optional --no-fund --no-audit \ +COPY --from=builder /out/*.tgz /tmp/agentmemory.tgz +# The runtime install resolves the tarball's dependencies against the registry, +# so the repo's `overrides` never reach it on their own. These are CVE pins +# (added by 91c78e7, "clear all 8 audit vulns"). Read them out of the packed +# tarball rather than retyping them, so a new pin in package.json cannot ship +# unpinned here without anyone noticing. +RUN OVERRIDES="$(tar -xzOf /tmp/agentmemory.tgz package/package.json | node -p 'JSON.stringify((JSON.parse(require("fs").readFileSync(0,"utf8")).overrides)||{})')" \ + && printf '{"name":"agentmemory-deploy","version":"1.0.0","private":true,"overrides":%s}\n' "$OVERRIDES" > package.json \ + && npm install /tmp/agentmemory.tgz --omit=optional --no-fund --no-audit \ + && rm -f /tmp/agentmemory.tgz \ && ln -s /opt/agentmemory/node_modules/.bin/agentmemory /usr/local/bin/agentmemory +# The previous Dockerfile pinned iii-sdk through an `overrides` block to stop a +# caret range resolving past the engine. Both this repo and the published package +# now pin iii-sdk exactly, so the override is redundant. Assert the resolved +# version instead of carrying a workaround for a condition that may not hold: a +# mismatch fails the build here rather than at runtime in production. +RUN RESOLVED="$(node -p "require('/opt/agentmemory/node_modules/iii-sdk/package.json').version")" \ + && echo "resolved iii-sdk: ${RESOLVED} (expected ${III_SDK_VERSION})" \ + && [ "${RESOLVED}" = "${III_SDK_VERSION}" ] + +# Prove the build produced a usable install at the path the entrypoint expects. +# Without this the image can ship a broken layout that only fails at boot, where +# failure costs a restart-budget entry instead of a build. +RUN test -f /opt/agentmemory/node_modules/@agentmemory/agentmemory/dist/iii-config.yaml \ + && test -x /usr/local/bin/agentmemory + ENV AGENTMEMORY_III_VERSION=${III_VERSION} \ TINI_SUBREAPER=1 -COPY --chmod=0755 entrypoint.sh /usr/local/bin/agentmemory-entrypoint.sh +COPY --chmod=0755 deploy/railway/entrypoint.sh /usr/local/bin/agentmemory-entrypoint.sh EXPOSE 3111 diff --git a/deploy/railway/README.md b/deploy/railway/README.md index 9aad4fb32e..9ec128ee34 100644 --- a/deploy/railway/README.md +++ b/deploy/railway/README.md @@ -65,7 +65,9 @@ For an authenticated call, your client must send `Authorization: Bearer ## Viewer access (port 3113 stays internal) Railway only exposes the single public port from your service's -`PORT` env var (which we map to 3111). The viewer stays bound to +`PORT` env var. The container always serves on 3111 (the app reads +`III_REST_PORT`, never `PORT`), so the service's target port must be set to +3111 in the Railway dashboard. The viewer stays bound to localhost inside the container. `railway ssh` is an interactive shell only — it does not support `-L`-style port forwarding, so reach the viewer with one of the following. @@ -90,6 +92,39 @@ expose that port through a second Railway TCP Proxy, then use a native `ssh -L 3113:localhost:3113 -p ` from your laptop. This is the heavier path; option A is what most users will want. +## Self-healing + +Railway only queries `healthcheckPath` at deploy time and only restarts on a +process **exit**, so a container that stops serving without exiting is invisible +to the platform. Every wedge observed so far has the same shape: the iii engine +process dies and the node process keeps running, reconnecting forever. The +engine owns the REST listener, the stream port, and the state store, so nothing +can be served once it is gone. + +| Variable | Default | Effect | +|---|---|---| +| `AGENTMEMORY_EXIT_ON_ENGINE_DEATH` | **on** | Exits non-zero when the engine dies more than 5s after it was spawned, so the platform restarts the container. Set `0` to disable. | +| `AGENTMEMORY_HEALTH_ESCALATE` | unset (**off**) | Exits after 10 consecutive failures of the 30s KV probe. | + +`AGENTMEMORY_EXIT_ON_ENGINE_DEATH` is on by default because it acts on a +process-exit event rather than a probe: there is no threshold to tune and no +false positive to trade against. Deaths within the first 5 seconds keep the +startup-failure path, which reports a clearer message. + +`AGENTMEMORY_HEALTH_ESCALATE` is a probe and stays off. Do not enable it until an +external uptime check exists against `/agentmemory/livez`. It forces process +exits and `restartPolicyMaxRetries` is 10, so a wedge recurring on every boot +reaches a stopped deployment in under an hour with nothing notifying you. +Truthy spellings are `1`, `true`, `TRUE`. **`yes` and `on` are rejected.** + +`restartPolicyType` is `ALWAYS` rather than `ON_FAILURE`: the SIGTERM shutdown +path ends in `process.exit(0)`, which `ON_FAILURE` reads as success and would not +restart. + +An earlier revision of this file documented an in-container shell watchdog +(`AGENTMEMORY_WATCHDOG*`). That was removed on 2026-08-26 in favour of the +engine-exit handler above plus external uptime monitoring. + ## Rotate the HMAC secret ```bash @@ -132,5 +167,5 @@ See for the current rate card. or use the dashboard's manual snapshot feature. - The Dockerfile builds on Railway's builder on every deploy. First deploy is ~2 minutes; cached layers make subsequent rebuilds quick. - Pin `AGENTMEMORY_VERSION` / `III_VERSION` build args in the + Pin the `III_VERSION` build arg in the service's *Variables* tab to lock a specific release. diff --git a/deploy/railway/entrypoint.sh b/deploy/railway/entrypoint.sh index ffdd633395..7cde41a29f 100755 --- a/deploy/railway/entrypoint.sh +++ b/deploy/railway/entrypoint.sh @@ -67,7 +67,9 @@ workers: file_path: /data/stream_store - name: iii-observability config: - enabled: true + # false, unlike the other three deploy targets: the in-memory OTEL + # exporter drove heap growth that crashed the container (2026-08-23). + enabled: false service_name: agentmemory exporter: memory sampling_ratio: 1.0 diff --git a/deploy/railway/health-validation.md b/deploy/railway/health-validation.md new file mode 100644 index 0000000000..b3efebf8dc --- /dev/null +++ b/deploy/railway/health-validation.md @@ -0,0 +1,244 @@ +# Health memory validation on Railway + +This experiment uses the existing `agentmemory` project and its isolated `sandbox` environment. It does not use production data or Railway's ephemeral sandbox product. + +## Environment and artifacts + +- Isolated `sandbox` deployment environment; no production data used. +- Linux cgroup v2, Node `v22.23.2`, iii engine and iii-sdk `0.11.2`. +- One Singapore replica, two vCPUs, `memory.max=8589934592` (8 GiB), `memory.high=max`, swap disabled. +- `NODE_OPTIONS=--max-old-space-size=6144`; observed V8 `heap_size_limit=6492782592`. +- Node, iii and the synthetic sibling allocator all report `0::/`. The visible cgroup mount is the namespace root. Hidden ancestor controls cannot be inspected from this container. +- Fresh ephemeral `/data`; the retained volume remained unattached and was never mounted by experimental code. +- No LLM/embedding API credentials, embedding provider or embedding shim: successful observation writes validate the provider-free path. +- Platform healthcheck remains `/agentmemory/livez`, with a 300-second startup timeout. + +The tested candidate runtime is commit `d16213721e2e0151a2904c08e8c2e9dde3ec3521`. Later commits include documentation and unrelated log/provider/GC changes validated separately by local regression tests; this live experiment does not claim their complete bundle matches the measured image. The baseline and current variants use source `42bcdce85eacb770ecf45c7315168f8be312d0b4`. The baseline changes only `heapCeiling` back to `snapshot.memory.heapTotal` in `evaluateHealth`, recreating the defect without comparing unrelated releases. Test-only code and builds stay outside the PR. + +Runtime dependency lock SHA-256: `9030232b4f6092ec74f351b61859b99f1cbf9e22a7bfe09ecce11152d6668ed2`. Subsequent builds use this lock; compare the resolved dependency version maps, because package tarball integrity changes with source. Record each image digest and installed `dist/index.mjs` hash separately. + +## Workload + +A test-only module imported by the application worker retains 640 MiB of JavaScript numbers in 8 MiB arrays, retains a touched 64 MiB Buffer, and starts a sibling Node process with another touched 64 MiB Buffer. Each second it replaces an 8 MiB array to create ordinary allocation/GC churn. The sibling's cgroup membership is verified rather than assumed. + +After at least 65 seconds of warm-up, take 31 samples at 30-second intervals: a full 15-minute first-to-last window. For every sample, capture `/health` HTTP status and JSON from the same response; capture `/livez`, a successful synthetic `/observe` write, worker memory, V8 limit, cgroup current/max/high/swap, memory.events and PSI. Verify health `uptimeSeconds` advances so repeated polls cannot masquerade as fresh collection. + +The initial single-array experiment is excluded from the comparison. One very large JS array caused transient V8 allocation overhead; every accepted run uses the same bounded 8 MiB chunks. + +## Results + +| Variant | Fresh samples / duration | Health | Liveness / observations | Heap used / committed | Heap used / V8 limit | +|---|---|---|---|---|---| +| Reintroduced baseline | 31 / 15m0.406s | 31 × 503, memory-critical | 31 × 200 / 31 × 201 | 95.058–96.972% | 10.807–18.682% | +| Existing PR #1224 | 31 / 15m10.920s | 31 × 200, healthy | 31 × 200 / 31 × 201 | 95.031–97.670% | 10.808–23.731% | +| Candidate `d16213721e2e0151a2904c08e8c2e9dde3ec3521` | 31 / 15m0.409s | 31 × 200, healthy | 31 × 200 / 31 × 201 | 95.203–97.482% | 10.939–19.725% | + +All three variants used identical Node/iii versions and the same 232 resolved package versions. Every paired response had matching body status and HTTP code, every health uptime was unique, and no cgroup memory event or PSI total increased. + +| Variant | Image digest | Installed `dist/index.mjs` SHA-256 | +|---|---|---| +| Baseline | `sha256:f48371808108061acb706f71760f082868919865871d1a65f91b0ad88318ef43` | `f555f9f2558e4a6f1d7795fe95ea904559d1954f719a68a7becdb3146421a048` | +| Current PR | `sha256:13b14bbb5d24ad737cee9b16ad134475a1c64b0509585873a826209ff8f63abb` | `e824533a643bf28b96eb110d8b51822c5f1fabbc33f85c7f349f57a44bccac46` | +| Candidate | `sha256:2fadeee1e7995a6ca4d90654022c15b7999ef2f63a196c814276fe89eb668d14` | `ec8af71563960e3ca0f4bd4c34d3a0c15d3e915b026a2e0e05e5d6c999610459` | + +Workload module SHA-256: `f8ace48d97343665518729861de6506deb80cdff7142489266382cce27676f9e`. + +## Controlled transition and file-cache checks + +The hold test produced 97 paired polls over eight fresh collector snapshots. Both heap and cgroup-max entered pending pressure at 06:38:34 UTC (degraded, HTTP 200, sample 1/2), became critical at 06:39:03 (HTTP 503), remained critical on the first clear snapshot at 06:40:04 (recovering, sample 1/2), and recovered at 06:40:33 (healthy, HTTP 200). Repeated polls within a snapshot did not advance either transition counter. Both runs use the same candidate image with sandbox-only warning/critical thresholds of 3%/5%, keeping actual use far below the 8 GiB cap. Preliminary transition recovery and cache output is excluded: the first test-only forced-GC helper used unsupported dynamic import in an inspector expression. The corrected helper calls `HeapProfiler.collectGarbage`; application code and the allocation module are unchanged. Clearing the control file drops bulk heap/native/sibling allocations but retains the last 8 MiB churn array until process stop, explaining the approximately 31 MiB recovery heap floor. + +The separate file-cache test produced 97 paired polls over seven fresh snapshots. Writing a 512 MiB disposable file moved cgroup occupancy to approximately 7.5% while heap use stayed approximately 0.5%. Cgroup-max entered pending pressure at 06:42:03 UTC, became critical at 06:42:33, entered recovery after unlinking the file at 06:43:34, and returned healthy at 06:44:04. Heap remained healthy throughout. Every controlled poll returned liveness 200 and observation 201; all body/HTTP mappings matched, and memory.events, PSI totals and swap remained zero. + +Controlled-phase helper SHA-256: + +- `force-gc.mjs`: `938c878c2158de38b2c55a1f36434ccd1444411747a35f9b49f1109dd26a44e9` +- `hold-samples.mjs`: `5453cdb068d5ed6fd0657ec23c57425ce658350c77931af221c6ad7921ffec52` +- `cache-samples.mjs`: `95f3948ddcd60f7303b0a57cdb7b99ce3b13f0abeb6180132145f95c0aa72321` + +### Operational correction + +Railway CLI 5.27 treats service source disconnect as service-wide even with an environment argument. Using it during sandbox setup removed a shared production source binding. It was restored with an environment-scoped patch and `skipDeploys:true`; configuration read-back and unchanged production deployment identities verified the correction without a production deployment. Private incident records retain the exact scope and evidence. + +## Limitations + +This is a synthetic workload on a fresh store. It does not establish retained-corpus capacity or production startup performance. The live environment exposes neither finite `memory.high` nor nested systemd ancestor controls; those cases are covered by filesystem fixtures. Cgroup v1 and hidden ancestors remain unsupported/unobservable as documented. Cgroup filesystem reads have no explicit timeout; the Docker engine observer itself has no retry mechanism. Lowered thresholds test transitions without deliberately approaching physical OOM. `memory.current` includes reclaimable file cache: current/max evaluates configured capacity occupancy, not proof that an allocation would fail or that OOM is imminent. + +## Cleanup + +Bulk synthetic allocations and the disposable cache file were removed, and test-only threshold overrides were deleted with `skipDeploys:true`. Railway normalized a last-region scale-to-zero request into its default region with one configured replica; the resulting sandbox deployment was canceled and the measured sandbox deployment stopped explicitly. Final read-back verifies no active sandbox deployment or running instance. The final 8 MiB churn array disappears with process stop. Service/configuration and artifact evidence remain; the retained volume stays READY and unattached. Production source/trigger configuration and both production deployment identities remain unchanged. Configured replica defaults remain present; this cleanup establishes an inactive sandbox rather than claiming a persisted zero-replica configuration. + +## Reproducing the deterministic checks + +```sh +npx vitest run test/health-memory.test.ts test/health-collection.test.ts test/health-monitor.test.ts test/health-thresholds.test.ts +npm test +npm run build +``` + +For another live comparison, export each recorded source to its own build context, reuse the dependency lock, add the same test-only worker allocation module, and deploy only to an isolated environment. Keep provider configuration, V8 limit, cgroup cap, replica count and warm-up unchanged. Test code must never be imported by a production build. API authentication must be read inside the container; do not print the generated HMAC secret. + +At the end of steady-state sampling, redeploy the exact candidate image with sandbox-only `AGENTMEMORY_HEALTH_MEM_WARN_PCT=3` and `AGENTMEMORY_HEALTH_MEM_CRITICAL_PCT=5`. Start before bulk test allocations, then apply the bounded workload. Polling faster than the collector must leave each snapshot's transition count unchanged. Remove bulk heap/native/sibling allocations and explicitly GC the worker only for this controlled test; verify recovery requires two fresh collector samples. Delete the test-only threshold overrides afterward. + +### Exact test-only allocation module + +The following is the frozen module used for the recorded measurements and workload hash above. In an isolated source export, save it as `src/repro-runtime.ts`, apply the required publication patch below, then add `import "./repro-runtime.js";` to `src/index.ts` before building. This import belongs only in the disposable experiment build, never in the application PR. The module collects diagnostics every second and adds bulk allocations only when its control file requests them. + +```typescript +import { readFileSync, writeFileSync, readdirSync } from 'node:fs'; +import { getHeapStatistics } from 'node:v8'; +import { spawn, type ChildProcess } from 'node:child_process'; +let heap: number[][] = []; +let native: Buffer | undefined; +let child: ChildProcess | undefined; +let previous = ''; +let churn: number[] = []; +const read = (p: string) => { try { return readFileSync(p, 'utf8').trim(); } catch { return null; } }; +setInterval(() => { + const control = read('/tmp/health-repro-control.json') || '{}'; + if (control !== previous) { + const config = JSON.parse(control); + heap = Array.from({length:Math.ceil((config.heapMiB || 0)/8)},()=>new Array(1024*1024).fill(0.25)); + native = config.nativeMiB ? Buffer.alloc(config.nativeMiB * 1024 * 1024, 0x5a) : undefined; + child?.kill(); child = undefined; + if (config.siblingMiB) child = spawn(process.execPath, ['-e', `const b=Buffer.alloc(${config.siblingMiB}*1024*1024,0x55);setInterval(()=>{if(!b[0])process.exit(1)},1000)`], { stdio: 'ignore', env: { ...process.env, NODE_OPTIONS: '' } }); + previous = control; + } + if (heap.length) churn = new Array(1024 * 1024).fill(Math.random()); + const processes = readdirSync('/proc').filter(p => /^\d+$/.test(p)).map(pid => ({pid, comm:read(`/proc/${pid}/comm`), cgroup:read(`/proc/${pid}/cgroup`), status:read(`/proc/${pid}/status`)?.split('\n').filter(l=>/^(VmRSS|VmSize|Name|PPid):/.test(l))})); + const cgroup = Object.fromEntries(['memory.current','memory.max','memory.high','memory.events','memory.pressure','memory.swap.current','memory.swap.max'].map(f => [f,read(`/sys/fs/cgroup/${f}`)])); + writeFileSync('/tmp/health-repro-diagnostics.json',JSON.stringify({timestamp:new Date().toISOString(),deploymentId:process.env.RAILWAY_DEPLOYMENT_ID,variant:process.env.HEALTH_REPRO_VARIANT,pid:process.pid,node:process.version,versions:process.versions,nodeOptions:process.env.NODE_OPTIONS,memory:process.memoryUsage(),heapSizeLimit:getHeapStatistics().heap_size_limit,allocations:{heapElements:heap.length * 1024 * 1024,nativeBytes:native?.length||0,churnElements:churn.length,siblingPid:child?.pid},cgroup,processes,mountinfo:read('/proc/self/mountinfo')?.split('\n').filter(l=>l.includes('cgroup'))})); +}, 1000).unref(); +``` + +### Required diagnostics publication patch for new runs + +The measured module truncates the diagnostics file while rewriting it, so another process can read partial JSON. Before building any new experiment, run this patch in its isolated source export. It writes diagnostics to a temporary file in the same directory, then atomically renames it over the reader-visible path. All diagnostics readers then receive a complete previous or current snapshot. + +This changes only diagnostics publication. The original module, hash and recorded measurements above remain historical evidence; the patched module has a different hash and has not been measured in those runs. Record the printed hash with each new run and use the patched module consistently across its variants. + +```sh +node --input-type=module <<'NODE' +import { readFileSync, writeFileSync } from 'node:fs'; +import { createHash } from 'node:crypto'; +const file = 'src/repro-runtime.ts'; +let source = readFileSync(file, 'utf8'); +if (createHash('sha256').update(source).digest('hex') !== 'f8ace48d97343665518729861de6506deb80cdff7142489266382cce27676f9e') { + throw new Error('Expected the exact measured allocation module before patching'); +} +source = source + .replace('writeFileSync, readdirSync', 'writeFileSync, renameSync, readdirSync') + .replace("writeFileSync('/tmp/health-repro-diagnostics.json',", "writeFileSync('/tmp/health-repro-diagnostics.next',") + .replace('\n}, 1000).unref();', "\n renameSync('/tmp/health-repro-diagnostics.next', '/tmp/health-repro-diagnostics.json');\n}, 1000).unref();"); +writeFileSync(file, source); +console.log('Atomic-publication workload SHA-256:', createHash('sha256').update(source).digest('hex')); +NODE +``` + +### Steady-state sampler + +Save the following as `/tmp/health-repro-sample.mjs` inside the sandbox container. It reads authentication locally and prints only response/diagnostic evidence. + +```javascript +import {readFileSync,writeFileSync} from 'node:fs'; +const secret=readFileSync('/data/.hmac','utf8').trim(); +const variant=process.argv[2]||'current'; +const count=Number(process.argv[3]||30); +const headers={authorization:`Bearer ${secret}`,'content-type':'application/json'}; +const request=async(path,body)=>{ + const at=new Date().toISOString();const response=await fetch('http://127.0.0.1:3111/agentmemory/'+path,{method:body?'POST':'GET',headers,body:body?JSON.stringify(body):undefined,signal:AbortSignal.timeout(15000)}); + return {at,status:response.status,body:await response.json()}; +}; +for(let n=0;nsetTimeout(r,Math.max(0,30000-(Date.now()-start)))); +} +``` + +After verifying the exact deployment is successful, run inside that container: + +```sh +node - <<'NODE' +const fs = require("node:fs"); +fs.writeFileSync("/tmp/health-repro-control.next", JSON.stringify({ heapMiB: 640, nativeMiB: 64, siblingMiB: 64 })); +fs.renameSync("/tmp/health-repro-control.next", "/tmp/health-repro-control.json"); +NODE +sleep 65 +node /tmp/health-repro-sample.mjs candidate 31 > /tmp/health-samples.jsonl +node - <<'NODE' +const fs = require("node:fs"); +fs.writeFileSync("/tmp/health-repro-control.next", "{}"); +fs.renameSync("/tmp/health-repro-control.next", "/tmp/health-repro-control.json"); +NODE +``` + +Changing the label only identifies the output; select baseline/current/candidate by deploying the corresponding source artifact. Archive the JSONL before stopping the disposable container. The baseline-only source change is `const heapCeiling = snapshot.memory.heapTotal;` in the recorded old `evaluateHealth` implementation; do not apply it to the candidate implementation. + +### Controlled checks (reproducible pseudocode) + +The following sequence describes the transition samplers. Use the same request helper and diagnostics as above; poll every two seconds while the application collector runs every 30 seconds. Every poll includes one health response, one liveness response and one synthetic observation write. + +```text +Redeploy identical candidate image with sandbox-only warning=3, critical=5. +Wait at least 70 seconds with control={} and verify healthy. +Record before sample. +Atomically set control={heapMiB:640,nativeMiB:64,siblingMiB:64}. +Record 48 entry polls, two seconds apart. +Atomically set control={heapMiB:0,nativeMiB:0,siblingMiB:0}; wait two seconds. +Force worker GC via inspector HeapProfiler.collectGarbage (helper below). +Record 48 recovery polls, two seconds apart. +Verify entry: entering(samples=1)/200, then critical/503 on next fresh snapshot. +Verify recovery: recovering(samples=1)/503, then healthy/200 on next fresh snapshot. +Verify repeated polls with identical uptimeSeconds never change transition state. + +Starting healthy after removing bulk heap/native/sibling load: +Record before sample. +Write /tmp/health-repro-cache.bin using 512 writes of one touched 1 MiB buffer. +Record 48 file-cache polls, two seconds apart; include memory.stat. +Unlink only that test file. +Record 48 cache-recovery polls, two seconds apart. +Verify cgroup occupancy can independently enter/recover while heap stays healthy. +``` + +The file-cache sampler reads `memory.stat` directly for each poll, separately from the unchanged allocation module. Save this exact helper as `/tmp/cache-samples.mjs` and run `node /tmp/cache-samples.mjs` inside the disposable container after recovery: + +```javascript +import {readFileSync,openSync,writeSync,closeSync,unlinkSync} from 'node:fs'; +const secret=readFileSync('/data/.hmac','utf8').trim(); +const headers={authorization:`Bearer ${secret}`,'content-type':'application/json'}; +const request=async(path,body)=>{const at=new Date().toISOString();const response=await fetch('http://127.0.0.1:3111/agentmemory/'+path,{method:body?'POST':'GET',headers,body:body?JSON.stringify(body):undefined,signal:AbortSignal.timeout(15000)});return {at,status:response.status,body:await response.json()};}; +const sample=async(phase,n)=>{const health=await request('health');const livez=await request('livez');const observe=await request('observe',{hookType:'PostToolUse',sessionId:'health-repro-cache',project:'health-repro',cwd:'/synthetic',timestamp:new Date().toISOString(),data:{tool_name:'Read',tool_input:{file_path:'/synthetic/cache-'+phase+'-'+n+'.txt'},tool_response:'Synthetic cache capacity sample'}});console.log(JSON.stringify({phase,n,deploymentId:process.env.RAILWAY_DEPLOYMENT_ID,health,livez,observe,diagnostics:JSON.parse(readFileSync('/tmp/health-repro-diagnostics.json','utf8')),memoryStat:readFileSync('/sys/fs/cgroup/memory.stat','utf8')}));}; +await sample('before',0); +const fd=openSync('/tmp/health-repro-cache.bin','w');const buffer=Buffer.alloc(1024*1024,0x5a); +for(let n=0;n<512;n++)writeSync(fd,buffer);closeSync(fd); +for(let n=0;n<48;n++){await sample('file-cache',n);await new Promise(r=>setTimeout(r,2000));} +unlinkSync('/tmp/health-repro-cache.bin'); +for(let n=0;n<48;n++){await sample('cache-recovery',n);await new Promise(r=>setTimeout(r,2000));} +``` + +The corrected GC helper runs only inside the disposable container. It opens the worker's loopback inspector, collects dead test allocations, records memory and closes the inspector. Save as `/tmp/force-gc.mjs`: + +```javascript +import {readFileSync} from 'node:fs'; +const pid=JSON.parse(readFileSync('/tmp/health-repro-diagnostics.json','utf8')).pid; +process.kill(pid,'SIGUSR1'); +await new Promise(r=>setTimeout(r,1000)); +const targets=await (await fetch('http://127.0.0.1:9229/json/list')).json(); +const ws=new WebSocket(targets[0].webSocketDebuggerUrl); +await new Promise((resolve,reject)=>{ws.addEventListener('open',resolve,{once:true});ws.addEventListener('error',reject,{once:true});}); +let id=0; +async function command(method,params={}){const requestId=++id;return new Promise((resolve,reject)=>{const timer=setTimeout(()=>reject(new Error('Inspector timeout')),10000);function handle(e){const r=JSON.parse(e.data);if(r.id!==requestId)return;clearTimeout(timer);ws.removeEventListener('message',handle);if(r.error||r.result?.exceptionDetails)reject(new Error(JSON.stringify(r)));else resolve(r.result);}ws.addEventListener('message',handle);ws.send(JSON.stringify({id:requestId,method,params}));});} +await command('HeapProfiler.collectGarbage'); +console.log(JSON.stringify({forcedGc:await command('Runtime.evaluate',{expression:'process.memoryUsage()',returnByValue:true})})); +await command('Runtime.evaluate',{expression:'setTimeout(() => process.getBuiltinModule("inspector").close(), 100)'}); +ws.close(); +``` + +## Docker ownership integration validation + +After integrating main's instance ownership handling, Node 22.23.2 passed 1,870 tests (one skipped), including 58 focused Docker/startup/launch tests; build and generated skill checks passed. TypeScript retained the same 30 normalized diagnostics as before the merge. Fresh Docker lookup uses the launch project and working directory, resumed engines use their validated container ID, Docker failures preserve ownership metadata, and intentional stop cancels supervision first. + +A separate live smoke used the actual merged watcher on Docker Engine 29.4.0 and Node 25.9.0 with Alpine 3.22 (`sha256:14358309a308569c32bdc37e2e0e9694be33a9d99e68afb0f5ff33cc1f695dce`). Two projects sharing one Compose file verified project-scoped lookup; direct-ID observation reported exit 7, while the scoped observer reported exit 0. Explicit cancellation and observer-parent exit left containers running; parent exit reaped the Docker wait process. Both disposable projects were then removed and label-scoped read-back was empty. Containers had no network, ports or host mounts and a 32 MiB limit. This verifies the Docker watcher; full CLI death-policy behavior has local regression coverage, not a live iii-engine restart test. The recorded Railway measurements above were not rerun for this integration. diff --git a/deploy/railway/railway.json b/deploy/railway/railway.json index 43f52173b9..4272988850 100644 --- a/deploy/railway/railway.json +++ b/deploy/railway/railway.json @@ -7,8 +7,8 @@ "deploy": { "numReplicas": 1, "healthcheckPath": "/agentmemory/livez", - "healthcheckTimeout": 30, - "restartPolicyType": "ON_FAILURE", + "healthcheckTimeout": 60, + "restartPolicyType": "ALWAYS", "restartPolicyMaxRetries": 10, "requiredMountPath": "/data" } diff --git a/docs/plans/2026-08-25-001-infra-agentmemory-hang-resilience-plan.md b/docs/plans/2026-08-25-001-infra-agentmemory-hang-resilience-plan.md new file mode 100644 index 0000000000..dcae6c7564 --- /dev/null +++ b/docs/plans/2026-08-25-001-infra-agentmemory-hang-resilience-plan.md @@ -0,0 +1,770 @@ +--- +artifact_contract: ce-unified-plan/v1 +artifact_readiness: implementation-ready +product_contract_source: ce-plan-bootstrap +execution: code +created: 2026-08-25 +depth: deep +branch: fix/1223-heap-severity-denominator +--- + +# agentmemory hang resilience + +## Goal Capsule + +**Objective.** A wedged agentmemory backend recovers without a human, and a human +finds out that it happened. + +**Means.** Two independent detection layers plus a notification layer, and the +deploy-path change that lets the in-process layer actually ship (KTD1, KTD6). + +**Authority hierarchy.** Product Contract outranks Planning Contract. A user +instruction in session outranks both. Where this plan and +`~/.claude/rules/destructive-commands.md` disagree about running a deploy, the +rule wins: draft, show target, confirm. + +**Stop conditions.** Stop and ask before any `railway up`, before any `railway +redeploy`, and before opening the upstream PR. Stop if U1's source build produces +a `dist/` that fails U1's parity check. + +**Execution profile.** Verification-first. Every unit that changes runtime +behaviour carries a check that fails if the behaviour breaks. + +**Tail ownership.** `ce-work` implements. Multi-lens subagent review follows, per +the user's instruction and `~/.claude/rules/code-review-methodology.md`. + +--- + +## Product Contract + +### Summary + +On 2026-08-25 the agentmemory Railway service stopped serving while its process +stayed alive. Railway reported `Online` for about 8 hours. This was the second +occurrence. Nothing detected it and nothing recovered it. This plan makes the +failure self-healing and visible. + +### Problem Frame + +Two platform mechanisms should have caught it. Neither can, and the gap between +them is exactly where this failure lives. + +- `healthcheckPath: /agentmemory/livez` gates a **new deploy** only. Railway's + docs state: "Railway does not monitor the healthcheck endpoint after the + deployment has gone live." +- `restartPolicyType: ON_FAILURE` fires when the process **exits**. This process + hangs and never exits. + +**CORRECTED 2026-08-26: the root cause below is wrong, and the correction is +load-bearing for every unit in this plan.** This is not an application hang. The +**iii engine process dies and the node process keeps running.** Verified by +`railway ssh` into a wedged container: `/proc` held only `tini` and +`node /usr/local/bin/agentmemory`, with **no `iii` process at all**; +`/proc/net/tcp` had one listener, `127.0.0.1:3113` (the viewer, owned by node), +while 3111, 3112 and 49134 were all unbound; `/data/state_store.db` mtime equalled +`last_ok` exactly. + +The 3111 REST listener lives **inside the engine**, so it dies first, with the +engine, not last. That inverts F1 and F2 below. The `state::set` timeout in the +table is a **consequence** of the engine already being gone: an invocation with +no peer to answer it, timing out at the engine's own 180000 ms setting. The +engine died at 05:59:11Z, where the `ws` Sender errors are. + +A dead engine is therefore invisible to the platform. Worse, the application +already collects the signal that would identify it and then discards it: +`src/health/monitor.ts` probes `kv.set`/`kv.get` every 30 seconds and measures +event-loop lag, writes the snapshot to KV, and returns. `src/health/thresholds.ts` +never reads `kvConnectivity`, and nothing acts on `status: "critical"`. + +The observed failure sequence, from the Railway logs: + +| Time (UTC) | Event | +|---|---| +| 08-24 20:45:00 | Deploy `a201e22d` succeeds | +| 08-24 20:45:46 | `[iii] Reconnecting` attempt 1 (starts at boot, unrelated to the fault) | +| 08-25 05:59:11 | `ws` Sender errors in `node_modules/ws/lib/sender.js` | +| 08-25 06:05:10 | `error Compression failed: "Invocation timeout after 180000ms: state::set"` | +| 08-25 06:08:19 | Last application log line, then silence | + +Memory sat at 663.9 MB against an 8192 MB limit with CPU idle, so this was a hang +and not an out-of-memory event. + +### Requirements + +- **R1.** A process that stops serving on 3111 must be restarted without human + action. +- **R2.** The restart mechanism must not exhaust `restartPolicyMaxRetries: 10` + and leave the deployment permanently `CRASHED`. That outcome is strictly worse + than the bug it replaces. +- **R3.** An outage, and a restart loop, must reach a human without a human + polling for it. +- **R4.** Source-level fixes held in this fork must reach production. Today they + cannot. +- **R5.** The health monitor must act on the failure signal it already collects, + rather than recording it and returning. +- **R6.** No detection path may depend on `/agentmemory/health` while upstream + issue #1223 is unfixed in production, because that route returns 503 on a + healthy process. +- **R7.** The source-level fixes must be offered upstream to + `rohitg00/agentmemory`, not held only in the fork. + +### Actors + +- **A1. The container process.** `agentmemory` under `tini`, PID equal to the + entrypoint shell's PID after `exec`. +- **A2. The Railway platform.** Owns deploy-time healthchecks and the + `ON_FAILURE` restart policy. Sees process exits, not hangs. +- **A3. The operator.** Currently the only detector. This plan removes that role + from the recovery path and leaves it in the notification path. +- **A4. MCP clients.** Claude Code, Codex, and OpenCode. All read + `AGENTMEMORY_URL=http://127.0.0.1:8899`, so they degrade and recover together. + +### Key Flows + +- **F1. Listener death.** The 3111 listener disappears while the process lives. + The out-of-process watchdog detects it and forces an exit. Railway restarts. +- **F2. KV stall with a live loop.** `state::set` begins timing out while the + event loop still runs. The in-process monitor detects it and exits before the + listener dies. This is the path the real outage took at 06:05Z. +- **F3. Restart loop.** A deterministic wedge recurs on every boot. The + once-per-boot cap bounds it, and the notification layer surfaces it. + +### Acceptance Examples + +- **AE1.** Given a live container, when the 3111 listener stops answering, then + the container exits and restarts within roughly 4 minutes, and + `/agentmemory/livez` returns 200 without operator action. Covers R1, F1. +- **AE2.** Given a live node process, when the KV probe fails on **10** + consecutive 30-second collections, then the process exits. The threshold is 10, + not 3: `src/index.ts` documents `state::set` exceeding the SDK's 30 s timeout + under sustained hook load, and a 5 s probe cannot tell a slow store from a dead + one. An earlier draft said "before the HTTP listener dies" -- **that is wrong**, + the listener belongs to the engine and is already gone. Covers R5, F2. +- **AE3.** Given a wedge that recurs on every boot, when the watchdog acts, then + it acts at most once per container lifetime, and watchdog action alone never + drives the deployment to `CRASHED`. Covers R2, F3. +- **AE4.** Given a source-built image, when it is deployed, then + `grep -rl heapSizeLimit /dist/` returns at least one file. + **`dist/` is bundled, not per-module** — `tsdown` emits hash-suffixed chunks + (`src-CzgoepGU.mjs`, `index.mjs`) and there is no `dist/health/` directory, so + the assertion must be a recursive grep over `dist/` and never a fixed module + path. Verified against the live container: `heapSizeLimit` appears **nowhere** + in the deployed `dist/`, which is the current failing state this proves out of. + Covers R4. +- **AE5.** Given the backend is down, when the uptime check next runs, then a + notification reaches the operator without the operator looking. Covers R3. + +### Success Criteria + +- No operator action is required to recover from a hang. +- No hang exceeds roughly 5 minutes of downtime. +- Every recovery event and every restart loop produces a notification. +- The four health commits on this branch run in production. + +### Scope Boundaries + +**In scope.** The Railway deploy target. The entrypoint watchdog. The in-process +health escalation. The Dockerfile source build. External uptime monitoring. An +upstream PR. + +**Out of scope.** +- Fixing the `iii` engine's `state::set` hang at its source. The cause sits in a + third-party engine, the payoff is uncertain, and every layer here makes the + hang survivable regardless. +- Restructuring the health module's architecture. +- `deploy/fly/`, `deploy/render/`, and `deploy/coolify/`. Each carries the same + gap in its own `entrypoint.sh`. None is deployed, so none can be verified, and + unverified copies drift. Recorded as follow-up per the user's decision. +- The MCP 7-tool degradation. It is a symptom of backend downtime, not a separate + defect. It resolves when the backend stays up, and it needs an MCP server + restart per host because clients fetch `tools/list` once at connect. + +### Dependencies + +- Railway CLI, authenticated, linked at `~/rw-agentmemory`. +- `railway ssh` access for in-container verification. +- An uptime-check provider for U5. +- A GitHub account able to open a PR against `rohitg00/agentmemory`. The fork + `inix-x/agentmemory` already exists as `origin`. + +### Outstanding Questions + +- **Q1 (deferred, with a resolution path).** The running image is 0.9.28 while + the Dockerfile ARG pins 0.9.29, so a rebuild on the current Dockerfile would + bump the app version alongside any entrypoint change. The user parked this + decision. **U1 dissolves it:** once the image builds from the repo, the app + version stops being an npm pin and becomes the repo state, so there is no + separate bump to decide. If U1 is dropped or deferred, Q1 becomes blocking + again and must return to the user. Do not decide it silently. + +### Sources + +- Railway docs, healthchecks: "Railway does not monitor the healthcheck endpoint + after the deployment has gone live." Retrieved via context7, 2026-08-25. +- Upstream issue #1223, heap severity denominator. Fixed locally on this branch, + absent from the deployed package. +- Live container inspection via `railway ssh`, 2026-08-25. + +--- + +## Planning Contract + +### Key Technical Decisions + +- **KTD1. RETIRED 2026-08-26. The out-of-process shell watchdog was removed.** + The original reasoning was that two layers cover each other's blind spots. Two + things retired it. + + First, the root cause turned out to be **engine process death**, not an + application hang, and `src/cli.ts` already registers `child.on("exit")` on the + detached engine carrying the exit code, the signal, and its dying stderr. That + is a direct event, detected in milliseconds, with no probe, no polling, and no + false-positive threshold. It replaces ~93 lines of shell with a handler that + already existed. + + Second, U5 external monitoring is mandatory before either killer may be enabled + (KTD5), so it exists either way. `livez` is unauthenticated and static, so an + external check sees exactly what an in-container `curl` sees, **and** survives a + whole-container wedge the in-container subshell cannot. Railway exposes + `deploymentRestart` on its public API. + + What is lost, stated rather than glossed: an external monitor needs a Railway + token held at the provider, `deploymentRestart` takes a per-deploy id so it is a + small script rather than a bare URL, and an egress failure between monitor and + Railway can cause a restart the in-container watchdog would not have. + Governs R1, R5. + +- **KTD2. Probe `/agentmemory/livez`, never `/agentmemory/health`.** `livez` is + unauthenticated (`src/triggers/api.ts:180-192` registers `api::liveness` with no + `checkAuth`, verified by plain curl returning 200) and returns a static payload. + `health` returns 503 on a healthy process in production because #1223 is + unfixed there. Governs R6. + +- **KTD3. Hardcode 3111 in the watchdog. Never use `$PORT`.** The application + never reads `PORT`; `entrypoint.sh` writes `port: 3111` into the iii-http worker + config directly. `PORT` happens to equal 3111 today, verified in-container. If + Railway ever injected a different value, `${PORT:-3111}` would poll a dead port, + never arm, and fail silently. The literal is the more correct choice here. + +- **KTD4. SIGTERM first, and know that the flush it buys never happens.** An + earlier draft justified the grace period as protecting `/data/state_store.db` + from corruption. **Two separate findings retire that reasoning.** First, the + engine is a detached process in its own session that owns the store, so a + single-PID SIGTERM cannot reach it and it dies by namespace teardown either + way. Second, `shutdown` awaits `indexPersistence.save()` -> `kv.set` -> + `state::set`, which parks for the engine's 180 s invocation timeout against a + dead engine, while the grace is 15 s. The hard exit always wins, so the flush + never completes in the failure this targets. + + The grace stays for the case where the engine is alive and only node is being + restarted, where the flush does complete. It is not a corruption guard, and + the plan should not claim it is. Separately, Railway already sends SIGKILL + after 0 s of draining on every deploy, so the store has survived mid-write + kills routinely. Governs R2. + +- **KTD5. Ship the notification layer first. The once-per-boot cap is weaker than + it looks.** An earlier draft claimed the cap "bounds a deterministic wedge to + one restart per deploy." **That is wrong.** The watchdog's `exit 0` / `exit 1` + are the *subshell's* status, not the container's, and by the time either runs + PID 1 has already been signalled. The cap bounds one kill per **container**, + and containers are unbounded up to `restartPolicyMaxRetries`. Every restart is + a fresh container with a fresh watchdog. + + Measured cycle times, not assumed: the shell watchdog loops in roughly 3.5 to 4 + minutes, reaching `CRASHED` in about 35 to 40 minutes; the in-process layer + loops in about 5.3 minutes, reaching it in roughly 53 minutes. The `armed` + guard and the `until curl` loop only close the never-healthy-at-boot case, not + wedge-after-boot. + + So U5 is not a nicety that makes enabling tidier. It is the only thing that + makes a restart loop visible before the budget is gone, which is why it gates + U6. Governs R2, R3. + +- **KTD6. Build from source, and accept that it is not reproducible.** An earlier + draft of this decision claimed the repo carries a `package-lock.json` so + `npm ci` would be reproducible. **That was wrong.** The lockfile exists only in + a working tree: `.gitignore:23` excludes it under the comment "Lock files — + never commit". It is absent from `git archive HEAD`, so `COPY package-lock.json` + fails every git-based build. Verified by building from a git-only context. + + The build therefore runs `npm install` with no lockfile, which floats every + caret dependency. That is the cost of the repo's lockfile policy, not a choice + made here, and it applies equally to the registry install running today. State + it plainly rather than claiming a reproducibility this build does not have. + + Two further constraints found by building it rather than reasoning about it: + `node:22-slim` ships npm 10.9.x, whose arborist fails this tree without a + lockfile (`Cannot read properties of null (reading 'edgesOut')`), so the builder + upgrades to npm 11 first; and `npm pack` ships only the `files:` allowlist, so + the runtime install resolves dependencies against the registry and the repo's + `overrides` must be carried into the runtime manifest explicitly or the CVE pins + from `91c78e7` never reach the container. Governs R4. + +- **KTD7. Evaluate and count before the persist, never after.** In + `collectHealth`, the KV probe is raced against a 5-second timeout, but the + subsequent `await kv.set(KV.health, "latest", snapshot)` at `monitor.ts:94` is + **not raced**. During the real outage `state::set` was timing out at 180000 ms. + An escalation counter placed after that `await` would never increment during the + exact failure it targets, because the function would be parked on the persist + while the 30-second interval spawned more hung collections. The counter lives in + memory and is incremented before the persist. Governs R5. + +- **KTD8. The `overrides` block in the Dockerfile is vestigial.** Its comment + describes agentmemory resolving `iii-sdk` through a caret range. Both the + published 0.9.28 and the repo's 0.9.29 pin `iii-sdk` at exactly `0.11.2`, + verified in-container and in `package.json`. U1 should drop the workaround, and + must verify the resolved version after the build rather than assume it. + +### Assumptions + +- ~~The `[iii] Reconnecting` stream is background noise.~~ **RETRACTED + 2026-08-26. This was arithmetically false, and believing it produced the wrong + root cause above.** Port 49134 is the *engine's* port, so `ECONNREFUSED` there + means the engine is gone. iii-sdk caps one reconnect attempt at + `maxDelayMs 30000 x jitter 1.3` = **39 s**. Deployment `a201e22d` booted + 08-24T20:45:00Z and logged attempt **807** at 08-25T12:39:06Z: 57,200 s / 807 = + **70.9 s per attempt**, above the ceiling, so the stream cannot have begun at + boot. At the measured 30.6 s/attempt the onset is ~05:48Z, within 11 minutes of + the `ws` Sender errors. + + **The reconnect counter dates the outage.** Divide elapsed seconds by the + attempt number and compare against the 39 s ceiling. It is the earliest and + cheapest signal in the log buffer. +- `tini` exits non-zero when its child dies by signal, so Railway classifies the + exit as a failure and `ON_FAILURE` restarts. **U2 must verify this**, because + a clean exit 0 would not trigger a restart and the watchdog would be inert. +- A hang can leave the event loop running. Grounded: the application logged a + completed smart search at 06:08:19Z, three minutes after the `state::set` + timeout. This is what makes the in-process layer worth having. + +### Implementation Constraints + +- Stay on `fix/1223-heap-severity-denominator`. No branch switch, no worktree, + per `~/.claude/rules/no-branch-switching.md`. +- `deploy/railway/entrypoint.sh` and `deploy/railway/railway.json` are currently + modified and uncommitted, and those edits are what production runs. U0 commits + that state before anything stacks on it. +- The entrypoint is POSIX `sh`, not bash. `node:22-slim` provides `dash`. +- Never print `AGENTMEMORY_SECRET` or any `/data/.hmac` content. Handle by path + and by metadata, per `~/.claude/rules/triggr-credential-handling.md`. + +### Sequencing + +U0 gates everything. U5 gates the enablement of U2 and U4 per KTD5. U1 gates U3 +and U4 reaching production. U8 is independent and may run any time after U3/U4. + +**U1 and U2 both edit `deploy/railway/entrypoint.sh`, so they must be +serialized.** Run U1 first: it may relocate the install root, and U2's watchdog is +inserted relative to the final `exec` line that U1 may rewrite. Running them as +parallel branches means whichever is written second clobbers the first. + +``` +U0 (commit + backup) + | + +--> U1 (source build, edits entrypoint.sh) + | + +--> U2 (watchdog, edits entrypoint.sh, shipped DISABLED) --+ + | | + +--> U3 --> U4 -------------------------------------------- + + | | + | U5 --> U6 (enable) --> U7 (verify) + | + +--> U8 (upstream PR) +``` + +--- + +## Implementation Units + +### U0. Commit the deployed state and back up the volume + +**Goal.** Establish a known-good rollback point before anything changes. + +**Requirements.** R2. + +**Files.** `deploy/railway/entrypoint.sh`, `deploy/railway/railway.json`. + +**Approach.** Both files are modified and uncommitted, and both are what +production runs. Commit them as-is on the current branch so later changes are +separable. Then back up `/data` using the command in `deploy/railway/README.md`. +Record the current deployment id and image digest so the rollback target is +written down rather than remembered. + +**Test Scenarios.** +- `git status --short -- deploy/railway/` reports clean after the commit. +- The backup archive exists locally and is non-empty. +- The recorded digest matches `railway deployment list --json` for the live + deployment. + +**Verification.** `git log -1 --stat -- deploy/railway/` shows both files. The +archive's size is within an order of magnitude of the volume's 0.8 GB. + +--- + +### U1. Build the image from repo source + +**Goal.** Make fork-level source changes reach production. Today they cannot. + +**Requirements.** R4. Dissolves Q1. + +**Files.** `deploy/railway/Dockerfile`, `deploy/railway/entrypoint.sh`, +`.dockerignore` (new). Plus one **Railway service setting**, which is not a file. + +**`rootDirectory` is NOT in `railway.json`.** Verified: the file's keys are +`$schema`, `build{builder,dockerfilePath}`, `deploy{...}`, and neither +`serviceManifest` nor `fileServiceManifest` carries `rootDirectory`, yet the +deployment meta reports `rootDirectory: deploy/railway`. It is a dashboard/API +service setting. **Editing `railway.json` will not change the build context.** If +this unit only edits files, the context stays `deploy/railway/`, `src/` never +uploads, and U1 silently no-ops while appearing to succeed. Change the service +setting to the repo root and repoint `dockerfilePath` accordingly. + +**Approach.** The current Dockerfile has two `COPY` lines, neither touching +`src/`, and no build step. It installs the published package from npm, which is +why `/opt/agentmemory/src` does not exist in the image and the deployed version +is 0.9.28. Use the KTD6 source-build procedure: repository-root context, copy +`package.json`, install with npm 11.19.0 and `npm install` (the repository does +not commit a lockfile), copy source, build, then `npm pack`. Install that +tarball into `/opt/agentmemory` in the runtime stage, carrying the package +security overrides into the runtime manifest. Preserve the entrypoint +install layout and verify the resolved `iii-sdk` version. `.dockerignore` +excludes local dependencies, git metadata, build output, and evaluation data. + +Note that `dist/` is gitignored, so the image must build it and must never expect +a prebuilt copy. + +**The entrypoint is hard-coupled to the npm install layout, and this is the way +U1 fails at boot.** `entrypoint.sh` runs under `set -eu` and does: + +```sh +III_CONFIG="/opt/agentmemory/node_modules/@agentmemory/agentmemory/dist/iii-config.yaml" +cat > "$III_CONFIG" <<'EOF' +``` + +If U1 moves the install root, that `cat >` writes into a directory that no longer +exists, `set -e` aborts, and the container dies **before the app starts**. Railway +reads that as a failure and retries, so a botched U1 burns all ten retries on the +first deploy and lands in `CRASHED` — precisely the R2 outcome this plan exists to +prevent. `npm run build` is `tsdown && cp iii-config.yaml dist/`, so the config +always lands at `/dist/iii-config.yaml`. + +Choose one deliberately and write down which: **either** preserve the exact +`/opt/agentmemory/node_modules/@agentmemory/agentmemory/` prefix, **or** update +`III_CONFIG` to the new root. The same applies to the final +`exec gosu "$RUN_AS" agentmemory "$@"`, which resolves `agentmemory` on PATH +through the Dockerfile's symlink into `node_modules/.bin/`. U1 must recreate an +equivalent or that line fails too. + +**Test Scenarios.** +- The image builds from a clean context. +- `node -e "require('/opt/agentmemory/.../package.json').version"` inside the + built image reports 0.9.29, matching the repo. +- `grep -rl heapSizeLimit /dist/` inside the image returns at least + one file. This is AE4. Use a recursive grep: `dist/` is bundled and + `dist/health/thresholds.js` does not exist. +- **The container boots at all.** `III_CONFIG` resolves and `agentmemory` is on + PATH. Test this before any deploy, because failure here is a boot loop that + eats the retry budget. +- The resolved `iii-sdk` is 0.11.2. +- `/agentmemory/livez` returns 200 from the built image. + +**Verification.** Build locally first. Only then consider a deploy, which is +gated by U7 and by the destructive-commands confirmation. + +--- + +### U2. REMOVED — out-of-process watchdog + +**Deleted 2026-08-26**, superseded by the engine-exit handler (U9) and U5. See +KTD1 for the reasoning. Removed: the ~93-line watchdog block from +`deploy/railway/entrypoint.sh`, four `AGENTMEMORY_WATCHDOG*` variables, their +validation guards, and the tests that exercised them. `curl` **stays** in the +image: `deploy/railway/README.md` documents `railway ssh` plus +`curl http://localhost:3113` as the in-container viewer check, and that path was +used for the 2026-08-26 forensics that corrected the root cause. + +### U9. Exit when the engine dies + +**Goal.** Turn the actual failure into a process exit, at the place that already +detects it. + +**Requirements.** R1, R2. Replaces F1. + +**Files.** `src/cli.ts`. + +**Approach.** The engine is spawned detached and already has a `child.on("exit")` +handler that captures the exit code, the signal, and up to 16KB of stderr, then +logs under `vlog` and returns. Past a 5-second startup grace, report the death on +`console.error` with that stderr and exit non-zero. Deaths inside the grace keep +the existing path, which renders a better startup message. + +Default **on**, unlike the probe-based layers: this acts on a process-exit event, +so there is no threshold to tune and no false positive to trade against. The +engine owns the REST listener, the stream port, and the state store, so nothing +can be served once it is gone. Opt out with `AGENTMEMORY_EXIT_ON_ENGINE_DEATH=0`. + +**Test Scenarios.** Death inside the grace keeps the startup path. Death after it +exits non-zero. The opt-out suppresses the exit. The stderr reaches the log. + +**Verification.** `npm test`, plus a container run that kills the engine and +observes the exit code. + +### U2-original (superseded, kept for the record) + +**Goal.** Convert a hang into an exit so the existing restart policy can act. + +**Requirements.** R1, R2. Implements F1. + +**Files.** `deploy/railway/entrypoint.sh`. + +**Approach.** Insert a backgrounded POSIX `sh` loop immediately before the final +`exec gosu`. Capture `$$` first: `exec` preserves the PID, so the shell's PID is +the PID the application will hold. The loop waits for a first successful `livez` +before arming, so a slow BM25 startup backfill can never trigger it, and a boot +that never succeeds is left to Railway's deploy-time healthcheck. After arming, it +polls every 60 seconds; on `WATCHDOG_FAILS` consecutive failures it sends SIGTERM, +waits `WATCHDOG_GRACE` seconds, then sends SIGKILL only if the process survives. +It then exits, which enforces the once-per-boot cap in KTD5 structurally rather +than by a flag. + +**Ship it disabled.** Default `AGENTMEMORY_WATCHDOG=0` in this unit. U6 flips it +after U5 exists. + +The loop's logic was drafted and exercised during planning, and all five +behaviours passed: never arms against a server that was never up; does not fire +at 2 of 3; fires at exactly 3 of 3; escalates SIGTERM to SIGKILL when SIGTERM is +ignored; a short blip resets the counter without a kill. `ce-work` should re-run +these rather than trust the record. + +**Test Scenarios.** +- Never arms when the endpoint has never returned 200. +- Does not fire at `WATCHDOG_FAILS - 1`. +- Fires at exactly `WATCHDOG_FAILS`. +- Escalates to SIGKILL when the target ignores SIGTERM. +- A recovery shorter than the threshold resets the counter and no kill occurs. +- **`AGENTMEMORY_WATCHDOG=0` produces no watchdog process at all.** +- **`tini` exits non-zero when its child is signalled**, so `ON_FAILURE` fires. + This is the Planning Contract assumption that must be proven, not assumed. A + clean exit 0 would make the whole unit inert. + +**Verification.** `sh -n` for syntax, then the behavioural harness against a +throwaway HTTP server. Confirm the exit code with a container-local run. + +--- + +### U3. Evaluate `kvConnectivity` in the health thresholds + +**Goal.** Stop discarding the signal that identifies this exact failure. + +**Requirements.** R5. Implements part of F2. + +**Files.** `src/health/thresholds.ts`, `test/health-thresholds.test.ts` +(**existing, append only**). + +**`test/health-thresholds.test.ts` ALREADY EXISTS** — 174 lines, 9 tests, +carrying the regression coverage for issue #158 and for the #1223 denominator +work in commit `44e5372`. **Append to it. Never overwrite it.** It already +defines a `snap()` fixture helper whose default includes `kvConnectivity`, so +reuse that helper rather than introducing a second fixture. + +An earlier draft of this plan claimed no health test file existed. That claim +came from a `find` whose output was swallowed by the `rtk` wrapper, and acting on +it destroyed the file (recovered from `HEAD`). Read a negative search result +through `rtk proxy ` before trusting it. + +**Approach.** `collectHealth` populates `snapshot.kvConnectivity` with `status`, +`latencyMs`, and an optional `error`, using a 5-second race. `evaluateHealth` +checks `connectionState`, `eventLoopLagMs`, `cpu`, and `memory`, and never reads +`kvConnectivity`. Add it: `status === "error"` is critical, with an alert string +matching the existing `snake_case_value` convention used by +`connection_disconnected` and `event_loop_lag_critical_NNNms`. Follow the existing +alert vocabulary rather than inventing a new shape. + +**Test Scenarios.** +- A snapshot with `kvConnectivity.status === "error"` evaluates to `critical` and + carries a `kv_*` alert. +- A snapshot with `status === "ok"` does not add an alert. +- An absent or malformed `kvConnectivity` does not throw and does not evaluate to + critical. The field is optional in older persisted snapshots, so this is a real + compatibility path, not a hypothetical. +- The existing heap, CPU, event-loop, and connection assertions still pass. + +**Verification.** `npm test`, scoped to the health tests. + +--- + +### U4. Escalate sustained critical health to a process exit + +**Goal.** Let the in-process layer act on what it detects, catching a wedge while +the event loop still runs. + +**Requirements.** R1, R2, R5. Implements F2. + +**Files.** `src/health/monitor.ts`, `test/health-monitor.test.ts` (new file). +`test/health-thresholds.test.ts` **already exists** — see U3. An earlier draft +claimed no health test file existed; that claim was wrong and acting on it +destroyed the file once already. + +**Approach.** Keep escalation gated on consecutive KV probe failures and arm +it only after a healthy KV probe has been observed. Decide before the snapshot +persist, so a stalled write cannot postpone the decision. CPU, event-loop, +connection, and memory severity must not increment this counter. + +With `AGENTMEMORY_HEALTH_ESCALATE` enabled, ten consecutive failures after +arming request SIGTERM; force a nonzero exit after 15 seconds if shutdown +stalls. Railway uses `ALWAYS`, because successful graceful shutdown exits zero. +The feature remains off by default. + +**Test Scenarios.** +- Failures before any healthy KV probe never arm escalation. +- Ten consecutive KV failures after arming trigger exactly one escalation. +- A healthy KV probe resets the counter. +- Critical memory/CPU/lag with healthy KV never escalates. +- With the flag off, no escalation occurs. +- The decision precedes snapshot persistence, including stalled writes. +- Escalation requests graceful shutdown before the forced-exit fallback. + +**Verification.** `npm test`. Assert on the escalation decision, not on a real +process exit. + +--- + +### U5. External uptime monitoring + +**Goal.** Make an outage and a restart loop reach a human without polling. + +**Requirements.** R3. Prerequisite for U6 per KTD5. + +**Files.** None in this repo. Configuration lives with the provider. Record the +setup in `deploy/railway/README.md`. + +**Approach.** Point an uptime check at +`https://agentmemory-production-0c12.up.railway.app/agentmemory/livez` on roughly +a 5-minute interval, alerting to a channel the user actually reads. The route +needs no auth, verified. This is the only layer that catches a restart loop +burning the retry budget, which is precisely the failure mode the other two +layers can create. + +**Test Scenarios.** +- The check reports healthy against the live service. +- A deliberate outage window produces an alert. +- The alert reaches the intended channel, not only the provider's dashboard. + +**Verification.** Confirm one real alert end to end. An untested alert path is +not a notification layer. + +--- + +### U6. Enable the watchdog and the escalation + +**Goal.** Turn on the killers, once notification exists. + +**Requirements.** R2, R3. + +**Files.** `deploy/railway/entrypoint.sh` default, or a Railway service variable. + +**Approach.** Flip `AGENTMEMORY_WATCHDOG` and U4's flag to on. Prefer Railway +service variables over a Dockerfile default so a rollback needs no rebuild. Do +not perform this unit until U5 has produced one verified alert. + +**Test Scenarios.** +- Both layers report armed in the deploy logs. +- A synthetic wedge in a throwaway environment produces exactly one restart. + +**Verification.** Read the deploy logs for both arming lines. + +--- + +### U7. Deploy and verify end to end + +**Goal.** Prove the whole chain on the live service. + +**Requirements.** All. + +**Files.** None. + +**Approach.** `railway up` from the repo root. **Confirm with the user first**, +per `~/.claude/rules/destructive-commands.md`: show the target project, +environment, service, and the rollback digest from U0. Watch the deployment to a +terminal state; never call it good on `railway status` alone, which reported +`Online` throughout the outage. + +**Test Scenarios.** +- AE4: `heapSizeLimit` is present in the deployed `dist/health/thresholds.js`. +- `/agentmemory/livez` returns 200 externally. +- The viewer at `http://127.0.0.1:8899/agentmemory/viewer` returns 200. +- `_proxy/status` shows a small `seconds_since_last_ok`. +- `memory_recall` through MCP returns real observations. +- Both watchdog layers log that they armed. + +**Verification.** Run every check above. Per +`~/.claude/rules/verification-no-dismiss.md`, trace any non-green item to ground +before declaring the deploy verified. + +--- + +### U8. Upstream PR to `rohitg00/agentmemory` + +**Goal.** Offer the fixes upstream instead of holding them only in the fork. + +**Requirements.** R7. + +**Files.** The U3 and U4 changes, plus the four existing health commits on this +branch. + +**Approach.** `origin` is `inix-x/agentmemory` and `upstream` is +`rohitg00/agentmemory`. Open a PR carrying the `kvConnectivity` evaluation and the +escalation. Reference issue #1223 for the heap work already committed here. +Confirm with the user before opening, per the destructive-commands rule: a PR is +outward-facing. + +**Test Scenarios.** Upstream CI passes. The PR description states the failure +mode, the evidence, and the mechanism. + +**Verification.** The PR URL is surfaced to the user. + +--- + +## Verification Contract + +Repo-specific commands: + +```bash +npm test # vitest, excludes test/integration.test.ts +npm run test:all # includes integration +npm run build # tsdown -> dist/ +sh -n deploy/railway/entrypoint.sh +``` + +Live-service checks: + +```bash +curl -sS -o /dev/null -w '%{http_code}\n' \ + https://agentmemory-production-0c12.up.railway.app/agentmemory/livez +curl -sS http://127.0.0.1:8899/_proxy/status +# Recursive: dist/ is BUNDLED, so dist/health/thresholds.js does not exist. +cd ~/rw-agentmemory && railway ssh --service agentmemory -- \ + "grep -rl heapSizeLimit /opt/agentmemory/node_modules/@agentmemory/agentmemory/dist/ | wc -l" +``` + +Quality gates: no unit is done while its own tests fail. No deploy is verified +while any check above is non-green or untraced. + +--- + +## Definition of Done + +**Global.** +- A hang recovers with no operator action, within roughly 5 minutes. +- A restart loop produces a notification rather than silence. +- The deployed image contains this branch's health fixes, proven by AE4. +- The upstream PR is open. +- No dead-end or experimental code remains in the diff. A long autonomous run + accumulates abandoned attempts; declaring done requires removing them. + +**Per unit.** Every Test Scenario passes, and every unit that changes runtime +behaviour leaves behind a check that fails if that behaviour breaks. + +**Explicitly not done** if: `railway status` alone is used as proof, the +notification path is configured but never fired once, or Q1 is resolved silently +in the event U1 is dropped. diff --git a/plugin/skills/agentmemory-config/REFERENCE.md b/plugin/skills/agentmemory-config/REFERENCE.md index ed128793dc..fe0bcb5375 100644 --- a/plugin/skills/agentmemory-config/REFERENCE.md +++ b/plugin/skills/agentmemory-config/REFERENCE.md @@ -3,7 +3,7 @@ Generated by scanning `src/` for `AGENTMEMORY_*` usage. Do not edit the block below by hand; run `npm run skills:gen` after adding or removing a variable. Internal markers ending in two underscores are excluded. -Configuration is read from the environment and from `~/.agentmemory/.env` (no `export` prefix). 42 recognized variables: +Configuration is read from the environment and from `~/.agentmemory/.env` (no `export` prefix). 50 recognized variables: - `AGENTMEMORY_AGENT_SCOPE` - `AGENTMEMORY_ALLOW_AGENT_SDK` @@ -18,10 +18,17 @@ Configuration is read from the environment and from `~/.agentmemory/.env` (no `e - `AGENTMEMORY_DOCKER_SKIP_CHOWN` - `AGENTMEMORY_DOCKER_UID` - `AGENTMEMORY_DROP_STALE_INDEX` +- `AGENTMEMORY_ENGINE_LOG` +- `AGENTMEMORY_EXIT_ON_ENGINE_DEATH` - `AGENTMEMORY_EXPORT_ROOT` - `AGENTMEMORY_FOLLOWUP_WINDOW_SECONDS` - `AGENTMEMORY_FORCE_PROXY` - `AGENTMEMORY_GRAPH_WEIGHT` +- `AGENTMEMORY_HEALTH_ESCALATE` +- `AGENTMEMORY_HEALTH_MEM_CRITICAL_PCT` +- `AGENTMEMORY_HEALTH_MEM_HOLD_SAMPLES` +- `AGENTMEMORY_HEALTH_MEM_RSS_BUDGET_MB` +- `AGENTMEMORY_HEALTH_MEM_WARN_PCT` - `AGENTMEMORY_III_CONFIG` - `AGENTMEMORY_III_VERSION` - `AGENTMEMORY_IMAGE_EMBEDDINGS` @@ -29,6 +36,7 @@ Configuration is read from the environment and from `~/.agentmemory/.env` (no `e - `AGENTMEMORY_INJECT_CONTEXT` - `AGENTMEMORY_LLM_NOTHINK` - `AGENTMEMORY_LLM_TIMEOUT_MS` +- `AGENTMEMORY_MAX_PROVIDER_CONCURRENCY` - `AGENTMEMORY_MCP_BLOCK` - `AGENTMEMORY_METRICS_PORT` - `AGENTMEMORY_PROBE_TIMEOUT_MS` diff --git a/src/cli.ts b/src/cli.ts index 1e5fb6b78c..93d2f76309 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -55,6 +55,8 @@ import { type ConnectManifest, type RemoveOptions, } from "./cli/remove-plan.js"; +import { createEngineLogForwarder } from "./cli/engine-log.js"; +import { watchDockerEngine, type DockerEngineTarget } from "./cli/docker-engine.js"; import { dockerComposeArgs, dockerProjectName, @@ -1426,6 +1428,30 @@ type StartupFailure = { }; let startupFailure: StartupFailure | null = null; +let stopDockerEngineWatch: (() => void) | undefined; + +// Off by default. The engine's own stdout is the only place a frame's +// `function_id` exists, but an unbounded log path here once wrote 137 GB +// (issue #519), so the forwarder is opt-in and hard-capped. +const ENGINE_LOG_ENABLED = + process.env["AGENTMEMORY_ENGINE_LOG"] === "1" || + process.env["AGENTMEMORY_ENGINE_LOG"] === "true"; + +function attachEngineLog( + stream: NodeJS.ReadableStream | null, + prefix: string, +): void { + if (!stream) return; + const forwarder = createEngineLogForwarder({ + prefix, + write: (line) => { + process.stderr.write(`${line}\n`); + }, + }); + stream.on("data", (chunk: Buffer) => forwarder.push(chunk)); + stream.on("end", () => forwarder.flush()); +} + let activeStartupStderr = createStartupStderrCapture(); function printCapturedStartupStderr(): void { @@ -1461,6 +1487,82 @@ function printDockerStartupLogs(): void { if (output) p.note(output.slice(-16 * 1024), "docker engine logs"); } +function createEngineExitHandler( + bin: string, + isDocker: boolean, + readStderr: () => string, + spawnedAt = Date.now(), +) { + return ( + code: number | null, + signal: NodeJS.Signals | null, + containerStopped = false, + ) => { + const abnormal = + containerStopped || + (code !== null && code !== 0) || (code === null && signal !== null); + if (abnormal) { + const stderr = readStderr(); + startupFailure = { + kind: isDocker ? "docker-crashed" : "engine-crashed", + stderr: + stderr.trim() || + (signal + ? `process killed by signal ${signal}` + : `process exited with code ${code}`), + binary: bin, + }; + vlog(`engine exited early: code=${code} signal=${signal}`); + if (IS_VERBOSE && stderr.trim()) { + p.log.error(`engine stderr:\n${stderr}`); + } + if (!isDocker) { + clearEnginePidfile(); + clearEngineState(); + } + + // The engine owns the REST listener, the stream port and the state store. + // When it dies the surviving node process cannot serve anything, but it + // stays up reconnecting forever, so the platform sees a healthy container + // and no HTTP. That is the shape of every wedge observed so far. + // + // Report it and exit, so a supervisor can restart the whole container. + // This is a process-exit event, not a heuristic probe, so there is no + // false-positive to tune. Death during startup keeps the old path: the + // startup code below reads `startupFailure` and renders a better message. + const engineRanFor = Date.now() - spawnedAt; + if ((!isDocker || containerStopped) && engineRanFor > ENGINE_STARTUP_GRACE_MS) { + console.error( + `[agentmemory] engine exited after ${Math.round(engineRanFor / 1000)}s ` + + `(code=${code} signal=${signal}); nothing can be served without it, exiting`, + ); + if (stderr.trim()) console.error(`[agentmemory] engine stderr:\n${stderr}`); + if (process.env["AGENTMEMORY_EXIT_ON_ENGINE_DEATH"] !== "0") { + process.exit(1); + } + } + } + }; +} + +function startDockerEngineWatch( + bin: string, + target: DockerEngineTarget, + spawnedAt = Date.now(), + readStderr: () => string = () => "", +): void { + const onEngineExit = createEngineExitHandler(bin, true, readStderr, spawnedAt); + stopDockerEngineWatch?.(); + stopDockerEngineWatch = watchDockerEngine( + bin, + target, + (code) => onEngineExit(code, null, true), + (error) => { + console.error(`[agentmemory] Docker engine supervision unavailable: ${error.message}`); + }, + ); +} + // Spawn a background engine and collect any startup stderr for a short // window. The process is unref'd so the CLI parent can exit cleanly; we // only care about stderr that shows up BEFORE the health check succeeds, @@ -1470,12 +1572,13 @@ function spawnEngineBackground( spawnArgs: string[], label: string, cwd?: string, + docker?: { composeFile: string; projectName: string }, ): ChildProcess { vlog(`spawn: ${bin} ${spawnArgs.join(" ")}${cwd ? ` (cwd: ${cwd})` : ""}`); - activeStartupStderr = createStartupStderrCapture(); + let stderrCapture = activeStartupStderr = createStartupStderrCapture(); const child = spawn(bin, spawnArgs, { detached: true, - stdio: ["ignore", "ignore", "pipe"], + stdio: ["ignore", ENGINE_LOG_ENABLED ? "pipe" : "ignore", "pipe"], windowsHide: true, ...(cwd ? { cwd } : {}), }); @@ -1483,37 +1586,44 @@ function spawnEngineBackground( if (!isDocker && typeof child.pid === "number") { writeEnginePidfile(child.pid); } + if (ENGINE_LOG_ENABLED) { + attachEngineLog(child.stdout, "[engine]"); + attachEngineLog(child.stderr, "[engine:err]"); + } + const spawnedAt = Date.now(); child.stderr?.on("data", (chunk: Buffer) => { - activeStartupStderr.append(chunk); + stderrCapture.append(chunk); }); + const onEngineExit = createEngineExitHandler( + bin, isDocker, () => stderrCapture.text(), spawnedAt, + ); child.on("exit", (code, signal) => { - const abnormal = - (code !== null && code !== 0) || (code === null && signal !== null); - if (abnormal) { - const stderr = activeStartupStderr.text(); - startupFailure = { - kind: isDocker ? "docker-crashed" : "engine-crashed", - stderr: - stderr.trim() || - (signal - ? `process killed by signal ${signal}` - : `process exited with code ${code}`), - binary: bin, - }; - vlog(`engine exited early: code=${code} signal=${signal}`); - if (IS_VERBOSE && stderr.trim()) { - p.log.error(`engine stderr:\n${stderr}`); - } - if (!isDocker) { - clearEnginePidfile(); - clearEngineState(); - } + if (isDocker && docker && code === 0) { + stderrCapture = activeStartupStderr = createStartupStderrCapture(); + startDockerEngineWatch(bin, { ...docker, cwd }, spawnedAt, () => stderrCapture.text()); + return; } + onEngineExit(code, signal); + }); + child.on("error", (error) => { + stderrCapture.append(Buffer.from(error.message)); + onEngineExit(1, null); }); child.unref(); return child; } +// Engine deaths inside this window are treated as startup failures, which the +// startup path reports with a better message. Later deaths mean a running +// deployment lost its engine, which is fatal to serving. +// +// Keep this SHORT. It is a hole in the only mechanism that catches engine death: +// an engine that dies inside the window after startup already completed leaves +// the container up and serving nothing, which is the original bug. A failed +// spawn surfaces within a second or two, so a few seconds is all the startup +// path needs. +const ENGINE_STARTUP_GRACE_MS = 5_000; + function prepareEngineLaunch(configPath: string): { configPath: string; cwd: string; @@ -1624,14 +1734,15 @@ async function startEngine(): Promise { return false; } persistDockerInspection(persistedState, inspection); - if (inspection.status === "running") return true; const dockerBin = whichBinary("docker"); - return Boolean( - dockerBin && - runCommand(dockerBin, ["start", inspection.containerId], { + if (!dockerBin) return false; + if (inspection.status === "stopped" && !runCommand( + dockerBin, ["start", inspection.containerId], { label: `Restarting owned Docker container ${inspection.containerId}`, - }), - ); + }, + )) return false; + startDockerEngineWatch(dockerBin, { containerId: inspection.containerId }); + return true; } const configPath = findIiiConfig(); warnIfRelocatedDataDir(); @@ -1776,6 +1887,8 @@ async function startEngine(): Promise { dockerBin, dockerComposeArgs(composeFile, projectName, ["up", "-d"]), "iii-engine via Docker", + process.cwd(), + { composeFile, projectName }, ); s.stop("Docker compose started"); return true; @@ -1815,18 +1928,19 @@ async function reconcilePersistedDockerEngine(): Promise { process.exit(1); } persistDockerInspection(state, inspection); - if (inspection.status === "stopped") { - const dockerBin = whichBinary("docker"); - if ( - !dockerBin || - !runCommand(dockerBin, ["start", inspection.containerId], { + const dockerBin = whichBinary("docker"); + if ( + !dockerBin || + (inspection.status === "stopped" && !runCommand( + dockerBin, ["start", inspection.containerId], { label: `Restarting owned Docker container ${inspection.containerId}`, - }) - ) { - p.log.error("The owned Docker container could not be restarted; lifecycle state was preserved."); - process.exit(1); - } + }, + )) + ) { + p.log.error("The owned Docker container could not be restarted; lifecycle state was preserved."); + process.exit(1); } + startDockerEngineWatch(dockerBin, { containerId: inspection.containerId }); if (!(await waitForEngine(5000))) { printDockerStartupLogs(); p.log.error( @@ -3423,6 +3537,7 @@ async function stopDockerEngine( process.exit(1); } clearWorkerPidfile(); + stopDockerEngineWatch?.(); if (inspection.status === "running" || inspection.status === "stopped") { if ( diff --git a/src/cli/docker-engine.ts b/src/cli/docker-engine.ts new file mode 100644 index 0000000000..35a39b9fe7 --- /dev/null +++ b/src/cli/docker-engine.ts @@ -0,0 +1,82 @@ +import { execFile, spawn, type ChildProcess } from "node:child_process"; +import { dockerComposeArgs } from "./engine-launch.js"; + +export type DockerEngineTarget = + | { containerId: string } + | { composeFile: string; projectName: string; cwd?: string }; + +export function watchDockerEngine( + dockerBin: string, + options: DockerEngineTarget, + onExit: (code: number) => void, + onError: (error: Error) => void, +): () => void { + let child: ChildProcess | undefined; + let stopped = false; + const cancel = () => { + if (stopped) return; + stopped = true; + process.removeListener("exit", cancel); + child?.kill(); + }; + const fail = (error: Error) => { + if (stopped) return; + stopped = true; + process.removeListener("exit", cancel); + onError(error); + }; + const observe = (containerId: string) => { + if (stopped) return; + if (!/^[a-f0-9]{12,64}$/.test(containerId)) { + fail(new Error("Expected exactly one iii-engine container ID")); + return; + } + try { + const observer = spawn(dockerBin, ["wait", containerId], { + stdio: ["ignore", "pipe", "ignore"], + windowsHide: true, + }); + child = observer; + let output = ""; + observer.stdout.on("data", (chunk: Buffer) => { + output = (output + chunk.toString("utf8")).slice(0, 64); + }); + observer.on("error", fail); + observer.on("close", (code, signal) => { + if (stopped) return; + const containerCode = output.trim(); + if (code !== 0 || !/^\d{1,3}$/.test(containerCode) || Number(containerCode) > 255) { + fail(new Error(`docker wait failed (code=${code} signal=${signal})`)); + return; + } + stopped = true; + process.removeListener("exit", cancel); + onExit(Number(containerCode)); + }); + observer.unref(); + (observer.stdout as typeof observer.stdout & { unref?: () => void }).unref?.(); + } catch (error) { + fail(error instanceof Error ? error : new Error(String(error))); + } + }; + process.once("exit", cancel); + try { + if ("containerId" in options) { + observe(options.containerId); + } else { + child = execFile( + dockerBin, + dockerComposeArgs(options.composeFile, options.projectName, ["ps", "--all", "--quiet", "iii-engine"]), + { encoding: "utf8", timeout: 5_000, maxBuffer: 1024, windowsHide: true, cwd: options.cwd }, + (error, stdout) => { + if (stopped) return; + if (error) { fail(error); return; } + observe(stdout.trim()); + }, + ); + } + } catch (error) { + fail(error instanceof Error ? error : new Error(String(error))); + } + return cancel; +} diff --git a/src/cli/engine-log.ts b/src/cli/engine-log.ts new file mode 100644 index 0000000000..773d3726df --- /dev/null +++ b/src/cli/engine-log.ts @@ -0,0 +1,112 @@ +import { StringDecoder } from "node:string_decoder"; + +export const ENGINE_LOG_MAX_TOTAL_BYTES = 32 * 1024 * 1024; +export const ENGINE_LOG_MAX_BYTES_PER_SECOND = 64 * 1024; +export const ENGINE_LOG_MAX_LINE_BYTES = 8 * 1024; +export const ENGINE_LOG_WINDOW_MS = 1000; + +export interface EngineLogForwarder { + push(chunk: Buffer): void; + flush(): void; +} + +export interface EngineLogForwarderOptions { + prefix: string; + write: (line: string) => void; + maxTotalBytes?: number; + maxBytesPerSecond?: number; + maxLineBytes?: number; + windowMs?: number; + now?: () => number; +} + +export function createEngineLogForwarder( + options: EngineLogForwarderOptions, +): EngineLogForwarder { + const { prefix, write } = options; + const maxTotalBytes = options.maxTotalBytes ?? ENGINE_LOG_MAX_TOTAL_BYTES; + const maxBytesPerSecond = + options.maxBytesPerSecond ?? ENGINE_LOG_MAX_BYTES_PER_SECOND; + const maxLineBytes = options.maxLineBytes ?? ENGINE_LOG_MAX_LINE_BYTES; + const windowMs = options.windowMs ?? ENGINE_LOG_WINDOW_MS; + const now = options.now ?? Date.now; + + const decoder = new StringDecoder("utf8"); + let pending = ""; + let totalBytes = 0; + let windowStart = now(); + let windowBytes = 0; + let droppedBytes = 0; + let stopped = false; + + function emitLine(line: string, exemptFromRateCap = false): void { + if (stopped) return; + const cost = Buffer.byteLength(line) + 1; + if (totalBytes + cost > maxTotalBytes) { + stopped = true; + write( + `${prefix} log forwarding stopped: ${maxTotalBytes} byte ceiling reached`, + ); + return; + } + if (!exemptFromRateCap && windowBytes + cost > maxBytesPerSecond) { + droppedBytes += cost; + return; + } + totalBytes += cost; + windowBytes += cost; + write(line); + } + + // The suppression notice is exempt from the rate cap so silent loss is + // impossible: it is one short line per window, still charged against + // both counters, so the lifetime ceiling still bounds it. + function reportDropped(): void { + if (droppedBytes === 0) return; + const dropped = droppedBytes; + droppedBytes = 0; + emitLine( + `${prefix} dropped ${dropped} bytes (rate cap ${maxBytesPerSecond} bytes/s)`, + true, + ); + } + + function rollWindow(): void { + if (now() - windowStart < windowMs) return; + windowStart = now(); + windowBytes = 0; + reportDropped(); + } + + function emitRecord(record: string): void { + emitLine(`${prefix} ${record.endsWith("\r") ? record.slice(0, -1) : record}`); + } + + return { + push(chunk: Buffer): void { + if (stopped) return; + rollWindow(); + pending += decoder.write(chunk); + let newline = pending.indexOf("\n"); + while (newline !== -1) { + emitRecord(pending.slice(0, newline)); + pending = pending.slice(newline + 1); + newline = pending.indexOf("\n"); + } + if (Buffer.byteLength(pending, "utf8") >= maxLineBytes) { + emitRecord(pending); + pending = ""; + } + }, + flush(): void { + if (stopped) return; + rollWindow(); + pending += decoder.end(); + if (pending.length > 0) { + emitRecord(pending); + pending = ""; + } + reportDropped(); + }, + }; +} diff --git a/src/health/memory.ts b/src/health/memory.ts new file mode 100644 index 0000000000..83034a3ef6 --- /dev/null +++ b/src/health/memory.ts @@ -0,0 +1,204 @@ +import { readFile } from "node:fs/promises"; +import { posix } from "node:path"; +import type { CgroupMemory, HealthSnapshot, MemoryEvaluation } from "../types.js"; + +export interface MemoryConfig { + memoryWarnPercent: number; + memoryCriticalPercent: number; + memoryRssBudgetBytes?: number; + memoryHoldSamples: number; +} + +export const MEMORY_DEFAULTS: MemoryConfig = { + memoryWarnPercent: 80, + memoryCriticalPercent: 95, + memoryHoldSamples: 2, +}; + +export function readMemoryConfig( + env: NodeJS.ProcessEnv = process.env, + overrides: Partial = {}, + warn: (message: string) => void = console.warn, +): MemoryConfig { + const number = (key: string, fallback: number) => + env[key] === undefined ? fallback : Number(env[key]); + const config: MemoryConfig = { + memoryWarnPercent: number("AGENTMEMORY_HEALTH_MEM_WARN_PCT", MEMORY_DEFAULTS.memoryWarnPercent), + memoryCriticalPercent: number("AGENTMEMORY_HEALTH_MEM_CRITICAL_PCT", MEMORY_DEFAULTS.memoryCriticalPercent), + memoryHoldSamples: number("AGENTMEMORY_HEALTH_MEM_HOLD_SAMPLES", MEMORY_DEFAULTS.memoryHoldSamples), + ...(env.AGENTMEMORY_HEALTH_MEM_RSS_BUDGET_MB !== undefined + ? { memoryRssBudgetBytes: Number(env.AGENTMEMORY_HEALTH_MEM_RSS_BUDGET_MB) * 1024 * 1024 } + : {}), + ...overrides, + }; + if (!(config.memoryWarnPercent > 0 && + config.memoryWarnPercent < config.memoryCriticalPercent && + config.memoryCriticalPercent <= 100)) { + warn("[agentmemory] invalid health memory thresholds: require 0 < warn < critical <= 100; using 80/95"); + config.memoryWarnPercent = MEMORY_DEFAULTS.memoryWarnPercent; + config.memoryCriticalPercent = MEMORY_DEFAULTS.memoryCriticalPercent; + } + if (!Number.isSafeInteger(config.memoryHoldSamples) || config.memoryHoldSamples < 1) { + warn("[agentmemory] invalid health memory hold count: using 2 samples"); + config.memoryHoldSamples = MEMORY_DEFAULTS.memoryHoldSamples; + } + if (config.memoryRssBudgetBytes !== undefined && + !(Number.isFinite(config.memoryRssBudgetBytes) && + config.memoryRssBudgetBytes > 0 && + config.memoryRssBudgetBytes <= Number.MAX_SAFE_INTEGER)) { + warn("[agentmemory] invalid health memory RSS budget: leaving budget unconfigured"); + delete config.memoryRssBudgetBytes; + } + return config; +} + +function parseBytes(raw: string, limit: boolean): number | undefined { + const value = raw.trim(); + if (!/^\d+$/.test(value)) return undefined; + const bytes = Number(value); + return Number.isSafeInteger(bytes) && (limit ? bytes > 0 : bytes >= 0) ? bytes : undefined; +} + +export async function collectCgroupMemory( + read: (path: string) => Promise = path => readFile(path, "utf8"), + platform: string = process.platform, +): Promise { + if (platform !== "linux") return { status: "unsupported", levels: [] }; + try { + const [cgroup, mountinfo] = await Promise.all([ + read("/proc/self/cgroup"), read("/proc/self/mountinfo"), + ]); + const group = cgroup.split("\n").find(line => line.startsWith("0::"))?.slice(3); + const decode = (path: string) => path.replace(/\\([0-7]{3})/g, (_, octal: string) => String.fromCharCode(parseInt(octal, 8))); + const mounts = mountinfo.split("\n").filter(line => line.split(" - ")[1]?.split(" ")[0] === "cgroup2").map(line => { + const fields = line.split(" - ")[0].split(" "); + return { root: decode(fields[3] ?? ""), mount: decode(fields[4] ?? "") }; + }).filter(({ root, mount }) => root.startsWith("/") && mount.startsWith("/")); + if (!group?.startsWith("/") || group.split("/").includes("..") || !mounts.length) { + return { status: "unavailable", levels: [] }; + } + const selected = mounts.sort((a, b) => a.root.length - b.root.length).find(({ root }) => + root === group || group.startsWith(root === "/" ? "/" : root + "/"), + ); + if (!selected) return { status: "unavailable", levels: [] }; + const relative = selected.root !== "/" && + (group === selected.root || group.startsWith(selected.root + "/")) + ? group.slice(selected.root.length) || "/" : group; + const boundary = posix.normalize(selected.mount); + let current = posix.resolve(boundary, "." + relative); + const paths: string[] = []; + while (true) { + paths.push(current); + if (current === boundary) break; + current = posix.dirname(current); + } + let partial = false; + const levels = await Promise.all(paths.map(async path => { + const level: CgroupMemory["levels"][number] = { path: "/" + posix.relative(boundary, path) }; + await Promise.all((["current", "max", "high"] as const).map(async key => { + try { + const raw = await read(posix.join(path, `memory.${key}`)); + if (key !== "current" && raw.trim() === "max") return; + const value = parseBytes(raw, key !== "current"); + if (value === undefined) partial = true; + else level[key] = value; + } catch { partial = true; } + })); + return level; + })); + return { status: partial ? "partial" : "available", levels }; + } catch { return { status: "unavailable", levels: [] }; } +} + +export function evaluateMemory( + memory: HealthSnapshot["memory"], + config: Pick, +): MemoryEvaluation[] { + const evaluations: MemoryEvaluation[] = []; + const add = ( + source: MemoryEvaluation["source"], usedBytes?: number, limitBytes?: number, path?: string, + ) => { + const validUsage = Number.isSafeInteger(usedBytes) && usedBytes! >= 0; + const validLimit = Number.isFinite(limitBytes) && limitBytes! > 0 && limitBytes! <= Number.MAX_SAFE_INTEGER; + const percent = validUsage && validLimit ? usedBytes! / limitBytes! * 100 : undefined; + const severity = percent === undefined ? "healthy" + : source !== "cgroup-high" && percent > config.memoryCriticalPercent ? "critical" + : percent > config.memoryWarnPercent ? "degraded" : "healthy"; + evaluations.push({ + source, + ...(path !== undefined ? { path } : {}), + ...(validUsage ? { usedBytes } : {}), + ...(validLimit ? { limitBytes } : {}), + ...(percent !== undefined ? { percent } : {}), + available: percent !== undefined, + severity, + }); + }; + add("heap", memory.heapUsed, memory.heapSizeLimit); + if (config.memoryRssBudgetBytes !== undefined) add("rss", memory.rss, config.memoryRssBudgetBytes); + for (const level of memory.cgroup?.levels ?? []) { + add("cgroup-max", level.current, level.max, level.path); + add("cgroup-high", level.current, level.high, level.path); + } + return evaluations; +} + +export interface MemoryHoldState { + entry: number; + clear: number; + latched: boolean; + source: MemoryEvaluation["source"]; + path?: string; +} + +export function holdMemoryCritical( + evaluations: MemoryEvaluation[], + states: Map, + requiredSamples: number, +): MemoryEvaluation[] { + const key = (evaluation: MemoryEvaluation) => `${evaluation.source}:${evaluation.path ?? ""}`; + const result = evaluations.map(evaluation => ({ ...evaluation })); + const present = new Set(result.map(key)); + for (const [id, state] of states) { + if (!present.has(id)) result.push({ + source: state.source, path: state.path, + available: false, severity: "healthy", + }); + } + for (const evaluation of result) { + if (evaluation.source === "cgroup-high") continue; + const id = key(evaluation); + const state = states.get(id) ?? { + entry: 0, clear: 0, latched: false, source: evaluation.source, path: evaluation.path, + }; + if (!evaluation.available) { + state.entry = state.clear = 0; + if (state.latched) { + evaluation.severity = "critical"; + evaluation.transition = "unavailable"; + } + } else if (evaluation.severity === "critical") { + state.clear = 0; + state.entry = Math.min(state.entry + 1, requiredSamples); + if (state.entry >= requiredSamples) state.latched = true; + if (!state.latched) { + evaluation.severity = "degraded"; + evaluation.transition = "entering"; + evaluation.samples = state.entry; + } + } else { + state.entry = 0; + state.clear = Math.min(state.clear + 1, requiredSamples); + if (state.clear >= requiredSamples) state.latched = false; + if (state.latched) { + evaluation.severity = "critical"; + evaluation.transition = "recovering"; + evaluation.samples = state.clear; + } + } + if (evaluation.transition) evaluation.requiredSamples = requiredSamples; + if (state.latched || state.entry > 0) states.set(id, state); + else states.delete(id); + } + return result; +} diff --git a/src/health/monitor.ts b/src/health/monitor.ts index 953c94a0f4..4a8d0cd420 100644 --- a/src/health/monitor.ts +++ b/src/health/monitor.ts @@ -1,13 +1,115 @@ +import { getHeapStatistics } from "node:v8"; import type { ISdk } from "iii-sdk"; import type { HealthSnapshot } from "../types.js"; import type { StateKV } from "../state/kv.js"; import { KV } from "../state/schema.js"; import { evaluateHealth } from "./thresholds.js"; +import { + collectCgroupMemory, evaluateMemory, holdMemoryCritical, readMemoryConfig, + type MemoryConfig, type MemoryHoldState, +} from "./memory.js"; + +export interface EscalationState { + consecutiveKvProbeFailures: number; + hasSeenHealthyKvProbe: boolean; + escalated: boolean; +} + +/** + * Advance the stall counter and report whether the process should exit. + * + * Gated on the KV probe specifically, NOT on `snapshot.status`. `evaluateHealth` + * raises `critical` from five independent conditions (connection, KV, event-loop + * lag, CPU, memory), so gating on the aggregate would let a CPU spike during a + * consolidation pass kill a process that is not wedged at all. + * + * `hasSeenHealthyKvProbe` mirrors the shell watchdog's "wait for a first success" rule. Without + * it, a store that is already stalled at boot escalates on the first minute of + * every container life, which spends the platform's restart budget in minutes + * and ends at a stopped deployment. + * + * Call this BEFORE persisting the snapshot. The persist is not raced against any + * timeout, and a stalled store parks it for the engine's full invocation timeout, + * so a counter behind it would never advance during the failure it exists to + * catch. + */ +export function bumpEscalation( + snapshot: HealthSnapshot, + state: EscalationState, + threshold: number, +): boolean { + const status = snapshot.kvConnectivity?.status; + if (status === "ok") { + state.hasSeenHealthyKvProbe = true; + state.consecutiveKvProbeFailures = 0; + return false; + } + if (status !== "error") return false; + if (!state.hasSeenHealthyKvProbe) return false; + state.consecutiveKvProbeFailures += 1; + if (state.escalated || state.consecutiveKvProbeFailures < threshold) return false; + state.escalated = true; + return true; +} export function registerHealthMonitor( sdk: ISdk, kv: StateKV, + config: Partial = {}, ): { stop: () => void } { + const memoryConfig = readMemoryConfig(process.env, config); + const memoryStates = new Map(); + let sequence = 0; + let acceptedSequence = 0; + let stopped = false; + let pendingSnapshot: HealthSnapshot | undefined; + let persisting = false; + + async function persistLatest(snapshot: HealthSnapshot): Promise { + pendingSnapshot = snapshot; + if (persisting) return; + persisting = true; + try { + while (pendingSnapshot && !stopped) { + const next = pendingSnapshot; + pendingSnapshot = undefined; + await kv.set(KV.health, "latest", next).catch(() => {}); + } + } finally { persisting = false; } + } + + const escalationState: EscalationState = { + consecutiveKvProbeFailures: 0, + hasSeenHealthyKvProbe: false, + escalated: false, + }; + // Default off. Enabling this arms an automatic process-killer, so it stays + // opt-in until external uptime monitoring exists to make a restart loop + // visible. The threshold spans 5 minutes at the 30s collection interval, + // deliberately wider than the write bursts this codebase already documents + // as able to exceed the engine's own 30s timeout (see src/index.ts). + const escalateEnabled = process.env["AGENTMEMORY_HEALTH_ESCALATE"] === "1" || + process.env["AGENTMEMORY_HEALTH_ESCALATE"]?.toLowerCase() === "true"; + const escalateAfter = 10; + + function escalate(alerts: string[]): void { + console.error( + `[agentmemory] health escalation: KV unreachable for ${escalateAfter} consecutive checks (${alerts.join(", ")}); exiting so the platform restarts`, + ); + // Prefer the registered SIGTERM shutdown so the index flushes. That handler + // ends in process.exit(0), so railway.json uses restartPolicyType ALWAYS — + // ON_FAILURE would read a graceful exit as success and never restart. + process.kill(process.pid, "SIGTERM"); + // NOT unref'd. If shutdown parks on the same stalled store, this is the only + // thing that still reaches an exit. + setTimeout(() => { + console.error( + "[agentmemory] health escalation: graceful shutdown did not finish; forcing exit", + ); + process.exit(1); + }, 15_000); + } + let connectionState = "connected"; let prevCpuUsage = process.cpuUsage(); let prevCpuTime = Date.now(); @@ -18,7 +120,8 @@ export function registerHealthMonitor( }); } - async function collectHealth(): Promise { + async function collectHealth(): Promise { + const sampleSequence = ++sequence; const mem = process.memoryUsage(); const currentCpu = process.cpuUsage(); const now = Date.now(); @@ -32,16 +135,27 @@ export function registerHealthMonitor( prevCpuUsage = currentCpu; prevCpuTime = now; + const cgroup = await collectCgroupMemory(); const startMark = performance.now(); await new Promise((resolve) => setImmediate(resolve)); const eventLoopLagMs = performance.now() - startMark; let workers: HealthSnapshot["workers"] = []; + // Raced, like the KV probe below. Unraced, this inherits the engine's + // invocation timeout (180s), so a dead engine parks the whole collection + // here -- upstream of the probe and of the escalation decision -- and + // detection slips from minutes to tens of minutes. + const WORKERS_PROBE_TIMEOUT = 5000; try { - const result = await sdk.trigger< - unknown, - { workers?: HealthSnapshot["workers"] } - >({ function_id: "engine::workers::list", payload: {} }); + const result = await Promise.race([ + sdk.trigger({ + function_id: "engine::workers::list", + payload: {}, + }), + new Promise((_, reject) => + setTimeout(() => reject(new Error("timeout")), WORKERS_PROBE_TIMEOUT), + ), + ]); if (result?.workers) workers = result.workers; } catch {} @@ -71,6 +185,8 @@ export function registerHealthMonitor( heapTotal: mem.heapTotal, rss: mem.rss, external: mem.external, + heapSizeLimit: getHeapStatistics().heap_size_limit, + cgroup, }, cpu: { userMicros: currentCpu.user, @@ -84,13 +200,23 @@ export function registerHealthMonitor( alerts: [], }; - const evaluated = evaluateHealth(snapshot); + if (stopped || sampleSequence <= acceptedSequence) return; + acceptedSequence = sampleSequence; + const memoryEvaluations = holdMemoryCritical( + evaluateMemory(snapshot.memory, memoryConfig), memoryStates, memoryConfig.memoryHoldSamples, + ); + snapshot.memory.evaluations = memoryEvaluations; + const evaluated = evaluateHealth(snapshot, memoryConfig, memoryEvaluations); snapshot.status = evaluated.status; snapshot.alerts = evaluated.alerts; snapshot.notes = evaluated.notes; - await kv.set(KV.health, "latest", snapshot).catch(() => {}); - return snapshot; + // Decide before the persist below, which races no timeout. + if (escalateEnabled && bumpEscalation(snapshot, escalationState, escalateAfter)) { + escalate(snapshot.alerts); + } + + await persistLatest(snapshot); } collectHealth().catch(() => {}); @@ -100,7 +226,7 @@ export function registerHealthMonitor( interval.unref(); return { - stop: () => clearInterval(interval), + stop: () => { stopped = true; clearInterval(interval); }, }; } diff --git a/src/health/thresholds.ts b/src/health/thresholds.ts index 7279afad16..0e2b0d2e6a 100644 --- a/src/health/thresholds.ts +++ b/src/health/thresholds.ts @@ -1,13 +1,12 @@ +import { evaluateMemory, MEMORY_DEFAULTS } from "./memory.js"; +import type { MemoryConfig } from "./memory.js"; import type { HealthSnapshot } from "../types.js"; -interface ThresholdConfig { +interface ThresholdConfig extends Omit { eventLoopLagWarnMs: number; eventLoopLagCriticalMs: number; cpuWarnPercent: number; cpuCriticalPercent: number; - memoryWarnPercent: number; - memoryCriticalPercent: number; - memoryRssFloorBytes: number; } const DEFAULTS: ThresholdConfig = { @@ -15,14 +14,13 @@ const DEFAULTS: ThresholdConfig = { eventLoopLagCriticalMs: 500, cpuWarnPercent: 80, cpuCriticalPercent: 90, - memoryWarnPercent: 80, - memoryCriticalPercent: 95, - memoryRssFloorBytes: 512 * 1024 * 1024, + ...MEMORY_DEFAULTS, }; export function evaluateHealth( snapshot: HealthSnapshot, config: Partial = {}, + memoryEvaluations = evaluateMemory(snapshot.memory, { ...DEFAULTS, ...config }), ): { status: "healthy" | "degraded" | "critical"; alerts: string[]; notes: string[] } { const cfg = { ...DEFAULTS, ...config }; const alerts: string[] = []; @@ -30,6 +28,12 @@ export function evaluateHealth( let critical = false; let degraded = false; + // NOTE: unreachable in production today. iii-sdk's setConnectionState only + // assigns a private field and emits nothing, so the "connection_state" + // listener in monitor.ts never fires and connectionState stays "connected" + // for the life of the process. Kept because the field is part of the + // snapshot contract and a future SDK may emit it; do not rely on it as a + // liveness signal until it does. if ( snapshot.connectionState === "disconnected" || snapshot.connectionState === "failed" @@ -41,6 +45,18 @@ export function evaluateHealth( degraded = true; } + // The KV probe in collectHealth is the only check that exercises the state + // store end to end (set then get, raced against a 5s timeout). A store that + // stops answering takes the HTTP workers down with it, so a failed probe is + // the earliest reliable signal of that failure and belongs at critical. + // kvConnectivity is optional on the snapshot, and older persisted snapshots + // predate it, so an absent or malformed value must read as "no signal" + // rather than as a failure. + if (snapshot.kvConnectivity?.status === "error") { + alerts.push("kv_probe_failed"); + critical = true; + } + if (snapshot.eventLoopLagMs > cfg.eventLoopLagCriticalMs) { alerts.push( `event_loop_lag_critical_${Math.round(snapshot.eventLoopLagMs)}ms`, @@ -59,21 +75,15 @@ export function evaluateHealth( degraded = true; } - const memPercent = - snapshot.memory.heapTotal > 0 - ? (snapshot.memory.heapUsed / snapshot.memory.heapTotal) * 100 - : 0; - const rss = snapshot.memory.rss ?? 0; - const rssAboveFloor = rss >= cfg.memoryRssFloorBytes; - const memMb = Math.round(rss / (1024 * 1024)); - if (memPercent > cfg.memoryCriticalPercent && rssAboveFloor) { - alerts.push(`memory_critical_${Math.round(memPercent)}%_rss${memMb}mb`); - critical = true; - } else if (memPercent > cfg.memoryWarnPercent && rssAboveFloor) { - alerts.push(`memory_warn_${Math.round(memPercent)}%_rss${memMb}mb`); - degraded = true; - } else if (memPercent > cfg.memoryWarnPercent) { - notes.push(`memory_heap_tight_${Math.round(memPercent)}%_rss${memMb}mb`); + for (const evaluation of memoryEvaluations) { + const signal = `${evaluation.source}${evaluation.path ?? ""}`; + if (!evaluation.available) notes.push(`memory_unavailable_${signal}`); + if (evaluation.transition) notes.push(`memory_${evaluation.transition}_${signal}`); + if (evaluation.severity === "critical") critical = true; + if (evaluation.severity === "degraded") degraded = true; + if (evaluation.severity !== "healthy") { + alerts.push(`memory_${evaluation.severity === "critical" ? "critical" : "warn"}_${signal}_${evaluation.percent === undefined ? "unavailable" : `${Math.round(evaluation.percent)}%`}`); + } } const status = critical ? "critical" : degraded ? "degraded" : "healthy"; diff --git a/src/mcp/server.ts b/src/mcp/server.ts index ef26427aaf..88142ea5d5 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -5,8 +5,7 @@ import type { SessionSummary, Memory, Session, - GraphNode, - GraphEdge, + GraphSnapshot, } from "../types.js"; import { getVisibleTools } from "./tools-registry.js"; import { timingSafeCompare } from "../auth.js"; @@ -1479,14 +1478,11 @@ export function registerMcpEndpoints( if (uri === "agentmemory://graph/stats") { try { - const nodes = await kv.list(KV.graphNodes); - const edges = await kv.list(KV.graphEdges); - const nodesByType: Record = {}; - for (const n of nodes) - nodesByType[n.type] = (nodesByType[n.type] || 0) + 1; - const edgesByType: Record = {}; - for (const e of edges) - edgesByType[e.type] = (edgesByType[e.type] || 0) + 1; + const snapshot = await kv.get( + KV.graphSnapshot, + "current", + ); + const stats = snapshot?.stats; return { status_code: 200, body: { @@ -1495,10 +1491,11 @@ export function registerMcpEndpoints( uri, mimeType: "application/json", text: JSON.stringify({ - totalNodes: nodes.length, - totalEdges: edges.length, - nodesByType, - edgesByType, + totalNodes: stats?.totalNodes ?? 0, + totalEdges: stats?.totalEdges ?? 0, + nodesByType: stats?.nodesByType ?? {}, + edgesByType: stats?.edgesByType ?? {}, + ...(snapshot ? {} : { pending: true }), }), }, ], diff --git a/src/providers/resilient.ts b/src/providers/resilient.ts index 95ece40c96..bad018a464 100644 --- a/src/providers/resilient.ts +++ b/src/providers/resilient.ts @@ -1,25 +1,93 @@ import type { MemoryProvider, CircuitBreakerState } from "../types.js"; import { CircuitBreaker } from "./circuit-breaker.js"; +import { getEnvVar } from "../config.js"; + +const DEFAULT_MAX_CONCURRENT = 4; + +/** + * Bounds how many calls are inside the provider at once. + * + * release() hands its slot directly to the next waiter rather than + * decrementing and letting waiters race for the opening, so the limit holds + * under a burst. + */ +class Semaphore { + private active = 0; + private waiting: Array<() => void> = []; + + constructor(private readonly limit: number) {} + + async acquire(): Promise { + if (this.active < this.limit) { + this.active++; + return; + } + await new Promise((resolve) => this.waiting.push(resolve)); + } + + release(): void { + const next = this.waiting.shift(); + if (next) next(); + else this.active--; + } +} + +// 429 is backpressure, not a fault. Counting it opened the breaker on a healthy +// provider and failed every compression for the recovery window. +function isRateLimited(err: unknown): boolean { + const message = err instanceof Error ? err.message : String(err); + return /\b429\b|rate.?limit|too many (concurrent )?requests/i.test(message); +} + +// Option is the test seam, env is the operator knob on Railway, constant is the +// default. getEnvVar rather than process.env so ~/.agentmemory/.env is honoured, +// which is how every sibling provider reads config. +function resolveMaxConcurrent(configured: number | undefined): number { + const candidate = + configured ?? + Number(getEnvVar("AGENTMEMORY_MAX_PROVIDER_CONCURRENCY") ?? NaN); + if (!Number.isFinite(candidate)) return DEFAULT_MAX_CONCURRENT; + const whole = Math.floor(candidate); + return whole >= 1 ? whole : DEFAULT_MAX_CONCURRENT; +} export class ResilientProvider implements MemoryProvider { - private breaker = new CircuitBreaker(); + private breaker: CircuitBreaker; + private gate: Semaphore; name: string; - constructor(private inner: MemoryProvider) { + constructor( + private inner: MemoryProvider, + options: { maxConcurrent?: number } = {}, + ) { + this.breaker = new CircuitBreaker(); + this.gate = new Semaphore(resolveMaxConcurrent(options.maxConcurrent)); this.name = `resilient(${inner.name})`; } private async call(fn: () => Promise): Promise { + // Checked before queueing. An open breaker should fail fast rather than + // occupy a slot that a call with a chance of succeeding could use. if (!this.breaker.isAllowed) { throw new Error("circuit_breaker_open"); } + await this.gate.acquire(); try { - const result = await fn(); - this.breaker.recordSuccess(); - return result; - } catch (err) { - this.breaker.recordFailure(); - throw err; + if (!this.breaker.isAllowed) { + throw new Error("circuit_breaker_open"); + } + try { + const result = await fn(); + this.breaker.recordSuccess(); + return result; + } catch (err) { + if (!isRateLimited(err)) this.breaker.recordFailure(); + throw err; + } + } finally { + // In `finally` so a throw cannot leak the slot and deadlock every call + // queued behind it. + this.gate.release(); } } diff --git a/src/state/index-persistence.ts b/src/state/index-persistence.ts index 6df0e2fdaf..3fedd1a91e 100644 --- a/src/state/index-persistence.ts +++ b/src/state/index-persistence.ts @@ -24,6 +24,27 @@ type IndexShardManifest = { chars: number; }; +// Suffix for the reclaim ledger that sits beside each manifest. +const GC_LEDGER_SUFFIX = ":gc"; + +// Stands in for a manifest written before generations were recorded, whose +// `generation` field is absent. Cannot collide with createIndexGeneration(). +const PRE_LEDGER_GENERATION = "pre-ledger"; + +// Every generation whose shards may still be on disk, live one included. The +// manifest alone cannot answer that: it names only the generation that is +// current, so a generation stranded by a failed read, a throw after commit, or +// a kill mid-cleanup becomes unreachable the moment the next manifest replaces +// it. Nothing else enumerates shard scopes — StateKV lists keys within a scope, +// not scopes by prefix — so what is not recorded here can never be found again. +type IndexGcLedger = { + v: 1; + generations: Array<{ + generation: string; + shards: Array<{ scope: string; key: string }>; + }>; +}; + type IndexPersistenceOptions = { shardChars?: number; createGeneration?: () => string; @@ -67,7 +88,8 @@ function isValidShardDescriptor( export class IndexPersistence { private timer: ReturnType | null = null; - private lastFailureLogAt = 0; + private lastFailureLogAt = new Map(); + private queue: Promise = Promise.resolve(); constructor( private kv: StateKV, @@ -83,7 +105,7 @@ export class IndexPersistence { // under sustained iii-engine write timeouts (issue #204). Funnel // rejections through logFailure() instead. this.timer = setTimeout(() => { - this.save().catch((err) => this.logFailure(err)); + this.save().catch((err) => this.logFailure("index", err)); }, DEBOUNCE_MS); } @@ -92,13 +114,52 @@ export class IndexPersistence { clearTimeout(this.timer); this.timer = null; } + return this.enqueue(() => this.runSave()); + } + + /** + * Serialise everything that read-modify-writes the gc ledger. + * + * Two saves genuinely overlap here: scheduleSave fires save() unawaited from + * a timer, flushIndexSave awaits save() on every delete path + * (src/functions/search.ts), and stop() clears the timer without awaiting a + * save already running. Overlapping saves drop each other's ledger entries, + * and worse, a save that publishes second reclaims the shards the first is + * still writing — leaving the first to publish a manifest naming data that + * is already gone, which fails the next load closed. + * + * Queue, never coalesce. flushIndexSave awaits this to make a delete + * durable, so handing back an in-flight promise that started before the + * delete would report success for a snapshot that does not contain it. The + * cost is real: a delete-path flush waits out the save ahead of it. + */ + private enqueue(work: () => Promise): Promise { + // runSave never rejects and the tail below always fulfils, so the queue + // cannot be left rejected and needs no rejection handler here. + const run = this.queue.then(work); + this.queue = run.then( + () => undefined, + () => undefined, + ); + return run; + } + + private async runSave(): Promise { + // Each index fails on its own. One try around both would let a BM25 + // failure stop the vector index persisting at all, and a lost vector index + // is never rebuilt: both rebuild triggers key on the BM25 size + // (src/index.ts, src/functions/search.ts). try { await this.saveBm25Index(this.bm25.serialize()); - if (this.vector) { + } catch (err) { + this.logFailure("BM25", err); + } + if (this.vector) { + try { await this.saveVectorIndex(this.vector.serialize()); + } catch (err) { + this.logFailure("vector", err); } - } catch (err) { - this.logFailure(err); } } @@ -129,16 +190,19 @@ export class IndexPersistence { } } - private logFailure(err: unknown): void { + private logFailure(index: string, err: unknown): void { const now = Date.now(); // Throttle: persistence failures under load arrive in bursts // (iii-engine queue pressure). Logging every debounce flush adds - // noise without information. - if (now - this.lastFailureLogAt < FAILURE_LOG_THROTTLE_MS) return; - this.lastFailureLogAt = now; + // noise without information. Throttled PER INDEX, so a vector failure + // right after a BM25 one is not swallowed — the two fail independently + // now, and at 3am you need to know which one stopped persisting. + const lastAt = this.lastFailureLogAt.get(index) ?? 0; + if (now - lastAt < FAILURE_LOG_THROTTLE_MS) return; + this.lastFailureLogAt.set(index, now); const code = (err as { code?: string })?.code; const message = err instanceof Error ? err.message : String(err); - logger.warn("index persistence: failed to save BM25/vector index", { + logger.warn(`index persistence: failed to save ${index} index`, { code, message, hint: @@ -192,6 +256,16 @@ export class IndexPersistence { chunks.push(chunk); } + // Record the generation BEFORE the first shard write. A kill anywhere from + // here to the manifest publish would otherwise leave shards on disk that + // nothing references and nothing can enumerate. + const tracked = await this.trackGeneration( + manifestKey, + generation, + shards, + previous, + ); + const writeResults = await Promise.allSettled( shards.map(async (shard, index) => { const chunk = chunks[index] ?? ""; @@ -211,7 +285,14 @@ export class IndexPersistence { (result): result is PromiseRejectedResult => result.status === "rejected", ); if (failedWrite) { - await this.deleteShards(shards, "shard_write_rollback"); + const allGone = await this.deleteShards(shards, "shard_write_rollback"); + // Drop the entry only if every shard actually went. A shard that survived + // its delete is unnameable once its entry is gone, which is the contract + // this file states for reclaim: a delete failure costs a retry, never a + // stranded generation. + if (tracked && allGone) { + await this.untrackGeneration(manifestKey, generation); + } throw failedWrite.reason; } @@ -249,23 +330,277 @@ export class IndexPersistence { error: errorMessage(err), }); } else { - await this.deleteShards(shards, "manifest_publish_rollback"); + const allGone = await this.deleteShards( + shards, + "manifest_publish_rollback", + ); + if (tracked && allGone) { + await this.untrackGeneration(manifestKey, generation); + } } throw err; } await this.deleteKey(KV.bm25Index, legacyKey, "legacy_cleanup"); - if (previous?.v === 1 && Array.isArray(previous.shards)) { - const currentShardIds = new Set( - shards.map((shard) => `${shard.scope}\0${shard.key}`), + // Only reclaim when this generation is actually in the ledger. Reclaiming + // against a ledger that does not list us would treat live shards as dead. + if (tracked) { + await this.reclaimGenerations(manifestKey, generation); + } else if (previous?.v === 1 && Array.isArray(previous.shards)) { + // The ledger was unusable this cycle, so nothing above will ever revisit + // `previous`. Fall back to the pre-ledger cleanup: the manifest just + // published supersedes it, and saves are serialised, so its shards are + // dead. Without this, a run of unusable cycles orphans one generation + // each — strictly worse than the code this replaced, which always had + // this path. + const liveIds = new Set( + shards.map((shard) => `${shard.scope}\u0000${shard.key}`), + ); + await this.deleteShards( + previous.shards.filter( + (shard) => + isValidShardDescriptor(shard) && + !liveIds.has(`${shard.scope}\u0000${shard.key}`), + ), + "previous_generation_cleanup", + ); + } + } + + /** Drop a generation's entry after its shards have been rolled back. */ + private async untrackGeneration( + manifestKey: string, + generation: string, + ): Promise { + try { + const ledger = await this.readLedger(manifestKey); + const generations = ledger.generations.filter( + (entry) => entry?.generation !== generation, ); - for (const shard of previous.shards) { - if (currentShardIds.has(`${shard.scope}\0${shard.key}`)) continue; - await this.deleteShards([shard], "previous_generation_cleanup"); + if (generations.length === ledger.generations.length) return; + await this.kv.set(KV.bm25Index, this.gcKey(manifestKey), { + v: 1, + generations, + }); + } catch { + // Best effort. A surviving entry costs a retry on the next reclaim, and + // this runs while a save is already failing — never make that worse. + } + } + + private gcKey(manifestKey: string): string { + return `${manifestKey}${GC_LEDGER_SUFFIX}`; + } + + private async readLedger(manifestKey: string): Promise { + // Throws rather than returning a blank ledger, because a blank one would + // be written straight back over whatever is stored. Callers catch it and + // skip tracking for that cycle; they must never treat it as "no ledger". + const stored = await this.kv.get( + KV.bm25Index, + this.gcKey(manifestKey), + ); + if (stored == null) return { v: 1, generations: [] }; + // Present but unrecognised: a newer version, or a rollback to this build + // after one that wrote a different shape. Overwriting drops every + // generation it tracked, so leave it exactly where it is. + if (stored.v !== 1 || !Array.isArray(stored.generations)) { + throw new Error( + `index gc ledger ${this.gcKey(manifestKey)} has an unrecognised shape ` + + `(v=${String((stored as { v?: unknown }).v)}); refusing to overwrite it`, + ); + } + return stored; + } + + private async trackGeneration( + manifestKey: string, + generation: string, + shards: IndexShardManifest["shards"], + previous: IndexShardManifest | null, + ): Promise { + try { + // The WHOLE body runs under this guard, not just the ledger read. A + // malformed ledger entry or manifest shard would otherwise throw a + // TypeError out of saveShardedIndex before any shard write, leaving one + // throttled log line per 60s as the only trace while BM25 stopped + // persisting for good. isValidShardDescriptor exists in this file + // because a per-shard-malformed manifest is already its threat model. + return await this.recordGeneration( + manifestKey, + generation, + shards, + previous, + ); + } catch (err) { + // A state::get brownout is the exact condition this bug appears under, + // so this path is not rare. Aborting the save here would stop persisting + // the index at all, which is worse than the leak being fixed. Writing a + // fresh ledger would drop every generation the stored one lists. Do + // neither: let the shards and manifest land, skip this cycle's tracking + // and its reclaim, and leak at most one generation instead of one per + // brownout. + // + // That leak can OUTLIVE the unreadable window. This generation publishes + // untracked; the next save re-seeds it from `previous`, but that read is + // itself `.catch(() => null)`, so if it also fails the generation is in + // no ledger and reclaim only ever looks at what precedes the live entry. + // With no scope enumeration in StateKV, nothing can find it again. + // Throttled for the same reason every other failure log here is: an + // unusable ledger stays unusable, so this fires on every debounce. + const throttleKey = `gc:${manifestKey}`; + const now = Date.now(); + if (now - (this.lastFailureLogAt.get(throttleKey) ?? 0) >= FAILURE_LOG_THROTTLE_MS) { + this.lastFailureLogAt.set(throttleKey, now); + logger.warn( + "index persistence: gc ledger unavailable, skipping reclaim", + { manifestKey, message: errorMessage(err) }, + ); } + return false; } } + private async recordGeneration( + manifestKey: string, + generation: string, + shards: IndexShardManifest["shards"], + previous: IndexShardManifest | null, + ): Promise { + const ledger = await this.readLedger(manifestKey); + // Optional-chained so one malformed entry costs that entry, not the whole + // cycle's tracking. reclaimGenerations tolerates them the same way. + const known = new Set(ledger.generations.map((entry) => entry?.generation)); + + // Seed the generation the pre-ledger code left live, so upgrading does not + // strand it. Only reachable while `previous` is readable, which is exactly + // the case the old cleanup already handled. + // + // `generation` is optional on the manifest and older stores really do omit + // it, so fall back to a sentinel rather than skipping the seed. The + // sentinel cannot collide with createIndexGeneration()'s `idx_` ids, and + // since every manifest written from here on carries a generation, the + // seeded entry always ends up preceding a live one and gets reclaimed. + if (previous?.v === 1 && Array.isArray(previous.shards)) { + const previousGeneration = previous.generation ?? PRE_LEDGER_GENERATION; + if (!known.has(previousGeneration)) { + ledger.generations.push({ + generation: previousGeneration, + shards: previous.shards + .filter(isValidShardDescriptor) + .map(({ scope, key }) => ({ scope, key })), + }); + known.add(previousGeneration); + } + } + + if (!known.has(generation)) { + ledger.generations.push({ + generation, + shards: shards.map(({ scope, key }) => ({ scope, key })), + }); + } + + await this.kv.set( + KV.bm25Index, + this.gcKey(manifestKey), + ledger, + ); + return true; + } + + /** + * Delete every tracked generation except the live one, then rewrite the + * ledger with whatever survived. A shard whose delete failed stays listed and + * is retried on the next save or load, so a delete failure costs a retry + * instead of stranding the generation for good. + */ + private async reclaimGenerations( + manifestKey: string, + liveGeneration: string | undefined, + ): Promise { + if (!liveGeneration) return; + const ledger = await this.readLedger(manifestKey).catch(() => null); + if (!ledger) return; + + // Reclaim strictly what precedes the live generation in the ledger, which + // trackGeneration appends to in creation order. Anything at or after the + // live entry is either live or a save still in flight: setIndexPersistence + // runs before load() in src/index.ts, so a request arriving during boot can + // have a save writing shards while this reclaim runs, and deleting those + // would publish a manifest whose shards are already half gone. A generation + // stranded after the live one is not lost, only deferred — it becomes + // reclaimable as soon as a newer generation is published. + const liveIndex = ledger.generations.findIndex( + (entry) => entry?.generation === liveGeneration, + ); + // Live generation untracked (a pre-ledger store, or a manifest written + // before this shipped). Nothing can be classified as superseded, so leave + // every entry alone rather than guess. + if (liveIndex < 0) return; + + const reclaimedPaths: string[] = []; + let failed = 0; + const survivors: IndexGcLedger["generations"] = []; + for (const [index, entry] of ledger.generations.entries()) { + // A malformed entry is kept, never iterated. This also runs on the load + // path, where a throw would null the loaded index and trigger the + // full-corpus rebuild. A GC step must not be able to fail a load. + if (index >= liveIndex || !entry || !Array.isArray(entry.shards)) { + survivors.push(entry); + continue; + } + const stranded: IndexGcLedger["generations"][number]["shards"] = []; + for (let offset = 0; offset < entry.shards.length; offset += 8) { + const batch = entry.shards.slice(offset, offset + 8); + const deleted = await Promise.all(batch.map(async shard => { + try { + await this.kv.delete(shard.scope, shard.key); + return true; + } catch { return false; } + })); + for (const [index, shard] of batch.entries()) { + if (deleted[index]) reclaimedPaths.push(statePath(shard.scope, shard.key)); + else { + failed += 1; + stranded.push(shard); + } + } + } + if (stranded.length > 0) { + survivors.push({ generation: entry.generation, shards: stranded }); + } + } + + // One audit row for the sweep, not one per shard. src/functions/audit.ts + // sets the policy: automatic bulk sweeps emit a single row listing every + // removed id, because per-item rows flood the log. A reclaim on a badly + // leaked store is well over a thousand shards, and this runs during boot. + if (reclaimedPaths.length > 0) { + await this.auditIndexPersistence("delete", reclaimedPaths, { + manifestKey, + reason: "generation_reclaim", + liveGeneration, + // `evicted` is the field name src/functions/audit.ts specifies for a + // sweep; retention.ts is the reference shape. `failed` surfaces deletes + // that will be retried, which would otherwise vanish silently. + evicted: reclaimedPaths.length, + failed, + }); + } + + // A successful delete is the only thing that shrinks the ledger, so with + // none there is nothing to rewrite. Without this, every boot writes the + // ledger back unchanged. + if (reclaimedPaths.length === 0) return; + await this.kv + .set(KV.bm25Index, this.gcKey(manifestKey), { + v: 1, + generations: survivors, + }) + .catch(() => undefined); + } + private async auditIndexPersistence( action: string, targetIds: string[], @@ -280,35 +615,41 @@ export class IndexPersistence { ); } + /** Reports whether the delete landed, so the reclaim path can retry the rest. */ private async deleteKey( scope: string, key: string, reason: string, - ): Promise { - let result = "deleted"; + ): Promise { + let ok = true; let error: string | undefined; try { await this.kv.delete(scope, key); } catch (err) { - result = "failed"; + ok = false; error = errorMessage(err); } await this.auditIndexPersistence("delete", [statePath(scope, key)], { scope, key, reason, - result, + result: ok ? "deleted" : "failed", error, }); + return ok; } private async deleteShards( shards: IndexShardManifest["shards"], reason: string, - ): Promise { + ): Promise { + let allGone = true; for (const shard of shards) { - await this.deleteKey(shard.scope, shard.key, reason); + if (!(await this.deleteKey(shard.scope, shard.key, reason))) { + allGone = false; + } } + return allGone; } private async isManifestPublished( @@ -368,7 +709,20 @@ export class IndexPersistence { manifest.value != null && typeof manifest.value === "object" ) { - return this.loadManifestData(manifest.value, label); + const data = await this.loadManifestData(manifest.value, label); + // Boot is the only point that sees a generation stranded by a kill: the + // save path only ever inspects its own predecessor. Reclaim once the live + // generation has actually loaded — a failed load must not authorise + // deleting anything. + if (data !== null) { + // Through the same queue as save(), or this sweep's ledger rewrite + // clobbers a concurrent save's entry. Never allowed to fail the load. + const live = manifest.value.generation; + await this + .enqueue(() => this.reclaimGenerations(manifestKey, live)) + .catch(() => undefined); + } + return data; } const legacy = await this.readIndexValue( diff --git a/src/types.ts b/src/types.ts index d2c63efa61..46482d2f88 100644 --- a/src/types.ts +++ b/src/types.ts @@ -219,6 +219,24 @@ export interface FunctionMetrics { avgQualityScore: number; } +export interface MemoryEvaluation { + source: "heap" | "rss" | "cgroup-max" | "cgroup-high"; + path?: string; + usedBytes?: number; + limitBytes?: number; + percent?: number; + severity: "healthy" | "degraded" | "critical"; + available: boolean; + transition?: "entering" | "recovering" | "unavailable"; + samples?: number; + requiredSamples?: number; +} + +export interface CgroupMemory { + status: "available" | "partial" | "unavailable" | "unsupported"; + levels: Array<{ path: string; current?: number; max?: number; high?: number }>; +} + export interface HealthSnapshot { connectionState: string; workers: Array<{ id: string; name: string; status: string }>; @@ -227,6 +245,9 @@ export interface HealthSnapshot { heapTotal: number; rss: number; external: number; + heapSizeLimit?: number; + cgroup?: CgroupMemory; + evaluations?: MemoryEvaluation[]; }; cpu: { userMicros: number; systemMicros: number; percent: number }; eventLoopLagMs: number; diff --git a/src/viewer/index.html b/src/viewer/index.html index de47f4f5f9..85b58e7e37 100644 --- a/src/viewer/index.html +++ b/src/viewer/index.html @@ -870,6 +870,11 @@ .gauge-bar { flex: 1; height: 6px; background: var(--bg-inset); overflow: hidden; } .gauge-fill { height: 100%; transition: width 0.5s; } .gauge-label { width: 90px; font-size: 10px; color: var(--ink-muted); font-family: var(--font-ui); text-transform: uppercase; letter-spacing: 0.08em; font-weight: 600; } + .memory-gauge .gauge { flex-wrap: wrap; } + .memory-gauge .gauge-label { width: 130px; } + .memory-gauge .gauge-value { width: auto; min-width: 155px; } + .memory-detail { font-size: 11px; color: var(--ink-muted); overflow-wrap: anywhere; margin: 4px 0 8px; } + @media (max-width: 600px) { .memory-gauge .gauge-value { width: 100%; } } .gauge-value { width: 70px; font-size: 11px; color: var(--ink); text-align: right; font-family: var(--font-mono); } .obs-type-icon { font-size: 16px; margin-right: 4px; } @@ -1202,6 +1207,11 @@

agentmemory

// into sentences; unknown slugs render as-is. function humanizeHealthFlag(f) { var m; + var memoryLabels = { heap: 'V8 heap', rss: 'RSS budget', 'cgroup-max': 'Container limit', 'cgroup-high': 'Container throttle' }; + if ((m = /^memory_(warn|critical)_(heap|rss|cgroup-max|cgroup-high)(.*?)_(\d+%|unavailable)$/.exec(f))) + return memoryLabels[m[2]] + (m[3] ? ' (' + m[3] + ')' : '') + ': ' + (m[1] === 'critical' ? 'critical' : 'elevated') + ', ' + (m[4] === 'unavailable' ? 'measurement unavailable' : m[4] + ' of budget') + '.'; + if ((m = /^memory_(entering|recovering|unavailable)_(heap|rss|cgroup-max|cgroup-high)(.*)$/.exec(f))) + return memoryLabels[m[2]] + (m[3] ? ' (' + m[3] + ')' : '') + ': ' + ({ entering: 'Critical pending', recovering: 'Recovery pending', unavailable: 'Measurement unavailable' })[m[1]] + '.'; if ((m = /^memory_heap_tight_(\d+)%_rss(\d+)mb$/.exec(f))) return 'Heap is running tight: ' + m[1] + '% of allocated heap in use (process memory ' + m[2] + ' MB). Informational — Node grows the heap on demand.'; if ((m = /^memory_(warn|critical)_(\d+)%_rss(\d+)mb$/.exec(f))) @@ -1210,7 +1220,7 @@

agentmemory

return 'CPU ' + (m[1] === 'critical' ? 'critically high' : 'elevated') + ': ' + m[2] + '%.'; if ((m = /^event_loop_lag_(warn|critical)_(\d+)ms$/.exec(f))) return 'Event loop ' + (m[1] === 'critical' ? 'severely delayed' : 'delayed') + ': ' + m[2] + ' ms behind. The worker is busy or blocked.'; - if (f === 'connection_reconnecting') + if (f === 'kv_probe_failed') return 'State store unreachable: the health probe could not read back what it wrote. Search and recall are likely stalled.'; if (f === 'connection_reconnecting') return 'Engine connection lost — reconnecting.'; if ((m = /^connection_(.+)$/.exec(f))) return 'Engine connection state: ' + m[1] + '.'; @@ -1467,6 +1477,29 @@

agentmemory

} } + function renderMemoryResources(memory) { + var labels = { heap: 'Heap', rss: 'RSS budget', 'cgroup-max': 'Container limit', 'cgroup-high': 'Container throttle' }; + var colors = { healthy: 'var(--green)', degraded: 'var(--yellow)', critical: 'var(--red)' }; + var transitions = { entering: 'Critical pending', recovering: 'Recovery pending', unavailable: 'Measurement unavailable' }; + var html = '
'; + var evaluations = Array.isArray(memory.evaluations) ? memory.evaluations : []; + evaluations.forEach(function(item) { + var available = item.available && Number.isFinite(item.percent) && item.percent >= 0 && Number.isFinite(item.usedBytes) && Number.isFinite(item.limitBytes) && item.limitBytes > 0; + var percent = available ? item.percent : 0; + var color = colors[item.severity] || 'var(--ink-muted)'; + var value = available ? Math.round(item.usedBytes / 1024 / 1024) + ' / ' + Math.round(item.limitBytes / 1024 / 1024) + ' MiB · ' + percent.toFixed(1) + '%' : 'Unavailable'; + html += '
' + (labels[item.source] || 'Memory') + '
' + value + '
'; + var detail = transitions[item.transition] || ''; + if (item.path) detail += (detail ? ' · ' : '') + item.path; + if (detail) html += '
' + esc(detail) + '
'; + }); + if (!evaluations.length) html += '
Memory pressure unavailable
'; + if (memory.cgroup && memory.cgroup.status !== 'available') html += '
Container measurements: ' + esc(memory.cgroup.status) + '
'; + html += '
Heap: ' + Math.round((memory.heapUsed || 0) / 1024 / 1024) + ' MiB · RSS: ' + Math.round((memory.rss || 0) / 1024 / 1024) + ' MiB'; + if (memory.external) html += ' · External: ' + Math.round(memory.external / 1024 / 1024) + ' MiB'; + return html + '
'; + } + async function loadDashboard() { var el = document.getElementById('view-dashboard'); if (!state.dashboard.loaded) el.innerHTML = '
Loading dashboard...
'; @@ -1575,20 +1608,7 @@

agentmemory

if (snap.memory || snap.cpu) { html += '
System Resources
'; - if (snap.memory) { - var heapUsed = Math.round((snap.memory.heapUsed || 0) / 1024 / 1024); - var heapTotal = Math.round((snap.memory.heapTotal || 0) / 1024 / 1024); - var rss = Math.round((snap.memory.rss || 0) / 1024 / 1024); - var heapPct = heapTotal > 0 ? Math.round((heapUsed / heapTotal) * 100) : 0; - var rssAboveFloor = rss >= 512; - var heapColor = (heapPct > 80 && rssAboveFloor) ? 'var(--red)' : (heapPct > 60 && rssAboveFloor) ? 'var(--yellow)' : 'var(--green)'; - html += '
Heap
' + heapUsed + ' / ' + heapTotal + ' MB
'; - html += '
RSS
' + rss + ' MB
'; - if (snap.memory.external) { - var ext = Math.round(snap.memory.external / 1024 / 1024); - html += '
External
' + ext + ' MB
'; - } - } + if (snap.memory) html += renderMemoryResources(snap.memory); if (snap.cpu) { var cpuPct = snap.cpu.percent || 0; var cpuColor = cpuPct > 80 ? 'var(--red)' : cpuPct > 50 ? 'var(--yellow)' : 'var(--green)'; diff --git a/test/cli-engine-startup.test.ts b/test/cli-engine-startup.test.ts index ceaeeacbe6..d98ff70867 100644 --- a/test/cli-engine-startup.test.ts +++ b/test/cli-engine-startup.test.ts @@ -1,5 +1,72 @@ import { readFileSync } from "node:fs"; import { describe, expect, it } from "vitest"; +import { ENGINE_LOG_MAX_BYTES_PER_SECOND } from "../src/cli/engine-log.js"; + +describe("engine spawn stdio and death detection", () => { + const source = readFileSync("src/cli.ts", "utf8"); + const spawnStart = source.indexOf("function spawnEngineBackground"); + const spawnEnd = source.indexOf("const ENGINE_STARTUP_GRACE_MS", spawnStart); + const spawnBody = source.slice(spawnStart, spawnEnd); + const exitBody = source.slice(source.indexOf("function createEngineExitHandler"), spawnStart); + + it("forwards engine stdout only behind the opt-in gate", () => { + expect(spawnBody).toContain( + 'stdio: ["ignore", ENGINE_LOG_ENABLED ? "pipe" : "ignore", "pipe"]', + ); + expect(spawnBody).not.toContain('stdio: ["ignore", "ignore", "pipe"]'); + expect(spawnBody).toContain( + 'if (ENGINE_LOG_ENABLED) {\n attachEngineLog(child.stdout, "[engine]");\n attachEngineLog(child.stderr, "[engine:err]");\n }', + ); + + expect(source).toContain( + 'const ENGINE_LOG_ENABLED =\n process.env["AGENTMEMORY_ENGINE_LOG"] === "1" ||\n process.env["AGENTMEMORY_ENGINE_LOG"] === "true"', + ); + expect(source).toContain('stream.on("end", () => forwarder.flush())'); + }); + + // Engine-death detection is the only thing that turns a dead engine into a + // container restart. Log forwarding shares the same spawn call and the same + // stderr stream, so these assertions exist to fail if forwarding is ever + // allowed to gate, replace, or reorder any part of it. + it("keeps engine-death detection ungated by the log forwarder", () => { + expect(spawnBody).toContain("const spawnedAt = Date.now()"); + expect(spawnBody).toContain("stderrCapture = activeStartupStderr = createStartupStderrCapture()"); + expect(spawnBody).toContain( + 'child.stderr?.on("data", (chunk: Buffer) => {\n stderrCapture.append(chunk);', + ); + expect(exitBody).toContain("const engineRanFor = Date.now() - spawnedAt"); + expect(exitBody).toContain("engineRanFor > ENGINE_STARTUP_GRACE_MS"); + expect(exitBody).toContain( + 'if (process.env["AGENTMEMORY_EXIT_ON_ENGINE_DEATH"] !== "0") {\n process.exit(1);', + ); + + // The capture listener and the exit handler must sit outside the gate. + const gateStart = spawnBody.indexOf("if (ENGINE_LOG_ENABLED) {"); + const gateEnd = spawnBody.indexOf("}", spawnBody.indexOf("[engine:err]")); + const gateBlock = spawnBody.slice(gateStart, gateEnd); + expect(gateBlock).not.toContain("stderrCapture.append"); + expect(gateBlock).not.toContain("process.exit(1)"); + expect(spawnBody.indexOf('child.on("exit"')).toBeGreaterThan(gateEnd); + }); + + // The forwarder and the death report share process.stderr, and process.exit(1) + // discards pending async writes on a pipe, so the rate cap is what bounds how + // much forwarder output can ever sit ahead of the death report inside one + // second. Measured non-displacing at 64 KiB against a no-forwarder control; + // raising this cap voids that measurement, so it is pinned here rather than + // left to the structural assertions above, which would all still pass. + it("keeps the rate cap low enough not to crowd out the death report", () => { + expect(ENGINE_LOG_MAX_BYTES_PER_SECOND).toBeLessThanOrEqual(64 * 1024); + }); + + it("still captures the dying engine's stderr for the death report", () => { + expect(spawnBody).toContain("() => stderrCapture.text()"); + expect(exitBody).toContain("const stderr = readStderr()"); + expect(exitBody).toContain( + "if (stderr.trim()) console.error(`[agentmemory] engine stderr:\\n${stderr}`)", + ); + }); +}); describe("fresh native engine startup", () => { const source = readFileSync("src/cli.ts", "utf8"); @@ -28,7 +95,7 @@ describe("fresh native engine startup", () => { expect(spawnBody).toContain( "activeStartupStderr = createStartupStderrCapture()", ); - expect(spawnBody).toContain("activeStartupStderr.append(chunk)"); + expect(spawnBody).toContain("stderrCapture.append(chunk)"); const timeoutStart = source.indexOf("const ready = await waitForEngine(15000)"); const timeoutEnd = source.indexOf('s.stop(c.ok("iii-engine is ready"))', timeoutStart); @@ -139,11 +206,34 @@ describe("fresh native engine startup", () => { }); it("preserves Docker ownership when compose startup fails", () => { - const spawnStart = source.indexOf("function spawnEngineBackground"); + const spawnStart = source.indexOf("function createEngineExitHandler"); const spawnEnd = source.indexOf("function prepareEngineLaunch", spawnStart); const spawnBody = source.slice(spawnStart, spawnEnd); expect(spawnBody).toContain("if (!isDocker)"); expect(spawnBody).toContain("clearEngineState()"); expect(spawnBody).not.toContain("if (!isDocker) clearEnginePidfile();\n clearEngineState();"); }); + + it.each([ + ["async function startEngine", "const configPath = findIiiConfig()"], + ["async function reconcilePersistedDockerEngine", "function installInstructions"], + ])("arms supervision for verified resumed containers in %s", (start, end) => { + const begin = source.indexOf(start); + const finish = source.indexOf(end, begin); + expect(begin).toBeGreaterThan(-1); + expect(finish).toBeGreaterThan(begin); + const body = source.slice(begin, finish); + expect(body).toContain("inspectOwnedDockerEngine("); + expect(body).toContain('["start", inspection.containerId]'); + expect(body).toContain("startDockerEngineWatch(dockerBin, { containerId: inspection.containerId })"); + expect(body.indexOf("startDockerEngineWatch(")).toBeGreaterThan(body.indexOf('["start", inspection.containerId]')); + }); + + it("cancels local supervision before intentionally stopping an owned container", () => { + const start = source.indexOf("async function stopDockerEngine("); + const end = source.indexOf("async function runStop(", start); + const body = source.slice(start, end); + expect(body.indexOf("stopDockerEngineWatch?.()")).toBeGreaterThanOrEqual(0); + expect(body.indexOf("stopDockerEngineWatch?.()")).toBeLessThan(body.indexOf('["stop", "--time", "10", inspection.containerId]')); + }); }); diff --git a/test/deploy-entrypoint-drift.test.ts b/test/deploy-entrypoint-drift.test.ts new file mode 100644 index 0000000000..21c3a9710f --- /dev/null +++ b/test/deploy-entrypoint-drift.test.ts @@ -0,0 +1,68 @@ +import { describe, it, expect } from "vitest"; +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; + +// The four deploy targets ship copies of one entrypoint, not a shared file: +// three of them build from their own directory and Docker cannot COPY from +// outside the build context. Copies drift silently -- the iii-observability flag +// sat at `true` in three files and `false` in a fourth with nothing recording +// that a decision had been made. +// +// This guard is deliberately not line-indexed. An earlier version pinned line +// numbers and broke on any insertion. +const read = (t: string) => + readFileSync( + fileURLToPath(new URL(`../deploy/${t}/entrypoint.sh`, import.meta.url)), + "utf8", + ); + +const TARGETS = ["railway", "fly", "render", "coolify"] as const; +const files = Object.fromEntries(TARGETS.map((t) => [t, read(t)])) as Record< + (typeof TARGETS)[number], + string +>; + +/** Body with comments and blank lines dropped, so a comment cannot mask drift. */ +const code = (s: string) => + s + .split("\n") + .map((l) => l.trim()) + .filter((l) => l.length > 0 && !l.startsWith("#")) + .join("\n"); + +describe("deploy entrypoint drift", () => { + it("coolify and render remain byte-identical", () => { + expect(files.coolify).toBe(files.render); + }); + + // Railway is the only target that disables the in-memory OTEL exporter. The + // reason lives in deploy/railway/entrypoint.sh next to the value, so deleting + // this test cannot lose it. + it("only railway disables iii-observability", () => { + expect(code(files.railway)).toMatch(/enabled: false/); + for (const t of ["fly", "render", "coolify"] as const) { + expect(code(files[t])).toMatch(/enabled: true/); + expect(code(files[t])).not.toMatch(/enabled: false/); + } + }); + + it("railway carries its reason next to the value", () => { + expect(files.railway).toMatch(/2026-08-23[\s\S]{0,120}enabled: false/); + }); + + // The real guard: with the observability line normalised away, railway's + // executable body must still equal render's. Any other divergence fails here, + // wherever it is inserted. + it("railway's executable body matches render apart from that one flag", () => { + const norm = (s: string) => code(s).replace(/enabled: (true|false)/, "enabled: X"); + expect(norm(files.railway)).toBe(norm(files.render)); + }); + + // Fly inserts its own block BEFORE the final exec rather than appending after + // it, so render's body up to that exec is the shared part. + it("fly shares render's body up to the final exec", () => { + const renderBody = code(files.render).split("\n").slice(0, -1); + const flyBody = code(files.fly).split("\n").slice(0, renderBody.length); + expect(flyBody).toEqual(renderBody); + }); +}); diff --git a/test/docker-engine.test.ts b/test/docker-engine.test.ts new file mode 100644 index 0000000000..b43a0e967c --- /dev/null +++ b/test/docker-engine.test.ts @@ -0,0 +1,398 @@ +import { EventEmitter } from "node:events"; +import { readFileSync } from "node:fs"; +import { PassThrough } from "node:stream"; +import { execFile, spawn } from "node:child_process"; +import { transpileModule } from "typescript"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { watchDockerEngine } from "../src/cli/docker-engine.js"; +import { createStartupStderrCapture } from "../src/cli/startup-stderr.js"; +import { dockerComposeArgs } from "../src/cli/engine-launch.js"; + +vi.mock("node:child_process", () => ({ spawn: vi.fn(), execFile: vi.fn() })); + +function child() { + return Object.assign(new EventEmitter(), { + pid: 123, + stdout: Object.assign(new PassThrough(), { unref: vi.fn() }), + stderr: new PassThrough(), + unref: vi.fn(), + kill: vi.fn(), + }); +} + +const source = readFileSync("src/cli.ts", "utf8"); +const functionStart = source.indexOf("function createEngineExitHandler("); +expect(functionStart, "CLI createEngineExitHandler source marker").toBeGreaterThanOrEqual(0); +const functionEnd = source.indexOf("const ENGINE_STARTUP_GRACE_MS", functionStart); +expect(functionEnd, "CLI engine startup grace source marker").toBeGreaterThan(functionStart); +const graceEnd = source.indexOf(";", functionEnd) + 1; +expect(graceEnd, "CLI engine startup grace declaration terminator").toBeGreaterThan(functionEnd); +const compiledSpawn = transpileModule(source.slice(functionStart, graceEnd), {}).outputText; + +// Run the CLI's actual spawn function without executing its command dispatcher. +function cliSpawn(logEnabled = false) { + const exit = vi.fn(); + const clearState = vi.fn(); + const writePid = vi.fn(); + const clearPid = vi.fn(); + const attachLog = vi.fn(); + const env: Record = {}; + const factory = new Function("spawn", "watchDockerEngine", "process", "clearEngineState", "writeEnginePidfile", "clearEnginePidfile", "createStartupStderrCapture", "ENGINE_LOG_ENABLED", "attachEngineLog", ` + const vlog = () => {}; + const IS_VERBOSE = false; + let startupFailure = null; + let stopDockerEngineWatch; + let activeStartupStderr = createStartupStderrCapture(); + ${compiledSpawn} + return { + start: spawnEngineBackground, + watch: startDockerEngineWatch, + failure: () => startupFailure, + stop: () => stopDockerEngineWatch?.(), + stderr: () => activeStartupStderr.text(), + }; + `); + return { ...factory(spawn, watchDockerEngine, { env, exit }, clearState, writePid, clearPid, createStartupStderrCapture, logEnabled, attachLog), exit, env, clearState, writePid, clearPid, attachLog }; +} + +type LookupCallback = (error: Error | null, stdout: string, stderr: string) => void; +let lookupCallback: LookupCallback; +let lookupChild: ReturnType; +function completeLookup(error: Error | null = null, ids = "a".repeat(64) + "\n") { + lookupCallback(error, ids, ""); +} +function finishStartup(compose: ReturnType) { + compose.emit("exit", 0, null); + completeLookup(); +} + +const cleanups: Array<() => void> = []; +beforeEach(() => { + vi.resetAllMocks(); + vi.useFakeTimers(); + vi.setSystemTime(0); + lookupChild = child(); + vi.mocked(execFile).mockImplementation((...args: unknown[]) => { + lookupCallback = args[3] as LookupCallback; + return lookupChild as never; + }); + vi.spyOn(console, "error").mockImplementation(() => {}); +}); +afterEach(() => { + for (const stop of cleanups.splice(0)) stop(); + vi.useRealTimers(); + vi.restoreAllMocks(); +}); + +function startDocker() { + const compose = child(); + const observer = child(); + vi.mocked(spawn).mockReturnValueOnce(compose as never).mockReturnValueOnce(observer as never); + const cli = cliSpawn(); + cleanups.push(cli.stop); + cli.start("docker", dockerComposeArgs("/own/compose.yml", "agentmemory-3121", ["up", "-d"]), "iii-engine via Docker", "/own/invocation", { composeFile: "/own/compose.yml", projectName: "agentmemory-3121" }); + return { compose, observer, cli }; +} + +function containerExit(observer: ReturnType, code: number) { + observer.emit("exit", 0, null); + observer.stdout.emit("data", Buffer.from(`${code}\n`)); + observer.emit("close", 0, null); +} + +describe("Docker engine lifetime supervision", () => { + it("keeps timers responsive while identity lookup is pending", () => { + const { compose, cli } = startDocker(); + const tick = vi.fn(); + setTimeout(tick, 100); + compose.emit("exit", 0, null); + expect(execFile).toHaveBeenCalledOnce(); + expect(spawn).toHaveBeenCalledTimes(1); + vi.advanceTimersByTime(100); + expect(tick).toHaveBeenCalledOnce(); + expect(cli.exit).not.toHaveBeenCalled(); + completeLookup(); + expect(spawn).toHaveBeenCalledTimes(2); + }); + + it.each([null, new Error("lookup cancelled")])("cancels pending lookup and ignores its late callback (%s)", (error) => { + const { compose, observer, cli } = startDocker(); + const before = process.listenerCount("exit"); + compose.emit("exit", 0, null); + expect(process.listenerCount("exit")).toBe(before + 1); + cli.stop(); + expect(lookupChild.kill).toHaveBeenCalledOnce(); + expect(process.listenerCount("exit")).toBe(before); + completeLookup(error); + expect(spawn).toHaveBeenCalledTimes(1); + expect(compose.kill).not.toHaveBeenCalled(); + expect(observer.kill).not.toHaveBeenCalled(); + expect(console.error).not.toHaveBeenCalled(); + expect(cli.exit).not.toHaveBeenCalled(); + cli.stop(); + expect(lookupChild.kill).toHaveBeenCalledOnce(); + }); + + it("cancels pending lookup on parent exit", () => { + const { compose } = startDocker(); + const before = process.listeners("exit"); + compose.emit("exit", 0, null); + const onExit = process.listeners("exit").find((listener) => !before.includes(listener)); + expect(onExit).toBeDefined(); + onExit!(0); + expect(lookupChild.kill).toHaveBeenCalledOnce(); + completeLookup(); + expect(spawn).toHaveBeenCalledTimes(1); + expect(process.listeners("exit")).toEqual(before); + }); + + it("cleans up when identity lookup throws synchronously", () => { + const { compose, cli } = startDocker(); + const before = process.listenerCount("exit"); + vi.mocked(execFile).mockImplementationOnce(() => { throw new Error("invalid executable"); }); + compose.emit("exit", 0, null); + expect(process.listenerCount("exit")).toBe(before); + expect(console.error).toHaveBeenCalledWith(expect.stringContaining("invalid executable")); + expect(cli.exit).not.toHaveBeenCalled(); + }); + + it.each([0, 137])("ignores detached client success, then reports actual container exit %i", (code) => { + const { compose, observer, cli } = startDocker(); + finishStartup(compose); + expect(cli.exit).not.toHaveBeenCalled(); + expect(cli.clearState).not.toHaveBeenCalled(); + expect(execFile).toHaveBeenCalledWith("docker", [ + "compose", "-p", "agentmemory-3121", "-f", "/own/compose.yml", "ps", "--all", "--quiet", "iii-engine", + ], expect.objectContaining({ timeout: 5000, cwd: "/own/invocation" }), expect.any(Function)); + expect(spawn).toHaveBeenLastCalledWith("docker", ["wait", "a".repeat(64)], expect.anything()); + vi.setSystemTime(6000); + containerExit(observer, code); + expect(cli.exit).toHaveBeenCalledWith(1); + expect(cli.clearState).not.toHaveBeenCalled(); + expect(cli.clearPid).not.toHaveBeenCalled(); + expect(cli.failure()).toMatchObject({ kind: "docker-crashed" }); + }); + + it("keeps early container failure in startup diagnostics", () => { + const { compose, observer, cli } = startDocker(); + compose.stderr.emit("data", Buffer.from("Container iii-engine started")); + finishStartup(compose); + containerExit(observer, 137); + expect(cli.failure()).toMatchObject({ kind: "docker-crashed", stderr: "process exited with code 137" }); + expect(cli.exit).not.toHaveBeenCalled(); + expect(cli.clearState).not.toHaveBeenCalled(); + expect(cli.clearPid).not.toHaveBeenCalled(); + }); + + it("preserves compose startup failures without starting an observer", () => { + const { compose, cli } = startDocker(); + compose.stderr.emit("data", Buffer.from("image pull failed")); + compose.emit("exit", 1, null); + expect(cli.failure()).toMatchObject({ kind: "docker-crashed", stderr: "image pull failed" }); + expect(spawn).toHaveBeenCalledTimes(1); + expect(cli.exit).not.toHaveBeenCalled(); + expect(cli.clearState).not.toHaveBeenCalled(); + expect(cli.clearPid).not.toHaveBeenCalled(); + }); + + it.each(["exit", "error"])("retains Docker ownership after a slow compose %s failure without declaring container death", (failure) => { + const { compose, cli } = startDocker(); + vi.setSystemTime(20_000); + if (failure === "exit") { + compose.stderr.emit("data", Buffer.from("image pull failed")); + compose.emit("exit", 1, null); + } else { + compose.emit("error", new Error("spawn ENOENT")); + } + expect(cli.failure()).toMatchObject({ kind: "docker-crashed" }); + expect(execFile).not.toHaveBeenCalled(); + expect(cli.exit).not.toHaveBeenCalled(); + expect(cli.clearState).not.toHaveBeenCalled(); + expect(cli.clearPid).not.toHaveBeenCalled(); + }); + + it("captures a Docker spawn error instead of leaving an unhandled error event", () => { + const { compose, cli } = startDocker(); + compose.emit("error", new Error("spawn ENOENT")); + expect(cli.failure()).toMatchObject({ kind: "docker-crashed", stderr: "spawn ENOENT" }); + expect(cli.exit).not.toHaveBeenCalled(); + expect(cli.clearState).not.toHaveBeenCalled(); + }); + + it("honors the engine-death opt-out", () => { + const { compose, observer, cli } = startDocker(); + cli.env.AGENTMEMORY_EXIT_ON_ENGINE_DEATH = "0"; + finishStartup(compose); + vi.setSystemTime(6000); + containerExit(observer, 137); + expect(cli.exit).not.toHaveBeenCalled(); + expect(cli.clearState).not.toHaveBeenCalled(); + expect(cli.clearPid).not.toHaveBeenCalled(); + }); + + it.each(["spawn", "exit", "output"])("reports observer %s errors without declaring the engine dead", (failure) => { + const { compose, observer, cli } = startDocker(); + finishStartup(compose); + vi.setSystemTime(6000); + if (failure === "spawn") observer.emit("error", new Error("spawn EACCES")); + observer.emit("close", failure === "output" ? 0 : 1, null); + expect(console.error).toHaveBeenCalledOnce(); + expect(console.error).toHaveBeenCalledWith(expect.stringContaining("supervision unavailable")); + expect(cli.exit).not.toHaveBeenCalled(); + expect(cli.clearState).not.toHaveBeenCalled(); + expect(cli.failure()).toBeNull(); + }); + + it.each(["", "a".repeat(64) + "\n" + "b".repeat(64)])("rejects missing or ambiguous engine identity", (ids) => { + const { compose, cli } = startDocker(); + compose.emit("exit", 0, null); + completeLookup(null, ids); + expect(spawn).toHaveBeenCalledTimes(1); + expect(console.error).toHaveBeenCalledWith(expect.stringContaining("exactly one iii-engine")); + expect(cli.exit).not.toHaveBeenCalled(); + }); + + it("reports identity lookup errors without stopping any container", () => { + const { compose, cli } = startDocker(); + const before = process.listenerCount("exit"); + compose.emit("exit", 0, null); + completeLookup(new Error("Docker unavailable")); + expect(console.error).toHaveBeenCalledWith(expect.stringContaining("Docker unavailable")); + expect(spawn).toHaveBeenCalledTimes(1); + expect(cli.exit).not.toHaveBeenCalled(); + expect(process.listenerCount("exit")).toBe(before); + }); + + it("reaps only the observer on intentional cleanup and allows the CLI to exit", () => { + const { compose, observer, cli } = startDocker(); + const before = process.listenerCount("exit"); + finishStartup(compose); + expect(process.listenerCount("exit")).toBe(before + 1); + expect(observer.unref).toHaveBeenCalledOnce(); + expect(observer.stdout.unref).toHaveBeenCalledOnce(); + cli.stop(); + expect(process.listenerCount("exit")).toBe(before); + expect(observer.kill).toHaveBeenCalledOnce(); + expect(compose.kill).not.toHaveBeenCalled(); + observer.emit("close", null, "SIGTERM"); + expect(cli.exit).not.toHaveBeenCalled(); + expect(cli.clearState).not.toHaveBeenCalled(); + expect(console.error).not.toHaveBeenCalled(); + }); + + it("reaps the observer when its parent exits", () => { + const { compose, observer, cli } = startDocker(); + const before = process.listeners("exit"); + finishStartup(compose); + const onExit = process.listeners("exit").find((listener) => !before.includes(listener)); + expect(onExit).toBeDefined(); + onExit!(0); + expect(observer.kill).toHaveBeenCalledOnce(); + observer.emit("close", null, "SIGTERM"); + expect(cli.exit).not.toHaveBeenCalled(); + expect(console.error).not.toHaveBeenCalled(); + }); +}); + +describe("native engine launch and stderr capture", () => { + it.each([false, true])("preserves native cwd, bounded stderr and fatal cleanup with log forwarding %s", logEnabled => { + const engine = child(); + vi.mocked(spawn).mockReturnValueOnce(engine as never); + const cli = cliSpawn(logEnabled); + cli.start("iii", ["--config", "/native/runtime.yaml"], "iii-engine", "/native/cwd"); + expect(spawn).toHaveBeenCalledWith("iii", ["--config", "/native/runtime.yaml"], expect.objectContaining({ + cwd: "/native/cwd", + stdio: ["ignore", logEnabled ? "pipe" : "ignore", "pipe"], + })); + expect(cli.writePid).toHaveBeenCalledWith(engine.pid); + expect(cli.attachLog).toHaveBeenCalledTimes(logEnabled ? 2 : 0); + engine.stderr.emit("data", Buffer.alloc(20 * 1024, "x")); + engine.stderr.emit("data", Buffer.from("discarded")); + expect(cli.stderr()).toBe("x".repeat(16 * 1024)); + vi.setSystemTime(6000); + engine.emit("exit", 1, null); + expect(cli.failure()).toMatchObject({ kind: "engine-crashed", stderr: "x".repeat(16 * 1024) }); + expect(cli.clearPid).toHaveBeenCalledOnce(); + expect(cli.clearState).toHaveBeenCalledOnce(); + expect(cli.exit).toHaveBeenCalledWith(1); + expect(execFile).not.toHaveBeenCalled(); + }); + + it("retains each launch's stderr while exposing the latest capture for startup timeouts", () => { + const first = child(); + const second = child(); + vi.mocked(spawn).mockReturnValueOnce(first as never).mockReturnValueOnce(second as never); + const cli = cliSpawn(); + cli.start("iii", [], "iii-engine", "/first"); + first.stderr.emit("data", Buffer.from("first startup error")); + cli.start("iii", [], "iii-engine", "/second"); + second.stderr.emit("data", Buffer.from("second startup output")); + first.emit("exit", 1, null); + expect(cli.failure()).toMatchObject({ stderr: "first startup error" }); + expect(cli.stderr()).toBe("second startup output"); + }); +}); + +describe("resumed Docker engine supervision", () => { + it.each([0, 137])("watches the verified ID without compose access and handles later exit %i", code => { + const observer = child(); + vi.mocked(spawn).mockReturnValueOnce(observer as never); + const cli = cliSpawn(); + cleanups.push(cli.stop); + cli.watch("docker", { containerId: "b".repeat(64) }); + expect(execFile).not.toHaveBeenCalled(); + expect(spawn).toHaveBeenCalledWith("docker", ["wait", "b".repeat(64)], expect.anything()); + vi.setSystemTime(6000); + containerExit(observer, code); + expect(cli.exit).toHaveBeenCalledWith(1); + expect(cli.clearState).not.toHaveBeenCalled(); + expect(cli.clearPid).not.toHaveBeenCalled(); + }); + + it("honors the opt-out for a resumed engine", () => { + const observer = child(); + vi.mocked(spawn).mockReturnValueOnce(observer as never); + const cli = cliSpawn(); + cleanups.push(cli.stop); + cli.env.AGENTMEMORY_EXIT_ON_ENGINE_DEATH = "0"; + cli.watch("docker", { containerId: "b".repeat(64) }); + vi.setSystemTime(6000); + containerExit(observer, 137); + expect(cli.exit).not.toHaveBeenCalled(); + expect(cli.clearState).not.toHaveBeenCalled(); + expect(cli.failure()).toMatchObject({ kind: "docker-crashed" }); + }); + + it("replaces an old observer without stopping either container or reporting cancellation as death", () => { + const first = child(); + const second = child(); + vi.mocked(spawn).mockReturnValueOnce(first as never).mockReturnValueOnce(second as never); + const cli = cliSpawn(); + cleanups.push(cli.stop); + cli.watch("docker", { containerId: "a".repeat(64) }); + cli.watch("docker", { containerId: "b".repeat(64) }); + expect(first.kill).toHaveBeenCalledOnce(); + vi.setSystemTime(6000); + first.emit("close", null, "SIGTERM"); + expect(cli.exit).not.toHaveBeenCalled(); + expect(console.error).not.toHaveBeenCalled(); + cli.stop(); + expect(second.kill).toHaveBeenCalledOnce(); + second.emit("close", null, "SIGTERM"); + expect(cli.exit).not.toHaveBeenCalled(); + expect(execFile).not.toHaveBeenCalled(); + expect(vi.mocked(spawn).mock.calls.every(([, args]) => args?.[0] === "wait")).toBe(true); + }); + + it("rejects invalid direct IDs without starting a Docker command", () => { + const cli = cliSpawn(); + const before = process.listenerCount("exit"); + cleanups.push(cli.stop); + cli.watch("docker", { containerId: "not-a-container-id" }); + expect(spawn).not.toHaveBeenCalled(); + expect(execFile).not.toHaveBeenCalled(); + expect(cli.exit).not.toHaveBeenCalled(); + expect(console.error).toHaveBeenCalledWith(expect.stringContaining("exactly one iii-engine")); + expect(process.listenerCount("exit")).toBe(before); + }); +}); diff --git a/test/engine-log.test.ts b/test/engine-log.test.ts new file mode 100644 index 0000000000..313b32d609 --- /dev/null +++ b/test/engine-log.test.ts @@ -0,0 +1,167 @@ +import { describe, expect, it } from "vitest"; +import { + ENGINE_LOG_MAX_BYTES_PER_SECOND, + ENGINE_LOG_MAX_TOTAL_BYTES, + createEngineLogForwarder, +} from "../src/cli/engine-log.js"; + +function harness(overrides: Record = {}) { + const lines: string[] = []; + let clock = 0; + const forwarder = createEngineLogForwarder({ + prefix: "[engine]", + write: (line) => lines.push(line), + now: () => clock, + ...overrides, + }); + return { + lines, + forwarder, + advance(ms: number) { + clock += ms; + }, + push(text: string) { + forwarder.push(Buffer.from(text, "utf8")); + }, + }; +} + +describe("engine log forwarder", () => { + it("prefixes each complete line and holds partial lines back", () => { + const h = harness(); + + h.push("registered worker\nWorker unregis"); + expect(h.lines).toEqual(["[engine] registered worker"]); + + h.push("tered\n"); + expect(h.lines).toEqual([ + "[engine] registered worker", + "[engine] Worker unregistered", + ]); + }); + + it("strips carriage returns and decodes utf8 split across chunks", () => { + const h = harness(); + const snowman = Buffer.from("☃", "utf8"); + + h.push("crlf line\r\n"); + h.forwarder.push(snowman.subarray(0, 1)); + h.forwarder.push(snowman.subarray(1)); + h.push("\n"); + + expect(h.lines).toEqual(["[engine] crlf line", "[engine] ☃"]); + }); + + it("flushes a trailing partial line when the stream ends", () => { + const h = harness(); + + h.push("panic: engine died"); + expect(h.lines).toEqual([]); + + h.forwarder.flush(); + expect(h.lines).toEqual(["[engine] panic: engine died"]); + }); + + it("stops permanently once the lifetime ceiling is reached", () => { + const h = harness({ maxTotalBytes: 64, maxBytesPerSecond: 1024 * 1024 }); + + for (let i = 0; i < 200; i += 1) h.push(`line ${i}\n`); + + const terminal = h.lines.filter((l) => l.includes("ceiling reached")); + expect(terminal).toHaveLength(1); + expect(terminal[0]).toBe( + "[engine] log forwarding stopped: 64 byte ceiling reached", + ); + expect(h.lines[h.lines.length - 1]).toBe(terminal[0]); + + const forwarded = h.lines.filter((l) => !l.includes("ceiling reached")); + const forwardedBytes = forwarded.reduce( + (sum, l) => sum + Buffer.byteLength(l) + 1, + 0, + ); + expect(forwardedBytes).toBeLessThanOrEqual(64); + + const before = h.lines.length; + h.push("still chatty\n"); + h.advance(10_000); + h.push("still chatty\n"); + h.forwarder.flush(); + expect(h.lines).toHaveLength(before); + }); + + it("stays silent for the rest of a single chunk that trips the ceiling", () => { + const h = harness({ maxTotalBytes: 64, maxBytesPerSecond: 1024 * 1024 }); + + h.push(Array.from({ length: 50 }, (_, i) => `line ${i}`).join("\n") + "\n"); + + expect(h.lines.filter((l) => l.includes("ceiling reached"))).toHaveLength(1); + expect(h.lines[h.lines.length - 1]).toBe( + "[engine] log forwarding stopped: 64 byte ceiling reached", + ); + }); + + it("drops over the rate cap and reports the drop once per window", () => { + const h = harness({ maxBytesPerSecond: 64, maxTotalBytes: 1024 * 1024 }); + const a = "a".repeat(30); + + h.push(`${a}\n`); + h.push(`${"b".repeat(30)}\n`); + h.push(`${"c".repeat(30)}\n`); + expect(h.lines).toEqual([`[engine] ${a}`]); + + h.advance(1000); + h.push(`${"d".repeat(30)}\n`); + expect(h.lines).toEqual([ + `[engine] ${a}`, + "[engine] dropped 80 bytes (rate cap 64 bytes/s)", + ]); + }); + + it("never suppresses silently, even with the window budget exhausted", () => { + const h = harness({ maxBytesPerSecond: 64, maxTotalBytes: 1024 * 1024 }); + + for (let i = 0; i < 20; i += 1) h.push(`${"a".repeat(30)}\n`); + h.forwarder.flush(); + + expect(h.lines).toEqual([ + `[engine] ${"a".repeat(30)}`, + "[engine] dropped 760 bytes (rate cap 64 bytes/s)", + ]); + }); + + it("charges its exempt notices against the lifetime ceiling", () => { + const h = harness({ maxBytesPerSecond: 8, maxTotalBytes: 200 }); + + for (let round = 0; round < 40; round += 1) { + h.push(`${"a".repeat(30)}\n`); + h.advance(1000); + } + + const forwardedBytes = h.lines + .filter((l) => !l.includes("ceiling reached")) + .reduce((sum, l) => sum + Buffer.byteLength(l) + 1, 0); + expect(forwardedBytes).toBeLessThanOrEqual(200); + expect(h.lines.filter((l) => l.includes("ceiling reached"))).toHaveLength(1); + }); + + it("force-flushes a newline-free stream instead of buffering it forever", () => { + const h = harness({ maxLineBytes: 16 }); + + h.push("0123456789abcdefghij"); + + expect(h.lines).toEqual(["[engine] 0123456789abcdefghij"]); + }); + + it("flushes a partial UTF-8 record at its byte limit", () => { + const h = harness({ maxLineBytes: 6 }); + h.push("☃"); + expect(h.lines).toEqual([]); + h.push("☃"); + expect(h.lines).toEqual(["[engine] ☃☃"]); + }); + + it("ships ceilings that bound a firehose without operator action", () => { + expect(ENGINE_LOG_MAX_TOTAL_BYTES).toBe(32 * 1024 * 1024); + expect(ENGINE_LOG_MAX_BYTES_PER_SECOND).toBe(64 * 1024); + }); +}); diff --git a/test/health-collection.test.ts b/test/health-collection.test.ts new file mode 100644 index 0000000000..843042d583 --- /dev/null +++ b/test/health-collection.test.ts @@ -0,0 +1,129 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { ISdk } from "iii-sdk"; +import type { StateKV } from "../src/state/kv.js"; +import type { HealthSnapshot } from "../src/types.js"; +import { KV } from "../src/state/schema.js"; +vi.mock("node:v8", () => ({ getHeapStatistics: vi.fn(() => ({ heap_size_limit: 100 })) })); +vi.mock("../src/health/memory.js", async importOriginal => ({ ...await importOriginal(), collectCgroupMemory: vi.fn(async () => ({ status: "unsupported", levels: [] })) })); +import { collectCgroupMemory } from "../src/health/memory.js"; +import { registerHealthMonitor } from "../src/health/monitor.js"; +import { registerApiTriggers } from "../src/triggers/api.js"; + +let used = 96; +let stop: (() => void) | undefined; +beforeEach(() => { + vi.useFakeTimers(); + used = 96; + vi.spyOn(process, "memoryUsage").mockImplementation(() => ({ heapUsed: used, heapTotal: 100, rss: 200, external: 0, arrayBuffers: 0 })); + vi.spyOn(process, "cpuUsage").mockReturnValue({ user: 0, system: 0 }); + vi.mocked(collectCgroupMemory).mockResolvedValue({ status: "unsupported", levels: [] }); +}); +afterEach(() => { stop?.(); vi.useRealTimers(); vi.restoreAllMocks(); vi.unstubAllEnvs(); }); + +function setup() { + const stored = new Map(); + const history: HealthSnapshot[] = []; + const functions = new Map Promise>(); + const sdk = { on: vi.fn(), trigger: vi.fn(async () => ({ workers: [] })), registerFunction: (id: string, fn: (payload: unknown) => Promise) => functions.set(id, fn), registerTrigger: vi.fn() }; + const kv = { set: vi.fn(async (_scope: string, key: string, value: unknown) => { stored.set(key, value); if (key === "latest") history.push(value as HealthSnapshot); }), get: vi.fn(async (_scope: string, key: string) => stored.get(key)), list: vi.fn(async () => []) }; + registerApiTriggers(sdk as unknown as ISdk, kv as unknown as StateKV); + stop = registerHealthMonitor(sdk as unknown as ISdk, kv as unknown as StateKV).stop; + return { sdk, kv, history, health: () => functions.get("api::health")!({}), latest: () => stored.get("latest") as HealthSnapshot }; +} +async function flush() { await vi.advanceTimersByTimeAsync(1); } +async function tick() { await vi.advanceTimersByTimeAsync(30_000); } + +describe("periodic memory hold and persisted API health", () => { + it("requires two observations to enter and clear; polls cannot advance the hold", async () => { + const app = setup(); await flush(); + expect(app.latest().status).toBe("degraded"); + expect(app.latest().memory.evaluations?.[0].transition).toBe("entering"); + for (let i = 0; i < 3; i++) expect((await app.health()).status_code).toBe(200); + await tick(); expect((await app.health()).status_code).toBe(503); + used = 20; await tick(); + expect(app.latest().memory.evaluations?.[0].transition).toBe("recovering"); + expect((await app.health()).status_code).toBe(503); + await tick(); expect((await app.health()).status_code).toBe(200); + }); + + it("interrupts spikes, missing entry samples and recovery samples", async () => { + const app = setup(); await flush(); + used = 20; await tick(); used = 96; await tick(); expect(app.latest().status).toBe("degraded"); + used = NaN; await tick(); used = 96; await tick(); expect(app.latest().status).toBe("degraded"); + await tick(); expect(app.latest().status).toBe("critical"); + used = 20; await tick(); used = NaN; await tick(); + expect(app.latest().memory.evaluations?.[0]).toMatchObject({ severity: "critical", available: false, transition: "unavailable" }); + used = 20; await tick(); expect(app.latest().status).toBe("critical"); + await tick(); expect(app.latest().status).toBe("healthy"); + }); + + it("retains a vanished critical cgroup source and resets on monitor restart", async () => { + used = 1; + vi.mocked(collectCgroupMemory).mockResolvedValue({ status: "available", levels: [{ path: "/parent", current: 96, max: 100 }] }); + const app = setup(); await flush(); await tick(); expect(app.latest().status).toBe("critical"); + vi.mocked(collectCgroupMemory).mockResolvedValue({ status: "unavailable", levels: [] }); + await tick(); expect(app.latest().memory.evaluations).toContainEqual(expect.objectContaining({ path: "/parent", severity: "critical", transition: "unavailable" })); + stop?.(); stop = registerHealthMonitor(app.sdk as unknown as ISdk, app.kv as unknown as StateKV).stop; + await flush(); expect(app.latest().status).toBe("healthy"); + }); + + it("reads configuration once and keeps KV failures immediate", async () => { + vi.stubEnv("AGENTMEMORY_HEALTH_MEM_HOLD_SAMPLES", "3"); + const app = setup(); await flush(); + vi.stubEnv("AGENTMEMORY_HEALTH_MEM_HOLD_SAMPLES", "1"); + await tick(); expect(app.latest().status).toBe("degraded"); + app.kv.get.mockRejectedValue(new Error("KV unavailable")); + used = 20; await tick(); + expect(app.latest().status).toBe("critical"); + expect(app.latest().alerts).toContain("kv_probe_failed"); + }); + + it("discards out-of-order probe completions without double-counting entry", async () => { + let release: (value: { status: "unsupported"; levels: [] }) => void; + vi.mocked(collectCgroupMemory).mockImplementationOnce(() => new Promise(resolve => { release = resolve; })); + const app = setup(); await flush(); await tick(); + expect(app.latest().status).toBe("degraded"); + release!({ status: "unsupported", levels: [] }); await flush(); + expect(app.history).toHaveLength(1); + await tick(); expect(app.latest().status).toBe("critical"); + }); + + it("escalates ten failed probes despite a stalled persist and times out workers", async () => { + vi.stubEnv("AGENTMEMORY_HEALTH_ESCALATE", "true"); + const kill = vi.spyOn(process, "kill").mockReturnValue(true); + const error = vi.spyOn(console, "error").mockImplementation(() => {}); + const app = setup(); await flush(); + const original = app.kv.set.getMockImplementation()!; + app.kv.set.mockImplementation((scope, key, value) => key === "latest" ? new Promise(() => {}) : original(scope, key, value)); + app.kv.get.mockRejectedValue(new Error("store unavailable")); + app.sdk.trigger.mockImplementation(() => new Promise(() => {})); + for (let i = 0; i < 9; i++) await tick(); + await vi.advanceTimersByTimeAsync(5000); + expect(kill).not.toHaveBeenCalled(); + await tick(); + expect(kill).toHaveBeenCalledExactlyOnceWith(process.pid, "SIGTERM"); + expect(error).toHaveBeenCalledWith(expect.stringContaining("KV unreachable")); + }); + + it("serializes slow persists while probes and escalation keep running", async () => { + vi.stubEnv("AGENTMEMORY_HEALTH_ESCALATE", "true"); + const kill = vi.spyOn(process, "kill").mockReturnValue(true); + const app = setup(); await flush(); + const original = app.kv.set.getMockImplementation()!; + let release: () => void; + let active = 0; + let maximum = 0; + app.kv.set.mockImplementation(async (scope, key, value) => { + if (key === "latest") { + active++; maximum = Math.max(maximum, active); + if (!release) await new Promise(resolve => { release = resolve; }); + await original(scope, key, value); active--; + } else await original(scope, key, value); + }); + await tick(); used = 20; await tick(); await tick(); + expect(maximum).toBe(1); + release!(); await flush(); expect(app.latest().status).toBe("healthy"); + expect(kill).not.toHaveBeenCalled(); + expect(app.kv.set).toHaveBeenCalledWith(KV.health, "_probe", expect.anything()); + }); +}); diff --git a/test/health-memory.test.ts b/test/health-memory.test.ts new file mode 100644 index 0000000000..ebb6b97f32 --- /dev/null +++ b/test/health-memory.test.ts @@ -0,0 +1,104 @@ +import { describe, expect, it, vi } from "vitest"; +import { collectCgroupMemory, readMemoryConfig } from "../src/health/memory.js"; + +const mount = "29 23 0:26 / /sys/fs/cgroup rw,nosuid - cgroup2 cgroup rw"; +function reader(files: Record) { + return async (path: string) => { + if (!(path in files)) throw new Error("unreadable"); + return files[path]; + }; +} +function files(group = "/service") { + return { "/proc/self/cgroup": `0::${group}\n`, "/proc/self/mountinfo": mount }; +} + +describe("cgroup v2 measurement", () => { + it("pairs each visible ancestor's usage with its own refreshed limits", async () => { + const data = { ...files(), "/sys/fs/cgroup/service/memory.current": "10", "/sys/fs/cgroup/service/memory.max": "1000", "/sys/fs/cgroup/service/memory.high": "max", "/sys/fs/cgroup/memory.current": "99", "/sys/fs/cgroup/memory.max": "100", "/sys/fs/cgroup/memory.high": "80" }; + expect(await collectCgroupMemory(reader(data), "linux")).toEqual({ status: "available", levels: [ { path: "/service", current: 10, max: 1000 }, { path: "/", current: 99, max: 100, high: 80 } ] }); + data["/sys/fs/cgroup/memory.max"] = "200"; + expect((await collectCgroupMemory(reader(data), "linux")).levels[1].max).toBe(200); + }); + + it.each([ + ["/tenant", "/tenant/service", "/service"], + ["/", "/", "/"], + ["/", "/service", "/service"], + ])("resolves mount root %s and process path %s", async (root, group, expected) => { + const result = await collectCgroupMemory(reader({ ...files(group), "/proc/self/mountinfo": mount.replace(" / /sys", ` ${root} /sys`), [`/sys/fs/cgroup${expected === "/" ? "" : expected}/memory.current`]: "10", [`/sys/fs/cgroup${expected === "/" ? "" : expected}/memory.max`]: "100", [`/sys/fs/cgroup${expected === "/" ? "" : expected}/memory.high`]: "max" }), "linux"); + expect(result.levels[0]).toMatchObject({ path: expected, current: 10, max: 100 }); + }); + + it.each(["/other", "/.."])("does not attribute an unmatched mount root %s to this process", async root => { + const read = vi.fn(reader({ + ...files("/app"), + "/proc/self/mountinfo": mount.replace(" / /sys", ` ${root} /sys`), + "/sys/fs/cgroup/app/memory.current": "99", + "/sys/fs/cgroup/app/memory.max": "100", + })); + expect(await collectCgroupMemory(read, "linux")).toEqual({ status: "unavailable", levels: [] }); + expect(read.mock.calls.every(([path]) => path.startsWith("/proc/"))).toBe(true); + }); + + it("uses the mount exposing the full visible ancestry", async () => { + const result = await collectCgroupMemory(reader({ + ...files("/tenant/service"), + "/proc/self/mountinfo": mount.replace(" / /sys/fs/cgroup", " /tenant /tenant-cgroup") + "\n" + mount, + "/sys/fs/cgroup/tenant/service/memory.current": "1", + "/sys/fs/cgroup/tenant/service/memory.max": "100", + "/sys/fs/cgroup/tenant/memory.current": "10", + "/sys/fs/cgroup/tenant/memory.max": "100", + "/sys/fs/cgroup/memory.current": "99", + "/sys/fs/cgroup/memory.max": "100", + }), "linux"); + expect(result.levels).toContainEqual({ path: "/", current: 99, max: 100 }); + }); + + it("decodes escaped mount paths and handles high independently", async () => { + const result = await collectCgroupMemory(reader({ ...files("/"), "/proc/self/mountinfo": mount.replace("/sys/fs/cgroup", "/cg\\040space"), "/cg space/memory.current": "90", "/cg space/memory.high": "80" }), "linux"); + expect(result).toEqual({ status: "partial", levels: [{ path: "/", current: 90, high: 80 }] }); + }); + + it.each(["", "-1", "NaN", "Infinity", "1e3", "12oops", "9007199254740992"])("rejects malformed usage %s without discarding valid limits", async current => { + const result = await collectCgroupMemory(reader({ ...files("/"), "/sys/fs/cgroup/memory.current": current, "/sys/fs/cgroup/memory.max": "100", "/sys/fs/cgroup/memory.high": "max" }), "linux"); + expect(result).toEqual({ status: "partial", levels: [{ path: "/", max: 100 }] }); + }); + + it.each(["0", "-1", "", "Infinity", "1024kb", "9007199254740992"])("rejects invalid hard limit %s while retaining high", async max => { + const result = await collectCgroupMemory(reader({ ...files("/"), "/sys/fs/cgroup/memory.current": "90", "/sys/fs/cgroup/memory.max": max, "/sys/fs/cgroup/memory.high": "80" }), "linux"); + expect(result).toEqual({ status: "partial", levels: [{ path: "/", current: 90, high: 80 }] }); + }); + + it("accepts zero usage and unlimited limits", async () => { + expect(await collectCgroupMemory(reader({ ...files("/"), "/sys/fs/cgroup/memory.current": "0\n", "/sys/fs/cgroup/memory.max": "max", "/sys/fs/cgroup/memory.high": "max" }), "linux")).toEqual({ status: "available", levels: [{ path: "/", current: 0 }] }); + }); + + it("handles unavailable and unsupported hosts", async () => { + expect((await collectCgroupMemory(reader({}), "linux")).status).toBe("unavailable"); + const read = vi.fn(); + expect((await collectCgroupMemory(read, "darwin")).status).toBe("unsupported"); + expect(read).not.toHaveBeenCalled(); + }); +}); + +describe("memory configuration", () => { + it("uses defaults and explicit overrides", () => { + expect(readMemoryConfig({}, {}, vi.fn())).toEqual({ memoryWarnPercent: 80, memoryCriticalPercent: 95, memoryHoldSamples: 2 }); + expect(readMemoryConfig({ AGENTMEMORY_HEALTH_MEM_WARN_PCT: "70", AGENTMEMORY_HEALTH_MEM_CRITICAL_PCT: "90", AGENTMEMORY_HEALTH_MEM_RSS_BUDGET_MB: "128", AGENTMEMORY_HEALTH_MEM_HOLD_SAMPLES: "3" }, { memoryHoldSamples: 1 }, vi.fn())).toEqual({ memoryWarnPercent: 70, memoryCriticalPercent: 90, memoryRssBudgetBytes: 128 * 1024 * 1024, memoryHoldSamples: 1 }); + }); + it.each(["", "0", "-1", "NaN", "Infinity", "101", "95"])("rejects invalid warning %s", value => { + const warn = vi.fn(); + expect(readMemoryConfig({ AGENTMEMORY_HEALTH_MEM_WARN_PCT: value }, {}, warn).memoryWarnPercent).toBe(80); + expect(warn).toHaveBeenCalledTimes(1); + }); + it("accepts critical 100 and rejects crossed thresholds", () => { + expect(readMemoryConfig({ AGENTMEMORY_HEALTH_MEM_CRITICAL_PCT: "100" }, {}, vi.fn()).memoryCriticalPercent).toBe(100); + expect(readMemoryConfig({ AGENTMEMORY_HEALTH_MEM_WARN_PCT: "90", AGENTMEMORY_HEALTH_MEM_CRITICAL_PCT: "80" }, {}, vi.fn())).toMatchObject({ memoryWarnPercent: 80, memoryCriticalPercent: 95 }); + }); + it.each(["0", "-1", "1.5", "NaN", "Infinity"])("rejects hold count %s", value => { + expect(readMemoryConfig({ AGENTMEMORY_HEALTH_MEM_HOLD_SAMPLES: value }, {}, vi.fn()).memoryHoldSamples).toBe(2); + }); + it.each(["", "0", "-1", "NaN", "Infinity", "1e300"])("rejects budget %s", value => { + expect(readMemoryConfig({ AGENTMEMORY_HEALTH_MEM_RSS_BUDGET_MB: value }, {}, vi.fn()).memoryRssBudgetBytes).toBeUndefined(); + }); +}); diff --git a/test/health-monitor.test.ts b/test/health-monitor.test.ts new file mode 100644 index 0000000000..f91d104982 --- /dev/null +++ b/test/health-monitor.test.ts @@ -0,0 +1,132 @@ +import { describe, it, expect } from "vitest"; +import type { HealthSnapshot } from "../src/types.js"; +import { bumpEscalation, type EscalationState } from "../src/health/monitor.js"; + +function snap(over: Partial = {}): HealthSnapshot { + return { + connectionState: "connected", + workers: [], + memory: { heapUsed: 0, heapTotal: 1, rss: 0, external: 0 }, + cpu: { userMicros: 0, systemMicros: 0, percent: 0 }, + eventLoopLagMs: 0, + uptimeSeconds: 600, + kvConnectivity: { status: "ok", latencyMs: 1 }, + status: "healthy", + alerts: [], + ...over, + }; +} + +const stalled = () => snap({ kvConnectivity: { status: "error", error: "kv_probe_failed" } }); +const noSignalProbes: Array<[string, unknown]> = [ + ["absent", undefined], + ["null", null], + ["missing status", {}], + ["unknown status", { status: "pending" }], + ["malformed status", { status: 123 }], +]; + +function fresh(): EscalationState { + return { consecutiveKvProbeFailures: 0, hasSeenHealthyKvProbe: false, escalated: false }; +} + +/** Arm the state the way a healthy first collection would. */ +function hasSeenHealthyKvProbe(): EscalationState { + const s = fresh(); + bumpEscalation(snap(), s, 3); + return s; +} + +describe("bumpEscalation arming", () => { + // Mirrors the shell watchdog's "wait for a first success" rule. Without it a + // store that is already stalled at boot escalates on the first minute of every + // container life, spending the platform's restart budget in minutes. + it("never escalates when the store is stalled from the very first check", () => { + const state = fresh(); + for (let i = 0; i < 50; i++) { + expect(bumpEscalation(stalled(), state, 3)).toBe(false); + } + expect(state.hasSeenHealthyKvProbe).toBe(false); + expect(state.consecutiveKvProbeFailures).toBe(0); + }); + + it("arms on the first healthy probe", () => { + const state = fresh(); + bumpEscalation(snap(), state, 3); + expect(state.hasSeenHealthyKvProbe).toBe(true); + }); + + it.each(noSignalProbes)("does not arm from %s probe data or subsequent errors", (_label, probe) => { + const state = fresh(); + expect(bumpEscalation(snap({ kvConnectivity: probe as HealthSnapshot["kvConnectivity"] }), state, 3)).toBe(false); + for (let i = 0; i < 10; i++) { + expect(bumpEscalation(stalled(), state, 3)).toBe(false); + } + expect(state).toEqual(fresh()); + }); +}); + +describe("bumpEscalation gating", () => { + it("escalates on the Nth consecutive stall, not before", () => { + const state = hasSeenHealthyKvProbe(); + expect(bumpEscalation(stalled(), state, 3)).toBe(false); + expect(bumpEscalation(stalled(), state, 3)).toBe(false); + expect(bumpEscalation(stalled(), state, 3)).toBe(true); + }); + + it("resets the counter on a healthy probe", () => { + const state = hasSeenHealthyKvProbe(); + bumpEscalation(stalled(), state, 3); + bumpEscalation(stalled(), state, 3); + bumpEscalation(snap(), state, 3); + expect(state.consecutiveKvProbeFailures).toBe(0); + expect(bumpEscalation(stalled(), state, 3)).toBe(false); + }); + + it.each(noSignalProbes)("preserves an existing failure streak across %s probe data", (_label, probe) => { + const state = hasSeenHealthyKvProbe(); + bumpEscalation(stalled(), state, 3); + bumpEscalation(stalled(), state, 3); + const before = { ...state }; + expect(bumpEscalation(snap({ kvConnectivity: probe as HealthSnapshot["kvConnectivity"] }), state, 3)).toBe(false); + expect(state).toEqual(before); + expect(bumpEscalation(stalled(), state, 3)).toBe(true); + }); + + it("escalates at most once per process", () => { + const state = hasSeenHealthyKvProbe(); + let fired = 0; + for (let i = 0; i < 20; i++) { + if (bumpEscalation(stalled(), state, 1)) fired++; + } + expect(fired).toBe(1); + }); + + // The gate is the KV probe, not snapshot.status. evaluateHealth raises + // `critical` from five independent conditions, so gating on the aggregate + // would let a CPU spike during a consolidation pass kill a healthy process. + it.each([ + ["cpu", { cpu: { userMicros: 0, systemMicros: 0, percent: 99 } }], + ["event loop lag", { eventLoopLagMs: 5000 }], + ["connection", { connectionState: "disconnected" }], + ["memory", { memory: { heapUsed: 99, heapTotal: 100, rss: 99, external: 0 } }], + ])("never escalates on a %s critical while the KV probe is healthy", (_label, over) => { + const state = hasSeenHealthyKvProbe(); + for (let i = 0; i < 50; i++) { + expect( + bumpEscalation(snap({ ...over, status: "critical" } as Partial), state, 3), + ).toBe(false); + } + expect(state.consecutiveKvProbeFailures).toBe(0); + }); + + it("treats an absent kvConnectivity as no signal, not as a stall", () => { + const state = hasSeenHealthyKvProbe(); + const s = snap(); + delete s.kvConnectivity; + for (let i = 0; i < 50; i++) { + expect(bumpEscalation(s, state, 3)).toBe(false); + } + expect(state.consecutiveKvProbeFailures).toBe(0); + }); +}); diff --git a/test/health-thresholds.test.ts b/test/health-thresholds.test.ts index 6f918a9174..c30f0e652e 100644 --- a/test/health-thresholds.test.ts +++ b/test/health-thresholds.test.ts @@ -17,81 +17,100 @@ function snap(over: Partial = {}): HealthSnapshot { }; } -describe("evaluateHealth memory severity", () => { - it("stays healthy when heap fills a tiny steady-state process (issue #158)", () => { - const s = snap({ - memory: { - heapUsed: 45 * 1024 * 1024, - heapTotal: 46 * 1024 * 1024, - rss: 120 * 1024 * 1024, - external: 0, - }, - }); - const { status, alerts, notes } = evaluateHealth(s); - expect(status).toBe("healthy"); - expect(alerts.find((a) => a.startsWith("memory_critical_"))).toBeUndefined(); - expect(alerts.find((a) => a.startsWith("memory_warn_"))).toBeUndefined(); - expect(alerts.find((a) => a.startsWith("memory_heap_tight_"))).toBeUndefined(); - expect(notes.find((n) => n.startsWith("memory_heap_tight_"))).toBeDefined(); +describe("independent real memory budgets", () => { + it.each([ + [390_771_624, 404_930_560, 6_492_782_592], + [267_970_328, 277_663_744, 2_197_815_296], + ])("keeps the reported committed heap %s healthy", (heapUsed, heapTotal, heapSizeLimit) => { + expect(evaluateHealth(snap({ memory: { heapUsed, heapTotal, heapSizeLimit, rss: 600_000_000, external: 0 } })).status).toBe("healthy"); + }); + + it("does not invent a budget for old snapshots", () => { + const result = evaluateHealth(snap({ memory: { heapUsed: 970, heapTotal: 1000, rss: 900_000_000, external: 0 } })); + expect(result.status).toBe("healthy"); + expect(result.notes).toContain("memory_unavailable_heap"); + }); + + it.each([[80, "healthy"], [81, "degraded"], [95, "degraded"], [96, "critical"]])("evaluates small heaps at %s percent without an RSS floor", (heapUsed, expected) => { + expect(evaluateHealth(snap({ memory: { heapUsed: Number(heapUsed), heapTotal: 100, heapSizeLimit: 100, rss: 200, external: 0 } })).status).toBe(expected); }); - it("goes critical when heap ratio is high AND RSS is above the floor", () => { + it("keeps heap and cgroup signals independent", () => { + const memory = { heapUsed: 96, heapTotal: 100, heapSizeLimit: 100, rss: 200, external: 0, cgroup: { status: "available" as const, levels: [{ path: "/", current: 100, max: 1000 }] } }; + expect(evaluateHealth(snap({ memory })).status).toBe("critical"); + memory.heapUsed = 10; + memory.cgroup.levels[0].current = 960; + expect(evaluateHealth(snap({ memory })).status).toBe("critical"); + }); + + it("reports memory.high as warning even beyond its boundary", () => { + const result = evaluateHealth(snap({ memory: { heapUsed: 1, heapTotal: 100, external: 0, rss: 2, cgroup: { status: "available", levels: [{ path: "/", current: 110, high: 100 }] } } })); + expect(result.status).toBe("degraded"); + expect(result.alerts).toContain("memory_warn_cgroup-high/_110%"); + }); + + it("evaluates an explicit RSS budget alongside low cgroup occupancy", () => { + const result = evaluateHealth(snap({ memory: { heapUsed: 1, heapTotal: 100, external: 0, rss: 110, cgroup: { status: "available", levels: [{ path: "/", current: 110, max: 1000 }] } } }), { memoryRssBudgetBytes: 100 }); + expect(result.status).toBe("critical"); + expect(result.alerts).toContain("memory_critical_rss_110%"); + }); + + it.each([0, -1, Infinity, NaN])("rejects invalid V8 limit %s", heapSizeLimit => { + expect(evaluateHealth(snap({ memory: { heapUsed: 99, heapTotal: 100, heapSizeLimit, rss: 1_000_000_000, external: 0 } })).status).toBe("healthy"); + }); +}); + +describe("non-memory signals", () => { + it.each([ + { cpu: { userMicros: 0, systemMicros: 0, percent: 91 } }, + { eventLoopLagMs: 501 }, + { connectionState: "failed" }, + ])("keeps critical conditions immediate during memory entry: %s", other => { + expect(evaluateHealth(snap(other), {}, [{ source: "heap", available: true, percent: 99, severity: "degraded", transition: "entering" }]).status).toBe("critical"); + }); +}); + +describe("evaluateHealth KV connectivity", () => { + it("goes critical when the KV probe fails", () => { const s = snap({ - memory: { - heapUsed: 970 * 1024 * 1024, - heapTotal: 1000 * 1024 * 1024, - rss: 1100 * 1024 * 1024, - external: 0, - }, + kvConnectivity: { status: "error", error: "kv_probe_failed", latencyMs: 5000 }, }); const { status, alerts } = evaluateHealth(s); expect(status).toBe("critical"); - expect(alerts.some((a) => a.startsWith("memory_critical_"))).toBe(true); + expect(alerts).toContain("kv_probe_failed"); }); - it("records heap_tight in the warn band when RSS is below the floor", () => { - const s = snap({ - memory: { - heapUsed: 85 * 1024 * 1024, - heapTotal: 100 * 1024 * 1024, - rss: 50 * 1024 * 1024, - external: 0, - }, - }); - const { status, alerts, notes } = evaluateHealth(s); + it("adds no KV alert when the probe succeeds", () => { + const { status, alerts } = evaluateHealth(snap()); expect(status).toBe("healthy"); - expect(notes.some((n) => n.startsWith("memory_heap_tight_"))).toBe(true); - expect(alerts.some((a) => a.startsWith("memory_heap_tight_"))).toBe(false); - expect(alerts.some((a) => a.startsWith("memory_warn_"))).toBe(false); - expect(alerts.some((a) => a.startsWith("memory_critical_"))).toBe(false); + expect(alerts.find((a) => a.startsWith("kv_"))).toBeUndefined(); }); - it("goes degraded when heap is above warn AND RSS is above the floor", () => { - const s = snap({ - memory: { - heapUsed: 850 * 1024 * 1024, - heapTotal: 1000 * 1024 * 1024, - rss: 900 * 1024 * 1024, - external: 0, - }, - }); - const { status, alerts } = evaluateHealth(s, { memoryRssFloorBytes: 800 * 1024 * 1024 }); - expect(status).toBe("degraded"); - expect(alerts.some((a) => a.startsWith("memory_warn_"))).toBe(true); + // kvConnectivity is optional on HealthSnapshot, and snapshots persisted before + // the field existed still come back from KV, so absence must read as "no + // signal" rather than as a failure. + it("stays healthy when kvConnectivity is absent", () => { + const s = snap(); + delete s.kvConnectivity; + const { status, alerts } = evaluateHealth(s); + expect(status).toBe("healthy"); + expect(alerts.find((a) => a.startsWith("kv_"))).toBeUndefined(); }); - it("respects caller-supplied memoryRssFloorBytes", () => { + it("does not throw or alert on a malformed kvConnectivity", () => { + const s = snap({ kvConnectivity: { status: undefined as unknown as string } }); + expect(() => evaluateHealth(s)).not.toThrow(); + expect(evaluateHealth(s).alerts.find((a) => a.startsWith("kv_"))).toBeUndefined(); + }); + + it("reports the KV alert alongside other critical signals", () => { const s = snap({ - memory: { - heapUsed: 98, - heapTotal: 100, - rss: 50 * 1024 * 1024, - external: 0, - }, + kvConnectivity: { status: "error", error: "kv_probe_failed" }, + eventLoopLagMs: 900, }); - const loose = evaluateHealth(s, { memoryRssFloorBytes: 10 * 1024 * 1024 }); - expect(loose.status).toBe("critical"); - const strict = evaluateHealth(s, { memoryRssFloorBytes: 1024 * 1024 * 1024 }); - expect(strict.status).toBe("healthy"); + const { status, alerts } = evaluateHealth(s); + expect(status).toBe("critical"); + expect(alerts).toContain("kv_probe_failed"); + expect(alerts.find((a) => a.startsWith("event_loop_lag_critical_"))).toBeDefined(); }); }); diff --git a/test/index-persistence.test.ts b/test/index-persistence.test.ts index 929791657f..4da40fd6f4 100644 --- a/test/index-persistence.test.ts +++ b/test/index-persistence.test.ts @@ -2,7 +2,7 @@ import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"; import { IndexPersistence } from "../src/state/index-persistence.js"; import { SearchIndex } from "../src/state/search-index.js"; import { VectorIndex } from "../src/state/vector-index.js"; -import type { CompressedObservation } from "../src/types.js"; +import type { AuditEntry, CompressedObservation } from "../src/types.js"; const BM25_SCOPE = "mem:index:bm25"; const BM25_LEGACY_KEY = "data"; @@ -563,6 +563,454 @@ describe("IndexPersistence", () => { expect(loaded.bm25!.search("alpha").length).toBe(0); }); + it("reclaims the previous generation when the previous manifest read fails (#1115)", async () => { + const previous = makeBm25("obs_old", "alpha previous snapshot"); + await new IndexPersistence(kv as never, previous, null, { + shardChars: 80, + createGeneration: () => "gen_old", + }).save(); + const oldShardScope = "mem:index:bm25:bm25:gen_old:00000"; + await expect(kv.get(oldShardScope, "data")).resolves.not.toBeNull(); + + // The manifest read that opens saveShardedIndex times out. It used to be + // swallowed into `previous = null`, which skipped the cleanup guard and + // stranded gen_old's shards with nothing left to ever revisit them. + const readFailsKv = { + ...kv, + get: vi.fn(async (scope: string, key: string): Promise => { + if (scope === BM25_SCOPE && key === BM25_MANIFEST_KEY) { + throw new Error("Invocation timeout after 180000ms: state::get"); + } + return kv.get(scope, key); + }), + }; + + const next = makeBm25("obs_new", "bravo new snapshot"); + await new IndexPersistence(readFailsKv as never, next, null, { + shardChars: 80, + createGeneration: () => "gen_new", + }).save(); + + const manifest = await getBm25Manifest(kv); + expect(manifest.generation).toBe("gen_new"); + await expect(kv.get(oldShardScope, "data")).resolves.toBeNull(); + }); + + it("reclaims a generation stranded by a failed cleanup on the next load (#1115)", async () => { + const previous = makeBm25("obs_old", "alpha previous snapshot"); + await new IndexPersistence(kv as never, previous, null, { + shardChars: 80, + createGeneration: () => "gen_old", + }).save(); + const oldShardScope = "mem:index:bm25:bm25:gen_old:00000"; + + const cleanupKv = { + ...kv, + delete: vi.fn(async () => { + throw new Error("cleanup failed"); + }), + }; + const next = makeBm25("obs_new", "bravo new snapshot"); + await new IndexPersistence(cleanupKv as never, next, null, { + shardChars: 80, + createGeneration: () => "gen_new", + }).save(); + + // Cleanup failed, so gen_old is still on disk. Nothing in the save path + // will revisit it — a later save only ever inspects its own predecessor. + await expect(kv.get(oldShardScope, "data")).resolves.not.toBeNull(); + + const loaded = await new IndexPersistence( + kv as never, + new SearchIndex(), + null, + ).load(); + expect(loaded.bm25!.search("bravo").length).toBe(1); + await expect(kv.get(oldShardScope, "data")).resolves.toBeNull(); + }); + + it("bounds concurrent reclaim deletes and retries failures without touching live or future shards", async () => { + await new IndexPersistence(kv as never, makeBm25("obs_live", "live snapshot"), null, { + createGeneration: () => "gen_live", + }).save(); + const live = await getBm25Manifest(kv); + const retired = Array.from({ length: 19 }, (_, i) => ({ scope: `retired:${i}`, key: "data" })); + const future = { scope: "future:0", key: "data" }; + for (const shard of [...retired, future]) await kv.set(shard.scope, shard.key, "shard"); + const gcKey = `${BM25_MANIFEST_KEY}:gc`; + const ledger = { + v: 1, + generations: [ + { generation: "gen_old", shards: retired }, + { generation: "gen_live", shards: live.shards.map(({ scope, key }) => ({ scope, key })) }, + { generation: "gen_future", shards: [future] }, + ], + }; + await kv.set(BM25_SCOPE, gcKey, ledger); + const failedScopes = new Set([retired[1].scope, retired[10].scope]); + const pending: Array<() => void> = []; + let active = 0; + let peak = 0; + const boundedKv = { + ...kv, + delete: vi.fn(async (scope: string, key: string) => { + active++; + peak = Math.max(peak, active); + try { + await new Promise(resolve => pending.push(resolve)); + if (failedScopes.has(scope)) throw new Error("retry delete"); + await kv.delete(scope, key); + } finally { active--; } + }), + }; + const loading = new IndexPersistence(boundedKv as never, new SearchIndex(), null).load(); + for (const size of [8, 8, 3]) { + await vi.waitFor(() => expect(pending).toHaveLength(size)); + expect(active).toBe(size); + for (const resolve of pending.splice(0).reverse()) resolve(); + } + const loaded = await loading; + expect(peak).toBe(8); + expect(loaded.bm25!.search("live")).toHaveLength(1); + expect(boundedKv.delete.mock.calls.map(([scope, key]) => ({ scope, key }))).toEqual(retired); + expect(await kv.get(BM25_SCOPE, gcKey)).toEqual({ + v: 1, + generations: [ + { generation: "gen_old", shards: [retired[1], retired[10]] }, + ...ledger.generations.slice(1), + ], + }); + const sweeps = (await kv.list("mem:audit")) + .filter(entry => entry.details.reason === "generation_reclaim"); + expect(sweeps).toHaveLength(1); + expect(sweeps[0].details).toMatchObject({ evicted: 17, failed: 2 }); + expect(sweeps[0].targetIds).toEqual(retired + .filter(shard => !failedScopes.has(shard.scope)) + .map(shard => `${shard.scope}/${shard.key}`)); + + const retryKv = { ...kv, delete: vi.fn(kv.delete) }; + await new IndexPersistence(retryKv as never, new SearchIndex(), null).load(); + expect(retryKv.delete.mock.calls).toEqual([ + [retired[1].scope, "data"], [retired[10].scope, "data"], + ]); + expect(await kv.get(BM25_SCOPE, gcKey)).toEqual({ v: 1, generations: ledger.generations.slice(1) }); + for (const shard of retired) expect(await kv.get(shard.scope, shard.key)).toBeNull(); + for (const shard of [...live.shards, future]) { + expect(await kv.get(shard.scope, shard.key)).not.toBeNull(); + } + }); + + it("reclaims the previous vector generation when the vector manifest read fails (#1115)", async () => { + await new IndexPersistence( + kv as never, + makeBm25("obs_old", "alpha previous snapshot"), + makeVector("obs_old"), + { shardChars: 80, createGeneration: () => "gen_old" }, + ).save(); + const oldVectorScope = "mem:index:bm25:vectors:gen_old:00000"; + await expect(kv.get(oldVectorScope, "data")).resolves.not.toBeNull(); + + const readFailsKv = { + ...kv, + get: vi.fn(async (scope: string, key: string): Promise => { + if (scope === BM25_SCOPE && key === VECTOR_MANIFEST_KEY) { + throw new Error("Invocation timeout after 180000ms: state::get"); + } + return kv.get(scope, key); + }), + }; + + await new IndexPersistence( + readFailsKv as never, + makeBm25("obs_new", "bravo new snapshot"), + makeVector("obs_new"), + { shardChars: 80, createGeneration: () => "gen_new" }, + ).save(); + + await expect(kv.get(oldVectorScope, "data")).resolves.toBeNull(); + }); + + it("leaves a generation recorded after the live one untouched on load (#1115)", async () => { + await new IndexPersistence( + kv as never, + makeBm25("obs_old", "alpha previous snapshot"), + null, + { shardChars: 80, createGeneration: () => "gen_live" }, + ).save(); + + // A concurrent save has recorded its generation and is mid-write, but has + // not published its manifest yet. setIndexPersistence runs before load() + // in src/index.ts, so a request arriving during boot produces exactly this. + const inflightScope = "mem:index:bm25:bm25:gen_inflight:00000"; + await kv.set(inflightScope, "data", "partial shard"); + const gcKey = `${BM25_MANIFEST_KEY}:gc`; + const ledger = await kv.get<{ + v: 1; + generations: Array<{ + generation: string; + shards: Array<{ scope: string; key: string }>; + }>; + }>(BM25_SCOPE, gcKey); + ledger!.generations.push({ + generation: "gen_inflight", + shards: [{ scope: inflightScope, key: "data" }], + }); + await kv.set(BM25_SCOPE, gcKey, ledger); + + await new IndexPersistence(kv as never, new SearchIndex(), null).load(); + + // Deleting it would leave the in-flight save publishing a manifest whose + // shards are already gone, which fails closed on the next load. + await expect(kv.get(inflightScope, "data")).resolves.toBe("partial shard"); + }); + + it("reclaims a pre-ledger manifest that carries no generation (#1115)", async () => { + const legacyScope = "mem:index:bm25:bm25:gen_legacy:00000"; + await kv.set(legacyScope, "data", "legacy shard"); + await kv.set(BM25_SCOPE, BM25_MANIFEST_KEY, { + v: 1, + shards: [{ scope: legacyScope, key: "data", chars: 12 }], + chars: 12, + }); + + await new IndexPersistence( + kv as never, + makeBm25("obs_new", "bravo new snapshot"), + null, + { shardChars: 80, createGeneration: () => "gen_new" }, + ).save(); + + await expect(kv.get(legacyScope, "data")).resolves.toBeNull(); + }); + + it("keeps a published manifest whole when two saves overlap (#1115)", async () => { + // scheduleSave fires save() unawaited from a timer while flushIndexSave + // awaits save() on every delete path, so two saves on ONE instance is the + // normal shape, not a contrivance. Without the queue, whichever published + // second reclaimed the other's shards and left it naming data already gone. + vi.useRealTimers(); + const store = new Map>(); + const slowKv = { + get: async (scope: string, key: string): Promise => + (store.get(scope)?.get(key) as T) ?? null, + set: async (scope: string, key: string, data: T): Promise => { + // Stall one of the first generation's shard writes so the second save + // overtakes it. + if (scope.includes(":gen_0:") && scope.endsWith("00005")) { + await new Promise((resolve) => setTimeout(resolve, 200)); + } + if (!store.has(scope)) store.set(scope, new Map()); + store.get(scope)!.set(key, data); + return data; + }, + delete: async (scope: string, key: string): Promise => { + store.get(scope)?.delete(key); + }, + list: async (scope: string): Promise => + Array.from((store.get(scope)?.values() ?? []) as Iterable), + }; + + const bm25 = new SearchIndex(); + for (let i = 0; i < 30; i++) { + bm25.add( + makeObs({ id: `obs_${i}`, title: `lorem ipsum dolor sit amet ${i}` }), + ); + } + let generation = 0; + const persistence = new IndexPersistence(slowKv as never, bm25, null, { + shardChars: 400, + createGeneration: () => `gen_${generation++}`, + }); + + // Let the first save get into its shard writes before the second starts, + // so the second is the one that publishes. + const first = persistence.save(); + await new Promise((resolve) => setTimeout(resolve, 20)); + const second = persistence.save(); + await Promise.all([first, second]); + + const manifest = await slowKv.get( + BM25_SCOPE, + BM25_MANIFEST_KEY, + ); + const missing: string[] = []; + for (const shard of manifest!.shards) { + if ((await slowKv.get(shard.scope, shard.key)) === null) { + missing.push(shard.scope); + } + } + expect(missing).toEqual([]); + + const loaded = await new IndexPersistence( + slowKv as never, + new SearchIndex(), + null, + ).load(); + expect(loaded.bm25).not.toBeNull(); + }); + + it("still persists the index when the gc ledger is unreadable (#1115)", async () => { + // state::get timing out is the condition this bug appears under. Aborting + // the save there would stop persisting entirely, which is worse than the + // leak: a lost BM25 index costs a full-corpus rebuild. + const failingKv = { + ...kv, + get: vi.fn(async (scope: string, key: string): Promise => { + if (scope === BM25_SCOPE && key === `${BM25_MANIFEST_KEY}:gc`) { + throw new Error("Invocation timeout after 180000ms: state::get"); + } + return kv.get(scope, key); + }), + }; + + await new IndexPersistence( + failingKv as never, + makeBm25("obs_new", "bravo new snapshot"), + null, + { shardChars: 80, createGeneration: () => "gen_new" }, + ).save(); + + const manifest = await getBm25Manifest(kv); + expect(manifest.generation).toBe("gen_new"); + await expect( + kv.get(manifest.shards[0].scope, manifest.shards[0].key), + ).resolves.not.toBeNull(); + }); + + it("still persists when the gc ledger holds a malformed entry (#1115)", async () => { + // A null entry used to throw a TypeError out of saveShardedIndex before any + // shard write, leaving one throttled log line per 60s while BM25 silently + // stopped persisting for good. + await kv.set(BM25_SCOPE, `${BM25_MANIFEST_KEY}:gc`, { + v: 1, + generations: [null], + }); + + await new IndexPersistence( + kv as never, + makeBm25("obs_new", "bravo new snapshot"), + null, + { shardChars: 80, createGeneration: () => "gen_new" }, + ).save(); + + const manifest = await getBm25Manifest(kv); + expect(manifest.generation).toBe("gen_new"); + await expect( + kv.get(manifest.shards[0].scope, manifest.shards[0].key), + ).resolves.not.toBeNull(); + }); + + it("still reclaims when the gc ledger holds a malformed entry (#1115)", async () => { + // Publishing is not enough. A malformed entry that silently disabled + // reclaim would leak a generation every cycle while this test stayed green. + await new IndexPersistence(kv as never, makeBm25("obs_old", "alpha"), null, { + shardChars: 80, + createGeneration: () => "gen_old", + }).save(); + const oldShardScope = "mem:index:bm25:bm25:gen_old:00000"; + + const gcKey = `${BM25_MANIFEST_KEY}:gc`; + const ledger = await kv.get<{ v: 1; generations: unknown[] }>( + BM25_SCOPE, + gcKey, + ); + ledger!.generations.unshift(null); + await kv.set(BM25_SCOPE, gcKey, ledger); + + await new IndexPersistence(kv as never, makeBm25("obs_new", "bravo"), null, { + shardChars: 80, + createGeneration: () => "gen_new", + }).save(); + + await expect(kv.get(oldShardScope, "data")).resolves.toBeNull(); + }); + + it("falls back to previous-generation cleanup when the ledger is unusable (#1115)", async () => { + // Measured regression guard: with no fallback, an unusable ledger leaked + // one generation per cycle — strictly worse than the code this replaced, + // which always had this path. Without this test the `else if` can be + // deleted with the whole suite green. + await new IndexPersistence(kv as never, makeBm25("obs_old", "alpha"), null, { + shardChars: 80, + createGeneration: () => "gen_old", + }).save(); + const oldShardScope = "mem:index:bm25:bm25:gen_old:00000"; + await expect(kv.get(oldShardScope, "data")).resolves.not.toBeNull(); + + // Ledger read fails, manifest read succeeds. Both are state::get with + // independent timeouts, so this split is ordinary, not contrived. + const ledgerFailsKv = { + ...kv, + get: vi.fn(async (scope: string, key: string): Promise => { + if (scope === BM25_SCOPE && key === `${BM25_MANIFEST_KEY}:gc`) { + throw new Error("Invocation timeout after 180000ms: state::get"); + } + return kv.get(scope, key); + }), + }; + + await new IndexPersistence( + ledgerFailsKv as never, + makeBm25("obs_new", "bravo"), + null, + { shardChars: 80, createGeneration: () => "gen_new" }, + ).save(); + + const manifest = await getBm25Manifest(kv); + expect(manifest.generation).toBe("gen_new"); + await expect(kv.get(oldShardScope, "data")).resolves.toBeNull(); + }); + + it("keeps the ledger entry when a rollback delete fails (#1115)", async () => { + // Shards that survived their rollback delete must keep the entry that + // names them, or nothing can ever reclaim them. + let shardWrites = 0; + const failingKv = { + ...kv, + set: vi.fn(async (scope: string, key: string, data: T): Promise => { + if (scope.includes(":gen_new:")) { + shardWrites += 1; + if (shardWrites === 2) throw new Error("shard write failed"); + } + return kv.set(scope, key, data); + }), + delete: vi.fn(async () => { + throw new Error("rollback delete failed"); + }), + }; + + await new IndexPersistence( + failingKv as never, + makeBm25("obs_new", "bravo new snapshot"), + null, + { shardChars: 80, createGeneration: () => "gen_new" }, + ).save(); + + const ledger = await kv.get<{ + v: 1; + generations: Array<{ generation: string }>; + }>(BM25_SCOPE, `${BM25_MANIFEST_KEY}:gc`); + expect(ledger!.generations.map((entry) => entry.generation)).toContain( + "gen_new", + ); + }); + + it("refuses to overwrite a gc ledger it does not recognise (#1115)", async () => { + const gcKey = `${BM25_MANIFEST_KEY}:gc`; + const future = { v: 2, generations: [], writtenByANewerBuild: true }; + await kv.set(BM25_SCOPE, gcKey, future); + + await new IndexPersistence( + kv as never, + makeBm25("obs_new", "bravo new snapshot"), + null, + { shardChars: 80, createGeneration: () => "gen_new" }, + ).save(); + + // Rewriting it as a v1 ledger would drop every generation it tracked. + await expect(kv.get(BM25_SCOPE, gcKey)).resolves.toEqual(future); + }); + it("keeps the previous vector generation when vector save fails after BM25 publish", async () => { const previousBm25 = makeBm25("obs_old", "alpha previous snapshot"); const previousVector = makeVector("obs_old"); diff --git a/test/resilient-provider.test.ts b/test/resilient-provider.test.ts new file mode 100644 index 0000000000..1b09e21c0c --- /dev/null +++ b/test/resilient-provider.test.ts @@ -0,0 +1,135 @@ +import { describe, it, expect, vi } from "vitest"; +import { ResilientProvider } from "../src/providers/resilient.js"; +import type { MemoryProvider } from "../src/types.js"; + +// Tracks how many calls are inside the provider at once, which is the thing the +// upstream API actually rejects. Counting total calls would not discriminate. +function countingProvider( + behaviour: (n: number) => Promise = async () => "ok", +): MemoryProvider & { peak: number; calls: number } { + let inFlight = 0; + const state = { + name: "counting", + peak: 0, + calls: 0, + async compress(): Promise { + inFlight++; + state.calls++; + if (inFlight > state.peak) state.peak = inFlight; + try { + return await behaviour(state.calls); + } finally { + inFlight--; + } + }, + async summarize(): Promise { + return state.compress(); + }, + }; + return state; +} + +function rateLimited(): Error { + return new Error('OpenAI API error (429): {"error":"too many concurrent requests"}'); +} + +describe("ResilientProvider concurrency", () => { + it("never runs more than the configured number of calls at once", async () => { + const inner = countingProvider( + () => new Promise((resolve) => setTimeout(() => resolve("ok"), 5)), + ); + const provider = new ResilientProvider(inner, { maxConcurrent: 3 }); + + const results = await Promise.all( + Array.from({ length: 24 }, () => provider.compress("sys", "user")), + ); + + expect(inner.peak).toBeLessThanOrEqual(3); + // Bounding must not drop work: every call still ran and still resolved. + expect(inner.calls).toBe(24); + expect(results.every((r) => r === "ok")).toBe(true); + }); + + it("releases its slot when a call throws", async () => { + const inner = countingProvider(async (n) => { + if (n <= 2) throw new Error("boom"); + return "ok"; + }); + // maxConcurrent 1 is load-bearing here: at the default of 4 a leaked slot + // would not deadlock, and the test would pass despite the bug. + const provider = new ResilientProvider(inner, { maxConcurrent: 1 }); + + const settled = await Promise.allSettled( + Array.from({ length: 5 }, () => provider.compress("sys", "user")), + ); + + expect(settled).toHaveLength(5); + expect(settled.filter((s) => s.status === "fulfilled")).toHaveLength(3); + }); + + it("rejects queued calls when earlier failures open the breaker without recording new failures", async () => { + const inner = countingProvider(async n => { + if (n <= 3) throw new Error("upstream exploded"); + return "recovered"; + }); + const provider = new ResilientProvider(inner, { maxConcurrent: 1 }); + let now = 1000; + const clock = vi.spyOn(Date, "now").mockImplementation(() => now++); + try { + const settled = await Promise.allSettled( + Array.from({ length: 6 }, (_, i) => i % 2 + ? provider.summarize("sys", "user") : provider.compress("sys", "user")), + ); + + expect(inner.calls).toBe(3); + expect(settled.map(result => result.status === "rejected" ? result.reason.message : result.value)).toEqual([ + "upstream exploded", "upstream exploded", "upstream exploded", + "circuit_breaker_open", "circuit_breaker_open", "circuit_breaker_open", + ]); + expect(provider.circuitState).toEqual({ + state: "open", failures: 3, lastFailureAt: 1002, openedAt: 1002, + }); + + now = 31_002; + await expect(provider.compress("sys", "user")).resolves.toBe("recovered"); + expect(inner.calls).toBe(4); + expect(provider.circuitState.state).toBe("closed"); + } finally { + clock.mockRestore(); + } + }); +}); + +describe("ResilientProvider rate limiting", () => { + it("does not open the breaker on 429s", async () => { + const inner = countingProvider(async (n) => { + if (n <= 5) throw rateLimited(); + return "ok"; + }); + const provider = new ResilientProvider(inner); + + // Five, against a default failure threshold of three. + for (let i = 0; i < 5; i++) { + await provider.compress("sys", "user").catch(() => undefined); + } + + expect(provider.circuitState.state).toBe("closed"); + await expect(provider.compress("sys", "user")).resolves.toBe("ok"); + }); + + it("still opens the breaker on genuine failures", async () => { + const inner = countingProvider(async () => { + throw new Error("upstream exploded"); + }); + const provider = new ResilientProvider(inner); + + for (let i = 0; i < 3; i++) { + await provider.compress("sys", "user").catch(() => undefined); + } + + expect(provider.circuitState.state).toBe("open"); + await expect(provider.compress("sys", "user")).rejects.toThrow( + "circuit_breaker_open", + ); + }); +}); diff --git a/test/viewer-heap-gauge.test.ts b/test/viewer-heap-gauge.test.ts new file mode 100644 index 0000000000..f07247e3ae --- /dev/null +++ b/test/viewer-heap-gauge.test.ts @@ -0,0 +1,80 @@ +import { describe, it, expect } from "vitest"; +import { readFileSync } from "node:fs"; +import { runInNewContext } from "node:vm"; + +const viewer = readFileSync("src/viewer/index.html", "utf8"); +function render(memory: Record): string { + const start = viewer.indexOf(" function renderMemoryResources(memory) {"); + const end = viewer.indexOf("\n }", start) + 6; + expect(start).toBeGreaterThan(-1); + return runInNewContext(`${viewer.slice(start, end)}; renderMemoryResources(memory)`, { + memory, + esc: (s: unknown) => String(s).replace(/[&<>"']/g, c => ({ + "&": "&", "<": "<", ">": ">", '"': """, "'": "'", + })[c]), + }); +} +const mb = 1024 * 1024; +const base = { heapUsed: 45 * mb, heapTotal: 46 * mb, rss: 511.5 * mb }; +const evaluation = { + source: "heap", usedBytes: 45 * mb, limitBytes: 100 * mb, + percent: 45, severity: "healthy", available: true, +}; + +describe("viewer memory resources", () => { + it("uses server severity even when RSS rounds across the former floor", () => { + const html = render({ ...base, evaluations: [{ ...evaluation, percent: 80.4, severity: "degraded" }] }); + expect(html).toContain("var(--yellow)"); + expect(html).toContain("80.4%"); + expect(html).not.toContain("var(--red)"); + }); + it("shows held recovery as critical rather than recoloring a low percentage", () => { + const html = render({ ...base, evaluations: [{ ...evaluation, severity: "critical", transition: "recovering" }] }); + expect(html).toContain("var(--red)"); + expect(html).toContain("Recovery pending"); + }); + it("shows entering critical as pending and caps only the drawn width", () => { + const html = render({ ...base, evaluations: [{ ...evaluation, percent: 102.2, severity: "degraded", transition: "entering" }] }); + expect(html).toContain("width:100%"); + expect(html).toContain("102.2%"); + expect(html).toContain("Critical pending"); + }); + it("renders old snapshots without an invented pressure limit", () => { + const html = render(base); + expect(html).toContain("Memory pressure unavailable"); + expect(html).toContain("Heap: 45 MiB"); + expect(html).not.toContain("var(--red)"); + }); + it("keeps unavailable latched signals visible and escapes their path", () => { + const html = render({ ...base, evaluations: [{ source: "cgroup-max", path: '', severity: "critical", available: false, transition: "unavailable" }] }); + expect(html).toContain("var(--red)"); + expect(html).toContain("Measurement unavailable"); + expect(html).toContain("<img"); + expect(html).not.toContain(" { + expect(render({ ...base, evaluations: [{ ...evaluation, source: "cgroup-high", severity: "degraded" }] })).toContain("Container throttle"); + }); +}); + + +describe("viewer memory alerts", () => { + const start = viewer.indexOf(" function humanizeHealthFlag(f) {"); + const end = viewer.indexOf("\n }", start) + 6; + const humanize = (f: string) => runInNewContext(`${viewer.slice(start, end)}; humanizeHealthFlag(f)`, { f }); + it.each([ + ["memory_critical_heap_96%", "V8 heap", "96%"], + ["memory_warn_cgroup-high/_110%", "Container throttle", "110%"], + ["memory_entering_rss", "RSS budget", "Critical pending"], + ["memory_recovering_heap", "V8 heap", "Recovery pending"], + ["memory_unavailable_cgroup-max/parent", "Container limit", "Measurement unavailable"], + ])("describes %s", (slug, source, detail) => { + expect(humanize(slug)).toContain(source); + expect(humanize(slug)).toContain(detail); + expect(humanize(slug)).not.toBe(slug); + }); + it("keeps legacy snapshots readable", () => { + expect(humanize("memory_critical_97%_rss557mb")).toContain("process memory 557 MB"); + }); +});