diff --git a/CHANGELOG.md b/CHANGELOG.md index b4fbed3102..26b1bd629c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ Versions follow the merge of a `*_release-v*` branch; CI publishes to npm on tag ## [Unreleased] +- **Session-file zstd compression, opt-in; decoupled from encryption (#1080)**: set `BILI_PERSIST_ZSTD=1` to compress session files — each file that actually shrinks is written as `BILIZSTD1`, a magic + format-version + mode-byte header over a JSON body that is zstd-compressed on Node ≥ 22.15 and stored raw on older runtimes; readers accept both bodies (native zstd, else the bundled fzstd WASM fallback), so files stay readable across versions and runtimes. **The default stays plain JSON** (owner decision: recoverability — jq/grep-debuggable, no downgrade tail risk — beats silent disk savings; the format change then only ever happens for deployments that opted in). `BILI_ENCRYPTION_KEY` (#708) is now an orthogonal knob: `BILIENC1` bytes are unchanged, its mode byte records whether the body is compressed (the opt-in applies inside it too), existing plain-JSON files are never rewritten at boot (downgrade safety) and convert organically on their next save. `createSessionCodec(key)` is replaced by `createStorageCodec({ key?, compress? })` (no codec when neither is active; both default off). Tests cover round-trip, mixed trees, opt-in/opt-out, raw fallback, cross-runtime decode, never-rewrite-at-boot, and organic conversion on next save. + - **Marker-integrity silence clause now follows `visibilityMarkers` (#913)**: `MARKER_INTEGRITY_NOTE` was one unconditional constant carrying two rules — the #717 anti-forgery segment (never write a marker-style line as your own text) and the #862 silence clause (execute compress-class calls silently, no preamble before / no summary after). With `compress.visibilityMarkers: false` the proxy stops emitting markers entirely, yet the silence clause kept riding every nudge and the injected compress prompt — an instruction about artifacts that deployment never shows. The constant is split (`MARKER_INTEGRITY_NOTE` base + `MARKER_SILENCE_CLAUSE`), and `withMarkerIntegrityNote(text, visibilityMarkers = true)` appends the clause only while markers are visible. The flag is resolved once per request at the prepare dispatch (`resolveCompress(...).visibilityMarkers ?? true`) and threaded through `prepareAnthropic` / `prepareOpenai` / `prepareGoogle` / `prepareResponses` / `injectSystem` plus the compress-loop systemPrompt build, so first request and loop re-requests agree (per-config byte-stable anchor preserved; default output byte-identical to before). The #717 anti-forgery segment stays unconditional in every configuration. Tests pin: clause dropped at `false`, base intact, pure-suffix byte-stability, default-arg legacy equality. - **Failed-compress receipts now name the live compressible span (#1026, slimmed port of PR #1008 by asuan-dev)**: a model whose compress call fails kept anchoring on refs an earlier fold already consumed — it re-issued the same dead range, burned retries, and eventually gave up and let the nudge re-fire every turn (observed live 2026-09-18: three consecutive failed compresses, two full T1 checkpoints wasted). Both failure receipts (`no valid ranges parsed` and `blocksCreated === 0`) now append `compressibleSpanHint(state)` — the raw-ref span NOT covered by an ACTIVE block right now (`mXXXXX–mYYYYY`, computed as highest mapped ref minus the newest active-block boundary) — so one retry lands on a valid range without another acp_status round-trip. Boundary counts ACTIVE blocks only: after a decompress, the restored span shows as compressible again, which is exactly the recoverable truth. When no raw refs are compressible (everything folded), the receipt names the active blocks' real ids (`startId b7, endId b11`) and suggests folding them into a higher-tier block — a kernel-first-class operation (b-ranges expand to their `effectiveMessageIds` and merge tiers). Coordinates with the #849 repeat-failure guard: the guard forbids repeating the same dead ranges, the hint hands over the live ones. Port keeps only PR #1008's head commit (e4a4700); its bundled #1003/#1004/#1006 content stays out (superseded by #740 / closed in the PR-queue triage). Tests pin: span past the highest ACTIVE block, inactive blocks not holding the boundary (decompress releases it), exhausted fallback naming real active block ids, and both receipt paths carrying the span. diff --git a/CONFIGURATION.md b/CONFIGURATION.md index 7407928e12..d8b112f44f 100644 --- a/CONFIGURATION.md +++ b/CONFIGURATION.md @@ -516,12 +516,13 @@ Environment variables take precedence over the config file. They are useful for | `BILI_PERSIST` | Set `0` to disable session persistence (in-memory only, lost on restart). | | `BILI_PERSIST_DEBOUNCE_MS` | Debounce window for persistence writes to disk, in ms (default `500`). | | `BILI_PERSIST_TAIL_TOKENS` | Token budget for the persisted conversation snapshot (#401). The disk record stores the **folded view** (block summaries in place of compressed ranges) truncated to the newest messages within this budget — never the raw full history. Default `16384`; `0` disables message persistence entirely (block summaries and compressed originals still persist; `bili export` falls back to block-only rendering). Live in-memory sessions are unaffected — `bili export` of a live session is always complete. | +| `BILI_PERSIST_ZSTD` | Set `1`/`true` to enable zstd compression of session files (#1080, owner decision: **default off** — plain JSON maximizes recoverability: jq/grep-debuggable, and immune to the downgrade tail risk). When enabled, every session file that actually shrinks is written as `BILIZSTD1` — a small header (magic + format version + mode byte) over the JSON body, zstd-compressed on Node ≥ 22.15 and stored raw on older runtimes; readers accept both bodies (native zstd, else the bundled WASM fallback), so files stay readable across runtimes and versions. Payloads the framing cannot shrink (tiny sessions) and key-less raw bodies stay bare JSON on disk. Independent of `BILI_ENCRYPTION_KEY` — with both active, the compression choice is recorded in the mode byte inside `BILIENC1`. Existing plain-JSON files are NEVER rewritten at boot (downgrade safety — a mass re-encode would make rolling back to an older bili read its own writes as "corrupt"); they convert organically on their next save. Note: once a session has been saved as `BILIZSTD1`, an OLDER bili (< 0.1.135) cannot read it — this caveat applies only to deployments that opted in; unset or any other value keeps plain JSON. | | `BILI_PERSIST_EPERM_ALERT_THRESHOLD` | N consecutive persist write failures (`EPERM`/`EBUSY`/`EACCES`) on one session before the one-time "add this dir to antivirus exclusions" alert fires (default `5`). Windows only. See [Windows: exclude the sessions dir](#windows-exclude-the-sessions-dir-from-antivirus-362). | | `BILI_PERSIST_EPERM_ALERT_REPEAT_MS` | Re-alert window for the persist EPERM alert, in ms. `0` (default) = alert once then stay silent; `>0` = re-alert at most every that many ms while the failures continue. | | `BILI_TUNNEL_ALLOWED_HOSTS` | `/bili/` tunnel admission for **remote clients** (#409): comma-separated `host` or `host:port` entries that unlock loopback/private destinations (e.g. a LAN relay or the machine's own sglang) for non-loopback clients. The proxy itself and link-local/metadata addresses are always denied; local (loopback) clients always pass. | | `BILI_MAX_SESSIONS` | Max sessions held in memory (default `256`; LRU eviction — disk is the source of truth). | | `BILI_SESSIONS_DIR` | Directory for persisted session state (default XDG data dir). | -| `BILI_ENCRYPTION_KEY` | Encrypt session files at rest (#708) for deployments on untrusted nodes. Exactly 32 bytes, hex (64 chars) or base64; unset = plain JSON files (default behavior, unchanged). When set: every session file is written as `BILIENC1` AES-256-GCM over zstd-compressed JSON (zstd on Node ≥ 22.15, raw body on older runtimes) — compression also shrinks the file ~5–10×. Existing unencoded files are taken over once at boot: each is re-encoded and atomically replaced in place (the rename IS the old-file deletion; a crash mid-migration self-heals on next boot). The key comes ONLY from this environment variable — never written to disk or logs — so keep it out of the same filesystem's reach. Invalid values abort startup (fail fast, never silently unencrypted). Booting with the wrong key skips the affected sessions as corrupt (logged, no crash). Losing the key makes encrypted sessions permanently unreadable. Symmetric by design (the same process encrypts and decrypts). Threat model (#708, owner-confirmed): it defends against **offline/mechanical** file acquisition — provider disk swaps, offline disk snapshots after node-image drift, stolen volumes, leaked backups, cloud-synced state dirs — where an offline third party cannot read the content without the key. It does NOT defend against targeted adversaries with live access to the node; for that tier move the trust root out of the proxy (KMS / TEE / confidential VMs, hardened permissions) instead of solving it inside the proxy — at that level far more than the key is exposed, so proxy-level key handling is not the boundary to hold (`BILI_PERSIST=0` disables persistence entirely). Double-encrypting the key with a second key from the same process/environment adds nothing: in every offline-theft scenario the attacker lacks exactly one artifact — your off-disk secret — whether it is called the data key or the wrapping key; only a wrapping key in a different trust domain (KMS/TPM/TEE) raises the bar, and that is scenario 2 above. | +| `BILI_ENCRYPTION_KEY` | Encrypt session files at rest (#708) for deployments on untrusted nodes. Exactly 32 bytes, hex (64 chars) or base64; unset = unencrypted plain-JSON files (or `BILIZSTD1` when `BILI_PERSIST_ZSTD=1` — see `BILI_PERSIST_ZSTD`). When set: every session file is written as `BILIENC1` AES-256-GCM over JSON that is zstd-compressed only when `BILI_PERSIST_ZSTD=1` (zstd on Node ≥ 22.15, raw body otherwise) — enabling compression also shrinks the file ~5–10×. Encryption and compression are now independent knobs (#1080). The key comes ONLY from this environment variable — never written to disk or logs — so keep it out of the same filesystem's reach. Invalid values abort startup (fail fast, never silently unencrypted). Booting with the wrong key skips the affected sessions as corrupt (logged, no crash). Losing the key makes encrypted sessions permanently unreadable. Symmetric by design (the same process encrypts and decrypts). Existing unencoded files are never rewritten at boot — they encrypt organically on their next save (downgrade safety; see `BILI_PERSIST_ZSTD`). Threat model (#708, owner-confirmed): it defends against **offline/mechanical** file acquisition — provider disk swaps, offline disk snapshots after node-image drift, stolen volumes, leaked backups, cloud-synced state dirs — where an offline third party cannot read the content without the key. It does NOT defend against targeted adversaries with live access to the node; for that tier move the trust root out of the proxy (KMS / TEE / confidential VMs, hardened permissions) instead of solving it inside the proxy — at that level far more than the key is exposed, so proxy-level key handling is not the boundary to hold (`BILI_PERSIST=0` disables persistence entirely). Double-encrypting the key with a second key from the same process/environment adds nothing: in every offline-theft scenario the attacker lacks exactly one artifact — your off-disk secret — whether it is called the data key or the wrapping key; only a wrapping key in a different trust domain (KMS/TPM/TEE) raises the bar, and that is scenario 2 above. | | `BILLION_CONTEXT_PROXY` | Exported by the launcher; client-side bili plugins/extensions detect it and self-disable their own compression (no double compression). | | `BILLION_CONTEXT_PLUGIN` | Set `0` to disable plugin mode entirely (wire-level tool injection resumes). | | `BILI_LAUNCHER_MODEL_WINDOWS` | Internal: the launcher hands the client's own per-model context windows (pi `models.json`, omp `models.yml`, opencode `models..limit`, codex `model_context_window`) to the spawned proxy as JSON, so the nudge denominator matches the real window for self-hosted models. Only the launcher sets it — no user configuration. | diff --git a/CONFIGURATION.zh-CN.md b/CONFIGURATION.zh-CN.md index 871e7ee341..33e611f02d 100644 --- a/CONFIGURATION.zh-CN.md +++ b/CONFIGURATION.zh-CN.md @@ -515,11 +515,12 @@ | `BILI_PERSIST` | 设 `0` 关闭会话持久化(仅内存,重启即丢)。 | | `BILI_PERSIST_DEBOUNCE_MS` | 持久化写盘的防抖窗口(毫秒,默认 `500`)。 | | `BILI_PERSIST_TAIL_TOKENS` | 持久化会话快照的 token 预算(#401)。盘上记录的是**折叠视图**(压缩范围以块摘要替代)并截断到该预算内的最新消息 —— 不再存全量原始历史。默认 `16384`;设 `0` 彻底不持久化消息(块摘要与压缩原件仍会持久化,`bili export` 退回块级渲染)。活会话内存不受影响 —— 活会话的 `bili export` 始终完整。 | +| `BILI_PERSIST_ZSTD` | 设为 `1`/`true` 启用会话文件的 zstd 压缩(#1080,owner 决定:**默认关闭**——纯 JSON 可恢复性最强:可用 jq/grep 调试,且无降级尾部风险)。启用后,每个确实能压缩变小的会话文件均以 `BILIZSTD1` 格式写入——即在 JSON 主体之上附加一个小头部(魔数 + 格式版本 + 模式字节),在 Node ≥ 22.15 上以 zstd 压缩存储,在旧版运行时上则原样存储;读取端兼容两种主体格式(优先使用原生 zstd,否则回退至内置的 WASM 实现),因此文件在不同运行时和版本间均可正常读取。压不小的小会话与无密钥的原始主体以裸 JSON 落盘。与 `BILI_ENCRYPTION_KEY` 相互独立——两者同时生效时,压缩方式会记录在 `BILIENC1` 内部的模式字节中。已有的纯 JSON 文件**永不在启动时改写**(降级安全——批量重编码会让回退到旧版 bili 时把自己的写入当成“损坏文件”);它们在其下一次保存时自然转换。注意:一旦会话已保存为 `BILIZSTD1`,旧版 bili(< 0.1.135)无法读取——该限制仅对显式启用的部署生效;未设置或其他值均保持纯 JSON。 | | `BILI_PERSIST_EPERM_ALERT_THRESHOLD` | 同一会话连续 N 次持久化写失败(`EPERM`/`EBUSY`/`EACCES`)后触发一次性「把该目录加入杀软排除项」告警的阈值(默认 `5`)。仅 Windows。见下文「Windows:把会话目录加入杀软排除项」章节。 | | `BILI_PERSIST_EPERM_ALERT_REPEAT_MS` | persist EPERM 告警的重复窗口(毫秒)。`0`(默认)= 只告警一次后静默;`>0` = 失败持续期间最多每这么久重复告警一次。 | | `BILI_MAX_SESSIONS` | 内存中最多保留的会话数(默认 `256`;LRU 淘汰 —— 磁盘是事实源)。 | | `BILI_SESSIONS_DIR` | 会话持久化目录(默认 XDG data 目录)。 | -| `BILI_ENCRYPTION_KEY` | 会话文件静态加密(#708),适用于部署在不可信节点的场景。密钥必须恰好 32 字节,hex(64 字符)或 base64;不设置 = 明文 JSON 文件(默认,行为不变)。设置后:每个会话文件以 `BILIENC1` AES-256-GCM 加密 zstd 压缩后的 JSON 写入(Node ≥ 22.15 启用 zstd,旧版本写原始数据)——压缩同时可使文件缩小约 5–10 倍。已有的未编码文件在启动时一次性接管:逐个重新编码并原子替换到原路径(rename 即旧文件的删除;迁移中途崩溃会在下次启动自愈)。密钥只从该环境变量读取——永不落盘、永不进日志——请确保它不受同一文件系统上的其他进程触及。非法值会导致启动中止(快速失败,绝不静默明文运行)。用错误的密钥启动时,受影响的会话按损坏文件跳过(有日志,不崩溃)。丢失密钥将使已加密的会话永久不可读。对称加密为刻意设计(同一进程既加密又解密)。威胁模型(#708,owner 确认):防的是**离线/机械性**的文件获取——云厂商换盘、节点镜像漂移后的离线磁盘快照、磁盘镜像失窃、备份泄露、被云同步的状态目录——离线第三方拿不到密钥即无法读取内容。不防御对活节点有访问权的定向攻击者;那一档应把信任根移出 proxy(KMS / TEE / 机密虚拟机 + 强化权限体系),而不是在 proxy 本身想办法——到了那个程度暴露的远不止密钥,proxy 层不是该守的边界(`BILI_PERSIST=0` 可彻底关闭持久化)。用同一进程/环境中的第二把密钥对密钥做二次加密不增加任何安全性:所有离线失窃场景里攻击者缺的始终只有一个工件——你的非落盘秘密——无论它叫数据密钥还是包裹密钥;只有把包裹密钥放进不同信任域(KMS/TPM/TEE)才能提高门槛,而那属于上面的场景 2。 | +| `BILI_ENCRYPTION_KEY` | 会话文件静态加密(#708),适用于部署在不可信节点的场景。密钥必须恰好 32 字节,hex(64 字符)或 base64;未设置 = 不加密的纯 JSON 文件(设 `BILI_PERSIST_ZSTD=1` 时为 `BILIZSTD1`——参见 `BILI_PERSIST_ZSTD`)。设置后:每个会话文件均以 `BILIENC1` 格式写入,即对 JSON 施加 AES-256-GCM 加密,JSON 仅在 `BILI_PERSIST_ZSTD=1` 时以 zstd 压缩(Node ≥ 22.15 使用 zstd,其余情况写入原始数据)——启用压缩还可将文件体积缩小约 5–10 倍。加密与压缩现为独立的配置项(#1080)。密钥只从该环境变量读取——永不落盘、永不进日志——请确保它不受同一文件系统上的其他进程触及。非法值会导致启动中止(快速失败,绝不静默明文运行)。用错误的密钥启动时,受影响的会话按损坏文件跳过(有日志,不崩溃)。丢失密钥将使已加密的会话永久不可读。对称加密为刻意设计(同一进程既加密又解密)。已有的未编码文件从不在启动时改写——在其下一次保存时自然加密(降级安全;参见 `BILI_PERSIST_ZSTD`)。威胁模型(#708,owner 确认):防的是**离线/机械性**的文件获取——云厂商换盘、节点镜像漂移后的离线磁盘快照、磁盘镜像失窃、备份泄露、被云同步的状态目录——离线第三方拿不到密钥即无法读取内容。不防御对活节点有访问权的定向攻击者;那一档应把信任根移出 proxy(KMS / TEE / 机密虚拟机 + 强化权限体系),而不是在 proxy 本身想办法——到了那个程度暴露的远不止密钥,proxy 层不是该守的边界(`BILI_PERSIST=0` 可彻底关闭持久化)。用同一进程/环境中的第二把密钥对密钥做二次加密不增加任何安全性:所有离线失窃场景里攻击者缺的始终只有一个工件——你的非落盘秘密——无论它叫数据密钥还是包裹密钥;只有把包裹密钥放进不同信任域(KMS/TPM/TEE)才能提高门槛,而那属于上面的场景 2。 | | `BILLION_CONTEXT_PROXY` | launcher 会导出它;客户端侧 bili 插件/扩展检测到后自禁用自身压缩(避免双重压缩)。 | | `BILLION_CONTEXT_PLUGIN` | 设 `0` 彻底关闭插件模式(恢复 wire 层工具注入)。 | | `BILI_LAUNCHER_MODEL_WINDOWS` | 内部使用:launcher 把客户端自身配置里的逐模型上下文窗口(pi `models.json`、omp `models.yml`、opencode `models..limit`、codex `model_context_window`)以 JSON 传给自己拉起的代理,让 nudge 分母对自托管模型也用真实窗口。只有 launcher 会设置,无需用户配置。 | diff --git a/src/encrypt.ts b/src/encrypt.ts index b7ffe707b8..ab46b693e8 100644 --- a/src/encrypt.ts +++ b/src/encrypt.ts @@ -1,38 +1,56 @@ import { createCipheriv, createDecipheriv, randomBytes } from "node:crypto"; import * as zlib from "node:zlib"; +import { Decompress as ZstdWasmDecompress } from "fzstd"; import type { StateStoreCodec } from "acp-kernel/persist"; /** - * Session-file encryption at rest (#708): AES-256-GCM over an optionally - * zstd-compressed JSON envelope, applied around every StateStore write/read. + * Session-file storage encoding at rest: AES-256-GCM encryption (#708) and + * zstd compression (#1080), applied around every StateStore write/read and + * configured INDEPENDENTLY of each other. * - * THREAT MODEL: the proxy may run on untrusted nodes; session files hold - * block summaries plus up to ~16k tokens of folded conversation per session - * — effectively full conversation content (code, pasted credentials). The - * key comes ONLY from the BILI_ENCRYPTION_KEY environment variable: a key - * file next to the data sits on the same untrusted filesystem and defeats - * the purpose. + * THREAT MODEL (#708): the proxy may run on untrusted nodes; session files + * hold block summaries plus up to ~16k tokens of folded conversation per + * session — effectively full conversation content (code, pasted credentials). + * The key comes ONLY from the BILI_ENCRYPTION_KEY environment variable: a key + * file next to the data sits on the same untrusted filesystem and defeats the + * purpose. * - * FORMAT (v1): - * offset 0..7 magic "BILIENC1" - * offset 8 format version (0x01) - * offset 9 body mode (0x00 raw, 0x01 zstd) - * offset 10..21 GCM nonce (random per write) - * offset 22.. AES-256-GCM ciphertext, final 16 bytes = auth tag + * COMPRESSION (#1080, owner decision): session JSON is zstd-compressed only + * when opted in (BILI_PERSIST_ZSTD=1/true); the default stays plain JSON for + * recoverability and downgrade safety. Clients and tools never see the + * on-disk format — the store decodes transparently and `bili export` renders + * plaintext. * - * Compression runs BEFORE encryption (GCM ciphertext is incompressible). - * node:zlib gained zstd in Node 22.15; on older runtimes the mode byte - * records a raw body so every supported version can still decrypt it. + * FORMATS (v1): + * encrypted (BILIENC1): + * offset 0..7 magic "BILIENC1" + * offset 8 format version (0x01) + * offset 9 body mode (0x00 raw, 0x01 zstd) + * offset 10..21 GCM nonce (random per write) + * offset 22.. AES-256-GCM ciphertext, final 16 bytes = auth tag + * compressed (BILIZSTD1): + * offset 0..8 magic "BILIZSTD1" + * offset 9 format version (0x01) + * offset 10 body mode (0x00 raw, 0x01 zstd) + * offset 11.. body (JSON or zstd stream) + * + * Compression runs BEFORE encryption (GCM ciphertext is incompressible); the + * mode byte records which, so each concern stays independently configurable. + * node:zlib gained zstd in Node 22.15; on older runtimes the writer falls + * back to a raw body and the reader uses the bundled fzstd WASM decoder, so a + * file written by any supported version reads back on every supported one. */ export const ENCRYPT_MAGIC = Buffer.from("BILIENC1", "utf8"); +export const ZSTD_MAGIC = Buffer.from("BILIZSTD1", "utf8"); const FORMAT_VERSION = 0x01; const MODE_RAW = 0x00; const MODE_ZSTD = 0x01; const NONCE_LEN = 12; const TAG_LEN = 16; -const HEADER_LEN = ENCRYPT_MAGIC.length + 2 + NONCE_LEN; -const MIN_ENCRYPTED_LEN = HEADER_LEN + TAG_LEN; +const ENCRYPT_HEADER_LEN = ENCRYPT_MAGIC.length + 2 + NONCE_LEN; +const MIN_ENCRYPTED_LEN = ENCRYPT_HEADER_LEN + TAG_LEN; +const PLAIN_HEADER_LEN = ZSTD_MAGIC.length + 2; /** Parse the BILI_ENCRYPTION_KEY value: hex or base64, must decode to * exactly 32 bytes. Hex wins when both parse (a base64 string made only of @@ -56,48 +74,116 @@ export function parseEncryptionKey(value: string): Buffer { return buf; } +let zstdOverride: false | null = null; + +/** Test hook: simulate runtimes without native zstd (pre-Node-22.15). Can + * only claim ABSENCE, never presence — encoding then falls back to a raw + * body and decoding uses the WASM path. */ +export function _setZstdAvailableForTest(v: false | null): void { + zstdOverride = v; +} + function zstdAvailable(): boolean { + if (zstdOverride === false) return false; return typeof zlib.zstdCompressSync === "function" && typeof zlib.zstdDecompressSync === "function"; } -/** Build the StateStore codec for encrypted session files. decode() passes - * buffers without the magic through untouched (legacy plaintext), so mixed - * plaintext+encrypted trees load fine and the boot-time migration can peek - * the magic before rewriting. A decode failure (wrong key, corruption) - * throws and the kernel store treats the file as corrupt (warn + skip). */ -export function createSessionCodec(key: Buffer): StateStoreCodec { +/** Decompress a zstd-mode body: native node:zlib when present, otherwise the + * bundled fzstd WASM decoder — keeps newer-runtime files readable on older + * ones (#1080). */ +function inflateZstd(body: Buffer): Buffer { + if (zstdAvailable()) return zlib.zstdDecompressSync(body); + const chunks: Buffer[] = []; + new ZstdWasmDecompress((chunk) => chunks.push(Buffer.from(chunk))).push(body, true); + return Buffer.concat(chunks); +} + +export interface StorageCodecOptions { + /** AES-256-GCM key (#708). Without one, BILIENC1 files cannot be read — + * they surface as an actionable corrupt-file error instead of garbage. */ + key?: Buffer | null; + /** zstd-compress bodies (#1080, default true) — applies to BOTH formats; + * the mode byte records it, so compression stays independent of + * encryption. */ + compress?: boolean; +} + +/** Build the StateStore codec for session files, or undefined when neither + * encryption nor compression applies (plain JSON on disk). decode() + * dispatches on magic — BILIENC1 → AES-256-GCM, BILIZSTD1 → zstd/raw, + * anything else passes through untouched (legacy plaintext) — so mixed + * trees load fine under any codec configuration. + * A decode failure throws and the kernel store treats the file as corrupt + * (warn + skip). */ +export function createStorageCodec(opts: StorageCodecOptions = {}): StateStoreCodec | undefined { + const key = opts.key ?? null; + const compress = opts.compress ?? false; + if (!key && !compress) return undefined; return { encode(data: string): Buffer { const plain = Buffer.from(data, "utf8"); - const useZstd = zstdAvailable(); - const body = useZstd ? zlib.zstdCompressSync(plain) : plain; + // Size guard (#1080 review): only frame-and-compress when it + // actually shrinks the payload. Tiny sessions would otherwise + // grow by the magic+header overhead, and — more important — an + // uncompressed MODE_RAW body has no reason to be framed at all + // when no key is set: unframed plain JSON maximizes backward + // compatibility (a downgrade reads it natively). + const useZstd = compress && zstdAvailable(); + let body = plain; + let mode = MODE_RAW; + if (useZstd) { + const z = zlib.zstdCompressSync(plain); + if (z.length < plain.length) { + body = z; + mode = MODE_ZSTD; + } + } + if (!key && mode === MODE_RAW) return plain; + const header = Buffer.from([FORMAT_VERSION, mode]); + if (!key) { + return Buffer.concat([ZSTD_MAGIC, header, body]); + } const nonce = randomBytes(NONCE_LEN); const cipher = createCipheriv("aes-256-gcm", key, nonce); const ct = Buffer.concat([cipher.update(body), cipher.final()]); - return Buffer.concat([ - ENCRYPT_MAGIC, - Buffer.from([FORMAT_VERSION, useZstd ? MODE_ZSTD : MODE_RAW]), - nonce, - ct, - cipher.getAuthTag(), - ]); + return Buffer.concat([ENCRYPT_MAGIC, header, nonce, ct, cipher.getAuthTag()]); }, decode(buf: Buffer): string { - if (buf.length < ENCRYPT_MAGIC.length || !buf.subarray(0, ENCRYPT_MAGIC.length).equals(ENCRYPT_MAGIC)) { - return buf.toString("utf8"); + if (buf.length >= ENCRYPT_MAGIC.length && buf.subarray(0, ENCRYPT_MAGIC.length).equals(ENCRYPT_MAGIC)) { + return decryptEnvelope(buf, key); } - if (buf.length < MIN_ENCRYPTED_LEN || buf[ENCRYPT_MAGIC.length] !== FORMAT_VERSION) { - throw new Error(`[encrypt] unsupported session file format version ${buf[ENCRYPT_MAGIC.length]}`); + if (buf.length >= ZSTD_MAGIC.length && buf.subarray(0, ZSTD_MAGIC.length).equals(ZSTD_MAGIC)) { + return decompressEnvelope(buf); } - const mode = buf[ENCRYPT_MAGIC.length + 1]; - const nonce = buf.subarray(HEADER_LEN - NONCE_LEN, HEADER_LEN); - const tag = buf.subarray(buf.length - TAG_LEN); - const ct = buf.subarray(HEADER_LEN, buf.length - TAG_LEN); - const decipher = createDecipheriv("aes-256-gcm", key, nonce); - decipher.setAuthTag(tag); - const body = Buffer.concat([decipher.update(ct), decipher.final()]); - const plain = mode === MODE_ZSTD ? zlib.zstdDecompressSync(body) : body; - return plain.toString("utf8"); + return buf.toString("utf8"); }, }; } + +function decryptEnvelope(buf: Buffer, key: Buffer | null): string { + if (!key) { + throw new Error("[storage] session file is BILIENC1-encrypted but no BILI_ENCRYPTION_KEY is set — cannot read it"); + } + if (buf.length < MIN_ENCRYPTED_LEN || buf[ENCRYPT_MAGIC.length] !== FORMAT_VERSION) { + throw new Error(`[encrypt] unsupported session file format version ${buf[ENCRYPT_MAGIC.length]}`); + } + const mode = buf[ENCRYPT_MAGIC.length + 1]; + const nonce = buf.subarray(ENCRYPT_HEADER_LEN - NONCE_LEN, ENCRYPT_HEADER_LEN); + const tag = buf.subarray(buf.length - TAG_LEN); + const ct = buf.subarray(ENCRYPT_HEADER_LEN, buf.length - TAG_LEN); + const decipher = createDecipheriv("aes-256-gcm", key, nonce); + decipher.setAuthTag(tag); + const body = Buffer.concat([decipher.update(ct), decipher.final()]); + const plain = mode === MODE_ZSTD ? inflateZstd(body) : body; + return plain.toString("utf8"); +} + +function decompressEnvelope(buf: Buffer): string { + if (buf.length < PLAIN_HEADER_LEN || buf[ZSTD_MAGIC.length] !== FORMAT_VERSION) { + throw new Error(`[storage] unsupported session file format version ${buf[ZSTD_MAGIC.length]}`); + } + const mode = buf[ZSTD_MAGIC.length + 1]; + const body = buf.subarray(PLAIN_HEADER_LEN); + const plain = mode === MODE_ZSTD ? inflateZstd(body) : body; + return plain.toString("utf8"); +} diff --git a/src/persist.ts b/src/persist.ts index 3137aa0b28..57bb2cf536 100644 --- a/src/persist.ts +++ b/src/persist.ts @@ -1,11 +1,11 @@ import { createHash } from "node:crypto"; import { existsSync, mkdirSync, renameSync, writeFileSync } from "node:fs"; -import { open, readdir, readFile, rm } from "node:fs/promises"; +import { readdir, readFile, rm } from "node:fs/promises"; import * as path from "node:path"; import { StateStore, flatFileNameFor, type PersistedEnvelope, type StateStoreCodec } from "acp-kernel/persist"; import { sessionsDir } from "./paths.js"; import { log as loggerLog } from "./logger.js"; -import { createSessionCodec, ENCRYPT_MAGIC, parseEncryptionKey } from "./encrypt.js"; +import { createStorageCodec, parseEncryptionKey } from "./encrypt.js"; import { PersistEpermAlert } from "./persist-eperm.js"; import { createInitialState, defaultCountTokens, prune, type CompressionState, type CoreMessage } from "acp-kernel"; import type { Session, BlockContent, BlockView } from "./session.js"; @@ -35,11 +35,16 @@ import type { WireProtocol } from "./util.js"; * - Forward-compat: `mergeState` fills any fields missing on a file written * by an older version, so a schema change never breaks old files. * - Disable with BILI_PERSIST=0 for ephemeral/test runs. - * - Encryption at rest (#708): BILI_ENCRYPTION_KEY (hex/base64, exactly 32 - * bytes) wraps every file as BILIENC1 AES-256-GCM(zstd(JSON)) via the - * kernel store's codec hook. Legacy plaintext files are re-encoded in - * place once at boot (migrateLegacyFiles); key material never touches - * disk or logs. + * - Storage encoding at rest: via the kernel store's codec hook, every file + * is optionally zstd-compressed when BILI_PERSIST_ZSTD=1/true (default + * off — plain JSON; #1080, BILIZSTD1 envelope) and, independently, + * AES-256-GCM-encrypted when + * BILI_ENCRYPTION_KEY (hex/base64, exactly 32 bytes) is set (#708, + * BILIENC1 envelope) — the body-mode byte records compression even inside + * BILIENC1, so the two stay separately configurable. Legacy plaintext + * files are NEVER rewritten at boot (downgrade safety — see + * sweepStaleTemps); they convert organically on their next save. Key + * material never touches disk or logs. * * MECHANISM lives in `acp-kernel/persist` (StateStore: atomic write, rename * retries, debounce, per-id serialization, corrupt-tolerant load, recursive @@ -203,13 +208,19 @@ export class SessionStore { this.dir = opts?.dir ?? defaultDir(); const baseLog = opts?.log ?? defaultLogger; this.log = baseLog; - // #708: env-only key (a key file next to the data sits on the same - // untrusted filesystem). Invalid values throw here — fail fast at - // startup instead of running silently unencrypted. + // #708/#1080: env-only storage policy (a key file next to the data + // sits on the same untrusted filesystem). Compression is OPT-IN + // (owner decision, #1080): BILI_PERSIST_ZSTD=1/true enables it, the + // default stays plain JSON. Invalid key values throw here — fail + // fast at startup instead of running silently unencrypted. const keyEnv = process.env.BILI_ENCRYPTION_KEY; - if (keyEnv) { - this.codec = createSessionCodec(parseEncryptionKey(keyEnv)); + let key: Buffer | null = null; + if (keyEnv) key = parseEncryptionKey(keyEnv); + this.codec = createStorageCodec({ key, compress: persistZstdEnabled() }); + if (key) { baseLog("info", "[persist] session-file encryption enabled (AES-256-GCM)"); + } else if (this.codec) { + baseLog("info", "[persist] session-file compression enabled (zstd, BILIZSTD1)"); } const epermAlert = new PersistEpermAlert({ dir: this.dir, @@ -267,7 +278,7 @@ export class SessionStore { * tree twice per start. */ async boot(): Promise> { if (!this.enabled) return new Map(); - await this.migrateLegacyFiles(); + await this.sweepStaleTemps(); const loaded = await this.store.loadAll(); await this.applyLegacyMigration(loaded); const out = new Map(); @@ -277,60 +288,25 @@ export class SessionStore { return out; } - /** #708: when encryption is enabled, take over legacy plaintext files: - * every .json under the sessions dir lacking the BILIENC1 magic is - * re-encoded in place — temp write + rename onto the SAME path, so the - * atomic replace IS the old-file deletion (no window where both, or - * neither, copy exists). A crash mid-run leaves each file either old or - * new; the next boot finishes the job and sweeps the crashed run's - * orphaned temps. Self-terminating: the 8-byte magic peek decides per - * file, so later boots cost O(files × 8 bytes). */ - private async migrateLegacyFiles(): Promise { - if (!this.codec) return; + /** #708/#1080 review: boot NEVER rewrites session file CONTENT. The first + * draft re-encoded every legacy plaintext file in place at boot — a mass + * format flip that a downgrade (rollback during incident triage) would + * meet with "skipping corrupt file" + empty-session resume + overwrite, + * silently destroying history. Legacy plaintext files keep loading via + * the codec's magic dispatch and convert organically on their next save; + * the only thing boot touches is orphaned `.tmp-enc-*` temps from crashed + * writes of previous runs (pure deletion of garbage, never content). */ + private async sweepStaleTemps(): Promise { let files: string[]; try { files = await walkJsonFiles(this.dir); } catch { return; } - let migrated = 0; - let failed = 0; for (const file of files) { if (STALE_ENC_TEMP_RE.test(path.basename(file))) { await rm(file, { force: true }).catch(() => {}); - continue; - } - let head: Buffer; - try { - head = await readFileHead(file); - } catch { - continue; } - if (head.equals(ENCRYPT_MAGIC)) continue; - let parsed: unknown; - try { - parsed = JSON.parse(await readFile(file, "utf8")); - } catch { - failed++; - this.log("warn", `[persist] encryption migration (#708): leaving unreadable file in place: ${file}`); - continue; - } - const tmp = `${file}.tmp-enc-${process.pid}-${Date.now()}`; - try { - writeFileSync(tmp, this.codec.encode(JSON.stringify(parsed))); - renameSync(tmp, file); - migrated++; - } catch { - failed++; - this.log("warn", `[persist] encryption migration (#708): failed to re-encode: ${file}`); - await rm(tmp, { force: true }).catch(() => {}); - } - } - if (migrated > 0 || failed > 0) { - this.log( - failed > 0 ? "warn" : "info", - `[persist] encryption migration (#708): re-encoded ${migrated} legacy session file(s)${failed > 0 ? `, ${failed} failed` : ""}`, - ); } } @@ -685,7 +661,16 @@ function persistEnabled(): boolean { return true; } -/** Temp name used by migrateLegacyFiles: `.tmp-enc--`. A +/** #1080 (owner decision): session files stay plain JSON by default — + * recoverability (jq/grep-debuggable, no downgrade tail risk) beats silent + * disk savings. Only BILI_PERSIST_ZSTD=1/true opts into zstd (BILIZSTD1); + * anything else (0/false/unset) keeps plain JSON. */ +function persistZstdEnabled(): boolean { + const env = process.env.BILI_PERSIST_ZSTD; + return env === "1" || env === "true"; +} + +/** Temp name used by atomic codec writes: `.tmp-enc--`. A * process death between write and rename orphans it; any such name present * at boot is stale by definition (the walk runs before this boot writes * anything) and gets swept. */ @@ -705,17 +690,6 @@ async function walkJsonFiles(dir: string): Promise { return out; } -async function readFileHead(file: string, len: number = ENCRYPT_MAGIC.length): Promise { - const fh = await open(file, "r"); - try { - const buf = Buffer.alloc(len); - const { bytesRead } = await fh.read(buf, 0, len, 0); - return buf.subarray(0, bytesRead); - } finally { - await fh.close(); - } -} - /** Token budget for the persisted folded-view snapshot (#401). The raw full * history is never persisted — prune() first replaces folded ranges with * their summaries (exactly what `bili export` renders by default), then the diff --git a/tests/fix-401-storage.test.ts b/tests/fix-401-storage.test.ts index 3ba117e79b..173cbf830a 100644 --- a/tests/fix-401-storage.test.ts +++ b/tests/fix-401-storage.test.ts @@ -9,6 +9,10 @@ import { SessionStore } from "../src/persist.ts"; import type { Session } from "../src/session.ts"; import { setLogCapture } from "../src/logger.ts"; +// These tests inspect raw on-disk session files directly (bypassing the store +// codec), so pin the plain-JSON format: #1080 made BILIZSTD1 the default. +process.env.BILI_PERSIST_ZSTD = "0"; + function jsonFilesUnder(dir: string): string[] { const out: string[] = []; for (const name of readdirSync(dir)) { diff --git a/tests/host-usage-postfold.test.ts b/tests/host-usage-postfold.test.ts index efbc383413..865efb62c3 100644 --- a/tests/host-usage-postfold.test.ts +++ b/tests/host-usage-postfold.test.ts @@ -16,6 +16,10 @@ import { startServer } from "../src/server.ts"; import type { ProxyOptions } from "../src/config.ts"; import { _setForTest as setRegistryForTest } from "../src/registry.ts"; +// Some tests inspect raw on-disk session files directly (bypassing the store +// codec), so pin the plain-JSON format: #1080 made BILIZSTD1 the default. +process.env.BILI_PERSIST_ZSTD = "0"; + function makeSession(id: string): Session { return { id, diff --git a/tests/persist-encryption.test.ts b/tests/persist-encryption.test.ts index 6d0e7649dd..ff89f0ac30 100644 --- a/tests/persist-encryption.test.ts +++ b/tests/persist-encryption.test.ts @@ -5,7 +5,7 @@ import { createHash, randomBytes } from "node:crypto"; import { mkdtempSync, rmSync, mkdirSync, writeFileSync, readFileSync, existsSync } from "node:fs"; import { join } from "node:path"; import { SessionStore } from "../src/persist.ts"; -import { createSessionCodec, ENCRYPT_MAGIC, parseEncryptionKey } from "../src/encrypt.ts"; +import { createStorageCodec, ENCRYPT_MAGIC, parseEncryptionKey } from "../src/encrypt.ts"; import type { Session } from "../src/session.ts"; import { createInitialState } from "acp-kernel"; @@ -43,6 +43,7 @@ async function withTempDir(name: string, fn: (h: Harness) => Promise): Pro await fn(h); } finally { delete process.env.BILI_ENCRYPTION_KEY; + delete process.env.BILI_PERSIST_ZSTD; rmSync(dir, { recursive: true, force: true }); } }); @@ -68,8 +69,8 @@ test("parseEncryptionKey accepts hex and base64, rejects everything else", () => }); test("codec: roundtrip, magic prefix, per-write nonce, tamper and wrong-key rejection", () => { - const codec = createSessionCodec(Buffer.from(KEY, "hex")); - const other = createSessionCodec(Buffer.from(KEY_OTHER, "hex")); + const codec = createStorageCodec({ key: Buffer.from(KEY, "hex") })!; + const other = createStorageCodec({ key: Buffer.from(KEY_OTHER, "hex") })!; const json = JSON.stringify({ hello: "world", n: [1, 2, 3] }); const enc = Buffer.isBuffer(codec.encode(json)) ? codec.encode(json) : Buffer.from(codec.encode(json)); @@ -87,9 +88,9 @@ test("codec: roundtrip, magic prefix, per-write nonce, tamper and wrong-key reje }); test("codec passes legacy plaintext through untouched", () => { - const codec = createSessionCodec(Buffer.from(KEY, "hex")); - const json = '{"version":3,"id":"x"}'; - assert.equal(codec.decode(Buffer.from(json, "utf8")), json); + const codec = createStorageCodec({ key: Buffer.from(KEY, "hex") })!; + const plain = JSON.stringify({ version: 3, savedAt: 1, id: "s", payload: { protocol: "openai" } }); + assert.equal(codec.decode(Buffer.from(plain, "utf8")), plain); }); await withTempDir("writes are encrypted on disk when the key is set", async (h) => { @@ -108,47 +109,57 @@ await withTempDir("writes are encrypted on disk when the key is set", async (h) } }); -await withTempDir("boot migrates legacy plaintext files to encrypted in place", async (h) => { - // Phase 1: legacy plaintext tree written by an unencrypted store. +await withTempDir("boot never rewrites legacy plaintext files (downgrade safety)", async (h) => { + // Phase 1: pre-#1080 plaintext tree (zstd opt-out = what older bili wrote). + process.env.BILI_PERSIST_ZSTD = "0"; const legacy = newStore(h); await legacy.writeNow(makeSession("s-1")); await legacy.writeNow(makeSession("s-2")); legacy.cancelAll(); + delete process.env.BILI_PERSIST_ZSTD; - // Phase 2: boot with the key — every unencoded file must be taken over. + // Phase 2: boot with the key — files must load but stay byte-identical + // plaintext. A boot-time mass rewrite (the #1083 draft) makes a + // downgrade destroy history: old bili reads its own framing as + // "corrupt", resumes into an empty session, and the next save + // overwrites the real one. process.env.BILI_ENCRYPTION_KEY = KEY; const store = newStore(h); try { const loaded = await store.boot(); - assert.equal(loaded.size, 2, "both legacy sessions load"); + assert.equal(loaded.size, 2, "both legacy sessions load through the codec fallback"); assert.ok(loaded.has("s-1") && loaded.has("s-2")); for (const id of ["s-1", "s-2"]) { const file = join(h.dir, "openai", "upstream_" + createHash("sha256").update(id, "utf8").digest("hex").slice(0, 24) + ".json"); const head = readFileSync(file).subarray(0, ENCRYPT_MAGIC.length); - assert.ok(head.equals(ENCRYPT_MAGIC), `${id} file re-encoded in place`); + assert.ok(!head.equals(ENCRYPT_MAGIC), `${id} file left untouched by boot`); } - assert.ok(h.logs.some((l) => l.msg.includes("re-encoded 2 legacy session file(s)")), "migration logged"); + assert.ok(!h.logs.some((l) => l.msg.includes("re-encoded")), "no migration log"); - // Phase 3: second boot is a no-op (no double-wrap, no re-migration log). + // Phase 3: organic conversion — the next real save of s-1 encodes it; + // s-2 stays plaintext until it is saved too. Second boot loads both. + await store.writeNow(loaded.get("s-1")!); store.cancelAll(); - const logs2: LogLine[] = []; - const again = new SessionStore({ dir: h.dir, debounceMs: 0, enabled: true, log: (level, msg) => logs2.push({ level, msg }) }); + const f1 = join(h.dir, "openai", "upstream_" + createHash("sha256").update("s-1", "utf8").digest("hex").slice(0, 24) + ".json"); + assert.ok(readFileSync(f1).subarray(0, ENCRYPT_MAGIC.length).equals(ENCRYPT_MAGIC), "s-1 converts on its next save"); + const again = newStore(h); try { const loaded2 = await again.boot(); - assert.equal(loaded2.size, 2, "still exactly 2 sessions — no double encoding"); - assert.ok(!logs2.some((l) => l.msg.includes("re-encoded")), "no migration on an already-encoded tree"); + assert.equal(loaded2.size, 2, "mixed-format tree loads together"); } finally { again.cancelAll(); } } finally { store.cancelAll(); + delete process.env.BILI_ENCRYPTION_KEY; } }); -await withTempDir("migration covers spill-style .fb.json files and leaves corrupt files alone", async (h) => { +await withTempDir("boot leaves foreign and corrupt files alone (no rewrites, no crashes)", async (h) => { mkdirSync(join(h.dir, "openai"), { recursive: true }); const spill = join(h.dir, "openai", "s-spill.fb.json"); - writeFileSync(spill, JSON.stringify({ version: 3, savedAt: Date.now(), id: "s-spill", payload: {} }), "utf8"); + const spillBody = JSON.stringify({ version: 3, savedAt: Date.now(), id: "s-spill", payload: {} }); + writeFileSync(spill, spillBody, "utf8"); const corrupt = join(h.dir, "openai", "garbage_deadbeef.json"); writeFileSync(corrupt, "%%% not json %%%", "utf8"); @@ -157,34 +168,38 @@ await withTempDir("migration covers spill-style .fb.json files and leaves corrup try { const loaded = await store.boot(); assert.equal(loaded.size, 0, "neither fake file is a valid session record"); - assert.ok(readFileSync(spill).subarray(0, ENCRYPT_MAGIC.length).equals(ENCRYPT_MAGIC), "spill file re-encoded"); + assert.equal(readFileSync(spill, "utf8"), spillBody, "foreign spill file byte-identical"); assert.equal(readFileSync(corrupt, "utf8"), "%%% not json %%%", "unreadable file left in place"); - assert.ok(h.logs.some((l) => l.level === "warn" && l.msg.includes("leaving unreadable file in place"))); } finally { store.cancelAll(); + delete process.env.BILI_ENCRYPTION_KEY; } }); -await withTempDir("boot sweeps orphaned .tmp-enc-* temps left by a crashed migration", async (h) => { +await withTempDir("boot sweeps orphaned .tmp-enc-* temps left by a crashed write", async (h) => { + // Pre-#1080 plaintext file (zstd opt-out), same as what older bili wrote. + process.env.BILI_PERSIST_ZSTD = "0"; const legacy = newStore(h); await legacy.writeNow(makeSession("s-crash")); legacy.cancelAll(); + delete process.env.BILI_PERSIST_ZSTD; // Simulate a process death between the temp write and the rename: a // stale temp sits next to an unencoded legacy file. const file = join(h.dir, "openai", "upstream_" + createHash("sha256").update("s-crash", "utf8").digest("hex").slice(0, 24) + ".json"); const orphan = `${file}.tmp-enc-99999-1700000000000`; - writeFileSync(orphan, "stale temp from a crashed migration"); + writeFileSync(orphan, "stale temp from a crashed write"); process.env.BILI_ENCRYPTION_KEY = KEY; const store = newStore(h); try { const loaded = await store.boot(); - assert.equal(loaded.size, 1, "legacy session loads after migration"); - assert.ok(readFileSync(file).subarray(0, ENCRYPT_MAGIC.length).equals(ENCRYPT_MAGIC), "file re-encoded in place"); + assert.equal(loaded.size, 1, "legacy session loads"); + assert.ok(!readFileSync(file).subarray(0, ENCRYPT_MAGIC.length).equals(ENCRYPT_MAGIC), "legacy file NOT rewritten by boot"); assert.equal(existsSync(orphan), false, "orphaned temp swept on next boot"); } finally { store.cancelAll(); + delete process.env.BILI_ENCRYPTION_KEY; } }); @@ -209,7 +224,8 @@ await withTempDir("invalid key fails fast at construction", async (h) => { assert.throws(() => newStore(h), /BILI_ENCRYPTION_KEY.*exactly 32 bytes/); }); -await withTempDir("without a key, files stay plaintext (zero behavior change)", async (h) => { +await withTempDir("without a key and zstd opted out, files stay plaintext", async (h) => { + process.env.BILI_PERSIST_ZSTD = "0"; const store = newStore(h); try { await store.writeNow(makeSession("s-plain")); diff --git a/tests/persist-tail.test.ts b/tests/persist-tail.test.ts index 1afb2dc35e..34b7dba6cd 100644 --- a/tests/persist-tail.test.ts +++ b/tests/persist-tail.test.ts @@ -8,6 +8,10 @@ import { Session, cacheBlockContent, snapshotMessages } from "../src/session.ts" import { renderHandoff } from "../src/export.ts"; import { createInitialState, defaultCountTokens } from "acp-kernel"; +// readRecord() inspects raw on-disk session files directly (bypassing the +// store codec), so pin the plain-JSON format: #1080 made BILIZSTD1 the default. +process.env.BILI_PERSIST_ZSTD = "0"; + // #401: the persisted record stores a BOUNDED FOLDED-VIEW snapshot — // prune() renders summaries in place of folded ranges, then the oldest // messages are dropped until the view fits BILI_PERSIST_TAIL_TOKENS. The raw diff --git a/tests/persist-zstd.test.ts b/tests/persist-zstd.test.ts new file mode 100644 index 0000000000..d5982265f2 --- /dev/null +++ b/tests/persist-zstd.test.ts @@ -0,0 +1,304 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { tmpdir } from "node:os"; +import { createHash, randomBytes } from "node:crypto"; +import { mkdtempSync, rmSync, readFileSync, statSync } from "node:fs"; +import { join } from "node:path"; +import * as zlib from "node:zlib"; +import { SessionStore } from "../src/persist.ts"; +import { createStorageCodec, ENCRYPT_MAGIC, ZSTD_MAGIC, _setZstdAvailableForTest } from "../src/encrypt.ts"; +import type { Session } from "../src/session.ts"; +import { createInitialState } from "acp-kernel"; + +const KEY = randomBytes(32).toString("hex"); + +// Static zstd frame produced by node:zlib zstdCompressSync; decoding it with +// the hook forced off exercises the bundled fzstd fallback for good. +const ZSTD_FIXTURE_B64 = "KLUv/SAlKQEAYWNwLWtlcm5lbC1maXh0dXJlLXBheWxvYWQtMDEyMzQ1Njc4OQ=="; +const ZSTD_FIXTURE_PLAIN = "acp-kernel-fixture-payload-0123456789"; + +const NATIVE_ZSTD = typeof zlib.zstdCompressSync === "function" && typeof zlib.zstdDecompressSync === "function"; + +type LogLine = { level: string; msg: string }; + +function makeSession(id: string, fill = 0, fillRandom = false): Session { + return { + id, + meta: { protocol: "openai", upstreamOrigin: "http://upstream" }, + stats: { requests: 0, tokensSaved: 0, inputTokens: 0, cachedTokens: 0, outputTokens: 0, cacheSamples: 0, lastInputTokens: 0, contextTokens: 0 }, + metadata: fill > 0 ? { fill: fillRandom ? randomBytes(fill).toString("hex") : "x".repeat(fill) } : {}, + state: createInitialState(), + createdAt: Date.now(), + lastSeen: Date.now(), + blockContents: new Map(), + inFlight: 0, + persisted: false, + }; +} + +interface Harness { + dir: string; + logs: LogLine[]; +} + +async function withTempDir(name: string, fn: (h: Harness) => Promise): Promise { + await test(name, async () => { + const dir = mkdtempSync(join(tmpdir(), "bili-zstd-")); + const logs: LogLine[] = []; + const h = { dir, logs }; + try { + await fn(h); + } finally { + delete process.env.BILI_ENCRYPTION_KEY; + delete process.env.BILI_PERSIST_ZSTD; + _setZstdAvailableForTest(null); + rmSync(dir, { recursive: true, force: true }); + } + }); +} + +function newStore(h: Harness, debounceMs = 0): SessionStore { + return new SessionStore({ + dir: h.dir, + debounceMs, + enabled: true, + log: (level, msg) => h.logs.push({ level, msg }), + }); +} + +function filePath(dir: string, id: string): string { + return join(dir, "openai", "upstream_" + createHash("sha256").update(id, "utf8").digest("hex").slice(0, 24) + ".json"); +} + +test("codec: plain roundtrip is deterministic and carries the BILIZSTD1 header", () => { + const codec = createStorageCodec({ compress: true })!; + // Compressible payload large enough that framing strictly shrinks it (the + // size guard frames only when compression actually wins). + const json = JSON.stringify({ hello: "world ".repeat(400) }); + const enc = Buffer.from(codec.encode(json)); + assert.ok(enc.subarray(0, ZSTD_MAGIC.length).equals(ZSTD_MAGIC), "magic prefix"); + assert.equal(enc[9], 0x01, "format version byte"); + assert.equal(codec.decode(enc), json, "roundtrip"); + assert.deepEqual(Buffer.from(codec.encode(json)), enc, "no nonce -> deterministic bytes"); +}); + +test("codec: config matrix — plain JSON by default, key or opt-in produce a codec", () => { + assert.equal(createStorageCodec({}), undefined, "default opts: no key, no compression -> plain JSON"); + assert.equal(createStorageCodec({ compress: false }), undefined); + assert.equal(createStorageCodec({ key: null, compress: false }), undefined); + assert.ok(createStorageCodec({ key: randomBytes(32) }), "key alone"); + assert.ok(createStorageCodec({ compress: true }), "compression alone"); +}); + +test("codec: legacy plaintext passthrough + magic dispatch across trees", () => { + const plain = createStorageCodec({ compress: true })!; + const keyed = createStorageCodec({ key: Buffer.from(KEY, "hex") })!; + const json = JSON.stringify({ a: 1 }); + assert.equal(plain.decode(Buffer.from(json, "utf8")), json, "legacy plaintext passes through"); + const zBuf = Buffer.from(plain.encode(json)); + assert.equal(keyed.decode(zBuf), json, "keyed codec still reads unencrypted BILIZSTD1 files"); + const eBuf = Buffer.from(keyed.encode(json)); + assert.ok(eBuf.subarray(0, 8).equals(ENCRYPT_MAGIC)); + assert.throws(() => plain.decode(eBuf), /BILI_ENCRYPTION_KEY/, "reading encrypted without a key says exactly that"); +}); + +test("codec: key + explicit compression keeps BILIENC1 in zstd mode (#708 behavior)", () => { + if (!NATIVE_ZSTD) return; + const keyed = createStorageCodec({ key: Buffer.from(KEY, "hex"), compress: true })!; + const json = JSON.stringify({ a: "value ".repeat(500) }); + const enc = Buffer.from(keyed.encode(json)); + assert.ok(enc.subarray(0, ENCRYPT_MAGIC.length).equals(ENCRYPT_MAGIC)); + assert.equal(enc[8], 0x01, "format version"); + assert.equal(enc[9], 0x01, "mode byte records the zstd body"); + assert.equal(keyed.decode(enc), json); + // Key alone defaults to an unframed raw body (compression is opt-in). + const keyedDefault = createStorageCodec({ key: Buffer.from(KEY, "hex") })!; + const encRaw = Buffer.from(keyedDefault.encode(json)); + assert.equal(encRaw[9], 0x00, "key alone -> raw body mode byte"); + assert.equal(keyedDefault.decode(encRaw), json); +}); + +test("fzstd fallback decodes node:zlib frames (static fixture)", () => { + _setZstdAvailableForTest(false); + try { + const codec = createStorageCodec({ compress: true })!; + const body = Buffer.from(ZSTD_FIXTURE_B64, "base64"); + const frame = Buffer.concat([ZSTD_MAGIC, Buffer.from([0x01, 0x01]), body]); + assert.equal(codec.decode(frame), ZSTD_FIXTURE_PLAIN); + } finally { + _setZstdAvailableForTest(null); + } +}); + +withTempDir("store: BILI_PERSIST_ZSTD=1 writes BILIZSTD1 and loads it back", async (h) => { + process.env.BILI_PERSIST_ZSTD = "1"; + const store = newStore(h); + await store.writeNow(makeSession("s-z")); + store.cancelAll(); + const buf = readFileSync(filePath(h.dir, "s-z")); + assert.ok(buf.subarray(0, 9).equals(ZSTD_MAGIC), "file on disk starts with BILIZSTD1"); + const loaded = await newStore(h).boot(); + assert.ok(loaded.has("s-z"), "loads back through the same codec"); + assert.ok(h.logs.some((l) => l.msg.includes("compression enabled")), "boot logs the opt-in codec"); +}); + +withTempDir("store: default (env unset) writes bare JSON and loads it back", async (h) => { + const store = newStore(h); + await store.writeNow(makeSession("s-def")); + store.cancelAll(); + const text = readFileSync(filePath(h.dir, "s-def"), "utf8"); + assert.equal(text[0], "{", "default stays plain JSON (compression is opt-in)"); + const loaded = await newStore(h).boot(); + assert.equal(loaded.size, 1); + assert.ok(!h.logs.some((l) => l.msg.includes("compression enabled"))); +}); + +withTempDir("store: large session shrinks vs the default plain-JSON baseline", async (h) => { + if (!NATIVE_ZSTD) return; + process.env.BILI_PERSIST_ZSTD = "1"; + const big = makeSession("s-big", 20_000); + big.blockContents.set("b1", "acp-block-summary-content ".repeat(20_000)); + const first = newStore(h); + await first.writeNow(big); + first.cancelAll(); + const compressedSize = statSync(filePath(h.dir, "s-big")).size; + process.env.BILI_PERSIST_ZSTD = "0"; + const second = newStore(h); + await second.writeNow(big); + second.cancelAll(); + const plainSize = statSync(filePath(h.dir, "s-big")).size; + assert.ok(compressedSize * 4 < plainSize, `zstd ${compressedSize}B must beat plain ${plainSize}B`); +}); + +withTempDir("store: BILI_PERSIST_ZSTD=0 keeps files as plain JSON", async (h) => { + process.env.BILI_PERSIST_ZSTD = "0"; + const store = newStore(h); + await store.writeNow(makeSession("s-plain")); + store.cancelAll(); + const text = readFileSync(filePath(h.dir, "s-plain"), "utf8"); + assert.equal(text[0], "{", "opt-out writes raw JSON"); + const loaded = await newStore(h).boot(); + assert.equal(loaded.size, 1); + assert.ok(!h.logs.some((l) => l.msg.includes("compression enabled"))); +}); + +withTempDir("store: opted-in runtime without native zstd falls back to unframed plain JSON", async (h) => { + process.env.BILI_PERSIST_ZSTD = "1"; + _setZstdAvailableForTest(false); + const store = newStore(h); + await store.writeNow(makeSession("s-oldrt")); + store.cancelAll(); + const buf = readFileSync(filePath(h.dir, "s-oldrt")); + assert.equal(buf[0], 0x7b, `bare JSON, got head ${buf.subarray(0, 9).toString("utf8")}`); + assert.ok(JSON.parse(buf.toString("utf8")), "parses as plain JSON"); + const loaded = await newStore(h).boot(); + assert.equal(loaded.size, 1, "plain file roundtrips"); +}); + +withTempDir("store: old runtime reads files written by a newer one (fzstd path)", async (h) => { + if (!NATIVE_ZSTD) return; + process.env.BILI_PERSIST_ZSTD = "1"; + const writer = newStore(h); + await writer.writeNow(makeSession("s-new")); + writer.cancelAll(); + const buf = readFileSync(filePath(h.dir, "s-new")); + assert.equal(buf[10], 0x01, "writer used the zstd body"); + _setZstdAvailableForTest(false); + const loaded = await newStore(h).boot(); + assert.equal(loaded.size, 1, "fallback decoder handles the zstd body"); +}); + +withTempDir("boot NEVER rewrites legacy plaintext files (downgrade safety, #1080 review)", async (h) => { + const legacy = newStore(h); + await legacy.writeNow(makeSession("s-1")); + await legacy.writeNow(makeSession("s-2")); + legacy.cancelAll(); + process.env.BILI_PERSIST_ZSTD = "1"; + const before = h.logs.length; + const store = newStore(h); + const loaded = await store.boot(); + assert.equal(loaded.size, 2, "both legacy sessions load under the compression-enabled codec"); + for (const id of ["s-1", "s-2"]) { + const raw = readFileSync(filePath(h.dir, id)); + assert.ok(raw[0] === 0x7b /* '{' */, `${id} still plain JSON on disk after boot`); + JSON.parse(raw.toString("utf8")); + } + assert.ok(!h.logs.slice(before).some((l) => l.msg.includes("re-encoded")), "no mass rewrite logged"); + store.cancelAll(); + // Organic conversion: the next real save re-encodes that one file. + await store.writeNow(makeSession("s-1", 8_000)); + store.cancelAll(); + assert.ok(readFileSync(filePath(h.dir, "s-1")).subarray(0, 9).equals(ZSTD_MAGIC), "next save converts organically"); + assert.ok(readFileSync(filePath(h.dir, "s-2"))[0] === 0x7b, "untouched file stays plaintext"); +}); + +withTempDir("mixed tree: legacy plaintext and BILIZSTD1 coexist until boot", async (h) => { + const legacy = newStore(h); + await legacy.writeNow(makeSession("s-old")); + legacy.cancelAll(); + process.env.BILI_PERSIST_ZSTD = "1"; + const store = newStore(h); + await store.writeNow(makeSession("s-new", 4_000)); + store.cancelAll(); + const loaded = await newStore(h).boot(); + assert.ok(loaded.has("s-old") && loaded.has("s-new"), "both formats readable in one tree"); + assert.ok(readFileSync(filePath(h.dir, "s-old"))[0] === 0x7b, "legacy side stays plaintext (no boot rewrite)"); + assert.ok(readFileSync(filePath(h.dir, "s-new")).subarray(0, 9).equals(ZSTD_MAGIC)); +}); + +withTempDir("keyed boot leaves BILIZSTD1 files untouched and readable", async (h) => { + process.env.BILI_PERSIST_ZSTD = "1"; + const store = newStore(h); + await store.writeNow(makeSession("s-z")); + store.cancelAll(); + const before = readFileSync(filePath(h.dir, "s-z")); + process.env.BILI_ENCRYPTION_KEY = KEY; + const loaded = await newStore(h).boot(); + assert.ok(loaded.has("s-z"), "unencrypted file loads fine under an encryption-enabled boot"); + assert.deepEqual(readFileSync(filePath(h.dir, "s-z")), before, "migration never rewrites what it can already read"); +}); + +withTempDir("no-key boot reports BILIENC1 files as unreadable instead of corrupt garbage", async (h) => { + process.env.BILI_PERSIST_ZSTD = "1"; + process.env.BILI_ENCRYPTION_KEY = KEY; + const writer = newStore(h); + await writer.writeNow(makeSession("s-secret")); + writer.cancelAll(); + delete process.env.BILI_ENCRYPTION_KEY; + const reader = newStore(h); + const loaded = await reader.boot(); + assert.equal(loaded.has("s-secret"), false, "undecodable file is skipped"); + assert.ok( + h.logs.some((l) => l.level === "warn" && l.msg.includes("BILI_ENCRYPTION_KEY")), + h.logs.map((l) => l.level + " " + l.msg).join("\n"), + ); + assert.ok(readFileSync(filePath(h.dir, "s-secret")).subarray(0, 8).equals(ENCRYPT_MAGIC), "file left untouched"); +}); + +withTempDir("decoupling: key + BILI_PERSIST_ZSTD=0 keeps BILIENC1 with a raw body", async (h) => { + process.env.BILI_ENCRYPTION_KEY = KEY; + process.env.BILI_PERSIST_ZSTD = "0"; + const store = newStore(h); + await store.writeNow(makeSession("s-rawenc")); + store.cancelAll(); + const buf = readFileSync(filePath(h.dir, "s-rawenc")); + assert.ok(buf.subarray(0, 8).equals(ENCRYPT_MAGIC)); + assert.equal(buf[8], 0x01, "format version"); + assert.equal(buf[9], 0x00, "mode byte records the raw body"); + const reloaded = newStore(h).loadSync("s-rawenc", { protocol: "openai", upstreamOrigin: "http://upstream" }); + assert.ok(reloaded, "raw-body encrypted file roundtrips"); +}); + +test("codec: payloads that do not shrink stay unframed plain JSON (size guard)", () => { + const codec = createStorageCodec({ compress: true })!; + // Small enough that zstd framing (magic + header + frame overhead) + // cannot beat the bare payload — the guard must return it verbatim. + const tiny = JSON.stringify({ hello: "world", n: [1, 2, 3] }); + const enc = Buffer.from(codec.encode(tiny)); + assert.equal(enc[0], 0x7b, `bare JSON head, got ${enc.subarray(0, 9).toString("utf8")}`); + assert.equal(enc.toString("utf8"), tiny, "byte-identical plaintext (no framing)"); + assert.equal(codec.decode(enc), tiny, "decode accepts unframed plaintext"); + // And the store still round-trips such a file (loadAll -> decode -> JSON). + const big = "x".repeat(20_000); + assert.notEqual(Buffer.from(codec.encode(big)).subarray(0, 9).toString("utf8"), big.slice(0, 9), "compressible payload still frames"); +}); diff --git a/tests/proxy-persist.test.ts b/tests/proxy-persist.test.ts index a475ce2c74..b86c70dae2 100644 --- a/tests/proxy-persist.test.ts +++ b/tests/proxy-persist.test.ts @@ -8,6 +8,11 @@ import { setLogCapture } from "../src/logger.ts"; import { dirname, join, relative, sep } from "node:path"; import type { Session, BlockContent } from "../src/session.ts"; import { createInitialState } from "acp-kernel"; + +// These tests inspect raw on-disk session files directly (bypassing the store +// codec), so pin the plain-JSON format: #1080 made BILIZSTD1 the default. +process.env.BILI_PERSIST_ZSTD = "0"; + /** Recursively collect *.json files under dir (sessions are namespaced into * protocol/ subdirs). */ function jsonFilesUnder(dir: string): string[] {