diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e73ff7075..adbe806916 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,15 @@ Versions follow the merge of a `*_release-v*` branch; CI publishes to npm on tag ## [Unreleased] +### Changed (storage correctness wave 3, #4242) + +- **State hydration can no longer drift from `mergeState`** (#4192): `COMPRESSION_STATE_FIELDS` in the new `src/state-hydration.ts` is derived mechanically from `Object.keys(mergeState(createInitialState()))` — the ingest/restore field list is generated from the same source as hydration, so a future field can never be restored-but-never-ingested (the `sweptOrphanCallIds`/`pendingOrphanSweepMinRef` loss class). Pinned by a parity test that fails the moment `createInitialState()` grows a key the list does not know. +- **CAS orphans are deleted the moment their refcount hits zero** (#3987): `releaseContent` now deletes the row on the same statement (`UPDATE … RETURNING` + delete on 0) on both the sqlite and mysql backends. Follows the FK discipline strictly — referencing rows are now always deleted *before* their content is released (five call sites reordered), so a zero-ref row can never be dropped while a surviving row still points at it. +- **Reap-shape sessions keep advancing instead of freezing** (#4216): blockContents that outlive their blocks (the #2695 reap shape) now ride the ingest plan as `sessions.orphans_json` — merged (`∪`, because LazyBlockContents may trim entries from memory) in the same transaction as everything else. The mirror skip, the async-twin skip, the importRecord skip and the #3818 writerOff JSON safety valve are all gone: the shape is representable, so `writerOff`'s "the index can represent everything I write" contract holds without an escape hatch. Cold loads after a reap now serve fresh state (the freeze-at-`requests=390` repro is pinned). +- **Raw wire sidecars join the media CAS** (#4069): `splitMediaContent` now also swaps every `data:;base64,` occurrence ≥2 KiB inside `rawOpenaiContent` / `rawOpenaiContentParts` / `rawResponsesItem` / `rawGoogleParts` for `{$media:}` markers; hydration restores the bytes byte-identically on both backends. A picture used to be stored twice (top-level `imageBase64` + the raw sidecar data URL), capping dedup at ~1.89× — the raw sidecar now references the same CAS row (33.8× on the issue's corpus shape). Pinned: no payload bytes in the stored JSON, byte-identical round-trip, shared-row refcount=2. +- **Full-mode authority over a tampered envelope is now pinned** (#4004): the four legacy-envelope integrity tests (`proxy-persist` ×3, `issue2129` ×1) pin `storage.unified=legacy` — they assert the envelope guard layer, which full mode legitimately supersedes; a new test pins the decided semantics instead (index serves while it holds the record; only when both carriers are gone/tampered does the load reject). These were flaky-by-construction under the #3818 full default. +- test: storage-render goldens regenerated once — the drift is inherited from master (#4198 headroom revert changed the stats/metadata shape inside fold-turn bodies), not from this PR; full==legacy==goldens still holds on every wire. + ### dsh 归因正面修:llm/stream 会话身份随请求对象(#4082) - dsh 的 LlmRuntime 在每次派发上盖章 `options.sessionId`(主循环/压缩/标题生成,重试重发也重新盖章),且把 llm/stream waterfall 文档化为官方拦截点。插件现在把每条流的迭代包进以该 sid 为键的 AsyncLocalStorage:适配器的 fetch 读到的会话身份**跟随请求对象,不再依赖异步链形状**——429 重试链、无 initiator 的 agentless 侧调用(compaction/标题)、已销毁的 initiator 作用域、宿主运行时缺口(#2975/#3918/#4082)全部免疫。 diff --git a/src/persist.ts b/src/persist.ts index 9e790fc5dc..1187e346f6 100644 --- a/src/persist.ts +++ b/src/persist.ts @@ -28,10 +28,11 @@ import { VERSION } from "./version.js"; import { createStorageCodec, parseEncryptionKey } from "./encrypt.js"; import { PersistEpermAlert } from "./persist-eperm.js"; import { createInitialState, defaultCountTokens, prune, type CompressionState, type CoreMessage, type MessageContentStore } from "acp-kernel"; -import { markDirty, scrubTurnSeparatorIds } from "./session.js"; +import { markDirty } from "./session.js"; +import { mergeState } from "./state-hydration.js"; import { LazyBlockContents, isDroppedMetadata, lazyMetadata } from "./session.js"; import type { Session, BlockContent, BlockView } from "./session.js"; -import { orphanContents, resolveStorageMode, UnifiedShadow, type StorageMode } from "./storage/shadow.js"; +import { resolveStorageMode, UnifiedShadow, type StorageMode } from "./storage/shadow.js"; import { resolveBackendName } from "./storage/backend.js"; import { toLegacyLike, type LegacySessionLike } from "./storage/ingest.js"; import { sha256Hex, type CheckpointRow } from "./storage/store.js"; @@ -215,55 +216,6 @@ interface PersistedSession { type Logger = (level: "info" | "warn" | "error", msg: string) => void; -/** Forward-compat: merge a parsed state with a fresh one so missing fields - * (added in later versions) get sane defaults instead of `undefined`. */ -function mergeState(parsed: CompressionState): CompressionState { - const fresh = createInitialState(); - return { - // #2627: heal persisted state written before the turn-separator fix — - // those blocks' coverage carries outbound-only acp_turn_sep_* ids that - // no resent history can ever contain; scrubbing at hydration is what - // stops live long sessions from alarming forever after an upgrade. - blocks: (() => { const blocks = parsed.blocks ?? fresh.blocks; scrubTurnSeparatorIds(blocks); return blocks; })(), - messageRefs: parsed.messageRefs ?? fresh.messageRefs, - nudge: { ...fresh.nudge, ...(parsed.nudge ?? {}) }, - stats: { ...fresh.stats, ...(parsed.stats ?? {}) }, - nextBlockId: parsed.nextBlockId ?? fresh.nextBlockId, - nextRunId: parsed.nextRunId ?? fresh.nextRunId, - tokenSnapshot: parsed.tokenSnapshot ?? fresh.tokenSnapshot, - lastPassIds: parsed.lastPassIds ?? fresh.lastPassIds, - hiddenOrphanRefs: parsed.hiddenOrphanRefs ?? fresh.hiddenOrphanRefs, - // #2362: without this a restart forgets which refs are dead and the - // model re-learns each one by failing a compress call again. - deadRefs: parsed.deadRefs ?? fresh.deadRefs, - terminalStreak: parsed.terminalStreak ?? fresh.terminalStreak, - nextRuleId: parsed.nextRuleId ?? fresh.nextRuleId, - // Without this, a restart re-exposes absorbed tool outputs: state - // resurrects with absorbed=[] and hideAbsorbedMessages has nothing to hide. - absorbed: parsed.absorbed ?? fresh.absorbed, - rules: parsed.rules ?? fresh.rules, - // #3769: fold-gated orphan-sweep bookkeeping must survive hydration — - // dropping sweptOrphanCallIds resurrects already-cleaned pairs after a - // restart (mid-history rewrite = cache break); the pending marker is - // transient, losing it only defers the sweep to the next fold. - sweptOrphanCallIds: parsed.sweptOrphanCallIds ?? fresh.sweptOrphanCallIds, - pendingOrphanSweepMinRef: parsed.pendingOrphanSweepMinRef ?? fresh.pendingOrphanSweepMinRef, - // #3939 ③/#3691: the emergency-truncate latch must survive hydration — - // dropping it re-exposes clipped bytes mid-history after a restart, - // rewriting the still-cached prefix on the first post-restart turn. - emergencyTruncateLatch: parsed.emergencyTruncateLatch ?? fresh.emergencyTruncateLatch, - // #3939 ②/#3805: the render deferral must survive hydration — losing - // it stamps migration tags onto still-cached carriers right after a - // restart (a silent mid-history rewrite with no fold in sight). - renderDeferredRefs: parsed.renderDeferredRefs ?? fresh.renderDeferredRefs, - // #3939 ⑤: the view latch must survive hydration — losing it - // re-derives transform output on still-cached carriers right after a - // restart (silent mid-history rewrite, no fold in sight). - viewLatches: parsed.viewLatches ?? fresh.viewLatches, - viewLatchConfigDigest: parsed.viewLatchConfigDigest ?? fresh.viewLatchConfigDigest, - }; -} - /** Extract a short host label from an upstream origin URL, safe for a * filename. Uses the full hostname (sanitized) rather than guessing the * registrable domain — a public-suffix-list lookup is overkill, and the full @@ -2323,21 +2275,12 @@ export class SessionStore { // Errors surface through the kernel write chain's own logging. }); } - // #3818 writerOff safety valve: writerOff's contract is - // "the index can represent everything I write". An - // orphan-contents record (the #2695 reap shape — block - // gone from state.blocks, text kept in blockContents) - // is SKIPPED by the mirror guard, so this tick would - // otherwise persist NOTHING at all (no rows, no file). - // Land the JSON envelope as the durable copy; the record - // stays envelope-served on every later read. - if (this.shadow && !this.shadow.stats.disabled && orphanContents(toLegacyLike(session))) { - this.log("warn", `[persist] unrepresentable record for ${session.id} under writerOff — landing a JSON envelope copy (index cannot carry its blockContents)`); - const build = this.guardedBuild(session); - void this.store.writeNow(session.id, build).then(() => this.stampOwnWrite(session.id)).catch(() => { - // Errors surface through the kernel write chain's own logging. - }); - } + // #3818 writerOff safety valve REMOVED (#4216/#4242): + // the orphan/reap shape is now representable — its + // contents ride sessions.orphans_json in the same + // transaction as everything else, so writerOff's contract + // ("the index can represent everything I write") holds + // without a JSON-envelope escape hatch. } catch (err) { // Kernel-store never-throw save discipline: an uncaughtException // escaping this microtask is fatal in embedded/plugin hosts diff --git a/src/session.ts b/src/session.ts index 8a0b5d1e89..13ee9cf544 100644 --- a/src/session.ts +++ b/src/session.ts @@ -10,6 +10,7 @@ import { durableMessageToolPaths, withDurableProtection } from "./durable-messag import type { AnonymousProvenance } from "./anon-provenance.js"; import type { WireProtocol } from "./util.js"; import type { InboundSlot, SeamSlot } from "./cache-ledger.js"; +import { isResponsesTurnSeparatorId } from "./turn-separator.js"; export type BlockView = { text: string; count: number }; @@ -1621,42 +1622,11 @@ export function foldCoverage( return matched < coveredBefore.size ? { expected: coveredBefore.size, matched } : null; } -/** #2627: bili-synthesized Responses turn separators (repairResponsesAssistantOrdering, - * src/server.ts) are OUTBOUND-ONLY ordering markers — no client's resent history - * ever contains them, so they can never anchor or be covered. When a - * separator-bearing view reached applyCompression (the plugin lane handed its - * processed view straight in), their ids were registered into new blocks' - * direct/effectiveMessageIds and inherited by higher folds via child merge — - * permanently "missing" on every reconcile pass: false [acp-drift]/fold- - * reconcile warns and stalled per-block ledger evidence against an unchanged - * history. They are protocol-ordering artifacts, not compressible raw history: - * exclude them everywhere coverage is stored or compared. */ -export const RESPONSES_TURN_SEPARATOR_ID_PREFIX = "acp_turn_sep_"; - -export function isResponsesTurnSeparatorId(id: string): boolean { - return id.startsWith(RESPONSES_TURN_SEPARATOR_ID_PREFIX); -} - -/** #2627: strip turn-separator ids from block coverage IN PLACE (load-time heal - * of persisted state + post-commit scrub). Returns how many ids were removed; - * 0 means the state was already clean. */ -export function scrubTurnSeparatorIds( - blocks: ReadonlyArray<{ directMessageIds?: string[]; effectiveMessageIds?: string[] }>, -): number { - let removed = 0; - for (const block of blocks) { - for (const key of ["directMessageIds", "effectiveMessageIds"] as const) { - const ids = block[key]; - if (!ids) continue; - const kept = ids.filter((id) => !isResponsesTurnSeparatorId(id)); - if (kept.length !== ids.length) { - removed += ids.length - kept.length; - block[key] = kept; - } - } - } - return removed; -} +// #2627 turn-separator helpers moved to ./turn-separator.ts (zero-dependency leaf +// — the storage seam needs them without session.ts's built-acp-kernel import; +// the doc comments live there now). Re-exported so existing consumers keep the +// session.js import path. +export { RESPONSES_TURN_SEPARATOR_ID_PREFIX, isResponsesTurnSeparatorId, scrubTurnSeparatorIds } from "./turn-separator.js"; /** #2627: the REAL-history coverage set — active blocks' effective ids minus * outbound-only turn separators. Single source for every drift consumer (the diff --git a/src/state-hydration.ts b/src/state-hydration.ts new file mode 100644 index 0000000000..37b9d697a4 --- /dev/null +++ b/src/state-hydration.ts @@ -0,0 +1,76 @@ +/** + * #4242 / #4192: state hydration is the single source of truth for which + * CompressionState fields survive persistence. mergeState() moved here so + * both the JSON envelope loader (persist.ts) and the unified view builder + * (storage/ingest.ts) consume ONE implementation, and the field list is + * DERIVED from it mechanically — Object.keys(mergeState(fresh)) — so a field + * added to the literal appears in kstate serialization, view rebuild and + * digest hashing automatically. The hand-kept MERGE_STATE_FIELDS list is + * gone; it had already drifted (sweptOrphanCallIds, + * pendingOrphanSweepMinRef — the #2388 explicit-field-clone disease). + */ +// Relative kernel-source import, NOT the "acp-kernel" package: this module is +// in the storage seam's load graph, which the CI mysql-contract lane runs with +// an UNBUILT kernel (install --ignore-scripts, no build step) — the package +// form resolves to kernel/dist and crashes that lane at import time (#4249). +import { createInitialState } from "../kernel/src/state.js"; +import type { CompressionState } from "../kernel/src/types.js"; +import { scrubTurnSeparatorIds } from "./turn-separator.js"; + +/** Forward-compat: merge a parsed state with a fresh one so missing fields + * (added in later versions) get sane defaults instead of `undefined`. */ +export function mergeState(parsed: CompressionState): CompressionState { + const fresh = createInitialState(); + return { + // #2627: heal persisted state written before the turn-separator fix — + // those blocks' coverage carries outbound-only acp_turn_sep_* ids that + // no resent history can ever contain; scrubbing at hydration is what + // stops live long sessions from alarming forever after an upgrade. + blocks: (() => { const blocks = parsed.blocks ?? fresh.blocks; scrubTurnSeparatorIds(blocks); return blocks; })(), + messageRefs: parsed.messageRefs ?? fresh.messageRefs, + nudge: { ...fresh.nudge, ...(parsed.nudge ?? {}) }, + stats: { ...fresh.stats, ...(parsed.stats ?? {}) }, + nextBlockId: parsed.nextBlockId ?? fresh.nextBlockId, + nextRunId: parsed.nextRunId ?? fresh.nextRunId, + tokenSnapshot: parsed.tokenSnapshot ?? fresh.tokenSnapshot, + lastPassIds: parsed.lastPassIds ?? fresh.lastPassIds, + hiddenOrphanRefs: parsed.hiddenOrphanRefs ?? fresh.hiddenOrphanRefs, + // #2362: without this a restart forgets which refs are dead and the + // model re-learns each one by failing a compress call again. + deadRefs: parsed.deadRefs ?? fresh.deadRefs, + terminalStreak: parsed.terminalStreak ?? fresh.terminalStreak, + nextRuleId: parsed.nextRuleId ?? fresh.nextRuleId, + // Without this, a restart re-exposes absorbed tool outputs: state + // resurrects with absorbed=[] and hideAbsorbedMessages has nothing to hide. + absorbed: parsed.absorbed ?? fresh.absorbed, + rules: parsed.rules ?? fresh.rules, + // #3769: fold-gated orphan-sweep bookkeeping must survive hydration — + // dropping sweptOrphanCallIds resurrects already-cleaned pairs after a + // restart (mid-history rewrite = cache break); the pending marker is + // transient, losing it only defers the sweep to the next fold. + sweptOrphanCallIds: parsed.sweptOrphanCallIds ?? fresh.sweptOrphanCallIds, + pendingOrphanSweepMinRef: parsed.pendingOrphanSweepMinRef ?? fresh.pendingOrphanSweepMinRef, + // #3939 ③/#3691: the emergency-truncate latch must survive hydration — + // dropping it re-exposes clipped bytes mid-history after a restart, + // rewriting the still-cached prefix on the first post-restart turn. + emergencyTruncateLatch: parsed.emergencyTruncateLatch ?? fresh.emergencyTruncateLatch, + // #3939 ②/#3805: the render deferral must survive hydration — losing + // it stamps migration tags onto still-cached carriers right after a + // restart (a silent mid-history rewrite with no fold in sight). + renderDeferredRefs: parsed.renderDeferredRefs ?? fresh.renderDeferredRefs, + // #3939 ⑤: the view latch must survive hydration — losing it + // re-derives transform output on still-cached carriers right after a + // restart (silent mid-history rewrite, no fold in sight). + viewLatches: parsed.viewLatches ?? fresh.viewLatches, + viewLatchConfigDigest: parsed.viewLatchConfigDigest ?? fresh.viewLatchConfigDigest, + }; +} + +/** + * Every CompressionState field mergeState() emits, in literal order. Derived + * at module load from the merge itself — no second list to keep in sync + * (#4192). Consumers: kstate_json serialization order (storage/ingest.ts), + * unified view rebuild, digest key order. + */ +export const COMPRESSION_STATE_FIELDS: readonly (keyof CompressionState)[] = + Object.keys(mergeState(createInitialState())) as (keyof CompressionState)[]; diff --git a/src/storage/ingest.ts b/src/storage/ingest.ts index 2f4e0d8499..d9470406b5 100644 --- a/src/storage/ingest.ts +++ b/src/storage/ingest.ts @@ -30,28 +30,12 @@ import type { BlockView } from "../session.js"; import { refToIndex } from "../../kernel/src/refs.js"; import { sha256Hex, type BlockRow, type CcrEntryRow, type IngestSessionInput, type MessageRow, type RefRow, type SessionRow } from "./store.js"; -/** Field order of mergeState() output (src/persist.ts) — the loader must - * rebuild the state object in exactly this order. Keep in sync. */ -const MERGE_STATE_FIELDS = [ - "blocks", - "messageRefs", - "nudge", - "stats", - "nextBlockId", - "nextRunId", - "tokenSnapshot", - "lastPassIds", - "hiddenOrphanRefs", - "emergencyTruncateLatch", - "renderDeferredRefs", - "viewLatches", - "viewLatchConfigDigest", - "deadRefs", - "terminalStreak", - "nextRuleId", - "absorbed", - "rules", -] as const; +/** + * Field order of the hydration merge output — DERIVED from mergeState itself + * (src/state-hydration.ts), not hand-kept: the hand list had already drifted + * (#4192). The loader rebuilds the state object in exactly this order. + */ +import { COMPRESSION_STATE_FIELDS as MERGE_STATE_FIELDS } from "../state-hydration.js"; /** Fields owned by dedicated tables/columns — never inside kstate_json. */ const TABLE_OWNED_FIELDS = new Set(["blocks", "messageRefs", "deadRefs", "nextBlockId", "nextRunId"]); @@ -103,20 +87,54 @@ export const ingestSerializations = { count: 0 }; * reader (getMessages hydrates before returning) sees the verbatim original. * No media (or empty) → passthrough with the plain content hash. */ +/** + * Media payloads below this size ride inline even inside raw sidecars — + * splitting them costs a CAS row + a join for a few hundred bytes (#4069). + */ +const MIN_MEDIA_URL_SPLIT_BYTES = 2048; +/** data:;base64, with the payload swapped for its CAS hash. */ +const MEDIA_URL_MARKER_RE = new RegExp( + `data:([a-zA-Z0-9.+\\/-]+);base64,([A-Za-z0-9+/=]{${MIN_MEDIA_URL_SPLIT_BYTES},})`, + "g", +); + function splitMediaContent(content: string): { content: string; hash: string; media: { hash: string; bytes: string }[] } { - if (!content.includes("\"imageBase64\"")) return { content, hash: sha256Hex(content), media: [] }; - let parsed: { imageBase64?: unknown }; - try { - parsed = JSON.parse(content) as { imageBase64?: unknown }; - } catch { - return { content, hash: sha256Hex(content), media: [] }; + let media: { hash: string; bytes: string }[] = []; + let split = content; + if (content.includes("\"imageBase64\"")) { + let parsed: { imageBase64?: unknown }; + try { + parsed = JSON.parse(content) as { imageBase64?: unknown }; + } catch { + parsed = undefined as unknown as { imageBase64?: unknown }; + } + if (parsed) { + const bytes = parsed.imageBase64; + if (typeof bytes === "string" && bytes.length > 0) { + const mediaHash = sha256Hex(bytes); + parsed.imageBase64 = { $media: mediaHash }; + split = JSON.stringify(parsed); + media.push({ hash: mediaHash, bytes }); + } + } } - const bytes = parsed.imageBase64; - if (typeof bytes !== "string" || bytes.length === 0) return { content, hash: sha256Hex(content), media: [] }; - const mediaHash = sha256Hex(bytes); - parsed.imageBase64 = { $media: mediaHash }; - const split = JSON.stringify(parsed); - return { content: split, hash: sha256Hex(split), media: [{ hash: mediaHash, bytes }] }; + // #4069: raw wire sidecars (rawOpenaiContent / rawOpenaiContentParts / + // rawResponsesItem / rawGoogleParts, …) carry the ORIGINAL part verbatim — + // for image parts that means the same base64 payload a second time, as a + // data URL. Swap every sufficiently large payload for its CAS hash, mime + // prefix and JSON structure untouched, so hydration is byte-identical. + if (split.includes(";base64,")) { + const byHash = new Map(media.map((m) => [m.hash, m.bytes])); + split = split.replace(MEDIA_URL_MARKER_RE, (_all: string, mime: string, payload: string) => { + const mediaHash = sha256Hex(payload); + if (!byHash.has(mediaHash)) { + byHash.set(mediaHash, payload); + media = [...media, { hash: mediaHash, bytes: payload }]; + } + return `data:${mime};base64,{$media:${mediaHash}}`; + }); + } + return { content: split, hash: sha256Hex(split), media }; } interface LogEntry { @@ -361,6 +379,19 @@ function finishPlan( }; }); + // #4216/#4242: the orphan journal — blockContents entries whose block is + // gone from state.blocks (#2695 reap keeps the contents for decompress). + // Rides every plan; the store merges it into sessions.orphans_json in the + // same transaction as the rest of the session. This is what makes the + // reap shape REPRESENTABLE, so record() no longer skips the ingest (the + // freeze in #4216: sessions stuck at hours-old rows). + const knownBlockIds = new Set(session.state.blocks.map((b) => b.blockId)); + const orphanEntries: Record = {}; + for (const [blockId, view] of session.blockContents) { + if (!knownBlockIds.has(blockId)) orphanEntries[blockId] = { one: view.one, full: view.full }; + } + const orphansJson = Object.keys(orphanEntries).length > 0 ? JSON.stringify(orphanEntries) : null; + const deadRefs = (session.state.deadRefs ?? []).map((ref, ord) => ({ ord, ref, @@ -401,6 +432,7 @@ function finishPlan( status: "active", createdAt: session.createdAt, updatedAt: session.lastSeen, + orphansJson, nextBlockId: session.state.nextBlockId, nextRunId: session.state.nextRunId, metaJson: JSON.stringify(session.meta), diff --git a/src/storage/mysql.ts b/src/storage/mysql.ts index 79de335a6c..6750f46b7e 100644 --- a/src/storage/mysql.ts +++ b/src/storage/mysql.ts @@ -297,7 +297,10 @@ export class MysqlBackend implements StorageBackend { private async releaseContent(c: PoolConnection | Pool, hash: string | null): Promise { if (!hash) return; - await c.query("UPDATE content SET refcount = GREATEST(refcount - 1, 0) WHERE hash = ?", [hash]); + // #3987 / #4242: delete-on-zero, same discipline as the sqlite twin — + // orphans must not structurally accumulate on either backend. + await c.query("UPDATE content SET refcount = refcount - 1 WHERE hash = ? AND refcount > 0", [hash]); + await c.query("DELETE FROM content WHERE hash = ? AND refcount <= 0", [hash]); } private async getContent(c: PoolConnection | Pool, hash: string): Promise { @@ -826,21 +829,40 @@ export class MysqlBackend implements StorageBackend { } private async hydrateMediaAsync(content: string): Promise { - if (!content.includes('"$media"')) return content; - try { - const parsed = JSON.parse(content) as { imageBase64?: unknown }; - const slot = parsed.imageBase64; - if (slot !== null && typeof slot === "object" && typeof (slot as { $media?: unknown }).$media === "string") { - const bytes = await this.getContent(this.pool, (slot as { $media: string }).$media); - if (typeof bytes === "string" && bytes.length > 0) { - parsed.imageBase64 = bytes; - return JSON.stringify(parsed); + let out = content; + if (out.includes('"$media"')) { + try { + const parsed = JSON.parse(out) as { imageBase64?: unknown }; + const slot = parsed.imageBase64; + if (slot !== null && typeof slot === "object" && typeof (slot as { $media?: unknown }).$media === "string") { + const bytes = await this.getContent(this.pool, (slot as { $media: string }).$media); + if (typeof bytes === "string" && bytes.length > 0) { + parsed.imageBase64 = bytes; + out = JSON.stringify(parsed); + } } + } catch { + // fall through: raw-sidecar pass below still runs } - return content; - } catch { - return content; } + // #4069: restore data-URL payloads swapped out of raw wire sidecars + // (mysql twin of UnifiedStore.hydrateMedia). + if (out.includes(";base64,{$media:")) { + const markerRe = /data:([a-zA-Z0-9.+\/\-]+);base64,\{\$media:([0-9a-f]{64})\}/g; + const hits: { mime: string; hash: string }[] = []; + for (const m of out.matchAll(markerRe)) hits.push({ mime: m[1], hash: m[2] }); + if (hits.length > 0) { + const cache = new Map(); + for (const h of new Set(hits.map((x) => x.hash))) { + cache.set(h, await this.getContent(this.pool, h)); + } + out = out.replace(markerRe, (all: string, mime: string, hash: string) => { + const bytes = cache.get(hash); + return typeof bytes === "string" && bytes.length > 0 ? `data:${mime};base64,${bytes}` : all; + }); + } + } + return out; } async getBlocks(sessionId: string): Promise<(BlockRow & { summaryHash: string; oneHash: string | null })[]> { diff --git a/src/storage/shadow.ts b/src/storage/shadow.ts index 0713128015..c13e7ebd31 100644 --- a/src/storage/shadow.ts +++ b/src/storage/shadow.ts @@ -269,40 +269,13 @@ export class UnifiedShadow { try { const loaded = this.legacyLoad(record); if (!loaded) return; - // #3818/#3135: a record carrying blockContents ids with no - // matching state.blocks entry (hand-built archives, web seeds) - // is a shape the unified ingest cannot represent — block rows - // land only for state.blocks entries, so a viewFor rebuild - // would silently drop those contents. Real kernel sessions - // never have orphans (state.blocks keeps consumed blocks), so - // skipping the mirror here is free; the JSON envelope stays - // the superset on disk. - if (orphanContents(loaded)) { - // #3841 reap journal: distinguish the REAL kernel reap shape - // (#2695: reaped block rows exist in the store but the - // incoming state no longer claims them) from never-ingested - // hand-built shapes. For reaps, drop exactly the reaped rows - // and journal the orphan contents — without this the rows - // resurrect into every viewFor rebuild after a restart and - // the next envelope save re-pollutes. Envelope stays the - // serving path either way (owner-pinned semantics). - const claimed = new Set(loaded.state.blocks.map((b) => b.blockId)); - const reapedIds = new Set(); - const orphans = new Map(); - try { - for (const row of this.sync.getBlocks(loaded.id)) { - if (!claimed.has(row.blockId)) reapedIds.add(row.blockId); - } - for (const [blockId, view] of loaded.blockContents) { - if (!claimed.has(blockId)) orphans.set(blockId, view); - } - if (reapedIds.size > 0 || orphans.size > 0) this.sync.reapBlocks(loaded.id, reapedIds, orphans); - } catch { - // journal failure must not break the save path - } - this.stats.orphanSkips++; - return; - } + // #4216/#4242: the reap shape (blockContents without their + // state.blocks entries) is now REPRESENTABLE — planIngest + // journals the orphan contents into sessions.orphans_json and + // the store merges it in the same transaction, while the blocks + // stale-sweep drops the reaped rows. The old early-return here + // froze every subsequent save for such sessions (rows stuck at + // hours-old state, #4216). Normal ingest for everyone. let legacy = loaded; let hasRawTail = false; if (rawTail && rawTail.lastMessages.length > 0) { @@ -483,10 +456,8 @@ export class UnifiedShadow { // #3818: same orphan-contents guard as record() — a record the // index cannot represent faithfully must not be served from a // viewFor rebuild; callers fall back to the envelope itself. - if (orphanContents(legacy)) { - this.stats.orphanSkips++; - return null; - } + // #4216/#4242: orphan shape is representable (journal rides the + // plan) — import normally instead of rejecting. this.sync.ingestSession(planIngest(legacy)); this.stats.imports++; loggerLog("info", `[storage] unified import on first open: ${legacy.id}`); @@ -766,20 +737,8 @@ export class UnifiedShadow { } if (liveCcr) legacy = { ...legacy, contentStore: liveCcr }; // #3818/#3841: same orphan-guard + reap-journal semantics. - if (orphanContents(legacy)) { - const claimed = new Set(legacy.state.blocks.map((b) => b.blockId)); - const reapedIds = new Set(); - const orphans = new Map(); - for (const row of await backend.getBlocks(legacy.id)) { - if (!claimed.has(row.blockId)) reapedIds.add(row.blockId); - } - for (const [blockId, view] of legacy.blockContents) { - if (!claimed.has(blockId)) orphans.set(blockId, view); - } - if (reapedIds.size > 0 || orphans.size > 0) await backend.reapBlocks(legacy.id, reapedIds, orphans); - this.stats.orphanSkips++; - return; - } + // #4216/#4242: orphan shape rides the plan journal — normal + // ingest (the async twin of the sync fix); no freeze. // #3133 watermark delta. let applied = false; try { @@ -869,10 +828,7 @@ export class UnifiedShadow { try { const legacy = this.legacyLoad(record); if (!legacy) return null; - if (orphanContents(legacy)) { - this.stats.orphanSkips++; - return null; - } + // #4216/#4242: orphan shape rides the plan journal — import normally. await this.enqueue(() => this.backend!.ingestSession(planIngest(legacy))); this.stats.imports++; loggerLog("info", `[storage] unified import on first open: ${legacy.id}`); diff --git a/src/storage/store.ts b/src/storage/store.ts index 140e042a46..df7fdfe78a 100644 --- a/src/storage/store.ts +++ b/src/storage/store.ts @@ -249,6 +249,13 @@ export interface IngestSessionInput { * absolute seqs before insert — see remapPlanSeqs. */ pluginSnapshotSeqs?: number[] | null; tailSeqs?: number[] | null; + /** Serialized orphan map (Record) — blockContents + * entries whose block is absent from state.blocks (#4216/#4242: the reap + * shape is REPRESENTABLE; rides the same transaction so the session row, + * blocks and tail keep advancing). MERGE semantics on write: existing + * journal entries survive, because LazyBlockContents means a live + * session's in-memory map may not carry entries trimmed earlier. */ + orphansJson?: string | null; } /** Extension-namespace guards (#3069): keys/namespaces are non-empty @@ -393,9 +400,59 @@ export class UnifiedStore { } /** refcount-aware release; the CAS body itself never leaves. */ + /** #4242: merge the incoming orphan map over the journal's current + * contents. Incoming entries win on key collision (same id, refreshed + * view); existing entries the live session no longer carries (trimmed + * from memory by LazyBlockContents) survive. */ + private mergeOrphansJson(sessionId: string, incomingJson: string | null): string | null { + if (incomingJson === null) { + // No orphans in this save — keep whatever the journal already + // holds (a previous reap's trimmed-away contents must survive). + const row = this.db + .prepare("SELECT orphans_json FROM sessions WHERE session_id = ?") + .get(sessionId) as { orphans_json: string | null } | undefined; + return row?.orphans_json ?? null; + } + let merged: Record = {}; + try { + const existing = this.db + .prepare("SELECT orphans_json FROM sessions WHERE session_id = ?") + .get(sessionId) as { orphans_json: string | null } | undefined; + if (existing?.orphans_json) { + const parsed = JSON.parse(existing.orphans_json) as Record; + if (parsed && typeof parsed === "object") merged = parsed; + } + } catch { + // corrupt journal → start fresh (loadLegacyView skips bad entries too) + } + try { + const incoming = JSON.parse(incomingJson) as Record; + if (incoming && typeof incoming === "object") merged = { ...merged, ...incoming }; + } catch { + // corrupt incoming → keep existing only + } + return Object.keys(merged).length > 0 ? JSON.stringify(merged) : null; + } + private releaseContent(hash: string | null): void { if (!hash) return; - this.db.prepare("UPDATE content SET refcount = MAX(refcount - 1, 0) WHERE hash = ?").run(hash); + // #3987 / #4242: delete-on-zero. The old decrement-only update left + // zero-refcount rows behind (11.32 GB of orphans on a real corpus) + // because the only sweeps lived in deleteSession()/gc(), which nobody + // runs on a schedule. Releasing the LAST reference now removes the + // row in the same statement — orphans cannot structurally accumulate. + // Re-putContent of the same bytes re-inserts cleanly (putContent is + // upsert), so no correctness change, only no-leak. + this.db + .prepare( + "UPDATE content SET refcount = refcount - 1 WHERE hash = ? AND refcount > 0 RETURNING refcount", + ) + .all(hash) + .forEach((row) => { + if ((row as { refcount: number }).refcount <= 0) { + this.db.prepare("DELETE FROM content WHERE hash = ? AND refcount <= 0").run(hash); + } + }); } /** Ingests one session plan as a single transaction. Full path (no @@ -505,18 +562,19 @@ export class UnifiedStore { .all(s.sessionId) as { seq: number; msg_id: string; content_hash: string }[]; for (const row of staleRows) { if (incomingIds.has(`${row.msg_id}\u0000${row.content_hash}`)) continue; - this.releaseContent(row.content_hash); - // #3762 review (#3901): media mappings ride the message - // sweep — release each referenced media hash and drop - // the rows with their message, or the CAS bytes leak - // forever (deleteSession/archiveSession already do). - for (const mm of this.db + // #4242 delete-on-zero discipline: remove the + // referencing rows FIRST, then release the hashes — + // a release that hits zero now DELETEs the content + // row, which an FK-referencing leftover row would + // reject. (Media mappings ride the message sweep, + // #3762 review.) + const staleMedia = this.db .prepare("SELECT media_hash FROM message_media WHERE session_id = ? AND seq = ?") - .all(s.sessionId, row.seq) as { media_hash: string }[]) { - this.releaseContent(mm.media_hash); - } + .all(s.sessionId, row.seq) as { media_hash: string }[]; this.db.prepare("DELETE FROM messages WHERE session_id = ? AND seq = ?").run(s.sessionId, row.seq); this.db.prepare("DELETE FROM message_media WHERE session_id = ? AND seq = ?").run(s.sessionId, row.seq); + this.releaseContent(row.content_hash); + for (const mm of staleMedia) this.releaseContent(mm.media_hash); } } } @@ -524,10 +582,10 @@ export class UnifiedStore { .prepare( `INSERT INTO sessions (session_id, parent_session_id, fork_point_seq, protocol, label, title, host_title, namespace, origin, status, created_at, updated_at, next_block_id, next_run_id, - meta_json, stats_json, metadata_json, kstate_json, tail_json, ccr_json, last_ingested_seq) + meta_json, stats_json, metadata_json, kstate_json, tail_json, ccr_json, last_ingested_seq, orphans_json) VALUES (@sessionId, @parentSessionId, @forkPointSeq, @protocol, @label, @title, @hostTitle, @namespace, @origin, @status, @createdAt, @updatedAt, @nextBlockId, @nextRunId, - @metaJson, @statsJson, @metadataJson, @kstateJson, @tailJson, @ccrJson, @lastIngestedSeq) + @metaJson, @statsJson, @metadataJson, @kstateJson, @tailJson, @ccrJson, @lastIngestedSeq, @orphansJson) ON CONFLICT(session_id) DO UPDATE SET parent_session_id = excluded.parent_session_id, fork_point_seq = excluded.fork_point_seq, @@ -547,7 +605,8 @@ export class UnifiedStore { kstate_json = excluded.kstate_json, tail_json = excluded.tail_json, ccr_json = excluded.ccr_json, - last_ingested_seq = excluded.last_ingested_seq`, + last_ingested_seq = excluded.last_ingested_seq, + orphans_json = excluded.orphans_json`, ) .run({ sessionId: s.sessionId, @@ -571,6 +630,10 @@ export class UnifiedStore { tailJson: s.tailJson, ccrJson: s.ccrJson, lastIngestedSeq: watermark, + // #4216/#4242: orphan journal rides the normal upsert with + // MERGE semantics (existing entries survive — LazyBlockContents + // means the live map may not carry trimmed entries). + orphansJson: this.mergeOrphansJson(s.sessionId, s.orphansJson ?? null), }); // Identity index (#2960 R2.1): claim (namespace, conversation_id) @@ -620,9 +683,10 @@ export class UnifiedStore { .all(s.sessionId) as Array<{ block_id: string; summary_hash: string; one_hash: string | null }>; for (const stale of staleBlocks) { if (incomingBlockIds.has(stale.block_id)) continue; + // referencing row first, then release (#4242 delete-on-zero) + this.db.prepare("DELETE FROM blocks WHERE session_id = ? AND block_id = ?").run(s.sessionId, stale.block_id); this.releaseContent(stale.summary_hash); if (stale.one_hash !== null) this.releaseContent(stale.one_hash); - this.db.prepare("DELETE FROM blocks WHERE session_id = ? AND block_id = ?").run(s.sessionId, stale.block_id); } for (const b of input.blocks) { const summaryHash = this.putContent(b.summary); @@ -636,12 +700,49 @@ export class UnifiedStore { .prepare("SELECT summary_hash, one_hash FROM blocks WHERE session_id = ? AND block_id = ?") .get(s.sessionId, b.blockId) as { summary_hash: string; one_hash: string | null } | undefined; if (existing) { + // #4242: redirect the row BEFORE the undo/release so a + // released-to-zero old hash is no longer FK-referenced by + // this blocks row when the content DELETE fires. The + // identical-re-ingest undo (same hash) is a pure -1 + // against the +1 bump and cannot reach zero while the row + // itself holds the reference. + this.db + .prepare( + `INSERT INTO blocks (block_id, session_id, ord, start_seq, end_seq, start_ref, end_ref, tier, + summary_hash, one_hash, supersedes_block_id, active, created_at, covered_hash, payload_json) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(session_id, block_id) DO UPDATE SET + ord = excluded.ord, start_seq = excluded.start_seq, end_seq = excluded.end_seq, + start_ref = excluded.start_ref, end_ref = excluded.end_ref, tier = excluded.tier, + summary_hash = excluded.summary_hash, one_hash = excluded.one_hash, + supersedes_block_id = excluded.supersedes_block_id, active = excluded.active, + created_at = excluded.created_at, covered_hash = excluded.covered_hash, + payload_json = excluded.payload_json`, + ) + .run( + b.blockId, + s.sessionId, + b.ord, + b.startSeq, + b.endSeq, + b.startRef, + b.endRef, + b.tier, + summaryHash, + oneHash, + b.supersedesBlockId, + b.active ? 1 : 0, + b.createdAt, + b.coveredHash, + b.payloadJson, + ); if (existing.summary_hash === summaryHash) this.releaseContent(summaryHash); else this.releaseContent(existing.summary_hash); if (existing.one_hash !== null) { if (existing.one_hash === oneHash && oneHash !== null) this.releaseContent(oneHash); else this.releaseContent(existing.one_hash); } + continue; } this.db .prepare( @@ -710,8 +811,8 @@ export class UnifiedStore { const oldCcr = this.db .prepare("SELECT content_hash FROM ccr_entries WHERE session_id = ?") .all(s.sessionId) as { content_hash: string }[]; - for (const row of oldCcr) this.releaseContent(row.content_hash); this.db.prepare("DELETE FROM ccr_entries WHERE session_id = ?").run(s.sessionId); + for (const row of oldCcr) this.releaseContent(row.content_hash); for (const e of input.ccrEntries) { const hash = this.putContent(e.content); this.db @@ -925,9 +1026,9 @@ export class UnifiedStore { .prepare("SELECT summary_hash, one_hash FROM blocks WHERE session_id = ? AND block_id = ?") .get(sessionId, blockId) as { summary_hash: string; one_hash: string | null } | undefined; if (!row) continue; + this.db.prepare("DELETE FROM blocks WHERE session_id = ? AND block_id = ?").run(sessionId, blockId); this.releaseContent(row.summary_hash); if (row.one_hash !== null) this.releaseContent(row.one_hash); - this.db.prepare("DELETE FROM blocks WHERE session_id = ? AND block_id = ?").run(sessionId, blockId); } if (orphanContents.size > 0) { const sess = this.db @@ -1799,21 +1900,33 @@ export class UnifiedStore { * serialization. A missing CAS row (corruption / manual surgery) falls * back to the split form rather than throwing — visible, not silent. */ private hydrateMedia(content: string): string { - if (!content.includes("\"$media\"")) return content; - try { - const parsed = JSON.parse(content) as { imageBase64?: unknown }; - const slot = parsed.imageBase64; - if (slot !== null && typeof slot === "object" && typeof (slot as { $media?: unknown }).$media === "string") { - const bytes = this.getContent((slot as { $media: string }).$media); - if (typeof bytes === "string" && bytes.length > 0) { - parsed.imageBase64 = bytes; - return JSON.stringify(parsed); + let out = content; + if (out.includes("\"$media\"")) { + try { + const parsed = JSON.parse(out) as { imageBase64?: unknown }; + const slot = parsed.imageBase64; + if (slot !== null && typeof slot === "object" && typeof (slot as { $media?: unknown }).$media === "string") { + const bytes = this.getContent((slot as { $media: string }).$media); + if (typeof bytes === "string" && bytes.length > 0) { + parsed.imageBase64 = bytes; + out = JSON.stringify(parsed); + } } + } catch { + // fall through: raw-sidecar pass below still runs } - return content; - } catch { - return content; } + // #4069: restore data-URL payloads swapped out of raw wire sidecars. + if (out.includes(";base64,{$media:")) { + out = out.replace( + /data:([a-zA-Z0-9.+\/\-]+);base64,\{\$media:([0-9a-f]{64})\}/g, + (all: string, mime: string, hash: string) => { + const bytes = this.getContent(hash); + return typeof bytes === "string" && bytes.length > 0 ? `data:${mime};base64,${bytes}` : all; + }, + ); + } + return out; } /** #3762 test/ops helper: refcount of the content row holding exactly diff --git a/src/turn-separator.ts b/src/turn-separator.ts new file mode 100644 index 0000000000..2ffce496bb --- /dev/null +++ b/src/turn-separator.ts @@ -0,0 +1,42 @@ +// #2627 turn-separator id helpers live in this ZERO-DEPENDENCY leaf so the +// storage seam can consume them without dragging session.ts (and its built- +// acp-kernel package import) into graphs that must load unbuilt kernel source +// (the mysql-contract CI lane installs with --ignore-scripts: no kernel/dist). +// session.ts re-exports these for its existing consumers. + +/** #2627: bili-synthesized Responses turn separators (repairResponsesAssistantOrdering, + * src/server.ts) are OUTBOUND-ONLY ordering markers — no client's resent history + * ever contains them, so they can never anchor or be covered. When a + * separator-bearing view reached applyCompression (the plugin lane handed its + * processed view straight in), their ids were registered into new blocks' + * direct/effectiveMessageIds and inherited by higher folds via child merge — + * permanently "missing" on every reconcile pass: false [acp-drift]/fold- + * reconcile warns and stalled per-block ledger evidence against an unchanged + * history. They are protocol-ordering artifacts, not compressible raw history: + * exclude them everywhere coverage is stored or compared. */ +export const RESPONSES_TURN_SEPARATOR_ID_PREFIX = "acp_turn_sep_"; + +export function isResponsesTurnSeparatorId(id: string): boolean { + return id.startsWith(RESPONSES_TURN_SEPARATOR_ID_PREFIX); +} + +/** #2627: strip turn-separator ids from block coverage IN PLACE (load-time heal + * of persisted state + post-commit scrub). Returns how many ids were removed; + * 0 means the state was already clean. */ +export function scrubTurnSeparatorIds( + blocks: ReadonlyArray<{ directMessageIds?: string[]; effectiveMessageIds?: string[] }>, +): number { + let removed = 0; + for (const block of blocks) { + for (const key of ["directMessageIds", "effectiveMessageIds"] as const) { + const ids = block[key]; + if (!ids) continue; + const kept = ids.filter((id) => !isResponsesTurnSeparatorId(id)); + if (kept.length !== ids.length) { + removed += ids.length - kept.length; + block[key] = kept; + } + } + } + return removed; +} diff --git a/tests/issue2129-restart-measurement-state.test.ts b/tests/issue2129-restart-measurement-state.test.ts index b147fbedaa..08a4e93a8a 100644 --- a/tests/issue2129-restart-measurement-state.test.ts +++ b/tests/issue2129-restart-measurement-state.test.ts @@ -7,6 +7,9 @@ import { mkdtempSync, readdirSync, readFileSync, writeFileSync } from "node:fs"; import { join } from "node:path"; process.env.NODE_ENV = "test"; +// #4004/#4242: envelope-corruption rejection is the legacy-layer guard; pin +// it so the unified index cannot mask the tampered envelope. +process.env.BILI_STORAGE_UNIFIED = "legacy"; import { createInitialState, defaultConfig } from "acp-kernel"; import { startServer } from "../src/server.ts"; diff --git a/tests/proxy-persist.test.ts b/tests/proxy-persist.test.ts index 19456628a2..e047a152da 100644 --- a/tests/proxy-persist.test.ts +++ b/tests/proxy-persist.test.ts @@ -1,4 +1,8 @@ import { test } from "node:test"; +// #4004/#4242: this file pins the LEGACY envelope layer (guard semantics, +// directory shape). Full mode adds index.db to the dir and serves +// unified-first — both would make these assertions nondeterministic. +process.env.BILI_STORAGE_UNIFIED = "legacy"; import assert from "node:assert/strict"; import { createHash } from "node:crypto"; import { tmpdir } from "node:os"; diff --git a/tests/storage-archive-gc.test.ts b/tests/storage-archive-gc.test.ts index 3ce3715c3b..4180076f30 100644 --- a/tests/storage-archive-gc.test.ts +++ b/tests/storage-archive-gc.test.ts @@ -158,7 +158,13 @@ test("gc(): age sweep archives only past-retention sessions; dormant shared rows assert.ok(store.getSessionRow("sess-old")); assert.equal((store.getSessionRow("sess-old") as unknown as { status?: string }).status, "archived"); assert.equal((store.getSessionRow("sess-new") as unknown as { status?: string }).status, "active"); - assert.equal(result.contentRowsDeleted >= 1, true, "released exclusive content collected"); + // #4242 delete-on-zero: the released row is deleted INLINE by the + // release itself, so gc's sweep counter may legitimately be 0 — the + // observable contract is that the exclusive content is GONE. + const contentLeft = ( + store as unknown as { db: { prepare: (q: string) => { all: () => { bytes?: unknown }[] } } } + ).db.prepare("SELECT bytes FROM content").all().length; + assert.ok(contentLeft === 1, `released exclusive content collected inline (left=${contentLeft})`); store.close(); } finally { try { fs.rmSync(dir, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 }); } catch { /* Windows stray handles */ } diff --git a/tests/storage-full-flip.test.ts b/tests/storage-full-flip.test.ts index b7583b5420..915c826265 100644 --- a/tests/storage-full-flip.test.ts +++ b/tests/storage-full-flip.test.ts @@ -307,7 +307,7 @@ test("#3818 ⑥ writerOff save that disables the sidecar still lands JSON the sa // up on NO copy at all (index skips it, writerOff wrote no file) // --------------------------------------------------------------------------- -test("#3818 ⑨ writerOff + reap-shape lands a JSON envelope anyway (no silent data death)", async () => { +test("#4242 ⑨ writerOff + reap-shape rides the index journal (orphans are representable, #4216)", async () => { _resetStorageRuntimeStatusForTest(); process.env.BILI_STORAGE_UNIFIED = "full"; process.env.BILI_STORAGE_JSON_WRITER = "0"; @@ -319,15 +319,19 @@ test("#3818 ⑨ writerOff + reap-shape lands a JSON envelope anyway (no silent d (session.blockContents as Map).set("b1", { one: null, full: { text: "retained zombie summary", count: 1 } }); store.scheduleSave(session); await settle(300); - assert.ok(countStateFiles(dir) >= 1, "the safety valve lands a JSON envelope even under writerOff"); - const sh = (store as unknown as { shadow?: { stats: { orphanSkips: number } } }).shadow; - assert.ok((sh?.stats.orphanSkips ?? 0) >= 1, "the mirror skipped the unrepresentable record"); + // writerOff stays off — the record's durability comes from the index + // (sessions.orphans_json), no JSON-envelope escape hatch is needed. store.closeQuietly(); const store2 = new SessionStore({ enabled: true, dir, debounceMs: 0 }); const reloaded = store2.loadSync("reap-writereoff-session"); - assert.equal((reloaded?.blockContents.get("b1") as { full: { text: string } } | undefined)?.full?.text, "retained zombie summary", - "the retained content survives the restart via the envelope"); + assert.ok(reloaded, "record reloads under full mode (index-served)"); + // The journal must carry the CONTENT, not just the key — pin the exact + // bytes so a placeholder/empty entry can't pass as "survived". + const reloadedB1 = (reloaded.blockContents as Map).get("b1") as + | { full?: { text?: string } } + | undefined; + assert.equal(reloadedB1?.full?.text, "retained zombie summary", "orphan content survives via the journal"); store2.closeQuietly(); rmrf(dir); } finally { @@ -340,38 +344,75 @@ test("#3818 ⑨ writerOff + reap-shape lands a JSON envelope anyway (no silent d // 8. #2695 reap-shape records stay envelope-served, never gutted by the index // --------------------------------------------------------------------------- -test("#3818 review: reap-shape records (content kept, block gone) stay envelope-served across restart", async () => { +test("#4242 review: reap-shape records ride the ingest journal and reload fresh (#4216 freeze fix)", async () => { _resetStorageRuntimeStatusForTest(); process.env.BILI_STORAGE_UNIFIED = "full"; try { - const dir = tmpDir("reap"); + const dir = tmpDir("reap-representable"); const store = new SessionStore({ enabled: true, dir, debounceMs: 0 }); - const session = makeSession("reap-session"); - // reapDestroyedSubstrate (#2695) shape: state.blocks loses the entry, - // blockContents keeps the text for the derived-decompress fallback. - (session.blockContents as Map).set("b1", { one: null, full: { text: "retained zombie summary", count: 1 } }); - store.scheduleSave(session); - await settle(300); - const sh = (store as unknown as { shadow?: { stats: { orphanSkips: number } } }).shadow; - assert.ok(sh, "full mode opens a sidecar"); - assert.ok((sh.stats.orphanSkips ?? 0) >= 1, "an unrepresentable record is skipped from the mirror, not gutted into it"); - const reloaded = store.loadSync("reap-session"); - assert.ok(reloaded, "session loads"); - assert.equal((reloaded.blockContents.get("b1") as { full: { text: string } }).full.text, "retained zombie summary", "same-process read keeps the retained content"); + const session = makeSession("reap-representable-session"); + const base = (session.stats as { requests: number }).requests; + (session.blockContents as Map).set("b9", { one: null, full: { text: "reaped original", count: 2 } }); + await store.writeNow(session); + // A LATER save (requests advanced) with the same orphan still carried. + session.stats.requests = base + 5; + await store.writeNow(session); store.closeQuietly(); - // Restart: nothing was ever ingested, so the cold read import-skips - // again and serves the envelope — the index never holds this shape. const store2 = new SessionStore({ enabled: true, dir, debounceMs: 0 }); - const reloaded2 = store2.loadSync("reap-session"); - assert.ok(reloaded2, "session loads after restart"); - assert.equal((reloaded2.blockContents.get("b1") as { full: { text: string } }).full.text, "retained zombie summary", "cross-restart read keeps the retained content"); - const sh2 = (store2 as unknown as { shadow?: { stats: { imports: number; orphanSkips: number } } }).shadow; - assert.equal(sh2?.stats.imports, 0, "the unrepresentable record never enters the index"); - assert.ok((sh2?.stats.orphanSkips ?? 0) >= 1); + const reloaded = store2.loadSync("reap-representable-session"); + assert.ok(reloaded, "reload works"); + assert.equal((reloaded.stats as { requests?: number }).requests, base + 5, "post-reap saves keep advancing (#4216)"); + assert.ok((reloaded.blockContents as Map).has("b9"), "orphan content survives"); store2.closeQuietly(); rmrf(dir); } finally { delete process.env.BILI_STORAGE_UNIFIED; } }); + +// --------------------------------------------------------------------------- +// #4004 (via #4242): full-mode authority over a tampered envelope. +// The unified index is the load path; a corrupted JSON envelope alone must +// NOT fail the load while the index holds the record. Corrupt BOTH carriers +// and the guard finally rejects. +// --------------------------------------------------------------------------- +test("#4242 ⑦ full mode serves the index when only the envelope id was tampered (#4004)", async () => { + const dir = tmpDir("authority"); + process.env.BILI_STORAGE_UNIFIED = "full"; + try { + const { SessionStore } = await import("../src/persist.ts"); + // #4004/#4242: _setStoreForTest lives on persist.ts; keep the peek-only + // discipline (loadSync must find the store the test just built). + const setStore = _setStoreForTest as unknown as ((s: unknown) => void) | undefined; + const store = new SessionStore({ dir, enabled: true, debounceMs: 0 }); + if (setStore) setStore(store); + const session = makeSession("authority-session"); + await store.writeNow(session); + const found = findStateFile(dir, "authority-session"); + assert.ok(found, "envelope written"); + + // Tamper ONLY the envelope body id — the index still holds the record. + const tampered = JSON.parse(JSON.stringify(found.payload)) as Record; + tampered.id = "not-the-real-id"; + fs.writeFileSync(found.file, JSON.stringify({ version: 1, payload: tampered })); + + const served = store.loadSync("authority-session", { protocol: "openai", upstreamOrigin: "https://api.openai.com" }); + assert.ok(served, "full mode serves the intact index record despite the tampered envelope"); + + // Corrupt BOTH carriers: drop the index row, keep the tampered file. + store.closeQuietly(); + const { UnifiedStore } = await import("../src/storage/store.ts"); + const uni = UnifiedStore.open(path.join(dir, "index.db")); + uni.deleteSession("authority-session"); + uni.close(); + const store2 = new SessionStore({ dir, enabled: true, debounceMs: 0 }); + if (setStore) setStore(store2); + const rejected = store2.loadSync("authority-session", { protocol: "openai", upstreamOrigin: "https://api.openai.com" }); + assert.equal(rejected, null, "with both carriers gone/tampered the load rejects"); + store2.closeQuietly(); + } finally { + process.env.BILI_STORAGE_UNIFIED = "full"; + rmrf(dir); + } +}); diff --git a/tests/storage-media-sidecar-split.test.ts b/tests/storage-media-sidecar-split.test.ts new file mode 100644 index 0000000000..83dfae911b --- /dev/null +++ b/tests/storage-media-sidecar-split.test.ts @@ -0,0 +1,127 @@ +/** + * #4069 / #4242: raw wire sidecars (rawOpenaiContent & friends) used to carry + * the original image payload a SECOND time as a data URL, capping media CAS + * dedup at ~1.89×. The split now swaps every payload ≥2 KiB for its CAS hash + * and hydration restores it byte-identically. + */ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { createHash } from "node:crypto"; +import { createInitialState } from "acp-kernel"; +import { planIngest, type LegacySessionLike } from "../src/storage/ingest.ts"; +import { UnifiedStore } from "../src/storage/store.ts"; + +process.env.BILI_PERSIST_ZSTD = "0"; +process.env.BILI_STORAGE_UNIFIED = "legacy"; +process.env.BILI_CONFIG_FILE = join(mkdtempSync(join(tmpdir(), "msplit-")), "missing.json"); + +const BIG = "QUJD".repeat(700); // 2800 base64 chars ≥ 2048 — gets split +const BIG_HASH = createHash("sha256").update(BIG).digest("hex"); +const URL = `data:image/png;base64,${BIG}`; + +function sha256(s: string): string { + return createHash("sha256").update(s).digest("hex"); +} + +function sessionOf(id: string, messages: unknown[]): LegacySessionLike { + const state = { ...createInitialState(), requests: messages.length }; + return { + id, + createdAt: Date.now(), + lastSeen: Date.now(), + meta: { label: id, protocol: "openai", upstreamOrigin: "https://api.openai.com" }, + stats: { requests: messages.length }, + metadata: { pluginAgent: "dsh" }, + state: state as never, + blockContents: new Map(), + lastMessages: messages as never, + lastMessagesFolded: false, + }; +} + +test("#4069: raw-sidecar data URLs ride the media CAS and hydrate byte-identically", () => { + const original = JSON.stringify({ + id: "m00001", + role: "user", + contentType: "text", + text: "look at this", + wireIndex: 0, + rawOpenaiContent: { type: "image_url", image_url: { url: URL } }, + }); + const dir = mkdtempSync(join(tmpdir(), "msp-")); + try { + const store = UnifiedStore.open(join(dir, "index.db")); + const plan = planIngest(sessionOf("media-split-1", [JSON.parse(original)])); + + // 1. the plan's stored content no longer embeds the payload + const entry = (plan.messages as unknown as { content: string }[])[0]; + assert.ok(!entry.content.includes(BIG), "payload is not stored inline"); + assert.ok(entry.content.includes(`data:image/png;base64,{$media:${BIG_HASH}}`), "marker carries the CAS hash"); + assert.deepEqual( + (plan.messages as unknown as { media?: { hash: string }[] }[])[0].media?.map((m) => m.hash), + [BIG_HASH], + "media rides the plan", + ); + + // 2. ingest: payload lands in the CAS exactly once + store.ingestSession(plan); + const raw = ( + store as unknown as { db: { prepare: (q: string) => { all: () => { bytes?: string; refcount?: number }[] } } } + ).db.prepare("SELECT bytes, refcount FROM content").all(); + assert.equal(raw.filter((r) => Buffer.from(r.bytes as unknown as Uint8Array).toString() === BIG).length, 1, "payload CAS row exists exactly once"); + + // 3. hydrate: round-trip is byte-identical + const msgs = store.getMessages("media-split-1"); + assert.equal(msgs.length, 1); + assert.equal(msgs[0].content, original, "hydrated content is byte-identical to the original"); + + // 4. dedup: the same payload as a top-level imageBase64 hits the SAME row + const second = JSON.stringify({ + id: "m00002", + role: "user", + contentType: "text", + text: "same image again", + wireIndex: 0, + imageBase64: BIG, + }); + store.ingestSession(planIngest(sessionOf("media-split-2", [JSON.parse(second)]))); + const after = ( + store as unknown as { db: { prepare: (q: string) => { all: () => { bytes?: string; refcount?: number }[] } } } + ).db.prepare("SELECT bytes, refcount FROM content").all(); + const payloadRows = after.filter((r) => Buffer.from(r.bytes as unknown as Uint8Array).toString() === BIG); + assert.equal(payloadRows.length, 1, "shared payload stays one CAS row"); + assert.equal(payloadRows[0].refcount, 2, "refcount covers both messages"); + + store.close(); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("#4069: small payloads (<2 KiB) stay inline — no CAS row, no join", () => { + const small = "QUJD".repeat(100); // 400 chars < 2048 + const original = JSON.stringify({ + id: "m00001", + role: "user", + contentType: "text", + text: "tiny icon", + wireIndex: 0, + rawOpenaiContent: { type: "image_url", image_url: { url: `data:image/png;base64,${small}` } }, + }); + const dir = mkdtempSync(join(tmpdir(), "mss-")); + try { + const store = UnifiedStore.open(join(dir, "index.db")); + const plan = planIngest(sessionOf("media-split-small", [JSON.parse(original)])); + const entry = (plan.messages as unknown as { content: string; media?: unknown[] }[])[0]; + assert.ok(entry.content.includes(small), "small payload stays inline"); + assert.equal(entry.media, undefined, "no media entries for small payloads"); + store.ingestSession(plan); + assert.equal(store.getMessages("media-split-small")[0].content, original); + store.close(); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); diff --git a/tests/storage-orphan-representable.test.ts b/tests/storage-orphan-representable.test.ts new file mode 100644 index 0000000000..4d2cc9a443 --- /dev/null +++ b/tests/storage-orphan-representable.test.ts @@ -0,0 +1,110 @@ +/** + * #4216 / #4242: a session whose blockContents outlives its state.blocks + * (the #2695 reap shape) must keep advancing in the unified store — no + * freeze, no hours-old cold loads. The orphan journal rides the ingest plan. + */ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { createInitialState } from "acp-kernel"; +import { planIngest, loadLegacyView, type LegacySessionLike } from "../src/storage/ingest.ts"; +import { UnifiedStore } from "../src/storage/store.ts"; + +process.env.BILI_PERSIST_ZSTD = "0"; +process.env.BILI_STORAGE_UNIFIED = "legacy"; +process.env.BILI_CONFIG_FILE = join(mkdtempSync(join(tmpdir(), "freeze-")), "missing.json"); + +function mkBlock(blockId: string, refs: string[]) { + return { + blockId, + runId: "r1", + tier: 1, + summary: `summary of ${blockId} — ` + "x".repeat(80), + directMessageIds: refs, + effectiveMessageIds: refs, + directBlockIds: [], + compressedTokens: 5000, + createdAt: Date.now(), + survivedCount: 0, + generation: "young", + active: true, + } as never; +} + +function legacyLike(id: string, requests: number, withOrphan: boolean): LegacySessionLike { + const state = { ...createInitialState(), requests }; + state.nextBlockId = withOrphan ? 4 : 3; + // b1..b2 stay claimed; b3 was REAPED (removed from state.blocks) but its + // original text survives in blockContents — the #2695 shape. + state.blocks = [mkBlock("b1", ["m00001"]), mkBlock("b2", ["m00002"])] as never; + const contents = new Map(); + contents.set("b1", { one: null, full: { text: "b1 summary", count: 3 } }); + contents.set("b2", { one: null, full: { text: "b2 summary", count: 3 } }); + if (withOrphan) contents.set("b3", { one: null, full: { text: "b3 orphaned original summary", count: 3 } }); + return { + id, + createdAt: Date.now(), + lastSeen: Date.now(), + meta: { label: id, protocol: "anthropic", upstreamOrigin: "https://api.anthropic.com" }, + stats: { requests }, + metadata: { pluginAgent: "dsh" }, + state: state as never, + blockContents: contents as never, + lastMessages: [ + { id: "m00001", role: "user", contentType: "text", text: "q1" }, + { id: "m00002", role: "assistant", contentType: "text", text: "a1" }, + ...(withOrphan ? ([{ id: "m00003", role: "user", contentType: "text", text: "q2 after reap" }] as never) : []), + ], + lastMessagesFolded: false, + }; +} + +test("#4216: post-reap saves keep advancing (no freeze) and cold loads are fresh", () => { + const dir = mkdtempSync(join(tmpdir(), "fz-")); + try { + const store = UnifiedStore.open(join(dir, "index.db")); + + // 1. normal two-block session + store.ingestSession(planIngest(legacyLike("sess", 390, false))); + assert.equal((store.getSessionRow("sess") as unknown as { status: string }).status, "active"); + + // 2. save with the reap shape: b3 orphaned, requests 876 + store.ingestSession(planIngest(legacyLike("sess", 876, true))); + + // The deterministic repro of #4216: the OLD code froze at requests=390. + const row = store.getSessionRow("sess")!; + assert.ok((row.statsJson as string).includes("876"), `session row advanced (stats=${(row.statsJson as string).slice(0, 60)})`); + + // 3. cold load: view carries the fresh state, orphan content survives + const view = loadLegacyView({ + session: row, + messages: store.getMessages("sess"), + blocks: store.getBlocks("sess"), + deadRefs: store.getDeadRefs("sess"), + refs: store.getRefs("sess"), + }); + const viewState = view.state as unknown as { blocks: { blockId: string }[] }; + assert.equal(viewState.blocks.length, 2, "reaped block does NOT resurrect"); + assert.ok(view.blockContents.has("b3"), "orphan original survives for decompress"); + assert.equal((view.blockContents.get("b3") as { full: { text: string } }).full.text, "b3 orphaned original summary"); + + // 4. journal merge: a later save (still carrying the orphan) is stable, + // and a save WITHOUT it (LazyBlockContents trimmed) keeps the journal. + store.ingestSession(planIngest(legacyLike("sess", 900, true))); + store.ingestSession(planIngest({ ...legacyLike("sess", 950, false), blockContents: new Map() } as LegacySessionLike)); + const view2 = loadLegacyView({ + session: store.getSessionRow("sess")!, + messages: store.getMessages("sess"), + blocks: store.getBlocks("sess"), + deadRefs: store.getDeadRefs("sess"), + refs: store.getRefs("sess"), + }); + assert.ok(view2.blockContents.has("b3"), "trimmed-away journal entry survives the merge"); + + store.close(); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); diff --git a/tests/storage-state-fields.test.ts b/tests/storage-state-fields.test.ts new file mode 100644 index 0000000000..19f3dd5850 --- /dev/null +++ b/tests/storage-state-fields.test.ts @@ -0,0 +1,70 @@ +/** + * #4192 / #4242: state fields survive the unified round-trip — the field + * list is DERIVED from mergeState, so hydration cannot silently drop fields + * the envelope loader keeps. + */ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { createInitialState } from "acp-kernel"; +import { COMPRESSION_STATE_FIELDS, mergeState } from "../src/state-hydration.ts"; +import { planIngest, loadLegacyView, type LegacySessionLike } from "../src/storage/ingest.ts"; +import { UnifiedStore } from "../src/storage/store.ts"; + +process.env.BILI_PERSIST_ZSTD = "0"; +process.env.BILI_STORAGE_UNIFIED = "legacy"; +process.env.BILI_CONFIG_FILE = join(mkdtempSync(join(tmpdir(), "statefields-")), "missing.json"); + +test("#4192: sweptOrphanCallIds / pendingOrphanSweepMinRef survive the unified round-trip", () => { + const dir = mkdtempSync(join(tmpdir(), "sf-db-")); + try { + const store = UnifiedStore.open(join(dir, "index.db")); + const state: Record = { + ...createInitialState(), + sweptOrphanCallIds: ["m00042", "m00043"], + pendingOrphanSweepMinRef: 42, + }; + const legacy = { + id: "sf-1", + createdAt: Date.now(), + lastSeen: Date.now(), + meta: { label: "sf-1", protocol: "anthropic", upstreamOrigin: "https://api.anthropic.com" }, + stats: { requests: 1 }, + metadata: { pluginAgent: "dsh" }, + state: state as never, + blockContents: new Map(), + lastMessages: [{ id: "m00001", role: "user", contentType: "text", text: "hello" }], + lastMessagesFolded: false, + } satisfies LegacySessionLike; + store.ingestSession(planIngest(legacy)); + + const rows = { + session: store.getSessionRow("sf-1")!, + messages: store.getMessages("sf-1"), + blocks: store.getBlocks("sf-1"), + deadRefs: store.getDeadRefs("sf-1"), + refs: store.getRefs("sf-1"), + }; + const view = loadLegacyView(rows); + // The two fields the hand list dropped (#4192) must round-trip intact. + assert.deepEqual((view.state as Record).sweptOrphanCallIds, ["m00042", "m00043"]); + assert.equal((view.state as Record).pendingOrphanSweepMinRef, 42); + store.close(); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("mechanism: every field of a fresh state is covered by the derived list (no future #4192)", () => { + const freshKeys = Object.keys(createInitialState()); + for (const key of freshKeys) { + assert.ok( + (COMPRESSION_STATE_FIELDS as readonly string[]).includes(key), + `createInitialState emits "${key}" but COMPRESSION_STATE_FIELDS lacks it — add it to mergeState's literal (src/state-hydration.ts)`, + ); + } + // And the list is exactly the merge output — nothing stale either. + assert.deepEqual([...COMPRESSION_STATE_FIELDS], Object.keys(mergeState(createInitialState()))); +}); diff --git a/tests/wire-exit-matrix.test.ts b/tests/wire-exit-matrix.test.ts index 18435cf078..bfb521b1ee 100644 --- a/tests/wire-exit-matrix.test.ts +++ b/tests/wire-exit-matrix.test.ts @@ -212,7 +212,11 @@ test("abort x proxy-openai-sse: client abort mid-stream destroys the upstream re const url = `http://127.0.0.1:${proxyPort}/bili/http://127.0.0.1:${up.port}/v1/chat/completions`; const outcome = await abortAfterFirstByte(url, { "content-type": "application/json", "x-acp-session": "mx-abort-oai" }, JSON.stringify({ model: "m-test", max_tokens: 1024, stream: true, messages: [{ role: "user", content: "hi" }] })); assert.ok(outcome === "first-byte" || outcome === "header"); - await new Promise((r) => setTimeout(r, 300)); + // Poll for the abort to propagate instead of a fixed 300ms sleep: + // slow runners (win/node24) destroy the upstream socket later. + for (let i = 0; i < 30 && up.socketsClosed() < 1; i += 1) { + await new Promise((r) => setTimeout(r, 100)); + } assert.ok(up.socketsClosed() >= 1, `upstream socket(s) must be destroyed after the client abort (closed=${up.socketsClosed()})`); assert.equal(up.requests(), 1, "exactly one upstream request"); } finally { @@ -233,7 +237,11 @@ test("abort x proxy-anthropic-sse: client abort mid-stream destroys the upstream try { const url = `http://127.0.0.1:${proxyPort}/bili/http://127.0.0.1:${up.port}/v1/messages`; await abortAfterFirstByte(url, { "content-type": "application/json", "x-acp-session": "mx-abort-ant" }, JSON.stringify({ model: "m-test", max_tokens: 1024, stream: true, messages: [{ role: "user", content: "hi" }] })); - await new Promise((r) => setTimeout(r, 300)); + // Poll for the abort to propagate instead of a fixed 300ms sleep: + // slow runners (win/node24) destroy the upstream socket later. + for (let i = 0; i < 30 && up.socketsClosed() < 1; i += 1) { + await new Promise((r) => setTimeout(r, 100)); + } assert.ok(up.socketsClosed() >= 1, `upstream socket(s) destroyed after client abort (closed=${up.socketsClosed()})`); } finally { proxy.close(); @@ -253,7 +261,11 @@ test("abort x proxy-responses-sse: client abort mid-stream destroys the upstream try { const url = `http://127.0.0.1:${proxyPort}/bili/http://127.0.0.1:${up.port}/v1/responses`; await abortAfterFirstByte(url, { "content-type": "application/json", "x-acp-session": "mx-abort-resp" }, JSON.stringify({ model: "m-test", max_tokens: 1024, stream: true, instructions: "You are a test agent.", input: [{ type: "message", role: "user", content: "hi" }] })); - await new Promise((r) => setTimeout(r, 300)); + // Poll for the abort to propagate instead of a fixed 300ms sleep: + // slow runners (win/node24) destroy the upstream socket later. + for (let i = 0; i < 30 && up.socketsClosed() < 1; i += 1) { + await new Promise((r) => setTimeout(r, 100)); + } assert.ok(up.socketsClosed() >= 1, `upstream socket(s) destroyed after client abort (closed=${up.socketsClosed()})`); } finally { proxy.close();