diff --git a/src/dsh-channel.ts b/src/dsh-channel.ts index 308c385f54..b04273ab25 100644 --- a/src/dsh-channel.ts +++ b/src/dsh-channel.ts @@ -407,7 +407,10 @@ export function planDshSpawn( "Set BILI_DSH_BIN to the dsh executable, or reword the argument.", ); }); - const comspec = env.COMSPEC?.trim() || "cmd.exe"; + // #4599: sanitized env copies keep original key casing (`ComSpec`) — an + // exact `env.COMSPEC` probe misses it on plain objects. + const comspecRaw = env.COMSPEC ?? env.ComSpec ?? env.comspec; + const comspec = comspecRaw?.trim() || "cmd.exe"; const line = `"${[command, ...args].map(quoteWinToken).join(" ")}"`; return { command: comspec, args: ["/d", "/s", "/c", line], windowsVerbatimArguments: true }; } diff --git a/src/launcher.ts b/src/launcher.ts index 42f66e34fd..d9d865c2ff 100644 --- a/src/launcher.ts +++ b/src/launcher.ts @@ -5418,7 +5418,12 @@ export function planClientSpawn( "Set BILI_CLIENT_BIN to the native executable (e.g. the real codex.exe) or to a script entry run under node, or reword the argument.", ); }); - const comspec = nonEmpty(env.COMSPEC) ? env.COMSPEC : "cmd.exe"; + // #4599: Windows env vars are case-insensitive, but sanitized copies + // (spread objects, e.g. stripInheritedProxy's inlineEnv) keep the original + // key casing — `ComSpec` — where an exact `env.COMSPEC` probe misses it + // and silently degrades to the bare-name fallback. Probe every spelling. + const comspecRaw = env.COMSPEC ?? env.ComSpec ?? env.comspec; + const comspec = nonEmpty(comspecRaw) ? comspecRaw : "cmd.exe"; return { command: comspec, args: ["/d", "/s", "/c", buildWindowsCommandLine(cmd, args)], diff --git a/src/persist.ts b/src/persist.ts index bdf9abc0d1..054472b61c 100644 --- a/src/persist.ts +++ b/src/persist.ts @@ -1610,6 +1610,30 @@ export class SessionStore { if (outcome.superseded.size > 0) { session.restampSuperseded = outcome.superseded; } + // The token snapshot is keyed by ref (#3488 render-refs): renumbered + // refs would otherwise inherit the stale count of whichever message + // held the new number pre-alignment — the wire tag then disagrees + // with the legacy-restart leg beyond citation renumbering (storage + // engine-proofs wire-parity, #4599). Two-pass moves: stage all + // (oldRef → newRef → count) first, then delete, then write — a + // single-pass swap would lose entries whose newRef is another + // move's oldRef. + const snap = (state as { tokenSnapshot?: Record }).tokenSnapshot; + if (snap && outcome.superseded.size > 0) { + const byRawNow = state.messageRefs.byRaw; + const moves: Array<[oldRef: string, newRef: string, count: number]> = []; + for (const [oldRef, rawId] of outcome.superseded) { + const count = snap[oldRef]; + const newRef = byRawNow[rawId]; + if (count === undefined || newRef === undefined || newRef === oldRef) continue; + moves.push([oldRef, newRef, count]); + } + if (moves.length > 0) { + for (const [oldRef] of moves) delete snap[oldRef]; + for (const [, newRef, count] of moves) snap[newRef] = count; + this.log("info", `[persist] remapped ${moves.length} token-snapshot entries onto ledger-aligned refs for ${session.id} (#4599)`); + } + } state.refLedgerAligned = true; this.log("info", `[persist] ledger-aligned refs for ${session.id}: kept=${outcome.kept} restamped=${outcome.restamped} rehomed=${outcome.rehomed} blocked=${outcome.blocked} superseded=${outcome.superseded.size} (#4316 PR3/3)`); this.scheduleSave(session); @@ -2560,6 +2584,63 @@ export class SessionStore { this.dropResidentMetadata(session); } + /** #4739: where this id's record lives RIGHT NOW — the one build-scoped + * source decision shared by both lazy-field restores after a save. + * Mirrors loadSync's routing exactly: no readable unified index (never + * opened, 3-strike disabled, shadow mode serving files) or a drift latch + * (the envelope outranks the index while it exists, #3135 ④) → file; + * else index. Bound at save time, deliberately NOT re-evaluated at + * restore time — a later racing writer converges on the next save, and + * re-routing mid-burst could flip the source between the two fields of + * one saved record. */ + private lazySource(id: string): "index" | "file" { + const sh = this.shadow; + if (!sh || !sh.readsUnified || sh.stats.disabled) return "file"; + if (sh.driftedSessions.has(id)) return "file"; + return "index"; + } + + /** #4739: narrow metadata restore for the dropped-metadata proxy — + * single-column index read or single-file-field envelope read instead + * of the full-session loadSync rebuild. Tries the bound source first, + * then the other copy; null only when BOTH miss/fail (keeps the proxy's + * refuse-empty throw semantics). Never imports, never schedules a save, + * never touches the drift latch. */ + private narrowMetadata(id: string, source: "index" | "file", meta?: { protocol?: string; upstreamOrigin?: string }): Record | null { + const fromIndex = (): Record | null => this.shadow?.getSessionMetadata(id) ?? null; + const fromFile = (): Record | null => { + const env = this.loadEnvelope(id, meta); + return env ? (env.payload.metadata ?? {}) : null; + }; + if (source === "index") { + const idx = fromIndex(); + return idx !== null ? idx : fromFile(); + } + const file = fromFile(); + return file !== null ? file : fromIndex(); + } + + /** #4739: narrow blockContents restore for the dropped-blockContents + * proxy — block rows + orphans journal (index) or the persisted + * record's map (file), never message rows. Same bound-source-first / + * other-copy-fallback / refuse-empty contract as narrowMetadata. */ + private narrowBlockContents(id: string, source: "index" | "file", meta?: { protocol?: string; upstreamOrigin?: string }): Record | null { + const fromIndex = (): Record | null => { + const rows = this.shadow?.getBlockContents(id); + return rows ? Object.fromEntries(rows) : null; + }; + const fromFile = (): Record | null => { + const env = this.loadEnvelope(id, meta); + return env ? (env.payload.blockContents ?? {}) : null; + }; + if (source === "index") { + const idx = fromIndex(); + return idx !== null ? idx : fromFile(); + } + const file = fromFile(); + return file !== null ? file : fromIndex(); + } + /** #3050 R3 ②a: once a save has captured the raw tail (envelope + the * unified messages table via the ⑤ rawTail override), an idle session * keeps only the bounded folded view resident — the exact shape a @@ -2587,9 +2668,10 @@ export class SessionStore { // Hand-built sessions (tests, seeds) may carry no meta — the lazy // loader degrades to the id-only lookup, same as a protocol-less file. const meta = session.meta ?? undefined; + // #4739: one narrow read per field instead of a full-session rebuild. + const source = this.lazySource(session.id); session.metadata = lazyMetadata(session.id, { protocol: meta?.protocol, upstreamOrigin: meta?.upstreamOrigin }, (id, m) => { - const loaded = this.loadSync(id, m); - return loaded ? loaded.metadata : null; + return this.narrowMetadata(id, source, m); }); } @@ -2606,9 +2688,10 @@ export class SessionStore { return; } if (existing.size === 0) return; + // #4739: one narrow read per field instead of a full-session rebuild. + const source = this.lazySource(session.id); const lazy = new LazyBlockContents(session.id, { protocol: session.meta.protocol, upstreamOrigin: session.meta.upstreamOrigin }, (id, meta) => { - const loaded = this.loadSync(id, meta); - return loaded ? Object.fromEntries(loaded.blockContents) : null; + return this.narrowBlockContents(id, source, meta); }); lazy.drop(); session.blockContents = lazy; diff --git a/src/storage/backend.ts b/src/storage/backend.ts index dd8473893a..99d07a96de 100644 --- a/src/storage/backend.ts +++ b/src/storage/backend.ts @@ -33,6 +33,7 @@ export type { SessionSummaryRow, WebSummaryRow, IngestSessionInput, + SessionViewRows, } from "./store.js"; import type { SessionRow, @@ -44,6 +45,7 @@ import type { SessionSummaryRow, WebSummaryRow, IngestSessionInput, + SessionViewRows, } from "./store.js"; /** Everything a storage backend must provide. Mirrors the operation surface @@ -74,6 +76,9 @@ export interface StorageBackend { /** #4316 PR3/3: fork adoption row clone (idempotent). */ cloneSessionRows(parentId: string, childId: string, throughSeq: number): Promise; getMessages(sessionId: string): Promise<(MessageRow & { contentHash: string })[]>; + /** #4739: the viewFor bundle from ONE consistent read snapshot (messages + * fetched only at the tail/pluginSnapshot pointer seqs). */ + sessionViewRows(sessionId: string): Promise; getBlocks(sessionId: string): Promise<(BlockRow & { summaryHash: string; oneHash: string | null })[]>; getDeadRefs(sessionId: string): Promise<{ ord: number; ref: string; refNum: number; recordedAt: number }[]>; getRefs(sessionId: string): Promise; @@ -189,6 +194,9 @@ export class SqliteBackend implements StorageBackend { getMessages(sessionId: string): Promise<(MessageRow & { contentHash: string })[]> { return Promise.resolve(this.store.getMessages(sessionId)); } + sessionViewRows(sessionId: string): Promise { + return Promise.resolve(this.store.sessionViewRows(sessionId)); + } getBlocks(sessionId: string): Promise<(BlockRow & { summaryHash: string; oneHash: string | null })[]> { return Promise.resolve(this.store.getBlocks(sessionId)); } diff --git a/src/storage/mysql.ts b/src/storage/mysql.ts index f3c46ea373..fdd28f636b 100644 --- a/src/storage/mysql.ts +++ b/src/storage/mysql.ts @@ -31,7 +31,7 @@ import type { Pool, PoolConnection, PoolOptions } from "mysql2/promise"; import { mkdirSync, writeFileSync, appendFileSync, readFileSync, existsSync } from "node:fs"; import { join } from "node:path"; import { log as loggerLog } from "../logger.js"; -import { sha256Hex } from "./store.js"; +import { sha256Hex, viewPointerSeqs } from "./store.js"; import { identityOfStoredContent } from "./identity.js"; import type { MessageContentStore } from "acp-kernel"; import type { StorageBackend } from "./backend.js"; @@ -45,6 +45,7 @@ import type { SessionSummaryRow, WebSummaryRow, IngestSessionInput, + SessionViewRows, } from "./store.js"; /** The mysql2/promise driver module, injected by openBackend() at open time. @@ -1007,6 +1008,50 @@ export class MysqlBackend implements StorageBackend { }; } + /** #4739: bounded view bundle (messages only at tail/pluginSnapshot + * pointer seqs). Per-statement reads like today's viewForAsync — no + * cross-query snapshot — which is safe here because ingest writes each + * session row and its pointed-at message rows in ONE transaction, so + * every pointer ever read resolves to rows that already exist. The + * sqlite lane carries the explicit read-transaction instead. */ + async sessionViewRows(sessionId: string): Promise { + const session = await this.getSessionRow(sessionId); + if (!session) return null; + const pointers = viewPointerSeqs(session.tailJson); + const messages: (MessageRow & { contentHash: string })[] = []; + const CHUNK = 500; + for (let i = 0; i < pointers.length; i += CHUNK) { + const chunk = [...new Set(pointers.slice(i, i + CHUNK))]; + if (chunk.length === 0) continue; + const [rows] = (await this.pool.query( + `SELECT seq, msg_id, ref, ref_num, role, kind, content_hash FROM messages + WHERE session_id = ? AND retired_at IS NULL AND seq IN (${chunk.map(() => "?").join(", ")})`, + [sessionId, ...chunk], + )) as unknown as Row[][]; + for (const r of rows) { + const contentHash = str(r.content_hash); + messages.push({ + seq: num(r.seq), + msgId: str(r.msg_id), + ref: nullableStr(r.ref), + refNum: nullableNum(r.ref_num), + role: str(r.role), + kind: str(r.kind), + contentHash, + content: await this.hydrateMediaAsync((await this.getContent(this.pool, contentHash)) ?? ""), + }); + } + } + messages.sort((a, b) => a.seq - b.seq); + return { + session, + messages, + blocks: await this.getBlocks(sessionId), + deadRefs: await this.getDeadRefs(sessionId), + refs: await this.getRefs(sessionId), + }; + } + async getMessages(sessionId: string, opts?: { includeRetired?: boolean }): Promise<(MessageRow & { contentHash: string; retiredAt: number | null })[]> { // #4390: retired rows stay out of the replay by default. const [rows] = (await this.pool.query( diff --git a/src/storage/shadow.ts b/src/storage/shadow.ts index d527be96c4..b97eb93e27 100644 --- a/src/storage/shadow.ts +++ b/src/storage/shadow.ts @@ -509,15 +509,11 @@ export class UnifiedShadow { if (this.sync === null) return null; if (this.stats.disabled) return null; try { - const sessionRow = this.sync.getSessionRow(id); - if (!sessionRow) return null; - const rows = { - session: sessionRow, - messages: this.sync.getMessages(id), - blocks: this.sync.getBlocks(id), - deadRefs: this.sync.getDeadRefs(id), - refs: this.sync.getRefs(id), - }; + // #4739: one consistent read snapshot; messages fetched ONLY at + // the tail/pluginSnapshot pointer seqs (the historical log is not + // decoded on ordinary loads). + const rows = this.sync.sessionViewRows(id); + if (!rows) return null; // The view's `state` object carries every MERGE_STATE_FIELDS key by // construction; the declared Record just needs a // widening cast to reach LegacySessionLike. @@ -778,6 +774,33 @@ export class UnifiedShadow { return this.sync.getCcr(sessionId); } + /** #4739: narrow single-column metadata read for lazy-field restore. + * Full-trust only (shadow mode's index may lag its envelopes) — callers + * gate on readsUnified; network backends degrade to null (the caller + * falls back to the envelope, exactly like today's sync loadSync). */ + getSessionMetadata(sessionId: string): Record | null { + if (this.sync === null) return null; + if (this.stats.disabled) return null; + try { + return this.sync.getSessionMetadata(sessionId); + } catch { + return null; + } + } + + /** #4739: narrow blockContents read for lazy-field restore (block rows + + * orphans journal, no message rows). Same trust/degrade contract as + * getSessionMetadata. */ + getBlockContents(sessionId: string): import("./store.ts").BlockContentsMap | null { + if (this.sync === null) return null; + if (this.stats.disabled) return null; + try { + return this.sync.getBlockContents(sessionId); + } catch { + return null; + } + } + /** #4316 PR3/3: raw id → canonical seq (single-row ids only) for the * one-time ledger re-stamp. Shadow mode's index may lag files it has * not re-recorded, so like getCcr this is full-trust only: sync store @@ -971,15 +994,10 @@ export class UnifiedShadow { if (this.stats.disabled) return null; try { const backend = this.backend!; - const sessionRow = await backend.getSessionRow(id); - if (!sessionRow) return null; - const loaded = loadLegacyView({ - session: sessionRow, - messages: await backend.getMessages(id), - blocks: await backend.getBlocks(id), - deadRefs: await backend.getDeadRefs(id), - refs: await backend.getRefs(id), - }) as LegacySessionLike; + // #4739: bounded view bundle (pointer-scoped message fetch). + const rows = await backend.sessionViewRows(id); + if (!rows) return null; + const loaded = loadLegacyView(rows) as LegacySessionLike; warnTailGaps(id, loaded as unknown as { tailGaps?: number }); return loaded; } catch (err) { diff --git a/src/storage/store.ts b/src/storage/store.ts index a13acf4edb..ed58c57a34 100644 --- a/src/storage/store.ts +++ b/src/storage/store.ts @@ -26,6 +26,7 @@ import { identityOfStoredContent } from "./identity.js"; import { recordIntegrityOutcome, recordSecondProcessOwner, recordBackupOutcome } from "./runtime-status.js"; import { listInstances, type RegistryEntry } from "../instance.js"; import type { MessageContentStore } from "acp-kernel"; +import type { BlockView } from "../session.js"; /** #3250: run a PASSIVE wal_checkpoint every N successful ingests. The * built-in autocheckpoint alone cannot guarantee reclamation in a long-lived @@ -36,6 +37,24 @@ import type { MessageContentStore } from "acp-kernel"; * forgotten — the next tick retries and journal_size_limit does the rest. */ const WAL_CHECKPOINT_EVERY_INGESTS = 20; +/** #4739 test instrumentation: message rows materialized by getMessages / + * getMessagesAt (each row = one CAS read + media hydration). Production + * code never reads this; tests assert ordinary loads stop decoding the + * whole historical log. */ +export const messageRowReads = { count: 0 }; + +/** #4739: the message seqs a view actually resolves — the union of the + * tail/pluginSnapshot pointers in a tail_json document. loadLegacyView + * consumes ONLY these rows; the rest of the log is decode waste on an + * ordinary load. Non-number entries are dropped (they can only ever miss + * in pointer resolution, i.e. count as gaps); malformed JSON throws — + * the view path's existing corrupt-fallback contract. */ +export function viewPointerSeqs(tailJson: string): number[] { + const tail = JSON.parse(tailJson) as { tailSeqs?: unknown; pluginSnapshotSeqs?: unknown }; + const pick = (v: unknown): number[] => (Array.isArray(v) ? v.filter((s): s is number => typeof s === "number") : []); + return [...pick(tail.tailSeqs), ...pick(tail.pluginSnapshotSeqs)]; +} + export function sha256Hex(bytes: string | Buffer): string { return createHash("sha256").update(bytes).digest("hex"); } @@ -67,7 +86,8 @@ export interface SessionRow { * state object in mergeState field order. */ kstateJson: string; /** View pointers: {tailSeqs: number[]|null, tailFolded: boolean, - * pluginSnapshotLen: number|null}. Content itself lives in messages. */ + * pluginSnapshotSeqs?: number[]|null} (loadLegacyView's contract). + * Content itself lives in messages. */ tailJson: string; /** Verbatim CCR store JSON (dying subsystem, Phase 3 removes it). */ ccrJson: string | null; @@ -2319,17 +2339,125 @@ export class UnifiedStore { : "SELECT seq, msg_id, ref, ref_num, role, kind, content_hash, retired_at FROM messages WHERE session_id = ? AND retired_at IS NULL ORDER BY seq", ) .all(sessionId) as Record[]; - return rows.map((r) => ({ - seq: r.seq as number, - msgId: r.msg_id as string, - ref: (r.ref as string | null) ?? null, - refNum: (r.ref_num as number | null) ?? null, - role: r.role as string, - kind: r.kind as string, - contentHash: r.content_hash as string, - retiredAt: (r.retired_at as number | null) ?? null, - content: this.hydrateMedia(this.getContent(r.content_hash as string) ?? ""), - })); + return rows.map((r) => { + messageRowReads.count++; + return { + seq: r.seq as number, + msgId: r.msg_id as string, + ref: (r.ref as string | null) ?? null, + refNum: (r.ref_num as number | null) ?? null, + role: r.role as string, + kind: r.kind as string, + contentHash: r.content_hash as string, + retiredAt: (r.retired_at as number | null) ?? null, + content: this.hydrateMedia(this.getContent(r.content_hash as string) ?? ""), + }; + }); + } + + /** #4739: live message rows for EXACTLY the given seqs — the bounded + * fetch for ordinary loads (the view resolves only tail/pluginSnapshot + * pointers). Same row shape and media hydration as getMessages; retired + * rows stay excluded and missing seqs simply don't come back (the view + * counts them as gaps). Chunked IN keeps the bound independent of + * pointer count. */ + getMessagesAt(sessionId: string, seqs: readonly number[]): (MessageRow & { contentHash: string })[] { + if (seqs.length === 0) return []; + const unique = [...new Set(seqs)]; + const CHUNK = 500; + const out: (MessageRow & { contentHash: string })[] = []; + for (let i = 0; i < unique.length; i += CHUNK) { + const chunk = unique.slice(i, i + CHUNK); + const rows = this.db + .prepare( + `SELECT seq, msg_id, ref, ref_num, role, kind, content_hash FROM messages + WHERE session_id = ? AND retired_at IS NULL AND seq IN (${chunk.map(() => "?").join(", ")})`, + ) + .all(sessionId, ...chunk) as Record[]; + for (const r of rows) { + messageRowReads.count++; + out.push({ + seq: r.seq as number, + msgId: r.msg_id as string, + ref: (r.ref as string | null) ?? null, + refNum: (r.ref_num as number | null) ?? null, + role: r.role as string, + kind: r.kind as string, + contentHash: r.content_hash as string, + content: this.hydrateMedia(this.getContent(r.content_hash as string) ?? ""), + }); + } + } + out.sort((a, b) => a.seq - b.seq); + return out; + } + + /** #4739: exactly what viewFor needs, from ONE consistent read snapshot. + * The deferred transaction pins the WAL snapshot across all five reads, + * so a concurrent ingest can no longer land between the session row and + * its pointer rows. Messages are fetched ONLY at the tail/pluginSnapshot + * pointer seqs — the historical log is not decoded on ordinary loads. + * Null on session miss; malformed tail_json throws into the caller's + * corrupt-fallback (same as before). */ + sessionViewRows(sessionId: string): { + session: SessionRow; + messages: (MessageRow & { contentHash: string })[]; + blocks: ReturnType; + deadRefs: ReturnType; + refs: RefRow[]; + } | null { + return this.db.transaction(() => { + const session = this.getSessionRow(sessionId); + if (!session) return null; + return { + session, + messages: this.getMessagesAt(sessionId, viewPointerSeqs(session.tailJson)), + blocks: this.getBlocks(sessionId), + deadRefs: this.getDeadRefs(sessionId), + refs: this.getRefs(sessionId), + }; + }); + } + + /** #4739: single-column metadata read for lazy-field restore (no block or + * message rows involved). Null when the session has no index row; + * corrupt JSON throws like every other column read — callers fall back + * to the envelope. */ + getSessionMetadata(sessionId: string): Record | null { + const row = this.db + .prepare("SELECT metadata_json FROM sessions WHERE session_id = ?") + .get(sessionId) as { metadata_json: string } | undefined; + if (!row) return null; + return JSON.parse(row.metadata_json) as Record; + } + + /** #4739: narrow blockContents read for lazy-field restore — the exact + * replay leg of loadLegacyView (block rows' summary/one CAS bytes plus + * the orphans_json journal merge, WITHOUT resurrecting state.blocks). + * Null when the session has no index row. */ + getBlockContents(sessionId: string): BlockContentsMap | null { + const row = this.db + .prepare("SELECT orphans_json FROM sessions WHERE session_id = ?") + .get(sessionId) as { orphans_json: string | null } | undefined; + if (!row) return null; + const out: BlockContentsMap = new Map(); + for (const b of this.getBlocks(sessionId)) { + out.set(b.blockId, { + one: b.one === null ? null : (JSON.parse(b.one) as BlockView), + full: JSON.parse(b.summary) as BlockView, + }); + } + if (row.orphans_json !== null && row.orphans_json !== undefined && row.orphans_json !== "") { + try { + const orphans = JSON.parse(row.orphans_json) as Record; + for (const [blockId, view] of Object.entries(orphans)) { + if (!out.has(blockId)) out.set(blockId, view); + } + } catch { + // corrupt journal → skip (loadLegacyView's same boundary) + } + } + return out; } /** #3762: swap {"$media": hash} references back to the CAS bytes. Only @@ -2455,3 +2583,10 @@ export class UnifiedStore { })); } } + +/** #4739: the viewFor bundle shape (see UnifiedStore.sessionViewRows). */ +export type SessionViewRows = ReturnType; + +/** #4739: blockContents map shape shared by UnifiedStore.getBlockContents + * and the shadow facade (Map is invariant — one alias keeps both sides exact). */ +export type BlockContentsMap = Map; diff --git a/src/web/blogfeed.ts b/src/web/blogfeed.ts index 6efe1560a7..bdf692f3dd 100644 --- a/src/web/blogfeed.ts +++ b/src/web/blogfeed.ts @@ -78,14 +78,17 @@ export function _setBlogRootForTest(root: string | null): void { function parseFrontmatter(text: string): { fm: Record; body: string } { const fm: Record = {}; let body = text; - if (text.startsWith("---\n")) { - const end = text.indexOf("\n---\n", 4); + // Git-for-Windows autocrlf checks .md posts out with CRLF; normalize so + // the exact-match fences below survive Windows checkouts (#4599). + const src = text.startsWith("---\r\n") ? text.replace(/\r\n/g, "\n") : text; + if (src.startsWith("---\n")) { + const end = src.indexOf("\n---\n", 4); if (end !== -1) { - for (const line of text.slice(4, end).split("\n")) { + for (const line of src.slice(4, end).split("\n")) { const j = line.indexOf(":"); if (j > 0) fm[line.slice(0, j).trim()] = line.slice(j + 1).trim().replace(/^["']|["']$/g, ""); } - body = text.slice(end + 5); + body = src.slice(end + 5); } } return { fm, body }; diff --git a/tests/corrupt-index-child.ts b/tests/corrupt-index-child.ts index 2d59e4a869..76eb4d5335 100644 --- a/tests/corrupt-index-child.ts +++ b/tests/corrupt-index-child.ts @@ -43,7 +43,7 @@ import type { ProxyOptions } from "../src/config.ts"; import { startServer } from "../src/server.ts"; import { SessionStore, _setStoreForTest } from "../src/persist.ts"; import { loadLegacyView, toLegacyLike } from "../src/storage/ingest.ts"; -import { canonicalize, sha256 } from "./golden-canonical.ts"; +import { canonicalize, sha256, sortObjectKeys } from "./golden-canonical.ts"; // --------------------------------------------------------------------------- // Shared with the parent test files (moved verbatim, single definition). @@ -81,6 +81,73 @@ export function viewStateDigest(view: ReturnType): string }))); } +/** #4316 layer 3 PR3/3 (380ef4dac): a unified full-mode cold load performs a + * ONE-TIME ledger alignment — citation numbers are re-homed onto ledger + * sequence (kernel restampRefsToLedger) and state.refLedgerAligned is stamped. + * Cross-mode / cross-read-path "identical state" proofs must therefore compare + * the state modulo that one-time renumbering: this strips the alignment stamp + * and re-homes every ref number to a canonical order (k-th smallest raw id -> + * m0000k), so two views differing ONLY in which numbering scheme produced + * their refs hash identically under viewDigest/viewStateDigest. Object key + * order is normalized too: different hydration paths build identical content + * under different insertion orders, and canonicalize preserves them. Pure + * function — returns a new object, input untouched. */ +export function stripOneTimeAlignment(view: T): T { + const v = view as { state?: unknown; blockContents?: unknown }; + if (!v.state || typeof v.state !== "object") return view; + const byRaw = (v.state as { messageRefs?: { byRaw?: Record } }).messageRefs?.byRaw; + const remap = new Map(); + if (byRaw) { + for (const [raw, ref] of Object.entries(byRaw).sort((a, b) => (a[0] < b[0] ? -1 : 1))) { + remap.set(ref, `m${String(remap.size + 1).padStart(5, "0")}`); + } + } + // The JSON pass below flattens Maps — pull blockContents out while it is + // still a Map, renumber its keys afterwards, rebuild it. Everything else + // in these views is JSON-native. + const bcIsMap = v.blockContents instanceof Map; + let bcEntries: Array<[string, string]> | null = null; + let source: unknown = view; + if (bcIsMap) { + bcEntries = [...(v.blockContents as Map).entries()]; + source = { ...(v as Record), blockContents: undefined }; + } + let json = JSON.stringify(canonicalize(source)); + if (remap.size > 0) { + // Single-pass alternation: sequential per-token replacement would chain + // (a target that is also another source gets re-mapped on the next pass). + const re = new RegExp(`\\b(${[...remap.keys()].join("|")})\\b`, "g"); + json = json.replace(re, (tok) => remap.get(tok)!); + } + const out = JSON.parse(json) as (T & { state?: Record; blockContents?: unknown }) & Record; + if (out.state && "refLedgerAligned" in out.state) delete out.state["refLedgerAligned"]; + // cacheLedger line `bd` digests hash the RAW outbound bodies — bodies whose + // only cross-leg delta is this same citation renumbering. The bodies are + // compared directly (per-turn shaCiteNorm), so masking the derived digests + // hides nothing; keeping them would fail every aligned comparison. + const ledger = (out.metadata as { cacheLedger?: { lines?: Array> } } | undefined)?.cacheLedger; + // Segment bookkeeping, not content, in the aligned view (#4599): a restart + // appends a fresh open tail line and re-anchors the closed-segment byte + // boundaries (be/bl/bs), so no-restart and restart legs legitimately differ + // there. Keep the closed segments' content; drop the unfinalized tail and + // the bookkeeping trio everywhere. + if (Array.isArray(ledger?.lines)) { + while (ledger.lines.length > 0 && ledger.lines[ledger.lines.length - 1]?.bl === undefined) ledger.lines.pop(); + for (const line of ledger.lines) { delete line["bd"]; delete line["be"]; delete line["bl"]; delete line["bs"]; delete line["prov"]; delete line["seam"]; } + } + // agg is the same experience telemetry summed: a restart re-partitions the + // provider-cache misses into seam buckets (seamMissed/seamSuspects vs + // providerSideMissed), so legs legitimately disagree. Per-line content + // fields stay; only the derived aggregate goes. + if (ledger) delete (ledger as { agg?: unknown }).agg; + // snapshotShadow is the shadow-copier's OWN health telemetry — it reports + // the ingest pipeline's experience of building this view (errors, watermark + // drops), which is inherently load-path-dependent, not conversation state. + if (out.metadata && "snapshotShadow" in (out.metadata as Record)) delete (out.metadata as Record)["snapshotShadow"]; + if (bcEntries) out.blockContents = new Map(bcEntries.map(([k, val]) => [remap.get(k) ?? k, val])); + return sortObjectKeys(out) as T; +} + export interface StorageWire { requested: string; effective: string; @@ -277,8 +344,12 @@ async function engproofScenario(argv: string[], run: CheckRun): Promise { // Regression (#2781): the failed open must have CLOSED its handle. await runGuard(run, ENGPROOF_CHECK_NAMES.midFlowGuard, dir, "-moved"); + // #4316 PR3/3: the clean full-mode read carried the one-time ledger + // alignment; the legacy fallback over the corrupt index did not. The + // parent passes its digest already modulo that alignment (single + // definition), so strip it here too before comparing. await run.run(ENGPROOF_CHECK_NAMES.stateDigest, () => - assert.equal(viewStateDigest(toLegacyLike(reloaded)), expectedStateDigest, + assert.equal(viewStateDigest(stripOneTimeAlignment(toLegacyLike(reloaded))), expectedStateDigest, "corrupt-index fallback state digest drift vs clean full-mode read")); // Writes must keep working (legacy path) — no throw, file refreshed. diff --git a/tests/golden-canonical.ts b/tests/golden-canonical.ts index 09f491f2a7..6736b33311 100644 --- a/tests/golden-canonical.ts +++ b/tests/golden-canonical.ts @@ -4,6 +4,22 @@ export { canonicalize, sha256 } from "../src/storage/canonical.ts"; import { canonicalize, sha256 } from "../src/storage/canonical.ts"; +/** Deterministic key order for cross-read-path digest comparisons: two views + * built by different hydration paths carry identical content under different + * map insertion orders, and canonicalize preserves insertion order — sorting + * here makes the digests path-independent. Maps/Sets pass through untouched + * (callers that need them iterable keep them intact; canonicalize flattens + * them downstream). */ +export function sortObjectKeys(v: unknown): unknown { + if (Array.isArray(v)) return v.map(sortObjectKeys); + if (v && typeof v === "object" && !(v instanceof Map) && !(v instanceof Set)) { + const out: Record = {}; + for (const k of Object.keys(v as Record).sort()) out[k] = sortObjectKeys((v as Record)[k]); + return out; + } + return v; +} + /** Canonical JSON → sha256, with a final pass of STRING-level environment * normalization (ports, session ids embedded inside text fields). */ function digestOf(value: unknown, extra?: { normalize?: (s: string) => string }): string { diff --git a/tests/golden-storage-drive.ts b/tests/golden-storage-drive.ts index a6b14205f8..4d65412aca 100644 --- a/tests/golden-storage-drive.ts +++ b/tests/golden-storage-drive.ts @@ -16,10 +16,11 @@ import http from "node:http"; import os from "node:os"; import path from "node:path"; import { once } from "node:events"; -import { canonicalize, sha256 } from "./golden-canonical.ts"; +import { canonicalize, sha256, sortObjectKeys } from "./golden-canonical.ts"; import { defaultConfig } from "acp-kernel"; import { startServer } from "../src/server.ts"; import { SessionStore, _setStoreForTest } from "../src/persist.ts"; +import { stripOneTimeAlignment } from "./corrupt-index-child.ts"; import { flushAllSessions, _resetSessionsForTest } from "../src/session.ts"; import type { ProxyOptions } from "../src/config.ts"; @@ -110,7 +111,7 @@ const GROW_TURNS = 20; // hard cap for the grow loop const SUMMARY_MARKER = "[Compressed conversation section]"; const EDIT_SENTINEL = "GOLDEN-EDIT-SENTINEL"; -interface GoldenTurn { i: number; sha: string; bytes: number } +interface GoldenTurn { i: number; sha: string; bytes: number; shaCiteNorm: string } interface GoldenRecord { wire: Wire; turns: GoldenTurn[]; @@ -118,6 +119,7 @@ interface GoldenRecord { editedAt: number; // body index after the mid-history edit summaryFrom: number; // first body index carrying the fold summary stateDigest: string; // sha256 of the canonicalized persisted session + stateDigestAligned: string; // #4316 PR3/3: same, modulo the one-time ledger alignment sectionDigests: Record; // drift localization (same normalization) stateKeyDigests: Record; // per-top-level-state-key digests stateShape: { // human-readable facts baked into the snapshot @@ -169,6 +171,18 @@ function canonicalBody(body: string): string { .replace(/msg-fix-[0-9a-f]+/g, "msg-fix-"); } +// #4316 PR3/3 (380ef4dac): a full-mode cold load re-homes citation numbers +// once (one-time ledger alignment) — that renumbering is the ONLY permitted +// wire delta vs the legacy restart. Wire-parity proofs therefore pin a second +// digest per turn with every citation number masked to : equality says +// "only the citation numbers may differ", everything else stays byte-exact. +// All m\d{5} tokens are masked (not just tag payloads) because summaries cite +// numbers in prose too; user content is identical across legs, so the wider +// mask cannot hide a real divergence. +function citeNormBody(body: string): string { + return body.replace(/\bm\d{5}\b/g, ""); +} + /** Timestamps (ms-epoch magnitude) and UUIDs become placeholders; everything * else — refs, block ids, coverage sets, stats — must be byte-stable. */ // canonicalize/sha256 live in ./golden-canonical.ts (shared with the storage-corpus pins). @@ -585,15 +599,7 @@ export async function driveWire(wire: Wire, sessionId = `golden-${wire}`, opts: // order, so a harmless field-reorder commit (#3093) red all four // goldens. Deep-sort object keys harness-side only; array order is // preserved (foldAnchorOrder & co are ordered contracts). - const sortObjectKeys = (v: unknown): unknown => { - if (Array.isArray(v)) return v.map(sortObjectKeys); - if (v && typeof v === "object") { - const o: Record = {}; - for (const k of Object.keys(v as Record).sort()) o[k] = sortObjectKeys((v as Record)[k]); - return o; - } - return v; - }; + const lastView = JSON.stringify(canonicalize(sortObjectKeys({ state: session.state, blockContents: [...session.blockContents.entries()], @@ -606,6 +612,18 @@ export async function driveWire(wire: Wire, sessionId = `golden-${wire}`, opts: .replace(/127\.0\.0\.1:\d+/g, "127.0.0.1:") .split(JSON.stringify(sessionId).slice(1, -1)).join(""); const stateDigest = sha256(lastView); + // #4316 PR3/3: parallel digest modulo the one-time ledger alignment — + // cross-mode proofs compare states that may differ only in ref numbering. + const stateDigestAligned = sha256( + JSON.stringify(canonicalize(sortObjectKeys(stripOneTimeAlignment({ + state: session.state, + blockContents: [...session.blockContents.entries()], + stats: session.stats, + metadata: session.metadata, + })))) + .replace(/127\.0\.0\.1:\d+/g, "127.0.0\.1:") + .split(JSON.stringify(sessionId).slice(1, -1)).join(""), + ); // Localization aid for cross-machine digest drift: per-section and // per-top-level-state-key digests under the SAME normalization, so a // CI failure message pinpoints the divergent subtree without needing @@ -629,11 +647,15 @@ export async function driveWire(wire: Wire, sessionId = `golden-${wire}`, opts: const active = st.blocks?.filter((b) => b.active) ?? []; return { wire, - turns: bodies.map((b, i) => ({ i, sha: sha256(canonicalBody(b)), bytes: Buffer.byteLength(b, "utf8") })), + turns: bodies.map((b, i) => { + const cb = canonicalBody(b); + return { i, sha: sha256(cb), bytes: Buffer.byteLength(b, "utf8"), shaCiteNorm: sha256(canonicalBody(citeNormBody(b))) }; + }), foldAt, editedAt, summaryFrom: bodies.findIndex((b) => b.includes(SUMMARY_MARKER)), stateDigest, + stateDigestAligned, sectionDigests, stateKeyDigests, stateShape: { diff --git a/tests/golden/storage-render/anthropic.json b/tests/golden/storage-render/anthropic.json index 5afa14b2c4..43e74c9457 100644 --- a/tests/golden/storage-render/anthropic.json +++ b/tests/golden/storage-render/anthropic.json @@ -4,88 +4,104 @@ { "i": 0, "sha": "daf2182687948ef53790608f8fe13f21f457552bd56c61e914699d71793a95d0", - "bytes": 22287 + "bytes": 22287, + "shaCiteNorm": "37a0f1ef30156301cae88e2dece8a03f8ba557440925d3f8fd4b6f654e9880aa" }, { "i": 1, "sha": "2894d8949694f953aef9b0a7d4c4ad03b3633b8cd90886852109468d10e72b45", - "bytes": 30997 + "bytes": 30997, + "shaCiteNorm": "ae251c0c242448c1d87f417194f38d42c901e563ca0234a616e22254d8d29352" }, { "i": 2, "sha": "911f0c635e145362329fef63920e7b4802f2f9c9528062173e867ad891832873", - "bytes": 36896 + "bytes": 36896, + "shaCiteNorm": "efbc1fcc13a7968e0dd8120bc1dc0e04b24d0aac997700fe52dd414e0270c351" }, { "i": 3, "sha": "425dee6b9938613b65f9c944aea28228eda5679915fd39597196876dda2915f6", - "bytes": 45569 + "bytes": 45569, + "shaCiteNorm": "0d5a15e5216f11dd47816ff0bc336532ffdaec00fecad51ff2c9995a4bb00254" }, { "i": 4, "sha": "e62cbddd7d1facbcf2efe3c781c09b14b5cf096067b7d7ae4818ee7a4639e0b6", - "bytes": 51431 + "bytes": 51431, + "shaCiteNorm": "43615d160671fcbd9408a80e0779f6cf122fdbde81b901e35a277d8e6ee8495a" }, { "i": 5, "sha": "03a6265741bdb3f74788ae479f5b1505e552e6082009dd3419605dff8143b5da", - "bytes": 60104 + "bytes": 60104, + "shaCiteNorm": "66411dc16ffa93bb1fbc28b167302372c7b170af10bfca6c95ef9466cf4b8628" }, { "i": 6, "sha": "c7f62d7a595006a08aaa71427f0bcdb3b620c8eac104723bc6ace7516aa3321c", - "bytes": 65966 + "bytes": 65966, + "shaCiteNorm": "c9d9ff30521f6d1fa5e3e94f9ed778a1491224fc6440cd8bc3dde9e6bd7f77e6" }, { "i": 7, "sha": "edb9fadc1d20813a24f45ea2d1e2ab0d92972d1cee22d45ac278d9dd1c17bd4c", - "bytes": 74639 + "bytes": 74639, + "shaCiteNorm": "4e4c14113531fb4152983af6c3596dd3944f30ca0ab4e720e92a31bd7a253a9d" }, { "i": 8, "sha": "d2be70c3953a492bf1b974ff4922001d74d0d61012446c59e732fec18852d713", - "bytes": 80501 + "bytes": 80501, + "shaCiteNorm": "ca749fa6f80100f404d828594314c8823dc2900aedb089264a057e72f54536fa" }, { "i": 9, "sha": "2ddad1b2186f7f0444543f1b72b2f4fe3dcfe6d8cf709211a54e820b7c3a6a4a", - "bytes": 49528 + "bytes": 49528, + "shaCiteNorm": "8008f1d0bbe5e604ca7b1201e6d60e04ab7a10c117373eda0412308319f5c6b2" }, { "i": 10, "sha": "7da7b1f5aed39a2af9df234e8ad97fff4fe1cf49623a3adc4a3c45cd22989be0", - "bytes": 57070 + "bytes": 57070, + "shaCiteNorm": "09e0e277a54ce862f28c41a40d4b1261a8b8198240a10191986f07ddb1b9ad34" }, { "i": 11, "sha": "fca40b7b4dae14bfa94542cb8d802ce7b097b7b2946f987437db98a1cbc0385c", - "bytes": 62892 + "bytes": 62892, + "shaCiteNorm": "58178314a73fd15478a8d1cef6e6f3f535e216ff3a27d5d5c0af85f3cc480fe5" }, { "i": 12, "sha": "8b5ff77e47604fe58cbb19234306863e3ec754c1d51a7a354bab40ffb12ef59a", - "bytes": 77030 + "bytes": 77030, + "shaCiteNorm": "0afc5df7cd478a9a127da7cb011c61a20cbf35ebe31409daad93e0e9c2944f24" }, { "i": 13, "sha": "375a80af22ca3da9763155fd5394cff9066e0695490105830137cdcd9432f7d1", - "bytes": 82815 + "bytes": 82815, + "shaCiteNorm": "c16a407ec4983f77cd62792e46a093c839853077d4b52cd799b81adb38a1ca20" }, { "i": 14, "sha": "3f23c183e08dfe04772ea8cf1b5196f8ad92a57868896d55aafc7eb9300ed53a", - "bytes": 91464 + "bytes": 91464, + "shaCiteNorm": "415619a3a81f1aad4f2790ffc63eaeb3a0410e27027e77609a01f0fbd1471abe" } ], "foldAt": 9, "editedAt": 12, "summaryFrom": 9, - "stateDigest": "6727f42bd91879a7aa370e1ce0c78963bcba0df92e50390d4a8011f8f0da334d", + "stateDigest": "0a241435d456ba994c92b0072d2173bf2a6d00ee73223458fb4302e170efb1e3", + "stateDigestAligned": "c7079cd68bf9eb0075c3844a5cb1ae54ff6ecad8a43da424a7ffebd033eceb7f", "sectionDigests": { "state": "f93dc1c876c672d4", "blockContents": "dc7d13d250c94bb4", "stats": "c6f1d434975d047d", - "metadata": "f1ec4aab3aff777a" + "metadata": "16cad648ccf41950" }, "stateKeyDigests": { "blocks": "3a454b9a16e4f0b9", diff --git a/tests/golden/storage-render/chat.json b/tests/golden/storage-render/chat.json index 46764f97db..79d5eb6b70 100644 --- a/tests/golden/storage-render/chat.json +++ b/tests/golden/storage-render/chat.json @@ -4,88 +4,104 @@ { "i": 0, "sha": "824973c12a04e4a9c6fc8a5b84555886597d1a41874e6b0da33a489fe5f9fe9b", - "bytes": 22397 + "bytes": 22397, + "shaCiteNorm": "40255e64416e7fbea076dfe1195d32eac811a33d6e48db44922f93bfd3dadc31" }, { "i": 1, "sha": "4261484dd0b52e37019fe840a217bf4940369c7cb3c6d1ac28cb6353d4a68a37", - "bytes": 30975 + "bytes": 30975, + "shaCiteNorm": "7630710b673b4aa2dd1a7aade6809f5a97bdcd525857a37e9ab253dffcb82f43" }, { "i": 2, "sha": "e8050d3ec8b0e3dd2f0ca67a4a0328a8e280e23b6f4a3e34380af58e8d38d678", - "bytes": 36787 + "bytes": 36787, + "shaCiteNorm": "ad6b3f4bda6cf0b9b779034ea1b0ceb715c7a763df6b33eab4d5df7862d84b14" }, { "i": 3, "sha": "73d69f8b8f1bd2899cfe2181b95bc2f3697695e9b85ad682d82f6cbd9e7355ff", - "bytes": 45365 + "bytes": 45365, + "shaCiteNorm": "d683426210363474b3723a9dfd4e0e58e9eaf8b747291106d0eb22b86d6b1466" }, { "i": 4, "sha": "d3a73b4a74f7975de837b7848f24d875866b1f2b0b15f4ef78064e3602da6665", - "bytes": 51177 + "bytes": 51177, + "shaCiteNorm": "7b8084737aabd65b5d9be87be1610819f658c14a632de7f3575b6fbf44860e0b" }, { "i": 5, "sha": "60ce463e424f124a2dc346479091cbd4b7a32c69696b798a1de6b1a3c0f3ef7b", - "bytes": 59755 + "bytes": 59755, + "shaCiteNorm": "7b96653787fc0adda16590f35f3cb70f47bbecfed5e8208bc9f9520f0c360330" }, { "i": 6, "sha": "1298d5f45146b76a42b5b75c712de8011fb091c5754f99d490a7df2e81ba2466", - "bytes": 65567 + "bytes": 65567, + "shaCiteNorm": "33e2e632f57a5aeec30801779d9e38fcc55247389ac83217e88e0c6500b6a783" }, { "i": 7, "sha": "b20457e685b4a1fecf0195f57b93e052a0d00a83feb8b1569ccb08d385663aa2", - "bytes": 74145 + "bytes": 74145, + "shaCiteNorm": "0c7a0b8156c69d5710d2e395b6b8c985878c9120de8ff6c8f1fbe957649b855d" }, { "i": 8, "sha": "a0ba24010c6faf10f11b66faca6f3aa915dde7ac006d66a434777cbd765328ca", - "bytes": 79957 + "bytes": 79957, + "shaCiteNorm": "180576856d481f74a2e1fd352cb1ee881644fe905fb3263f16b12c90a7f5b86a" }, { "i": 9, "sha": "f1e08e530a96ecb4996c0a63e840e3f374a82b095e285ffea94e70391eb32fe0", - "bytes": 49458 + "bytes": 49458, + "shaCiteNorm": "7f1a433d2f553cc122e877912c99464485726c3d0707c5ffd12acd2478475176" }, { "i": 10, "sha": "8f3e444b1951f67e4f86abf6e727162108ba39d389b1ee4c90eea69df15bada0", - "bytes": 56859 + "bytes": 56859, + "shaCiteNorm": "90d0ab02fca529d89c1e045ec2bb157872e5b70508975b6d1782fc6c0a525cc9" }, { "i": 11, "sha": "94a8e322c3c0822a18bccd6295c3d2bfb4c7961d897298361612f286fb6108fe", - "bytes": 62594 + "bytes": 62594, + "shaCiteNorm": "9ca56f6252eb19787f5577ed01833ea816d242299c4f7a3b24351b7fb767860e" }, { "i": 12, "sha": "2c0a965647d8832dc74ebe5edc8cfe59b67abc936714119e89c93410f985e1af", - "bytes": 76640 + "bytes": 76640, + "shaCiteNorm": "39c2ab72032ed2149fdbb66698cffc18f6e0d96f713fb37181e08cb0cadc993a" }, { "i": 13, "sha": "a7497d037a324f1fdb6f605db0b8161e4e183f1e2354bb68035fb9f1373ee0d0", - "bytes": 82375 + "bytes": 82375, + "shaCiteNorm": "001e2a63c5352cd3230a30bc82dd6f9fbb8c6421f42e1bc8dd87551f50b8ae9f" }, { "i": 14, "sha": "abd73cec691568d84e72384aa5facdc71c759eeafc865d62df7b4eca6683675a", - "bytes": 90929 + "bytes": 90929, + "shaCiteNorm": "0f6af62ddbc1c0a39009524d581f52f728e928d438e998a716be5d054a8c6609" } ], "foldAt": 9, "editedAt": 12, "summaryFrom": 9, - "stateDigest": "e149c301f382438ee063a38f70dd794207b88e94bdf3c14c5191da0eeaa99bc9", + "stateDigest": "9f1b8258a916fcf63af14625def9ed0236eb8ec9c542f0027206985598d2a5b8", + "stateDigestAligned": "86e45900d61e81f0c8239bfe4b4019451b600c2a5258e89e53c68bc09fd2d5ca", "sectionDigests": { "state": "8390f21336885863", "blockContents": "94a58f96d7c5cfc3", "stats": "24d705b0e39bd13c", - "metadata": "55733fb81ccf7d69" + "metadata": "2198f47054c8acae" }, "stateKeyDigests": { "blocks": "d9770feee91dafc6", diff --git a/tests/golden/storage-render/google.json b/tests/golden/storage-render/google.json index ba999cadec..a0fa7d5830 100644 --- a/tests/golden/storage-render/google.json +++ b/tests/golden/storage-render/google.json @@ -4,88 +4,104 @@ { "i": 0, "sha": "45998d29279880c0b19002f5ca40489e1e9dfcbdb53b9203a5ffe868ea3e51b8", - "bytes": 20510 + "bytes": 20510, + "shaCiteNorm": "a80634db66b3e2f5c96a66d48a7baf753f089d0af76e1c7370e44a22be3e2c50" }, { "i": 1, "sha": "f61275d77406c502c11238e2764ea84c8bf03c8302856749b8f5b73572fb5331", - "bytes": 29065 + "bytes": 29065, + "shaCiteNorm": "bbb40518ce7a359b6fb4d9ccebf356a433dee955daf8676d937fc2b1e2c493e0" }, { "i": 2, "sha": "98b77dc7c79575218c8e52ddec9386d5db7a7d30587dd992cb64c18536fb7667", - "bytes": 34891 + "bytes": 34891, + "shaCiteNorm": "3c18b9c31b2c19266a4f006c368008aa6dd0dbacdeb1bc00c86f3a76cd8fd8b5" }, { "i": 3, "sha": "2f3024ee57f441a1c0d1b06d38ccf1f4c74ef297aa3d58622bc29f69c21c93d6", - "bytes": 43446 + "bytes": 43446, + "shaCiteNorm": "8e074da69eef79658d01adab2bed04d55a232eebf3e437c0a94b4afefb52bb8c" }, { "i": 4, "sha": "27f6f223e7682e0e0790a69e43c982c3fe0e505153be7fa4d3ec6cf4532ff2e2", - "bytes": 49272 + "bytes": 49272, + "shaCiteNorm": "63b04fd0418ab224c41ee6e9ee2068a73b973c8df07e4e7f2ffbe6e19373ec7a" }, { "i": 5, "sha": "6a7e23920335f91762a8ac72b177a1cd68538ce1872325b6aa9e3b44191f3ca4", - "bytes": 57827 + "bytes": 57827, + "shaCiteNorm": "723ccf3a211d8b750959b479939ded61e4c9818473bde3a7364eae85ca9ad5da" }, { "i": 6, "sha": "5b770ce0c9456af96904cd5b007fbb07fb4194f2636db8b6ad2602ef71d3834d", - "bytes": 63653 + "bytes": 63653, + "shaCiteNorm": "4c20a67e2e4df13e061f8f094b92774d2c53c60a0711b6456731b943e510eec4" }, { "i": 7, "sha": "9bab6f659c0dcd05e6576e6d47b1d1d2378a6ace1a7effec721958f6f7476cd8", - "bytes": 72208 + "bytes": 72208, + "shaCiteNorm": "c807dfe30868d78848a846469e99e7d40fe7b866717974886c30ba8def051063" }, { "i": 8, "sha": "c179b940520413ba2a0e267165a9914712c3a9ff3c045761d61da876aeccedd7", - "bytes": 78034 + "bytes": 78034, + "shaCiteNorm": "2091c5306aba0a16a47547029fe8dd6b57cc8e31b3c6a2abea7d9a8ba7b63839" }, { "i": 9, "sha": "1b1448a075964dae98ad3b09565d31590cc9fdadb4d37300c4460030f67990e6", - "bytes": 50620 + "bytes": 50620, + "shaCiteNorm": "6ef485631895d0a448bb8d52235b9f3a00ba15143b587ac7253d119c83c61441" }, { "i": 10, "sha": "eddf6ad2081f5eb1e4822bf9805a7953eff3241269ed5f3eec346909c9da5bf2", - "bytes": 58023 + "bytes": 58023, + "shaCiteNorm": "caec6185725a235e023982a9d7f6b4fcb284bb6a2e21b89bb8f2b7e36bb1bf5c" }, { "i": 11, "sha": "0ae5c9b7b31dbb0d52c1b4a157bcc08c549ff435229383ec33fb0225b3c93f4e", - "bytes": 63772 + "bytes": 63772, + "shaCiteNorm": "894e9c4f827fb398b7a6c9bc91d430633e51bd552585fb5086c2cf3c18d6097f" }, { "i": 12, "sha": "d9a62f5c48f849bdc9514ae4586f2204ef5ac624aaf97c50dbc8ed2c1bee9706", - "bytes": 77777 + "bytes": 77777, + "shaCiteNorm": "2f1449ede69c9ee7db22f39c49584f64d35a29541097d78f524a4f22c5998acc" }, { "i": 13, "sha": "f60319947e77097d08edfcc5fdddd46057c44596a30d1d6bb33c5af4bad5001f", - "bytes": 83526 + "bytes": 83526, + "shaCiteNorm": "ee505218214a0e58875762c06c79e717164b89cfe29cdd61630fdd1fecb41a92" }, { "i": 14, "sha": "275f3b3e103dbfbf545448f8c421254cc7b025f8c081d941cd39bef4a90023be", - "bytes": 92056 + "bytes": 92056, + "shaCiteNorm": "c434408819f41c9e943e1cd60b80189f1844ba2c8f11898043f158f8485aee9b" } ], "foldAt": 9, "editedAt": 12, "summaryFrom": 9, - "stateDigest": "80e83935e5fa7b052a3cb0f1d641bd81cb6ee59f91f3c1f2f31c204de29368a5", + "stateDigest": "c4979d4f58cfa49bc8be8c43b8e1f1b0861b3b9d44cdc415ee09e4ca21d9aa5a", + "stateDigestAligned": "43e4aa88c25bcb08f0435e60e0e37270061479086893d8148749d09c839db65b", "sectionDigests": { "state": "2e1def3a3234afdc", "blockContents": "473264d9df8ba338", "stats": "900481685c848462", - "metadata": "88cd02a467b4457e" + "metadata": "939dc0c09b6cbcbf" }, "stateKeyDigests": { "blocks": "0cfd0b9c0fc53073", diff --git a/tests/golden/storage-render/responses.json b/tests/golden/storage-render/responses.json index 7004d90435..3ece8b999d 100644 --- a/tests/golden/storage-render/responses.json +++ b/tests/golden/storage-render/responses.json @@ -4,88 +4,104 @@ { "i": 0, "sha": "57662dd5517499b0833b00df645aacd9ec25a1add3b6efa94703cc40006824a8", - "bytes": 22467 + "bytes": 22467, + "shaCiteNorm": "dd2d60a2354cb8bb76834fa946ad985964def531c400f15a9f0370ee948fd039" }, { "i": 1, "sha": "ade76466ddbffb25c3eba9e06896c110c147ce1daf7f1e231fe054d9d1832c4b", - "bytes": 31395 + "bytes": 31395, + "shaCiteNorm": "9a1ea902b7287c23566da611dbab379637998536dc37365d47cd8577fa5f6a8c" }, { "i": 2, "sha": "09ee3e5a15865e5cc5c10c28026c1501f565212134c64a03214fbf367f63b881", - "bytes": 37517 + "bytes": 37517, + "shaCiteNorm": "99c6597f2bc37b03f6935208eb4be452619d599ce21b06f118f15657c5494a0b" }, { "i": 3, "sha": "cd3e38d97e64660eb60f1e387a1e53de8723ae1aa916db4d0a7b8f7ec181c266", - "bytes": 46445 + "bytes": 46445, + "shaCiteNorm": "947f17b43bc46a8f1ad828172f394e003f499af7ca16196f907620662a3a7929" }, { "i": 4, "sha": "b8ff4f16ca15cc1b8589251a849e3e30ec7d7d82b92abc0bb262bd2f8422db38", - "bytes": 52567 + "bytes": 52567, + "shaCiteNorm": "c0ce07681a604168b19c4e585112d5644555b8283a7c92c620336d0ec1453806" }, { "i": 5, "sha": "70f002ab5e7c3301ec440fd6863b7635c38874842445794bf87229f783ce2124", - "bytes": 61495 + "bytes": 61495, + "shaCiteNorm": "20c8f20fe476b978fcb07d34f4311a2712e8ea04e829b8ee0912e7741d6f95d1" }, { "i": 6, "sha": "aa63e444c4b86e426d949d2030a2b2a4f84a1018ebb152f0dd429c9747f357b3", - "bytes": 67617 + "bytes": 67617, + "shaCiteNorm": "91c714ad3016518bef0e07f9a7d227c9adcd164b5c3b1e62ee24b20bc38524d2" }, { "i": 7, "sha": "4cef1346cac328fc82bd552007051118cc95751e62d4b3f10a4432297a6801e5", - "bytes": 76545 + "bytes": 76545, + "shaCiteNorm": "68db839f454054f55c426b3e173d9baa9d4120a6f963cfa4856d4bad2b11e318" }, { "i": 8, "sha": "adee16938078b6c6cac3ca423ffe427d2199493d6cbbbc01059b223de3bba077", - "bytes": 82667 + "bytes": 82667, + "shaCiteNorm": "f8ca121812fc632d1b2cb140ac8ec41ceeb4f82916256386aa308d2a4272b09f" }, { "i": 9, "sha": "a6cefd10bbb51de65d7c363ea6f299a2301f4d8104a5ce8b59f4de87e711130d", - "bytes": 50498 + "bytes": 50498, + "shaCiteNorm": "266499f0528988fed9680ffef231be1162e6501857511a3de6a5f95ba1564256" }, { "i": 10, "sha": "ed8590ef249a0756d077fcea24230dd83a7f805a03da6f91457c607b27351c5e", - "bytes": 58283 + "bytes": 58283, + "shaCiteNorm": "5ed641acfb444fed79d86b5104b176d11250d309a649a2230ba2f3f08950453c" }, { "i": 11, "sha": "461eb77a6cc41abbf8046fb3dc5858f18c8d848a8dff4a2c888a38a1d16df707", - "bytes": 64328 + "bytes": 64328, + "shaCiteNorm": "c674b6af035c1b786fd0f47c0c33abc4179337e3df595391f3fcf9b6973ef531" }, { "i": 12, "sha": "b356956d375e951d2d5a05fe55c1e28b09c616f9b2d7eca6fea5e62924dbe53e", - "bytes": 78746 + "bytes": 78746, + "shaCiteNorm": "f8d20e84c64dfd8e224496a6feb949d16edc9de40f5100243d0514ec86eeddec" }, { "i": 13, "sha": "1b3d93417fce2ba8f70c52d0425626398a571d7afbd287ffb89a6ba7ae37bd29", - "bytes": 84794 + "bytes": 84794, + "shaCiteNorm": "b24c9b065471e5a264cb73d58b2383d9f8fc5b4feeb1f7d0fea05aeb96d1ff01" }, { "i": 14, "sha": "9c82e4523129ef8df61cf7622bb637c7379cce153c78e3814bd3000853034323", - "bytes": 93703 + "bytes": 93703, + "shaCiteNorm": "cd5c51970f832bcaa5f27406e353c8e67a53706c2bc8249e35634b6339dc893a" } ], "foldAt": 9, "editedAt": 12, "summaryFrom": 9, - "stateDigest": "08526d3552fca7ac7552cd16a221893a652fbe740c9b8dccac6db57b3d6302ee", + "stateDigest": "b3bddfa75ce3f8b94424be9a0164c1b9a183faa9daf705e6175cabdd671c95ff", + "stateDigestAligned": "ae78cdd8a1784474f1ffa5e9c8e8871871e4498b75df568b614a977e60af2339", "sectionDigests": { "state": "db313e8eba2d65fc", "blockContents": "b6701268a0bdcd8a", "stats": "327d3967b8371aa8", - "metadata": "b5c39d84db67f424" + "metadata": "a9eecd3816738158" }, "stateKeyDigests": { "blocks": "34b0dc17f8c24595", diff --git a/tests/issue4739-bounded-session-load.test.ts b/tests/issue4739-bounded-session-load.test.ts new file mode 100644 index 0000000000..bc9906b5f0 --- /dev/null +++ b/tests/issue4739-bounded-session-load.test.ts @@ -0,0 +1,264 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import { SessionStore } from "../src/persist.ts"; +import type { Session } from "../src/session.ts"; +import { isDroppedMetadata, snapshotMessages } from "../src/session.ts"; +import { createInitialState, type CoreMessage } from "acp-kernel"; +import { planIngest, planIngestDelta, loadLegacyView, type LegacySessionLike } from "../src/storage/ingest.ts"; +import { UnifiedStore, messageRowReads, viewPointerSeqs } from "../src/storage/store.ts"; +import type { UnifiedShadow } from "../src/storage/shadow.ts"; +import { rmrf } from "./tmp-rm.ts"; + +// #4739: ordinary loads must not materialize the whole historical log — the +// view only consumes tail/pluginSnapshot pointer rows — and the dropped +// lazy fields must restore through narrow reads instead of two full-session +// rebuilds per saved record. + +process.env.BILI_PERSIST_ZSTD = "0"; +process.env.BILI_CONFIG_FILE = path.join( + fs.mkdtempSync(path.join(os.tmpdir(), "bili-4739-hermetic-")), + "no-config.json", +); +process.env.BILI_STORAGE_UNIFIED = "legacy"; +// Keep the persisted tail SMALL relative to the seeded history so the +// pointer set is a strict subset of the message table. +process.env.BILI_PERSIST_TAIL_TOKENS = "600"; + +function makeSession(id: string): Session { + return { + id, + meta: { label: id, protocol: "openai", upstreamOrigin: "http://up:1" }, + stats: { requests: 3, tokensSaved: 0, inputTokens: 10, cachedTokens: 0, outputTokens: 5, cacheSamples: 0, lastInputTokens: 10, contextTokens: 1000, compressCreditTokens: 0, retrieveCalls: 0, retrieveHits: 0, retrieveMisses: 0, storedBytes: 0, storeBytesSaved: 0, rangeRestores: 0 }, + metadata: {}, + state: createInitialState(), + createdAt: Date.now() - 1000, + lastSeen: Date.now(), + blockContents: new Map(), + inFlight: 0, + persisted: false, + pendingRetrievals: [], + }; +} + +function rawTail(n: number): CoreMessage[] { + const out: CoreMessage[] = []; + for (let i = 0; i < n; i++) { + out.push({ id: `big${i}`, contentType: "text", role: i % 2 === 0 ? "user" : "assistant", text: `bounded load probe message number ${i} with filler words to spend tokens` }); + } + return out; +} + +/** Finds the persisted envelope for `id` under `root` (payload.id match). */ +function findEnvelope(root: string, id: string): string | null { + const hits: string[] = []; + const walk = (d: string): void => { + for (const f of fs.readdirSync(d, { withFileTypes: true })) { + const fp = path.join(d, f.name); + if (f.isDirectory()) walk(fp); + else if (f.name.endsWith(".json")) hits.push(fp); + } + }; + walk(root); + for (const hit of hits) { + try { + const envelope = JSON.parse(fs.readFileSync(hit, "utf8")) as Record; + const payload = (envelope["payload"] ?? envelope) as Record; + if (payload["id"] === id) return hit; + } catch { + // non-JSON or partial write — skip + } + } + return null; +} + +test("full mode: viewFor decodes only pointer rows and matches the full-log decode", async () => { + const prev = process.env.BILI_STORAGE_UNIFIED; + process.env.BILI_STORAGE_UNIFIED = "full"; + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "bili-4739-view-")); + const store = new SessionStore({ dir, debounceMs: 60_000, enabled: true }); + try { + const s = makeSession("big-1"); + s.metadata.foldAnchors = { m00001: { c: "text", frags: ["a", "b"] } }; + const N = 400; + snapshotMessages(s, rawTail(N)); + await store.writeNow(s); + + const sh = (store as unknown as { shadow?: UnifiedShadow }).shadow; + assert.ok(sh && sh.readsUnified, "full-mode sidecar live"); + + const direct = UnifiedStore.open(path.join(dir, "index.db")); + // Generation 2 — the production shape after an idle fold (#3050 R3): + // the table keeps every row, tail_json points at the bounded raw + // suffix only (an equal-length raw tail is a valid empty delta). + const K = 40; + const allRows = direct.getMessages("big-1"); + assert.equal(allRows.length, N, "generation 1 ingested the full history"); + const maxSeq = allRows[allRows.length - 1]!.seq; + const suffixSeqs = allRows.slice(-K).map((m) => m.seq); + const boundaryRow = allRows[allRows.length - 1]!; + const deltaLegacy: LegacySessionLike = { + id: "big-1", + createdAt: s.createdAt, + lastSeen: s.lastSeen, + meta: { protocol: "openai" }, + stats: { requests: 4 }, + metadata: {}, + state: { blocks: [], messageRefs: { byRaw: {}, byRef: {} }, nextBlockId: 1, nextRunId: 1 }, + blockContents: new Map(), + lastMessages: rawTail(N).slice(-K), + }; + const plan = planIngestDelta(deltaLegacy, maxSeq, suffixSeqs, { msgId: boundaryRow.msgId, contentHash: boundaryRow.contentHash }, true); + assert.ok(plan, "empty delta accepted for the shrunken raw tail"); + direct.ingestSession(plan, { trustedSeq: maxSeq, boundaryMsgId: boundaryRow.msgId, boundaryHash: boundaryRow.contentHash, trustedTailSeqs: suffixSeqs }); + + // Reference decode: the OLD algorithm — every message row through + // loadLegacyView on the second handle of the same database. + const sessionRow = direct.getSessionRow("big-1"); + assert.ok(sessionRow, "session row present"); + const liveSeqs = new Set(direct.getMessages("big-1").map((m) => m.seq)); + assert.equal(liveSeqs.size, N, "table still holds the full history"); + const expectedPointers = [...new Set(viewPointerSeqs(sessionRow.tailJson))].filter((seq) => liveSeqs.has(seq)).length; + assert.equal(expectedPointers, K, `tail now points at ${K} of ${N} rows`); + + const reference = loadLegacyView({ + session: sessionRow, + messages: direct.getMessages("big-1"), + blocks: direct.getBlocks("big-1"), + deadRefs: direct.getDeadRefs("big-1"), + refs: direct.getRefs("big-1"), + }); + const before = messageRowReads.count; + const via = sh.viewFor("big-1"); + const delta = messageRowReads.count - before; + direct.close(); + + assert.ok(via, "viewFor resolved"); + assert.deepEqual(via, reference, "bounded decode byte-equal to the full-log decode"); + assert.equal(delta, expectedPointers, `viewFor read ${delta} message rows for ${expectedPointers} pointer seqs (full log = ${liveSeqs.size})`); + } finally { + store.cancelAll(); + if (prev === undefined) delete process.env.BILI_STORAGE_UNIFIED; + else process.env.BILI_STORAGE_UNIFIED = prev; + rmrf(dir); + } +}); + +test("full mode: dropped metadata restores from the index alone (narrow read, no envelope)", async () => { + const prev = process.env.BILI_STORAGE_UNIFIED; + process.env.BILI_STORAGE_UNIFIED = "full"; + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "bili-4739-md-")); + const store = new SessionStore({ dir, debounceMs: 60_000, enabled: true }); + try { + const s = makeSession("md-idx"); + s.metadata.foldAnchors = { m00001: { c: "tool-call" } }; + s.metadata.cacheLedger = { restartDrops: { count: 2 } }; + snapshotMessages(s, rawTail(50)); + await store.writeNow(s); + assert.equal(isDroppedMetadata(s.metadata), true, "metadata folded after idle save"); + + // The persisted shape (what a restart restores) — captured while the + // envelope still exists. + const envelopePath = findEnvelope(dir, "md-idx"); + assert.ok(envelopePath, "envelope written in full mode"); + const envelopePayload = JSON.parse(fs.readFileSync(envelopePath, "utf8")).payload as Record; + const persistedKeys = Object.keys(envelopePayload.metadata as object).sort(); + + // Delete the file copy: restore must come from the index alone. + fs.unlinkSync(envelopePath); + assert.deepEqual(s.metadata.cacheLedger, { restartDrops: { count: 2 } }, "narrow index restore serves the value"); + assert.deepEqual(Object.keys(s.metadata).sort(), persistedKeys, "materialized shape is the persisted (restart) shape"); + assert.equal(isDroppedMetadata(s.metadata), false, "materialized on read"); + + // A later save persists the same keys — no thinned erase. + await store.writeNow(s); + const again = findEnvelope(dir, "md-idx"); + assert.ok(again, "envelope rewritten by later save"); + assert.deepEqual(Object.keys(JSON.parse(fs.readFileSync(again, "utf8")).payload.metadata as object).sort(), persistedKeys); + } finally { + store.cancelAll(); + if (prev === undefined) delete process.env.BILI_STORAGE_UNIFIED; + else process.env.BILI_STORAGE_UNIFIED = prev; + rmrf(dir); + } +}); + +test("store: getBlockContents/getSessionMetadata equal the loadLegacyView replay leg", async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "bili-4739-narrow-")); + const store = UnifiedStore.open(path.join(dir, "index.db")); + try { + // Real folded corpus payload (one active block + contents + refs). + const corpusFile = path.resolve(import.meta.dirname, "golden/storage-corpus/folded/anthropic/127.0.0.1_29e4587af02d1ed9ae4b0054.json"); + const envelope = JSON.parse(fs.readFileSync(corpusFile, "utf8")) as Record; + const payload = (envelope["payload"] ?? envelope) as Record; + const legacy = payload as unknown as LegacySessionLike; + // The persisted record carries blockContents as a plain object; the + // ingest planner takes the in-memory Map shape. + legacy.blockContents = new Map(Object.entries((payload["blockContents"] ?? {}) as Record)); + // The pre-#lastSeen corpus files carry savedAt only; the planner stamps + // sessions.updated_at from lastSeen. + if (typeof legacy.lastSeen !== "number") legacy.lastSeen = Number(payload["savedAt"] ?? Date.now()); + store.ingestSession(planIngest(legacy)); + + const sessionRow = store.getSessionRow("corpus-folded"); + assert.ok(sessionRow, "corpus ingested"); + const rows = { + session: sessionRow, + messages: store.getMessages("corpus-folded"), + blocks: store.getBlocks("corpus-folded"), + deadRefs: store.getDeadRefs("corpus-folded"), + refs: store.getRefs("corpus-folded"), + }; + const reference = loadLegacyView(rows); + assert.ok(reference.blockContents.size > 0, "corpus carries block contents"); + + assert.deepEqual(store.getBlockContents("corpus-folded"), reference.blockContents, "narrow blockContents == replay leg"); + assert.deepEqual(store.getSessionMetadata("corpus-folded"), reference.metadata, "narrow metadata == replay leg"); + assert.equal(store.getBlockContents("no-such-session"), null, "row miss degrades to null"); + assert.equal(store.getSessionMetadata("no-such-session"), null, "row miss degrades to null"); + } finally { + store.close(); + rmrf(dir); + } +}); + +test("store: getMessagesAt returns exactly the requested live seqs, in order", async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "bili-4739-at-")); + const store = UnifiedStore.open(path.join(dir, "index.db")); + try { + const N = 600; + const messages: CoreMessage[] = []; + for (let i = 0; i < N; i++) { + messages.push({ id: `at${i}`, contentType: "text", role: i % 2 === 0 ? "user" : "assistant", text: `getMessagesAt probe row ${i}` }); + } + const legacy: LegacySessionLike = { + id: "at-1", + createdAt: Date.now(), + lastSeen: Date.now(), + meta: { protocol: "openai" }, + stats: { requests: 1 }, + metadata: {}, + state: { blocks: [], messageRefs: { byRaw: {}, byRef: {} }, nextBlockId: 1, nextRunId: 1 }, + blockContents: new Map(), + lastMessages: messages, + }; + store.ingestSession(planIngest(legacy)); + const all = store.getMessages("at-1"); + assert.equal(all.length, N, "all rows ingested"); + + // Sparse request spanning both IN-chunks (>500) plus a missing seq. + const wanted: number[] = [1, 2, 3]; + for (let seq = 10; seq <= N; seq += 7) wanted.push(seq); + wanted.push(99_999); + const got = store.getMessagesAt("at-1", wanted); + const expected = wanted.filter((seq) => seq <= N); + assert.deepEqual(got.map((m) => m.seq), expected, "exactly the requested live seqs, ordered"); + assert.ok(got.every((m) => m.content.length > 0), "content hydrated"); + assert.deepEqual(store.getMessagesAt("at-1", []), [], "empty request → empty result"); + } finally { + store.close(); + rmrf(dir); + } +}); diff --git a/tests/launcher-opencode-inline.test.ts b/tests/launcher-opencode-inline.test.ts index 7ff04b2ace..09f433c205 100644 --- a/tests/launcher-opencode-inline.test.ts +++ b/tests/launcher-opencode-inline.test.ts @@ -28,6 +28,7 @@ interface SpawnRecord { command: string; args: string[]; env: NodeJS.ProcessEnv | undefined; + verbatim: boolean | undefined; } // `bili opencode-inline` (#4669): the launcher must start NO proxy, hand the @@ -75,7 +76,7 @@ test("runLaunch opencode-inline: no proxy spawned, BILI_OPENCODE_INLINE=1, attac const spawns: SpawnRecord[] = []; const spawnImpl: SpawnFn = (cmd, args, options) => { - spawns.push({ command: cmd, args: [...args], env: options?.env }); + spawns.push({ command: cmd, args: [...args], env: options?.env, verbatim: options?.windowsVerbatimArguments }); const child = makeFakeChild(0); const orig = child.on!.bind(child); (child as { on: SpawnChild["on"] }).on = (event, listener) => { @@ -109,7 +110,25 @@ test("runLaunch opencode-inline: no proxy spawned, BILI_OPENCODE_INLINE=1, attac { fetchImpl, fetchHealthInfo, spawnImpl, sleep: () => Promise.resolve() }, ); assert.equal(spawns.length, 1, "exactly one child (opencode itself) — no proxy spawn"); - assert.equal(spawns[0].command, fakeOc); + // #679: on Windows the .cmd shim rides comspec /d /s /c "" + // (CreateProcess cannot execute a batch file); elsewhere it spawns directly. + // Expectations derive from the child's captured env, not ambient + // process.env — concurrent tests in one CI shard rewrite globals mid-flight (#4599). + const cap = spawns[0]; + if (process.platform === "win32") { + // Mirror planClientSpawn's case-insensitive probe: sanitized env + // copies keep the OS key casing (`ComSpec`), plain objects don't (#4599). + const ce = cap.env ?? {}; + const childComspec = ce.COMSPEC ?? ce.ComSpec ?? ce.comspec; + const expectedComspec = typeof childComspec === "string" && childComspec.trim().length > 0 ? childComspec : "cmd.exe"; + assert.equal(cap.command, expectedComspec, "#679 comspec wrap uses the child env's COMSPEC"); + assert.deepEqual(cap.args.slice(0, 3), ["/d", "/s", "/c"]); + assert.equal(cap.verbatim, true, "#679 verbatim argv keeps cmd.exe from re-splitting"); + assert.match(cap.args[3] ?? "", /^".*\.cmd"$/, "comspec line wraps the quoted .cmd shim"); + } else { + assert.equal(cap.command, fakeOc); + assert.deepEqual(cap.args, []); + } const env = spawns[0].env!; assert.equal(env.BILI_OPENCODE_INLINE, "1"); assert.equal(env.BILLION_CONTEXT_PROXY, undefined, "inherited BILLION_CONTEXT_PROXY must be stripped (attach would win)"); @@ -179,7 +198,7 @@ test("runLaunch opencode-inline: refuses (exit 2) when the bili opencode plugin const spawns: SpawnRecord[] = []; const spawnImpl: SpawnFn = (cmd, args, options) => { - spawns.push({ command: cmd, args: [...args], env: options?.env }); + spawns.push({ command: cmd, args: [...args], env: options?.env, verbatim: options?.windowsVerbatimArguments }); return makeFakeChild(0); }; const prevExit = process.exit; diff --git a/tests/plugin-fork.test.ts b/tests/plugin-fork.test.ts index d15bc89fe9..1c4801377d 100644 --- a/tests/plugin-fork.test.ts +++ b/tests/plugin-fork.test.ts @@ -747,15 +747,26 @@ test("HTTP fork revision and originals survive a new proxy process", { timeout: let child: ReturnType | undefined; try { await compress(h); + // CI-only race (#4599): the compress fold lands in the live session + // state asynchronously after the tool response (#3573 replay + // semantics). On slow runners flushSync below would otherwise persist + // the pre-fold state and the cold load reports "0 block(s) + // (compressions=0)". Bounded-wait for the fold first. + for (let i = 0; i < 100 && !(resolveConversation("parent").session?.state.blocks?.length ?? 0); i++) { + await new Promise((r) => setTimeout(r, 50)); + } const parent = resolveConversation("parent").session!; parent.metadata.effectiveCcr = { ...DEFAULT_CCR_CONFIG, enabled: true }; parent.contentStore = storeOriginal(createContentStore(), { ref: "m00001", rawId: parent.pluginSnapshot![0].id, text: "CCR original retained independently after restart", kind: "original", tokens: 10, head: "CCR original" }); parent.contentStoreDirty = true; assert(h.store.flushSync(parent)); - // #4316 PR3/3: cold load runs the one-time ledger alignment — the loaded - // state now carries refLedgerAligned. Mirror it in the expectation. + // #4316 PR3/3: cold load runs the one-time ledger alignment, but only + // when the unified store is available — config-clean machines (CI) run + // legacy mode where the alignment is an honest skip and the loaded + // state never carries refLedgerAligned. Mirror the mode in the + // expectation instead of asserting the flag unconditionally. const persistedState = JSON.parse(JSON.stringify(parent.state)); - persistedState.refLedgerAligned = true; + if (h.store.replicasAvailable()) persistedState.refLedgerAligned = true; assert.deepEqual(JSON.parse(JSON.stringify(h.store.loadSync("parent")!.state)), persistedState); const snapshot = (await h.request("/__bili/plugin/snapshot?conversationId=parent")).body; const req = forkRequest(snapshot); diff --git a/tests/storage-backend-contract.test.ts b/tests/storage-backend-contract.test.ts index ba36a1a71b..672c64b9b7 100644 --- a/tests/storage-backend-contract.test.ts +++ b/tests/storage-backend-contract.test.ts @@ -3,7 +3,7 @@ import assert from "node:assert/strict"; import * as fs from "node:fs"; import * as os from "node:os"; import * as path from "node:path"; -import { planIngest, type LegacySessionLike } from "../src/storage/ingest.ts"; +import { planIngest, planIngestDelta, type LegacySessionLike } from "../src/storage/ingest.ts"; // The contract suite deliberately imports NOTHING from store.ts: every // assertion goes through the StorageBackend interface only. When the mysql // backend lands (#3388 step 2) this same file runs against it unchanged. @@ -110,6 +110,53 @@ test("contract: ingest → reads round-trip through the interface", async () => } }); +// #4739: ordinary loads must fetch ONLY the tail/pluginSnapshot pointer rows — +// the historical log stays on disk but is not decoded. Both lanes carry this +// contract (sqlite pins it here; the mysql lane below pins the same shape). +test("contract: sessionViewRows scopes message fetch to tail pointers (#4739)", async () => { + const dir = scratch(); + try { + const backend = await openBackend("sqlite", path.join(dir, "index.db")); + try { + const N = 6; + const msgs = Array.from({ length: N }, (_, i) => mkMsg(`m${i + 1}`, `body-${i + 1} ` + "z".repeat(120))); + assert.ok(await backend.ingestSession(planIngest(mkSession("sess-view", msgs))), "gen-1 ingest ok"); + + // Generation 2 — the production shape after an idle fold: the + // table keeps every row, tail_json points at the bounded suffix. + const all = await backend.getMessages("sess-view"); + assert.equal(all.length, N, "generation 1 ingested the full history"); + const maxSeq = all[all.length - 1]!.seq; + const K = 3; + const suffixSeqs = all.slice(-K).map((m) => m.seq); + const boundary = all[all.length - 1]!; + const plan = planIngestDelta(mkSession("sess-view", msgs.slice(-K)), maxSeq, suffixSeqs, { msgId: boundary.msgId, contentHash: boundary.contentHash }, true); + assert.ok(plan, "empty delta accepted for the shrunken raw tail"); + assert.ok( + await backend.ingestSession(plan, { trustedSeq: maxSeq, boundaryMsgId: boundary.msgId, boundaryHash: boundary.contentHash, trustedTailSeqs: suffixSeqs }), + "gen-2 delta ingest ok", + ); + + const view = await backend.sessionViewRows("sess-view"); + assert.ok(view, "view bundle present"); + assert.equal(view!.session.sessionId, "sess-view"); + const stillAll = await backend.getMessages("sess-view"); + assert.equal(stillAll.length, N, "full history retained on disk"); + assert.deepEqual(view!.messages.map((m) => m.seq), suffixSeqs, "only the pointer seqs were fetched"); + const bySeq = new Map(stillAll.map((m) => [m.seq, m])); + for (const row of view!.messages) { + assert.equal(row.content, bySeq.get(row.seq)!.content, "pointer row byte-equal to the full-log row"); + assert.equal(row.contentHash, bySeq.get(row.seq)!.contentHash); + } + assert.equal(await backend.sessionViewRows("no-such-session"), null, "null on session miss"); + } finally { + await backend.close(); + } + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + test("contract: session_ext + watermark + tailSeqs", async () => { const dir = scratch(); try { @@ -248,6 +295,25 @@ test("mysql lane: contract surface against MySQL 8.0", { skip: MYSQL_DSN.length const rows2 = await backend.getMessages(sid); assert.equal(rows2.length, 2, "no duplicate rows on identical re-ingest"); + // #4739: sessionViewRows pointer scoping on the mysql lane (per-statement + // reads are safe because ingestSession commits message rows + session row + // in one transaction; the sqlite twin pins the same shape above) + const vwSid = `sess-mysql-view-${run}`; + const vwMsgs = [mkMsg("m1", "view " + "v".repeat(80)), mkMsg("m2", "view " + "w".repeat(80)), mkMsg("m3", "view " + "q".repeat(80)), mkMsg("m4", "view " + "e".repeat(80))]; + assert.ok(await backend.ingestSession(planIngest(mkSession(vwSid, vwMsgs))), "view gen-1 ok"); + const vwAll = await backend.getMessages(vwSid); + const vwMaxSeq = vwAll[vwAll.length - 1]!.seq; + const vwSuffix = vwAll.slice(-2).map((m) => m.seq); + const vwBoundary = vwAll[vwAll.length - 1]!; + const vwPlan = planIngestDelta(mkSession(vwSid, vwMsgs.slice(-2)), vwMaxSeq, vwSuffix, { msgId: vwBoundary.msgId, contentHash: vwBoundary.contentHash }, true); + assert.ok(vwPlan, "empty delta accepted for the shrunken raw tail"); + assert.ok(await backend.ingestSession(vwPlan, { trustedSeq: vwMaxSeq, boundaryMsgId: vwBoundary.msgId, boundaryHash: vwBoundary.contentHash, trustedTailSeqs: vwSuffix }), "view gen-2 ok"); + const vwView = await backend.sessionViewRows(vwSid); + assert.ok(vwView, "view bundle present"); + assert.deepEqual(vwView!.messages.map((m) => m.seq), vwSuffix, "only the pointer seqs were fetched"); + assert.equal(await backend.sessionViewRows(`no-such-${run}`), null, "null on session miss"); + await backend.deleteSession(vwSid); + // reap journal await backend.reapBlocks(sid, new Set(["b2"]), new Map([["b2", { one: null, full: { text: "orphaned", count: 0 } }]])); await backend.reapBlocks(sid, new Set(["b2"]), new Map(), ); diff --git a/tests/storage-engine-proofs.test.ts b/tests/storage-engine-proofs.test.ts index 118ca0bde7..d9eb3334b2 100644 --- a/tests/storage-engine-proofs.test.ts +++ b/tests/storage-engine-proofs.test.ts @@ -11,7 +11,11 @@ * render goldens, both with a mid-conversation restart * and with the legacy file DELETED at the restart (the * index alone carries the session — the migration's - * end-state promise). + * end-state promise). #4316 PR3/3 (380ef4dac): a + * full-mode cold load additionally re-homes citation + * numbers once (one-time ledger alignment) — comparisons + * run modulo that renumbering (citation-normalized + * digests), which is the ONLY permitted delta. * STALE REJECT — a multi-instance stale write that the guarded build * rejects must never reach index.db; the converged disk * record is what gets ingested. @@ -45,7 +49,7 @@ import { planIngest, loadLegacyView, toLegacyLike, type LegacySessionLike } from import { canonicalize, sha256 } from "./golden-canonical.ts"; import { driveWire, WIRES, type Wire } from "./golden-storage-drive.ts"; import { rmrf } from "./tmp-rm.ts"; -import { ENGPROOF_CHECKS, spawnCorruptIndexChild, viewDigest, viewStateDigest } from "./corrupt-index-child.ts"; +import { ENGPROOF_CHECKS, spawnCorruptIndexChild, stripOneTimeAlignment, viewDigest, viewStateDigest } from "./corrupt-index-child.ts"; process.env.BILI_PERSIST_ZSTD = "0"; // #3400: pin the legacy wire shape. This suite proves STORAGE restart @@ -145,11 +149,12 @@ function unifiedRows(store: UnifiedStore, id: string): ReturnType { + test(`storage engine: full-mode restart is invisible on the ${wire} wire (== legacy restart / committed goldens, modulo the #4316 one-time citation renumbering)`, { timeout: 240_000 }, async () => { const golden = JSON.parse(fs.readFileSync(path.join(RENDER_ROOT, `${wire}.json`), "utf8")) as { - turns: Array<{ i: number; sha: string }>; + turns: Array<{ i: number; sha: string; shaCiteNorm: string }>; foldAt: number; editedAt: number; summaryFrom: number; stateDigest: string; + stateDigestAligned: string; stateShape: Record; }; // The restart oracle: a legacy-only process restart. One caveat is @@ -162,16 +167,18 @@ for (const wire of WIRES) { const record = await driveWire(wire as Wire, `golden-${wire}`, { storageUnified: "full", reloadAfterEdit: leg }); assert.equal(record.turns.length, legacy.turns.length, `${wire}/${leg}: body count drifted vs legacy restart`); for (let i = 0; i < legacy.turns.length; i++) { - assert.equal(record.turns[i]!.sha, legacy.turns[i]!.sha, `${wire}/${leg}: outbound body #${i} digest drift vs legacy restart`); - assert.equal(record.turns[i]!.sha, golden.turns[i]!.sha, `${wire}/${leg}: outbound body #${i} digest drift vs committed goldens (restart must not change the wire)`); + // #4316 PR3/3 (380ef4dac): a full-mode cold load re-homes citation + // numbers once (one-time ledger alignment) — the ONLY permitted + // wire delta. Citation-normalized digests must match turn-for-turn. + assert.equal(record.turns[i]!.shaCiteNorm, legacy.turns[i]!.shaCiteNorm, `${wire}/${leg}: outbound body #${i} drifts beyond citation renumbering vs legacy restart`); + assert.equal(record.turns[i]!.shaCiteNorm, golden.turns[i]!.shaCiteNorm, `${wire}/${leg}: outbound body #${i} drifts beyond citation renumbering vs committed goldens (restart-inherent bs-field loss aside)`); } assert.equal(record.foldAt, legacy.foldAt, `${wire}/${leg}: fold landmark drifted`); assert.equal(record.editedAt, legacy.editedAt, `${wire}/${leg}: edit landmark drifted`); assert.equal(record.summaryFrom, legacy.summaryFrom, `${wire}/${leg}: summary carrier landmark drifted`); - assert.equal(record.stateDigest, legacy.stateDigest, `${wire}/${leg}: persisted state digest drift vs legacy restart`); + assert.equal(record.stateDigestAligned, legacy.stateDigestAligned, `${wire}/${leg}: persisted state drifts beyond the one-time ledger alignment vs legacy restart`); + assert.equal(record.stateDigestAligned, golden.stateDigestAligned, `${wire}/${leg}: persisted state drifts beyond the one-time ledger alignment vs committed goldens`); assert.deepEqual(record.stateShape, legacy.stateShape, `${wire}/${leg}: state shape drift`); - assert.deepEqual(record.sectionDigests, legacy.sectionDigests, `${wire}/${leg}: state section digests drift`); - assert.deepEqual(record.stateKeyDigests, legacy.stateKeyDigests, `${wire}/${leg}: state key digests drift`); } }); } @@ -293,20 +300,25 @@ test("storage engine: rollback contract — unset the env and the legacy read pa // (The load-path digest is the stable oracle: an in-memory session vs // a loaded one differs benignly by the storageMode stamp + one-time // tail normalization, identically under both paths.) + // #3818 flip: absent now resolves to FULL, so the legacy read path has + // to be spelled out explicitly (the knob's one-line escape hatch) — + // merely deleting the env would read unified and vacate this leg. const storeFull = new SessionStore({ dir, debounceMs: 1, enabled: true }); const viaFull = storeFull.loadSync("corpus-folded", { protocol: "anthropic", upstreamOrigin: "http://127.0.0.1:8199" }); storeFull.closeQuietly(); assert.ok(viaFull, "full-mode reload returned null"); const digestFull = viewDigest(toLegacyLike(viaFull)); - delete process.env.BILI_STORAGE_UNIFIED; + process.env.BILI_STORAGE_UNIFIED = "legacy"; const store2 = new SessionStore({ dir, debounceMs: 1, enabled: true }); const legacy = store2.loadSync("corpus-folded", { protocol: "anthropic", upstreamOrigin: "http://127.0.0.1:8199" }); store2.closeQuietly(); assert.ok(legacy, "legacy-only reload returned null after rollback"); // #2960 ⑤: cross-mode equality is on the STATE proper — full mode // serves the raw table tail, legacy the envelope's folded view. - assert.equal(viewStateDigest(toLegacyLike(legacy)), viewStateDigest(toLegacyLike(viaFull)), "rollback: legacy-only load state digest drift vs full-mode load"); + // #4316 PR3/3: modulo the one-time ledger alignment (full-side cold + // load re-homes citation numbers; legacy keeps counter numbers). + assert.equal(viewStateDigest(stripOneTimeAlignment(toLegacyLike(legacy))), viewStateDigest(stripOneTimeAlignment(toLegacyLike(viaFull))), "rollback: legacy-only load state digest drift vs full-mode load"); assert.ok(viaFull.lastMessages && viaFull.lastMessages.length > 0, "⑤: full-mode load serves the table tail"); assert.ok(legacy.lastMessages && legacy.lastMessages.length > 0, "⑤: rollback keeps the envelope tail"); const disk = findWrittenRecord(dir, "corpus-folded"); @@ -381,7 +393,10 @@ test("storage engine: corrupt index.db degrades to legacy-only instead of bricki // executed in the child, verified in the parent). The child also runs // the #2781 mid-flow rename guard plus a post-close guard, so future // handle-leak drift trips there by name instead of at teardown. - const stateDigestClean = viewStateDigest(toLegacyLike(viaFull)); + // #4316 PR3/3: expected digest computed modulo the one-time ledger + // alignment — the clean read above already carried it in memory, and + // the child's fresh load applies it again; both sides must normalize. + const stateDigestClean = viewStateDigest(stripOneTimeAlignment(toLegacyLike(viaFull))); const childEnv: NodeJS.ProcessEnv = { ...process.env, BILI_STORAGE_UNIFIED: "full", @@ -509,5 +524,5 @@ test("storage engine: no SQLite engine -> SessionStore runs legacy-only, writes // Engine note: these legs run on the process-default engine; the corpus // suites (storage-unified.test.ts) cover BOTH engines at the state level, -// and the wire legs here assert against the same committed goldens the -// legacy baseline produced. +// and the wire legs here assert citation-normalized equality against the same +// committed goldens the legacy baseline produced. diff --git a/tests/storage-orphan-recovery.test.ts b/tests/storage-orphan-recovery.test.ts index c63f3f63fb..9fd6f13b0a 100644 --- a/tests/storage-orphan-recovery.test.ts +++ b/tests/storage-orphan-recovery.test.ts @@ -69,7 +69,12 @@ test("cold load recovers the pre-fix orphan freeze before processing a request", assert.equal(recovered.stats.requests, 876); assert.equal(recovered.state.nextBlockId, 4); assert.deepEqual(recovered.state.blocks.map(b => b.blockId), ["b3"]); - assert.equal(recovered.state.messageRefs.byRef.m00003, "raw-new"); + // #4316 PR3/3: a full-mode cold load re-homes citation numbers once + // (ledger alignment) — pin the content-addressable round-trip, not the + // counter-era number, so the assertion holds under any numbering scheme. + const alignedRef = recovered.state.messageRefs.byRaw["raw-new"]; + assert.ok(alignedRef, "raw-new must keep a citation number after recovery"); + assert.equal(recovered.state.messageRefs.byRef[alignedRef], "raw-new"); assert.equal(recovered.lastMessages?.at(-1)?.text, "new turn"); assert.equal(recovered.metadata.recoveryMarker, "fresh"); assert.equal(recovered.blockContents.get("b1")?.full.text, "retained original"); diff --git a/tests/web-blogfeed.test.ts b/tests/web-blogfeed.test.ts index 0d1d820907..46204193a9 100644 --- a/tests/web-blogfeed.test.ts +++ b/tests/web-blogfeed.test.ts @@ -144,6 +144,18 @@ test("a root without content degrades to an empty list instead of throwing", () assert.deepEqual(getBlogFeed(), { items: [] }); }); +test("CRLF frontmatter (Windows autocrlf checkout) still yields title/excerpt/date", () => { + const posts = join(pkgRoot, "website", "blog", "posts"); + writeFileSync(join(posts, "2026-10-08_crlf.md"), "---\r\ntitle: \"CRLF post\"\r\nexcerpt: \"EN excerpt.\"\r\n---\r\n\r\n## Body\r\n\r\nText.\r\n"); + _setBlogRootForTest(pkgRoot); + const data = getBlogFeed(); + const crlf = data.items.find((i) => i.id === "crlf"); + assert.ok(crlf, "crlf post listed"); + assert.equal(crlf.title, "CRLF post"); + assert.equal(crlf.excerpt, "EN excerpt."); + assert.equal(crlf.date, "2026-10-08"); +}); + test("default probing finds the repository tree from the TS source layout", () => { _setBlogRootForTest(null); const data = getBlogFeed();