diff --git a/src/fork-adoption.ts b/src/fork-adoption.ts index 8bc295b144..3e005817a3 100644 --- a/src/fork-adoption.ts +++ b/src/fork-adoption.ts @@ -342,7 +342,9 @@ function persistForkLineageNow(session: Session, log: (level: string, msg: strin * for life. The link copies nothing at link time; adoption still judges per * block in raw-id set semantics, so a mis-guessed parent can at most be * searched read-only, never adopted from. */ -export function maybeAdoptForkBlocks(args: { +// #4942: async — the parent load below must reach the DB on network storage +// backends, or an anonymous fork of a DB-only parent starts fresh. +export async function maybeAdoptForkBlocks(args: { session: Session; parentId: string; protocol: WireProtocol; @@ -350,7 +352,7 @@ export function maybeAdoptForkBlocks(args: { upstreamOrigin: string; enabled: boolean; log: (level: string, msg: string) => void; -}): void { +}): Promise { const { session, parentId, protocol, parsed, upstreamOrigin, enabled, log } = args; const incomingMsgs = incomingCoreMessages(protocol, parsed); if (!incomingMsgs) { @@ -358,7 +360,7 @@ export function maybeAdoptForkBlocks(args: { return; } let parent = peekSession(parentId); - if (!parent) parent = getStore().loadSync(parentId, { protocol, upstreamOrigin }) ?? undefined; + if (!parent) parent = (await getStore().loadAsync(parentId, { protocol, upstreamOrigin })) ?? undefined; if (!parent) { log("info", `[fork-adoption] fork parent ${parentId} not found (evicted and not persisted); starting fresh (#629)`); return; @@ -516,10 +518,12 @@ export function maybeAdoptDerivedFork(args: { * (the parent id came from the plugin register) and belongs with the * byte-prefix strong path, which is on by default. */ -export function reconcileDerivedLineage( +// #4942: async — parent loads below must reach the DB on network storage +// backends, or lineage of DB-only parents is never restored at boot. +export async function reconcileDerivedLineage( sessions: Session[], opts: { log: (level: string, msg: string) => void; adoptionEnabled: boolean }, -): number { +): Promise { const { log, adoptionEnabled } = opts; if (!adoptionEnabled) return 0; let adopted = 0; @@ -534,10 +538,10 @@ export function reconcileDerivedLineage( let parent = peekSession(parentId); if (!parent) { - parent = getStore().loadSync(parentId, { + parent = (await getStore().loadAsync(parentId, { protocol: s.meta.protocol, upstreamOrigin: s.meta.upstreamOrigin, - }) ?? undefined; + })) ?? undefined; } if (!parent) { // Evicted before it was persisted — or restored later. Leave the diff --git a/src/persist.ts b/src/persist.ts index 5aa2e6f80f..951c1484e2 100644 --- a/src/persist.ts +++ b/src/persist.ts @@ -440,7 +440,13 @@ export class SessionStore { * failed to — the sidecar just stays undefined). Resolved immediately * for sqlite/legacy boots. */ private readonly mysqlReady: Promise = Promise.resolve(); - /** Await the storage backend before first use (mysql pool + DDL). */ + /** #4942: true when a network backend (mysql) was selected for + * unified-full mode — the sidecar attaches asynchronously, so before + * ready() resolves this.shadow is still undefined while the DB already + * carries rows. */ + private readonly networkFullExpected: boolean = false; + /** Await the storage backend before first use (mysql pool + DDL). openAsync + * never rejects (it degrades to null), so awaiting this cannot throw. */ ready(): Promise { return this.mysqlReady; } @@ -484,7 +490,7 @@ export class SessionStore { private readonly capWarnAt = new Map(); private readonly debounceMs: number; - constructor(opts?: { dir?: string; debounceMs?: number; enabled?: boolean; log?: Logger }) { + constructor(opts?: { dir?: string; debounceMs?: number; enabled?: boolean; log?: Logger; shadow?: UnifiedShadow }) { const debounceMs = opts?.debounceMs ?? defaultDebounce(); this.enabled = (opts?.enabled ?? true) && debounceMs >= 0; this.debounceMs = Math.max(0, debounceMs); @@ -541,17 +547,18 @@ export class SessionStore { if (wantJsonWriterOff && !this.jsonWriterOff) { baseLog("warn", "[storage] storage.jsonWriter=off ignored — requires storage.unified=full on the sqlite backend (network backends keep the JSON envelope as the sync read path)"); } - const legacyLoad = (rec: unknown): LegacySessionLike | null => { - const s = buildSession(rec as PersistedSession); - return toLegacyLike(s); - }; + const legacyLoad = recordToLegacyView; // #3931 step 3: network backends (mysql) can only open asynchronously // (pool + DDL), so the sidecar attaches when it resolves — until then // this.shadow stays undefined and every surface keeps its documented // no-sidecar posture (envelope reads/writes, banner, replicas off). // `ready()` gates the async consumers (saves, boot, admin). const backendName = resolveBackendName(process.env, fileStorageBackend()); - if (this.enabled && !unifiedBlockedByEncryption && backendName === "mysql") { + if (opts?.shadow !== undefined) { + // #4942 test seam: a pre-built sidecar skips the env-driven open. + this.shadow = opts.shadow; + this.networkFullExpected = !!(opts.shadow.readsUnified && opts.shadow.isAsyncBackend); + } else if (this.enabled && !unifiedBlockedByEncryption && backendName === "mysql") { this.mysqlReady = UnifiedShadow.openAsync( this.sessionsDir, process.env, @@ -564,6 +571,7 @@ export class SessionStore { this.shadow = sh ?? undefined; }); this.shadow = undefined; + this.networkFullExpected = storageMode === "full"; } else { if (this.enabled && !unifiedBlockedByEncryption) { // #5188: remember the open args when this sync attempt actually @@ -765,11 +773,14 @@ export class SessionStore { // #2960 R2.1 boot union (full mode): rows whose legacy file is gone // (deleted, or written by a Phase 3 future that stops writing JSON) // remain enumerable — the index is self-sufficient. Shadow stays - // file-walk: there JSON is the truth base. - if (this.shadow?.readsUnified) { - for (const id of this.shadow.listSessionIds()) { + // file-walk: there JSON is the truth base. #4942: on network backends + // the sidecar attaches asynchronously and the sync facade misses by + // design — enumerate through the async twins instead. + if (this.shadow?.readsUnified || this.networkFullExpected) { + if (this.networkFullExpected) await this.ready(); + for (const id of await this.unionIds()) { if (out.has(id)) continue; - const session = this.loadSync(id); + const session = await this.loadAsync(id); if (session) out.set(id, session); } } @@ -798,11 +809,13 @@ export class SessionStore { // without a legacy file stay enumerable (the index is // self-sufficient). bootBounded (limits path) intentionally stays // file-based: bounded boot is the cold-start triage path for huge - // legacy trees. - if (this.shadow?.readsUnified) { - for (const id of this.shadow.listSessionIds()) { + // legacy trees. #4942: network backends enumerate through the async + // twins (the sync facade misses by design there). + if (this.shadow?.readsUnified || this.networkFullExpected) { + if (this.networkFullExpected) await this.ready(); + for (const id of await this.unionIds()) { if (out.has(id)) continue; - const session = this.loadSync(id); + const session = await this.loadAsync(id); if (session) out.set(id, session); } } @@ -821,6 +834,21 @@ export class SessionStore { return false; } + /** #4942: enumeration for the boot/loadAll unions. sqlite keeps the sync + * listSessionIds(); network backends enumerate through sessionSummaries — + * one query carries both the ids and updatedAt, so a budget-bounded + * pre-warm can order newest-first (matching the savedAt-desc arbitration + * of the file path) instead of alphabetical row order. Both sources + * return the SAME row set (neither filters archived tombstones). */ + private async unionIds(): Promise { + const shadow = this.shadow; + if (!shadow) return []; + if (!shadow.isAsyncBackend) return shadow.listSessionIds(); + const rows = await shadow.sessionSummariesAsync(); + rows.sort((a, b) => b.updatedAt - a.updatedAt); + return rows.map((r) => r.sessionId); + } + private async bootBounded(limits: { maxSessions: number; maxBytes: number }): Promise> { if (!Number.isFinite(limits.maxSessions) || limits.maxSessions < 0 || !Number.isFinite(limits.maxBytes) || limits.maxBytes < 0) { throw new Error("Invalid session boot limits"); @@ -936,15 +964,19 @@ export class SessionStore { // The pure-index ids that do enter are measured by their SERIALIZED // record — the same proxy readRawFileWithBytes gives the file path // (#2803: the budget is a RAM proxy, not a disk one). - if (this.shadow?.readsUnified) { + // #4942: network backends pre-warm through the async twins under the + // SAME budget (the sync facade misses by design there); newest-first + // ordering comes from unionIds. + if (this.shadow?.readsUnified || this.networkFullExpected) { + if (this.networkFullExpected) await this.ready(); let warmBytes = 0; for (const item of warm) warmBytes += item.bytes; - for (const id of this.shadow.listSessionIds()) { + for (const id of await this.unionIds()) { if (out.size >= limits.maxSessions) break; if (warmBytes >= limits.maxBytes) break; if (out.has(id)) continue; if (this.stateFileOnDisk(id)) continue; - const session = this.loadSync(id); + const session = await this.loadAsync(id); if (!session) continue; const bytes = this.estimateDecodedBytes(session); if (warmBytes + bytes > limits.maxBytes) continue; @@ -1665,6 +1697,28 @@ export class SessionStore { return session; } + /** #4942: async reload seam for network backends — on mysql the sync + * facade has NO read path (viewFor/listSessionIds miss by design, #3931 + * step 3), so a cold load with no local envelope used to return null and + * the request path silently started a fresh session, losing every folded + * block at each restart. Serves exactly what loadSync serves whenever + * loadSync finds something (zero behavior change); only when the sync + * path has no copy does it fall to the DB row via viewForAsync. + * Envelope-first precedence is deliberate: envelope writes are + * synchronous per save while DB writes ride the coalesced record queue, + * so an existing envelope is never older than the DB in single-writer + * deployments. Null when neither copy exists. */ + async loadAsync(id: string, meta?: { protocol?: string; upstreamOrigin?: string }): Promise { + const sync = this.loadSync(id, meta); + if (sync) return sync; + const shadow = this.shadow; + if (!shadow || !shadow.readsUnified || !shadow.isAsyncBackend) return null; + await this.ready(); + const view = await shadow.viewForAsync(id); + if (!view) return null; + return this.afterLoad(id, view); + } + /** Shared post-load fixups for the unified read path: rebuilds the * Session from a store view (viewToRecord → buildSession, the same * forward-compat pass as a legacy load) and applies the #408 clamp @@ -3473,6 +3527,14 @@ function restoreBlockContents(entries: Iterable): Ma return blockContents; } +/** #4942: the record→legacy-view adapter the sidecar ingest path runs on + * every committed record — exported so the hermetic seam suite can build a + * sidecar against a stub backend with the SAME adapter the store uses. */ +export function recordToLegacyView(record: unknown): LegacySessionLike | null { + const s = buildSession(record as PersistedSession); + return toLegacyLike(s); +} + function buildSession(parsed: PersistedSession): Session { const blockContents = restoreBlockContents(Object.entries(parsed.blockContents ?? {})); // Read grouped shape (v2+); fall back to flat fields for v1 files. diff --git a/src/plugin.ts b/src/plugin.ts index 2feb2e5c31..7168d33e84 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -8,7 +8,7 @@ import path from "node:path"; import { createHash } from "node:crypto"; import { getStore, storageStatusLabel } from "./persist.js"; import { cloneStoreForRefs } from "./store.js"; -import { acquireInFlight, createSession, getSession, publishForkSession, diagnoseSuccessWithoutUsage, effectiveConfig, findSessionByCanonicalId, listSessions, markCompactionBoundary, markDirty, peekSession, releaseInFlight, statusInputBaseline, withSessionLock, type Session } from "./session.js"; +import { acquireInFlight, createSession, getSessionAsync, publishForkSession, diagnoseSuccessWithoutUsage, effectiveConfig, findSessionByCanonicalId, listSessions, markCompactionBoundary, markDirty, peekSession, releaseInFlight, statusInputBaseline, withSessionLock, type Session } from "./session.js"; import { clientConversationHeader } from "./session-id.js"; import { buildContextView } from "./context-view.js"; import { ABSORB_TOOL_NAME, BILI_ACP_TOOLS_ANTHROPIC, BILI_ACP_TOOLS_ANTHROPIC_NO_RANGE, BILI_ACP_TOOLS_OPENAI, BILI_ACP_TOOLS_OPENAI_NO_RANGE, BILI_ACP_TOOLS_RESPONSES_NO_RANGE, PROXY_TOOL_NAMES, READ_ORIGINAL_IMAGE_TOOL, READ_ORIGINAL_IMAGE_TOOL_NAME, READ_ORIGINAL_IMAGE_TOOL_OPENAI, READ_ORIGINAL_IMAGE_TOOL_RESPONSES, RETRIEVE_TOOL_NAME, RULE_TOOL, RULE_TOOL_NAME, RULE_TOOL_OPENAI, RULE_TOOL_RESPONSES, SEARCH_CONTEXT_TOOL_NAME, absorbToolsFor, retrieveToolsFor, withAcpNotes, withAcpNotesOpenAI } from "./compress-tool.js"; @@ -1310,14 +1310,19 @@ export function forkParentRevisionLegacy(session: Session): string { return forkHash({ sessionId: session.id, messages, state, blockContents: Object.fromEntries(session.blockContents), contentStore: store }); } -function resolveForkConversation(id: string): Session | undefined { +// #4942: async — on network storage backends the existence probes must be +// able to reach the DB (the sync facade misses by design there). Materialization +// must ride the SAME async seam: a sync getSession after an async probe would +// mint an EMPTY resident ghost for a DB-only id, and that ghost then poisons +// every later getSessionAsync of the id for the process lifetime. +async function resolveForkConversation(id: string): Promise { if (peekSession(id)) return peekSession(id); - if (getStore().loadSync(id)) return getSession(id); + if (await getStore().loadAsync(id)) return getSessionAsync(id); const { session, entry } = resolveConversation(id); if (session) return session; const persistedId = entry?.sessionId ?? id; - if (!getStore().loadSync(persistedId)) return undefined; - return getSession(persistedId); + if (!(await getStore().loadAsync(persistedId))) return undefined; + return getSessionAsync(persistedId); } /** #2601 freshness window for persona-lineage following — same family as @@ -1448,7 +1453,7 @@ function forkReply(res: ServerResponse, status: number, body: unknown): void { export async function handlePluginSnapshot(conversationId: string, res: ServerResponse): Promise { if (!conversationId) return forkReply(res, 400, { ok: false, code: "INVALID_REQUEST", error: "conversationId is required" }); - const session = resolveForkConversation(conversationId); + const session = await resolveForkConversation(conversationId); if (!session) return forkReply(res, 404, { ok: false, code: "PARENT_NOT_FOUND", error: "unknown plugin conversation" }); acquireInFlight(session); try { @@ -1502,7 +1507,7 @@ export async function handlePluginFork(payload: string, res: ServerResponse): Pr const requestHash = forkHash(request); const replay = async (): Promise => { await getStore().recoverCompletedFork(request.childConversationId); - const existing = resolveForkConversation(request.childConversationId); + const existing = await resolveForkConversation(request.childConversationId); if (!existing) return false; const receipt = existing.metadata.publicForkReceipt as { requestHash?: unknown; response?: unknown } | undefined; if (receipt?.requestHash === requestHash) forkReply(res, 200, { ...(receipt.response as Record), replayed: true }); @@ -1512,7 +1517,7 @@ export async function handlePluginFork(payload: string, res: ServerResponse): Pr try { if (await replay()) return; } catch (err) { return forkReply(res, 503, { ok: false, status: "unavailable", code: "FORK_FAILED", error: String(err) }); } if (conversations.has(request.childConversationId) || pendingRegisters.some((r) => r.conversationId === request.childConversationId) || (registeredIds.has(request.childConversationId) && !forkChildHeldBySameParent(request.childConversationId, request.parentConversationId))) return forkReply(res, 409, { ok: false, code: "CHILD_CONFLICT", error: "child conversation already registered" }); - const parent = resolveForkConversation(request.parentConversationId); + const parent = await resolveForkConversation(request.parentConversationId); if (!parent) return forkReply(res, 404, { ok: false, code: "PARENT_NOT_FOUND", error: "parent conversation not found" }); acquireInFlight(parent); try { @@ -1747,7 +1752,9 @@ function webSessionUrl(origin: string | undefined, sessionId: string): string | return `${o}/__bili/#/session/${encodeURIComponent(sessionId)}`; } -export function handlePluginStatus(conversationId: string, res: import("node:http").ServerResponse, deps: PluginToolDeps, fallbackLatest = false, trustedOrigin?: string): void { +// #4942: async — the session resolution below must reach the DB on network +// storage backends (the caller, handleAdminRoute, is already async). +export async function handlePluginStatus(conversationId: string, res: import("node:http").ServerResponse, deps: PluginToolDeps, fallbackLatest = false, trustedOrigin?: string): Promise { // #3862: the dsh client's Context tab reads this route CROSS-ORIGIN from // the dsh web app (its own loopback port / the dsh-app://app scheme). The // trusted-origin gate upstream of dispatch already admitted the request; @@ -1758,7 +1765,7 @@ export function handlePluginStatus(conversationId: string, res: import("node:htt if (trustedOrigin !== undefined && trustedOrigin.length > 0) h["access-control-allow-origin"] = trustedOrigin; return h; }; - let session = resolveForkConversation(conversationId); + let session = await resolveForkConversation(conversationId); const entry = conversations.get(conversationId); let viaFallback = false; let resolvedConversationId = conversationId; @@ -2034,7 +2041,7 @@ export async function handlePluginTool( routedBy = "native"; deps.log("info", `[plugin] tool "${tool}": routed by host call id (#4580) to "${session.id}"`); } else if (conversationId) { - session = resolveForkConversation(conversationId); + session = await resolveForkConversation(conversationId); entry = conversations.get(conversationId); if (session && witnessIds.size > 0 && !witnessIds.has(session.id)) { // #2601: a persona-lineage witness is not a spoof — the host stamps @@ -2135,7 +2142,7 @@ export async function handlePluginTool( } } if (!session) { - session = resolveForkConversation(conversationId); + session = await resolveForkConversation(conversationId); entry = conversations.get(conversationId); } // #760: the verbatim-id fallback above can resolve a session with NO map diff --git a/src/server.ts b/src/server.ts index 5270ab1223..84dc0014d3 100644 --- a/src/server.ts +++ b/src/server.ts @@ -567,7 +567,7 @@ export async function createEngineContext(opts: ProxyOptions): Promise>(); + +function stampExisting(existing: Session, meta?: SessionMeta): void { + existing.lastSeen = Date.now(); + existing.restored = false; + // Fill in protocol/upstream/label meta on an existing session if the caller + // now knows it (e.g. a session was created by loadAll without meta). + if (meta?.protocol && !existing.meta.protocol) existing.meta.protocol = meta.protocol; + if (meta?.upstreamOrigin && !existing.meta.upstreamOrigin) existing.meta.upstreamOrigin = meta.upstreamOrigin; + if (meta?.label && !existing.meta.label) existing.meta.label = meta.label; +} + +/** Shared tail of getSession / getSessionAsync: adopt the reloaded session + * into the resident pool, or create a fresh one under the cap guard. */ +function adoptOrCreate(id: string, meta: SessionMeta | undefined, reloaded: Session | null): Session { if (reloaded) { // A memory-miss reload is triggered by a real request: stamp fresh // activity, not the restored-from-disk state (#404). @@ -1190,6 +1193,46 @@ export function getSession(id: string, meta?: { protocol?: Session["meta"]["prot return session; } +export function getSession(id: string, meta?: SessionMeta): Session { + if (publishingForks.has(id)) throw new Error("child fork publication in progress"); + const existing = sessions.get(id); + if (existing) { + stampExisting(existing, meta); + return existing; + } + // Memory miss: try reload from disk (e.g. after LRU eviction). + const store = getStore(); + makeResidentRoom(); + return adoptOrCreate(id, meta, store.loadSync(id, meta)); +} + +/** #4942: async twin of getSession for the request path — on network storage + * backends (mysql) the sync facade cannot read session state back, so a + * memory miss must be able to fall to the DB row (store.loadAsync) instead + * of silently starting a fresh session and losing every folded block at each + * restart. On sqlite this is loadSync plus a microtask: identical behavior. + * Concurrent first-touches of the same id share ONE load (single-flight). */ +export async function getSessionAsync(id: string, meta?: SessionMeta): Promise { + // #5044 publication race ported to the async seam: a request landing on a + // child id while its fork is mid-publication must fail loudly instead of + // adopting a fresh empty session that would wedge the publication. + if (publishingForks.has(id)) throw new Error("child fork publication in progress"); + const existing = sessions.get(id); + if (existing) { + stampExisting(existing, meta); + return existing; + } + const store = getStore(); + makeResidentRoom(); + let pending = pendingSessionLoads.get(id); + if (!pending) { + pending = (async () => adoptOrCreate(id, meta, await store.loadAsync(id, meta)))() + .finally(() => { pendingSessionLoads.delete(id); }); + pendingSessionLoads.set(id, pending); + } + return pending; +} + export function createSession(id: string, meta?: Session["meta"]): Session { return { id, @@ -1210,9 +1253,11 @@ export function createSession(id: string, meta?: Session["meta"]): Session { const publishingForks = new Set(); +// #4942: async — the existence probe must reach the DB on network storage +// backends, or a fork could silently clobber a DB-only child row. export async function publishForkSession(session: Session, parentId: string): Promise { if (publishingForks.has(session.id)) throw new Error("child fork publication in progress"); - if (sessions.has(session.id) || getStore().loadSync(session.id)) throw new Error("child session already exists"); + if (sessions.has(session.id) || (await getStore().loadAsync(session.id))) throw new Error("child session already exists"); publishingForks.add(session.id); try { makeResidentRoom(sessionWeight(session)); diff --git a/src/storage/shadow.ts b/src/storage/shadow.ts index db9efa522b..71e78ef2a8 100644 --- a/src/storage/shadow.ts +++ b/src/storage/shadow.ts @@ -386,6 +386,13 @@ export class UnifiedShadow { } } + /** #4942 test seam: build a shadow directly on an async backend without + * env-driven open or sidecar-instance registration (hermetic suites + * stub the StorageBackend contract; see tests/storage-mysql-seam.test.ts). */ + static fromBackend(mode: StorageMode, backend: StorageBackend, legacyLoad?: (record: unknown) => LegacySessionLike | null): UnifiedShadow { + return new UnifiedShadow(mode, null, legacyLoad ?? (() => null), backend); + } + /** True when reads should go unified-first (full mode, sidecar healthy). */ get readsUnified(): boolean { return this.mode === "full" && !this.stats.disabled; diff --git a/src/web/sessions-data.ts b/src/web/sessions-data.ts index ca32ab84be..6f916eccb0 100644 --- a/src/web/sessions-data.ts +++ b/src/web/sessions-data.ts @@ -1294,7 +1294,9 @@ const detailInFlight = new Map>(); async function loadSessionById(id: string): Promise<{ session: Session; live: boolean } | null> { const live = listSessions().find((s) => s.id === id); if (live) return { session: live, live: true }; - const synced = getStore().loadSync(id); + // #4942: async seam — DB-only sessions are invisible to the sync facade + // (rides the persist process singleton per #4990). + const synced = await getStore().loadAsync(id); if (synced) return { session: synced, live: false }; const files = await ensureIndex(); const abs = byId?.get(id); diff --git a/tests/advisory-panel.test.ts b/tests/advisory-panel.test.ts index 5192be407d..04da836ef8 100644 --- a/tests/advisory-panel.test.ts +++ b/tests/advisory-panel.test.ts @@ -22,7 +22,7 @@ function mockRes(): { res: http.ServerResponse; status: number; body: string } { return out; } -test("#1577: /acp panel surfaces an active advisory before the footer (stripper-safe), byte-exact when clean", () => { +test("#1577: /acp panel surfaces an active advisory before the footer (stripper-safe), byte-exact when clean", async () => { _resetSessionsForTest(); _resetAdvisoryWatcherForTest(); try { @@ -30,7 +30,7 @@ test("#1577: /acp panel surfaces an active advisory before the footer (stripper- const deps = { core: createCore(), config: defaultConfig(200000), log: (_l: string, _m: string) => {} }; const r0 = mockRes(); - handlePluginStatus("never-seen", r0.res, deps, true); + await handlePluginStatus("never-seen", r0.res, deps, true); assert.equal(r0.status, 200); const p0 = JSON.parse(r0.body).panel as string | undefined; assert.ok(p0, "panel renders for the live session"); @@ -39,7 +39,7 @@ test("#1577: /acp panel surfaces an active advisory before the footer (stripper- _setAdvisoryStateForTest({ active: { id: "bc-2026-001", affected: ">=1.0.0", target: "0.1.157", reason: "corrupts tool-call arguments", currentVersion: "0.1.156" } }); const r1 = mockRes(); - handlePluginStatus("never-seen", r1.res, deps, true); + await handlePluginStatus("never-seen", r1.res, deps, true); assert.equal(r1.status, 200); const p1 = JSON.parse(r1.body).panel as string; assert.ok(p1.includes("⚠️ CRITICAL ADVISORY:"), "advisory line present"); @@ -53,7 +53,7 @@ test("#1577: /acp panel surfaces an active advisory before the footer (stripper- const dollarReason = "costs $5M; echo $& back"; _setAdvisoryStateForTest({ active: { id: "bc-2026-002", affected: ">=1.0.0", target: "1.0.9", reason: dollarReason, currentVersion: "1.0.1" } }); const r2 = mockRes(); - handlePluginStatus("never-seen", r2.res, deps, true); + await handlePluginStatus("never-seen", r2.res, deps, true); const p2 = JSON.parse(r2.body).panel as string; assert.ok(p2.includes(dollarReason), "dollar-sign reason survives byte-exact"); assert.equal((p2.match(new RegExp(PANEL_BOX_FOOTER.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"), "g")) ?? []).length, 1, "footer appears exactly once"); diff --git a/tests/fork-ccr-adoption.test.ts b/tests/fork-ccr-adoption.test.ts index 66617304f8..65ef3dfee7 100644 --- a/tests/fork-ccr-adoption.test.ts +++ b/tests/fork-ccr-adoption.test.ts @@ -98,9 +98,9 @@ function armParent(body: Body, fold: { startId: string; endId: string }, ccr: bo return { parent, core, config, blockId: block.blockId, turnMsgs: turn.messages }; } -function forkChild(a: Armed, forkBody: Body): Session { +async function forkChild(a: Armed, forkBody: Body): Promise { const child = getSession(sid("child"), META); - maybeAdoptForkBlocks({ + await maybeAdoptForkBlocks({ session: child, parentId: a.parent.id, protocol: "openai", @@ -137,7 +137,7 @@ test("cloneStoreForRefs: ref-filtered independent clone", () => { assert.equal(cloneStoreForRefs(createContentStore(), []), null); }); -test("fork of armed parent resolves adopted covered refs via acp_retrieve (#1341)", () => { +test("fork of armed parent resolves adopted covered refs via acp_retrieve (#1341)", async () => { const body: Body = { model: "test-model", messages: [ @@ -160,7 +160,7 @@ test("fork of armed parent resolves adopted covered refs via acp_retrieve (#1341 const forkBody = structuredClone(body); forkBody.messages[7] = { role: "tool", tool_call_id: "call_b", content: `${BIG_B}\n(edited branch)` }; - const child = forkChild(armed, forkBody); + const child = await forkChild(armed, forkBody); assert.equal(child.state.blocks.length, 1); assert.ok(child.state.blocks[0].active); @@ -193,7 +193,7 @@ test("fork of armed parent resolves adopted covered refs via acp_retrieve (#1341 assert.equal(child.contentStoreDirty, true); }); -test("refs cited by placeholder-shaped incoming messages are adopted too (#1341)", () => { +test("refs cited by placeholder-shaped incoming messages are adopted too (#1341)", async () => { // Small tool output in the shared prefix (never placeholderized, so the // block stays adoptable); the big result sits OUTSIDE the fold span and // is stored at arrival time as m00007. @@ -241,7 +241,7 @@ test("refs cited by placeholder-shaped incoming messages are adopted too (#1341) { role: "assistant", content: "yes, fixed" }, ], }; - const child = forkChild(armed, forkBody); + const child = await forkChild(armed, forkBody); const cstore = contentStoreOf(child); assert.ok(cstore.byRef["m00007"], "cited ref adopted into child store"); @@ -282,7 +282,7 @@ test("refs cited by placeholder-shaped incoming messages are adopted too (#1341) assert.ok(reAck.includes(BIG_B.slice(0, 200)), "still retrievable in the tool result after the fold"); }); -test("child persists its own companion and survives a proxy restart (#1341)", () => { +test("child persists its own companion and survives a proxy restart (#1341)", async () => { const dir = mkdtempSync(path.join(tmpdir(), "bili-fork-ccr-")); _setStoreForTest(new SessionStore({ dir, debounceMs: 0 })); try { @@ -300,7 +300,7 @@ test("child persists its own companion and survives a proxy restart (#1341)", () const armed = armParent(body, { startId: "m00001", endId: "m00005" }, true); const forkBody = structuredClone(body); forkBody.messages[7] = { role: "tool", tool_call_id: "call_b", content: `${BIG_B}\n(edited branch)` }; - const child = forkChild(armed, forkBody); + const child = await forkChild(armed, forkBody); assert.equal(contentStoreOf(child).byRef["m00003"] ? true : false, true); assert.ok(getStore().flushSync(child), "flush wrote the child's records"); @@ -321,7 +321,7 @@ test("child persists its own companion and survives a proxy restart (#1341)", () } }); -test("fork of a CCR-less parent is unchanged: no store, no artifact (#1341)", () => { +test("fork of a CCR-less parent is unchanged: no store, no artifact (#1341)", async () => { const body: Body = { model: "test-model", messages: [ @@ -336,14 +336,14 @@ test("fork of a CCR-less parent is unchanged: no store, no artifact (#1341)", () }; const armed = armParent(body, { startId: "m00001", endId: "m00005" }, false); assert.equal(Object.keys(contentStoreOf(armed.parent).byRef).length, 0, "CCR off: nothing stored"); - const child = forkChild(armed, structuredClone(body)); + const child = await forkChild(armed, structuredClone(body)); assert.equal(child.state.blocks.length, 1, "block adoption itself still happens"); assert.equal(child.state.blocks[0].blockId, armed.blockId); assert.equal(child.contentStore, undefined, "no store materialized on the child"); assert.notEqual(child.contentStoreDirty, true, "no dirty flag, hence no envelope write"); }); -test("adoption disabled leaves the fork without blocks or store (#1341)", () => { +test("adoption disabled leaves the fork without blocks or store (#1341)", async () => { const body: Body = { model: "test-model", messages: [ @@ -357,7 +357,7 @@ test("adoption disabled leaves the fork without blocks or store (#1341)", () => }; const armed = armParent(body, { startId: "m00001", endId: "m00005" }, true); const child = getSession(sid("child"), META); - maybeAdoptForkBlocks({ + await maybeAdoptForkBlocks({ session: child, parentId: armed.parent.id, protocol: "openai", diff --git a/tests/issue2029-status-baseline-provenance.test.ts b/tests/issue2029-status-baseline-provenance.test.ts index 975bb559f2..3dbbb7c6de 100644 --- a/tests/issue2029-status-baseline-provenance.test.ts +++ b/tests/issue2029-status-baseline-provenance.test.ts @@ -134,11 +134,12 @@ function mockRes(): { res: http.ServerResponse; status: number; body: string } { return out; } -test("#2029: plugin status nudge + panel feed the provenance-aware baseline", () => { +test("#2029: plugin status nudge + panel feed the provenance-aware baseline", async () => { _resetSessionsForTest(); _resetPluginStateForTest(); try { - SCENARIOS.forEach((sc, i) => { + for (let i = 0; i < SCENARIOS.length; i++) { + const sc = SCENARIOS[i]; const session = getSession(`issue2029-plugin-${i}`, { protocol: "anthropic" }); session.metadata.effectiveContextLimit = WINDOW; session.stats = { ...zeroStats(), ...sc.stats } as Session["stats"]; @@ -150,7 +151,7 @@ test("#2029: plugin status nudge + panel feed the provenance-aware baseline", () const deps = { core: createCore(), config: defaultConfig(WINDOW), log: () => {} }; const seen = captureTokenCounts(deps.core); const r = mockRes(); - handlePluginStatus(session.id, r.res, deps); + await handlePluginStatus(session.id, r.res, deps); assert.equal(r.status, 200, `${sc.name}: status answered`); assert.deepStrictEqual(seen, [sc.expected], `${sc.name}: nudge recompute tokenCount`); const panel = JSON.parse(r.body).panel as string | undefined; @@ -168,7 +169,7 @@ test("#2029: plugin status nudge + panel feed the provenance-aware baseline", () assert.ok(!nudgeLine.includes("EMERGENCY") && !nudgeLine.includes("OVER-LIMIT"), `${sc.name}: no phantom pressure (got: ${nudgeLine})`); } - }); + } } finally { _resetSessionsForTest(); _resetPluginStateForTest(); diff --git a/tests/issue404-restore-timestamps.test.ts b/tests/issue404-restore-timestamps.test.ts index dc03a35aff..3e823e9099 100644 --- a/tests/issue404-restore-timestamps.test.ts +++ b/tests/issue404-restore-timestamps.test.ts @@ -119,7 +119,7 @@ await test("boot restore: lastSeen=savedAt, restored flag, freshness-keyed trunc assert.notEqual(active.restored, true, "a session created by a live request is not restored"); const statusDeps = { core: createCore(), config: defaultConfig(200000), log: (_l: string, _m: string) => {} }; const res1 = mockRes(); - handlePluginStatus("never-seen", res1.res, statusDeps, true); + await handlePluginStatus("never-seen", res1.res, statusDeps, true); const body1 = JSON.parse(res1.body) as { ok: boolean; fallback?: boolean; label: string | null }; assert.equal(res1.status, 200); assert.equal(body1.ok, true); @@ -129,7 +129,7 @@ await test("boot restore: lastSeen=savedAt, restored flag, freshness-keyed trunc // All sessions restored (no activity since boot): refuse to guess. _resetSessionsForTest(); const res2 = mockRes(); - handlePluginStatus("never-seen", res2.res, statusDeps, true); + await handlePluginStatus("never-seen", res2.res, statusDeps, true); assert.equal(res2.status, 404); assert.ok(res2.body.includes("since boot"), "explicit no-post-boot-activity error instead of a readdir-order guess"); } finally { diff --git a/tests/lineage-reconcile-boot.test.ts b/tests/lineage-reconcile-boot.test.ts index 7c5a1d637f..64e6bb123b 100644 --- a/tests/lineage-reconcile-boot.test.ts +++ b/tests/lineage-reconcile-boot.test.ts @@ -363,7 +363,7 @@ test("AC10: reconcile is idempotent across repeated boots", async () => { derivedBlocksAdopted: undefined, }, } as typeof child; - reconcileDerivedLineage([orphan], { log: (lvl, msg) => logs.push([lvl, msg]), adoptionEnabled: true }); + await reconcileDerivedLineage([orphan], { log: (lvl, msg) => logs.push([lvl, msg]), adoptionEnabled: true }); assert.equal(orphan.state.blocks.length, 0, "an unreachable parent must seed nothing"); assert.notEqual(orphan.metadata.derivedBlocksAdopted, true, "an unreachable parent must not close the door"); assert.ok( diff --git a/tests/plugin-fork.test.ts b/tests/plugin-fork.test.ts index 96d9dfeb0f..44db0d8a8d 100644 --- a/tests/plugin-fork.test.ts +++ b/tests/plugin-fork.test.ts @@ -1477,6 +1477,10 @@ test("HTTP a mid-history rewrite replaces the snapshot wholesale with an exact b }); test("HTTP a cold-restored non-retained parent serves its intact raw tail immediately (#5140)", { timeout: 60_000 }, async () => { + // Pin the mode this raw-tail proof requires: under the runner's legacy + // baseline no unified rows exist and forkSnapshot fails closed (#4599). + const oldMode = process.env.BILI_STORAGE_UNIFIED; + process.env.BILI_STORAGE_UNIFIED = "full"; const h = await harness(true); try { const parent = resolveConversation("parent").session!; @@ -1495,7 +1499,11 @@ test("HTTP a cold-restored non-retained parent serves its intact raw tail immedi await sendModel(h, "parent", h.messages); assert.equal((await h.request("/__bili/plugin/snapshot?conversationId=parent")).status, 200); assert.equal(resolveConversation("parent").session!.pluginSnapshot!.length, 3); - } finally { await h.close(); } + } finally { + await h.close(); + if (oldMode === undefined) delete process.env.BILI_STORAGE_UNIFIED; + else process.env.BILI_STORAGE_UNIFIED = oldMode; + } }); test("fork protocol v2: snapshot serves the light revision, a stale v1 whole-store revision is still accepted (#2671 Phase 2b)", { timeout: 60_000 }, async () => { diff --git a/tests/storage-mysql-seam.test.ts b/tests/storage-mysql-seam.test.ts new file mode 100644 index 0000000000..7abc01a72f --- /dev/null +++ b/tests/storage-mysql-seam.test.ts @@ -0,0 +1,557 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import { SessionStore, _setStoreForTest, recordToLegacyView } from "../src/persist.ts"; +import { _resetSessionsForTest, getSessionAsync, peekSession } from "../src/session.ts"; +import { handlePluginStatus } from "../src/plugin.ts"; +import type { Session } from "../src/session.ts"; +import { UnifiedShadow } from "../src/storage/shadow.ts"; +import type { + StorageBackend, + IngestSessionInput, + MessageRow, + BlockRow, + RefRow, + CcrEntryRow, + SessionSummaryRow, + StorageReadView, + SessionViewRows, +} from "../src/storage/backend.ts"; +import { sha256 } from "../src/storage/canonical.ts"; +import { viewPointerSeqs } from "../src/storage/store.ts"; + +/** + * #4942: the async reload seam on network storage backends — hermetic twin of + * tests/storage-backend-mysql-e2e.test.ts (which skips in CI without a DSN, + * which is exactly why #4936's writerOff breakage sailed through green CI). + * A typed stub stands in for the mysql2 backend so the seam's contract is + * pinned on every CI run: + * - loadAsync serves what loadSync serves whenever loadSync finds something + * (zero behavior change), and only falls to the DB row when the sync path + * has no copy (envelope-first precedence); + * - boot / bootBounded / loadAll enumerate DB-only ids through the async + * twins under the same budgets; + * - getSessionAsync single-flights concurrent first-touches of one id; + * - the jsonWriter=off gate stays sqlite-only on mysql (the #4936 gate). + */ + +process.env.BILI_PERSIST_ZSTD = "0"; +process.env.BILI_CONFIG_FILE = path.join( + fs.mkdtempSync(path.join(os.tmpdir(), "bili-mysql-seam-hermetic-")), + "no-config.json", +); +for (const k of ["BILI_STORAGE_BACKEND", "BILI_STORAGE_UNIFIED", "BILI_STORAGE_JSON_WRITER", "BILI_MYSQL_DSN"]) delete process.env[k]; + +const CORPUS_ROOT = path.resolve(import.meta.dirname, "golden/storage-corpus"); +const CORPUS_ID = "corpus-folded"; +const UPSTREAM_ORIGIN = "http://127.0.0.1:8199"; + +interface StubRows { + session: IngestSessionInput["session"]; + messages: MessageRow[]; + blocks: BlockRow[]; + deadRefs: IngestSessionInput["deadRefs"]; + refs: RefRow[]; + ccrEntries: CcrEntryRow[]; +} + +/** Map-shaped stand-in for MysqlBackend: stores exactly what ingestSession + * receives and hands it back through the read twins, so viewForAsync round- + * trips the same rows the save path produced (last write wins per session). */ +class StubMysqlBackend implements StorageBackend { + readonly rows = new Map(); + readonly viewCalls = { count: 0 }; + + ingestSession(input: IngestSessionInput): Promise { + this.rows.set(input.session.sessionId, { + session: input.session, + messages: input.messages, + blocks: input.blocks, + deadRefs: input.deadRefs, + refs: input.refs, + ccrEntries: input.ccrEntries, + }); + return Promise.resolve(true); + } + sessionCounters(sessionId: string): Promise<{ requests: number; nextBlockId: number } | null> { + const row = this.rows.get(sessionId)?.session; + if (!row) return Promise.resolve(null); + let requests = 0; + try { requests = (JSON.parse(row.statsJson) as { requests?: number }).requests ?? 0; } catch { /* malformed statsJson counts as zero */ } + return Promise.resolve({ requests, nextBlockId: row.nextBlockId }); + } + sessionUpdatedAt(sessionId: string): Promise { + return Promise.resolve(this.rows.get(sessionId)?.session.updatedAt ?? null); + } + deleteSession(sessionId: string): Promise { + this.rows.delete(sessionId); + return Promise.resolve(); + } + listSessionIds(): Promise { + return Promise.resolve([...this.rows.keys()]); + } + lookupConversation(_namespace: string, _conversationId: string): Promise { + return Promise.resolve(null); + } + listConversationBindings(): Promise<{ namespace: string; conversationId: string; sessionId: string; claimedAt: number }[]> { + return Promise.resolve([]); + } + getSessionRow(sessionId: string): Promise { + this.viewCalls.count++; + return Promise.resolve(this.rows.get(sessionId)?.session ?? null); + } + liveRowCount(sessionId: string): Promise { + return Promise.resolve(this.rows.get(sessionId)?.messages.length ?? 0); + } + lastLiveSeq(sessionId: string): Promise { + const rows = this.rows.get(sessionId)?.messages ?? []; + return Promise.resolve(rows.length > 0 ? Math.max(...rows.map((r) => r.seq)) : null); + } + seqByRaw(sessionId: string): Promise | null> { + const refs = this.rows.get(sessionId)?.refs; + if (!refs || refs.length === 0) return Promise.resolve(null); + const out = new Map(); + for (const r of refs) out.set(r.rawId, r.ord); + return Promise.resolve(out); + } + cloneSessionRows(): Promise { + return Promise.resolve(true); + } + getMessages(sessionId: string, opts?: { includeRetired?: boolean; seqs?: readonly number[] }): Promise<(MessageRow & { contentHash: string })[]> { + const rows = this.rows.get(sessionId)?.messages ?? []; + const live = opts?.seqs ? rows.filter((r) => opts.seqs!.includes(r.seq)) : rows; + return Promise.resolve(live.map((r) => ({ ...r, contentHash: r.contentHash ?? sha256(r.content) }))); + } + getBlocks(sessionId: string, opts?: { limit: number; offset?: number }): Promise<(BlockRow & { summaryHash: string; oneHash: string | null })[]> { + const rows = this.rows.get(sessionId)?.blocks ?? []; + const start = opts?.offset ?? 0; + const slice = opts ? rows.slice(start, start + opts.limit) : rows; + return Promise.resolve(slice.map((b) => ({ ...b, summaryHash: sha256(b.summary), oneHash: b.one === null ? null : sha256(b.one) }))); + } + getDeadRefs(sessionId: string): Promise { + return Promise.resolve(this.rows.get(sessionId)?.deadRefs ?? []); + } + getRefs(sessionId: string): Promise { + return Promise.resolve(this.rows.get(sessionId)?.refs ?? []); + } + sessionViewRows(sessionId: string): Promise { + const row = this.rows.get(sessionId); + if (!row) return Promise.resolve(null); + const pointers = new Set(viewPointerSeqs(row.session.tailJson)); + return Promise.resolve({ + session: row.session, + messages: row.messages.filter((r) => pointers.has(r.seq)).map((r) => ({ ...r, contentHash: r.contentHash ?? sha256(r.content) })), + blocks: row.blocks.map((b) => ({ ...b, summaryHash: sha256(b.summary), oneHash: b.one === null ? null : sha256(b.one) })), + deadRefs: row.deadRefs, + refs: row.refs, + }); + } + readSnapshot(read: (reader: StorageReadView) => Promise): Promise { + return read({ + getSessionRow: (id) => this.getSessionRow(id), + getMessages: (id, opts) => this.getMessages(id, opts), + getBlocks: (id, opts) => this.getBlocks(id, opts), + getDeadRefs: (id) => this.getDeadRefs(id), + getRefs: (id) => this.getRefs(id), + }); + } + getCcr(): Promise { + return Promise.resolve(null); + } + getCcrEntries(sessionId: string): Promise { + return Promise.resolve(this.rows.get(sessionId)?.ccrEntries ?? []); + } + // No watermark on the stub → every record takes the full-ingest path + // (deterministic seeding; the delta probe would need boundary state). + ingestWatermark(): Promise { + return Promise.resolve(null); + } + tailSeqs(): Promise { + return Promise.resolve(null); + } + clearIngestWatermark(): Promise { + return Promise.resolve(); + } + resetAllIngestWatermarks(): Promise { + return Promise.resolve(); + } + probeIngestBoundary(): Promise { + return Promise.resolve(null); + } + reapBlocks(): Promise { + return Promise.resolve(); + } + archiveSession(): Promise<{ ok: boolean; exportPath?: string; reason?: string; contentRowsDeleted?: number }> { + return Promise.resolve({ ok: false, reason: "stub" }); + } + restoreArchived(): Promise<{ ok: boolean; reason?: string }> { + return Promise.resolve({ ok: false, reason: "stub" }); + } + gc(): Promise<{ archived: number; contentRowsDeleted: number }> { + return Promise.resolve({ archived: 0, contentRowsDeleted: 0 }); + } + contentRefCountOfBody(): Promise { + return Promise.resolve(0); + } + mediaRefCount(): Promise { + return Promise.resolve(0); + } + insertCheckpoint(): Promise { + return Promise.resolve(); + } + listCheckpoints(): Promise { + return Promise.resolve([]); + } + getCheckpoint(): Promise { + return Promise.resolve(undefined); + } + checkpointState(): Promise { + throw new Error("stub-mysql backend: checkpoints not implemented"); + } + deleteCheckpoint(): Promise { + return Promise.resolve(false); + } + setSessionExt(): Promise { + return Promise.resolve(); + } + getSessionExt(): Promise { + return Promise.resolve(null); + } + listSessionExt(): Promise<{ key: string; value: unknown; updatedAt: number }[]> { + return Promise.resolve([]); + } + deleteSessionExt(): Promise { + return Promise.resolve(0); + } + setKv(): Promise { + return Promise.resolve(); + } + getKv(): Promise { + return Promise.resolve(null); + } + listKv(): Promise<{ namespace: string; key: string; value: unknown; updatedAt: number }[]> { + return Promise.resolve([]); + } + deleteKv(): Promise { + return Promise.resolve(0); + } + sessionSummaries(): Promise { + const out: SessionSummaryRow[] = []; + for (const stored of this.rows.values()) { + const s = stored.session; + let contextTokens: number | undefined; + try { + const stats = JSON.parse(s.statsJson) as { contextEstimateTokens?: unknown }; + if (typeof stats.contextEstimateTokens === "number") contextTokens = stats.contextEstimateTokens; + } catch { /* malformed statsJson leaves contextTokens unset */ } + out.push({ + sessionId: s.sessionId, + title: s.title ?? undefined, + label: s.label ?? undefined, + protocol: s.protocol ?? undefined, + origin: s.origin ?? undefined, + namespace: s.namespace ?? "", + blocks: stored.blocks.length, + latestBlockAt: stored.blocks.length > 0 ? Math.max(...stored.blocks.map((b) => b.createdAt)) : undefined, + contextTokens, + updatedAt: s.updatedAt, + status: s.status ?? "active", + }); + } + return Promise.resolve(out); + } + webSummaryRows(): Promise { + return Promise.resolve([]); + } + verifyProjections(): Promise<{ ok: true; checks: number } | { ok: false; error: string }> { + return Promise.resolve({ ok: true, checks: 0 }); + } + counts(): Promise> { + return Promise.resolve({ sessions: this.rows.size }); + } + contentDedupReport(): Promise<{ rows: number; bytesTotal: number; refcounted: number }> { + return Promise.resolve({ rows: 0, bytesTotal: 0, refcounted: 0 }); + } + close(): Promise { + return Promise.resolve(); + } +} + +function tmpDir(prefix: string): string { + return fs.mkdtempSync(path.join(os.tmpdir(), prefix)); +} + +/** Loads the folded corpus case into its own legacy store and returns the + * Session — the seed for every seam store below. */ +async function seedCorpusSession(baseDir: string): Promise { + fs.cpSync(path.join(CORPUS_ROOT, "folded"), path.join(baseDir, "folded"), { recursive: true }); + const loader = new SessionStore({ dir: path.join(baseDir, "folded"), debounceMs: 1, enabled: true }); + const loaded = loader.loadSync(CORPUS_ID, { protocol: "anthropic", upstreamOrigin: UPSTREAM_ORIGIN }); + loader.cancelAll(); + await loader.closeQuietlyAsync(); + if (!loaded) throw new Error(`corpus case ${CORPUS_ID} failed to load`); + return loaded; +} + +/** One full-mode sidecar over a stub backend, wired with the SAME + * record→view adapter the real store uses (recordToLegacyView) so ingest + * lands rows exactly as production would. */ +function makeShadow(stub: StubMysqlBackend): UnifiedShadow { + return UnifiedShadow.fromBackend("full", stub, recordToLegacyView); +} + +/** Seam store: empty dir, injected sidecar. */ +function makeSeamStore(dir: string, shadow: UnifiedShadow): SessionStore { + return new SessionStore({ dir, debounceMs: 1, enabled: true, shadow }); +} + +/** Drives the seeded session onto the stub through the real save path + * (guardedBuild → shadow.record → serial queue → ingestSession). */ +async function seedOnSeam(store: SessionStore, shadow: UnifiedShadow, session: Session): Promise { + assert.equal(store.flushSync(session), true); + const deadline = Date.now() + 5000; + for (;;) { + if ((await shadow.viewForAsync(CORPUS_ID)) !== null) return; + if (Date.now() > deadline) throw new Error(`timed out waiting for ${CORPUS_ID} to land on the stub backend`); + await new Promise((r) => setTimeout(r, 25)); + } +} + +function findEnvelopeFiles(dir: string): string[] { + const out: string[] = []; + const walk = (d: string): void => { + for (const ent of fs.readdirSync(d, { withFileTypes: true })) { + const p = path.join(d, ent.name); + if (ent.isDirectory()) walk(p); + else if (ent.name.endsWith(".json")) out.push(p); + } + }; + walk(dir); + return out; +} + +function removeEnvelopes(dir: string): void { + for (const f of findEnvelopeFiles(dir)) fs.rmSync(f, { force: true }); +} + +test("seam: DB-only session is unreachable via loadSync but reachable via loadAsync", async () => { + const baseDir = tmpDir("bili-seam-dbonly-"); + const loaded = await seedCorpusSession(baseDir); + const stub = new StubMysqlBackend(); + const shadow = makeShadow(stub); + const seamDir = tmpDir("bili-seam-dbonly-store-"); + const store = makeSeamStore(seamDir, shadow); + try { + await seedOnSeam(store, shadow, loaded); + removeEnvelopes(seamDir); + assert.equal(findEnvelopeFiles(seamDir).length, 0); + + // The old contract, pinned: the sync facade misses by design on + // network backends — this is the exact read that returned null and + // started a fresh (empty) session before #4942. + assert.equal(store.loadSync(CORPUS_ID), null); + + const reloaded = await store.loadAsync(CORPUS_ID); + assert.ok(reloaded, "loadAsync must serve the DB row when no envelope exists"); + assert.equal(reloaded.id, CORPUS_ID); + assert.ok(loaded.state.blocks.length > 0, "corpus case must carry folded blocks"); + assert.deepEqual( + reloaded!.state.blocks.map((b) => b.blockId).sort(), + loaded.state.blocks.map((b) => b.blockId).sort(), + ); + } finally { + store.cancelAll(); + await store.closeQuietlyAsync(); + } +}); + +test("seam: an existing envelope wins over the DB row (zero behavior change)", async () => { + const baseDir = tmpDir("bili-seam-envfirst-"); + const loaded = await seedCorpusSession(baseDir); + const stub = new StubMysqlBackend(); + const shadow = makeShadow(stub); + const seamDir = tmpDir("bili-seam-envfirst-store-"); + const store = makeSeamStore(seamDir, shadow); + try { + await seedOnSeam(store, shadow, loaded); + assert.ok(findEnvelopeFiles(seamDir).length > 0, "mysql keeps writing envelopes (writerOff is sqlite-only)"); + + const viaSync = store.loadSync(CORPUS_ID); + assert.ok(viaSync, "envelope load must keep working"); + const before = stub.viewCalls.count; + const viaAsync = await store.loadAsync(CORPUS_ID); + assert.deepEqual(viaAsync, viaSync, "loadAsync serves content-identical to the sync result when the sync path finds a copy (each cold load builds a fresh object; reference identity was never part of the contract)"); + assert.equal(stub.viewCalls.count, before, "no DB read may happen while an envelope exists"); + } finally { + store.cancelAll(); + await store.closeQuietlyAsync(); + } +}); + +test("seam: without a sidecar at all, loadAsync is exactly loadSync", async () => { + const baseDir = tmpDir("bili-seam-parity-"); + const loaded = await seedCorpusSession(baseDir); + const store = new SessionStore({ dir: baseDir, debounceMs: 1, enabled: true }); + try { + store.flushSync(loaded); + const a = store.loadSync(CORPUS_ID); + const b = await store.loadAsync(CORPUS_ID); + assert.ok(a && b); + assert.deepEqual(b, a, "no sidecar → loadAsync serves content-identical to the sync result"); + } finally { + store.cancelAll(); + await store.closeQuietlyAsync(); + } +}); + +test("seam: bounded boot pre-warms DB-only sessions within the budget", async () => { + const baseDir = tmpDir("bili-seam-boot-"); + const loaded = await seedCorpusSession(baseDir); + const stub = new StubMysqlBackend(); + const shadow = makeShadow(stub); + const seamDir = tmpDir("bili-seam-boot-store-"); + const store = makeSeamStore(seamDir, shadow); + try { + await seedOnSeam(store, shadow, loaded); + removeEnvelopes(seamDir); + + const warmed = await store.boot({ maxSessions: 8, maxBytes: 64 * 1024 * 1024 }); + const warm = warmed.get(CORPUS_ID); + assert.ok(warm, "boot union must enumerate the DB-only id through the async twins"); + assert.deepEqual( + warm!.state.blocks.map((b) => b.blockId).sort(), + loaded.state.blocks.map((b) => b.blockId).sort(), + ); + } finally { + store.cancelAll(); + await store.closeQuietlyAsync(); + } +}); + +test("seam: loadAll unions in DB-only sessions", async () => { + const baseDir = tmpDir("bili-seam-loadall-"); + const loaded = await seedCorpusSession(baseDir); + const stub = new StubMysqlBackend(); + const shadow = makeShadow(stub); + const seamDir = tmpDir("bili-seam-loadall-store-"); + const store = makeSeamStore(seamDir, shadow); + try { + await seedOnSeam(store, shadow, loaded); + removeEnvelopes(seamDir); + + const all = await store.loadAll(); + assert.ok(all.has(CORPUS_ID), "loadAll must see the DB-only id"); + assert.ok(all.get(CORPUS_ID)!.state.blocks.length === loaded.state.blocks.length); + } finally { + store.cancelAll(); + await store.closeQuietlyAsync(); + } +}); + +test("seam: getSessionAsync reloads a DB-only session into the resident pool, single-flight", async () => { + const baseDir = tmpDir("bili-seam-pool-"); + const loaded = await seedCorpusSession(baseDir); + const stub = new StubMysqlBackend(); + const shadow = makeShadow(stub); + const seamDir = tmpDir("bili-seam-pool-store-"); + const store = makeSeamStore(seamDir, shadow); + _setStoreForTest(store); + try { + await seedOnSeam(store, shadow, loaded); + removeEnvelopes(seamDir); + assert.equal(store.loadSync(CORPUS_ID), null, "precondition: nothing local to serve"); + + const [a, b] = await Promise.all([getSessionAsync(CORPUS_ID), getSessionAsync(CORPUS_ID)]); + assert.equal(a, b, "concurrent first-touches must share one load (single-flight)"); + assert.equal(a.id, CORPUS_ID); + assert.equal(a.state.blocks.length, loaded.state.blocks.length); + assert.equal(await getSessionAsync(CORPUS_ID), a, "subsequent calls hit the resident pool"); + } finally { + store.cancelAll(); + await store.closeQuietlyAsync(); + } +}); + +test("seam: plugin conversation resolution materializes DB-only sessions through the async seam", async () => { + const baseDir = tmpDir("bili-seam-pluginres-"); + const loaded = await seedCorpusSession(baseDir); + const stub = new StubMysqlBackend(); + const shadow = makeShadow(stub); + const seamDir = tmpDir("bili-seam-pluginres-store-"); + const store = makeSeamStore(seamDir, shadow); + _setStoreForTest(store); + // The earlier tests leave CORPUS_ID resident in the module-level pool — + // this scenario needs a cold process view of the conversation. + _resetSessionsForTest(); + try { + await seedOnSeam(store, shadow, loaded); + removeEnvelopes(seamDir); + assert.equal(store.loadSync(CORPUS_ID), null, "precondition: DB-only"); + assert.equal(peekSession(CORPUS_ID), undefined, "precondition: not resident"); + + const chunks: string[] = []; + let status = 0; + const res = { + writeHead: (st: number): void => { status = st; }, + end: (body?: string): void => { if (body) chunks.push(body); }, + } as unknown as import("node:http").ServerResponse; + // The kernel/config deps are never reached for a DB-reloaded session + // (no in-memory plugin snapshot → the nudge path stays empty), so a + // throwing stub doubles as a guard against accidental use. + const deps = { + core: { processTurn(): never { throw new Error("deps.core must not be called"); } }, + config: {}, + log: (): void => {}, + } as unknown as Parameters[2]; + await handlePluginStatus(CORPUS_ID, res, deps); + + assert.equal(status, 200, "status must resolve the DB-only conversation"); + const body = JSON.parse(chunks[chunks.length - 1]); + assert.ok(body.ok === true); + assert.equal(body.sessionId, CORPUS_ID, "must answer from the resolved session, not a ghost"); + const resident = peekSession(CORPUS_ID); + assert.ok(resident, "resolution must land the session in the resident pool"); + // The pre-fix shape: a sync getSession after an async probe minted an + // EMPTY ghost here (0 blocks) that then poisoned later getSessionAsync. + assert.deepEqual( + resident!.state.blocks.map((b) => b.blockId).sort(), + loaded.state.blocks.map((b) => b.blockId).sort(), + ); + } finally { + store.cancelAll(); + await store.closeQuietlyAsync(); + } +}); + +test("gate: jsonWriter=off stays ignored on mysql (the #4936 gate)", async () => { + const prev = { + BILI_STORAGE_BACKEND: process.env.BILI_STORAGE_BACKEND, + BILI_STORAGE_UNIFIED: process.env.BILI_STORAGE_UNIFIED, + BILI_STORAGE_JSON_WRITER: process.env.BILI_STORAGE_JSON_WRITER, + }; + process.env.BILI_STORAGE_BACKEND = "mysql"; + process.env.BILI_STORAGE_UNIFIED = "full"; + process.env.BILI_STORAGE_JSON_WRITER = "off"; + const logs: string[] = []; + const baseDir = tmpDir("bili-seam-gate-"); + const loaded = await seedCorpusSession(baseDir); + const seamDir = tmpDir("bili-seam-gate-store-"); + const store = new SessionStore({ dir: seamDir, debounceMs: 1, enabled: true, log: (_l, m) => logs.push(m) }); + try { + store.flushSync(loaded); + assert.ok( + logs.some((m) => m.includes("storage.jsonWriter=off ignored")), + `expected the sqlite-only gate warning, got: ${logs.join(" | ")}`, + ); + assert.ok(findEnvelopeFiles(seamDir).length > 0, "the envelope MUST keep being written on mysql"); + } finally { + store.cancelAll(); + await store.closeQuietlyAsync(); + for (const [k, v] of Object.entries(prev)) { + if (v === undefined) delete process.env[k]; + else process.env[k] = v; + } + } +}); diff --git a/tests/update-notes.test.ts b/tests/update-notes.test.ts index 31cfc6193c..8ebf1cd6bb 100644 --- a/tests/update-notes.test.ts +++ b/tests/update-notes.test.ts @@ -331,7 +331,7 @@ test("#1977 /acp panel: one critical-update line BEFORE the footer, stripper-saf const deps = { core: createCore(), config: defaultConfig(200000), log: (_l: string, _m: string) => {} }; const r0 = mockRes(); - handlePluginStatus("never-seen", r0.res, deps, true); + await handlePluginStatus("never-seen", r0.res, deps, true); assert.equal(r0.status, 200); const p0 = JSON.parse(r0.body).panel as string; assert.ok(!p0.includes("Update ready") && !p0.includes("Update available") && !p0.includes("Critical update"), "no update line when clean"); @@ -339,7 +339,7 @@ test("#1977 /acp panel: one critical-update line BEFORE the footer, stripper-saf _setReleaseNotesStateForTest({ entries: SURFACE_ENTRIES, diskVersion: NEXT2 }); const r1 = mockRes(); - handlePluginStatus("never-seen", r1.res, deps, true); + await handlePluginStatus("never-seen", r1.res, deps, true); const p1 = JSON.parse(r1.body).panel as string; assert.ok(p1.includes(`Critical update ready: ${NEXT2} — restart to finish.`), "panel line present"); assert.ok(p1.includes(PANEL_BOX_FOOTER), "footer still present with the update line"); @@ -355,7 +355,7 @@ test("#1977 /acp panel: one critical-update line BEFORE the footer, stripper-saf diskVersion: NEXT, }); const r2 = mockRes(); - handlePluginStatus("never-seen", r2.res, deps, true); + await handlePluginStatus("never-seen", r2.res, deps, true); const p2 = JSON.parse(r2.body).panel as string; assert.ok(p2.includes(dollarSummary), "dollar-sign summary survives byte-exact"); } finally { @@ -430,7 +430,7 @@ test("#4295 status and panel agree when only some critical updates are installed : "CRITICAL UPDATE AVAILABLE (instance-level):"), status); assert.equal(status.includes("UPDATE READY"), ready, "restart must deliver the critical fixes"); const res = mockRes(); - handlePluginStatus("never-seen", res.res, { core: createCore(), config: defaultConfig(200000), log: () => {} }, true); + await handlePluginStatus("never-seen", res.res, { core: createCore(), config: defaultConfig(200000), log: () => {} }, true); const body = JSON.parse(res.body) as { panel: string }; assert.ok(body.panel.includes(ready ? `Critical update ready: ${disk} — restart to finish.`