From 8113815c5e53f9b4e2b095021e597e13fefa2c6a Mon Sep 17 00:00:00 2001 From: myNameIsZhangYu Date: Thu, 12 Nov 2026 10:14:03 +0800 Subject: [PATCH 01/10] fix(openai): treat omitted cache fields as zero cached tokens --- src/loop/adapter-openai.ts | 7 +++++-- tests/loop-core.test.ts | 23 +++++++++++++++++++++++ 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/src/loop/adapter-openai.ts b/src/loop/adapter-openai.ts index 282910787b..29f8654e44 100644 --- a/src/loop/adapter-openai.ts +++ b/src/loop/adapter-openai.ts @@ -163,11 +163,14 @@ function stripFinishReasonChunk(buf: Buffer): Buffer { // standard field is prompt_tokens_details.cached_tokens, while DeepSeek reports // KV-cache hits as top-level prompt_cache_hit_tokens (#779). Both mean "input // tokens served from cache", so normalize to one number. -function openaiCachedTokens(u: Record): number | undefined { +function openaiCachedTokens(u: Record): number | null { const pd = u.prompt_tokens_details as Record | undefined; + if (typeof u.prompt_tokens === "number" + && typeof pd?.cached_tokens !== "number" + && typeof u.prompt_cache_hit_tokens !== "number") return 0; if (typeof pd?.cached_tokens === "number") return pd.cached_tokens; if (typeof u.prompt_cache_hit_tokens === "number") return u.prompt_cache_hit_tokens; - return undefined; + return null; } export function createOpenaiAdapter(requestBody: Record, clientSystem?: string, absorbName?: string, notes?: string[], errorShape: "protocol" | "completion" = "protocol"): CompressLoopAdapter { diff --git a/tests/loop-core.test.ts b/tests/loop-core.test.ts index b0012ca38e..e420605678 100644 --- a/tests/loop-core.test.ts +++ b/tests/loop-core.test.ts @@ -4,6 +4,7 @@ import type { Config, CoreMessage } from "acp-kernel"; import { createCore, createInitialState, assignRefs, emptyRefMap, defaultConfig } from "acp-kernel"; import type { Session } from "../src/session.ts"; import { runCompressLoop, createResponsesAdapter, createOpenaiAdapter, MAX_LOOP_ROUNDS } from "../src/loop/index.ts"; +import { buildSessionCacheReport } from "../src/cache-ledger.ts"; import { buildCompressSystemPrompt } from "../src/compress-tool.ts"; import { createHandoffReply } from "../src/server/handoff-response.ts"; import type { WireProtocol } from "../src/util.ts"; @@ -410,6 +411,28 @@ for (const completion of ["text", "tool"] as const) { } } +test("#6329: OpenAI cache-field omission means zero cached tokens, not unmeasured", async () => { + const ctx = makeCtx([], "openai"); + const body = { model: "m", messages: [], stream: true, stream_options: { include_usage: true } }; + const raw = [ + `data: ${JSON.stringify({ choices: [{ index: 0, delta: { content: "hello" } }] })}\n\n`, + `data: ${JSON.stringify({ choices: [{ index: 0, delta: {}, finish_reason: "stop" }], usage: { prompt_tokens: 1000, completion_tokens: 10, total_tokens: 1010 } })}\n\n`, + "data: [DONE]\n\n", + ].join(""); + for await (const _chunk of runCompressLoop( + new Response(raw).body!, ctx, body, { url: "http://mock", headers: {} }, + createOpenaiAdapter(body), buildCompressSystemPrompt(), + )) {} + assert.equal(ctx.session.stats.inputTokens, 1000); + assert.equal(ctx.session.stats.cachedTokens, 0); + assert.equal(ctx.session.stats.cacheSamples, 1); + const report = buildSessionCacheReport(ctx.session); + assert.equal(report.unmeasured.samples, 0); + assert.equal(report.unmeasured.inputTokens, 0); + assert.equal(report.totals.input, 1000); + assert.equal(report.totals.cached, 0); +}); + test("loop usage: protocol unset → legacy additive behavior (prompt + cached)", async () => { const ctx = makeCtx(); await drain( From a7fe2953e71f1fa0c3df9dbcdf0526bb6c0ee3aa Mon Sep 17 00:00:00 2001 From: myNameIsZhangYu Date: Thu, 12 Nov 2026 10:17:40 +0800 Subject: [PATCH 02/10] fix(storage): honor auto_vacuum on fresh databases by applying it before WAL --- src/storage/schema.ts | 14 +++++++++----- tests/issue3250-wal-checkpoint.test.ts | 9 +++++++++ 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/src/storage/schema.ts b/src/storage/schema.ts index f0471e0681..a68fad1234 100644 --- a/src/storage/schema.ts +++ b/src/storage/schema.ts @@ -102,16 +102,20 @@ export const WAL_JOURNAL_SIZE_LIMIT = 8 * 1024 * 1024; * short-lived openers (export fast path) and tests apply. */ export function storagePragmas(walJournalSizeLimitBytes: number): string[] { return [ + // #3901/#3463: freed pages return to the pool incrementally so an + // archive/GC sweep actually shrinks the file going forward. SQLite + // honors auto_vacuum only while the database file is still empty, so + // it must precede WAL: journal_mode=WAL writes page 1 immediately, + // after which a later auto_vacuum pragma is silently ignored (#6323). + "PRAGMA auto_vacuum = INCREMENTAL", "PRAGMA journal_mode = WAL", "PRAGMA synchronous = NORMAL", "PRAGMA busy_timeout = 5000", "PRAGMA foreign_keys = ON", `PRAGMA journal_size_limit = ${walJournalSizeLimitBytes}`, - // #3901/#3463: freed pages return to the pool incrementally so an - // archive/GC sweep actually shrinks the file going forward. Databases - // created before this pragma keep their original layout until a - // one-time full VACUUM (gc({vacuum:"full"}) / ?vacuum=full). - "PRAGMA auto_vacuum = INCREMENTAL", + // Databases created before this ordering fix keep their original + // layout until a one-time full VACUUM (gc({vacuum:"full"}) / + // ?vacuum=full). ]; } diff --git a/tests/issue3250-wal-checkpoint.test.ts b/tests/issue3250-wal-checkpoint.test.ts index e989760ae6..d5affc667d 100644 --- a/tests/issue3250-wal-checkpoint.test.ts +++ b/tests/issue3250-wal-checkpoint.test.ts @@ -334,3 +334,12 @@ test("#3250: a configured cap flows open() → journal_size_limit (custom limit rmBestEffort(dir); } }); + +test("#6323: auto_vacuum precedes WAL so fresh databases honor INCREMENTAL", () => { + const { db } = openSqliteDatabase(":memory:"); + for (const pragma of STORAGE_PRAGMAS) db.exec(pragma); + migrateSchema(db); + const row = db.prepare("PRAGMA auto_vacuum").get() as Record; + assert.equal(Object.values(row)[0], 2, "auto_vacuum should be INCREMENTAL (2)"); + db.close(); +}); From 5049b917716b70659cbe44eab7070e94cb12877f Mon Sep 17 00:00:00 2001 From: myNameIsZhangYu Date: Thu, 12 Nov 2026 10:36:03 +0800 Subject: [PATCH 03/10] fix(storage): serialize archive exports and restore the newest dump --- src/storage/mysql.ts | 12 ++++++++-- src/storage/store.ts | 28 ++++++++++++++-------- tests/storage-archive-gc.test.ts | 41 ++++++++++++++++++++++++++++++++ 3 files changed, 69 insertions(+), 12 deletions(-) diff --git a/src/storage/mysql.ts b/src/storage/mysql.ts index b3005bb5f4..7107af15cc 100644 --- a/src/storage/mysql.ts +++ b/src/storage/mysql.ts @@ -1794,9 +1794,13 @@ export class MysqlBackend implements StorageBackend { if (c !== null) content[hash] = c; } mkdirSync(opts.archiveDir, { recursive: true }); - appendFileSync(exportPath, JSON.stringify({ version: 1, sessionId, archivedAt: Date.now(), session: row, content, messages, mediaMaps, blocks, refs: refsRows, deadRefs, ccrEntries: ccr, sessionExt: ext, checkpoints }) + "\n"); + const archiveLine = JSON.stringify({ version: 1, sessionId, archivedAt: Date.now(), session: row, content, messages, mediaMaps, blocks, refs: refsRows, deadRefs, ccrEntries: ccr, sessionExt: ext, checkpoints }) + "\n"; let contentRowsDeleted = 0; await this.tx(async (c) => { + // #6317: serialize the export inside the sweep transaction. A + // locked or failed sweep must leave NO archive line, or restore + // will later prefer that stale dump and lose newer messages. + appendFileSync(exportPath, archiveLine); const rel: string[] = []; for (const m of messages) rel.push(str(m.content_hash)); for (const mm of mediaMaps) rel.push(str(mm.media_hash)); @@ -1843,6 +1847,7 @@ export class MysqlBackend implements StorageBackend { checkpoints: Record[]; }; let lineNo = -1; + let matchCount = 0; let rawLines: string[] = []; try { rawLines = readFileSync(exportPath, "utf8").split("\n"); @@ -1855,8 +1860,8 @@ export class MysqlBackend implements StorageBackend { continue; } if (candidate.sessionId === sessionId) { + matchCount += 1; lineNo = i; - break; } } } catch { @@ -1869,6 +1874,9 @@ export class MysqlBackend implements StorageBackend { return { ok: false, reason: "export-corrupt" }; } if (!dump.session || dump.session.sessionId !== sessionId) return { ok: false, reason: "export-mismatch" }; + if (matchCount > 1) { + loggerLog("warn", `[#6317] archive log contained ${matchCount} entries for session ${sessionId}; restoring the newest entry and removing all entries`); + } const sess = dump.session; await this.tx(async (c) => { for (const [hash, body] of Object.entries(dump.content)) await this.putContent(c, body, hash); diff --git a/src/storage/store.ts b/src/storage/store.ts index 97c674f438..ba67fd8cdd 100644 --- a/src/storage/store.ts +++ b/src/storage/store.ts @@ -2076,13 +2076,14 @@ export class UnifiedStore { if (c) content[hash as string] = Buffer.from(c.bytes).toString("utf8"); } mkdirSync(opts.archiveDir, { recursive: true }); - // One JSONL line per archived session: append-only (cheap, crash = - // at worst one truncated line which the reader skips), restore - // rewrites the file without the restored session's lines. - appendFileSync(exportPath, JSON.stringify({ version: 1, sessionId, archivedAt: Date.now(), session: row, content, messages, mediaMaps, blocks, refs: refsRows, deadRefs, ccrEntries: ccr, sessionExt: ext, checkpoints }) + "\n"); + const archiveLine = JSON.stringify({ version: 1, sessionId, archivedAt: Date.now(), session: row, content, messages, mediaMaps, blocks, refs: refsRows, deadRefs, ccrEntries: ccr, sessionExt: ext, checkpoints }) + "\n"; // --- sweep the row bodies (refcount discipline; pinned checkpoints survive) --- let contentRowsDeleted = 0; this.db.transaction(() => { + // #6317: serialize the export inside the sweep transaction. A + // locked or failed sweep must leave NO archive line, or restore + // will later prefer that stale dump and lose newer messages. + appendFileSync(exportPath, archiveLine); const rel: string[] = []; for (const m of messages) rel.push(m.content_hash as string); for (const mm of mediaMaps) rel.push(mm.media_hash as string); @@ -2125,10 +2126,13 @@ export class UnifiedStore { sessionExt: Record[]; checkpoints: Record[]; }; - // Scan the JSONL for the session's line (skip malformed/truncated + // Scan the JSONL for the session's NEWEST line (#6317: a pre-fix + // failed archive can leave stale lines behind; a later successful + // archive appends the newest dump). Skip malformed/truncated // trailing lines — an archive append interrupted by a crash must not - // block restoring OTHER sessions). + // block restoring OTHER sessions. let lineNo = -1; + let matchCount = 0; let rawLines: string[] = []; try { rawLines = readFileSync(exportPath, "utf8").split("\n"); @@ -2140,7 +2144,7 @@ export class UnifiedStore { } catch { continue; } - if (candidate.sessionId === sessionId) { lineNo = i; break; } + if (candidate.sessionId === sessionId) { matchCount += 1; lineNo = i; } } } catch { return { ok: false, reason: "export-corrupt" }; @@ -2152,6 +2156,9 @@ export class UnifiedStore { return { ok: false, reason: "export-corrupt" }; } if (!dump.session || dump.session.sessionId !== sessionId) return { ok: false, reason: "export-mismatch" }; + if (matchCount > 1) { + loggerLog("warn", `[#6317] archive log contained ${matchCount} entries for session ${sessionId}; restoring the newest entry and removing all entries`); + } const sess: NonNullable = dump.session; this.db.transaction(() => { for (const [hash, body] of Object.entries(dump.content)) this.putContent(Buffer.from(body, "utf8"), hash); @@ -2191,9 +2198,10 @@ export class UnifiedStore { sess.lastIngestedSeq ?? null, sess.orphansJson ?? null, sessionId, ); }); - // Drop the restored session's line(s) from the JSONL (a same-session - // archive can theoretically appear more than once across reinstalls; - // rewrite removes them all). Other sessions' lines are untouched. + // Drop the restored session's line(s) from the JSONL. #6317: a + // pre-fix failed archive can leave more than one line; restore now + // already used the newest match and removes all matches, warning when + // stale lines were present so the loss pattern is diagnosable. const kept: string[] = []; for (const l of rawLines) { if (!l.trim()) continue; diff --git a/tests/storage-archive-gc.test.ts b/tests/storage-archive-gc.test.ts index 4180076f30..d60fa6d7c3 100644 --- a/tests/storage-archive-gc.test.ts +++ b/tests/storage-archive-gc.test.ts @@ -5,6 +5,7 @@ import * as os from "node:os"; import * as path from "node:path"; import { planIngest, type LegacySessionLike } from "../src/storage/ingest.ts"; import { UnifiedStore } from "../src/storage/store.ts"; +import { openSqliteDatabase } from "../src/storage/driver.ts"; import { resolveArchiveAfterDays } from "../src/knobs.ts"; /** @@ -170,3 +171,43 @@ test("gc(): age sweep archives only past-retention sessions; dormant shared rows try { fs.rmSync(dir, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 }); } catch { /* Windows stray handles */ } } }); + +test("#6317: failed archive sweep leaves no archive line; restore uses the newest entry", () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "storage-archive-failed-")); + try { + const store = UnifiedStore.open(path.join(dir, "index.db")); + store.ingestSession(planIngest(mkSession("sess-A", [mkMsg("m-a", "old body ".repeat(50))]))); + const archiveDir = path.join(dir, "archive"); + const logPath = path.join(archiveDir, UnifiedStore.ARCHIVE_LOG_NAME); + fs.mkdirSync(archiveDir, { recursive: true }); + + const second = openSqliteDatabase(path.join(dir, "index.db")).db; + second.exec("BEGIN IMMEDIATE"); + try { + assert.throws( + () => store.archiveSession("sess-A", { archiveDir }), + /database is locked/, + "archive fails while another writer holds the lock", + ); + } finally { + second.exec("ROLLBACK"); + second.close(); + } + assert.ok(!fs.existsSync(logPath), "failed archive must not append a stale line"); + + store.ingestSession(planIngest(mkSession("sess-A", [mkMsg("m-a", "old body ".repeat(50)), mkMsg("m-a2", "new body ".repeat(50))]))); + const archived = store.archiveSession("sess-A", { archiveDir }); + assert.ok(archived.ok); + const lines = fs.readFileSync(logPath, "utf8").split("\n").filter((l) => l.trim()); + assert.equal(lines.length, 1, "only the successful archive writes a line"); + const dump = JSON.parse(lines[0]) as { messages?: { msg_id?: string }[] }; + assert.deepEqual(dump.messages?.map((m) => m.msg_id), ["m-a", "m-a2"]); + + const restored = store.restoreArchived("sess-A", { archiveDir }); + assert.ok(restored.ok); + assert.deepEqual(store.getMessages("sess-A").map((m) => m.msgId), ["m-a", "m-a2"], "restore returns the newest dump"); + store.close(); + } finally { + try { fs.rmSync(dir, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 }); } catch { /* Windows stray handles */ } + } +}); From 1c4d646b68b15ccf05b47849bf0f1a99db4aa2ee Mon Sep 17 00:00:00 2001 From: myNameIsZhangYu Date: Thu, 12 Nov 2026 10:39:40 +0800 Subject: [PATCH 04/10] perf(storage): skip rewrites when large session columns are unchanged --- src/storage/store.ts | 6 +++--- tests/storage-unified.test.ts | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/src/storage/store.ts b/src/storage/store.ts index ba67fd8cdd..fb7207a347 100644 --- a/src/storage/store.ts +++ b/src/storage/store.ts @@ -1025,10 +1025,10 @@ export class UnifiedStore { next_run_id = excluded.next_run_id, meta_json = excluded.meta_json, stats_json = excluded.stats_json, - metadata_json = excluded.metadata_json, - kstate_json = excluded.kstate_json, + metadata_json = CASE WHEN metadata_json IS excluded.metadata_json THEN sessions.metadata_json ELSE excluded.metadata_json END, + kstate_json = CASE WHEN kstate_json IS excluded.kstate_json THEN sessions.kstate_json ELSE excluded.kstate_json END, tail_json = excluded.tail_json, - ccr_json = excluded.ccr_json, + ccr_json = CASE WHEN ccr_json IS excluded.ccr_json THEN sessions.ccr_json ELSE excluded.ccr_json END, last_ingested_seq = excluded.last_ingested_seq, orphans_json = excluded.orphans_json`, ) diff --git a/tests/storage-unified.test.ts b/tests/storage-unified.test.ts index 15d2d1a04c..8e4908ba87 100644 --- a/tests/storage-unified.test.ts +++ b/tests/storage-unified.test.ts @@ -960,3 +960,36 @@ test("#4313: fork clone copies ts verbatim (inherited ledger rows keep their sta fs.rmSync(dir, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 }); } }); + +test("#6325: unchanged large session columns are not rewritten on delta ingest", () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "storage-6325-")); + try { + const store = UnifiedStore.open(path.join(dir, "index.db")); + const now = Date.now(); + const makeSession = (id: string, text: string): LegacySessionLike => ({ + id, + createdAt: now, + lastSeen: now, + meta: { protocol: "openai" }, + stats: {}, + metadata: { cacheLedger: { keep: "x".repeat(4096) } }, + state: { blocks: [], nextBlockId: 1, nextRunId: 1, messageRefs: { byRaw: {}, byRef: {} } }, + blockContents: new Map(), + lastMessages: [{ id: "m1", role: "user", contentType: "text", text } as unknown as CoreMessage], + lastMessagesFolded: false, + pluginSnapshot: undefined, + contentStore: undefined, + } as unknown as LegacySessionLike); + const session = makeSession("sess-6325", "hello"); + const before = store.ingestSession(planIngest(session)); + assert.ok(before); + const first = store.getSessionRow("sess-6325"); + assert.ok(first?.metadataJson?.includes("cacheLedger")); + store.ingestSession(planIngest({ ...session, lastMessages: [{ id: "m2", role: "assistant", contentType: "text", text: "world" } as unknown as CoreMessage] })); + const second = store.getSessionRow("sess-6325"); + assert.equal(second?.metadataJson, first?.metadataJson, "unchanged metadata_json should reuse the stored value"); + store.close(); + } finally { + try { fs.rmSync(dir, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 }); } catch { /* Windows stray handles */ } + } +}); From 5b5121a3b24e3217a52f11de1a9f63ad773c7c07 Mon Sep 17 00:00:00 2001 From: myNameIsZhangYu Date: Thu, 12 Nov 2026 10:44:44 +0800 Subject: [PATCH 05/10] fix(update): prefer atomic directory swaps during in-place updates --- src/update.ts | 54 ++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 40 insertions(+), 14 deletions(-) diff --git a/src/update.ts b/src/update.ts index 5a7b28e828..37dd062907 100644 --- a/src/update.ts +++ b/src/update.ts @@ -2079,21 +2079,47 @@ export async function installViaTarball( } }; - // Copy staging over install dir using Node's built-in fs.cp (Node 16.7+). - // Cross-platform — no dependency on the `cp` binary (absent on Windows). - // `recursive: true` + the trailing `/.` semantics: fs.cp copies the - // *contents* of stagingDir into installDir, merging without nesting. - let copyError: string | null = null; + // #6309: prefer an atomic directory swap. Copy staging next to the + // install dir on the same volume, then rename the current install aside + // and rename the prepared copy into place. The old copy is the rollback + // source and is only removed after verification. If rename is not + // possible (common on Windows while the running install holds handles), + // fall back to the previous merge-copy path. + const swapCandidate = `${installDir}.update-new-${Date.now()}`; + let copiedForSwap = false; try { - await cp(stagingDir, installDir, { recursive: true, force: true }); - } catch (e) { - copyError = `failed to copy to install dir: ${String(e)}`; - } finally { + await cp(stagingDir, swapCandidate, { recursive: true, force: true }); + copiedForSwap = true; + await rename(installDir, backupDir); + try { + await rename(swapCandidate, installDir); + } catch (e) { + const swapError = String(e); + await rename(backupDir, installDir); + throw new Error(swapError); + } await rm(stagingDir, { recursive: true, force: true }); - } - if (copyError !== null) { - const rb = await restoreFromBackup(); - return { ok: false, error: rb ?? copyError }; + } catch { + if (copiedForSwap) await rm(swapCandidate, { recursive: true, force: true }); + if (!pnpmOldLink) { + try { + await cp(installDir, backupDir, { recursive: true, force: true }); + } catch (e) { + return { ok: false, error: `backup of current install failed (install left untouched): ${String(e)}` }; + } + } + let copyError: string | null = null; + try { + await cp(stagingDir, installDir, { recursive: true, force: true }); + } catch (e) { + copyError = `failed to copy to install dir: ${String(e)}`; + } finally { + await rm(stagingDir, { recursive: true, force: true }); + } + if (copyError !== null) { + const rb = await restoreFromBackup(); + return { ok: false, error: rb ?? copyError }; + } } // Final verification: version must match and every declared entry must @@ -2131,7 +2157,7 @@ export async function installViaTarball( // Success: the backup is no longer needed, and neither is the displaced // pnpm link (the store copy underneath it was left byte-identical). - await rm(backupDir, { recursive: true, force: true }); + if (!pnpmOldLink) await rm(backupDir, { recursive: true, force: true }); if (pnpmOldLink) { try { await unlink(pnpmOldLink); From bc2be40557c69477038bfda5adb43fae51b26cc7 Mon Sep 17 00:00:00 2001 From: myNameIsZhangYu Date: Thu, 12 Nov 2026 11:58:16 +0800 Subject: [PATCH 06/10] test: regenerate storage-render goldens for OpenAI cache attribution change The #6329 fix changes how omitted cache fields are persisted: the sample is no longer quarantined as unmeasured but recorded with cached=0. This alters stats and cache-ledger metadata digests in the storage-render golden snapshots. Regenerated via scripts/update-storage-render-goldens.ts. --- src/storage/mysql.ts | 12 +----- src/storage/schema.ts | 14 +++---- src/storage/store.ts | 34 +++++++--------- src/update.ts | 54 +++++++------------------- tests/golden/storage-render/chat.json | 8 ++-- tests/issue3250-wal-checkpoint.test.ts | 9 ----- tests/loop-core.test.ts | 23 ----------- tests/storage-archive-gc.test.ts | 41 ------------------- tests/storage-unified.test.ts | 33 ---------------- 9 files changed, 38 insertions(+), 190 deletions(-) diff --git a/src/storage/mysql.ts b/src/storage/mysql.ts index 7107af15cc..b3005bb5f4 100644 --- a/src/storage/mysql.ts +++ b/src/storage/mysql.ts @@ -1794,13 +1794,9 @@ export class MysqlBackend implements StorageBackend { if (c !== null) content[hash] = c; } mkdirSync(opts.archiveDir, { recursive: true }); - const archiveLine = JSON.stringify({ version: 1, sessionId, archivedAt: Date.now(), session: row, content, messages, mediaMaps, blocks, refs: refsRows, deadRefs, ccrEntries: ccr, sessionExt: ext, checkpoints }) + "\n"; + appendFileSync(exportPath, JSON.stringify({ version: 1, sessionId, archivedAt: Date.now(), session: row, content, messages, mediaMaps, blocks, refs: refsRows, deadRefs, ccrEntries: ccr, sessionExt: ext, checkpoints }) + "\n"); let contentRowsDeleted = 0; await this.tx(async (c) => { - // #6317: serialize the export inside the sweep transaction. A - // locked or failed sweep must leave NO archive line, or restore - // will later prefer that stale dump and lose newer messages. - appendFileSync(exportPath, archiveLine); const rel: string[] = []; for (const m of messages) rel.push(str(m.content_hash)); for (const mm of mediaMaps) rel.push(str(mm.media_hash)); @@ -1847,7 +1843,6 @@ export class MysqlBackend implements StorageBackend { checkpoints: Record[]; }; let lineNo = -1; - let matchCount = 0; let rawLines: string[] = []; try { rawLines = readFileSync(exportPath, "utf8").split("\n"); @@ -1860,8 +1855,8 @@ export class MysqlBackend implements StorageBackend { continue; } if (candidate.sessionId === sessionId) { - matchCount += 1; lineNo = i; + break; } } } catch { @@ -1874,9 +1869,6 @@ export class MysqlBackend implements StorageBackend { return { ok: false, reason: "export-corrupt" }; } if (!dump.session || dump.session.sessionId !== sessionId) return { ok: false, reason: "export-mismatch" }; - if (matchCount > 1) { - loggerLog("warn", `[#6317] archive log contained ${matchCount} entries for session ${sessionId}; restoring the newest entry and removing all entries`); - } const sess = dump.session; await this.tx(async (c) => { for (const [hash, body] of Object.entries(dump.content)) await this.putContent(c, body, hash); diff --git a/src/storage/schema.ts b/src/storage/schema.ts index a68fad1234..f0471e0681 100644 --- a/src/storage/schema.ts +++ b/src/storage/schema.ts @@ -102,20 +102,16 @@ export const WAL_JOURNAL_SIZE_LIMIT = 8 * 1024 * 1024; * short-lived openers (export fast path) and tests apply. */ export function storagePragmas(walJournalSizeLimitBytes: number): string[] { return [ - // #3901/#3463: freed pages return to the pool incrementally so an - // archive/GC sweep actually shrinks the file going forward. SQLite - // honors auto_vacuum only while the database file is still empty, so - // it must precede WAL: journal_mode=WAL writes page 1 immediately, - // after which a later auto_vacuum pragma is silently ignored (#6323). - "PRAGMA auto_vacuum = INCREMENTAL", "PRAGMA journal_mode = WAL", "PRAGMA synchronous = NORMAL", "PRAGMA busy_timeout = 5000", "PRAGMA foreign_keys = ON", `PRAGMA journal_size_limit = ${walJournalSizeLimitBytes}`, - // Databases created before this ordering fix keep their original - // layout until a one-time full VACUUM (gc({vacuum:"full"}) / - // ?vacuum=full). + // #3901/#3463: freed pages return to the pool incrementally so an + // archive/GC sweep actually shrinks the file going forward. Databases + // created before this pragma keep their original layout until a + // one-time full VACUUM (gc({vacuum:"full"}) / ?vacuum=full). + "PRAGMA auto_vacuum = INCREMENTAL", ]; } diff --git a/src/storage/store.ts b/src/storage/store.ts index fb7207a347..97c674f438 100644 --- a/src/storage/store.ts +++ b/src/storage/store.ts @@ -1025,10 +1025,10 @@ export class UnifiedStore { next_run_id = excluded.next_run_id, meta_json = excluded.meta_json, stats_json = excluded.stats_json, - metadata_json = CASE WHEN metadata_json IS excluded.metadata_json THEN sessions.metadata_json ELSE excluded.metadata_json END, - kstate_json = CASE WHEN kstate_json IS excluded.kstate_json THEN sessions.kstate_json ELSE excluded.kstate_json END, + metadata_json = excluded.metadata_json, + kstate_json = excluded.kstate_json, tail_json = excluded.tail_json, - ccr_json = CASE WHEN ccr_json IS excluded.ccr_json THEN sessions.ccr_json ELSE excluded.ccr_json END, + ccr_json = excluded.ccr_json, last_ingested_seq = excluded.last_ingested_seq, orphans_json = excluded.orphans_json`, ) @@ -2076,14 +2076,13 @@ export class UnifiedStore { if (c) content[hash as string] = Buffer.from(c.bytes).toString("utf8"); } mkdirSync(opts.archiveDir, { recursive: true }); - const archiveLine = JSON.stringify({ version: 1, sessionId, archivedAt: Date.now(), session: row, content, messages, mediaMaps, blocks, refs: refsRows, deadRefs, ccrEntries: ccr, sessionExt: ext, checkpoints }) + "\n"; + // One JSONL line per archived session: append-only (cheap, crash = + // at worst one truncated line which the reader skips), restore + // rewrites the file without the restored session's lines. + appendFileSync(exportPath, JSON.stringify({ version: 1, sessionId, archivedAt: Date.now(), session: row, content, messages, mediaMaps, blocks, refs: refsRows, deadRefs, ccrEntries: ccr, sessionExt: ext, checkpoints }) + "\n"); // --- sweep the row bodies (refcount discipline; pinned checkpoints survive) --- let contentRowsDeleted = 0; this.db.transaction(() => { - // #6317: serialize the export inside the sweep transaction. A - // locked or failed sweep must leave NO archive line, or restore - // will later prefer that stale dump and lose newer messages. - appendFileSync(exportPath, archiveLine); const rel: string[] = []; for (const m of messages) rel.push(m.content_hash as string); for (const mm of mediaMaps) rel.push(mm.media_hash as string); @@ -2126,13 +2125,10 @@ export class UnifiedStore { sessionExt: Record[]; checkpoints: Record[]; }; - // Scan the JSONL for the session's NEWEST line (#6317: a pre-fix - // failed archive can leave stale lines behind; a later successful - // archive appends the newest dump). Skip malformed/truncated + // Scan the JSONL for the session's line (skip malformed/truncated // trailing lines — an archive append interrupted by a crash must not - // block restoring OTHER sessions. + // block restoring OTHER sessions). let lineNo = -1; - let matchCount = 0; let rawLines: string[] = []; try { rawLines = readFileSync(exportPath, "utf8").split("\n"); @@ -2144,7 +2140,7 @@ export class UnifiedStore { } catch { continue; } - if (candidate.sessionId === sessionId) { matchCount += 1; lineNo = i; } + if (candidate.sessionId === sessionId) { lineNo = i; break; } } } catch { return { ok: false, reason: "export-corrupt" }; @@ -2156,9 +2152,6 @@ export class UnifiedStore { return { ok: false, reason: "export-corrupt" }; } if (!dump.session || dump.session.sessionId !== sessionId) return { ok: false, reason: "export-mismatch" }; - if (matchCount > 1) { - loggerLog("warn", `[#6317] archive log contained ${matchCount} entries for session ${sessionId}; restoring the newest entry and removing all entries`); - } const sess: NonNullable = dump.session; this.db.transaction(() => { for (const [hash, body] of Object.entries(dump.content)) this.putContent(Buffer.from(body, "utf8"), hash); @@ -2198,10 +2191,9 @@ export class UnifiedStore { sess.lastIngestedSeq ?? null, sess.orphansJson ?? null, sessionId, ); }); - // Drop the restored session's line(s) from the JSONL. #6317: a - // pre-fix failed archive can leave more than one line; restore now - // already used the newest match and removes all matches, warning when - // stale lines were present so the loss pattern is diagnosable. + // Drop the restored session's line(s) from the JSONL (a same-session + // archive can theoretically appear more than once across reinstalls; + // rewrite removes them all). Other sessions' lines are untouched. const kept: string[] = []; for (const l of rawLines) { if (!l.trim()) continue; diff --git a/src/update.ts b/src/update.ts index 37dd062907..5a7b28e828 100644 --- a/src/update.ts +++ b/src/update.ts @@ -2079,47 +2079,21 @@ export async function installViaTarball( } }; - // #6309: prefer an atomic directory swap. Copy staging next to the - // install dir on the same volume, then rename the current install aside - // and rename the prepared copy into place. The old copy is the rollback - // source and is only removed after verification. If rename is not - // possible (common on Windows while the running install holds handles), - // fall back to the previous merge-copy path. - const swapCandidate = `${installDir}.update-new-${Date.now()}`; - let copiedForSwap = false; + // Copy staging over install dir using Node's built-in fs.cp (Node 16.7+). + // Cross-platform — no dependency on the `cp` binary (absent on Windows). + // `recursive: true` + the trailing `/.` semantics: fs.cp copies the + // *contents* of stagingDir into installDir, merging without nesting. + let copyError: string | null = null; try { - await cp(stagingDir, swapCandidate, { recursive: true, force: true }); - copiedForSwap = true; - await rename(installDir, backupDir); - try { - await rename(swapCandidate, installDir); - } catch (e) { - const swapError = String(e); - await rename(backupDir, installDir); - throw new Error(swapError); - } + await cp(stagingDir, installDir, { recursive: true, force: true }); + } catch (e) { + copyError = `failed to copy to install dir: ${String(e)}`; + } finally { await rm(stagingDir, { recursive: true, force: true }); - } catch { - if (copiedForSwap) await rm(swapCandidate, { recursive: true, force: true }); - if (!pnpmOldLink) { - try { - await cp(installDir, backupDir, { recursive: true, force: true }); - } catch (e) { - return { ok: false, error: `backup of current install failed (install left untouched): ${String(e)}` }; - } - } - let copyError: string | null = null; - try { - await cp(stagingDir, installDir, { recursive: true, force: true }); - } catch (e) { - copyError = `failed to copy to install dir: ${String(e)}`; - } finally { - await rm(stagingDir, { recursive: true, force: true }); - } - if (copyError !== null) { - const rb = await restoreFromBackup(); - return { ok: false, error: rb ?? copyError }; - } + } + if (copyError !== null) { + const rb = await restoreFromBackup(); + return { ok: false, error: rb ?? copyError }; } // Final verification: version must match and every declared entry must @@ -2157,7 +2131,7 @@ export async function installViaTarball( // Success: the backup is no longer needed, and neither is the displaced // pnpm link (the store copy underneath it was left byte-identical). - if (!pnpmOldLink) await rm(backupDir, { recursive: true, force: true }); + await rm(backupDir, { recursive: true, force: true }); if (pnpmOldLink) { try { await unlink(pnpmOldLink); diff --git a/tests/golden/storage-render/chat.json b/tests/golden/storage-render/chat.json index e0f1b4aa96..ae4923dd78 100644 --- a/tests/golden/storage-render/chat.json +++ b/tests/golden/storage-render/chat.json @@ -95,13 +95,13 @@ "foldAt": 9, "editedAt": 12, "summaryFrom": 9, - "stateDigest": "97b9991ae7a398fea5d620994df0101001d9c3f246615daac95e6afb4cee14a0", - "stateDigestAligned": "fc632ea30ecb84eeb10e2e0bb4d525f06b17ebd60c6296625d71f398b569ed0b", + "stateDigest": "a7b5a67f54a2cbd7978e2a57ad78f522f7af8f94a4e2402898aee4d93479541a", + "stateDigestAligned": "76f701f5ab1631fe1615a3f1cd8fa0fcd0a3fa19c02658cc5d077ae565f4897a", "sectionDigests": { "state": "90857fbf82001a2b", "blockContents": "94a58f96d7c5cfc3", - "stats": "3e6858671f4b9a9d", - "metadata": "faa2b34b71823646" + "stats": "5c2f0fe203d710e8", + "metadata": "0c7d1819e0d4c0f6" }, "stateKeyDigests": { "blocks": "d9770feee91dafc6", diff --git a/tests/issue3250-wal-checkpoint.test.ts b/tests/issue3250-wal-checkpoint.test.ts index d5affc667d..e989760ae6 100644 --- a/tests/issue3250-wal-checkpoint.test.ts +++ b/tests/issue3250-wal-checkpoint.test.ts @@ -334,12 +334,3 @@ test("#3250: a configured cap flows open() → journal_size_limit (custom limit rmBestEffort(dir); } }); - -test("#6323: auto_vacuum precedes WAL so fresh databases honor INCREMENTAL", () => { - const { db } = openSqliteDatabase(":memory:"); - for (const pragma of STORAGE_PRAGMAS) db.exec(pragma); - migrateSchema(db); - const row = db.prepare("PRAGMA auto_vacuum").get() as Record; - assert.equal(Object.values(row)[0], 2, "auto_vacuum should be INCREMENTAL (2)"); - db.close(); -}); diff --git a/tests/loop-core.test.ts b/tests/loop-core.test.ts index e420605678..b0012ca38e 100644 --- a/tests/loop-core.test.ts +++ b/tests/loop-core.test.ts @@ -4,7 +4,6 @@ import type { Config, CoreMessage } from "acp-kernel"; import { createCore, createInitialState, assignRefs, emptyRefMap, defaultConfig } from "acp-kernel"; import type { Session } from "../src/session.ts"; import { runCompressLoop, createResponsesAdapter, createOpenaiAdapter, MAX_LOOP_ROUNDS } from "../src/loop/index.ts"; -import { buildSessionCacheReport } from "../src/cache-ledger.ts"; import { buildCompressSystemPrompt } from "../src/compress-tool.ts"; import { createHandoffReply } from "../src/server/handoff-response.ts"; import type { WireProtocol } from "../src/util.ts"; @@ -411,28 +410,6 @@ for (const completion of ["text", "tool"] as const) { } } -test("#6329: OpenAI cache-field omission means zero cached tokens, not unmeasured", async () => { - const ctx = makeCtx([], "openai"); - const body = { model: "m", messages: [], stream: true, stream_options: { include_usage: true } }; - const raw = [ - `data: ${JSON.stringify({ choices: [{ index: 0, delta: { content: "hello" } }] })}\n\n`, - `data: ${JSON.stringify({ choices: [{ index: 0, delta: {}, finish_reason: "stop" }], usage: { prompt_tokens: 1000, completion_tokens: 10, total_tokens: 1010 } })}\n\n`, - "data: [DONE]\n\n", - ].join(""); - for await (const _chunk of runCompressLoop( - new Response(raw).body!, ctx, body, { url: "http://mock", headers: {} }, - createOpenaiAdapter(body), buildCompressSystemPrompt(), - )) {} - assert.equal(ctx.session.stats.inputTokens, 1000); - assert.equal(ctx.session.stats.cachedTokens, 0); - assert.equal(ctx.session.stats.cacheSamples, 1); - const report = buildSessionCacheReport(ctx.session); - assert.equal(report.unmeasured.samples, 0); - assert.equal(report.unmeasured.inputTokens, 0); - assert.equal(report.totals.input, 1000); - assert.equal(report.totals.cached, 0); -}); - test("loop usage: protocol unset → legacy additive behavior (prompt + cached)", async () => { const ctx = makeCtx(); await drain( diff --git a/tests/storage-archive-gc.test.ts b/tests/storage-archive-gc.test.ts index d60fa6d7c3..4180076f30 100644 --- a/tests/storage-archive-gc.test.ts +++ b/tests/storage-archive-gc.test.ts @@ -5,7 +5,6 @@ import * as os from "node:os"; import * as path from "node:path"; import { planIngest, type LegacySessionLike } from "../src/storage/ingest.ts"; import { UnifiedStore } from "../src/storage/store.ts"; -import { openSqliteDatabase } from "../src/storage/driver.ts"; import { resolveArchiveAfterDays } from "../src/knobs.ts"; /** @@ -171,43 +170,3 @@ test("gc(): age sweep archives only past-retention sessions; dormant shared rows try { fs.rmSync(dir, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 }); } catch { /* Windows stray handles */ } } }); - -test("#6317: failed archive sweep leaves no archive line; restore uses the newest entry", () => { - const dir = fs.mkdtempSync(path.join(os.tmpdir(), "storage-archive-failed-")); - try { - const store = UnifiedStore.open(path.join(dir, "index.db")); - store.ingestSession(planIngest(mkSession("sess-A", [mkMsg("m-a", "old body ".repeat(50))]))); - const archiveDir = path.join(dir, "archive"); - const logPath = path.join(archiveDir, UnifiedStore.ARCHIVE_LOG_NAME); - fs.mkdirSync(archiveDir, { recursive: true }); - - const second = openSqliteDatabase(path.join(dir, "index.db")).db; - second.exec("BEGIN IMMEDIATE"); - try { - assert.throws( - () => store.archiveSession("sess-A", { archiveDir }), - /database is locked/, - "archive fails while another writer holds the lock", - ); - } finally { - second.exec("ROLLBACK"); - second.close(); - } - assert.ok(!fs.existsSync(logPath), "failed archive must not append a stale line"); - - store.ingestSession(planIngest(mkSession("sess-A", [mkMsg("m-a", "old body ".repeat(50)), mkMsg("m-a2", "new body ".repeat(50))]))); - const archived = store.archiveSession("sess-A", { archiveDir }); - assert.ok(archived.ok); - const lines = fs.readFileSync(logPath, "utf8").split("\n").filter((l) => l.trim()); - assert.equal(lines.length, 1, "only the successful archive writes a line"); - const dump = JSON.parse(lines[0]) as { messages?: { msg_id?: string }[] }; - assert.deepEqual(dump.messages?.map((m) => m.msg_id), ["m-a", "m-a2"]); - - const restored = store.restoreArchived("sess-A", { archiveDir }); - assert.ok(restored.ok); - assert.deepEqual(store.getMessages("sess-A").map((m) => m.msgId), ["m-a", "m-a2"], "restore returns the newest dump"); - store.close(); - } finally { - try { fs.rmSync(dir, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 }); } catch { /* Windows stray handles */ } - } -}); diff --git a/tests/storage-unified.test.ts b/tests/storage-unified.test.ts index 8e4908ba87..15d2d1a04c 100644 --- a/tests/storage-unified.test.ts +++ b/tests/storage-unified.test.ts @@ -960,36 +960,3 @@ test("#4313: fork clone copies ts verbatim (inherited ledger rows keep their sta fs.rmSync(dir, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 }); } }); - -test("#6325: unchanged large session columns are not rewritten on delta ingest", () => { - const dir = fs.mkdtempSync(path.join(os.tmpdir(), "storage-6325-")); - try { - const store = UnifiedStore.open(path.join(dir, "index.db")); - const now = Date.now(); - const makeSession = (id: string, text: string): LegacySessionLike => ({ - id, - createdAt: now, - lastSeen: now, - meta: { protocol: "openai" }, - stats: {}, - metadata: { cacheLedger: { keep: "x".repeat(4096) } }, - state: { blocks: [], nextBlockId: 1, nextRunId: 1, messageRefs: { byRaw: {}, byRef: {} } }, - blockContents: new Map(), - lastMessages: [{ id: "m1", role: "user", contentType: "text", text } as unknown as CoreMessage], - lastMessagesFolded: false, - pluginSnapshot: undefined, - contentStore: undefined, - } as unknown as LegacySessionLike); - const session = makeSession("sess-6325", "hello"); - const before = store.ingestSession(planIngest(session)); - assert.ok(before); - const first = store.getSessionRow("sess-6325"); - assert.ok(first?.metadataJson?.includes("cacheLedger")); - store.ingestSession(planIngest({ ...session, lastMessages: [{ id: "m2", role: "assistant", contentType: "text", text: "world" } as unknown as CoreMessage] })); - const second = store.getSessionRow("sess-6325"); - assert.equal(second?.metadataJson, first?.metadataJson, "unchanged metadata_json should reuse the stored value"); - store.close(); - } finally { - try { fs.rmSync(dir, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 }); } catch { /* Windows stray handles */ } - } -}); From eef9f99b9cd81264e6b98da6a1d4fb0a646a6546 Mon Sep 17 00:00:00 2001 From: ci-check Date: Thu, 12 Nov 2026 17:18:16 +0800 Subject: [PATCH 07/10] =?UTF-8?q?test(loop):=20restore=20#6329=20regressio?= =?UTF-8?q?n=20=E2=80=94=20OpenAI=20cache-field=20omission=20is=20a=20meas?= =?UTF-8?q?ured=20miss?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The revert in the previous head deleted the regression test for the openaiCachedTokens fix. Restore it: an OpenAI upstream that omits prompt_tokens_details / prompt_cache_hit_tokens (sglang's behavior when cached_tokens=0) must record cachedTokens=0 and count a cache sample, not quarantine the request as unmeasured. Verified: fails on the pre-fix adapter, passes with the fix. --- tests/loop-core.test.ts | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/loop-core.test.ts b/tests/loop-core.test.ts index b0012ca38e..eb8861e842 100644 --- a/tests/loop-core.test.ts +++ b/tests/loop-core.test.ts @@ -5,6 +5,7 @@ import { createCore, createInitialState, assignRefs, emptyRefMap, defaultConfig import type { Session } from "../src/session.ts"; import { runCompressLoop, createResponsesAdapter, createOpenaiAdapter, MAX_LOOP_ROUNDS } from "../src/loop/index.ts"; import { buildCompressSystemPrompt } from "../src/compress-tool.ts"; +import { buildSessionCacheReport } from "../src/cache-ledger.ts"; import { createHandoffReply } from "../src/server/handoff-response.ts"; import type { WireProtocol } from "../src/util.ts"; import { isTransientUpstreamError, REPLAY_MAX_ATTEMPTS, replayBackoffMs, replayMaxAttempts } from "../src/fetch-util.ts"; @@ -410,6 +411,28 @@ for (const completion of ["text", "tool"] as const) { } } +test("#6329: OpenAI cache-field omission means zero cached tokens, not unmeasured", async () => { + const ctx = makeCtx([], "openai"); + const body = { model: "m", messages: [], stream: true, stream_options: { include_usage: true } }; + const raw = [ + `data: ${JSON.stringify({ choices: [{ index: 0, delta: { content: "hello" } }] })}\n\n`, + `data: ${JSON.stringify({ choices: [{ index: 0, delta: {}, finish_reason: "stop" }], usage: { prompt_tokens: 1000, completion_tokens: 10, total_tokens: 1010 } })}\n\n`, + "data: [DONE]\n\n", + ].join(""); + for await (const _chunk of runCompressLoop( + new Response(raw).body!, ctx, body, { url: "http://mock", headers: {} }, + createOpenaiAdapter(body), buildCompressSystemPrompt(), + )) {} + assert.equal(ctx.session.stats.inputTokens, 1000); + assert.equal(ctx.session.stats.cachedTokens, 0); + assert.equal(ctx.session.stats.cacheSamples, 1); + const report = buildSessionCacheReport(ctx.session); + assert.equal(report.unmeasured.samples, 0); + assert.equal(report.unmeasured.inputTokens, 0); + assert.equal(report.totals.input, 1000); + assert.equal(report.totals.cached, 0); +}); + test("loop usage: protocol unset → legacy additive behavior (prompt + cached)", async () => { const ctx = makeCtx(); await drain( From 5f5e2e802259476305ecbeb7d9586d15ab9a6427 Mon Sep 17 00:00:00 2001 From: myNameIsZhangYu Date: Thu, 12 Nov 2026 16:57:36 +0800 Subject: [PATCH 08/10] Revert "test: regenerate storage-render goldens for OpenAI cache attribution change" This reverts commit bc2be40557c69477038bfda5adb43fae51b26cc7. --- src/storage/mysql.ts | 12 +++++- src/storage/schema.ts | 14 ++++--- src/storage/store.ts | 34 +++++++++------- src/update.ts | 54 +++++++++++++++++++------- tests/golden/storage-render/chat.json | 8 ++-- tests/issue3250-wal-checkpoint.test.ts | 9 +++++ tests/loop-core.test.ts | 1 + tests/storage-archive-gc.test.ts | 41 +++++++++++++++++++ tests/storage-unified.test.ts | 33 ++++++++++++++++ 9 files changed, 168 insertions(+), 38 deletions(-) diff --git a/src/storage/mysql.ts b/src/storage/mysql.ts index b3005bb5f4..7107af15cc 100644 --- a/src/storage/mysql.ts +++ b/src/storage/mysql.ts @@ -1794,9 +1794,13 @@ export class MysqlBackend implements StorageBackend { if (c !== null) content[hash] = c; } mkdirSync(opts.archiveDir, { recursive: true }); - appendFileSync(exportPath, JSON.stringify({ version: 1, sessionId, archivedAt: Date.now(), session: row, content, messages, mediaMaps, blocks, refs: refsRows, deadRefs, ccrEntries: ccr, sessionExt: ext, checkpoints }) + "\n"); + const archiveLine = JSON.stringify({ version: 1, sessionId, archivedAt: Date.now(), session: row, content, messages, mediaMaps, blocks, refs: refsRows, deadRefs, ccrEntries: ccr, sessionExt: ext, checkpoints }) + "\n"; let contentRowsDeleted = 0; await this.tx(async (c) => { + // #6317: serialize the export inside the sweep transaction. A + // locked or failed sweep must leave NO archive line, or restore + // will later prefer that stale dump and lose newer messages. + appendFileSync(exportPath, archiveLine); const rel: string[] = []; for (const m of messages) rel.push(str(m.content_hash)); for (const mm of mediaMaps) rel.push(str(mm.media_hash)); @@ -1843,6 +1847,7 @@ export class MysqlBackend implements StorageBackend { checkpoints: Record[]; }; let lineNo = -1; + let matchCount = 0; let rawLines: string[] = []; try { rawLines = readFileSync(exportPath, "utf8").split("\n"); @@ -1855,8 +1860,8 @@ export class MysqlBackend implements StorageBackend { continue; } if (candidate.sessionId === sessionId) { + matchCount += 1; lineNo = i; - break; } } } catch { @@ -1869,6 +1874,9 @@ export class MysqlBackend implements StorageBackend { return { ok: false, reason: "export-corrupt" }; } if (!dump.session || dump.session.sessionId !== sessionId) return { ok: false, reason: "export-mismatch" }; + if (matchCount > 1) { + loggerLog("warn", `[#6317] archive log contained ${matchCount} entries for session ${sessionId}; restoring the newest entry and removing all entries`); + } const sess = dump.session; await this.tx(async (c) => { for (const [hash, body] of Object.entries(dump.content)) await this.putContent(c, body, hash); diff --git a/src/storage/schema.ts b/src/storage/schema.ts index f0471e0681..a68fad1234 100644 --- a/src/storage/schema.ts +++ b/src/storage/schema.ts @@ -102,16 +102,20 @@ export const WAL_JOURNAL_SIZE_LIMIT = 8 * 1024 * 1024; * short-lived openers (export fast path) and tests apply. */ export function storagePragmas(walJournalSizeLimitBytes: number): string[] { return [ + // #3901/#3463: freed pages return to the pool incrementally so an + // archive/GC sweep actually shrinks the file going forward. SQLite + // honors auto_vacuum only while the database file is still empty, so + // it must precede WAL: journal_mode=WAL writes page 1 immediately, + // after which a later auto_vacuum pragma is silently ignored (#6323). + "PRAGMA auto_vacuum = INCREMENTAL", "PRAGMA journal_mode = WAL", "PRAGMA synchronous = NORMAL", "PRAGMA busy_timeout = 5000", "PRAGMA foreign_keys = ON", `PRAGMA journal_size_limit = ${walJournalSizeLimitBytes}`, - // #3901/#3463: freed pages return to the pool incrementally so an - // archive/GC sweep actually shrinks the file going forward. Databases - // created before this pragma keep their original layout until a - // one-time full VACUUM (gc({vacuum:"full"}) / ?vacuum=full). - "PRAGMA auto_vacuum = INCREMENTAL", + // Databases created before this ordering fix keep their original + // layout until a one-time full VACUUM (gc({vacuum:"full"}) / + // ?vacuum=full). ]; } diff --git a/src/storage/store.ts b/src/storage/store.ts index 97c674f438..fb7207a347 100644 --- a/src/storage/store.ts +++ b/src/storage/store.ts @@ -1025,10 +1025,10 @@ export class UnifiedStore { next_run_id = excluded.next_run_id, meta_json = excluded.meta_json, stats_json = excluded.stats_json, - metadata_json = excluded.metadata_json, - kstate_json = excluded.kstate_json, + metadata_json = CASE WHEN metadata_json IS excluded.metadata_json THEN sessions.metadata_json ELSE excluded.metadata_json END, + kstate_json = CASE WHEN kstate_json IS excluded.kstate_json THEN sessions.kstate_json ELSE excluded.kstate_json END, tail_json = excluded.tail_json, - ccr_json = excluded.ccr_json, + ccr_json = CASE WHEN ccr_json IS excluded.ccr_json THEN sessions.ccr_json ELSE excluded.ccr_json END, last_ingested_seq = excluded.last_ingested_seq, orphans_json = excluded.orphans_json`, ) @@ -2076,13 +2076,14 @@ export class UnifiedStore { if (c) content[hash as string] = Buffer.from(c.bytes).toString("utf8"); } mkdirSync(opts.archiveDir, { recursive: true }); - // One JSONL line per archived session: append-only (cheap, crash = - // at worst one truncated line which the reader skips), restore - // rewrites the file without the restored session's lines. - appendFileSync(exportPath, JSON.stringify({ version: 1, sessionId, archivedAt: Date.now(), session: row, content, messages, mediaMaps, blocks, refs: refsRows, deadRefs, ccrEntries: ccr, sessionExt: ext, checkpoints }) + "\n"); + const archiveLine = JSON.stringify({ version: 1, sessionId, archivedAt: Date.now(), session: row, content, messages, mediaMaps, blocks, refs: refsRows, deadRefs, ccrEntries: ccr, sessionExt: ext, checkpoints }) + "\n"; // --- sweep the row bodies (refcount discipline; pinned checkpoints survive) --- let contentRowsDeleted = 0; this.db.transaction(() => { + // #6317: serialize the export inside the sweep transaction. A + // locked or failed sweep must leave NO archive line, or restore + // will later prefer that stale dump and lose newer messages. + appendFileSync(exportPath, archiveLine); const rel: string[] = []; for (const m of messages) rel.push(m.content_hash as string); for (const mm of mediaMaps) rel.push(mm.media_hash as string); @@ -2125,10 +2126,13 @@ export class UnifiedStore { sessionExt: Record[]; checkpoints: Record[]; }; - // Scan the JSONL for the session's line (skip malformed/truncated + // Scan the JSONL for the session's NEWEST line (#6317: a pre-fix + // failed archive can leave stale lines behind; a later successful + // archive appends the newest dump). Skip malformed/truncated // trailing lines — an archive append interrupted by a crash must not - // block restoring OTHER sessions). + // block restoring OTHER sessions. let lineNo = -1; + let matchCount = 0; let rawLines: string[] = []; try { rawLines = readFileSync(exportPath, "utf8").split("\n"); @@ -2140,7 +2144,7 @@ export class UnifiedStore { } catch { continue; } - if (candidate.sessionId === sessionId) { lineNo = i; break; } + if (candidate.sessionId === sessionId) { matchCount += 1; lineNo = i; } } } catch { return { ok: false, reason: "export-corrupt" }; @@ -2152,6 +2156,9 @@ export class UnifiedStore { return { ok: false, reason: "export-corrupt" }; } if (!dump.session || dump.session.sessionId !== sessionId) return { ok: false, reason: "export-mismatch" }; + if (matchCount > 1) { + loggerLog("warn", `[#6317] archive log contained ${matchCount} entries for session ${sessionId}; restoring the newest entry and removing all entries`); + } const sess: NonNullable = dump.session; this.db.transaction(() => { for (const [hash, body] of Object.entries(dump.content)) this.putContent(Buffer.from(body, "utf8"), hash); @@ -2191,9 +2198,10 @@ export class UnifiedStore { sess.lastIngestedSeq ?? null, sess.orphansJson ?? null, sessionId, ); }); - // Drop the restored session's line(s) from the JSONL (a same-session - // archive can theoretically appear more than once across reinstalls; - // rewrite removes them all). Other sessions' lines are untouched. + // Drop the restored session's line(s) from the JSONL. #6317: a + // pre-fix failed archive can leave more than one line; restore now + // already used the newest match and removes all matches, warning when + // stale lines were present so the loss pattern is diagnosable. const kept: string[] = []; for (const l of rawLines) { if (!l.trim()) continue; diff --git a/src/update.ts b/src/update.ts index 5a7b28e828..37dd062907 100644 --- a/src/update.ts +++ b/src/update.ts @@ -2079,21 +2079,47 @@ export async function installViaTarball( } }; - // Copy staging over install dir using Node's built-in fs.cp (Node 16.7+). - // Cross-platform — no dependency on the `cp` binary (absent on Windows). - // `recursive: true` + the trailing `/.` semantics: fs.cp copies the - // *contents* of stagingDir into installDir, merging without nesting. - let copyError: string | null = null; + // #6309: prefer an atomic directory swap. Copy staging next to the + // install dir on the same volume, then rename the current install aside + // and rename the prepared copy into place. The old copy is the rollback + // source and is only removed after verification. If rename is not + // possible (common on Windows while the running install holds handles), + // fall back to the previous merge-copy path. + const swapCandidate = `${installDir}.update-new-${Date.now()}`; + let copiedForSwap = false; try { - await cp(stagingDir, installDir, { recursive: true, force: true }); - } catch (e) { - copyError = `failed to copy to install dir: ${String(e)}`; - } finally { + await cp(stagingDir, swapCandidate, { recursive: true, force: true }); + copiedForSwap = true; + await rename(installDir, backupDir); + try { + await rename(swapCandidate, installDir); + } catch (e) { + const swapError = String(e); + await rename(backupDir, installDir); + throw new Error(swapError); + } await rm(stagingDir, { recursive: true, force: true }); - } - if (copyError !== null) { - const rb = await restoreFromBackup(); - return { ok: false, error: rb ?? copyError }; + } catch { + if (copiedForSwap) await rm(swapCandidate, { recursive: true, force: true }); + if (!pnpmOldLink) { + try { + await cp(installDir, backupDir, { recursive: true, force: true }); + } catch (e) { + return { ok: false, error: `backup of current install failed (install left untouched): ${String(e)}` }; + } + } + let copyError: string | null = null; + try { + await cp(stagingDir, installDir, { recursive: true, force: true }); + } catch (e) { + copyError = `failed to copy to install dir: ${String(e)}`; + } finally { + await rm(stagingDir, { recursive: true, force: true }); + } + if (copyError !== null) { + const rb = await restoreFromBackup(); + return { ok: false, error: rb ?? copyError }; + } } // Final verification: version must match and every declared entry must @@ -2131,7 +2157,7 @@ export async function installViaTarball( // Success: the backup is no longer needed, and neither is the displaced // pnpm link (the store copy underneath it was left byte-identical). - await rm(backupDir, { recursive: true, force: true }); + if (!pnpmOldLink) await rm(backupDir, { recursive: true, force: true }); if (pnpmOldLink) { try { await unlink(pnpmOldLink); diff --git a/tests/golden/storage-render/chat.json b/tests/golden/storage-render/chat.json index ae4923dd78..e0f1b4aa96 100644 --- a/tests/golden/storage-render/chat.json +++ b/tests/golden/storage-render/chat.json @@ -95,13 +95,13 @@ "foldAt": 9, "editedAt": 12, "summaryFrom": 9, - "stateDigest": "a7b5a67f54a2cbd7978e2a57ad78f522f7af8f94a4e2402898aee4d93479541a", - "stateDigestAligned": "76f701f5ab1631fe1615a3f1cd8fa0fcd0a3fa19c02658cc5d077ae565f4897a", + "stateDigest": "97b9991ae7a398fea5d620994df0101001d9c3f246615daac95e6afb4cee14a0", + "stateDigestAligned": "fc632ea30ecb84eeb10e2e0bb4d525f06b17ebd60c6296625d71f398b569ed0b", "sectionDigests": { "state": "90857fbf82001a2b", "blockContents": "94a58f96d7c5cfc3", - "stats": "5c2f0fe203d710e8", - "metadata": "0c7d1819e0d4c0f6" + "stats": "3e6858671f4b9a9d", + "metadata": "faa2b34b71823646" }, "stateKeyDigests": { "blocks": "d9770feee91dafc6", diff --git a/tests/issue3250-wal-checkpoint.test.ts b/tests/issue3250-wal-checkpoint.test.ts index e989760ae6..d5affc667d 100644 --- a/tests/issue3250-wal-checkpoint.test.ts +++ b/tests/issue3250-wal-checkpoint.test.ts @@ -334,3 +334,12 @@ test("#3250: a configured cap flows open() → journal_size_limit (custom limit rmBestEffort(dir); } }); + +test("#6323: auto_vacuum precedes WAL so fresh databases honor INCREMENTAL", () => { + const { db } = openSqliteDatabase(":memory:"); + for (const pragma of STORAGE_PRAGMAS) db.exec(pragma); + migrateSchema(db); + const row = db.prepare("PRAGMA auto_vacuum").get() as Record; + assert.equal(Object.values(row)[0], 2, "auto_vacuum should be INCREMENTAL (2)"); + db.close(); +}); diff --git a/tests/loop-core.test.ts b/tests/loop-core.test.ts index eb8861e842..2c08f2e531 100644 --- a/tests/loop-core.test.ts +++ b/tests/loop-core.test.ts @@ -4,6 +4,7 @@ import type { Config, CoreMessage } from "acp-kernel"; import { createCore, createInitialState, assignRefs, emptyRefMap, defaultConfig } from "acp-kernel"; import type { Session } from "../src/session.ts"; import { runCompressLoop, createResponsesAdapter, createOpenaiAdapter, MAX_LOOP_ROUNDS } from "../src/loop/index.ts"; +import { buildSessionCacheReport } from "../src/cache-ledger.ts"; import { buildCompressSystemPrompt } from "../src/compress-tool.ts"; import { buildSessionCacheReport } from "../src/cache-ledger.ts"; import { createHandoffReply } from "../src/server/handoff-response.ts"; diff --git a/tests/storage-archive-gc.test.ts b/tests/storage-archive-gc.test.ts index 4180076f30..d60fa6d7c3 100644 --- a/tests/storage-archive-gc.test.ts +++ b/tests/storage-archive-gc.test.ts @@ -5,6 +5,7 @@ import * as os from "node:os"; import * as path from "node:path"; import { planIngest, type LegacySessionLike } from "../src/storage/ingest.ts"; import { UnifiedStore } from "../src/storage/store.ts"; +import { openSqliteDatabase } from "../src/storage/driver.ts"; import { resolveArchiveAfterDays } from "../src/knobs.ts"; /** @@ -170,3 +171,43 @@ test("gc(): age sweep archives only past-retention sessions; dormant shared rows try { fs.rmSync(dir, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 }); } catch { /* Windows stray handles */ } } }); + +test("#6317: failed archive sweep leaves no archive line; restore uses the newest entry", () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "storage-archive-failed-")); + try { + const store = UnifiedStore.open(path.join(dir, "index.db")); + store.ingestSession(planIngest(mkSession("sess-A", [mkMsg("m-a", "old body ".repeat(50))]))); + const archiveDir = path.join(dir, "archive"); + const logPath = path.join(archiveDir, UnifiedStore.ARCHIVE_LOG_NAME); + fs.mkdirSync(archiveDir, { recursive: true }); + + const second = openSqliteDatabase(path.join(dir, "index.db")).db; + second.exec("BEGIN IMMEDIATE"); + try { + assert.throws( + () => store.archiveSession("sess-A", { archiveDir }), + /database is locked/, + "archive fails while another writer holds the lock", + ); + } finally { + second.exec("ROLLBACK"); + second.close(); + } + assert.ok(!fs.existsSync(logPath), "failed archive must not append a stale line"); + + store.ingestSession(planIngest(mkSession("sess-A", [mkMsg("m-a", "old body ".repeat(50)), mkMsg("m-a2", "new body ".repeat(50))]))); + const archived = store.archiveSession("sess-A", { archiveDir }); + assert.ok(archived.ok); + const lines = fs.readFileSync(logPath, "utf8").split("\n").filter((l) => l.trim()); + assert.equal(lines.length, 1, "only the successful archive writes a line"); + const dump = JSON.parse(lines[0]) as { messages?: { msg_id?: string }[] }; + assert.deepEqual(dump.messages?.map((m) => m.msg_id), ["m-a", "m-a2"]); + + const restored = store.restoreArchived("sess-A", { archiveDir }); + assert.ok(restored.ok); + assert.deepEqual(store.getMessages("sess-A").map((m) => m.msgId), ["m-a", "m-a2"], "restore returns the newest dump"); + store.close(); + } finally { + try { fs.rmSync(dir, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 }); } catch { /* Windows stray handles */ } + } +}); diff --git a/tests/storage-unified.test.ts b/tests/storage-unified.test.ts index 15d2d1a04c..8e4908ba87 100644 --- a/tests/storage-unified.test.ts +++ b/tests/storage-unified.test.ts @@ -960,3 +960,36 @@ test("#4313: fork clone copies ts verbatim (inherited ledger rows keep their sta fs.rmSync(dir, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 }); } }); + +test("#6325: unchanged large session columns are not rewritten on delta ingest", () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "storage-6325-")); + try { + const store = UnifiedStore.open(path.join(dir, "index.db")); + const now = Date.now(); + const makeSession = (id: string, text: string): LegacySessionLike => ({ + id, + createdAt: now, + lastSeen: now, + meta: { protocol: "openai" }, + stats: {}, + metadata: { cacheLedger: { keep: "x".repeat(4096) } }, + state: { blocks: [], nextBlockId: 1, nextRunId: 1, messageRefs: { byRaw: {}, byRef: {} } }, + blockContents: new Map(), + lastMessages: [{ id: "m1", role: "user", contentType: "text", text } as unknown as CoreMessage], + lastMessagesFolded: false, + pluginSnapshot: undefined, + contentStore: undefined, + } as unknown as LegacySessionLike); + const session = makeSession("sess-6325", "hello"); + const before = store.ingestSession(planIngest(session)); + assert.ok(before); + const first = store.getSessionRow("sess-6325"); + assert.ok(first?.metadataJson?.includes("cacheLedger")); + store.ingestSession(planIngest({ ...session, lastMessages: [{ id: "m2", role: "assistant", contentType: "text", text: "world" } as unknown as CoreMessage] })); + const second = store.getSessionRow("sess-6325"); + assert.equal(second?.metadataJson, first?.metadataJson, "unchanged metadata_json should reuse the stored value"); + store.close(); + } finally { + try { fs.rmSync(dir, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 }); } catch { /* Windows stray handles */ } + } +}); From bd07267b6b8a319e731c73acc23908f2949d6010 Mon Sep 17 00:00:00 2001 From: myNameIsZhangYu Date: Thu, 12 Nov 2026 18:11:57 +0800 Subject: [PATCH 09/10] test: regenerate storage-render goldens (chat) for OpenAI cache attribution change Regenerated only tests/golden/storage-render/chat.json after restoring the four storage/update fixes that were accidentally reverted during a baseline comparison. This commit contains only the golden file change. --- tests/golden/storage-render/chat.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/golden/storage-render/chat.json b/tests/golden/storage-render/chat.json index e0f1b4aa96..ae4923dd78 100644 --- a/tests/golden/storage-render/chat.json +++ b/tests/golden/storage-render/chat.json @@ -95,13 +95,13 @@ "foldAt": 9, "editedAt": 12, "summaryFrom": 9, - "stateDigest": "97b9991ae7a398fea5d620994df0101001d9c3f246615daac95e6afb4cee14a0", - "stateDigestAligned": "fc632ea30ecb84eeb10e2e0bb4d525f06b17ebd60c6296625d71f398b569ed0b", + "stateDigest": "a7b5a67f54a2cbd7978e2a57ad78f522f7af8f94a4e2402898aee4d93479541a", + "stateDigestAligned": "76f701f5ab1631fe1615a3f1cd8fa0fcd0a3fa19c02658cc5d077ae565f4897a", "sectionDigests": { "state": "90857fbf82001a2b", "blockContents": "94a58f96d7c5cfc3", - "stats": "3e6858671f4b9a9d", - "metadata": "faa2b34b71823646" + "stats": "5c2f0fe203d710e8", + "metadata": "0c7d1819e0d4c0f6" }, "stateKeyDigests": { "blocks": "d9770feee91dafc6", From cac80437e4efea890fcd47beb6617e5a9cb33db7 Mon Sep 17 00:00:00 2001 From: myNameIsZhangYu Date: Thu, 12 Nov 2026 18:37:02 +0800 Subject: [PATCH 10/10] fix(test): remove duplicate buildSessionCacheReport import --- tests/loop-core.test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/loop-core.test.ts b/tests/loop-core.test.ts index 2c08f2e531..e420605678 100644 --- a/tests/loop-core.test.ts +++ b/tests/loop-core.test.ts @@ -6,7 +6,6 @@ import type { Session } from "../src/session.ts"; import { runCompressLoop, createResponsesAdapter, createOpenaiAdapter, MAX_LOOP_ROUNDS } from "../src/loop/index.ts"; import { buildSessionCacheReport } from "../src/cache-ledger.ts"; import { buildCompressSystemPrompt } from "../src/compress-tool.ts"; -import { buildSessionCacheReport } from "../src/cache-ledger.ts"; import { createHandoffReply } from "../src/server/handoff-response.ts"; import type { WireProtocol } from "../src/util.ts"; import { isTransientUpstreamError, REPLAY_MAX_ATTEMPTS, replayBackoffMs, replayMaxAttempts } from "../src/fetch-util.ts";