diff --git a/src/persist.ts b/src/persist.ts index 2746ad49f4..cbc0059ea2 100644 --- a/src/persist.ts +++ b/src/persist.ts @@ -202,6 +202,8 @@ interface PersistedSession { cacheSamples?: number; lastInputTokens?: number; lastInputTokensSource?: string; + emergencyShrinkAt?: number; + emergencyShrinkEstimate?: number; lastInputTokensOrigin?: string; overflowArmTokens?: number; overflowArmCompletionReserve?: number; @@ -222,7 +224,8 @@ interface PersistedSession { lastUsageGradeTokens?: number; calibratedEstimate?: number; calibratedEstimateOrigin?: string; - calibrationRing?: { origin: string; values: number[] }; + calibratedEstimateModel?: string; + calibrationRing?: { origin: string; model?: string; values: number[] }; pendingFoldUsage?: boolean; lastLocalTextEstimate?: number; lastLocalTextEstimateOrigin?: string; @@ -4198,14 +4201,15 @@ function restoreCalibratedEstimate(v: unknown): number | undefined { // itself would admit (finite, within the plausibility band) and only the newest // window-size entries; anything else drops the ring wholesale so a malformed // file can neither teach a factor nor mask a fresh learning run on restart. -function restoreCalibrationRing(v: unknown): { origin: string; values: number[] } | undefined { +function restoreCalibrationRing(v: unknown): { origin: string; model?: string; values: number[] } | undefined { if (!v || typeof v !== "object") return undefined; - const ring = v as { origin?: unknown; values?: unknown }; + const ring = v as { origin?: unknown; model?: unknown; values?: unknown }; if (typeof ring.origin !== "string" || ring.origin.length === 0) return undefined; if (!Array.isArray(ring.values)) return undefined; const values = ring.values.filter((x): x is number => typeof x === "number" && Number.isFinite(x) && x >= CALIBRATION_SAMPLE_MIN && x <= CALIBRATION_SAMPLE_MAX); const recent = values.slice(-CALIBRATION_SAMPLE_WINDOW); - return recent.length > 0 ? { origin: ring.origin, values: recent } : undefined; + const model = typeof ring.model === "string" && ring.model.trim().length > 0 ? ring.model : undefined; + return recent.length > 0 ? { origin: ring.origin, ...(model === undefined ? {} : { model }), values: recent } : undefined; } function restoreBlockContents(entries: Iterable): Map { @@ -4266,6 +4270,10 @@ function buildSession(parsed: PersistedSession): Session { // #857: provenance — legacy files lack it; absent stays absent and // evidence-grade consumers treat absent as untrusted. lastInputTokensSource: stats.lastInputTokensSource === "usage" || stats.lastInputTokensSource === "estimate" || stats.lastInputTokensSource === "overflow-arm" ? stats.lastInputTokensSource : undefined, + // Preserve the explanation saved by armFailureShrink alongside its + // raised baseline. A real usage settlement clears both markers. + emergencyShrinkAt: typeof stats.emergencyShrinkAt === "number" && Number.isFinite(stats.emergencyShrinkAt) && stats.emergencyShrinkAt >= 0 ? stats.emergencyShrinkAt : undefined, + emergencyShrinkEstimate: typeof stats.emergencyShrinkEstimate === "number" && Number.isFinite(stats.emergencyShrinkEstimate) && stats.emergencyShrinkEstimate > 0 ? stats.emergencyShrinkEstimate : undefined, // #1110: one-shot overflow arm — legacy files lack it; absent = no arm. overflowArmTokens: typeof stats.overflowArmTokens === "number" && Number.isFinite(stats.overflowArmTokens) && stats.overflowArmTokens > 0 ? stats.overflowArmTokens : undefined, // #3450: proven completion reservation + one-time warn flag — legacy files lack both. @@ -4291,6 +4299,7 @@ function buildSession(parsed: PersistedSession): Session { lastInputTokensOrigin: typeof stats.lastInputTokensOrigin === "string" && stats.lastInputTokensOrigin.trim().length > 0 ? stats.lastInputTokensOrigin : undefined, calibratedEstimate: restoreCalibratedEstimate(stats.calibratedEstimate), calibratedEstimateOrigin: typeof stats.calibratedEstimateOrigin === "string" && stats.calibratedEstimateOrigin.trim().length > 0 ? stats.calibratedEstimateOrigin : undefined, + calibratedEstimateModel: typeof stats.calibratedEstimateModel === "string" && stats.calibratedEstimateModel.trim().length > 0 ? stats.calibratedEstimateModel : undefined, calibrationRing: restoreCalibrationRing(stats.calibrationRing), // Intentionally NOT restored (#2129 triage): pendingFoldUsage is set at // compress execution and consumed only as a log-suffix attribution by the diff --git a/tests/golden/storage-render/anthropic.json b/tests/golden/storage-render/anthropic.json index 2ebb2f40a2..779a27e556 100644 --- a/tests/golden/storage-render/anthropic.json +++ b/tests/golden/storage-render/anthropic.json @@ -95,12 +95,12 @@ "foldAt": 9, "editedAt": 12, "summaryFrom": 9, - "stateDigest": "97c49de03bb6c42c99a9d0f03ce7cef247b4cb034ee1da24d5954207225bb341", - "stateDigestAligned": "2f913ebef14626bd7923a11fffc9676e2919664f2a5e1649b456bf03ec09cb29", + "stateDigest": "42c35ba8831b61047e7e978190e716a1a33ead46eeafc1a11e3c93edb5290b99", + "stateDigestAligned": "c2919947d05d5e7a4bd87d953b873a15a1c9f8eb8c87855d345cd5041c4c83de", "sectionDigests": { "state": "56e66f6b72c7f0dd", "blockContents": "dc7d13d250c94bb4", - "stats": "e83198a55e6b2c29", + "stats": "7bab3f342a5398ab", "metadata": "5d34f4fe94270d09" }, "stateKeyDigests": { diff --git a/tests/golden/storage-render/chat.json b/tests/golden/storage-render/chat.json index 0469c7da94..91a3e3f0b7 100644 --- a/tests/golden/storage-render/chat.json +++ b/tests/golden/storage-render/chat.json @@ -95,12 +95,12 @@ "foldAt": 9, "editedAt": 12, "summaryFrom": 9, - "stateDigest": "accd7fe100acd1e3adf3b6186b4c709085f4e229ea2d8d553f903cf30f97b42c", - "stateDigestAligned": "4d6b17bddffef0fbaf73bd585a8a702c34a866c105719da5fa16bc90da1be38b", + "stateDigest": "11146c571e821244043e57cc91fa880c904423a1e1976a2d9d606f608a74a979", + "stateDigestAligned": "5f60be1a36988f35de1944bcaa30178c9ff46d4dc099760cc7e84640a81243df", "sectionDigests": { "state": "90857fbf82001a2b", "blockContents": "94a58f96d7c5cfc3", - "stats": "5c2f0fe203d710e8", + "stats": "fecf844aec250fc2", "metadata": "5e869987612d599b" }, "stateKeyDigests": { diff --git a/tests/golden/storage-render/google.json b/tests/golden/storage-render/google.json index 7278d9ff2d..8df3785946 100644 --- a/tests/golden/storage-render/google.json +++ b/tests/golden/storage-render/google.json @@ -95,12 +95,12 @@ "foldAt": 9, "editedAt": 12, "summaryFrom": 9, - "stateDigest": "a9769a0169874c9b501b27a73cf721dcb57097969e974f050a3fa68a741b17fd", - "stateDigestAligned": "3db9797c6435542e793cfd27d4d5d9a76d49474289601d07c90d65fa9d567112", + "stateDigest": "0c38240af94862bb1dde24ca679098f914e2b4f48b809e6f5fd89a9cd50e572a", + "stateDigestAligned": "f40a87a4be7dfc050b95b30f7cc7ba21326dbceea69e2ed76637783fd39b51c3", "sectionDigests": { "state": "c5243ba17364d3ac", "blockContents": "473264d9df8ba338", - "stats": "a3c84817f1d0c3dd", + "stats": "794b12c5699ae15a", "metadata": "631282b79ac9672c" }, "stateKeyDigests": { diff --git a/tests/golden/storage-render/responses.json b/tests/golden/storage-render/responses.json index c1a8cdf618..4a062a4b5b 100644 --- a/tests/golden/storage-render/responses.json +++ b/tests/golden/storage-render/responses.json @@ -95,12 +95,12 @@ "foldAt": 9, "editedAt": 12, "summaryFrom": 9, - "stateDigest": "afe47188cdeafd5b6670fe757632e67d04bb05cb6e7c2a6971f0d498199e86a8", - "stateDigestAligned": "ed7a20e60d65949c92e27963ca373fc3481ef423ee9f6e33f67bcc213f964e50", + "stateDigest": "10642618204e57f051b7784d9ab87428a986d40cc6f99ce99e6a9e85b0b4377d", + "stateDigestAligned": "52674f6fc691140ed81bd5843babcb9fa4a4243795714c0a818c819965a599b5", "sectionDigests": { "state": "e686a14f31d69ce5", "blockContents": "b6701268a0bdcd8a", - "stats": "099bde0a9cf64cad", + "stats": "e8ebcb92d0bb0878", "metadata": "5b0d05c2e6677cac" }, "stateKeyDigests": { diff --git a/tests/persist-calibration-model.test.ts b/tests/persist-calibration-model.test.ts new file mode 100644 index 0000000000..85de57705c --- /dev/null +++ b/tests/persist-calibration-model.test.ts @@ -0,0 +1,80 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import {mkdtempSync, readdirSync, readFileSync, writeFileSync} from 'node:fs'; +import {tmpdir} from 'node:os'; +import path from 'node:path'; +import {createInitialState} from 'acp-kernel'; +import {SessionStore} from '../src/persist.ts'; +import type {Session} from '../src/session.ts'; +import {currentCalibrationFactor} from '../src/util.ts'; +import {rmrf} from './tmp-rm.ts'; + +process.env.BILI_CONFIG_FILE = path.join(tmpdir(), 'calibration-model-no-config.json'); +process.env.BILI_PERSIST_ZSTD = '0'; + +function session(id: string, model?: string): Session { + return {id, meta:{protocol:'anthropic',upstreamOrigin:'http://synthetic.invalid'}, + stats:{requests:3,tokensSaved:0,inputTokens:10,cachedTokens:0,outputTokens:5, + cacheSamples:0,lastInputTokens:10,contextTokens:1000,compressCreditTokens:0, + retrieveCalls:0,retrieveHits:0,retrieveMisses:0,storedBytes:0,storeBytesSaved:0,rangeRestores:0, + calibratedEstimate:0.8,calibratedEstimateOrigin:'http://synthetic.invalid', + calibratedEstimateModel:model, + calibrationRing:{origin:'http://synthetic.invalid',...(model === undefined ? {} : {model}),values:[0.8,0.8]}}, + metadata:{},createdAt:1700000000000,lastSeen:1700000001000,state:createInitialState(), + blockContents:new Map(),inFlight:0,persisted:false,pendingRetrievals:[]}; +} + +for (const mode of ['legacy','full']) { + for (const model of ['synthetic-model-a', undefined]) { + test(`calibration provenance survives ${mode} cold load (${model ?? 'legacy unknown model'})`, async () => { + process.env.BILI_STORAGE_UNIFIED = mode; + const dir = mkdtempSync(path.join(tmpdir(),'bili-calibration-model-')); + const original = session('calibration-roundtrip',model); + let store = new SessionStore({dir,enabled:true,debounceMs:1}); + try { + await store.writeNow(original); + store.closeQuietly(); + store = new SessionStore({dir,enabled:true,debounceMs:1}); + const restored = store.loadSync(original.id,original.meta); + assert.ok(restored); + assert.deepEqual(restored.stats.calibrationRing,original.stats.calibrationRing, + 'Cold load must retain the ring including its exact model provenance'); + assert.equal(restored.stats.calibratedEstimateModel,model); + assert.equal(currentCalibrationFactor(restored.stats,'synthetic-model-a'),0.8); + assert.equal(currentCalibrationFactor(restored.stats,'synthetic-model-b'), + model === undefined ? 0.8 : undefined, + 'Known-model factors must not cross a model switch; unknown legacy models retain existing behavior'); + } finally {store.closeQuietly();rmrf(dir);} + }); + } +} + +for (const invalid of [123,' ',{},null]) { + test(`invalid persisted model provenance is rejected (${JSON.stringify(invalid)})`, async () => { + process.env.BILI_STORAGE_UNIFIED = 'legacy'; + const dir = mkdtempSync(path.join(tmpdir(),'bili-calibration-invalid-model-')); + const original = session('invalid-provenance','synthetic-model-a'); + let store = new SessionStore({dir,enabled:true,debounceMs:1}); + try { + await store.writeNow(original); + store.closeQuietly(); + const files = readdirSync(dir,{recursive:true}).filter(x => typeof x === 'string' && x.endsWith('.json')); + const file = files.map(x => path.join(dir,String(x))).find(x => { + const raw = JSON.parse(readFileSync(x,'utf8')); + return (raw.payload ?? raw).id === original.id; + }); + assert.ok(file,'Fixture must have written its session envelope'); + const raw = JSON.parse(readFileSync(file,'utf8')); + const payload = raw.payload ?? raw; + payload.stats.calibrationRing.model = invalid; + payload.stats.calibratedEstimateModel = invalid; + writeFileSync(file,JSON.stringify(raw)); + store = new SessionStore({dir,enabled:true,debounceMs:1}); + const restored = store.loadSync(original.id,original.meta); + assert.ok(restored); + assert.equal(restored.stats.calibrationRing?.model,undefined); + assert.equal(restored.stats.calibratedEstimateModel,undefined); + assert.deepEqual(restored.stats.calibrationRing?.values,[0.8,0.8]); + } finally {store.closeQuietly();rmrf(dir);} + }); +} diff --git a/tests/persist-emergency-marker.test.ts b/tests/persist-emergency-marker.test.ts new file mode 100644 index 0000000000..b5ddd9f650 --- /dev/null +++ b/tests/persist-emergency-marker.test.ts @@ -0,0 +1,41 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import {mkdtempSync} from 'node:fs'; +import {tmpdir} from 'node:os'; +import path from 'node:path'; +import {createInitialState} from 'acp-kernel'; +import {SessionStore} from '../src/persist.ts'; +import type {Session} from '../src/session.ts'; +import {rmrf} from './tmp-rm.ts'; + +for(const mode of ['legacy','full']){ + for(const armed of [true,false]){ + test(`failure-shrink status evidence survives ${mode} restart (armed=${armed})`,async()=>{ + process.env.BILI_STORAGE_UNIFIED=mode; + process.env.BILI_PERSIST_ZSTD='0'; + const dir=mkdtempSync(path.join(tmpdir(),'bili-shrink-marker-test-')); + process.env.BILI_CONFIG_FILE=path.join(dir,'absent-config'); + const session:Session={id:'synthetic-shrink-marker',meta:{protocol:'anthropic',upstreamOrigin:'http://synthetic.invalid'}, + stats:{requests:1,tokensSaved:0,inputTokens:1000,cachedTokens:0,outputTokens:5,cacheSamples:0, + lastInputTokens:10000,lastInputTokensSource:armed?'estimate':'usage',contextTokens:10000,compressCreditTokens:0, + retrieveCalls:0,retrieveHits:0,retrieveMisses:0,storedBytes:0,storeBytesSaved:0,rangeRestores:0, + ...(armed?{emergencyShrinkAt:1700000000000,emergencyShrinkEstimate:10000}:{})},metadata:{}, + createdAt:1700000000000,lastSeen:1700000001000,state:createInitialState(),blockContents:new Map(), + inFlight:0,persisted:false,pendingRetrievals:[]}; + let store=new SessionStore({dir,enabled:true,debounceMs:1}); + try{ + await store.writeNow(session);store.closeQuietly();store=new SessionStore({dir,enabled:true,debounceMs:1}); + const restored=store.loadSync(session.id,session.meta);assert.ok(restored); + assert.equal(restored.stats.emergencyShrinkAt,armed?1700000000000:undefined); + assert.equal(restored.stats.emergencyShrinkEstimate,armed?10000:undefined); + assert.equal(restored.stats.lastInputTokens,10000); + assert.equal(restored.stats.lastInputTokensSource,armed?'estimate':'usage'); + delete restored.stats.emergencyShrinkAt;delete restored.stats.emergencyShrinkEstimate; + await store.writeNow(restored);store.closeQuietly();store=new SessionStore({dir,enabled:true,debounceMs:1}); + const cleared=store.loadSync(session.id,session.meta);assert.ok(cleared); + assert.equal(cleared.stats.emergencyShrinkAt,undefined); + assert.equal(cleared.stats.emergencyShrinkEstimate,undefined); + }finally{store.closeQuietly();rmrf(dir);} + }); + } +}