Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 13 additions & 4 deletions src/persist.ts
Original file line number Diff line number Diff line change
Expand Up @@ -202,6 +202,8 @@ interface PersistedSession {
cacheSamples?: number;
lastInputTokens?: number;
lastInputTokensSource?: string;
emergencyShrinkAt?: number;
emergencyShrinkEstimate?: number;
lastInputTokensOrigin?: string;
overflowArmTokens?: number;
overflowArmCompletionReserve?: number;
Expand All @@ -222,7 +224,8 @@ interface PersistedSession {
lastUsageGradeTokens?: number;
calibratedEstimate?: number;
calibratedEstimateOrigin?: string;
calibrationRing?: { origin: string; values: number[] };
calibratedEstimateModel?: string;
calibrationRing?: { origin: string; model?: string; values: number[] };
pendingFoldUsage?: boolean;
lastLocalTextEstimate?: number;
lastLocalTextEstimateOrigin?: string;
Expand Down Expand Up @@ -4198,14 +4201,15 @@ function restoreCalibratedEstimate(v: unknown): number | undefined {
// itself would admit (finite, within the plausibility band) and only the newest
// window-size entries; anything else drops the ring wholesale so a malformed
// file can neither teach a factor nor mask a fresh learning run on restart.
function restoreCalibrationRing(v: unknown): { origin: string; values: number[] } | undefined {
function restoreCalibrationRing(v: unknown): { origin: string; model?: string; values: number[] } | undefined {
if (!v || typeof v !== "object") return undefined;
const ring = v as { origin?: unknown; values?: unknown };
const ring = v as { origin?: unknown; model?: unknown; values?: unknown };
if (typeof ring.origin !== "string" || ring.origin.length === 0) return undefined;
if (!Array.isArray(ring.values)) return undefined;
const values = ring.values.filter((x): x is number => typeof x === "number" && Number.isFinite(x) && x >= CALIBRATION_SAMPLE_MIN && x <= CALIBRATION_SAMPLE_MAX);
const recent = values.slice(-CALIBRATION_SAMPLE_WINDOW);
return recent.length > 0 ? { origin: ring.origin, values: recent } : undefined;
const model = typeof ring.model === "string" && ring.model.trim().length > 0 ? ring.model : undefined;
return recent.length > 0 ? { origin: ring.origin, ...(model === undefined ? {} : { model }), values: recent } : undefined;
}

function restoreBlockContents(entries: Iterable<readonly [string, unknown]>): Map<string, BlockContent> {
Expand Down Expand Up @@ -4266,6 +4270,10 @@ function buildSession(parsed: PersistedSession): Session {
// #857: provenance — legacy files lack it; absent stays absent and
// evidence-grade consumers treat absent as untrusted.
lastInputTokensSource: stats.lastInputTokensSource === "usage" || stats.lastInputTokensSource === "estimate" || stats.lastInputTokensSource === "overflow-arm" ? stats.lastInputTokensSource : undefined,
// Preserve the explanation saved by armFailureShrink alongside its
// raised baseline. A real usage settlement clears both markers.
emergencyShrinkAt: typeof stats.emergencyShrinkAt === "number" && Number.isFinite(stats.emergencyShrinkAt) && stats.emergencyShrinkAt >= 0 ? stats.emergencyShrinkAt : undefined,
emergencyShrinkEstimate: typeof stats.emergencyShrinkEstimate === "number" && Number.isFinite(stats.emergencyShrinkEstimate) && stats.emergencyShrinkEstimate > 0 ? stats.emergencyShrinkEstimate : undefined,
// #1110: one-shot overflow arm — legacy files lack it; absent = no arm.
overflowArmTokens: typeof stats.overflowArmTokens === "number" && Number.isFinite(stats.overflowArmTokens) && stats.overflowArmTokens > 0 ? stats.overflowArmTokens : undefined,
// #3450: proven completion reservation + one-time warn flag — legacy files lack both.
Expand All @@ -4291,6 +4299,7 @@ function buildSession(parsed: PersistedSession): Session {
lastInputTokensOrigin: typeof stats.lastInputTokensOrigin === "string" && stats.lastInputTokensOrigin.trim().length > 0 ? stats.lastInputTokensOrigin : undefined,
calibratedEstimate: restoreCalibratedEstimate(stats.calibratedEstimate),
calibratedEstimateOrigin: typeof stats.calibratedEstimateOrigin === "string" && stats.calibratedEstimateOrigin.trim().length > 0 ? stats.calibratedEstimateOrigin : undefined,
calibratedEstimateModel: typeof stats.calibratedEstimateModel === "string" && stats.calibratedEstimateModel.trim().length > 0 ? stats.calibratedEstimateModel : undefined,
calibrationRing: restoreCalibrationRing(stats.calibrationRing),
// Intentionally NOT restored (#2129 triage): pendingFoldUsage is set at
// compress execution and consumed only as a log-suffix attribution by the
Expand Down
6 changes: 3 additions & 3 deletions tests/golden/storage-render/anthropic.json
Original file line number Diff line number Diff line change
Expand Up @@ -95,12 +95,12 @@
"foldAt": 9,
"editedAt": 12,
"summaryFrom": 9,
"stateDigest": "97c49de03bb6c42c99a9d0f03ce7cef247b4cb034ee1da24d5954207225bb341",
"stateDigestAligned": "2f913ebef14626bd7923a11fffc9676e2919664f2a5e1649b456bf03ec09cb29",
"stateDigest": "42c35ba8831b61047e7e978190e716a1a33ead46eeafc1a11e3c93edb5290b99",
"stateDigestAligned": "c2919947d05d5e7a4bd87d953b873a15a1c9f8eb8c87855d345cd5041c4c83de",
"sectionDigests": {
"state": "56e66f6b72c7f0dd",
"blockContents": "dc7d13d250c94bb4",
"stats": "e83198a55e6b2c29",
"stats": "7bab3f342a5398ab",
"metadata": "5d34f4fe94270d09"
},
"stateKeyDigests": {
Expand Down
6 changes: 3 additions & 3 deletions tests/golden/storage-render/chat.json
Original file line number Diff line number Diff line change
Expand Up @@ -95,12 +95,12 @@
"foldAt": 9,
"editedAt": 12,
"summaryFrom": 9,
"stateDigest": "accd7fe100acd1e3adf3b6186b4c709085f4e229ea2d8d553f903cf30f97b42c",
"stateDigestAligned": "4d6b17bddffef0fbaf73bd585a8a702c34a866c105719da5fa16bc90da1be38b",
"stateDigest": "11146c571e821244043e57cc91fa880c904423a1e1976a2d9d606f608a74a979",
"stateDigestAligned": "5f60be1a36988f35de1944bcaa30178c9ff46d4dc099760cc7e84640a81243df",
"sectionDigests": {
"state": "90857fbf82001a2b",
"blockContents": "94a58f96d7c5cfc3",
"stats": "5c2f0fe203d710e8",
"stats": "fecf844aec250fc2",
"metadata": "5e869987612d599b"
},
"stateKeyDigests": {
Expand Down
6 changes: 3 additions & 3 deletions tests/golden/storage-render/google.json
Original file line number Diff line number Diff line change
Expand Up @@ -95,12 +95,12 @@
"foldAt": 9,
"editedAt": 12,
"summaryFrom": 9,
"stateDigest": "a9769a0169874c9b501b27a73cf721dcb57097969e974f050a3fa68a741b17fd",
"stateDigestAligned": "3db9797c6435542e793cfd27d4d5d9a76d49474289601d07c90d65fa9d567112",
"stateDigest": "0c38240af94862bb1dde24ca679098f914e2b4f48b809e6f5fd89a9cd50e572a",
"stateDigestAligned": "f40a87a4be7dfc050b95b30f7cc7ba21326dbceea69e2ed76637783fd39b51c3",
"sectionDigests": {
"state": "c5243ba17364d3ac",
"blockContents": "473264d9df8ba338",
"stats": "a3c84817f1d0c3dd",
"stats": "794b12c5699ae15a",
"metadata": "631282b79ac9672c"
},
"stateKeyDigests": {
Expand Down
6 changes: 3 additions & 3 deletions tests/golden/storage-render/responses.json
Original file line number Diff line number Diff line change
Expand Up @@ -95,12 +95,12 @@
"foldAt": 9,
"editedAt": 12,
"summaryFrom": 9,
"stateDigest": "afe47188cdeafd5b6670fe757632e67d04bb05cb6e7c2a6971f0d498199e86a8",
"stateDigestAligned": "ed7a20e60d65949c92e27963ca373fc3481ef423ee9f6e33f67bcc213f964e50",
"stateDigest": "10642618204e57f051b7784d9ab87428a986d40cc6f99ce99e6a9e85b0b4377d",
"stateDigestAligned": "52674f6fc691140ed81bd5843babcb9fa4a4243795714c0a818c819965a599b5",
"sectionDigests": {
"state": "e686a14f31d69ce5",
"blockContents": "b6701268a0bdcd8a",
"stats": "099bde0a9cf64cad",
"stats": "e8ebcb92d0bb0878",
"metadata": "5b0d05c2e6677cac"
},
"stateKeyDigests": {
Expand Down
80 changes: 80 additions & 0 deletions tests/persist-calibration-model.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import {mkdtempSync, readdirSync, readFileSync, writeFileSync} from 'node:fs';
import {tmpdir} from 'node:os';
import path from 'node:path';
import {createInitialState} from 'acp-kernel';
import {SessionStore} from '../src/persist.ts';
import type {Session} from '../src/session.ts';
import {currentCalibrationFactor} from '../src/util.ts';
import {rmrf} from './tmp-rm.ts';

process.env.BILI_CONFIG_FILE = path.join(tmpdir(), 'calibration-model-no-config.json');
process.env.BILI_PERSIST_ZSTD = '0';

function session(id: string, model?: string): Session {
return {id, meta:{protocol:'anthropic',upstreamOrigin:'http://synthetic.invalid'},
stats:{requests:3,tokensSaved:0,inputTokens:10,cachedTokens:0,outputTokens:5,
cacheSamples:0,lastInputTokens:10,contextTokens:1000,compressCreditTokens:0,
retrieveCalls:0,retrieveHits:0,retrieveMisses:0,storedBytes:0,storeBytesSaved:0,rangeRestores:0,
calibratedEstimate:0.8,calibratedEstimateOrigin:'http://synthetic.invalid',
calibratedEstimateModel:model,
calibrationRing:{origin:'http://synthetic.invalid',...(model === undefined ? {} : {model}),values:[0.8,0.8]}},
metadata:{},createdAt:1700000000000,lastSeen:1700000001000,state:createInitialState(),
blockContents:new Map(),inFlight:0,persisted:false,pendingRetrievals:[]};
}

for (const mode of ['legacy','full']) {
for (const model of ['synthetic-model-a', undefined]) {
test(`calibration provenance survives ${mode} cold load (${model ?? 'legacy unknown model'})`, async () => {
process.env.BILI_STORAGE_UNIFIED = mode;
const dir = mkdtempSync(path.join(tmpdir(),'bili-calibration-model-'));
const original = session('calibration-roundtrip',model);
let store = new SessionStore({dir,enabled:true,debounceMs:1});
try {
await store.writeNow(original);
store.closeQuietly();
store = new SessionStore({dir,enabled:true,debounceMs:1});
const restored = store.loadSync(original.id,original.meta);
assert.ok(restored);
assert.deepEqual(restored.stats.calibrationRing,original.stats.calibrationRing,
'Cold load must retain the ring including its exact model provenance');
assert.equal(restored.stats.calibratedEstimateModel,model);
assert.equal(currentCalibrationFactor(restored.stats,'synthetic-model-a'),0.8);
assert.equal(currentCalibrationFactor(restored.stats,'synthetic-model-b'),
model === undefined ? 0.8 : undefined,
'Known-model factors must not cross a model switch; unknown legacy models retain existing behavior');
} finally {store.closeQuietly();rmrf(dir);}
});
}
}

for (const invalid of [123,' ',{},null]) {
test(`invalid persisted model provenance is rejected (${JSON.stringify(invalid)})`, async () => {
process.env.BILI_STORAGE_UNIFIED = 'legacy';
const dir = mkdtempSync(path.join(tmpdir(),'bili-calibration-invalid-model-'));
const original = session('invalid-provenance','synthetic-model-a');
let store = new SessionStore({dir,enabled:true,debounceMs:1});
try {
await store.writeNow(original);
store.closeQuietly();
const files = readdirSync(dir,{recursive:true}).filter(x => typeof x === 'string' && x.endsWith('.json'));
const file = files.map(x => path.join(dir,String(x))).find(x => {
const raw = JSON.parse(readFileSync(x,'utf8'));
return (raw.payload ?? raw).id === original.id;
});
assert.ok(file,'Fixture must have written its session envelope');
const raw = JSON.parse(readFileSync(file,'utf8'));
const payload = raw.payload ?? raw;
payload.stats.calibrationRing.model = invalid;
payload.stats.calibratedEstimateModel = invalid;
writeFileSync(file,JSON.stringify(raw));
store = new SessionStore({dir,enabled:true,debounceMs:1});
const restored = store.loadSync(original.id,original.meta);
assert.ok(restored);
assert.equal(restored.stats.calibrationRing?.model,undefined);
assert.equal(restored.stats.calibratedEstimateModel,undefined);
assert.deepEqual(restored.stats.calibrationRing?.values,[0.8,0.8]);
} finally {store.closeQuietly();rmrf(dir);}
});
}
41 changes: 41 additions & 0 deletions tests/persist-emergency-marker.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import {mkdtempSync} from 'node:fs';
import {tmpdir} from 'node:os';
import path from 'node:path';
import {createInitialState} from 'acp-kernel';
import {SessionStore} from '../src/persist.ts';
import type {Session} from '../src/session.ts';
import {rmrf} from './tmp-rm.ts';

for(const mode of ['legacy','full']){
for(const armed of [true,false]){
test(`failure-shrink status evidence survives ${mode} restart (armed=${armed})`,async()=>{
process.env.BILI_STORAGE_UNIFIED=mode;
process.env.BILI_PERSIST_ZSTD='0';
const dir=mkdtempSync(path.join(tmpdir(),'bili-shrink-marker-test-'));
process.env.BILI_CONFIG_FILE=path.join(dir,'absent-config');
const session:Session={id:'synthetic-shrink-marker',meta:{protocol:'anthropic',upstreamOrigin:'http://synthetic.invalid'},
stats:{requests:1,tokensSaved:0,inputTokens:1000,cachedTokens:0,outputTokens:5,cacheSamples:0,
lastInputTokens:10000,lastInputTokensSource:armed?'estimate':'usage',contextTokens:10000,compressCreditTokens:0,
retrieveCalls:0,retrieveHits:0,retrieveMisses:0,storedBytes:0,storeBytesSaved:0,rangeRestores:0,
...(armed?{emergencyShrinkAt:1700000000000,emergencyShrinkEstimate:10000}:{})},metadata:{},
createdAt:1700000000000,lastSeen:1700000001000,state:createInitialState(),blockContents:new Map(),
inFlight:0,persisted:false,pendingRetrievals:[]};
let store=new SessionStore({dir,enabled:true,debounceMs:1});
try{
await store.writeNow(session);store.closeQuietly();store=new SessionStore({dir,enabled:true,debounceMs:1});
const restored=store.loadSync(session.id,session.meta);assert.ok(restored);
assert.equal(restored.stats.emergencyShrinkAt,armed?1700000000000:undefined);
assert.equal(restored.stats.emergencyShrinkEstimate,armed?10000:undefined);
assert.equal(restored.stats.lastInputTokens,10000);
assert.equal(restored.stats.lastInputTokensSource,armed?'estimate':'usage');
delete restored.stats.emergencyShrinkAt;delete restored.stats.emergencyShrinkEstimate;
await store.writeNow(restored);store.closeQuietly();store=new SessionStore({dir,enabled:true,debounceMs:1});
const cleared=store.loadSync(session.id,session.meta);assert.ok(cleared);
assert.equal(cleared.stats.emergencyShrinkAt,undefined);
assert.equal(cleared.stats.emergencyShrinkEstimate,undefined);
}finally{store.closeQuietly();rmrf(dir);}
});
}
}
Loading