diff --git a/src/cache-ledger.ts b/src/cache-ledger.ts index abd5888910..ddf0b40885 100644 --- a/src/cache-ledger.ts +++ b/src/cache-ledger.ts @@ -20,7 +20,7 @@ import { METADATA_CANONICAL_SYS_FP, METADATA_FOLD_COVERAGE, METADATA_LAST_RESPON import { clearPostRebuildAnchor, markDirty, type Session } from "./session.js"; import { isForeignMembership } from "./request-membership.js"; import { billInputUsage } from "./billing.js"; -import { normalizeUpstreamOrigin } from "./util.js"; +import { CALIBRATION_PROVENANCE_TEXT_ONLY, normalizeUpstreamOrigin } from "./util.js"; import { toolFail, toolOk, type ProxyToolResult } from "./proxy-tool-result.js"; // Render window for handleAcpCache's detail:"full" text view (#1489). The @@ -529,6 +529,11 @@ type Capture = { sentBody?: SeamSlot; inBody?: InboundSlot; imageFacts?: ForwardedImageFacts; + // #5952: the calibration pair for THIS send — final wire text estimate + // (no image reserve) + the image reserve of the same body, noted at the + // relay main-lane send chokepoint. The settle that bills this body + // consumes exactly this entry instead of the shared prepare slot. + calibration?: { estimate: number; imageTokens: number; origin?: string; model?: string }; }; const captures = new AsyncLocalStorage(); let captureOrder = 0; @@ -546,6 +551,7 @@ export async function withCacheCapture(session: Session | undefined, fn: () = delete capture.sentBody; delete capture.inBody; delete capture.imageFacts; + delete capture.calibration; } } }); @@ -861,6 +867,22 @@ export function noteForwardedImageFacts(session: Session, facts: ForwardedImageF imageFactsLastSent.set(session, facts); } +/** #5952: the per-request calibration pair for THIS send — the final wire + * text estimate (textOverhead, no image reserve) + the image reserve of + * the same body, noted at the relay main-lane send chokepoint next to + * noteOutboundContextEstimates. The next settleUsageReport / + * settleOverflowCalibration consumes exactly this entry (same pairing + * guarantee as the seam forensics): a concurrent request's prepare can no + * longer overwrite the estimate a settle reads, and an image-bearing + * sample can no longer train the text multiplier. Outside a capture there + * is no per-request owner for the pair, so it is dropped, never parked on + * the session where a later settle could misread it as this request's — + * lanes outside the capture (loop/plugin) keep the legacy shared slot. */ +export function noteCalibrationPair(session: Session, pair: { estimate: number; imageTokens: number; origin?: string; model?: string }): void { + const capture = captures.getStore(); + if (capture?.session === session) { capture.calibration = pair; return; } +} + function settleImageLearning(session: Session, billedTotal: number): void { const capture = captures.getStore(); const facts = capture?.session === session ? capture.imageFacts : imageFactsLastSent.get(session); @@ -1831,10 +1853,25 @@ function ingestCalibrationPair(session: Session, billed: number, upstream: strin const settleOrigin = normalizeUpstreamOrigin(upstream); const pendingCalibOrigin = session.stats.lastCalibrationPolicyOrigin !== undefined ? normalizeUpstreamOrigin(session.stats.lastCalibrationPolicyOrigin) : undefined; const calibPolicy = pendingCalibOrigin === settleOrigin ? session.stats.lastCalibrationPolicy : undefined; - const pendingEst = session.stats.lastLocalTextEstimate; - const pendingOrigin = normalizeUpstreamOrigin(session.stats.lastLocalTextEstimateOrigin); + // #5952: the per-request capture pair wins over the shared prepare slot — + // it is the final wire estimate of the request THIS report bills, noted + // at the send chokepoint. Consumed once: the settle that reads it owns + // it, so a concurrent settle can never re-read it as its own. + const capture = captures.getStore(); + const capturePair = capture?.session === session ? capture.calibration : undefined; + if (capture?.session === session) delete capture.calibration; + // An image-bearing sample carries no text-only signal: it must not train + // the text multiplier at all — not even through the legacy slot, which a + // concurrent prepare may have left holding another request's estimate. + if (capturePair !== undefined && capturePair.imageTokens > 0) return; + const pendingEst = capturePair !== undefined ? capturePair.estimate : session.stats.lastLocalTextEstimate; + const pendingOrigin = capturePair !== undefined ? normalizeUpstreamOrigin(capturePair.origin) : normalizeUpstreamOrigin(session.stats.lastLocalTextEstimateOrigin); if (pendingEst !== undefined && pendingEst >= CALIBRATION_MIN_ESTIMATE && settleOrigin !== undefined && pendingOrigin === settleOrigin) { const rawSample = billed / pendingEst; + // #5952: provenance of THIS sample — text-only only when it arrived + // on a capture pair. A legacy shared-slot sample carries none, so a + // ring holding one stays mixed until the window flushes it out. + const sampleProvenance = capturePair !== undefined ? CALIBRATION_PROVENANCE_TEXT_ONLY : undefined; // #2117 B: the ring is keyed by route AND model — tokenizers bill // differently across models, so a mid-session model switch starts a // FRESH ring instead of blending two models' billing scales into one @@ -1844,8 +1881,10 @@ function ingestCalibrationPair(session: Session, billed: number, upstream: strin // whose load whitelist drops the model fields (#2141) — continues // and ADOPTS the first known model. Discarding it would clear a // provenance-matched published k̂ on the first post-restart settle - // (the #2129 regression class). - const settleModel = typeof session.metadata?.lastModel === "string" && session.metadata.lastModel !== "" ? session.metadata.lastModel : undefined; + // (the #2129 regression class). The capture pair's model, when + // known, is this request's own — it outranks the session's + // last-known model (#5952). + const settleModel = capturePair?.model !== undefined ? capturePair.model : (typeof session.metadata?.lastModel === "string" && session.metadata.lastModel !== "" ? session.metadata.lastModel : undefined); let ring = session.stats.calibrationRing; if (!ring || ring.origin !== settleOrigin || (ring.model !== undefined && settleModel !== undefined && ring.model !== settleModel)) { ring = { origin: settleOrigin, model: settleModel, values: [] }; @@ -1854,7 +1893,16 @@ function ingestCalibrationPair(session: Session, billed: number, upstream: strin } if (rawSample >= CALIBRATION_SAMPLE_MIN && rawSample <= CALIBRATION_SAMPLE_MAX) { ring.values.push(rawSample); - if (ring.values.length > CALIBRATION_SAMPLE_WINDOW) ring.values.shift(); + // #5952: provenances ride in lockstep with values — a legacy + // ring restored without the array pads with undefined, so the + // publish verdict below stays aligned with the window. + if (ring.provenances === undefined) ring.provenances = []; + while (ring.provenances.length < ring.values.length - 1) ring.provenances.push(undefined); + ring.provenances.push(sampleProvenance); + if (ring.values.length > CALIBRATION_SAMPLE_WINDOW) { + ring.values.shift(); + ring.provenances.shift(); + } const spread = Math.max(...ring.values) / Math.min(...ring.values); if (ring.values.length >= 2 && spread <= CALIBRATION_CONSISTENCY_RATIO) { const mean = ring.values.reduce((a, b) => a + b, 0) / ring.values.length; @@ -1864,6 +1912,7 @@ function ingestCalibrationPair(session: Session, billed: number, upstream: strin delete session.stats.calibratedEstimate; delete session.stats.calibratedEstimateOrigin; delete session.stats.calibratedEstimateModel; + delete session.stats.calibratedEstimateProvenance; } else { const clamped = Math.min(CALIBRATION_CLAMP_MAX, Math.max(CALIBRATION_CLAMP_MIN, mean)); session.stats.calibratedEstimate = calibPolicy?.max !== undefined ? Math.min(clamped, calibPolicy.max) : clamped; @@ -1871,14 +1920,22 @@ function ingestCalibrationPair(session: Session, billed: number, upstream: strin // Provenance is the ring's model key, not this sample's stamp — // an unknown-model settle continues a model-keyed ring without // blanking the established provenance. Stays inside the publish - // branch: the disabled path above clears all three fields and + // branch: the disabled path above clears all four fields and // must not re-stamp one of them (#3211 review). session.stats.calibratedEstimateModel = ring.model; + // #5952: the published factor's provenance is the ring's — + // "text-only-v1" only when every value in the window + // carries it. A single legacy sample in the window keeps + // the factor inert until the window flushes it out. + const ringProvenance = ring.provenances !== undefined && ring.provenances.length === ring.values.length && ring.provenances.every((p) => p === CALIBRATION_PROVENANCE_TEXT_ONLY) ? CALIBRATION_PROVENANCE_TEXT_ONLY : undefined; + if (ringProvenance !== undefined) session.stats.calibratedEstimateProvenance = ringProvenance; + else delete session.stats.calibratedEstimateProvenance; } } else { delete session.stats.calibratedEstimate; delete session.stats.calibratedEstimateOrigin; delete session.stats.calibratedEstimateModel; + delete session.stats.calibratedEstimateProvenance; } } session.stats.calibrationRing = ring; diff --git a/src/persist.ts b/src/persist.ts index cbc0059ea2..334da9e279 100644 --- a/src/persist.ts +++ b/src/persist.ts @@ -44,7 +44,7 @@ import { resolveBackendName } from "./storage/backend.js"; import { toLegacyLike, type LegacySessionLike } from "./storage/ingest.js"; import { sha256Hex, type CheckpointRow, type MessageRefContentRow, type MessageIdContentRow } from "./storage/store.js"; import { contentStoreOf } from "./store.js"; -import type { WireProtocol } from "./util.js"; +import { CALIBRATION_PROVENANCE_TEXT_ONLY, type WireProtocol } from "./util.js"; import { currentContextObservation, canonicalJson, CALIBRATION_CLAMP_MAX, CALIBRATION_CLAMP_MIN, CALIBRATION_SAMPLE_MAX, CALIBRATION_SAMPLE_MIN, CALIBRATION_SAMPLE_WINDOW } from "./cache-ledger.js"; /** @@ -225,7 +225,8 @@ interface PersistedSession { calibratedEstimate?: number; calibratedEstimateOrigin?: string; calibratedEstimateModel?: string; - calibrationRing?: { origin: string; model?: string; values: number[] }; + calibratedEstimateProvenance?: string; + calibrationRing?: { origin: string; model?: string; values: number[]; provenances?: (string | undefined)[] }; pendingFoldUsage?: boolean; lastLocalTextEstimate?: number; lastLocalTextEstimateOrigin?: string; @@ -4201,15 +4202,25 @@ function restoreCalibratedEstimate(v: unknown): number | undefined { // itself would admit (finite, within the plausibility band) and only the newest // window-size entries; anything else drops the ring wholesale so a malformed // file can neither teach a factor nor mask a fresh learning run on restart. -function restoreCalibrationRing(v: unknown): { origin: string; model?: string; values: number[] } | undefined { +// #5952: (value, provenance) pairs stay aligned through the filter + window +// slice, so a restored ring's provenance verdict matches its values; a stored +// provenance entry counts only on an exact "text-only-v1" match, and the key +// is omitted entirely when no entry survives (legacy files keep their exact +// restored shape). +function restoreCalibrationRing(v: unknown): { origin: string; model?: string; values: number[]; provenances?: (string | undefined)[] } | undefined { if (!v || typeof v !== "object") return undefined; - const ring = v as { origin?: unknown; model?: unknown; values?: unknown }; + const ring = v as { origin?: unknown; model?: unknown; values?: unknown; provenances?: unknown }; if (typeof ring.origin !== "string" || ring.origin.length === 0) return undefined; if (!Array.isArray(ring.values)) return undefined; - const values = ring.values.filter((x): x is number => typeof x === "number" && Number.isFinite(x) && x >= CALIBRATION_SAMPLE_MIN && x <= CALIBRATION_SAMPLE_MAX); - const recent = values.slice(-CALIBRATION_SAMPLE_WINDOW); + const storedProvenances = Array.isArray(ring.provenances) ? ring.provenances : undefined; + const kept = ring.values + .map((x, i) => ({ value: x, provenance: storedProvenances?.[i] })) + .filter((e): e is { value: number; provenance: unknown } => typeof e.value === "number" && Number.isFinite(e.value) && e.value >= CALIBRATION_SAMPLE_MIN && e.value <= CALIBRATION_SAMPLE_MAX); + const recent = kept.slice(-CALIBRATION_SAMPLE_WINDOW); + if (recent.length === 0) return undefined; + const provenances = recent.map((e) => (e.provenance === CALIBRATION_PROVENANCE_TEXT_ONLY ? CALIBRATION_PROVENANCE_TEXT_ONLY : undefined)); const model = typeof ring.model === "string" && ring.model.trim().length > 0 ? ring.model : undefined; - return recent.length > 0 ? { origin: ring.origin, ...(model === undefined ? {} : { model }), values: recent } : undefined; + return { origin: ring.origin, ...(model === undefined ? {} : { model }), values: recent.map((e) => e.value), ...(provenances.some((p) => p !== undefined) ? { provenances } : {}) }; } function restoreBlockContents(entries: Iterable): Map { @@ -4300,6 +4311,10 @@ function buildSession(parsed: PersistedSession): Session { calibratedEstimate: restoreCalibratedEstimate(stats.calibratedEstimate), calibratedEstimateOrigin: typeof stats.calibratedEstimateOrigin === "string" && stats.calibratedEstimateOrigin.trim().length > 0 ? stats.calibratedEstimateOrigin : undefined, calibratedEstimateModel: typeof stats.calibratedEstimateModel === "string" && stats.calibratedEstimateModel.trim().length > 0 ? stats.calibratedEstimateModel : undefined, + // #5952: restored on an exact "text-only-v1" match only, like the + // ring's provenance entries — a stale mixed-modal multiplier must + // not survive a restart wearing a provenance it never earned. + calibratedEstimateProvenance: stats.calibratedEstimateProvenance === CALIBRATION_PROVENANCE_TEXT_ONLY ? CALIBRATION_PROVENANCE_TEXT_ONLY : undefined, calibrationRing: restoreCalibrationRing(stats.calibrationRing), // Intentionally NOT restored (#2129 triage): pendingFoldUsage is set at // compress execution and consumed only as a log-suffix attribution by the diff --git a/src/server/relay.ts b/src/server/relay.ts index 7ce093b91b..46abf2673b 100644 --- a/src/server/relay.ts +++ b/src/server/relay.ts @@ -13,7 +13,7 @@ import { type BiliMessage } from "acp-kernel/wire"; import { absorbToolName, applyAbsorbView } from "../absorb.js"; import { APIG_RESIGN_SCHEME, resignApig } from "../apig-resign.js"; import { noteSuccessfulForward } from "../cache-keepalive.js"; -import { credentialFingerprint, generationParamFp, lastForwardedBody, noteClientAbort, noteForwardedBody, noteForwardedImageFacts, noteInboundMessages, noteOutboundContextEstimates, settleOverflowCalibration, settleUsageReport, withCacheCapture } from "../cache-ledger.js"; +import { credentialFingerprint, generationParamFp, lastForwardedBody, noteCalibrationPair, noteClientAbort, noteForwardedBody, noteForwardedImageFacts, noteInboundMessages, noteOutboundContextEstimates, settleOverflowCalibration, settleUsageReport, withCacheCapture } from "../cache-ledger.js"; import { warnCacheCollapse } from "../cache-warn.js"; import { stampOutbound } from "../chain-checkpoint.js"; import { rewrapCommandcodeBody } from "../commandcode-wire.js"; @@ -1185,6 +1185,17 @@ const reply = prepared && !prepared.sidePassthrough ? createHandoffReply(prepare imageTokens: est.imageTokens, calibrated: scaledText !== est.textOverhead, }); + // #5952: the calibration pair for THIS send — the final wire text + // estimate (no image reserve) + the image reserve of the same body, + // so the settle that bills this body trains the text multiplier only + // from text-only evidence, even when a concurrent request overwrote + // the shared prepare slot in between. + noteCalibrationPair(prepared.session, { + estimate: est.textOverhead, + imageTokens: est.imageTokens, + origin: normalizeUpstreamOrigin(upstreamUrl), + model: typeof sentParsed?.model === "string" && sentParsed.model !== "" ? sentParsed.model : (prepared.model ?? prepared.google?.model), + }); // #2131: exact message count for seam forensics (sentParsed is the one // shared parse above — Responses carries "input", Google native "contents"). const sentArr = sentParsed !== null ? (sentParsed.messages ?? sentParsed.input ?? sentParsed.contents) : null; diff --git a/src/session.ts b/src/session.ts index ca5e7cddf3..c249fc923b 100644 --- a/src/session.ts +++ b/src/session.ts @@ -588,11 +588,27 @@ export type Session = { * Absent on factors published before this field existed → reads * treat it as compatible and retire it through normal rollover. */ calibratedEstimateModel?: string; + /** #5952: provenance of the published calibratedEstimate — + * "text-only-v1" when every sample in calibrationRing came from a + * text-only final wire estimate (per-request capture pair); absent + * for factors trained on the legacy shared prepare slot, whose + * evidence may include image-billed usage. The read gate + * (currentCalibrationFactor) only applies a factor carrying this + * stamp. Persisted so a restart doesn't silently re-arm a stale + * mixed-modal multiplier; reset at native-compaction boundaries + * with the rest of the baseline stats. */ + calibratedEstimateProvenance?: string; /** #1933 F1 / #2117 B: evidence ring behind calibratedEstimate — * the recent admitted raw samples for ONE origin+model (max 3). * Persisted so a restart doesn't re-arm the warmup delay; reset at - * native-compaction boundaries with the rest of the baseline stats. */ - calibrationRing?: { origin: string; model?: string; values: number[] }; + * native-compaction boundaries with the rest of the baseline stats. + * #5952: provenances[i] is the provenance of values[i] — + * "text-only-v1" for a sample from a text-only final wire + * estimate, undefined for a legacy shared-slot sample. The + * published factor's provenance is "text-only-v1" only when every + * value in the window carries it; one legacy sample in the window + * keeps the factor inert until the window flushes it out. */ + calibrationRing?: { origin: string; model?: string; values: number[]; provenances?: (string | undefined)[] }; /** #1933 F1: pending pairing input — local estimate of the LAST * prepared outbound in BILLED caliber (estimateCoreMessages + * system/tools overhead + image reserve, defaultCountTokens rate; @@ -1811,6 +1827,8 @@ export function resetSessionCompression(session: Session): void { delete session.stats.calibratedEstimateOrigin; // #2117 B: same boundary — the factor's model provenance dies with it. delete session.stats.calibratedEstimateModel; + // #5952: same boundary — the factor's text-only provenance dies with it. + delete session.stats.calibratedEstimateProvenance; delete session.stats.calibrationRing; delete session.stats.lastLocalTextEstimate; delete session.stats.lastLocalTextEstimateOrigin; diff --git a/src/util.ts b/src/util.ts index 9f05351d4c..acccab0b09 100644 --- a/src/util.ts +++ b/src/util.ts @@ -129,6 +129,12 @@ export function applyEstimateCalibration(raw: number, k: number | undefined, kOr return raw * k; } +/** #5952: the provenance stamp a published k̂ carries when every sample in + * its ring came from a text-only final wire estimate (per-request capture + * pair, #4430 discipline). Only this value ever decides: a factor whose + * evidence may include image-billed usage must not scale a text estimate. */ +export const CALIBRATION_PROVENANCE_TEXT_ONLY = "text-only-v1"; + /** #2117: read-side gate for the learned estimator scale — the model dimension * of k̂ provenance. A factor published against model X must not decide on * model Y: tokenizers bill differently, and a stale cross-model factor can be @@ -138,10 +144,19 @@ export function applyEstimateCalibration(raw: number, k: number | undefined, kOr * string → factor kept as-is, so factors published before this field existed * (or requests without a model stamp) are never invalidated by missing * information — they retire through the normal ring rollover when new samples - * land. Invalid factors degrade to absent. */ -export function currentCalibrationFactor(stats: { calibratedEstimate?: number; calibratedEstimateModel?: string }, lastModel?: unknown): number | undefined { + * land. Invalid factors degrade to absent. + * #5952: the provenance dimension is STRICT where the model dimension is + * lenient — a factor with no text-only stamp is absent, however well it + * matches on route+model. The incident proved a legacy multiplier can be + * image-contaminated (the shared prepare slot records text + image reserve + * under a "TextEstimate" name), and no amount of route/model matching can + * vouch for its evidence. Sessions re-train within two fresh text-only + * settles; a ring that still holds a legacy sample stays inert until the + * window flushes it out. */ +export function currentCalibrationFactor(stats: { calibratedEstimate?: number; calibratedEstimateModel?: string; calibratedEstimateProvenance?: string }, lastModel?: unknown): number | undefined { const k = stats.calibratedEstimate; if (k === undefined || !Number.isFinite(k) || k <= 0) return undefined; + if (stats.calibratedEstimateProvenance !== CALIBRATION_PROVENANCE_TEXT_ONLY) return undefined; if (stats.calibratedEstimateModel !== undefined && typeof lastModel === "string" && lastModel !== "" && stats.calibratedEstimateModel !== lastModel) return undefined; return k; } diff --git a/tests/golden/storage-render/anthropic.json b/tests/golden/storage-render/anthropic.json index 7c01ccf504..8b82ea71a1 100644 --- a/tests/golden/storage-render/anthropic.json +++ b/tests/golden/storage-render/anthropic.json @@ -95,12 +95,12 @@ "foldAt": 9, "editedAt": 12, "summaryFrom": 9, - "stateDigest": "ba0f75f6ddc882d00d5c456facd79166b62e3f2cacadd60744aa370d6b247105", - "stateDigestAligned": "66991d5cb8864f3ca029914aa9afc20cc8b9687cfd4391ffe200017c68057946", + "stateDigest": "7d9352ae810c93dca3092e0b1154d085e0c65573adc709337b4d553b8c8fd9d6", + "stateDigestAligned": "418815c041d86df080144b3390fd46c00b62ff3d7eaea99cba1416b6fbb7f2ca", "sectionDigests": { "state": "37b26b4f6898afff", "blockContents": "dc7d13d250c94bb4", - "stats": "7caf981f99159be9", + "stats": "f5b9c609de8f63d7", "metadata": "ba51746c6749d0cf" }, "stateKeyDigests": { diff --git a/tests/golden/storage-render/chat.json b/tests/golden/storage-render/chat.json index 5ecde5722b..2df8de3655 100644 --- a/tests/golden/storage-render/chat.json +++ b/tests/golden/storage-render/chat.json @@ -95,12 +95,12 @@ "foldAt": 9, "editedAt": 12, "summaryFrom": 9, - "stateDigest": "2c0b36206756617ae1e32cf0d221ddd8bf7f3e06994b49d11578bd1a35c90f26", - "stateDigestAligned": "3011800706759958cfeab6520289f1ffe257cf4f7c3275d846c03a991f9ed6b8", + "stateDigest": "3b026f0757532a0f4d4864c8475e35084112514de77da6d05c39f9212850a426", + "stateDigestAligned": "324937200dc49ebae5deded743842f16db2b81acef3fb883497458ef92e5aa76", "sectionDigests": { "state": "90857fbf82001a2b", "blockContents": "94a58f96d7c5cfc3", - "stats": "7b6162aef3873d31", + "stats": "640d24d4646ef7aa", "metadata": "7218c1e3430c3d21" }, "stateKeyDigests": { diff --git a/tests/golden/storage-render/google.json b/tests/golden/storage-render/google.json index 26c5f73b94..e956f587ce 100644 --- a/tests/golden/storage-render/google.json +++ b/tests/golden/storage-render/google.json @@ -95,12 +95,12 @@ "foldAt": 9, "editedAt": 12, "summaryFrom": 9, - "stateDigest": "c5eefef6525f180ab54a7ea57c4ddf004a568db0fc2d6d227ede31dc960706db", - "stateDigestAligned": "5217aa02331256a419837cd91105d3e3172f2ff38f0f3914c0e01128e221f012", + "stateDigest": "8d50ca57d589a147582678391dcc3c5e77187870d4784d93c9b3cf9c7c65b623", + "stateDigestAligned": "a1cb621439651bbc7b4a64e0d5497207edbdbb418464619c658480513ca22cc4", "sectionDigests": { "state": "c5243ba17364d3ac", "blockContents": "473264d9df8ba338", - "stats": "c90671e484dc5d37", + "stats": "63ebd0a52a0fb760", "metadata": "2980a8959aa0a39c" }, "stateKeyDigests": { diff --git a/tests/golden/storage-render/responses.json b/tests/golden/storage-render/responses.json index 8041cd5155..f47a7efca8 100644 --- a/tests/golden/storage-render/responses.json +++ b/tests/golden/storage-render/responses.json @@ -95,12 +95,12 @@ "foldAt": 9, "editedAt": 12, "summaryFrom": 9, - "stateDigest": "ff092bc66e11c35b15ad98acf6206629115825b62168da2e5542e06297f6d47a", - "stateDigestAligned": "db5f08a7d57c14952c31571e45b4dbc6c2ed35fe8a97c58cda6fa854d411eb25", + "stateDigest": "c17b172920ea1b06e6be198729a7ed99acd82b9fc01d52a830843d287b399d82", + "stateDigestAligned": "e3d92373f526bafee62ea638bc692070f2b029a6dd20ac776d0e36ff8c7a263d", "sectionDigests": { "state": "bee6e1ba0c6fba62", "blockContents": "b6701268a0bdcd8a", - "stats": "1b56c993f03cd122", + "stats": "5c319add8a5aadab", "metadata": "77714417e484ba89" }, "stateKeyDigests": { diff --git a/tests/issue2117-context-estimate-provenance.test.ts b/tests/issue2117-context-estimate-provenance.test.ts index 67f65a5e94..bb47c1f47b 100644 --- a/tests/issue2117-context-estimate-provenance.test.ts +++ b/tests/issue2117-context-estimate-provenance.test.ts @@ -11,7 +11,7 @@ import type { ProxyOptions } from "../src/config.ts"; import { SessionStore, _setStoreForTest } from "../src/persist.ts"; import { _setForTest as setRegistryForTest } from "../src/registry.ts"; import { settleUsageReport } from "../src/cache-ledger.ts"; -import { currentCalibrationFactor } from "../src/util.ts"; +import { CALIBRATION_PROVENANCE_TEXT_ONLY, currentCalibrationFactor } from "../src/util.ts"; import { displayContextBest, getSession, listSessions, resetSessionCompression, zeroStats, type Session } from "../src/session.ts"; // #2117: the admin bar showed the char-count UPPER BOUND of the in-flight send @@ -46,9 +46,11 @@ test("currentCalibrationFactor: invalid factors degrade to absent", () => { }); test("currentCalibrationFactor: cross-model factors act as absent, missing info keeps them", () => { - const st = { calibratedEstimate: 0.5 }; + const st = { calibratedEstimate: 0.5, calibratedEstimateProvenance: CALIBRATION_PROVENANCE_TEXT_ONLY }; // Legacy factor (never recorded a model) or unknown current model — missing // information must NOT invalidate evidence; rollover retires it naturally. + // (#5952: the provenance stamp is the exception — a factor without a + // text-only stamp never decides, however well it matches on model.) assert.equal(currentCalibrationFactor(st, "gpt-x"), 0.5); assert.equal(currentCalibrationFactor(st, undefined), 0.5); assert.equal(currentCalibrationFactor({ ...st, calibratedEstimateModel: "gpt-x" }, undefined), 0.5); @@ -234,11 +236,13 @@ test("resetSessionCompression drops the new provenance/display fields at the bou st.calibratedEstimate = 0.7; st.calibratedEstimateOrigin = "http://a"; st.calibratedEstimateModel = "gpt-x"; + st.calibratedEstimateProvenance = CALIBRATION_PROVENANCE_TEXT_ONLY; st.calibrationRing = { origin: "http://a", model: "gpt-x", values: [0.7] }; st.contextEstimateTokens = 12_000; st.contextEstimateCalibrated = true; resetSessionCompression(s); assert.equal(st.calibratedEstimateModel, undefined); + assert.equal(st.calibratedEstimateProvenance, undefined); assert.equal(st.calibrationRing, undefined); assert.equal(st.contextEstimateTokens, undefined); assert.equal(st.contextEstimateCalibrated, undefined); diff --git a/tests/issue3211-calibration-policy.test.ts b/tests/issue3211-calibration-policy.test.ts index d3c80f11b8..b6238a724c 100644 --- a/tests/issue3211-calibration-policy.test.ts +++ b/tests/issue3211-calibration-policy.test.ts @@ -6,6 +6,7 @@ process.env.NODE_ENV = "test"; import { defaultConfig } from "acp-kernel"; import type { Session } from "../src/session.ts"; import { settleUsageReport } from "../src/cache-ledger.ts"; +import { CALIBRATION_PROVENANCE_TEXT_ONLY } from "../src/util.ts"; import { applyCompressSettings, resolveCompress } from "../src/compress-settings.ts"; import { findRoute, parseCalibrationPolicy, parseCompressSettings, parseRouteEntry, type ProviderRoutes } from "../src/config.ts"; @@ -134,6 +135,11 @@ test("#3211 A: explicit max below the publication floor is honored (user intent test("#3211 A: policy {disabled:true} suppresses publication but keeps the ring", () => { const s = makeSession(); + // A previously published text-only factor must clear with the rest of + // its stamps when the route opts out (#5952). + s.stats.calibratedEstimate = 0.5; + s.stats.calibratedEstimateOrigin = ORIGIN; + s.stats.calibratedEstimateProvenance = CALIBRATION_PROVENANCE_TEXT_ONLY; stampPolicy(s, { disabled: true }, ORIGIN); pair(s, 10000, ORIGIN, 18600); stampPolicy(s, { disabled: true }, ORIGIN); @@ -141,6 +147,7 @@ test("#3211 A: policy {disabled:true} suppresses publication but keeps the ring" assert.equal(s.stats.calibratedEstimate, undefined, "no factor may be applied on this route"); assert.equal(s.stats.calibratedEstimateOrigin, undefined); assert.equal(s.stats.calibratedEstimateModel, undefined, "provenance stamp clears with the factor"); + assert.equal(s.stats.calibratedEstimateProvenance, undefined, "text-only provenance clears with the factor"); const ring = s.stats.calibrationRing; assert.ok(ring && ring.values.length === 2, "raw ring stays auditable"); }); diff --git a/tests/issue3645-calibrated-futility.test.ts b/tests/issue3645-calibrated-futility.test.ts index f900b4b0f4..7a034891ef 100644 --- a/tests/issue3645-calibrated-futility.test.ts +++ b/tests/issue3645-calibrated-futility.test.ts @@ -11,7 +11,7 @@ import type { ProxyOptions } from "../src/config.ts"; import { SessionStore, _setStoreForTest } from "../src/persist.ts"; import { getSession } from "../src/session.ts"; import { _setForTest as setRegistryForTest } from "../src/registry.ts"; -import { normalizeUpstreamOrigin } from "../src/util.ts"; +import { CALIBRATION_PROVENANCE_TEXT_ONLY, normalizeUpstreamOrigin } from "../src/util.ts"; // #4185: an identified fresh session whose preflight ends in deterministic // exhaustion now forwards once for upstream arbitration instead of failing @@ -191,6 +191,8 @@ function seedCalibratedSession(sessionId: string, upstreamPort: number): void { const s = getSession(sessionId); s.stats.calibratedEstimate = K; s.stats.calibratedEstimateOrigin = `http://127.0.0.1:${upstreamPort}`; // == currentOrigin → ×K applies + // #5952: the seeded factor is text-only evidence, so it may decide. + s.stats.calibratedEstimateProvenance = CALIBRATION_PROVENANCE_TEXT_ONLY; } test("#3645: matching-origin k̂>1 no longer skips a raw fold that fits the window", async () => { diff --git a/tests/issue5952-image-calibration-contamination.test.ts b/tests/issue5952-image-calibration-contamination.test.ts new file mode 100644 index 0000000000..407f7c333b --- /dev/null +++ b/tests/issue5952-image-calibration-contamination.test.ts @@ -0,0 +1,242 @@ +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync as rmrf } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; + +process.env.NODE_ENV = "test"; +// #2129: the measurement-state round-trip pins the legacy storage layer; +// the unified default hangs these bare-store tests. +process.env.BILI_STORAGE_UNIFIED = "legacy"; + +import { createCore, createInitialState, defaultConfig, defaultPrompts, type CoreMessage } from "acp-kernel"; +import { noteCalibrationPair, settleOverflowCalibration, settleUsageReport, withCacheCapture } from "../src/cache-ledger.ts"; +import { SessionStore } from "../src/persist.ts"; +import { preflightCompress, type PreflightDeps } from "../src/preflight.ts"; +import type { Session } from "../src/session.ts"; +import { CALIBRATION_PROVENANCE_TEXT_ONLY, currentCalibrationFactor } from "../src/util.ts"; + +// #5952: request usage containing images contaminated the text-only estimate +// calibration, and concurrent settlement read shared prepare facts — so a +// stale mixed-modal multiplier could reject a fitting payload after a model +// switch. The fix pairs each settle with the final wire estimate of ITS OWN +// request (per-request capture, #4430 discipline): image-bearing samples +// never train the text multiplier, and a published factor only decides when +// every sample in its ring came from a text-only wire estimate +// (provenance "text-only-v1"). + +const ORIGIN = "http://a"; + +let seq = 0; +function makeSession(id?: string): Session { + seq += 1; + return { + id: id ?? `i5952-${seq}`, + metadata: { lastModel: "gpt-x" }, + stats: {}, + state: { blocks: [], messageRefs: { byRaw: {}, byRef: {} }, tokenSnapshot: {} }, + } as unknown as Session; +} + +type Pair = { estimate: number; imageTokens: number; origin: string; model?: string }; + +async function settleWithPair(s: Session, pair: Pair, billed: number, prepareEst?: number): Promise { + // prepareEst: what prepare-*.ts wrote into the shared slot — the + // contaminated value in the bug (text + image reserve), or a concurrent + // request's prepare value. + if (prepareEst !== undefined) { + s.stats.lastLocalTextEstimate = prepareEst; + s.stats.lastLocalTextEstimateOrigin = pair.origin; + } + await withCacheCapture(s, async () => { + noteCalibrationPair(s, pair); + settleUsageReport(s, { total: billed, reportedCached: null, upstream: pair.origin }); + }); +} + +test("#5952: image-bearing usage never trains the text multiplier", async () => { + const s = makeSession(); + const st = s.stats; + // Two consistent image-bearing samples: prepare recorded the contaminated + // estimate (text 10000 + image reserve 17000); billed 15000. Pre-fix the + // shared slot trained k̂ = 15000/27000 ≈ 0.556. + await settleWithPair(s, { estimate: 10000, imageTokens: 17000, origin: ORIGIN, model: "gpt-x" }, 15000, 27000); + await settleWithPair(s, { estimate: 10000, imageTokens: 17000, origin: ORIGIN, model: "gpt-x" }, 15000, 27000); + assert.equal(st.calibratedEstimate, undefined, "image-contaminated samples must not publish a text factor"); + assert.equal(st.calibratedEstimateProvenance, undefined); + assert.equal(st.calibrationRing, undefined, "skipped samples leave no ring evidence"); +}); + +test("#5952: a text-only final wire estimate trains the multiplier from its own pair", async () => { + const s = makeSession(); + const st = s.stats; + // The shared slot holds a DIFFERENT (concurrent/stale) prepare value: the + // settle must use its own capture pair (10000), not the slot (20000). + await settleWithPair(s, { estimate: 10000, imageTokens: 0, origin: ORIGIN, model: "gpt-x" }, 5000, 20000); + assert.equal(st.calibratedEstimate, undefined, "one sample must not publish"); + await settleWithPair(s, { estimate: 10000, imageTokens: 0, origin: ORIGIN, model: "gpt-x" }, 5000, 20000); + assert.equal(st.calibratedEstimate, 0.5, "two agreeing text-only samples publish their own ratio, not the shared slot's 0.25"); + assert.equal(st.calibratedEstimateProvenance, CALIBRATION_PROVENANCE_TEXT_ONLY); + assert.equal(currentCalibrationFactor(st, "gpt-x"), 0.5, "a text-only factor applies on its route+model"); + assert.deepEqual(st.calibrationRing?.provenances, [CALIBRATION_PROVENANCE_TEXT_ONLY, CALIBRATION_PROVENANCE_TEXT_ONLY]); +}); + +test("#5952: interleaved image/text settles — only the text request trains", async () => { + const s = makeSession(); + const st = s.stats; + let releaseBPrepared!: () => void; + let releaseASettled!: () => void; + const bPrepared = new Promise((r) => { releaseBPrepared = r; }); + const aSettled = new Promise((r) => { releaseASettled = r; }); + const runA = withCacheCapture(s, async () => { + // Image request notes its final wire estimate, then waits: a + // concurrent text request prepares (overwriting the shared slot) + // before A settles. + noteCalibrationPair(s, { estimate: 10000, imageTokens: 17000, origin: ORIGIN, model: "gpt-x" }); + await bPrepared; + settleUsageReport(s, { total: 15000, reportedCached: null, upstream: ORIGIN }); + releaseASettled(); + }); + const runB = withCacheCapture(s, async () => { + s.stats.lastLocalTextEstimate = 8000; + s.stats.lastLocalTextEstimateOrigin = ORIGIN; + noteCalibrationPair(s, { estimate: 8000, imageTokens: 0, origin: ORIGIN, model: "gpt-x" }); + releaseBPrepared(); + await aSettled; + settleUsageReport(s, { total: 4000, reportedCached: null, upstream: ORIGIN }); + }); + await Promise.all([runA, runB]); + // A's image sample must not count as the second sample, even though the + // shared slot held a matching estimate (8000) when it settled: pre-fix + // the ring held [1.875] (15000/8000) here. + assert.deepEqual(st.calibrationRing?.values, [0.5], "only the text request's sample may enter the ring"); + assert.equal(st.calibratedEstimate, undefined, "one text sample alone must not publish"); + // A second text-only settle publishes with text-only provenance. + await settleWithPair(s, { estimate: 8000, imageTokens: 0, origin: ORIGIN, model: "gpt-x" }, 4000, 8000); + assert.equal(st.calibratedEstimate, 0.5); + assert.equal(st.calibratedEstimateProvenance, CALIBRATION_PROVENANCE_TEXT_ONLY); +}); + +test("currentCalibrationFactor: a factor without text-only provenance never decides (#5952)", () => { + assert.equal(currentCalibrationFactor({ calibratedEstimate: 0.5 }, "gpt-x"), undefined, "legacy factor without provenance is inert"); + assert.equal(currentCalibrationFactor({ calibratedEstimate: 0.5, calibratedEstimateProvenance: "other" }, "gpt-x"), undefined, "unknown provenance is inert"); + assert.equal( + currentCalibrationFactor({ calibratedEstimate: 0.5, calibratedEstimateProvenance: CALIBRATION_PROVENANCE_TEXT_ONLY }, "gpt-x"), + 0.5, + "a text-only factor applies", + ); + assert.equal( + currentCalibrationFactor({ calibratedEstimate: 0.5, calibratedEstimateProvenance: CALIBRATION_PROVENANCE_TEXT_ONLY, calibratedEstimateModel: "gpt-y" }, "gpt-x"), + undefined, + "cross-model still absent", + ); +}); + +function makePreflightFixture() { + const core = createCore(); + const config = defaultConfig(262144); + // No range can qualify, so a non-fitting verdict never spends a + // summarization call: the test judges the estimate, not the fold walk. + config.compress.minCompressRange = 300000; + const messages: CoreMessage[] = [{ id: "m1", role: "user", contentType: "text", text: "f".repeat(160000) }]; + const session: Session = { + id: "i5952-preflight", + meta: {}, + metadata: { lastModel: "gpt-x" }, + state: createInitialState(), + stats: { requests: 1, tokensSaved: 0, inputTokens: 0, cachedTokens: 0, outputTokens: 0, cacheSamples: 0, lastInputTokens: 0, compressCreditTokens: 0, contextTokens: 0, retrieveCalls: 0, retrieveHits: 0, retrieveMisses: 0, storedBytes: 0, storeBytesSaved: 0, rangeRestores: 0 }, + pendingRetrievals: [], + createdAt: Date.now(), + lastSeen: Date.now(), + blockContents: new Map(), + inFlight: 0, + persisted: false, + }; + const deps: PreflightDeps = { + core, session, config, prompts: defaultPrompts, protocol: "openai", + url: "http://127.0.0.1:9/v1/chat/completions", headers: {}, model: "gpt-x", + log: () => {}, + upstreamOrigin: ORIGIN, wireOverhead: 22000, imageReserve: 17000, + }; + return { messages, session, deps }; +} + +test("#5952: a stale multiplier without text-only provenance never rejects a fitting payload", async () => { + const { messages, session, deps } = makePreflightFixture(); + // The incident's stale factor: k̂=4, route+model match, NO provenance + // (pre-upgrade/legacy shape) — a model switch just happened. + session.stats.calibratedEstimate = 4; + session.stats.calibratedEstimateOrigin = ORIGIN; + session.stats.calibratedEstimateModel = "gpt-x"; + const result = await preflightCompress(deps, messages); + // The kernel's rendered view adds a few ref-tag tokens over the raw + // input estimate (~79K raw vs ~265K calibrated), so judge by the wide + // gap, not the exact count. + assert.ok(result.payloadEstimate < 100000, `stale k̂=4 must not scale the estimate (got ${result.payloadEstimate})`); + assert.equal(result.fitsWindow, true, "the unchanged payload fits the 262144 window on the raw scale"); +}); + +test("#5952: a text-only-provenance multiplier still decides preflight", async () => { + const { messages, session, deps } = makePreflightFixture(); + session.stats.calibratedEstimate = 4; + session.stats.calibratedEstimateOrigin = ORIGIN; + session.stats.calibratedEstimateModel = "gpt-x"; + session.stats.calibratedEstimateProvenance = CALIBRATION_PROVENANCE_TEXT_ONLY; + const result = await preflightCompress(deps, messages); + assert.ok(result.payloadEstimate > 240000, `a proven text-only factor must still scale the estimate (got ${result.payloadEstimate})`); + assert.equal(result.fitsWindow, false, "the calibrated payload genuinely exceeds the window"); +}); + +test("#5952: overflow calibration inherits the capture-pair provenance", async () => { + const s = makeSession(); + const st = s.stats; + await withCacheCapture(s, async () => { + noteCalibrationPair(s, { estimate: 10000, imageTokens: 17000, origin: ORIGIN, model: "gpt-x" }); + settleOverflowCalibration(s, 15000, ORIGIN); + }); + assert.equal(st.calibratedEstimate, undefined, "an image-bearing overflow must not teach"); + assert.equal(st.calibrationRing, undefined); + await withCacheCapture(s, async () => { + noteCalibrationPair(s, { estimate: 10000, imageTokens: 0, origin: ORIGIN, model: "gpt-x" }); + settleOverflowCalibration(s, 5000, ORIGIN); + }); + await withCacheCapture(s, async () => { + noteCalibrationPair(s, { estimate: 20000, imageTokens: 0, origin: ORIGIN, model: "gpt-x" }); + settleOverflowCalibration(s, 11000, ORIGIN); + }); + assert.equal(st.calibratedEstimate, 0.525, "two agreeing text-only overflow samples publish their mean"); + assert.equal(st.calibratedEstimateProvenance, CALIBRATION_PROVENANCE_TEXT_ONLY); +}); + +test("#5952: text-only provenance survives a reload round-trip", async () => { + const dir = mkdtempSync(join(tmpdir(), "bili-5952-rt-")); + try { + const store = new SessionStore({ dir, debounceMs: 5, enabled: true }); + const s: Session = { + id: "i5952-rt", + meta: {}, + metadata: {}, + state: createInitialState(), + stats: { requests: 0, tokensSaved: 0, inputTokens: 0, cachedTokens: 0, outputTokens: 0, cacheSamples: 0, lastInputTokens: 0, compressCreditTokens: 0, contextTokens: 0, retrieveCalls: 0, retrieveHits: 0, retrieveMisses: 0, storedBytes: 0, storeBytesSaved: 0, rangeRestores: 0 }, + createdAt: Date.now(), + lastSeen: Date.now(), + blockContents: new Map(), + inFlight: 0, + persisted: false, + pendingRetrievals: [], + }; + s.stats.calibratedEstimate = 0.5; + s.stats.calibratedEstimateOrigin = ORIGIN; + s.stats.calibratedEstimateProvenance = CALIBRATION_PROVENANCE_TEXT_ONLY; + s.stats.calibrationRing = { origin: ORIGIN, model: "gpt-x", values: [0.5, 0.5], provenances: [CALIBRATION_PROVENANCE_TEXT_ONLY, CALIBRATION_PROVENANCE_TEXT_ONLY] }; + await store.writeNow(s); + await store.flushAll([]); + const back = new SessionStore({ dir, debounceMs: 5, enabled: true }).loadSync("i5952-rt"); + assert.ok(back, "session reloaded"); + assert.equal(back.stats.calibratedEstimateProvenance, CALIBRATION_PROVENANCE_TEXT_ONLY, "published provenance round-trips"); + assert.deepEqual(back.stats.calibrationRing?.provenances, [CALIBRATION_PROVENANCE_TEXT_ONLY, CALIBRATION_PROVENANCE_TEXT_ONLY], "ring provenance round-trips"); + assert.equal(currentCalibrationFactor(back.stats, "gpt-x"), 0.5, "the restored factor still decides"); + } finally { + rmrf(dir, { recursive: true, force: true }); + } +}); diff --git a/tests/persist-calibration-model.test.ts b/tests/persist-calibration-model.test.ts index 85de57705c..a48546d45f 100644 --- a/tests/persist-calibration-model.test.ts +++ b/tests/persist-calibration-model.test.ts @@ -6,7 +6,7 @@ import path from 'node:path'; import {createInitialState} from 'acp-kernel'; import {SessionStore} from '../src/persist.ts'; import type {Session} from '../src/session.ts'; -import {currentCalibrationFactor} from '../src/util.ts'; +import {CALIBRATION_PROVENANCE_TEXT_ONLY, currentCalibrationFactor} from '../src/util.ts'; import {rmrf} from './tmp-rm.ts'; process.env.BILI_CONFIG_FILE = path.join(tmpdir(), 'calibration-model-no-config.json'); @@ -19,6 +19,8 @@ function session(id: string, model?: string): Session { retrieveCalls:0,retrieveHits:0,retrieveMisses:0,storedBytes:0,storeBytesSaved:0,rangeRestores:0, calibratedEstimate:0.8,calibratedEstimateOrigin:'http://synthetic.invalid', calibratedEstimateModel:model, + // #5952: the seeded factor is text-only evidence, so it may decide. + calibratedEstimateProvenance:CALIBRATION_PROVENANCE_TEXT_ONLY, calibrationRing:{origin:'http://synthetic.invalid',...(model === undefined ? {} : {model}),values:[0.8,0.8]}}, metadata:{},createdAt:1700000000000,lastSeen:1700000001000,state:createInitialState(), blockContents:new Map(),inFlight:0,persisted:false,pendingRetrievals:[]}; diff --git a/tests/preflight-zero-fold-calibration.test.ts b/tests/preflight-zero-fold-calibration.test.ts index 7837ff72ba..fbb51659f3 100644 --- a/tests/preflight-zero-fold-calibration.test.ts +++ b/tests/preflight-zero-fold-calibration.test.ts @@ -9,6 +9,7 @@ import type { ProxyOptions } from "../src/config.ts"; import { SessionStore, _setStoreForTest } from "../src/persist.ts"; import { getSession } from "../src/session.ts"; import { outboundPayloadBreakdown, preflightCompressIfNeeded, type Prepared } from "../src/server.ts"; +import { CALIBRATION_PROVENANCE_TEXT_ONLY } from "../src/util.ts"; import { _liveUpstreamTimersForTest, _resetFetchUtilForTest } from "../src/fetch-util.ts"; process.env.NODE_ENV = "test"; @@ -90,6 +91,9 @@ async function run(s: Scenario) { session.stats.calibratedEstimate = s.k; session.stats.calibratedEstimateOrigin = s.otherOrigin ? "http://other.invalid" : origin; session.stats.calibratedEstimateModel = s.otherModel ? "another-model" : "test-model"; + // #5952: the synthetic factor is text-only evidence, so it may decide — + // the origin/model mismatch cases still ignore it via their own gates. + session.stats.calibratedEstimateProvenance = CALIBRATION_PROVENANCE_TEXT_ONLY; session.metadata.lastModel = "test-model"; if (s.localRefusal) session.metadata.localRefusals = [{ origin, model: "test-model", window: limit, at: Date.now() }]; // Keep the normal no-ranges shortcut out of the way: this is the caller's