diff --git a/docs/CACHING_AND_SAVINGS.md b/docs/CACHING_AND_SAVINGS.md index 8866c9a0..7ea5ac19 100644 --- a/docs/CACHING_AND_SAVINGS.md +++ b/docs/CACHING_AND_SAVINGS.md @@ -24,7 +24,7 @@ Pricing relative to base input rate: | bucket | meaning | rate | |---|---|---:| | `input_tokens` | uncached input | `1.0x` | -| `cache_creation_input_tokens` (`cc`) | cache write | `1.25x` | +| `cache_creation_input_tokens` (`cc`) | cache write | `1.25x` (5m) / `2x` (1h) | | `cache_read_input_tokens` (`cr`) | cache read | `0.1x` | | output | model reply | `5x` input on Fable 5 | @@ -97,7 +97,7 @@ For each `/v1/messages` request, pxpipe records three measurements: The actual input cost is: ```text -actual_eff = input_tokens + cc * 1.25 + cr * 0.10 +actual_eff = input_tokens + cc_5m * 1.25 + cc_1h * 2.0 + cr * 0.10 ``` The text baseline first splits the measured text tokens: @@ -110,7 +110,7 @@ coldTail = baseline_tokens - cacheable If the actual request is cold (`cr === 0`): ```text -baseline_eff = cacheable * 1.25 + coldTail +baseline_eff = cacheable * observed_write_rate + coldTail ``` If the actual request is warm (`cr > 0`): @@ -119,10 +119,10 @@ If the actual request is warm (`cr > 0`): reused = min(prevCacheable, cacheable) grown = cacheable - reused -baseline_eff = reused * 0.10 + grown * 1.25 + coldTail +baseline_eff = reused * 0.10 + grown * observed_write_rate + coldTail ``` -`prevCacheable` is used only after `cr > 0` proves a warm read. It refines how much of the text baseline was reused vs newly grown. If there is no completed same-session prior with the same static-prefix hash, pxpipe assumes full reuse for the text baseline: `prevCacheable = cacheable`. That is conservative for savings because it makes the text baseline cheaper. +`observed_write_rate` is the request's server-reported mix of 5-minute writes at `1.25x` and 1-hour writes at `2x`. If the API omits the tier split, pxpipe falls back to `1.25x` for backward compatibility. `prevCacheable` is used only after `cr > 0` proves a warm read. It refines how much of the text baseline was reused vs newly grown. If there is no completed same-session prior with the same static-prefix hash, pxpipe assumes full reuse for the text baseline: `prevCacheable = cacheable`. That is conservative for savings because it makes the text baseline cheaper. Replay uses request start time (`ts - duration_ms`) to avoid overlapping requests refining each other's `prevCacheable` split before the earlier request had completed. @@ -190,6 +190,8 @@ Every row in `~/.pxpipe/events.jsonl` carries the fields needed to reproduce the - `baseline_cacheable_tokens` - `input_tokens` - `cache_create_tokens` +- `cache_create_5m_tokens` +- `cache_create_1h_tokens` - `cache_read_tokens` - `first_user_sha8` - `system_sha8` diff --git a/src/core/baseline.ts b/src/core/baseline.ts index ea2dfa11..c85b9e4b 100644 --- a/src/core/baseline.ts +++ b/src/core/baseline.ts @@ -4,10 +4,31 @@ * See docs/CACHING_AND_SAVINGS.md for the full derivation and audit history. */ -/** Documented Anthropic price ratios: cc_5m = 1.25×, cr = 0.1× base input. One-line change if rates change. */ +/** Documented Anthropic price ratios: cc_5m = 1.25×, cc_1h = 2×, cr = 0.1× base input. */ export const CACHE_CREATE_RATE = 1.25; +const CACHE_CREATE_1H_RATE = 2.0; export const CACHE_READ_RATE = 0.1; +/** Effective cache-write rate for this request. Older usage payloads do not + * expose the tier split; preserve the historical/conservative 5-minute rate + * in that case. The text counterfactual uses the same observed tier mix as + * the transformed request because pxpipe relocates, rather than invents, the + * caller's cache-control markers. */ +function cacheCreateRate(cc: number, cc5m?: number, cc1h?: number): number { + if (!(cc > 0)) return CACHE_CREATE_RATE; + const splitReported = cc5m !== undefined || cc1h !== undefined; + if (!splitReported) return CACHE_CREATE_RATE; + const fiveMinute = Math.max(0, cc5m ?? 0); + const oneHour = Math.max(0, cc1h ?? 0); + const splitTotal = fiveMinute + oneHour; + if (!(splitTotal > 0)) return CACHE_CREATE_RATE; + // The API contract says splitTotal === cc. Normalize malformed/mismatched + // payloads to the aggregate so telemetry never invents extra write tokens. + return ( + fiveMinute * CACHE_CREATE_RATE + oneHour * CACHE_CREATE_1H_RATE + ) / splitTotal; +} + /** Anthropic prompt-cache TTL (seconds). Kept for callers that display provider * docs, but savings math does not use TTL to infer a hypothetical text-cache * hit: text is considered warm only when the actual request reports cr > 0. */ @@ -103,23 +124,46 @@ export function computeBaselineInputEff( cr: number, warm = false, prevCacheable = 0, +): number { + return computeBaselineInputEffWithCacheTier( + baseline, baselineCacheable, inputTokens, cc, cr, warm, prevCacheable, 0, + ); +} + +/** Tier-aware variant for internal telemetry accounting. */ +export function computeBaselineInputEffWithCacheTier( + baseline: number, + baselineCacheable: number, + inputTokens: number, + cc: number, + cr: number, + warm: boolean, + prevCacheable: number, + cacheCreate1hTokens: number, + cacheCreate5mTokens?: number, ): number { if (baseline <= 0) return 0; // Probe miss: can't split prefix from tail, so credit nothing (same as actual). - if (baselineCacheable <= 0) return computeActualInputEff(inputTokens, cc, cr); + if (baselineCacheable <= 0) { + return computeActualInputEffWithCacheTier( + inputTokens, cc, cr, cacheCreate1hTokens, cacheCreate5mTokens, + ); + } const cacheable = Math.min(baselineCacheable, baseline); const coldTail = baseline - cacheable; + const createRate = cacheCreateRate(cc, cacheCreate5mTokens, cacheCreate1hTokens); if (warm) { // Text reads the prefix it already had cached (0.10×) and creates only the - // growth since last turn (1.25×). Independent of the image path's cache. + // growth since last turn (at the observed write-tier rate). Independent of + // the image path's cache. const reused = Math.min(Math.max(prevCacheable, 0), cacheable); const grown = cacheable - reused; - return reused * CACHE_READ_RATE + grown * CACHE_CREATE_RATE + coldTail * 1.0; + return reused * CACHE_READ_RATE + grown * createRate + coldTail * 1.0; } // Cold (first turn / TTL expiry): no warm cache for text either, so it // re-creates the whole cacheable prefix at the create rate — same event the // imaged path pays. Removes the phantom "free read" that fabricated a loss. - return cacheable * CACHE_CREATE_RATE + coldTail * 1.0; + return cacheable * createRate + coldTail * 1.0; } /** Weighted input cost pxpipe actually paid this turn. */ @@ -128,5 +172,18 @@ export function computeActualInputEff( cc: number, cr: number, ): number { - return inputTokens + cc * CACHE_CREATE_RATE + cr * CACHE_READ_RATE; + return computeActualInputEffWithCacheTier(inputTokens, cc, cr, 0); +} + +/** Tier-aware variant for internal telemetry accounting. */ +export function computeActualInputEffWithCacheTier( + inputTokens: number, + cc: number, + cr: number, + cacheCreate1hTokens: number, + cacheCreate5mTokens?: number, +): number { + return inputTokens + + cc * cacheCreateRate(cc, cacheCreate5mTokens, cacheCreate1hTokens) + + cr * CACHE_READ_RATE; } diff --git a/src/dashboard.ts b/src/dashboard.ts index ef6e126e..909f90f3 100644 --- a/src/dashboard.ts +++ b/src/dashboard.ts @@ -33,8 +33,8 @@ import * as readline from 'node:readline'; import type { ProxyEvent } from './core/proxy.js'; import type { TrackEvent } from './core/tracker.js'; import { - computeActualInputEff, - computeBaselineInputEff, + computeActualInputEffWithCacheTier, + computeBaselineInputEffWithCacheTier, deriveBaselineWarmth, } from './core/baseline.js'; import { @@ -167,10 +167,10 @@ export interface RecentRow { * cache_read>0 ⇒ a warm cache existed for both paths; cache_read===0 ⇒ cold * for both, so text re-creates its prefix too (no phantom warm read on a * cold turn — that would fabricate a loss out of a real token win): - * warm: baseline_input_eff = reused×0.10 + grown×1.25 + cold_tail×1.0 + * warm: baseline_input_eff = reused×0.10 + grown×observed_create_rate + cold_tail×1.0 * where reused = min(prevCacheable, cacheable), grown = cacheable − reused - * cold: baseline_input_eff = cacheable×1.25 + cold_tail×1.0 - * actual_input_eff = input + cache_create×1.25 + cache_read×0.10 + * cold: baseline_input_eff = cacheable×observed_create_rate + cold_tail×1.0 + * actual_input_eff = input + cache_create_5m×1.25 + cache_create_1h×2 + cache_read×0.10 * output_equiv = output × 5 (input-token-equivalent at the 5× output rate) * saved = baseline_input_eff − actual_input_eff * baseline_total = baseline_input_eff + output_equiv @@ -607,6 +607,8 @@ export class DashboardState { const inp = u?.input_tokens ?? 0; const out = u?.output_tokens ?? 0; const cc = u?.cache_creation_input_tokens ?? 0; + const cc5m = u?.cache_creation?.ephemeral_5m_input_tokens; + const cc1h = u?.cache_creation?.ephemeral_1h_input_tokens ?? 0; const cr = u?.cache_read_input_tokens ?? 0; const gpt = isOpenAIEvent(ev.path, ev.accountingProvider); @@ -671,7 +673,9 @@ export class DashboardState { haveBaseline = typeof baseline === 'number' && baseline > 0 && probeOk; // Weighted INPUT cost we actually paid this turn. - actualInputEff = haveUsage ? computeActualInputEff(inp, cc, cr) : 0; + actualInputEff = haveUsage + ? computeActualInputEffWithCacheTier(inp, cc, cr, cc1h, cc5m) + : 0; // pxpipe only reduces input by imaging the static slab. An UNCOMPRESSED // row had its body forwarded untouched, so its unproxied counterfactual @@ -710,7 +714,7 @@ export class DashboardState { prefixShaNow, ); baselineInputEff = creditSaving - ? computeBaselineInputEff( + ? computeBaselineInputEffWithCacheTier( baseline as number, cacheable, inp, @@ -718,6 +722,8 @@ export class DashboardState { cr, warm, prevCacheable, + cc1h, + cc5m, ) : actualInputEff; // Record this completed turn's prefix size for future cr>0 split estimates. @@ -953,6 +959,8 @@ export class DashboardState { const inp = t.input_tokens ?? 0; const out = t.output_tokens ?? 0; const cc = t.cache_create_tokens ?? 0; + const cc5m = t.cache_create_5m_tokens; + const cc1h = t.cache_create_1h_tokens ?? 0; const cr = t.cache_read_tokens ?? 0; const compressed = t.compressed === true; const gpt = isOpenAIEvent(t.path, t.accounting_provider); @@ -1003,7 +1011,9 @@ export class DashboardState { const probeOk = probeStatus === 'ok' || (probeStatus === undefined && typeof baseline === 'number' && baseline > 0); haveBaseline = typeof baseline === 'number' && baseline > 0 && probeOk; - actualInputEff = haveUsage ? computeActualInputEff(inp, cc, cr) : 0; + actualInputEff = haveUsage + ? computeActualInputEffWithCacheTier(inp, cc, cr, cc1h, cc5m) + : 0; // Mirror update(): only credit the cache-modeled counterfactual on // compressed rows. Uncompressed/passthrough rows fall back to the // actual cost so they show zero saved (no fabricated savings). @@ -1027,7 +1037,7 @@ export class DashboardState { prefixShaR, ); baselineInputEff = creditSaving - ? computeBaselineInputEff( + ? computeBaselineInputEffWithCacheTier( baseline as number, cacheable, inp, @@ -1035,6 +1045,8 @@ export class DashboardState { cr, warmR, prevCacheableR, + cc1h, + cc5m, ) : actualInputEff; if (typeof sidR === 'string' && sidR.length > 0 && haveUsage) { diff --git a/src/dashboard/fragments.ts b/src/dashboard/fragments.ts index 3e5971e3..18887522 100644 --- a/src/dashboard/fragments.ts +++ b/src/dashboard/fragments.ts @@ -275,9 +275,9 @@ export function renderHeaderFragment(s: StatsPayload, port: number): string { statTile( 'Estimated saved', `$${(s.saved_usd ?? 0).toFixed(2)}`, - `at $${pa.input_per_mtok}/M input tokens`, + `at $${pa.input_per_mtok}/M base input price`, '', - 'A rough dollar figure: saved tokens × the input price. Actual savings depend on your plan and caching — see the math drawer.', + 'Cache-aware estimate using the server-reported 5-minute/1-hour write split when available (1.25×/2×), cache reads (0.10×), and the base input price.', ) + costTile + ``; @@ -285,10 +285,10 @@ export function renderHeaderFragment(s: StatsPayload, port: number): string { // math drawer const savedMath = `
formula: saved = baseline − actual
` + - `
weights: input×1.0, cache_create×1.25, cache_read×0.10
` + + `
weights: input×1.0, cache_write_5m×1.25, cache_write_1h×2.0, cache_read×0.10
` + `
` + mathRow('baseline', s.baseline_input_weighted, '(cache-aware: cacheable×weight + cold_tail)') + - mathRow('actual', s.actual_input_weighted, '(input + cc×1.25 + cr×0.10 from usage)') + + mathRow('actual', s.actual_input_weighted, '(input + cc_5m×1.25 + cc_1h×2.0 + cr×0.10)') + mathRow('saved', s.saved_input_tokens, `= baseline − actual`) + `output excluded — identical with/without compression`; diff --git a/src/dashboard/types.ts b/src/dashboard/types.ts index 27338b72..92962264 100644 --- a/src/dashboard/types.ts +++ b/src/dashboard/types.ts @@ -31,6 +31,7 @@ export interface StatsPayload { compressed_minus_passthrough_avg_usd: number; split_sufficient_sample: boolean; split_min_sample_per_bucket: number; + /** Cache-tier-aware input-side dollar savings. */ saved_usd: number; output_weighted: number; baseline_token_equivalent: number; diff --git a/src/sessions.ts b/src/sessions.ts index a436d393..9e798ccc 100644 --- a/src/sessions.ts +++ b/src/sessions.ts @@ -30,8 +30,8 @@ import * as crypto from 'node:crypto'; import * as readline from 'node:readline'; import type { TrackEvent } from './core/tracker.js'; import { - computeActualInputEff, - computeBaselineInputEff, + computeActualInputEffWithCacheTier, + computeBaselineInputEffWithCacheTier, deriveBaselineWarmth, } from './core/baseline.js'; @@ -179,13 +179,17 @@ export async function aggregateSessions( // Events missing either probe stay out of the rollup — no estimation. const inp = ev.input_tokens ?? 0; const cc = ev.cache_create_tokens ?? 0; + const cc5m = ev.cache_create_5m_tokens; + const cc1h = ev.cache_create_1h_tokens ?? 0; const cr = ev.cache_read_tokens ?? 0; const haveUsage = inp > 0 || cc > 0 || cr > 0; const baseline = ev.baseline_tokens; if ( typeof baseline === 'number' && baseline > 0 && - haveUsage + haveUsage && + ev.compressed === true && + (ev.baseline_probe_status === 'ok' || ev.baseline_probe_status === undefined) ) { const cacheable = ev.baseline_cacheable_tokens ?? 0; const prefixSha = ev.system_sha8; @@ -202,7 +206,7 @@ export async function aggregateSessions( CACHE_TTL_SEC, prefixSha, ); - const baselineEff = computeBaselineInputEff( + const baselineEff = computeBaselineInputEffWithCacheTier( baseline, cacheable, inp, @@ -210,8 +214,10 @@ export async function aggregateSessions( cr, warm, prevCacheable, + cc1h, + cc5m, ); - const actualEff = computeActualInputEff(inp, cc, cr); + const actualEff = computeActualInputEffWithCacheTier(inp, cc, cr, cc1h, cc5m); const tokensSaved = baselineEff - actualEff; s.tokensSavedEst += Math.round(tokensSaved); s.charsSaved += Math.round(tokensSaved * 4); diff --git a/tests/baseline.test.ts b/tests/baseline.test.ts index 2f83dc16..fce6481a 100644 --- a/tests/baseline.test.ts +++ b/tests/baseline.test.ts @@ -1,7 +1,9 @@ import { describe, it, expect } from 'vitest'; import { computeBaselineInputEff, + computeBaselineInputEffWithCacheTier, computeActualInputEff, + computeActualInputEffWithCacheTier, deriveBaselineWarmth, CACHE_CREATE_RATE, CACHE_READ_RATE, @@ -67,6 +69,35 @@ describe('computeBaselineInputEff (warmth-aware)', () => { const warm = computeBaselineInputEff(5000, 4000, inp, cc, cr, true, 4000); expect(cold).toBeGreaterThan(warm); }); + + it('prices server-reported 1-hour cache writes at 2x on both sides', () => { + const actual = computeActualInputEffWithCacheTier(1000, 4000, 0, 4000); + const baseline = computeBaselineInputEffWithCacheTier(5000, 4000, 1000, 4000, 0, false, 0, 4000); + expect(actual).toBe(1000 + 4000 * 2); + expect(baseline).toBe(4000 * 2 + 1000); + }); + + it('uses the observed weighted rate for mixed 5-minute and 1-hour writes', () => { + const actual = computeActualInputEffWithCacheTier(1000, 4000, 0, 1000, 3000); + const baseline = computeBaselineInputEffWithCacheTier( + 5000, 4000, 1000, 4000, 0, false, 0, 1000, 3000, + ); + expect(actual).toBe(1000 + 3000 * 1.25 + 1000 * 2); + expect(baseline).toBe(1000 + 4000 * ((3000 * 1.25 + 1000 * 2) / 4000)); + }); + + it('preserves zero savings on a probe miss with 1-hour writes', () => { + const actual = computeActualInputEffWithCacheTier(1000, 4000, 0, 4000, 0); + const baseline = computeBaselineInputEffWithCacheTier( + 5000, 0, 1000, 4000, 0, false, 0, 4000, 0, + ); + expect(baseline).toBe(actual); + }); + + it('uses both reported tiers and normalizes a mismatched split to aggregate writes', () => { + const actual = computeActualInputEffWithCacheTier(0, 4000, 0, 1000, 1000); + expect(actual).toBe(4000 * ((1000 * 1.25 + 1000 * 2) / 2000)); + }); }); /** diff --git a/tests/dashboard-api.test.ts b/tests/dashboard-api.test.ts index edeff2b7..4e88df17 100644 --- a/tests/dashboard-api.test.ts +++ b/tests/dashboard-api.test.ts @@ -424,6 +424,61 @@ describe('GPT savings split', () => { expect(recent.recent.at(-1)!.session_saved_so_far_delta ?? 0).toBe(0); }); + it('uses the Anthropic 1-hour cache-write split in dashboard savings', async () => { + setAllowedModelBases(['claude-fable-5']); + dash.update({ + method: 'POST', path: '/v1/messages', model: 'claude-fable-5-20260609', status: 200, + durationMs: 1, + usage: { + input_tokens: 1000, + output_tokens: 0, + cache_creation_input_tokens: 2000, + cache_read_input_tokens: 0, + cache_creation: { + ephemeral_5m_input_tokens: 0, + ephemeral_1h_input_tokens: 2000, + }, + }, + info: { + compressed: true, + baselineTokens: 10000, + baselineCacheableTokens: 9000, + baselineProbeStatus: 'ok', + firstUserSha8: 'onehour', + }, + } as never); + + const stats = (await dash.serveStats().json()) as StatsPayload; + expect(stats.actual_input_weighted).toBe(5000); + expect(stats.baseline_input_weighted).toBe(19000); + expect(stats.saved_input_tokens).toBe(14000); + expect(stats.saved_usd).toBe(0.14); + }); + + it('replay() preserves Anthropic 1-hour cache-write accounting', async () => { + writeEvents(tmp, [ + ev({ + model: 'claude-fable-5-20260609', + compressed: true, + input_tokens: 1000, + cache_create_tokens: 2000, + cache_create_5m_tokens: 0, + cache_create_1h_tokens: 2000, + baseline_tokens: 10000, + baseline_cacheable_tokens: 9000, + baseline_probe_status: 'ok', + first_user_sha8: 'onehour-replay', + }), + ]); + await dash.replay(tmp.eventsFile); + + const recent = (await dash.serveRecent().json()) as RecentPayload; + const row = recent.recent.at(-1)!; + expect(row.actual_input).toBe(5000); + expect(row.baseline_input).toBe(19000); + expect(row.session_saved_so_far_delta).toBe(14000); + }); + it('replay() reconstructs GPT recent rows byte-identically to the live path', async () => { writeEvents(tmp, [ ev({ diff --git a/tests/sessions.test.ts b/tests/sessions.test.ts index f4bb075f..97efebc3 100644 --- a/tests/sessions.test.ts +++ b/tests/sessions.test.ts @@ -214,6 +214,24 @@ describe('aggregateSessions', () => { expect(s.charsSaved).toBe(-6_775 * 4); }); + it('excludes passthrough and failed-probe rows from session savings', async () => { + const measured = { + first_user_sha8: 'gatedrows', + baseline_tokens: 10_000, + baseline_cacheable_tokens: 9_000, + input_tokens: 1_000, + cache_create_tokens: 2_000, + } as const; + writeEvents(tmp, [ + ev({ ...measured, compressed: false, baseline_probe_status: 'ok' }), + ev({ ...measured, compressed: true, baseline_probe_status: 'partial' }), + ev({ ...measured, compressed: true, baseline_probe_status: 'failed' }), + ]); + + const { sessions } = await aggregateSessions(tmp); + expect(sessions.get('gatedrows')!.tokensSavedEst).toBe(0); + }); + it('prices text cold when the actual request has cache_read=0', async () => { // Turn 2 is 60s after turn 1, but cr=0 means the server did not report a // cache read for the actual request. The imagined text path gets the same