diff --git a/CHANGELOG.md b/CHANGELOG.md index b1e6d95a..af44f421 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ Cross-package release notes for relayburn. Package changelogs contain package-le ## [Unreleased] - Pricing recognizes Claude 5 and GPT-5.6 models, prefers first-party tariffs over reseller duplicates, and applies long-context price tiers. +- Pricing reports retired first-party models, including the `codex-auto-review` alias, as unpriced instead of inheriting reseller tariffs; summary buckets expose their unpriced-turn count, while a ledger-home `models.dev.json` override can explicitly restore pricing. - `burn hotspots --findings` surfaces unknown model pricing explicitly and ranks unpriced sessions by token volume instead of treating them as $0.00. ## [4.0.0] - 2026-06-23 diff --git a/crates/relayburn-cli/src/commands/summary/human.rs b/crates/relayburn-cli/src/commands/summary/human.rs index 35b13299..8f94ded8 100644 --- a/crates/relayburn-cli/src/commands/summary/human.rs +++ b/crates/relayburn-cli/src/commands/summary/human.rs @@ -594,7 +594,7 @@ pub(super) fn emit_human( report.unpriced_turns, models, ); eprintln!( - " Update the snapshot (pnpm run pricing:update) or add an override at $RELAYBURN_HOME/models.dev.json.", + " Update the snapshot (pnpm run pricing:update) or add an override at /models.dev.json.", ); } } diff --git a/crates/relayburn-cli/src/commands/summary/json.rs b/crates/relayburn-cli/src/commands/summary/json.rs index 6e448833..56d16efd 100644 --- a/crates/relayburn-cli/src/commands/summary/json.rs +++ b/crates/relayburn-cli/src/commands/summary/json.rs @@ -119,6 +119,10 @@ pub(super) fn grouped_json_value( "stopReasons".into(), stop_reasons_to_json(&report.stop_reasons), ); + payload.insert("unpricedTurns".into(), json!(report.unpriced_turns)); + if !report.unpriced_models.is_empty() { + payload.insert("unpricedModels".into(), json!(report.unpriced_models)); + } if !report.subagents.is_empty() { // `subagents: {paired, orphan, total}` (issue #435). Skipped // when both buckets are zero so the JSON shape stays compact diff --git a/crates/relayburn-cli/src/commands/summary/mod.rs b/crates/relayburn-cli/src/commands/summary/mod.rs index 28c5babf..60127b60 100644 --- a/crates/relayburn-cli/src/commands/summary/mod.rs +++ b/crates/relayburn-cli/src/commands/summary/mod.rs @@ -457,9 +457,8 @@ mod tests { assert!(format!("{duplicate}").contains("duplicate --tag filter")); } - #[test] - fn grouped_json_includes_quality_when_report_has_it() { - let report = SummaryGroupedReport { + fn empty_grouped_report() -> SummaryGroupedReport { + SummaryGroupedReport { group_by: SummaryGroupBy::Model, tag_key: None, tag_values: Vec::new(), @@ -479,16 +478,35 @@ mod tests { replacement_savings: relayburn_sdk::ReplacementSavingsSummary::default(), stop_reasons: relayburn_sdk::StopReasonCounts::default(), subagents: SubagentCounts::default(), - quality: Some(QualityResult::default()), + quality: None, unpriced_turns: 0, unpriced_models: Vec::new(), - }; + } + } + + #[test] + fn grouped_json_includes_quality_when_report_has_it() { + let mut report = empty_grouped_report(); + report.quality = Some(QualityResult::default()); let value = grouped_json_value(&report, &relayburn_sdk::IngestReport::empty()); assert_eq!(value["quality"], json!({"outcomes": [], "oneShot": []})); } + #[test] + fn grouped_json_surfaces_unpriced_turns_and_models() { + let mut report = empty_grouped_report(); + report.turn_count = 1; + report.unpriced_turns = 1; + report.unpriced_models = vec!["gpt-5-codex".into()]; + + let value = grouped_json_value(&report, &relayburn_sdk::IngestReport::empty()); + + assert_eq!(value["unpricedTurns"], 1); + assert_eq!(value["unpricedModels"], json!(["gpt-5-codex"])); + } + #[test] fn subagents_line_renders_only_when_counts_nonzero() { // Empty bucket → skipped, line absent (so old summaries keep @@ -514,30 +532,7 @@ mod tests { fn subagents_json_payload_includes_total_and_omits_when_empty() { // Empty bucket → key absent in JSON so `summary.json | jq` for // pre-#435 callers still passes without a `?.` guard. - let mut report = SummaryGroupedReport { - group_by: SummaryGroupBy::Model, - tag_key: None, - tag_values: Vec::new(), - turn_count: 0, - rows: Vec::new(), - total_cost: CostBreakdown { - model: String::new().into(), - total: 0.0, - input: 0.0, - output: 0.0, - reasoning: 0.0, - cache_read: 0.0, - cache_create: 0.0, - }, - fidelity: relayburn_sdk::summarize_fidelity(&[]), - per_cell_fidelity: json!({"groupBy": "model"}), - replacement_savings: relayburn_sdk::ReplacementSavingsSummary::default(), - stop_reasons: relayburn_sdk::StopReasonCounts::default(), - subagents: SubagentCounts::default(), - quality: None, - unpriced_turns: 0, - unpriced_models: Vec::new(), - }; + let mut report = empty_grouped_report(); let value = grouped_json_value(&report, &relayburn_sdk::IngestReport::empty()); assert!( value.get("subagents").is_none(), diff --git a/crates/relayburn-sdk/data/primary-model-ids.json b/crates/relayburn-sdk/data/primary-model-ids.json new file mode 100644 index 00000000..4c9a3886 --- /dev/null +++ b/crates/relayburn-sdk/data/primary-model-ids.json @@ -0,0 +1,206 @@ +[ + "chatgpt-image-latest", + "claude-3-5-haiku-20241022", + "claude-3-5-haiku-latest", + "claude-3-5-haiku@20241022", + "claude-3-5-sonnet-20240620", + "claude-3-5-sonnet-20241022", + "claude-3-5-sonnet@20241022", + "claude-3-7-sonnet-20250219", + "claude-3-7-sonnet@20250219", + "claude-3-haiku-20240307", + "claude-3-opus-20240229", + "claude-3-sonnet-20240229", + "claude-fable-5", + "claude-haiku-4-5", + "claude-haiku-4-5-20251001", + "claude-haiku-4-5@20251001", + "claude-opus-4-0", + "claude-opus-4-1", + "claude-opus-4-1-20250805", + "claude-opus-4-1@20250805", + "claude-opus-4-20250514", + "claude-opus-4-5", + "claude-opus-4-5-20251101", + "claude-opus-4-5@20251101", + "claude-opus-4-6", + "claude-opus-4-6@default", + "claude-opus-4-7", + "claude-opus-4-7@default", + "claude-opus-4-8", + "claude-opus-4-8@default", + "claude-opus-4@20250514", + "claude-opus-5", + "claude-opus-5@default", + "claude-sonnet-4-0", + "claude-sonnet-4-20250514", + "claude-sonnet-4-5", + "claude-sonnet-4-5-20250929", + "claude-sonnet-4-5@20250929", + "claude-sonnet-4-6", + "claude-sonnet-4-6@default", + "claude-sonnet-4@20250514", + "claude-sonnet-5", + "claude-sonnet-5@default", + "codex-auto-review", + "deep-research-max-preview-04-2026", + "deep-research-preview-04-2026", + "deepseek-ai/deepseek-v3.1-maas", + "deepseek-ai/deepseek-v3.2-maas", + "gemini-1.5-flash", + "gemini-1.5-flash-8b", + "gemini-1.5-pro", + "gemini-2.0-flash", + "gemini-2.0-flash-lite", + "gemini-2.5-computer-use-preview-10-2025", + "gemini-2.5-flash", + "gemini-2.5-flash-image", + "gemini-2.5-flash-image-preview", + "gemini-2.5-flash-lite", + "gemini-2.5-flash-lite-preview-06-17", + "gemini-2.5-flash-lite-preview-09-2025", + "gemini-2.5-flash-preview-04-17", + "gemini-2.5-flash-preview-05-20", + "gemini-2.5-flash-preview-09-2025", + "gemini-2.5-flash-preview-tts", + "gemini-2.5-flash-tts", + "gemini-2.5-pro", + "gemini-2.5-pro-preview-05-06", + "gemini-2.5-pro-preview-06-05", + "gemini-2.5-pro-preview-tts", + "gemini-2.5-pro-tts", + "gemini-3-flash-preview", + "gemini-3-pro-image", + "gemini-3-pro-image-preview", + "gemini-3-pro-preview", + "gemini-3.1-flash-image", + "gemini-3.1-flash-image-preview", + "gemini-3.1-flash-lite", + "gemini-3.1-flash-lite-image", + "gemini-3.1-flash-lite-preview", + "gemini-3.1-flash-live-preview", + "gemini-3.1-flash-tts-preview", + "gemini-3.1-pro-preview", + "gemini-3.1-pro-preview-customtools", + "gemini-3.5-flash", + "gemini-3.5-flash-lite", + "gemini-3.5-live-translate-preview", + "gemini-3.6-flash", + "gemini-embedding-001", + "gemini-embedding-2", + "gemini-flash-latest", + "gemini-flash-lite-latest", + "gemini-live-2.5-flash", + "gemini-live-2.5-flash-preview-native-audio", + "gemini-omni-flash-preview", + "gemini-robotics-er-1.6-preview", + "gemma-3-12b-it", + "gemma-3-27b-it", + "gemma-3-4b-it", + "gemma-3n-e2b-it", + "gemma-3n-e4b-it", + "gemma-4-26b-a4b-it", + "gemma-4-31b-it", + "gpt-3.5-turbo", + "gpt-4", + "gpt-4-turbo", + "gpt-4.1", + "gpt-4.1-mini", + "gpt-4.1-nano", + "gpt-4o", + "gpt-4o-2024-05-13", + "gpt-4o-2024-08-06", + "gpt-4o-2024-11-20", + "gpt-4o-mini", + "gpt-5", + "gpt-5-chat-latest", + "gpt-5-codex", + "gpt-5-mini", + "gpt-5-nano", + "gpt-5-pro", + "gpt-5.1", + "gpt-5.1-chat-latest", + "gpt-5.1-codex", + "gpt-5.1-codex-max", + "gpt-5.1-codex-mini", + "gpt-5.2", + "gpt-5.2-chat-latest", + "gpt-5.2-codex", + "gpt-5.2-pro", + "gpt-5.3-chat-latest", + "gpt-5.3-codex", + "gpt-5.3-codex-spark", + "gpt-5.4", + "gpt-5.4-mini", + "gpt-5.4-nano", + "gpt-5.4-pro", + "gpt-5.5", + "gpt-5.5-pro", + "gpt-5.6", + "gpt-5.6-luna", + "gpt-5.6-sol", + "gpt-5.6-terra", + "gpt-image-1", + "gpt-image-1-mini", + "gpt-image-1.5", + "gpt-image-2", + "gpt-realtime-2.1", + "grok-2", + "grok-2-1212", + "grok-2-latest", + "grok-2-vision", + "grok-2-vision-1212", + "grok-2-vision-latest", + "grok-3", + "grok-3-fast", + "grok-3-fast-latest", + "grok-3-latest", + "grok-3-mini", + "grok-3-mini-fast", + "grok-3-mini-fast-latest", + "grok-3-mini-latest", + "grok-4", + "grok-4-1-fast", + "grok-4-1-fast-non-reasoning", + "grok-4-fast", + "grok-4-fast-non-reasoning", + "grok-4.20-0309-non-reasoning", + "grok-4.20-0309-reasoning", + "grok-4.20-multi-agent-0309", + "grok-4.3", + "grok-4.5", + "grok-beta", + "grok-build-0.1", + "grok-code-fast-1", + "grok-imagine-image", + "grok-imagine-image-quality", + "grok-imagine-video", + "grok-imagine-video-1.5", + "grok-vision-beta", + "lyria-3-clip-preview", + "lyria-3-pro-preview", + "meta/llama-3.3-70b-instruct-maas", + "meta/llama-4-maverick-17b-128e-instruct-maas", + "moonshotai/kimi-k2-thinking-maas", + "o1", + "o1-mini", + "o1-preview", + "o1-pro", + "o3", + "o3-deep-research", + "o3-mini", + "o3-pro", + "o4-mini", + "o4-mini-deep-research", + "openai/gpt-oss-120b-maas", + "openai/gpt-oss-20b-maas", + "qwen/qwen3-235b-a22b-instruct-2507-maas", + "text-embedding-3-large", + "text-embedding-3-small", + "text-embedding-ada-002", + "veo-3.1-fast-generate-preview", + "veo-3.1-generate-preview", + "veo-3.1-lite-generate-preview", + "zai-org/glm-4.7-maas", + "zai-org/glm-5-maas" +] diff --git a/crates/relayburn-sdk/src/analyze/pricing.rs b/crates/relayburn-sdk/src/analyze/pricing.rs index 07246a98..2b7f7f1c 100644 --- a/crates/relayburn-sdk/src/analyze/pricing.rs +++ b/crates/relayburn-sdk/src/analyze/pricing.rs @@ -130,6 +130,14 @@ type ModelsDevRoot = IndexMap; const PRIMARY_PRICING_PROVIDERS: &[&str] = &["anthropic", "openai", "google", "google-vertex", "xai"]; +/// Bare model IDs owned by a primary pricing provider, plus burn's aliases for +/// those IDs. Unlike the models.dev snapshot, this set is append-only: +/// `pnpm run pricing:update` retains the existing catalog and unions the +/// outgoing and incoming primary-provider IDs before replacing the snapshot. +/// That history prevents a retired first-party model or its logged alias from +/// silently inheriting a reseller tariff after its primary entry disappears. +const BUILTIN_PRIMARY_MODEL_IDS_JSON: &str = include_str!("../../data/primary-model-ids.json"); + /// Bundled `models.dev.json` snapshot. Refreshed via `pnpm run pricing:update`, /// which writes through to the SDK crate's `data/` copy. Vendoring inside the /// crate is required so `cargo package` / `cargo publish --dry-run` can verify @@ -140,7 +148,10 @@ const BUILTIN_PRICING_JSON: &str = include_str!("../../data/models.dev.json"); /// `HashMap` of several hundred entries, and `load_builtin_pricing` is on the /// hot path of multiple SDK verbs that each used to re-parse it. static BUILTIN_PRICING: LazyLock = LazyLock::new(|| { - parse_pricing(BUILTIN_PRICING_JSON).expect("bundled models.dev.json must parse") + let protected_models: HashSet = serde_json::from_str(BUILTIN_PRIMARY_MODEL_IDS_JSON) + .expect("bundled primary-model-ids.json must parse"); + parse_pricing_with_protected_models(BUILTIN_PRICING_JSON, &protected_models) + .expect("bundled models.dev.json must parse") }); /// Load the bundled `models.dev` snapshot. No I/O — the JSON is embedded at @@ -172,15 +183,22 @@ fn load_from_file(path: &Path) -> io::Result { } fn parse_pricing(raw: &str) -> serde_json::Result { + parse_pricing_with_protected_models(raw, &HashSet::new()) +} + +fn parse_pricing_with_protected_models( + raw: &str, + protected_models: &HashSet, +) -> serde_json::Result { let parsed: ModelsDevRoot = serde_json::from_str(raw)?; - Ok(flatten(&parsed)) + Ok(flatten(&parsed, protected_models)) } /// Flatten a nested `provider → model → cost` map into the flat /// `model_id → ModelCost` table burn uses for lookup. Skips entries that lack /// either `input` or `output` — matches the TS guard so we don't surface /// half-priced models. -fn flatten(root: &ModelsDevRoot) -> PricingTable { +fn flatten(root: &ModelsDevRoot, protected_models: &HashSet) -> PricingTable { let mut out = PricingTable::new(); let mut primary_models: HashSet = HashSet::new(); for (provider_id, provider) in root { @@ -189,7 +207,7 @@ fn flatten(root: &ModelsDevRoot) -> PricingTable { }; for (id, model) in models { let primary_provider = PRIMARY_PRICING_PROVIDERS.contains(&provider_id.as_str()); - if !primary_provider && primary_models.contains(id) { + if !primary_provider && (primary_models.contains(id) || protected_models.contains(id)) { continue; } let Some(cost) = model.cost.as_ref() else { @@ -302,6 +320,43 @@ mod tests { assert_eq!(sol.context_tiers[0].output, 45.0); } + #[test] + fn builtin_snapshot_does_not_price_retired_primary_models_from_resellers() { + let table = load_builtin_pricing(); + for model in [ + "claude-sonnet-4-20250514", + "gemini-2.5-flash-preview-05-20", + "gpt-5-codex", + ] { + assert!( + !table.contains_key(model), + "retired first-party model {model} must be unpriced" + ); + } + } + + #[test] + fn builtin_primary_models_are_all_recorded_in_ownership_history() { + let root: ModelsDevRoot = serde_json::from_str(BUILTIN_PRICING_JSON).unwrap(); + let protected: HashSet = + serde_json::from_str(BUILTIN_PRIMARY_MODEL_IDS_JSON).unwrap(); + + for provider_id in PRIMARY_PRICING_PROVIDERS { + let Some(models) = root + .get(*provider_id) + .and_then(|provider| provider.models.as_ref()) + else { + continue; + }; + for model_id in models.keys() { + assert!( + protected.contains(model_id), + "primary model {provider_id}/{model_id} is missing from primary-model-ids.json" + ); + } + } + } + #[test] fn flatten_preserves_separate_reasoning_tariff() { let raw = r#"{ @@ -435,6 +490,57 @@ mod tests { assert_eq!(cost.output, 60.0); } + #[test] + fn protected_primary_model_never_defaults_cache_fields_from_reseller() { + let raw = r#"{ + "reseller": { + "models": { + "retired-primary": { "cost": { "input": 2.7, "output": 13.5 } } + } + } + }"#; + let protected = HashSet::from(["retired-primary".to_string()]); + let table = parse_pricing_with_protected_models(raw, &protected).unwrap(); + assert!( + !table.contains_key("retired-primary"), + "a reseller entry without cache tariffs must not become a ModelCost" + ); + } + + #[test] + fn protected_internal_alias_never_takes_a_direct_reseller_price() { + let raw = r#"{ + "reseller": { + "models": { + "codex-auto-review": { "cost": { "input": 2.7, "output": 13.5 } } + } + } + }"#; + let protected: HashSet = + serde_json::from_str(BUILTIN_PRIMARY_MODEL_IDS_JSON).unwrap(); + assert!(protected.contains("codex-auto-review")); + let table = parse_pricing_with_protected_models(raw, &protected).unwrap(); + assert!(!table.contains_key("codex-auto-review")); + } + + #[test] + fn flatten_keeps_reseller_exclusive_model_that_was_never_primary() { + let raw = r#"{ + "reseller": { + "models": { + "reseller-exclusive": { "cost": { "input": 1.2, "output": 4.8 } } + } + } + }"#; + let protected = HashSet::from(["some-other-model".to_string()]); + let table = parse_pricing_with_protected_models(raw, &protected).unwrap(); + let cost = table + .get("reseller-exclusive") + .expect("never-primary reseller model remains priced"); + assert_eq!(cost.input, 1.2); + assert_eq!(cost.output, 4.8); + } + #[test] fn flatten_skips_models_without_input_or_output() { let raw = r#"{ @@ -498,4 +604,37 @@ mod tests { // Other builtin entries are still present. assert!(table.contains_key("claude-sonnet-4-6")); } + + #[test] + fn load_pricing_override_can_price_a_retired_primary_model() { + let override_path = std::env::temp_dir().join(format!( + "relayburn-retired-pricing-test-{}.json", + std::process::id() + )); + let raw = r#"{ + "user": { + "models": { + "gpt-5-codex": { + "cost": { + "input": 7, + "output": 11, + "cache_read": 0.7, + "cache_write": 8 + } + } + } + } + }"#; + fs::write(&override_path, raw).unwrap(); + let table = load_pricing(Some(&override_path)); + let _ = fs::remove_file(&override_path); + + let cost = table + .get("gpt-5-codex") + .expect("user override restores retired model pricing"); + assert_eq!(cost.input, 7.0); + assert_eq!(cost.output, 11.0); + assert_eq!(cost.cache_read, 0.7); + assert_eq!(cost.cache_write, 8.0); + } } diff --git a/crates/relayburn-sdk/src/query_verbs/compare.rs b/crates/relayburn-sdk/src/query_verbs/compare.rs index 7b689151..f202d27e 100644 --- a/crates/relayburn-sdk/src/query_verbs/compare.rs +++ b/crates/relayburn-sdk/src/query_verbs/compare.rs @@ -126,7 +126,7 @@ impl LedgerHandle { turns.retain(|t| has_minimum_fidelity(t.turn.fidelity.as_ref(), min_fidelity)); } - let pricing = load_pricing(None); + let pricing = load_pricing_for_ledger(self); let table = build_compare_table( &turns, &AnalyzeCompareOptions { @@ -183,7 +183,7 @@ impl LedgerHandle { filter.contains(&provider.to_ascii_lowercase()) }); } - let pricing = load_pricing(None); + let pricing = load_pricing_for_ledger(self); let Some((buckets, per_bucket)) = super::partition_into_buckets(turns, q.since.as_deref(), bucket_secs, |t| &t.turn.ts)? diff --git a/crates/relayburn-sdk/src/query_verbs/flow.rs b/crates/relayburn-sdk/src/query_verbs/flow.rs index 394714b9..8809e24d 100644 --- a/crates/relayburn-sdk/src/query_verbs/flow.rs +++ b/crates/relayburn-sdk/src/query_verbs/flow.rs @@ -427,7 +427,7 @@ impl LedgerHandle { /// span trees get loaded. The same window is then applied to the /// returned [`Vec`] cap. pub fn context_delta(&self, opts: ContextDeltaOpts) -> Result> { - let pricing = load_pricing(None); + let pricing = load_pricing_for_ledger(self); // Build the seed `since` filter from `opts.since`. We always have a // sensible `effective_since()` default, but only apply it when the diff --git a/crates/relayburn-sdk/src/query_verbs/hotspots.rs b/crates/relayburn-sdk/src/query_verbs/hotspots.rs index ec0b2871..065017fb 100644 --- a/crates/relayburn-sdk/src/query_verbs/hotspots.rs +++ b/crates/relayburn-sdk/src/query_verbs/hotspots.rs @@ -208,7 +208,7 @@ impl LedgerHandle { filter.contains(&provider.to_ascii_lowercase()) }); } - let pricing = load_pricing(None); + let pricing = load_pricing_for_ledger(self); if matches!(opts.group_by, Some(HotspotsGroupBy::Findings)) { let patterns = match opts.patterns { diff --git a/crates/relayburn-sdk/src/query_verbs/mod.rs b/crates/relayburn-sdk/src/query_verbs/mod.rs index 0a683f8a..318456ad 100644 --- a/crates/relayburn-sdk/src/query_verbs/mod.rs +++ b/crates/relayburn-sdk/src/query_verbs/mod.rs @@ -514,6 +514,20 @@ fn open_with(ledger_home: Option<&Path>) -> Result { Ledger::open(opts) } +/// Load built-in pricing plus the override adjacent to this handle's ledger. +/// +/// `LedgerOpenOptions::with_home(home)` stores `burn.sqlite` in `home`, so the +/// ledger path remains the single source of truth even for embedders that do +/// not configure `RELAYBURN_HOME` in the process environment. +fn load_pricing_for_ledger(handle: &LedgerHandle) -> PricingTable { + let override_path = handle + .inner + .burn_path() + .parent() + .map(|home| home.join("models.dev.json")); + load_pricing(override_path.as_deref()) +} + fn normalize_provider_filter(provider: Option>) -> Option { let filter: ProviderFilter = provider .unwrap_or_default() diff --git a/crates/relayburn-sdk/src/query_verbs/overhead.rs b/crates/relayburn-sdk/src/query_verbs/overhead.rs index c67ab98c..c95a2a12 100644 --- a/crates/relayburn-sdk/src/query_verbs/overhead.rs +++ b/crates/relayburn-sdk/src/query_verbs/overhead.rs @@ -167,7 +167,7 @@ fn gather_overhead( ..Default::default() }; let turns = collect_turns(handle, &q)?; - let pricing = load_pricing(None); + let pricing = load_pricing_for_ledger(handle); let attribution = attribute_overhead(AttributeOverheadInput { files: &parsed_files, turns: &turns, diff --git a/crates/relayburn-sdk/src/query_verbs/sessions.rs b/crates/relayburn-sdk/src/query_verbs/sessions.rs index 1241a718..bace89b0 100644 --- a/crates/relayburn-sdk/src/query_verbs/sessions.rs +++ b/crates/relayburn-sdk/src/query_verbs/sessions.rs @@ -48,7 +48,7 @@ impl LedgerHandle { note: Some("no turns recorded for this session yet".to_string()), }); } - let pricing = load_pricing(None); + let pricing = load_pricing_for_ledger(self); let mut models = std::collections::BTreeSet::new(); let mut total_tokens: u64 = 0; let mut costs = Vec::with_capacity(turns.len()); @@ -197,7 +197,7 @@ impl LedgerHandle { let q = build_query(None, opts.project.as_deref(), opts.since.as_deref())?; let turns = collect_turns(self, &q)?; - let pricing = load_pricing(None); + let pricing = load_pricing_for_ledger(self); // Aggregate per-session in a single pass over the turn stream. let mut acc: BTreeMap = BTreeMap::new(); for turn in &turns { diff --git a/crates/relayburn-sdk/src/query_verbs/summary/mod.rs b/crates/relayburn-sdk/src/query_verbs/summary/mod.rs index cf692e26..37a456b4 100644 --- a/crates/relayburn-sdk/src/query_verbs/summary/mod.rs +++ b/crates/relayburn-sdk/src/query_verbs/summary/mod.rs @@ -158,7 +158,7 @@ impl LedgerHandle { } let enriched = self.inner.query_turns(&q)?; let turns: Vec = enriched.iter().map(|e| e.turn.clone()).collect(); - let pricing = load_pricing(None); + let pricing = load_pricing_for_ledger(self); let mut summary = compute_summary(&turns, &pricing); if let Some(tag) = group_by_tag { summary.by_tag = Some(compute_summary_by_tag(&enriched, &tag, &pricing)); @@ -542,6 +542,7 @@ pub struct SummaryBucket { pub start: String, pub end: String, pub turn_count: u64, + pub unpriced_turns: u64, pub total_tokens: u64, pub total_cost: CostBreakdown, pub group_by: SummaryGroupBy, @@ -587,7 +588,7 @@ impl LedgerHandle { let q = build_summary_report_query(&opts)?; let provider_filter = normalize_summary_provider_filter(opts.providers.as_deref()); - let pricing = load_pricing(None); + let pricing = load_pricing_for_ledger(self); let agent_session_ids = match opts.agent.as_deref() { Some(agent_id) => Some(resolve_summary_agent_session_tree(&self.inner, agent_id)?), None => None, @@ -620,6 +621,7 @@ impl LedgerHandle { .into_iter() .enumerate() .map(|(i, bturns)| { + let (unpriced_turns, _) = tally_unpriced(&bturns, &pricing); let rows = if by_provider { aggregate_by_provider(&bturns, AggregateByProviderOptions::new(&pricing)) .into_iter() @@ -634,6 +636,7 @@ impl LedgerHandle { start: buckets.start_iso(i), end: buckets.end_iso(i), turn_count: bturns.len() as u64, + unpriced_turns, total_tokens, total_cost, group_by, @@ -651,7 +654,7 @@ impl LedgerHandle { pub fn summary_report(&self, opts: SummaryReportOptions) -> Result { let q = build_summary_report_query(&opts)?; let provider_filter = normalize_summary_provider_filter(opts.providers.as_deref()); - let pricing = load_pricing(None); + let pricing = load_pricing_for_ledger(self); let agent_session_ids = match opts.agent.as_deref() { Some(agent_id) => Some(resolve_summary_agent_session_tree(&self.inner, agent_id)?), None => None, diff --git a/crates/relayburn-sdk/src/query_verbs/tests.rs b/crates/relayburn-sdk/src/query_verbs/tests.rs index 5f345ae6..60793c68 100644 --- a/crates/relayburn-sdk/src/query_verbs/tests.rs +++ b/crates/relayburn-sdk/src/query_verbs/tests.rs @@ -476,6 +476,44 @@ fn summary_report_grouped_tracks_unpriced_turns_and_models() { ); } +#[test] +fn ledger_home_pricing_override_restores_retired_model_cost() { + let dir = tempfile::tempdir().unwrap(); + std::fs::write( + dir.path().join("models.dev.json"), + r#"{ + "user": { + "models": { + "gpt-5-codex": { + "cost": { "input": 7, "output": 11, "cache_read": 0.7, "cache_write": 8 } + } + } + } + }"#, + ) + .unwrap(); + let mut handle = Ledger::open(LedgerOpenOptions::with_home(dir.path())).unwrap(); + let mut turn = bucket_test_turn( + "override-session", + "override-message", + "2026-05-01T00:00:00.000Z", + 1_000_000, + ); + turn.model = "gpt-5-codex".into(); + turn.usage.output = 1_000_000; + handle.raw_mut().append_turns(&[turn]).unwrap(); + + let SummaryReport::Grouped(report) = handle + .summary_report(SummaryReportOptions::default()) + .unwrap() + else { + panic!("expected grouped report"); + }; + assert_eq!(report.unpriced_turns, 0); + assert!(report.unpriced_models.is_empty()); + assert_eq!(report.total_cost.total, 18.0); +} + /// Acceptance test for issue #437: the legacy `LedgerHandle::summary` /// surface (the slim one) also exposes the new counts. Verifies a turn /// without a stop_reason field round-trips to `None`/`none` rather @@ -1274,6 +1312,36 @@ fn compare_returns_flat_cells_and_absent_models() { assert!(json["fidelity"]["summary"]["missingCoverage"].is_object()); } +#[test] +fn compare_existing_priced_turns_signal_exposes_unpriced_model() { + let (_dir, mut handle) = fixture_handle(); + let mut turn = bucket_test_turn( + "retired-session", + "retired-message", + "2026-04-23T00:02:00.000Z", + 1_000, + ); + turn.model = "gpt-5-codex".into(); + handle.raw_mut().append_turns(&[turn]).unwrap(); + + let result = handle + .compare(CompareOptions { + models: vec!["claude-sonnet-4-6".into(), "gpt-5-codex".into()], + min_fidelity: Some(FidelityClass::Partial), + ..CompareOptions::default() + }) + .unwrap(); + + let cell = result + .cells + .iter() + .find(|cell| cell.model == "gpt-5-codex") + .unwrap(); + assert_eq!(cell.turns, 1); + assert_eq!(cell.priced_turns, 0); + assert_eq!(cell.cost_per_turn, None); +} + #[test] fn compare_metadata_counts_all_matched_turns_pre_models_filter() { // TS-parity contract: `analyzedTurns` and `fidelity.summary` describe @@ -2820,12 +2888,11 @@ fn summary_timeseries_places_turns_in_buckets_and_sums_to_total() { let now = super::system_now_secs() as i64; let ts_recent = super::format_iso_z_ms(now - 180, 0); // 3m ago let ts_older = super::format_iso_z_ms(now - 720, 0); // 12m ago + let mut retired = bucket_test_turn("s1", "m1", &ts_recent, 1_000); + retired.model = "gpt-5-codex".into(); handle .raw_mut() - .append_turns(&[ - bucket_test_turn("s1", "m1", &ts_recent, 1_000), - bucket_test_turn("s1", "m2", &ts_older, 2_000), - ]) + .append_turns(&[retired, bucket_test_turn("s1", "m2", &ts_older, 2_000)]) .expect("append"); let series = handle @@ -2847,6 +2914,21 @@ fn summary_timeseries_places_turns_in_buckets_and_sums_to_total() { "two turns 9m apart -> two distinct 5m buckets" ); assert!(nonempty.iter().all(|b| b.turn_count == 1)); + let retired_bucket = nonempty + .iter() + .find(|bucket| bucket.rows.iter().any(|row| row.label == "gpt-5-codex")) + .expect("retired model bucket"); + assert_eq!(retired_bucket.unpriced_turns, 1); + let priced_bucket = nonempty + .iter() + .find(|bucket| { + bucket + .rows + .iter() + .any(|row| row.label == "claude-sonnet-4-6") + }) + .expect("priced model bucket"); + assert_eq!(priced_bucket.unpriced_turns, 0); // Per-bucket totals reconcile with the un-bucketed total. let total_tokens: u64 = series.buckets.iter().map(|b| b.total_tokens).sum(); diff --git a/scripts/update-pricing.mjs b/scripts/update-pricing.mjs index 981dfa50..728f833f 100644 --- a/scripts/update-pricing.mjs +++ b/scripts/update-pricing.mjs @@ -1,5 +1,5 @@ #!/usr/bin/env node -import { writeFile } from 'node:fs/promises'; +import { readFile, writeFile } from 'node:fs/promises'; import * as path from 'node:path'; import { fileURLToPath } from 'node:url'; @@ -7,6 +7,27 @@ const __dirname = path.dirname(fileURLToPath(import.meta.url)); const OUTS = [ path.resolve(__dirname, '..', 'crates', 'relayburn-sdk', 'data', 'models.dev.json'), ]; +const PRIMARY_MODEL_IDS = path.resolve( + __dirname, + '..', + 'crates', + 'relayburn-sdk', + 'data', + 'primary-model-ids.json', +); +const PRIMARY_PRICING_PROVIDERS = [ + 'anthropic', + 'openai', + 'google', + 'google-vertex', + 'xai', +]; + +function primaryModelIds(snapshot) { + return PRIMARY_PRICING_PROVIDERS.flatMap((provider) => + Object.keys(snapshot[provider]?.models ?? {}), + ); +} const res = await fetch('https://models.dev/api.json'); if (!res.ok) { @@ -14,6 +35,21 @@ if (!res.ok) { process.exit(1); } const body = await res.text(); +const incoming = JSON.parse(body); +const outgoing = JSON.parse(await readFile(OUTS[0], 'utf8')); +// Retain the full ownership catalog, including burn-defined aliases such as +// codex-auto-review, then add every outgoing/incoming first-party snapshot ID. +const retained = JSON.parse(await readFile(PRIMARY_MODEL_IDS, 'utf8')); +const primaryIds = [ + ...new Set([ + ...retained, + ...primaryModelIds(outgoing), + ...primaryModelIds(incoming), + ]), +].sort(); + +await writeFile(PRIMARY_MODEL_IDS, `${JSON.stringify(primaryIds, null, 2)}\n`, 'utf8'); +console.log(`wrote ${PRIMARY_MODEL_IDS} (${primaryIds.length} primary model ids)`); for (const out of OUTS) { await writeFile(out, body, 'utf8'); console.log(`wrote ${out} (${body.length} bytes)`); diff --git a/tests/fixtures/cli-golden/snapshots/state-status-json.stdout.txt b/tests/fixtures/cli-golden/snapshots/state-status-json.stdout.txt index f36069be..e422a7ab 100644 --- a/tests/fixtures/cli-golden/snapshots/state-status-json.stdout.txt +++ b/tests/fixtures/cli-golden/snapshots/state-status-json.stdout.txt @@ -21,7 +21,7 @@ "rows": 0 }, "archive": { - "schemaVersion": 5 + "schemaVersion": 6 }, "config": { "store": "off", diff --git a/tests/fixtures/cli-golden/snapshots/state-status.stdout.txt b/tests/fixtures/cli-golden/snapshots/state-status.stdout.txt index 36954c47..9e3e7401 100644 --- a/tests/fixtures/cli-golden/snapshots/state-status.stdout.txt +++ b/tests/fixtures/cli-golden/snapshots/state-status.stdout.txt @@ -14,7 +14,7 @@ content DB (content.sqlite): path: ${RELAYBURN_HOME}/content.sqlite rows: 0 archive state: - schema version: 5 + schema version: 6 last built: never last rebuild: never config: diff --git a/tests/fixtures/cli-golden/snapshots/summary-json.stdout.txt b/tests/fixtures/cli-golden/snapshots/summary-json.stdout.txt index 9721c222..10b8718b 100644 --- a/tests/fixtures/cli-golden/snapshots/summary-json.stdout.txt +++ b/tests/fixtures/cli-golden/snapshots/summary-json.stdout.txt @@ -6,9 +6,9 @@ "turns": 7, "totalCost": { "model": "aggregate", - "total": 0.0441, - "input": 0.0195, - "output": 0.01755, + "total": 0.0376, + "input": 0.017, + "output": 0.01355, "reasoning": 0, "cacheRead": 0.0063, "cacheCreate": 0.00075 @@ -35,27 +35,6 @@ "cacheCreate": 0.00075 } }, - { - "model": "gpt-5-codex", - "turns": 1, - "usage": { - "input": 2000, - "output": 400, - "reasoning": 350, - "cacheRead": 0, - "cacheCreate5m": 0, - "cacheCreate1h": 0 - }, - "cost": { - "model": "gpt-5-codex", - "total": 0.006500000000000001, - "input": 0.0025, - "output": 0.004, - "reasoning": 0, - "cacheRead": 0, - "cacheCreate": 0 - } - }, { "model": "claude-haiku-4-5", "turns": 2, @@ -76,6 +55,27 @@ "cacheRead": 0.00045, "cacheCreate": 0 } + }, + { + "model": "gpt-5-codex", + "turns": 1, + "usage": { + "input": 2000, + "output": 400, + "reasoning": 350, + "cacheRead": 0, + "cacheCreate5m": 0, + "cacheCreate1h": 0 + }, + "cost": { + "model": "gpt-5-codex", + "total": 0, + "input": 0, + "output": 0, + "reasoning": 0, + "cacheRead": 0, + "cacheCreate": 0 + } } ], "fidelity": { @@ -137,54 +137,54 @@ } }, { - "label": "gpt-5-codex", - "partial": true, + "label": "claude-haiku-4-5", + "partial": false, "fields": { "input": { - "known": 1, + "known": 2, "missing": 0 }, "output": { - "known": 1, + "known": 2, "missing": 0 }, "reasoning": { - "known": 1, + "known": 2, "missing": 0 }, "cacheRead": { - "known": 0, - "missing": 1 + "known": 2, + "missing": 0 }, "cacheCreate": { - "known": 0, - "missing": 1 + "known": 2, + "missing": 0 } } }, { - "label": "claude-haiku-4-5", - "partial": false, + "label": "gpt-5-codex", + "partial": true, "fields": { "input": { - "known": 2, + "known": 1, "missing": 0 }, "output": { - "known": 2, + "known": 1, "missing": 0 }, "reasoning": { - "known": 2, + "known": 1, "missing": 0 }, "cacheRead": { - "known": 2, - "missing": 0 + "known": 0, + "missing": 1 }, "cacheCreate": { - "known": 2, - "missing": 0 + "known": 0, + "missing": 1 } } } @@ -200,5 +200,9 @@ "refusal": 0, "silent": 0, "none": 7 - } + }, + "unpricedTurns": 1, + "unpricedModels": [ + "gpt-5-codex" + ] } diff --git a/tests/fixtures/cli-golden/snapshots/summary.stderr.txt b/tests/fixtures/cli-golden/snapshots/summary.stderr.txt new file mode 100644 index 00000000..25a7ca11 --- /dev/null +++ b/tests/fixtures/cli-golden/snapshots/summary.stderr.txt @@ -0,0 +1,2 @@ +warning: 1 turn(s) had no pricing for model(s): gpt-5-codex — their cost is reported as $0. + Update the snapshot (pnpm run pricing:update) or add an override at /models.dev.json. diff --git a/tests/fixtures/cli-golden/snapshots/summary.stdout.txt b/tests/fixtures/cli-golden/snapshots/summary.stdout.txt index 83e70e7d..1f2932f4 100644 --- a/tests/fixtures/cli-golden/snapshots/summary.stdout.txt +++ b/tests/fixtures/cli-golden/snapshots/summary.stdout.txt @@ -5,11 +5,11 @@ turns analyzed: 7 model turns input output reasoning cacheRead cacheCreate cost claude-sonnet-4-6 4 5,100 830 0 19,500 200 $0.034 -gpt-5-codex 1 2,000 400 350 — — $0.0065 claude-haiku-4-5 2 1,700 220 0 4,500 0 $0.0033 +gpt-5-codex 1 2,000 400 350 — — $0.00 -total cost: $0.044 - input $0.019 / output $0.018 / reasoning $0.00 / cacheRead $0.0063 / cacheCreate $0.0008 +total cost: $0.038 + input $0.017 / output $0.014 / reasoning $0.00 / cacheRead $0.0063 / cacheCreate $0.0008 Turn outcomes: 0 end_turn, 0 max_tokens, 0 refusal, 0 pause, 7 none