From ce8f69e74ec7e160e2319221964fb876ff5b8fd9 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Sun, 11 Oct 2026 23:44:08 +0100 Subject: [PATCH] fix(tests): derive the priced Claude model from litellm's catalog by capability The first-party claude-* rows in litellm's price map no longer carry the *_above_200k_tokens tier fields, so no id in the hardcoded candidate list satisfied a test asking for a long-context tier and two test modules failed at collection. Only the Bedrock and Vertex AI Claude rows still carry the tier. anthropic_pricing_model now scans litellm.model_cost for Claude rows that have the requested fields, preferring Anthropic's own rows, then ids without a provider or region qualifier, then alphabetical order, so retiring an id or a field never needs a list edit. The long-context tests in test_counterfactual_pricing and TestLongContextPricing::test_litellm_path ask for a tiered row instead of naming claude-sonnet-4-5, and test_models compares the registry against the catalog instead of a literal $3/$15. --- tests/_pricing_models.py | 66 +++++++++++++++----------- tests/test_counterfactual_pricing.py | 7 ++- tests/test_models.py | 12 +++-- tests/test_providers/test_anthropic.py | 11 ++++- 4 files changed, 61 insertions(+), 35 deletions(-) diff --git a/tests/_pricing_models.py b/tests/_pricing_models.py index 21dfa36f52..824eb77478 100644 --- a/tests/_pricing_models.py +++ b/tests/_pricing_models.py @@ -3,13 +3,19 @@ ``litellm.model_cost`` is downloaded from GitHub at import time, so it is live third-party data. BerriAI prunes retired models from it: on 2026-09-23 ``claude-sonnet-4-20250514`` disappeared and every test that priced it began -failing with ``KeyError: 'input_cost_per_token'`` — on every open pull request -at once, with no change on our side. +failing with ``KeyError: 'input_cost_per_token'``, on every open pull request +at once, with no change on our side. Later the first-party ``claude-*`` entries +lost their ``*_above_200k_tokens`` tier fields altogether (Anthropic ended the +1M-context beta that carried that premium), so a hardcoded list of first-party +ids cannot be refreshed to satisfy a test that needs a long-context tier: the +only ``claude`` entries still carrying it are the Bedrock and Vertex AI rows. The model id in those tests is incidental. They assert that Headroom's cost arithmetic agrees with litellm's numbers, not that any particular model is -priced correctly, so the fix is to stop naming a specific release and instead -ask for *a* model carrying the fields the test needs. +priced correctly, so the helper asks the catalog for *a* Claude model carrying +the fields the test needs, preferring Anthropic's own rows, then ids with no +provider or region qualifier, then alphabetical order, so selection is deterministic and needs no +upkeep as ids retire. Pinning to the copy of the table vendored in the litellm wheel is not the alternative it looks like: the two maps are complementary, not ordered. The @@ -22,40 +28,44 @@ import pytest -#: Preference order, newest first. A test takes the first entry that carries -#: every field it needs, so retiring one is a no-op until the list runs dry. -_CANDIDATES: tuple[str, ...] = ( - "claude-sonnet-4-5-20250929", - "claude-sonnet-4-5", - "claude-sonnet-4-20250514", - "claude-opus-4-5-20251101", - "claude-opus-4-5", -) - _BASE_FIELDS = ("input_cost_per_token", "output_cost_per_token") +_FIRST_PARTY_PROVIDER = "anthropic" + + +def _candidate_order(item: tuple[str, dict[str, object]]) -> tuple[bool, bool, str]: + """Sort key: first-party rows, then unqualified ids, then alphabetical.""" + key, info = item + return ( + info.get("litellm_provider") != _FIRST_PARTY_PROVIDER, + "/" in key or "." in key, + key, + ) + def anthropic_pricing_model(*required_fields: str) -> str: - """Return a currently-priced Anthropic model carrying ``required_fields``. + """Return a currently-priced Claude model carrying ``required_fields``. Always includes the base input/output costs. Raises with an actionable - message rather than skipping: if litellm prices none of these, the pricing - tests are not measuring anything and that should be loud. + message rather than skipping: if litellm prices no Claude model with these + fields, the pricing tests are not measuring anything and that should be + loud. """ import litellm needed = set(_BASE_FIELDS) | set(required_fields) - for model in _CANDIDATES: - info = litellm.model_cost.get(model) - if isinstance(info, dict) and needed <= set(info): - return model - - raise AssertionError( - "litellm prices none of the candidate models with the fields " - f"{sorted(needed)}. It most likely retired them from " - "model_prices_and_context_window.json; add a current model id to " - "_CANDIDATES in tests/_pricing_models.py (newest first)." - ) + matches = [ + (key, info) + for key, info in litellm.model_cost.items() + if "claude" in key and isinstance(info, dict) and needed <= set(info) + ] + if not matches: + raise AssertionError( + f"litellm prices no Claude model with the fields {sorted(needed)}. " + "It most likely dropped them from model_prices_and_context_window.json; " + "check which fields current Claude rows carry before changing the test." + ) + return min(matches, key=_candidate_order)[0] @pytest.fixture(scope="session") diff --git a/tests/test_counterfactual_pricing.py b/tests/test_counterfactual_pricing.py index 9b8a10bdbf..1b10b28be0 100644 --- a/tests/test_counterfactual_pricing.py +++ b/tests/test_counterfactual_pricing.py @@ -172,12 +172,15 @@ def test_untagged_writes_default_to_the_five_minute_ttl(): def test_long_context_derives_the_one_hour_rate_and_says_so(): """No catalog publishes a combined 1h + above-200k rate, so it derives.""" - rates = resolve_rates(SONNET, long_context=True) + tiered = anthropic_pricing_model( + "input_cost_per_token_above_200k_tokens", "cache_creation_input_token_cost" + ) + rates = resolve_rates(tiered, long_context=True) assert rates.basis == BASIS_CATALOG_TTL_RATIO assert rates.write_1h / rates.uncached == pytest.approx(2.00, abs=1e-6) # And it is the EXPENSIVE tier, not the base one. - assert rates.uncached > resolve_rates(SONNET).uncached + assert rates.uncached > resolve_rates(tiered).uncached # ── Provider agnosticism: every harness, every backend ──────────────────── diff --git a/tests/test_models.py b/tests/test_models.py index d8c7e105c3..685a3e6378 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -288,10 +288,16 @@ def test_claude_info(self): assert info.context_window == 200000 # Pricing comes from litellm's live table, so name a model it currently # prices; the retired-id path is the MODEL_ALIASES assertion below. - pricing = ModelRegistry.get_pricing(anthropic_pricing_model()) + import litellm + + priced = anthropic_pricing_model() + pricing = ModelRegistry.get_pricing(priced) assert pricing is not None - assert pricing[0] == 3.00 # input cost per 1M - assert pricing[1] == 15.00 # output cost per 1M + # The registry reports per-1M rates read from litellm's table. + assert pricing[0] == pytest.approx(litellm.model_cost[priced]["input_cost_per_token"] * 1e6) + assert pricing[1] == pytest.approx( + litellm.model_cost[priced]["output_cost_per_token"] * 1e6 + ) # Retired model alias should also resolve alias_pricing = ModelRegistry.get_pricing("claude-3-5-sonnet-20241022") assert alias_pricing is not None diff --git a/tests/test_providers/test_anthropic.py b/tests/test_providers/test_anthropic.py index 45c696f44a..92b64b1d18 100644 --- a/tests/test_providers/test_anthropic.py +++ b/tests/test_providers/test_anthropic.py @@ -2,6 +2,8 @@ import pytest +from tests._pricing_models import anthropic_pricing_model + class TestAnthropicModelSanitization: def test_sanitize_model_id_removes_ansi_escape_sequences(self): @@ -122,9 +124,14 @@ def manual_provider(self, monkeypatch): @pytest.mark.parametrize(("input_tokens", "output_tokens", "cached_tokens", "expected"), _CASES) def test_litellm_path(self, provider, input_tokens, output_tokens, cached_tokens, expected): - cost = provider.estimate_cost( - input_tokens, output_tokens, "claude-sonnet-4-5", cached_tokens + # The first-party catalog rows no longer carry the above-200k tier, so + # the LiteLLM path is exercised through a Claude Sonnet 4.x row that does. + model = anthropic_pricing_model( + "input_cost_per_token_above_200k_tokens", + "output_cost_per_token_above_200k_tokens", + "cache_read_input_token_cost_above_200k_tokens", ) + cost = provider.estimate_cost(input_tokens, output_tokens, model, cached_tokens) assert cost == pytest.approx(expected, rel=1e-4) @pytest.mark.parametrize(("input_tokens", "output_tokens", "cached_tokens", "expected"), _CASES)