diff --git a/CHANGELOG.d/models-dev-metadata-field-union.md b/CHANGELOG.d/models-dev-metadata-field-union.md new file mode 100644 index 000000000..7d5a316e8 --- /dev/null +++ b/CHANGELOG.d/models-dev-metadata-field-union.md @@ -0,0 +1 @@ +Fixed `_merge_models_dev_metadata` silently discarding a provider's own catalog-reported `architecture`/`context_window`/`max_output_tokens` when Models.dev matched the model by id but had no `modalities`/`limit` data for it. The join now applies Models.dev's value only when Models.dev actually reports one for that field, falling back to the provider's own already-discovered value otherwise; free-model classification remains Models.dev-authoritative per ADR 0041's cost-safety argument. Added `test_models_dev_merge_unions_fields_instead_of_clobbering_provider_evidence` in `tests/test_model_discovery.py`. diff --git a/contextual_orchestrator/model_discovery.py b/contextual_orchestrator/model_discovery.py index b4ae2f135..34b5f3dfa 100644 --- a/contextual_orchestrator/model_discovery.py +++ b/contextual_orchestrator/model_discovery.py @@ -904,7 +904,21 @@ def collect(value: object, key: str = "") -> None: def _merge_models_dev_metadata(payload: Any, metadata: Any, provider: str) -> Any: - """Join an availability catalog with Models.dev cost and modality evidence.""" + """Join an availability catalog with Models.dev cost and modality evidence. + + Free-model classification is always taken from Models.dev (see + ``_models_dev_cost_is_free`` and ADR 0041's cost-safety argument), so a + provider cannot certify itself as free. Modality and capacity evidence carries no such safety + argument -- it is not used to certify a model as free -- so those fields + are a field-level union instead: Models.dev's value wins only when + Models.dev actually reports one, and the provider's own catalog value + (already present on ``row``) survives untouched whenever Models.dev is + silent on that specific field. Without this fallback, a model matched in + Models.dev but missing ``modalities``/``limit`` data there would have its + own provider-reported architecture/context window/max output tokens + silently discarded in favor of nothing, even though nothing about that + absence casts any doubt on the provider's own value. + """ rows = payload.get("data") if isinstance(payload, dict) else None provider_row = metadata.get(provider) if isinstance(metadata, dict) else None models = provider_row.get("models") if isinstance(provider_row, dict) else None @@ -919,6 +933,7 @@ def _merge_models_dev_metadata(payload: Any, metadata: Any, provider: str) -> An continue cost = model.get("cost") pricing: dict[str, str] = {} + original_pricing = row.get("pricing") if isinstance(row.get("pricing"), dict) else {} if isinstance(cost, dict): for source_key, target_key in (("input", "prompt"), ("output", "completion")): value = cost.get(source_key) @@ -926,17 +941,40 @@ def _merge_models_dev_metadata(payload: Any, metadata: Any, provider: str) -> An pricing[target_key] = str(Decimal(str(value)) / Decimal(1_000_000)) modalities = model.get("modalities") if isinstance(model.get("modalities"), dict) else {} limits = model.get("limit") if isinstance(model.get("limit"), dict) else {} + model_provider = model.get("provider") + models_dev_npm = ( + model_provider.get("npm") + if isinstance(model_provider, dict) + else provider_row.get("npm") + ) + original_architecture = row.get("architecture") if isinstance(row.get("architecture"), dict) else {} + merged_max_output_tokens = _positive_int_metadata(limits.get("output")) + if merged_max_output_tokens is None: + merged_max_output_tokens = row.get("max_output_tokens") + merged_context_window = _positive_int_metadata(limits.get("context")) + if merged_context_window is None: + merged_context_window = row.get("context_window", row.get("context_length")) enriched.append( { **row, - "pricing": pricing, + "pricing": {**original_pricing, **pricing}, "architecture": { - "input_modalities": modalities.get("input"), - "output_modalities": modalities.get("output"), + **original_architecture, + "input_modalities": ( + modalities.get("input") + if modalities.get("input") is not None + else original_architecture.get("input_modalities") + ), + "output_modalities": ( + modalities.get("output") + if modalities.get("output") is not None + else original_architecture.get("output_modalities") + ), }, - "max_output_tokens": _positive_int_metadata(limits.get("output")), - "context_window": _positive_int_metadata(limits.get("context")), + "max_output_tokens": merged_max_output_tokens, + "context_window": merged_context_window, "is_free": _models_dev_cost_is_free(cost), + "_models_dev_npm": models_dev_npm, } ) return {**payload, "data": enriched} diff --git a/docs/planning/adrs/0041-generalize-models-dev-cost-classification.md b/docs/planning/adrs/0041-generalize-models-dev-cost-classification.md index f4f2eaa7b..3691f5fb9 100644 --- a/docs/planning/adrs/0041-generalize-models-dev-cost-classification.md +++ b/docs/planning/adrs/0041-generalize-models-dev-cost-classification.md @@ -85,9 +85,11 @@ declared configuration. `ProviderModelSource` gains `"opencode"` (replacing the deleted `_MODELS_DEV_OPENCODE_PROVIDER` module constant and its `provider_name == "opencode_zen"` special case with the same value, now expressed as data), `nvidia_nim` and `nvidia_nim_sub` both set it -to `"nvidia"`, and `openai` sets it to `"openai"`. `openrouter` and `bytez` -keep the `None` default: OpenRouter already reports its own real per-token -pricing, and there is no +to `"nvidia"`, `openai` sets it to `"openai"`, and `openrouter` sets it to +`"openrouter"`. OpenRouter's availability row and provider-reported fields are +unioned with Models.dev metadata; neither source erases fields omitted by the +other. Models.dev remains authoritative only for `is_free`. `bytez` keeps the +`None` default because there is no Models.dev signal to join for Bytez. The invocation site in `discover_provider_models` becomes `if @@ -177,6 +179,15 @@ nothing about the retry can turn a paid model free. Motivated by the `orchestrator/free` review-sidecar reliability gap in `ContextualWisdomLab/.github` PR #1433. +## Amendment (2026-09-05): preserve both OpenRouter and Models.dev evidence + +The join now unions pricing and architecture at field granularity. OpenRouter +fields survive when Models.dev omits them, while a present Models.dev field +wins for the same key. The independent `is_free` decision remains derived only +from the complete Models.dev cost object, so preserving provider data cannot +self-certify a model as free. Models.dev's per-model SDK override is retained as +protocol evidence for consumers that expose more than one wire protocol. + ## References Models.dev. (2026). *Models.dev API*. https://models.dev/api.json diff --git a/tests/test_model_discovery.py b/tests/test_model_discovery.py index 1cd2b5e2d..fee7283f8 100644 --- a/tests/test_model_discovery.py +++ b/tests/test_model_discovery.py @@ -734,6 +734,118 @@ def test_models_dev_merge_preserves_limit_metadata() -> None: assert merged["data"][0]["max_output_tokens"] == 32768 +def test_models_dev_merge_preserves_model_protocol_override() -> None: + payload = {"data": [{"id": "chat-model"}, {"id": "messages-model"}]} + metadata = { + "openrouter": { + "npm": "@ai-sdk/openai-compatible", + "models": { + "chat-model": {"cost": {"input": 0, "output": 0}}, + "messages-model": { + "cost": {"input": 0, "output": 0}, + "provider": {"npm": "@ai-sdk/anthropic"}, + }, + }, + } + } + + merged = _merge_models_dev_metadata(payload, metadata, "openrouter") + + assert [row["_models_dev_npm"] for row in merged["data"]] == [ + "@ai-sdk/openai-compatible", + "@ai-sdk/anthropic", + ] + + +def test_models_dev_merge_unions_fields_instead_of_clobbering_provider_evidence() -> None: + """Neither source may silently erase the other's field-level evidence. + + Free-model classification stays Models.dev-authoritative (ADR 0041's + cost-safety argument: a compromised provider must never be able to + self-report "free"). Modality and capacity metadata carry no such safety argument, so + they are a field-level union: two partial records, each missing what the + other supplies, must combine rather than have the later source blank out + the earlier one's evidence. + """ + # The provider's own catalog row reports real architecture/capacity + # evidence that Models.dev does not have for this model at all. + payload = { + "data": [ + { + "id": "vendor/only-provider-knows-capacity", + "context_window": 128000, + "max_output_tokens": 4096, + "architecture": { + "input_modalities": ["text", "image"], + "output_modalities": ["text"], + "tokenizer": "provider-tokenizer", + }, + "pricing": {"prompt": "0.000001", "image": "0.02"}, + } + ] + } + metadata = { + "openai": { + "models": { + # Matched by id, but Models.dev only has cost evidence here -- + # no "modalities" or "limit" key at all for this model. + "vendor/only-provider-knows-capacity": {"cost": {"input": 0, "output": 0}}, + } + } + } + + merged = _merge_models_dev_metadata(payload, metadata, "openai") + row = merged["data"][0] + + # Models.dev's cost evidence is applied (is_free is third-party-verified)... + assert row["is_free"] is True + # ...while the provider's own architecture/capacity evidence, which + # Models.dev is silent on, survives instead of being blanked to None. + assert row["architecture"] == { + "input_modalities": ["text", "image"], + "output_modalities": ["text"], + "tokenizer": "provider-tokenizer", + } + assert row["pricing"] == { + "prompt": "0", + "completion": "0", + "image": "0.02", + } + assert row["context_window"] == 128000 + assert row["max_output_tokens"] == 4096 + + # And when Models.dev *does* report a field, its value still wins over a + # provider's own (e.g. stale) value for that same field. + payload_with_stale = { + "data": [ + { + "id": "vendor/models-dev-knows-more", + "context_window": 8000, + "architecture": {"input_modalities": ["text"], "output_modalities": ["text"]}, + } + ] + } + metadata_with_fresh = { + "openai": { + "models": { + "vendor/models-dev-knows-more": { + "cost": {"input": 0, "output": 0}, + "modalities": {"input": ["text", "audio"], "output": ["text"]}, + "limit": {"context": 200000}, + } + } + } + } + merged_fresh = _merge_models_dev_metadata(payload_with_stale, metadata_with_fresh, "openai") + row_fresh = merged_fresh["data"][0] + assert row_fresh["context_window"] == 200000 + assert row_fresh["architecture"]["input_modalities"] == ["text", "audio"] + # Models.dev did not report max_output_tokens for this model: the + # provider's own catalog row had none either, so the field stays absent + # rather than being fabricated. + assert row_fresh["max_output_tokens"] is None + + @pytest.fixture(autouse=True) def _fresh_backend(): set_backend(InMemoryCredentialBackend())