From 120160590e2eae908b866013db3d2f8d1193bd46 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 10:28:32 +0900 Subject: [PATCH 1/5] fix(discovery): union Models.dev metadata with provider evidence, not clobber _merge_models_dev_metadata unconditionally overwrote a matched row's architecture/max_output_tokens/context_window with Models.dev's values even when Models.dev had no modalities/limit data for that model, silently discarding the provider's own already-discovered catalog evidence for nothing. Reproduced live with a synthetic partial record, then fixed: these capacity/modality fields are now a field-level union (Models.dev's value wins only when Models.dev actually reports one; the provider's own value survives otherwise). Cost (pricing/is_free) is intentionally left untouched and stays exclusively Models.dev-sourced per ADR 0041's cost-safety argument -- a compromised provider must never be able to self-report "free". Also records live verification of two adjacent gaps that were already correct on main and needed only regression coverage, not a fix: - OpenRouter is already a live PROVIDER_MODEL_SOURCES entry feeding orchestrator/free with no provider-specific carve-out; verified live against the real OpenRouter API (566 models discovered, 10 correctly classified free). - TaskOrchestrator._zdr_agent_allowed's `not zdr_required or has_zdr_tag` boolean makes it mathematically impossible for a non-ZDR-requiring pool to exclude a ZDR-capable agent; added an end-to-end regression test through bootstrap activation covering ZDR-capable free models from two provider families at once. Co-Authored-By: Claude Sonnet 5 --- .../models-dev-metadata-field-union.md | 1 + contextual_orchestrator/model_discovery.py | 42 ++++++++-- docs/product-technical-gap-baseline.md | 50 ++++++++++++ tests/test_model_discovery.py | 81 +++++++++++++++++++ tests/test_provider_bootstrap.py | 55 +++++++++++++ 5 files changed, 224 insertions(+), 5 deletions(-) create mode 100644 CHANGELOG.d/models-dev-metadata-field-union.md diff --git a/CHANGELOG.d/models-dev-metadata-field-union.md b/CHANGELOG.d/models-dev-metadata-field-union.md new file mode 100644 index 000000000..ea3b2f3ba --- /dev/null +++ b/CHANGELOG.d/models-dev-metadata-field-union.md @@ -0,0 +1 @@ +Fixed `_merge_models_dev_metadata` silently discarding a provider's own catalog-reported `architecture`/`context_window`/`max_output_tokens` when Models.dev matched the model by id but had no `modalities`/`limit` data for it. The join now applies Models.dev's value only when Models.dev actually reports one for that field, falling back to the provider's own already-discovered value otherwise; cost (`pricing`/`is_free`) stays exclusively Models.dev-sourced per ADR 0041's cost-safety argument, since that is the one field a compromised provider could otherwise lie about. Added `test_models_dev_merge_unions_fields_instead_of_clobbering_provider_evidence` in `tests/test_model_discovery.py`. diff --git a/contextual_orchestrator/model_discovery.py b/contextual_orchestrator/model_discovery.py index 6d1617830..ea8d8a5c2 100644 --- a/contextual_orchestrator/model_discovery.py +++ b/contextual_orchestrator/model_discovery.py @@ -886,7 +886,24 @@ def collect(value: object, key: str = "") -> None: def _merge_models_dev_metadata(payload: Any, metadata: Any, provider: str) -> Any: - """Join an availability catalog with Models.dev cost and modality evidence.""" + """Join an availability catalog with Models.dev cost and modality evidence. + + Cost is always taken solely from Models.dev (see ``_models_dev_cost_is_free``'s + docstring and ADR 0041's cost-safety argument): none of these four joined + providers reports its own per-model price today, so there is nothing on the + provider side for a compromised or lying upstream to assert instead, and + ``pricing``/``is_free`` must stay third-party-verified rather than + self-reported. Modality and capacity evidence carries no such safety + argument -- it is not used to certify a model as free -- so those fields + are a field-level union instead: Models.dev's value wins only when + Models.dev actually reports one, and the provider's own catalog value + (already present on ``row``) survives untouched whenever Models.dev is + silent on that specific field. Without this fallback, a model matched in + Models.dev but missing ``modalities``/``limit`` data there would have its + own provider-reported architecture/context window/max output tokens + silently discarded in favor of nothing, even though nothing about that + absence casts any doubt on the provider's own value. + """ rows = payload.get("data") if isinstance(payload, dict) else None provider_row = metadata.get(provider) if isinstance(metadata, dict) else None models = provider_row.get("models") if isinstance(provider_row, dict) else None @@ -908,16 +925,31 @@ def _merge_models_dev_metadata(payload: Any, metadata: Any, provider: str) -> An pricing[target_key] = str(Decimal(str(value)) / Decimal(1_000_000)) modalities = model.get("modalities") if isinstance(model.get("modalities"), dict) else {} limits = model.get("limit") if isinstance(model.get("limit"), dict) else {} + original_architecture = row.get("architecture") if isinstance(row.get("architecture"), dict) else {} + merged_max_output_tokens = _positive_int_metadata(limits.get("output")) + if merged_max_output_tokens is None: + merged_max_output_tokens = row.get("max_output_tokens") + merged_context_window = _positive_int_metadata(limits.get("context")) + if merged_context_window is None: + merged_context_window = row.get("context_window", row.get("context_length")) enriched.append( { **row, "pricing": pricing, "architecture": { - "input_modalities": modalities.get("input"), - "output_modalities": modalities.get("output"), + "input_modalities": ( + modalities.get("input") + if modalities.get("input") is not None + else original_architecture.get("input_modalities") + ), + "output_modalities": ( + modalities.get("output") + if modalities.get("output") is not None + else original_architecture.get("output_modalities") + ), }, - "max_output_tokens": _positive_int_metadata(limits.get("output")), - "context_window": _positive_int_metadata(limits.get("context")), + "max_output_tokens": merged_max_output_tokens, + "context_window": merged_context_window, "is_free": _models_dev_cost_is_free(cost), } ) diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index d145a0b1d..eb37651bd 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -1,5 +1,55 @@ # Contextual Orchestrator: Product & Technical Gap Baseline +## 2026-09-02 live verification: OpenRouter free discovery, ZDR non-exclusion, and Models.dev field synthesis + +Observation time: 2026-09-02 Asia/Seoul. Ran the actual discovery code (not +just PR descriptions) against the real OpenRouter API with a live +`OPENROUTER_API_KEY` in read-only/dry-run mode, and read the current `main` +tip (`88390816`) directly. + +- **OpenRouter auto-discovery genuinely feeds `orchestrator/free`.** `openrouter` + is a live entry in `PROVIDER_MODEL_SOURCES` (`contextual_orchestrator/model_discovery.py:402-408`), + reachable through `discover_all_models` with no provider-specific carve-out + in `_is_general_free_agent`/`general_free_serving_candidates`/`agent_from_discovered`. + Live run against `https://openrouter.ai/api/v1/models` on 2026-09-02 + discovered 566 OpenRouter models, correctly classified 10 as `is_free` + (e.g. `z-ai/glm-5.2:free`, `nvidia/nemotron-3-ultra-550b-a55b:free`), with + zero errors. This is the real routing path, not only the opt-in canary CLI + added in PR #985. PR #971/#985 are open and currently `mergeable_state: + behind`; their production-code content already substantially overlaps what + is on `main` today (their live diff against `main` touches only tests/docs), + so the mechanical merge scheduler updating them once approved is expected + to be sufficient -- no further production change was needed here. +- **ZDR-true models are never excluded by a `zdr: false` (not-required) pool.** + `TaskOrchestrator._zdr_agent_allowed` (`contextual_orchestrator/orchestrator.py:3992-3995`) + is `not zdr_required or "privacy:zdr" in agent.tags` -- when a pool/request + does not require ZDR, the expression is unconditionally `True` regardless + of an agent's own tags, so a ZDR-capable agent can mathematically never be + dropped by a non-ZDR-requiring pool. `_apply_discovered_model_evidence` + already propagates OpenRouter's ZDR endpoint evidence onto matching model + ids from every other discovered provider, proven by + `tests/test_model_discovery.py::test_discover_all_models_applies_model_zdr_evidence_to_other_sources`. + Added `tests/test_provider_bootstrap.py::test_orchestrator_free_pool_never_excludes_zdr_true_models_from_multiple_providers` + to prove this end-to-end through bootstrap activation into the live + `orchestrator/free` candidate set for ZDR-capable free models from two + provider families at once. No production change was needed for this item. +- **Models.dev metadata could silently clobber a provider's own metadata -- + fixed.** `_merge_models_dev_metadata` unconditionally overwrote a matched + row's `architecture`/`max_output_tokens`/`context_window` with Models.dev's + values even when Models.dev had no `modalities`/`limit` data for that + model, discarding the provider's own already-discovered evidence for + nothing. Reproduced live with a synthetic partial record before fixing. + Cost (`pricing`/`is_free`) intentionally stays exclusively Models.dev- + sourced (ADR 0041's cost-safety argument: a compromised provider must never + be able to self-report "free"); architecture/capacity fields now union + field-by-field, preferring Models.dev's value only when Models.dev actually + reports one. `openrouter` itself still does not join Models.dev at all + (`models_dev_provider_id=None`, ADR 0041's deliberate choice, since + OpenRouter already reports real per-token pricing) -- that decision is + unchanged; this fix closes the join's real clobbering bug for the four + providers (`openai`, `nvidia_nim`, `nvidia_nim_sub`, `opencode_zen`) that do + join. Added `tests/test_model_discovery.py::test_models_dev_merge_unions_fields_instead_of_clobbering_provider_evidence`. + ## 2026-09-01 Autonomous Commercialization Loop: PR #970 Merge, Token Accounting & Cost Gateway Harmonization Observation time: 2026-09-01 Asia/Seoul. diff --git a/tests/test_model_discovery.py b/tests/test_model_discovery.py index 15f611402..41eb23418 100644 --- a/tests/test_model_discovery.py +++ b/tests/test_model_discovery.py @@ -667,6 +667,87 @@ def test_models_dev_merge_preserves_limit_metadata() -> None: assert merged["data"][0]["max_output_tokens"] == 32768 +def test_models_dev_merge_unions_fields_instead_of_clobbering_provider_evidence() -> None: + """Neither source may silently erase the other's field-level evidence. + + Cost stays solely Models.dev-authoritative (ADR 0041's cost-safety + argument: a compromised provider must never be able to self-report + "free"). Modality and capacity metadata carry no such safety argument, so + they are a field-level union: two partial records, each missing what the + other supplies, must combine rather than have the later source blank out + the earlier one's evidence. + """ + # The provider's own catalog row reports real architecture/capacity + # evidence that Models.dev does not have for this model at all. + payload = { + "data": [ + { + "id": "vendor/only-provider-knows-capacity", + "context_window": 128000, + "max_output_tokens": 4096, + "architecture": { + "input_modalities": ["text", "image"], + "output_modalities": ["text"], + }, + } + ] + } + metadata = { + "openai": { + "models": { + # Matched by id, but Models.dev only has cost evidence here -- + # no "modalities" or "limit" key at all for this model. + "vendor/only-provider-knows-capacity": {"cost": {"input": 0, "output": 0}}, + } + } + } + + merged = _merge_models_dev_metadata(payload, metadata, "openai") + row = merged["data"][0] + + # Models.dev's cost evidence is applied (is_free is third-party-verified)... + assert row["is_free"] is True + # ...while the provider's own architecture/capacity evidence, which + # Models.dev is silent on, survives instead of being blanked to None. + assert row["architecture"] == { + "input_modalities": ["text", "image"], + "output_modalities": ["text"], + } + assert row["context_window"] == 128000 + assert row["max_output_tokens"] == 4096 + + # And when Models.dev *does* report a field, its value still wins over a + # provider's own (e.g. stale) value for that same field. + payload_with_stale = { + "data": [ + { + "id": "vendor/models-dev-knows-more", + "context_window": 8000, + "architecture": {"input_modalities": ["text"], "output_modalities": ["text"]}, + } + ] + } + metadata_with_fresh = { + "openai": { + "models": { + "vendor/models-dev-knows-more": { + "cost": {"input": 0, "output": 0}, + "modalities": {"input": ["text", "audio"], "output": ["text"]}, + "limit": {"context": 200000}, + } + } + } + } + merged_fresh = _merge_models_dev_metadata(payload_with_stale, metadata_with_fresh, "openai") + row_fresh = merged_fresh["data"][0] + assert row_fresh["context_window"] == 200000 + assert row_fresh["architecture"]["input_modalities"] == ["text", "audio"] + # Models.dev did not report max_output_tokens for this model: the + # provider's own catalog row had none either, so the field stays absent + # rather than being fabricated. + assert row_fresh["max_output_tokens"] is None + + @pytest.fixture(autouse=True) def _fresh_backend(): set_backend(InMemoryCredentialBackend()) diff --git a/tests/test_provider_bootstrap.py b/tests/test_provider_bootstrap.py index 442bfa46f..f879165b2 100644 --- a/tests/test_provider_bootstrap.py +++ b/tests/test_provider_bootstrap.py @@ -338,6 +338,61 @@ def test_active_agent_from_discovered_free_vision_model_is_not_free_pool_eligibl assert orchestrator._is_general_free_agent(agent) is False +def test_orchestrator_free_pool_never_excludes_zdr_true_models_from_multiple_providers(): + """A ``zdr: false`` (not-required) pool must still surface ``zdr:true`` members. + + ``zdr: false`` on a request/pool means "ZDR is not *required*", never + "ZDR-capable models are excluded". ``TaskOrchestrator._zdr_agent_allowed`` + encodes this as ``not zdr_required or has_zdr_tag`` -- when + ``zdr_required`` is False, the expression is unconditionally True and + cannot depend on the agent's own tags, so a ``privacy:zdr`` agent can + never be dropped by a pool that merely does not *require* ZDR (see + ``contextual_orchestrator.orchestrator.TaskOrchestrator._zdr_agent_allowed`` + and its use in ``_ranked_agents``). This also proves the discovery-level + guarantee (``test_discover_all_models_applies_model_zdr_evidence_to_other_sources`` + in ``tests/test_model_discovery.py``) survives all the way through + bootstrap activation into the live ``orchestrator/free`` candidate set, + for ZDR-capable free models sourced from more than one provider family. + """ + openrouter_zdr = replace( + _model("openrouter", "OPENROUTER_API_KEY", "openrouter/zdr-free-model", 0.0), + is_free=True, + zdr_capable=True, + ) + nvidia_zdr = replace( + _model("nvidia_nim", "NVIDIA_NIM_API_KEY", "nvidia/zdr-free-model", 0.0), + is_free=True, + zdr_capable=True, + ) + plain_free = replace( + _model("bytez", "BYTEZ_API_KEY", "bytez/plain-free-model", 0.0), + is_free=True, + ) + + agents = [ + provider_bootstrap._active_agent_from_discovered(model) + for model in (openrouter_zdr, nvidia_zdr, plain_free) + ] + orchestrator = TaskOrchestrator(agents) + + # Sanity: the fixture models really do carry the tags this test exercises. + assert {"privacy:zdr", "cost:free"} <= set(agents[0].tags) + assert {"privacy:zdr", "cost:free"} <= set(agents[1].tags) + assert "privacy:zdr" not in agents[2].tags + + # Default (request does not require ZDR): every free agent, ZDR-tagged or + # not, from every provider family, remains an eligible orchestrator/free + # candidate -- ZDR-true is never filtered out by a non-ZDR-requiring pool. + default_pool = orchestrator._ranked_agents("", "worker", free_only=True) + assert {agent.id for agent in default_pool} == {agent.id for agent in agents} + + # Explicit zdr_only=True narrows to just the ZDR-capable members, proving + # the ZDR tag is doing real filtering work (not merely inert metadata). + with orchestrator.request_policy(True): + zdr_only_pool = orchestrator._ranked_agents("", "worker", free_only=True) + assert {agent.id for agent in zdr_only_pool} == {agents[0].id, agents[1].id} + + def test_serving_tags_preserve_explicit_no_zdr_evidence(): """Explicit unsupported zero-data retention survives tag normalization.""" model = replace( From 5fd496166fe215f0ec4842a15d28c699efc1ddd6 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 4 Sep 2026 16:28:56 +0900 Subject: [PATCH 2/5] fix(discovery): retain only metadata union delta Remove OpenRouter and ZDR evidence already superseded by protected main.\n\nCommit-Message-Assisted-by: Codex Signed-off-by: Seongho Bae --- docs/product-technical-gap-baseline.md | 50 ----------------------- tests/test_provider_bootstrap.py | 55 -------------------------- 2 files changed, 105 deletions(-) diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index eb37651bd..d145a0b1d 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -1,55 +1,5 @@ # Contextual Orchestrator: Product & Technical Gap Baseline -## 2026-09-02 live verification: OpenRouter free discovery, ZDR non-exclusion, and Models.dev field synthesis - -Observation time: 2026-09-02 Asia/Seoul. Ran the actual discovery code (not -just PR descriptions) against the real OpenRouter API with a live -`OPENROUTER_API_KEY` in read-only/dry-run mode, and read the current `main` -tip (`88390816`) directly. - -- **OpenRouter auto-discovery genuinely feeds `orchestrator/free`.** `openrouter` - is a live entry in `PROVIDER_MODEL_SOURCES` (`contextual_orchestrator/model_discovery.py:402-408`), - reachable through `discover_all_models` with no provider-specific carve-out - in `_is_general_free_agent`/`general_free_serving_candidates`/`agent_from_discovered`. - Live run against `https://openrouter.ai/api/v1/models` on 2026-09-02 - discovered 566 OpenRouter models, correctly classified 10 as `is_free` - (e.g. `z-ai/glm-5.2:free`, `nvidia/nemotron-3-ultra-550b-a55b:free`), with - zero errors. This is the real routing path, not only the opt-in canary CLI - added in PR #985. PR #971/#985 are open and currently `mergeable_state: - behind`; their production-code content already substantially overlaps what - is on `main` today (their live diff against `main` touches only tests/docs), - so the mechanical merge scheduler updating them once approved is expected - to be sufficient -- no further production change was needed here. -- **ZDR-true models are never excluded by a `zdr: false` (not-required) pool.** - `TaskOrchestrator._zdr_agent_allowed` (`contextual_orchestrator/orchestrator.py:3992-3995`) - is `not zdr_required or "privacy:zdr" in agent.tags` -- when a pool/request - does not require ZDR, the expression is unconditionally `True` regardless - of an agent's own tags, so a ZDR-capable agent can mathematically never be - dropped by a non-ZDR-requiring pool. `_apply_discovered_model_evidence` - already propagates OpenRouter's ZDR endpoint evidence onto matching model - ids from every other discovered provider, proven by - `tests/test_model_discovery.py::test_discover_all_models_applies_model_zdr_evidence_to_other_sources`. - Added `tests/test_provider_bootstrap.py::test_orchestrator_free_pool_never_excludes_zdr_true_models_from_multiple_providers` - to prove this end-to-end through bootstrap activation into the live - `orchestrator/free` candidate set for ZDR-capable free models from two - provider families at once. No production change was needed for this item. -- **Models.dev metadata could silently clobber a provider's own metadata -- - fixed.** `_merge_models_dev_metadata` unconditionally overwrote a matched - row's `architecture`/`max_output_tokens`/`context_window` with Models.dev's - values even when Models.dev had no `modalities`/`limit` data for that - model, discarding the provider's own already-discovered evidence for - nothing. Reproduced live with a synthetic partial record before fixing. - Cost (`pricing`/`is_free`) intentionally stays exclusively Models.dev- - sourced (ADR 0041's cost-safety argument: a compromised provider must never - be able to self-report "free"); architecture/capacity fields now union - field-by-field, preferring Models.dev's value only when Models.dev actually - reports one. `openrouter` itself still does not join Models.dev at all - (`models_dev_provider_id=None`, ADR 0041's deliberate choice, since - OpenRouter already reports real per-token pricing) -- that decision is - unchanged; this fix closes the join's real clobbering bug for the four - providers (`openai`, `nvidia_nim`, `nvidia_nim_sub`, `opencode_zen`) that do - join. Added `tests/test_model_discovery.py::test_models_dev_merge_unions_fields_instead_of_clobbering_provider_evidence`. - ## 2026-09-01 Autonomous Commercialization Loop: PR #970 Merge, Token Accounting & Cost Gateway Harmonization Observation time: 2026-09-01 Asia/Seoul. diff --git a/tests/test_provider_bootstrap.py b/tests/test_provider_bootstrap.py index f879165b2..442bfa46f 100644 --- a/tests/test_provider_bootstrap.py +++ b/tests/test_provider_bootstrap.py @@ -338,61 +338,6 @@ def test_active_agent_from_discovered_free_vision_model_is_not_free_pool_eligibl assert orchestrator._is_general_free_agent(agent) is False -def test_orchestrator_free_pool_never_excludes_zdr_true_models_from_multiple_providers(): - """A ``zdr: false`` (not-required) pool must still surface ``zdr:true`` members. - - ``zdr: false`` on a request/pool means "ZDR is not *required*", never - "ZDR-capable models are excluded". ``TaskOrchestrator._zdr_agent_allowed`` - encodes this as ``not zdr_required or has_zdr_tag`` -- when - ``zdr_required`` is False, the expression is unconditionally True and - cannot depend on the agent's own tags, so a ``privacy:zdr`` agent can - never be dropped by a pool that merely does not *require* ZDR (see - ``contextual_orchestrator.orchestrator.TaskOrchestrator._zdr_agent_allowed`` - and its use in ``_ranked_agents``). This also proves the discovery-level - guarantee (``test_discover_all_models_applies_model_zdr_evidence_to_other_sources`` - in ``tests/test_model_discovery.py``) survives all the way through - bootstrap activation into the live ``orchestrator/free`` candidate set, - for ZDR-capable free models sourced from more than one provider family. - """ - openrouter_zdr = replace( - _model("openrouter", "OPENROUTER_API_KEY", "openrouter/zdr-free-model", 0.0), - is_free=True, - zdr_capable=True, - ) - nvidia_zdr = replace( - _model("nvidia_nim", "NVIDIA_NIM_API_KEY", "nvidia/zdr-free-model", 0.0), - is_free=True, - zdr_capable=True, - ) - plain_free = replace( - _model("bytez", "BYTEZ_API_KEY", "bytez/plain-free-model", 0.0), - is_free=True, - ) - - agents = [ - provider_bootstrap._active_agent_from_discovered(model) - for model in (openrouter_zdr, nvidia_zdr, plain_free) - ] - orchestrator = TaskOrchestrator(agents) - - # Sanity: the fixture models really do carry the tags this test exercises. - assert {"privacy:zdr", "cost:free"} <= set(agents[0].tags) - assert {"privacy:zdr", "cost:free"} <= set(agents[1].tags) - assert "privacy:zdr" not in agents[2].tags - - # Default (request does not require ZDR): every free agent, ZDR-tagged or - # not, from every provider family, remains an eligible orchestrator/free - # candidate -- ZDR-true is never filtered out by a non-ZDR-requiring pool. - default_pool = orchestrator._ranked_agents("", "worker", free_only=True) - assert {agent.id for agent in default_pool} == {agent.id for agent in agents} - - # Explicit zdr_only=True narrows to just the ZDR-capable members, proving - # the ZDR tag is doing real filtering work (not merely inert metadata). - with orchestrator.request_policy(True): - zdr_only_pool = orchestrator._ranked_agents("", "worker", free_only=True) - assert {agent.id for agent in zdr_only_pool} == {agents[0].id, agents[1].id} - - def test_serving_tags_preserve_explicit_no_zdr_evidence(): """Explicit unsupported zero-data retention survives tag normalization.""" model = replace( From ef53030675bf0a809462bd928d3d1d00b521d9f5 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 4 Sep 2026 16:30:01 +0900 Subject: [PATCH 3/5] fix(discovery): align union contract with OpenRouter composition Commit-Message-Assisted-by: Codex Signed-off-by: Seongho Bae --- CHANGELOG.d/models-dev-metadata-field-union.md | 2 +- contextual_orchestrator/model_discovery.py | 9 +++------ tests/test_model_discovery.py | 6 +++--- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.d/models-dev-metadata-field-union.md b/CHANGELOG.d/models-dev-metadata-field-union.md index ea3b2f3ba..7d5a316e8 100644 --- a/CHANGELOG.d/models-dev-metadata-field-union.md +++ b/CHANGELOG.d/models-dev-metadata-field-union.md @@ -1 +1 @@ -Fixed `_merge_models_dev_metadata` silently discarding a provider's own catalog-reported `architecture`/`context_window`/`max_output_tokens` when Models.dev matched the model by id but had no `modalities`/`limit` data for it. The join now applies Models.dev's value only when Models.dev actually reports one for that field, falling back to the provider's own already-discovered value otherwise; cost (`pricing`/`is_free`) stays exclusively Models.dev-sourced per ADR 0041's cost-safety argument, since that is the one field a compromised provider could otherwise lie about. Added `test_models_dev_merge_unions_fields_instead_of_clobbering_provider_evidence` in `tests/test_model_discovery.py`. +Fixed `_merge_models_dev_metadata` silently discarding a provider's own catalog-reported `architecture`/`context_window`/`max_output_tokens` when Models.dev matched the model by id but had no `modalities`/`limit` data for it. The join now applies Models.dev's value only when Models.dev actually reports one for that field, falling back to the provider's own already-discovered value otherwise; free-model classification remains Models.dev-authoritative per ADR 0041's cost-safety argument. Added `test_models_dev_merge_unions_fields_instead_of_clobbering_provider_evidence` in `tests/test_model_discovery.py`. diff --git a/contextual_orchestrator/model_discovery.py b/contextual_orchestrator/model_discovery.py index 8fa75644a..355854a89 100644 --- a/contextual_orchestrator/model_discovery.py +++ b/contextual_orchestrator/model_discovery.py @@ -906,12 +906,9 @@ def collect(value: object, key: str = "") -> None: def _merge_models_dev_metadata(payload: Any, metadata: Any, provider: str) -> Any: """Join an availability catalog with Models.dev cost and modality evidence. - Cost is always taken solely from Models.dev (see ``_models_dev_cost_is_free``'s - docstring and ADR 0041's cost-safety argument): none of these four joined - providers reports its own per-model price today, so there is nothing on the - provider side for a compromised or lying upstream to assert instead, and - ``pricing``/``is_free`` must stay third-party-verified rather than - self-reported. Modality and capacity evidence carries no such safety + Free-model classification is always taken from Models.dev (see + ``_models_dev_cost_is_free`` and ADR 0041's cost-safety argument), so a + provider cannot certify itself as free. Modality and capacity evidence carries no such safety argument -- it is not used to certify a model as free -- so those fields are a field-level union instead: Models.dev's value wins only when Models.dev actually reports one, and the provider's own catalog value diff --git a/tests/test_model_discovery.py b/tests/test_model_discovery.py index 0353d6903..78ad3953a 100644 --- a/tests/test_model_discovery.py +++ b/tests/test_model_discovery.py @@ -737,9 +737,9 @@ def test_models_dev_merge_preserves_limit_metadata() -> None: def test_models_dev_merge_unions_fields_instead_of_clobbering_provider_evidence() -> None: """Neither source may silently erase the other's field-level evidence. - Cost stays solely Models.dev-authoritative (ADR 0041's cost-safety - argument: a compromised provider must never be able to self-report - "free"). Modality and capacity metadata carry no such safety argument, so + Free-model classification stays Models.dev-authoritative (ADR 0041's + cost-safety argument: a compromised provider must never be able to + self-report "free"). Modality and capacity metadata carry no such safety argument, so they are a field-level union: two partial records, each missing what the other supplies, must combine rather than have the later source blank out the earlier one's evidence. From 654482ddc037752ba4c6c38e8220f00c7e4ab40b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 01:40:44 +0900 Subject: [PATCH 4/5] fix(discovery): retain models.dev protocol evidence Signed-off-by: Seongho Bae --- contextual_orchestrator/model_discovery.py | 7 +++++++ tests/test_model_discovery.py | 23 ++++++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/contextual_orchestrator/model_discovery.py b/contextual_orchestrator/model_discovery.py index 355854a89..2ed0cdf13 100644 --- a/contextual_orchestrator/model_discovery.py +++ b/contextual_orchestrator/model_discovery.py @@ -940,6 +940,12 @@ def _merge_models_dev_metadata(payload: Any, metadata: Any, provider: str) -> An pricing[target_key] = str(Decimal(str(value)) / Decimal(1_000_000)) modalities = model.get("modalities") if isinstance(model.get("modalities"), dict) else {} limits = model.get("limit") if isinstance(model.get("limit"), dict) else {} + model_provider = model.get("provider") + models_dev_npm = ( + model_provider.get("npm") + if isinstance(model_provider, dict) + else provider_row.get("npm") + ) original_architecture = row.get("architecture") if isinstance(row.get("architecture"), dict) else {} merged_max_output_tokens = _positive_int_metadata(limits.get("output")) if merged_max_output_tokens is None: @@ -966,6 +972,7 @@ def _merge_models_dev_metadata(payload: Any, metadata: Any, provider: str) -> An "max_output_tokens": merged_max_output_tokens, "context_window": merged_context_window, "is_free": _models_dev_cost_is_free(cost), + "_models_dev_npm": models_dev_npm, } ) return {**payload, "data": enriched} diff --git a/tests/test_model_discovery.py b/tests/test_model_discovery.py index 78ad3953a..221e3c994 100644 --- a/tests/test_model_discovery.py +++ b/tests/test_model_discovery.py @@ -734,6 +734,29 @@ def test_models_dev_merge_preserves_limit_metadata() -> None: assert merged["data"][0]["max_output_tokens"] == 32768 +def test_models_dev_merge_preserves_model_protocol_override() -> None: + payload = {"data": [{"id": "chat-model"}, {"id": "messages-model"}]} + metadata = { + "openrouter": { + "npm": "@ai-sdk/openai-compatible", + "models": { + "chat-model": {"cost": {"input": 0, "output": 0}}, + "messages-model": { + "cost": {"input": 0, "output": 0}, + "provider": {"npm": "@ai-sdk/anthropic"}, + }, + }, + } + } + + merged = _merge_models_dev_metadata(payload, metadata, "openrouter") + + assert [row["_models_dev_npm"] for row in merged["data"]] == [ + "@ai-sdk/openai-compatible", + "@ai-sdk/anthropic", + ] + + def test_models_dev_merge_unions_fields_instead_of_clobbering_provider_evidence() -> None: """Neither source may silently erase the other's field-level evidence. From bec466d2c97c0c20dc47f25bec6e9afd75765eb4 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 01:44:37 +0900 Subject: [PATCH 5/5] fix(discovery): union OpenRouter and models.dev fields Signed-off-by: Seongho Bae --- contextual_orchestrator/model_discovery.py | 4 +++- ...generalize-models-dev-cost-classification.md | 17 ++++++++++++++--- tests/test_model_discovery.py | 8 ++++++++ 3 files changed, 25 insertions(+), 4 deletions(-) diff --git a/contextual_orchestrator/model_discovery.py b/contextual_orchestrator/model_discovery.py index 2ed0cdf13..34b5f3dfa 100644 --- a/contextual_orchestrator/model_discovery.py +++ b/contextual_orchestrator/model_discovery.py @@ -933,6 +933,7 @@ def _merge_models_dev_metadata(payload: Any, metadata: Any, provider: str) -> An continue cost = model.get("cost") pricing: dict[str, str] = {} + original_pricing = row.get("pricing") if isinstance(row.get("pricing"), dict) else {} if isinstance(cost, dict): for source_key, target_key in (("input", "prompt"), ("output", "completion")): value = cost.get(source_key) @@ -956,8 +957,9 @@ def _merge_models_dev_metadata(payload: Any, metadata: Any, provider: str) -> An enriched.append( { **row, - "pricing": pricing, + "pricing": {**original_pricing, **pricing}, "architecture": { + **original_architecture, "input_modalities": ( modalities.get("input") if modalities.get("input") is not None diff --git a/docs/planning/adrs/0041-generalize-models-dev-cost-classification.md b/docs/planning/adrs/0041-generalize-models-dev-cost-classification.md index f4f2eaa7b..3691f5fb9 100644 --- a/docs/planning/adrs/0041-generalize-models-dev-cost-classification.md +++ b/docs/planning/adrs/0041-generalize-models-dev-cost-classification.md @@ -85,9 +85,11 @@ declared configuration. `ProviderModelSource` gains `"opencode"` (replacing the deleted `_MODELS_DEV_OPENCODE_PROVIDER` module constant and its `provider_name == "opencode_zen"` special case with the same value, now expressed as data), `nvidia_nim` and `nvidia_nim_sub` both set it -to `"nvidia"`, and `openai` sets it to `"openai"`. `openrouter` and `bytez` -keep the `None` default: OpenRouter already reports its own real per-token -pricing, and there is no +to `"nvidia"`, `openai` sets it to `"openai"`, and `openrouter` sets it to +`"openrouter"`. OpenRouter's availability row and provider-reported fields are +unioned with Models.dev metadata; neither source erases fields omitted by the +other. Models.dev remains authoritative only for `is_free`. `bytez` keeps the +`None` default because there is no Models.dev signal to join for Bytez. The invocation site in `discover_provider_models` becomes `if @@ -177,6 +179,15 @@ nothing about the retry can turn a paid model free. Motivated by the `orchestrator/free` review-sidecar reliability gap in `ContextualWisdomLab/.github` PR #1433. +## Amendment (2026-09-05): preserve both OpenRouter and Models.dev evidence + +The join now unions pricing and architecture at field granularity. OpenRouter +fields survive when Models.dev omits them, while a present Models.dev field +wins for the same key. The independent `is_free` decision remains derived only +from the complete Models.dev cost object, so preserving provider data cannot +self-certify a model as free. Models.dev's per-model SDK override is retained as +protocol evidence for consumers that expose more than one wire protocol. + ## References Models.dev. (2026). *Models.dev API*. https://models.dev/api.json diff --git a/tests/test_model_discovery.py b/tests/test_model_discovery.py index 221e3c994..fee7283f8 100644 --- a/tests/test_model_discovery.py +++ b/tests/test_model_discovery.py @@ -778,7 +778,9 @@ def test_models_dev_merge_unions_fields_instead_of_clobbering_provider_evidence( "architecture": { "input_modalities": ["text", "image"], "output_modalities": ["text"], + "tokenizer": "provider-tokenizer", }, + "pricing": {"prompt": "0.000001", "image": "0.02"}, } ] } @@ -802,6 +804,12 @@ def test_models_dev_merge_unions_fields_instead_of_clobbering_provider_evidence( assert row["architecture"] == { "input_modalities": ["text", "image"], "output_modalities": ["text"], + "tokenizer": "provider-tokenizer", + } + assert row["pricing"] == { + "prompt": "0", + "completion": "0", + "image": "0.02", } assert row["context_window"] == 128000 assert row["max_output_tokens"] == 4096