diff --git a/contextual_orchestrator/model_discovery.py b/contextual_orchestrator/model_discovery.py index a4d28c819..077725914 100644 --- a/contextual_orchestrator/model_discovery.py +++ b/contextual_orchestrator/model_discovery.py @@ -197,7 +197,6 @@ def configured_gateway_source( list_url="https://openrouter.ai/api/v1/models?output_modalities=all", chat_base_url="https://openrouter.ai/api/v1", capabilities=("chat",), - evidence_only=True, ), ProviderModelSource( provider_name="opencode_zen", @@ -1279,9 +1278,12 @@ def discover_all_models( ) except ProviderDiscoveryError as exc: errors.append(exc) - # The OpenRouter catalog is evidence-only; its public ZDR endpoint supplies - # matching privacy evidence for discovered models from other providers. It - # is never selected as an inference upstream here. + # OpenRouter's public ZDR endpoint supplies exact-match, route-level privacy + # evidence (zdr_capable) for every discovered model, OpenRouter's own rows + # included. ZDR is model-level evidence, not grounds to bar an entire + # provider account from serving: a caller that needs ZDR-only routes reads + # zdr_capable / the privacy:zdr tag per model, same as for any other + # provider. return _apply_discovered_model_evidence( _deduplicate_discovered_models(discovered), _openrouter_zdr_model_ids(timeout=timeout), diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index e10aad199..a74640868 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -1,5 +1,42 @@ # Contextual Orchestrator: Product & Technical Gap Baseline +## 2026-08-31 OpenRouter blanket-excluded from serving on ZDR grounds (PR #951) + +Reported directly (product-loop session, Korean): `orchestrator/free` never surfaces an OpenRouter +model, even with `OPENROUTER_API_KEY` registered and OpenRouter's own catalog carrying complete +per-token pricing (unlike Bytez, which never reports pricing at all — a separate, already-tracked +gap in [#940](https://github.com/ContextualWisdomLab/contextual-orchestrator/issues/940)). The +report's own diagnosis was exactly right: "ZDR status is not grounds to ban the whole OpenRouter +provider-account from serving — it must be per-route/model evidence." + +**Root cause, traced through the full commit history** (`721ae519` → `588aef3`/`984fe91`/`6dcfc07` +→ `caab2ed` → `952996ec`): `PROVIDER_MODEL_SOURCES`'s `openrouter` entry has carried +`evidence_only=True` since `952996ec` ("fix(discovery): keep OpenRouter catalog evidence-only", +empty commit body, no linked issue). `evidence_only=True` means *no* OpenRouter model can ever +become a serving agent (`agent_from_discovered` hard-refuses; `is_routable_discovered_model` +excludes it), regardless of that model's own ZDR status. The genuinely real, well-documented +security fix in this trail is `721ae519` — it fixed cross-provider ZDR evidence *contamination* +(an earlier version suffix-matched OpenRouter's ZDR feed against every provider's models, so an +unrelated model sharing a display name could inherit OpenRouter's attestation; fixed by requiring +an exact canonical-id match). `952996ec` is a separate, later, much broader change with no +independent rationale on record, and it made things worse: `_apply_discovered_model_evidence`'s +`zdr_capable=not model.evidence_only and matches(model.model_id)` meant OpenRouter's own rows could +never carry their own exact-match ZDR attestation into a serving decision at all. + +`ContextualWisdomLab/.github`'s own `scripts/ci/zdr_policy.py` (the central Strix/Noema/OpenCode +review sidecar's policy layer) was already built correctly for this: `is_zdr_model()` evaluates +OpenRouter per exact `"provider/model"` route against the same `/api/v1/endpoints/zdr` feed, with +full `PROVIDER_ZDR_SCOPE`/`PROVIDER_CREDENTIAL_NAMES`/`PROVIDER_BASE_URLS` entries for `openrouter` +already in place — it never depended on this blanket exclusion, it just never had anything to +evaluate because this repo's own discovery layer never let an OpenRouter model become servable. + +**Fix**: [PR #951](https://github.com/ContextualWisdomLab/contextual-orchestrator/pull/951) removes +`evidence_only=True` from OpenRouter's `PROVIDER_MODEL_SOURCES` entry (no other source ever set it; +the `evidence_only` guard itself stays as general infrastructure for a genuinely evidence-only +future source). New regression test proves a free, ZDR-attested OpenRouter model is now routable, +serving-agent-eligible, and carries `privacy:zdr` — RED confirmed against the pre-fix source before +restoring the fix. Full suite: `2824 passed, 1 skipped`; `interrogate`: 100%. + ## 2026-08-30 provider-catalog-sync: no scheduled run has succeeded in 5 days over one provider; workflow check was too strict `provider-catalog-sync.yml` (run `33312773022`, job `99260685380`) failed with `credential diff --git a/tests/test_discover_models_cli.py b/tests/test_discover_models_cli.py index f9a333d31..81affc7a2 100644 --- a/tests/test_discover_models_cli.py +++ b/tests/test_discover_models_cli.py @@ -380,6 +380,7 @@ def test_enable_cheapest_requires_agents_db() -> None: def test_enable_cheapest_activates_the_lowest_priced_discovered_agent(tmp_path) -> None: + """OpenRouter is a normal routable source, so the actual cheapest row wins.""" from contextual_orchestrator import TaskOrchestrator from contextual_orchestrator.orchestrator import ModelAgent @@ -412,14 +413,14 @@ def urlopen(request, timeout=None): set_backend(None) report = json.loads(stdout.getvalue()) - assert report["enabled_agent_ids"] == ["openai_pricey_model"] + assert report["enabled_agent_ids"] == ["openrouter_cheap_model"] reloaded = TaskOrchestrator([ModelAgent("seed_agent", "seed-model")], agents_db=db_path) by_id = {agent.id: agent for agent in reloaded.candidates} - assert by_id["openai_pricey_model"].disabled is False + assert by_id["openrouter_cheap_model"].disabled is False -def test_enable_cheapest_bootstraps_independent_provider_families(tmp_path) -> None: +def test_enable_cheapest_bootstraps_independent_provider_accounts(tmp_path) -> None: """CLI bootstrap must use the provider-diverse selector, not only the cheapest vendor.""" from contextual_orchestrator import TaskOrchestrator from contextual_orchestrator.orchestrator import ModelAgent @@ -456,6 +457,7 @@ def urlopen(request, timeout=None): report = json.loads(stdout.getvalue()) assert report["enabled_agent_ids"] == [ + "openrouter_router_model", "nvidia_nim_nim_model", "openai_openai_model", ] diff --git a/tests/test_model_discovery.py b/tests/test_model_discovery.py index 0b059fd39..b8382028c 100644 --- a/tests/test_model_discovery.py +++ b/tests/test_model_discovery.py @@ -43,6 +43,7 @@ discover_provider_models, free_discovered_models, general_free_serving_candidates, + is_routable_discovered_model, openrouter_paid_inference_available, refresh_price_book, select_cheapest_discovered_agent, @@ -1505,12 +1506,13 @@ def urlopen(request, timeout=None): def test_default_sources_request_openrouter_full_modality_catalog() -> None: + """OpenRouter is a normal routable source: ZDR is per-model evidence, not a blanket ban.""" sources = {source.provider_name: source for source in PROVIDER_MODEL_SOURCES} assert sources["openai"].capabilities == () assert sources["openrouter"].capabilities == ("chat",) assert sources["openrouter"].list_url.endswith("?output_modalities=all") - assert sources["openrouter"].evidence_only is True + assert sources["openrouter"].evidence_only is False assert sources["opencode_zen"].list_url == "https://opencode.ai/zen/v1/models" assert sources["nvidia_nim"].capabilities == ("chat",) assert sources["nvidia_nim_sub"].capabilities == ("chat",) @@ -1628,6 +1630,52 @@ def urlopen(request, timeout=None): ] +def test_openrouter_free_zdr_verified_model_is_routable_and_servable() -> None: + """ZDR is per-model evidence, not grounds to bar the whole account from serving. + + OpenRouter is no longer ``evidence_only`` in ``PROVIDER_MODEL_SOURCES`` + (see ContextualWisdomLab/contextual-orchestrator#941/#945 and the + ``.github`` sidecar's own ``zdr_policy.is_zdr_model``, which already + evaluates OpenRouter per exact route, never as a blanket exclusion). A + free, ZDR-attested OpenRouter model must become a normal routable, + servable candidate carrying honest ``zdr_capable``/``privacy:zdr`` + evidence, exactly like any other provider's free models. + """ + register_credential("OPENROUTER_API_KEY", "sk-openrouter") + openrouter_source = next( + source for source in PROVIDER_MODEL_SOURCES if source.provider_name == "openrouter" + ) + assert openrouter_source.evidence_only is False + + def urlopen(request, timeout=None): + return _Response( + {"data": [{"id": "free/zdr-model", "pricing": {"prompt": "0", "completion": "0"}}]} + ) + + with ( + patch( + "contextual_orchestrator.model_discovery.urllib.request.urlopen", + side_effect=urlopen, + ), + patch( + "contextual_orchestrator.model_discovery._fetch_json_same_host_https", + return_value={"data": [{"model_id": "free/zdr-model"}]}, + ), + ): + discovered, errors = discover_all_models((openrouter_source,)) + + assert errors == [] + model = next(m for m in discovered if m.model_id == "free/zdr-model") + assert model.evidence_only is False + assert model.zdr_capable is True + assert model.is_free is True + assert is_routable_discovered_model(model) is True + + agent = agent_from_discovered(model) + assert "privacy:zdr" in agent.tags + assert model in general_free_serving_candidates(discovered) + + def test_openrouter_zdr_evidence_uses_the_registered_kv_credential() -> None: register_credential("OPENROUTER_API_KEY", "sk-openrouter") seen_calls = []