Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 6 additions & 4 deletions contextual_orchestrator/model_discovery.py
Original file line number Diff line number Diff line change
Expand Up @@ -197,7 +197,6 @@ def configured_gateway_source(
list_url="https://openrouter.ai/api/v1/models?output_modalities=all",
chat_base_url="https://openrouter.ai/api/v1",
capabilities=("chat",),
evidence_only=True,
),
ProviderModelSource(
provider_name="opencode_zen",
Expand Down Expand Up @@ -1279,9 +1278,12 @@ def discover_all_models(
)
except ProviderDiscoveryError as exc:
errors.append(exc)
# The OpenRouter catalog is evidence-only; its public ZDR endpoint supplies
# matching privacy evidence for discovered models from other providers. It
# is never selected as an inference upstream here.
# OpenRouter's public ZDR endpoint supplies exact-match, route-level privacy
# evidence (zdr_capable) for every discovered model, OpenRouter's own rows
# included. ZDR is model-level evidence, not grounds to bar an entire
# provider account from serving: a caller that needs ZDR-only routes reads
# zdr_capable / the privacy:zdr tag per model, same as for any other
# provider.
return _apply_discovered_model_evidence(
_deduplicate_discovered_models(discovered),
_openrouter_zdr_model_ids(timeout=timeout),
Expand Down
37 changes: 37 additions & 0 deletions docs/product-technical-gap-baseline.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,42 @@
# Contextual Orchestrator: Product & Technical Gap Baseline

## 2026-08-31 OpenRouter blanket-excluded from serving on ZDR grounds (PR #951)

Reported directly (product-loop session, Korean): `orchestrator/free` never surfaces an OpenRouter
model, even with `OPENROUTER_API_KEY` registered and OpenRouter's own catalog carrying complete
per-token pricing (unlike Bytez, which never reports pricing at all — a separate, already-tracked
gap in [#940](https://github.com/ContextualWisdomLab/contextual-orchestrator/issues/940)). The
report's own diagnosis was exactly right: "ZDR status is not grounds to ban the whole OpenRouter
provider-account from serving — it must be per-route/model evidence."

**Root cause, traced through the full commit history** (`721ae519` → `588aef3`/`984fe91`/`6dcfc07`
→ `caab2ed` → `952996ec`): `PROVIDER_MODEL_SOURCES`'s `openrouter` entry has carried
`evidence_only=True` since `952996ec` ("fix(discovery): keep OpenRouter catalog evidence-only",
empty commit body, no linked issue). `evidence_only=True` means *no* OpenRouter model can ever
become a serving agent (`agent_from_discovered` hard-refuses; `is_routable_discovered_model`
excludes it), regardless of that model's own ZDR status. The genuinely real, well-documented
security fix in this trail is `721ae519` — it fixed cross-provider ZDR evidence *contamination*
(an earlier version suffix-matched OpenRouter's ZDR feed against every provider's models, so an
unrelated model sharing a display name could inherit OpenRouter's attestation; fixed by requiring
an exact canonical-id match). `952996ec` is a separate, later, much broader change with no
independent rationale on record, and it made things worse: `_apply_discovered_model_evidence`'s
`zdr_capable=not model.evidence_only and matches(model.model_id)` meant OpenRouter's own rows could
never carry their own exact-match ZDR attestation into a serving decision at all.

`ContextualWisdomLab/.github`'s own `scripts/ci/zdr_policy.py` (the central Strix/Noema/OpenCode
review sidecar's policy layer) was already built correctly for this: `is_zdr_model()` evaluates
OpenRouter per exact `"provider/model"` route against the same `/api/v1/endpoints/zdr` feed, with
full `PROVIDER_ZDR_SCOPE`/`PROVIDER_CREDENTIAL_NAMES`/`PROVIDER_BASE_URLS` entries for `openrouter`
already in place — it never depended on this blanket exclusion, it just never had anything to
evaluate because this repo's own discovery layer never let an OpenRouter model become servable.

**Fix**: [PR #951](https://github.com/ContextualWisdomLab/contextual-orchestrator/pull/951) removes
`evidence_only=True` from OpenRouter's `PROVIDER_MODEL_SOURCES` entry (no other source ever set it;
the `evidence_only` guard itself stays as general infrastructure for a genuinely evidence-only
future source). New regression test proves a free, ZDR-attested OpenRouter model is now routable,
serving-agent-eligible, and carries `privacy:zdr` — RED confirmed against the pre-fix source before
restoring the fix. Full suite: `2824 passed, 1 skipped`; `interrogate`: 100%.

## 2026-08-30 provider-catalog-sync: no scheduled run has succeeded in 5 days over one provider; workflow check was too strict

`provider-catalog-sync.yml` (run `33312773022`, job `99260685380`) failed with `credential
Expand Down
8 changes: 5 additions & 3 deletions tests/test_discover_models_cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -380,6 +380,7 @@ def test_enable_cheapest_requires_agents_db() -> None:


def test_enable_cheapest_activates_the_lowest_priced_discovered_agent(tmp_path) -> None:
"""OpenRouter is a normal routable source, so the actual cheapest row wins."""
from contextual_orchestrator import TaskOrchestrator
from contextual_orchestrator.orchestrator import ModelAgent

Expand Down Expand Up @@ -412,14 +413,14 @@ def urlopen(request, timeout=None):
set_backend(None)

report = json.loads(stdout.getvalue())
assert report["enabled_agent_ids"] == ["openai_pricey_model"]
assert report["enabled_agent_ids"] == ["openrouter_cheap_model"]

reloaded = TaskOrchestrator([ModelAgent("seed_agent", "seed-model")], agents_db=db_path)
by_id = {agent.id: agent for agent in reloaded.candidates}
assert by_id["openai_pricey_model"].disabled is False
assert by_id["openrouter_cheap_model"].disabled is False


def test_enable_cheapest_bootstraps_independent_provider_families(tmp_path) -> None:
def test_enable_cheapest_bootstraps_independent_provider_accounts(tmp_path) -> None:
"""CLI bootstrap must use the provider-diverse selector, not only the cheapest vendor."""
from contextual_orchestrator import TaskOrchestrator
from contextual_orchestrator.orchestrator import ModelAgent
Expand Down Expand Up @@ -456,6 +457,7 @@ def urlopen(request, timeout=None):

report = json.loads(stdout.getvalue())
assert report["enabled_agent_ids"] == [
"openrouter_router_model",
"nvidia_nim_nim_model",
"openai_openai_model",
]
Expand Down
50 changes: 49 additions & 1 deletion tests/test_model_discovery.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@
discover_provider_models,
free_discovered_models,
general_free_serving_candidates,
is_routable_discovered_model,
openrouter_paid_inference_available,
refresh_price_book,
select_cheapest_discovered_agent,
Expand Down Expand Up @@ -1505,12 +1506,13 @@ def urlopen(request, timeout=None):


def test_default_sources_request_openrouter_full_modality_catalog() -> None:
"""OpenRouter is a normal routable source: ZDR is per-model evidence, not a blanket ban."""
sources = {source.provider_name: source for source in PROVIDER_MODEL_SOURCES}

assert sources["openai"].capabilities == ()
assert sources["openrouter"].capabilities == ("chat",)
assert sources["openrouter"].list_url.endswith("?output_modalities=all")
assert sources["openrouter"].evidence_only is True
assert sources["openrouter"].evidence_only is False
assert sources["opencode_zen"].list_url == "https://opencode.ai/zen/v1/models"
assert sources["nvidia_nim"].capabilities == ("chat",)
assert sources["nvidia_nim_sub"].capabilities == ("chat",)
Expand Down Expand Up @@ -1628,6 +1630,52 @@ def urlopen(request, timeout=None):
]


def test_openrouter_free_zdr_verified_model_is_routable_and_servable() -> None:
"""ZDR is per-model evidence, not grounds to bar the whole account from serving.

OpenRouter is no longer ``evidence_only`` in ``PROVIDER_MODEL_SOURCES``
(see ContextualWisdomLab/contextual-orchestrator#941/#945 and the
``.github`` sidecar's own ``zdr_policy.is_zdr_model``, which already
evaluates OpenRouter per exact route, never as a blanket exclusion). A
free, ZDR-attested OpenRouter model must become a normal routable,
servable candidate carrying honest ``zdr_capable``/``privacy:zdr``
evidence, exactly like any other provider's free models.
"""
register_credential("OPENROUTER_API_KEY", "sk-openrouter")
openrouter_source = next(
source for source in PROVIDER_MODEL_SOURCES if source.provider_name == "openrouter"
)
assert openrouter_source.evidence_only is False

def urlopen(request, timeout=None):
return _Response(
{"data": [{"id": "free/zdr-model", "pricing": {"prompt": "0", "completion": "0"}}]}
)

with (
patch(
"contextual_orchestrator.model_discovery.urllib.request.urlopen",
side_effect=urlopen,
),
patch(
"contextual_orchestrator.model_discovery._fetch_json_same_host_https",
return_value={"data": [{"model_id": "free/zdr-model"}]},
),
):
discovered, errors = discover_all_models((openrouter_source,))

assert errors == []
model = next(m for m in discovered if m.model_id == "free/zdr-model")
assert model.evidence_only is False
assert model.zdr_capable is True
assert model.is_free is True
assert is_routable_discovered_model(model) is True

agent = agent_from_discovered(model)
assert "privacy:zdr" in agent.tags
assert model in general_free_serving_candidates(discovered)


def test_openrouter_zdr_evidence_uses_the_registered_kv_credential() -> None:
register_credential("OPENROUTER_API_KEY", "sk-openrouter")
seen_calls = []
Expand Down
Loading