diff --git a/README.md b/README.md index c710d07..da4d843 100644 --- a/README.md +++ b/README.md @@ -95,6 +95,24 @@ Requirements and caveats: - A withheld response has already been billed by OpenRouter. - The hosted router cannot choose local or non-OpenRouter providers. +## Jev + +A `jev` alias makes Jev Cerberus's model-selection intelligence while Cerberus +keeps execution. The pool may span every provider Cerberus knows, local hosts +included. Before each request Cerberus filters it, asks Jev through OpenRouter's +Decisions API which remaining model is the cheapest one strong enough, and runs +that model first through its own failover loop, the rest in configured order. + +- Each pool model needs a registry `strength` (`basic`, `standard`, `strong`, + `frontier`); Jev weighs it against cost tier. +- Jev reads the latest user message by default (`jev.input`), or the whole + conversation, or request metadata only. Provider names never leave. +- `typesafe/jev-1.13` is pinned by default so an upstream model update cannot + silently change routing. +- If no usable decision comes back, the request runs in configured order; a + decision outage never takes the alias down. +- Streams, fails over and cools down like any dispatch alias. + ## Dispatch Dispatch resolves a stable Cerberus alias through a validated, identity-scoped diff --git a/config/cerberus.example.yaml b/config/cerberus.example.yaml index ac748f9..2253b44 100644 --- a/config/cerberus.example.yaml +++ b/config/cerberus.example.yaml @@ -18,7 +18,8 @@ providers: primary: { api_key_env: OPENROUTER_API_KEY } models: openrouter/free: { cost_tier: free, capabilities: [chat, tools] } - anthropic/claude-sonnet: { cost_tier: paid, capabilities: [chat, tools, coding] } + # strength is read only by jev aliases: Jev weighs it against cost + anthropic/claude-sonnet: { cost_tier: paid, strength: frontier, capabilities: [chat, tools, coding] } google/gemini-pro: { cost_tier: paid, capabilities: [chat, tools, coding] } google: @@ -27,7 +28,7 @@ providers: credentials: free-1: { api_key_env: GEMINI_KEY_1 } models: - gemini-flash: { cost_tier: free, capabilities: [chat, tools] } + gemini-flash: { cost_tier: free, strength: standard, capabilities: [chat, tools] } quota_cooldown_seconds: 14400 # daily free-tier quota transport_cooldown_seconds: 30 # network blips never brick a provider for hours quota_scope: model # credential = one 429 cools every model on that key @@ -77,6 +78,24 @@ aliases: timeout_seconds: 120 allow_paid_pool: true # Jev often picks a paid model; opt in explicitly + cerberus/smart: + mode: jev + # Jev as Cerberus's model-selection intelligence; Cerberus executes. The + # pool may span any providers (local hosts included). Before each request + # Cerberus filters it, asks Jev through OpenRouter's Decisions API which + # remaining model is the cheapest one strong enough, then runs that model + # first through its own failover loop. No usable answer -> configured order. + candidates: # configured order is the fallback order + - { provider: google, credential: free-1, model: gemini-flash } + - { provider: openrouter, credential: primary, model: anthropic/claude-sonnet } + jev: + decider: { provider: openrouter, credential: primary } + model: typesafe/jev-1.13 # pinned; bump deliberately after testing + input: last_user_message # or full_conversation, or metadata (no text) + max_input_chars: 4000 + timeout_seconds: 5 # decision deadline; then configured order + allow_paid_pool: true + identities: example-reader: credential_env: CB_KEY_EXAMPLE_READER # holds a cb- client key diff --git a/docs/architecture.md b/docs/architecture.md index 703c3bb..ccf36d6 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -9,6 +9,8 @@ client → cerberus-api → identity + alias authorization + cost policy → filtered pool → Jev Router → OpenRouter /chat/completions → typesafe/jev-router chooses one pool model → decision verified against the pool → response + → filtered pool → Jev decision → OpenRouter /api/alpha/decisions + → chosen model first → Dispatch → any provider ``` Cerberus owns routing policy, identity authorization, configured free/paid @@ -68,10 +70,51 @@ calls. Readiness is the backend's presence and a non-empty pool. The hosted router can only choose among models one OpenRouter credential reaches; it cannot place a request on a local or non-OpenRouter provider. A -native strategy in which Cerberus asks Jev for a decision over every eligible -backend and executes the winner itself is the intended next step, and would -reuse the same pool filter. There is no per-request cost ceiling: Cerberus does -not hold model prices, and a ceiling checked after a routed call is too late. +`jev` alias does. There is no per-request cost ceiling: Cerberus does not hold +model prices, and a ceiling checked after a routed call is too late. + +## Jev + +A `jev` alias uses Jev as Cerberus's model-selection intelligence and keeps +execution in Cerberus. Three layers, three questions: Cerberus answers "what may +run?", Jev answers "what should run?", and the provider runs it. + +The pool is the alias's candidates on any provider — a local llama.cpp host, +Cloudflare, OpenRouter — each a registry model with a `strength`. Before every +request `router/pool.py` filters it exactly as it does for jev-router (cost, then +health, cooldown, credential). The distinct models that survive are offered to +Jev under opaque ids (`m1`, `m2`, …) and described by registry facts only: model +id, strength, cost tier, context window, capabilities, description. Provider and +credential names never leave. One `choice` question goes to OpenRouter's +Decisions API (`POST /api/alpha/decisions`, model `typesafe/jev-1.13` pinned by +default) under the decider credential the policy names: which candidate is the +least costly one still strong enough for this request. + +The decision service is a third party even when the winner is local, so what it +reads is policy (`jev.input`): the latest user message (default), the whole +conversation, or request metadata with no prompt text at all — always truncated +to `max_input_chars`, always with the request's shape (turns, size, tools, +response format, requested output length). With one model or none left there is +nothing to choose, and no decision is asked, so no text leaves. + +Jev's answer can only reorder: the chosen model goes first, the rest of the +cost-admitted pool follows in configured order, and the ordinary failover loop +executes that order — health, cooldown and credential gates, 429 cooldowns, +streaming and telemetry included. An answer that is not exactly an offered id is +never interpreted. Every way of not getting a usable answer — no decider backend +or credential, the decider's provider marked down, timeout, transport or HTTP +error, an unreadable response, an absent or unknown choice — runs the request in +configured order instead, which every pool member already satisfies; the alias +stays routable while no decision can be had. + +Each routing event carries a `jev` record: options offered, whether the decision +was `chosen`, `skipped` or a `fallback`, the reason, the choice and its +probability, decision latency, status and usage. It never carries request text. +The confirmed Decisions API contract covers the endpoint, the model, the +`model`/`state`/`questions` request and `answers` keyed by question id; the +fields inside one question and one answer are not yet confirmed, so the answer +is read tolerantly and a misread degrades to configured order rather than to an +unvetted model. ## Dispatch control plane diff --git a/docs/persistence.md b/docs/persistence.md index 7032135..d280f32 100644 --- a/docs/persistence.md +++ b/docs/persistence.md @@ -1,7 +1,7 @@ # Persistence review for 0.01 SQLite stores two state classes. Revisioned control state contains service -aliases, bindings, ordered paths, Fusion and jev-router pool members, identity policy and credential +aliases, bindings, ordered paths, Fusion, jev-router and jev pool members, identity policy and credential references plus one atomic active-revision pointer. Operational state contains provider health, cooldowns, routing events, usage projections and audit records; with OIDC enabled it also includes pending login and session state. Credential diff --git a/src/cerberus/app.py b/src/cerberus/app.py index 9153940..203d780 100644 --- a/src/cerberus/app.py +++ b/src/cerberus/app.py @@ -48,6 +48,7 @@ from cerberus.fusion import FusionBackend, OpenRouterFusionBackend, fusion_dispatch, fusion_status from cerberus.identity.session_store import InMemorySessionStore, SqliteSessionStore from cerberus.identity.sso import AdminSSO +from cerberus.jev import DecisionBackend, OpenRouterJevDecider, jev_dispatch from cerberus.jev_router import JevRouterBackend, OpenRouterJevRouterBackend, jev_router_dispatch from cerberus.registry import CerberusConfig, ConfigDocument, load_config_document from cerberus.registry.schema import Alias, Candidate @@ -164,6 +165,7 @@ def create_app( fusion_transport: httpx.AsyncBaseTransport | None = None, sso_transport: httpx.AsyncBaseTransport | None = None, jev_router_transport: httpx.AsyncBaseTransport | None = None, + jev_transport: httpx.AsyncBaseTransport | None = None, ) -> FastAPI: if config is None: document = load_config_document() @@ -217,6 +219,9 @@ def create_app( jev_router_backends: dict[str, JevRouterBackend] = { "openrouter": OpenRouterJevRouterBackend(jev_router_transport) } + # Decision services a jev alias asks which pool model to run, keyed by + # JevPolicy.backend; fixed at boot the same way. + jev_deciders: dict[str, DecisionBackend] = {"openrouter": OpenRouterJevDecider(jev_transport)} if boot_config.admin_sso is not None: # sessions + pending logins share the state DB when one is configured, so # they survive restarts and are consistent across workers; else in-memory @@ -238,6 +243,8 @@ async def lifespan(app: FastAPI): await backend.aclose() for router in jev_router_backends.values(): await router.aclose() + for decider in jev_deciders.values(): + await decider.aclose() if verifier is not None: await verifier.aclose() if admin_sso is not None: @@ -654,7 +661,7 @@ async def admin_routes(request: Request) -> Response: cost gate, the same health/cooldown/credential order the failover loop walks, the same authorization predicate the inference path applies, and the same three gates fusion dispatch refuses on, and the same filtered - pool a jev-router request would send. + pool a jev-router request would send or a jev decision would be offered. Read-only, and behind the same boundary as /admin/status: the projection names every provider, model and credential reference in the revision, so @@ -672,6 +679,7 @@ async def admin_routes(request: Request) -> Response: store=store, backend_names=fusion_backends.keys(), jev_router_backend_names=jev_router_backends.keys(), + jev_decider_names=jev_deciders.keys(), ), headers=_ADMIN_UI_HEADERS, ) @@ -987,6 +995,21 @@ async def chat_completions(request: Request) -> JSONResponse | StreamingResponse telemetry=telemetry, control_plane=control_plane, ) + if alias.mode == "jev": + # Jev chooses the order; Cerberus's own loop executes it, so jev + # aliases stream, fail over and cool down like any dispatch alias. + return await jev_dispatch( + body=body, + alias_name=alias_name, + alias=alias, + identity_name=context.name if context else None, + document=document, + backends=jev_deciders, + store=store, + client=request.app.state.http_client, + telemetry=telemetry, + control_plane=control_plane, + ) return await dispatch( body=body, alias_name=alias_name, diff --git a/src/cerberus/control/routes.py b/src/cerberus/control/routes.py index f8c4f3b..82b9331 100644 --- a/src/cerberus/control/routes.py +++ b/src/cerberus/control/routes.py @@ -10,8 +10,10 @@ runtime_exclusion() health, then cooldown, then credential presence authorization_error() whether a configured identity may use the alias fusion_readiness() backend, credential and provider gates for fusion - resolve_pool() the filtered pool a jev-router request would send + resolve_pool() the filtered pool a jev-router request would send, + or a jev decision would be offered jev_router_readiness() backend presence and a non-empty pool + decider_readiness() whether a jev alias could ask for a decision now Fusion is projected differently on purpose. Its panel never enters the failover loop — Cerberus resolves one backend, one credential and one provider and sends @@ -24,6 +26,10 @@ the pool the next request would send (eligible) or filtered out before Jev sees it (excluded, with the first filter that refused it). +A jev alias is the same kind of pool, across any providers, plus a decider. The +alias stays routable while no decision can be had — it then runs in configured +order — so its readiness and its decider's are reported separately. + Read-only: no upstream connection, no cooldown applied, no lifecycle state, and no secret value or credential locator — a path names its credential, never where that credential is read from. @@ -36,6 +42,7 @@ from cerberus.fusion.dispatch import fusion_readiness from cerberus.identity.auth import IdentityContext, authorization_error +from cerberus.jev.dispatch import decider_readiness from cerberus.jev_router.backend import JEV_ROUTER_MODEL from cerberus.jev_router.dispatch import jev_router_readiness from cerberus.registry.loader import ConfigDocument @@ -203,6 +210,54 @@ def _jev_router( } +def _jev( + config: CerberusConfig, + alias_name: str, + alias: Alias, + *, + decider_names: Collection[str], + control_plane: Any | None, + store: Any, + now: float | None, +) -> dict[str, Any]: + assert alias.jev is not None + policy = alias.jev + members = resolve_pool( + config, alias_name, allow_paid=policy.allow_paid_pool, control_plane=control_plane, store=store, now=now + ) + pool = [ + _path( + ordinal, + member.target, + STATE_EXCLUDED if member.exclusion is not None else STATE_ELIGIBLE, + ( + {"reason": member.exclusion.reason, "scope": member.exclusion.scope, "retry_at": member.exclusion.retry_at} + if member.exclusion is not None + else None + ), + ) + for ordinal, member in enumerate(members) + ] + attemptable = sum(1 for member in members if member.exclusion is None) + return { + "decider": { + "backend": policy.backend, + "provider": policy.decider.provider, + "credential_ref": policy.decider.credential, + "model": policy.model, + "input": policy.input, + "max_input_chars": policy.max_input_chars, + "timeout_seconds": policy.timeout_seconds, + "readiness": decider_readiness( + config, alias, backend_names=decider_names, control_plane=control_plane + ), + }, + "allow_paid_pool": policy.allow_paid_pool, + "pool": pool, + "readiness": {"pool_available": attemptable, "pool_size": len(members), "available": attemptable > 0}, + } + + def route_projection( document: ConfigDocument, *, @@ -210,6 +265,7 @@ def route_projection( store: Any, backend_names: Collection[str] = (), jev_router_backend_names: Collection[str] = (), + jev_decider_names: Collection[str] = (), now: float | None = None, ) -> dict[str, Any]: """The active revision's routes, with every state decided server-side.""" @@ -236,6 +292,16 @@ def route_projection( store=store, now=now, ) + elif alias.mode == "jev": + projected["jev"] = _jev( + config, + alias_name, + alias, + decider_names=jev_decider_names, + control_plane=control_plane, + store=store, + now=now, + ) else: projected["allow_paid_fallback"] = alias.allow_paid_fallback projected["paths"] = _paths( diff --git a/src/cerberus/control/sqlite.py b/src/cerberus/control/sqlite.py index 16f3d03..e8d054d 100644 --- a/src/cerberus/control/sqlite.py +++ b/src/cerberus/control/sqlite.py @@ -309,7 +309,7 @@ def _materialize(self, document: ConfigDocument) -> None: ) self._connection.execute( "INSERT INTO alias_bindings VALUES (?, ?, ?)", - (revision, alias_name, alias.mode if alias.mode in ("fusion", "jev-router") else "dedicated"), + (revision, alias_name, alias.mode if alias.mode in ("fusion", "jev-router", "jev") else "dedicated"), ) for ordinal, candidate in enumerate(alias.candidates): model = config.resolve_model(candidate, context=f"alias {alias_name!r} candidate") @@ -317,7 +317,7 @@ def _materialize(self, document: ConfigDocument) -> None: "INSERT INTO route_paths VALUES (?, ?, ?, ?, ?, ?, ?)", (revision, alias_name, ordinal, candidate.provider, candidate.credential, candidate.model, model.cost_tier), ) - if alias.mode in ("fusion", "jev-router"): + if alias.mode in ("fusion", "jev-router", "jev"): self._connection.execute( "INSERT INTO route_members VALUES (?, ?, ?, ?, ?, ?, ?)", ( diff --git a/src/cerberus/jev/__init__.py b/src/cerberus/jev/__init__.py new file mode 100644 index 0000000..917e3b6 --- /dev/null +++ b/src/cerberus/jev/__init__.py @@ -0,0 +1,27 @@ +"""jev mode: Jev as Cerberus's model-selection intelligence, Cerberus as executor. + +Cerberus filters a pool that may span every provider it knows — local hosts, +Cloudflare, OpenRouter — and asks Jev, through OpenRouter's Decisions API, one +question: which surviving model is the cheapest one strong enough for this +request. Cerberus then runs that model through its own failover loop. Jev +decides; Cerberus governs and executes. +""" + +from cerberus.jev.backend import ( + Decision, + DecisionBackend, + DecisionError, + DecisionRequest, + OpenRouterJevDecider, +) +from cerberus.jev.dispatch import decider_readiness, jev_dispatch + +__all__ = [ + "Decision", + "DecisionBackend", + "DecisionError", + "DecisionRequest", + "OpenRouterJevDecider", + "decider_readiness", + "jev_dispatch", +] diff --git a/src/cerberus/jev/backend.py b/src/cerberus/jev/backend.py new file mode 100644 index 0000000..9f54c95 --- /dev/null +++ b/src/cerberus/jev/backend.py @@ -0,0 +1,180 @@ +"""Decision-service seam plus the OpenRouter Decisions API implementation. + +A jev alias asks one bounded question — which pool model should serve this +request — and gets back one of the option ids it offered, or nothing usable. +Nothing outside this module knows the wire shape of a decision service. + +Confirmed contract for OpenRouter's Decisions API: ``POST /api/alpha/decisions`` +with a bearer key; the body carries ``model``, ``state`` and ``questions``; +Jev answers ``choice``, ``noul`` and ``score`` questions; the response carries +``answers`` keyed by question id, with typed results and probabilities, and +``usage``. The field names inside one question and one answer are not yet +confirmed, so the request uses the plainest form and the answer is read +tolerantly. A misread can only ever produce "no usable answer", because the +choice must equal an option id Cerberus sent; the request then runs in +configured order, which every pool member already satisfies. +""" + +from __future__ import annotations + +import asyncio +import time +from dataclasses import dataclass +from typing import Any, Literal, Protocol + +import httpx + +DecisionFailure = Literal["timeout", "unreachable", "http_error", "invalid_response"] + +# the one question a jev alias asks, by id +ROUTE_QUESTION_ID = "route" +ROUTE_QUESTION = ( + "Which candidate model is the least costly one that is still strong enough " + "to handle this request well?" +) +# keys an answer's chosen option may sit under, most specific first +_CHOICE_KEYS = ("choice", "result", "value", "answer", "selected") +_USAGE_KEYS = ("prompt_tokens", "completion_tokens", "total_tokens", "cost") + + +@dataclass(frozen=True, slots=True) +class DecisionRequest: + endpoint: str + api_key: str + model: str + state: str # everything the decision service may read, already scoped by policy + options: list[str] # opaque option ids; never a provider name + timeout_seconds: float + + +@dataclass(frozen=True, slots=True) +class Decision: + choice: str | None # an option id exactly as answered, or None when absent + confidence: float | None + usage: dict[str, float] | None + http_status: int + latency_ms: float + + +class DecisionError(Exception): + def __init__(self, reason: DecisionFailure, *, latency_ms: float, http_status: int | None = None) -> None: + super().__init__(reason) + self.reason = reason + self.latency_ms = latency_ms + self.http_status = http_status + + +class DecisionBackend(Protocol): + name: str + + async def decide(self, request: DecisionRequest) -> Decision: ... + + async def aclose(self) -> None: ... + + +class OpenRouterJevDecider: + """Ask Jev through OpenRouter's Decisions API. Never chat completions.""" + + name = "openrouter" + + def __init__(self, transport: httpx.AsyncBaseTransport | None = None) -> None: + self._client = httpx.AsyncClient(transport=transport) + + @staticmethod + def build_body(request: DecisionRequest) -> dict[str, Any]: + return { + "model": request.model, + "state": request.state, + "questions": [ + { + "id": ROUTE_QUESTION_ID, + "type": "choice", + "question": ROUTE_QUESTION, + "options": list(request.options), + } + ], + } + + async def decide(self, request: DecisionRequest) -> Decision: + started = time.perf_counter() + + def elapsed() -> float: + return (time.perf_counter() - started) * 1000 + + try: + async with asyncio.timeout(request.timeout_seconds): + response = await self._client.post( + request.endpoint, + json=self.build_body(request), + headers={"authorization": f"Bearer {request.api_key}", "content-type": "application/json"}, + timeout=request.timeout_seconds, + ) + except TimeoutError: + raise DecisionError("timeout", latency_ms=elapsed()) from None + except httpx.TimeoutException: + raise DecisionError("timeout", latency_ms=elapsed()) from None + except httpx.HTTPError: + raise DecisionError("unreachable", latency_ms=elapsed()) from None + + if response.status_code >= 400: + raise DecisionError("http_error", latency_ms=elapsed(), http_status=response.status_code) + try: + payload = response.json() + except ValueError: + raise DecisionError("invalid_response", latency_ms=elapsed(), http_status=response.status_code) from None + if not isinstance(payload, dict): + raise DecisionError("invalid_response", latency_ms=elapsed(), http_status=response.status_code) + + choice, confidence = _route_answer(payload.get("answers")) + return Decision( + choice=choice, + confidence=confidence, + usage=_usage(payload.get("usage")), + http_status=response.status_code, + latency_ms=elapsed(), + ) + + async def aclose(self) -> None: + await self._client.aclose() + + +def _route_answer(answers: Any) -> tuple[str | None, float | None]: + """The chosen option and its probability from the route answer, if present.""" + + if isinstance(answers, list): + answer = next((a for a in answers if isinstance(a, dict) and a.get("id") == ROUTE_QUESTION_ID), None) + elif isinstance(answers, dict): + answer = answers.get(ROUTE_QUESTION_ID) + else: + return None, None + if isinstance(answer, str): + return answer, None + if not isinstance(answer, dict): + return None, None + + probabilities = answer.get("probabilities") + probabilities = ( + {k: float(v) for k, v in probabilities.items() if isinstance(v, (int, float)) and not isinstance(v, bool)} + if isinstance(probabilities, dict) + else {} + ) + choice = next((answer[key] for key in _CHOICE_KEYS if isinstance(answer.get(key), str)), None) + if choice is None and probabilities: + choice = max(probabilities, key=probabilities.__getitem__) + confidence = probabilities.get(choice) if choice is not None else None + if confidence is None: + stated = answer.get("confidence", answer.get("probability")) + if isinstance(stated, (int, float)) and not isinstance(stated, bool): + confidence = float(stated) + return choice, confidence + + +def _usage(usage: Any) -> dict[str, float] | None: + if not isinstance(usage, dict): + return None + kept = { + key: value + for key in _USAGE_KEYS + if isinstance((value := usage.get(key)), (int, float)) and not isinstance(value, bool) and value >= 0 + } + return kept or None diff --git a/src/cerberus/jev/dispatch.py b/src/cerberus/jev/dispatch.py new file mode 100644 index 0000000..4b48b56 --- /dev/null +++ b/src/cerberus/jev/dispatch.py @@ -0,0 +1,195 @@ +"""Route a jev request: Cerberus decides what may run, Jev decides what should. + + alias candidates (any provider) + → cost, health, cooldown, credential router/pool.py + → distinct models offered under opaque ids jev/state.py + → one decision: which is cheapest-sufficient jev/backend.py + → chosen model first, rest in config order DispatchPlan + → the ordinary failover loop executes router/dispatch.py + +Jev never executes anything and never widens the pool: its answer can only +reorder models policy already admitted. Any failure to get a usable answer — +no credential, timeout, transport or HTTP error, an unreadable response, a +choice that is not an offered id — runs the request in configured order and +records why. With one model or none left to choose from, no decision is asked. +""" + +from __future__ import annotations + +import os +from collections.abc import Collection +from typing import Any + +# cerberus.router before anything importing cerberus.egress: egress.client +# imports router.engine, and the router package imports egress.client in turn +from cerberus.router.dispatch import DispatchPlan, dispatch +from cerberus.router.pool import PoolMember, resolve_pool +from cerberus.jev.backend import DecisionBackend, DecisionError, DecisionRequest +from cerberus.jev.state import Option, decision_state, options_for +from cerberus.registry.loader import ConfigDocument +from cerberus.registry.schema import Alias, CerberusConfig + +COST_REFUSAL = "paid_pool_prohibited" + + +def decider_readiness( + config: CerberusConfig, + alias: Alias, + *, + backend_names: Collection[str], + control_plane: Any | None, +) -> dict[str, bool]: + """Whether a decision could be asked right now, by its own three gates. + + Not whether the alias can route: a jev alias still serves, in configured + order, while no decision can be had. ``/admin/routes`` reports both. + """ + + assert alias.jev is not None + decider = alias.jev.decider + backend_present = alias.jev.backend in backend_names + credential_present = bool(_decider_key(config, alias)) + provider_available = control_plane is None or control_plane.provider_available(decider.provider) + return { + "backend_present": backend_present, + "credential_present": credential_present, + "provider_available": provider_available, + "decides": backend_present and credential_present and provider_available, + } + + +def _decider_key(config: CerberusConfig, alias: Alias) -> str: + assert alias.jev is not None + decider = alias.jev.decider + env_name = config.providers[decider.provider].credentials[decider.credential].api_key_env + return os.environ.get(env_name, "").strip() + + +def _cost_refused(member: PoolMember) -> bool: + return member.exclusion is not None and member.exclusion.reason == COST_REFUSAL + + +async def jev_dispatch( + *, + body: dict[str, Any], + alias_name: str, + alias: Alias, + identity_name: str | None, + document: ConfigDocument, + backends: dict[str, DecisionBackend], + store: Any, + client: Any, + telemetry: Any, + control_plane: Any | None = None, +): + policy = alias.jev + assert policy is not None + config = document.config + + members = resolve_pool( + config, alias_name, allow_paid=policy.allow_paid_pool, control_plane=control_plane, store=store + ) + # The loop re-applies health, cooldown and credential itself, so the plan + # carries every cost-admitted target; only what is attemptable right now is + # offered to the decision, so it cannot pick a model the loop would skip. + admitted = [member.target for member in members if not _cost_refused(member)] + cost_exclusions = [ + {**member.target.describe(), **member.exclusion.telemetry()} + for member in members + if _cost_refused(member) and member.exclusion is not None + ] + options = options_for([member.target for member in members if member.exclusion is None]) + + record: dict[str, Any] = { + "decider": policy.backend, + "model": policy.model, + "input": policy.input, + "options": [option.label for option in options], + "decision": None, # chosen | skipped | fallback + "reason": None, + "choice": None, + "confidence": None, + "latency_ms": None, + "http_status": None, + "usage": None, + } + chosen = await _decide(body, options, config, alias, policy, backends, control_plane, record) + + if chosen is None: + order = admitted + else: + first = [t for t in admitted if (t.provider_id, t.model) == (chosen.provider, chosen.model)] + order = first + [t for t in admitted if (t.provider_id, t.model) != (chosen.provider, chosen.model)] + + return await dispatch( + body=body, + alias_name=alias_name, + identity_name=identity_name, + document=document, + store=store, + client=client, + telemetry=telemetry, + control_plane=control_plane, + plan=DispatchPlan(targets=order, exclusions=cost_exclusions, jev=record), + ) + + +async def _decide( + body: dict[str, Any], + options: list[Option], + config: CerberusConfig, + alias: Alias, + policy: Any, + backends: dict[str, DecisionBackend], + control_plane: Any | None, + record: dict[str, Any], +) -> Option | None: + """The option Jev chose, or None with ``record`` saying why there is none.""" + + def without(decision: str, reason: str) -> None: + record.update(decision=decision, reason=reason) + + if len(options) <= 1: + # nothing to choose between: no request text leaves for a decision + without("skipped", "single_option" if options else "no_option") + return None + readiness = decider_readiness(config, alias, backend_names=backends.keys(), control_plane=control_plane) + if not readiness["backend_present"]: + without("fallback", "backend_missing") + return None + if not readiness["credential_present"]: + without("fallback", "credential_missing") + return None + if not readiness["provider_available"]: + without("fallback", "decider_unavailable") + return None + + request = DecisionRequest( + endpoint=str(policy.endpoint), + api_key=_decider_key(config, alias), + model=policy.model, + state=decision_state(body, options, config, input_mode=policy.input, max_chars=policy.max_input_chars), + options=[option.id for option in options], + timeout_seconds=float(policy.timeout_seconds), + ) + try: + decision = await backends[policy.backend].decide(request) + except DecisionError as exc: + record.update(latency_ms=round(exc.latency_ms, 3), http_status=exc.http_status) + without("fallback", exc.reason) + return None + + record.update( + latency_ms=round(decision.latency_ms, 3), + http_status=decision.http_status, + usage=decision.usage, + confidence=decision.confidence, + ) + chosen = next((option for option in options if option.id == decision.choice), None) + if chosen is None: + # absent, or not an id Cerberus offered: never guessed at + without("fallback", "choice_outside_pool" if decision.choice is not None else "no_choice") + record["confidence"] = None + return None + record.update(decision="chosen", choice=chosen.label) + return chosen diff --git a/src/cerberus/jev/state.py b/src/cerberus/jev/state.py new file mode 100644 index 0000000..9b05404 --- /dev/null +++ b/src/cerberus/jev/state.py @@ -0,0 +1,126 @@ +"""What a jev decision may read: the policy-scoped decision state. + +The decision service is a third party even when the model it picks runs on a +local host, so everything it receives is built here and nowhere else, under the +alias's ``jev.input`` policy: + + last_user_message the latest user turn, truncated + full_conversation every turn, keeping the most recent text when truncated + metadata request shape only; no prompt text at all + +Every mode also states request shape (turn count, size, tools, response format, +requested output length) and describes each pool model by registry facts — +model id, strength, cost tier, context window, capabilities, description. A +model is offered under an opaque id; provider and credential names never leave. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +from cerberus.registry.schema import CerberusConfig, JevInput +from cerberus.router.engine import Target + +_TRUNCATED = " …[truncated]" + + +@dataclass(frozen=True, slots=True) +class Option: + id: str # opaque, what the decision service sees and answers with + provider: str + model: str + + @property + def label(self) -> str: + return f"{self.provider}/{self.model}" + + +def options_for(targets: list[Target]) -> list[Option]: + """One option per distinct provider/model, in configured order. + + Two credentials for one model are the same choice to a decision; which + credential serves it stays the failover loop's business. + """ + + seen: dict[tuple[str, str], Option] = {} + for target in targets: + key = (target.provider_id, target.model) + if key not in seen: + seen[key] = Option(id=f"m{len(seen) + 1}", provider=target.provider_id, model=target.model) + return list(seen.values()) + + +def decision_state( + body: dict[str, Any], + options: list[Option], + config: CerberusConfig, + *, + input_mode: JevInput, + max_chars: int, +) -> str: + messages = [m for m in body.get("messages") or [] if isinstance(m, dict)] + sections = [_request_shape(body, messages)] + if input_mode == "last_user_message": + last = next((m for m in reversed(messages) if m.get("role") == "user"), None) + text = _truncate_head(_text(last.get("content")), max_chars) if last is not None else "" + sections.append(f"Latest user message:\n<<<\n{text}\n>>>") + elif input_mode == "full_conversation": + transcript = "\n\n".join(f"{m.get('role', 'unknown')}: {_text(m.get('content'))}" for m in messages) + sections.append(f"Conversation:\n<<<\n{_truncate_tail(transcript, max_chars)}\n>>>") + sections.append("Candidate models:\n" + "\n".join(_describe(option, config) for option in options)) + return "\n\n".join(sections) + + +def _request_shape(body: dict[str, Any], messages: list[dict[str, Any]]) -> str: + size = sum(len(_text(m.get("content"))) for m in messages) + tools = body.get("tools") + response_format = body.get("response_format") + max_out = body.get("max_completion_tokens", body.get("max_tokens")) + lines = [ + "Request:", + f"- turns: {len(messages)} (user turns: {sum(1 for m in messages if m.get('role') == 'user')})", + f"- approximate prompt size: {size} characters", + f"- tools offered: {len(tools) if isinstance(tools, list) else 0}", + f"- response format: {response_format.get('type', 'unspecified') if isinstance(response_format, dict) else 'text'}", + ] + if isinstance(max_out, int) and not isinstance(max_out, bool): + lines.append(f"- requested maximum output: {max_out} tokens") + return "\n".join(lines) + + +def _describe(option: Option, config: CerberusConfig) -> str: + entry = config.providers[option.provider].models[option.model] + facts = [ + f"model {option.model}", + f"strength {entry.strength}", + f"cost {entry.cost_tier}", + f"context window {entry.context_window if entry.context_window is not None else 'unknown'}", + f"capabilities {', '.join(entry.capabilities) if entry.capabilities else 'unspecified'}", + ] + line = f"- {option.id}: " + "; ".join(facts) + return line + (f". {entry.description}" if entry.description else "") + + +def _text(content: Any) -> str: + """Message text; non-text parts are named, never inlined.""" + + if isinstance(content, str): + return content + if not isinstance(content, list): + return "" + parts: list[str] = [] + for part in content: + if isinstance(part, dict) and part.get("type") == "text" and isinstance(part.get("text"), str): + parts.append(part["text"]) + elif isinstance(part, dict): + parts.append(f"[{part.get('type', 'part')}]") + return "\n".join(parts) + + +def _truncate_head(text: str, limit: int) -> str: + return text if len(text) <= limit else text[:limit] + _TRUNCATED + + +def _truncate_tail(text: str, limit: int) -> str: + return text if len(text) <= limit else "[earlier turns truncated] …" + text[-limit:] diff --git a/src/cerberus/registry/schema.py b/src/cerberus/registry/schema.py index e6fbf64..2e184f1 100644 --- a/src/cerberus/registry/schema.py +++ b/src/cerberus/registry/schema.py @@ -12,13 +12,22 @@ from pydantic import BaseModel, ConfigDict, Field, HttpUrl, model_validator CostTier = Literal["free", "paid"] -AliasMode = Literal["direct", "dispatch", "free", "fusion", "jev-router"] +AliasMode = Literal["direct", "dispatch", "free", "fusion", "jev-router", "jev"] # Managed deliberation backends Cerberus can hand a fusion request to. FusionBackendName = Literal["openrouter"] # Managed model-selection routers Cerberus can hand a jev-router request to. JevRouterBackendName = Literal["openrouter"] # OpenRouter's jev-router plugin accepts at most this many include patterns. JEV_ROUTER_MAX_POOL = 1024 +# Decision services Cerberus can ask which pool model should serve a jev request. +JevDeciderName = Literal["openrouter"] +# What of a request a jev alias lets the decision service read. +JevInput = Literal["last_user_message", "full_conversation", "metadata"] +# How strong a model is, relative to the others an operator registers. Jev +# weighs it against cost to pick the cheapest model strong enough for a task. +ModelStrength = Literal["basic", "standard", "strong", "frontier"] +# Distinct models one decision may choose between; bounds the decision input. +JEV_MAX_OPTIONS = 64 ALIAS_PREFIX = "cerberus/" VERSION_PATTERN = r"^cerberus-\d{4}-\d{2}-\d{2}\.\d+$" @@ -138,6 +147,11 @@ class ModelEntry(BaseModel): cost_tier: CostTier capabilities: list[str] = Field(default_factory=list) context_window: int | None = Field(default=None, ge=1) + # Read only by a jev alias, which describes each pool model to the decision + # service by these and never by its provider name. Required for a model in + # a jev pool: without a strength there is nothing to weigh cost against. + strength: ModelStrength | None = None + description: str | None = Field(default=None, min_length=1, max_length=200) class CredentialEntry(BaseModel): @@ -269,6 +283,53 @@ class JevRouterPolicy(BaseModel): allow_paid_pool: bool = False +class JevDecider(BaseModel): + """Where a jev alias asks for its decision: a registered credential, by name.""" + + model_config = ConfigDict(extra="forbid") + + provider: str = Field(min_length=1) + credential: str = Field(min_length=1) + + +class JevPolicy(BaseModel): + """Policy for a jev alias: Cerberus asks Jev which model, then runs it itself. + + The pool is the alias's candidates on any provider — local, Cloudflare, + OpenRouter — filtered by cost, health, cooldown and credential before every + request. Jev, a decision model, is asked one bounded question: which of the + surviving models is the cheapest one strong enough for this request. Cerberus + then executes that model through its own failover loop, the rest of the pool + behind it in configured order. If Jev cannot answer, or answers with + anything but a pool model, the request runs in configured order instead. + Jev decides; it never executes, and it never sees a provider name. + """ + + model_config = ConfigDict(extra="forbid") + + decider: JevDecider + backend: JevDeciderName = "openrouter" + endpoint: HttpUrl = HttpUrl("https://openrouter.ai/api/alpha/decisions") + # Pinned, so an upstream model update cannot silently change routing; bump + # it deliberately after testing. "~typesafe/jev-latest" follows upstream. + model: str = Field(default="typesafe/jev-1.13", min_length=1) + # The decision service is a third party even when the winner is local, so + # what it reads is policy: the latest user message by default, the whole + # conversation only by choice, or request metadata and no prompt text. + input: JevInput = "last_user_message" + max_input_chars: int = Field(default=4000, ge=1, le=100_000) + # Deadline for the decision alone; on expiry the request runs in configured + # order. Generation keeps the provider's own timeouts. + timeout_seconds: float = Field(default=5.0, gt=0, le=60) + allow_paid_pool: bool = False + + @model_validator(mode="after") + def validate_endpoint(self) -> "JevPolicy": + if self.endpoint.scheme != "https": + raise ValueError("jev endpoint must use https: the decision input carries request text") + return self + + class Alias(BaseModel): model_config = ConfigDict(extra="forbid") @@ -278,6 +339,7 @@ class Alias(BaseModel): allow_experimental: bool = False fusion: FusionPolicy | None = None jev_router: JevRouterPolicy | None = None + jev: JevPolicy | None = None @model_validator(mode="after") def validate_mode_coherence(self) -> "Alias": @@ -293,6 +355,14 @@ def validate_mode_coherence(self) -> "Alias": # there is no ordered fallback for it to govern: Jev chooses from the # pool, and paid membership is jev_router.allow_paid_pool raise ValueError("jev-router alias must not set allow_paid_fallback; use jev_router.allow_paid_pool") + if self.mode == "jev" and self.jev is None: + raise ValueError("jev alias requires a jev policy block") + if self.mode != "jev" and self.jev is not None: + raise ValueError("jev policy block is only valid on a jev alias") + if self.mode == "jev" and self.allow_paid_fallback: + # Jev may choose a paid model first, so paid is pool membership, not + # a fallback + raise ValueError("jev alias must not set allow_paid_fallback; use jev.allow_paid_pool") if self.mode == "free" and self.allow_paid_fallback: raise ValueError("free-mode alias must not set allow_paid_fallback") return self @@ -399,6 +469,29 @@ def _validate_jev_router_pool(self, alias_name: str, alias: Alias, tiers: list[C if not policy.allow_paid_pool and any(tier == "paid" for tier in tiers): raise ValueError(f"alias {alias_name!r}: paid jev-router pool member requires allow_paid_pool") + def _validate_jev_pool(self, alias_name: str, alias: Alias, tiers: list[CostTier]) -> None: + assert alias.jev is not None + policy = alias.jev + decider = self.providers.get(policy.decider.provider) + if decider is None or policy.decider.credential not in decider.credentials: + raise ValueError( + f"alias {alias_name!r}: jev decider {policy.decider.provider}/{policy.decider.credential} " + f"is not a registered provider credential" + ) + options = {(candidate.provider, candidate.model) for candidate in alias.candidates} + if len(options) > JEV_MAX_OPTIONS: + raise ValueError( + f"alias {alias_name!r}: {len(options)} distinct pool models exceed the jev limit of {JEV_MAX_OPTIONS}" + ) + for candidate in alias.candidates: + if self.providers[candidate.provider].models[candidate.model].strength is None: + raise ValueError( + f"alias {alias_name!r}: jev pool model {candidate.provider}/{candidate.model} needs a " + f"registry strength for Jev to weigh against its cost" + ) + if not policy.allow_paid_pool and any(tier == "paid" for tier in tiers): + raise ValueError(f"alias {alias_name!r}: paid jev pool member requires allow_paid_pool") + @model_validator(mode="after") def validate_references(self) -> "CerberusConfig": if self.server.host not in LOOPBACK_HOSTS and not self.server.api_token_env: @@ -446,6 +539,8 @@ def validate_references(self) -> "CerberusConfig": if alias.jev_router is not None: self._validate_jev_router_pool(alias_name, alias, tiers) + if alias.jev is not None: + self._validate_jev_pool(alias_name, alias, tiers) for identity_name, identity in self.identities.items(): if identity.jwt_client_id is not None and self.authentik is None: diff --git a/src/cerberus/router/dispatch.py b/src/cerberus/router/dispatch.py index af6bcfe..b295e20 100644 --- a/src/cerberus/router/dispatch.py +++ b/src/cerberus/router/dispatch.py @@ -3,6 +3,8 @@ One loop serves every alias mode (the heads share one policy engine). Fusion-mode aliases divert to the fusion backend (cerberus.fusion.dispatch); jev-router aliases divert to the hosted Jev Router (cerberus.jev_router.dispatch). +jev aliases run through this loop too, in an order a decision chose +(cerberus.jev.dispatch hands it a DispatchPlan). """ from __future__ import annotations @@ -11,6 +13,7 @@ import os import time import uuid +from dataclasses import dataclass from datetime import datetime, timezone from typing import Any @@ -32,6 +35,21 @@ from cerberus.telemetry import RoutingAttempt, RoutingEvent, RoutingOutcome, TelemetryEmitter +@dataclass(frozen=True, slots=True) +class DispatchPlan: + """An attempt order decided before the loop, by a model-selection strategy. + + The loop still owns everything after the order: health, cooldown and + credential gates, failover, cooldown writes and telemetry. A plan can only + reorder what policy already admitted; it carries no target the alias does + not configure, and the cost gate it was built under is recorded with it. + """ + + targets: list[Target] # attempt order, cost-gated + exclusions: list[dict] # what cost policy removed before ordering + jev: dict[str, Any] # the decision record, for telemetry and the response + + def _attempt( target: Target, outcome: str, @@ -72,6 +90,7 @@ def _event( candidates: list[str] | None = None, exclusions: list[dict] | None = None, upstream_reported_cost: float | None = None, + jev: dict[str, Any] | None = None, ) -> RoutingEvent: return RoutingEvent( request_id=request_id, @@ -96,6 +115,7 @@ def _event( identity=identity, config_version=config_version, config_checksum=config_checksum, + jev=jev, ) @@ -191,6 +211,7 @@ async def dispatch( client: httpx.AsyncClient, telemetry: TelemetryEmitter, control_plane: Any | None = None, + plan: DispatchPlan | None = None, ) -> JSONResponse | StreamingResponse: request_id = str(uuid.uuid4()) config = document.config @@ -200,7 +221,14 @@ async def dispatch( targets = ordered_targets(config, alias_name) ordered = [f"{t.provider_id}/{t.credential_id}/{t.model}" for t in targets] - eligible, exclusions = cost_eligible(alias, targets) + if plan is None: + eligible, exclusions = cost_eligible(alias, targets) + else: + # a plan may only reorder what this alias configures, never add to it + if any(target not in targets for target in plan.targets): + raise ValueError(f"dispatch plan for {alias_name!r} names a target the alias does not configure") + eligible, exclusions = list(plan.targets), list(plan.exclusions) + jev_record = plan.jev if plan is not None else None attempts: list[RoutingAttempt] = [] attempted = 0 @@ -283,6 +311,8 @@ async def dispatch( "config_version": document.version, "config_checksum": document.checksum, } + if jev_record is not None: + metadata["jev"] = jev_record if streaming: collector = StreamingUsageCollector() @@ -323,6 +353,7 @@ async def proxy() -> Any: identity=identity_name, config_version=document.version, config_checksum=document.checksum, + jev=jev_record, candidates=ordered, exclusions=list(exclusions), ) @@ -361,6 +392,7 @@ async def proxy() -> Any: identity=identity_name, config_version=document.version, config_checksum=document.checksum, + jev=jev_record, candidates=ordered, exclusions=list(exclusions), ) @@ -387,6 +419,7 @@ async def proxy() -> Any: identity=identity_name, config_version=document.version, config_checksum=document.checksum, + jev=jev_record, candidates=ordered, exclusions=list(exclusions), ) @@ -413,6 +446,7 @@ async def proxy() -> Any: identity=identity_name, config_version=document.version, config_checksum=document.checksum, + jev=jev_record, candidates=ordered, exclusions=list(exclusions), ) diff --git a/src/cerberus/telemetry/emitter.py b/src/cerberus/telemetry/emitter.py index c3c9fa9..f838059 100644 --- a/src/cerberus/telemetry/emitter.py +++ b/src/cerberus/telemetry/emitter.py @@ -17,7 +17,7 @@ logger = logging.getLogger(__name__) -SCHEMA_VERSION = 6 +SCHEMA_VERSION = 7 AttemptOutcome = Literal[ "missing_credentials", "transport_error", @@ -91,6 +91,10 @@ class RoutingEvent: # jev-router only: the pool Cerberus sent, what the router reported choosing, # and why a response was withheld when it was. None outside jev-router mode. jev_router: dict[str, Any] | None = None + # jev only: which models were offered, what the decision chose and how sure + # it was, or why the request ran in configured order instead. Never any of + # the request text the decision read. None outside jev mode. + jev: dict[str, Any] | None = None schema_version: int = SCHEMA_VERSION def as_payload(self) -> dict[str, Any]: @@ -101,6 +105,7 @@ def as_payload(self) -> dict[str, Any]: "reasoning_effort": self.reasoning_effort, "fusion": self.fusion, "jev_router": self.jev_router, + "jev": self.jev, "identity": self.identity, "config_version": self.config_version, "config_checksum": self.config_checksum, diff --git a/src/cerberus/ui/static/app.js b/src/cerberus/ui/static/app.js index 9c4bc1c..95aac71 100644 --- a/src/cerberus/ui/static/app.js +++ b/src/cerberus/ui/static/app.js @@ -145,6 +145,7 @@ function aliasEntries() { function aliasRoutable(entry) { if (entry.fusion) return entry.fusion.readiness.available === true; if (entry.jev_router) return entry.jev_router.readiness.available === true; + if (entry.jev) return entry.jev.readiness.available === true; return (entry.paths || []).some((path) => path.state === "eligible" || path.state === "standby"); } @@ -696,11 +697,11 @@ function describeState(path) { : { short: "excluded", note: reason.replace(/_/g, " "), tone: "t-warn" }; } -// A jev-router pool is not a ladder: Jev chooses among the eligible members, so -// none is "first" and none is standby. Exclusions read exactly as they do for a +// A jev-router or jev pool is not a ladder: Jev chooses among the eligible +// members, so none is "first" and none is standby. Exclusions read exactly as they do for a // route path, because the same filters produced them. function describePoolState(path) { - if (path.state === "eligible") return { short: "in pool", note: "sent to Jev, which may choose it", tone: "t-up" }; + if (path.state === "eligible") return { short: "in pool", note: "offered to Jev, which may choose it", tone: "t-up" }; return describeState(path); } @@ -724,7 +725,7 @@ function aliasRail(aliases, current) { const on = current && entry.alias === current.alias; const count = entry.paths ? entry.paths.length + " route paths" : entry.fusion ? entry.fusion.panel.length + " panel members" - : entry.jev_router.pool.length + " pool models"; + : (entry.jev_router || entry.jev).pool.length + " pool models"; return el("button", { class: "alias-tab" + (on ? " alias-tab-on" : ""), attrs: { type: "button", id: "alias-tab-" + index, "aria-pressed": on ? "true" : "false" }, @@ -732,11 +733,13 @@ function aliasRail(aliases, current) { }, el("code", { class: "alias-name", text: entry.alias }), el("span", { class: "alias-meta" }, - el("span", { class: "pill", text: entry.mode === "fusion" ? "Fusion" : entry.mode === "jev-router" ? "Jev Router" : "dedicated" }), + el("span", { class: "pill", text: MODE_PILL[entry.mode] || "dedicated" }), el("code", { text: count }))); })); } +const MODE_PILL = { fusion: "Fusion", "jev-router": "Jev Router", jev: "Jev" }; + function pathRow(entry, path, pool) { const state = pool ? describePoolState(path) : describeState(path); const on = SELECTION.alias === entry.alias && SELECTION.ordinal === path.ordinal; @@ -798,6 +801,28 @@ function jevRouterPool(entry) { codeText(ready.available ? "yes" : "no", ready.available ? "t-up" : "t-bad")))); } +function jevPool(entry) { + const j = entry.jev; + const d = j.decider; + const ready = d.readiness; + const yes = (v) => codeText(v ? "yes" : "no", v ? "t-up" : "t-bad"); + return el("div", { class: "stack" }, + el("p", { class: "chain-intro", text: "Cerberus filters this pool by cost, health, cooldown and credential, asks Jev which remaining model is the cheapest one strong enough, then runs that model first through its own failover loop. Without a usable decision the request runs in configured order." }), + el("div", null, + el("div", { class: "path-head" }, + el("span", { text: "Member" }), el("span", { text: "Pool model" }), + el("span", { text: "Credential" }), el("span", { text: "Cost" }), el("span", { text: "State" })), + ...j.pool.map((p) => pathRow(entry, p, true))), + el("div", { class: "card" }, + el("h3", { text: "Decider" }), + el("div", { class: "card-row" }, el("span", { text: "model" }), codeText(d.model)), + el("div", { class: "card-row" }, el("span", { text: "reads" }), codeText(d.input.replace(/_/g, " "))), + el("div", { class: "card-row" }, el("span", { text: "backend configured" }), yes(ready.backend_present)), + el("div", { class: "card-row" }, el("span", { text: "credential present" }), yes(ready.credential_present)), + el("div", { class: "card-row" }, el("span", { text: "can decide now" }), yes(ready.decides)), + el("div", { class: "card-row" }, el("span", { text: "can route now" }), yes(j.readiness.available)))); +} + function inspectorRows(entry) { const projection = STATE.routes; const base = [ @@ -836,6 +861,26 @@ function inspectorRows(entry) { ["reason", member.exclusion ? memberState.note : "—", member.exclusion ? memberState.tone : "t-mono"], ]); } + if (entry.jev) { + const j = entry.jev; + const rows = base.concat([ + ["decider", j.decider.model, "t-ink"], + ["decision reads", j.decider.input.replace(/_/g, " "), "t-mono"], + ["pool", j.readiness.pool_available + " of " + j.readiness.pool_size + " offered to Jev", "t-mono"], + ["paid pool", j.allow_paid_pool ? "allowed" : "prohibited", "t-mono"], + ["deciding", j.decider.readiness.decides ? "ready" : "configured order", j.decider.readiness.decides ? "t-up" : "t-warn"], + ]); + const member = j.pool.find((p) => p.ordinal === SELECTION.ordinal); + if (!member) return rows.concat([["pool model", "select a pool model", "t-mono"]]); + const memberState = describePoolState(member); + return rows.concat([ + ["pool model", member.provider + "/" + member.model, "t-ink"], + ["credential", member.credential_ref, "t-mono"], + ["cost tier", member.cost_tier, member.cost_tier === "paid" ? "t-warn" : "t-mono"], + ["state", memberState.short, memberState.tone], + ["reason", member.exclusion ? memberState.note : "—", member.exclusion ? memberState.tone : "t-mono"], + ]); + } const path = (entry.paths || []).find((p) => p.ordinal === SELECTION.ordinal); if (!path) return base.concat([["route path", "select a route path", "t-mono"]]); const state = describeState(path); @@ -853,7 +898,7 @@ function inspectorRows(entry) { function inspector(entry) { const rows = inspectorRows(entry); return el("aside", { class: "inspector", attrs: { id: "inspector", "aria-live": "polite", tabindex: "-1" } }, - el("span", { class: "fact-label", text: entry.fusion ? "Fusion inspector" : entry.jev_router ? "Jev Router inspector" : "Route path inspector" }), + el("span", { class: "fact-label", text: entry.fusion ? "Fusion inspector" : entry.jev_router ? "Jev Router inspector" : entry.jev ? "Jev inspector" : "Route path inspector" }), el("div", { class: "inspect-rows" }, ...rows.map(([k, v, tone]) => el("div", { class: "inspect-row" }, el("span", { text: k }), codeText(v, tone)))), @@ -876,6 +921,8 @@ function viewRoutes() { ? fusionChain(entry) : entry.jev_router ? jevRouterPool(entry) + : entry.jev + ? jevPool(entry) : el("div", null, el("div", { class: "path-head" }, el("span", { text: "Order" }), el("span", { text: "Route path" }), @@ -1032,6 +1079,10 @@ function jevRouterEvidence(event) { return event.jev_router ? evidenceRows("Jev Router evidence", event.jev_router) : null; } +function jevEvidence(event) { + return event.jev ? evidenceRows("Jev decision", event.jev) : null; +} + function decisionInspector(event) { const rows = [ ["request id", text(event.request_id), "t-ink"], @@ -1060,6 +1111,7 @@ function decisionInspector(event) { exclusionList(event), fusionEvidence(event), jevRouterEvidence(event), + jevEvidence(event), ); } diff --git a/tests/test_admin_ui.py b/tests/test_admin_ui.py index c3cf00f..2bf717b 100644 --- a/tests/test_admin_ui.py +++ b/tests/test_admin_ui.py @@ -737,7 +737,7 @@ async def test_history_is_rendered_from_the_event_never_from_current_route_state app = make_app(monkeypatch, tmp_path) script = (await fetch(app, "/admin/ui/app.js")).text - for name in ("attemptLadder", "exclusionList", "fusionEvidence", "jevRouterEvidence", "decisionInspector"): + for name in ("attemptLadder", "exclusionList", "fusionEvidence", "jevRouterEvidence", "jevEvidence", "decisionInspector"): source = _function_source(script, name) for current in ("STATE.routes", "projectedAliases", "aliasRoutable", "STATE.status"): assert current not in source, f"{name} must not explain history with current state: {current}" diff --git a/tests/test_example_config.py b/tests/test_example_config.py index 65e9939..efd87fd 100644 --- a/tests/test_example_config.py +++ b/tests/test_example_config.py @@ -15,4 +15,5 @@ def test_example_config_validates(): "cerberus/dispatch-standard", "cerberus/fusion-code", "cerberus/auto", + "cerberus/smart", } diff --git a/tests/test_jev_dispatch.py b/tests/test_jev_dispatch.py new file mode 100644 index 0000000..fcb054d --- /dev/null +++ b/tests/test_jev_dispatch.py @@ -0,0 +1,489 @@ +"""jev mode: Cerberus asks Jev which model, then runs it through its own loop. + +Both the Decisions API (jev_transport) and every generation upstream +(http_transport) are mocked at the HTTP boundary; no live traffic. These prove: +the decision request's wire shape and what it may read; that Jev's choice only +reorders models policy already admitted, across providers; that any failure to +get a usable answer runs the request in configured order; and that no request +text reaches telemetry. +""" + +import json + +import httpx +import pytest + +from cerberus.app import create_app +from cerberus.jev import DecisionRequest, OpenRouterJevDecider +from cerberus.jev.backend import _route_answer +from cerberus.registry import load_config_document +from tests.test_control import write_config + +LOCAL = ("edgebox", "gemma-4-12b") +MID = ("openrouter", "vendor/mid") +FRONTIER = ("openrouter", "anthropic/claude-sonnet") +SECRET_TURN = "first-turn-private-detail" +LATEST = "latest-question-text" + + +def jev_raw(*, jev=None, candidates=(LOCAL, MID, FRONTIER), state_path=None) -> dict: + raw = { + "metadata": {"version": "cerberus-2026-10-05.2"}, + "providers": { + "edgebox": { + "base_url": "http://edgebox.test/v1", + "credentials": {"none": {"api_key_env": "EDGE_KEY"}}, + "models": { + "gemma-4-12b": {"cost_tier": "free", "strength": "standard", "context_window": 32768, + "capabilities": ["chat", "coding"], "description": "Local 12B, fast."}, + }, + }, + "openrouter": { + "base_url": "https://openrouter.ai/api/v1", + "credentials": { + "main": {"api_key_env": "OPENROUTER_API_KEY"}, + "decisions": {"api_key_env": "JEV_DECISIONS_KEY"}, + }, + "models": { + "vendor/mid": {"cost_tier": "free", "strength": "strong"}, + "anthropic/claude-sonnet": {"cost_tier": "paid", "strength": "frontier"}, + }, + }, + }, + "identities": { + "dev": { + "credential_env": "CB_KEY_DEV", + "allowed_modes": ["jev"], + "allowed_aliases": ["cerberus/smart"], + }, + }, + "aliases": { + "cerberus/smart": { + "mode": "jev", + "candidates": [ + {"provider": p, "credential": "none" if p == "edgebox" else "main", "model": m} + for p, m in candidates + ], + "jev": { + "decider": {"provider": "openrouter", "credential": "decisions"}, + "allow_paid_pool": True, + **(jev or {}), + }, + }, + }, + } + if state_path is not None: + raw["state"] = {"path": str(state_path)} + return raw + + +class Capture: + def __init__(self, respond): + self.requests: list[httpx.Request] = [] + self._respond = respond + + def __call__(self, request: httpx.Request) -> httpx.Response: + self.requests.append(request) + return self._respond(request) + + @property + def body(self) -> dict: + return json.loads(self.requests[-1].content) + + def models(self) -> list[str]: + return [json.loads(r.content)["model"] for r in self.requests] + + +def answer(payload, status=200): + def respond(_request): + return httpx.Response(status, json=payload) + + return respond + + +def chooses(option, probability=0.91): + return answer({ + "answers": {"route": {"type": "choice", "choice": option, "probabilities": {option: probability}}}, + "usage": {"prompt_tokens": 120, "completion_tokens": 1, "total_tokens": 121, "cost": 0.00004}, + }) + + +def generation_ok(request: httpx.Request) -> httpx.Response: + model = json.loads(request.content)["model"] + return httpx.Response(200, json={ + "model": model, + "choices": [{"message": {"role": "assistant", "content": f"answer from {model}"}}], + "usage": {"prompt_tokens": 9, "completion_tokens": 3, "total_tokens": 12}, + }) + + +def make_app(monkeypatch, tmp_path, *, raw=None, decide=None, generate=generation_ok, decider_key="jev-key"): + monkeypatch.setenv("EDGE_KEY", "local-no-auth") + monkeypatch.setenv("OPENROUTER_API_KEY", "or-key") + monkeypatch.setenv("CB_KEY_DEV", "cb-dev") + if decider_key is None: + monkeypatch.delenv("JEV_DECISIONS_KEY", raising=False) + else: + monkeypatch.setenv("JEV_DECISIONS_KEY", decider_key) + doc = load_config_document(write_config(tmp_path, "v.yaml", raw or jev_raw()), validate_credentials=False) + decisions = Capture(decide or chooses("m2")) + upstream = Capture(generate) + app = create_app(doc, http_transport=httpx.MockTransport(upstream), jev_transport=httpx.MockTransport(decisions)) + return app, decisions, upstream + + +AUTH = {"authorization": "Bearer cb-dev"} +CONVERSATION = [ + {"role": "user", "content": SECRET_TURN}, + {"role": "assistant", "content": "an earlier answer"}, + {"role": "user", "content": LATEST}, +] +REQ = {"model": "cerberus/smart", "messages": CONVERSATION} + + +def client_for(app): + return httpx.AsyncClient(transport=httpx.ASGITransport(app=app, client=("127.0.0.1", 40001)), base_url="http://t") + + +async def call(app, body=REQ): + async with app.router.lifespan_context(app): + async with client_for(app) as client: + resp = await client.post("/v1/chat/completions", json=body, headers=AUTH) + if body.get("stream"): + await resp.aread() + events = (await client.get("/admin/events")).json()["events"] + return resp, events + + +# ---- the decision request ---- + + +@pytest.mark.asyncio +async def test_the_decision_is_one_decisions_api_call_with_the_pinned_model(monkeypatch, tmp_path): + app, decisions, _ = make_app(monkeypatch, tmp_path) + await call(app) + + assert len(decisions.requests) == 1 + req = decisions.requests[0] + assert str(req.url) == "https://openrouter.ai/api/alpha/decisions" + assert req.headers["authorization"] == "Bearer jev-key", "the decider's own credential" + body = decisions.body + assert body["model"] == "typesafe/jev-1.13" + assert body["questions"] == [{ + "id": "route", + "type": "choice", + "question": "Which candidate model is the least costly one that is still strong enough " + "to handle this request well?", + "options": ["m1", "m2", "m3"], + }] + assert set(body) == {"model", "state", "questions"} + + +@pytest.mark.asyncio +async def test_the_decision_reads_only_the_latest_user_message_by_default(monkeypatch, tmp_path): + app, decisions, _ = make_app(monkeypatch, tmp_path) + await call(app) + state = decisions.body["state"] + + assert LATEST in state + assert SECRET_TURN not in state and "an earlier answer" not in state + # models are described by registry facts, under opaque ids + assert "m1: model gemma-4-12b; strength standard; cost free; context window 32768; capabilities chat, coding. Local 12B, fast." in state + assert "m3: model anthropic/claude-sonnet; strength frontier; cost paid" in state + # provider, credential and secret locations never leave + for private in ("edgebox", "openrouter/", "main", "decisions", "EDGE_KEY", "OPENROUTER_API_KEY", "or-key"): + assert private not in state, private + + +@pytest.mark.asyncio +async def test_full_conversation_and_metadata_inputs_are_opt_in(monkeypatch, tmp_path): + app, decisions, _ = make_app(monkeypatch, tmp_path, raw=jev_raw(jev={"input": "full_conversation"})) + await call(app) + assert SECRET_TURN in decisions.body["state"] and LATEST in decisions.body["state"] + + app, decisions, _ = make_app(monkeypatch, tmp_path, raw=jev_raw(jev={"input": "metadata"})) + await call(app) + state = decisions.body["state"] + assert SECRET_TURN not in state and LATEST not in state and "an earlier answer" not in state + assert "turns: 3 (user turns: 2)" in state + + +@pytest.mark.asyncio +async def test_the_decision_input_is_truncated_to_policy(monkeypatch, tmp_path): + app, decisions, _ = make_app(monkeypatch, tmp_path, raw=jev_raw(jev={"max_input_chars": 10})) + await call(app, {**REQ, "messages": [{"role": "user", "content": "0123456789-overflow-not-sent"}]}) + state = decisions.body["state"] + assert "0123456789 …[truncated]" in state and "overflow-not-sent" not in state + + +# ---- the choice reorders; Cerberus executes ---- + + +@pytest.mark.asyncio +async def test_the_chosen_model_runs_first_through_cerberus(monkeypatch, tmp_path): + app, _, upstream = make_app(monkeypatch, tmp_path, decide=chooses("m3")) + resp, events = await call(app) + + assert resp.status_code == 200 + assert upstream.models() == ["anthropic/claude-sonnet"] + assert str(upstream.requests[0].url) == "https://openrouter.ai/api/v1/chat/completions" + assert upstream.requests[0].headers["authorization"] == "Bearer or-key", "the pool member's credential" + assert resp.json()["cerberus"]["jev"]["choice"] == "openrouter/anthropic/claude-sonnet" + + event = events[0] + assert (event["mode"], event["outcome"], event["model"], event["cost_tier"]) == ( + "jev", "success", "anthropic/claude-sonnet", "paid" + ) + assert event["used_fallback"] is False + assert event["jev"] == { + "decider": "openrouter", + "model": "typesafe/jev-1.13", + "input": "last_user_message", + "options": ["edgebox/gemma-4-12b", "openrouter/vendor/mid", "openrouter/anthropic/claude-sonnet"], + "decision": "chosen", + "reason": None, + "choice": "openrouter/anthropic/claude-sonnet", + "confidence": 0.91, + "latency_ms": event["jev"]["latency_ms"], + "http_status": 200, + "usage": {"prompt_tokens": 120, "completion_tokens": 1, "total_tokens": 121, "cost": 0.00004}, + } + + +@pytest.mark.asyncio +async def test_a_local_model_can_be_the_choice(monkeypatch, tmp_path): + app, _, upstream = make_app(monkeypatch, tmp_path, decide=chooses("m1")) + resp, _ = await call(app) + assert upstream.models() == ["gemma-4-12b"] + assert str(upstream.requests[0].url) == "http://edgebox.test/v1/chat/completions" + assert resp.json()["choices"][0]["message"]["content"] == "answer from gemma-4-12b" + + +@pytest.mark.asyncio +async def test_a_failing_choice_fails_over_to_the_pool_in_config_order(monkeypatch, tmp_path): + def frontier_rate_limited(request): + if json.loads(request.content)["model"] == "anthropic/claude-sonnet": + return httpx.Response(429, json={"error": "rate"}) + return generation_ok(request) + + app, _, upstream = make_app(monkeypatch, tmp_path, decide=chooses("m3"), generate=frontier_rate_limited) + resp, events = await call(app) + + assert resp.status_code == 200 + assert upstream.models() == ["anthropic/claude-sonnet", "gemma-4-12b"] + assert events[0]["used_fallback"] is True + assert events[0]["jev"]["decision"] == "chosen" + + +@pytest.mark.asyncio +async def test_jev_aliases_stream_like_any_dispatch_alias(monkeypatch, tmp_path): + def sse(request): + return httpx.Response(200, content=b'data: {"choices":[{"delta":{"content":"hi"}}]}\n\ndata: [DONE]\n\n', + headers={"content-type": "text/event-stream"}) + + app, decisions, upstream = make_app(monkeypatch, tmp_path, decide=chooses("m2"), generate=sse) + resp, events = await call(app, {**REQ, "stream": True}) + assert resp.status_code == 200 and resp.headers["x-cerberus-model"] == "vendor/mid" + assert len(decisions.requests) == 1 + assert events[0]["streaming"] is True and events[0]["jev"]["choice"] == "openrouter/vendor/mid" + + +# ---- no usable answer: configured order ---- + + +def _raise(exc): + def respond(request): + raise exc("down", request=request) + + return respond + + +@pytest.mark.parametrize( + ("decide", "reason", "status"), + [ + (answer({"error": "boom"}, status=500), "http_error", 500), + (answer({"error": "bad request"}, status=400), "http_error", 400), + (_raise(httpx.ReadTimeout), "timeout", None), + (_raise(httpx.ConnectError), "unreachable", None), + (lambda r: httpx.Response(200, content=b"", headers={"content-type": "text/html"}), + "invalid_response", 200), + (chooses("m9"), "choice_outside_pool", 200), + # a provider/model name is not an option id, however plausible + (chooses("anthropic/claude-sonnet"), "choice_outside_pool", 200), + (answer({"answers": {}, "usage": {}}), "no_choice", 200), + ], + ids=["http-500", "http-400", "timeout", "unreachable", "invalid-json", "unknown-id", "model-name", "no-answer"], +) +@pytest.mark.asyncio +async def test_without_a_usable_answer_the_request_runs_in_config_order( + monkeypatch, tmp_path, decide, reason, status +): + app, _, upstream = make_app(monkeypatch, tmp_path, decide=decide) + resp, events = await call(app) + + assert resp.status_code == 200, "a decision failure never fails the request" + assert upstream.models() == ["gemma-4-12b"], "configured order: the first pool model" + record = events[0]["jev"] + assert (record["decision"], record["reason"], record["http_status"]) == ("fallback", reason, status) + assert record["choice"] is None and record["confidence"] is None + + +@pytest.mark.asyncio +async def test_a_missing_decider_credential_never_calls_the_decision_service(monkeypatch, tmp_path): + app, decisions, upstream = make_app(monkeypatch, tmp_path, decider_key=None) + resp, events = await call(app) + assert resp.status_code == 200 + assert decisions.requests == [] + assert upstream.models() == ["gemma-4-12b"] + assert (events[0]["jev"]["decision"], events[0]["jev"]["reason"]) == ("fallback", "credential_missing") + + +@pytest.mark.asyncio +async def test_with_one_model_left_no_decision_is_asked(monkeypatch, tmp_path): + """Nothing to choose between, so no request text leaves for a decision.""" + app, decisions, upstream = make_app(monkeypatch, tmp_path) + for model in ("vendor/mid", "anthropic/claude-sonnet"): + app.state.cooldowns.apply(scope="model", provider="openrouter", credential="main", model=model, + reason="quota_429", duration_seconds=600) + resp, events = await call(app) + + assert resp.status_code == 200 + assert decisions.requests == [] + assert upstream.models() == ["gemma-4-12b"] + record = events[0]["jev"] + assert (record["decision"], record["reason"], record["options"]) == ("skipped", "single_option", ["edgebox/gemma-4-12b"]) + + +@pytest.mark.asyncio +async def test_a_cooled_down_model_is_never_offered(monkeypatch, tmp_path): + app, decisions, _ = make_app(monkeypatch, tmp_path, decide=chooses("m1")) + app.state.cooldowns.apply(scope="model", provider="openrouter", credential="main", model="vendor/mid", + reason="quota_429", duration_seconds=600) + await call(app) + assert decisions.body["questions"][0]["options"] == ["m1", "m2"] + assert "vendor/mid" not in decisions.body["state"] + assert "m2: model anthropic/claude-sonnet" in decisions.body["state"] + + +@pytest.mark.asyncio +async def test_an_empty_pool_is_exhausted_without_a_decision(monkeypatch, tmp_path): + app, decisions, upstream = make_app(monkeypatch, tmp_path, raw=jev_raw(candidates=(MID, FRONTIER))) + app.state.cooldowns.apply(scope="credential", provider="openrouter", credential="main", model=None, + reason="quota_429", duration_seconds=600) + resp, events = await call(app) + assert resp.status_code == 503 + assert decisions.requests == [] and upstream.requests == [] + assert events[0]["outcome"] == "routing_exhausted" + assert (events[0]["jev"]["decision"], events[0]["jev"]["reason"]) == ("skipped", "no_option") + + +# ---- the trust boundary ---- + + +@pytest.mark.asyncio +async def test_no_request_text_reaches_telemetry(monkeypatch, tmp_path): + app, _, _ = make_app(monkeypatch, tmp_path, raw=jev_raw(jev={"input": "full_conversation"})) + _, events = await call(app) + serialized = json.dumps(events) + assert LATEST not in serialized and SECRET_TURN not in serialized + + +@pytest.mark.asyncio +async def test_server_tools_are_refused_before_any_decision(monkeypatch, tmp_path): + app, decisions, upstream = make_app(monkeypatch, tmp_path) + resp, _ = await call(app, {**REQ, "tools": [{"type": "openrouter:fusion", "parameters": {}}]}) + assert resp.status_code == 400 + assert decisions.requests == [] and upstream.requests == [] + + +# ---- admin projection and control-plane records ---- + + +@pytest.mark.asyncio +async def test_routes_project_the_pool_and_the_decider_separately(monkeypatch, tmp_path): + app, _, _ = make_app(monkeypatch, tmp_path, decider_key=None) + app.state.cooldowns.apply(scope="model", provider="openrouter", credential="main", model="vendor/mid", + reason="quota_429", duration_seconds=600) + async with app.router.lifespan_context(app): + async with client_for(app) as client: + projection = (await client.get("/admin/routes")).json() + + entry = next(a for a in projection["aliases"] if a["alias"] == "cerberus/smart") + jev = entry["jev"] + assert [(m["provider"], m["model"], m["state"]) for m in jev["pool"]] == [ + ("edgebox", "gemma-4-12b", "eligible"), + ("openrouter", "vendor/mid", "excluded"), + ("openrouter", "anthropic/claude-sonnet", "eligible"), + ] + assert jev["readiness"] == {"pool_available": 2, "pool_size": 3, "available": True} + # no decision can be had, and the alias still routes in configured order + assert jev["decider"]["readiness"] == { + "backend_present": True, "credential_present": False, "provider_available": True, "decides": False, + } + assert (jev["decider"]["model"], jev["decider"]["input"]) == ("typesafe/jev-1.13", "last_user_message") + assert "JEV_DECISIONS_KEY" not in json.dumps(projection) + + +@pytest.mark.asyncio +async def test_revision_materializes_the_jev_pool(monkeypatch, tmp_path): + import sqlite3 + + state = tmp_path / "state.sqlite3" + app, _, _ = make_app(monkeypatch, tmp_path, raw=jev_raw(state_path=state)) + async with app.router.lifespan_context(app): + pass + connection = sqlite3.connect(state) + try: + binding = connection.execute("SELECT binding_type FROM alias_bindings WHERE alias='cerberus/smart'").fetchone() + roles = connection.execute("SELECT DISTINCT role FROM route_members WHERE alias='cerberus/smart'").fetchall() + finally: + connection.close() + assert binding == ("jev",) and roles == [("pool",)] + + +# ---- backend unit: wire shape and tolerant answer reading ---- + + +def test_decider_builds_the_documented_body(): + request = DecisionRequest(endpoint="https://openrouter.ai/api/alpha/decisions", api_key="k", + model="typesafe/jev-1.13", state="s", options=["m1", "m2"], timeout_seconds=5) + body = OpenRouterJevDecider.build_body(request) + assert body["model"] == "typesafe/jev-1.13" and body["state"] == "s" + assert body["questions"][0]["options"] == ["m1", "m2"] and body["questions"][0]["type"] == "choice" + + +@pytest.mark.parametrize( + ("answers", "expected"), + [ + ({"route": {"choice": "m2", "probabilities": {"m1": 0.2, "m2": 0.8}}}, ("m2", 0.8)), + ({"route": {"result": "m1"}}, ("m1", None)), + ({"route": {"value": "m1", "confidence": 0.7}}, ("m1", 0.7)), + ({"route": "m2"}, ("m2", None)), + ([{"id": "route", "type": "choice", "result": "m2"}], ("m2", None)), + ({"route": {"probabilities": {"m1": 0.1, "m2": 0.9}}}, ("m2", 0.9)), + ({"other": {"choice": "m1"}}, (None, None)), + ("m1", (None, None)), + (None, (None, None)), + ], +) +def test_the_route_answer_is_read_tolerantly(answers, expected): + assert _route_answer(answers) == expected + + +@pytest.mark.asyncio +async def test_a_plan_cannot_add_a_target_the_alias_does_not_configure(monkeypatch, tmp_path): + from dataclasses import replace + + from cerberus.router.dispatch import DispatchPlan, dispatch + from cerberus.router.engine import ordered_targets + + app, _, upstream = make_app(monkeypatch, tmp_path) + document = app.state.lifecycle.active + stranger = replace(ordered_targets(document.config, "cerberus/smart")[0], model="elsewhere/unlisted") + with pytest.raises(ValueError, match="does not configure"): + await dispatch( + body=REQ, alias_name="cerberus/smart", document=document, store=app.state.cooldowns, + client=httpx.AsyncClient(transport=httpx.MockTransport(upstream)), telemetry=None, + plan=DispatchPlan(targets=[stranger], exclusions=[], jev={}), + ) + assert upstream.requests == [] diff --git a/tests/test_schema.py b/tests/test_schema.py index f1c4d4f..c816ec1 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -389,3 +389,84 @@ def test_jev_router_identity_must_allow_the_mode(): CerberusConfig.model_validate(raw) raw["identities"]["recon"]["allowed_modes"] = ["free", "jev-router"] assert CerberusConfig.model_validate(raw) + + +# -- jev: Cerberus asks, Jev decides, Cerberus executes ------------------------- + + +def _jev_native_raw(**alias_overrides) -> dict: + raw = make() + raw["providers"]["openrouter"]["models"]["openrouter/free"]["strength"] = "standard" + raw["providers"]["openrouter"]["models"]["anthropic/claude-sonnet"]["strength"] = "frontier" + raw["providers"]["google"]["models"]["gemini-flash"]["strength"] = "standard" + alias = { + "mode": "jev", + "candidates": [ + {"provider": "google", "credential": "free-1", "model": "gemini-flash"}, + {"provider": "openrouter", "credential": "primary", "model": "openrouter/free"}, + ], + "jev": {"decider": {"provider": "openrouter", "credential": "primary"}}, + } + alias.update(alias_overrides) + raw["aliases"]["cerberus/smart"] = alias + return raw + + +def test_jev_alias_valid_with_pinned_defaults(): + """Unlike jev-router, the pool may span providers and credentials.""" + config = CerberusConfig.model_validate(_jev_native_raw()) + policy = config.aliases["cerberus/smart"].jev + assert policy is not None + assert policy.model == "typesafe/jev-1.13" + assert str(policy.endpoint) == "https://openrouter.ai/api/alpha/decisions" + assert (policy.input, policy.max_input_chars, policy.allow_paid_pool) == ("last_user_message", 4000, False) + + +def test_jev_mode_and_policy_block_must_agree(): + with pytest.raises(ValidationError, match="jev policy block"): + CerberusConfig.model_validate(_jev_native_raw(jev=None)) + raw = _jev_native_raw() + raw["aliases"]["cerberus/free"]["jev"] = {"decider": {"provider": "openrouter", "credential": "primary"}} + with pytest.raises(ValidationError, match="only valid on a jev alias"): + CerberusConfig.model_validate(raw) + + +def test_jev_decider_must_be_a_registered_credential(): + raw = _jev_native_raw(jev={"decider": {"provider": "openrouter", "credential": "ghost"}}) + with pytest.raises(ValidationError, match="registered provider credential"): + CerberusConfig.model_validate(raw) + + +def test_jev_pool_models_need_a_strength(): + raw = _jev_native_raw() + del raw["providers"]["google"]["models"]["gemini-flash"]["strength"] + with pytest.raises(ValidationError, match="needs a registry strength"): + CerberusConfig.model_validate(raw) + + +def test_jev_paid_pool_member_requires_opt_in_and_refuses_paid_fallback(): + paid = {"provider": "openrouter", "credential": "primary", "model": "anthropic/claude-sonnet"} + raw = _jev_native_raw() + raw["aliases"]["cerberus/smart"]["candidates"].append(paid) + with pytest.raises(ValidationError, match="allow_paid_pool"): + CerberusConfig.model_validate(raw) + raw["aliases"]["cerberus/smart"]["jev"]["allow_paid_pool"] = True + assert CerberusConfig.model_validate(raw) + raw["aliases"]["cerberus/smart"]["allow_paid_fallback"] = True + with pytest.raises(ValidationError, match="allow_paid_pool"): + CerberusConfig.model_validate(raw) + + +def test_jev_endpoint_must_be_https(): + raw = _jev_native_raw(jev={ + "decider": {"provider": "openrouter", "credential": "primary"}, + "endpoint": "http://decisions.example/api/alpha/decisions", + }) + with pytest.raises(ValidationError, match="https"): + CerberusConfig.model_validate(raw) + + +def test_jev_input_policy_is_a_closed_vocabulary(): + raw = _jev_native_raw(jev={"decider": {"provider": "openrouter", "credential": "primary"}, "input": "everything"}) + with pytest.raises(ValidationError): + CerberusConfig.model_validate(raw) diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py index 2488d19..fba9e6d 100644 --- a/tests/test_telemetry.py +++ b/tests/test_telemetry.py @@ -93,7 +93,7 @@ async def telemetry(request: httpx.Request) -> httpx.Response: assert len(events) == 1 assert events[0].headers["authorization"] == "Bearer telemetry-test-value" event = json.loads(events[0].content) - assert event["schema_version"] == 6 + assert event["schema_version"] == 7 assert event["alias"] == "cerberus/controlled" assert event["attempts"][0]["used_fallback"] is False assert event["attempts"][0]["cooldown_scope"] is None