From 19a44ff3efdc2e5920caebd850b2c56a908a9de7 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 5 Oct 2026 18:09:51 +0000 Subject: [PATCH] Ask Jev for reasoning effort as a second decision question A jev alias with `jev.reasoning_efforts` set asks a second choice question in the same Decisions API call: how much reasoning effort the request needs, answered from that list only. - The answer replaces the configured reasoning_effort of every planned candidate that sets one, and is never added to one that sets none: a route without an effort may not accept the parameter at all. The list keeps Jev to values every such route accepts. A caller's own stated effort still wins unless the candidate sets reasoning_effort_override. - Each answer is vetted on its own: an effort outside the list leaves the configured effort standing whatever the model answer was. With one model left the call is still made, for the effort alone; with neither question to ask, no call and no request text leaves. - DispatchPlan may change the effort of a target that sets one and nothing else; dispatch refuses any other difference from a configured target. - Validation refuses an efforts list with fewer than two values, a repeat, or no pool candidate whose effort it could replace. - The decision adapter now carries a list of questions and returns one answer per question. The jev telemetry record gains effort, effort_confidence and effort_reason; /admin/routes and the console show which efforts Jev may choose. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01P4Nm4MVSMmG6JyZi8WumtK --- README.md | 3 + config/cerberus.example.yaml | 4 +- docs/architecture.md | 16 ++- src/cerberus/control/routes.py | 1 + src/cerberus/jev/__init__.py | 12 +- src/cerberus/jev/backend.py | 56 ++++++---- src/cerberus/jev/dispatch.py | 128 +++++++++++++++------- src/cerberus/registry/schema.py | 28 ++++- src/cerberus/router/dispatch.py | 25 ++++- src/cerberus/ui/static/app.js | 2 + tests/test_jev_dispatch.py | 188 +++++++++++++++++++++++++++++++- tests/test_schema.py | 14 +++ 12 files changed, 398 insertions(+), 79 deletions(-) diff --git a/README.md b/README.md index da4d843..a76b646 100644 --- a/README.md +++ b/README.md @@ -111,6 +111,9 @@ that model first through its own failover loop, the rest in configured order. silently change routing. - If no usable decision comes back, the request runs in configured order; a decision outage never takes the alias down. +- Optionally (`jev.reasoning_efforts`), the same call also asks how much + reasoning effort the request needs; the answer replaces the effort of pool + candidates that already set one, and a caller's own effort still wins. - Streams, fails over and cools down like any dispatch alias. ## Dispatch diff --git a/config/cerberus.example.yaml b/config/cerberus.example.yaml index 2253b44..054de31 100644 --- a/config/cerberus.example.yaml +++ b/config/cerberus.example.yaml @@ -87,7 +87,8 @@ aliases: # first through its own failover loop. No usable answer -> configured order. candidates: # configured order is the fallback order - { provider: google, credential: free-1, model: gemini-flash } - - { provider: openrouter, credential: primary, model: anthropic/claude-sonnet } + # a candidate that sets reasoning_effort is one Jev's effort answer may replace + - { provider: openrouter, credential: primary, model: anthropic/claude-sonnet, reasoning_effort: medium } jev: decider: { provider: openrouter, credential: primary } model: typesafe/jev-1.13 # pinned; bump deliberately after testing @@ -95,6 +96,7 @@ aliases: max_input_chars: 4000 timeout_seconds: 5 # decision deadline; then configured order allow_paid_pool: true + reasoning_efforts: [low, medium, high] # also ask Jev for an effort, among these only identities: example-reader: diff --git a/docs/architecture.md b/docs/architecture.md index ccf36d6..6d041d7 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -107,9 +107,23 @@ error, an unreadable response, an absent or unknown choice — runs the request configured order instead, which every pool member already satisfies; the alias stays routable while no decision can be had. +With `jev.reasoning_efforts` set, the same call asks a second question: how much +reasoning effort the request needs, answered from that list only. The answer +replaces the configured `reasoning_effort` of every planned candidate that sets +one, and is never added to a candidate that sets none, because a route without +one may not accept the parameter at all; the list is how an operator keeps Jev +to values every such route accepts. The egress rule is unchanged: a caller's own +stated effort wins unless the candidate sets `reasoning_effort_override`. The +effort is vetted on its own — an answer outside the list leaves the configured +effort standing, whatever the model answer was — and with one model left the +call is still made for the effort alone. A plan may change the effort of a +target that sets one and nothing else about it; the dispatch loop refuses any +other difference from a configured target. + Each routing event carries a `jev` record: options offered, whether the decision was `chosen`, `skipped` or a `fallback`, the reason, the choice and its -probability, decision latency, status and usage. It never carries request text. +probability, the effort applied with its probability or why none was, decision +latency, status and usage. It never carries request text. The confirmed Decisions API contract covers the endpoint, the model, the `model`/`state`/`questions` request and `answers` keyed by question id; the fields inside one question and one answer are not yet confirmed, so the answer diff --git a/src/cerberus/control/routes.py b/src/cerberus/control/routes.py index 82b9331..ac8c1e8 100644 --- a/src/cerberus/control/routes.py +++ b/src/cerberus/control/routes.py @@ -246,6 +246,7 @@ def _jev( "credential_ref": policy.decider.credential, "model": policy.model, "input": policy.input, + "reasoning_efforts": policy.reasoning_efforts, "max_input_chars": policy.max_input_chars, "timeout_seconds": policy.timeout_seconds, "readiness": decider_readiness( diff --git a/src/cerberus/jev/__init__.py b/src/cerberus/jev/__init__.py index 917e3b6..233f32d 100644 --- a/src/cerberus/jev/__init__.py +++ b/src/cerberus/jev/__init__.py @@ -1,27 +1,31 @@ """jev mode: Jev as Cerberus's model-selection intelligence, Cerberus as executor. Cerberus filters a pool that may span every provider it knows — local hosts, -Cloudflare, OpenRouter — and asks Jev, through OpenRouter's Decisions API, one -question: which surviving model is the cheapest one strong enough for this -request. Cerberus then runs that model through its own failover loop. Jev -decides; Cerberus governs and executes. +Cloudflare, OpenRouter — and asks Jev, through OpenRouter's Decisions API, +which surviving model is the cheapest one strong enough for this request and, +by policy, how much reasoning effort it needs. Cerberus then runs that model +through its own failover loop. Jev decides; Cerberus governs and executes. """ from cerberus.jev.backend import ( + Answer, Decision, DecisionBackend, DecisionError, DecisionRequest, OpenRouterJevDecider, + Question, ) from cerberus.jev.dispatch import decider_readiness, jev_dispatch __all__ = [ + "Answer", "Decision", "DecisionBackend", "DecisionError", "DecisionRequest", "OpenRouterJevDecider", + "Question", "decider_readiness", "jev_dispatch", ] diff --git a/src/cerberus/jev/backend.py b/src/cerberus/jev/backend.py index 9f54c95..d3977e7 100644 --- a/src/cerberus/jev/backend.py +++ b/src/cerberus/jev/backend.py @@ -1,8 +1,9 @@ """Decision-service seam plus the OpenRouter Decisions API implementation. -A jev alias asks one bounded question — which pool model should serve this -request — and gets back one of the option ids it offered, or nothing usable. -Nothing outside this module knows the wire shape of a decision service. +A jev alias asks bounded questions in one call — which pool model should serve +this request and, when its policy says so, how much reasoning effort the request +needs — and gets back, per question, one of the options it offered or nothing +usable. Nothing outside this module knows the wire shape of a decision service. Confirmed contract for OpenRouter's Decisions API: ``POST /api/alpha/decisions`` with a bearer key; the body carries ``model``, ``state`` and ``questions``; @@ -10,9 +11,9 @@ ``answers`` keyed by question id, with typed results and probabilities, and ``usage``. The field names inside one question and one answer are not yet confirmed, so the request uses the plainest form and the answer is read -tolerantly. A misread can only ever produce "no usable answer", because the -choice must equal an option id Cerberus sent; the request then runs in -configured order, which every pool member already satisfies. +tolerantly. A misread can only ever produce "no usable answer", because each +choice must equal an option Cerberus sent; the request then runs in configured +order and configured effort, which policy already allows. """ from __future__ import annotations @@ -26,31 +27,45 @@ DecisionFailure = Literal["timeout", "unreachable", "http_error", "invalid_response"] -# the one question a jev alias asks, by id +# the questions a jev alias asks, by id ROUTE_QUESTION_ID = "route" ROUTE_QUESTION = ( "Which candidate model is the least costly one that is still strong enough " "to handle this request well?" ) +EFFORT_QUESTION_ID = "effort" +EFFORT_QUESTION = "How much reasoning effort should the chosen model spend on this request?" # keys an answer's chosen option may sit under, most specific first _CHOICE_KEYS = ("choice", "result", "value", "answer", "selected") _USAGE_KEYS = ("prompt_tokens", "completion_tokens", "total_tokens", "cost") +@dataclass(frozen=True, slots=True) +class Question: + id: str + question: str + options: list[str] # opaque model ids or effort levels; never a provider name + + @dataclass(frozen=True, slots=True) class DecisionRequest: endpoint: str api_key: str model: str state: str # everything the decision service may read, already scoped by policy - options: list[str] # opaque option ids; never a provider name + questions: list[Question] timeout_seconds: float @dataclass(frozen=True, slots=True) -class Decision: - choice: str | None # an option id exactly as answered, or None when absent +class Answer: + choice: str | None # exactly as answered, or None when absent; not yet vetted confidence: float | None + + +@dataclass(frozen=True, slots=True) +class Decision: + answers: dict[str, Answer] # one per question asked, keyed by question id usage: dict[str, float] | None http_status: int latency_ms: float @@ -86,12 +101,8 @@ def build_body(request: DecisionRequest) -> dict[str, Any]: "model": request.model, "state": request.state, "questions": [ - { - "id": ROUTE_QUESTION_ID, - "type": "choice", - "question": ROUTE_QUESTION, - "options": list(request.options), - } + {"id": q.id, "type": "choice", "question": q.question, "options": list(q.options)} + for q in request.questions ], } @@ -125,10 +136,9 @@ def elapsed() -> float: if not isinstance(payload, dict): raise DecisionError("invalid_response", latency_ms=elapsed(), http_status=response.status_code) - choice, confidence = _route_answer(payload.get("answers")) + answers = payload.get("answers") return Decision( - choice=choice, - confidence=confidence, + answers={q.id: Answer(*_answer(answers, q.id)) for q in request.questions}, usage=_usage(payload.get("usage")), http_status=response.status_code, latency_ms=elapsed(), @@ -138,13 +148,13 @@ async def aclose(self) -> None: await self._client.aclose() -def _route_answer(answers: Any) -> tuple[str | None, float | None]: - """The chosen option and its probability from the route answer, if present.""" +def _answer(answers: Any, question_id: str) -> tuple[str | None, float | None]: + """The chosen option and its probability for one question, if present.""" if isinstance(answers, list): - answer = next((a for a in answers if isinstance(a, dict) and a.get("id") == ROUTE_QUESTION_ID), None) + answer = next((a for a in answers if isinstance(a, dict) and a.get("id") == question_id), None) elif isinstance(answers, dict): - answer = answers.get(ROUTE_QUESTION_ID) + answer = answers.get(question_id) else: return None, None if isinstance(answer, str): diff --git a/src/cerberus/jev/dispatch.py b/src/cerberus/jev/dispatch.py index 4b48b56..976b758 100644 --- a/src/cerberus/jev/dispatch.py +++ b/src/cerberus/jev/dispatch.py @@ -4,6 +4,7 @@ → cost, health, cooldown, credential router/pool.py → distinct models offered under opaque ids jev/state.py → one decision: which is cheapest-sufficient jev/backend.py + (and, by policy, how much reasoning effort) → chosen model first, rest in config order DispatchPlan → the ordinary failover loop executes router/dispatch.py @@ -11,23 +12,40 @@ reorder models policy already admitted. Any failure to get a usable answer — no credential, timeout, transport or HTTP error, an unreadable response, a choice that is not an offered id — runs the request in configured order and -records why. With one model or none left to choose from, no decision is asked. +records why. With one model or none left to choose from, no model is asked +for; with no effort question either, no decision call is made at all. + +The effort answer, when policy asks for one, replaces the configured +reasoning_effort of every planned candidate that sets one, and of no other. The +egress keeps its rule that a caller's own stated effort wins unless the +candidate sets reasoning_effort_override. """ from __future__ import annotations import os from collections.abc import Collection +from dataclasses import replace from typing import Any # cerberus.router before anything importing cerberus.egress: egress.client # imports router.engine, and the router package imports egress.client in turn from cerberus.router.dispatch import DispatchPlan, dispatch +from cerberus.router.engine import Target from cerberus.router.pool import PoolMember, resolve_pool -from cerberus.jev.backend import DecisionBackend, DecisionError, DecisionRequest +from cerberus.jev.backend import ( + EFFORT_QUESTION, + EFFORT_QUESTION_ID, + ROUTE_QUESTION, + ROUTE_QUESTION_ID, + DecisionBackend, + DecisionError, + DecisionRequest, + Question, +) from cerberus.jev.state import Option, decision_state, options_for from cerberus.registry.loader import ConfigDocument -from cerberus.registry.schema import Alias, CerberusConfig +from cerberus.registry.schema import Alias, CerberusConfig, JevPolicy COST_REFUSAL = "paid_pool_prohibited" @@ -98,7 +116,8 @@ async def jev_dispatch( for member in members if _cost_refused(member) and member.exclusion is not None ] - options = options_for([member.target for member in members if member.exclusion is None]) + attemptable = [member.target for member in members if member.exclusion is None] + options = options_for(attemptable) record: dict[str, Any] = { "decider": policy.backend, @@ -109,17 +128,24 @@ async def jev_dispatch( "reason": None, "choice": None, "confidence": None, + "effort": None, # the reasoning effort Jev chose and Cerberus applied + "effort_confidence": None, + "effort_reason": None, # why no effort was applied, when none was "latency_ms": None, "http_status": None, "usage": None, } - chosen = await _decide(body, options, config, alias, policy, backends, control_plane, record) + chosen, effort = await _decide(body, options, attemptable, config, alias, policy, backends, control_plane, record) if chosen is None: order = admitted else: first = [t for t in admitted if (t.provider_id, t.model) == (chosen.provider, chosen.model)] order = first + [t for t in admitted if (t.provider_id, t.model) != (chosen.provider, chosen.model)] + if effort is not None: + # only where the route already sets an effort: a route without one may + # not accept the parameter at all + order = [replace(t, reasoning_effort=effort) if t.reasoning_effort is not None else t for t in order] return await dispatch( body=body, @@ -137,59 +163,83 @@ async def jev_dispatch( async def _decide( body: dict[str, Any], options: list[Option], + attemptable: list[Target], config: CerberusConfig, alias: Alias, - policy: Any, + policy: JevPolicy, backends: dict[str, DecisionBackend], control_plane: Any | None, record: dict[str, Any], -) -> Option | None: - """The option Jev chose, or None with ``record`` saying why there is none.""" +) -> tuple[Option | None, str | None]: + """The model and the effort Jev chose, each None with ``record`` saying why. + + The model is asked for only with two or more to choose between, and the + effort only when policy lists efforts and an attemptable candidate sets + one for it to replace. With neither to ask, no call is made and no request + text leaves. + """ - def without(decision: str, reason: str) -> None: - record.update(decision=decision, reason=reason) + ask_route = len(options) > 1 + ask_effort = policy.reasoning_efforts is not None and any(t.reasoning_effort is not None for t in attemptable) + if not ask_route: + record.update(decision="skipped", reason="single_option" if options else "no_option") + if not ask_effort: + record["effort_reason"] = "not_requested" if policy.reasoning_efforts is None else "not_applicable" + if not (ask_route or ask_effort): + return None, None + + def failed(reason: str) -> tuple[None, None]: + if ask_route: + record.update(decision="fallback", reason=reason) + if ask_effort: + record["effort_reason"] = reason + return None, None - if len(options) <= 1: - # nothing to choose between: no request text leaves for a decision - without("skipped", "single_option" if options else "no_option") - return None readiness = decider_readiness(config, alias, backend_names=backends.keys(), control_plane=control_plane) if not readiness["backend_present"]: - without("fallback", "backend_missing") - return None + return failed("backend_missing") if not readiness["credential_present"]: - without("fallback", "credential_missing") - return None + return failed("credential_missing") if not readiness["provider_available"]: - without("fallback", "decider_unavailable") - return None - + return failed("decider_unavailable") + + questions = [] + if ask_route: + questions.append(Question(ROUTE_QUESTION_ID, ROUTE_QUESTION, [option.id for option in options])) + if ask_effort: + assert policy.reasoning_efforts is not None + questions.append(Question(EFFORT_QUESTION_ID, EFFORT_QUESTION, list(policy.reasoning_efforts))) request = DecisionRequest( endpoint=str(policy.endpoint), api_key=_decider_key(config, alias), model=policy.model, state=decision_state(body, options, config, input_mode=policy.input, max_chars=policy.max_input_chars), - options=[option.id for option in options], + questions=questions, timeout_seconds=float(policy.timeout_seconds), ) try: decision = await backends[policy.backend].decide(request) except DecisionError as exc: record.update(latency_ms=round(exc.latency_ms, 3), http_status=exc.http_status) - without("fallback", exc.reason) - return None - - record.update( - latency_ms=round(decision.latency_ms, 3), - http_status=decision.http_status, - usage=decision.usage, - confidence=decision.confidence, - ) - chosen = next((option for option in options if option.id == decision.choice), None) - if chosen is None: - # absent, or not an id Cerberus offered: never guessed at - without("fallback", "choice_outside_pool" if decision.choice is not None else "no_choice") - record["confidence"] = None - return None - record.update(decision="chosen", choice=chosen.label) - return chosen + return failed(exc.reason) + record.update(latency_ms=round(decision.latency_ms, 3), http_status=decision.http_status, usage=decision.usage) + + # Each answer is vetted on its own and never guessed at: it must be exactly + # an option Cerberus offered for that question. + chosen = None + if ask_route: + answer = decision.answers[ROUTE_QUESTION_ID] + chosen = next((option for option in options if option.id == answer.choice), None) + if chosen is None: + record.update(decision="fallback", reason="choice_outside_pool" if answer.choice is not None else "no_choice") + else: + record.update(decision="chosen", choice=chosen.label, confidence=answer.confidence) + effort = None + if ask_effort: + answer = decision.answers[EFFORT_QUESTION_ID] + if answer.choice in (policy.reasoning_efforts or []): + effort = answer.choice + record.update(effort=effort, effort_confidence=answer.confidence) + else: + record["effort_reason"] = "choice_outside_options" if answer.choice is not None else "no_choice" + return chosen, effort diff --git a/src/cerberus/registry/schema.py b/src/cerberus/registry/schema.py index 2e184f1..1d7b985 100644 --- a/src/cerberus/registry/schema.py +++ b/src/cerberus/registry/schema.py @@ -28,6 +28,7 @@ ModelStrength = Literal["basic", "standard", "strong", "frontier"] # Distinct models one decision may choose between; bounds the decision input. JEV_MAX_OPTIONS = 64 +ReasoningEffort = Literal["minimal", "low", "medium", "high"] ALIAS_PREFIX = "cerberus/" VERSION_PATTERN = r"^cerberus-\d{4}-\d{2}-\d{2}\.\d+$" @@ -201,7 +202,7 @@ class Candidate(BaseModel): # reported none at ~1.5s. Google bills thinking as output, so this is a real # cost and latency lever — but a single observation must not be generalised # to a whole corpus. - reasoning_effort: Literal["minimal", "low", "medium", "high"] | None = None + reasoning_effort: ReasoningEffort | None = None # A caller that sets reasoning_effort itself keeps it unless the route # explicitly claims precedence. Silently overwriting a caller's stated # reasoning budget would hide the substitution from whoever asked for it. @@ -322,11 +323,27 @@ class JevPolicy(BaseModel): # order. Generation keeps the provider's own timeouts. timeout_seconds: float = Field(default=5.0, gt=0, le=60) allow_paid_pool: bool = False + # When set, the same decision also asks Jev how much reasoning effort the + # request needs, choosing only among these values. The answer replaces the + # configured reasoning_effort of a pool candidate that sets one, and of no + # other: a route without one may not accept the parameter at all, and a + # value outside this list may be one a route rejects. A caller's own stated + # effort still wins unless the candidate sets reasoning_effort_override. + # Absent means the question is never asked. + reasoning_efforts: list[ReasoningEffort] | None = None @model_validator(mode="after") def validate_endpoint(self) -> "JevPolicy": if self.endpoint.scheme != "https": raise ValueError("jev endpoint must use https: the decision input carries request text") + if self.reasoning_efforts is not None: + if len(set(self.reasoning_efforts)) != len(self.reasoning_efforts): + raise ValueError("jev.reasoning_efforts lists a value twice") + if len(self.reasoning_efforts) < 2: + raise ValueError( + "jev.reasoning_efforts needs at least two values to choose between; " + "set reasoning_effort on the candidates instead" + ) return self @@ -491,6 +508,15 @@ def _validate_jev_pool(self, alias_name: str, alias: Alias, tiers: list[CostTier ) if not policy.allow_paid_pool and any(tier == "paid" for tier in tiers): raise ValueError(f"alias {alias_name!r}: paid jev pool member requires allow_paid_pool") + if policy.reasoning_efforts is not None and not any( + candidate.reasoning_effort is not None for candidate in alias.candidates + ): + # the answer only ever replaces a configured effort, so with none + # configured it would be asked for and never applied + raise ValueError( + f"alias {alias_name!r}: jev.reasoning_efforts is set but no pool candidate sets " + f"reasoning_effort, so Jev's answer would never be applied" + ) @model_validator(mode="after") def validate_references(self) -> "CerberusConfig": diff --git a/src/cerberus/router/dispatch.py b/src/cerberus/router/dispatch.py index b295e20..1de4414 100644 --- a/src/cerberus/router/dispatch.py +++ b/src/cerberus/router/dispatch.py @@ -13,7 +13,7 @@ import os import time import uuid -from dataclasses import dataclass +from dataclasses import dataclass, replace from datetime import datetime, timezone from typing import Any @@ -41,8 +41,9 @@ class DispatchPlan: The loop still owns everything after the order: health, cooldown and credential gates, failover, cooldown writes and telemetry. A plan can only - reorder what policy already admitted; it carries no target the alias does - not configure, and the cost gate it was built under is recorded with it. + reorder what policy already admitted, and change the reasoning effort of a + target that already sets one; it carries no target the alias does not + configure, and the cost gate it was built under is recorded with it. """ targets: list[Target] # attempt order, cost-gated @@ -50,6 +51,22 @@ class DispatchPlan: jev: dict[str, Any] # the decision record, for telemetry and the response +def _configured(planned: Target, targets: list[Target]) -> bool: + """Whether a planned target is a configured one, its reasoning effort aside + where the configured target sets an effort of its own.""" + + for target in targets: + if planned == target: + return True + if ( + target.reasoning_effort is not None + and planned.reasoning_effort is not None + and replace(planned, reasoning_effort=target.reasoning_effort) == target + ): + return True + return False + + def _attempt( target: Target, outcome: str, @@ -225,7 +242,7 @@ async def dispatch( eligible, exclusions = cost_eligible(alias, targets) else: # a plan may only reorder what this alias configures, never add to it - if any(target not in targets for target in plan.targets): + if not all(_configured(target, targets) for target in plan.targets): raise ValueError(f"dispatch plan for {alias_name!r} names a target the alias does not configure") eligible, exclusions = list(plan.targets), list(plan.exclusions) jev_record = plan.jev if plan is not None else None diff --git a/src/cerberus/ui/static/app.js b/src/cerberus/ui/static/app.js index 95aac71..e35b3aa 100644 --- a/src/cerberus/ui/static/app.js +++ b/src/cerberus/ui/static/app.js @@ -817,6 +817,8 @@ function jevPool(entry) { el("h3", { text: "Decider" }), el("div", { class: "card-row" }, el("span", { text: "model" }), codeText(d.model)), el("div", { class: "card-row" }, el("span", { text: "reads" }), codeText(d.input.replace(/_/g, " "))), + el("div", { class: "card-row" }, el("span", { text: "reasoning effort" }), + codeText(d.reasoning_efforts ? "Jev chooses: " + d.reasoning_efforts.join(", ") : "configured")), el("div", { class: "card-row" }, el("span", { text: "backend configured" }), yes(ready.backend_present)), el("div", { class: "card-row" }, el("span", { text: "credential present" }), yes(ready.credential_present)), el("div", { class: "card-row" }, el("span", { text: "can decide now" }), yes(ready.decides)), diff --git a/tests/test_jev_dispatch.py b/tests/test_jev_dispatch.py index fcb054d..3344ef1 100644 --- a/tests/test_jev_dispatch.py +++ b/tests/test_jev_dispatch.py @@ -14,8 +14,8 @@ import pytest from cerberus.app import create_app -from cerberus.jev import DecisionRequest, OpenRouterJevDecider -from cerberus.jev.backend import _route_answer +from cerberus.jev import DecisionRequest, OpenRouterJevDecider, Question +from cerberus.jev.backend import _answer from cerberus.registry import load_config_document from tests.test_control import write_config @@ -244,6 +244,9 @@ async def test_the_chosen_model_runs_first_through_cerberus(monkeypatch, tmp_pat "reason": None, "choice": "openrouter/anthropic/claude-sonnet", "confidence": 0.91, + "effort": None, + "effort_confidence": None, + "effort_reason": "not_requested", "latency_ms": event["jev"]["latency_ms"], "http_status": 200, "usage": {"prompt_tokens": 120, "completion_tokens": 1, "total_tokens": 121, "cost": 0.00004}, @@ -446,10 +449,18 @@ async def test_revision_materializes_the_jev_pool(monkeypatch, tmp_path): def test_decider_builds_the_documented_body(): request = DecisionRequest(endpoint="https://openrouter.ai/api/alpha/decisions", api_key="k", - model="typesafe/jev-1.13", state="s", options=["m1", "m2"], timeout_seconds=5) + model="typesafe/jev-1.13", state="s", + questions=[Question("route", "q?", ["m1", "m2"]), Question("effort", "e?", ["low", "high"])], + timeout_seconds=5) body = OpenRouterJevDecider.build_body(request) - assert body["model"] == "typesafe/jev-1.13" and body["state"] == "s" - assert body["questions"][0]["options"] == ["m1", "m2"] and body["questions"][0]["type"] == "choice" + assert body == { + "model": "typesafe/jev-1.13", + "state": "s", + "questions": [ + {"id": "route", "type": "choice", "question": "q?", "options": ["m1", "m2"]}, + {"id": "effort", "type": "choice", "question": "e?", "options": ["low", "high"]}, + ], + } @pytest.mark.parametrize( @@ -467,7 +478,7 @@ def test_decider_builds_the_documented_body(): ], ) def test_the_route_answer_is_read_tolerantly(answers, expected): - assert _route_answer(answers) == expected + assert _answer(answers, "route") == expected @pytest.mark.asyncio @@ -487,3 +498,168 @@ async def test_a_plan_cannot_add_a_target_the_alias_does_not_configure(monkeypat plan=DispatchPlan(targets=[stranger], exclusions=[], jev={}), ) assert upstream.requests == [] + + +# ---- the second question: reasoning effort ---- + +EFFORTS = ["low", "medium", "high"] + + +def effort_raw(**jev) -> dict: + """The frontier candidate sets an effort for Jev's answer to replace; the + others set none, so nothing may be added to them.""" + raw = jev_raw(jev={"reasoning_efforts": EFFORTS, **jev}) + for candidate in raw["aliases"]["cerberus/smart"]["candidates"]: + if candidate["model"] == FRONTIER[1]: + candidate["reasoning_effort"] = "medium" + return raw + + +def decides(route="m3", effort="high"): + answers = {} + if route is not None: + answers["route"] = {"choice": route, "probabilities": {route: 0.8}} + if effort is not None: + answers["effort"] = {"choice": effort, "probabilities": {effort: 0.7}} + return answer({"answers": answers, "usage": {"total_tokens": 130}}) + + +def sent_effort(upstream, index=0): + return json.loads(upstream.requests[index].content).get("reasoning_effort") + + +@pytest.mark.asyncio +async def test_effort_is_a_second_question_in_the_same_call(monkeypatch, tmp_path): + app, decisions, _ = make_app(monkeypatch, tmp_path, raw=effort_raw(), decide=decides()) + await call(app) + + assert len(decisions.requests) == 1 + assert [q["id"] for q in decisions.body["questions"]] == ["route", "effort"] + assert decisions.body["questions"][1] == { + "id": "effort", + "type": "choice", + "question": "How much reasoning effort should the chosen model spend on this request?", + "options": EFFORTS, + } + + +@pytest.mark.asyncio +async def test_the_effort_answer_replaces_a_configured_effort(monkeypatch, tmp_path): + app, _, upstream = make_app(monkeypatch, tmp_path, raw=effort_raw(), decide=decides("m3", "high")) + resp, events = await call(app) + + assert resp.status_code == 200 + assert upstream.models() == ["anthropic/claude-sonnet"] + assert sent_effort(upstream) == "high", "Jev's answer, not the configured medium" + event = events[0] + assert event["reasoning_effort"] == "high" + assert (event["jev"]["effort"], event["jev"]["effort_confidence"], event["jev"]["effort_reason"]) == ("high", 0.7, None) + assert resp.json()["cerberus"]["reasoning_effort"] == "high" + + +@pytest.mark.asyncio +async def test_the_effort_is_never_added_to_a_route_without_one(monkeypatch, tmp_path): + app, _, upstream = make_app(monkeypatch, tmp_path, raw=effort_raw(), decide=decides("m2", "high")) + await call(app) + assert upstream.models() == ["vendor/mid"] + assert sent_effort(upstream) is None, "a route without an effort may not accept the parameter" + + +@pytest.mark.asyncio +async def test_a_caller_stated_effort_still_wins(monkeypatch, tmp_path): + app, _, upstream = make_app(monkeypatch, tmp_path, raw=effort_raw(), decide=decides("m3", "high")) + await call(app, {**REQ, "reasoning_effort": "low"}) + assert sent_effort(upstream) == "low" + + +@pytest.mark.parametrize("effort", ["maximum", "minimal"], ids=["unknown", "not-in-policy"]) +@pytest.mark.asyncio +async def test_an_effort_outside_the_options_is_never_applied(monkeypatch, tmp_path, effort): + app, _, upstream = make_app(monkeypatch, tmp_path, raw=effort_raw(), decide=decides("m3", effort)) + _, events = await call(app) + assert sent_effort(upstream) == "medium", "the configured effort stands" + record = events[0]["jev"] + assert (record["decision"], record["effort"], record["effort_reason"]) == ("chosen", None, "choice_outside_options") + + +@pytest.mark.asyncio +async def test_a_missing_effort_answer_leaves_the_route_choice_standing(monkeypatch, tmp_path): + app, _, upstream = make_app(monkeypatch, tmp_path, raw=effort_raw(), decide=decides("m3", None)) + _, events = await call(app) + assert upstream.models() == ["anthropic/claude-sonnet"] and sent_effort(upstream) == "medium" + assert (events[0]["jev"]["decision"], events[0]["jev"]["effort_reason"]) == ("chosen", "no_choice") + + +@pytest.mark.asyncio +async def test_effort_alone_is_asked_when_one_model_is_left(monkeypatch, tmp_path): + app, decisions, upstream = make_app(monkeypatch, tmp_path, raw=effort_raw(), decide=decides(None, "low")) + app.state.cooldowns.apply(scope="model", provider="openrouter", credential="main", model="vendor/mid", + reason="quota_429", duration_seconds=600) + app.state.cooldowns.apply(scope="model", provider="edgebox", credential="none", model="gemma-4-12b", + reason="quota_429", duration_seconds=600) + _, events = await call(app) + + assert [q["id"] for q in decisions.body["questions"]] == ["effort"] + assert upstream.models() == ["anthropic/claude-sonnet"] and sent_effort(upstream) == "low" + record = events[0]["jev"] + assert (record["decision"], record["reason"], record["effort"]) == ("skipped", "single_option", "low") + + +@pytest.mark.asyncio +async def test_effort_is_not_asked_when_no_attemptable_route_sets_one(monkeypatch, tmp_path): + app, decisions, _ = make_app(monkeypatch, tmp_path, raw=effort_raw(), decide=decides("m1", None)) + app.state.cooldowns.apply(scope="model", provider="openrouter", credential="main", + model="anthropic/claude-sonnet", reason="quota_429", duration_seconds=600) + _, events = await call(app) + assert [q["id"] for q in decisions.body["questions"]] == ["route"] + assert events[0]["jev"]["effort_reason"] == "not_applicable" + + +@pytest.mark.asyncio +async def test_a_failed_decision_records_why_no_effort_was_applied(monkeypatch, tmp_path): + app, _, upstream = make_app(monkeypatch, tmp_path, raw=effort_raw(), decide=answer({"error": "x"}, status=503)) + _, events = await call(app) + record = events[0]["jev"] + assert (record["decision"], record["reason"], record["effort_reason"]) == ("fallback", "http_error", "http_error") + assert upstream.models() == ["gemma-4-12b"] + + +@pytest.mark.asyncio +async def test_a_plan_may_change_a_configured_effort_but_never_add_one(monkeypatch, tmp_path): + from dataclasses import replace + + from cerberus.router.dispatch import DispatchPlan, dispatch + from cerberus.router.engine import ordered_targets + + app, _, upstream = make_app(monkeypatch, tmp_path, raw=effort_raw()) + document = app.state.lifecycle.active + local, _mid, frontier = ordered_targets(document.config, "cerberus/smart") + + async def run(target): + async with httpx.AsyncClient(transport=httpx.MockTransport(upstream)) as client: + return await dispatch( + body=REQ, alias_name="cerberus/smart", document=document, store=app.state.cooldowns, + client=client, telemetry=_NullTelemetry(), + plan=DispatchPlan(targets=[target], exclusions=[], jev={}), + ) + + with pytest.raises(ValueError, match="does not configure"): + await run(replace(local, reasoning_effort="high")) + assert upstream.requests == [] + resp = await run(replace(frontier, reasoning_effort="high")) + assert resp.status_code == 200 and sent_effort(upstream) == "high" + + +class _NullTelemetry: + def emit(self, _event): + pass + + +@pytest.mark.asyncio +async def test_routes_show_which_efforts_jev_may_choose(monkeypatch, tmp_path): + app, _, _ = make_app(monkeypatch, tmp_path, raw=effort_raw()) + async with app.router.lifespan_context(app): + async with client_for(app) as client: + projection = (await client.get("/admin/routes")).json() + entry = next(a for a in projection["aliases"] if a["alias"] == "cerberus/smart") + assert entry["jev"]["decider"]["reasoning_efforts"] == EFFORTS diff --git a/tests/test_schema.py b/tests/test_schema.py index c816ec1..ba0e7db 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -470,3 +470,17 @@ def test_jev_input_policy_is_a_closed_vocabulary(): raw = _jev_native_raw(jev={"decider": {"provider": "openrouter", "credential": "primary"}, "input": "everything"}) with pytest.raises(ValidationError): CerberusConfig.model_validate(raw) + + +def test_jev_reasoning_efforts_need_a_choice_and_a_route_to_apply_to(): + raw = _jev_native_raw() + policy = raw["aliases"]["cerberus/smart"]["jev"] + policy["reasoning_efforts"] = ["low", "high"] + with pytest.raises(ValidationError, match="would never be applied"): + CerberusConfig.model_validate(raw) + raw["aliases"]["cerberus/smart"]["candidates"][0]["reasoning_effort"] = "low" + assert CerberusConfig.model_validate(raw).aliases["cerberus/smart"].jev.reasoning_efforts == ["low", "high"] + for bad, message in ((["low"], "at least two"), (["low", "low"], "twice"), (["low", "extreme"], "literal_error")): + policy["reasoning_efforts"] = bad + with pytest.raises(ValidationError, match=message): + CerberusConfig.model_validate(raw)