From 06ab4b4202018320354ac52520a2cae273aed65d Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 6 Oct 2026 02:35:53 +0000 Subject: [PATCH] Send jev decisions in the Decisions API's documented shape A live probe against OpenRouter rejected the adapter's request with HTTP 400: `questions` must be a record keyed by question id, not a list. OpenRouter's published reference gives the rest of the contract: each choice question is {type, instructions, criteria}, with criteria mapping every offered option to its guidance or null, and usage reports input_tokens, output_tokens and cost. - build_body sends questions as {id: {type, instructions, criteria}}; the options carry no guidance of their own since the state describes each one. The live validator accepts this form (the next refusal was the key's budget). - Usage is read from input_tokens/output_tokens/cost; the old chat-completions names never matched and only cost survived. - The probe reports a 401/402/403 as a refused key or credit, not a rejected request shape. - The probe's recording transport re-served a decoded body under its original content-encoding, so any compressed response surfaced as "unreachable". Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01NB8ieibwKFDVRzG8jXS6F6 --- README.md | 3 ++- docs/architecture.md | 14 ++++++---- scripts/probe-jev-decisions.py | 10 +++++-- src/cerberus/jev/backend.py | 30 ++++++++++++--------- tests/test_jev_dispatch.py | 45 ++++++++++++++++--------------- tests/test_probe_jev_decisions.py | 33 ++++++++++++++++------- 6 files changed, 82 insertions(+), 53 deletions(-) diff --git a/README.md b/README.md index 6b3de39..4ffcfac 100644 --- a/README.md +++ b/README.md @@ -119,7 +119,8 @@ that model first through its own failover loop, the rest in configured order. To check a real Decisions API against what a jev alias sends, run the live probe. It sends one synthetic decision through Cerberus's own adapter, prints the raw response beside what Cerberus read from it, and exits non-zero when the -request is rejected or an answer cannot be read (one decision call is billed): +request is rejected or an answer cannot be read (one decision call is billed, so +the key needs a credit limit above zero): ```sh OPENROUTER_API_KEY=... uv run --frozen python scripts/probe-jev-decisions.py diff --git a/docs/architecture.md b/docs/architecture.md index 9fae7f8..3637d1c 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -124,11 +124,15 @@ Each routing event carries a `jev` record: options offered, whether the decision was `chosen`, `skipped` or a `fallback`, the reason, the choice and its probability, the effort applied with its probability or why none was, decision latency, status and usage. It never carries request text. -The confirmed Decisions API contract covers the endpoint, the model, the -`model`/`state`/`questions` request and `answers` keyed by question id; the -fields inside one question and one answer are not yet confirmed, so the answer -is read tolerantly and a misread degrades to configured order rather than to an -unvetted model. `scripts/probe-jev-decisions.py` settles it against a live +The request follows OpenRouter's published Decisions API reference: `questions` +is a record keyed by question id, and each `choice` question carries +`instructions` and `criteria`, a record of the offered options (with no guidance +of their own; the state describes each model). The live validator rejects a list +of questions and accepts this form. A choice answer is documented as `choice`, +`confidence` and per-option `probabilities`; the record keeps the probability of +the option chosen. Until a live call confirms the answer, it is read tolerantly, +and a misread degrades to configured order rather than to an unvetted model. +`scripts/probe-jev-decisions.py` settles it against a live endpoint: it sends one synthetic decision through the production adapter and reports whether the request was accepted and each answer read as an offered option. diff --git a/scripts/probe-jev-decisions.py b/scripts/probe-jev-decisions.py index 0aceec8..c6a30ca 100644 --- a/scripts/probe-jev-decisions.py +++ b/scripts/probe-jev-decisions.py @@ -96,9 +96,10 @@ def __init__(self, inner: httpx.AsyncBaseTransport) -> None: async def handle_async_request(self, request: httpx.Request) -> httpx.Response: response = await self._inner.handle_async_request(request) - content = await response.aread() + content = await response.aread() # decoded, so its encoding headers no longer apply self.status, self.body = response.status_code, content - return httpx.Response(response.status_code, headers=response.headers, content=content, request=request) + headers = [(k, v) for k, v in response.headers.multi_items() if k not in ("content-encoding", "content-length")] + return httpx.Response(response.status_code, headers=headers, content=content, request=request) async def aclose(self) -> None: await self._inner.aclose() @@ -197,6 +198,11 @@ def render(report: Report, *, endpoint: str, model: str) -> str: def verdict(report: Report) -> str: if report.ok: return "OK. The request is accepted and every answer reads as an offered option." + if report.failure == "http_error" and report.status in (401, 402, 403): + return ( + f"KEY REFUSED (HTTP {report.status}). OpenRouter refused the key or its credit before Jev answered; " + "the error above says which. Use a key with a credit limit above zero and retry." + ) if report.failure == "http_error": return ( f"REQUEST REJECTED (HTTP {report.status}). Compare the error body with the request body; " diff --git a/src/cerberus/jev/backend.py b/src/cerberus/jev/backend.py index d3977e7..ea12574 100644 --- a/src/cerberus/jev/backend.py +++ b/src/cerberus/jev/backend.py @@ -5,15 +5,18 @@ needs — and gets back, per question, one of the options it offered or nothing usable. Nothing outside this module knows the wire shape of a decision service. -Confirmed contract for OpenRouter's Decisions API: ``POST /api/alpha/decisions`` -with a bearer key; the body carries ``model``, ``state`` and ``questions``; -Jev answers ``choice``, ``noul`` and ``score`` questions; the response carries -``answers`` keyed by question id, with typed results and probabilities, and -``usage``. The field names inside one question and one answer are not yet -confirmed, so the request uses the plainest form and the answer is read -tolerantly. A misread can only ever produce "no usable answer", because each -choice must equal an option Cerberus sent; the request then runs in configured -order and configured effort, which policy already allows. +Contract for OpenRouter's Decisions API, from its published reference: ``POST +/api/alpha/decisions`` with a bearer key; the body carries ``model``, ``state`` +and ``questions``, a record keyed by question id. A ``choice`` question is +``{type, instructions, criteria}``, where ``criteria`` maps each option to its +guidance or ``null``; the live validator rejects a list of questions and +accepts this form. The response carries ``answers`` keyed by question id — a +choice answer is ``{type, choice, confidence?, probabilities?}`` — and ``usage`` +with ``input_tokens``, ``output_tokens`` and ``cost``. The answer is still read +tolerantly until a live call confirms it. A misread can only ever produce "no +usable answer", because each choice must equal an option Cerberus sent; the +request then runs in configured order and configured effort, which policy +already allows. """ from __future__ import annotations @@ -37,7 +40,7 @@ EFFORT_QUESTION = "How much reasoning effort should the chosen model spend on this request?" # keys an answer's chosen option may sit under, most specific first _CHOICE_KEYS = ("choice", "result", "value", "answer", "selected") -_USAGE_KEYS = ("prompt_tokens", "completion_tokens", "total_tokens", "cost") +_USAGE_KEYS = ("input_tokens", "output_tokens", "cost") @dataclass(frozen=True, slots=True) @@ -97,13 +100,14 @@ def __init__(self, transport: httpx.AsyncBaseTransport | None = None) -> None: @staticmethod def build_body(request: DecisionRequest) -> dict[str, Any]: + # an option needs no guidance of its own: the state already describes each one return { "model": request.model, "state": request.state, - "questions": [ - {"id": q.id, "type": "choice", "question": q.question, "options": list(q.options)} + "questions": { + q.id: {"type": "choice", "instructions": q.question, "criteria": dict.fromkeys(q.options)} for q in request.questions - ], + }, } async def decide(self, request: DecisionRequest) -> Decision: diff --git a/tests/test_jev_dispatch.py b/tests/test_jev_dispatch.py index 3344ef1..c0fb42d 100644 --- a/tests/test_jev_dispatch.py +++ b/tests/test_jev_dispatch.py @@ -103,8 +103,8 @@ def respond(_request): def chooses(option, probability=0.91): return answer({ - "answers": {"route": {"type": "choice", "choice": option, "probabilities": {option: probability}}}, - "usage": {"prompt_tokens": 120, "completion_tokens": 1, "total_tokens": 121, "cost": 0.00004}, + "answers": {"route": {"type": "choice", "choice": option, "confidence": 0.6, "probabilities": {option: probability}}}, + "usage": {"input_tokens": 120, "output_tokens": 1, "cost": 0.00004}, }) @@ -169,13 +169,12 @@ async def test_the_decision_is_one_decisions_api_call_with_the_pinned_model(monk assert req.headers["authorization"] == "Bearer jev-key", "the decider's own credential" body = decisions.body assert body["model"] == "typesafe/jev-1.13" - assert body["questions"] == [{ - "id": "route", + assert body["questions"] == {"route": { "type": "choice", - "question": "Which candidate model is the least costly one that is still strong enough " - "to handle this request well?", - "options": ["m1", "m2", "m3"], - }] + "instructions": "Which candidate model is the least costly one that is still strong enough " + "to handle this request well?", + "criteria": {"m1": None, "m2": None, "m3": None}, + }} assert set(body) == {"model", "state", "questions"} @@ -249,7 +248,7 @@ async def test_the_chosen_model_runs_first_through_cerberus(monkeypatch, tmp_pat "effort_reason": "not_requested", "latency_ms": event["jev"]["latency_ms"], "http_status": 200, - "usage": {"prompt_tokens": 120, "completion_tokens": 1, "total_tokens": 121, "cost": 0.00004}, + "usage": {"input_tokens": 120, "output_tokens": 1, "cost": 0.00004}, } @@ -363,7 +362,7 @@ async def test_a_cooled_down_model_is_never_offered(monkeypatch, tmp_path): app.state.cooldowns.apply(scope="model", provider="openrouter", credential="main", model="vendor/mid", reason="quota_429", duration_seconds=600) await call(app) - assert decisions.body["questions"][0]["options"] == ["m1", "m2"] + assert list(decisions.body["questions"]["route"]["criteria"]) == ["m1", "m2"] assert "vendor/mid" not in decisions.body["state"] assert "m2: model anthropic/claude-sonnet" in decisions.body["state"] @@ -456,16 +455,19 @@ def test_decider_builds_the_documented_body(): assert body == { "model": "typesafe/jev-1.13", "state": "s", - "questions": [ - {"id": "route", "type": "choice", "question": "q?", "options": ["m1", "m2"]}, - {"id": "effort", "type": "choice", "question": "e?", "options": ["low", "high"]}, - ], + "questions": { + "route": {"type": "choice", "instructions": "q?", "criteria": {"m1": None, "m2": None}}, + "effort": {"type": "choice", "instructions": "e?", "criteria": {"low": None, "high": None}}, + }, } @pytest.mark.parametrize( ("answers", "expected"), [ + # the documented shape: confidence summarizes the spread; the record keeps the choice's probability + ({"route": {"type": "choice", "choice": "m2", "confidence": 0.6, "probabilities": {"m1": 0.2, "m2": 0.8}}}, + ("m2", 0.8)), ({"route": {"choice": "m2", "probabilities": {"m1": 0.2, "m2": 0.8}}}, ("m2", 0.8)), ({"route": {"result": "m1"}}, ("m1", None)), ({"route": {"value": "m1", "confidence": 0.7}}, ("m1", 0.7)), @@ -521,7 +523,7 @@ def decides(route="m3", effort="high"): answers["route"] = {"choice": route, "probabilities": {route: 0.8}} if effort is not None: answers["effort"] = {"choice": effort, "probabilities": {effort: 0.7}} - return answer({"answers": answers, "usage": {"total_tokens": 130}}) + return answer({"answers": answers, "usage": {"input_tokens": 130, "output_tokens": 2}}) def sent_effort(upstream, index=0): @@ -534,12 +536,11 @@ async def test_effort_is_a_second_question_in_the_same_call(monkeypatch, tmp_pat await call(app) assert len(decisions.requests) == 1 - assert [q["id"] for q in decisions.body["questions"]] == ["route", "effort"] - assert decisions.body["questions"][1] == { - "id": "effort", + assert list(decisions.body["questions"]) == ["route", "effort"] + assert decisions.body["questions"]["effort"] == { "type": "choice", - "question": "How much reasoning effort should the chosen model spend on this request?", - "options": EFFORTS, + "instructions": "How much reasoning effort should the chosen model spend on this request?", + "criteria": dict.fromkeys(EFFORTS), } @@ -599,7 +600,7 @@ async def test_effort_alone_is_asked_when_one_model_is_left(monkeypatch, tmp_pat reason="quota_429", duration_seconds=600) _, events = await call(app) - assert [q["id"] for q in decisions.body["questions"]] == ["effort"] + assert list(decisions.body["questions"]) == ["effort"] assert upstream.models() == ["anthropic/claude-sonnet"] and sent_effort(upstream) == "low" record = events[0]["jev"] assert (record["decision"], record["reason"], record["effort"]) == ("skipped", "single_option", "low") @@ -611,7 +612,7 @@ async def test_effort_is_not_asked_when_no_attemptable_route_sets_one(monkeypatc app.state.cooldowns.apply(scope="model", provider="openrouter", credential="main", model="anthropic/claude-sonnet", reason="quota_429", duration_seconds=600) _, events = await call(app) - assert [q["id"] for q in decisions.body["questions"]] == ["route"] + assert list(decisions.body["questions"]) == ["route"] assert events[0]["jev"]["effort_reason"] == "not_applicable" diff --git a/tests/test_probe_jev_decisions.py b/tests/test_probe_jev_decisions.py index b62b384..758ccdd 100644 --- a/tests/test_probe_jev_decisions.py +++ b/tests/test_probe_jev_decisions.py @@ -6,6 +6,7 @@ never prints the key. """ +import gzip import importlib.util import json import sys @@ -38,10 +39,10 @@ def respond(request): def answered(route="m1", effort="low"): return { "answers": { - "route": {"choice": route, "probabilities": {route: 0.84}}, - "effort": {"choice": effort, "probabilities": {effort: 0.66}}, + "route": {"type": "choice", "choice": route, "confidence": 0.7, "probabilities": {route: 0.84}}, + "effort": {"type": "choice", "choice": effort, "confidence": 0.5, "probabilities": {effort: 0.66}}, }, - "usage": {"prompt_tokens": 210, "completion_tokens": 2, "total_tokens": 212}, + "usage": {"input_tokens": 210, "output_tokens": 2, "cost": 0.0000088}, } @@ -61,9 +62,9 @@ async def test_the_probe_sends_the_production_request_shape(): body = json.loads(request.content) assert body == report.request_body assert body["model"] == "typesafe/jev-1.13" - assert [q["id"] for q in body["questions"]] == ["route", "effort"] - assert body["questions"][0]["options"] == ["m1", "m2", "m3"] - assert body["questions"][1]["options"] == ["low", "medium", "high"] + assert list(body["questions"]) == ["route", "effort"] + assert body["questions"]["route"]["criteria"] == {"m1": None, "m2": None, "m3": None} + assert body["questions"]["effort"]["criteria"] == {"low": None, "medium": None, "high": None} # synthetic state only: the probe's own made-up pool and prompt assert "probe/small-local" in body["state"] and probe.DEFAULT_PROMPT in body["state"] @@ -74,7 +75,7 @@ async def test_an_accepted_request_with_readable_answers_passes(): assert report.ok assert report.answers["route"] == {"choice": "m3", "confidence": 0.84, "offered": True} assert report.answers["effort"] == {"choice": "high", "confidence": 0.66, "offered": True} - assert report.usage == {"prompt_tokens": 210, "completion_tokens": 2, "total_tokens": 212} + assert report.usage == {"input_tokens": 210, "output_tokens": 2, "cost": 0.0000088} assert probe.verdict(report).startswith("OK") @@ -82,19 +83,20 @@ async def test_an_accepted_request_with_readable_answers_passes(): async def test_no_effort_asks_only_the_route(): sent = [] report = await run(transport(payload=answered(), capture=sent), ask_effort=False) - assert [q["id"] for q in json.loads(sent[0].content)["questions"]] == ["route"] + assert list(json.loads(sent[0].content)["questions"]) == ["route"] assert list(report.answers) == ["route"] and report.ok @pytest.mark.parametrize( ("t", "start", "status"), [ - (transport(400, {"error": {"message": "questions[0].options: unknown field"}}), "REQUEST REJECTED (HTTP 400)", 400), + (transport(400, {"error": {"message": "questions: expected record, received array"}}), "REQUEST REJECTED (HTTP 400)", 400), + (transport(403, {"error": {"message": "Key limit exceeded (total limit)."}}), "KEY REFUSED (HTTP 403)", 403), (transport(200, content=b""), "UNREADABLE RESPONSE", 200), (transport(200, {"answers": {"route": {"label": "m1"}}}), "ANSWER NOT READ for route, effort", 200), (transport(200, answered("anthropic/claude", "extreme")), "ANSWERED OUTSIDE THE OPTIONS", 200), ], - ids=["rejected", "not-json", "unread", "outside"], + ids=["rejected", "key-refused", "not-json", "unread", "outside"], ) @pytest.mark.asyncio async def test_each_failure_is_reported_as_what_it_is(t, start, status): @@ -104,6 +106,17 @@ async def test_each_failure_is_reported_as_what_it_is(t, start, status): assert probe.verdict(report).startswith(start) +@pytest.mark.asyncio +async def test_a_compressed_response_is_reported_as_what_it_is(): + error = gzip.compress(json.dumps({"error": {"message": "Key limit exceeded (total limit)."}}).encode()) + report = await run(httpx.MockTransport( + lambda _request: httpx.Response(403, content=error, headers={"content-encoding": "gzip"}) + )) + assert (report.status, report.failure) == (403, "http_error") + assert report.raw == {"error": {"message": "Key limit exceeded (total limit)."}} + assert probe.verdict(report).startswith("KEY REFUSED (HTTP 403)") + + @pytest.mark.asyncio async def test_the_rendered_report_shows_the_raw_response_and_never_the_key(): report = await run(transport(400, {"error": {"message": "bad question"}}))