Live Anthropic + OpenAI judges + canary preflight

.github/workflows/ci.yml

@@ -31,7 +31,7 @@ jobs:
       - uses: actions/setup-python@v5
         with:
           python-version: "3.12"
-      - run: pip install -e .[dev]
+      - run: pip install -e .[dev,judges]
       - run: pytest -m "not integration" --cov=medimage_eval --cov-fail-under=70
 
   secrets-scan:

pyproject.toml

@@ -60,6 +60,7 @@ ignore = ["S101"]
 
 [tool.ruff.lint.per-file-ignores]
 "tests/**/*.py" = ["S", "E501"]
+"src/medimage_eval/judges/clinical_prompts.py" = ["E501"]
 
 [tool.pytest.ini_options]
 testpaths = ["tests"]

src/medimage_eval/judges/__init__.py

@@ -1,13 +1,31 @@
-"""Judge orchestration: dual-judge clinical accuracy with κ floor."""
+"""Judge orchestration: dual-judge clinical accuracy with κ floor.
 
+The Anthropic and OpenAI adapters are lazily importable — calling them
+without the optional SDK installed raises only when constructed, not when
+the package is imported.
+"""
+
+from medimage_eval.judges.anthropic_judge import AnthropicJudge, JudgeAPIError
 from medimage_eval.judges.base import Judge, JudgeQuery, JudgeVerdict
+from medimage_eval.judges.decision import (
+    GRADING_DECISION_SCHEMA,
+    DecisionParseError,
+    parse_decision_text,
+)
 from medimage_eval.judges.dual_judge import DualJudge, DualJudgeItem, DualJudgeResult
+from medimage_eval.judges.openai_judge import OpenAIJudge
 
 __all__ = [
+    "GRADING_DECISION_SCHEMA",
+    "AnthropicJudge",
+    "DecisionParseError",
     "DualJudge",
     "DualJudgeItem",
     "DualJudgeResult",
     "Judge",
+    "JudgeAPIError",
     "JudgeQuery",
     "JudgeVerdict",
+    "OpenAIJudge",
+    "parse_decision_text",
 ]

src/medimage_eval/judges/anthropic_judge.py

@@ -0,0 +1,141 @@
+"""AnthropicJudge — Claude Opus 4.7 wrapper that implements the `Judge` Protocol.
+
+Design notes (per the claude-api guidance):
+
+* Default model is `claude-opus-4-7`. The model ID is the bare alias — do not
+  append a date suffix.
+* Opus 4.7 removes `temperature`, `top_p`, `top_k`, and `budget_tokens`. We
+  pass none of them. Adaptive thinking is off by default on 4.7 (matches
+  4.6 behaviour) — we leave it off for grading because the rubric is short
+  and we want deterministic-ish judgments.
+* The system prompt is sent with `cache_control: {type: "ephemeral"}` so it
+  caches across the many calls a single eval run makes. Below the 4096-token
+  cache minimum, the marker is silently inert — it costs nothing and starts
+  paying off once the rubric grows.
+* Structured outputs via `output_config.format` (not a prefill — prefills
+  400 on Opus 4.7).
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from medimage_eval.judges.base import JudgeQuery, JudgeVerdict
+from medimage_eval.judges.clinical_prompts import (
+    DEFAULT_CLINICAL_GRADING_PROMPT,
+    build_user_message,
+)
+from medimage_eval.judges.decision import GRADING_DECISION_SCHEMA, parse_decision_text
+
+if TYPE_CHECKING:
+    import anthropic
+
+DEFAULT_MODEL_ID = "claude-opus-4-7"
+DEFAULT_MAX_TOKENS = 400
+DEFAULT_TIMEOUT_SECONDS = 30.0
+
+
+class JudgeAPIError(RuntimeError):
+    """Raised when the underlying provider returned a non-retryable error."""
+
+
+class AnthropicJudge:
+    """Adapter from the Anthropic SDK to the `Judge` Protocol.
+
+    Args:
+        model_id: Anthropic model ID. Defaults to ``"claude-opus-4-7"``.
+        system_prompt: Override the default clinical-grading rubric.
+        max_tokens: Output cap. Grading responses are short, so 400 is plenty.
+        timeout_seconds: Per-request timeout. Fails fast if a judge call hangs.
+        client: Inject an ``anthropic.Anthropic`` (or test double). When None,
+            a default client is constructed lazily on first ``grade`` call.
+    """
+
+    def __init__(
+        self,
+        *,
+        model_id: str = DEFAULT_MODEL_ID,
+        system_prompt: str = DEFAULT_CLINICAL_GRADING_PROMPT,
+        max_tokens: int = DEFAULT_MAX_TOKENS,
+        timeout_seconds: float = DEFAULT_TIMEOUT_SECONDS,
+        client: anthropic.Anthropic | None = None,
+    ) -> None:
+        if not model_id:
+            raise ValueError("model_id is required")
+        self.model_id = model_id
+        self._system_prompt = system_prompt
+        self._max_tokens = max_tokens
+        self._timeout_seconds = timeout_seconds
+        self._client = client
+
+    def _ensure_client(self) -> anthropic.Anthropic:
+        if self._client is not None:
+            return self._client
+        try:
+            import anthropic
+        except ImportError as exc:  # pragma: no cover - import failure path
+            raise RuntimeError(
+                "anthropic SDK is not installed. Install with `pip install medimage-eval[judges]`."
+            ) from exc
+        self._client = anthropic.Anthropic(timeout=self._timeout_seconds)
+        return self._client
+
+    def grade(self, query: JudgeQuery) -> JudgeVerdict:
+        client = self._ensure_client()
+        try:
+            response = client.messages.create(
+                model=self.model_id,
+                max_tokens=self._max_tokens,
+                system=[
+                    {
+                        "type": "text",
+                        "text": self._system_prompt,
+                        "cache_control": {"type": "ephemeral"},
+                    }
+                ],
+                messages=[
+                    {
+                        "role": "user",
+                        "content": build_user_message(
+                            gold=query.gold,
+                            candidate=query.candidate,
+                            instructions=query.instructions,
+                        ),
+                    }
+                ],
+                output_config={
+                    "format": {
+                        "type": "json_schema",
+                        "schema": GRADING_DECISION_SCHEMA,
+                    }
+                },
+            )
+        except Exception as exc:
+            raise JudgeAPIError(f"anthropic judge call failed: {exc}") from exc
+
+        if getattr(response, "stop_reason", None) == "refusal":
+            raise JudgeAPIError(
+                f"judge refused to grade prompt_id={query.prompt_id}: "
+                f"{getattr(response, 'stop_details', None)}"
+            )
+
+        text = _extract_first_text_block(response)
+        label, rationale = parse_decision_text(text)
+        return JudgeVerdict(
+            prompt_id=query.prompt_id,
+            label=label,
+            rationale=rationale,
+            raw=text,
+        )
+
+
+def _extract_first_text_block(response: Any) -> str:
+    content = getattr(response, "content", None)
+    if not content:
+        raise JudgeAPIError("anthropic response had no content blocks")
+    for block in content:
+        if getattr(block, "type", None) == "text":
+            text = getattr(block, "text", "")
+            if text:
+                return text
+    raise JudgeAPIError("anthropic response had no non-empty text block")

src/medimage_eval/judges/clinical_prompts.py

@@ -0,0 +1,45 @@
+"""Default grading prompts for clinical-accuracy judges.
+
+The prompts here are deliberately conservative — they bias toward FAIL when
+the candidate misses an actionable finding, hallucinates one, or would lead
+to a different clinical decision. Override by passing a custom `system_prompt`
+to the judge constructors when a benchmark needs a different rubric.
+
+Keep the prompt deterministic (no timestamps, no UUIDs) so prompt caching on
+the Anthropic side hits a stable prefix.
+"""
+
+from __future__ import annotations
+
+DEFAULT_CLINICAL_GRADING_PROMPT = """You are a board-certified radiologist evaluating an AI model's clinical output against a reference standard.
+
+You will be shown a GOLD report (or label) and a CANDIDATE model output. Decide whether the CANDIDATE correctly captures the clinically actionable content of the GOLD.
+
+Decision rule:
+- Label 1 (PASS) if the CANDIDATE identifies the same primary findings as the GOLD and would lead to the same clinical decision.
+- Label 0 (FAIL) if any of the following are true:
+  * The CANDIDATE misses an actionable finding present in the GOLD (e.g. acute hemorrhage, midline shift, free air, pneumothorax, large vessel occlusion, mass effect, hydrocephalus, status epilepticus pattern).
+  * The CANDIDATE hallucinates a finding not in the GOLD.
+  * The CANDIDATE would lead to a materially different clinical decision than the GOLD.
+- When uncertain between PASS and FAIL, default to FAIL — under-calling is a costlier error than over-calling in this evaluation.
+
+Return your decision as a single JSON object with two fields:
+- `label`: integer 0 or 1
+- `rationale`: 1-2 sentences explaining the decision; cite the specific finding(s) that drove the call
+
+Do not include any text outside the JSON object."""
+
+
+GRADING_USER_TEMPLATE = """GOLD:
+{gold}
+
+CANDIDATE:
+{candidate}
+
+{instructions}"""
+
+
+def build_user_message(gold: str, candidate: str, instructions: str = "") -> str:
+    """Render the user-message body for a grading call."""
+    suffix = f"\nADDITIONAL INSTRUCTIONS:\n{instructions}" if instructions else ""
+    return GRADING_USER_TEMPLATE.format(gold=gold, candidate=candidate, instructions=suffix).strip()

src/medimage_eval/judges/decision.py

@@ -0,0 +1,84 @@
+"""Structured-output schema for judge decisions.
+
+A judge always emits `{label: 0 or 1, rationale: str}`. This schema is used:
+
+* As the JSON-schema constraint sent to the provider API (Anthropic
+  `output_config.format` + OpenAI `response_format`) so the model returns
+  parseable JSON without prefill tricks (prefills 400 on Claude Opus 4.7).
+* As a lenient parser for free-text responses (with `parse_decision_text`)
+  in case a provider returns text outside the schema constraint.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from typing import Any
+
+GRADING_DECISION_SCHEMA: dict[str, Any] = {
+    "type": "object",
+    "properties": {
+        "label": {"type": "integer", "enum": [0, 1]},
+        "rationale": {"type": "string"},
+    },
+    "required": ["label", "rationale"],
+    "additionalProperties": False,
+}
+
+
+class DecisionParseError(ValueError):
+    """Raised when a judge response cannot be parsed into a GradingDecision."""
+
+
+def parse_decision_text(text: str) -> tuple[int, str]:
+    """Parse a judge response into (label, rationale).
+
+    Accepts either a strict JSON object or a JSON object embedded in text
+    (e.g. wrapped in a Markdown code fence). Raises `DecisionParseError`
+    on anything that does not yield `{label: 0|1, rationale: str}`.
+    """
+    if not text or not text.strip():
+        raise DecisionParseError("empty judge response")
+
+    obj = _extract_json_object(text)
+    if "label" not in obj or "rationale" not in obj:
+        raise DecisionParseError(f"missing required fields in judge response: {obj!r}")
+
+    label = obj["label"]
+    if isinstance(label, bool) or not isinstance(label, int) or label not in (0, 1):
+        raise DecisionParseError(f"label must be 0 or 1, got {label!r}")
+
+    rationale = obj["rationale"]
+    if not isinstance(rationale, str):
+        raise DecisionParseError(f"rationale must be a string, got {type(rationale).__name__}")
+
+    return label, rationale
+
+
+_JSON_FENCE_RE = re.compile(r"```(?:json)?\s*(\{.*?\})\s*```", re.DOTALL)
+
+
+def _extract_json_object(text: str) -> dict[str, Any]:
+    """Pull the first JSON object out of `text`. Tolerates fences and surrounding prose."""
+    try:
+        return json.loads(text)
+    except json.JSONDecodeError:
+        pass
+
+    fence_match = _JSON_FENCE_RE.search(text)
+    if fence_match:
+        try:
+            return json.loads(fence_match.group(1))
+        except json.JSONDecodeError as exc:
+            raise DecisionParseError(f"fenced JSON failed to parse: {exc}") from exc
+
+    first_brace = text.find("{")
+    last_brace = text.rfind("}")
+    if first_brace != -1 and last_brace > first_brace:
+        candidate = text[first_brace : last_brace + 1]
+        try:
+            return json.loads(candidate)
+        except json.JSONDecodeError as exc:
+            raise DecisionParseError(f"object slice failed to parse: {exc}") from exc
+
+    raise DecisionParseError(f"no JSON object found in judge response: {text[:120]!r}")