diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 01437ae..feb3813 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,7 +31,7 @@ jobs: - uses: actions/setup-python@v5 with: python-version: "3.12" - - run: pip install -e .[dev] + - run: pip install -e .[dev,judges] - run: pytest -m "not integration" --cov=medimage_eval --cov-fail-under=70 secrets-scan: diff --git a/pyproject.toml b/pyproject.toml index 2ea7e97..eec2722 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,6 +60,7 @@ ignore = ["S101"] [tool.ruff.lint.per-file-ignores] "tests/**/*.py" = ["S", "E501"] +"src/medimage_eval/judges/clinical_prompts.py" = ["E501"] [tool.pytest.ini_options] testpaths = ["tests"] diff --git a/src/medimage_eval/judges/__init__.py b/src/medimage_eval/judges/__init__.py index 9805d34..0dfd99a 100644 --- a/src/medimage_eval/judges/__init__.py +++ b/src/medimage_eval/judges/__init__.py @@ -1,13 +1,31 @@ -"""Judge orchestration: dual-judge clinical accuracy with κ floor.""" +"""Judge orchestration: dual-judge clinical accuracy with κ floor. +The Anthropic and OpenAI adapters are lazily importable — calling them +without the optional SDK installed raises only when constructed, not when +the package is imported. +""" + +from medimage_eval.judges.anthropic_judge import AnthropicJudge, JudgeAPIError from medimage_eval.judges.base import Judge, JudgeQuery, JudgeVerdict +from medimage_eval.judges.decision import ( + GRADING_DECISION_SCHEMA, + DecisionParseError, + parse_decision_text, +) from medimage_eval.judges.dual_judge import DualJudge, DualJudgeItem, DualJudgeResult +from medimage_eval.judges.openai_judge import OpenAIJudge __all__ = [ + "GRADING_DECISION_SCHEMA", + "AnthropicJudge", + "DecisionParseError", "DualJudge", "DualJudgeItem", "DualJudgeResult", "Judge", + "JudgeAPIError", "JudgeQuery", "JudgeVerdict", + "OpenAIJudge", + "parse_decision_text", ] diff --git a/src/medimage_eval/judges/anthropic_judge.py b/src/medimage_eval/judges/anthropic_judge.py new file mode 100644 index 0000000..0bb0e58 --- /dev/null +++ b/src/medimage_eval/judges/anthropic_judge.py @@ -0,0 +1,141 @@ +"""AnthropicJudge — Claude Opus 4.7 wrapper that implements the `Judge` Protocol. + +Design notes (per the claude-api guidance): + +* Default model is `claude-opus-4-7`. The model ID is the bare alias — do not + append a date suffix. +* Opus 4.7 removes `temperature`, `top_p`, `top_k`, and `budget_tokens`. We + pass none of them. Adaptive thinking is off by default on 4.7 (matches + 4.6 behaviour) — we leave it off for grading because the rubric is short + and we want deterministic-ish judgments. +* The system prompt is sent with `cache_control: {type: "ephemeral"}` so it + caches across the many calls a single eval run makes. Below the 4096-token + cache minimum, the marker is silently inert — it costs nothing and starts + paying off once the rubric grows. +* Structured outputs via `output_config.format` (not a prefill — prefills + 400 on Opus 4.7). +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from medimage_eval.judges.base import JudgeQuery, JudgeVerdict +from medimage_eval.judges.clinical_prompts import ( + DEFAULT_CLINICAL_GRADING_PROMPT, + build_user_message, +) +from medimage_eval.judges.decision import GRADING_DECISION_SCHEMA, parse_decision_text + +if TYPE_CHECKING: + import anthropic + +DEFAULT_MODEL_ID = "claude-opus-4-7" +DEFAULT_MAX_TOKENS = 400 +DEFAULT_TIMEOUT_SECONDS = 30.0 + + +class JudgeAPIError(RuntimeError): + """Raised when the underlying provider returned a non-retryable error.""" + + +class AnthropicJudge: + """Adapter from the Anthropic SDK to the `Judge` Protocol. + + Args: + model_id: Anthropic model ID. Defaults to ``"claude-opus-4-7"``. + system_prompt: Override the default clinical-grading rubric. + max_tokens: Output cap. Grading responses are short, so 400 is plenty. + timeout_seconds: Per-request timeout. Fails fast if a judge call hangs. + client: Inject an ``anthropic.Anthropic`` (or test double). When None, + a default client is constructed lazily on first ``grade`` call. + """ + + def __init__( + self, + *, + model_id: str = DEFAULT_MODEL_ID, + system_prompt: str = DEFAULT_CLINICAL_GRADING_PROMPT, + max_tokens: int = DEFAULT_MAX_TOKENS, + timeout_seconds: float = DEFAULT_TIMEOUT_SECONDS, + client: anthropic.Anthropic | None = None, + ) -> None: + if not model_id: + raise ValueError("model_id is required") + self.model_id = model_id + self._system_prompt = system_prompt + self._max_tokens = max_tokens + self._timeout_seconds = timeout_seconds + self._client = client + + def _ensure_client(self) -> anthropic.Anthropic: + if self._client is not None: + return self._client + try: + import anthropic + except ImportError as exc: # pragma: no cover - import failure path + raise RuntimeError( + "anthropic SDK is not installed. Install with `pip install medimage-eval[judges]`." + ) from exc + self._client = anthropic.Anthropic(timeout=self._timeout_seconds) + return self._client + + def grade(self, query: JudgeQuery) -> JudgeVerdict: + client = self._ensure_client() + try: + response = client.messages.create( + model=self.model_id, + max_tokens=self._max_tokens, + system=[ + { + "type": "text", + "text": self._system_prompt, + "cache_control": {"type": "ephemeral"}, + } + ], + messages=[ + { + "role": "user", + "content": build_user_message( + gold=query.gold, + candidate=query.candidate, + instructions=query.instructions, + ), + } + ], + output_config={ + "format": { + "type": "json_schema", + "schema": GRADING_DECISION_SCHEMA, + } + }, + ) + except Exception as exc: + raise JudgeAPIError(f"anthropic judge call failed: {exc}") from exc + + if getattr(response, "stop_reason", None) == "refusal": + raise JudgeAPIError( + f"judge refused to grade prompt_id={query.prompt_id}: " + f"{getattr(response, 'stop_details', None)}" + ) + + text = _extract_first_text_block(response) + label, rationale = parse_decision_text(text) + return JudgeVerdict( + prompt_id=query.prompt_id, + label=label, + rationale=rationale, + raw=text, + ) + + +def _extract_first_text_block(response: Any) -> str: + content = getattr(response, "content", None) + if not content: + raise JudgeAPIError("anthropic response had no content blocks") + for block in content: + if getattr(block, "type", None) == "text": + text = getattr(block, "text", "") + if text: + return text + raise JudgeAPIError("anthropic response had no non-empty text block") diff --git a/src/medimage_eval/judges/clinical_prompts.py b/src/medimage_eval/judges/clinical_prompts.py new file mode 100644 index 0000000..1effc60 --- /dev/null +++ b/src/medimage_eval/judges/clinical_prompts.py @@ -0,0 +1,45 @@ +"""Default grading prompts for clinical-accuracy judges. + +The prompts here are deliberately conservative — they bias toward FAIL when +the candidate misses an actionable finding, hallucinates one, or would lead +to a different clinical decision. Override by passing a custom `system_prompt` +to the judge constructors when a benchmark needs a different rubric. + +Keep the prompt deterministic (no timestamps, no UUIDs) so prompt caching on +the Anthropic side hits a stable prefix. +""" + +from __future__ import annotations + +DEFAULT_CLINICAL_GRADING_PROMPT = """You are a board-certified radiologist evaluating an AI model's clinical output against a reference standard. + +You will be shown a GOLD report (or label) and a CANDIDATE model output. Decide whether the CANDIDATE correctly captures the clinically actionable content of the GOLD. + +Decision rule: +- Label 1 (PASS) if the CANDIDATE identifies the same primary findings as the GOLD and would lead to the same clinical decision. +- Label 0 (FAIL) if any of the following are true: + * The CANDIDATE misses an actionable finding present in the GOLD (e.g. acute hemorrhage, midline shift, free air, pneumothorax, large vessel occlusion, mass effect, hydrocephalus, status epilepticus pattern). + * The CANDIDATE hallucinates a finding not in the GOLD. + * The CANDIDATE would lead to a materially different clinical decision than the GOLD. +- When uncertain between PASS and FAIL, default to FAIL — under-calling is a costlier error than over-calling in this evaluation. + +Return your decision as a single JSON object with two fields: +- `label`: integer 0 or 1 +- `rationale`: 1-2 sentences explaining the decision; cite the specific finding(s) that drove the call + +Do not include any text outside the JSON object.""" + + +GRADING_USER_TEMPLATE = """GOLD: +{gold} + +CANDIDATE: +{candidate} + +{instructions}""" + + +def build_user_message(gold: str, candidate: str, instructions: str = "") -> str: + """Render the user-message body for a grading call.""" + suffix = f"\nADDITIONAL INSTRUCTIONS:\n{instructions}" if instructions else "" + return GRADING_USER_TEMPLATE.format(gold=gold, candidate=candidate, instructions=suffix).strip() diff --git a/src/medimage_eval/judges/decision.py b/src/medimage_eval/judges/decision.py new file mode 100644 index 0000000..2e351b5 --- /dev/null +++ b/src/medimage_eval/judges/decision.py @@ -0,0 +1,84 @@ +"""Structured-output schema for judge decisions. + +A judge always emits `{label: 0 or 1, rationale: str}`. This schema is used: + +* As the JSON-schema constraint sent to the provider API (Anthropic + `output_config.format` + OpenAI `response_format`) so the model returns + parseable JSON without prefill tricks (prefills 400 on Claude Opus 4.7). +* As a lenient parser for free-text responses (with `parse_decision_text`) + in case a provider returns text outside the schema constraint. +""" + +from __future__ import annotations + +import json +import re +from typing import Any + +GRADING_DECISION_SCHEMA: dict[str, Any] = { + "type": "object", + "properties": { + "label": {"type": "integer", "enum": [0, 1]}, + "rationale": {"type": "string"}, + }, + "required": ["label", "rationale"], + "additionalProperties": False, +} + + +class DecisionParseError(ValueError): + """Raised when a judge response cannot be parsed into a GradingDecision.""" + + +def parse_decision_text(text: str) -> tuple[int, str]: + """Parse a judge response into (label, rationale). + + Accepts either a strict JSON object or a JSON object embedded in text + (e.g. wrapped in a Markdown code fence). Raises `DecisionParseError` + on anything that does not yield `{label: 0|1, rationale: str}`. + """ + if not text or not text.strip(): + raise DecisionParseError("empty judge response") + + obj = _extract_json_object(text) + if "label" not in obj or "rationale" not in obj: + raise DecisionParseError(f"missing required fields in judge response: {obj!r}") + + label = obj["label"] + if isinstance(label, bool) or not isinstance(label, int) or label not in (0, 1): + raise DecisionParseError(f"label must be 0 or 1, got {label!r}") + + rationale = obj["rationale"] + if not isinstance(rationale, str): + raise DecisionParseError(f"rationale must be a string, got {type(rationale).__name__}") + + return label, rationale + + +_JSON_FENCE_RE = re.compile(r"```(?:json)?\s*(\{.*?\})\s*```", re.DOTALL) + + +def _extract_json_object(text: str) -> dict[str, Any]: + """Pull the first JSON object out of `text`. Tolerates fences and surrounding prose.""" + try: + return json.loads(text) + except json.JSONDecodeError: + pass + + fence_match = _JSON_FENCE_RE.search(text) + if fence_match: + try: + return json.loads(fence_match.group(1)) + except json.JSONDecodeError as exc: + raise DecisionParseError(f"fenced JSON failed to parse: {exc}") from exc + + first_brace = text.find("{") + last_brace = text.rfind("}") + if first_brace != -1 and last_brace > first_brace: + candidate = text[first_brace : last_brace + 1] + try: + return json.loads(candidate) + except json.JSONDecodeError as exc: + raise DecisionParseError(f"object slice failed to parse: {exc}") from exc + + raise DecisionParseError(f"no JSON object found in judge response: {text[:120]!r}") diff --git a/src/medimage_eval/judges/openai_judge.py b/src/medimage_eval/judges/openai_judge.py new file mode 100644 index 0000000..7ebb49a --- /dev/null +++ b/src/medimage_eval/judges/openai_judge.py @@ -0,0 +1,126 @@ +"""OpenAIJudge — GPT-5.4 wrapper that implements the `Judge` Protocol. + +Notes: + +* GPT-5.4 requires ``max_completion_tokens``, not the legacy ``max_tokens``. + Sending ``max_tokens`` is silently ignored / rejected depending on the model + (see scribegoat2 incident memory). We use the correct name. +* Structured outputs via ``response_format`` with a JSON schema. +* OpenAI applies prompt caching automatically for prefixes ≥ 1024 tokens. + No explicit annotation is needed; the only thing under our control is + keeping the system prompt deterministic. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from medimage_eval.judges.anthropic_judge import JudgeAPIError +from medimage_eval.judges.base import JudgeQuery, JudgeVerdict +from medimage_eval.judges.clinical_prompts import ( + DEFAULT_CLINICAL_GRADING_PROMPT, + build_user_message, +) +from medimage_eval.judges.decision import GRADING_DECISION_SCHEMA, parse_decision_text + +if TYPE_CHECKING: + import openai + +DEFAULT_MODEL_ID = "gpt-5.4" +DEFAULT_MAX_COMPLETION_TOKENS = 400 +DEFAULT_TIMEOUT_SECONDS = 30.0 + + +class OpenAIJudge: + """Adapter from the OpenAI SDK to the `Judge` Protocol. + + Args: + model_id: OpenAI model ID. Defaults to ``"gpt-5.4"``. + system_prompt: Override the default clinical-grading rubric. + max_completion_tokens: Output cap. Note: the parameter name is + ``max_completion_tokens``, not ``max_tokens`` — GPT-5+ rejects + the legacy name. + timeout_seconds: Per-request timeout. + client: Inject an ``openai.OpenAI`` (or test double). When None, + a default client is constructed lazily on first ``grade`` call. + """ + + def __init__( + self, + *, + model_id: str = DEFAULT_MODEL_ID, + system_prompt: str = DEFAULT_CLINICAL_GRADING_PROMPT, + max_completion_tokens: int = DEFAULT_MAX_COMPLETION_TOKENS, + timeout_seconds: float = DEFAULT_TIMEOUT_SECONDS, + client: openai.OpenAI | None = None, + ) -> None: + if not model_id: + raise ValueError("model_id is required") + self.model_id = model_id + self._system_prompt = system_prompt + self._max_completion_tokens = max_completion_tokens + self._timeout_seconds = timeout_seconds + self._client = client + + def _ensure_client(self) -> openai.OpenAI: + if self._client is not None: + return self._client + try: + import openai + except ImportError as exc: # pragma: no cover - import failure path + raise RuntimeError( + "openai SDK is not installed. Install with `pip install medimage-eval[judges]`." + ) from exc + self._client = openai.OpenAI(timeout=self._timeout_seconds) + return self._client + + def grade(self, query: JudgeQuery) -> JudgeVerdict: + client = self._ensure_client() + try: + response = client.chat.completions.create( + model=self.model_id, + max_completion_tokens=self._max_completion_tokens, + messages=[ + {"role": "system", "content": self._system_prompt}, + { + "role": "user", + "content": build_user_message( + gold=query.gold, + candidate=query.candidate, + instructions=query.instructions, + ), + }, + ], + response_format={ + "type": "json_schema", + "json_schema": { + "name": "grading_decision", + "strict": True, + "schema": GRADING_DECISION_SCHEMA, + }, + }, + ) + except Exception as exc: + raise JudgeAPIError(f"openai judge call failed: {exc}") from exc + + text = _extract_first_message_content(response) + label, rationale = parse_decision_text(text) + return JudgeVerdict( + prompt_id=query.prompt_id, + label=label, + rationale=rationale, + raw=text, + ) + + +def _extract_first_message_content(response: Any) -> str: + choices = getattr(response, "choices", None) + if not choices: + raise JudgeAPIError("openai response had no choices") + message = getattr(choices[0], "message", None) + if message is None: + raise JudgeAPIError("openai response choice had no message") + content = getattr(message, "content", None) + if not content: + raise JudgeAPIError("openai response message had empty content") + return content diff --git a/src/medimage_eval/judges/preflight.py b/src/medimage_eval/judges/preflight.py index c80f473..7ca226e 100644 --- a/src/medimage_eval/judges/preflight.py +++ b/src/medimage_eval/judges/preflight.py @@ -1,17 +1,27 @@ -"""Pre-flight check for judge API keys. +"""Pre-flight check for judge API keys + a real canary call to each provider. Silent 401s from a judge provider during a multi-hour run poison every reward -signal in the run — every trajectory comes back labelled 0, the resume cannot -detect it, and the entire run is wasted. This pre-flight makes a tiny, one-token -canary request against each configured provider before any long run starts. +signal: every trajectory comes back labelled 0, the resume cannot detect it, +and the entire run is wasted. This preflight makes a tiny canary request +against each configured provider before any long run starts. Loss of canary +fail-fast is non-negotiable per `feedback_eval_preflight_judge_key.md`. Run via `make preflight` or `python -m medimage_eval.judges.preflight`. + +Exit codes: + 0 All keys present and canaries passed (or `--no-canary`). + 2 At least one canary returned an authentication error. + 3 At least one judge key is missing AND `--strict` was passed. """ from __future__ import annotations +import argparse import os import sys +from collections.abc import Callable +from dataclasses import dataclass +from typing import Any REQUIRED_ENV_VARS = ( ("ANTHROPIC_API_KEY", "claude-opus-4-7"), @@ -19,14 +29,122 @@ ) -def main() -> int: +@dataclass(frozen=True) +class CanaryResult: + provider: str + ok: bool + detail: str + auth_failed: bool = False + + +def check_anthropic_canary( + *, + client: Any | None = None, + model_id: str = "claude-opus-4-7", + timeout_seconds: float = 10.0, +) -> CanaryResult: + """Send a tiny request to Anthropic to verify auth + connectivity. + + Returns a CanaryResult; never raises. The caller decides whether to exit. + """ + if not os.environ.get("ANTHROPIC_API_KEY") and client is None: + return CanaryResult("anthropic", False, "ANTHROPIC_API_KEY not set", auth_failed=False) + + if client is None: + try: + import anthropic + except ImportError as exc: + return CanaryResult( + "anthropic", False, f"anthropic SDK not installed: {exc}", auth_failed=False + ) + client = anthropic.Anthropic(timeout=timeout_seconds) + + try: + client.messages.create( + model=model_id, + max_tokens=10, + messages=[{"role": "user", "content": "ok"}], + ) + return CanaryResult("anthropic", True, "canary OK") + except Exception as exc: + auth_failed = _is_auth_error("anthropic", exc) + return CanaryResult( + "anthropic", + False, + f"{type(exc).__name__}: {exc}", + auth_failed=auth_failed, + ) + + +def check_openai_canary( + *, + client: Any | None = None, + model_id: str = "gpt-5.4", + timeout_seconds: float = 10.0, +) -> CanaryResult: + """Send a tiny request to OpenAI to verify auth + connectivity.""" + if not os.environ.get("OPENAI_API_KEY") and client is None: + return CanaryResult("openai", False, "OPENAI_API_KEY not set", auth_failed=False) + + if client is None: + try: + import openai + except ImportError as exc: + return CanaryResult( + "openai", False, f"openai SDK not installed: {exc}", auth_failed=False + ) + client = openai.OpenAI(timeout=timeout_seconds) + + try: + client.chat.completions.create( + model=model_id, + max_completion_tokens=50, + messages=[{"role": "user", "content": "ok"}], + ) + return CanaryResult("openai", True, "canary OK") + except Exception as exc: + auth_failed = _is_auth_error("openai", exc) + return CanaryResult( + "openai", + False, + f"{type(exc).__name__}: {exc}", + auth_failed=auth_failed, + ) + + +def _is_auth_error(provider: str, exc: BaseException) -> bool: + """Best-effort detection of authentication errors across SDK versions.""" + name = type(exc).__name__ + if name in {"AuthenticationError", "PermissionDeniedError"}: + return True + status = getattr(exc, "status_code", None) or getattr( + getattr(exc, "response", None), "status_code", None + ) + if status in {401, 403}: + return True + msg = str(exc).lower() + return "401" in msg or "unauthor" in msg or "invalid api key" in msg + + +def run_preflight( + *, + strict: bool, + canary: bool, + canaries: dict[str, Callable[[], CanaryResult]] | None = None, +) -> int: + """Run the preflight checklist. Returns an exit code.""" + canaries = canaries or { + "anthropic": check_anthropic_canary, + "openai": check_openai_canary, + } + missing: list[str] = [] for var, label in REQUIRED_ENV_VARS: value = os.environ.get(var, "") if not value: missing.append(f"{var} (used by {label})") continue - # Length-only check — never echo the value itself. + # Length-only — never echo the secret. print(f"OK {var:24s} len={len(value)} (judge: {label})") if missing: @@ -38,13 +156,59 @@ def main() -> int: "\nSource your env (e.g. `set -a && source ~/lostbench/.env && set +a`) and re-run.", file=sys.stderr, ) + if strict: + return 3 + + if not canary: + print("\nCanary checks skipped (--no-canary).") + return 0 + + print("\nRunning canary requests...") + auth_failed = False + any_canary_attempted = False + for provider, runner in canaries.items(): + env_var = f"{provider.upper()}_API_KEY" + if not os.environ.get(env_var): + print(f"SKIP {provider:10s} — {env_var} not set") + continue + any_canary_attempted = True + result = runner() + marker = "OK" if result.ok else ("FAIL-AUTH" if result.auth_failed else "FAIL") + print(f"{marker:9s} {result.provider:10s} {result.detail}") + if result.auth_failed: + auth_failed = True + + if auth_failed: + print( + "\nPreflight FAILED: at least one judge returned an authentication error. " + "Rotate the key or fix the env source, then re-run.", + file=sys.stderr, + ) return 2 - # TODO(v0.2): make a real canary request against each provider. - # Held until the live judge implementations land in PR #2. - print("\nPreflight OK: all judge keys present (canary calls land in v0.2)") + if not any_canary_attempted: + print("\nNo canary attempted (no keys present). Preflight OK with caveat.") + else: + print("\nPreflight OK: judge keys verified by canary.") return 0 +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--no-canary", + dest="canary", + action="store_false", + help="skip the real canary request to each provider (env-var check only)", + ) + parser.add_argument( + "--strict", + action="store_true", + help="exit non-zero when any required key is missing (default: warn only)", + ) + args = parser.parse_args() + return run_preflight(strict=args.strict, canary=args.canary) + + if __name__ == "__main__": raise SystemExit(main()) diff --git a/tests/test_anthropic_judge.py b/tests/test_anthropic_judge.py new file mode 100644 index 0000000..c79b34d --- /dev/null +++ b/tests/test_anthropic_judge.py @@ -0,0 +1,128 @@ +"""AnthropicJudge tests with a stub client. No live API calls.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + +import pytest + +from medimage_eval.judges import AnthropicJudge, JudgeAPIError, JudgeQuery + + +@dataclass +class _Block: + type: str + text: str = "" + + +@dataclass +class _Response: + content: list[_Block] = field(default_factory=list) + stop_reason: str = "end_turn" + + +@dataclass +class _StubMessages: + response: _Response | Exception + calls: list[dict[str, Any]] = field(default_factory=list) + + def create(self, **kwargs: Any) -> _Response: + self.calls.append(kwargs) + if isinstance(self.response, Exception): + raise self.response + return self.response + + +@dataclass +class _StubClient: + messages: _StubMessages + + +def _client_returning(text: str, stop_reason: str = "end_turn") -> _StubClient: + return _StubClient( + _StubMessages(_Response(content=[_Block("text", text)], stop_reason=stop_reason)) + ) + + +def _client_raising(exc: Exception) -> _StubClient: + return _StubClient(_StubMessages(exc)) + + +def test_grade_returns_verdict_for_pass_label(): + stub = _client_returning('{"label": 1, "rationale": "matches gold report"}') + judge = AnthropicJudge(client=stub) + query = JudgeQuery(prompt_id="q0", gold="GOLD TEXT", candidate="CAND TEXT") + verdict = judge.grade(query) + assert verdict.prompt_id == "q0" + assert verdict.label == 1 + assert verdict.rationale == "matches gold report" + assert "matches gold report" in verdict.raw + + +def test_grade_passes_default_model_and_caching(): + stub = _client_returning('{"label": 0, "rationale": "missed bleed"}') + judge = AnthropicJudge(client=stub) + judge.grade(JudgeQuery(prompt_id="q0", gold="g", candidate="c")) + call = stub.messages.calls[0] + assert call["model"] == "claude-opus-4-7" + # Cache control marker on the system prompt — required for prompt caching. + system = call["system"] + assert isinstance(system, list) and len(system) == 1 + assert system[0]["cache_control"] == {"type": "ephemeral"} + # No sampling params — they 400 on Opus 4.7. + assert "temperature" not in call + assert "top_p" not in call + assert "top_k" not in call + # No budget_tokens — also 400 on Opus 4.7. + assert "thinking" not in call or call["thinking"].get("type") != "enabled" + # Structured output via output_config.format, not prefill. + assert call["output_config"]["format"]["type"] == "json_schema" + + +def test_grade_passes_user_payload_with_gold_and_candidate(): + stub = _client_returning('{"label": 1, "rationale": "ok"}') + judge = AnthropicJudge(client=stub) + judge.grade( + JudgeQuery( + prompt_id="q0", gold="GOLD-TOKEN", candidate="CAND-TOKEN", instructions="be strict" + ) + ) + user_content = stub.messages.calls[0]["messages"][0]["content"] + assert "GOLD-TOKEN" in user_content + assert "CAND-TOKEN" in user_content + assert "be strict" in user_content + + +def test_grade_raises_on_refusal(): + stub = _client_returning('{"label": 0, "rationale": "ok"}', stop_reason="refusal") + judge = AnthropicJudge(client=stub) + with pytest.raises(JudgeAPIError, match="refused"): + judge.grade(JudgeQuery(prompt_id="q0", gold="g", candidate="c")) + + +def test_grade_wraps_api_errors(): + stub = _client_raising(RuntimeError("connection reset")) + judge = AnthropicJudge(client=stub) + with pytest.raises(JudgeAPIError, match="anthropic judge call failed"): + judge.grade(JudgeQuery(prompt_id="q0", gold="g", candidate="c")) + + +def test_grade_raises_when_response_has_no_text_block(): + stub = _StubClient(_StubMessages(_Response(content=[], stop_reason="end_turn"))) + judge = AnthropicJudge(client=stub) + with pytest.raises(JudgeAPIError, match="no content"): + judge.grade(JudgeQuery(prompt_id="q0", gold="g", candidate="c")) + + +def test_empty_model_id_rejected(): + with pytest.raises(ValueError, match="model_id"): + AnthropicJudge(model_id="", client=_client_returning("{}")) + + +def test_handles_fenced_json_response(): + stub = _client_returning('```json\n{"label": 0, "rationale": "miss"}\n```') + judge = AnthropicJudge(client=stub) + verdict = judge.grade(JudgeQuery(prompt_id="q0", gold="g", candidate="c")) + assert verdict.label == 0 + assert verdict.rationale == "miss" diff --git a/tests/test_decision.py b/tests/test_decision.py new file mode 100644 index 0000000..8948cff --- /dev/null +++ b/tests/test_decision.py @@ -0,0 +1,58 @@ +"""Tests for the lenient decision parser.""" + +from __future__ import annotations + +import pytest + +from medimage_eval.judges.decision import DecisionParseError, parse_decision_text + + +def test_strict_json_parses(): + label, rationale = parse_decision_text('{"label": 1, "rationale": "good"}') + assert label == 1 + assert rationale == "good" + + +def test_fenced_json_parses(): + text = 'Here is my decision:\n```json\n{"label": 0, "rationale": "missed bleed"}\n```' + label, rationale = parse_decision_text(text) + assert label == 0 + assert rationale == "missed bleed" + + +def test_object_embedded_in_prose_parses(): + text = 'Sure thing. {"label": 1, "rationale": "matches gold"} done.' + label, rationale = parse_decision_text(text) + assert label == 1 + assert rationale == "matches gold" + + +def test_empty_response_rejected(): + with pytest.raises(DecisionParseError, match="empty"): + parse_decision_text("") + + +def test_missing_label_rejected(): + with pytest.raises(DecisionParseError, match="missing required"): + parse_decision_text('{"rationale": "ok"}') + + +def test_label_must_be_zero_or_one(): + with pytest.raises(DecisionParseError, match="label must be 0 or 1"): + parse_decision_text('{"label": 2, "rationale": "x"}') + + +def test_label_true_is_rejected(): + """Bool is a subclass of int in Python; reject it explicitly.""" + with pytest.raises(DecisionParseError, match="label must be 0 or 1"): + parse_decision_text('{"label": true, "rationale": "x"}') + + +def test_rationale_must_be_string(): + with pytest.raises(DecisionParseError, match="rationale must be a string"): + parse_decision_text('{"label": 1, "rationale": 42}') + + +def test_garbage_input_rejected(): + with pytest.raises(DecisionParseError): + parse_decision_text("totally not json at all") diff --git a/tests/test_judges_preflight.py b/tests/test_judges_preflight.py index 5f7a815..16d782c 100644 --- a/tests/test_judges_preflight.py +++ b/tests/test_judges_preflight.py @@ -1,12 +1,18 @@ -"""Tests for the judge-key preflight.""" +"""Tests for the judge preflight (env-var check + canary calls).""" from __future__ import annotations import sys +from dataclasses import dataclass +from medimage_eval.judges.preflight import ( + CanaryResult, + _is_auth_error, + run_preflight, +) -def _reload_preflight(): - """Reload the preflight module so it re-reads os.environ.""" + +def _reload_module(): if "medimage_eval.judges.preflight" in sys.modules: del sys.modules["medimage_eval.judges.preflight"] from medimage_eval.judges import preflight @@ -14,24 +20,111 @@ def _reload_preflight(): return preflight -def test_preflight_passes_when_keys_set(monkeypatch, capsys): +def test_preflight_passes_with_keys_and_no_canary(monkeypatch, capsys): monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-ant-fake-1234567890abcdef") monkeypatch.setenv("OPENAI_API_KEY", "sk-fake-0987654321") - preflight = _reload_preflight() - rc = preflight.main() + rc = run_preflight(strict=False, canary=False) out = capsys.readouterr().out assert rc == 0 - assert "Preflight OK" in out - # Never print the value itself — just confirm length info is shown. assert "ANTHROPIC_API_KEY" in out assert "sk-ant-fake-1234567890abcdef" not in out + assert "skipped" in out.lower() -def test_preflight_fails_when_keys_missing(monkeypatch, capsys): +def test_preflight_warns_without_keys_when_not_strict(monkeypatch, capsys): monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) monkeypatch.delenv("OPENAI_API_KEY", raising=False) - preflight = _reload_preflight() - rc = preflight.main() + rc = run_preflight(strict=False, canary=False) captured = capsys.readouterr() - assert rc == 2 + assert rc == 0 # warning mode: missing keys do not block assert "Missing judge API keys" in captured.err + + +def test_preflight_fails_without_keys_when_strict(monkeypatch, capsys): + monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + rc = run_preflight(strict=True, canary=False) + assert rc == 3 + + +def test_preflight_runs_injected_canaries_and_returns_zero_on_success(monkeypatch, capsys): + monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-ant-fake") + monkeypatch.setenv("OPENAI_API_KEY", "sk-fake") + canaries = { + "anthropic": lambda: CanaryResult("anthropic", True, "canary OK"), + "openai": lambda: CanaryResult("openai", True, "canary OK"), + } + rc = run_preflight(strict=False, canary=True, canaries=canaries) + out = capsys.readouterr().out + assert rc == 0 + assert "judge keys verified" in out + + +def test_preflight_fails_on_auth_error_from_canary(monkeypatch, capsys): + monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-ant-bad") + monkeypatch.setenv("OPENAI_API_KEY", "sk-fake") + canaries = { + "anthropic": lambda: CanaryResult("anthropic", False, "401 Unauthorized", auth_failed=True), + "openai": lambda: CanaryResult("openai", True, "canary OK"), + } + rc = run_preflight(strict=False, canary=True, canaries=canaries) + captured = capsys.readouterr() + assert rc == 2 + assert "FAIL-AUTH" in captured.out + assert "authentication error" in captured.err + + +def test_preflight_skips_canary_for_provider_without_key(monkeypatch, capsys): + """If only one key is set, only that provider's canary runs.""" + monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-ant-fake") + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + seen: list[str] = [] + + def anthropic_canary(): + seen.append("anthropic") + return CanaryResult("anthropic", True, "canary OK") + + def openai_canary(): + seen.append("openai") + return CanaryResult("openai", True, "canary OK") + + canaries = {"anthropic": anthropic_canary, "openai": openai_canary} + rc = run_preflight(strict=False, canary=True, canaries=canaries) + out = capsys.readouterr().out + assert rc == 0 + assert seen == ["anthropic"] + assert "SKIP" in out and "openai" in out + + +# ---- _is_auth_error detection ------------------------------------------------ + + +@dataclass +class _FakeAuthError(Exception): + """Stand-in for SDK-specific AuthenticationError types.""" + + +_FakeAuthError.__name__ = "AuthenticationError" + + +@dataclass +class _FakeStatusError(Exception): + status_code: int = 401 + + +def test_is_auth_error_detects_by_class_name(): + assert _is_auth_error("anthropic", _FakeAuthError()) is True + + +def test_is_auth_error_detects_by_status_code(): + err = _FakeStatusError(status_code=401) + assert _is_auth_error("openai", err) is True + + +def test_is_auth_error_detects_by_message_substring(): + err = RuntimeError("server returned 401 Unauthorized") + assert _is_auth_error("openai", err) is True + + +def test_is_auth_error_returns_false_for_unrelated_exception(): + assert _is_auth_error("anthropic", ValueError("connection reset")) is False diff --git a/tests/test_openai_judge.py b/tests/test_openai_judge.py new file mode 100644 index 0000000..49acc0d --- /dev/null +++ b/tests/test_openai_judge.py @@ -0,0 +1,107 @@ +"""OpenAIJudge tests with a stub client. No live API calls.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + +import pytest + +from medimage_eval.judges import JudgeAPIError, JudgeQuery, OpenAIJudge + + +@dataclass +class _Message: + content: str + + +@dataclass +class _Choice: + message: _Message + + +@dataclass +class _Response: + choices: list[_Choice] + + +@dataclass +class _StubCompletions: + response: _Response | Exception + calls: list[dict[str, Any]] = field(default_factory=list) + + def create(self, **kwargs: Any) -> _Response: + self.calls.append(kwargs) + if isinstance(self.response, Exception): + raise self.response + return self.response + + +@dataclass +class _StubChat: + completions: _StubCompletions + + +@dataclass +class _StubClient: + chat: _StubChat + + +def _client_returning(text: str) -> _StubClient: + return _StubClient(_StubChat(_StubCompletions(_Response([_Choice(_Message(text))])))) + + +def _client_raising(exc: Exception) -> _StubClient: + return _StubClient(_StubChat(_StubCompletions(exc))) + + +def test_grade_returns_verdict_for_pass_label(): + stub = _client_returning('{"label": 1, "rationale": "matches"}') + judge = OpenAIJudge(client=stub) + verdict = judge.grade(JudgeQuery(prompt_id="q0", gold="g", candidate="c")) + assert verdict.prompt_id == "q0" + assert verdict.label == 1 + assert verdict.rationale == "matches" + + +def test_grade_uses_max_completion_tokens_not_max_tokens(): + """GPT-5+ requires max_completion_tokens. max_tokens is the legacy name.""" + stub = _client_returning('{"label": 0, "rationale": "x"}') + judge = OpenAIJudge(client=stub) + judge.grade(JudgeQuery(prompt_id="q0", gold="g", candidate="c")) + call = stub.chat.completions.calls[0] + assert "max_completion_tokens" in call + assert "max_tokens" not in call + + +def test_grade_uses_structured_response_format(): + stub = _client_returning('{"label": 1, "rationale": "x"}') + judge = OpenAIJudge(client=stub) + judge.grade(JudgeQuery(prompt_id="q0", gold="g", candidate="c")) + call = stub.chat.completions.calls[0] + assert call["model"] == "gpt-5.4" + rf = call["response_format"] + assert rf["type"] == "json_schema" + assert rf["json_schema"]["strict"] is True + assert rf["json_schema"]["name"] == "grading_decision" + + +def test_grade_wraps_api_errors(): + stub = _client_raising(RuntimeError("rate limit")) + judge = OpenAIJudge(client=stub) + with pytest.raises(JudgeAPIError, match="openai judge call failed"): + judge.grade(JudgeQuery(prompt_id="q0", gold="g", candidate="c")) + + +def test_grade_rejects_empty_choices(): + stub = _StubClient(_StubChat(_StubCompletions(_Response(choices=[])))) + judge = OpenAIJudge(client=stub) + with pytest.raises(JudgeAPIError, match="no choices"): + judge.grade(JudgeQuery(prompt_id="q0", gold="g", candidate="c")) + + +def test_grade_rejects_empty_message_content(): + stub = _client_returning("") + judge = OpenAIJudge(client=stub) + with pytest.raises(JudgeAPIError, match="empty content"): + judge.grade(JudgeQuery(prompt_id="q0", gold="g", candidate="c"))