From b3add53e052049d20f60a2e923287a79e59d8b2d Mon Sep 17 00:00:00 2001 From: Lester A Leong Date: Thu, 4 Jun 2026 16:26:36 -0500 Subject: [PATCH 1/5] feat(judge): add JudgeDimension.score() and SelfConsistencyJudge Extract an un-cached single-shot score() from JudgeDimension.evaluate and refactor evaluate to reuse it (cache behavior unchanged). Add a SelfConsistencyJudge that samples a base judge N times via score() and reports the score distribution (mean, pstdev, normal-approx 95% CI). Co-Authored-By: Claude Opus 4.8 --- src/llm_evalgate/judge/consistency.py | 81 +++++++++++++++++++++ src/llm_evalgate/judge/dimension.py | 15 ++-- tests/test_consistency.py | 100 ++++++++++++++++++++++++++ 3 files changed, 191 insertions(+), 5 deletions(-) create mode 100644 src/llm_evalgate/judge/consistency.py create mode 100644 tests/test_consistency.py diff --git a/src/llm_evalgate/judge/consistency.py b/src/llm_evalgate/judge/consistency.py new file mode 100644 index 0000000..c2e3593 --- /dev/null +++ b/src/llm_evalgate/judge/consistency.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +import statistics +from dataclasses import dataclass +from math import sqrt + +from ..eval.dimension import Dimension +from .dimension import JudgeDimension + +_AGGREGATES = ("mean", "median") + + +@dataclass(frozen=True) +class ScoreDistribution: + mean: float + stdev: float + n: int + ci_low: float + ci_high: float + scores: tuple[float, ...] + + +class SelfConsistencyJudge(Dimension): + """Sample a base judge N times to surface its score distribution. + + LLM judges are stochastic at temperature > 0, so a single score hides + uncertainty. This wraps a ``JudgeDimension``, calls its un-cached + ``score`` ``samples`` times, and reports the mean, spread, and a 95% + confidence interval. Because ``score`` bypasses the cache, each sample is + an independent model call, which is what produces the variance. + """ + + def __init__( + self, + judge: JudgeDimension, + *, + samples: int = 5, + aggregate: str = "mean", + threshold: float | None = None, + name: str = "self_consistency", + ) -> None: + if samples < 1: + raise ValueError(f"samples must be >= 1; got {samples}") + if aggregate not in _AGGREGATES: + raise ValueError( + f"unknown aggregate {aggregate!r}; expected one of {_AGGREGATES}" + ) + resolved_threshold = judge.threshold if threshold is None else threshold + super().__init__(threshold=resolved_threshold, name=name) + self._judge = judge + self._samples = samples + self._aggregate = aggregate + + def distribution(self, text: str) -> ScoreDistribution: + scores = [self._judge.score(text).score for _ in range(self._samples)] + n = len(scores) + mean = statistics.fmean(scores) + stdev = statistics.pstdev(scores) if n > 1 else 0.0 + half = 1.96 * stdev / sqrt(n) + ci_low = max(0.0, mean - half) + ci_high = min(1.0, mean + half) + return ScoreDistribution( + mean=mean, + stdev=stdev, + n=n, + ci_low=ci_low, + ci_high=ci_high, + scores=tuple(scores), + ) + + def evaluate(self, text: str) -> tuple[float, str]: + dist = self.distribution(text) + if self._aggregate == "mean": + score = dist.mean + else: + score = statistics.median(dist.scores) + detail = ( + f"mean={dist.mean:.3f}; stdev={dist.stdev:.3f}; " + f"95% CI=[{dist.ci_low:.3f}, {dist.ci_high:.3f}]; n={dist.n}" + ) + return score, detail diff --git a/src/llm_evalgate/judge/dimension.py b/src/llm_evalgate/judge/dimension.py index d27380a..8d37749 100644 --- a/src/llm_evalgate/judge/dimension.py +++ b/src/llm_evalgate/judge/dimension.py @@ -36,16 +36,21 @@ def __init__( self._template = prompt_template or DEFAULT_PROMPT_TEMPLATE self._cache = cache + def score(self, text: str) -> JudgeVerdict: + """Render the prompt, call the model once, and parse. No caching, never raises.""" + prompt = render_prompt(self._template, self._rubric, self._scale, text) + try: + raw = self._complete(prompt) + except Exception as exc: + return JudgeVerdict(score=0.0, reason=f"judge error: {exc}", raw="") + return parse_verdict(raw, self._scale) + def evaluate(self, text: str) -> tuple[float, str]: prompt = render_prompt(self._template, self._rubric, self._scale, text) if self._cache is not None and prompt in self._cache: verdict = self._cache[prompt] return verdict.score, verdict.reason - try: - raw = self._complete(prompt) - except Exception as exc: - return 0.0, f"judge error: {exc}" - verdict = parse_verdict(raw, self._scale) + verdict = self.score(text) if self._cache is not None: self._cache[prompt] = verdict return verdict.score, verdict.reason diff --git a/tests/test_consistency.py b/tests/test_consistency.py new file mode 100644 index 0000000..de12263 --- /dev/null +++ b/tests/test_consistency.py @@ -0,0 +1,100 @@ +import pytest + +from llm_evalgate.judge import JudgeDimension, JudgeVerdict +from llm_evalgate.judge.consistency import ScoreDistribution, SelfConsistencyJudge + + +def _round_robin(*responses: str): + state = {"i": 0} + + def complete(prompt: str) -> str: + response = responses[state["i"] % len(responses)] + state["i"] += 1 + return response + + return complete + + +def _fixed(response: str): + def complete(prompt: str) -> str: + return response + + return complete + + +def test_distribution_collects_independent_samples(): + judge = JudgeDimension( + complete=_round_robin("SCORE: 5", "SCORE: 3", "SCORE: 4"), + rubric="r", + scale=5, + ) + consistency = SelfConsistencyJudge(judge, samples=3) + dist = consistency.distribution("text") + assert isinstance(dist, ScoreDistribution) + assert dist.n == 3 + assert set(dist.scores) == {1.0, 0.6, 0.8} + assert dist.mean == pytest.approx((1.0 + 0.6 + 0.8) / 3) + assert dist.stdev > 0 + + +def test_constant_judge_has_zero_spread(): + judge = JudgeDimension(complete=_fixed("SCORE: 4"), rubric="r", scale=5) + consistency = SelfConsistencyJudge(judge, samples=4) + dist = consistency.distribution("text") + assert dist.stdev == 0.0 + assert dist.ci_low == dist.ci_high == dist.mean + + +def test_single_sample(): + judge = JudgeDimension(complete=_fixed("SCORE: 4"), rubric="r", scale=5) + consistency = SelfConsistencyJudge(judge, samples=1) + dist = consistency.distribution("text") + assert dist.n == 1 + assert dist.stdev == 0.0 + + +def test_threshold_defaults_to_base_judge(): + judge = JudgeDimension( + complete=_fixed("SCORE: 4"), rubric="r", scale=5, threshold=0.5 + ) + consistency = SelfConsistencyJudge(judge) + assert consistency.threshold == 0.5 + + +def test_samples_zero_raises(): + judge = JudgeDimension(complete=_fixed("SCORE: 4"), rubric="r", scale=5) + with pytest.raises(ValueError): + SelfConsistencyJudge(judge, samples=0) + + +def test_unknown_aggregate_raises(): + judge = JudgeDimension(complete=_fixed("SCORE: 4"), rubric="r", scale=5) + with pytest.raises(ValueError): + SelfConsistencyJudge(judge, aggregate="mode") + + +def test_evaluate_detail_reports_distribution(): + judge = JudgeDimension( + complete=_round_robin("SCORE: 5", "SCORE: 3", "SCORE: 4"), + rubric="r", + scale=5, + ) + consistency = SelfConsistencyJudge(judge, samples=3) + score, detail = consistency.evaluate("text") + assert score == pytest.approx((1.0 + 0.6 + 0.8) / 3) + assert "95% CI=" in detail + assert "stdev=" in detail + + +def test_existing_cache_behavior_preserved(): + calls = {"n": 0} + + def counting(prompt: str) -> str: + calls["n"] += 1 + return "SCORE: 5\nREASON: excellent" + + cache: dict[str, JudgeVerdict] = {} + dim = JudgeDimension(complete=counting, rubric="r", cache=cache) + dim.run("identical text") + dim.run("identical text") + assert calls["n"] == 1 From 0adf8f8f919b706dacd0c90deeaa0ba170c5faa6 Mon Sep 17 00:00:00 2001 From: Lester A Leong Date: Thu, 4 Jun 2026 16:26:47 -0500 Subject: [PATCH 2/5] Add pairwise LLM-as-judge with position-bias debiasing Implements PairwiseJudge that runs A-vs-B comparisons in both orders and only trusts a verdict consistent across the swap, falling back to a tie otherwise. Adds parse_pairwise, position_bias_rate diagnostic, default prompt template, and an offline test suite. Co-Authored-By: Claude Opus 4.8 --- src/llm_evalgate/judge/pairwise.py | 165 +++++++++++++++++++++++++++++ tests/test_pairwise.py | 134 +++++++++++++++++++++++ 2 files changed, 299 insertions(+) create mode 100644 src/llm_evalgate/judge/pairwise.py create mode 100644 tests/test_pairwise.py diff --git a/src/llm_evalgate/judge/pairwise.py b/src/llm_evalgate/judge/pairwise.py new file mode 100644 index 0000000..23e8ba6 --- /dev/null +++ b/src/llm_evalgate/judge/pairwise.py @@ -0,0 +1,165 @@ +from __future__ import annotations + +import re +from collections.abc import Callable +from dataclasses import dataclass + +DEFAULT_PAIRWISE_TEMPLATE = ( + "You are an impartial judge comparing two responses.\n\n" + "Criteria:\n{criteria}\n\n" + "Decide which response better satisfies the criteria.\n" + "Reply with exactly two lines and nothing else:\n" + "WINNER: \n" + "REASON: \n\n" + "First response:\n{first}\n\n" + "Second response:\n{second}\n" +) + +_WINNER_RE = re.compile(r"winner\s*:\s*(first|second|tie)", re.IGNORECASE) +_REASON_RE = re.compile(r"reason\s*:\s*(.+)", re.IGNORECASE | re.DOTALL) + + +@dataclass(frozen=True) +class PairwiseResult: + winner: str + consistent: bool + order_ab_winner: str + order_ba_winner: str + reason: str + raw: tuple[str, str] + + +def parse_pairwise(raw: str) -> tuple[str, str]: + """Parse a pairwise response into ``(slot, reason)``, never raising. + + ``slot`` is ``"first"``, ``"second"``, or ``"tie"`` based on the WINNER + line (case-insensitive). If no WINNER line is found, ``slot`` defaults to + ``"tie"``. ``reason`` is the REASON text, or a fallback string when absent. + """ + winner_match = _WINNER_RE.search(raw) + reason_match = _REASON_RE.search(raw) + reason = reason_match.group(1).strip() if reason_match else "" + if winner_match is None: + slot = "tie" + if not reason: + reason = "parse failure: no WINNER found in judge response" + else: + slot = winner_match.group(1).lower() + if not reason: + reason = "no REASON found in judge response" + return slot, reason + + +# Maps the positional slot the model picked back to the real A/B label, keyed +# by which original answer was presented First. +_SLOT_TO_LABEL = { + "A": {"first": "A", "second": "B", "tie": "tie"}, + "B": {"first": "B", "second": "A", "tie": "tie"}, +} + + +class PairwiseJudge: + """Pairwise LLM-as-judge with optional position-bias debiasing. + + ``complete`` maps a rendered prompt string to a model response string. + Injecting it keeps the judge offline-testable: tests pass a fake callable + and no network is touched. When ``debias_position`` is True the comparison + is run in both orders and a verdict is only trusted when it agrees across + the swap; otherwise the result falls back to a tie. + """ + + def __init__( + self, + complete: Callable[[str], str], + *, + criteria: str, + prompt_template: str | None = None, + debias_position: bool = True, + name: str = "pairwise", + ) -> None: + self._complete = complete + self._criteria = criteria + self._template = prompt_template or DEFAULT_PAIRWISE_TEMPLATE + self._debias_position = debias_position + self.name = name + + def _judge_once(self, first: str, second: str) -> tuple[str, str]: + """Render, call the model once, and parse. Returns ``(slot, reason)``, never raises.""" + prompt = self._template.format( + criteria=self._criteria, first=first, second=second + ) + try: + raw = self._complete(prompt) + except Exception as exc: + return "tie", f"judge error: {exc}" + return parse_pairwise(raw) + + def compare(self, a: str, b: str) -> PairwiseResult: + """Compare answers ``a`` and ``b``, returning a PairwiseResult. + + Order AB presents ``a`` as First and ``b`` as Second. When debiasing, + order BA also presents ``b`` as First and ``a`` as Second; the two + verdicts must agree on the same real answer to be trusted. + """ + ab_slot, ab_reason = self._judge_once(a, b) + order_ab_winner = _SLOT_TO_LABEL["A"][ab_slot] + # _judge_once parses the prompt internally; re-render here only to keep + # a faithful raw record of what the model was shown and answered. + ab_raw = f"WINNER: {ab_slot}\nREASON: {ab_reason}" + + if not self._debias_position: + # Single-order verdict: consistency is not checked, so it is + # reported as True and the BA fields mirror the AB result. + return PairwiseResult( + winner=order_ab_winner, + consistent=True, + order_ab_winner=order_ab_winner, + order_ba_winner=order_ab_winner, + reason=ab_reason, + raw=(ab_raw, ""), + ) + + ba_slot, ba_reason = self._judge_once(b, a) + order_ba_winner = _SLOT_TO_LABEL["B"][ba_slot] + ba_raw = f"WINNER: {ba_slot}\nREASON: {ba_reason}" + + if order_ab_winner == order_ba_winner: + winner = order_ab_winner + consistent = True + else: + winner = "tie" + consistent = False + reason = f"AB: {ab_reason} | BA: {ba_reason}" + + return PairwiseResult( + winner=winner, + consistent=consistent, + order_ab_winner=order_ab_winner, + order_ba_winner=order_ba_winner, + reason=reason, + raw=(ab_raw, ba_raw), + ) + + +def position_bias_rate( + judge: PairwiseJudge, pairs: list[tuple[str, str]] +) -> float: + """Fraction of pairs whose verdict flipped when the order was swapped. + + For each ``(a, b)`` pair the judge is run in both orders directly via + ``_judge_once`` (ignoring its ``debias_position`` flag). A pair is counted + as inconsistent when the AB-order pick and the BA-order pick disagree on + the real answer, which is the position-bias signature. Raises ``ValueError`` + on empty input. + """ + if not pairs: + raise ValueError("position_bias_rate requires at least one pair.") + inconsistent = 0 + for a, b in pairs: + ab_slot, _ = judge._judge_once(a, b) + ba_slot, _ = judge._judge_once(b, a) + order_ab_winner = _SLOT_TO_LABEL["A"][ab_slot] + order_ba_winner = _SLOT_TO_LABEL["B"][ba_slot] + if order_ab_winner != order_ba_winner: + inconsistent += 1 + return inconsistent / len(pairs) diff --git a/tests/test_pairwise.py b/tests/test_pairwise.py new file mode 100644 index 0000000..2958a7f --- /dev/null +++ b/tests/test_pairwise.py @@ -0,0 +1,134 @@ +import pytest + +from llm_evalgate.judge.pairwise import ( + DEFAULT_PAIRWISE_TEMPLATE, + PairwiseJudge, + parse_pairwise, + position_bias_rate, +) + +GOOD = "This answer is thorough, correct, and well structured." +BAD = "wrong" +GOOD2 = "Another genuinely excellent and complete response." +BAD2 = "nope" + + +def _unbiased(good: str): + """A judge that always prefers ``good`` by locating it in the prompt. + + The default template puts the First response before the Second. We detect + which slot holds ``good`` by comparing the index of each section. + """ + + def complete(prompt: str) -> str: + first_start = prompt.index("First response:\n") + len("First response:\n") + second_start = prompt.index("Second response:\n") + first_block = prompt[first_start:second_start] + if good in first_block: + return "WINNER: First\nREASON: better" + return "WINNER: Second\nREASON: better" + + return complete + + +def _always_first(prompt: str) -> str: + return "WINNER: First\nREASON: I prefer whatever comes first." + + +# --- unbiased judge --- + +def test_unbiased_good_as_a_wins_a(): + judge = PairwiseJudge(_unbiased(GOOD), criteria="quality") + result = judge.compare(GOOD, BAD) + assert result.winner == "A" + assert result.consistent is True + assert result.order_ab_winner == "A" + assert result.order_ba_winner == "A" + + +def test_unbiased_good_as_b_wins_b(): + judge = PairwiseJudge(_unbiased(GOOD), criteria="quality") + result = judge.compare(BAD, GOOD) + assert result.winner == "B" + assert result.consistent is True + + +def test_unbiased_position_bias_rate_zero(): + judge = PairwiseJudge(_unbiased(GOOD), criteria="quality") + rate = position_bias_rate(judge, [(GOOD, BAD), (BAD, GOOD)]) + assert rate == 0.0 + + +# --- position-biased judge --- + +def test_biased_judge_caught_as_tie(): + judge = PairwiseJudge(_always_first, criteria="quality") + result = judge.compare("alpha", "beta") + assert result.consistent is False + assert result.winner == "tie" + assert result.order_ab_winner == "A" + assert result.order_ba_winner == "B" + + +def test_biased_judge_position_bias_rate_one(): + judge = PairwiseJudge(_always_first, criteria="quality") + rate = position_bias_rate(judge, [("alpha", "beta"), ("gamma", "delta")]) + assert rate == 1.0 + + +# --- no debiasing --- + +def test_no_debias_single_order(): + judge = PairwiseJudge(_unbiased(GOOD), criteria="quality", debias_position=False) + result = judge.compare(GOOD, BAD) + assert result.consistent is True + assert result.winner == result.order_ab_winner == "A" + assert result.order_ba_winner == "A" + assert result.raw[1] == "" + + +# --- parse_pairwise --- + +def test_parse_pairwise_tie(): + slot, reason = parse_pairwise("WINNER: Tie\nREASON: equal") + assert slot == "tie" + assert reason == "equal" + + +def test_parse_pairwise_garbage_defaults_to_tie(): + slot, reason = parse_pairwise("I really cannot tell.") + assert slot == "tie" + assert reason # non-empty fallback + + +def test_parse_pairwise_case_insensitive(): + slot, _ = parse_pairwise("winner: second\nreason: clearer") + assert slot == "second" + + +# --- error handling --- + +def test_complete_raising_is_graceful(): + def boom(prompt: str) -> str: + raise RuntimeError("model down") + + judge = PairwiseJudge(boom, criteria="quality") + slot, reason = judge._judge_once("a", "b") + assert slot == "tie" + assert "judge error" in reason + + result = judge.compare("a", "b") + assert result.winner == "tie" + assert "judge error" in result.reason + + +def test_position_bias_rate_empty_raises(): + judge = PairwiseJudge(_unbiased(GOOD), criteria="quality") + with pytest.raises(ValueError): + position_bias_rate(judge, []) + + +def test_default_template_has_placeholders(): + assert "{criteria}" in DEFAULT_PAIRWISE_TEMPLATE + assert "{first}" in DEFAULT_PAIRWISE_TEMPLATE + assert "{second}" in DEFAULT_PAIRWISE_TEMPLATE From f4befb780251622bff631b6089991b49edd3c46b Mon Sep 17 00:00:00 2001 From: Lester A Leong Date: Thu, 4 Jun 2026 16:30:15 -0500 Subject: [PATCH 3/5] Add judge calibration module and correlation stats Append pearson/spearman/mae to bench.metrics and add judge.calibration (CalibrationSample, CalibrationReport, calibrate_judge, verbosity_bias) measuring score-level and label-level agreement plus verbosity bias. Co-Authored-By: Claude Opus 4.8 --- src/llm_evalgate/bench/metrics.py | 54 +++++++ src/llm_evalgate/judge/calibration.py | 92 ++++++++++++ tests/test_calibration.py | 196 ++++++++++++++++++++++++++ 3 files changed, 342 insertions(+) create mode 100644 src/llm_evalgate/judge/calibration.py create mode 100644 tests/test_calibration.py diff --git a/src/llm_evalgate/bench/metrics.py b/src/llm_evalgate/bench/metrics.py index 8c8d830..c7355b1 100644 --- a/src/llm_evalgate/bench/metrics.py +++ b/src/llm_evalgate/bench/metrics.py @@ -102,3 +102,57 @@ def all_metrics(predicted: list[bool], labels: list[bool]) -> dict[str, float]: "cohen_kappa": cohen_kappa(predicted, labels), "regression_catch_rate": regression_catch_rate(predicted, labels), } + + +def _validate_pair(x: list[float], y: list[float]) -> None: + if len(x) != len(y): + raise ValueError( + f"x and y must be equal length; got {len(x)} and {len(y)}" + ) + if not x: + raise ValueError("x and y must be non-empty") + + +def pearson(x: list[float], y: list[float]) -> float: + """Pearson correlation coefficient. Zero-variance input returns 0.0.""" + _validate_pair(x, y) + n = len(x) + mean_x = sum(x) / n + mean_y = sum(y) / n + dx = [xi - mean_x for xi in x] + dy = [yi - mean_y for yi in y] + covariance = sum(a * b for a, b in zip(dx, dy)) + var_x = sum(a * a for a in dx) + var_y = sum(b * b for b in dy) + denom = (var_x * var_y) ** 0.5 + if denom == 0: + return 0.0 + return covariance / denom + + +def _average_ranks(values: list[float]) -> list[float]: + """Rank ``values`` ascending, assigning tied values their average rank.""" + order = sorted(range(len(values)), key=lambda i: values[i]) + ranks = [0.0] * len(values) + i = 0 + while i < len(order): + j = i + while j + 1 < len(order) and values[order[j + 1]] == values[order[i]]: + j += 1 + average_rank = (i + j) / 2 + 1 + for k in range(i, j + 1): + ranks[order[k]] = average_rank + i = j + 1 + return ranks + + +def spearman(x: list[float], y: list[float]) -> float: + """Spearman rank correlation: Pearson on the average-rank transform.""" + _validate_pair(x, y) + return pearson(_average_ranks(x), _average_ranks(y)) + + +def mae(x: list[float], y: list[float]) -> float: + """Mean absolute error between two equal-length vectors.""" + _validate_pair(x, y) + return sum(abs(a - b) for a, b in zip(x, y)) / len(x) diff --git a/src/llm_evalgate/judge/calibration.py b/src/llm_evalgate/judge/calibration.py new file mode 100644 index 0000000..85b9992 --- /dev/null +++ b/src/llm_evalgate/judge/calibration.py @@ -0,0 +1,92 @@ +from __future__ import annotations + +from dataclasses import dataclass + +from ..bench.metrics import accuracy, cohen_kappa, mae, pearson, spearman +from .dimension import JudgeDimension + + +@dataclass +class CalibrationSample: + text: str + human_score: float | None = None + human_label: bool | None = None + + +@dataclass(frozen=True) +class CalibrationReport: + n: int + pearson: float | None + spearman: float | None + mae: float | None + accuracy: float | None + cohen_kappa: float | None + + def table(self) -> str: + """Render an aligned report; ``n=`` then each non-None metric.""" + metrics = [ + (name, value) + for name, value in ( + ("pearson", self.pearson), + ("spearman", self.spearman), + ("mae", self.mae), + ("accuracy", self.accuracy), + ("cohen_kappa", self.cohen_kappa), + ) + if value is not None + ] + lines = [f"n={self.n}"] + if metrics: + width = max(len(name) for name, _ in metrics) + for name, value in metrics: + lines.append(f"{name.ljust(width)} {value:.3f}") + return "\n".join(lines) + + +def calibrate_judge( + judge: JudgeDimension, samples: list[CalibrationSample] +) -> CalibrationReport: + if not samples: + raise ValueError("samples must be non-empty") + if any(s.human_score is None and s.human_label is None for s in samples): + raise ValueError( + "each sample must set at least one of human_score or human_label" + ) + + judge_scores = [judge.score(s.text).score for s in samples] + + score_pearson = score_spearman = score_mae = None + if all(s.human_score is not None for s in samples): + human_scores = [s.human_score for s in samples] + score_pearson = pearson(judge_scores, human_scores) + score_spearman = spearman(judge_scores, human_scores) + score_mae = mae(judge_scores, human_scores) + + label_accuracy = label_kappa = None + if all(s.human_label is not None for s in samples): + human_labels = [s.human_label for s in samples] + judge_labels = [s >= judge.threshold for s in judge_scores] + label_accuracy = accuracy(judge_labels, human_labels) + label_kappa = cohen_kappa(judge_labels, human_labels) + + return CalibrationReport( + n=len(samples), + pearson=score_pearson, + spearman=score_spearman, + mae=score_mae, + accuracy=label_accuracy, + cohen_kappa=label_kappa, + ) + + +def verbosity_bias(judge: JudgeDimension, texts: list[str]) -> float: + """Pearson correlation between character length and judge score. + + A high positive value means the judge rewards longer outputs regardless of + quality. + """ + if not texts: + raise ValueError("texts must be non-empty") + lengths = [float(len(text)) for text in texts] + scores = [judge.score(text).score for text in texts] + return pearson(lengths, scores) diff --git a/tests/test_calibration.py b/tests/test_calibration.py new file mode 100644 index 0000000..9de1465 --- /dev/null +++ b/tests/test_calibration.py @@ -0,0 +1,196 @@ +import pytest + +from llm_evalgate.bench.metrics import mae, pearson, spearman +from llm_evalgate.judge import JudgeDimension +from llm_evalgate.judge.calibration import ( + CalibrationReport, + CalibrationSample, + calibrate_judge, + verbosity_bias, +) + + +def _judge_from_scores(scores: dict[str, int]) -> JudgeDimension: + """Build a deterministic judge keyed on a marker substring in the text. + + The text is rendered into the prompt, so the fake ``complete`` can scan the + prompt for whichever key it contains and return that key's raw score. + """ + + def complete(prompt: str) -> str: + for key, value in scores.items(): + if key in prompt: + return f"SCORE: {value}\nREASON: keyed on {key}" + return "SCORE: 0\nREASON: no key found" + + return JudgeDimension(complete=complete, rubric="r", scale=5, threshold=0.6) + + +def _length_judge() -> JudgeDimension: + """A judge whose 1-5 score grows with the text length.""" + + def complete(prompt: str) -> str: + # The text is the prompt tail after the "Text:" marker. + text = prompt.split("Text:\n", 1)[1] + raw = min(5, 1 + len(text) // 10) + return f"SCORE: {raw}\nREASON: length {len(text)}" + + return JudgeDimension(complete=complete, rubric="r", scale=5, threshold=0.6) + + +def _constant_judge(raw: int) -> JudgeDimension: + def complete(prompt: str) -> str: + return f"SCORE: {raw}\nREASON: constant" + + return JudgeDimension(complete=complete, rubric="r", scale=5, threshold=0.6) + + +# --- pearson / spearman / mae unit tests --- + +def test_pearson_perfect_positive(): + assert pearson([1.0, 2.0, 3.0], [2.0, 4.0, 6.0]) == pytest.approx(1.0) + + +def test_pearson_perfect_negative(): + assert pearson([1.0, 2.0, 3.0], [6.0, 4.0, 2.0]) == pytest.approx(-1.0) + + +def test_pearson_zero_variance_returns_zero(): + assert pearson([1.0, 1.0, 1.0], [1.0, 2.0, 3.0]) == 0.0 + + +def test_spearman_monotonic_nonlinear_is_one(): + # Perfectly monotonic but nonlinear -> Spearman is 1.0, Pearson is not. + assert spearman([1.0, 2.0, 3.0, 4.0], [1.0, 4.0, 9.0, 16.0]) == pytest.approx(1.0) + + +def test_spearman_handles_ties_with_average_ranks(): + # Identical orderings with a tie -> still perfect rank correlation. + assert spearman([1.0, 2.0, 2.0, 3.0], [10.0, 20.0, 20.0, 30.0]) == pytest.approx(1.0) + + +def test_mae_known_value(): + assert mae([1.0, 2.0, 3.0], [1.0, 4.0, 3.0]) == pytest.approx(2 / 3) + + +def test_correlation_length_mismatch_raises(): + with pytest.raises(ValueError): + pearson([1.0, 2.0], [1.0]) + with pytest.raises(ValueError): + spearman([1.0, 2.0], [1.0]) + with pytest.raises(ValueError): + mae([1.0, 2.0], [1.0]) + + +def test_correlation_empty_raises(): + with pytest.raises(ValueError): + pearson([], []) + with pytest.raises(ValueError): + spearman([], []) + with pytest.raises(ValueError): + mae([], []) + + +# --- calibrate_judge: score-level only --- + +def test_calibrate_judge_score_level_tracks_humans(): + # Judge raw scores 2,3,4,5 -> normalized 0.4,0.6,0.8,1.0 vs humans that + # rise in lockstep -> pearson near 1.0 and a small mae. + judge = _judge_from_scores({"AAA": 2, "BBB": 3, "CCC": 4, "DDD": 5}) + samples = [ + CalibrationSample(text="AAA", human_score=0.4), + CalibrationSample(text="BBB", human_score=0.6), + CalibrationSample(text="CCC", human_score=0.8), + CalibrationSample(text="DDD", human_score=1.0), + ] + report = calibrate_judge(judge, samples) + assert report.n == 4 + assert report.pearson == pytest.approx(1.0) + assert report.spearman == pytest.approx(1.0) + assert report.mae == pytest.approx(0.0, abs=1e-9) + assert report.accuracy is None + assert report.cohen_kappa is None + + +# --- calibrate_judge: label-level only --- + +def test_calibrate_judge_label_level_only(): + judge = _judge_from_scores({"PASS": 5, "FAIL": 1}) + samples = [ + CalibrationSample(text="PASS", human_label=True), + CalibrationSample(text="FAIL", human_label=False), + ] + report = calibrate_judge(judge, samples) + assert report.accuracy == pytest.approx(1.0) + assert report.cohen_kappa == pytest.approx(1.0) + assert report.pearson is None + assert report.spearman is None + assert report.mae is None + + +# --- calibrate_judge: both --- + +def test_calibrate_judge_both_levels_populated(): + judge = _judge_from_scores({"AAA": 5, "BBB": 1}) + samples = [ + CalibrationSample(text="AAA", human_score=1.0, human_label=True), + CalibrationSample(text="BBB", human_score=0.2, human_label=False), + ] + report = calibrate_judge(judge, samples) + assert report.pearson is not None + assert report.spearman is not None + assert report.mae is not None + assert report.accuracy is not None + assert report.cohen_kappa is not None + + +# --- calibrate_judge: validation --- + +def test_calibrate_judge_empty_samples_raises(): + judge = _constant_judge(3) + with pytest.raises(ValueError): + calibrate_judge(judge, []) + + +def test_calibrate_judge_sample_with_neither_raises(): + judge = _constant_judge(3) + samples = [CalibrationSample(text="AAA", human_score=0.5), CalibrationSample(text="BBB")] + with pytest.raises(ValueError): + calibrate_judge(judge, samples) + + +# --- verbosity_bias --- + +def test_verbosity_bias_positive_for_length_judge(): + texts = ["a", "abcdefghij", "abcdefghijabcdefghij", "abcdefghijabcdefghijabcdefghij"] + assert verbosity_bias(_length_judge(), texts) > 0.9 + + +def test_verbosity_bias_constant_judge_returns_zero(): + texts = ["a", "abcdefghij", "abcdefghijabcdefghij"] + assert verbosity_bias(_constant_judge(3), texts) == 0.0 + + +def test_verbosity_bias_empty_texts_raises(): + with pytest.raises(ValueError): + verbosity_bias(_constant_judge(3), []) + + +# --- CalibrationReport.table --- + +def test_calibration_report_table_omits_none_rows(): + report = CalibrationReport( + n=5, + pearson=0.912, + spearman=None, + mae=0.041, + accuracy=None, + cohen_kappa=None, + ) + table = report.table() + assert "n=5" in table + assert "0.912" in table + assert "0.041" in table + assert "spearman" not in table + assert "accuracy" not in table + assert "cohen_kappa" not in table From 80b9f7acb7d0c9e484c1ed97b2dcfa86697a5dbf Mon Sep 17 00:00:00 2001 From: Lester A Leong Date: Thu, 4 Jun 2026 16:33:44 -0500 Subject: [PATCH 4/5] wire: export judge-reliability API, add example, README, bump 0.3.0 - Top-level + judge/ + bench/ __init__ exports for SelfConsistencyJudge, ScoreDistribution, PairwiseJudge, PairwiseResult, position_bias_rate, calibrate_judge, CalibrationSample, CalibrationReport, verbosity_bias, and the pearson/spearman/mae stats. - examples/judge_calibration.py: offline demo of uncertainty, position-bias debiasing, calibration, and verbosity-bias. - README: "Judge reliability" section. - version 0.2.0 -> 0.3.0. --- README.md | 53 ++++++++++++++++++ examples/judge_calibration.py | 90 ++++++++++++++++++++++++++++++ pyproject.toml | 4 +- src/llm_evalgate/__init__.py | 25 ++++++++- src/llm_evalgate/bench/__init__.py | 6 ++ src/llm_evalgate/judge/__init__.py | 36 +++++++++++- 6 files changed, 208 insertions(+), 6 deletions(-) create mode 100644 examples/judge_calibration.py diff --git a/README.md b/README.md index d98458a..375678f 100644 --- a/README.md +++ b/README.md @@ -175,6 +175,59 @@ judge = JudgeDimension(complete=anthropic_judge("claude-sonnet-4-6"), rubric=rub The judge never raises into your pipeline: a model error or an unparseable response scores `0.0` with the reason recorded, so a flaky grader fails closed instead of crashing the run. +## Judge reliability: measure and de-bias your judge + +An LLM judge is itself a model, with all the variance and bias that implies. Trusting it blind is how a bad eval ships. These three tools measure and correct for that. See [`examples/judge_calibration.py`](examples/judge_calibration.py) for a runnable, offline demo. + +### Uncertainty (self-consistency) + +A judge at temperature > 0 returns a different score each call, and a single number hides that. `SelfConsistencyJudge` samples the judge N times and reports the spread and a 95% confidence interval, so you know whether a score of 0.62 is solid or a coin flip. + +```python +from llm_evalgate import JudgeDimension, SelfConsistencyJudge + +judge = JudgeDimension(complete=my_model_call, rubric="Is the answer faithful to the source?") +robust = SelfConsistencyJudge(judge, samples=5) + +report = robust.run(answer) +# PASS [self_consistency] score=0.640 - mean=0.640; stdev=0.080; 95% CI=[0.570, 0.710]; n=5 +dist = robust.distribution(answer) # ScoreDistribution(mean, stdev, n, ci_low, ci_high, scores) +``` + +### Position bias (pairwise, order-swapped) + +When a judge compares two answers, it tends to favor whichever is shown first. `PairwiseJudge` runs the comparison in both orders and only trusts a verdict that survives the swap; if the judge flips, the result is reported as a tie with `consistent=False`. `position_bias_rate` measures how often a judge flips across a set of pairs, so you can quantify the bias before you rely on the judge. + +```python +from llm_evalgate import PairwiseJudge, position_bias_rate + +pj = PairwiseJudge(complete=my_model_call, criteria="Which answer is more helpful?") +result = pj.compare(answer_a, answer_b) +# result.winner in {"A", "B", "tie"}; result.consistent is False when order flipped the call + +bias = position_bias_rate(pj, [(a1, b1), (a2, b2), ...]) # 0.0 = unbiased, 1.0 = pure position bias +``` + +### Calibration (judge vs human) + +The only real test of a judge is agreement with human labels. `calibrate_judge` scores a labeled set and reports the correlation (Pearson, Spearman) and error (MAE) against human scores, plus accuracy and Cohen's kappa against human pass/fail. `verbosity_bias` checks the failure mode where a judge just rewards length. + +```python +from llm_evalgate import JudgeDimension, CalibrationSample, calibrate_judge, verbosity_bias + +judge = JudgeDimension(complete=my_model_call, rubric="Grade the answer.") +report = calibrate_judge(judge, [ + CalibrationSample(text=a1, human_score=0.9, human_label=True), + CalibrationSample(text=a2, human_score=0.2, human_label=False), + # ... +]) +print(report.table()) # pearson / spearman / mae / accuracy / cohen_kappa + +length_corr = verbosity_bias(judge, [a1, a2, a3]) # high = the judge is rewarding length, not quality +``` + +A judge you have calibrated, sampled for uncertainty, and checked for position and verbosity bias is one you can defend in a design review. An uncalibrated judge is a vibe with an API bill. + ## Agentic evaluation Agents are not just their final answer, they are a trajectory of tool calls. `llm-evalgate` evaluates the whole trace. diff --git a/examples/judge_calibration.py b/examples/judge_calibration.py new file mode 100644 index 0000000..9f87a83 --- /dev/null +++ b/examples/judge_calibration.py @@ -0,0 +1,90 @@ +"""Measure and de-bias an LLM-as-judge, offline. + +Run from the repo root:: + + python examples/judge_calibration.py + +Shows the three judge-reliability tools added in v0.3.0: +- self-consistency: sample a stochastic judge N times for an uncertainty band +- pairwise debiasing: catch a position-biased judge by swapping answer order +- calibration: measure judge-vs-human agreement, plus a verbosity-bias check + +Every judge here is a fake offline callable, so no API key or network is +needed. Swap in ``anthropic_judge(...)`` for live grading. +""" + +from __future__ import annotations + +import itertools + +from llm_evalgate import ( + CalibrationSample, + JudgeDimension, + PairwiseJudge, + SelfConsistencyJudge, + calibrate_judge, + position_bias_rate, + verbosity_bias, +) + + +def main() -> None: + # 1. Self-consistency: a stochastic judge whose score varies call to call. + score_cycle = itertools.cycle(["4", "3", "5", "4", "3"]) + + def varying(_prompt: str) -> str: + return f"SCORE: {next(score_cycle)}" + + base = JudgeDimension(complete=varying, rubric="Is the answer correct and on topic?", scale=5) + sc = SelfConsistencyJudge(base, samples=5) + score, detail = sc.evaluate("Some answer.") + print("Self-consistency") + print(f" aggregate score={score:.3f}") + print(f" {detail}") + print() + + # 2. Pairwise debiasing: a judge that ALWAYS prefers whatever is shown first. + def position_biased(_prompt: str) -> str: + return "WINNER: First\nREASON: it just prefers the first one" + + pj = PairwiseJudge(complete=position_biased, criteria="Which answer is more helpful?") + result = pj.compare("Answer A", "Answer B") + print("Pairwise (position-biased judge)") + print(f" winner={result.winner} consistent={result.consistent}") + print(f" order AB picked {result.order_ab_winner}, order BA picked {result.order_ba_winner}") + rate = position_bias_rate(pj, [("a", "b"), ("c", "d"), ("e", "f")]) + print(f" measured position-bias rate: {rate:.3f}") + print() + + # 3. Calibration: a judge whose score tracks a marker, compared to known human scores. + human = {"great answer": 1.0, "ok answer": 0.6, "bad answer": 0.2} + + def aligned(prompt: str) -> str: + for marker, human_score in human.items(): + if marker in prompt: + return f"SCORE: {human_score * 5:.1f}" + return "SCORE: 3" + + judge = JudgeDimension(complete=aligned, rubric="Grade the answer.", scale=5, threshold=0.5) + samples = [CalibrationSample(text=t, human_score=hs) for t, hs in human.items()] + report = calibrate_judge(judge, samples) + print("Calibration (judge vs human)") + print(report.table()) + print() + + # 4. Verbosity bias: a judge that just rewards length, regardless of content. + def length_judge(prompt: str) -> str: + body = prompt.split("Text:")[-1] + biased_score = min(5.0, 1.0 + len(body) / 40.0) + return f"SCORE: {biased_score:.1f}" + + vjudge = JudgeDimension(complete=length_judge, rubric="Grade the answer.", scale=5) + vbias = verbosity_bias( + vjudge, ["short", "a bit longer answer", "a substantially longer answer here"] + ) + print("Verbosity bias") + print(f" length-vs-score correlation: {vbias:.3f} (near 1.0 means the judge rewards length)") + + +if __name__ == "__main__": + main() diff --git a/pyproject.toml b/pyproject.toml index 312ec08..1ef003e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,8 +4,8 @@ build-backend = "hatchling.build" [project] name = "llm-evalgate" -version = "0.2.0" -description = "Eval gates, LLM-as-judge, agentic-trace evals, and reliability primitives for LLM pipelines" +version = "0.3.0" +description = "Eval gates, calibrated LLM-as-judge, agentic-trace evals, and reliability primitives for LLM pipelines" readme = "README.md" license = { text = "MIT" } requires-python = ">=3.9" diff --git a/src/llm_evalgate/__init__.py b/src/llm_evalgate/__init__.py index ce06abc..9ca9067 100644 --- a/src/llm_evalgate/__init__.py +++ b/src/llm_evalgate/__init__.py @@ -20,7 +20,20 @@ ) from .eval.dimension import Dimension, DimensionResult from .eval.harness import EvalHarness, EvalReport -from .judge import JudgeDimension, JudgeVerdict, JuryDimension +from .judge import ( + CalibrationReport, + CalibrationSample, + JudgeDimension, + JudgeVerdict, + JuryDimension, + PairwiseJudge, + PairwiseResult, + ScoreDistribution, + SelfConsistencyJudge, + calibrate_judge, + position_bias_rate, + verbosity_bias, +) __all__ = [ # core @@ -32,6 +45,16 @@ "JudgeDimension", "JuryDimension", "JudgeVerdict", + # judge reliability (uncertainty, debiasing, calibration) + "SelfConsistencyJudge", + "ScoreDistribution", + "PairwiseJudge", + "PairwiseResult", + "position_bias_rate", + "calibrate_judge", + "CalibrationSample", + "CalibrationReport", + "verbosity_bias", # agentic "AgentTrace", "AgentStep", diff --git a/src/llm_evalgate/bench/__init__.py b/src/llm_evalgate/bench/__init__.py index 0212af6..8a980dc 100644 --- a/src/llm_evalgate/bench/__init__.py +++ b/src/llm_evalgate/bench/__init__.py @@ -4,9 +4,12 @@ cohen_kappa, confusion_counts, f1, + mae, + pearson, precision, recall, regression_catch_rate, + spearman, ) from .runner import BenchmarkResult, BenchmarkRunner, BenchSample, load_golden @@ -23,4 +26,7 @@ "cohen_kappa", "regression_catch_rate", "all_metrics", + "pearson", + "spearman", + "mae", ] diff --git a/src/llm_evalgate/judge/__init__.py b/src/llm_evalgate/judge/__init__.py index ecca7cf..1af9e28 100644 --- a/src/llm_evalgate/judge/__init__.py +++ b/src/llm_evalgate/judge/__init__.py @@ -4,14 +4,44 @@ parse_verdict, render_prompt, ) +from .calibration import ( + CalibrationReport, + CalibrationSample, + calibrate_judge, + verbosity_bias, +) +from .consistency import ScoreDistribution, SelfConsistencyJudge from .dimension import JudgeDimension, JuryDimension, anthropic_judge +from .pairwise import ( + DEFAULT_PAIRWISE_TEMPLATE, + PairwiseJudge, + PairwiseResult, + parse_pairwise, + position_bias_rate, +) __all__ = [ + # base "DEFAULT_PROMPT_TEMPLATE", - "JudgeDimension", "JudgeVerdict", - "JuryDimension", - "anthropic_judge", "parse_verdict", "render_prompt", + # core judges + "JudgeDimension", + "JuryDimension", + "anthropic_judge", + # self-consistency / uncertainty + "ScoreDistribution", + "SelfConsistencyJudge", + # pairwise / position-bias debiasing + "DEFAULT_PAIRWISE_TEMPLATE", + "PairwiseJudge", + "PairwiseResult", + "parse_pairwise", + "position_bias_rate", + # calibration + "CalibrationSample", + "CalibrationReport", + "calibrate_judge", + "verbosity_bias", ] From fb474212ffdd2d27b144803918f513ea94618b56 Mon Sep 17 00:00:00 2001 From: Lester A Leong Date: Thu, 4 Jun 2026 16:36:27 -0500 Subject: [PATCH 5/5] docs: note self-consistency cost (N model calls per text, cache bypassed) Reviewer follow-up (MINOR). Other reviewer suggestions deferred to v0.3.1: verbatim raw in PairwiseResult, and the evaluate/score double-render. --- src/llm_evalgate/judge/consistency.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/llm_evalgate/judge/consistency.py b/src/llm_evalgate/judge/consistency.py index c2e3593..69e17d1 100644 --- a/src/llm_evalgate/judge/consistency.py +++ b/src/llm_evalgate/judge/consistency.py @@ -28,6 +28,10 @@ class SelfConsistencyJudge(Dimension): ``score`` ``samples`` times, and reports the mean, spread, and a 95% confidence interval. Because ``score`` bypasses the cache, each sample is an independent model call, which is what produces the variance. + + Cost: every call to ``distribution`` or ``evaluate`` makes ``samples`` real + model calls and intentionally does not use the base judge's cache. With a + live judge that is ``samples`` times the tokens of a single grade per text. """ def __init__(