feat: judge reliability - calibration, uncertainty, and bias debiasing (v0.3.0)

README.md

@@ -175,6 +175,59 @@ judge = JudgeDimension(complete=anthropic_judge("claude-sonnet-4-6"), rubric=rub
 
 The judge never raises into your pipeline: a model error or an unparseable response scores `0.0` with the reason recorded, so a flaky grader fails closed instead of crashing the run.
 
+## Judge reliability: measure and de-bias your judge
+
+An LLM judge is itself a model, with all the variance and bias that implies. Trusting it blind is how a bad eval ships. These three tools measure and correct for that. See [`examples/judge_calibration.py`](examples/judge_calibration.py) for a runnable, offline demo.
+
+### Uncertainty (self-consistency)
+
+A judge at temperature > 0 returns a different score each call, and a single number hides that. `SelfConsistencyJudge` samples the judge N times and reports the spread and a 95% confidence interval, so you know whether a score of 0.62 is solid or a coin flip.
+
+```python
+from llm_evalgate import JudgeDimension, SelfConsistencyJudge
+
+judge = JudgeDimension(complete=my_model_call, rubric="Is the answer faithful to the source?")
+robust = SelfConsistencyJudge(judge, samples=5)
+
+report = robust.run(answer)
+# PASS [self_consistency] score=0.640 - mean=0.640; stdev=0.080; 95% CI=[0.570, 0.710]; n=5
+dist = robust.distribution(answer)   # ScoreDistribution(mean, stdev, n, ci_low, ci_high, scores)
+```
+
+### Position bias (pairwise, order-swapped)
+
+When a judge compares two answers, it tends to favor whichever is shown first. `PairwiseJudge` runs the comparison in both orders and only trusts a verdict that survives the swap; if the judge flips, the result is reported as a tie with `consistent=False`. `position_bias_rate` measures how often a judge flips across a set of pairs, so you can quantify the bias before you rely on the judge.
+
+```python
+from llm_evalgate import PairwiseJudge, position_bias_rate
+
+pj = PairwiseJudge(complete=my_model_call, criteria="Which answer is more helpful?")
+result = pj.compare(answer_a, answer_b)
+# result.winner in {"A", "B", "tie"}; result.consistent is False when order flipped the call
+
+bias = position_bias_rate(pj, [(a1, b1), (a2, b2), ...])   # 0.0 = unbiased, 1.0 = pure position bias
+```
+
+### Calibration (judge vs human)
+
+The only real test of a judge is agreement with human labels. `calibrate_judge` scores a labeled set and reports the correlation (Pearson, Spearman) and error (MAE) against human scores, plus accuracy and Cohen's kappa against human pass/fail. `verbosity_bias` checks the failure mode where a judge just rewards length.
+
+```python
+from llm_evalgate import JudgeDimension, CalibrationSample, calibrate_judge, verbosity_bias
+
+judge = JudgeDimension(complete=my_model_call, rubric="Grade the answer.")
+report = calibrate_judge(judge, [
+    CalibrationSample(text=a1, human_score=0.9, human_label=True),
+    CalibrationSample(text=a2, human_score=0.2, human_label=False),
+    # ...
+])
+print(report.table())   # pearson / spearman / mae / accuracy / cohen_kappa
+
+length_corr = verbosity_bias(judge, [a1, a2, a3])   # high = the judge is rewarding length, not quality
+```
+
+A judge you have calibrated, sampled for uncertainty, and checked for position and verbosity bias is one you can defend in a design review. An uncalibrated judge is a vibe with an API bill.
+
 ## Agentic evaluation
 
 Agents are not just their final answer, they are a trajectory of tool calls. `llm-evalgate` evaluates the whole trace.

examples/judge_calibration.py

@@ -0,0 +1,90 @@
+"""Measure and de-bias an LLM-as-judge, offline.
+
+Run from the repo root::
+
+    python examples/judge_calibration.py
+
+Shows the three judge-reliability tools added in v0.3.0:
+- self-consistency: sample a stochastic judge N times for an uncertainty band
+- pairwise debiasing: catch a position-biased judge by swapping answer order
+- calibration: measure judge-vs-human agreement, plus a verbosity-bias check
+
+Every judge here is a fake offline callable, so no API key or network is
+needed. Swap in ``anthropic_judge(...)`` for live grading.
+"""
+
+from __future__ import annotations
+
+import itertools
+
+from llm_evalgate import (
+    CalibrationSample,
+    JudgeDimension,
+    PairwiseJudge,
+    SelfConsistencyJudge,
+    calibrate_judge,
+    position_bias_rate,
+    verbosity_bias,
+)
+
+
+def main() -> None:
+    # 1. Self-consistency: a stochastic judge whose score varies call to call.
+    score_cycle = itertools.cycle(["4", "3", "5", "4", "3"])
+
+    def varying(_prompt: str) -> str:
+        return f"SCORE: {next(score_cycle)}"
+
+    base = JudgeDimension(complete=varying, rubric="Is the answer correct and on topic?", scale=5)
+    sc = SelfConsistencyJudge(base, samples=5)
+    score, detail = sc.evaluate("Some answer.")
+    print("Self-consistency")
+    print(f"  aggregate score={score:.3f}")
+    print(f"  {detail}")
+    print()
+
+    # 2. Pairwise debiasing: a judge that ALWAYS prefers whatever is shown first.
+    def position_biased(_prompt: str) -> str:
+        return "WINNER: First\nREASON: it just prefers the first one"
+
+    pj = PairwiseJudge(complete=position_biased, criteria="Which answer is more helpful?")
+    result = pj.compare("Answer A", "Answer B")
+    print("Pairwise (position-biased judge)")
+    print(f"  winner={result.winner}  consistent={result.consistent}")
+    print(f"  order AB picked {result.order_ab_winner}, order BA picked {result.order_ba_winner}")
+    rate = position_bias_rate(pj, [("a", "b"), ("c", "d"), ("e", "f")])
+    print(f"  measured position-bias rate: {rate:.3f}")
+    print()
+
+    # 3. Calibration: a judge whose score tracks a marker, compared to known human scores.
+    human = {"great answer": 1.0, "ok answer": 0.6, "bad answer": 0.2}
+
+    def aligned(prompt: str) -> str:
+        for marker, human_score in human.items():
+            if marker in prompt:
+                return f"SCORE: {human_score * 5:.1f}"
+        return "SCORE: 3"
+
+    judge = JudgeDimension(complete=aligned, rubric="Grade the answer.", scale=5, threshold=0.5)
+    samples = [CalibrationSample(text=t, human_score=hs) for t, hs in human.items()]
+    report = calibrate_judge(judge, samples)
+    print("Calibration (judge vs human)")
+    print(report.table())
+    print()
+
+    # 4. Verbosity bias: a judge that just rewards length, regardless of content.
+    def length_judge(prompt: str) -> str:
+        body = prompt.split("Text:")[-1]
+        biased_score = min(5.0, 1.0 + len(body) / 40.0)
+        return f"SCORE: {biased_score:.1f}"
+
+    vjudge = JudgeDimension(complete=length_judge, rubric="Grade the answer.", scale=5)
+    vbias = verbosity_bias(
+        vjudge, ["short", "a bit longer answer", "a substantially longer answer here"]
+    )
+    print("Verbosity bias")
+    print(f"  length-vs-score correlation: {vbias:.3f}  (near 1.0 means the judge rewards length)")
+
+
+if __name__ == "__main__":
+    main()

pyproject.toml

@@ -4,8 +4,8 @@ build-backend = "hatchling.build"
 
 [project]
 name = "llm-evalgate"
-version = "0.2.0"
-description = "Eval gates, LLM-as-judge, agentic-trace evals, and reliability primitives for LLM pipelines"
+version = "0.3.0"
+description = "Eval gates, calibrated LLM-as-judge, agentic-trace evals, and reliability primitives for LLM pipelines"
 readme = "README.md"
 license = { text = "MIT" }
 requires-python = ">=3.9"

src/llm_evalgate/__init__.py

@@ -20,7 +20,20 @@
 )
 from .eval.dimension import Dimension, DimensionResult
 from .eval.harness import EvalHarness, EvalReport
-from .judge import JudgeDimension, JudgeVerdict, JuryDimension
+from .judge import (
+    CalibrationReport,
+    CalibrationSample,
+    JudgeDimension,
+    JudgeVerdict,
+    JuryDimension,
+    PairwiseJudge,
+    PairwiseResult,
+    ScoreDistribution,
+    SelfConsistencyJudge,
+    calibrate_judge,
+    position_bias_rate,
+    verbosity_bias,
+)
 
 __all__ = [
     # core
@@ -32,6 +45,16 @@
     "JudgeDimension",
     "JuryDimension",
     "JudgeVerdict",
+    # judge reliability (uncertainty, debiasing, calibration)
+    "SelfConsistencyJudge",
+    "ScoreDistribution",
+    "PairwiseJudge",
+    "PairwiseResult",
+    "position_bias_rate",
+    "calibrate_judge",
+    "CalibrationSample",
+    "CalibrationReport",
+    "verbosity_bias",
     # agentic
     "AgentTrace",
     "AgentStep",

src/llm_evalgate/bench/__init__.py

@@ -4,9 +4,12 @@
     cohen_kappa,
     confusion_counts,
     f1,
+    mae,
+    pearson,
     precision,
     recall,
     regression_catch_rate,
+    spearman,
 )
 from .runner import BenchmarkResult, BenchmarkRunner, BenchSample, load_golden
 
@@ -23,4 +26,7 @@
     "cohen_kappa",
     "regression_catch_rate",
     "all_metrics",
+    "pearson",
+    "spearman",
+    "mae",
 ]

src/llm_evalgate/bench/metrics.py

@@ -102,3 +102,57 @@ def all_metrics(predicted: list[bool], labels: list[bool]) -> dict[str, float]:
         "cohen_kappa": cohen_kappa(predicted, labels),
         "regression_catch_rate": regression_catch_rate(predicted, labels),
     }
+
+
+def _validate_pair(x: list[float], y: list[float]) -> None:
+    if len(x) != len(y):
+        raise ValueError(
+            f"x and y must be equal length; got {len(x)} and {len(y)}"
+        )
+    if not x:
+        raise ValueError("x and y must be non-empty")
+
+
+def pearson(x: list[float], y: list[float]) -> float:
+    """Pearson correlation coefficient. Zero-variance input returns 0.0."""
+    _validate_pair(x, y)
+    n = len(x)
+    mean_x = sum(x) / n
+    mean_y = sum(y) / n
+    dx = [xi - mean_x for xi in x]
+    dy = [yi - mean_y for yi in y]
+    covariance = sum(a * b for a, b in zip(dx, dy))
+    var_x = sum(a * a for a in dx)
+    var_y = sum(b * b for b in dy)
+    denom = (var_x * var_y) ** 0.5
+    if denom == 0:
+        return 0.0
+    return covariance / denom
+
+
+def _average_ranks(values: list[float]) -> list[float]:
+    """Rank ``values`` ascending, assigning tied values their average rank."""
+    order = sorted(range(len(values)), key=lambda i: values[i])
+    ranks = [0.0] * len(values)
+    i = 0
+    while i < len(order):
+        j = i
+        while j + 1 < len(order) and values[order[j + 1]] == values[order[i]]:
+            j += 1
+        average_rank = (i + j) / 2 + 1
+        for k in range(i, j + 1):
+            ranks[order[k]] = average_rank
+        i = j + 1
+    return ranks
+
+
+def spearman(x: list[float], y: list[float]) -> float:
+    """Spearman rank correlation: Pearson on the average-rank transform."""
+    _validate_pair(x, y)
+    return pearson(_average_ranks(x), _average_ranks(y))
+
+
+def mae(x: list[float], y: list[float]) -> float:
+    """Mean absolute error between two equal-length vectors."""
+    _validate_pair(x, y)
+    return sum(abs(a - b) for a, b in zip(x, y)) / len(x)

src/llm_evalgate/judge/__init__.py

@@ -4,14 +4,44 @@
     parse_verdict,
     render_prompt,
 )
+from .calibration import (
+    CalibrationReport,
+    CalibrationSample,
+    calibrate_judge,
+    verbosity_bias,
+)
+from .consistency import ScoreDistribution, SelfConsistencyJudge
 from .dimension import JudgeDimension, JuryDimension, anthropic_judge
+from .pairwise import (
+    DEFAULT_PAIRWISE_TEMPLATE,
+    PairwiseJudge,
+    PairwiseResult,
+    parse_pairwise,
+    position_bias_rate,
+)
 
 __all__ = [
+    # base
     "DEFAULT_PROMPT_TEMPLATE",
-    "JudgeDimension",
     "JudgeVerdict",
-    "JuryDimension",
-    "anthropic_judge",
     "parse_verdict",
     "render_prompt",
+    # core judges
+    "JudgeDimension",
+    "JuryDimension",
+    "anthropic_judge",
+    # self-consistency / uncertainty
+    "ScoreDistribution",
+    "SelfConsistencyJudge",
+    # pairwise / position-bias debiasing
+    "DEFAULT_PAIRWISE_TEMPLATE",
+    "PairwiseJudge",
+    "PairwiseResult",
+    "parse_pairwise",
+    "position_bias_rate",
+    # calibration
+    "CalibrationSample",
+    "CalibrationReport",
+    "calibrate_judge",
+    "verbosity_bias",
 ]