diff --git a/README.md b/README.md index 98294e7..d98458a 100644 --- a/README.md +++ b/README.md @@ -9,19 +9,28 @@ Deterministic eval gates and reliability primitives for LLM pipelines. --- -Most LLM eval tooling is either LLM-as-judge (non-deterministic, expensive, not CI-friendly) or a heavy enterprise suite. `llm-evalgate` is neither. +Most LLM eval tooling is either heavy enterprise SaaS or a pile of one-off scripts. `llm-evalgate` is a small, composable library that covers the full eval surface and runs in CI. -It gives you two things: +It gives you four things: -- **Eval gates**: code-only quality dimensions that run the same way every time. Drop them into any pipeline, run them in CI, get a pass/fail with a reason. -- **Reliability primitives**: retry with backoff, model fallback chains, and a circuit breaker. The building blocks for LLM pipelines that hold up in production. +- **Eval gates**: code-only quality dimensions that run the same way every time. Drop them into any pipeline, run them in CI, get a pass/fail with a reason. No tokens burned, no model drift. +- **LLM-as-judge and jury**: when a check needs semantic judgment a regex cannot give, grade with a model, or a panel of models, and read the agreement back. Same `Dimension` interface, so judges compose with gates in one harness. +- **Agentic evals**: score agent traces, not just final text. Tool selection, argument validity, step efficiency, goal completion, and judge-backed trajectory coherence for long-horizon reasoning. +- **Benchmarking**: measure your eval against labeled data. Accuracy, precision/recall, Cohen's kappa, and regression-catch-rate, so you know whether your gate actually catches the failures that matter. + +Plus **reliability primitives** (retry with backoff, model fallback chains, circuit breaker) for pipelines that hold up in production. + +Design rule: deterministic gates are the cheap, CI-friendly backbone; reach for a judge only where semantic nuance demands it; benchmark both so the eval is trustworthy rather than assumed. ## Install ```bash -pip install llm-evalgate +pip install llm-evalgate # core: gates, agentic evals, benchmarking +pip install "llm-evalgate[judge]" # adds the Anthropic adapter for live LLM-as-judge ``` +The judge and jury work with any callable that maps a prompt string to a response string, so the core package needs no model SDK and the test suite runs fully offline. The `[judge]` extra only adds a convenience adapter for live calls. + ## Quickstart ### Eval gates @@ -41,9 +50,9 @@ report = harness.run(llm_output) if not report.passed: print(report) # EvalReport: FAIL - # FAIL [blocklist] score=0.000 — prohibited terms found: ['confidential'] - # PASS [readability] score=0.612 — Flesch ease=61.2, FK grade=8.4 - # PASS [schema_compliance] score=1.000 — all 2 required fields present + # FAIL [blocklist] score=0.000 - prohibited terms found: ['confidential'] + # PASS [readability] score=0.612 - Flesch ease=61.2, FK grade=8.4 + # PASS [schema_compliance] score=1.000 - all 2 required fields present ``` ### Custom dimension @@ -86,7 +95,7 @@ result = with_fallback( fallback=lambda: call_model("claude-sonnet-4-6", prompt), ) -# ordered chain — first success wins +# ordered chain - first success wins result = with_fallback_chain([ lambda: call_model("claude-opus-4-8", prompt), lambda: call_model("claude-sonnet-4-6", prompt), @@ -119,15 +128,141 @@ except CircuitOpenError: All dimensions follow the same interface: `evaluate(text) -> (score, detail)`. Writing a new one is ten lines. -## Why deterministic? +## LLM-as-judge and jury + +Some quality dimensions cannot be checked with code: tone, helpfulness, faithfulness to a brief. For those, grade with a model. `JudgeDimension` is a regular `Dimension`, so it drops into the same harness next to your deterministic gates. + +```python +from llm_evalgate import JudgeDimension + +# `complete` is any callable mapping a prompt to a response string. +# Inject your own, or use the optional Anthropic adapter (see below). +judge = JudgeDimension( + complete=my_model_call, + rubric="Is the answer factually correct and on-topic for the question?", + scale=5, # model grades 1..5; normalized to [0, 1] + threshold=0.6, +) + +report = judge.run("The capital of France is Paris.") +# PASS [judge] score=1.000 - clear and correct +``` + +Want a panel instead of a single grader? `JuryDimension` runs several judges and aggregates the verdicts, and reports how much they agreed (a tight spread means the eval is stable; a wide one means the dimension is ambiguous). + +```python +from llm_evalgate import JudgeDimension, JuryDimension + +jury = JuryDimension( + judges=[ + JudgeDimension(complete=opus_call, rubric=rubric, name="opus"), + JudgeDimension(complete=sonnet_call, rubric=rubric, name="sonnet"), + JudgeDimension(complete=haiku_call, rubric=rubric, name="haiku"), + ], + aggregate="mean", # or "median", "majority" +) +report = jury.run(answer) +``` + +For live grading without writing the adapter yourself, install `llm-evalgate[judge]` and use `anthropic_judge`: + +```python +from llm_evalgate.judge import anthropic_judge +from llm_evalgate import JudgeDimension + +judge = JudgeDimension(complete=anthropic_judge("claude-sonnet-4-6"), rubric=rubric) +``` + +The judge never raises into your pipeline: a model error or an unparseable response scores `0.0` with the reason recorded, so a flaky grader fails closed instead of crashing the run. + +## Agentic evaluation + +Agents are not just their final answer, they are a trajectory of tool calls. `llm-evalgate` evaluates the whole trace. + +```python +from llm_evalgate import ( + AgentTrace, AgentStep, ToolCall, AgentEvalHarness, + ToolSelectionDimension, ToolArgValidityDimension, + StepEfficiencyDimension, GoalCompletionDimension, +) + +trace = AgentTrace( + goal="Find the weather in Paris and convert it to Fahrenheit.", + steps=[ + AgentStep( + thought="Look up the weather.", + tool_calls=[ToolCall(name="get_weather", args={"city": "Paris"}, result="18C")], + ), + AgentStep( + thought="Convert to Fahrenheit.", + tool_calls=[ToolCall(name="celsius_to_f", args={"c": 18}, result="64.4F")], + ), + ], + final_answer="It is about 64.4F in Paris.", +) + +harness = AgentEvalHarness([ + ToolSelectionDimension(expected_tools=["get_weather", "celsius_to_f"], mode="order"), + ToolArgValidityDimension(validators={"celsius_to_f": lambda a: "c" in a}), + StepEfficiencyDimension(max_steps=3), + GoalCompletionDimension(required_substrings=["F"]), +]) + +report = harness.run(trace) +if not report.passed: + print(report) +``` + +| Trace dimension | What it scores | +|---|---| +| `ToolSelectionDimension` | Did the agent call the expected tools (as a subset, exact set, or in order)? | +| `ToolArgValidityDimension` | Were tool arguments well-formed, and did any call error? | +| `StepEfficiencyDimension` | Did it finish within a step budget, without looping on repeated calls? | +| `GoalCompletionDimension` | Did the final answer satisfy a checker or required content? | +| `TrajectoryCoherenceDimension` | Judge-backed: does the reasoning path hang together over long horizons? | + +`TrajectoryCoherenceDimension` wraps a `JudgeDimension`, so long-horizon reasoning quality is scored by a model while the structural checks above stay deterministic. + +## Benchmarking: is your eval any good? + +An eval you have not measured is a guess. Point a `BenchmarkRunner` at a labeled dataset and it tells you how well your gate matches human judgment, including the metric that matters most in production: the fraction of real regressions it actually catches. + +```python +from llm_evalgate import BenchmarkRunner, EvalHarness, load_golden +from llm_evalgate.eval.dimensions import ( + BlocklistDimension, SchemaComplianceDimension, ReadabilityDimension, +) + +harness = EvalHarness([ + BlocklistDimension(terms=["confidential", "internal use only", "[REDACTED]"]), + SchemaComplianceDimension(required_fields=["title:", "summary:"]), + ReadabilityDimension(threshold=0.2), +]) + +result = BenchmarkRunner(harness).run(load_golden()) +print(result.table()) +``` + +Run against the bundled 24-sample golden set (see [`examples/benchmark.py`](examples/benchmark.py)), the deterministic harness alone scores: + +``` +n=24 +accuracy 0.833 +precision 0.750 +recall 1.000 +f1 0.857 +cohen_kappa 0.667 +regression_catch_rate 0.667 +``` + +It catches every formatting and policy violation but misses 4 of 12 regressions, because those failures are semantic (a fluent, well-formatted answer that is simply wrong). Add an LLM judge to the same harness and the catch rate closes to `1.000`. That is the whole thesis: deterministic gates are necessary but not sufficient, and the benchmark is how you prove where the line is. -LLM-as-judge eval is useful for research. In production pipelines, you need: +## Deterministic gates first, judge when you need to -- The same input to produce the same pass/fail result every run -- CI to catch regressions without burning tokens on every commit -- An audit trail that doesn't depend on a model that may drift +Reach for the cheapest check that works: -`llm-evalgate` eval dimensions are pure functions. No model calls, no network, no randomness. +- **Deterministic gates** (`BlocklistDimension`, `SchemaComplianceDimension`, and your own) are pure functions: same input, same pass/fail, every run. No model calls, no network, no randomness. Run them on every commit in CI without burning tokens, and keep an audit trail that does not depend on a model that may drift. +- **Judges and juries** are the opt-in layer for the semantic checks code cannot express. They cost tokens and carry variance, so use them where they earn it, and benchmark them so the variance is known rather than assumed. ## Composing with a pipeline diff --git a/examples/agentic_eval.py b/examples/agentic_eval.py new file mode 100644 index 0000000..199fd54 --- /dev/null +++ b/examples/agentic_eval.py @@ -0,0 +1,85 @@ +"""Evaluate a multi-step tool-using agent trace, offline and runnable. + +Run from the repo root:: + + python examples/agentic_eval.py +""" + +from __future__ import annotations + +from llm_evalgate.agentic import ( + AgentEvalHarness, + AgentStep, + AgentTrace, + GoalCompletionDimension, + StepEfficiencyDimension, + ToolArgValidityDimension, + ToolCall, + ToolSelectionDimension, +) + + +def build_trace() -> AgentTrace: + return AgentTrace( + goal="Find the population of France and convert it to millions.", + steps=[ + AgentStep( + thought="I need the current population figure, so I will search.", + tool_calls=[ + ToolCall( + name="search", + args={"query": "population of France"}, + result="France population is approximately 68000000.", + ) + ], + observation="France population is approximately 68000000.", + ), + AgentStep( + thought="Now I convert the raw figure to millions.", + tool_calls=[ + ToolCall( + name="calculator", + args={"expression": "68000000 / 1000000"}, + result=68.0, + ) + ], + observation="68.0", + ), + ], + final_answer="The population of France is about 68 million.", + ) + + +def main() -> None: + trace = build_trace() + harness = AgentEvalHarness( + [ + ToolSelectionDimension( + expected_tools=["search", "calculator"], mode="order" + ), + ToolArgValidityDimension( + validators={"search": lambda args: bool(args.get("query"))} + ), + StepEfficiencyDimension(max_steps=4), + GoalCompletionDimension(required_substrings=["68 million"]), + ] + ) + report = harness.run(trace) + print(report) + + # To grade long-horizon reasoning coherence with an LLM-as-judge, add a + # TrajectoryCoherenceDimension wrapping a JudgeDimension. The ``complete`` + # callable is injected, so this stays offline in tests: + # + # from llm_evalgate.agentic import TrajectoryCoherenceDimension + # from llm_evalgate.judge import JudgeDimension + # + # def complete(prompt: str) -> str: + # return "SCORE: 5\nREASON: the steps follow logically toward the goal" + # + # judge = JudgeDimension(complete=complete, rubric="Is the trajectory coherent?") + # harness = AgentEvalHarness([TrajectoryCoherenceDimension(judge)]) + + +if __name__ == "__main__": + main() diff --git a/examples/benchmark.py b/examples/benchmark.py new file mode 100644 index 0000000..1da953d --- /dev/null +++ b/examples/benchmark.py @@ -0,0 +1,76 @@ +"""Benchmark a deterministic eval harness, then a judge-augmented one, offline. + +Run from the repo root:: + + python examples/benchmark.py + +The deterministic harness catches blocklist, schema, and readability failures +but misses semantic regressions (factually wrong but well-formed text). Adding +an LLM-as-judge dimension raises the regression catch rate. The judge here is a +fake offline ``complete`` so the example needs no API key and no network. +""" + +from __future__ import annotations + +from llm_evalgate.bench import BenchmarkRunner, load_golden +from llm_evalgate.eval import EvalHarness +from llm_evalgate.eval.dimensions import ( + BlocklistDimension, + ReadabilityDimension, + SchemaComplianceDimension, +) +from llm_evalgate.judge import JudgeDimension + +# Distinctive substrings from the four semantic-fail summaries in the golden +# dataset. A real judge reads the text and reasons; this fake one keys on these +# so the example is deterministic. +_SEMANTIC_MARKERS = ( + "capital of France is Berlin", + "three parts gold", + "ten trillion dollars", + "freeze the flour", +) + + +def fake_complete(prompt: str) -> str: + """Stand in for an LLM judge: flag known semantic regressions, else pass. + + Replace this with a real Anthropic-backed callable:: + + from llm_evalgate.judge import anthropic_judge + complete = anthropic_judge(model="claude-3-5-haiku-latest") + """ + if any(marker in prompt for marker in _SEMANTIC_MARKERS): + return "SCORE: 1\nREASON: semantic issue" + return "SCORE: 5\nREASON: ok" + + +def build_deterministic_dimensions() -> list: + return [ + BlocklistDimension(terms=["confidential", "internal use only", "[REDACTED]"]), + SchemaComplianceDimension(required_fields=["title:", "summary:"]), + ReadabilityDimension(threshold=0.2), + ] + + +def main() -> None: + samples = load_golden() + + deterministic = EvalHarness(build_deterministic_dimensions()) + deterministic_result = BenchmarkRunner(deterministic).run(samples) + print("Deterministic harness") + print(deterministic_result.table()) + print() + + judge = JudgeDimension( + complete=fake_complete, + rubric="Is the text factually correct, on topic, and coherent?", + ) + augmented = EvalHarness(build_deterministic_dimensions() + [judge]) + augmented_result = BenchmarkRunner(augmented).run(samples) + print("Judge-augmented harness") + print(augmented_result.table()) + + +if __name__ == "__main__": + main() diff --git a/examples/judge_eval.py b/examples/judge_eval.py new file mode 100644 index 0000000..28204a1 --- /dev/null +++ b/examples/judge_eval.py @@ -0,0 +1,42 @@ +"""Minimal LLM-as-judge demo with a fake offline ``complete``. + +Run from the repo root:: + + python examples/judge_eval.py +""" + +from __future__ import annotations + +from llm_evalgate.judge import JudgeDimension + + +def fake_complete(prompt: str) -> str: + """Stand in for an LLM judge based on a marker in the text. + + Replace this with a real Anthropic-backed callable:: + + from llm_evalgate.judge import anthropic_judge + complete = anthropic_judge(model="claude-3-5-haiku-latest") + """ + if "capital of France is Berlin" in prompt: + return "SCORE: 1\nREASON: the capital of France is Paris, not Berlin" + return "SCORE: 5\nREASON: factually correct and on topic" + + +def main() -> None: + judge = JudgeDimension( + complete=fake_complete, + rubric="Is the text factually correct, on topic, and coherent?", + ) + + good = "title: Capital of France\nsummary: The capital of France is Paris." + bad = "title: Capital of France\nsummary: The capital of France is Berlin." + + for label, text in (("good", good), ("bad", bad)): + result = judge.run(text) + status = "PASS" if result.passed else "FAIL" + print(f"[{label}] {status} score={result.score:.3f} - {result.detail}") + + +if __name__ == "__main__": + main() diff --git a/pyproject.toml b/pyproject.toml index 1f319ce..312ec08 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,15 +4,26 @@ build-backend = "hatchling.build" [project] name = "llm-evalgate" -version = "0.1.1" -description = "Deterministic eval gates and reliability primitives for LLM pipelines" +version = "0.2.0" +description = "Eval gates, LLM-as-judge, agentic-trace evals, and reliability primitives for LLM pipelines" readme = "README.md" license = { text = "MIT" } requires-python = ">=3.9" authors = [{ name = "Lester Leong", email = "lester.leong89@gmail.com" }] -keywords = ["llm", "eval", "evaluation", "reliability", "ai", "agents", "pipeline"] +keywords = [ + "llm", + "eval", + "evaluation", + "llm-as-judge", + "agent", + "agentic", + "benchmark", + "reliability", + "ai", + "pipeline", +] classifiers = [ - "Development Status :: 3 - Alpha", + "Development Status :: 4 - Beta", "Intended Audience :: Developers", "License :: OSI Approved :: MIT License", "Programming Language :: Python :: 3", @@ -28,6 +39,9 @@ dependencies = [ ] [project.optional-dependencies] +judge = [ + "anthropic>=0.40", +] dev = [ "pytest>=8.0", "pytest-cov>=5.0", @@ -42,6 +56,17 @@ Issues = "https://github.com/LesterALeong/llm-evalgate/issues" [tool.hatch.build.targets.wheel] packages = ["src/llm_evalgate"] +artifacts = ["*.jsonl"] + +[tool.hatch.build.targets.sdist] +include = [ + "src/llm_evalgate", + "tests", + "examples", + "README.md", + "LICENSE", +] +artifacts = ["*.jsonl"] [tool.ruff] line-length = 100 diff --git a/src/llm_evalgate/__init__.py b/src/llm_evalgate/__init__.py index ebd660c..ce06abc 100644 --- a/src/llm_evalgate/__init__.py +++ b/src/llm_evalgate/__init__.py @@ -1,4 +1,53 @@ +from .agentic import ( + AgentEvalHarness, + AgentEvalReport, + AgentStep, + AgentTrace, + GoalCompletionDimension, + StepEfficiencyDimension, + ToolArgValidityDimension, + ToolCall, + ToolSelectionDimension, + TraceDimension, + TrajectoryCoherenceDimension, +) +from .bench import ( + BenchmarkResult, + BenchmarkRunner, + BenchSample, + all_metrics, + load_golden, +) from .eval.dimension import Dimension, DimensionResult from .eval.harness import EvalHarness, EvalReport +from .judge import JudgeDimension, JudgeVerdict, JuryDimension -__all__ = ["Dimension", "DimensionResult", "EvalHarness", "EvalReport"] +__all__ = [ + # core + "Dimension", + "DimensionResult", + "EvalHarness", + "EvalReport", + # judge + "JudgeDimension", + "JuryDimension", + "JudgeVerdict", + # agentic + "AgentTrace", + "AgentStep", + "ToolCall", + "TraceDimension", + "AgentEvalHarness", + "AgentEvalReport", + "ToolSelectionDimension", + "ToolArgValidityDimension", + "StepEfficiencyDimension", + "GoalCompletionDimension", + "TrajectoryCoherenceDimension", + # bench + "BenchmarkRunner", + "BenchSample", + "BenchmarkResult", + "load_golden", + "all_metrics", +] diff --git a/src/llm_evalgate/agentic/__init__.py b/src/llm_evalgate/agentic/__init__.py new file mode 100644 index 0000000..1c1d2d8 --- /dev/null +++ b/src/llm_evalgate/agentic/__init__.py @@ -0,0 +1,24 @@ +from .dimensions import ( + GoalCompletionDimension, + StepEfficiencyDimension, + ToolArgValidityDimension, + ToolSelectionDimension, + TraceDimension, + TrajectoryCoherenceDimension, +) +from .harness import AgentEvalHarness, AgentEvalReport +from .trace import AgentStep, AgentTrace, ToolCall + +__all__ = [ + "AgentEvalHarness", + "AgentEvalReport", + "AgentStep", + "AgentTrace", + "GoalCompletionDimension", + "StepEfficiencyDimension", + "ToolArgValidityDimension", + "ToolCall", + "ToolSelectionDimension", + "TraceDimension", + "TrajectoryCoherenceDimension", +] diff --git a/src/llm_evalgate/agentic/dimensions/__init__.py b/src/llm_evalgate/agentic/dimensions/__init__.py new file mode 100644 index 0000000..b1361c2 --- /dev/null +++ b/src/llm_evalgate/agentic/dimensions/__init__.py @@ -0,0 +1,15 @@ +from .base import TraceDimension +from .goal_completion import GoalCompletionDimension +from .step_efficiency import StepEfficiencyDimension +from .tool_arg_validity import ToolArgValidityDimension +from .tool_selection import ToolSelectionDimension +from .trajectory_coherence import TrajectoryCoherenceDimension + +__all__ = [ + "GoalCompletionDimension", + "StepEfficiencyDimension", + "ToolArgValidityDimension", + "ToolSelectionDimension", + "TraceDimension", + "TrajectoryCoherenceDimension", +] diff --git a/src/llm_evalgate/agentic/dimensions/base.py b/src/llm_evalgate/agentic/dimensions/base.py new file mode 100644 index 0000000..e3c880c --- /dev/null +++ b/src/llm_evalgate/agentic/dimensions/base.py @@ -0,0 +1,26 @@ +from __future__ import annotations + +from abc import ABC, abstractmethod + +from ...eval.dimension import DimensionResult +from ..trace import AgentTrace + + +class TraceDimension(ABC): + """Base class for a single eval dimension over an agent trace. + + Subclass this and implement ``evaluate``. The harness calls ``run``, + which applies the threshold and returns a ``DimensionResult``. + """ + + def __init__(self, threshold: float = 1.0, name: str | None = None) -> None: + self.threshold = threshold + self.name = name or self.__class__.__name__ + + @abstractmethod + def evaluate(self, trace: AgentTrace) -> tuple[float, str]: + """Return (score, detail). Score is in [0.0, 1.0].""" + + def run(self, trace: AgentTrace) -> DimensionResult: + score, detail = self.evaluate(trace) + return DimensionResult(score=score, passed=score >= self.threshold, detail=detail) diff --git a/src/llm_evalgate/agentic/dimensions/goal_completion.py b/src/llm_evalgate/agentic/dimensions/goal_completion.py new file mode 100644 index 0000000..614c9aa --- /dev/null +++ b/src/llm_evalgate/agentic/dimensions/goal_completion.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +from collections.abc import Callable + +from ..trace import AgentTrace +from .base import TraceDimension + + +class GoalCompletionDimension(TraceDimension): + """Score whether the agent completed its goal. + + If ``checker`` is given it decides pass (1.0) or fail (0.0). Otherwise if + ``required_substrings`` is given the score is the fraction present in the + final answer (case-insensitive). Otherwise the score is 1.0 when the + final answer is non-empty, else 0.0. + """ + + def __init__( + self, + checker: Callable[[AgentTrace], bool] | None = None, + required_substrings: list[str] | None = None, + *, + threshold: float = 1.0, + name: str = "goal_completion", + ) -> None: + super().__init__(threshold=threshold, name=name) + self._checker = checker + self._required_substrings = required_substrings + + def evaluate(self, trace: AgentTrace) -> tuple[float, str]: + if self._checker is not None: + passed = self._checker(trace) + return (1.0 if passed else 0.0), f"checker -> {passed}" + if self._required_substrings: + answer = (trace.final_answer or "").lower() + present = [s for s in self._required_substrings if s.lower() in answer] + score = len(present) / len(self._required_substrings) + return score, f"{len(present)}/{len(self._required_substrings)} substrings present" + has_answer = bool(trace.final_answer) + return (1.0 if has_answer else 0.0), f"final answer present: {has_answer}" diff --git a/src/llm_evalgate/agentic/dimensions/step_efficiency.py b/src/llm_evalgate/agentic/dimensions/step_efficiency.py new file mode 100644 index 0000000..8ca9aad --- /dev/null +++ b/src/llm_evalgate/agentic/dimensions/step_efficiency.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +from ..trace import AgentTrace +from .base import TraceDimension + + +class StepEfficiencyDimension(TraceDimension): + """Score the agent on staying within a step budget without repeating work. + + Base score is 1.0 when ``num_steps <= max_steps``, else ``max_steps / + num_steps``. When ``penalize_repeats`` is set, repeated identical + ``(name, sorted-args)`` tool calls subtract ``repeats / total_calls`` + from the score, floored at 0.0. + """ + + def __init__( + self, + max_steps: int, + *, + penalize_repeats: bool = True, + threshold: float = 1.0, + name: str = "step_efficiency", + ) -> None: + super().__init__(threshold=threshold, name=name) + self._max_steps = max_steps + self._penalize_repeats = penalize_repeats + + def evaluate(self, trace: AgentTrace) -> tuple[float, str]: + num_steps = trace.num_steps + if num_steps <= self._max_steps: + score = 1.0 + else: + score = self._max_steps / num_steps + repeats = 0 + if self._penalize_repeats: + calls = trace.all_tool_calls() + seen: set[tuple[str, tuple[tuple[str, object], ...]]] = set() + for call in calls: + key = (call.name, tuple(sorted(call.args.items()))) + if key in seen: + repeats += 1 + else: + seen.add(key) + if repeats and calls: + score = max(0.0, score - repeats / len(calls)) + detail = f"{num_steps} steps vs budget {self._max_steps}; {repeats} repeated calls" + return score, detail diff --git a/src/llm_evalgate/agentic/dimensions/tool_arg_validity.py b/src/llm_evalgate/agentic/dimensions/tool_arg_validity.py new file mode 100644 index 0000000..62590c3 --- /dev/null +++ b/src/llm_evalgate/agentic/dimensions/tool_arg_validity.py @@ -0,0 +1,44 @@ +from __future__ import annotations + +from collections.abc import Callable + +from ..trace import AgentTrace +from .base import TraceDimension + + +class ToolArgValidityDimension(TraceDimension): + """Score the fraction of tool calls with valid arguments. + + A tool call is invalid if ``call.error`` is set, or a validator exists + for ``call.name`` and returns False on ``call.args``. Score is the + fraction of valid calls. With no tool calls the dimension passes. + """ + + def __init__( + self, + validators: dict[str, Callable[[dict], bool]] | None = None, + *, + threshold: float = 1.0, + name: str = "tool_arg_validity", + ) -> None: + super().__init__(threshold=threshold, name=name) + self._validators = validators or {} + + def evaluate(self, trace: AgentTrace) -> tuple[float, str]: + calls = trace.all_tool_calls() + if not calls: + return 1.0, "no tool calls" + reasons: list[str] = [] + for call in calls: + if call.error is not None: + reasons.append(f"{call.name}: error {call.error}") + continue + validator = self._validators.get(call.name) + if validator is not None and not validator(call.args): + reasons.append(f"{call.name}: invalid args {call.args}") + invalid = len(reasons) + score = (len(calls) - invalid) / len(calls) + detail = f"{invalid}/{len(calls)} invalid" + if reasons: + detail += "; " + "; ".join(reasons) + return score, detail diff --git a/src/llm_evalgate/agentic/dimensions/tool_selection.py b/src/llm_evalgate/agentic/dimensions/tool_selection.py new file mode 100644 index 0000000..16e9038 --- /dev/null +++ b/src/llm_evalgate/agentic/dimensions/tool_selection.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +from ..trace import AgentTrace +from .base import TraceDimension + +_MODES = ("subset", "exact", "order") + + +class ToolSelectionDimension(TraceDimension): + """Score whether the agent used the expected tools. + + ``mode`` controls the comparison: + + - ``"subset"``: fraction of expected tools present in the trace. + - ``"exact"``: 1.0 iff the set of used tools equals the expected set. + - ``"order"``: 1.0 iff expected tools appear as an in-order subsequence. + """ + + def __init__( + self, + expected_tools: list[str], + *, + mode: str = "subset", + threshold: float = 1.0, + name: str = "tool_selection", + ) -> None: + if mode not in _MODES: + raise ValueError(f"unknown mode {mode!r}; expected one of {_MODES}") + super().__init__(threshold=threshold, name=name) + self._expected = expected_tools + self._mode = mode + + def evaluate(self, trace: AgentTrace) -> tuple[float, str]: + actual = trace.tool_names() + actual_set = set(actual) + matched = [tool for tool in self._expected if tool in actual_set] + missing = [tool for tool in self._expected if tool not in actual_set] + if self._mode == "exact": + score = 1.0 if set(self._expected) == actual_set else 0.0 + elif self._mode == "order": + score = 1.0 if self._is_subsequence(self._expected, actual) else 0.0 + else: + score = len(matched) / len(self._expected) if self._expected else 1.0 + detail = f"mode={self._mode}; matched={matched}; missing={missing}" + return score, detail + + @staticmethod + def _is_subsequence(expected: list[str], actual: list[str]) -> bool: + iterator = iter(actual) + return all(tool in iterator for tool in expected) diff --git a/src/llm_evalgate/agentic/dimensions/trajectory_coherence.py b/src/llm_evalgate/agentic/dimensions/trajectory_coherence.py new file mode 100644 index 0000000..5c2a9bf --- /dev/null +++ b/src/llm_evalgate/agentic/dimensions/trajectory_coherence.py @@ -0,0 +1,27 @@ +from __future__ import annotations + +from ...judge import JudgeDimension +from ..trace import AgentTrace +from .base import TraceDimension + + +class TrajectoryCoherenceDimension(TraceDimension): + """Grade long-horizon reasoning coherence with an LLM-as-judge. + + The trace is serialized with ``AgentTrace.to_text`` and passed to the + injected ``JudgeDimension``, whose ``(score, detail)`` is returned + directly. Injecting the judge keeps this dimension offline-testable. + """ + + def __init__( + self, + judge: JudgeDimension, + *, + threshold: float = 0.6, + name: str = "trajectory_coherence", + ) -> None: + super().__init__(threshold=threshold, name=name) + self._judge = judge + + def evaluate(self, trace: AgentTrace) -> tuple[float, str]: + return self._judge.evaluate(trace.to_text()) diff --git a/src/llm_evalgate/agentic/harness.py b/src/llm_evalgate/agentic/harness.py new file mode 100644 index 0000000..1356073 --- /dev/null +++ b/src/llm_evalgate/agentic/harness.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +from dataclasses import dataclass + +from ..eval.dimension import DimensionResult +from .dimensions.base import TraceDimension +from .trace import AgentTrace + + +@dataclass +class AgentEvalReport: + passed: bool + results: dict[str, DimensionResult] + trace: AgentTrace + + @property + def failures(self) -> dict[str, DimensionResult]: + return {name: r for name, r in self.results.items() if not r.passed} + + def __str__(self) -> str: + lines = [f"AgentEvalReport: {'PASS' if self.passed else 'FAIL'}"] + for name, result in self.results.items(): + status = "PASS" if result.passed else "FAIL" + lines.append(f" {status} [{name}] score={result.score:.3f} - {result.detail}") + return "\n".join(lines) + + +class AgentEvalHarness: + """Run a list of trace dimensions against a trace and produce a report. + + Usage:: + + harness = AgentEvalHarness([ + ToolSelectionDimension(expected_tools=["search", "calculator"]), + StepEfficiencyDimension(max_steps=5), + ]) + report = harness.run(trace) + if not report.passed: + raise ValueError(str(report)) + """ + + def __init__(self, dimensions: list[TraceDimension]) -> None: + if not dimensions: + raise ValueError("AgentEvalHarness requires at least one dimension.") + self._dimensions = dimensions + + def run(self, trace: AgentTrace) -> AgentEvalReport: + results: dict[str, DimensionResult] = {} + for dim in self._dimensions: + results[dim.name] = dim.run(trace) + passed = all(r.passed for r in results.values()) + return AgentEvalReport(passed=passed, results=results, trace=trace) diff --git a/src/llm_evalgate/agentic/trace.py b/src/llm_evalgate/agentic/trace.py new file mode 100644 index 0000000..4d906b6 --- /dev/null +++ b/src/llm_evalgate/agentic/trace.py @@ -0,0 +1,82 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + + +@dataclass +class ToolCall: + name: str + args: dict[str, Any] = field(default_factory=dict) + result: Any = None + error: str | None = None + + +@dataclass +class AgentStep: + thought: str | None = None + tool_calls: list[ToolCall] = field(default_factory=list) + observation: str | None = None + + +@dataclass +class AgentTrace: + goal: str + steps: list[AgentStep] = field(default_factory=list) + final_answer: str | None = None + + @property + def num_steps(self) -> int: + return len(self.steps) + + def tool_names(self) -> list[str]: + """Tool names in execution order across all steps.""" + return [call.name for step in self.steps for call in step.tool_calls] + + def all_tool_calls(self) -> list[ToolCall]: + return [call for step in self.steps for call in step.tool_calls] + + def to_text(self) -> str: + """Readable serialization a judge can read.""" + lines = [f"Goal: {self.goal}"] + for index, step in enumerate(self.steps, start=1): + lines.append(f"Step {index}:") + if step.thought is not None: + lines.append(f" Thought: {step.thought}") + for call in step.tool_calls: + lines.append(f" Tool: {call.name} args={call.args}") + if call.error is not None: + lines.append(f" Error: {call.error}") + elif call.result is not None: + lines.append(f" Result: {call.result}") + if step.observation is not None: + lines.append(f" Observation: {step.observation}") + if self.final_answer is not None: + lines.append(f"Final answer: {self.final_answer}") + return "\n".join(lines) + + @classmethod + def from_dict(cls, data: dict) -> AgentTrace: + steps = [] + for step_data in data.get("steps", []): + tool_calls = [ + ToolCall( + name=call["name"], + args=call.get("args", {}), + result=call.get("result"), + error=call.get("error"), + ) + for call in step_data.get("tool_calls", []) + ] + steps.append( + AgentStep( + thought=step_data.get("thought"), + tool_calls=tool_calls, + observation=step_data.get("observation"), + ) + ) + return cls( + goal=data["goal"], + steps=steps, + final_answer=data.get("final_answer"), + ) diff --git a/src/llm_evalgate/bench/__init__.py b/src/llm_evalgate/bench/__init__.py new file mode 100644 index 0000000..0212af6 --- /dev/null +++ b/src/llm_evalgate/bench/__init__.py @@ -0,0 +1,26 @@ +from .metrics import ( + accuracy, + all_metrics, + cohen_kappa, + confusion_counts, + f1, + precision, + recall, + regression_catch_rate, +) +from .runner import BenchmarkResult, BenchmarkRunner, BenchSample, load_golden + +__all__ = [ + "BenchSample", + "BenchmarkResult", + "BenchmarkRunner", + "load_golden", + "confusion_counts", + "accuracy", + "precision", + "recall", + "f1", + "cohen_kappa", + "regression_catch_rate", + "all_metrics", +] diff --git a/src/llm_evalgate/bench/datasets/golden_eval.jsonl b/src/llm_evalgate/bench/datasets/golden_eval.jsonl new file mode 100644 index 0000000..e63c7f3 --- /dev/null +++ b/src/llm_evalgate/bench/datasets/golden_eval.jsonl @@ -0,0 +1,24 @@ +{"text": "title: Weekly Update\nsummary: The team shipped the new login flow and fixed two bugs. Users can now reset passwords on their own.", "label": true} +{"text": "title: Release Notes\nsummary: We added dark mode and made the app start faster. Tap the gear to turn it on.", "label": true} +{"text": "title: Status Report\nsummary: Sales rose last week. The store sold more shoes than hats. The team is happy.", "label": true} +{"text": "title: Meeting Recap\nsummary: We met on Monday and picked a date. The launch is set for June. Everyone agreed.", "label": true} +{"text": "title: Project Brief\nsummary: This tool sends a daily email. It lists open tasks. You can reply to mark them done.", "label": true} +{"text": "title: Help Article\nsummary: To add a card, open settings and tap add card. Then type the number and save it.", "label": true} +{"text": "title: Onboarding Note\nsummary: Welcome to the team. Read the guide, set up your laptop, and say hi in the chat.", "label": true} +{"text": "title: Bug Summary\nsummary: The app crashed when the list was empty. We added a check. It works now.", "label": true} +{"text": "title: Support FAQ\nsummary: If your order is late, check the tracking link first. If it still does not show, write to us and we will help.", "label": true} +{"text": "title: Sprint Goal\nsummary: This sprint we will finish the search box and the filter menu. We also want to fix the slow page load.", "label": true} +{"text": "title: Welcome Email\nsummary: Thanks for joining. Set your password, add a photo, and pick your topics. Then you are all set to start.", "label": true} +{"text": "title: Change Log\nsummary: We fixed a typo on the home page and made the buttons larger. The app should feel easier to tap now.", "label": true} +{"text": "title: Vendor Note\nsummary: This pricing is confidential and should not be shared with the client during the call today.", "label": false, "meta": {"reason": "blocklist"}} +{"text": "title: Draft Email\nsummary: Please review the attached file. It is marked internal use only so keep it within the team.", "label": false, "meta": {"reason": "blocklist"}} +{"text": "title: Legal Notice\nsummary: The settlement amount was [REDACTED] and the parties agreed to keep the terms private for now.", "label": false, "meta": {"reason": "blocklist"}} +{"text": "summary: The quarterly numbers look strong and the team hit every target this period without much trouble.", "label": false, "meta": {"reason": "schema"}} +{"text": "title: Standalone Heading\nThis paragraph has a title but no summary field, so the schema check should flag it as missing.", "label": false, "meta": {"reason": "schema"}} +{"text": "Just a plain paragraph of text with no structured fields at all, which the schema gate should reject outright.", "label": false, "meta": {"reason": "schema"}} +{"text": "title: Architecture Overview\nsummary: Notwithstanding the aforementioned heterogeneous infrastructural interdependencies, the organizationally instantiated microservice orchestration necessitates comprehensively multifaceted reconfiguration methodologies.", "label": false, "meta": {"reason": "readability"}} +{"text": "title: Compliance Memo\nsummary: Pursuant to the aforementioned regulatory promulgations, stakeholders shall, contemporaneously and irrespective of jurisdictional heterogeneity, operationalize the enumerated remediations.", "label": false, "meta": {"reason": "readability"}} +{"text": "title: Capital of France\nsummary: The capital of France is Berlin, and it sits on the banks of the river Thames near the Alps.", "label": false, "meta": {"reason": "semantic"}} +{"text": "title: Water Facts\nsummary: Water is made of three parts gold and one part helium. It boils at five degrees and freezes when hot.", "label": false, "meta": {"reason": "semantic"}} +{"text": "title: Company Update\nsummary: Our sales last year were ten trillion dollars. That is more than the whole world earns. We also have no real customers at all.", "label": false, "meta": {"reason": "semantic"}} +{"text": "title: Recipe Note\nsummary: To bake bread, freeze the flour for a week, then microwave the oven and serve the soup cold.", "label": false, "meta": {"reason": "semantic"}} diff --git a/src/llm_evalgate/bench/metrics.py b/src/llm_evalgate/bench/metrics.py new file mode 100644 index 0000000..8c8d830 --- /dev/null +++ b/src/llm_evalgate/bench/metrics.py @@ -0,0 +1,104 @@ +from __future__ import annotations + + +def _validate(predicted: list[bool], labels: list[bool]) -> None: + if len(predicted) != len(labels): + raise ValueError( + f"predicted and labels must be equal length; " + f"got {len(predicted)} and {len(labels)}" + ) + if not predicted: + raise ValueError("predicted and labels must be non-empty") + + +def confusion_counts(predicted: list[bool], labels: list[bool]) -> dict[str, int]: + """Return tp/fp/tn/fn counts with the positive class being ``True`` (pass).""" + _validate(predicted, labels) + tp = fp = tn = fn = 0 + for pred, label in zip(predicted, labels): + if pred and label: + tp += 1 + elif pred and not label: + fp += 1 + elif not pred and not label: + tn += 1 + else: + fn += 1 + return {"tp": tp, "fp": fp, "tn": tn, "fn": fn} + + +def accuracy(predicted: list[bool], labels: list[bool]) -> float: + _validate(predicted, labels) + correct = sum(1 for pred, label in zip(predicted, labels) if pred == label) + return correct / len(labels) + + +def precision(predicted: list[bool], labels: list[bool]) -> float: + """Precision for the positive class (pass). Zero denominator returns 0.0.""" + counts = confusion_counts(predicted, labels) + denom = counts["tp"] + counts["fp"] + if denom == 0: + return 0.0 + return counts["tp"] / denom + + +def recall(predicted: list[bool], labels: list[bool]) -> float: + """Recall for the positive class (pass). Zero denominator returns 0.0.""" + counts = confusion_counts(predicted, labels) + denom = counts["tp"] + counts["fn"] + if denom == 0: + return 0.0 + return counts["tp"] / denom + + +def f1(predicted: list[bool], labels: list[bool]) -> float: + """Harmonic mean of precision and recall. Zero denominator returns 0.0.""" + prec = precision(predicted, labels) + rec = recall(predicted, labels) + denom = prec + rec + if denom == 0: + return 0.0 + return 2 * prec * rec / denom + + +def cohen_kappa(predicted: list[bool], labels: list[bool]) -> float: + """Cohen's kappa for two binary raters, corrected for chance agreement. + + Zero expected-disagreement denominator returns 0.0. + """ + counts = confusion_counts(predicted, labels) + n = len(labels) + observed = (counts["tp"] + counts["tn"]) / n + pred_pos = (counts["tp"] + counts["fp"]) / n + label_pos = (counts["tp"] + counts["fn"]) / n + expected = pred_pos * label_pos + (1 - pred_pos) * (1 - label_pos) + denom = 1 - expected + if denom == 0: + return 0.0 + return (observed - expected) / denom + + +def regression_catch_rate(predicted: list[bool], labels: list[bool]) -> float: + """Fraction of regressions (label ``False``) correctly predicted ``False``. + + This is recall on the negative class. If there are no regressions in + ``labels`` the denominator is zero and this returns 0.0; it is the + caller's job to note that no regressions were present. + """ + counts = confusion_counts(predicted, labels) + denom = counts["tn"] + counts["fp"] + if denom == 0: + return 0.0 + return counts["tn"] / denom + + +def all_metrics(predicted: list[bool], labels: list[bool]) -> dict[str, float]: + """Compute the full metric suite in one pass-style call.""" + return { + "accuracy": accuracy(predicted, labels), + "precision": precision(predicted, labels), + "recall": recall(predicted, labels), + "f1": f1(predicted, labels), + "cohen_kappa": cohen_kappa(predicted, labels), + "regression_catch_rate": regression_catch_rate(predicted, labels), + } diff --git a/src/llm_evalgate/bench/runner.py b/src/llm_evalgate/bench/runner.py new file mode 100644 index 0000000..eb4bc48 --- /dev/null +++ b/src/llm_evalgate/bench/runner.py @@ -0,0 +1,83 @@ +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +from .metrics import all_metrics + + +@dataclass +class BenchSample: + text: str + label: bool + meta: dict[str, Any] = field(default_factory=dict) + + +@dataclass +class BenchmarkResult: + predicted: list[bool] + labels: list[bool] + metrics: dict[str, float] + n: int + + def table(self) -> str: + """Render an aligned metric/value table with values to 3 decimals.""" + width = max(len(name) for name in self.metrics) + lines = [f"n={self.n}"] + for name, value in self.metrics.items(): + lines.append(f"{name.ljust(width)} {value:.3f}") + return "\n".join(lines) + + +class BenchmarkRunner: + """Grade labeled samples and score the grader against the human labels. + + ``grader`` is anything with a ``run(text)`` method returning an object + that has a ``.passed`` bool, i.e. a ``Dimension`` or an ``EvalHarness``. + """ + + def __init__(self, grader: Any) -> None: + self._grader = grader + + def run(self, samples: list[BenchSample]) -> BenchmarkResult: + if not samples: + raise ValueError("BenchmarkRunner requires at least one sample.") + predicted = [self._grader.run(sample.text).passed for sample in samples] + labels = [sample.label for sample in samples] + metrics = all_metrics(predicted, labels) + return BenchmarkResult( + predicted=predicted, + labels=labels, + metrics=metrics, + n=len(samples), + ) + + +def load_golden(path: str | None = None) -> list[BenchSample]: + """Load a JSONL golden dataset into ``BenchSample`` objects. + + Defaults to the bundled ``datasets/golden_eval.jsonl``. Each line is a + JSON object ``{"text": ..., "label": bool, "meta": {...}}`` where ``meta`` + is optional. + """ + if path is None: + dataset_path = Path(__file__).resolve().parent / "datasets" / "golden_eval.jsonl" + else: + dataset_path = Path(path) + samples: list[BenchSample] = [] + with dataset_path.open(encoding="utf-8") as handle: + for line in handle: + line = line.strip() + if not line: + continue + record = json.loads(line) + samples.append( + BenchSample( + text=record["text"], + label=record["label"], + meta=record.get("meta", {}), + ) + ) + return samples diff --git a/src/llm_evalgate/eval/harness.py b/src/llm_evalgate/eval/harness.py index f3df2ae..f2797e5 100644 --- a/src/llm_evalgate/eval/harness.py +++ b/src/llm_evalgate/eval/harness.py @@ -19,7 +19,7 @@ def __str__(self) -> str: lines = [f"EvalReport: {'PASS' if self.passed else 'FAIL'}"] for name, result in self.results.items(): status = "PASS" if result.passed else "FAIL" - lines.append(f" {status} [{name}] score={result.score:.3f} — {result.detail}") + lines.append(f" {status} [{name}] score={result.score:.3f} - {result.detail}") return "\n".join(lines) diff --git a/src/llm_evalgate/judge/__init__.py b/src/llm_evalgate/judge/__init__.py new file mode 100644 index 0000000..ecca7cf --- /dev/null +++ b/src/llm_evalgate/judge/__init__.py @@ -0,0 +1,17 @@ +from .base import ( + DEFAULT_PROMPT_TEMPLATE, + JudgeVerdict, + parse_verdict, + render_prompt, +) +from .dimension import JudgeDimension, JuryDimension, anthropic_judge + +__all__ = [ + "DEFAULT_PROMPT_TEMPLATE", + "JudgeDimension", + "JudgeVerdict", + "JuryDimension", + "anthropic_judge", + "parse_verdict", + "render_prompt", +] diff --git a/src/llm_evalgate/judge/base.py b/src/llm_evalgate/judge/base.py new file mode 100644 index 0000000..c405cd5 --- /dev/null +++ b/src/llm_evalgate/judge/base.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +import re +from dataclasses import dataclass + +DEFAULT_PROMPT_TEMPLATE = ( + "You are an impartial grader. Grade the text below against the rubric.\n\n" + "Rubric:\n{rubric}\n\n" + "Grade on a 1 to {scale} scale, where 1 is worst and {scale} is best.\n" + "Reply with exactly two lines and nothing else:\n" + "SCORE: \n" + "REASON: \n\n" + "Text:\n{text}\n" +) + +_SCORE_RE = re.compile(r"score\s*:\s*([-+]?\d*\.?\d+)", re.IGNORECASE) +_REASON_RE = re.compile(r"reason\s*:\s*(.+)", re.IGNORECASE | re.DOTALL) + + +@dataclass(frozen=True) +class JudgeVerdict: + score: float + reason: str + raw: str + + +def parse_verdict(raw: str, scale: int) -> JudgeVerdict: + """Parse a model response into a JudgeVerdict, never raising. + + Looks for ``SCORE:`` followed by a number and ``REASON:`` text. The score + is normalized to [0.0, 1.0] by dividing by ``scale`` and clamping. When + ``scale`` is 1 the value is treated as an already-normalized 0-1 float. + If no score can be parsed, returns score 0.0 with an explanatory reason. + """ + score_match = _SCORE_RE.search(raw) + reason_match = _REASON_RE.search(raw) + reason = reason_match.group(1).strip() if reason_match else "" + if score_match is None: + return JudgeVerdict( + score=0.0, + reason="parse failure: no SCORE found in judge response", + raw=raw, + ) + value = float(score_match.group(1)) + normalized = value if scale == 1 else value / scale + clamped = max(0.0, min(1.0, normalized)) + if not reason: + reason = "no REASON found in judge response" + return JudgeVerdict(score=clamped, reason=reason, raw=raw) + + +def render_prompt(template: str, rubric: str, scale: int, text: str) -> str: + return template.format(rubric=rubric, scale=scale, text=text) diff --git a/src/llm_evalgate/judge/dimension.py b/src/llm_evalgate/judge/dimension.py new file mode 100644 index 0000000..d27380a --- /dev/null +++ b/src/llm_evalgate/judge/dimension.py @@ -0,0 +1,123 @@ +from __future__ import annotations + +import statistics +from collections.abc import Callable +from typing import Any + +from ..eval.dimension import Dimension +from .base import DEFAULT_PROMPT_TEMPLATE, JudgeVerdict, parse_verdict, render_prompt + +_AGGREGATES = ("mean", "median", "majority") + + +class JudgeDimension(Dimension): + """Grade text with an injected model callable (LLM-as-judge). + + ``complete`` maps a rendered prompt string to a model response string. + Injecting it keeps the dimension offline-testable: tests pass a fake + callable and no network is touched. + """ + + def __init__( + self, + complete: Callable[[str], str], + rubric: str, + *, + scale: int = 5, + threshold: float = 0.6, + name: str = "judge", + prompt_template: str | None = None, + cache: dict[str, JudgeVerdict] | None = None, + ) -> None: + super().__init__(threshold=threshold, name=name) + self._complete = complete + self._rubric = rubric + self._scale = scale + self._template = prompt_template or DEFAULT_PROMPT_TEMPLATE + self._cache = cache + + def evaluate(self, text: str) -> tuple[float, str]: + prompt = render_prompt(self._template, self._rubric, self._scale, text) + if self._cache is not None and prompt in self._cache: + verdict = self._cache[prompt] + return verdict.score, verdict.reason + try: + raw = self._complete(prompt) + except Exception as exc: + return 0.0, f"judge error: {exc}" + verdict = parse_verdict(raw, self._scale) + if self._cache is not None: + self._cache[prompt] = verdict + return verdict.score, verdict.reason + + +class JuryDimension(Dimension): + """Aggregate several JudgeDimension verdicts into one score. + + ``aggregate`` is one of ``"mean"``, ``"median"``, or ``"majority"``. The + detail string reports the aggregate score, the per-judge scores, and an + agreement signal: the population standard deviation of the scores (lower + means more agreement), or the vote split for ``"majority"``. + """ + + def __init__( + self, + judges: list[JudgeDimension], + *, + aggregate: str = "mean", + threshold: float = 0.6, + name: str = "jury", + ) -> None: + if not judges: + raise ValueError("JuryDimension requires at least one judge.") + if aggregate not in _AGGREGATES: + raise ValueError( + f"unknown aggregate {aggregate!r}; expected one of {_AGGREGATES}" + ) + super().__init__(threshold=threshold, name=name) + self._judges = judges + self._aggregate = aggregate + + def evaluate(self, text: str) -> tuple[float, str]: + scores = [judge.evaluate(text)[0] for judge in self._judges] + per_judge = ", ".join(f"{s:.3f}" for s in scores) + if self._aggregate == "mean": + score = statistics.fmean(scores) + agreement = f"stdev={statistics.pstdev(scores):.3f}" + elif self._aggregate == "median": + score = statistics.median(scores) + agreement = f"stdev={statistics.pstdev(scores):.3f}" + else: + passing = sum(1 for s in scores if s >= self.threshold) + failing = len(scores) - passing + score = passing / len(scores) + agreement = f"votes={passing} pass / {failing} fail" + detail = ( + f"{self._aggregate} score={score:.3f}; " + f"per-judge=[{per_judge}]; {agreement}" + ) + return score, detail + + +def anthropic_judge(model: str, client: Any = None) -> Callable[[str], str]: + """Build a ``complete`` callable backed by the Anthropic Messages API. + + Requires the ``[judge]`` extra (the ``anthropic`` package). The import is + deferred to call time so the base package never hard-depends on it. If no + ``client`` is given, a default ``anthropic.Anthropic()`` is constructed. + The returned callable sends ``prompt`` as a single user message and returns + the text of the first content block. + """ + import anthropic + + active_client = client if client is not None else anthropic.Anthropic() + + def complete(prompt: str) -> str: + message = active_client.messages.create( + model=model, + max_tokens=512, + messages=[{"role": "user", "content": prompt}], + ) + return message.content[0].text + + return complete diff --git a/tests/test_agentic.py b/tests/test_agentic.py new file mode 100644 index 0000000..bafaf72 --- /dev/null +++ b/tests/test_agentic.py @@ -0,0 +1,253 @@ +import pytest + +from llm_evalgate.agentic import ( + AgentEvalHarness, + AgentStep, + AgentTrace, + GoalCompletionDimension, + StepEfficiencyDimension, + ToolArgValidityDimension, + ToolCall, + ToolSelectionDimension, + TrajectoryCoherenceDimension, +) +from llm_evalgate.judge import JudgeDimension + + +def _trace(steps, *, goal="goal", final_answer="done"): + return AgentTrace(goal=goal, steps=steps, final_answer=final_answer) + + +def _step(*names, **kwargs): + args = kwargs.get("args", {}) + return AgentStep(tool_calls=[ToolCall(name=n, args=args) for n in names]) + + +# --- ToolSelectionDimension --- + +def test_tool_selection_subset_partial(): + trace = _trace([_step("search")]) + dim = ToolSelectionDimension(["search", "calculator"], mode="subset") + result = dim.run(trace) + assert result.score == 0.5 + assert not result.passed + assert "calculator" in result.detail + + +def test_tool_selection_subset_full_passes(): + trace = _trace([_step("search", "calculator")]) + dim = ToolSelectionDimension(["search", "calculator"], mode="subset") + result = dim.run(trace) + assert result.passed + assert result.score == 1.0 + + +def test_tool_selection_exact_pass_and_fail(): + expected = ["search", "calculator"] + passing = _trace([_step("calculator", "search")]) + failing = _trace([_step("search", "calculator", "browser")]) + dim = ToolSelectionDimension(expected, mode="exact") + assert dim.run(passing).passed + assert not dim.run(failing).passed + + +def test_tool_selection_order_pass_and_fail(): + dim = ToolSelectionDimension(["search", "calculator"], mode="order") + in_order = _trace([_step("search"), _step("browser"), _step("calculator")]) + out_of_order = _trace([_step("calculator"), _step("search")]) + assert dim.run(in_order).passed + assert not dim.run(out_of_order).passed + + +def test_tool_selection_unknown_mode_raises(): + with pytest.raises(ValueError): + ToolSelectionDimension(["search"], mode="bogus") + + +# --- ToolArgValidityDimension --- + +def test_tool_arg_validity_validator_fails(): + trace = AgentTrace( + goal="g", + steps=[ + AgentStep(tool_calls=[ToolCall(name="search", args={"query": "x"})]), + AgentStep(tool_calls=[ToolCall(name="search", args={})]), + ], + ) + dim = ToolArgValidityDimension({"search": lambda args: bool(args.get("query"))}) + result = dim.run(trace) + assert result.score == 0.5 + assert not result.passed + assert "invalid" in result.detail + + +def test_tool_arg_validity_errored_call(): + trace = AgentTrace( + goal="g", + steps=[ + AgentStep(tool_calls=[ToolCall(name="search", args={"query": "x"})]), + AgentStep(tool_calls=[ToolCall(name="search", error="timeout")]), + ], + ) + dim = ToolArgValidityDimension() + result = dim.run(trace) + assert result.score == 0.5 + assert "timeout" in result.detail + + +def test_tool_arg_validity_no_calls_passes(): + trace = AgentTrace(goal="g", steps=[AgentStep(thought="thinking")]) + result = ToolArgValidityDimension().run(trace) + assert result.passed + assert "no tool calls" in result.detail + + +# --- StepEfficiencyDimension --- + +def test_step_efficiency_under_budget_passes(): + trace = _trace([_step("a"), _step("b")]) + result = StepEfficiencyDimension(max_steps=4).run(trace) + assert result.passed + assert result.score == 1.0 + + +def test_step_efficiency_over_budget_fails(): + trace = _trace([_step("a"), _step("b"), _step("c"), _step("d")]) + result = StepEfficiencyDimension(max_steps=2).run(trace) + assert not result.passed + assert result.score == 0.5 + + +def test_step_efficiency_penalizes_repeats(): + trace = AgentTrace( + goal="g", + steps=[ + AgentStep(tool_calls=[ToolCall(name="search", args={"q": "x"})]), + AgentStep(tool_calls=[ToolCall(name="search", args={"q": "x"})]), + ], + ) + result = StepEfficiencyDimension(max_steps=4).run(trace) + assert result.score == 0.5 + assert "1 repeated calls" in result.detail + + +def test_step_efficiency_no_penalty_when_disabled(): + trace = AgentTrace( + goal="g", + steps=[ + AgentStep(tool_calls=[ToolCall(name="search", args={"q": "x"})]), + AgentStep(tool_calls=[ToolCall(name="search", args={"q": "x"})]), + ], + ) + result = StepEfficiencyDimension(max_steps=4, penalize_repeats=False).run(trace) + assert result.score == 1.0 + + +# --- GoalCompletionDimension --- + +def test_goal_completion_checker(): + trace = _trace([_step("a")], final_answer="anything") + passing = GoalCompletionDimension(checker=lambda t: t.num_steps == 1) + failing = GoalCompletionDimension(checker=lambda t: t.num_steps == 5) + assert passing.run(trace).passed + assert not failing.run(trace).passed + + +def test_goal_completion_substrings(): + trace = _trace([_step("a")], final_answer="The answer is 68 Million people.") + dim = GoalCompletionDimension(required_substrings=["68 million", "missing"]) + result = dim.run(trace) + assert result.score == 0.5 + assert not result.passed + + +def test_goal_completion_empty_final_answer(): + trace = AgentTrace(goal="g", steps=[_step("a")], final_answer=None) + result = GoalCompletionDimension().run(trace) + assert not result.passed + assert result.score == 0.0 + + +# --- TrajectoryCoherenceDimension --- + +def test_trajectory_coherence_with_fake_judge(): + def complete(prompt: str) -> str: + return "SCORE: 5\nREASON: coherent" + + judge = JudgeDimension(complete=complete, rubric="Is the trajectory coherent?") + trace = _trace([_step("search"), _step("calculator")]) + result = TrajectoryCoherenceDimension(judge).run(trace) + assert result.passed + assert result.score == 1.0 + assert "coherent" in result.detail + + +# --- AgentEvalHarness --- + +def test_harness_empty_raises(): + with pytest.raises(ValueError): + AgentEvalHarness([]) + + +def test_harness_combined_pass(): + trace = _trace( + [_step("search"), _step("calculator")], + final_answer="68 million", + ) + harness = AgentEvalHarness( + [ + ToolSelectionDimension(["search", "calculator"], mode="subset"), + StepEfficiencyDimension(max_steps=4), + GoalCompletionDimension(required_substrings=["68 million"]), + ] + ) + report = harness.run(trace) + assert report.passed + assert not report.failures + + +def test_harness_combined_fail_and_str(): + trace = _trace([_step("search")], final_answer="68 million") + harness = AgentEvalHarness( + [ + ToolSelectionDimension(["search", "calculator"], mode="subset"), + GoalCompletionDimension(required_substrings=["68 million"]), + ] + ) + report = harness.run(trace) + assert not report.passed + assert "tool_selection" in report.failures + text = str(report) + assert text.startswith("AgentEvalReport: FAIL") + assert "[tool_selection]" in text + assert "[goal_completion]" in text + + +# --- AgentTrace round-trip --- + +def test_trace_from_dict_to_text_round_trip(): + data = { + "goal": "Find the answer.", + "steps": [ + { + "thought": "search first", + "tool_calls": [ + {"name": "search", "args": {"query": "answer"}, "result": "42"} + ], + "observation": "42", + }, + { + "tool_calls": [{"name": "calculator", "error": "boom"}], + }, + ], + "final_answer": "The answer is 42.", + } + trace = AgentTrace.from_dict(data) + assert trace.num_steps == 2 + assert trace.tool_names() == ["search", "calculator"] + assert trace.steps[1].thought is None + text = trace.to_text() + assert "Goal: Find the answer." in text + assert "Tool: search args=" in text + assert "Error: boom" in text + assert "Final answer: The answer is 42." in text diff --git a/tests/test_bench.py b/tests/test_bench.py new file mode 100644 index 0000000..04cee0e --- /dev/null +++ b/tests/test_bench.py @@ -0,0 +1,175 @@ +import math + +import pytest + +from llm_evalgate.bench import ( + BenchmarkResult, + BenchmarkRunner, + BenchSample, + accuracy, + all_metrics, + cohen_kappa, + confusion_counts, + f1, + load_golden, + precision, + recall, + regression_catch_rate, +) +from llm_evalgate.eval import EvalHarness +from llm_evalgate.eval.dimensions import ( + BlocklistDimension, + ReadabilityDimension, + SchemaComplianceDimension, +) + +# Hand-computed vectors: tp=2, fp=1, tn=2, fn=1. +PREDICTED = [True, True, False, False, True, False] +LABELS = [True, False, False, True, True, False] + + +# --- metrics correctness --- + +def test_confusion_counts(): + assert confusion_counts(PREDICTED, LABELS) == {"tp": 2, "fp": 1, "tn": 2, "fn": 1} + + +def test_accuracy(): + assert accuracy(PREDICTED, LABELS) == pytest.approx(4 / 6) + + +def test_precision_recall_f1(): + assert precision(PREDICTED, LABELS) == pytest.approx(2 / 3) + assert recall(PREDICTED, LABELS) == pytest.approx(2 / 3) + assert f1(PREDICTED, LABELS) == pytest.approx(2 / 3) + + +def test_regression_catch_rate(): + # Negative-class recall: tn / (tn + fp) = 2 / 3. + assert regression_catch_rate(PREDICTED, LABELS) == pytest.approx(2 / 3) + + +def test_cohen_kappa_partial(): + # observed=4/6, expected=0.5 -> kappa = (0.6667 - 0.5) / 0.5 = 0.3333. + assert cohen_kappa(PREDICTED, LABELS) == pytest.approx(1 / 3, abs=1e-9) + + +def test_cohen_kappa_perfect_agreement(): + preds = [True, False, True, False] + assert cohen_kappa(preds, preds) == pytest.approx(1.0) + + +def test_all_metrics_keys(): + metrics = all_metrics(PREDICTED, LABELS) + assert set(metrics) == { + "accuracy", + "precision", + "recall", + "f1", + "cohen_kappa", + "regression_catch_rate", + } + assert all(isinstance(v, float) for v in metrics.values()) + + +# --- zero-denominator behavior --- + +def test_precision_zero_denominator_returns_zero(): + # No positive predictions -> precision denominator is zero. + assert precision([False, False], [True, False]) == 0.0 + + +def test_regression_catch_rate_no_regressions_returns_zero(): + assert regression_catch_rate([True, True], [True, True]) == 0.0 + + +# --- input validation --- + +def test_length_mismatch_raises(): + with pytest.raises(ValueError): + accuracy([True, False], [True]) + + +def test_empty_inputs_raise(): + with pytest.raises(ValueError): + accuracy([], []) + + +# --- BenchmarkRunner over an in-memory dataset --- + +def test_runner_with_blocklist_dimension(): + samples = [ + BenchSample(text="This is a public document.", label=True), + BenchSample(text="This is an internal document.", label=False), + BenchSample(text="Another clean public note.", label=True), + ] + runner = BenchmarkRunner(BlocklistDimension(terms=["internal"])) + result = runner.run(samples) + assert result.predicted == [True, False, True] + assert result.labels == [True, False, True] + # All three predictions match their labels -> perfect scores. + assert result.metrics["accuracy"] == pytest.approx(1.0) + assert result.metrics["regression_catch_rate"] == pytest.approx(1.0) + assert result.n == 3 + + +def test_runner_empty_samples_raises(): + runner = BenchmarkRunner(BlocklistDimension(terms=["internal"])) + with pytest.raises(ValueError): + runner.run([]) + + +def test_benchmark_result_table_is_aligned(): + result = BenchmarkResult( + predicted=[True], + labels=[True], + metrics={"accuracy": 1.0, "f1": 0.5}, + n=1, + ) + table = result.table() + assert "n=1" in table + assert "1.000" in table + assert "0.500" in table + + +# --- golden dataset end to end --- + +def test_load_golden_non_empty(): + dataset = load_golden() + assert isinstance(dataset, list) + assert len(dataset) > 0 + assert all(isinstance(s, BenchSample) for s in dataset) + + +def test_runner_over_golden_dataset(): + dataset = load_golden() + runner = BenchmarkRunner(BlocklistDimension(terms=["confidential"])) + result = runner.run(dataset) + assert isinstance(result, BenchmarkResult) + assert result.n == len(dataset) + assert len(result.predicted) == len(dataset) + assert all(not math.isnan(v) for v in result.metrics.values()) + + +def test_golden_deterministic_metrics_are_locked(): + """Pin the README/article benchmark numbers to CI. + + The deterministic harness here is the canonical one from + examples/benchmark.py. These six values are quoted in the README and the + article, so an unpinned textstat bump or a dataset edit that moved them must + fail the suite rather than silently invalidate the writeup. + """ + harness = EvalHarness([ + BlocklistDimension(terms=["confidential", "internal use only", "[REDACTED]"]), + SchemaComplianceDimension(required_fields=["title:", "summary:"]), + ReadabilityDimension(threshold=0.2), + ]) + result = BenchmarkRunner(harness).run(load_golden()) + + assert result.n == 24 + assert result.metrics["accuracy"] == pytest.approx(0.833, abs=1e-3) + assert result.metrics["precision"] == pytest.approx(0.750, abs=1e-3) + assert result.metrics["recall"] == pytest.approx(1.000, abs=1e-3) + assert result.metrics["f1"] == pytest.approx(0.857, abs=1e-3) + assert result.metrics["cohen_kappa"] == pytest.approx(0.667, abs=1e-3) + assert result.metrics["regression_catch_rate"] == pytest.approx(0.667, abs=1e-3) diff --git a/tests/test_judge.py b/tests/test_judge.py new file mode 100644 index 0000000..826443a --- /dev/null +++ b/tests/test_judge.py @@ -0,0 +1,163 @@ +import pytest + +from llm_evalgate.eval import EvalHarness +from llm_evalgate.eval.dimensions import BlocklistDimension +from llm_evalgate.judge import ( + JudgeDimension, + JudgeVerdict, + JuryDimension, + parse_verdict, +) + + +def _fixed(response: str): + def complete(prompt: str) -> str: + return response + return complete + + +# --- JudgeDimension --- + +def test_judge_scores_and_passes(): + dim = JudgeDimension( + complete=_fixed("SCORE: 4\nREASON: clear and correct"), + rubric="Is the text clear and correct?", + scale=5, + threshold=0.6, + ) + result = dim.run("Some text under review.") + assert result.score == pytest.approx(0.8) + assert result.passed + assert "clear and correct" in result.detail + + +def test_judge_unparseable_scores_zero(): + dim = JudgeDimension( + complete=_fixed("I cannot grade this."), + rubric="Is the text clear?", + threshold=0.6, + ) + result = dim.run("Some text under review.") + assert result.score == 0.0 + assert not result.passed + assert "parse failure" in result.detail + + +def test_judge_callable_raising_is_graceful(): + def boom(prompt: str) -> str: + raise RuntimeError("model down") + + dim = JudgeDimension(complete=boom, rubric="Is the text clear?", threshold=0.6) + result = dim.run("Some text under review.") + assert result.score == 0.0 + assert not result.passed + assert "judge error" in result.detail + + +def test_judge_cache_dedupes_by_prompt(): + calls = {"n": 0} + + def counting(prompt: str) -> str: + calls["n"] += 1 + return "SCORE: 5\nREASON: excellent" + + cache: dict[str, JudgeVerdict] = {} + dim = JudgeDimension( + complete=counting, + rubric="Is the text excellent?", + cache=cache, + ) + dim.run("identical text") + dim.run("identical text") + assert calls["n"] == 1 + + +# --- parse_verdict --- + +def test_parse_verdict_scale_one_keeps_float(): + verdict = parse_verdict("SCORE: 0.9\nREASON: strong", scale=1) + assert verdict.score == pytest.approx(0.9) + assert verdict.reason == "strong" + assert verdict.raw == "SCORE: 0.9\nREASON: strong" + + +# --- JuryDimension --- + +def test_jury_mean_passes(): + judges = [ + JudgeDimension(complete=_fixed("SCORE: 5\nREASON: a"), rubric="r"), + JudgeDimension(complete=_fixed("SCORE: 4\nREASON: b"), rubric="r"), + JudgeDimension(complete=_fixed("SCORE: 3\nREASON: c"), rubric="r"), + ] + jury = JuryDimension(judges, aggregate="mean", threshold=0.6) + result = jury.run("text") + assert result.score == pytest.approx(0.8) + assert result.passed + assert "stdev=" in result.detail + + +def test_jury_median(): + judges = [ + JudgeDimension(complete=_fixed("SCORE: 5\nREASON: a"), rubric="r"), + JudgeDimension(complete=_fixed("SCORE: 4\nREASON: b"), rubric="r"), + JudgeDimension(complete=_fixed("SCORE: 3\nREASON: c"), rubric="r"), + ] + jury = JuryDimension(judges, aggregate="median", threshold=0.6) + result = jury.run("text") + assert result.score == pytest.approx(0.8) + assert result.passed + + +def test_jury_majority(): + judges = [ + JudgeDimension(complete=_fixed("SCORE: 5\nREASON: a"), rubric="r"), + JudgeDimension(complete=_fixed("SCORE: 4\nREASON: b"), rubric="r"), + JudgeDimension(complete=_fixed("SCORE: 3\nREASON: c"), rubric="r"), + ] + jury = JuryDimension(judges, aggregate="majority", threshold=0.6) + result = jury.run("text") + # All three judge scores (1.0, 0.8, 0.6) clear the 0.6 threshold. + assert result.score == pytest.approx(1.0) + assert result.passed + assert "votes=3 pass / 0 fail" in result.detail + + +def test_jury_majority_split(): + judges = [ + JudgeDimension(complete=_fixed("SCORE: 5\nREASON: a"), rubric="r"), + JudgeDimension(complete=_fixed("SCORE: 4\nREASON: b"), rubric="r"), + JudgeDimension(complete=_fixed("SCORE: 1\nREASON: c"), rubric="r"), + ] + jury = JuryDimension(judges, aggregate="majority", threshold=0.6) + result = jury.run("text") + # Scores 1.0 and 0.8 pass, 0.2 fails -> 2 of 3. + assert result.score == pytest.approx(2 / 3) + assert result.passed + assert "votes=2 pass / 1 fail" in result.detail + + +def test_jury_empty_raises(): + with pytest.raises(ValueError): + JuryDimension([]) + + +def test_jury_unknown_aggregate_raises(): + judges = [JudgeDimension(complete=_fixed("SCORE: 5\nREASON: a"), rubric="r")] + with pytest.raises(ValueError): + JuryDimension(judges, aggregate="harmonic") + + +# --- composition with EvalHarness --- + +def test_judge_composes_in_harness(): + judge = JudgeDimension( + complete=_fixed("SCORE: 4\nREASON: clear"), + rubric="Is the text clear?", + threshold=0.6, + ) + blocklist = BlocklistDimension(terms=["secret"]) + harness = EvalHarness([judge, blocklist]) + report = harness.run("This is a public document.") + assert report.passed + assert "judge" in report.results + assert report.results["judge"].score == pytest.approx(0.8)