diff --git a/src/agenteval/graders/__init__.py b/src/agenteval/graders/__init__.py index 6ed19fe..84ca3dd 100644 --- a/src/agenteval/graders/__init__.py +++ b/src/agenteval/graders/__init__.py @@ -29,12 +29,14 @@ def _ensure_registry() -> None: from agenteval.graders.regex import RegexGrader from agenteval.graders.semantic import SemanticGrader from agenteval.graders.tool_check import ToolCheckGrader + from agenteval.graders.trajectory import TrajectoryGrader _GRADER_REGISTRY.update({ "exact": ExactGrader, "contains": ContainsGrader, "regex": RegexGrader, "tool-check": ToolCheckGrader, + "trajectory": TrajectoryGrader, "llm-judge": LLMJudgeGrader, "custom": CustomGrader, "json_schema": JsonSchemaGrader, diff --git a/src/agenteval/graders/trajectory.py b/src/agenteval/graders/trajectory.py new file mode 100644 index 0000000..1655d9e --- /dev/null +++ b/src/agenteval/graders/trajectory.py @@ -0,0 +1,68 @@ +"""Trajectory (multi-step path) grader (#9). + +Grades the agent's tool-call PATH against an expected ordered trajectory. Unlike +``tool-check`` (presence / subsequence of tool *names*), this scores **path +adherence**: the ordered sequence of steps, penalizing extra/unexpected steps +and (optionally) inefficient paths. The score is graded (0..1) via the +longest-common-subsequence ratio, so a near-miss path gets partial credit — the +"Agent GPA" style of trajectory evaluation. The resulting GradeResult flows into +the run and (when federated) is recorded in AgentLens's hash chain like any other +verdict. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import List, Optional, Sequence + +from agenteval.models import AgentResult, EvalCase, GradeResult + + +def _lcs_len(a: Sequence, b: Sequence) -> int: + """Length of the longest common subsequence of two sequences (DP).""" + if not a or not b: + return 0 + prev = [0] * (len(b) + 1) + for x in a: + curr = [0] + for j, y in enumerate(b): + curr.append(prev[j] + 1 if x == y else max(prev[j + 1], curr[j])) + prev = curr + return prev[-1] + + +@dataclass +class TrajectoryGrader: + """Score the tool-call path against ``expected`` (ordered tool names). + + Falls back to ``case.expected['trajectory']`` then ``['tools_called']`` when + ``expected`` isn't set in config. ``allow_extra`` lets the actual path contain + extra steps without penalty (only the expected steps must appear in order); + otherwise extra steps lower the score. ``max_steps`` fails an over-long path. + """ + + expected: List[str] = field(default_factory=list) + allow_extra: bool = False + max_steps: Optional[int] = None + + async def grade(self, case: EvalCase, result: AgentResult) -> GradeResult: + expected = list(self.expected) or case.expected.get("trajectory") or case.expected.get("tools_called", []) + if not expected: + return GradeResult(passed=True, score=1.0, reason="No expected trajectory to check") + + actual = [t.get("name") for t in result.tools_called] + lcs = _lcs_len(expected, actual) + denom = len(expected) if self.allow_extra else max(len(expected), len(actual)) + score = round(lcs / denom, 4) if denom else 1.0 + + path_ok = (lcs == len(expected)) if self.allow_extra else (actual == expected) + steps_ok = self.max_steps is None or len(actual) <= self.max_steps + passed = path_ok and steps_ok + + if passed: + reason = "Trajectory matches expected path" + elif not steps_ok: + reason = f"Path too long: {len(actual)} steps > max {self.max_steps}" + else: + reason = f"Path adherence {score:.2f}; expected {expected}, got {actual}" + return GradeResult(passed=passed, score=score, reason=reason) diff --git a/tests/test_trajectory_grader.py b/tests/test_trajectory_grader.py new file mode 100644 index 0000000..6e88341 --- /dev/null +++ b/tests/test_trajectory_grader.py @@ -0,0 +1,69 @@ +"""Trajectory (multi-step path) grader (#9).""" + +import asyncio + +from agenteval.graders import get_grader +from agenteval.graders.trajectory import TrajectoryGrader, _lcs_len +from agenteval.models import AgentResult, EvalCase + + +def _case(expected=None) -> EvalCase: + return EvalCase(name="t", input="x", expected=expected or {}, grader="trajectory") + + +def _result(names) -> AgentResult: + return AgentResult(output="", tools_called=[{"name": n} for n in names]) + + +def _grade(grader, case, result): + return asyncio.run(grader.grade(case, result)) + + +def test_exact_path_passes(): + r = _grade(TrajectoryGrader(expected=["search", "read", "write"]), _case(), _result(["search", "read", "write"])) + assert r.passed and r.score == 1.0 + + +def test_reordered_path_fails_with_partial_credit(): + r = _grade(TrajectoryGrader(expected=["search", "read", "write"]), _case(), _result(["read", "search", "write"])) + assert not r.passed and 0.0 < r.score < 1.0 + + +def test_extra_steps_penalized_by_default(): + r = _grade(TrajectoryGrader(expected=["a", "b"]), _case(), _result(["a", "x", "b", "y"])) + assert not r.passed and r.score == 0.5 # LCS 2 / max(2,4) + + +def test_allow_extra_passes_when_expected_in_order(): + r = _grade(TrajectoryGrader(expected=["a", "b"], allow_extra=True), _case(), _result(["a", "x", "b", "y"])) + assert r.passed and r.score == 1.0 + + +def test_max_steps_fails_an_overlong_path(): + r = _grade(TrajectoryGrader(expected=["a"], allow_extra=True, max_steps=2), _case(), _result(["a", "b", "c"])) + assert not r.passed and "too long" in r.reason + + +def test_missing_step_partial_credit(): + r = _grade(TrajectoryGrader(expected=["a", "b", "c"]), _case(), _result(["a", "c"])) + assert not r.passed and r.score == round(2 / 3, 4) + + +def test_empty_expected_passes(): + assert _grade(TrajectoryGrader(), _case(), _result([])).passed + + +def test_falls_back_to_case_expected_trajectory(): + r = _grade(TrajectoryGrader(), _case({"trajectory": ["a", "b"]}), _result(["a", "b"])) + assert r.passed + + +def test_registered_in_grader_registry(): + g = get_grader("trajectory", {"expected": ["a"]}) + assert isinstance(g, TrajectoryGrader) + + +def test_lcs_len(): + assert _lcs_len(["a", "b", "c"], ["a", "c"]) == 2 + assert _lcs_len([], ["a"]) == 0 + assert _lcs_len(["a", "b"], ["b", "a"]) == 1