feat(graders): trajectory / multi-step path grader (#9)

src/agenteval/graders/__init__.py

@@ -29,12 +29,14 @@ def _ensure_registry() -> None:
     from agenteval.graders.regex import RegexGrader
     from agenteval.graders.semantic import SemanticGrader
     from agenteval.graders.tool_check import ToolCheckGrader
+    from agenteval.graders.trajectory import TrajectoryGrader
 
     _GRADER_REGISTRY.update({
         "exact": ExactGrader,
         "contains": ContainsGrader,
         "regex": RegexGrader,
         "tool-check": ToolCheckGrader,
+        "trajectory": TrajectoryGrader,
         "llm-judge": LLMJudgeGrader,
         "custom": CustomGrader,
         "json_schema": JsonSchemaGrader,

src/agenteval/graders/trajectory.py

@@ -0,0 +1,68 @@
+"""Trajectory (multi-step path) grader (#9).
+
+Grades the agent's tool-call PATH against an expected ordered trajectory. Unlike
+``tool-check`` (presence / subsequence of tool *names*), this scores **path
+adherence**: the ordered sequence of steps, penalizing extra/unexpected steps
+and (optionally) inefficient paths. The score is graded (0..1) via the
+longest-common-subsequence ratio, so a near-miss path gets partial credit — the
+"Agent GPA" style of trajectory evaluation. The resulting GradeResult flows into
+the run and (when federated) is recorded in AgentLens's hash chain like any other
+verdict.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import List, Optional, Sequence
+
+from agenteval.models import AgentResult, EvalCase, GradeResult
+
+
+def _lcs_len(a: Sequence, b: Sequence) -> int:
+    """Length of the longest common subsequence of two sequences (DP)."""
+    if not a or not b:
+        return 0
+    prev = [0] * (len(b) + 1)
+    for x in a:
+        curr = [0]
+        for j, y in enumerate(b):
+            curr.append(prev[j] + 1 if x == y else max(prev[j + 1], curr[j]))
+        prev = curr
+    return prev[-1]
+
+
+@dataclass
+class TrajectoryGrader:
+    """Score the tool-call path against ``expected`` (ordered tool names).
+
+    Falls back to ``case.expected['trajectory']`` then ``['tools_called']`` when
+    ``expected`` isn't set in config. ``allow_extra`` lets the actual path contain
+    extra steps without penalty (only the expected steps must appear in order);
+    otherwise extra steps lower the score. ``max_steps`` fails an over-long path.
+    """
+
+    expected: List[str] = field(default_factory=list)
+    allow_extra: bool = False
+    max_steps: Optional[int] = None
+
+    async def grade(self, case: EvalCase, result: AgentResult) -> GradeResult:
+        expected = list(self.expected) or case.expected.get("trajectory") or case.expected.get("tools_called", [])
+        if not expected:
+            return GradeResult(passed=True, score=1.0, reason="No expected trajectory to check")
+
+        actual = [t.get("name") for t in result.tools_called]
+        lcs = _lcs_len(expected, actual)
+        denom = len(expected) if self.allow_extra else max(len(expected), len(actual))
+        score = round(lcs / denom, 4) if denom else 1.0
+
+        path_ok = (lcs == len(expected)) if self.allow_extra else (actual == expected)
+        steps_ok = self.max_steps is None or len(actual) <= self.max_steps
+        passed = path_ok and steps_ok
+
+        if passed:
+            reason = "Trajectory matches expected path"
+        elif not steps_ok:
+            reason = f"Path too long: {len(actual)} steps > max {self.max_steps}"
+        else:
+            reason = f"Path adherence {score:.2f}; expected {expected}, got {actual}"
+        return GradeResult(passed=passed, score=score, reason=reason)

tests/test_trajectory_grader.py

@@ -0,0 +1,69 @@
+"""Trajectory (multi-step path) grader (#9)."""
+
+import asyncio
+
+from agenteval.graders import get_grader
+from agenteval.graders.trajectory import TrajectoryGrader, _lcs_len
+from agenteval.models import AgentResult, EvalCase
+
+
+def _case(expected=None) -> EvalCase:
+    return EvalCase(name="t", input="x", expected=expected or {}, grader="trajectory")
+
+
+def _result(names) -> AgentResult:
+    return AgentResult(output="", tools_called=[{"name": n} for n in names])
+
+
+def _grade(grader, case, result):
+    return asyncio.run(grader.grade(case, result))
+
+
+def test_exact_path_passes():
+    r = _grade(TrajectoryGrader(expected=["search", "read", "write"]), _case(), _result(["search", "read", "write"]))
+    assert r.passed and r.score == 1.0
+
+
+def test_reordered_path_fails_with_partial_credit():
+    r = _grade(TrajectoryGrader(expected=["search", "read", "write"]), _case(), _result(["read", "search", "write"]))
+    assert not r.passed and 0.0 < r.score < 1.0
+
+
+def test_extra_steps_penalized_by_default():
+    r = _grade(TrajectoryGrader(expected=["a", "b"]), _case(), _result(["a", "x", "b", "y"]))
+    assert not r.passed and r.score == 0.5  # LCS 2 / max(2,4)
+
+
+def test_allow_extra_passes_when_expected_in_order():
+    r = _grade(TrajectoryGrader(expected=["a", "b"], allow_extra=True), _case(), _result(["a", "x", "b", "y"]))
+    assert r.passed and r.score == 1.0
+
+
+def test_max_steps_fails_an_overlong_path():
+    r = _grade(TrajectoryGrader(expected=["a"], allow_extra=True, max_steps=2), _case(), _result(["a", "b", "c"]))
+    assert not r.passed and "too long" in r.reason
+
+
+def test_missing_step_partial_credit():
+    r = _grade(TrajectoryGrader(expected=["a", "b", "c"]), _case(), _result(["a", "c"]))
+    assert not r.passed and r.score == round(2 / 3, 4)
+
+
+def test_empty_expected_passes():
+    assert _grade(TrajectoryGrader(), _case(), _result([])).passed
+
+
+def test_falls_back_to_case_expected_trajectory():
+    r = _grade(TrajectoryGrader(), _case({"trajectory": ["a", "b"]}), _result(["a", "b"]))
+    assert r.passed
+
+
+def test_registered_in_grader_registry():
+    g = get_grader("trajectory", {"expected": ["a"]})
+    assert isinstance(g, TrajectoryGrader)
+
+
+def test_lcs_len():
+    assert _lcs_len(["a", "b", "c"], ["a", "c"]) == 2
+    assert _lcs_len([], ["a"]) == 0
+    assert _lcs_len(["a", "b"], ["b", "a"]) == 1