feat(provenance): suite/dataset content hashing for reproducibility (#11)

src/agenteval/commands/__init__.py

@@ -21,6 +21,7 @@
     profile,
     report,
     run,
+    suite_hash,
     trends,
     verify,
     worker,
@@ -46,6 +47,7 @@
     dashboard,
     trends,
     verify,
+    suite_hash,
 ]
 
 

src/agenteval/commands/suite_hash.py

@@ -0,0 +1,23 @@
+"""The 'suite-hash' command — print a suite's provenance hash (#11)."""
+
+from __future__ import annotations
+
+import click
+
+
+def register(cli: click.Group, helpers: dict) -> None:
+    """Register the suite-hash command on the CLI group."""
+
+    @cli.command(name="suite-hash")
+    @click.argument("suite_file", type=click.Path(exists=True))
+    def suite_hash(suite_file: str) -> None:
+        """Print the reproducibility content-hash of an eval suite (Art.10).
+
+        Pin this in CI to detect dataset drift between approved runs:
+
+          agenteval suite-hash suite.yaml
+        """
+        from agenteval.loader import load_suite
+        from agenteval.provenance import suite_content_hash
+
+        click.echo(suite_content_hash(load_suite(suite_file)))

src/agenteval/provenance.py

@@ -0,0 +1,45 @@
+"""Suite/dataset provenance hashing for reproducibility (EU AI Act Art.10) (#11).
+
+A deterministic content hash of an EvalSuite — its cases
+(name/input/expected/grader/grader_config/tags) plus name/agent/defaults — so a
+run can record EXACTLY which suite version produced it, and CI can detect drift
+("the dataset changed since the last approved run"). Sorted-key canonical JSON
+makes the hash reproducible across machines and runs.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from typing import Any
+
+from agenteval.models import EvalSuite
+
+
+def _canonical(obj: Any) -> str:
+    return json.dumps(obj, sort_keys=True, separators=(",", ":"), default=str)
+
+
+def suite_content_hash(suite: EvalSuite) -> str:
+    """Reproducible ``sha256:…`` fingerprint of a suite's content.
+
+    Order-sensitive over cases (a reorder is a real change). Independent of how
+    the suite was loaded, so the same YAML/JSON always yields the same hash.
+    """
+    body = {
+        "name": suite.name,
+        "agent": suite.agent,
+        "defaults": suite.defaults,
+        "cases": [
+            {
+                "name": c.name,
+                "input": c.input,
+                "expected": c.expected,
+                "grader": c.grader,
+                "grader_config": c.grader_config,
+                "tags": list(c.tags),
+            }
+            for c in suite.cases
+        ],
+    }
+    return "sha256:" + hashlib.sha256(_canonical(body).encode("utf-8")).hexdigest()

src/agenteval/runner.py

@@ -187,11 +187,16 @@ async def _run_with_sem(index: int, case: EvalCase) -> None:
     total_tokens_out = sum(r.tokens_out for r in results)
     avg_latency = sum(r.latency_ms for r in results) / total if total else 0
 
+    # Stamp the suite provenance hash (#11) so the run records exactly which
+    # suite version produced it (Art.10 reproducibility); flows through to the
+    # EU AI Act evidence module via run.config.
+    from agenteval.provenance import suite_content_hash
+
     run = EvalRun(
         id=run_id or uuid.uuid4().hex[:12],
         suite=suite.name,
         agent_ref=suite.agent,
-        config=run_config or {},
+        config={**(run_config or {}), "suite_hash": suite_content_hash(suite)},
         results=results,
         summary={
             "total": total,

tests/test_provenance.py

@@ -0,0 +1,49 @@
+"""Suite/dataset provenance hashing (#11)."""
+
+from agenteval.models import EvalCase, EvalSuite
+from agenteval.provenance import suite_content_hash
+
+
+def _case(name="c1", expected=None, grader="exact"):
+    return EvalCase(name=name, input="hi", expected=expected or {"value": "ok"}, grader=grader)
+
+
+def _suite(cases=None, name="s", agent="a", defaults=None):
+    return EvalSuite(name=name, agent=agent, cases=cases or [_case()], defaults=defaults or {})
+
+
+def test_hash_is_sha256_prefixed_and_reproducible():
+    s = _suite()
+    h1 = suite_content_hash(s)
+    h2 = suite_content_hash(_suite())  # rebuilt identical
+    assert h1.startswith("sha256:")
+    assert h1 == h2
+
+
+def test_hash_changes_when_a_case_changes():
+    base = suite_content_hash(_suite([_case(expected={"value": "ok"})]))
+    changed = suite_content_hash(_suite([_case(expected={"value": "DIFFERENT"})]))
+    assert base != changed
+
+
+def test_hash_changes_on_case_reorder():
+    a = suite_content_hash(_suite([_case("c1"), _case("c2")]))
+    b = suite_content_hash(_suite([_case("c2"), _case("c1")]))
+    assert a != b  # order-sensitive
+
+
+def test_hash_changes_on_name_agent_defaults():
+    base = suite_content_hash(_suite())
+    assert suite_content_hash(_suite(name="other")) != base
+    assert suite_content_hash(_suite(agent="other")) != base
+    assert suite_content_hash(_suite(defaults={"timeout": 5})) != base
+
+
+def test_command_registered():
+    from click.testing import CliRunner
+
+    from agenteval.cli import cli
+
+    res = CliRunner().invoke(cli, ["suite-hash", "--help"])
+    assert res.exit_code == 0
+    assert "content-hash" in res.output.lower()