diff --git a/src/agenteval/commands/__init__.py b/src/agenteval/commands/__init__.py index 3c52daf..9d756aa 100644 --- a/src/agenteval/commands/__init__.py +++ b/src/agenteval/commands/__init__.py @@ -21,6 +21,7 @@ profile, report, run, + suite_hash, trends, verify, worker, @@ -46,6 +47,7 @@ dashboard, trends, verify, + suite_hash, ] diff --git a/src/agenteval/commands/suite_hash.py b/src/agenteval/commands/suite_hash.py new file mode 100644 index 0000000..22d1e20 --- /dev/null +++ b/src/agenteval/commands/suite_hash.py @@ -0,0 +1,23 @@ +"""The 'suite-hash' command — print a suite's provenance hash (#11).""" + +from __future__ import annotations + +import click + + +def register(cli: click.Group, helpers: dict) -> None: + """Register the suite-hash command on the CLI group.""" + + @cli.command(name="suite-hash") + @click.argument("suite_file", type=click.Path(exists=True)) + def suite_hash(suite_file: str) -> None: + """Print the reproducibility content-hash of an eval suite (Art.10). + + Pin this in CI to detect dataset drift between approved runs: + + agenteval suite-hash suite.yaml + """ + from agenteval.loader import load_suite + from agenteval.provenance import suite_content_hash + + click.echo(suite_content_hash(load_suite(suite_file))) diff --git a/src/agenteval/provenance.py b/src/agenteval/provenance.py new file mode 100644 index 0000000..a3d69ec --- /dev/null +++ b/src/agenteval/provenance.py @@ -0,0 +1,45 @@ +"""Suite/dataset provenance hashing for reproducibility (EU AI Act Art.10) (#11). + +A deterministic content hash of an EvalSuite — its cases +(name/input/expected/grader/grader_config/tags) plus name/agent/defaults — so a +run can record EXACTLY which suite version produced it, and CI can detect drift +("the dataset changed since the last approved run"). Sorted-key canonical JSON +makes the hash reproducible across machines and runs. +""" + +from __future__ import annotations + +import hashlib +import json +from typing import Any + +from agenteval.models import EvalSuite + + +def _canonical(obj: Any) -> str: + return json.dumps(obj, sort_keys=True, separators=(",", ":"), default=str) + + +def suite_content_hash(suite: EvalSuite) -> str: + """Reproducible ``sha256:…`` fingerprint of a suite's content. + + Order-sensitive over cases (a reorder is a real change). Independent of how + the suite was loaded, so the same YAML/JSON always yields the same hash. + """ + body = { + "name": suite.name, + "agent": suite.agent, + "defaults": suite.defaults, + "cases": [ + { + "name": c.name, + "input": c.input, + "expected": c.expected, + "grader": c.grader, + "grader_config": c.grader_config, + "tags": list(c.tags), + } + for c in suite.cases + ], + } + return "sha256:" + hashlib.sha256(_canonical(body).encode("utf-8")).hexdigest() diff --git a/src/agenteval/runner.py b/src/agenteval/runner.py index bfb8225..968a348 100644 --- a/src/agenteval/runner.py +++ b/src/agenteval/runner.py @@ -187,11 +187,16 @@ async def _run_with_sem(index: int, case: EvalCase) -> None: total_tokens_out = sum(r.tokens_out for r in results) avg_latency = sum(r.latency_ms for r in results) / total if total else 0 + # Stamp the suite provenance hash (#11) so the run records exactly which + # suite version produced it (Art.10 reproducibility); flows through to the + # EU AI Act evidence module via run.config. + from agenteval.provenance import suite_content_hash + run = EvalRun( id=run_id or uuid.uuid4().hex[:12], suite=suite.name, agent_ref=suite.agent, - config=run_config or {}, + config={**(run_config or {}), "suite_hash": suite_content_hash(suite)}, results=results, summary={ "total": total, diff --git a/tests/test_provenance.py b/tests/test_provenance.py new file mode 100644 index 0000000..5659bb6 --- /dev/null +++ b/tests/test_provenance.py @@ -0,0 +1,49 @@ +"""Suite/dataset provenance hashing (#11).""" + +from agenteval.models import EvalCase, EvalSuite +from agenteval.provenance import suite_content_hash + + +def _case(name="c1", expected=None, grader="exact"): + return EvalCase(name=name, input="hi", expected=expected or {"value": "ok"}, grader=grader) + + +def _suite(cases=None, name="s", agent="a", defaults=None): + return EvalSuite(name=name, agent=agent, cases=cases or [_case()], defaults=defaults or {}) + + +def test_hash_is_sha256_prefixed_and_reproducible(): + s = _suite() + h1 = suite_content_hash(s) + h2 = suite_content_hash(_suite()) # rebuilt identical + assert h1.startswith("sha256:") + assert h1 == h2 + + +def test_hash_changes_when_a_case_changes(): + base = suite_content_hash(_suite([_case(expected={"value": "ok"})])) + changed = suite_content_hash(_suite([_case(expected={"value": "DIFFERENT"})])) + assert base != changed + + +def test_hash_changes_on_case_reorder(): + a = suite_content_hash(_suite([_case("c1"), _case("c2")])) + b = suite_content_hash(_suite([_case("c2"), _case("c1")])) + assert a != b # order-sensitive + + +def test_hash_changes_on_name_agent_defaults(): + base = suite_content_hash(_suite()) + assert suite_content_hash(_suite(name="other")) != base + assert suite_content_hash(_suite(agent="other")) != base + assert suite_content_hash(_suite(defaults={"timeout": 5})) != base + + +def test_command_registered(): + from click.testing import CliRunner + + from agenteval.cli import cli + + res = CliRunner().invoke(cli, ["suite-hash", "--help"]) + assert res.exit_code == 0 + assert "content-hash" in res.output.lower()