diff --git a/src/agenteval/commands/__init__.py b/src/agenteval/commands/__init__.py index 0a5425f..3c52daf 100644 --- a/src/agenteval/commands/__init__.py +++ b/src/agenteval/commands/__init__.py @@ -22,6 +22,7 @@ report, run, trends, + verify, worker, ) @@ -44,6 +45,7 @@ examples, dashboard, trends, + verify, ] diff --git a/src/agenteval/commands/verify.py b/src/agenteval/commands/verify.py new file mode 100644 index 0000000..74a96f7 --- /dev/null +++ b/src/agenteval/commands/verify.py @@ -0,0 +1,65 @@ +"""The 'verify-evidence' command — independently verify eval evidence (#10).""" + +from __future__ import annotations + +import json + +import click + + +def register(cli: click.Group, helpers: dict) -> None: + """Register the verify-evidence command on the CLI group.""" + + @cli.command(name="verify-evidence") + @click.option("--run-id", default=None, help="Eval run id (resolves to session eval-).") + @click.option("--session-id", default=None, help="Explicit AgentLens session id (overrides --run-id).") + @click.option("--agentlens-server", envvar="AGENTLENS_SERVER", default="", help="AgentLens base URL.") + @click.option("--agentlens-api-key", envvar="AGENTLENS_API_KEY", default="", help="AgentLens API key (audit/admin role).") + @click.option("--format", "fmt", default="text", type=click.Choice(["text", "json"]), show_default=True) + def verify_evidence(run_id, session_id, agentlens_server, agentlens_api_key, fmt): + """Independently verify a run's eval evidence in AgentLens. + + Re-walks the session's hash chain server-side and confirms it is intact and + non-empty. Because ``eval_result`` can't be client-inserted, a verified + eval session proves the evidence is server-authored — you can verify it, + but you can't forge it. Exits non-zero if verification fails. + + agenteval verify-evidence --run-id RUN_ID \\ + --agentlens-server https://lens --agentlens-api-key $KEY + """ + import agenteval.cli as _cli_mod + _fail = _cli_mod._fail + + from agenteval.verify import ( + VerifyError, + session_id_for_run, + verify_eval_evidence, + ) + + sid = session_id or (session_id_for_run(run_id) if run_id else None) + if not sid: + _fail("Provide --run-id or --session-id.") + if not agentlens_server: + _fail("Provide --agentlens-server (or set AGENTLENS_SERVER).") + if not agentlens_api_key: + _fail("Provide --agentlens-api-key (or set AGENTLENS_API_KEY).") + + assert sid is not None # _fail() raises; narrows the Optional for type-checkers + try: + result = verify_eval_evidence(server=agentlens_server, api_key=agentlens_api_key, session_id=sid) + except VerifyError as e: + _fail(str(e)) + + if fmt == "json": + click.echo(json.dumps(result, indent=2)) + else: + mark = "OK" if result["verified"] else "FAIL" + click.echo( + f"[{mark}] session {sid}: verified={result['verified']} " + f"(chain={result['chainVerified']}, sessions={result['sessionsVerified']})" + ) + if result["brokenChains"]: + click.echo(f" broken chains: {result['brokenChains']}") + + if not result["verified"]: + raise SystemExit(1) diff --git a/src/agenteval/verify.py b/src/agenteval/verify.py new file mode 100644 index 0000000..3227fe4 --- /dev/null +++ b/src/agenteval/verify.py @@ -0,0 +1,71 @@ +"""Independently verify federated eval evidence in AgentLens (#10). + +The agenteval->AgentLens federation records each run as a server-authoritative, +hash-chained ``eval_result`` under session ``eval-``. That event type is +excluded from AgentLens's client ingest enum, so a client CANNOT forge one — a +verified eval session is therefore genuine eval evidence. This module asks +AgentLens to re-walk the session's hash chain (``GET /api/audit/verify``) and +confirms it is intact and non-empty. You can verify; you can't forge. +""" + +from __future__ import annotations + +from typing import Any, Dict + +import httpx + + +class VerifyError(Exception): + """Raised when the verification request to AgentLens cannot be completed.""" + + +def session_id_for_run(run_id: str) -> str: + """The synthetic AgentLens session id the federation uses for a run.""" + return f"eval-{run_id}" + + +def verify_eval_evidence( + *, + server: str, + api_key: str, + session_id: str, + timeout: float = 10.0, +) -> Dict[str, Any]: + """Verify one session's eval-evidence chain via AgentLens. + + Calls ``GET /api/audit/verify?sessionId=...`` (needs an AgentLens API key with + an audit/admin role). Returns a dict with ``verified`` (the chain is intact + AND the session exists — ``sessionsVerified >= 1``, so an absent session that + verifies *vacuously* is reported unverified), plus the underlying chain + fields. Raises :class:`VerifyError` on a transport/HTTP failure. + """ + url = server.rstrip("/") + "/api/audit/verify" + try: + resp = httpx.get( + url, + params={"sessionId": session_id}, + headers={"Authorization": f"Bearer {api_key}"}, + timeout=timeout, + ) + except httpx.HTTPError as e: + raise VerifyError(f"request to AgentLens failed: {e}") from e + if resp.status_code != 200: + raise VerifyError(f"AgentLens returned HTTP {resp.status_code}: {resp.text[:200]}") + try: + data = resp.json() + except ValueError as e: + raise VerifyError(f"AgentLens response was not JSON: {e}") from e + + chain_verified = bool(data.get("verified")) + sessions = int(data.get("sessionsVerified") or 0) + return { + "sessionId": session_id, + # A session that doesn't exist verifies vacuously — require >= 1 so + # "verify a run that was never federated" is reported as unverified. + "verified": chain_verified and sessions >= 1, + "chainVerified": chain_verified, + "sessionsVerified": sessions, + "firstHash": data.get("firstHash"), + "lastHash": data.get("lastHash"), + "brokenChains": data.get("brokenChains") or [], + } diff --git a/tests/test_verify_evidence.py b/tests/test_verify_evidence.py new file mode 100644 index 0000000..c28f30f --- /dev/null +++ b/tests/test_verify_evidence.py @@ -0,0 +1,71 @@ +"""Independent eval-evidence verify (#10).""" + +import httpx +import pytest + +from agenteval import verify as V + + +class _Resp: + def __init__(self, status_code=200, payload=None, text=""): + self.status_code = status_code + self._payload = payload if payload is not None else {} + self.text = text + + def json(self): + return self._payload + + +def test_session_id_for_run(): + assert V.session_id_for_run("abc123") == "eval-abc123" + + +def test_verified_chain_with_session(monkeypatch): + monkeypatch.setattr(V.httpx, "get", lambda *a, **k: _Resp(payload={ + "verified": True, "sessionsVerified": 1, "firstHash": "h0", "lastHash": "h1", "brokenChains": [], + })) + r = V.verify_eval_evidence(server="https://lens", api_key="k", session_id="eval-x") + assert r["verified"] is True + assert r["chainVerified"] is True and r["sessionsVerified"] == 1 + assert r["lastHash"] == "h1" + + +def test_absent_session_is_unverified_not_vacuous(monkeypatch): + # Chain "verifies" with no sessions → must be reported unverified (nothing to attest). + monkeypatch.setattr(V.httpx, "get", lambda *a, **k: _Resp(payload={"verified": True, "sessionsVerified": 0})) + r = V.verify_eval_evidence(server="https://lens", api_key="k", session_id="eval-missing") + assert r["verified"] is False + assert r["chainVerified"] is True + + +def test_broken_chain_is_unverified(monkeypatch): + monkeypatch.setattr(V.httpx, "get", lambda *a, **k: _Resp(payload={ + "verified": False, "sessionsVerified": 1, "brokenChains": [{"sessionId": "eval-x"}], + })) + r = V.verify_eval_evidence(server="https://lens", api_key="k", session_id="eval-x") + assert r["verified"] is False + assert r["brokenChains"] + + +def test_non_200_raises_verifyerror(monkeypatch): + monkeypatch.setattr(V.httpx, "get", lambda *a, **k: _Resp(status_code=403, text="Forbidden")) + with pytest.raises(V.VerifyError): + V.verify_eval_evidence(server="https://lens", api_key="bad", session_id="eval-x") + + +def test_transport_error_raises_verifyerror(monkeypatch): + def boom(*a, **k): + raise httpx.ConnectError("down") + monkeypatch.setattr(V.httpx, "get", boom) + with pytest.raises(V.VerifyError): + V.verify_eval_evidence(server="https://lens", api_key="k", session_id="eval-x") + + +def test_command_registered(): + from click.testing import CliRunner + + from agenteval.cli import cli + + res = CliRunner().invoke(cli, ["verify-evidence", "--help"]) + assert res.exit_code == 0 + assert "verify a run's eval evidence" in res.output.lower()