feat(verify): independent eval-evidence verify command (#10)

src/agenteval/commands/__init__.py

@@ -22,6 +22,7 @@
     report,
     run,
     trends,
+    verify,
     worker,
 )
 
@@ -44,6 +45,7 @@
     examples,
     dashboard,
     trends,
+    verify,
 ]
 
 

src/agenteval/commands/verify.py

@@ -0,0 +1,65 @@
+"""The 'verify-evidence' command — independently verify eval evidence (#10)."""
+
+from __future__ import annotations
+
+import json
+
+import click
+
+
+def register(cli: click.Group, helpers: dict) -> None:
+    """Register the verify-evidence command on the CLI group."""
+
+    @cli.command(name="verify-evidence")
+    @click.option("--run-id", default=None, help="Eval run id (resolves to session eval-<run-id>).")
+    @click.option("--session-id", default=None, help="Explicit AgentLens session id (overrides --run-id).")
+    @click.option("--agentlens-server", envvar="AGENTLENS_SERVER", default="", help="AgentLens base URL.")
+    @click.option("--agentlens-api-key", envvar="AGENTLENS_API_KEY", default="", help="AgentLens API key (audit/admin role).")
+    @click.option("--format", "fmt", default="text", type=click.Choice(["text", "json"]), show_default=True)
+    def verify_evidence(run_id, session_id, agentlens_server, agentlens_api_key, fmt):
+        """Independently verify a run's eval evidence in AgentLens.
+
+        Re-walks the session's hash chain server-side and confirms it is intact and
+        non-empty. Because ``eval_result`` can't be client-inserted, a verified
+        eval session proves the evidence is server-authored — you can verify it,
+        but you can't forge it. Exits non-zero if verification fails.
+
+          agenteval verify-evidence --run-id RUN_ID \\
+            --agentlens-server https://lens --agentlens-api-key $KEY
+        """
+        import agenteval.cli as _cli_mod
+        _fail = _cli_mod._fail
+
+        from agenteval.verify import (
+            VerifyError,
+            session_id_for_run,
+            verify_eval_evidence,
+        )
+
+        sid = session_id or (session_id_for_run(run_id) if run_id else None)
+        if not sid:
+            _fail("Provide --run-id or --session-id.")
+        if not agentlens_server:
+            _fail("Provide --agentlens-server (or set AGENTLENS_SERVER).")
+        if not agentlens_api_key:
+            _fail("Provide --agentlens-api-key (or set AGENTLENS_API_KEY).")
+
+        assert sid is not None  # _fail() raises; narrows the Optional for type-checkers
+        try:
+            result = verify_eval_evidence(server=agentlens_server, api_key=agentlens_api_key, session_id=sid)
+        except VerifyError as e:
+            _fail(str(e))
+
+        if fmt == "json":
+            click.echo(json.dumps(result, indent=2))
+        else:
+            mark = "OK" if result["verified"] else "FAIL"
+            click.echo(
+                f"[{mark}] session {sid}: verified={result['verified']} "
+                f"(chain={result['chainVerified']}, sessions={result['sessionsVerified']})"
+            )
+            if result["brokenChains"]:
+                click.echo(f"  broken chains: {result['brokenChains']}")
+
+        if not result["verified"]:
+            raise SystemExit(1)

src/agenteval/verify.py

@@ -0,0 +1,71 @@
+"""Independently verify federated eval evidence in AgentLens (#10).
+
+The agenteval->AgentLens federation records each run as a server-authoritative,
+hash-chained ``eval_result`` under session ``eval-<run id>``. That event type is
+excluded from AgentLens's client ingest enum, so a client CANNOT forge one — a
+verified eval session is therefore genuine eval evidence. This module asks
+AgentLens to re-walk the session's hash chain (``GET /api/audit/verify``) and
+confirms it is intact and non-empty. You can verify; you can't forge.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict
+
+import httpx
+
+
+class VerifyError(Exception):
+    """Raised when the verification request to AgentLens cannot be completed."""
+
+
+def session_id_for_run(run_id: str) -> str:
+    """The synthetic AgentLens session id the federation uses for a run."""
+    return f"eval-{run_id}"
+
+
+def verify_eval_evidence(
+    *,
+    server: str,
+    api_key: str,
+    session_id: str,
+    timeout: float = 10.0,
+) -> Dict[str, Any]:
+    """Verify one session's eval-evidence chain via AgentLens.
+
+    Calls ``GET /api/audit/verify?sessionId=...`` (needs an AgentLens API key with
+    an audit/admin role). Returns a dict with ``verified`` (the chain is intact
+    AND the session exists — ``sessionsVerified >= 1``, so an absent session that
+    verifies *vacuously* is reported unverified), plus the underlying chain
+    fields. Raises :class:`VerifyError` on a transport/HTTP failure.
+    """
+    url = server.rstrip("/") + "/api/audit/verify"
+    try:
+        resp = httpx.get(
+            url,
+            params={"sessionId": session_id},
+            headers={"Authorization": f"Bearer {api_key}"},
+            timeout=timeout,
+        )
+    except httpx.HTTPError as e:
+        raise VerifyError(f"request to AgentLens failed: {e}") from e
+    if resp.status_code != 200:
+        raise VerifyError(f"AgentLens returned HTTP {resp.status_code}: {resp.text[:200]}")
+    try:
+        data = resp.json()
+    except ValueError as e:
+        raise VerifyError(f"AgentLens response was not JSON: {e}") from e
+
+    chain_verified = bool(data.get("verified"))
+    sessions = int(data.get("sessionsVerified") or 0)
+    return {
+        "sessionId": session_id,
+        # A session that doesn't exist verifies vacuously — require >= 1 so
+        # "verify a run that was never federated" is reported as unverified.
+        "verified": chain_verified and sessions >= 1,
+        "chainVerified": chain_verified,
+        "sessionsVerified": sessions,
+        "firstHash": data.get("firstHash"),
+        "lastHash": data.get("lastHash"),
+        "brokenChains": data.get("brokenChains") or [],
+    }

tests/test_verify_evidence.py

@@ -0,0 +1,71 @@
+"""Independent eval-evidence verify (#10)."""
+
+import httpx
+import pytest
+
+from agenteval import verify as V
+
+
+class _Resp:
+    def __init__(self, status_code=200, payload=None, text=""):
+        self.status_code = status_code
+        self._payload = payload if payload is not None else {}
+        self.text = text
+
+    def json(self):
+        return self._payload
+
+
+def test_session_id_for_run():
+    assert V.session_id_for_run("abc123") == "eval-abc123"
+
+
+def test_verified_chain_with_session(monkeypatch):
+    monkeypatch.setattr(V.httpx, "get", lambda *a, **k: _Resp(payload={
+        "verified": True, "sessionsVerified": 1, "firstHash": "h0", "lastHash": "h1", "brokenChains": [],
+    }))
+    r = V.verify_eval_evidence(server="https://lens", api_key="k", session_id="eval-x")
+    assert r["verified"] is True
+    assert r["chainVerified"] is True and r["sessionsVerified"] == 1
+    assert r["lastHash"] == "h1"
+
+
+def test_absent_session_is_unverified_not_vacuous(monkeypatch):
+    # Chain "verifies" with no sessions → must be reported unverified (nothing to attest).
+    monkeypatch.setattr(V.httpx, "get", lambda *a, **k: _Resp(payload={"verified": True, "sessionsVerified": 0}))
+    r = V.verify_eval_evidence(server="https://lens", api_key="k", session_id="eval-missing")
+    assert r["verified"] is False
+    assert r["chainVerified"] is True
+
+
+def test_broken_chain_is_unverified(monkeypatch):
+    monkeypatch.setattr(V.httpx, "get", lambda *a, **k: _Resp(payload={
+        "verified": False, "sessionsVerified": 1, "brokenChains": [{"sessionId": "eval-x"}],
+    }))
+    r = V.verify_eval_evidence(server="https://lens", api_key="k", session_id="eval-x")
+    assert r["verified"] is False
+    assert r["brokenChains"]
+
+
+def test_non_200_raises_verifyerror(monkeypatch):
+    monkeypatch.setattr(V.httpx, "get", lambda *a, **k: _Resp(status_code=403, text="Forbidden"))
+    with pytest.raises(V.VerifyError):
+        V.verify_eval_evidence(server="https://lens", api_key="bad", session_id="eval-x")
+
+
+def test_transport_error_raises_verifyerror(monkeypatch):
+    def boom(*a, **k):
+        raise httpx.ConnectError("down")
+    monkeypatch.setattr(V.httpx, "get", boom)
+    with pytest.raises(V.VerifyError):
+        V.verify_eval_evidence(server="https://lens", api_key="k", session_id="eval-x")
+
+
+def test_command_registered():
+    from click.testing import CliRunner
+
+    from agenteval.cli import cli
+
+    res = CliRunner().invoke(cli, ["verify-evidence", "--help"])
+    assert res.exit_code == 0
+    assert "verify a run's eval evidence" in res.output.lower()