From 4714a1d51ab6189da7a6813088f7b06d2708974b Mon Sep 17 00:00:00 2001
From: Jonathan Amponsah <82057176+mgalore@users.noreply.github.com>
Date: Sat, 5 Sep 2026 10:06:17 +0000
Subject: [PATCH] feat(pytest): write failed suite artifacts

---
 README.md                     | 11 +++++++++++
 src/spanlite/pytest_plugin.py | 23 +++++++++++++++++++++++
 tests/test_pytest_plugin.py   | 34 ++++++++++++++++++++++++++++++++++
 3 files changed, 68 insertions(+)
 create mode 100644 tests/test_pytest_plugin.py

diff --git a/README.md b/README.md
index eb4b088..481cf3a 100644
--- a/README.md
+++ b/README.md
@@ -62,6 +62,17 @@ suite.run([RagCase("q1", "which tree", gold_ids=("doc-neem",))], my_agent)
 assert suite.pass_rate() == 1.0
 ```
 
+When an eval row fails, the bundled pytest plugin can save the stable JSON
+artifact for CI inspection without writing artifacts for passing suites:
+
+```python
+def test_retriever_quality(suite_artifact, tmp_path):
+    suite = Suite("rag", [RecallJudge()])
+    suite.run([RagCase("q1", "which tree", gold_ids=("doc-neem",))], my_agent)
+    suite_artifact(suite, tmp_path / "rag.json")
+    assert suite.pass_rate() == 1.0
+```
+
 Trace an existing call path the same way:
 
 ```python
diff --git a/src/spanlite/pytest_plugin.py b/src/spanlite/pytest_plugin.py
index 66d5371..dd18f7b 100644
--- a/src/spanlite/pytest_plugin.py
+++ b/src/spanlite/pytest_plugin.py
@@ -1,7 +1,12 @@
 from __future__ import annotations
 
+import json
+from collections.abc import Callable
+from pathlib import Path
+
 import pytest
 
+from spanlite.evals.base import Suite
 from spanlite.trace import MemorySink, Tracer
 
 
@@ -9,3 +14,21 @@
 def tracer() -> Tracer:
     """A tracer with a memory sink. Fake the clock in tests that care about time."""
     return Tracer("pytest", sinks=[MemorySink()])
+
+
+@pytest.fixture
+def suite_artifact() -> Callable[[Suite, str | Path], Path | None]:
+    """Write a readable Suite artifact only when one or more rows have failed."""
+
+    def write(suite: Suite, path: str | Path) -> Path | None:
+        if all(row.passed for row in suite.rows):
+            return None
+        artifact = Path(path)
+        artifact.parent.mkdir(parents=True, exist_ok=True)
+        artifact.write_text(
+            json.dumps(suite.to_json(), indent=2, sort_keys=True) + "\n",
+            encoding="utf-8",
+        )
+        return artifact
+
+    return write
diff --git a/tests/test_pytest_plugin.py b/tests/test_pytest_plugin.py
new file mode 100644
index 0000000..ac81742
--- /dev/null
+++ b/tests/test_pytest_plugin.py
@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+import json
+
+from spanlite import AgentCase, Output, Suite, TaskJudge
+
+
+def test_suite_artifact_writes_pretty_json_for_failed_rows(
+    tmp_path, suite_artifact
+) -> None:
+    suite = Suite("agent", [TaskJudge()])
+    suite.run(
+        [AgentCase("capital", "capital", expect="Delhi")],
+        lambda _case: Output(text="Mumbai"),
+    )
+
+    path = suite_artifact(suite, tmp_path / "agent.json")
+
+    assert path == tmp_path / "agent.json"
+    assert path.read_text(encoding="utf-8").startswith("{\n  \"cases\": 1,")
+    assert json.loads(path.read_text(encoding="utf-8")) == suite.to_json()
+
+
+def test_suite_artifact_skips_passing_rows(tmp_path, suite_artifact) -> None:
+    suite = Suite("agent", [TaskJudge()])
+    suite.run(
+        [AgentCase("capital", "capital", expect="Delhi")],
+        lambda _case: Output(text="Delhi"),
+    )
+
+    path = tmp_path / "agent.json"
+
+    assert suite_artifact(suite, path) is None
+    assert not path.exists()