feat(promote): freeze failing suite runs for local CI replay

README.md

@@ -86,6 +86,60 @@ with t.span("answer", "llm") as span:
 print(t.summary())
 ```
 
+## Promote failures → CI replay
+
+When a suite row fails in CI or locally, freeze the cases and recorded outputs
+into a **regression pack**, then replay them after you change the agent:
+
+```python
+from pathlib import Path
+from spanlite import (
+    AgentCase,
+    Output,
+    Suite,
+    TaskJudge,
+    assert_no_worse,
+    load_regression,
+    promote,
+    replay,
+)
+
+cases = [AgentCase("capital", "capital of India", expect="Delhi")]
+outputs = {}
+
+def agent(case):
+    out = Output(text="Mumbai")  # buggy
+    outputs[case.id] = out
+    return out
+
+suite = Suite("geo", [TaskJudge()])
+suite.run(cases, agent)
+pack_path = promote(suite, cases, outputs, Path("tests/regressions"))
+
+# Later — same judges, new agent (or frozen replay):
+pack = load_regression(pack_path)
+# Deterministic re-score of the recorded failure:
+replayed = replay(pack, [TaskJudge()])
+assert not replayed.rows[0].passed
+
+# Or run your fixed agent and gate CI:
+fixed = Suite("geo", [TaskJudge()])
+fixed.run(
+    [AgentCase("capital", "capital of India", expect="Delhi")],
+    lambda _c: Output(text="Delhi"),
+)
+assert_no_worse(fixed, pack)
+```
+
+Inspect a pack without Python:
+
+```bash
+python -m spanlite show tests/regressions/geo-capital.json
+python -m spanlite check tests/regressions/geo-capital.json
+```
+
+Packs are stable JSON (`sort_keys`, no timestamps). No new dependencies.
+
 ## What it looks like
 
 `t.summary()`:

examples/promote_replay.py

@@ -0,0 +1,49 @@
+"""Promote a failing suite row, then replay it locally (no cloud)."""
+
+from pathlib import Path
+
+from spanlite import (
+    AgentCase,
+    Output,
+    Suite,
+    TaskJudge,
+    assert_no_worse,
+    load_regression,
+    promote,
+    replay,
+)
+
+
+def buggy_agent(case):
+    return Output(text="Mumbai")
+
+
+def fixed_agent(case):
+    return Output(text="New Delhi")
+
+
+def main() -> None:
+    cases = [AgentCase("capital", "capital of India?", expect="Delhi")]
+    outputs = {}
+
+    def capturing(case):
+        out = buggy_agent(case)
+        outputs[case.id] = out
+        return out
+
+    suite = Suite("geo", [TaskJudge()])
+    suite.run(cases, capturing)
+    pack_path = promote(suite, cases, outputs, Path("/tmp/spanlite-regressions"))
+    print("wrote", pack_path)
+
+    pack = load_regression(pack_path)
+    print("replay pass_rate", replay(pack, [TaskJudge()]).pass_rate())
+
+    fixed = Suite("geo", [TaskJudge()])
+    fixed.run(cases, fixed_agent)
+    assert_no_worse(fixed, pack)
+    print("fixed agent cleared the regression gate")
+
+
+if __name__ == "__main__":
+    main()

pyproject.toml

@@ -30,6 +30,9 @@ Issues = "https://github.com/vk-ai/spanlite/issues"
 [project.entry-points.pytest11]
 spanlite = "spanlite.pytest_plugin"
 
+[project.scripts]
+spanlite-promote = "spanlite.promote:main"
+
 [tool.setuptools.packages.find]
 where = ["src"]
 

src/spanlite/__init__.py

@@ -6,6 +6,14 @@
 from spanlite.evals.skill import SkillCase, SchemaJudge, SelectionJudge, RefusalJudge
 from spanlite.evals.rag import RagCase, FaithfulnessJudge, MrrJudge, RecallJudge
 from spanlite.report import html_report, summary_table
+from spanlite.promote import (
+    PromoteError,
+    RegressionPack,
+    assert_no_worse,
+    load_regression,
+    promote,
+    replay,
+)
 
 __all__ = [
     "AgentCase",
@@ -19,9 +27,11 @@
     "MemorySink",
     "MrrJudge",
     "Output",
+    "PromoteError",
     "RagCase",
     "RecallJudge",
     "RefusalJudge",
+    "RegressionPack",
     "SchemaJudge",
     "Score",
     "SelectionJudge",
@@ -30,7 +40,11 @@
     "Suite",
     "TaskJudge",
     "Tracer",
+    "assert_no_worse",
     "html_report",
+    "load_regression",
+    "promote",
+    "replay",
     "summary_table",
 ]
 __version__ = "0.1.0"

src/spanlite/__main__.py

@@ -0,0 +1,10 @@
+"""python -m spanlite — currently exposes regression-pack utilities."""
+
+from __future__ import annotations
+
+from spanlite.promote import _cli, main
+
+__all__ = ["main", "_cli"]
+
+if __name__ == "__main__":
+    main()