Upgrade/ci and prompt edit

.github/workflows/ci.yml

@@ -0,0 +1,59 @@
+name: CI
+
+# Offline checks only: the API tests use a fake database and a fake model, so this needs no secrets
+# and spends no tokens. Live-database tests stay behind RUN_LIVE=1 and are skipped here.
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ci-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  api:
+    name: API and SDK tests (Python 3.11)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+          cache: pip
+          cache-dependency-path: |
+            api/pyproject.toml
+            sdk/pyproject.toml
+      - name: Install
+        run: pip install -e sdk -e 'api[dev]'
+      - name: API tests
+        working-directory: api
+        run: pytest -q
+      - name: SDK tests
+        run: pytest sdk/ -q
+
+  web:
+    name: Web lint, tests and build (Node 22)
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: web
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 22
+          cache: npm
+          cache-dependency-path: web/package-lock.json
+      - run: npm ci
+      - run: npm run lint
+      - run: npm test
+      - name: Build
+        run: npm run build
+        env:
+          # Placeholders so the build can run without real accounts. Nothing here is a secret.
+          NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY: pk_test_Y2xlcmsuZXhhbXBsZS5jb20k
+          CLERK_SECRET_KEY: sk_test_placeholderplaceholderplaceholder

README.md

@@ -120,7 +120,7 @@ Redaction at the source and again on ingest (`[REDACTED:<type>:<hash8>]`), 32 KB
 
 - **N is small.** Results are directional. The UI says so and shows Wilson intervals.
 - **Only registered workflows replay.** There is no endpoint that runs arbitrary code, and the recorder currently targets Python and the Anthropic SDK.
-- **Edit-output mode only.** You can replace a step's output; editing a prompt and re-running that step live is not built.
+- **Two kinds of edit.** You can replace a step's output, or (for workflows registered on this server) change an agent's instructions: that agent then runs live again in both arms, with your instructions and with the recorded ones. Instructions of agents that run in your own code (forked through a worker) cannot be changed from here. Instruction edits are covered by offline tests with a fake model; they have not been run against the real model yet.
 - **Experiments live in the API process** and are lost on a restart; the UI polls (no streaming).
 - **Rate limits are per process**, and there is no retention sweep or key-rotation endpoint (see SECURITY.md).
 - **Auto-verify tests only candidates with a concrete edit.** On the demo run that was one of three.

api/app/experiments/runner.py

@@ -56,6 +56,9 @@ def view(doc: dict[str, Any]) -> dict[str, Any]:
     return {
         "id": doc["id"], "status": doc["status"], "parent_run": doc["parent_run"],
         "fork_node_run_idx": doc["fork_idx"], "n": doc["n"], "edit_patch": doc["edit_patch"],
+        # "output": the agent's output was replaced. "prompt": the agent ran again with new instructions,
+        # and edit_patch is {"prompt": <the new instructions>}.
+        "edit_mode": doc.get("edit_mode", "output"),
         "control": doc["control"], "treatment": doc["treatment"], "tiles": tiles,
         "stats": doc.get("stats") or _stats(doc),
         "tokens_used": doc["tokens_used"], "token_ceiling": doc["token_ceiling"],
@@ -67,45 +70,50 @@ def view(doc: dict[str, Any]) -> dict[str, Any]:
 
 async def create_experiment(
     workspace: str, parent_run_id: str, fork_node_run_idx: int, edit: dict[str, Any], n: int,
-    *, client: Any, tools: Any, created_by: str,
+    *, client: Any, tools: Any, created_by: str, prompt: str | None = None,
 ) -> str:
     """Validate, store the experiment as 'queued', start it in the background, return its id.
 
-    Raises what prepare_fork raises (LookupError, ValueError, ModelPinError, SideEffectBlocked),
-    and ValueError if n is outside 1..MAX_N, all before anything is created.
+    Give `edit` (replace what the agent produced) or `prompt` (new instructions, the agent runs
+    again). Raises what prepare_fork raises (LookupError, ValueError, ModelPinError,
+    SideEffectBlocked), and ValueError if n is outside 1..MAX_N, all before anything is created.
     """
     settings = get_settings()
     if not 1 <= n <= settings.max_n:
         raise ValueError(f"n must be between 1 and {settings.max_n}")
     parent = await db.get_run_with_steps(workspace, parent_run_id)
     if parent is not None and parent["workflow"] not in REGISTRY:
+        if prompt is not None:
+            # The agents of an outside workflow live in the customer's code, which this server cannot change.
+            raise ValueError("new instructions can only be tested on workflows registered on this server; change a value instead")
         # Not one of ours to run: the customer's worker runs it. Same checks on the fork point, no live run here.
         from app.experiments import worker  # noqa: PLC0415 (worker imports this module)
 
         return await worker.create(workspace, {**parent, "id": parent_run_id}, fork_node_run_idx, edit, n, created_by)
-    await prepare_fork(workspace, parent_run_id, fork_node_run_idx, edit, tools)
+    await prepare_fork(workspace, parent_run_id, fork_node_run_idx, None if prompt is not None else edit, tools, prompt=prompt)
 
     experiment_id = uuid4().hex
     # Control and treatment interleaved, so a token cap leaves the two arms balanced.
     tiles = [{"arm": arm, "i": i, "status": "queued"} for i in range(1, n + 1) for arm in ("control", "treatment")]
-    stored_edit, _ = redact(edit)
+    stored_edit, _ = redact({"prompt": prompt} if prompt is not None else edit)
     doc = {
-        "parent_run": parent_run_id, "fork_idx": fork_node_run_idx, "edit_mode": "output",
+        "parent_run": parent_run_id, "fork_idx": fork_node_run_idx, "edit_mode": "prompt" if prompt is not None else "output",
         "edit_patch": stored_edit, "n": n, "control": {"pass": 0, "total": 0},
         "treatment": {"pass": 0, "total": 0}, "tiles": tiles, "status": "queued",
         "tokens_used": 0, "token_ceiling": settings.max_tokens_per_experiment,
         "created_by": created_by, "created_at": _now(),
     }
     await db.create_experiment(workspace, experiment_id, doc)
 
-    task = asyncio.create_task(_run(workspace, experiment_id, doc, edit, client, tools))
+    task = asyncio.create_task(_run(workspace, experiment_id, doc, edit, client, tools, prompt))
     _TASKS[experiment_id] = task
     task.add_done_callback(lambda _t: _TASKS.pop(experiment_id, None))
     return experiment_id
 
 
 async def _run(
-    workspace: str, experiment_id: str, doc: dict[str, Any], edit: dict[str, Any], client: Any, tools: Any
+    workspace: str, experiment_id: str, doc: dict[str, Any], edit: dict[str, Any], client: Any, tools: Any,
+    prompt: str | None = None,
 ) -> None:
     gate, lock = asyncio.Semaphore(CONCURRENCY), asyncio.Lock()
     skipped = False
@@ -129,8 +137,9 @@ async def one(tile: dict[str, Any]) -> None:
             arm, outcome, tokens = tile["arm"], "error", 0
             try:
                 run_id = await run_fork(
-                    workspace, doc["parent_run"], doc["fork_idx"], edit if arm == "treatment" else None,
-                    arm, client=client, tools=tools, experiment_id=experiment_id,
+                    workspace, doc["parent_run"], doc["fork_idx"],
+                    edit if arm == "treatment" and prompt is None else None,
+                    arm, client=client, tools=tools, experiment_id=experiment_id, prompt=prompt,
                 )
                 tile["run_id"] = run_id
                 summary = await db.get_run_summary(workspace, run_id) or {}

api/app/models.py

@@ -5,11 +5,13 @@
 
 from typing import Any, Literal
 
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, model_validator
 
 # Becomes part of a SurrealDB record id, so keep it boring.
 _ID = r"^[A-Za-z0-9_-]{8,64}$"
 
+MAX_PROMPT_CHARS = 8000  # an agent's instructions; far above any prompt in the demo workflow
+
 
 class Step(BaseModel):
     run: str | None = None  # taken from the URL, never trusted from the body
@@ -56,11 +58,22 @@ class RunFinish(BaseModel):
 class ExperimentCreate(BaseModel):
     run_id: str
     fork_node_run_idx: int = Field(ge=1)
-    # Treatment edit: top-level state keys to overwrite at the fork point. The control arm
-    # forks at the same point with no edit, so an experiment needs one.
-    edit: dict[str, Any] = Field(min_length=1)
+    # Treatment edit, one of two kinds. `edit`: top-level state keys to overwrite at the fork point.
+    # `prompt`: new instructions for the agent at the fork point, which then runs live again (the
+    # control arm re-runs it with the recorded instructions). The control arm otherwise forks at the
+    # same point with no change, so an experiment needs exactly one of them.
+    edit: dict[str, Any] = {}
+    prompt: str | None = Field(default=None, min_length=1, max_length=MAX_PROMPT_CHARS)
     n: int = Field(ge=1, le=100)  # the real ceiling is MAX_N, enforced with a clear message
 
+    @model_validator(mode="after")
+    def _exactly_one_change(self) -> "ExperimentCreate":
+        if self.prompt is not None and self.edit:
+            raise ValueError("send either edit or prompt, not both")
+        if self.prompt is None and not self.edit:
+            raise ValueError("send edit (a value to change) or prompt (new instructions)")
+        return self
+
 
 class WorkerClaim(BaseModel):
     workflows: list[str] = Field(min_length=1, max_length=50)  # what this worker can run

api/app/rca/memory.py

@@ -108,7 +108,8 @@ async def remember_experiment(
     )
     text = pattern_text(
         workflow=run["workflow"], agent=agent, node_run_idx=k, fields=failing_fields(run),
-        patch_keys=sorted(experiment["edit_patch"]), hypothesis=hypothesis, experiment_id=experiment["id"],
+        patch_keys=["the agent's instructions"] if experiment.get("edit_mode") == "prompt" else sorted(experiment["edit_patch"]),
+        hypothesis=hypothesis, experiment_id=experiment["id"],
         run_id=run["id"], control=experiment["control"], treatment=experiment["treatment"],
         p=experiment["stats"]["fisher_p"],
     )

api/app/replay/fork.py

@@ -14,6 +14,10 @@
 State at the fork point is the parent's recorded state_after of node run k, with the edit's
 top-level keys overwritten. The control arm is the same fork with no edit, so control and
 treatment differ in exactly one thing.
+
+The other kind of edit changes an agent's instructions instead of its output. Then node run k is
+not copied at all: it runs live again from the state it started with, in both arms, with the new
+instructions in the treatment arm and the recorded ones in the control arm.
 """
 
 import inspect
@@ -90,6 +94,30 @@ class ForkPlan:
     prefix: list[dict[str, Any]]  # recorded steps of node runs before k
     node_k: list[dict[str, Any]]  # recorded steps of node run k
     start_idx: int  # numbering the live steps continue from
+    # Instruction edit only: node run k runs live again, so its agent and the instructions it was
+    # recorded with (the control arm keeps them) are needed. Both None for an output edit.
+    agent: str | None = None
+    recorded_system: str | None = None
+
+
+def _state_before(steps: list[dict[str, Any]], k: int, node_k: list[dict[str, Any]]) -> dict[str, Any]:
+    """The state node run k started from: what node run k-1 left behind. For the first node run
+    there is no earlier step, so it is the state after it minus the keys it wrote (the inputs)."""
+    if k > 1:
+        before = [s for s in steps if s["node_run_idx"] == k - 1]
+        if not before or not isinstance(before[-1]["state_after"], dict):
+            raise ValueError(f"node run {k - 1} has no recorded state_after to restore")
+        return before[-1]["state_after"]
+    after = node_k[-1]["state_after"]
+    return {key: value for key, value in after.items() if key not in node_k[-1]["writes"]}
+
+
+def _recorded_system(node_k: list[dict[str, Any]]) -> str:
+    llm = next((s for s in node_k if s["kind"] == "llm"), None)
+    system = ((llm or {}).get("input") or {}).get("system")
+    if not isinstance(system, str) or not system.strip():
+        raise ValueError("this agent has no plain-text instructions recorded, so they cannot be edited")
+    return system
 
 
 async def prepare_fork(
@@ -99,9 +127,13 @@ async def prepare_fork(
     edit: dict[str, Any] | None,
     tools: Any,
     mocks: dict[str, Any] | None = None,
+    prompt: str | None = None,
 ) -> ForkPlan:
     """Validate a fork and work out what it will copy and restore. Creates nothing.
 
+    `prompt` is the other kind of edit: instead of replacing what node run k produced, node run k
+    runs live again with these instructions (the control arm with the recorded ones).
+
     Raises LookupError (parent not in workspace, or unregistered workflow), ValueError (bad
     fork point or edit), ModelPinError, SideEffectBlocked.
     """
@@ -120,6 +152,8 @@ async def prepare_fork(
     recorded_state = node_k[-1]["state_after"]
     if not isinstance(recorded_state, dict):
         raise ValueError(f"node run {k} has no recorded state_after to restore")
+    if prompt is not None and edit is not None:
+        raise ValueError("change either what the agent produced or its instructions, not both")
     if edit is not None:
         undeclared = sorted(set(edit) - set(node_k[-1]["writes"]))
         if undeclared:
@@ -132,6 +166,13 @@ async def prepare_fork(
 
     prefix = [s for s in steps if s["node_run_idx"] < k]
     prefix_last_idx = max((s["idx"] for s in prefix), default=0)
+    if prompt is not None:
+        recorded_system = _recorded_system(node_k)
+        before = _state_before(steps, k, node_k)
+        if workflow.next_node(before) != node_k[-1]["agent"]:
+            raise ValueError(f"node run {k} cannot be re-run from the recorded state")
+        # Node run k is not copied: it runs live, so its steps are numbered right after the prefix.
+        return ForkPlan(workflow, k, before, prefix, node_k, prefix_last_idx, node_k[-1]["agent"], recorded_system)
     # Live steps continue numbering after what was copied: node run k is either kept as-is
     # (its steps keep their idx) or collapsed into one edited step right after the prefix.
     start_idx = prefix_last_idx + 1 if edit is not None else max(s["idx"] for s in prefix + node_k)
@@ -149,30 +190,40 @@ async def run_fork(
     tools: Any,
     mocks: dict[str, Any] | None = None,
     experiment_id: str | None = None,
+    prompt: str | None = None,
 ) -> str:
     """Create a fork run and return its id. The run's outcome ('pass' | 'fail' | 'error') is
     stored on the run; a failure after the run exists is recorded, not raised. Validation
-    errors (see prepare_fork) are raised before any run is created."""
+    errors (see prepare_fork) are raised before any run is created.
+
+    With `prompt`, node run k runs live in both arms: the treatment arm with the new instructions,
+    the control arm with the ones it was recorded with. Nothing about node run k is copied."""
     mocks = mocks or {}
-    plan = await prepare_fork(workspace, parent_run_id, fork_node_run_idx, edit, tools, mocks)
+    plan = await prepare_fork(workspace, parent_run_id, fork_node_run_idx, edit, tools, mocks, prompt)
     workflow, k = plan.workflow, plan.k
+    systems = None
+    if prompt is not None and plan.agent is not None:
+        systems = {plan.agent: prompt if arm == "treatment" else plan.recorded_system}
 
     run_id: str | None = None
     try:
         async with trace_run(
             workflow.name, DbSink(workspace), origin="live", mocks=mocks,
-            start_idx=plan.start_idx, start_node_run_idx=k,
+            # An instruction edit re-runs node run k itself, so the live numbering starts one earlier.
+            start_idx=plan.start_idx, start_node_run_idx=k - 1 if systems is not None else k,
         ) as run:
             run_id = run.id
             copied = [_copy(s, run.id, "replayed") for s in plan.prefix]
-            if edit is None:
+            if systems is not None:
+                pass  # node run k runs live below
+            elif edit is None:
                 copied += [_copy(s, run.id, "replayed") for s in plan.node_k]
             else:
                 copied.append(_edited_step(run.id, plan.start_idx, k, plan.node_k[-1], edit, plan.state))
             await db.add_steps(workspace, run.id, copied)
             await db.create_fork_edge(workspace, run.id, parent_run_id, k, arm, experiment_id)
 
-            final = await run_to_end(workflow.build_nodes(), workflow.next_node, plan.state, client, tools)
+            final = await run_to_end(workflow.build_nodes(systems), workflow.next_node, plan.state, client, tools)
             passed, detail = workflow.check(final, await workflow.ground_truth(final, tools))
             run.outcome, run.oracle_detail = ("pass" if passed else "fail"), detail
     except Exception as exc:

api/app/routes/agent.py

@@ -96,7 +96,9 @@ async def rerun_experiment(
     doc = await db.get_experiment(key.workspace, experiment_id)
     if doc is None:
         raise HTTPException(status_code=404, detail="Experiment not found")
-    body = ExperimentCreate(run_id=doc["parent_run"], fork_node_run_idx=doc["fork_idx"], edit=doc["edit_patch"], n=doc["n"])
+    patch = doc["edit_patch"]
+    change = {"prompt": patch["prompt"]} if doc.get("edit_mode") == "prompt" else {"edit": patch}
+    body = ExperimentCreate(run_id=doc["parent_run"], fork_node_run_idx=doc["fork_idx"], n=doc["n"], **change)
     return {"id": await launch(key.workspace, key.user_id, body, runtime)}
 
 

api/app/routes/experiments.py

@@ -29,7 +29,7 @@ async def launch(workspace: str, user_id: str, body: ExperimentCreate, runtime:
             await enforce_daily_cap(user_id)
             experiment_id = await create_experiment(
                 workspace, body.run_id, body.fork_node_run_idx, body.edit, body.n,
-                client=client, tools=tools, created_by=user_id,
+                client=client, tools=tools, created_by=user_id, prompt=body.prompt,
             )
     except LookupError as exc:
         raise HTTPException(status_code=404, detail=str(exc)) from None
@@ -39,7 +39,7 @@ async def launch(workspace: str, user_id: str, body: ExperimentCreate, runtime:
         raise HTTPException(status_code=409, detail=str(exc)) from None
     await audit.record(workspace, user_id, "experiment.launch", experiment_id, {
         "parent_run": body.run_id, "fork_node_run_idx": body.fork_node_run_idx, "n": body.n,
-        "edit_keys": sorted(body.edit),
+        "edit_keys": ["prompt"] if body.prompt is not None else sorted(body.edit),
     })
     return experiment_id
 

api/app/workflows/registry.py

@@ -15,7 +15,8 @@
 @dataclass(frozen=True)
 class Workflow:
     name: str
-    build_nodes: Callable[[], dict[str, Any]]
+    # `systems` replaces an agent's instructions by agent name (a fork with edited instructions).
+    build_nodes: Callable[[dict[str, str] | None], dict[str, Any]]
     next_node: Callable[[dict[str, Any]], str]
     ground_truth: Callable[[dict[str, Any], Any], Awaitable[dict[str, Any]]]
     check: Callable[[dict[str, Any], dict[str, Any]], tuple[bool, dict[str, Any]]]
@@ -33,9 +34,9 @@ async def _renewal_truth(state: dict[str, Any], tools: Any) -> dict[str, Any]:
 REGISTRY: dict[str, Workflow] = {
     "renewal_brief": Workflow(
         name="renewal_brief",
-        # A fork never re-runs the planner (it is node run 1: copied or edited), so the
-        # planner prompt here is irrelevant to forks; the good one is the safe default.
-        build_nodes=lambda: build_nodes(PLANNER_SYSTEM_GOOD),
+        # An output-edit fork never re-runs the agent it forks at, so the planner prompt here only
+        # matters to a prompt-edit fork, which always passes the planner's instructions in `systems`.
+        build_nodes=lambda systems=None: build_nodes(PLANNER_SYSTEM_GOOD, systems),
         next_node=next_node,
         ground_truth=_renewal_truth,
         check=check,