diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..0738281 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,59 @@ +name: CI + +# Offline checks only: the API tests use a fake database and a fake model, so this needs no secrets +# and spends no tokens. Live-database tests stay behind RUN_LIVE=1 and are skipped here. +on: + push: + branches: [main] + pull_request: + +permissions: + contents: read + +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: true + +jobs: + api: + name: API and SDK tests (Python 3.11) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + cache: pip + cache-dependency-path: | + api/pyproject.toml + sdk/pyproject.toml + - name: Install + run: pip install -e sdk -e 'api[dev]' + - name: API tests + working-directory: api + run: pytest -q + - name: SDK tests + run: pytest sdk/ -q + + web: + name: Web lint, tests and build (Node 22) + runs-on: ubuntu-latest + defaults: + run: + working-directory: web + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: npm + cache-dependency-path: web/package-lock.json + - run: npm ci + - run: npm run lint + - run: npm test + - name: Build + run: npm run build + env: + # Placeholders so the build can run without real accounts. Nothing here is a secret. + NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY: pk_test_Y2xlcmsuZXhhbXBsZS5jb20k + CLERK_SECRET_KEY: sk_test_placeholderplaceholderplaceholder diff --git a/README.md b/README.md index e405b47..1328ad5 100644 --- a/README.md +++ b/README.md @@ -120,7 +120,7 @@ Redaction at the source and again on ingest (`[REDACTED::]`), 32 KB - **N is small.** Results are directional. The UI says so and shows Wilson intervals. - **Only registered workflows replay.** There is no endpoint that runs arbitrary code, and the recorder currently targets Python and the Anthropic SDK. -- **Edit-output mode only.** You can replace a step's output; editing a prompt and re-running that step live is not built. +- **Two kinds of edit.** You can replace a step's output, or (for workflows registered on this server) change an agent's instructions: that agent then runs live again in both arms, with your instructions and with the recorded ones. Instructions of agents that run in your own code (forked through a worker) cannot be changed from here. Instruction edits are covered by offline tests with a fake model; they have not been run against the real model yet. - **Experiments live in the API process** and are lost on a restart; the UI polls (no streaming). - **Rate limits are per process**, and there is no retention sweep or key-rotation endpoint (see SECURITY.md). - **Auto-verify tests only candidates with a concrete edit.** On the demo run that was one of three. diff --git a/api/app/experiments/runner.py b/api/app/experiments/runner.py index c789920..4f82525 100644 --- a/api/app/experiments/runner.py +++ b/api/app/experiments/runner.py @@ -56,6 +56,9 @@ def view(doc: dict[str, Any]) -> dict[str, Any]: return { "id": doc["id"], "status": doc["status"], "parent_run": doc["parent_run"], "fork_node_run_idx": doc["fork_idx"], "n": doc["n"], "edit_patch": doc["edit_patch"], + # "output": the agent's output was replaced. "prompt": the agent ran again with new instructions, + # and edit_patch is {"prompt": }. + "edit_mode": doc.get("edit_mode", "output"), "control": doc["control"], "treatment": doc["treatment"], "tiles": tiles, "stats": doc.get("stats") or _stats(doc), "tokens_used": doc["tokens_used"], "token_ceiling": doc["token_ceiling"], @@ -67,30 +70,34 @@ def view(doc: dict[str, Any]) -> dict[str, Any]: async def create_experiment( workspace: str, parent_run_id: str, fork_node_run_idx: int, edit: dict[str, Any], n: int, - *, client: Any, tools: Any, created_by: str, + *, client: Any, tools: Any, created_by: str, prompt: str | None = None, ) -> str: """Validate, store the experiment as 'queued', start it in the background, return its id. - Raises what prepare_fork raises (LookupError, ValueError, ModelPinError, SideEffectBlocked), - and ValueError if n is outside 1..MAX_N, all before anything is created. + Give `edit` (replace what the agent produced) or `prompt` (new instructions, the agent runs + again). Raises what prepare_fork raises (LookupError, ValueError, ModelPinError, + SideEffectBlocked), and ValueError if n is outside 1..MAX_N, all before anything is created. """ settings = get_settings() if not 1 <= n <= settings.max_n: raise ValueError(f"n must be between 1 and {settings.max_n}") parent = await db.get_run_with_steps(workspace, parent_run_id) if parent is not None and parent["workflow"] not in REGISTRY: + if prompt is not None: + # The agents of an outside workflow live in the customer's code, which this server cannot change. + raise ValueError("new instructions can only be tested on workflows registered on this server; change a value instead") # Not one of ours to run: the customer's worker runs it. Same checks on the fork point, no live run here. from app.experiments import worker # noqa: PLC0415 (worker imports this module) return await worker.create(workspace, {**parent, "id": parent_run_id}, fork_node_run_idx, edit, n, created_by) - await prepare_fork(workspace, parent_run_id, fork_node_run_idx, edit, tools) + await prepare_fork(workspace, parent_run_id, fork_node_run_idx, None if prompt is not None else edit, tools, prompt=prompt) experiment_id = uuid4().hex # Control and treatment interleaved, so a token cap leaves the two arms balanced. tiles = [{"arm": arm, "i": i, "status": "queued"} for i in range(1, n + 1) for arm in ("control", "treatment")] - stored_edit, _ = redact(edit) + stored_edit, _ = redact({"prompt": prompt} if prompt is not None else edit) doc = { - "parent_run": parent_run_id, "fork_idx": fork_node_run_idx, "edit_mode": "output", + "parent_run": parent_run_id, "fork_idx": fork_node_run_idx, "edit_mode": "prompt" if prompt is not None else "output", "edit_patch": stored_edit, "n": n, "control": {"pass": 0, "total": 0}, "treatment": {"pass": 0, "total": 0}, "tiles": tiles, "status": "queued", "tokens_used": 0, "token_ceiling": settings.max_tokens_per_experiment, @@ -98,14 +105,15 @@ async def create_experiment( } await db.create_experiment(workspace, experiment_id, doc) - task = asyncio.create_task(_run(workspace, experiment_id, doc, edit, client, tools)) + task = asyncio.create_task(_run(workspace, experiment_id, doc, edit, client, tools, prompt)) _TASKS[experiment_id] = task task.add_done_callback(lambda _t: _TASKS.pop(experiment_id, None)) return experiment_id async def _run( - workspace: str, experiment_id: str, doc: dict[str, Any], edit: dict[str, Any], client: Any, tools: Any + workspace: str, experiment_id: str, doc: dict[str, Any], edit: dict[str, Any], client: Any, tools: Any, + prompt: str | None = None, ) -> None: gate, lock = asyncio.Semaphore(CONCURRENCY), asyncio.Lock() skipped = False @@ -129,8 +137,9 @@ async def one(tile: dict[str, Any]) -> None: arm, outcome, tokens = tile["arm"], "error", 0 try: run_id = await run_fork( - workspace, doc["parent_run"], doc["fork_idx"], edit if arm == "treatment" else None, - arm, client=client, tools=tools, experiment_id=experiment_id, + workspace, doc["parent_run"], doc["fork_idx"], + edit if arm == "treatment" and prompt is None else None, + arm, client=client, tools=tools, experiment_id=experiment_id, prompt=prompt, ) tile["run_id"] = run_id summary = await db.get_run_summary(workspace, run_id) or {} diff --git a/api/app/models.py b/api/app/models.py index c1107ba..ccfa4c3 100644 --- a/api/app/models.py +++ b/api/app/models.py @@ -5,11 +5,13 @@ from typing import Any, Literal -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, model_validator # Becomes part of a SurrealDB record id, so keep it boring. _ID = r"^[A-Za-z0-9_-]{8,64}$" +MAX_PROMPT_CHARS = 8000 # an agent's instructions; far above any prompt in the demo workflow + class Step(BaseModel): run: str | None = None # taken from the URL, never trusted from the body @@ -56,11 +58,22 @@ class RunFinish(BaseModel): class ExperimentCreate(BaseModel): run_id: str fork_node_run_idx: int = Field(ge=1) - # Treatment edit: top-level state keys to overwrite at the fork point. The control arm - # forks at the same point with no edit, so an experiment needs one. - edit: dict[str, Any] = Field(min_length=1) + # Treatment edit, one of two kinds. `edit`: top-level state keys to overwrite at the fork point. + # `prompt`: new instructions for the agent at the fork point, which then runs live again (the + # control arm re-runs it with the recorded instructions). The control arm otherwise forks at the + # same point with no change, so an experiment needs exactly one of them. + edit: dict[str, Any] = {} + prompt: str | None = Field(default=None, min_length=1, max_length=MAX_PROMPT_CHARS) n: int = Field(ge=1, le=100) # the real ceiling is MAX_N, enforced with a clear message + @model_validator(mode="after") + def _exactly_one_change(self) -> "ExperimentCreate": + if self.prompt is not None and self.edit: + raise ValueError("send either edit or prompt, not both") + if self.prompt is None and not self.edit: + raise ValueError("send edit (a value to change) or prompt (new instructions)") + return self + class WorkerClaim(BaseModel): workflows: list[str] = Field(min_length=1, max_length=50) # what this worker can run diff --git a/api/app/rca/memory.py b/api/app/rca/memory.py index 43f1e81..6b92494 100644 --- a/api/app/rca/memory.py +++ b/api/app/rca/memory.py @@ -108,7 +108,8 @@ async def remember_experiment( ) text = pattern_text( workflow=run["workflow"], agent=agent, node_run_idx=k, fields=failing_fields(run), - patch_keys=sorted(experiment["edit_patch"]), hypothesis=hypothesis, experiment_id=experiment["id"], + patch_keys=["the agent's instructions"] if experiment.get("edit_mode") == "prompt" else sorted(experiment["edit_patch"]), + hypothesis=hypothesis, experiment_id=experiment["id"], run_id=run["id"], control=experiment["control"], treatment=experiment["treatment"], p=experiment["stats"]["fisher_p"], ) diff --git a/api/app/replay/fork.py b/api/app/replay/fork.py index 98a6e1e..2ffad9f 100644 --- a/api/app/replay/fork.py +++ b/api/app/replay/fork.py @@ -14,6 +14,10 @@ State at the fork point is the parent's recorded state_after of node run k, with the edit's top-level keys overwritten. The control arm is the same fork with no edit, so control and treatment differ in exactly one thing. + +The other kind of edit changes an agent's instructions instead of its output. Then node run k is +not copied at all: it runs live again from the state it started with, in both arms, with the new +instructions in the treatment arm and the recorded ones in the control arm. """ import inspect @@ -90,6 +94,30 @@ class ForkPlan: prefix: list[dict[str, Any]] # recorded steps of node runs before k node_k: list[dict[str, Any]] # recorded steps of node run k start_idx: int # numbering the live steps continue from + # Instruction edit only: node run k runs live again, so its agent and the instructions it was + # recorded with (the control arm keeps them) are needed. Both None for an output edit. + agent: str | None = None + recorded_system: str | None = None + + +def _state_before(steps: list[dict[str, Any]], k: int, node_k: list[dict[str, Any]]) -> dict[str, Any]: + """The state node run k started from: what node run k-1 left behind. For the first node run + there is no earlier step, so it is the state after it minus the keys it wrote (the inputs).""" + if k > 1: + before = [s for s in steps if s["node_run_idx"] == k - 1] + if not before or not isinstance(before[-1]["state_after"], dict): + raise ValueError(f"node run {k - 1} has no recorded state_after to restore") + return before[-1]["state_after"] + after = node_k[-1]["state_after"] + return {key: value for key, value in after.items() if key not in node_k[-1]["writes"]} + + +def _recorded_system(node_k: list[dict[str, Any]]) -> str: + llm = next((s for s in node_k if s["kind"] == "llm"), None) + system = ((llm or {}).get("input") or {}).get("system") + if not isinstance(system, str) or not system.strip(): + raise ValueError("this agent has no plain-text instructions recorded, so they cannot be edited") + return system async def prepare_fork( @@ -99,9 +127,13 @@ async def prepare_fork( edit: dict[str, Any] | None, tools: Any, mocks: dict[str, Any] | None = None, + prompt: str | None = None, ) -> ForkPlan: """Validate a fork and work out what it will copy and restore. Creates nothing. + `prompt` is the other kind of edit: instead of replacing what node run k produced, node run k + runs live again with these instructions (the control arm with the recorded ones). + Raises LookupError (parent not in workspace, or unregistered workflow), ValueError (bad fork point or edit), ModelPinError, SideEffectBlocked. """ @@ -120,6 +152,8 @@ async def prepare_fork( recorded_state = node_k[-1]["state_after"] if not isinstance(recorded_state, dict): raise ValueError(f"node run {k} has no recorded state_after to restore") + if prompt is not None and edit is not None: + raise ValueError("change either what the agent produced or its instructions, not both") if edit is not None: undeclared = sorted(set(edit) - set(node_k[-1]["writes"])) if undeclared: @@ -132,6 +166,13 @@ async def prepare_fork( prefix = [s for s in steps if s["node_run_idx"] < k] prefix_last_idx = max((s["idx"] for s in prefix), default=0) + if prompt is not None: + recorded_system = _recorded_system(node_k) + before = _state_before(steps, k, node_k) + if workflow.next_node(before) != node_k[-1]["agent"]: + raise ValueError(f"node run {k} cannot be re-run from the recorded state") + # Node run k is not copied: it runs live, so its steps are numbered right after the prefix. + return ForkPlan(workflow, k, before, prefix, node_k, prefix_last_idx, node_k[-1]["agent"], recorded_system) # Live steps continue numbering after what was copied: node run k is either kept as-is # (its steps keep their idx) or collapsed into one edited step right after the prefix. start_idx = prefix_last_idx + 1 if edit is not None else max(s["idx"] for s in prefix + node_k) @@ -149,30 +190,40 @@ async def run_fork( tools: Any, mocks: dict[str, Any] | None = None, experiment_id: str | None = None, + prompt: str | None = None, ) -> str: """Create a fork run and return its id. The run's outcome ('pass' | 'fail' | 'error') is stored on the run; a failure after the run exists is recorded, not raised. Validation - errors (see prepare_fork) are raised before any run is created.""" + errors (see prepare_fork) are raised before any run is created. + + With `prompt`, node run k runs live in both arms: the treatment arm with the new instructions, + the control arm with the ones it was recorded with. Nothing about node run k is copied.""" mocks = mocks or {} - plan = await prepare_fork(workspace, parent_run_id, fork_node_run_idx, edit, tools, mocks) + plan = await prepare_fork(workspace, parent_run_id, fork_node_run_idx, edit, tools, mocks, prompt) workflow, k = plan.workflow, plan.k + systems = None + if prompt is not None and plan.agent is not None: + systems = {plan.agent: prompt if arm == "treatment" else plan.recorded_system} run_id: str | None = None try: async with trace_run( workflow.name, DbSink(workspace), origin="live", mocks=mocks, - start_idx=plan.start_idx, start_node_run_idx=k, + # An instruction edit re-runs node run k itself, so the live numbering starts one earlier. + start_idx=plan.start_idx, start_node_run_idx=k - 1 if systems is not None else k, ) as run: run_id = run.id copied = [_copy(s, run.id, "replayed") for s in plan.prefix] - if edit is None: + if systems is not None: + pass # node run k runs live below + elif edit is None: copied += [_copy(s, run.id, "replayed") for s in plan.node_k] else: copied.append(_edited_step(run.id, plan.start_idx, k, plan.node_k[-1], edit, plan.state)) await db.add_steps(workspace, run.id, copied) await db.create_fork_edge(workspace, run.id, parent_run_id, k, arm, experiment_id) - final = await run_to_end(workflow.build_nodes(), workflow.next_node, plan.state, client, tools) + final = await run_to_end(workflow.build_nodes(systems), workflow.next_node, plan.state, client, tools) passed, detail = workflow.check(final, await workflow.ground_truth(final, tools)) run.outcome, run.oracle_detail = ("pass" if passed else "fail"), detail except Exception as exc: diff --git a/api/app/routes/agent.py b/api/app/routes/agent.py index 97ed6a3..188f77d 100644 --- a/api/app/routes/agent.py +++ b/api/app/routes/agent.py @@ -96,7 +96,9 @@ async def rerun_experiment( doc = await db.get_experiment(key.workspace, experiment_id) if doc is None: raise HTTPException(status_code=404, detail="Experiment not found") - body = ExperimentCreate(run_id=doc["parent_run"], fork_node_run_idx=doc["fork_idx"], edit=doc["edit_patch"], n=doc["n"]) + patch = doc["edit_patch"] + change = {"prompt": patch["prompt"]} if doc.get("edit_mode") == "prompt" else {"edit": patch} + body = ExperimentCreate(run_id=doc["parent_run"], fork_node_run_idx=doc["fork_idx"], n=doc["n"], **change) return {"id": await launch(key.workspace, key.user_id, body, runtime)} diff --git a/api/app/routes/experiments.py b/api/app/routes/experiments.py index 5dcf99b..486f183 100644 --- a/api/app/routes/experiments.py +++ b/api/app/routes/experiments.py @@ -29,7 +29,7 @@ async def launch(workspace: str, user_id: str, body: ExperimentCreate, runtime: await enforce_daily_cap(user_id) experiment_id = await create_experiment( workspace, body.run_id, body.fork_node_run_idx, body.edit, body.n, - client=client, tools=tools, created_by=user_id, + client=client, tools=tools, created_by=user_id, prompt=body.prompt, ) except LookupError as exc: raise HTTPException(status_code=404, detail=str(exc)) from None @@ -39,7 +39,7 @@ async def launch(workspace: str, user_id: str, body: ExperimentCreate, runtime: raise HTTPException(status_code=409, detail=str(exc)) from None await audit.record(workspace, user_id, "experiment.launch", experiment_id, { "parent_run": body.run_id, "fork_node_run_idx": body.fork_node_run_idx, "n": body.n, - "edit_keys": sorted(body.edit), + "edit_keys": ["prompt"] if body.prompt is not None else sorted(body.edit), }) return experiment_id diff --git a/api/app/workflows/registry.py b/api/app/workflows/registry.py index e1203c6..a49748d 100644 --- a/api/app/workflows/registry.py +++ b/api/app/workflows/registry.py @@ -15,7 +15,8 @@ @dataclass(frozen=True) class Workflow: name: str - build_nodes: Callable[[], dict[str, Any]] + # `systems` replaces an agent's instructions by agent name (a fork with edited instructions). + build_nodes: Callable[[dict[str, str] | None], dict[str, Any]] next_node: Callable[[dict[str, Any]], str] ground_truth: Callable[[dict[str, Any], Any], Awaitable[dict[str, Any]]] check: Callable[[dict[str, Any], dict[str, Any]], tuple[bool, dict[str, Any]]] @@ -33,9 +34,9 @@ async def _renewal_truth(state: dict[str, Any], tools: Any) -> dict[str, Any]: REGISTRY: dict[str, Workflow] = { "renewal_brief": Workflow( name="renewal_brief", - # A fork never re-runs the planner (it is node run 1: copied or edited), so the - # planner prompt here is irrelevant to forks; the good one is the safe default. - build_nodes=lambda: build_nodes(PLANNER_SYSTEM_GOOD), + # An output-edit fork never re-runs the agent it forks at, so the planner prompt here only + # matters to a prompt-edit fork, which always passes the planner's instructions in `systems`. + build_nodes=lambda systems=None: build_nodes(PLANNER_SYSTEM_GOOD, systems), next_node=next_node, ground_truth=_renewal_truth, check=check, diff --git a/api/app/workflows/renewal_brief/nodes.py b/api/app/workflows/renewal_brief/nodes.py index 455b155..47c4f7f 100644 --- a/api/app/workflows/renewal_brief/nodes.py +++ b/api/app/workflows/renewal_brief/nodes.py @@ -31,13 +31,15 @@ async def _ask(llm: Any, system: str, user: str, max_tokens: int) -> str: return next(b.text for b in reply.content if b.type == "text") -def build_nodes(planner_system: str) -> dict[str, Any]: +def build_nodes(planner_system: str, systems: dict[str, str] | None = None) -> dict[str, Any]: """Nodes keyed by name. The planner prompt is a parameter so the same code records the - good run and the planted-bug run.""" + good run and the planted-bug run. `systems` replaces an agent's instructions by agent name: + this is how a fork re-runs one agent with edited instructions.""" + systems = systems or {} @node("planner", reads=["request"], writes=["plan"]) async def planner(state, llm, tools): - return {"plan": parse_json(await _ask(llm, planner_system, state["request"], 4000))} + return {"plan": parse_json(await _ask(llm, systems.get("planner", planner_system), state["request"], 4000))} @node("researcher", reads=["plan"], writes=["findings"]) async def researcher(state, llm, tools): @@ -48,7 +50,7 @@ async def researcher(state, llm, tools): tickets = await tools.zendesk_search_tickets(plan["account_id"], statuses) pipeline = await tools.salesforce_list_open_opportunities(plan["account_id"]) summary = await _ask( - llm, RESEARCHER_SYSTEM, json.dumps({"tickets": tickets, "pipeline": pipeline}), 4000 + llm, systems.get("researcher", RESEARCHER_SYSTEM), json.dumps({"tickets": tickets, "pipeline": pipeline}), 4000 ) return {"findings": {"tickets": tickets, "pipeline": pipeline, "summary": summary}} @@ -61,7 +63,7 @@ async def writer(state, llm, tools): parts = [f"Request: {state['request']}", "Findings:\n" + json.dumps(state["findings"])] if state.get("draft") and state.get("feedback"): parts += [f"Previous draft:\n{state['draft']}", f"Reviewer feedback: {state['feedback']}"] - draft = await _ask(llm, WRITER_SYSTEM, "\n\n".join(parts), 8000) + draft = await _ask(llm, systems.get("writer", WRITER_SYSTEM), "\n\n".join(parts), 8000) return {"draft": draft, "drafts": state.get("drafts", 0) + 1} @node( @@ -71,7 +73,7 @@ async def writer(state, llm, tools): ) async def critic(state, llm, tools): user = f"Findings:\n{json.dumps(state['findings'])}\n\nBrief:\n{state['draft']}" - review = parse_json(await _ask(llm, CRITIC_SYSTEM, user, 4000)) + review = parse_json(await _ask(llm, systems.get("critic", CRITIC_SYSTEM), user, 4000)) if review.get("verdict") not in ("approve", "revise"): raise ValueError(f"critic verdict must be approve or revise: {review!r}") return { diff --git a/api/scripts/seed_more.py b/api/scripts/seed_more.py new file mode 100644 index 0000000..b846935 --- /dev/null +++ b/api/scripts/seed_more.py @@ -0,0 +1,180 @@ +"""A second batch of sample data, so the graphs have something good in them. Same rules as seed_org.py: additive, +tagged seed = "presentation", removable with `seed_org.py --remove`, no tokens, nothing written to Agent Memory, and +NOTHING posted to Slack (the real worker code tells webhooks when a fork finishes; this script switches that off). + + cd api && .venv/bin/python scripts/seed_more.py --workspace org_XXXX --dry-run + cd api && .venv/bin/python scripts/seed_more.py --workspace org_XXXX + +It adds a third workflow, "invoice-review (sample data)", with eight runs over about three days and five experiments +run through the real worker code, one for each kind of result the app can show: + + fixed it the extractor fix, 5 of 5 against 0 of 5 + no difference a fix to the wrong agent, 0 of 5 against 0 of 5 + helped, but not every run the same fix with a flaky approver, 5 of 8 against 0 of 8 + only a hint 3 of 3 against 0 of 3, too few runs to be sure + made things worse a fork made INSIDE the first fork's run, 0 of 3 against 3 of 3 + +The last one is a fork of a fork, so the run page draws a branch off a branch. +""" + +import argparse +import asyncio +import sys +from datetime import timedelta + +from seed_org import TAG, InProcess, ago, set_times, tag # noqa: E402 (also puts api/ and sdk/ on the path) + +from app import alerts, db # noqa: E402 +from app.experiments import worker as server # noqa: E402 +from app.importer import ImportRun, ImportStep, import_run # noqa: E402 +from app.rca import memory # noqa: E402 +from agenttrace_sdk import Worker # noqa: E402 + +FLOW = "invoice-review (sample data)" + +# (hours ago, invoice, net amount, VAT). The extractor reads the net amount and forgets the VAT: the bug. +INVOICES = [ + (70, "INV-2041", 1000.0, 180.0), (62, "INV-2047", 640.0, 0.0), (54, "INV-2052", 2500.0, 450.0), (46, "INV-2058", 310.0, 55.8), + (38, "INV-2063", 980.0, 0.0), (27, "INV-2071", 1500.0, 270.0), (16, "INV-2079", 720.0, 129.6), (6, "INV-2084", 1200.0, 0.0), +] + + +def approved_is_right(state): + """The customer's check: the amount that was approved must be the invoice total, VAT included.""" + if state.get("approved_total") is None: + return False, "nothing was approved, so the invoice was not paid" + if abs(state["approved_total"] - state["expected"]) < 0.005: + return True, "the approved amount matches the invoice total" + return False, f"approved {state['approved_total']:.2f} but the invoice total is {state['expected']:.2f} (VAT left out)" + + +approved_is_right.__name__ = "approved_amount_is_the_total" + + +def invoice_run(hours: float, number: str, net: float, vat: float) -> ImportRun: + expected = round(net + vat, 2) + read = net # the bug: the extractor reads the net amount only + s1 = {"invoice": number, "net": net, "vat": vat, "expected": expected, "total": read} + s2 = {**s1, "valid": read > 0} + s3 = {**s2, "approved_total": read} + ok, reason = approved_is_right(s3) + return ImportRun( + workflow=FLOW, started_at=ago(hours).isoformat(), outcome="pass" if ok else "fail", + oracle_detail={"verdict": {"kind": "code", "name": "approved_amount_is_the_total", "passed": ok, "reason": reason}}, + steps=[ + ImportStep(agent="extractor", node=1, model="claude-haiku-5-5", input={"invoice": number}, output={"total": read}, + state_after=s1, writes=["total"], tokens_in=420, tokens_out=24), + ImportStep(agent="validator", node=2, model="claude-haiku-5-5", input={"total": read}, output={"valid": s2["valid"]}, + state_after=s2, writes=["valid"], tokens_in=60, tokens_out=6), + ImportStep(agent="approver", node=3, kind="tool", tool_name="approve_payment", tool_args={"amount": read}, tool_result={"status": "queued"}), + ImportStep(agent="approver", node=3, model="claude-haiku-5-5", input={"valid": s2["valid"]}, output={"approved_total": read}, + state_after=s3, writes=["approved_total"], tokens_in=70, tokens_out=10), + ], + ) + + +# One experiment per kind of result. (index of the run, turn to fork, the edit, n, minutes after the run, how to read it) +EXPERIMENTS = [ + (0, 1, {"total": 1180.0}, 5, 35), # fixed it + (2, 2, {"valid": True}, 5, 25), # no difference: the validator was never the problem + (3, 1, {"total": 365.8}, 8, 40), # helped, but not every run: a flaky approver + (5, 1, {"total": 1770.0}, 3, 20), # only a hint +] +NESTED = (2, {"valid": False}, 3) # a fork of the first experiment's fixed run: makes things worse + +FLAKY_CALLS = {"INV-2058": {3, 6, 8}} # the approver sometimes bounces a payment for manual review +_calls: dict[str, int] = {} + + +async def resume_invoice(state, rec): + """The customer's code from the fork point: the validator if it has not run, then the approver.""" + if "valid" not in state: + state = {**state, "valid": state["total"] > 0} + await rec.turn("validator", input={"total": state["total"]}, output={"valid": state["valid"]}, state_after=state, writes=["valid"], tokens_in=60, tokens_out=6) + amount = state["total"] if state["valid"] else None + fixed = abs(state["total"] - state["expected"]) < 0.005 + if fixed and state["invoice"] in FLAKY_CALLS: # only counts the runs that had the fix + _calls[state["invoice"]] = _calls.get(state["invoice"], 0) + 1 + if _calls[state["invoice"]] in FLAKY_CALLS[state["invoice"]]: + amount = None + state = {**state, "approved_total": amount} + await rec.turn("approver", input={"valid": state["valid"]}, output={"approved_total": amount}, state_after=state, writes=["approved_total"], + tool_calls=[{"name": "approve_payment", "args": {"amount": amount}, "result": {"status": "queued" if amount is not None else "held"}}], + tokens_in=70, tokens_out=10) + return state + + +async def run_one(workspace: str, worker: Worker, parent_id: str, turn: int, edit: dict, n: int, parent_started, minutes_after: int) -> str: + parent = await db.get_run_with_steps(workspace, parent_id) + exp_id = await server.create(workspace, {**parent, "id": parent_id}, turn, edit, n, "seed") + jobs = 0 + while await worker.run_once(): + jobs += 1 + started = parent_started + timedelta(minutes=minutes_after) + await db.update_experiment(workspace, exp_id, {"created_at": started.isoformat(), "finished_at": (started + timedelta(minutes=5)).isoformat()}) + done = await db.get_experiment(workspace, exp_id) + print(f" fork at turn {turn} (n={n}): {jobs} jobs, without {done['control']['pass']}/{done['control']['total']}, with {done['treatment']['pass']}/{done['treatment']['total']}") + return exp_id + + +async def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--workspace", required=True) + ap.add_argument("--dry-run", action="store_true") + args = ap.parse_args() + ws = args.workspace + if not ws.startswith("org_"): + print("The workspace must be a Clerk organization id (org_...).") + return 2 + if args.dry_run: + print(f"would add: {len(INVOICES)} runs of {FLOW!r}, {len(EXPERIMENTS)} experiments and 1 fork of a fork, through the real worker code") + return 0 + + # Sample data must not reach the real world. The worker code saves a verified fix to Agent Memory and tells + # webhooks (your Slack) when a fork finishes: both are switched off here. + memory._open = lambda: None + + async def silent(*_a, **_k): + return None + alerts.notify_experiment = silent + alerts.notify_failure = silent + db.enable_pool() # one process, one loop: reuse connections, so a job takes seconds, not tens of seconds + + await db.apply_schema() + async with db.connect() as c: + if await c.query("SELECT id FROM run WHERE workspace = $ws AND workflow = $f LIMIT 1", {"ws": ws, "f": FLOW}): + print("This batch is already in the workspace. Run seed_org.py --remove first if you want to start again.") + return 1 + runs_before = {r["id"].id for r in await c.query("SELECT id FROM run WHERE workspace = $ws", {"ws": ws})} + exps_before = {e["id"].id for e in await c.query("SELECT id FROM experiment WHERE workspace = $ws", {"ws": ws})} + print(f"workspace {ws}: {len(runs_before)} runs, {len(exps_before)} experiments now") + + try: + print(f"adding {len(INVOICES)} runs of {FLOW}") + ids = [] + for hours, number, net, vat in INVOICES: + made = await import_run(ws, invoice_run(hours, number, net, vat)) + await set_times(made["id"], ago(hours), 29) + ids.append(made["id"]) + worker = Worker(client=InProcess(ws), name="seed") + worker.register(FLOW, resume_invoice, oracle=approved_is_right, safe_to_rerun=True) + print("running the experiments through the worker") + first = None + for index, turn, edit, n, minutes in EXPERIMENTS: + exp_id = await run_one(ws, worker, ids[index], turn, edit, n, ago(INVOICES[index][0]), minutes) + first = first or (index, exp_id) + # a fork made inside the first fork: from the run its treatment arm made + index, edit, n = NESTED + parent_exp = await db.get_experiment(ws, first[1]) + inner = next(t["run_id"] for t in parent_exp["tiles"] if t["arm"] == "treatment" and t.get("run_id")) + print("a fork of a fork (from the first experiment's fixed run)") + await run_one(ws, worker, inner, 2, edit, n, ago(INVOICES[first[0]][0]), 80) + finally: + runs, exps = await tag(ws, runs_before, exps_before) + await db.close_pool() + print(f"done: {len(ids)} runs, {exps} experiments, {runs} runs in all (fork attempts included), all tagged seed={TAG}") + return 0 + + +if __name__ == "__main__": + sys.exit(asyncio.run(main())) diff --git a/api/tests/test_prompt_edit.py b/api/tests/test_prompt_edit.py new file mode 100644 index 0000000..f5c22e2 --- /dev/null +++ b/api/tests/test_prompt_edit.py @@ -0,0 +1,175 @@ +"""Forking with new instructions instead of a new output. Offline: a fake model answers from the +instructions it is shown, so the recorded (buggy) planner instructions fail and the fixed ones pass, +and the tests can see which instructions each arm really ran with.""" + +import asyncio +import json +from types import SimpleNamespace + +import pytest +from agenttrace_sdk import trace_run +from pydantic import ValidationError + +from app.experiments import runner +from app.models import ExperimentCreate +from app.replay.fork import prepare_fork, run_fork +from app.runner.run import run_to_end +from app.runner.sink import DbSink +from app.workflows.renewal_brief.graph import next_node +from app.workflows.renewal_brief.nodes import build_nodes +from app.workflows.renewal_brief.prompts import BUG_LINE, PLANNER_SYSTEM_BUGGY, PLANNER_SYSTEM_GOOD +from app.workflows.renewal_brief.tools import FixtureTools +from test_experiments import FakeLLM +from test_fork import WS, steps_of +from test_renewal_brief import APPROVE, BUGGY_PLAN, GOOD_PLAN, REQUEST, draft + + +class PromptAwareLLM(FakeLLM): + """Like FakeLLM, but the planner follows the instructions it is given, and every system prompt is kept.""" + + def __init__(self) -> None: + super().__init__() + self.systems: list[str] = [] + self.messages = SimpleNamespace(create=self._ask) + + async def _ask(self, **kw): + self.systems.append(kw["system"]) + if kw["system"].startswith("You are the planning agent"): + plan = BUGGY_PLAN if BUG_LINE in kw["system"] else GOOD_PLAN + text = json.dumps(plan) + block = SimpleNamespace(type="text", text=text, model_dump=lambda mode="python": {"type": "text", "text": text}) + return SimpleNamespace(content=[block], stop_reason="end_turn", + usage=SimpleNamespace(input_tokens=10, output_tokens=5)) + return await self._create(**kw) + + +@pytest.fixture +def parent(store): + """A run recorded with the planted-bug planner instructions: it says 0 open tickets and fails.""" + async def go(): + async with trace_run("renewal_brief", DbSink(WS)) as r: + await run_to_end(build_nodes(PLANNER_SYSTEM_BUGGY), next_node, {"request": REQUEST}, + PromptAwareLLM(), FixtureTools()) + return r.id + + return asyncio.run(go()) + + +def fork(parent, k, prompt, arm, client=None): + return asyncio.run(run_fork(WS, parent, k, None, arm, client=client or PromptAwareLLM(), + tools=FixtureTools(), prompt=prompt)) + + +def test_the_agent_runs_again_in_both_arms_with_its_own_instructions(store, parent): + client = PromptAwareLLM() + treated = fork(parent, 1, PLANNER_SYSTEM_GOOD, "treatment", client) + control = fork(parent, 1, PLANNER_SYSTEM_GOOD, "control", client) + + assert store.runs[treated]["outcome"] == "pass" + assert store.runs[control]["outcome"] == "fail" # the control arm kept the recorded, buggy instructions + assert client.systems[0] == PLANNER_SYSTEM_GOOD and client.systems[len(client.systems) // 2] == PLANNER_SYSTEM_BUGGY + + +def test_node_run_k_is_not_copied_or_edited_it_runs_live_and_numbering_continues(store, parent): + steps = steps_of(store, fork(parent, 1, PLANNER_SYSTEM_GOOD, "treatment")) + assert [s["origin"] for s in steps] == ["live"] * 6 # nothing to copy before the first agent + assert [s["node_run_idx"] for s in steps] == [1, 2, 2, 2, 3, 4] + assert [s["idx"] for s in steps] == [1, 2, 3, 4, 5, 6] + assert steps[0]["input"]["system"] == PLANNER_SYSTEM_GOOD and steps[0]["tokens_in"] > 0 + + +def test_forking_a_later_agent_replays_everything_before_it_exactly(store, parent): + steps = steps_of(store, fork(parent, 3, "You are the writing agent. Say 4 tickets.", "treatment")) + assert [s["origin"] for s in steps] == ["replayed"] * 4 + ["live"] * 2 + assert [s["node_run_idx"] for s in steps] == [1, 2, 2, 2, 3, 4] + assert [s["idx"] for s in steps] == [1, 2, 3, 4, 5, 6] + assert all(s["tokens_in"] == 0 for s in steps[:4]) + assert steps[4]["input"]["system"].startswith("You are the writing agent. Say 4") + + +def test_one_change_at_a_time(store, parent): + with pytest.raises(ValueError, match="either what the agent produced or its instructions"): + asyncio.run(prepare_fork(WS, parent, 1, {"plan": GOOD_PLAN}, FixtureTools(), prompt="x")) + with pytest.raises(ValidationError): + ExperimentCreate(run_id="r", fork_node_run_idx=1, n=3, edit={"plan": {}}, prompt="x") + with pytest.raises(ValidationError): + ExperimentCreate(run_id="r", fork_node_run_idx=1, n=3) + with pytest.raises(ValidationError): + ExperimentCreate(run_id="r", fork_node_run_idx=1, n=3, prompt="") + with pytest.raises(ValidationError): + ExperimentCreate(run_id="r", fork_node_run_idx=1, n=3, prompt="x" * 8001) + + +def test_an_agent_without_plain_text_instructions_cannot_be_edited(store, parent): + store.steps[(parent, 1)]["input"] = {"system": [{"type": "text", "text": "blocks"}]} + with pytest.raises(ValueError, match="no plain-text instructions"): + asyncio.run(prepare_fork(WS, parent, 1, None, FixtureTools(), prompt="x")) + + +def test_an_experiment_tests_new_instructions_against_the_recorded_ones(store, parent): + client = PromptAwareLLM() + + async def go(): + exp_id = await runner.create_experiment( + WS, parent, 1, {}, 3, client=client, tools=FixtureTools(), created_by="t", prompt=PLANNER_SYSTEM_GOOD, + ) + await runner._TASKS[exp_id] + return exp_id + + exp_id = asyncio.run(go()) + doc = runner.view(store.experiments[exp_id]) + assert doc["status"] == "done" and doc["edit_mode"] == "prompt" + assert doc["control"] == {"pass": 0, "total": 3} and doc["treatment"] == {"pass": 3, "total": 3} + assert doc["edit_patch"] == {"prompt": PLANNER_SYSTEM_GOOD} + assert doc["stats"]["label"].startswith("directional") + + +def test_the_stored_instructions_are_redacted(store, parent): + async def go(): + exp_id = await runner.create_experiment( + WS, parent, 1, {}, 1, client=PromptAwareLLM(), tools=FixtureTools(), created_by="t", + prompt=PLANNER_SYSTEM_GOOD + "\nEscalate to a@b.io", + ) + await runner._TASKS[exp_id] + return exp_id + + assert "a@b.io" not in str(store.experiments[asyncio.run(go())]["edit_patch"]) + + +def test_a_workflow_this_server_does_not_run_cannot_take_new_instructions(store): + async def go(): + async with trace_run("outside_flow", DbSink(WS)) as r: + pass + with pytest.raises(ValueError, match="registered on this server"): + await runner.create_experiment( + WS, r.id, 1, {}, 3, client=PromptAwareLLM(), tools=FixtureTools(), created_by="t", prompt="x", + ) + + asyncio.run(go()) + + +def test_the_api_takes_new_instructions_and_reports_the_result(store, parent, login): + import time + + from fastapi.testclient import TestClient + + from app.main import app + from app.runtime import get_runtime + + login(WS) + app.dependency_overrides[get_runtime] = lambda: (PromptAwareLLM(), FixtureTools()) + try: + with TestClient(app) as api: + resp = api.post("/v1/experiments", json={"run_id": parent, "fork_node_run_idx": 1, "prompt": PLANNER_SYSTEM_GOOD, "n": 2}) + assert resp.status_code == 202 + deadline = time.time() + 5 + while time.time() < deadline: + body = api.get(f"/v1/experiments/{resp.json()['id']}").json() + if body["status"] == "done": + break + time.sleep(0.02) + assert body["edit_mode"] == "prompt" and body["treatment"] == {"pass": 2, "total": 2} + both = api.post("/v1/experiments", json={"run_id": parent, "fork_node_run_idx": 1, "prompt": "x", "edit": {"plan": {}}, "n": 2}) + assert both.status_code == 422 + finally: + app.dependency_overrides.pop(get_runtime, None) diff --git a/docs/integrations.md b/docs/integrations.md index 5271146..4a66f34 100644 --- a/docs/integrations.md +++ b/docs/integrations.md @@ -157,3 +157,5 @@ Findings from running it for real: every database call opened its own connection ## Sample data for a presentation `api/scripts/seed_org.py --workspace org_... ` adds runs to a workspace so its Runs page is busy: copies of the workspace's own failing renewal run at other times, and runs of a sample workflow named "support-triage (sample data)" with three experiments, run through the real worker code (a fix that works, one that does not, and a hint). Everything it makes is tagged `seed = "presentation"`, and `--remove` deletes exactly that. It spends no tokens and never writes to Agent Memory. `--dry-run` says what it would do. The public `/demo` pages (the Examples dropdown) use fixed sample data and need no sign-in. + +`api/scripts/seed_more.py --workspace org_...` adds a second batch on top: a third workflow ("invoice-review (sample data)"), eight runs over about three days, and five experiments with one of each kind of result (fixed it, no difference, helped but not every run, only a hint, and one that made things worse, made as a fork of a fork). It is tagged the same way, so `seed_org.py --remove` takes out both batches. It never writes to Agent Memory and never posts to a webhook. diff --git a/docs/submission/devpost-fields.md b/docs/submission/devpost-fields.md new file mode 100644 index 0000000..b1d1d83 --- /dev/null +++ b/docs/submission/devpost-fields.md @@ -0,0 +1,111 @@ +# Devpost fields: copy each block into its field + +Lines in *[brackets]* are for you to fill or confirm before submitting. + +## Elevator pitch (200 characters max; this one is 195) + +``` +AgentTrace finds the step that broke your agent run, then proves the fix: fork it, rerun live with and without the change (0 of 8 becomes 8 of 8), with Slack alerts. Works on top of your harness. +``` + +## Project story (paste this into "About the project"; it is Markdown) + +```markdown +## Inspiration + +An AI engineer ships a research → write → critique workflow. It fails nine agent steps deep with a confident, wrong answer: *"0 open tickets"* when the real number is five. Traces show **what happened**. They don't show **what caused it**, and re-running the workflow is noisy, costs tokens, and one lucky re-run proves nothing. We wanted a debugger that treats a fix like an experiment. + +## What it does + +AgentTrace records a multi-agent run, then lets you **fork it at any step**, edit that step's output, and re-run everything after it **live, N times, in two arms**: a control with no edit and a treatment with your edit. It reports **k of N right per arm plus a Fisher exact p-value, always labelled "directional"**. + +On our demo workflow (a four-agent renewal brief): + +- **Decoy:** make the critic stricter. **0/5 → 0/5.** The obvious fix does nothing. +- **Hero:** fix the planner's ticket filter. **0/8 → 8/8** (Fisher p = 0.00016, directional). +- A **root-cause agent** ranked the planner first (80%) and a small test supported it (0/3 → 3/3, too few runs to be sure). +- **Generalization:** on five other Era accounts, 3 fresh runs each, the original planner prompt scored **0/15** and the patched one **13/15** (two patched runs crashed and count as failures). + +Recorded steps replay exactly from the recording. Forked steps run live, so they are **sampled, never deterministic**. That is why every arm is repeated. + +## It works on top of the harness you already run + +AgentTrace is built as an observability and testing layer, not a replacement for your orchestration. Working today: + +- **LangGraph** and **any Python loop**: a few lines record each agent turn, with secrets redacted before they leave your process. +- **OpenTelemetry**: anything that emits GenAI spans can post them (through an OpenTelemetry Collector for programs that only send protobuf). +- **Your own definition of "wrong"**: a check you write in code gives each run a right/wrong verdict. +- **A worker next to your code** lets you fork agents AgentTrace didn't build. It only connects out, and you state `safe_to_rerun=True` because a fork runs your agents live. +- **Slack or any webhook** tells you when a run fails and when a fork finishes, with what changed and the result. +- **A CI gate** re-runs a fix you already proved against the code you're about to ship and exits non-zero if it stopped working. +- **MCP server and command line** so an assistant or a terminal can list failed runs, rank suspects and start experiments, under keys that say exactly what the caller may do. + +Adapters for more harnesses (CrewAI, the OpenAI Agents SDK, the Claude Agent SDK) are **in progress and not part of this submission**. + +## How we built it + +- **SDK (Python, stdlib only):** records, redacts, and ships the worker, the MCP server and the CLI. +- **API (FastAPI, Pydantic v2):** ingest, forks, experiments, root cause, alerts, workers. +- **SurrealDB Cloud:** runs, steps and the fork graph (`has_step`, `next`, `forked_from`, `tested` edges) plus **Agent Memory** for verified failure patterns. +- **Claude Haiku 5.5:** the demo agents and the root-cause agent (a forced tool call, schema-validated, with evidence quotes checked word for word against the trace). +- **Clerk:** organizations are workspaces; the workspace comes only from the verified identity or key, never from a request body. +- **Era by Eon:** a simulated Zendesk + Salesforce company gives the workflow its tools **and** exact ground truth, so a pass or fail is a fact. +- **Next.js + Tailwind** on **Render**. + +## Challenges + +- **Honest statistics from small samples.** N is 3 to 8, so we show k/N, a Fisher p-value and the word *directional* everywhere, and the code never calls live steps deterministic. +- **A debugger that can't trust its own measurements.** We ran the whole add-on end to end with nothing faked (real API, real database, real SDK over HTTP). It found that every worker job took 21 seconds, because closing a database connection takes about a second. Reusing connections took a job to 1.5 seconds and the health check from 1.55 s to 0.05 s. +- **Reality vs the spec.** The real Python OpenTelemetry exporter ignores `http/json` and always sends protobuf, so we say so and document the Collector route instead of pretending. +- **Safe re-runs.** A fork runs agents again. Side-effecting tools need a mock, workers must state `safe_to_rerun=True`, and every experiment has per-key, per-user and per-minute limits. + +## Accomplishments we're proud of + +- A causal claim from a handful of runs, with the uncertainty on screen. +- Every addition is covered by tests (over 300 API tests, 90 SDK tests and over 65 web tests) and the full flow was run for real. +- Security by default: redaction before storage, key scopes, escaped Slack messages, webhook addresses checked against private networks, workspace isolation proven by a test. + +## What we learned + +A trace tells you where to look; an experiment tells you whether you were right. The decoy result (0/5 → 0/5) taught us the most: the obvious suspect was innocent, and only a controlled re-run could say so. + +## What's next + +Adapters for CrewAI, the OpenAI Agents SDK and the Claude Agent SDK; OpenTelemetry protobuf directly; per-step start times so parallel agents can be drawn; editing an agent's instructions and re-running it; a clearly labelled lower-trust LLM judge. + +*[Prior work: say what existed before Oct 16, if anything. Team: list everyone and who built what.]* +``` + +## Built with (up to 25 tags) + +``` +python, fastapi, pydantic, next.js, typescript, react, tailwindcss, surrealdb, surrealdb-agent-memory, claude, anthropic-api, clerk, render, era-by-eon, opentelemetry, langgraph, model-context-protocol, slack, pytest, node.js +``` + +## "Try it out" links + +| Label | Link | +| --- | --- | +| Live demo (no sign-in), with the Examples dropdown | https://agenttrace-web.onrender.com/demo | +| All your runs by time (the Gantt) | https://agenttrace-web.onrender.com/demo/overview | +| Source code | https://github.com/AjayMaan13/AgentTrace | +| API health | https://agenttrace-api-apaa.onrender.com/health | + +*[Open each link logged out before you submit. If you upgrade Render, they won't sleep; if not, open them once right before judging.]* + +## Project media (JPG/PNG/GIF, 5 MB max, 3:2 ratio, up to 15) + +Capture at 1440×900 and crop to 3:2 (1350×900). In this order: + +1. **The headline run page**: "The brief says 0 open tickets, not 5." with the line of four agents. `/demo` +2. **The line diagram with the cause marked**: planner "Likely cause 80%", next nodes "Affected by it". +3. **The fork result**: 0/8 → 8/8 with the p-value and "directional". +4. **The decoy**: 0/5 → 0/5 (shows honesty). +5. **All runs by time (Gantt)**: `/demo/overview`, both workflows, forks as diamonds. +6. **The Slack failure message**: *[a real screenshot from your channel]*. +7. **The Slack "Fork finished" message**: *[a real screenshot; test it first]*. +8. **Forking your own agents**: `/demo/worker` (the worker code and the result). +9. **Keys and alerts**: `/demo/connect` with the key picker. +10. **Terminal: the CI gate** passing (exit 0) and failing (exit 1). +11. **The Examples dropdown** open, showing all five examples. +12. **Architecture diagram** from the README. diff --git a/docs/submission/video-script-4min.md b/docs/submission/video-script-4min.md new file mode 100644 index 0000000..0dbef14 --- /dev/null +++ b/docs/submission/video-script-4min.md @@ -0,0 +1,90 @@ +# Video script, 4:00: three slides (1:00) then the live demo (3:00) + +**Story in one line:** *Your agents already run somewhere. AgentTrace sits on top, tells you when a run is wrong, tests which step broke it, and proves whether the fix works.* + +Everything here is real and in the repo. Anything sped up says "sped up" on screen. Nobody says "deterministic", "any harness" or "everything". + +## Screens to set up before you record + +| Screen | What it is | +| --- | --- | +| **S. Slides** | `https://agenttrace-web.onrender.com/demo/pitch` (or `http://localhost:3100/demo/pitch`). Press **F** for full screen, **→** for the next slide, **N** for speaker notes (leave notes off while recording). | +| **A. Browser, tab 1** | `/demo` (public, signed out). Zoom 110%, 1440×900. | +| **B. Browser, tab 2** | `/runs` **signed in**, in your organisation: the Gantt with both workflows. | +| **C. Terminal** | In `api/`, large font, one tab, command ready: `.venv/bin/python scripts/record_demo.py` | +| **D. Slack** | The alerts channel, scrolled to the bottom, notifications off. | +| **E. Browser, tab 3** | `/demo` → **Examples** dropdown, ready on *Forking your own agents*. | + +## Part 1: the slides (0:00 – 1:00) + +| Time | Slide | What to say | Notes | +| --- | --- | --- | --- | +| 0:00 – 0:12 | **1. AgentTrace** | "Hi, I'm *[your name]*. This is AgentTrace. It finds the step that broke your AI agents, and proves whether the fix works." | Stay on the slide, don't read the tagline aloud. | +| 0:12 – 0:38 | **2. Meet Maya** | "Meet Maya, an AI engineer. Her renewal-brief agents just told a customer there were zero open tickets. There were five. Her traces show what happened, not why. Re-running is noisy and costs tokens, so one lucky run proves nothing. And she can't move her whole team to a new framework." | Pause after "There were five." | +| 0:38 – 1:00 | **3. Our solution** | "AgentTrace records the run, follows the data back to the step that caused it, then tests the fix: fork that step, rerun everything after it live, with and without the change. Zero of eight becomes eight of eight. It sits on top of the harness she already uses, and tells her in Slack when something breaks." | The dots draw themselves in. Let the green row finish before you move on. | + +Maya is a **composite persona**; the slide says so. Don't present her as a real customer. + +## Part 2: the demo (1:00 – 4:00) + +| Time | Screen | What to say | On-screen text | Real or edited | +| --- | --- | --- | --- | --- | +| 1:00 – 1:20 | **A.** `/demo` top: the headline and the line of four agents. Click **Planner**. Point to **"Likely cause 80%"** and the downstream nodes marked **"Affected by it"**. | "Back to Maya's problem. This is the real run, recorded. The writer says zero open tickets, but the writer is innocent. The planner filtered the tickets to 'solved', so everything after it worked from nothing. A root-cause agent marks it at eighty percent, and a small test supports that." | "Supported by a small test" | real | +| 1:20 – 1:55 | **C.** Run `record_demo.py` until it prints `FAIL`. **Cut to B.** The new row appears on the Gantt in its own colour. **Cut to D.** The Slack failure message arrives. | "Now a live run from our own pipeline. It lands on the sheet as it finishes. That's every run in the workspace by time, each in its own colour, with forks hanging under the run they came from. A check I wrote in code judged it wrong, and the failure goes straight to Slack with the reason and a link. That's the observability layer: it records any run it can reach, whether or not you can fork it." | "live" while the terminal runs | real (cut the waiting) | +| 1:55 – 2:35 | **A.** The decoy (0/5 → 0/5) for 5 s. Then the hero experiment: the squares fill (if you forked live, label it **sped up**), then the result card. | "Hypotheses are cheap, so we test them. Making the critic stricter changes nothing: zero of five, either way. Now the planner. Same fork point twice: a control with no change, a treatment with the fix. Steps before the fork replay exactly from the recording. Everything after runs live, so each side is repeated. Zero of eight becomes eight of eight, a Fisher p-value of 0.00016. Directional, not proof." | "0/8 → 8/8 · p = 0.00016 · directional" | stored results; if you fork live, "sped up" | +| 2:35 – 2:55 | **D.** The Slack **"Fork finished"** message. | "When a fork finishes, Slack gets one message: what changed, how many runs came out right with and without it, and the p-value, marked directional. Maya doesn't have to watch a dashboard." | none | real *(test it before recording)* | +| 2:55 – 3:35 | **E.** *Forking your own agents* (the worker code, then the result). Then *Keys, alerts and agents* (make a key for a worker, show the scopes). Then *From your terminal and CI* (the gate: exit 0, then exit 1). | "Maya's agents don't run on our server, so there's a worker. It runs next to her code, connects out only, and reruns her own agents from the fork point; AgentTrace counts the results. LangGraph and plain Python loops connect in a few lines, and anything that emits OpenTelemetry works through a collector. Keys say exactly what each caller may do. In CI, a gate reruns a proven fix against new code: exit zero when it holds, one when it doesn't. Adapters for more harnesses are in progress." | "LangGraph · Python loops · OpenTelemetry" | real | +| 3:35 – 3:50 | **A.** The benchmark card (0/15 vs 13/15), then back to `/demo/overview` for one second. | "What's real: the renewal brief, its experiments, and this whole flow, tested against a real database. What's sample: the support-triage workflow. And with small N, every result is directional." | "sample data" on the triage rows | real | +| 3:50 – 4:00 | **S.** Slide 1 again, or the partner names. | "Zero of fifteen to thirteen of fifteen across five other accounts. Built on Claude, SurrealDB, Clerk, Era and Render. This is AgentTrace." | "AgentTrace" | real | + +About 540 spoken words in all. If you run long, shorten the 2:55 part first, then drop the decoy. + +## Recording order (shortest path) + +1. Warm both Render services and wait for a reply (open the site once). +2. Slides first, one take per slide. +3. Record **C → B → D** in one go (1:20 – 1:55): it needs the live run. +4. Record the browser shots (1:00, 1:55, 2:55, 3:35). +5. Fork once for real (N=3) if you want it live on camera. Label "sped up" when you cut. +6. After that fork finishes, record the Slack **"Fork finished"** message. +7. Voice-over last, one take per section. + +## Slack messages the viewer should see + +**Failed run** +``` +Renewal brief gave a wrong answer. +Check : +Open the run +``` + +**Fork finished** +``` +Fork finished on renewal_brief: changing the planner. +Without the change 0 of 5 right, with it 5 of 5. The change fixed it. +Directional only (p = 0.008): live steps are sampled. +Open it +``` +Record both from your real channel. If one doesn't arrive, don't edit one in: show the **Send a test** result on the Connect page. + +## Before you export + +- [ ] No notifications, no secrets on screen (never show `.env` or a key). +- [ ] Every sped-up part says "sped up". Maya is called a composite persona. +- [ ] 4:00 or less. Upload **unlisted**, then open the link **logged out**. + +## If something goes wrong + +| Problem | Do this | +| --- | --- | +| The live API is slow | Skip "live" in 1:20 – 1:55; use the saved results and say nothing about live. | +| Slack doesn't show the message | Show the Send-a-test result; don't fake the message. | +| `/runs` looks empty | You're in the wrong organisation. The workspace is your Clerk organisation. | +| A live fork takes too long | Use the stored hero experiment (0/8 → 8/8). | +| Over time | Drop the decoy, then trim the 2:55 part. | + +## Answers if someone asks + +- *Does it work with any harness?* "It's built to sit on top of whichever one you run. LangGraph and any Python loop work today, anything that emits OpenTelemetry works through a collector, and adapters for more harnesses are in progress." +- *Is it deterministic?* "No. Recorded steps replay exactly; forked steps run live and are sampled, which is why we repeat them and say directional." +- *Why trust N = 8?* "You shouldn't treat it as proof. It shows direction, with a p-value, and we say so everywhere." diff --git a/docs/submission/video-script.md b/docs/submission/video-script.md new file mode 100644 index 0000000..aeba41f --- /dev/null +++ b/docs/submission/video-script.md @@ -0,0 +1,80 @@ +# Video script: AgentTrace (1:56 version) + +*A 4-minute version with three slides first is in `video-script-4min.md`.* + +**Story in one line:** *Your agents already run somewhere. AgentTrace sits on top, tells you when a run is wrong, tests which step broke it, and proves whether the fix works.* + +Everything in this script is real and in the repo. Anything sped up is labelled "sped up" on screen. Nobody says "deterministic". + +## Screens you need open (set these up before you record) + +| Screen | What it is | +| --- | --- | +| **A. Browser, tab 1** | `https://agenttrace-web.onrender.com/demo` (public, signed out). Zoom 110%, 1440×900. | +| **B. Browser, tab 2** | `/runs` **signed in** as yourself, in your organisation (the Gantt with both workflows). | +| **C. Terminal** | In `api/`, font large, one tab, nothing else visible. Command ready: `.venv/bin/python scripts/record_demo.py` | +| **D. Slack** | The alerts channel, scrolled to the bottom, notifications off. | +| **E. Browser, tab 3** | `/demo/worker` and `/demo/terminal` (from the Examples dropdown). | + +## Timeline + +| # | Time | Screen to show | What to say (voice-over) | On-screen text | Real or edited | +| --- | --- | --- | --- | --- | --- | +| 1 | 0:00 – 0:09 | **A.** `/demo` top: the headline *"The brief says 0 open tickets, not 5."* and the line of four agents. | "Four AI agents wrote a renewal brief that says zero open tickets. The real number is five. It's wrong, and nothing in the trace says why." | none | real | +| 2 | 0:09 – 0:22 | **A.** Click the **Planner** node: show what went in and out. Point to **"Likely cause 80%"** and the downstream nodes marked **"Affected by it"**. | "A tracer would blame the writer. AgentTrace follows the data upstream. The planner filtered the tickets to 'solved', so everything after it was built on nothing. A root-cause agent marks it, and a small test supports it." | "Supported by a small test" | real | +| 3 | 0:22 – 0:44 | **C.** Run `record_demo.py`; the line `run …: FAIL` prints. **Cut to B.** The new row appears on the Gantt in its own colour. **Cut to D.** The Slack failure message arrives. | "This is a live run from our own pipeline. It lands on the sheet as it finishes, judged by a check I wrote in code, and the failure goes straight to Slack with the reason and a link. Every run has its own colour, and forks hang under the run they came from." | "live" in the corner during the terminal part | real (cut the waiting) | +| 4 | 0:44 – 1:08 | **A.** The decoy experiment (0/5 → 0/5) for 4 seconds, then the hero experiment: the 8 + 8 squares, then the result card. | "Hypotheses are cheap, so we test them. Making the critic stricter changes nothing: zero of five, either way. Now the planner. Same fork point twice: one run without the fix, one with it. Steps before the fork replay exactly; everything after runs live, so we repeat it. Zero of eight becomes eight of eight." | "0/8 → 8/8 · Fisher p = 0.00016 · directional" | stored results; say "saved" if asked | +| 5 | 1:08 – 1:20 | **D.** The Slack **"Fork finished"** message. | "When a fork finishes, Slack gets one message: what changed, how many runs came out right, and a p-value, marked directional. Small N shows direction, not proof." | none | real *(test it before recording)* | +| 6 | 1:20 – 1:42 | **E.** `/demo` → **Examples** dropdown → *Forking your own agents* (show the worker code, then the result). Then *From your terminal and CI* (the gate: exit 0, then exit 1). | "It sits on top of the agents you already run. LangGraph and plain Python loops connect in a few lines, anything that emits OpenTelemetry works through a collector, and a worker next to your code lets you fork your own agents. In CI, a gate re-runs a proven fix and fails the build if it stops working. More adapters are on the way." | "LangGraph · Python loops · OpenTelemetry" | real | +| 7 | 1:42 – 1:56 | **A.** The benchmark line (0/15 vs 13/15), then the partner names. | "Zero of fifteen to thirteen of fifteen across five other accounts. Directional, but measured. Built on Claude, SurrealDB with Agent Memory, Clerk, Era and Render. This is AgentTrace." | "AgentTrace" and the partner names | stored results | + +About 290 spoken words. If you run long, cut shot 4's decoy first, then shorten shot 2. + +## The Slack messages (what the viewer should see) + +**Failed run:** +``` +Renewal brief gave a wrong answer. +Check : +Open the run +``` + +**Fork finished:** +``` +Fork finished on renewal_brief: changing the planner. +Without the change 0 of 5 right, with it 5 of 5. The change fixed it. +Directional only (p = 0.008): live steps are sampled. +Open it +``` +Record both from your real channel. If a message doesn't arrive, don't fake it: show the **Send a test** result on the Connect page instead. + +## Recording order (shortest path) + +1. Warm both Render services and wait for a reply. +2. Record **C → B → D** in one go (shot 3), because it needs the live run. +3. Record shots 1, 2, 4, 6 and 7 in the browser. +4. Fork once for real (N=3, sped up) if you want a live fork on camera. Say "sped up" on screen when you cut. +5. Record shot 5 after a fork finishes, from the same Slack channel. +6. Voice-over last, in one take per shot. + +## Checks before you export + +- [ ] No notification pop-ups, no secrets on screen (never show `.env` or a key). +- [ ] Every sped-up part says "sped up". +- [ ] Nobody says "deterministic", "any harness" or "everything". +- [ ] 1:56 or less. Upload **unlisted**, then open the link **logged out**. + +## If something goes wrong + +| Problem | Do this | +| --- | --- | +| The live API is slow | Record shot 3 without the live part and say nothing about "live". Use the saved results. | +| Slack doesn't show the message | Show the Send-a-test result instead; don't edit one in. | +| `/runs` looks empty | You're in the wrong organisation. The workspace is your Clerk organisation. | +| You run over time | Drop the decoy in shot 4, then trim shot 2. | + +## Answers if someone asks + +- *Does it work with any harness?* "It's built to sit on top of whichever one you run. LangGraph and any Python loop work today, anything that emits OpenTelemetry works through a collector, and adapters for more harnesses are in progress." +- *Is it deterministic?* "No. Recorded steps replay exactly; forked steps run live and are sampled, which is why we repeat them and say directional." +- *Why trust N = 8?* "You shouldn't treat it as proof. It shows direction with a p-value, and we say so everywhere." diff --git a/web/app/demo/pitch/page.tsx b/web/app/demo/pitch/page.tsx new file mode 100644 index 0000000..e24b56c --- /dev/null +++ b/web/app/demo/pitch/page.tsx @@ -0,0 +1,7 @@ +import { PitchDeck } from "@/components/pitch-deck"; + +export const metadata = { title: "Pitch" }; + +export default function Page() { + return ; +} diff --git a/web/app/globals.css b/web/app/globals.css index 192072b..99ee289 100644 --- a/web/app/globals.css +++ b/web/app/globals.css @@ -89,3 +89,8 @@ h1, h2, h3 { font-family: var(--font-newsreader), Georgia, serif; font-weight: 4 @keyframes draw { from { stroke-dashoffset: var(--len, 400); } to { stroke-dashoffset: 0; } } .draw { stroke-dasharray: var(--len, 400); animation: draw 0.6s ease-out both; } @media (prefers-reduced-motion: reduce) { .draw { animation: none; stroke-dasharray: none; } } + +/* The pitch slides ease in once. Opacity and a small rise only. */ +@keyframes deck-in { from { opacity: 0; transform: translateY(10px); } to { opacity: 1; transform: none; } } +.deck-in { animation: deck-in 0.5s ease-out both; } +@media (prefers-reduced-motion: reduce) { .deck-in { animation: none; } } diff --git a/web/components/experiment-view.tsx b/web/components/experiment-view.tsx index a93952a..034680c 100644 --- a/web/components/experiment-view.tsx +++ b/web/components/experiment-view.tsx @@ -60,7 +60,9 @@ export function ExperimentView({
What was changed
- + {e.edit_mode === "prompt" + ? + : }
diff --git a/web/components/pitch-deck.tsx b/web/components/pitch-deck.tsx new file mode 100644 index 0000000..5be5c8b --- /dev/null +++ b/web/components/pitch-deck.tsx @@ -0,0 +1,172 @@ +"use client"; + +import { useCallback, useEffect, useState, type ReactNode } from "react"; +import { Mark } from "@/components/brand"; + +/** Three slides, one minute, then the live demo. Keys: right arrow / space = next, left arrow = back, + * F = full screen, N = speaker notes (hidden by default so a recording is clean). */ +const BYLINE = "Built for Build North · Ship something real"; + +type Slide = { title: string; seconds: number; say: string; body: ReactNode }; + +const NODES = ["Planner", "Researcher", "Writer", "Critic"]; + +function Solution() { + return ( +
+ + + {NODES.map((n, i) => ( + + + {n} + + ))} + likely cause + + {[0, 1].map((row) => + Array.from({ length: 8 }, (_, i) => ( + + )), + )} + without the fix: 0 of 8 right + with the fix: 8 of 8 right + the same fork point, run live eight times each way + +
    + {[ + ["Record", "from the harness she already uses: LangGraph, a Python loop, or OpenTelemetry."], + ["Test", "Fork the suspect step and rerun everything after it, live, with and without the fix."], + ["Know", "The result lands in Slack, and a CI gate guards the fix from then on."], + ].map(([head, text], i) => ( +
  1. +

    {head}

    +

    {text}

    +
    2. + ))} +
+
+ ); +} + +const SLIDES: Slide[] = [ + { + title: "AgentTrace", + seconds: 12, + say: "Hi, I'm [your name]. This is AgentTrace. It finds the step that broke your AI agents, and proves whether the fix works.", + body: ( +
+
+ +
+

AgentTrace

+

+ Find the step that broke your agents. Prove the fix. +

+

{BYLINE}

+
+ ), + }, + { + title: "Meet Maya", + seconds: 26, + say: "Meet Maya, an AI engineer. Her renewal-brief agents just told a customer there were zero open tickets. There were five. Her traces show what happened, not why. Re-running is noisy and costs tokens, so one lucky run proves nothing. And she can't move her whole team to a new framework.", + body: ( +
+
+ M +
+

Maya

+

AI engineer · a composite persona

+
+

Runs her agents on LangGraph and plain Python. Lives in Slack and CI.

+
+
+

+ “The brief says zero open tickets. There are five. Which of my four agents broke it?” +

+
    + {[ + "Traces show what happened, not what caused it.", + "Re-running is noisy and costs tokens. One lucky run proves nothing.", + "She can't move her team to a new framework.", + ].map((t, i) => ( +
  • + + {t} +
    • + ))} +
+
+
+ ), + }, + { + title: "Our solution", + seconds: 22, + say: "AgentTrace records the run, follows the data back to the step that caused it, then tests the fix: fork that step, rerun everything after it live, with and without the change. Zero of eight becomes eight of eight. It sits on top of the harness she already uses, and tells her in Slack when something breaks.", + body: ( +
+

Find the step. Prove the fix.

+ +

Directional evidence, not proof: live steps are sampled, so every side is repeated.

+
+ ), + }, +]; + +export function PitchDeck() { + const [i, setI] = useState(0); + const [notes, setNotes] = useState(false); + const go = useCallback((n: number) => setI(Math.max(0, Math.min(SLIDES.length - 1, n))), []); + + useEffect(() => { + function onKey(e: KeyboardEvent) { + if (e.metaKey || e.ctrlKey || e.altKey) return; + if (["ArrowRight", " ", "PageDown", "Enter"].includes(e.key)) { e.preventDefault(); setI((c) => Math.min(SLIDES.length - 1, c + 1)); } + else if (["ArrowLeft", "PageUp", "Backspace"].includes(e.key)) { e.preventDefault(); setI((c) => Math.max(0, c - 1)); } + else if (e.key === "Home") setI(0); + else if (e.key === "End") setI(SLIDES.length - 1); + else if (e.key.toLowerCase() === "n") setNotes((v) => !v); + else if (e.key.toLowerCase() === "f") { + if (document.fullscreenElement) void document.exitFullscreen(); + else void document.documentElement.requestFullscreen?.(); + } + } + window.addEventListener("keydown", onKey); + return () => window.removeEventListener("keydown", onKey); + }, []); + + const slide = SLIDES[i]; + return ( +
+
+ AgentTrace + +
+ +
+ {slide.body} +
+ + {notes && ( + + )} +

← → move · F full screen · N notes

+
+ ); +} diff --git a/web/components/test-fix-panel.tsx b/web/components/test-fix-panel.tsx index 44be091..0dc7760 100644 --- a/web/components/test-fix-panel.tsx +++ b/web/components/test-fix-panel.tsx @@ -3,7 +3,7 @@ import { useEffect, useMemo, useState } from "react"; import { ApiError, useApi } from "@/lib/api"; import { diffState } from "@/lib/diff"; -import { forkPrefill, nodeRuns } from "@/lib/runs"; +import { forkPrefill, nodeRuns, recordedInstructions } from "@/lib/runs"; import { brief } from "@/lib/story"; import type { Run, State } from "@/lib/types"; @@ -12,8 +12,9 @@ const N_CHOICES = [3, 5, 8]; const asText = (v: unknown) => JSON.stringify(v, null, 2); /** The fork editor, beside the details of the agent you selected. You see what went in and what came out - * on the left; here you change what came out, choose how many runs, and fork. Everything before the agent - * is replayed exactly; everything after it runs again, with your change and without it. */ + * on the left; here you change what came out (or, for an agent this server runs, its instructions), choose + * how many runs, and fork. Everything before the agent is replayed exactly; everything after it runs again, + * with your change and without it. */ export function TestFixPanel({ run, request, onLaunched, baseLabel, onDirty, }: { @@ -28,10 +29,15 @@ export function TestFixPanel({ // The agent to change is the one that was selected on the line. There is no second place to choose it. const selected = forkable.find((r) => r.idx === request?.nodeRunIdx); const prefill = useMemo(() => (selected ? forkPrefill(selected) : {}), [selected]); + // New instructions need this server to run the agent again, so a run forked through your own worker only changes values. + const instructions = useMemo(() => (selected ? recordedInstructions(selected) : null), [selected]); + const canEditInstructions = instructions !== null && run.fork_mode !== "worker"; // One editor per value the agent wrote. A suggested fix arrives already applied. const [texts, setTexts] = useState>(() => Object.fromEntries(Object.entries(selected ? forkPrefill(selected) : {}).map(([k, v]) => [k, asText(request?.patch?.[k] ?? v)])), ); + const [mode, setMode] = useState<"output" | "prompt">("output"); + const [promptText, setPromptText] = useState(instructions ?? ""); const [n, setN] = useState(5); const [submitting, setSubmitting] = useState(false); const [apiError, setApiError] = useState(null); @@ -54,7 +60,8 @@ export function TestFixPanel({ after: c.kind === "removed" ? "nothing" : brief(c.after, 50), })), ); - const dirty = lines.length > 0; + const promptChanged = mode === "prompt" && promptText.trim() !== "" && promptText !== instructions; + const dirty = mode === "prompt" ? promptChanged : lines.length > 0; useEffect(() => { onDirty?.(dirty); return () => onDirty?.(false); @@ -69,8 +76,12 @@ export function TestFixPanel({ ); } - const invalid = parsed.find((p) => p.error); - const problem = invalid ? `Fix the JSON in "${invalid.key}" first.` : !dirty ? "Change a value above to see what it does." : null; + const invalid = mode === "output" ? parsed.find((p) => p.error) : undefined; + const problem = invalid + ? `Fix the JSON in "${invalid.key}" first.` + : mode === "prompt" && promptText.trim() === "" ? "The instructions cannot be empty." + : !dirty ? (mode === "prompt" ? "Change the instructions above to see what they do." : "Change a value above to see what it does.") + : null; const forkAt = selected.idx; async function launch() { setSubmitting(true); @@ -78,7 +89,11 @@ export function TestFixPanel({ try { const { id } = await api<{ id: string }>("/v1/experiments", { method: "POST", - body: JSON.stringify({ run_id: run.id, fork_node_run_idx: forkAt, edit, n }), + body: JSON.stringify( + mode === "prompt" + ? { run_id: run.id, fork_node_run_idx: forkAt, prompt: promptText, n } + : { run_id: run.id, fork_node_run_idx: forkAt, edit, n }, + ), }); setSubmitting(false); onLaunched(id); @@ -93,9 +108,26 @@ export function TestFixPanel({

Change it, then fork

- Edit what the {selected.agent} produced. A new line opens on the graph and everything after it - runs again, with your change and without it. + {mode === "prompt" + ? <>Change the instructions the {selected.agent} is given. It runs again, and everything after it, with your instructions and with the recorded ones. + : <>Edit what the {selected.agent} produced. A new line opens on the graph and everything after it + runs again, with your change and without it.}

+ {canEditInstructions && ( +
+ {([["output", "What it produced"], ["prompt", "Its instructions"]] as const).map(([value, label]) => ( + + ))} +
+ )} {baseLabel && (

Starts from one run of {baseLabel}, with that fork's change already applied. @@ -103,7 +135,21 @@ export function TestFixPanel({ )}

- {Object.keys(prefill).map((key) => { + {mode === "prompt" && ( +
+ +