From 39a3aafd20bac5074422e24b909c8a33ddca3e33 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jason=20K=C3=B6lker?= <jason@koelker.net>
Date: Wed, 26 Aug 2026 17:16:31 +0000
Subject: [PATCH 1/2] fix(experiments): require queued scripts

Commit 07e60bec changed every new experiment to queue automatically
while leaving script_content optional. That admitted queued records with
no runnable script and no API path for attaching one later.

Require nonblank scripts for single and batch creation, reject blank
rerun sources before copying, and align agent instructions with the
mandatory queue contract.
---
 .../tests/t5_discovery/PROMPT_api_stripped.md |  2 +-
 tests/test_scriptless_experiments.py          | 76 +++++++++++++++++++
 the_lab/PROMPT_api.md                         |  4 +-
 the_lab/routes/experiments.py                 | 44 +++++------
 the_lab/schemas.py                            |  2 +-
 5 files changed, 101 insertions(+), 27 deletions(-)
 create mode 100644 tests/test_scriptless_experiments.py

diff --git a/optimization/tests/t5_discovery/PROMPT_api_stripped.md b/optimization/tests/t5_discovery/PROMPT_api_stripped.md
index 78a966e..8456c92 100644
--- a/optimization/tests/t5_discovery/PROMPT_api_stripped.md
+++ b/optimization/tests/t5_discovery/PROMPT_api_stripped.md
@@ -22,7 +22,7 @@ The core loop (aim for 5-7 API calls per iteration):
 1. **Orient** → `GET /orient` — current state + recommended next action. Follow `next_step`.
 2. **Leaderboard** → `GET /leaderboard?metric=score` — rankings of ideas by metric.
 3. **Create idea** → `POST /ideas/new {parent_ids, description}` — creates git branch and **auto-checkouts** (no separate checkout call needed).
-4. **Create + start experiment** → `POST /ideas/<id>/experiments {description, script_content, meta, tags}` — when `script_content` is provided, the experiment **auto-starts** (no separate start call needed).
+4. **Create experiment** → `POST /ideas/<id>/experiments {description, script_content, meta, tags}` — `script_content` is required and the experiment is queued automatically.
 5. **Wait** → `GET /wait?experiment_id=<id>` — blocks until finished, returns the **full experiment result** with metrics, branch diff, and progress comparison.
 6. **Note findings** → `POST /ideas/<id>/note {text, level}`
 7. **Conclude** → `POST /ideas/<id>/conclude {conclusion}` — then branch into next idea
diff --git a/tests/test_scriptless_experiments.py b/tests/test_scriptless_experiments.py
new file mode 100644
index 0000000..ce96325
--- /dev/null
+++ b/tests/test_scriptless_experiments.py
@@ -0,0 +1,76 @@
+import tempfile
+import unittest
+from pathlib import Path
+
+from fastapi import FastAPI
+from fastapi.testclient import TestClient
+
+from the_lab import deps
+from the_lab.routes import experiments
+from the_lab.store import Store
+
+
+class _Runner:
+    def wake_scheduler(self):
+        pass
+
+
+class ExperimentRouteTest(unittest.TestCase):
+    def setUp(self):
+        self.temp_dir = tempfile.TemporaryDirectory()
+        self.repo_dir = Path(self.temp_dir.name)
+        (self.repo_dir / ".git" / "info").mkdir(parents=True)
+        self.store = Store(self.repo_dir)
+        idea = self.store.create_idea("test idea", [], "idea/1")
+        self.store.save_idea(idea)
+
+        self.original_deps = experiments.store, experiments.runner, experiments.REPO_DIR, deps.store
+        experiments.store = self.store
+        experiments.runner = _Runner()
+        experiments.REPO_DIR = self.repo_dir
+        deps.store = self.store
+
+        app = FastAPI()
+        app.include_router(experiments.router)
+        self.client = TestClient(app)
+
+    def tearDown(self):
+        experiments.store, experiments.runner, experiments.REPO_DIR, deps.store = self.original_deps
+        self.temp_dir.cleanup()
+
+    def test_create_rejects_missing_or_blank_script_without_creating_record(self):
+        cases = [
+            ({"description": "missing script"}, 422),
+            ({"description": "empty script", "script_content": ""}, 400),
+            ({"description": "blank script", "script_content": " \n"}, 400),
+        ]
+        for payload, status_code in cases:
+            with self.subTest(payload=payload):
+                response = self.client.post("/api/v1/ideas/1/experiments", json=payload)
+                self.assertEqual(status_code, response.status_code)
+                self.assertEqual([], self.store.list_all_experiments())
+
+    def test_batch_create_rejects_missing_or_blank_script_without_creating_records(self):
+        for script_content in (None, "", " \n"):
+            body = {"metas": [{"description": "invalid script"}]}
+            if script_content is not None:
+                body["shared_script_content"] = script_content
+            with self.subTest(script_content=script_content):
+                response = self.client.post("/api/v1/ideas/1/experiments/batch", json=body)
+                self.assertEqual(400, response.status_code)
+                self.assertEqual([], self.store.list_all_experiments())
+
+    def test_rerun_rejects_blank_script_without_creating_record(self):
+        source = self.store.create_experiment(1, "blank script")
+        script_path = self.repo_dir / source["script"]
+        script_path.parent.mkdir(parents=True, exist_ok=True)
+        script_path.write_text(" \n")
+
+        response = self.client.post(f"/api/v1/experiments/{source['label']}/rerun")
+
+        self.assertEqual(400, response.status_code)
+        labels = [exp["label"] for exp in self.store.list_all_experiments()]
+        self.assertEqual([source["label"]], labels)
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/the_lab/PROMPT_api.md b/the_lab/PROMPT_api.md
index c2e258e..d02ffd6 100644
--- a/the_lab/PROMPT_api.md
+++ b/the_lab/PROMPT_api.md
@@ -27,7 +27,7 @@ The core loop (aim for 5-7 API calls per iteration):
 1. **Orient** → `GET /orient` — current state + recommended next action. Follow `next_step`.
 2. **Leaderboard + Search** → `GET /leaderboard/search?metric=score&q=keyword` — rankings AND search in one call. Includes the best idea's details — no need to GET individual ideas separately.
 3. **Create idea** → `POST /ideas/new {parent_ids, description}` — creates git branch and **auto-checkouts** (no separate checkout call needed).
-4. **Create + start experiment** → `POST /ideas/<id>/experiments {description, script_content, meta, tags}` — when `script_content` is provided, the experiment **auto-starts** (no separate start call needed).
+4. **Create experiment** → `POST /ideas/<id>/experiments {description, script_content, meta, tags}` — `script_content` is required and the experiment is queued automatically.
 5. **Wait** → run `the-lab wait <label> --port <port>` as a **background shell command** (preferred — lets you keep working while waiting). Exits 0 on success, 1 on failure, prints compact JSON. Or call `GET /wait?experiment_id=<id>` directly for simple sequential flows.
 5b. **Wait for messages** → run `the-lab messages --port <port>` as a **background shell command** to block until a message arrives in your inbox. Prints a JSON array of unread messages on exit.
 6. **Note findings** → `POST /ideas/<id>/note {text, level}`
@@ -142,7 +142,7 @@ use them directly instead of curl. The tools map 1:1 to API endpoints.
 1. **orient** → current state + recommended next action
 2. **leaderboard_search** → rankings + related ideas in one call
 3. **create_idea** → creates branch + auto-checkout
-4. **create_experiment** → provide `script_content` to auto-start
+4. **create_experiment** → provide required `script_content`; the experiment is queued automatically
 5. **wait_for_experiment** → blocks until done, returns full result
 6. **add_note** → record findings
 7. **conclude_idea** → then branch into next idea
diff --git a/the_lab/routes/experiments.py b/the_lab/routes/experiments.py
index 07dcb23..65ded40 100644
--- a/the_lab/routes/experiments.py
+++ b/the_lab/routes/experiments.py
@@ -125,17 +125,18 @@ def _create_and_queue_experiment(
     """Shared create+queue path used by single-create and batch-create.
 
     Mirrors the exact store calls of the single-create handler so scoring and
-    queueing behave identically: create the record, optionally write the wrapped
-    script, flip to ``queued``, broadcast, and wake the scheduler. Returns the
-    (queued) experiment record.
+    queueing behave identically: validate the script, create the record, write
+    the wrapped script, flip to ``queued``, broadcast, and wake the scheduler.
+    Returns the queued experiment record.
     """
-    exp = store.create_experiment(idea_id, description, meta=meta, tags=tags)
+    if not script_content or not script_content.strip():
+        raise HTTPException(400, "script_content must be a non-empty string")
 
-    if script_content is not None:
-        script_path = REPO_DIR / exp["script"]
-        script_path.parent.mkdir(parents=True, exist_ok=True)
-        script_path.write_text(_wrap_script(script_content))
-        os.chmod(script_path, 0o755)
+    exp = store.create_experiment(idea_id, description, meta=meta, tags=tags)
+    script_path = REPO_DIR / exp["script"]
+    script_path.parent.mkdir(parents=True, exist_ok=True)
+    script_path.write_text(_wrap_script(script_content))
+    os.chmod(script_path, 0o755)
 
     # The queue takes it from here. New experiments default to "queued";
     # the scheduler will start them when capacity permits and dependencies
@@ -181,22 +182,17 @@ def _queue_position(exp: dict) -> int | None:
 async def create_experiment(idea_id: int, req: NewExperimentRequest):
     """Create a new experiment under an idea.
 
-    Registers an experiment record and, if ``script_content`` is provided,
-    writes the script file to disk with an auto-injected guard and preamble.
-    The idea must be in ``active`` status. Use ``meta`` to store arbitrary
-    hyperparameters or configuration, and ``tags`` to categorize the
-    experiment for filtering and comparison.
-
-    Set ``auto_start: true`` to immediately start the experiment after creation,
-    saving a separate ``POST /experiments/<id>/start`` call.
+    Requires ``script_content`` and writes it with an auto-injected guard and
+    preamble before queueing the experiment. The idea must be in ``active``
+    status. Use ``meta`` to store arbitrary hyperparameters or configuration,
+    and ``tags`` to categorize the experiment for filtering and comparison.
 
     Example:
         POST /api/v1/ideas/1/experiments {"description": "baseline run",
                                            "script_content": "#!/bin/bash\\npython train.py",
                                            "tags": ["baseline", "v1"],
-                                           "meta": {"lr": 0.001, "epochs": 50},
-                                           "auto_start": true}
-        -> {"id": 4, "idea_id": 1, "status": "running", "script": ".the_lab/scripts/4.sh", ...}
+                                           "meta": {"lr": 0.001, "epochs": 50}}
+        -> {"id": "1.4", "idea_id": 1, "status": "queued", ...}
     """
     idea = store.get_idea(idea_id)
     if not idea:
@@ -241,9 +237,9 @@ async def create_experiment(idea_id: int, req: NewExperimentRequest):
 async def batch_create_experiments(idea_id: int, request: Request):
     """Batch-create queued experiments that share one script (feedback M1).
 
-    Creates one queued experiment per entry in ``metas``, each reusing the same
-    ``shared_script_content`` and the exact single-create store path (so
-    scoring/queueing behave identically). Body:
+    Requires ``shared_script_content`` and creates one queued experiment per
+    entry in ``metas``, each reusing the same script and the exact single-create
+    store path (so scoring/queueing behave identically). Body:
 
         {"shared_script_content": "#!/bin/bash\\npython train.py",
          "metas": [
@@ -1101,6 +1097,8 @@ async def rerun_experiment(exp_ref: str):
     if not script_path.exists():
         raise HTTPException(400, f"script file not found: {source['script']}")
     script_content = script_path.read_text()
+    if not script_content.strip():
+        raise HTTPException(400, f"script file is empty: {source['script']}")
 
     # Create the new experiment — strip stale "rerun: " prefixes so they don't accumulate
     base_desc = re.sub(r'^(rerun: )+', '', source['description'])
diff --git a/the_lab/schemas.py b/the_lab/schemas.py
index 4fa615e..ead68d3 100644
--- a/the_lab/schemas.py
+++ b/the_lab/schemas.py
@@ -44,7 +44,7 @@ class ResourceRequirements(BaseModel):
 class NewExperimentRequest(BaseModel):
     description: str
     meta: dict | None = None
-    script_content: str | None = None
+    script_content: str
     tags: list[str] = []
     auto_start: bool | None = None  # legacy, still accepted (ignored — queue handles it)
     priority: int = 0

From 0ee1ef03fb36cd38de21da5e4ef16742ec5221e9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jason=20K=C3=B6lker?= <jason@koelker.net>
Date: Wed, 26 Aug 2026 17:17:50 +0000
Subject: [PATCH 2/2] fix(runner): fail missing scripts cleanly

Commit cba73ea4 made the missing-script path persist a failed state,
but used label before assigning it. The resulting NameError escaped
recovery, left the record queued, and caused indefinite scheduler retries.

Commit 39a3aaf prevents new scriptless records. Resolve the canonical
label before checking the script path so existing malformed records still
terminate as failed instead of spinning.
---
 tests/test_scriptless_experiments.py | 21 +++++++++++++++++++++
 the_lab/runner.py                    |  1 +
 2 files changed, 22 insertions(+)

diff --git a/tests/test_scriptless_experiments.py b/tests/test_scriptless_experiments.py
index ce96325..803f248 100644
--- a/tests/test_scriptless_experiments.py
+++ b/tests/test_scriptless_experiments.py
@@ -1,3 +1,4 @@
+import asyncio
 import tempfile
 import unittest
 from pathlib import Path
@@ -7,6 +8,7 @@
 
 from the_lab import deps
 from the_lab.routes import experiments
+from the_lab.runner import ExperimentRunner
 from the_lab.store import Store
 
 
@@ -72,5 +74,24 @@ def test_rerun_rejects_blank_script_without_creating_record(self):
         labels = [exp["label"] for exp in self.store.list_all_experiments()]
         self.assertEqual([source["label"]], labels)
 
+
+class ExperimentRunnerTest(unittest.TestCase):
+    def test_start_marks_missing_script_failed(self):
+        with tempfile.TemporaryDirectory() as temp_dir:
+            repo_dir = Path(temp_dir)
+            (repo_dir / ".git" / "info").mkdir(parents=True)
+            store = Store(repo_dir)
+            exp = store.create_experiment(1, "missing script")
+            store.update_experiment(exp["label"], status="queued")
+            runner = ExperimentRunner(store)
+
+            result = asyncio.run(runner.start(exp["label"]))
+
+            saved = store.get_experiment(exp["label"])
+            self.assertEqual("error", result["status"])
+            self.assertEqual("failed", saved["status"])
+            self.assertIn("script not found", saved["error"])
+
+
 if __name__ == "__main__":
     unittest.main()
diff --git a/the_lab/runner.py b/the_lab/runner.py
index 651647e..ca9477c 100644
--- a/the_lab/runner.py
+++ b/the_lab/runner.py
@@ -664,6 +664,7 @@ async def start(
         if exp["status"] not in ("queued", "pending", "failed", "cancelled"):
             return {"status": "error", "reason": f"experiment is {exp['status']}, expected queued, failed, or cancelled"}
 
+        label = exp.get("label") or str(exp_id)
         script_path = self._store.repo_dir / exp["script"]
         if not script_path.exists():
             msg = (