fix(experiments): reject scriptless runs

optimization/tests/t5_discovery/PROMPT_api_stripped.md

@@ -22,7 +22,7 @@ The core loop (aim for 5-7 API calls per iteration):
 1. **Orient** → `GET /orient` — current state + recommended next action. Follow `next_step`.
 2. **Leaderboard** → `GET /leaderboard?metric=score` — rankings of ideas by metric.
 3. **Create idea** → `POST /ideas/new {parent_ids, description}` — creates git branch and **auto-checkouts** (no separate checkout call needed).
-4. **Create + start experiment** → `POST /ideas/<id>/experiments {description, script_content, meta, tags}` — when `script_content` is provided, the experiment **auto-starts** (no separate start call needed).
+4. **Create experiment** → `POST /ideas/<id>/experiments {description, script_content, meta, tags}` — `script_content` is required and the experiment is queued automatically.
 5. **Wait** → `GET /wait?experiment_id=<id>` — blocks until finished, returns the **full experiment result** with metrics, branch diff, and progress comparison.
 6. **Note findings** → `POST /ideas/<id>/note {text, level}`
 7. **Conclude** → `POST /ideas/<id>/conclude {conclusion}` — then branch into next idea

tests/test_scriptless_experiments.py

@@ -0,0 +1,97 @@
+import asyncio
+import tempfile
+import unittest
+from pathlib import Path
+
+from fastapi import FastAPI
+from fastapi.testclient import TestClient
+
+from the_lab import deps
+from the_lab.routes import experiments
+from the_lab.runner import ExperimentRunner
+from the_lab.store import Store
+
+
+class _Runner:
+    def wake_scheduler(self):
+        pass
+
+
+class ExperimentRouteTest(unittest.TestCase):
+    def setUp(self):
+        self.temp_dir = tempfile.TemporaryDirectory()
+        self.repo_dir = Path(self.temp_dir.name)
+        (self.repo_dir / ".git" / "info").mkdir(parents=True)
+        self.store = Store(self.repo_dir)
+        idea = self.store.create_idea("test idea", [], "idea/1")
+        self.store.save_idea(idea)
+
+        self.original_deps = experiments.store, experiments.runner, experiments.REPO_DIR, deps.store
+        experiments.store = self.store
+        experiments.runner = _Runner()
+        experiments.REPO_DIR = self.repo_dir
+        deps.store = self.store
+
+        app = FastAPI()
+        app.include_router(experiments.router)
+        self.client = TestClient(app)
+
+    def tearDown(self):
+        experiments.store, experiments.runner, experiments.REPO_DIR, deps.store = self.original_deps
+        self.temp_dir.cleanup()
+
+    def test_create_rejects_missing_or_blank_script_without_creating_record(self):
+        cases = [
+            ({"description": "missing script"}, 422),
+            ({"description": "empty script", "script_content": ""}, 400),
+            ({"description": "blank script", "script_content": " \n"}, 400),
+        ]
+        for payload, status_code in cases:
+            with self.subTest(payload=payload):
+                response = self.client.post("/api/v1/ideas/1/experiments", json=payload)
+                self.assertEqual(status_code, response.status_code)
+                self.assertEqual([], self.store.list_all_experiments())
+
+    def test_batch_create_rejects_missing_or_blank_script_without_creating_records(self):
+        for script_content in (None, "", " \n"):
+            body = {"metas": [{"description": "invalid script"}]}
+            if script_content is not None:
+                body["shared_script_content"] = script_content
+            with self.subTest(script_content=script_content):
+                response = self.client.post("/api/v1/ideas/1/experiments/batch", json=body)
+                self.assertEqual(400, response.status_code)
+                self.assertEqual([], self.store.list_all_experiments())
+
+    def test_rerun_rejects_blank_script_without_creating_record(self):
+        source = self.store.create_experiment(1, "blank script")
+        script_path = self.repo_dir / source["script"]
+        script_path.parent.mkdir(parents=True, exist_ok=True)
+        script_path.write_text(" \n")
+
+        response = self.client.post(f"/api/v1/experiments/{source['label']}/rerun")
+
+        self.assertEqual(400, response.status_code)
+        labels = [exp["label"] for exp in self.store.list_all_experiments()]
+        self.assertEqual([source["label"]], labels)
+
+
+class ExperimentRunnerTest(unittest.TestCase):
+    def test_start_marks_missing_script_failed(self):
+        with tempfile.TemporaryDirectory() as temp_dir:
+            repo_dir = Path(temp_dir)
+            (repo_dir / ".git" / "info").mkdir(parents=True)
+            store = Store(repo_dir)
+            exp = store.create_experiment(1, "missing script")
+            store.update_experiment(exp["label"], status="queued")
+            runner = ExperimentRunner(store)
+
+            result = asyncio.run(runner.start(exp["label"]))
+
+            saved = store.get_experiment(exp["label"])
+            self.assertEqual("error", result["status"])
+            self.assertEqual("failed", saved["status"])
+            self.assertIn("script not found", saved["error"])
+
+
+if __name__ == "__main__":
+    unittest.main()

the_lab/PROMPT_api.md

@@ -27,7 +27,7 @@ The core loop (aim for 5-7 API calls per iteration):
 1. **Orient** → `GET /orient` — current state + recommended next action. Follow `next_step`.
 2. **Leaderboard + Search** → `GET /leaderboard/search?metric=score&q=keyword` — rankings AND search in one call. Includes the best idea's details — no need to GET individual ideas separately.
 3. **Create idea** → `POST /ideas/new {parent_ids, description}` — creates git branch and **auto-checkouts** (no separate checkout call needed).
-4. **Create + start experiment** → `POST /ideas/<id>/experiments {description, script_content, meta, tags}` — when `script_content` is provided, the experiment **auto-starts** (no separate start call needed).
+4. **Create experiment** → `POST /ideas/<id>/experiments {description, script_content, meta, tags}` — `script_content` is required and the experiment is queued automatically.
 5. **Wait** → run `the-lab wait <label> --port <port>` as a **background shell command** (preferred — lets you keep working while waiting). Exits 0 on success, 1 on failure, prints compact JSON. Or call `GET /wait?experiment_id=<id>` directly for simple sequential flows.
 5b. **Wait for messages** → run `the-lab messages --port <port>` as a **background shell command** to block until a message arrives in your inbox. Prints a JSON array of unread messages on exit.
 6. **Note findings** → `POST /ideas/<id>/note {text, level}`
@@ -142,7 +142,7 @@ use them directly instead of curl. The tools map 1:1 to API endpoints.
 1. **orient** → current state + recommended next action
 2. **leaderboard_search** → rankings + related ideas in one call
 3. **create_idea** → creates branch + auto-checkout
-4. **create_experiment** → provide `script_content` to auto-start
+4. **create_experiment** → provide required `script_content`; the experiment is queued automatically
 5. **wait_for_experiment** → blocks until done, returns full result
 6. **add_note** → record findings
 7. **conclude_idea** → then branch into next idea

the_lab/routes/experiments.py

@@ -125,17 +125,18 @@ def _create_and_queue_experiment(
     """Shared create+queue path used by single-create and batch-create.
 
     Mirrors the exact store calls of the single-create handler so scoring and
-    queueing behave identically: create the record, optionally write the wrapped
-    script, flip to ``queued``, broadcast, and wake the scheduler. Returns the
-    (queued) experiment record.
+    queueing behave identically: validate the script, create the record, write
+    the wrapped script, flip to ``queued``, broadcast, and wake the scheduler.
+    Returns the queued experiment record.
     """
-    exp = store.create_experiment(idea_id, description, meta=meta, tags=tags)
+    if not script_content or not script_content.strip():
+        raise HTTPException(400, "script_content must be a non-empty string")
 
-    if script_content is not None:
-        script_path = REPO_DIR / exp["script"]
-        script_path.parent.mkdir(parents=True, exist_ok=True)
-        script_path.write_text(_wrap_script(script_content))
-        os.chmod(script_path, 0o755)
+    exp = store.create_experiment(idea_id, description, meta=meta, tags=tags)
+    script_path = REPO_DIR / exp["script"]
+    script_path.parent.mkdir(parents=True, exist_ok=True)
+    script_path.write_text(_wrap_script(script_content))
+    os.chmod(script_path, 0o755)
 
     # The queue takes it from here. New experiments default to "queued";
     # the scheduler will start them when capacity permits and dependencies
@@ -181,22 +182,17 @@ def _queue_position(exp: dict) -> int | None:
 async def create_experiment(idea_id: int, req: NewExperimentRequest):
     """Create a new experiment under an idea.
 
-    Registers an experiment record and, if ``script_content`` is provided,
-    writes the script file to disk with an auto-injected guard and preamble.
-    The idea must be in ``active`` status. Use ``meta`` to store arbitrary
-    hyperparameters or configuration, and ``tags`` to categorize the
-    experiment for filtering and comparison.
-
-    Set ``auto_start: true`` to immediately start the experiment after creation,
-    saving a separate ``POST /experiments/<id>/start`` call.
+    Requires ``script_content`` and writes it with an auto-injected guard and
+    preamble before queueing the experiment. The idea must be in ``active``
+    status. Use ``meta`` to store arbitrary hyperparameters or configuration,
+    and ``tags`` to categorize the experiment for filtering and comparison.
 
     Example:
         POST /api/v1/ideas/1/experiments {"description": "baseline run",
                                            "script_content": "#!/bin/bash\\npython train.py",
                                            "tags": ["baseline", "v1"],
-                                           "meta": {"lr": 0.001, "epochs": 50},
-                                           "auto_start": true}
-        -> {"id": 4, "idea_id": 1, "status": "running", "script": ".the_lab/scripts/4.sh", ...}
+                                           "meta": {"lr": 0.001, "epochs": 50}}
+        -> {"id": "1.4", "idea_id": 1, "status": "queued", ...}
     """
     idea = store.get_idea(idea_id)
     if not idea:
@@ -241,9 +237,9 @@ async def create_experiment(idea_id: int, req: NewExperimentRequest):
 async def batch_create_experiments(idea_id: int, request: Request):
     """Batch-create queued experiments that share one script (feedback M1).
 
-    Creates one queued experiment per entry in ``metas``, each reusing the same
-    ``shared_script_content`` and the exact single-create store path (so
-    scoring/queueing behave identically). Body:
+    Requires ``shared_script_content`` and creates one queued experiment per
+    entry in ``metas``, each reusing the same script and the exact single-create
+    store path (so scoring/queueing behave identically). Body:
 
         {"shared_script_content": "#!/bin/bash\\npython train.py",
          "metas": [
@@ -1101,6 +1097,8 @@ async def rerun_experiment(exp_ref: str):
     if not script_path.exists():
         raise HTTPException(400, f"script file not found: {source['script']}")
     script_content = script_path.read_text()
+    if not script_content.strip():
+        raise HTTPException(400, f"script file is empty: {source['script']}")
 
     # Create the new experiment — strip stale "rerun: " prefixes so they don't accumulate
     base_desc = re.sub(r'^(rerun: )+', '', source['description'])

the_lab/runner.py

@@ -664,6 +664,7 @@ async def start(
         if exp["status"] not in ("queued", "pending", "failed", "cancelled"):
             return {"status": "error", "reason": f"experiment is {exp['status']}, expected queued, failed, or cancelled"}
 
+        label = exp.get("label") or str(exp_id)
         script_path = self._store.repo_dir / exp["script"]
         if not script_path.exists():
             msg = (

the_lab/schemas.py

@@ -44,7 +44,7 @@ class ResourceRequirements(BaseModel):
 class NewExperimentRequest(BaseModel):
     description: str
     meta: dict | None = None
-    script_content: str | None = None
+    script_content: str
     tags: list[str] = []
     auto_start: bool | None = None  # legacy, still accepted (ignored — queue handles it)
     priority: int = 0