diff --git a/optimization/tests/t5_discovery/PROMPT_api_stripped.md b/optimization/tests/t5_discovery/PROMPT_api_stripped.md index 78a966e..8456c92 100644 --- a/optimization/tests/t5_discovery/PROMPT_api_stripped.md +++ b/optimization/tests/t5_discovery/PROMPT_api_stripped.md @@ -22,7 +22,7 @@ The core loop (aim for 5-7 API calls per iteration): 1. **Orient** → `GET /orient` — current state + recommended next action. Follow `next_step`. 2. **Leaderboard** → `GET /leaderboard?metric=score` — rankings of ideas by metric. 3. **Create idea** → `POST /ideas/new {parent_ids, description}` — creates git branch and **auto-checkouts** (no separate checkout call needed). -4. **Create + start experiment** → `POST /ideas//experiments {description, script_content, meta, tags}` — when `script_content` is provided, the experiment **auto-starts** (no separate start call needed). +4. **Create experiment** → `POST /ideas//experiments {description, script_content, meta, tags}` — `script_content` is required and the experiment is queued automatically. 5. **Wait** → `GET /wait?experiment_id=` — blocks until finished, returns the **full experiment result** with metrics, branch diff, and progress comparison. 6. **Note findings** → `POST /ideas//note {text, level}` 7. **Conclude** → `POST /ideas//conclude {conclusion}` — then branch into next idea diff --git a/tests/test_scriptless_experiments.py b/tests/test_scriptless_experiments.py new file mode 100644 index 0000000..803f248 --- /dev/null +++ b/tests/test_scriptless_experiments.py @@ -0,0 +1,97 @@ +import asyncio +import tempfile +import unittest +from pathlib import Path + +from fastapi import FastAPI +from fastapi.testclient import TestClient + +from the_lab import deps +from the_lab.routes import experiments +from the_lab.runner import ExperimentRunner +from the_lab.store import Store + + +class _Runner: + def wake_scheduler(self): + pass + + +class ExperimentRouteTest(unittest.TestCase): + def setUp(self): + self.temp_dir = tempfile.TemporaryDirectory() + self.repo_dir = Path(self.temp_dir.name) + (self.repo_dir / ".git" / "info").mkdir(parents=True) + self.store = Store(self.repo_dir) + idea = self.store.create_idea("test idea", [], "idea/1") + self.store.save_idea(idea) + + self.original_deps = experiments.store, experiments.runner, experiments.REPO_DIR, deps.store + experiments.store = self.store + experiments.runner = _Runner() + experiments.REPO_DIR = self.repo_dir + deps.store = self.store + + app = FastAPI() + app.include_router(experiments.router) + self.client = TestClient(app) + + def tearDown(self): + experiments.store, experiments.runner, experiments.REPO_DIR, deps.store = self.original_deps + self.temp_dir.cleanup() + + def test_create_rejects_missing_or_blank_script_without_creating_record(self): + cases = [ + ({"description": "missing script"}, 422), + ({"description": "empty script", "script_content": ""}, 400), + ({"description": "blank script", "script_content": " \n"}, 400), + ] + for payload, status_code in cases: + with self.subTest(payload=payload): + response = self.client.post("/api/v1/ideas/1/experiments", json=payload) + self.assertEqual(status_code, response.status_code) + self.assertEqual([], self.store.list_all_experiments()) + + def test_batch_create_rejects_missing_or_blank_script_without_creating_records(self): + for script_content in (None, "", " \n"): + body = {"metas": [{"description": "invalid script"}]} + if script_content is not None: + body["shared_script_content"] = script_content + with self.subTest(script_content=script_content): + response = self.client.post("/api/v1/ideas/1/experiments/batch", json=body) + self.assertEqual(400, response.status_code) + self.assertEqual([], self.store.list_all_experiments()) + + def test_rerun_rejects_blank_script_without_creating_record(self): + source = self.store.create_experiment(1, "blank script") + script_path = self.repo_dir / source["script"] + script_path.parent.mkdir(parents=True, exist_ok=True) + script_path.write_text(" \n") + + response = self.client.post(f"/api/v1/experiments/{source['label']}/rerun") + + self.assertEqual(400, response.status_code) + labels = [exp["label"] for exp in self.store.list_all_experiments()] + self.assertEqual([source["label"]], labels) + + +class ExperimentRunnerTest(unittest.TestCase): + def test_start_marks_missing_script_failed(self): + with tempfile.TemporaryDirectory() as temp_dir: + repo_dir = Path(temp_dir) + (repo_dir / ".git" / "info").mkdir(parents=True) + store = Store(repo_dir) + exp = store.create_experiment(1, "missing script") + store.update_experiment(exp["label"], status="queued") + runner = ExperimentRunner(store) + + result = asyncio.run(runner.start(exp["label"])) + + saved = store.get_experiment(exp["label"]) + self.assertEqual("error", result["status"]) + self.assertEqual("failed", saved["status"]) + self.assertIn("script not found", saved["error"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/the_lab/PROMPT_api.md b/the_lab/PROMPT_api.md index c2e258e..d02ffd6 100644 --- a/the_lab/PROMPT_api.md +++ b/the_lab/PROMPT_api.md @@ -27,7 +27,7 @@ The core loop (aim for 5-7 API calls per iteration): 1. **Orient** → `GET /orient` — current state + recommended next action. Follow `next_step`. 2. **Leaderboard + Search** → `GET /leaderboard/search?metric=score&q=keyword` — rankings AND search in one call. Includes the best idea's details — no need to GET individual ideas separately. 3. **Create idea** → `POST /ideas/new {parent_ids, description}` — creates git branch and **auto-checkouts** (no separate checkout call needed). -4. **Create + start experiment** → `POST /ideas//experiments {description, script_content, meta, tags}` — when `script_content` is provided, the experiment **auto-starts** (no separate start call needed). +4. **Create experiment** → `POST /ideas//experiments {description, script_content, meta, tags}` — `script_content` is required and the experiment is queued automatically. 5. **Wait** → run `the-lab wait