From e1548cbe140b620e370e9605460da77823ee24da Mon Sep 17 00:00:00 2001 From: Pran-Ker Date: Fri, 5 Jun 2026 14:27:51 -0700 Subject: [PATCH] Add Harbor mode: run SIA self-improvement on external benchmarks Opt-in via --harbor_dataset / --harbor_task_dir. Each generation's target_agent.py is run inside the benchmark's Docker containers and scored by Harbor's native verifiers; per-task rewards + trajectories feed the existing feedback loop. Local behavior and the golden-master prompts are unchanged (Harbor text is injected only when the flag is set). - sia/harbor_agent.py: BaseAgent adapter that runs the generated agent in-container - sia/harbor_runner.py: drives the harbor CLI, folds results into results.json + trajectories - sia/tasks/_shared/reference_harbor_agent.py: in-container reference template - cli/prompts/run_setup/layout/orchestrator: flags, prompt injection, routing - docs/harbor.md, README, pyproject [harbor] extra, tests --- README.md | 31 +++ docs/harbor.md | 91 ++++++++ pyproject.toml | 4 + sia/cli.py | 27 ++- sia/harbor_agent.py | 115 ++++++++++ sia/harbor_runner.py | 234 ++++++++++++++++++++ sia/layout.py | 7 + sia/orchestrator.py | 209 +++++++++++++---- sia/prompts.py | 56 +++++ sia/run_setup.py | 44 +++- sia/tasks/_shared/reference_harbor_agent.py | 169 ++++++++++++++ tests/test_harbor.py | 88 ++++++++ 12 files changed, 1019 insertions(+), 56 deletions(-) create mode 100644 docs/harbor.md create mode 100644 sia/harbor_agent.py create mode 100644 sia/harbor_runner.py create mode 100644 sia/tasks/_shared/reference_harbor_agent.py create mode 100644 tests/test_harbor.py diff --git a/README.md b/README.md index 5f50848..5f47655 100644 --- a/README.md +++ b/README.md @@ -217,6 +217,36 @@ Full step-by-step for both paths: [docs/walkthrough.md](docs/walkthrough.md). --- +## Run on a Harbor benchmark + +Instead of a local task, SIA can self-improve against any external benchmark from the +[Harbor](https://www.harborframework.com/docs) registry (Terminal-Bench, SWE-Bench, AIME, …). Each +generation's agent is run **inside the benchmark's own Docker containers** and scored by Harbor's +native verifiers — no local `evaluate.py` required. + +```bash +uv tool install harbor # the Harbor CLI (ships its own interpreter) +harbor auth login # one-time +pip install 'sia-agent[claude,harbor]' +export ANTHROPIC_API_KEY="..." + +sia run --harbor_dataset terminal-bench-sample@2.0 --max_gen 3 --run_id 1 +``` + +| Flag | Description | +|---|---| +| `--harbor_dataset NAME@VERSION` | Benchmark to download from the Harbor registry (enables Harbor mode) | +| `--harbor_task_dir PATH` | Use a pre-downloaded benchmark directory instead | +| `--harbor_include_task NAME` | Restrict the run to specific task(s); repeatable (cheap for testing) | +| `--harbor_working_dir DIR` | Container working directory the agent operates in (default `/app`) | + +In Harbor mode the meta/feedback prompts gain an injected in-container contract (the base prompts are +unchanged): the generated `target_agent.py` takes `--working_dir / --instruction_file / --log_dir`, +runs an agentic bash loop inside the container, and leaves it in the state the verifier expects. See +[docs/harbor.md](docs/harbor.md) for architecture and details. + +--- + ## Evaluation After every generation the orchestrator scores the target agent automatically and @@ -247,6 +277,7 @@ Full contract, return-format rules, and a complete example: [EVALUATION_GUIDE.md - [docs/architecture.md](docs/architecture.md) — directory layout, generation flow, prompt customization - [docs/walkthrough.md](docs/walkthrough.md) — detailed custom-task walkthrough +- [docs/harbor.md](docs/harbor.md) — run SIA against external Harbor benchmarks - [docs/configuration.md](docs/configuration.md) — agent impls, models, API keys, CLI reference - [EVALUATION_GUIDE.md](EVALUATION_GUIDE.md) — writing `evaluate.py` for a custom task - [docs/troubleshooting.md](docs/troubleshooting.md) — common errors and fixes diff --git a/docs/harbor.md b/docs/harbor.md new file mode 100644 index 0000000..aa48d8c --- /dev/null +++ b/docs/harbor.md @@ -0,0 +1,91 @@ +# Running SIA on Harbor benchmarks + +SIA can run its self-improvement loop against external benchmarks from the +[Harbor](https://www.harborframework.com/docs) registry. Each generation's `target_agent.py` is +attached to the benchmark, executed **inside the benchmark's own Docker containers**, and scored by +Harbor's **native verifiers** — there is no local dataset or `evaluate.py`. + +## How it maps onto SIA + +| SIA concept | Local mode | Harbor mode | +|---|---|---| +| Task | a `tasks//` directory | a downloaded Harbor benchmark (many container tasks) | +| Running a generation | `python target_agent.py --dataset_dir … --working_dir …` | one Harbor **job**: the agent runs in every task's container | +| Scoring | local `evaluate.py` → `results.json` | each task's verifier → reward → aggregated `results.json` | +| Feedback input | execution log + `results.json` | per-task trajectories + per-task rewards (same files) | + +The self-improvement loop itself is unchanged: meta-agent writes gen 1, the feedback agent reads the +scores + trajectories and rewrites the agent for gen 2, and so on. + +## Quick start + +```bash +uv tool install harbor # Harbor CLI (isolated; SIA calls it as a subprocess) +harbor auth login # one-time, for the registry +pip install 'sia-agent[claude,harbor]' +export ANTHROPIC_API_KEY="..." + +# Cheap smoke test: one tiny task +sia run --harbor_dataset terminal-bench-sample@2.0 --harbor_include_task log-summary-date-ranges \ + --max_gen 2 --run_id 1 + +# Full benchmark +sia run --harbor_dataset terminal-bench-sample@2.0 --max_gen 3 --run_id 2 +``` + +## Flags + +| Flag | Description | +|---|---| +| `--harbor_dataset NAME@VERSION` | Benchmark downloaded from the Harbor registry (enables Harbor mode) | +| `--harbor_task_dir PATH` | Use a pre-downloaded benchmark directory (a dir whose children are task folders) | +| `--harbor_include_task NAME` | Restrict to specific task name(s); repeatable. Keeps test runs cheap. | +| `--harbor_working_dir DIR` | Container working directory the agent operates in (default `/app`) | + +`SIA_HARBOR_BIN` overrides the path to the `harbor` CLI if it is not on `PATH`. + +## The in-container agent contract + +In Harbor mode the meta/feedback prompts are **appended** (not edited) with an in-container contract. +The generated `target_agent.py` must follow: + +``` +python3 target_agent.py --working_dir --instruction_file --log_dir +``` + +Inside the container it is guaranteed: internet access, an LLM API key in the environment, the model +id in `SIA_TASK_MODEL`, and the `anthropic` SDK installed. The agent runs an agentic bash loop — +explore the working dir, edit files, verify — and leaves the container in the state the task's +verifier checks. It must **not** write a submission file. The bundled reference implementation is +[`sia/tasks/_shared/reference_harbor_agent.py`](../sia/tasks/_shared/reference_harbor_agent.py); the +meta-agent models its output on it. + +## Architecture + +Two small modules implement the bridge, with no changes to the agent harness: + +- **`sia/harbor_agent.py`** — `SIATargetAgent`, a Harbor `BaseAgent`. For each task it uploads the + generation's `target_agent.py` into the container, runs it against the task instruction, downloads + the trajectory back to the host, and lets the task's verifier score the container. +- **`sia/harbor_runner.py`** — drives the `harbor` CLI as a subprocess (Harbor ships its own + interpreter), then parses the job's `result.json` into SIA's `results.json` plus per-task + `agent_execution/execution_q{i}.json` trajectories that the feedback agent already understands. + +## Output + +Per generation, under `runs/run_{id}/gen_{n}/`: + +- `target_agent.py` — the agent for that generation +- `results.json` — `{score, mean_reward, n_tasks, n_solved, n_errors, per_task: [...]}` +- `agent_execution/execution_q{i}.json` — one trajectory per benchmark task +- `harbor_jobs/` — the raw Harbor job output (per-trial `result.json`, verifier reward, logs) +- `harbor_run.log` — stdout/stderr from the Harbor CLI + +## Notes & limits + +- Benchmark images must allow internet (the default) so the agent can install the SDK and call the + model. The adapter installs `anthropic` best-effort; on a base image without Python it `apt`-installs + it first. +- A task that scores `0` is still a valid run — that is exactly the signal the feedback agent uses to + improve the next generation. +- Harbor removes task containers/images after each run, so every run re-pulls images. diff --git a/pyproject.toml b/pyproject.toml index a041a8e..e7c2f90 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,6 +37,10 @@ claude = ["claude-agent-sdk>=0.1.50"] openhands = ["openhands-ai>=1.6.0"] pydantic-ai = ["pydantic-ai>=1.0"] mlebench = ["google-generativeai>=0.8"] +# Harbor mode drives the `harbor` CLI as a subprocess (it ships its own interpreter), +# so no Python dependency is pinned here. Install the CLI separately, e.g. +# `uv tool install harbor`, or set SIA_HARBOR_BIN to its path. +harbor = [] dev = [ "pytest>=7.0", "ruff>=0.11", diff --git a/sia/cli.py b/sia/cli.py index 27520d7..812058e 100644 --- a/sia/cli.py +++ b/sia/cli.py @@ -39,7 +39,7 @@ def _add_run_args(parser: argparse.ArgumentParser, env_config: Config) -> None: help="Maximum number of generations to run (default: 3)", ) parser.add_argument("--run_id", type=int, default=1, help="Run ID for this experiment (default: 1)") - task_group = parser.add_mutually_exclusive_group(required=True) + task_group = parser.add_mutually_exclusive_group(required=False) task_group.add_argument( "--task", type=str, @@ -51,6 +51,31 @@ def _add_run_args(parser: argparse.ArgumentParser, env_config: Config) -> None: type=str, help="Path to an external task directory (e.g., ./tasks/my-task)", ) + parser.add_argument( + "--harbor_dataset", + type=str, + default=None, + help="Harbor benchmark to run on, e.g. 'terminal-bench-sample@2.0' (enables Harbor mode)", + ) + parser.add_argument( + "--harbor_task_dir", + type=str, + default=None, + help="Path to a pre-downloaded Harbor benchmark directory (enables Harbor mode)", + ) + parser.add_argument( + "--harbor_include_task", + action="append", + default=None, + metavar="TASK_NAME", + help="Restrict the Harbor run to these task name(s); repeatable", + ) + parser.add_argument( + "--harbor_working_dir", + type=str, + default="/app", + help="Container working directory the agent operates in (default: /app)", + ) parser.add_argument( "--meta-agent-profile", dest="meta_agent_profile", diff --git a/sia/harbor_agent.py b/sia/harbor_agent.py new file mode 100644 index 0000000..956ad37 --- /dev/null +++ b/sia/harbor_agent.py @@ -0,0 +1,115 @@ +"""Harbor agent that runs a SIA-generated target agent inside a benchmark task container.""" + +import json +import os +import shlex +from pathlib import Path + +from harbor.agents.base import BaseAgent +from harbor.environments.base import BaseEnvironment +from harbor.models.agent.context import AgentContext + +FORWARDED_KEY_VARS = ( + "ANTHROPIC_API_KEY", + "GEMINI_API_KEY", + "GOOGLE_API_KEY", + "OPENAI_API_KEY", +) +AGENT_DIR = "/tmp/sia_agent" +OUTPUT_DIR = "/tmp/sia_agent/out" + + +async def _maybe_await(value): + return await value if hasattr(value, "__await__") else value + + +class SIATargetAgent(BaseAgent): + def __init__( + self, + *args, + agent_path: str | None = None, + working_dir: str = "/app", + task_model: str | None = None, + run_timeout: int = 900, + pip_packages: str = "anthropic", + **kwargs, + ): + super().__init__(*args, **kwargs) + if not agent_path: + raise ValueError("SIATargetAgent requires agent_path (the generated target_agent.py)") + self._agent_path = str(Path(agent_path).expanduser().resolve()) + self._working_dir = working_dir + self._task_model = task_model or self.model_name + self._run_timeout = int(run_timeout) + self._pip_packages = pip_packages.strip() + + @staticmethod + def name() -> str: + return "sia-target-agent" + + def version(self) -> str: + return "0.1.0" + + def _agent_env(self) -> dict: + env = {k: os.environ[k] for k in FORWARDED_KEY_VARS if os.environ.get(k)} + if self._task_model: + env["SIA_TASK_MODEL"] = self._task_model + return env + + async def setup(self, environment: BaseEnvironment) -> None: + await _maybe_await( + environment.exec( + "command -v python3 >/dev/null 2>&1 || (apt-get update && apt-get install -y python3 python3-pip)", + timeout_sec=600, + ) + ) + if self._pip_packages: + await _maybe_await( + environment.exec( + f"python3 -m pip install --quiet {self._pip_packages} --break-system-packages " + f"|| python3 -m pip install --quiet {self._pip_packages}", + timeout_sec=600, + ) + ) + await _maybe_await(environment.exec(f"mkdir -p {AGENT_DIR} {OUTPUT_DIR}")) + + async def run(self, instruction: str, environment: BaseEnvironment, context: AgentContext) -> None: + await _maybe_await(environment.upload_file(self._agent_path, f"{AGENT_DIR}/target_agent.py")) + instruction_host = Path(self.logs_dir) / "INSTRUCTION.md" + instruction_host.write_text(instruction, encoding="utf-8") + await _maybe_await(environment.upload_file(str(instruction_host), f"{AGENT_DIR}/INSTRUCTION.md")) + + cmd = ( + f"cd {shlex.quote(self._working_dir)} && " + f"python3 {AGENT_DIR}/target_agent.py " + f"--working_dir {shlex.quote(self._working_dir)} " + f"--instruction_file {AGENT_DIR}/INSTRUCTION.md " + f"--log_dir {OUTPUT_DIR}" + ) + result = await _maybe_await(environment.exec(cmd, env=self._agent_env(), timeout_sec=self._run_timeout)) + + stdout = getattr(result, "stdout", "") or "" + stderr = getattr(result, "stderr", "") or "" + return_code = getattr(result, "return_code", None) + (Path(self.logs_dir) / "agent_stdout.log").write_text(stdout + "\n" + stderr, encoding="utf-8") + + try: + await _maybe_await(environment.download_dir(OUTPUT_DIR, str(Path(self.logs_dir) / "sia_out"))) + except Exception as exc: + (Path(self.logs_dir) / "download_error.txt").write_text(str(exc), encoding="utf-8") + + self._populate_context(context, return_code) + + def _populate_context(self, context: AgentContext, return_code) -> None: + traj = Path(self.logs_dir) / "sia_out" / "agent_execution.json" + if not traj.is_file(): + return + try: + usage = json.loads(traj.read_text(encoding="utf-8")).get("usage", {}) + except Exception: + return + if usage.get("input_tokens"): + context.n_input_tokens = usage["input_tokens"] + if usage.get("output_tokens"): + context.n_output_tokens = usage["output_tokens"] + context.metadata = {**(context.metadata or {}), "agent_return_code": return_code} diff --git a/sia/harbor_runner.py b/sia/harbor_runner.py new file mode 100644 index 0000000..62b9e56 --- /dev/null +++ b/sia/harbor_runner.py @@ -0,0 +1,234 @@ +"""Drive a Harbor job for one SIA generation and fold the result back into SIA.""" + +import json +import logging +import os +import shutil +import subprocess +import tempfile +from dataclasses import dataclass +from pathlib import Path + +logger = logging.getLogger(__name__) + +ADAPTER_IMPORT_NAME = "sia_harbor_adapter" +ADAPTER_CLASS = "SIATargetAgent" +JOB_NAME = "sia" + + +@dataclass +class HarborRun: + benchmark_path: str + task_model: str + working_dir: str + include_tasks: list[str] | None + + +def resolve_harbor_bin() -> str: + harbor_bin = os.getenv("SIA_HARBOR_BIN") or shutil.which("harbor") + if not harbor_bin: + raise SystemExit( + "Harbor CLI not found. Install it (e.g. `uv tool install harbor`) and ensure " + "`harbor` is on PATH, or set SIA_HARBOR_BIN to its full path." + ) + return harbor_bin + + +def parse_benchmark(spec: str) -> dict: + path = Path(spec).expanduser() + if path.exists(): + return {"path": str(path.resolve())} + if "@" in spec: + name, version = spec.split("@", 1) + return {"name": name, "version": version} + return {"name": spec} + + +def prepare_harbor_benchmark(benchmark: str, dest_dir: str, n_samples: int = 3) -> tuple[str, str]: + """Resolve a benchmark to a local task-parent dir + sample instructions. + + A local directory is used as-is; a ``name@version`` spec is downloaded (export + layout) into ``dest_dir/benchmark/``. Returns ``(benchmark_path, sample_text)`` + where benchmark_path is a directory whose children are task folders. + """ + p = Path(benchmark).expanduser() + if p.exists(): + parent = p.resolve() + else: + out = Path(dest_dir, "benchmark") + out.mkdir(parents=True, exist_ok=True) + harbor_bin = resolve_harbor_bin() + logger.info("Downloading Harbor benchmark '%s' to %s", benchmark, out) + subprocess.run([harbor_bin, "download", benchmark, "--export", "-o", str(out)], check=True) + instr = sorted(out.glob("*/*/instruction.md")) or sorted(out.glob("**/instruction.md")) + if not instr: + raise SystemExit(f"No tasks (instruction.md) found after downloading '{benchmark}'") + parent = instr[0].parent.parent + + samples = [] + for ip in sorted(parent.glob("*/instruction.md"))[:n_samples]: + text = ip.read_text(encoding="utf-8") + if len(text) > 1500: + text = text[:1500] + "\n...(truncated)" + samples.append(f"### Task: {ip.parent.name}\n{text}") + sample_text = "\n\n".join(samples) if samples else "(no sample instructions found)" + return str(parent), sample_text + + +def _build_config( + *, + target_agent_path, + task_model, + benchmark, + jobs_dir, + working_dir, + include_tasks, + n_concurrent, + run_timeout, +) -> dict: + dataset = parse_benchmark(benchmark) + if include_tasks: + dataset["task_names"] = include_tasks + return { + "job_name": JOB_NAME, + "jobs_dir": jobs_dir, + "n_concurrent_trials": n_concurrent, + "agents": [ + { + "import_path": f"{ADAPTER_IMPORT_NAME}:{ADAPTER_CLASS}", + "model_name": task_model, + "kwargs": { + "agent_path": target_agent_path, + "task_model": task_model, + "working_dir": working_dir, + "run_timeout": run_timeout, + }, + } + ], + "datasets": [dataset], + } + + +def _stage_adapter(staging_dir: Path) -> None: + src = Path(__file__).with_name("harbor_agent.py") + shutil.copy(src, staging_dir / f"{ADAPTER_IMPORT_NAME}.py") + + +def _primary_reward(rewards: dict): + if not rewards: + return None + if "reward" in rewards: + return rewards["reward"] + numeric = [v for v in rewards.values() if isinstance(v, (int, float))] + return sum(numeric) / len(numeric) if numeric else None + + +def _collect_results(job_dir: Path, gen_dir: Path, benchmark: str) -> dict: + exec_out_dir = gen_dir / "agent_execution" + exec_out_dir.mkdir(parents=True, exist_ok=True) + + per_task = [] + trials = sorted(p for p in job_dir.glob("*/result.json") if p.parent.name != job_dir.name) + for idx, trial_path in enumerate(trials): + try: + trial = json.loads(trial_path.read_text(encoding="utf-8")) + except Exception as exc: + per_task.append({"task": trial_path.parent.name, "error": f"unreadable result: {exc}"}) + continue + rewards = (trial.get("verifier_result") or {}).get("rewards") or {} + reward = _primary_reward(rewards) + per_task.append( + { + "task": trial.get("task_name", trial_path.parent.name), + "reward": reward, + "rewards": rewards, + "solved": bool(reward and reward >= 1.0), + "exception": trial.get("exception_info"), + } + ) + traj = trial_path.parent / "agent" / "sia_out" / "agent_execution.json" + if traj.is_file(): + shutil.copy(traj, exec_out_dir / f"execution_q{idx}.json") + + rewards_vals = [t["reward"] for t in per_task if isinstance(t.get("reward"), (int, float))] + mean_reward = sum(rewards_vals) / len(rewards_vals) if rewards_vals else 0.0 + results = { + "benchmark": benchmark, + "score": mean_reward, + "mean_reward": mean_reward, + "n_tasks": len(per_task), + "n_solved": sum(1 for t in per_task if t.get("solved")), + "n_errors": sum(1 for t in per_task if t.get("exception")), + "per_task": per_task, + } + (gen_dir / "results.json").write_text(json.dumps(results, indent=2), encoding="utf-8") + return results + + +def run_generation_on_harbor( + target_agent_path: str, + gen_dir: str, + task_model: str, + benchmark: str, + *, + working_dir: str = "/app", + include_tasks: list[str] | None = None, + n_concurrent: int = 2, + run_timeout: int = 900, +) -> dict: + harbor_bin = resolve_harbor_bin() + gen_path = Path(gen_dir).resolve() + jobs_dir = gen_path / "harbor_jobs" + target_agent_path = str(Path(target_agent_path).resolve()) + + with tempfile.TemporaryDirectory(prefix="sia_harbor_") as staging: + staging_dir = Path(staging) + _stage_adapter(staging_dir) + config = _build_config( + target_agent_path=target_agent_path, + task_model=task_model, + benchmark=benchmark, + jobs_dir=str(jobs_dir), + working_dir=working_dir, + include_tasks=include_tasks, + n_concurrent=n_concurrent, + run_timeout=run_timeout, + ) + config_path = staging_dir / "job.json" + config_path.write_text(json.dumps(config, indent=2), encoding="utf-8") + + env = dict(os.environ) + env["PYTHONPATH"] = os.pathsep.join(filter(None, [str(staging_dir), env.get("PYTHONPATH", "")])) + + logger.info("Launching Harbor job on benchmark '%s' (model=%s)", benchmark, task_model) + proc = subprocess.run([harbor_bin, "run", "-c", str(config_path)], env=env, text=True, capture_output=True) + (gen_path / "harbor_run.log").write_text( + (proc.stdout or "") + "\n--- STDERR ---\n" + (proc.stderr or ""), encoding="utf-8" + ) + if proc.returncode != 0: + logger.error("Harbor run exited with code %s (see harbor_run.log)", proc.returncode) + + job_dir = jobs_dir / JOB_NAME + if not (job_dir / "result.json").is_file(): + results = { + "benchmark": benchmark, + "score": 0.0, + "mean_reward": 0.0, + "n_tasks": 0, + "n_solved": 0, + "n_errors": 0, + "per_task": [], + "error": f"Harbor job did not produce result.json (exit {proc.returncode})", + } + (gen_path / "results.json").write_text(json.dumps(results, indent=2), encoding="utf-8") + return results + + results = _collect_results(job_dir, gen_path, benchmark) + logger.info( + "Harbor job complete: score=%.3f solved=%d/%d errors=%d", + results["score"], + results["n_solved"], + results["n_tasks"], + results["n_errors"], + ) + return results diff --git a/sia/layout.py b/sia/layout.py index 1efbc01..d2710fb 100644 --- a/sia/layout.py +++ b/sia/layout.py @@ -29,6 +29,7 @@ class Names: STDOUT_LOG = "target_agent_stdout.log" TRAIN_STDOUT_LOG = "train_stdout.log" EVAL_LOG = "evaluation.log" + HARBOR_RUN_LOG = "harbor_run.log" RESULTS_JSON = "results.json" CONTEXT_MD = "context.md" IMPROVEMENT_MD = "improvement.md" @@ -47,9 +48,15 @@ class Names: REFERENCE_AGENT_FILE = "reference_target_agent.py" SAMPLE_TASK_DESCRIPTIONS = f"{REFERENCE_DIR}/SAMPLE_TASK_DESCRIPTIONS.md" REFERENCE_AGENT = f"{REFERENCE_DIR}/{REFERENCE_AGENT_FILE}" + REFERENCE_HARBOR_AGENT = "reference_harbor_agent.py" SHARED_DIR = "_shared" +def bundled_shared_dir() -> str: + """Absolute path to the bundled ``sia/tasks/_shared`` directory.""" + return str(resource_files("sia.tasks") / Names.SHARED_DIR) + + def venv_python_path(venv_dir: str) -> str: """Path to the python executable inside a venv.""" return os.path.join(venv_dir, "bin", "python") diff --git a/sia/orchestrator.py b/sia/orchestrator.py index 0411565..a1284d3 100644 --- a/sia/orchestrator.py +++ b/sia/orchestrator.py @@ -55,6 +55,7 @@ from sia import __version__, cli from sia.agent_reference import ResolvedAgentReference, copy_reference_into, resolve_agent_reference from sia.config import Config +from sia.harbor_runner import HarborRun, prepare_harbor_benchmark, run_generation_on_harbor from sia.io_utils import file_size_ok, write_text from sia.layout import BUNDLED_TASKS, Names, RunLayout, TaskLayout, resolve_task_dir, venv_python_path from sia.logging_setup import configure_logging, get_logger @@ -62,7 +63,14 @@ from sia.prompts import build_feedback_prompt, build_meta_prompt from sia.providers import Provider from sia.results import FeedbackContext, TargetAgentResult -from sia.run_setup import RunSetup, TaskFiles, install_requirements, load_task_files, setup_run_directory +from sia.run_setup import ( + RunSetup, + TaskFiles, + install_requirements, + load_harbor_task_files, + load_task_files, + setup_run_directory, +) from sia.util import run_agent __all__ = [ @@ -422,6 +430,36 @@ def _run_target_agent( return TargetAgentResult(False, stdout, "", error_msg).as_tuple() +def _run_target_agent_on_harbor(target_agent_path: str, gen_dir: str, harbor: HarborRun) -> tuple[bool, str, str]: + """Run this generation's agent on a Harbor benchmark. Returns (success, stdout, error_msg). + + run_generation_on_harbor writes results.json + agent_execution/ into gen_dir, so the + downstream feedback path consumes them exactly as in local mode. + """ + try: + results = run_generation_on_harbor( + target_agent_path=target_agent_path, + gen_dir=gen_dir, + task_model=harbor.task_model, + benchmark=harbor.benchmark_path, + working_dir=harbor.working_dir, + include_tasks=harbor.include_tasks, + ) + except Exception as e: + msg = f"Harbor run failed: {e!s}" + logger.exception(f" ✗ {msg}") + return False, msg, msg + + success = "error" not in results and results["n_tasks"] > 0 + summary = ( + f"Harbor benchmark: {results['benchmark']}\n" + f"score (mean reward): {results['score']:.4f}\n" + f"solved: {results['n_solved']}/{results['n_tasks']} | errors: {results['n_errors']}" + ) + logger.info(f" ✓ Harbor run complete | {summary.splitlines()[-1]}") + return success, summary, results.get("error", "") + + def _build_feedback_context( current_gen: int, gen_dir: str, @@ -566,11 +604,15 @@ def _run_feedback_agent( target_provider: Provider, focus: str = "harness", resolved_ref: ResolvedAgentReference | None = None, + task_text: str | None = None, + harbor: bool = False, ) -> None: """Run the feedback agent to create an improved target agent or train.py. Args: focus: "harness" (default) for code improvement or "weights" for RL-based tuning + task_text: task description to use directly (Harbor mode has no local task.md) + harbor: when True, append the Harbor in-container contract to the feedback prompt """ # Read the appropriate agent file based on focus mode gen_dir = os.path.join(run_dir, f"gen_{current_gen}") @@ -580,7 +622,7 @@ def _run_feedback_agent( agent_file = os.path.join(gen_dir, Names.TARGET_AGENT) agent_py = Path(agent_file).read_text(encoding="utf-8") - task = Path(dataset_dir, "task.md").read_text(encoding="utf-8") + task = task_text if task_text is not None else Path(dataset_dir, "task.md").read_text(encoding="utf-8") previous_gens_list = list(range(1, current_gen)) if current_gen > 1 else [] previous_gens_text = ", ".join(map(str, previous_gens_list)) if previous_gens_list else "None" @@ -604,6 +646,7 @@ def _run_feedback_agent( provider=target_provider, requirements_dir=requirements_dir, focus=focus, + harbor=harbor, ) os.makedirs(next_gen_dir, exist_ok=True) @@ -647,12 +690,15 @@ def run_generation( focus: str = "harness", training_sandbox: str = "modal", resolved_ref: ResolvedAgentReference | None = None, + harbor: HarborRun | None = None, ) -> None: """Execute one generation: run target agent, evaluate, optionally run feedback agent. Args: focus: "harness" for code improvement or "weights" for RL-based tuning training_sandbox: "modal" (default) or "sandboxfusion" for train.py code execution + harbor: when set, run the generation on a Harbor benchmark (scored by the + benchmark's own verifiers) instead of locally; the rest of the loop is unchanged. """ run_dir = run_setup.run_directory layout = RunLayout(run_dir) @@ -676,24 +722,34 @@ def run_generation( generation_start_time = time.time() - # Run target agent - target_agent_success, target_agent_stdout, target_agent_stderr, target_agent_error_msg = _run_target_agent( - venv_dir=run_setup.venv_dir, - target_agent_path=target_agent_path, - abs_dataset_dir=abs_dataset_dir, - gen_dir=gen_dir, - stdout_log_file=stdout_log_file, - sandbox=sandbox, - env_config=env_config, - ) + if harbor is not None: + # Harbor mode: run the agent in the benchmark's containers; the runner writes + # results.json + agent_execution/ into gen_dir, so no local evaluation step. + target_agent_success, target_agent_stdout, target_agent_error_msg = _run_target_agent_on_harbor( + target_agent_path, gen_dir, harbor + ) + target_agent_stderr = "" + stdout_log_file = os.path.join(gen_dir, Names.HARBOR_RUN_LOG) + generation_duration = time.time() - generation_start_time + else: + # Run target agent + target_agent_success, target_agent_stdout, target_agent_stderr, target_agent_error_msg = _run_target_agent( + venv_dir=run_setup.venv_dir, + target_agent_path=target_agent_path, + abs_dataset_dir=abs_dataset_dir, + gen_dir=gen_dir, + stdout_log_file=stdout_log_file, + sandbox=sandbox, + env_config=env_config, + ) - generation_duration = time.time() - generation_start_time + generation_duration = time.time() - generation_start_time - # Run evaluation (if evaluate.py exists) - logger.info("=" * 60) - logger.info("Running evaluation (if available)...") - run_evaluation(gen_dir, dataset_dir, run_setup.venv_dir, config=env_config) - logger.info("=" * 60) + # Run evaluation (if evaluate.py exists) + logger.info("=" * 60) + logger.info("Running evaluation (if available)...") + run_evaluation(gen_dir, dataset_dir, run_setup.venv_dir, config=env_config) + logger.info("=" * 60) # Add generation to context improvement_md_path = layout.improvement_md(current_gen) @@ -748,6 +804,8 @@ def run_generation( target_provider=target_provider, focus=focus, resolved_ref=resolved_ref, + task_text=task_files.task_md if harbor is not None else None, + harbor=harbor is not None, ) else: logger.info(f"Generation {current_gen} is the final generation. Skipping feedback agent.") @@ -785,9 +843,19 @@ def main(): serve_in_background(host=args.web_host, port=args.web_port, runs_dir=Names.RUNS_ROOT) max_gen = args.max_gen - task_dir, shared_dir = resolve_task_dir(args.task, args.task_dir) run_id = args.run_id + # Resolve execution mode: exactly one of the four task selectors must be given. + harbor_mode = bool(args.harbor_dataset or args.harbor_task_dir) + n_selectors = sum(bool(x) for x in (args.task, args.task_dir, args.harbor_dataset, args.harbor_task_dir)) + if n_selectors != 1: + raise SystemExit("Provide exactly one of --task, --task_dir, --harbor_dataset, or --harbor_task_dir") + if harbor_mode: + task_dir, shared_dir, benchmark = None, None, (args.harbor_dataset or args.harbor_task_dir) + else: + task_dir, shared_dir = resolve_task_dir(args.task, args.task_dir) + benchmark = None + # Resolve agent profiles: the meta profile bundles agent_impl + model + provider; # the target profile bundles model + provider + agent_reference (the seed code). meta_profile = load_meta_agent_profile(args.meta_agent_profile) @@ -797,12 +865,22 @@ def main(): agent_impl = meta_profile.agent_impl target_provider = target_profile.provider - task_layout = TaskLayout(task_dir, shared_dir) - resolved_ref = resolve_agent_reference(target_profile.agent_reference, task_layout) + if harbor_mode: + task_layout = None + resolved_ref = None + else: + assert task_dir is not None and shared_dir is not None + task_layout = TaskLayout(task_dir, shared_dir) + resolved_ref = resolve_agent_reference(target_profile.agent_reference, task_layout) logger.info("Configuration:") logger.info(f" - Maximum generations: {max_gen}") - logger.info(f" - Task directory: {task_dir}") + if harbor_mode: + logger.info(f" - Harbor benchmark: {benchmark}") + if args.harbor_include_task: + logger.info(f" - Harbor tasks (filter): {', '.join(args.harbor_include_task)}") + else: + logger.info(f" - Task directory: {task_dir}") logger.info(f" - Run ID: {run_id}") logger.info( f" - Meta agent profile: {meta_profile.profile_id} (agent_impl={agent_impl}, model={meta_model}, " @@ -844,36 +922,57 @@ def main(): ) # ======================== - # SECTION 1: Load Files from Task Directory - # ======================== - - task_files = load_task_files(task_dir, shared_dir, resolved_ref) - - # ======================== - # SECTION 2: Setup Run Directories + # SECTION 1+2: Load Task Files + Setup Run Directories # ======================== - - run_setup = setup_run_directory( - run_id, - task_dir, - meta_model, - task_model, - agent_impl, - max_gen, - focus=args.focus, - config=env_config, - meta_profile=meta_profile, - target_profile=target_profile, - ) + # Harbor mode needs the run directory before it can download the benchmark, and + # runs the target agent in the benchmark's containers (no local venv). + + if harbor_mode: + assert benchmark is not None + run_setup = setup_run_directory( + run_id, + benchmark, + meta_model, + task_model, + agent_impl, + max_gen, + focus=args.focus, + config=env_config, + meta_profile=meta_profile, + target_profile=target_profile, + create_venv=False, + ) + benchmark_path, sample_descriptions = prepare_harbor_benchmark(benchmark, run_setup.run_directory) + task_files = load_harbor_task_files(benchmark, sample_descriptions) + logger.info(f" ✓ Harbor benchmark prepared at: {benchmark_path}") + else: + assert task_dir is not None and shared_dir is not None + task_files = load_task_files(task_dir, shared_dir, resolved_ref) + run_setup = setup_run_directory( + run_id, + task_dir, + meta_model, + task_model, + agent_impl, + max_gen, + focus=args.focus, + config=env_config, + meta_profile=meta_profile, + target_profile=target_profile, + ) # ======================== # SECTION 3: Build Initial Prompt # ======================== # A multi-file directory reference is read by the agent from disk (copied into its - # working dir) rather than embedded in the prompt. - copy_reference_into(resolved_ref, run_setup.meta_agent_working_directory) - reference_dir = run_setup.meta_agent_working_directory if resolved_ref.ref_dir is not None else None + # working dir) rather than embedded in the prompt. Harbor mode has no resolved_ref: + # its reference agent is bundled and already embedded in task_files. + if resolved_ref is not None: + copy_reference_into(resolved_ref, run_setup.meta_agent_working_directory) + reference_dir = run_setup.meta_agent_working_directory if resolved_ref.ref_dir is not None else None + else: + reference_dir = None # Log focus mode and training sandbox logger.info("Configuration (continued):") @@ -890,6 +989,7 @@ def main(): reference_dir=reference_dir, focus=args.focus, training_sandbox=args.training_sandbox, + harbor=harbor_mode, ) # ======================== @@ -916,9 +1016,21 @@ def main(): # SECTION 5: Main Loop - Run Target Agent and Feedback Agent # ======================== - dataset_directory = task_layout.dataset_dir - abs_dataset_directory = task_layout.abs_dataset_dir - logger.info(f"Dataset directory: {abs_dataset_directory}") + if harbor_mode: + # Harbor mode has no local dataset dir; these are unused when harbor_run is set. + dataset_directory = abs_dataset_directory = "" + harbor_run = HarborRun( + benchmark_path=benchmark_path, + task_model=task_model, + working_dir=args.harbor_working_dir, + include_tasks=args.harbor_include_task, + ) + else: + assert task_layout is not None + dataset_directory = task_layout.dataset_dir + abs_dataset_directory = task_layout.abs_dataset_dir + harbor_run = None + logger.info(f"Dataset directory: {abs_dataset_directory}") for current_gen in range(1, max_gen + 1): logger.info("=" * 80) @@ -940,6 +1052,7 @@ def main(): focus=args.focus, training_sandbox=args.training_sandbox, resolved_ref=resolved_ref, + harbor=harbor_run, ) # Early stopping for weights mode: if feedback agent signaled completion diff --git a/sia/prompts.py b/sia/prompts.py index 4ca47a7..869cf26 100644 --- a/sia/prompts.py +++ b/sia/prompts.py @@ -609,6 +609,56 @@ async def main(dataset_dir, working_dir): """ +# Appended ONLY in Harbor mode. The base prompts stay byte-identical (golden-master +# safe); this block redefines the target agent's contract for in-container execution so it +# satisfies a Harbor task's native verifier instead of writing a local submission file. +HARBOR_META_PROMPT = """ + +================================================================================ +HARBOR MODE — IN-CONTAINER EXECUTION CONTRACT (OVERRIDES THE DATASET/SUBMISSION RULES ABOVE) +================================================================================ +Your target_agent.py is NOT run against a local dataset directory and there is NO +submission file or evaluate.py. Instead it is uploaded into the Docker container of +each benchmark task and executed INSIDE that container; the task's own verifier +inspects the final state of the container to score you. + +Use THIS command-line contract (ignore the earlier --dataset_dir / submission rules): + + python3 target_agent.py --working_dir --instruction_file --log_dir + + --working_dir directory to operate in; create the required output files here + --instruction_file path to the task instruction describing what to accomplish + --log_dir write your trajectory to /agent_execution.json + +Runtime guarantees inside the container: + * internet is available, + * an LLM API key is exported (ANTHROPIC_API_KEY), + * the solver model id is in env var SIA_TASK_MODEL (use it; it equals {task_model}), + * the `anthropic` SDK is installed. + +Build an agentic loop: read the instruction, use the model with a bash tool to explore +the working directory, create/edit files, and verify your work, then stop when done. +The "reference target agent" shown above already follows THIS exact contract — model +your target_agent.py on it. Record every step to /agent_execution.json. Do NOT +write a submission file; satisfy the task by leaving the container in the requested state. +""" + +HARBOR_FEEDBACK_PROMPT = """ + +================================================================================ +HARBOR MODE +================================================================================ +The target agent runs INSIDE each benchmark task's Docker container (no dataset dir, +no submission file). Its CLI contract is: + python3 target_agent.py --working_dir --instruction_file --log_dir +The score is the mean reward from the benchmark's own verifiers — see the EVALUATION +RESULTS (results.json): per_task rewards, n_solved, n_errors. When you rewrite +target_agent.py, KEEP this exact CLI contract and the in-container agentic-loop pattern; +focus on making the agent explore, edit files, and verify more reliably across diverse +tasks. Do NOT introduce a submission-file workflow. +""" + + def build_meta_prompt( task_files: TaskFiles, task_model: str, @@ -617,6 +667,7 @@ def build_meta_prompt( reference_dir: str | None = None, focus: str = "harness", training_sandbox: str = "modal", + harbor: bool = False, ) -> str: """Build the meta-agent prompt for creating the initial target agent. @@ -705,6 +756,8 @@ def build_meta_prompt( Example invocation (paths will vary at runtime): python target_agent.py --dataset_dir /path/to/dataset --working_dir /path/to/working """ + if harbor: + base = base + HARBOR_META_PROMPT.replace("{task_model}", task_model) if provider is None or provider.client_kind != "openai": return base return build_target_client_setup(provider, task_model) + base @@ -755,6 +808,7 @@ def build_feedback_prompt( provider: Provider | None = None, requirements_dir: str | None = None, focus: str = "harness", + harbor: bool = False, ) -> str: """Build the feedback agent prompt for improving the target agent or train.py. @@ -951,6 +1005,8 @@ def build_feedback_prompt( "(one package per line) to declare third-party packages your target_agent.py needs; they are " 'installed before the target agent runs. This is the one exception to the "only two files" rule above.\n' ) + if harbor: + base += HARBOR_FEEDBACK_PROMPT if provider is None or provider.client_kind != "openai": return base return build_target_client_setup(provider, task_model) + base diff --git a/sia/run_setup.py b/sia/run_setup.py index 7757ad7..c0c78aa 100644 --- a/sia/run_setup.py +++ b/sia/run_setup.py @@ -19,7 +19,7 @@ from sia.config import Config from sia.context_manager import ContextManager -from sia.layout import RunLayout, TaskLayout, venv_pip_path, venv_python_path +from sia.layout import Names, RunLayout, TaskLayout, bundled_shared_dir, venv_pip_path, venv_python_path from sia.logging_setup import get_logger if TYPE_CHECKING: @@ -88,6 +88,32 @@ def load_task_files( ) +def load_harbor_task_files(benchmark: str, sample_descriptions: str) -> TaskFiles: + """Build TaskFiles for Harbor mode from the bundled in-container reference agent. + + The reference agent and task overview describe the in-container contract; the + sample descriptions are real task instructions sampled from the benchmark. + """ + shared = bundled_shared_dir() + reference_target_agent_py = Path(shared, Names.REFERENCE_HARBOR_AGENT).read_text() + with open(os.path.join(shared, Names.SHARED_SAMPLE_EXECUTION)) as f: + sample_agent_execution = json.load(f) + + task_md = ( + f"You are building ONE general-purpose agent to solve tasks from the Harbor " + f"benchmark '{benchmark}'. Your agent runs inside each task's Docker container " + f"and must leave the container in the state the task's verifier expects. Below " + f"are representative task instructions your agent must be able to handle:\n\n" + f"{sample_descriptions}" + ) + return TaskFiles( + sample_task_descriptions=sample_descriptions, + reference_target_agent_py=reference_target_agent_py, + sample_agent_execution=sample_agent_execution, + task_md=task_md, + ) + + def _create_venv(venv_dir: str, packages: list[str]) -> None: """Create a virtual environment and install packages.""" if shutil.which("uv"): @@ -150,11 +176,14 @@ def setup_run_directory( config: Config | None = None, meta_profile: MetaAgentProfile | None = None, target_profile: TargetAgentProfile | None = None, + create_venv: bool = True, ) -> RunSetup: """Create run directories, venv, and context manager. Args: focus: "harness" (default) or "weights" for RL-based tuning. Determines which packages are installed. + create_venv: when False, skip the local venv (Harbor mode runs the target agent in + the benchmark's own containers, so no local interpreter is needed). """ cfg = config or Config() layout = RunLayout.for_run_id(run_id) @@ -173,12 +202,13 @@ def setup_run_directory( os.makedirs(meta_agent_working_directory, exist_ok=False) venv_dir = layout.venv_dir - logger.info(f"Creating virtual environment at: {venv_dir}") - # Build packages list conditionally based on focus mode - packages = cfg.VENV_PACKAGES.copy() - if focus == "weights": - packages.extend(cfg.WEIGHTS_VENV_PACKAGES) - _create_venv(venv_dir, packages) + if create_venv: + logger.info(f"Creating virtual environment at: {venv_dir}") + # Build packages list conditionally based on focus mode + packages = cfg.VENV_PACKAGES.copy() + if focus == "weights": + packages.extend(cfg.WEIGHTS_VENV_PACKAGES) + _create_venv(venv_dir, packages) _write_run_profiles(run_directory, meta_profile, target_profile) diff --git a/sia/tasks/_shared/reference_harbor_agent.py b/sia/tasks/_shared/reference_harbor_agent.py new file mode 100644 index 0000000..283aa6e --- /dev/null +++ b/sia/tasks/_shared/reference_harbor_agent.py @@ -0,0 +1,169 @@ +#!/usr/bin/env python3 +"""Reference Harbor target agent — the in-container template the meta-agent imitates. + +Runs inside the benchmark task's container and leaves it in the state the verifier accepts +(no dataset, no submission file). CLI contract, provided by sia/harbor_agent.py at runtime: + + python3 target_agent.py --working_dir --instruction_file --log_dir + +Guarantees in-container: internet, an LLM API key in env, the model id in SIA_TASK_MODEL, +and the `anthropic` SDK installed. Drives a bash tool-use loop and records every step to +/agent_execution.json for the SIA feedback agent. +""" + +import argparse +import json +import os +import subprocess +from datetime import datetime +from pathlib import Path + +DEFAULT_MODEL = "claude-haiku-4-5-20251001" +MAX_TURNS = 20 +COMMAND_TIMEOUT = 300 + +BASH_TOOL = { + "name": "run_bash", + "description": ( + "Run a bash command inside the task's working directory and get back its " + "stdout, stderr and exit code. Use this to explore files, edit code, and " + "produce the outputs the task requires." + ), + "input_schema": { + "type": "object", + "properties": { + "command": {"type": "string", "description": "The bash command to execute."}, + }, + "required": ["command"], + }, +} + +SYSTEM_PROMPT = ( + "You are an autonomous software engineer working inside a Linux container. " + "You are given a task instruction and a working directory. Accomplish the task " + "by running shell commands with the run_bash tool: inspect the environment, edit " + "or create files, and verify your work. The task is graded by an automated " + "verifier that checks the final state of the container, so make sure the required " + "output files exist at the exact paths requested. When you are confident the task " + "is fully complete, reply with a short final message and DO NOT call any more tools." +) + + +def _clip(text: str, limit: int = 6000) -> str: + return text if len(text) <= limit else text[:limit] + f"\n...(truncated, {len(text)} chars)" + + +def run_bash(command: str, working_dir: str) -> dict: + try: + proc = subprocess.run( + command, + shell=True, + cwd=working_dir, + capture_output=True, + text=True, + timeout=COMMAND_TIMEOUT, + executable="/bin/bash", + ) + stdout, stderr, code = proc.stdout, proc.stderr, proc.returncode + except subprocess.TimeoutExpired: + stdout, stderr, code = "", f"command timed out after {COMMAND_TIMEOUT}s", 124 + except Exception as exc: + stdout, stderr, code = "", f"failed to run command: {exc}", 1 + return {"stdout": _clip(stdout), "stderr": _clip(stderr), "exit_code": code} + + +def main() -> None: + parser = argparse.ArgumentParser(description="Reference Harbor target agent") + parser.add_argument("--working_dir", required=True, help="Directory to operate in") + parser.add_argument("--instruction_file", required=True, help="Path to the task instruction") + parser.add_argument("--log_dir", required=True, help="Where to write agent_execution.json") + parser.add_argument("--task_model", default=None, help="Solver model id (overrides SIA_TASK_MODEL)") + args = parser.parse_args() + + working_dir = args.working_dir + log_dir = Path(args.log_dir) + log_dir.mkdir(parents=True, exist_ok=True) + log_path = log_dir / "agent_execution.json" + model = args.task_model or os.getenv("SIA_TASK_MODEL") or DEFAULT_MODEL + instruction = Path(args.instruction_file).read_text(encoding="utf-8") + + trajectory = { + "model": model, + "working_dir": working_dir, + "started_at": datetime.now().isoformat(), + "instruction": instruction, + "steps": [], + "usage": {"input_tokens": 0, "output_tokens": 0}, + } + + def flush() -> None: + trajectory["finished_at"] = datetime.now().isoformat() + log_path.write_text(json.dumps(trajectory, indent=2), encoding="utf-8") + + try: + import anthropic + except ImportError: + trajectory["error"] = "anthropic SDK not available in container" + flush() + raise SystemExit("anthropic SDK not installed") + + client = anthropic.Anthropic() + messages = [ + { + "role": "user", + "content": ( + f"Task working directory: {working_dir}\n\n" + f"Task instruction:\n{instruction}\n\n" + "Begin by exploring the working directory, then complete the task." + ), + } + ] + + for turn in range(MAX_TURNS): + response = client.messages.create( + model=model, + max_tokens=4096, + system=SYSTEM_PROMPT, + tools=[BASH_TOOL], + messages=messages, + ) + usage = getattr(response, "usage", None) + if usage: + trajectory["usage"]["input_tokens"] += getattr(usage, "input_tokens", 0) or 0 + trajectory["usage"]["output_tokens"] += getattr(usage, "output_tokens", 0) or 0 + + assistant_content = [] + tool_uses = [] + for block in response.content: + if block.type == "text": + assistant_content.append({"type": "text", "text": block.text}) + trajectory["steps"].append({"turn": turn, "type": "assistant_text", "text": block.text}) + elif block.type == "tool_use": + assistant_content.append({"type": "tool_use", "id": block.id, "name": block.name, "input": block.input}) + tool_uses.append(block) + + messages.append({"role": "assistant", "content": assistant_content}) + + if response.stop_reason != "tool_use" or not tool_uses: + trajectory["stop_reason"] = response.stop_reason + break + + tool_results = [] + for tu in tool_uses: + command = tu.input.get("command", "") + result = run_bash(command, working_dir) + trajectory["steps"].append({"turn": turn, "type": "tool_use", "command": command, "result": result}) + tool_results.append({"type": "tool_result", "tool_use_id": tu.id, "content": json.dumps(result)}) + messages.append({"role": "user", "content": tool_results}) + else: + trajectory["stop_reason"] = "max_turns" + + flush() + print( + f"[reference_harbor_agent] done | turns={len(trajectory['steps'])} " + f"| in={trajectory['usage']['input_tokens']} out={trajectory['usage']['output_tokens']}" + ) + + +if __name__ == "__main__": + main() diff --git a/tests/test_harbor.py b/tests/test_harbor.py new file mode 100644 index 0000000..522fed8 --- /dev/null +++ b/tests/test_harbor.py @@ -0,0 +1,88 @@ +"""Unit tests for the Harbor integration (pure logic; no Docker/network).""" + +import json + +from sia import harbor_runner + + +def test_parse_benchmark_name_version(): + assert harbor_runner.parse_benchmark("terminal-bench-sample@2.0") == { + "name": "terminal-bench-sample", + "version": "2.0", + } + + +def test_parse_benchmark_bare_name(): + assert harbor_runner.parse_benchmark("hello-world") == {"name": "hello-world"} + + +def test_parse_benchmark_local_path(tmp_path): + out = harbor_runner.parse_benchmark(str(tmp_path)) + assert out == {"path": str(tmp_path.resolve())} + + +def test_primary_reward_variants(): + assert harbor_runner._primary_reward({"reward": 1.0}) == 1.0 + assert harbor_runner._primary_reward({"a": 0.5, "b": 1.5}) == 1.0 + assert harbor_runner._primary_reward({}) is None + assert harbor_runner._primary_reward({"note": "x"}) is None + + +def _make_trial(job_dir, name, reward, with_traj=True): + trial = job_dir / name + (trial / "agent" / "sia_out").mkdir(parents=True, exist_ok=True) + result = { + "task_name": name, + "verifier_result": {"rewards": {"reward": reward}} if reward is not None else {"rewards": {}}, + "exception_info": None, + } + (trial / "result.json").write_text(json.dumps(result), encoding="utf-8") + if with_traj: + (trial / "agent" / "sia_out" / "agent_execution.json").write_text( + json.dumps({"steps": [], "usage": {}}), encoding="utf-8" + ) + + +def test_collect_results_aggregates_and_copies_trajectories(tmp_path): + job_dir = tmp_path / "sia" + job_dir.mkdir() + (job_dir / "result.json").write_text("{}", encoding="utf-8") + _make_trial(job_dir, "task_a", 1.0) + _make_trial(job_dir, "task_b", 0.0) + + gen_dir = tmp_path / "gen_1" + gen_dir.mkdir() + + results = harbor_runner._collect_results(job_dir, gen_dir, "bench@1.0") + + assert results["n_tasks"] == 2 + assert results["n_solved"] == 1 + assert results["score"] == 0.5 + assert results["benchmark"] == "bench@1.0" + + written = json.loads((gen_dir / "results.json").read_text()) + assert written["score"] == 0.5 + assert {t["task"] for t in written["per_task"]} == {"task_a", "task_b"} + + traj_files = sorted((gen_dir / "agent_execution").glob("execution_q*.json")) + assert len(traj_files) == 2 + + +def test_harbor_prompt_injections_exist_and_define_contract(): + from sia import prompts + + assert "--working_dir" in prompts.HARBOR_META_PROMPT + assert "--instruction_file" in prompts.HARBOR_META_PROMPT + assert "--log_dir" in prompts.HARBOR_META_PROMPT + assert "submission" in prompts.HARBOR_META_PROMPT.lower() + assert "HARBOR MODE" in prompts.HARBOR_FEEDBACK_PROMPT + + +def test_build_meta_prompt_default_has_no_harbor_block(): + """Default (non-harbor) path must stay byte-identical for golden snapshots.""" + from sia.prompts import build_meta_prompt + from sia.run_setup import TaskFiles + + tf = TaskFiles("desc", "ref", {}, "task") + assert "HARBOR MODE" not in build_meta_prompt(tf, "m", "/wd") + assert "HARBOR MODE" in build_meta_prompt(tf, "m", "/wd", harbor=True)