Add SWE-bench agent runner

.gitignore

@@ -8,3 +8,9 @@ build/
 # Regenerable SWE-bench artifacts (sample contains gold patches — keep out of git)
 data/swebench_verified_sample.jsonl
 predictions/swebench_predictions_template.jsonl
+
+# SWE-bench runner artifacts (repo clones, per-attempt logs, raw prediction runs)
+workspaces/
+attempts/
+predictions/*.jsonl
+!predictions/*_v*.jsonl

docs/swebench_experiment.md

@@ -63,41 +63,78 @@ sampled task in the official SWE-bench harness format:
 
 ### 3. Run agents to generate patches
 
-For each task, give the agent only the agent-safe fields (`task.description`,
-`task.repo`, `base_commit`, tags). The agent checks out `repo` at
-`base_commit`, works on the issue, and produces a unified diff. Fill that diff
-into the `model_patch` field of the corresponding template line. (Full agent
-automation is intentionally not implemented yet.)
+`scripts/run_swebench_agent.py` automates real attempts. Per task × attempt it
+checks out the target repo at `base_commit` in an isolated git worktree
+(clones are cached under `workspaces/repos/`), runs your agent command there,
+captures `git diff` against `base_commit` as `model_patch`, and writes one
+official predictions JSONL per attempt index.
+
+```bash
+# dry-run the whole pipeline first (no real agent, empty patches)
+python scripts/run_swebench_agent.py --agent-cmd noop --run-name dryrun
+
+# 3 independent attempts per task with Claude Code
+python scripts/run_swebench_agent.py \
+    --model-name anote-code --attempts 3 --timeout 1800 --run-name v1 \
+    --agent-cmd 'claude -p "$(cat {prompt_file})" --permission-mode acceptEdits'
+# → predictions/v1_attempt1.jsonl, v1_attempt2.jsonl, v1_attempt3.jsonl
+```
+
+`--agent-cmd` is a shell template with `{prompt_file}`, `{workdir}`,
+`{instance_id}`, and `{repo}` placeholders, so any agent CLI plugs in. The
+agent's prompt and working directory contain only agent-safe data — the
+runner strips `hidden_reference` at load time (`load_agent_safe_tasks`) and
+never writes gold fields into prompts, logs, or attempt records
+(`attempts/{run}/{instance}/attempt-{k}/`). Failed or timed-out attempts are
+recorded with an empty patch so they still count as rollouts. Use `--resume`
+to continue an interrupted run and `--attempts N` for multiple independent
+rollouts per task (the harness allows one prediction per instance per file,
+hence one predictions file per attempt index).
 
 ### 4. Evaluate with the official SWE-bench harness
 
 ```bash
-pip install swebench
-python -m swebench.harness.run_evaluation \
-    --dataset_name SWE-bench/SWE-bench_Verified \
-    --predictions_path predictions/swebench_predictions_template.jsonl \
-    --max_workers 4 \
-    --run_id codebench-swebench-v0
+pip install swebench   # or: pip install -e ".[eval]"
+for k in 1 2 3; do
+  python -m swebench.harness.run_evaluation \
+      --dataset_name SWE-bench/SWE-bench_Verified \
+      --predictions_path predictions/v1_attempt$k.jsonl \
+      --max_workers 2 \
+      --run_id v1-attempt$k
+done
 ```
 
-The harness applies each `model_patch` in a containerized environment and runs
-the hidden FAIL_TO_PASS / PASS_TO_PASS tests.
+The harness applies each `model_patch` in a containerized environment
+(Docker must be running) and runs the hidden FAIL_TO_PASS / PASS_TO_PASS
+tests. Each run writes a summary `{model}.{run_id}.json` plus per-instance
+reports under `logs/run_evaluation/{run_id}/{model}/`.
 
 ### 5. Convert results back into CodeBench metrics
 
-From the harness report, build `ExecutionResult` objects per (task, agent):
+```bash
+python scripts/convert_swebench_report.py \
+    --run-ids v1-attempt1 v1-attempt2 v1-attempt3 \
+    --model-name anote-code -k 3 \
+    --output data/swebench_results_anote-code_v1.json
+```
+
+Each evaluated (instance, attempt) becomes one `ExecutionResult` rollout:
 
 - `tests_passed` / `tests_total` — from the FAIL_TO_PASS + PASS_TO_PASS outcomes
 - `regression_count` — PASS_TO_PASS tests that now fail
 - `execution_success` — the instance is marked *resolved*
 
-These feed directly into the existing `codebench.evaluate` leaderboard
-(pass@k, regression rate, cost-adjusted score) without any API changes.
+Pooling one harness run per attempt gives exactly the rollout shape
+`codebench.evaluate.reliability_at_k` consumes (n = attempts, c = resolved
+attempts), replacing synthetic rollouts with real ones in the H-experiment
+pipeline — no API changes.
 
 ## Testing
 
-Adapter tests run offline with fake instances (no Hugging Face download):
+Adapter, runner, and results tests all run offline (fake instances, local
+git fixture repos, fixture harness reports — no Hugging Face download, no
+Docker):
 
 ```bash
-python -m pytest tests/test_swebench_adapter.py -q
+python -m pytest tests/test_swebench_adapter.py tests/test_swebench_runner.py tests/test_swebench_results.py -q
 ```

pyproject.toml

@@ -17,6 +17,9 @@ dependencies = [
     "tqdm>=4.65",
 ]
 
+[project.optional-dependencies]
+eval = ["swebench>=2.1"]
+
 [tool.setuptools.packages.find]
 where = ["src"]
 

scripts/convert_swebench_report.py

@@ -0,0 +1,93 @@
+#!/usr/bin/env python3
+"""Convert official SWE-bench harness reports into CodeBench metrics.
+
+Pools one or more harness runs (one per attempt index) into ExecutionResult
+rollouts, computes reliability@k / pass-rate / regression aggregates, and
+writes a versioned results JSON.
+
+Usage:
+    python scripts/convert_swebench_report.py \\
+        --run-ids v1-attempt1 v1-attempt2 v1-attempt3 \\
+        --model-name anote-code -k 3 \\
+        --output data/swebench_results_anote-code_v1.json
+"""
+
+import argparse
+import json
+import os
+import sys
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "src"))
+
+from codebench.swebench_results import (
+    DEFAULT_LOGS_DIR,
+    aggregate_swebench_results,
+    harness_report_to_execution_results,
+)
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--run-ids", nargs="+", required=True,
+        help="Harness run ids to pool (one per attempt index)",
+    )
+    parser.add_argument("--model-name", required=True, help="model_name_or_path used in the runs")
+    parser.add_argument("-k", type=int, default=5, help="k for reliability@k (default: 5)")
+    parser.add_argument(
+        "--summary-dir", default=".",
+        help="Directory holding {model}.{run_id}.json summaries (default: cwd)",
+    )
+    parser.add_argument(
+        "--logs-dir", default=DEFAULT_LOGS_DIR,
+        help=f"Harness evaluation logs dir (default: {DEFAULT_LOGS_DIR})",
+    )
+    parser.add_argument("--output", default=None, help="Results JSON output path")
+    args = parser.parse_args()
+
+    safe_model = args.model_name.replace("/", "__")
+    all_results = []
+    per_run = {}
+    for run_id in args.run_ids:
+        summary_path = os.path.join(args.summary_dir, f"{safe_model}.{run_id}.json")
+        if not os.path.exists(summary_path):
+            sys.exit(f"error: harness summary not found: {summary_path}")
+        results = harness_report_to_execution_results(
+            summary_path=summary_path,
+            run_id=run_id,
+            model_name=args.model_name,
+            logs_dir=args.logs_dir,
+        )
+        per_run[run_id] = {
+            "n_instances": len(results),
+            "resolved": sum(1 for r in results if r.execution_success),
+        }
+        all_results.extend(results)
+
+    aggregates = aggregate_swebench_results(all_results, k=args.k)
+
+    print(f"Model: {args.model_name}   runs: {', '.join(args.run_ids)}")
+    for run_id, stats in per_run.items():
+        print(f"  {run_id}: {stats['resolved']}/{stats['n_instances']} resolved")
+    print(f"reliability@{args.k}:     {aggregates[f'reliability@{args.k}']:.4f}")
+    print(f"resolve rate:         {aggregates['resolve_rate']:.4f}")
+    print(f"mean pass rate:       {aggregates['mean_pass_rate']:.4f}")
+    print(f"mean regression rate: {aggregates['mean_regression_rate']:.4f}")
+
+    if args.output:
+        payload = {
+            "model": args.model_name,
+            "run_ids": args.run_ids,
+            "k": args.k,
+            "per_run": per_run,
+            "aggregates": aggregates,
+            "rollouts": [r.model_dump(mode="json") for r in all_results],
+        }
+        os.makedirs(os.path.dirname(os.path.abspath(args.output)), exist_ok=True)
+        with open(args.output, "w", encoding="utf-8") as fh:
+            json.dump(payload, fh, indent=2)
+        print(f"\nWrote results to {os.path.abspath(args.output)}")
+
+
+if __name__ == "__main__":
+    main()

scripts/run_swebench_agent.py

@@ -0,0 +1,112 @@
+#!/usr/bin/env python3
+"""Run real agent attempts against a SWE-bench Verified sample.
+
+For each task x attempt: checks out the target repo at base_commit in an
+isolated git worktree, runs the agent command there, captures git diff as
+model_patch, and writes one official predictions JSONL per attempt index.
+
+The agent only ever sees the issue description, repo name, base commit,
+difficulty, and tags — never the gold patch or hidden tests.
+
+Usage:
+    # dry-run the pipeline (no real agent, empty patches)
+    python scripts/run_swebench_agent.py --agent-cmd noop --run-name dryrun
+
+    # 3 attempts per task with Claude Code
+    python scripts/run_swebench_agent.py \\
+        --model-name anote-code --attempts 3 --run-name v1 \\
+        --agent-cmd 'claude -p "$(cat {prompt_file})" --permission-mode acceptEdits'
+"""
+
+import argparse
+import os
+import sys
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "src"))
+
+from codebench.swebench_runner import (
+    DEFAULT_GIT_BASE_URL,
+    DEFAULT_TIMEOUT_S,
+    run_swebench_attempts,
+)
+
+ROOT = os.path.join(os.path.dirname(__file__), "..")
+DEFAULT_SAMPLE = os.path.join(ROOT, "data", "swebench_verified_sample.jsonl")
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--sample",
+        default=DEFAULT_SAMPLE,
+        help="Task sample JSONL from prepare_swebench_sample.py",
+    )
+    parser.add_argument(
+        "--agent-cmd",
+        default="noop",
+        help=(
+            "Shell command template with {prompt_file}/{workdir}/{instance_id}/{repo} "
+            "placeholders, or 'noop' for a dry run (default: noop)"
+        ),
+    )
+    parser.add_argument("--model-name", default="anote-code", help="model_name_or_path value")
+    parser.add_argument("--attempts", type=int, default=1, help="Attempts per task (default: 1)")
+    parser.add_argument("--run-name", default="run", help="Name for this run's artifacts")
+    parser.add_argument(
+        "--timeout", type=int, default=DEFAULT_TIMEOUT_S,
+        help=f"Per-attempt agent timeout in seconds (default: {DEFAULT_TIMEOUT_S})",
+    )
+    parser.add_argument(
+        "--workspaces-dir", default=os.path.join(ROOT, "workspaces"),
+        help="Where repo clones and worktrees live",
+    )
+    parser.add_argument(
+        "--attempts-dir", default=os.path.join(ROOT, "attempts"),
+        help="Where per-attempt records (prompt/log/patch/meta) are written",
+    )
+    parser.add_argument(
+        "--predictions-dir", default=os.path.join(ROOT, "predictions"),
+        help="Where prediction JSONL files are written",
+    )
+    parser.add_argument(
+        "--git-base-url", default=DEFAULT_GIT_BASE_URL,
+        help="Base URL for cloning task repos (default: https://github.com)",
+    )
+    parser.add_argument(
+        "--resume", action="store_true",
+        help="Skip (task, attempt) pairs that already have a recorded meta.json",
+    )
+    parser.add_argument(
+        "--keep-workspaces", action="store_true",
+        help="Keep per-attempt worktrees for debugging",
+    )
+    args = parser.parse_args()
+
+    prediction_files = run_swebench_attempts(
+        sample_path=args.sample,
+        agent_cmd=args.agent_cmd,
+        model_name=args.model_name,
+        attempts=args.attempts,
+        run_name=args.run_name,
+        workspaces_dir=args.workspaces_dir,
+        attempts_dir=args.attempts_dir,
+        predictions_dir=args.predictions_dir,
+        timeout_s=args.timeout,
+        resume=args.resume,
+        keep_workspaces=args.keep_workspaces,
+        git_base_url=args.git_base_url,
+    )
+    print(f"Run '{args.run_name}' complete: {len(prediction_files)} prediction file(s)")
+    for attempt, path in sorted(prediction_files.items()):
+        print(f"  attempt {attempt}: {os.path.abspath(path)}")
+    print(
+        "\nNext: evaluate each file with the official harness, e.g.\n"
+        "  python -m swebench.harness.run_evaluation "
+        "--dataset_name SWE-bench/SWE-bench_Verified \\\n"
+        f"      --predictions_path {os.path.abspath(prediction_files[1])} \\\n"
+        f"      --max_workers 2 --run_id {args.run_name}-attempt1"
+    )
+
+
+if __name__ == "__main__":
+    main()

src/codebench/__init__.py

@@ -37,6 +37,8 @@
     write_swebench_sample,
     write_predictions_jsonl,
 )
+from .swebench_runner import run_swebench_attempts
+from .swebench_results import harness_report_to_execution_results
 
 __all__ = [
     "TaskDifficulty",
@@ -68,4 +70,6 @@
     "swebench_instance_to_codetask",
     "write_swebench_sample",
     "write_predictions_jsonl",
+    "run_swebench_attempts",
+    "harness_report_to_execution_results",
 ]