Add SWE-bench Verified adapter

.gitignore

@@ -4,3 +4,7 @@ __pycache__/
 *.pyc
 dist/
 build/
+
+# Regenerable SWE-bench artifacts (sample contains gold patches — keep out of git)
+data/swebench_verified_sample.jsonl
+predictions/swebench_predictions_template.jsonl

README.md

@@ -12,6 +12,21 @@ pip install -e .
 python scripts/run_demo.py
 ```
 
+## SWE-bench Verified Tasks
+
+CodeBench can load real-world tasks from
+[SWE-bench Verified](https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified)
+in place of the synthetic samples:
+
+```bash
+python scripts/prepare_swebench_sample.py            # sample 5 tasks
+python scripts/create_swebench_predictions_template.py
+```
+
+See [docs/swebench_experiment.md](docs/swebench_experiment.md) for the full
+workflow, including the leakage policy (gold patches are never shown to agents)
+and evaluation with the official SWE-bench harness.
+
 ## Benchmark Design
 
 Each **CodeTask** is drawn from a real or synthetic repository and includes:

docs/swebench_experiment.md

@@ -0,0 +1,103 @@
+# SWE-bench Verified Experiment
+
+CodeBench now supports real-world tasks from
+[SWE-bench Verified](https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified),
+replacing the synthetic sample tasks for benchmark runs. The adapter lives in
+`src/codebench/swebench_adapter.py`.
+
+## Leakage policy (important)
+
+The gold `patch`, `test_patch`, `FAIL_TO_PASS`, and `PASS_TO_PASS` fields must
+**never** be shown to a coding agent. The adapter enforces this:
+
+- `swebench_instance_to_codetask()` produces a fully agent-safe `CodeTask`:
+  `test_file` and `reference_solution` hold placeholder strings, never gold data.
+- The agent-facing view (`agent_prompt_payload()`) contains only the issue
+  description (`problem_statement`), repo name, base commit, difficulty, and tags.
+- Gold data is kept only in the clearly-marked `hidden_reference` block of the
+  sample file, for internal sanity checks and harness evaluation — not inference.
+
+## Workflow
+
+### 1. Prepare a sample
+
+```bash
+python scripts/prepare_swebench_sample.py                 # 5 tasks (default)
+python scripts/prepare_swebench_sample.py --limit 30      # scale up later
+python scripts/prepare_swebench_sample.py --limit 50 --difficulty easy
+```
+
+This downloads `SWE-bench/SWE-bench_Verified` from Hugging Face (requires the
+`datasets` package) and writes `data/swebench_verified_sample.jsonl`. Each line
+contains:
+
+| Key | Contents | Agent-visible? |
+|-----|----------|----------------|
+| `task` | Agent-safe `CodeTask` fields | Yes |
+| `base_commit` | Commit to check the repo out at | Yes |
+| `environment_setup_commit` | Commit for environment setup | No (infra only) |
+| `hidden_reference` | Gold patch, test patch, FAIL_TO_PASS, PASS_TO_PASS | **Never** |
+
+SWE-bench difficulty labels map to CodeBench difficulties:
+
+| SWE-bench label | CodeBench difficulty |
+|-----------------|----------------------|
+| `<15 min fix` | easy |
+| `15 min - 1 hour` | medium |
+| `1-4 hours` | hard |
+| `>4 hours` | hard |
+| missing/unknown | medium |
+
+### 2. Create a predictions template
+
+```bash
+python scripts/create_swebench_predictions_template.py --model-name anote-code
+```
+
+Writes `predictions/swebench_predictions_template.jsonl` with one stub per
+sampled task in the official SWE-bench harness format:
+
+```json
+{"instance_id": "astropy__astropy-12907", "model_name_or_path": "anote-code", "model_patch": ""}
+```
+
+### 3. Run agents to generate patches
+
+For each task, give the agent only the agent-safe fields (`task.description`,
+`task.repo`, `base_commit`, tags). The agent checks out `repo` at
+`base_commit`, works on the issue, and produces a unified diff. Fill that diff
+into the `model_patch` field of the corresponding template line. (Full agent
+automation is intentionally not implemented yet.)
+
+### 4. Evaluate with the official SWE-bench harness
+
+```bash
+pip install swebench
+python -m swebench.harness.run_evaluation \
+    --dataset_name SWE-bench/SWE-bench_Verified \
+    --predictions_path predictions/swebench_predictions_template.jsonl \
+    --max_workers 4 \
+    --run_id codebench-swebench-v0
+```
+
+The harness applies each `model_patch` in a containerized environment and runs
+the hidden FAIL_TO_PASS / PASS_TO_PASS tests.
+
+### 5. Convert results back into CodeBench metrics
+
+From the harness report, build `ExecutionResult` objects per (task, agent):
+
+- `tests_passed` / `tests_total` — from the FAIL_TO_PASS + PASS_TO_PASS outcomes
+- `regression_count` — PASS_TO_PASS tests that now fail
+- `execution_success` — the instance is marked *resolved*
+
+These feed directly into the existing `codebench.evaluate` leaderboard
+(pass@k, regression rate, cost-adjusted score) without any API changes.
+
+## Testing
+
+Adapter tests run offline with fake instances (no Hugging Face download):
+
+```bash
+python -m pytest tests/test_swebench_adapter.py -q
+```

pyproject.toml

@@ -9,6 +9,7 @@ description = "Agent code-generation benchmark harness"
 readme = "README.md"
 requires-python = ">=3.10"
 dependencies = [
+    "datasets>=2.19",
     "numpy>=1.24",
     "pandas>=2.0",
     "pydantic>=2.0",

scripts/create_swebench_predictions_template.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+"""Create an empty SWE-bench predictions JSONL template from a task sample.
+
+Reads the sampled tasks (see prepare_swebench_sample.py) and writes one
+prediction stub per task in the official SWE-bench harness format:
+instance_id, model_name_or_path, model_patch (empty by default).
+
+Usage:
+    python scripts/create_swebench_predictions_template.py
+    python scripts/create_swebench_predictions_template.py --model-name my-agent
+"""
+
+import argparse
+import os
+import sys
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "src"))
+
+from codebench.swebench_adapter import read_swebench_sample, write_predictions_jsonl
+
+DEFAULT_INPUT = os.path.join(
+    os.path.dirname(__file__), "..", "data", "swebench_verified_sample.jsonl"
+)
+DEFAULT_OUTPUT = os.path.join(
+    os.path.dirname(__file__), "..", "predictions", "swebench_predictions_template.jsonl"
+)
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--input",
+        default=DEFAULT_INPUT,
+        help="Sampled task JSONL (default: data/swebench_verified_sample.jsonl)",
+    )
+    parser.add_argument(
+        "--output",
+        default=DEFAULT_OUTPUT,
+        help="Template JSONL path (default: predictions/swebench_predictions_template.jsonl)",
+    )
+    parser.add_argument(
+        "--model-name",
+        default="anote-code",
+        help="Value for model_name_or_path in each stub (default: anote-code)",
+    )
+    args = parser.parse_args()
+
+    records = read_swebench_sample(args.input)
+    predictions = [
+        {
+            "instance_id": record["task"]["task_id"],
+            "model_name_or_path": args.model_name,
+            "model_patch": "",
+        }
+        for record in records
+    ]
+    path = write_predictions_jsonl(predictions, args.output)
+    print(f"Wrote {len(predictions)} prediction stubs to {os.path.abspath(path)}")
+
+
+if __name__ == "__main__":
+    main()

scripts/prepare_swebench_sample.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env python3
+"""Sample SWE-bench Verified into a CodeBench task file.
+
+Loads SWE-bench/SWE-bench_Verified from Hugging Face and writes a small
+JSONL sample (default 5 tasks) to data/swebench_verified_sample.jsonl.
+
+Usage:
+    python scripts/prepare_swebench_sample.py
+    python scripts/prepare_swebench_sample.py --limit 30 --difficulty easy
+    python scripts/prepare_swebench_sample.py --limit 50 --output data/my_sample.jsonl
+"""
+
+import argparse
+import os
+import sys
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "src"))
+
+from codebench.swebench_adapter import write_swebench_sample
+
+DEFAULT_OUTPUT = os.path.join(
+    os.path.dirname(__file__), "..", "data", "swebench_verified_sample.jsonl"
+)
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--limit",
+        type=int,
+        default=5,
+        help="Number of tasks to sample (default: 5; try 30 or 50 for larger runs)",
+    )
+    parser.add_argument(
+        "--difficulty",
+        default=None,
+        help="Filter by difficulty: easy/medium/hard or a raw SWE-bench label",
+    )
+    parser.add_argument(
+        "--output",
+        default=DEFAULT_OUTPUT,
+        help="Output JSONL path (default: data/swebench_verified_sample.jsonl)",
+    )
+    parser.add_argument(
+        "--split",
+        default="test",
+        help="Dataset split (default: test)",
+    )
+    args = parser.parse_args()
+
+    records = write_swebench_sample(
+        output_path=args.output,
+        limit=args.limit,
+        difficulty=args.difficulty,
+        split=args.split,
+    )
+    print(f"Wrote {len(records)} SWE-bench Verified tasks to {os.path.abspath(args.output)}")
+    for record in records:
+        task = record["task"]
+        print(f"  - {task['task_id']} ({task['difficulty']}) {task['repo']}")
+
+
+if __name__ == "__main__":
+    main()

src/codebench/__init__.py

@@ -31,6 +31,12 @@
     make_complexity_score,
     make_benchmark,
 )
+from .swebench_adapter import (
+    load_swebench_verified,
+    swebench_instance_to_codetask,
+    write_swebench_sample,
+    write_predictions_jsonl,
+)
 
 __all__ = [
     "TaskDifficulty",
@@ -58,4 +64,8 @@
     "make_test_suite",
     "make_complexity_score",
     "make_benchmark",
+    "load_swebench_verified",
+    "swebench_instance_to_codetask",
+    "write_swebench_sample",
+    "write_predictions_jsonl",
 ]