From 883d8d5933c8713e70b00d354c6c0f2dd22b7572 Mon Sep 17 00:00:00 2001 From: Bjjj834 Date: Mon, 13 Jul 2026 15:53:50 -0400 Subject: [PATCH] Add SWE-bench Verified adapter --- .gitignore | 4 + README.md | 15 ++ docs/swebench_experiment.md | 103 ++++++++ pyproject.toml | 1 + .../create_swebench_predictions_template.py | 62 +++++ scripts/prepare_swebench_sample.py | 64 +++++ src/codebench/__init__.py | 10 + src/codebench/swebench_adapter.py | 236 ++++++++++++++++++ tests/test_swebench_adapter.py | 204 +++++++++++++++ 9 files changed, 699 insertions(+) create mode 100644 docs/swebench_experiment.md create mode 100644 scripts/create_swebench_predictions_template.py create mode 100644 scripts/prepare_swebench_sample.py create mode 100644 src/codebench/swebench_adapter.py create mode 100644 tests/test_swebench_adapter.py diff --git a/.gitignore b/.gitignore index e368cbf..a7e9bea 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,7 @@ __pycache__/ *.pyc dist/ build/ + +# Regenerable SWE-bench artifacts (sample contains gold patches — keep out of git) +data/swebench_verified_sample.jsonl +predictions/swebench_predictions_template.jsonl diff --git a/README.md b/README.md index 4c54a92..6339916 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,21 @@ pip install -e . python scripts/run_demo.py ``` +## SWE-bench Verified Tasks + +CodeBench can load real-world tasks from +[SWE-bench Verified](https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified) +in place of the synthetic samples: + +```bash +python scripts/prepare_swebench_sample.py # sample 5 tasks +python scripts/create_swebench_predictions_template.py +``` + +See [docs/swebench_experiment.md](docs/swebench_experiment.md) for the full +workflow, including the leakage policy (gold patches are never shown to agents) +and evaluation with the official SWE-bench harness. + ## Benchmark Design Each **CodeTask** is drawn from a real or synthetic repository and includes: diff --git a/docs/swebench_experiment.md b/docs/swebench_experiment.md new file mode 100644 index 0000000..cce232f --- /dev/null +++ b/docs/swebench_experiment.md @@ -0,0 +1,103 @@ +# SWE-bench Verified Experiment + +CodeBench now supports real-world tasks from +[SWE-bench Verified](https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified), +replacing the synthetic sample tasks for benchmark runs. The adapter lives in +`src/codebench/swebench_adapter.py`. + +## Leakage policy (important) + +The gold `patch`, `test_patch`, `FAIL_TO_PASS`, and `PASS_TO_PASS` fields must +**never** be shown to a coding agent. The adapter enforces this: + +- `swebench_instance_to_codetask()` produces a fully agent-safe `CodeTask`: + `test_file` and `reference_solution` hold placeholder strings, never gold data. +- The agent-facing view (`agent_prompt_payload()`) contains only the issue + description (`problem_statement`), repo name, base commit, difficulty, and tags. +- Gold data is kept only in the clearly-marked `hidden_reference` block of the + sample file, for internal sanity checks and harness evaluation — not inference. + +## Workflow + +### 1. Prepare a sample + +```bash +python scripts/prepare_swebench_sample.py # 5 tasks (default) +python scripts/prepare_swebench_sample.py --limit 30 # scale up later +python scripts/prepare_swebench_sample.py --limit 50 --difficulty easy +``` + +This downloads `SWE-bench/SWE-bench_Verified` from Hugging Face (requires the +`datasets` package) and writes `data/swebench_verified_sample.jsonl`. Each line +contains: + +| Key | Contents | Agent-visible? | +|-----|----------|----------------| +| `task` | Agent-safe `CodeTask` fields | Yes | +| `base_commit` | Commit to check the repo out at | Yes | +| `environment_setup_commit` | Commit for environment setup | No (infra only) | +| `hidden_reference` | Gold patch, test patch, FAIL_TO_PASS, PASS_TO_PASS | **Never** | + +SWE-bench difficulty labels map to CodeBench difficulties: + +| SWE-bench label | CodeBench difficulty | +|-----------------|----------------------| +| `<15 min fix` | easy | +| `15 min - 1 hour` | medium | +| `1-4 hours` | hard | +| `>4 hours` | hard | +| missing/unknown | medium | + +### 2. Create a predictions template + +```bash +python scripts/create_swebench_predictions_template.py --model-name anote-code +``` + +Writes `predictions/swebench_predictions_template.jsonl` with one stub per +sampled task in the official SWE-bench harness format: + +```json +{"instance_id": "astropy__astropy-12907", "model_name_or_path": "anote-code", "model_patch": ""} +``` + +### 3. Run agents to generate patches + +For each task, give the agent only the agent-safe fields (`task.description`, +`task.repo`, `base_commit`, tags). The agent checks out `repo` at +`base_commit`, works on the issue, and produces a unified diff. Fill that diff +into the `model_patch` field of the corresponding template line. (Full agent +automation is intentionally not implemented yet.) + +### 4. Evaluate with the official SWE-bench harness + +```bash +pip install swebench +python -m swebench.harness.run_evaluation \ + --dataset_name SWE-bench/SWE-bench_Verified \ + --predictions_path predictions/swebench_predictions_template.jsonl \ + --max_workers 4 \ + --run_id codebench-swebench-v0 +``` + +The harness applies each `model_patch` in a containerized environment and runs +the hidden FAIL_TO_PASS / PASS_TO_PASS tests. + +### 5. Convert results back into CodeBench metrics + +From the harness report, build `ExecutionResult` objects per (task, agent): + +- `tests_passed` / `tests_total` — from the FAIL_TO_PASS + PASS_TO_PASS outcomes +- `regression_count` — PASS_TO_PASS tests that now fail +- `execution_success` — the instance is marked *resolved* + +These feed directly into the existing `codebench.evaluate` leaderboard +(pass@k, regression rate, cost-adjusted score) without any API changes. + +## Testing + +Adapter tests run offline with fake instances (no Hugging Face download): + +```bash +python -m pytest tests/test_swebench_adapter.py -q +``` diff --git a/pyproject.toml b/pyproject.toml index 5fe9f68..d29eb9b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,6 +9,7 @@ description = "Agent code-generation benchmark harness" readme = "README.md" requires-python = ">=3.10" dependencies = [ + "datasets>=2.19", "numpy>=1.24", "pandas>=2.0", "pydantic>=2.0", diff --git a/scripts/create_swebench_predictions_template.py b/scripts/create_swebench_predictions_template.py new file mode 100644 index 0000000..bf7ace4 --- /dev/null +++ b/scripts/create_swebench_predictions_template.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +"""Create an empty SWE-bench predictions JSONL template from a task sample. + +Reads the sampled tasks (see prepare_swebench_sample.py) and writes one +prediction stub per task in the official SWE-bench harness format: +instance_id, model_name_or_path, model_patch (empty by default). + +Usage: + python scripts/create_swebench_predictions_template.py + python scripts/create_swebench_predictions_template.py --model-name my-agent +""" + +import argparse +import os +import sys + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "src")) + +from codebench.swebench_adapter import read_swebench_sample, write_predictions_jsonl + +DEFAULT_INPUT = os.path.join( + os.path.dirname(__file__), "..", "data", "swebench_verified_sample.jsonl" +) +DEFAULT_OUTPUT = os.path.join( + os.path.dirname(__file__), "..", "predictions", "swebench_predictions_template.jsonl" +) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--input", + default=DEFAULT_INPUT, + help="Sampled task JSONL (default: data/swebench_verified_sample.jsonl)", + ) + parser.add_argument( + "--output", + default=DEFAULT_OUTPUT, + help="Template JSONL path (default: predictions/swebench_predictions_template.jsonl)", + ) + parser.add_argument( + "--model-name", + default="anote-code", + help="Value for model_name_or_path in each stub (default: anote-code)", + ) + args = parser.parse_args() + + records = read_swebench_sample(args.input) + predictions = [ + { + "instance_id": record["task"]["task_id"], + "model_name_or_path": args.model_name, + "model_patch": "", + } + for record in records + ] + path = write_predictions_jsonl(predictions, args.output) + print(f"Wrote {len(predictions)} prediction stubs to {os.path.abspath(path)}") + + +if __name__ == "__main__": + main() diff --git a/scripts/prepare_swebench_sample.py b/scripts/prepare_swebench_sample.py new file mode 100644 index 0000000..b0fd83e --- /dev/null +++ b/scripts/prepare_swebench_sample.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +"""Sample SWE-bench Verified into a CodeBench task file. + +Loads SWE-bench/SWE-bench_Verified from Hugging Face and writes a small +JSONL sample (default 5 tasks) to data/swebench_verified_sample.jsonl. + +Usage: + python scripts/prepare_swebench_sample.py + python scripts/prepare_swebench_sample.py --limit 30 --difficulty easy + python scripts/prepare_swebench_sample.py --limit 50 --output data/my_sample.jsonl +""" + +import argparse +import os +import sys + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "src")) + +from codebench.swebench_adapter import write_swebench_sample + +DEFAULT_OUTPUT = os.path.join( + os.path.dirname(__file__), "..", "data", "swebench_verified_sample.jsonl" +) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--limit", + type=int, + default=5, + help="Number of tasks to sample (default: 5; try 30 or 50 for larger runs)", + ) + parser.add_argument( + "--difficulty", + default=None, + help="Filter by difficulty: easy/medium/hard or a raw SWE-bench label", + ) + parser.add_argument( + "--output", + default=DEFAULT_OUTPUT, + help="Output JSONL path (default: data/swebench_verified_sample.jsonl)", + ) + parser.add_argument( + "--split", + default="test", + help="Dataset split (default: test)", + ) + args = parser.parse_args() + + records = write_swebench_sample( + output_path=args.output, + limit=args.limit, + difficulty=args.difficulty, + split=args.split, + ) + print(f"Wrote {len(records)} SWE-bench Verified tasks to {os.path.abspath(args.output)}") + for record in records: + task = record["task"] + print(f" - {task['task_id']} ({task['difficulty']}) {task['repo']}") + + +if __name__ == "__main__": + main() diff --git a/src/codebench/__init__.py b/src/codebench/__init__.py index 502ab10..7abb304 100644 --- a/src/codebench/__init__.py +++ b/src/codebench/__init__.py @@ -31,6 +31,12 @@ make_complexity_score, make_benchmark, ) +from .swebench_adapter import ( + load_swebench_verified, + swebench_instance_to_codetask, + write_swebench_sample, + write_predictions_jsonl, +) __all__ = [ "TaskDifficulty", @@ -58,4 +64,8 @@ "make_test_suite", "make_complexity_score", "make_benchmark", + "load_swebench_verified", + "swebench_instance_to_codetask", + "write_swebench_sample", + "write_predictions_jsonl", ] diff --git a/src/codebench/swebench_adapter.py b/src/codebench/swebench_adapter.py new file mode 100644 index 0000000..4609782 --- /dev/null +++ b/src/codebench/swebench_adapter.py @@ -0,0 +1,236 @@ +"""Adapter for SWE-bench Verified (https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified). + +Converts SWE-bench Verified instances into CodeBench :class:`CodeTask` objects +and prepares prediction files for the official SWE-bench evaluation harness. + +Leakage policy +-------------- +The gold ``patch``, ``test_patch``, ``FAIL_TO_PASS``, and ``PASS_TO_PASS`` +fields must NEVER reach a coding agent's prompt. The :class:`CodeTask` +produced here is fully agent-safe: ``test_file`` and ``reference_solution`` +hold placeholders, never gold data. Gold data is only available through +:func:`extract_hidden_reference` and is stored under the clearly-marked +``hidden_reference`` key in sample files, for sanity checks — not inference. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any, Dict, Iterable, List, Optional, Union + +from .core import CodeTask, TaskDifficulty + +SWEBENCH_DATASET_NAME = "SWE-bench/SWE-bench_Verified" + +#: Gold fields that must never appear in agent-facing output. +HIDDEN_FIELDS = ("patch", "test_patch", "FAIL_TO_PASS", "PASS_TO_PASS") + +#: Instance fields the agent is allowed to see. +AGENT_VISIBLE_FIELDS = ("instance_id", "repo", "base_commit", "problem_statement") + +#: Placeholder values — CodeTask requires these fields, but for SWE-bench the +#: real tests and gold patch live in the hidden reference, not on the task. +TEST_FILE_PLACEHOLDER = "swebench://hidden-tests (evaluated by the SWE-bench harness)" +REFERENCE_SOLUTION_PLACEHOLDER = ( + "swebench://hidden-gold-patch (reference-only; stored separately, never for inference)" +) + +#: SWE-bench Verified annotates difficulty as estimated fix time. +SWEBENCH_DIFFICULTY_MAP: Dict[str, TaskDifficulty] = { + "<15 min fix": TaskDifficulty.EASY, + "15 min - 1 hour": TaskDifficulty.MEDIUM, + "1-4 hours": TaskDifficulty.HARD, + ">4 hours": TaskDifficulty.HARD, +} + +#: Required keys for the official SWE-bench prediction format. +PREDICTION_KEYS = ("instance_id", "model_name_or_path", "model_patch") + + +def map_swebench_difficulty(raw: Optional[str]) -> TaskDifficulty: + """Map a SWE-bench Verified difficulty label to :class:`TaskDifficulty`. + + Unknown or missing labels default to MEDIUM. + """ + if raw is None: + return TaskDifficulty.MEDIUM + label = str(raw).strip() + if label in SWEBENCH_DIFFICULTY_MAP: + return SWEBENCH_DIFFICULTY_MAP[label] + try: + return TaskDifficulty(label.lower()) + except ValueError: + return TaskDifficulty.MEDIUM + + +def _load_raw_dataset(split: str) -> Iterable[Dict[str, Any]]: + """Load the raw SWE-bench Verified split from Hugging Face.""" + try: + from datasets import load_dataset + except ImportError as exc: # pragma: no cover - exercised only without extras + raise ImportError( + "The 'datasets' package is required to load SWE-bench Verified. " + "Install it with: pip install datasets" + ) from exc + return load_dataset(SWEBENCH_DATASET_NAME, split=split) + + +def load_swebench_verified( + split: str = "test", + limit: Optional[int] = None, + difficulty: Optional[Union[str, TaskDifficulty]] = None, +) -> List[Dict[str, Any]]: + """Load SWE-bench Verified instances as plain dicts. + + Args: + split: Dataset split (SWE-bench Verified only publishes "test"). + limit: Keep at most this many instances (applied after filtering). + difficulty: Filter by difficulty — accepts a CodeBench level + ("easy"/"medium"/"hard") or a raw SWE-bench label ("<15 min fix", ...). + + Returns: + Raw instance dicts, including gold fields. Callers preparing + agent-facing data must go through :func:`swebench_instance_to_codetask` + or :func:`agent_prompt_payload`, never hand these dicts to an agent. + """ + if limit is not None and limit < 1: + raise ValueError(f"limit must be >= 1, got {limit}") + + wanted: Optional[TaskDifficulty] = None + if difficulty is not None: + wanted = ( + difficulty + if isinstance(difficulty, TaskDifficulty) + else map_swebench_difficulty(str(difficulty)) + ) + + instances: List[Dict[str, Any]] = [] + for row in _load_raw_dataset(split): + instance = dict(row) + if wanted is not None and map_swebench_difficulty(instance.get("difficulty")) != wanted: + continue + instances.append(instance) + if limit is not None and len(instances) >= limit: + break + return instances + + +def swebench_instance_to_codetask(instance: Dict[str, Any]) -> CodeTask: + """Convert one SWE-bench Verified instance into an agent-safe CodeTask. + + The returned task never contains gold data: ``test_file`` and + ``reference_solution`` are placeholders, so the task can be serialized + into prompts without leaking the solution. + """ + difficulty = map_swebench_difficulty(instance.get("difficulty")) + tags = ["swebench", instance["repo"], difficulty.value] + if instance.get("base_commit"): + tags.append(f"base_commit:{instance['base_commit']}") + return CodeTask( + task_id=instance["instance_id"], + repo=instance["repo"], + description=instance["problem_statement"], + difficulty=difficulty, + test_file=TEST_FILE_PLACEHOLDER, + reference_solution=REFERENCE_SOLUTION_PLACEHOLDER, + tags=tags, + ) + + +def extract_hidden_reference(instance: Dict[str, Any]) -> Dict[str, Any]: + """Extract gold fields for internal sanity checks. NEVER show to agents.""" + hidden = {field: instance.get(field) for field in HIDDEN_FIELDS} + hidden["_note"] = ( + "Reference-only gold data for sanity checks and harness evaluation. " + "Must never be included in agent prompts." + ) + return hidden + + +def agent_prompt_payload(instance: Dict[str, Any]) -> Dict[str, Any]: + """Build the agent-facing view of an instance: only allowed fields. + + Includes the issue description, repo name, base commit, and safe metadata. + """ + task = swebench_instance_to_codetask(instance) + return { + "task_id": task.task_id, + "repo": task.repo, + "base_commit": instance.get("base_commit"), + "description": task.description, + "difficulty": task.difficulty.value, + "tags": task.tags, + } + + +def write_swebench_sample( + output_path: Union[str, Path], + limit: int = 5, + difficulty: Optional[Union[str, TaskDifficulty]] = None, + split: str = "test", +) -> List[Dict[str, Any]]: + """Sample SWE-bench Verified and write a JSONL task file. + + Each line holds an agent-safe ``task`` (CodeTask fields), the + ``base_commit`` / ``environment_setup_commit`` needed to check out the + repo, and a clearly-marked ``hidden_reference`` block with gold data for + internal sanity checks only. + + Returns the written records. + """ + instances = load_swebench_verified(split=split, limit=limit, difficulty=difficulty) + records: List[Dict[str, Any]] = [] + for instance in instances: + task = swebench_instance_to_codetask(instance) + records.append( + { + "task": task.model_dump(mode="json"), + "base_commit": instance.get("base_commit"), + "environment_setup_commit": instance.get("environment_setup_commit"), + "hidden_reference": extract_hidden_reference(instance), + } + ) + + path = Path(output_path) + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8") as fh: + for record in records: + fh.write(json.dumps(record, ensure_ascii=False) + "\n") + return records + + +def read_swebench_sample(sample_path: Union[str, Path]) -> List[Dict[str, Any]]: + """Read a JSONL sample file written by :func:`write_swebench_sample`.""" + records: List[Dict[str, Any]] = [] + with Path(sample_path).open("r", encoding="utf-8") as fh: + for line in fh: + line = line.strip() + if line: + records.append(json.loads(line)) + return records + + +def write_predictions_jsonl( + predictions: Iterable[Dict[str, Any]], + output_path: Union[str, Path], +) -> Path: + """Write predictions in the official SWE-bench harness JSONL format. + + Each prediction must provide ``instance_id``, ``model_name_or_path``, + and ``model_patch``. Extra keys are dropped so the output stays + harness-compatible. + """ + path = Path(output_path) + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8") as fh: + for prediction in predictions: + missing = [key for key in PREDICTION_KEYS if key not in prediction] + if missing: + raise ValueError( + f"Prediction for {prediction.get('instance_id', '')!r} " + f"is missing required keys: {missing}" + ) + row = {key: prediction[key] for key in PREDICTION_KEYS} + fh.write(json.dumps(row, ensure_ascii=False) + "\n") + return path diff --git a/tests/test_swebench_adapter.py b/tests/test_swebench_adapter.py new file mode 100644 index 0000000..fe1c0f2 --- /dev/null +++ b/tests/test_swebench_adapter.py @@ -0,0 +1,204 @@ +"""Tests for codebench.swebench_adapter (no network access required).""" + +import json + +import pytest + +from codebench.core import CodeTask, TaskDifficulty +from codebench import swebench_adapter +from codebench.swebench_adapter import ( + HIDDEN_FIELDS, + PREDICTION_KEYS, + REFERENCE_SOLUTION_PLACEHOLDER, + TEST_FILE_PLACEHOLDER, + agent_prompt_payload, + extract_hidden_reference, + load_swebench_verified, + map_swebench_difficulty, + read_swebench_sample, + swebench_instance_to_codetask, + write_predictions_jsonl, + write_swebench_sample, +) + +GOLD_PATCH = "diff --git a/astropy/io.py b/astropy/io.py\n--- SECRET GOLD PATCH ---\n" +GOLD_TEST_PATCH = "diff --git a/tests/test_io.py b/tests/test_io.py\n--- SECRET TEST PATCH ---\n" + + +def make_instance(i=0, difficulty="15 min - 1 hour"): + """A fake SWE-bench Verified instance with all relevant fields.""" + return { + "instance_id": f"astropy__astropy-{1000 + i}", + "repo": "astropy/astropy", + "base_commit": f"abc{i:03d}def", + "environment_setup_commit": f"env{i:03d}", + "problem_statement": f"Issue {i}: reading FITS files crashes with a TypeError.", + "hints_text": "", + "difficulty": difficulty, + "patch": GOLD_PATCH, + "test_patch": GOLD_TEST_PATCH, + "FAIL_TO_PASS": '["tests/test_io.py::test_fits_read"]', + "PASS_TO_PASS": '["tests/test_io.py::test_fits_write"]', + } + + +@pytest.fixture +def fake_dataset(monkeypatch): + """Route the adapter's HF loader to 12 fake instances of mixed difficulty.""" + labels = ["<15 min fix", "15 min - 1 hour", "1-4 hours", ">4 hours"] + instances = [make_instance(i, difficulty=labels[i % len(labels)]) for i in range(12)] + monkeypatch.setattr(swebench_adapter, "_load_raw_dataset", lambda split: instances) + return instances + + +# ---------------------------------------------------------------- mapping + +def test_instance_maps_to_codetask(): + instance = make_instance(difficulty="15 min - 1 hour") + task = swebench_instance_to_codetask(instance) + assert isinstance(task, CodeTask) + assert task.task_id == instance["instance_id"] + assert task.repo == "astropy/astropy" + assert task.description == instance["problem_statement"] + assert task.difficulty == TaskDifficulty.MEDIUM + assert task.test_file == TEST_FILE_PLACEHOLDER + assert task.reference_solution == REFERENCE_SOLUTION_PLACEHOLDER + assert "swebench" in task.tags + assert "astropy/astropy" in task.tags + assert "medium" in task.tags + assert f"base_commit:{instance['base_commit']}" in task.tags + + +@pytest.mark.parametrize( + "raw,expected", + [ + ("<15 min fix", TaskDifficulty.EASY), + ("15 min - 1 hour", TaskDifficulty.MEDIUM), + ("1-4 hours", TaskDifficulty.HARD), + (">4 hours", TaskDifficulty.HARD), + ("easy", TaskDifficulty.EASY), + ("HARD", TaskDifficulty.HARD), + (None, TaskDifficulty.MEDIUM), + ("something new", TaskDifficulty.MEDIUM), + ], +) +def test_difficulty_mapping(raw, expected): + assert map_swebench_difficulty(raw) == expected + + +# ------------------------------------------------------------ no leakage + +def test_codetask_contains_no_gold_data(): + task = swebench_instance_to_codetask(make_instance()) + dumped = json.dumps(task.model_dump(mode="json")) + assert "SECRET GOLD PATCH" not in dumped + assert "SECRET TEST PATCH" not in dumped + assert "FAIL_TO_PASS" not in dumped + assert "PASS_TO_PASS" not in dumped + assert "test_fits_read" not in dumped + + +def test_agent_prompt_payload_excludes_hidden_fields(): + instance = make_instance() + payload = agent_prompt_payload(instance) + for field in HIDDEN_FIELDS: + assert field not in payload + dumped = json.dumps(payload) + assert "SECRET GOLD PATCH" not in dumped + assert "SECRET TEST PATCH" not in dumped + # allowed fields are present + assert payload["task_id"] == instance["instance_id"] + assert payload["repo"] == instance["repo"] + assert payload["base_commit"] == instance["base_commit"] + assert payload["description"] == instance["problem_statement"] + + +def test_extract_hidden_reference_keeps_gold_for_sanity_checks(): + hidden = extract_hidden_reference(make_instance()) + assert hidden["patch"] == GOLD_PATCH + assert hidden["test_patch"] == GOLD_TEST_PATCH + assert "never" in hidden["_note"].lower() + + +# --------------------------------------------------------------- loading + +def test_load_limit(fake_dataset): + assert len(load_swebench_verified(limit=5)) == 5 + assert len(load_swebench_verified(limit=30)) == len(fake_dataset) # capped by data + assert len(load_swebench_verified()) == len(fake_dataset) + + +def test_load_invalid_limit(fake_dataset): + with pytest.raises(ValueError): + load_swebench_verified(limit=0) + + +def test_load_difficulty_filter(fake_dataset): + easy = load_swebench_verified(difficulty="easy") + assert easy and all(i["difficulty"] == "<15 min fix" for i in easy) + hard = load_swebench_verified(difficulty="1-4 hours") # raw label accepted too + assert hard and all( + map_swebench_difficulty(i["difficulty"]) == TaskDifficulty.HARD for i in hard + ) + + +# ---------------------------------------------------------- sample files + +def test_write_swebench_sample_default_limit(fake_dataset, tmp_path): + out = tmp_path / "data" / "sample.jsonl" + records = write_swebench_sample(out) + assert len(records) == 5 # default limit + lines = out.read_text(encoding="utf-8").strip().splitlines() + assert len(lines) == 5 + + for line in lines: + record = json.loads(line) + task = record["task"] + # agent-safe task: placeholders only, no gold text anywhere in it + assert task["test_file"] == TEST_FILE_PLACEHOLDER + assert task["reference_solution"] == REFERENCE_SOLUTION_PLACEHOLDER + assert "SECRET GOLD PATCH" not in json.dumps(task) + # gold data lives only in the clearly-marked hidden block + assert record["hidden_reference"]["patch"] == GOLD_PATCH + assert record["base_commit"] + + +def test_write_and_read_sample_roundtrip(fake_dataset, tmp_path): + out = tmp_path / "sample.jsonl" + written = write_swebench_sample(out, limit=3) + assert read_swebench_sample(out) == written + + +# ------------------------------------------------------------ predictions + +def test_predictions_jsonl_official_format(tmp_path): + out = tmp_path / "predictions" / "preds.jsonl" + predictions = [ + { + "instance_id": "astropy__astropy-1000", + "model_name_or_path": "anote-code", + "model_patch": "", + "extra_key": "should be dropped", + }, + { + "instance_id": "astropy__astropy-1001", + "model_name_or_path": "anote-code", + "model_patch": "diff --git a/x b/x\n", + }, + ] + write_predictions_jsonl(predictions, out) + + lines = out.read_text(encoding="utf-8").strip().splitlines() + assert len(lines) == 2 + for line in lines: + row = json.loads(line) + assert set(row.keys()) == set(PREDICTION_KEYS) + assert json.loads(lines[0])["model_patch"] == "" + + +def test_predictions_jsonl_rejects_missing_keys(tmp_path): + with pytest.raises(ValueError, match="model_patch"): + write_predictions_jsonl( + [{"instance_id": "x", "model_name_or_path": "y"}], + tmp_path / "preds.jsonl", + )