diff --git a/scripts/build-terminal-bench-4-images.py b/scripts/build-terminal-bench-4-images.py new file mode 100644 index 0000000..1484902 --- /dev/null +++ b/scripts/build-terminal-bench-4-images.py @@ -0,0 +1,148 @@ +"""Build Terminal-Bench 4.0 task images into Modal's image store. + +Usage: + python3 scripts/build-terminal-bench-4-images.py --tasks-dir /path/to/terminal-bench/tasks \ + [--task NAME ...] [--out src/benchmarks/terminal-bench-4/image-ids.json] [--dry-run] + +Requires MODAL_TOKEN_ID and MODAL_TOKEN_SECRET (or a Modal CLI profile). For each +runnable (non docker-compose) task this builds environment/Dockerfile and +tests/Dockerfile with their directories as build context, then records the +resulting Modal image IDs keyed by task id. The output JSON is what the harness +reads at run time, so re-run this whenever TERMINAL_BENCH_4_SOURCE_COMMIT changes. + +MODAL_IMAGE_BUILDER_VERSION defaults to 2025.06. Older builder versions pip-install +Modal's client dependencies into the image, which would alter task environments. +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import subprocess +import sys +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +os.environ.setdefault("MODAL_IMAGE_BUILDER_VERSION", "2025.06") + +import modal # noqa: E402 + +APP_NAME = "terminal-bench-4-images" +COMPOSE_FILE = "environment/docker-compose.yaml" +DEFAULT_OUT = Path("src/benchmarks/terminal-bench-4/image-ids.json") +SOURCE_RE = re.compile(r'TERMINAL_BENCH_4_SOURCE_COMMIT\s*=\s*"([0-9a-f]{40})"') + + +def pinned_commit() -> str: + text = Path("src/benchmarks/terminal-bench-4/tasks-source.ts").read_text() + match = SOURCE_RE.search(text) + if match is None: + sys.exit("could not read TERMINAL_BENCH_4_SOURCE_COMMIT from tasks-source.ts") + return match.group(1) + + +def checkout_commit(tasks_dir: Path) -> str: + out = subprocess.run( + ["git", "-C", str(tasks_dir), "rev-parse", "HEAD"], + check=True, + capture_output=True, + text=True, + ) + return out.stdout.strip() + + +def runnable_tasks(tasks_dir: Path) -> list[str]: + return sorted( + p.name + for p in tasks_dir.iterdir() + if p.is_dir() + and not p.name.startswith(".") + and (p / "task.toml").is_file() + and not (p / COMPOSE_FILE).is_file() + ) + + +def load_existing(out: Path, commit: str) -> dict[str, dict[str, str]]: + if not out.is_file(): + return {} + data = json.loads(out.read_text()) + if data.get("sourceCommit") != commit: + return {} + return dict(data.get("images", {})) + + +def write_out(out: Path, commit: str, images: dict[str, dict[str, str]]) -> None: + payload = { + "sourceCommit": commit, + "images": {k: images[k] for k in sorted(images)}, + } + out.write_text(json.dumps(payload, indent=2) + "\n") + + +def build_task(app: modal.App, tasks_dir: Path, task: str) -> dict[str, str]: + task_dir = tasks_dir / task + agent = modal.Image.from_dockerfile( + task_dir / "environment" / "Dockerfile", + context_dir=task_dir / "environment", + ) + verifier = modal.Image.from_dockerfile( + task_dir / "tests" / "Dockerfile", + context_dir=task_dir / "tests", + ) + agent = agent.build(app) + verifier = verifier.build(app) + return {"agent": agent.object_id, "verifier": verifier.object_id} + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--tasks-dir", type=Path, required=True) + parser.add_argument("--task", action="append", default=[]) + parser.add_argument("--out", type=Path, default=DEFAULT_OUT) + parser.add_argument("--concurrency", type=int, default=4) + parser.add_argument("--dry-run", action="store_true") + args = parser.parse_args() + + tasks_dir: Path = args.tasks_dir.resolve() + commit = pinned_commit() + actual = checkout_commit(tasks_dir.parent) + if actual != commit: + sys.exit(f"tasks dir is at {actual}, expected pinned {commit}") + + runnable = runnable_tasks(tasks_dir) + selected = args.task or runnable + unknown = sorted(set(selected) - set(runnable)) + if unknown: + sys.exit(f"unknown or docker-compose tasks: {', '.join(unknown)}") + + if args.dry_run: + for task in selected: + print(task) + return 0 + + images = load_existing(args.out, commit) + failures: dict[str, str] = {} + app = modal.App.lookup(APP_NAME, create_if_missing=True) + + def run(task: str) -> None: + try: + images[task] = build_task(app, tasks_dir, task) + print(f"built {task}: {images[task]}", flush=True) + except Exception as error: # noqa: BLE001 + failures[task] = str(error) + print(f"FAILED {task}: {error}", file=sys.stderr, flush=True) + + with modal.enable_output(), ThreadPoolExecutor(max_workers=args.concurrency) as pool: + list(pool.map(run, selected)) + + write_out(args.out, commit, images) + if failures: + print(f"{len(failures)} task(s) failed: {', '.join(sorted(failures))}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/benchmarks/agent-cli/harness.ts b/src/benchmarks/agent-cli/harness.ts index 173450b..7e5490f 100644 --- a/src/benchmarks/agent-cli/harness.ts +++ b/src/benchmarks/agent-cli/harness.ts @@ -89,6 +89,18 @@ export interface OriHarnessDef { readonly parseRun: (stdout: string) => OriAgentRun; } +export function installSystemPackagesStep(packages: readonly string[]): string { + const list = packages.join(" "); + return [ + "RUN if command -v apt-get >/dev/null", + `then apt-get update && apt-get install -y --no-install-recommends ${list}`, + `elif command -v dnf >/dev/null; then dnf install -y --setopt=install_weak_deps=False --allowerasing ${list}`, + `elif command -v apk >/dev/null; then apk add --no-cache bash ${list}`, + 'else echo "no supported package manager (apt-get, dnf, apk)" >&2 && exit 1', + "fi", + ].join("; "); +} + function buildImageSteps(opts: { agentPackage: string; binaryName: string; @@ -100,7 +112,7 @@ function buildImageSteps(opts: { `npm install -g ${JSON.stringify(opts.agentPackage)}`; const nvmScript = "/tmp/nvm-install.sh"; return [ - "RUN apt-get update && apt-get install -y --no-install-recommends curl ca-certificates git", + installSystemPackagesStep(["curl", "ca-certificates", "git"]), "ENV NVM_DIR=/root/.nvm", `RUN ${verifiedDownload(NVM_INSTALL_URL, NVM_INSTALL_SHA256, nvmScript)} && bash ${nvmScript} && rm -f ${nvmScript}`, `RUN . /root/.nvm/nvm.sh && nvm install ${NODE_VERSION} && ${installCommand} && ln -sf $(which ${opts.binaryName}) /usr/local/bin/${opts.binaryName} && ln -sf $(which node) /usr/local/bin/node && ln -sf $(which npm) /usr/local/bin/npm`, @@ -117,7 +129,7 @@ function buildAgentImageSteps(opts: { if (opts.agentPackage === opts.defaultPackage) { const archivePath = "/tmp/agent-runtime.tar.zst"; return [ - "RUN apt-get update && apt-get install -y --no-install-recommends curl ca-certificates git zstd", + installSystemPackagesStep(["curl", "ca-certificates", "git", "zstd"]), `RUN ${verifiedDownload(DEFAULT_AGENT_RUNTIME_URL, DEFAULT_AGENT_RUNTIME_SHA256, archivePath)} && zstd -dc ${archivePath} | tar -x -C / && rm -f ${archivePath}`, 'ENV PATH="/root/.local/bin:$PATH"', "RUN ln -sf /opt/agent-runtime/app/node_modules/.bin/claude /usr/local/bin/claude && ln -sf /opt/agent-runtime/app/node_modules/.bin/pi /usr/local/bin/pi && ln -sf /opt/agent-runtime/app/node_modules/.bin/prime-agent /usr/local/bin/prime-agent && ln -sf /opt/agent-runtime/node/bin/node /usr/local/bin/node && ln -sf /opt/agent-runtime/node/bin/npm /usr/local/bin/npm && ln -sf /opt/agent-runtime/node/bin/npx /usr/local/bin/npx", @@ -137,7 +149,7 @@ function buildOmpImageSteps(agentPackage: string): string[] { assertValidAgentPackage(agentPackage); const bunZip = "/tmp/bun.zip"; return [ - "RUN apt-get update && apt-get install -y --no-install-recommends curl ca-certificates git unzip", + installSystemPackagesStep(["curl", "ca-certificates", "git", "unzip"]), "ENV BUN_INSTALL=/root/.bun", `RUN ${verifiedDownload(BUN_RELEASE_URL, BUN_RELEASE_SHA256, bunZip)} && unzip -q ${bunZip} -d /tmp && install -m 0755 /tmp/bun-linux-x64/bun /usr/local/bin/bun && rm -rf ${bunZip} /tmp/bun-linux-x64`, `RUN bun install -g ${JSON.stringify(agentPackage)} && ln -sf /root/.bun/bin/omp /usr/local/bin/omp`, diff --git a/src/benchmarks/benchmark-config.ts b/src/benchmarks/benchmark-config.ts index 8dbbe57..d5e4afd 100644 --- a/src/benchmarks/benchmark-config.ts +++ b/src/benchmarks/benchmark-config.ts @@ -168,6 +168,16 @@ export const TerminalBenchConfigSchema = z.object({ export type TerminalBenchConfig = z.infer; +export const TerminalBench4OptionsSchema = TerminalBenchOptionsSchema; + +export const TerminalBench4ConfigSchema = z.object({ + benchmarkId: z.literal("terminal_bench_4"), + ...ModelBenchmarkBaseSchema.shape, + ...TerminalBench4OptionsSchema.shape, +}); + +export type TerminalBench4Config = z.infer; + export const DracoBenchmarkConfigSchema = z.object({ benchmarkId: z.literal("draco"), panelConfig: DracoPanelConfigSchema, @@ -330,6 +340,7 @@ export const NativeBenchmarkRunConfigSchema = z.discriminatedUnion( Tau3BenchBankingConfigSchema, MmmuProVisionBenchmarkConfigSchema, TerminalBenchConfigSchema, + TerminalBench4ConfigSchema, DracoBenchmarkConfigSchema, IfStructBenchmarkConfigSchema, SweAtlasQaConfigSchema, @@ -365,6 +376,7 @@ export const BENCHMARK_OPTIONS_SCHEMAS = { tau3_bench_banking: Tau3BenchBankingOptionsSchema, mmmu_pro_vision: MmmuProVisionOptionsSchema, terminal_bench: TerminalBenchOptionsSchema, + terminal_bench_4: TerminalBench4OptionsSchema, ifstruct: IfStructOptionsSchema, swe_atlas_qa: SweAtlasOptionsSchema, swe_atlas_tw: SweAtlasOptionsSchema, diff --git a/src/benchmarks/benchmark-meta.ts b/src/benchmarks/benchmark-meta.ts index 87e41eb..6c84185 100644 --- a/src/benchmarks/benchmark-meta.ts +++ b/src/benchmarks/benchmark-meta.ts @@ -39,6 +39,11 @@ export const TERMINAL_BENCH_META = { defaultEpochs: 1, } as const satisfies BenchmarkMeta; +export const TERMINAL_BENCH_4_META = { + id: "terminal_bench_4", + defaultEpochs: 1, +} as const satisfies BenchmarkMeta; + export const DRACO_META = { id: "draco", defaultEpochs: 1, @@ -106,6 +111,7 @@ const BENCHMARK_META: Readonly> = { [TAU_BENCH_AIRLINE_META.id]: TAU_BENCH_AIRLINE_META, [TAU3_BENCH_BANKING_META.id]: TAU3_BENCH_BANKING_META, [TERMINAL_BENCH_META.id]: TERMINAL_BENCH_META, + [TERMINAL_BENCH_4_META.id]: TERMINAL_BENCH_4_META, [DRACO_META.id]: DRACO_META, [IFSTRUCT_META.id]: IFSTRUCT_META, [SWE_ATLAS_QA_META.id]: SWE_ATLAS_QA_META, diff --git a/src/benchmarks/harbor/modal-sandbox.ts b/src/benchmarks/harbor/modal-sandbox.ts index dd32534..11a03b5 100644 --- a/src/benchmarks/harbor/modal-sandbox.ts +++ b/src/benchmarks/harbor/modal-sandbox.ts @@ -16,7 +16,12 @@ import type { SandboxSessionInstance, UploadSpec, } from "./sandbox"; -import { SandboxSession, makeSessionInstance, toSolverError } from "./sandbox"; +import { + SANDBOX_IMAGE_KINDS, + SandboxSession, + makeSessionInstance, + toSolverError, +} from "./sandbox"; export interface ModalSandboxConfig { readonly appName: string; @@ -64,7 +69,17 @@ export function makeModalSandboxLayer( try: () => getApp(), catch: (e) => toSolverError("Failed to resolve Modal app", e), }); - const baseImage = client.images.fromRegistry(input.imageTag); + const baseImage = + input.imageKind === SANDBOX_IMAGE_KINDS.ModalImageId + ? yield* tryPromise({ + try: () => client.images.fromId(input.imageTag), + catch: (e: unknown) => + toSolverError( + `Failed to resolve Modal image ${input.imageTag}`, + e + ), + }) + : client.images.fromRegistry(input.imageTag); const image = input.imageBuildSteps !== undefined && input.imageBuildSteps.length > 0 ? baseImage.dockerfileCommands([...input.imageBuildSteps]) @@ -76,14 +91,20 @@ export function makeModalSandboxLayer( }); const sandbox = yield* tryPromise({ try: () => - client.sandboxes.create(app, builtImage, { - timeoutMs: input.timeoutSec * 1000, - cpu: input.cpus, - memoryMiB: input.memoryMb, - blockNetwork: !input.allowInternet, - workdir: input.workdir, - command: [...input.keepAliveCommand], - }), + client.sandboxes.create( + app, + builtImage, + definedValues({ + timeoutMs: input.timeoutSec * 1000, + cpu: input.cpus, + memoryMiB: input.memoryMb, + gpu: input.gpu, + env: input.env === undefined ? undefined : { ...input.env }, + blockNetwork: !input.allowInternet, + workdir: input.workdir, + command: [...input.keepAliveCommand], + }) + ), catch: (e) => toSolverError("Failed to create Modal sandbox", e), }); let handedOff = false; diff --git a/src/benchmarks/harbor/sandbox.ts b/src/benchmarks/harbor/sandbox.ts index 83610bc..111162d 100644 --- a/src/benchmarks/harbor/sandbox.ts +++ b/src/benchmarks/harbor/sandbox.ts @@ -46,12 +46,23 @@ export interface SandboxSessionInstance { readonly destroy: () => Effect; } +export const SANDBOX_IMAGE_KINDS = { + Registry: "registry", + ModalImageId: "modal-image-id", +} as const; + +export type SandboxImageKind = + (typeof SANDBOX_IMAGE_KINDS)[keyof typeof SANDBOX_IMAGE_KINDS]; + export interface CreateSessionInput { readonly imageTag: string; + readonly imageKind?: SandboxImageKind; readonly imageBuildSteps?: readonly string[]; readonly timeoutSec: number; readonly cpus: number; readonly memoryMb: number; + readonly gpu?: string; + readonly env?: Readonly>; readonly allowInternet: boolean; readonly workdir: string; readonly keepAliveCommand: readonly string[]; diff --git a/src/benchmarks/registry.ts b/src/benchmarks/registry.ts index 0a4509a..e25b0c3 100644 --- a/src/benchmarks/registry.ts +++ b/src/benchmarks/registry.ts @@ -15,6 +15,7 @@ import { } from "./swe-atlas/benchmark"; import { TAU_BENCH_AIRLINE_BENCHMARK } from "./tau-bench-airline/benchmark"; import { TAU3_BENCH_BANKING_BENCHMARK } from "./tau3-bench-banking/benchmark"; +import { TERMINAL_BENCH_4_BENCHMARK } from "./terminal-bench-4/benchmark"; import { TERMINAL_BENCH_BENCHMARK } from "./terminal-bench/benchmark"; import type { Benchmark } from "./types"; import { VGI_BENCH_BENCHMARK } from "./vgi-bench/benchmark"; @@ -27,6 +28,7 @@ const BENCHMARKS: Record = { [TAU3_BENCH_BANKING_BENCHMARK.id]: TAU3_BENCH_BANKING_BENCHMARK, [MMMU_PRO_VISION_BENCHMARK.id]: MMMU_PRO_VISION_BENCHMARK, [TERMINAL_BENCH_BENCHMARK.id]: TERMINAL_BENCH_BENCHMARK, + [TERMINAL_BENCH_4_BENCHMARK.id]: TERMINAL_BENCH_4_BENCHMARK, [DRACO_BENCHMARK.id]: DRACO_BENCHMARK, [IFSTRUCT_BENCHMARK.id]: IFSTRUCT_BENCHMARK, [SWE_ATLAS_QA_BENCHMARK.id]: SWE_ATLAS_QA_BENCHMARK, diff --git a/src/benchmarks/terminal-bench-4/README.md b/src/benchmarks/terminal-bench-4/README.md new file mode 100644 index 0000000..5b7d97c --- /dev/null +++ b/src/benchmarks/terminal-bench-4/README.md @@ -0,0 +1,62 @@ +# Terminal-Bench 4.0 + +[Terminal-Bench 4.0](https://www.tbench.ai/news/terminal-bench-4-0) is the current major release of the continuous Terminal-Bench line that began with 3.0. It is a different task set from Terminal-Bench 2.1 (`terminal_bench` in this catalog), so scores are not comparable across the two benchmarks. This benchmark is registered separately as `terminal_bench_4`. + +## Source & license + +- Repository: , pinned to the `v4.0.0` tag, commit `452bf305c6daa62fc59061d22133a7cbc7c1572e` (`TERMINAL_BENCH_4_SOURCE_COMMIT`). Override the checkout with `BENCH_TERMINAL_BENCH_4_TASKS_DIR=`. +- Harbor dataset id: `terminal-bench/terminal-bench@4.0.0` (). +- License: Apache-2.0 (repository `LICENSE`). +- The checkout contains 66 task manifests. This harness exposes **55** of them, see the exclusion below. + +## Task format + +Each task ships `task.toml`, `instruction.md`, `environment/Dockerfile` (agent image), `tests/Dockerfile` and `tests/test.sh` (verifier image). Only the manifest fields the harness reads are validated (`schema.ts`): agent and verifier timeouts, agent and verifier resources (`cpus`, `memory_mb`, `storage_mb`, `gpus`, `gpu_types`, `allow_internet`, `env`), `artifacts`, `verifier.collect`, `verifier.env` and `metadata.category`. All 66 manifests declare `verifier.environment_mode = "separate"`. + +## Execution model + +Unlike 2.1, the verifier does not run inside the agent container. + +1. Agent sandbox is created from the task's prebuilt agent image with the task's `cpus`, `memory_mb`, GPU and `environment.env`. Internet is always enabled so the agent CLI can reach OpenRouter, and tasks that declare `allow_internet = false` record `agentNetworkForced: true` in sample metadata. Modal sandboxes ignore the image's `USER` directive and execute commands as root, which also lets the root-only agent CLI layering steps (system packages, nvm under `/root`) apply to every task image. The package step detects `apt-get`, `dnf` or `apk`, so the Fedora-based `retro-console-soc` image builds alongside the Debian and Ubuntu ones. The three tasks whose agent Dockerfile ends in a non-root `USER` (`fp8-rmsnorm-gemm` and `rs-archive-clone` as `agent`, `risk-scorer-replay` as `nobody`) therefore run the agent as root, and their samples record `agentRunsAsRoot: true` with `taskImageUser` in metadata. +2. The ori agent runs against `/instruction.md` with the task's `agent.timeout_sec`. +3. `verifier.collect` hooks run in the agent sandbox. Hooks that target a compose sidecar service fail the sample. +4. `/logs/artifacts` plus every declared artifact path is bundled into a tarball, downloaded, and extracted at the same absolute paths in a fresh verifier sandbox built from the task's verifier image with `verifier.environment` resources (falling back to the agent resources) and `verifier.env`. +5. `/tests/test.sh` runs with `verifier.timeout_sec`. Reward is read from `/logs/verifier/reward.txt` and scored 1 only when the reward is 1, otherwise 0 (`harbor/reward.ts`). + +Per-task sandbox timeouts are the task's declared timeouts plus a fixed margin, so an 8 hour task gets an 8 hour agent sandbox. + +## GPUs + +Three tasks declare one H100 each: `fp8-rmsnorm-gemm`, `jax-speedrun-gpu`, `math-eval-grader`. `toModalGpu` maps `gpus`/`gpu_types` to the Modal `gpu` string (`"H100"`, or `"H100:2"` for multiple) and refuses tasks that request GPUs without naming a type. `storage_mb` is parsed but not enforced because the Modal sandbox API used here has no disk-size parameter. + +## Images + +4.0 tasks ship Dockerfiles with local-context `COPY` steps rather than prebuilt image names, and the Modal JS SDK rejects those Dockerfiles. Images are built once per pinned commit with the Modal Python SDK (which supports a local build context) and stored in Modal's image store, so no external registry is involved: + +```bash +pip install modal +MODAL_TOKEN_ID=... MODAL_TOKEN_SECRET=... \ + python3 scripts/build-terminal-bench-4-images.py --tasks-dir /tasks [--task ] [--concurrency 4] [--dry-run] +``` + +The script writes `image-ids.json` (`{ sourceCommit, images: { : { agent, verifier } } }`) with Modal image IDs. `images.ts` validates it at load time and refuses a map whose `sourceCommit` differs from `TERMINAL_BENCH_4_SOURCE_COMMIT`; the solver fails a sample whose task has no entry. Sandboxes are created with `imageKind: "modal-image-id"`, which resolves the ID with `images.fromId` before layering the agent install steps. Images built into the `terminal-bench-4-images` Modal app are visible to the run-time app because Modal image IDs are workspace-scoped. + +## Excluded tasks + +Eleven tasks use `environment/docker-compose.yaml` and need more than one container. The Modal sandbox abstraction here is single-container, so they are excluded from the dataset and reported by `listComposeTaskIds`: + +`ctr-optimization`, `cumulative-layout-shift`, `freight-dispatch-shift`, `heat-pump-warranty`, `intrastat-meldung`, `kv-live-surgery`, `legacy-utility-triage`, `live-database-cutover`, `medical-claims-processing`, `nextjs-performance`, `payments-pipeline-fix`. + +A score over the 55 runnable tasks is not an official Terminal-Bench 4.0 number. + +## Sandbox lifetimes + +The agent sandbox outlives `maxAgentTimeoutSec` by the sum of collect-hook timeouts plus the artifact bundle, transfer and extract budgets plus a fixed margin (`agentSandboxTimeoutSec`). The verifier sandbox is created before artifact transfer, so its lifetime covers the same artifact budgets plus `maxTestTimeoutSec` (`verifierSandboxTimeoutSec`). + +## Artifact semantics + +Harbor empties directory artifact targets in the verifier before uploading the agent's copy. `artifactExtractCommand` mirrors this: any declared source that appears as a directory in the bundle is removed on the verifier before extraction, so files the agent deleted do not survive. Excluded patterns are simply absent from the bundle; verifiers that need a pristine copy of an excluded file restore it themselves (see `vpp-loss-divergence`). + +## Config options + +Same agent options as `terminal_bench` (`TerminalBenchOptionsSchema`). diff --git a/src/benchmarks/terminal-bench-4/benchmark.test.ts b/src/benchmarks/terminal-bench-4/benchmark.test.ts new file mode 100644 index 0000000..2373ab1 --- /dev/null +++ b/src/benchmarks/terminal-bench-4/benchmark.test.ts @@ -0,0 +1,86 @@ +import { describe, expect, it } from "bun:test"; + +import { assertRight } from "../../internal/testing"; +import { parseSchema } from "../../internal/zod"; +import { BenchmarkRunConfigSchema } from "../benchmark-config"; +import { getBenchmarkMeta, TERMINAL_BENCH_4_META } from "../benchmark-meta"; +import { getBenchmark } from "../registry"; +import { TERMINAL_BENCH_BENCHMARK } from "../terminal-bench/benchmark"; +import { TERMINAL_BENCH_4_BENCHMARK } from "./benchmark"; +import { buildImageMap, TERMINAL_BENCH_4_IMAGES, taskImages } from "./images"; +import { TERMINAL_BENCH_4_SOURCE_COMMIT } from "./tasks-source"; + +describe("terminal-bench-4 registry wiring", () => { + it("registers a benchmark distinct from terminal-bench 2.1", () => { + expect(TERMINAL_BENCH_4_BENCHMARK.id).toBe("terminal_bench_4"); + expect(TERMINAL_BENCH_BENCHMARK.id).toBe("terminal_bench"); + expect(getBenchmark("terminal_bench_4")).toBe(TERMINAL_BENCH_4_BENCHMARK); + expect(getBenchmark("terminal_bench")).toBe(TERMINAL_BENCH_BENCHMARK); + }); + + it("agrees with its metadata on id, epochs and temperature", () => { + expect(TERMINAL_BENCH_4_META.id).toBe(TERMINAL_BENCH_4_BENCHMARK.id); + expect(TERMINAL_BENCH_4_BENCHMARK.defaultEpochs).toBe( + TERMINAL_BENCH_4_META.defaultEpochs + ); + expect(getBenchmarkMeta(TERMINAL_BENCH_4_META.id)).toBe( + TERMINAL_BENCH_4_META + ); + expect(TERMINAL_BENCH_4_BENCHMARK.temperature).toBe(0); + expect(TERMINAL_BENCH_4_BENCHMARK.degradeSolverErrors).toBe(true); + }); + + it("parses a run config with the default agent", () => { + const result = parseSchema(BenchmarkRunConfigSchema, { + benchmarkId: "terminal_bench_4", + model: "anthropic/claude-opus-5", + reasoningEffort: "high", + agentReasoningEffort: "high", + }); + assertRight(result); + expect(result.right.benchmarkId).toBe("terminal_bench_4"); + expect( + result.right.benchmarkId === "terminal_bench_4" && result.right.agent + ).toBe("pi"); + }); +}); + +describe("terminal-bench-4 image map", () => { + const raw = { + sourceCommit: TERMINAL_BENCH_4_SOURCE_COMMIT, + images: { + "hello-world": { agent: "im-abc123", verifier: "im-def456" }, + }, + }; + + it("exposes Modal image ids per task and undefined for unknown tasks", () => { + const map = buildImageMap(raw); + expect(taskImages(map, "hello-world")).toEqual({ + agent: "im-abc123", + verifier: "im-def456", + }); + expect(taskImages(map, "missing")).toBeUndefined(); + }); + + it("rejects a map built from a different source commit", () => { + expect(() => + buildImageMap({ ...raw, sourceCommit: "0".repeat(40) }) + ).toThrow(/pinned to/); + }); + + it("rejects ids that are not Modal image ids", () => { + expect(() => + buildImageMap({ + ...raw, + images: { x: { agent: "ghcr.io/x:y", verifier: "im-1" } }, + }) + ).toThrow(/invalid/); + }); + + it("ships a committed map for the pinned commit", () => { + expect(TERMINAL_BENCH_4_IMAGES.sourceCommit).toBe( + TERMINAL_BENCH_4_SOURCE_COMMIT + ); + expect(TERMINAL_BENCH_4_IMAGES.images.size).toBeGreaterThan(0); + }); +}); diff --git a/src/benchmarks/terminal-bench-4/benchmark.ts b/src/benchmarks/terminal-bench-4/benchmark.ts new file mode 100644 index 0000000..322a2d1 --- /dev/null +++ b/src/benchmarks/terminal-bench-4/benchmark.ts @@ -0,0 +1,88 @@ +import type { HttpClient } from "@effect/platform"; +import { gen } from "effect/Effect"; +import type { Layer } from "effect/Layer"; +import { + fail as layerFail, + effect as layerEffect, + provide as layerProvide, + mergeAll as layerMergeAll, + succeed as layerSucceed, +} from "effect/Layer"; + +import type { Dataset } from "../../harness/dataset"; +import { Scorer } from "../../harness/scorer"; +import { Solver } from "../../harness/solver"; +import { definedValues } from "../../internal/guards"; +import { getOriHarness } from "../agent-cli/harness"; +import { TERMINAL_BENCH_4_META } from "../benchmark-meta"; +import { makeModalSandboxLayer } from "../harbor/modal-sandbox"; +import { SandboxSession } from "../harbor/sandbox"; +import type { Benchmark, BenchmarkRunInput } from "../types"; +import { makeTerminalBench4DatasetLayer } from "./dataset"; +import { terminalBench4Scorer } from "./scorer"; +import type { TerminalBench4SolverOpts } from "./solver"; +import { terminalBench4Solver } from "./solver"; + +export const TERMINAL_BENCH_4_ID = TERMINAL_BENCH_4_META.id; + +const TERMINAL_BENCH_4_APP_NAME = "openrouter-terminal-bench-4" as const; + +function makeTerminalBench4Layer( + input: BenchmarkRunInput +): Layer { + const { benchmarkConfig } = input; + if (benchmarkConfig.benchmarkId !== "terminal_bench_4") { + return layerFail( + new Error("terminal_bench_4 received mismatched benchmarkConfig") + ); + } + const { agent } = benchmarkConfig; + const solverOpts: TerminalBench4SolverOpts = definedValues({ + model: benchmarkConfig.model, + apiKey: input.apiKey, + sessionId: input.sessionId, + endpointId: benchmarkConfig.endpointId, + agentPackage: benchmarkConfig.agentPackage, + oriInstallUrl: benchmarkConfig.oriInstallUrl, + appendSystemPrompt: benchmarkConfig.appendSystemPrompt, + systemPrompt: benchmarkConfig.systemPrompt, + agentReasoningEffort: benchmarkConfig.agentReasoningEffort, + oriChannel: benchmarkConfig.oriChannel, + allowedTools: benchmarkConfig.allowedTools, + disallowedTools: benchmarkConfig.disallowedTools, + isolateAgentConfig: benchmarkConfig.isolateAgentConfig, + }); + const datasetLayer = makeTerminalBench4DatasetLayer( + definedValues({ + taskSubset: benchmarkConfig.taskSubset, + maxAgentTimeoutSec: benchmarkConfig.maxAgentTimeoutSec, + }) + ); + const sandboxLayer: Layer = makeModalSandboxLayer({ + appName: TERMINAL_BENCH_4_APP_NAME, + environment: benchmarkConfig.modalEnv, + }); + const solverLayer = layerEffect(Solver)( + gen(function* () { + const sessionFactory = yield* SandboxSession; + return Solver.of( + terminalBench4Solver(sessionFactory, solverOpts, getOriHarness(agent)) + ); + }) + ); + const scorerLayer = layerSucceed(Scorer, Scorer.of(terminalBench4Scorer)); + return layerMergeAll( + datasetLayer, + solverLayer.pipe(layerProvide(sandboxLayer)), + scorerLayer + ); +} + +export const TERMINAL_BENCH_4_BENCHMARK: Benchmark = { + id: TERMINAL_BENCH_4_ID, + makeDatasetLayer: () => makeTerminalBench4DatasetLayer(), + temperature: 0, + defaultEpochs: TERMINAL_BENCH_4_META.defaultEpochs, + degradeSolverErrors: true, + makeLayer: makeTerminalBench4Layer, +}; diff --git a/src/benchmarks/terminal-bench-4/dataset.test.ts b/src/benchmarks/terminal-bench-4/dataset.test.ts new file mode 100644 index 0000000..38ec812 --- /dev/null +++ b/src/benchmarks/terminal-bench-4/dataset.test.ts @@ -0,0 +1,351 @@ +import { afterAll, beforeAll, describe, expect, it } from "bun:test"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { assertLeft, assertRight } from "../../internal/testing"; +import { parseSchema } from "../../internal/zod"; +import { + TERMINAL_BENCH_4_DATASET_ID, + dockerfileUser, + listComposeTaskIds, + listTaskIds, + loadTask, + readTerminalBench4Meta, + taskToSample, + toModalGpu, + toSandboxResources, +} from "./dataset"; +import { TERMINAL_BENCH_4_IMAGES } from "./images"; +import { TaskTomlSchema } from "./schema"; +import { + ensureTasksCheckedOut, + resetCheckoutCache, + tasksDir as tasksDirOf, +} from "./tasks-source"; + +const NETWORK = describe.skipIf(Boolean(process.env["CI"])); + +const BASE_TOML = ` +[task] +name = "fixture" + +[metadata] +category = "software-engineering" + +[agent] +timeout_sec = 3600 + +[verifier] +timeout_sec = 600 +environment_mode = "separate" + +[environment] +cpus = 2 +memory_mb = 4096 +storage_mb = 10240 +gpus = 0 +`; + +const GPU_TOML = ` +artifacts = ["/app/out", { source = "/data", exclude = ["*.tmp"] }] + +[task] +name = "gpu-fixture" + +[metadata] +category = "machine-learning" + +[agent] +timeout_sec = 7200 + +[verifier] +timeout_sec = 900 +environment_mode = "separate" +env = { TB_SEED = "1" } + +[[verifier.collect]] +command = "cp /var/log/app.log /logs/artifacts/" + +[verifier.environment] +cpus = 16 +memory_mb = 32768 +storage_mb = 1024000 +gpus = 1 +gpu_types = ["H100"] + +[environment] +cpus = 8 +memory_mb = 16384 +storage_mb = 102400 +gpus = 1 +gpu_types = ["H100"] +allow_internet = false +env = { HF_HOME = "/cache" } +`; + +function writeTask( + root: string, + id: string, + toml: string, + compose = false, + dockerfile = "FROM ubuntu:24.04\nWORKDIR /app\n" +) { + const dir = join(root, id); + mkdirSync(join(dir, "environment"), { recursive: true }); + writeFileSync(join(dir, "environment", "Dockerfile"), dockerfile); + writeFileSync(join(dir, "task.toml"), toml); + writeFileSync(join(dir, "instruction.md"), `Solve ${id}.\n`); + if (compose) { + writeFileSync( + join(dir, "environment", "docker-compose.yaml"), + "services: {}\n" + ); + } +} + +let fixtureRoot = ""; +beforeAll(() => { + fixtureRoot = mkdtempSync(join(tmpdir(), "tb4-fixture-")); + writeTask(fixtureRoot, "b-cpu", BASE_TOML); + writeTask(fixtureRoot, "a-gpu", GPU_TOML); + writeTask(fixtureRoot, "c-compose", BASE_TOML, true); + writeTask( + fixtureRoot, + "d-nonroot", + BASE_TOML, + false, + "FROM ubuntu:24.04\nUSER root\nRUN useradd agent\nuser agent\n" + ); + mkdirSync(join(fixtureRoot, ".hidden")); + writeFileSync(join(fixtureRoot, "README.md"), "not a task\n"); +}); +afterAll(() => { + rmSync(fixtureRoot, { recursive: true, force: true }); +}); + +describe("terminal-bench-4 task.toml schema", () => { + it("rejects a manifest whose verifier is not in separate mode", () => { + assertLeft( + parseSchema(TaskTomlSchema, { + task: { name: "x" }, + metadata: { category: "c" }, + agent: { timeout_sec: 10 }, + verifier: { timeout_sec: 10, environment_mode: "same" }, + environment: { cpus: 1, memory_mb: 1, storage_mb: 1 }, + }) + ); + }); + + it("defaults gpus, gpu_types, allow_internet, env, artifacts and collect", () => { + const result = parseSchema(TaskTomlSchema, { + task: { name: "x" }, + metadata: { category: "c" }, + agent: { timeout_sec: 10 }, + verifier: { timeout_sec: 10, environment_mode: "separate" }, + environment: { cpus: 1, memory_mb: 1, storage_mb: 1 }, + }); + assertRight(result); + expect(result.right.environment.gpus).toBe(0); + expect(result.right.environment.gpu_types).toEqual([]); + expect(result.right.environment.allow_internet).toBe(true); + expect(result.right.environment.env).toEqual({}); + expect(result.right.artifacts).toEqual([]); + expect(result.right.verifier.collect).toEqual([]); + expect(result.right.verifier.env).toEqual({}); + }); +}); + +describe("terminal-bench-4 task listing", () => { + it("lists non-compose task directories sorted and skips hidden dirs and files", () => { + expect(listTaskIds(fixtureRoot)).toEqual(["a-gpu", "b-cpu", "d-nonroot"]); + }); + + it("lists compose tasks separately", () => { + expect(listComposeTaskIds(fixtureRoot)).toEqual(["c-compose"]); + }); + + it("honors a task subset in the requested order and drops unknown or compose ids", () => { + expect( + listTaskIds(fixtureRoot, ["b-cpu", "nope", "c-compose", "a-gpu"]) + ).toEqual(["b-cpu", "a-gpu"]); + }); +}); + +describe("terminal-bench-4 dockerfileUser", () => { + it("returns undefined when there is no USER or the last USER is root", () => { + expect(dockerfileUser("FROM x\nRUN true\n")).toBeUndefined(); + expect(dockerfileUser("FROM x\nUSER agent\nUSER root\n")).toBeUndefined(); + expect(dockerfileUser("FROM x\nUSER 0\n")).toBeUndefined(); + }); + + it("returns the last USER directive, ignoring case and indentation", () => { + expect(dockerfileUser("FROM x\nUSER root\n user nobody\n")).toBe("nobody"); + expect(dockerfileUser("FROM x\nUSER agent:agent\n")).toBe("agent:agent"); + }); +}); + +describe("terminal-bench-4 GPU mapping", () => { + const env = { + cpus: 1, + memory_mb: 1, + storage_mb: 1, + allow_internet: true, + env: {}, + }; + + it("returns no gpu for cpu-only tasks", () => { + expect(toModalGpu({ ...env, gpus: 0, gpu_types: [] })).toBeUndefined(); + }); + + it("maps a single accelerator to its bare type", () => { + expect(toModalGpu({ ...env, gpus: 1, gpu_types: ["H100"] })).toBe("H100"); + }); + + it("maps multiple accelerators with a count suffix", () => { + expect(toModalGpu({ ...env, gpus: 2, gpu_types: ["A100"] })).toBe("A100:2"); + }); + + it("refuses to guess an accelerator when gpu_types is empty", () => { + expect(() => toModalGpu({ ...env, gpus: 1, gpu_types: [] })).toThrow( + /refusing to guess/ + ); + }); + + it("omits the gpu key entirely for cpu-only sandbox resources", () => { + const resources = toSandboxResources({ ...env, gpus: 0, gpu_types: [] }); + expect(Object.hasOwn(resources, "gpu")).toBe(false); + }); +}); + +describe("terminal-bench-4 taskToSample", () => { + it("builds a stable id, instruction input and resource metadata for a cpu task", () => { + const sample = taskToSample(loadTask("b-cpu", fixtureRoot)); + expect(sample.id).toBe(`${TERMINAL_BENCH_4_DATASET_ID}-b-cpu`); + expect(sample.input).toBe("Solve b-cpu.\n"); + expect(sample.target).toEqual({ text: "b-cpu" }); + const meta = readTerminalBench4Meta(sample.metadata); + expect(meta).toBeDefined(); + expect(meta?.maxAgentTimeoutSec).toBe(3600); + expect(meta?.maxTestTimeoutSec).toBe(600); + expect(meta?.agentEnv).toEqual({ + cpus: 2, + memoryMb: 4096, + env: {}, + allowInternet: true, + }); + expect(meta?.verifierEnv).toEqual(meta?.agentEnv); + expect(meta?.imageUser).toBeUndefined(); + }); + + it("records the image's final non-root USER so the run can report the root deviation", () => { + const meta = readTerminalBench4Meta( + taskToSample(loadTask("d-nonroot", fixtureRoot)).metadata + ); + expect(meta?.imageUser).toBe("agent"); + }); + + it("propagates gpu, env vars, artifacts and collect hooks for a gpu task", () => { + const meta = readTerminalBench4Meta( + taskToSample(loadTask("a-gpu", fixtureRoot)).metadata + ); + expect(meta?.agentEnv).toEqual({ + cpus: 8, + memoryMb: 16384, + gpu: "H100", + env: { HF_HOME: "/cache" }, + allowInternet: false, + }); + expect(meta?.verifierEnv).toEqual({ + cpus: 16, + memoryMb: 32768, + gpu: "H100", + env: { TB_SEED: "1" }, + allowInternet: true, + }); + expect(meta?.artifacts).toEqual([ + "/app/out", + { source: "/data", exclude: ["*.tmp"] }, + ]); + expect(meta?.collect).toEqual([ + { command: "cp /var/log/app.log /logs/artifacts/" }, + ]); + }); + + it("applies an agent timeout override without touching the verifier timeout", () => { + const meta = readTerminalBench4Meta( + taskToSample(loadTask("b-cpu", fixtureRoot), 60).metadata + ); + expect(meta?.maxAgentTimeoutSec).toBe(60); + expect(meta?.maxTestTimeoutSec).toBe(600); + }); + + it("refuses to build a sample for a compose task", () => { + expect(() => taskToSample(loadTask("c-compose", fixtureRoot))).toThrow( + /docker-compose/ + ); + }); + + it("returns undefined for metadata that is not terminal-bench-4 metadata", () => { + expect(readTerminalBench4Meta(undefined)).toBeUndefined(); + expect(readTerminalBench4Meta({ taskId: "x" })).toBeUndefined(); + }); +}); + +NETWORK("terminal-bench-4 pinned checkout", () => { + let tasksDir = ""; + let savedCacheDisable: string | undefined; + beforeAll(async () => { + resetCheckoutCache(); + savedCacheDisable = process.env.BENCH_DATASET_CACHE_DISABLE; + process.env.BENCH_DATASET_CACHE_DISABLE ??= "0"; + tasksDir = tasksDirOf(await ensureTasksCheckedOut()); + }, 300_000); + afterAll(() => { + if (savedCacheDisable === undefined) { + delete process.env.BENCH_DATASET_CACHE_DISABLE; + } else { + process.env.BENCH_DATASET_CACHE_DISABLE = savedCacheDisable; + } + }); + + it("exposes 55 runnable tasks and 11 compose tasks", () => { + expect(listTaskIds(tasksDir)).toHaveLength(55); + expect(listComposeTaskIds(tasksDir)).toHaveLength(11); + }); + + it("parses every manifest and maps exactly three H100 tasks", () => { + const gpuTasks = listTaskIds(tasksDir) + .map((id) => taskToSample(loadTask(id, tasksDir))) + .flatMap((sample) => { + const meta = readTerminalBench4Meta(sample.metadata); + return meta?.agentEnv.gpu === undefined ? [] : [meta.taskId]; + }); + expect(gpuTasks).toEqual([ + "fp8-rmsnorm-gemm", + "jax-speedrun-gpu", + "math-eval-grader", + ]); + for (const id of listComposeTaskIds(tasksDir)) { + expect(loadTask(id, tasksDir).composeFile).toBeDefined(); + } + }); + + it("ships Modal images for exactly the runnable task set", () => { + expect([...TERMINAL_BENCH_4_IMAGES.images.keys()].sort()).toEqual( + listTaskIds(tasksDir) + ); + }); + + it("identifies exactly three tasks whose agent image runs as a non-root user", () => { + const nonRoot = listTaskIds(tasksDir).flatMap((id) => { + const user = loadTask(id, tasksDir).imageUser; + return user === undefined ? [] : [`${id}:${user}`]; + }); + expect(nonRoot).toEqual([ + "fp8-rmsnorm-gemm:agent", + "risk-scorer-replay:nobody", + "rs-archive-clone:agent", + ]); + }); +}); diff --git a/src/benchmarks/terminal-bench-4/dataset.ts b/src/benchmarks/terminal-bench-4/dataset.ts new file mode 100644 index 0000000..dd78b4e --- /dev/null +++ b/src/benchmarks/terminal-bench-4/dataset.ts @@ -0,0 +1,277 @@ +import { existsSync, readFileSync, readdirSync, statSync } from "node:fs"; +import { join } from "node:path"; + +import { fromIterable } from "effect/Chunk"; +import type { Effect } from "effect/Effect"; +import { fail, flatMap, mapError, succeed } from "effect/Effect"; +import type { Layer } from "effect/Layer"; +import { effect } from "effect/Layer"; +import { none, some } from "effect/Option"; +import type { Stream } from "effect/Stream"; +import { + flatMap as flatMapStream, + fromEffect, + paginateChunkEffect, +} from "effect/Stream"; +import { parse as tomlParse } from "smol-toml"; + +import type { Sample } from "../../harness/core"; +import { DatasetError } from "../../harness/core"; +import type { DatasetStreamOptions } from "../../harness/dataset"; +import { Dataset } from "../../harness/dataset"; +import { Either } from "../../internal/either"; +import { definedValues } from "../../internal/guards"; +import { parseSchema, z } from "../../internal/zod"; +import type { TaskEnvironment, TerminalBench4Task } from "./schema"; +import { ArtifactSchema, CollectHookSchema, TaskTomlSchema } from "./schema"; +import { ensureTasksCheckedOutEffect, tasksDir } from "./tasks-source"; + +export const TERMINAL_BENCH_4_DATASET_ID = "terminal_bench_4" as const; + +const COMPOSE_FILE = join("environment", "docker-compose.yaml"); +const AGENT_DOCKERFILE = join("environment", "Dockerfile"); +const USER_DIRECTIVE = /^\s*USER\s+(\S+)/i; + +export function dockerfileUser(dockerfile: string): string | undefined { + const users = dockerfile + .split("\n") + .map((line) => USER_DIRECTIVE.exec(line)?.[1]) + .filter((user): user is string => user !== undefined); + const last = users.at(-1); + return last === undefined || last === "root" || last === "0" + ? undefined + : last; +} + +export function loadTask( + taskId: string, + tasksDirPath: string +): TerminalBench4Task { + const taskDir = join(tasksDirPath, taskId); + const raw = readFileSync(join(taskDir, "task.toml"), "utf8"); + const tomlObj = Either.try(() => tomlParse(raw)); + if (Either.isLeft(tomlObj)) { + throw new Error( + `terminal-bench-4 task "${taskId}" task.toml failed to parse: ${String(tomlObj.left)}` + ); + } + const parsed = parseSchema(TaskTomlSchema, tomlObj.right); + if (Either.isLeft(parsed)) { + throw new Error( + `terminal-bench-4 task "${taskId}" task.toml failed validation: ${parsed.left.message}` + ); + } + const composePath = join(taskDir, COMPOSE_FILE); + return { + id: taskId, + taskToml: parsed.right, + taskDir, + instructionPath: join(taskDir, "instruction.md"), + composeFile: existsSync(composePath) ? composePath : undefined, + imageUser: dockerfileUser( + readFileSync(join(taskDir, AGENT_DOCKERFILE), "utf8") + ), + }; +} + +function isTaskDir(tasksDirPath: string, entry: string): boolean { + if (entry.startsWith(".")) { + return false; + } + try { + return ( + statSync(join(tasksDirPath, entry)).isDirectory() && + existsSync(join(tasksDirPath, entry, "task.toml")) + ); + } catch { + return false; + } +} + +export function listComposeTaskIds(tasksDirPath: string): readonly string[] { + return readdirSync(tasksDirPath) + .filter( + (entry) => + isTaskDir(tasksDirPath, entry) && + existsSync(join(tasksDirPath, entry, COMPOSE_FILE)) + ) + .sort(); +} + +export function listTaskIds( + tasksDirPath: string, + taskSubset?: readonly string[] +): readonly string[] { + const onDisk = readdirSync(tasksDirPath).filter( + (entry) => + isTaskDir(tasksDirPath, entry) && + !existsSync(join(tasksDirPath, entry, COMPOSE_FILE)) + ); + if (taskSubset !== undefined && taskSubset.length > 0) { + const diskSet = new Set(onDisk); + return taskSubset.filter((id) => diskSet.has(id)); + } + return [...onDisk].sort(); +} + +const SandboxResourcesSchema = z.object({ + cpus: z.number().int().positive(), + memoryMb: z.number().int().positive(), + gpu: z.string().min(1).optional(), + env: z.record(z.string(), z.string()), + allowInternet: z.boolean(), +}); + +export type SandboxResources = z.infer; + +export const TerminalBench4SampleMetaSchema = z.object({ + taskId: z.string().min(1), + maxAgentTimeoutSec: z.number().positive(), + maxTestTimeoutSec: z.number().positive(), + category: z.string(), + agentEnv: SandboxResourcesSchema, + verifierEnv: SandboxResourcesSchema, + imageUser: z.string().min(1).optional(), + artifacts: z.array(ArtifactSchema), + collect: z.array(CollectHookSchema), + reward: z.number().optional(), + testOutput: z.string().optional(), +}); + +export type TerminalBench4SampleMeta = z.infer< + typeof TerminalBench4SampleMetaSchema +>; + +export function toSandboxResources( + env: TaskEnvironment, + extraEnvVars: Readonly> = {} +): SandboxResources { + return definedValues({ + cpus: env.cpus, + memoryMb: env.memory_mb, + gpu: toModalGpu(env), + env: { ...env.env, ...extraEnvVars }, + allowInternet: env.allow_internet, + }); +} + +export function toModalGpu(env: TaskEnvironment): string | undefined { + if (env.gpus === 0) { + return undefined; + } + const type = env.gpu_types[0]; + if (type === undefined) { + throw new Error( + `task requests ${env.gpus} GPU(s) but declares no gpu_types; refusing to guess an accelerator` + ); + } + return env.gpus === 1 ? type : `${type}:${env.gpus}`; +} + +export function taskToSample( + task: TerminalBench4Task, + maxAgentTimeoutSecOverride?: number +): Sample { + if (task.composeFile !== undefined) { + throw new Error( + `terminal-bench-4 task "${task.id}" uses docker-compose, which this harness does not support` + ); + } + const instruction = readFileSync(task.instructionPath, "utf8"); + const { environment, verifier, agent, metadata, artifacts } = task.taskToml; + const meta: TerminalBench4SampleMeta = definedValues({ + taskId: task.id, + maxAgentTimeoutSec: maxAgentTimeoutSecOverride ?? agent.timeout_sec, + maxTestTimeoutSec: verifier.timeout_sec, + category: metadata.category, + agentEnv: toSandboxResources(environment), + verifierEnv: toSandboxResources( + verifier.environment ?? environment, + verifier.env + ), + imageUser: task.imageUser, + artifacts, + collect: verifier.collect, + }); + return { + id: `${TERMINAL_BENCH_4_DATASET_ID}-${task.id}`, + input: instruction, + target: { text: task.id }, + metadata: meta, + }; +} + +export function readTerminalBench4Meta( + metadata: Readonly> | undefined +): TerminalBench4SampleMeta | undefined { + if (metadata === undefined) { + return undefined; + } + const parsed = parseSchema(TerminalBench4SampleMetaSchema, metadata); + return Either.isLeft(parsed) ? undefined : parsed.right; +} + +export interface TerminalBench4DatasetConfig { + readonly taskSubset?: readonly string[]; + readonly maxAgentTimeoutSec?: number; + readonly pageSize?: number; +} + +export function makeTerminalBench4DatasetLayer( + config?: TerminalBench4DatasetConfig +): Layer { + const pageSize = config?.pageSize ?? 20; + const taskSubset = config?.taskSubset; + const maxAgentTimeoutSec = config?.maxAgentTimeoutSec; + return effect( + Dataset, + succeed(buildDatasetService({ pageSize, taskSubset, maxAgentTimeoutSec })) + ); +} + +function buildDatasetService(opts: { + readonly pageSize: number; + readonly taskSubset?: readonly string[]; + readonly maxAgentTimeoutSec?: number; +}): ReturnType { + const { pageSize, taskSubset, maxAgentTimeoutSec } = opts; + const tasksDirEffect = ensureTasksCheckedOutEffect().pipe( + mapError((e) => new DatasetError({ message: e.message })), + flatMap((root) => succeed(tasksDir(root))) + ); + const sizeEffect: Effect = tasksDirEffect.pipe( + flatMap((dir) => succeed(listTaskIds(dir, taskSubset).length)) + ); + const stream = (opts2?: DatasetStreamOptions): Stream => + fromEffect(tasksDirEffect).pipe( + flatMapStream((dir: string) => { + const allIds = listTaskIds(dir, taskSubset); + const start = opts2?.start ?? 0; + const requestedEnd = opts2?.end ?? allIds.length; + const end = Math.min(requestedEnd, allIds.length); + const pageIds = allIds.slice(start, end); + return paginateChunkEffect(0, (offset: number) => { + const pageSlice = pageIds.slice(offset, offset + pageSize); + const built = Either.try(() => + pageSlice.map((id) => + taskToSample(loadTask(id, dir), maxAgentTimeoutSec) + ) + ); + if (Either.isLeft(built)) { + return fail( + new DatasetError({ + message: `Failed to load terminal-bench-4 tasks at offset ${offset}: ${String(built.left)}`, + }) + ); + } + const nextOffset = offset + pageSize; + const hasMore = nextOffset < pageIds.length; + return succeed([ + fromIterable(built.right), + hasMore ? some(nextOffset) : none(), + ] as const); + }); + }) + ); + return Dataset.of({ stream, size: sizeEffect }); +} diff --git a/src/benchmarks/terminal-bench-4/image-ids.json b/src/benchmarks/terminal-bench-4/image-ids.json new file mode 100644 index 0000000..afd14fb --- /dev/null +++ b/src/benchmarks/terminal-bench-4/image-ids.json @@ -0,0 +1,225 @@ +{ + "sourceCommit": "452bf305c6daa62fc59061d22133a7cbc7c1572e", + "images": { + "atrx-vep-crispr": { + "agent": "im-5mzBHPjvYN8zIeS6JeEKmR", + "verifier": "im-soU60kfnSJpxarw5z0Xsls" + }, + "batched-eval-parity": { + "agent": "im-kBn6DlBhHSrVN8JT9WgqbE", + "verifier": "im-arsDZFQI9fohCwpftm4Xmo" + }, + "biped-contact-dynamics": { + "agent": "im-nxaATwoff5WzVgqWC3ueUg", + "verifier": "im-aI2kfN2sYYh7qYYuvjf5Ki" + }, + "bun-sourcemap-leak": { + "agent": "im-VnmCRNhZKb7HmJvvSmpvSS", + "verifier": "im-Vl88ZMvXHbhJ6S02LyvIap" + }, + "cad-model": { + "agent": "im-61M5ZsDKFh5kSZ2tCZWVdB", + "verifier": "im-H0ScSMfg12YuFmxArZsWls" + }, + "cargo-flight-dispatch": { + "agent": "im-3ar3uyEVkz5JG01K03FSGL", + "verifier": "im-ewcfQ4RReECHrtBRgbIozd" + }, + "coq-block-bound": { + "agent": "im-04T13UpbEAMId2NSJUp6SF", + "verifier": "im-aFOGNEMzDXXs1v53C1EBn2" + }, + "data-anonymization": { + "agent": "im-X8mNLloN7NLDM0HER40G8k", + "verifier": "im-4GyZC9MkFZv2VmsUiZYygH" + }, + "distributed-dedup": { + "agent": "im-WnqarejRmceVgqXZG0WS4y", + "verifier": "im-uRC5MnMRmgBoMqFeqtk3nU" + }, + "embedding-drift-monitor": { + "agent": "im-PpFyIuYjqgY7lVH5zeWn8W", + "verifier": "im-GFLxtRFzTejj1HA8IuRQho" + }, + "fin-saccr-rwa": { + "agent": "im-CaLvuH98JYvektFWObLx9D", + "verifier": "im-zFbl2JjMfiRThTG467pr73" + }, + "foodstuff-beta-activity": { + "agent": "im-0D8mQR2fRuJ4CXlJ8WSU9H", + "verifier": "im-QZsMu4kmz3O4asRkrrjNMA" + }, + "formal-crypto": { + "agent": "im-oB4I8OQeHKxpydnII7FSv7", + "verifier": "im-W1MgXXhDW3Xd9IS3O2FcaJ" + }, + "fp8-rmsnorm-gemm": { + "agent": "im-FBlniwzVOyNSzbofwXX7Sf", + "verifier": "im-aRPhi7NBmOCMhMQxVOT2E6" + }, + "freecad-impeller": { + "agent": "im-bKnL2lcYxiJvy7oCmBC8kJ", + "verifier": "im-LSPLtMLQE1ZKgOLfrk2uh3" + }, + "freecad-platform-drawing": { + "agent": "im-mifytSREMlerQ7D73Vc3kp", + "verifier": "im-giErBLPDEx5peOyYKzGr2n" + }, + "freecad-spring-clip": { + "agent": "im-bKnL2lcYxiJvy7oCmBC8kJ", + "verifier": "im-D6lcNYtlx1O7yg0pLocbAK" + }, + "glycan-ms2-elucidation": { + "agent": "im-tVkTPoGRtm54VS5UxHye0p", + "verifier": "im-a6N7eouCDSEhvo7ePaG39C" + }, + "gsea-proteomics": { + "agent": "im-uzrdxyBsF9lXgsBzNsolX3", + "verifier": "im-nkIpSvrBVlb5648LWMndAK" + }, + "hof-topology-interpenetration": { + "agent": "im-EMSua2mY5hrV84ODUQio6O", + "verifier": "im-kHF4rEeFTCiLo3gYSAvs5X" + }, + "html-js-filter": { + "agent": "im-QIU15HnCYJ1n5Hrp44ovlp", + "verifier": "im-JyPSKpY5Cci0XXtr9yEDXG" + }, + "interleaved-vigenere": { + "agent": "im-u9b0sHMJDOnlaxKT6Txe5l", + "verifier": "im-zYsvvdjqBcynmj7duY40xg" + }, + "jax-speedrun-gpu": { + "agent": "im-N6iQ3hzwZC8peLicLzgL9M", + "verifier": "im-WFYkmZNiX4qKi0URADBJwt" + }, + "ks-solver-cpp": { + "agent": "im-AWBP7mR4qUflFwFvw5f2Vb", + "verifier": "im-Odr1CRDUAjeo1BhRdNVqqq" + }, + "lake-temp-glm": { + "agent": "im-i3rLQTsqFSYEOT92nur7qt", + "verifier": "im-it6BKyg4odryjD8OcPmXy9" + }, + "layout-config-recreation": { + "agent": "im-h1pPUIY5Ytp6u6fwhMjaq1", + "verifier": "im-g89G0VUFlmkOuWMDodvyQZ" + }, + "layout-config-recreation2": { + "agent": "im-KnCKkgI6t0Gpja6WXqKba4", + "verifier": "im-1ETetaATJPqnFtf6oCAm4U" + }, + "math-eval-grader": { + "agent": "im-rSssB1KmnzIxwD1Rs93NrI", + "verifier": "im-yQbArTXAtUudEH4ZjJoeab" + }, + "mp-checkpoint-consolidation": { + "agent": "im-sWyivSwRZANmAmiaPguZCK", + "verifier": "im-iPJtn3o95kUUq1O3HD8qb0" + }, + "music-harmony": { + "agent": "im-h4eLdJF4FdDeGLlrKSdS2D", + "verifier": "im-N1tgE2XbCSM8h5eJVr69tv" + }, + "mvcc-lsm-compaction": { + "agent": "im-2gXstFD5uCfDJCcmnLtIeZ", + "verifier": "im-LfmhJztcZwKbyHI2DFujA5" + }, + "ontology-kg-querying": { + "agent": "im-2hI0nvFliMMqAFMA5llA3d", + "verifier": "im-AfLrc8nkL1xBFSob8CJ1bp" + }, + "photonic-waveguide-routing": { + "agent": "im-pCYVVyjBavAEYkuJgozWUi", + "verifier": "im-vGf57QW6Br4K2RW8l6uO00" + }, + "pretrain-shard-corruption": { + "agent": "im-Bv99gtPti1VZ961QSW4moe", + "verifier": "im-cRiRtyN0jiljrSuCMDjHS9" + }, + "production-planning": { + "agent": "im-iDZVsFKcthAsrgI0ZAnlPM", + "verifier": "im-5LTiyQOWuyHcMWoWnXl0Th" + }, + "protein-autointerp-disulfide": { + "agent": "im-si3dXWMicfkPd4VZDvVCAJ", + "verifier": "im-QUzqvygh1T8DjPDJZqoDwt" + }, + "react-lead-form": { + "agent": "im-LKft15TMrpUL9cveUjPEv8", + "verifier": "im-cgFLZ0elW4ScsRhPLNBKHP" + }, + "retro-console-soc": { + "agent": "im-PDekqRInPMIT5hA5ibSsqx", + "verifier": "im-l9VMHqhZwCChc9jReDlifb" + }, + "risk-scorer-replay": { + "agent": "im-Q9ALFBHyhVNWlslfsJ6cvq", + "verifier": "im-mVQSAczLppxJhFWJ8q22c9" + }, + "roy-polymorph-cn": { + "agent": "im-mpoKHymb4pQS0PRWMmXMJM", + "verifier": "im-LwNVFOAvkuKuYvXeMejMQG" + }, + "rs-archive-clone": { + "agent": "im-9O5meSz6MvfwQpMkRShZ4w", + "verifier": "im-dD0YhMM2uiNKvb3kM9Ts8R" + }, + "satb-audio-transcription": { + "agent": "im-ayMOwN81rBC0IYGlnCgChh", + "verifier": "im-ZHu8g2cFqoN7ZWm8CY8jWh" + }, + "session-window-debug": { + "agent": "im-PvFjlbPMy4TK8t78B85qRH", + "verifier": "im-jbGBZHefGrA7YB4yR55Jwq" + }, + "sglang-qwen-burst": { + "agent": "im-oGI1JG2N5ATgqomxqM2ZNa", + "verifier": "im-5ySAc9FIK3MD6slsX047gD" + }, + "shadow-relay": { + "agent": "im-QEpRRr0q9Cu4axYMOuDtBh", + "verifier": "im-DLOI58H0O9lewdcjaDu0wb" + }, + "sound-change-cascade": { + "agent": "im-MmUWHFroYghOfNIMBPlq0U", + "verifier": "im-TZsHjXKVzikNheZrbiXTqU" + }, + "takens-embedding-lean": { + "agent": "im-YQ7KtwWLhnbbOS7IhEudJK", + "verifier": "im-mL2UnKl0T1kTZdQ15RC9FZ" + }, + "telecom-entity-resolution": { + "agent": "im-xMkee5jCTN5uo6g2sb4a8Z", + "verifier": "im-HkAgySdmfJS99o1l12ZAwd" + }, + "uefi-bootkit": { + "agent": "im-IuXJpvadCgAxNnD1mNngYn", + "verifier": "im-YmviJTLkykmRUFzJzjLLsR" + }, + "vba-userform-port": { + "agent": "im-If0Gw9lp55OmbkJBANDhtp", + "verifier": "im-iRE4ikpfNdbayPgfalIGb9" + }, + "vf2-speedup-networkx": { + "agent": "im-4RHL1D2JzIPOaREGkR8aZu", + "verifier": "im-9hv1GCOv9a01qdfGKyrRns" + }, + "vllm-deepseek-streaming": { + "agent": "im-lZ9QozXsV4i7jNH9PNWH98", + "verifier": "im-DnbP8OnC3Mxvd8T3d2k4L6" + }, + "vpp-loss-divergence": { + "agent": "im-53aeOBx7q6eVGTRYW5NikY", + "verifier": "im-Rn4eJ2D7g0Kck62cKgtnJR" + }, + "wal-recovery-ordering": { + "agent": "im-xV4icf0Bdmezzi1PzQvqLe", + "verifier": "im-PeMankuM3Nw8YntS0AsdBA" + }, + "wdm-design": { + "agent": "im-9cQ83P2bxI3JfLSAg0duHg", + "verifier": "im-dpyIXtQ258y6uA4hjOSTD5" + } + } +} diff --git a/src/benchmarks/terminal-bench-4/images.ts b/src/benchmarks/terminal-bench-4/images.ts new file mode 100644 index 0000000..10e84b4 --- /dev/null +++ b/src/benchmarks/terminal-bench-4/images.ts @@ -0,0 +1,52 @@ +import { Either } from "../../internal/either"; +import { firstZodIssueMessage, parseSchema, z } from "../../internal/zod"; +import imageIdsJson from "./image-ids.json"; +import { TERMINAL_BENCH_4_SOURCE_COMMIT } from "./tasks-source"; + +const ModalImageIdSchema = z.string().regex(/^im-[A-Za-z0-9]+$/); + +const TaskImagesSchema = z.object({ + agent: ModalImageIdSchema, + verifier: ModalImageIdSchema, +}); + +export type TerminalBench4TaskImages = z.infer; + +const ImageIdsSchema = z.object({ + sourceCommit: z.string().regex(/^[0-9a-f]{40}$/), + images: z.record(z.string().min(1), TaskImagesSchema), +}); + +export interface TerminalBench4ImageMap { + readonly sourceCommit: string; + readonly images: ReadonlyMap; +} + +export function buildImageMap( + raw: unknown, + expectedCommit: string = TERMINAL_BENCH_4_SOURCE_COMMIT +): TerminalBench4ImageMap { + const parsed = parseSchema(ImageIdsSchema, raw); + if (Either.isLeft(parsed)) { + throw new TypeError( + `terminal-bench-4 image-ids.json is invalid: ${firstZodIssueMessage(parsed.left)}` + ); + } + const { sourceCommit, images } = parsed.right; + if (sourceCommit !== expectedCommit) { + throw new TypeError( + `terminal-bench-4 image-ids.json was built from ${sourceCommit} but tasks are pinned to ${expectedCommit}; rerun scripts/build-terminal-bench-4-images.py` + ); + } + return { sourceCommit, images: new Map(Object.entries(images)) }; +} + +export function taskImages( + map: TerminalBench4ImageMap, + taskId: string +): TerminalBench4TaskImages | undefined { + return map.images.get(taskId); +} + +export const TERMINAL_BENCH_4_IMAGES: TerminalBench4ImageMap = + buildImageMap(imageIdsJson); diff --git a/src/benchmarks/terminal-bench-4/schema.ts b/src/benchmarks/terminal-bench-4/schema.ts new file mode 100644 index 0000000..6e5ea11 --- /dev/null +++ b/src/benchmarks/terminal-bench-4/schema.ts @@ -0,0 +1,68 @@ +import type { ValueOf } from "../../internal/guards"; +import { z } from "../../internal/zod"; +import { ORI_AGENTS } from "../agent-cli/schema"; + +export const TERMINAL_BENCH_4_VERSION = "4.0.0" as const; + +export const TERMINAL_BENCH_4_AGENTS = ORI_AGENTS; + +export type TerminalBench4Agent = ValueOf; + +export const DEFAULT_TERMINAL_BENCH_4_AGENT: TerminalBench4Agent = "pi"; + +export const ArtifactSchema = z.union([ + z.string().min(1), + z.object({ + source: z.string().min(1), + exclude: z.array(z.string()).default([]), + service: z.string().optional(), + }), +]); + +export type Artifact = z.infer; + +const EnvironmentSchema = z.object({ + cpus: z.number().int().positive(), + memory_mb: z.number().int().positive(), + storage_mb: z.number().int().positive(), + gpus: z.number().int().nonnegative().default(0), + gpu_types: z.array(z.string().min(1)).default([]), + allow_internet: z.boolean().default(true), + env: z.record(z.string(), z.string()).default({}), +}); + +export type TaskEnvironment = z.infer; + +export const CollectHookSchema = z.object({ + command: z.string().min(1), + service: z.string().optional(), + timeout_sec: z.number().positive().optional(), +}); + +export type CollectHook = z.infer; + +export const TaskTomlSchema = z.object({ + artifacts: z.array(ArtifactSchema).default([]), + task: z.object({ name: z.string().min(1) }), + metadata: z.object({ category: z.string().min(1) }), + agent: z.object({ timeout_sec: z.number().positive() }), + verifier: z.object({ + timeout_sec: z.number().positive(), + environment_mode: z.literal("separate"), + environment: EnvironmentSchema.optional(), + collect: z.array(CollectHookSchema).default([]), + env: z.record(z.string(), z.string()).default({}), + }), + environment: EnvironmentSchema, +}); + +export type TaskToml = z.infer; + +export interface TerminalBench4Task { + readonly id: string; + readonly taskToml: TaskToml; + readonly taskDir: string; + readonly instructionPath: string; + readonly composeFile: string | undefined; + readonly imageUser: string | undefined; +} diff --git a/src/benchmarks/terminal-bench-4/scorer.test.ts b/src/benchmarks/terminal-bench-4/scorer.test.ts new file mode 100644 index 0000000..141a9fc --- /dev/null +++ b/src/benchmarks/terminal-bench-4/scorer.test.ts @@ -0,0 +1,71 @@ +import { describe, expect, it } from "bun:test"; + +import { runPromise } from "effect/Effect"; + +import type { Sample } from "../../harness/core"; +import { initialTaskState, ScoreValue } from "../../harness/core"; +import type { TerminalBench4SampleMeta } from "./dataset"; +import { terminalBench4Scorer } from "./scorer"; + +const META: TerminalBench4SampleMeta = { + taskId: "music-harmony", + maxAgentTimeoutSec: 28_800, + maxTestTimeoutSec: 600, + category: "software-engineering", + agentEnv: { cpus: 2, memoryMb: 4096, env: {}, allowInternet: true }, + verifierEnv: { cpus: 2, memoryMb: 4096, env: {}, allowInternet: false }, + artifacts: [], + collect: [], +}; + +function sampleWith(reward: number | undefined, testOutput?: string): Sample { + return { + id: "terminal_bench_4-music-harmony", + input: "harmonize", + target: { text: "music-harmony" }, + metadata: { + ...META, + ...(reward !== undefined && { reward }), + ...(testOutput !== undefined && { testOutput }), + }, + }; +} + +describe("terminal-bench-4 scorer (pure)", () => { + it("scores Correct when the stashed reward is 1", async () => { + const state = initialTaskState(sampleWith(1, "1 passed")); + const score = await runPromise( + terminalBench4Scorer(state, state.sample.target) + ); + expect(score.value).toBe(ScoreValue.Correct); + expect(score.answer).toBe("music-harmony"); + expect(score.explanation).toBe("1 passed"); + expect(score.trajectory).toEqual({ kind: "verifier_log", log: "1 passed" }); + }); + it("scores Incorrect when the stashed reward is 0", async () => { + const state = initialTaskState(sampleWith(0, "1 failed")); + const score = await runPromise( + terminalBench4Scorer(state, state.sample.target) + ); + expect(score.value).toBe(ScoreValue.Incorrect); + }); + it("scores Incorrect when no reward is stashed", async () => { + const state = initialTaskState(sampleWith(undefined)); + const score = await runPromise( + terminalBench4Scorer(state, state.sample.target) + ); + expect(score.value).toBe(ScoreValue.Incorrect); + expect(score.explanation).toBe(""); + expect(score.trajectory).toBeUndefined(); + }); + it("scores Incorrect when metadata is not terminal-bench-4 metadata", async () => { + const state = initialTaskState({ + ...sampleWith(1), + metadata: { taskId: "music-harmony", reward: 1 }, + }); + const score = await runPromise( + terminalBench4Scorer(state, state.sample.target) + ); + expect(score.value).toBe(ScoreValue.Incorrect); + }); +}); diff --git a/src/benchmarks/terminal-bench-4/scorer.ts b/src/benchmarks/terminal-bench-4/scorer.ts new file mode 100644 index 0000000..e943e51 --- /dev/null +++ b/src/benchmarks/terminal-bench-4/scorer.ts @@ -0,0 +1,38 @@ +import { succeed } from "effect/Effect"; + +import type { Score, Target, TaskState } from "../../harness/core"; +import { ScoreValue } from "../../harness/core"; +import type { ScorerService } from "../../harness/scorer"; +import { definedValues } from "../../internal/guards"; +import { readTerminalBench4Meta } from "./dataset"; + +function readReward(state: TaskState): { + reward: number; + testOutput?: string; +} { + const meta = readTerminalBench4Meta(state.sample.metadata); + if (meta === undefined) { + return { reward: 0 }; + } + return definedValues({ + reward: meta.reward ?? 0, + testOutput: meta.testOutput, + }); +} + +export const terminalBench4Scorer: ScorerService = ( + state: TaskState, + target: Target +) => { + const { reward, testOutput } = readReward(state); + const score: Score = definedValues({ + value: reward >= 1 ? ScoreValue.Correct : ScoreValue.Incorrect, + answer: target.text, + explanation: testOutput ?? "", + trajectory: + testOutput !== undefined + ? ({ kind: "verifier_log", log: testOutput } as const) + : undefined, + }); + return succeed(score); +}; diff --git a/src/benchmarks/terminal-bench-4/session.test.ts b/src/benchmarks/terminal-bench-4/session.test.ts new file mode 100644 index 0000000..8d3a0e2 --- /dev/null +++ b/src/benchmarks/terminal-bench-4/session.test.ts @@ -0,0 +1,482 @@ +import { describe, expect, it } from "bun:test"; +import { spawnSync } from "node:child_process"; +import { + existsSync, + mkdirSync, + mkdtempSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import type { Effect } from "effect/Effect"; +import { either, gen, provide, runPromise } from "effect/Effect"; +import type { Layer } from "effect/Layer"; + +import type { SolverError } from "../../harness/core"; +import { assertLeft, assertRight } from "../../internal/testing"; +import type { + CreateSessionInput, + ExecResult, + SandboxSessionFactory, +} from "../harbor/sandbox"; +import { makeFakeSandboxLayer, SandboxSession } from "../harbor/sandbox"; +import type { TerminalBench4SampleMeta } from "./dataset"; +import { + agentNetworkDeviation, + agentUserDeviation, + agentSandboxTimeoutSec, + ARTIFACT_BUNDLE_TIMEOUT_MS, + ARTIFACT_TRANSFER_TIMEOUT_MS, + artifactBundleCommand, + artifactExtractCommand, + createAgentSession, + DEFAULT_COLLECT_TIMEOUT_SEC, + SANDBOX_TIMEOUT_MARGIN_SEC, + verifierSandboxTimeoutSec, + createVerifierSession, + runCollectHooks, + runVerifier, + sandboxCollectHooks, + transferArtifacts, +} from "./session"; + +const OK: ExecResult = { exitCode: 0, stdout: "", stderr: "" }; + +const META: TerminalBench4SampleMeta = { + taskId: "fp8-rmsnorm-gemm", + maxAgentTimeoutSec: 100, + maxTestTimeoutSec: 50, + category: "machine-learning", + agentEnv: { + cpus: 8, + memoryMb: 16384, + gpu: "H100", + env: { HF_HOME: "/cache" }, + allowInternet: false, + }, + verifierEnv: { + cpus: 4, + memoryMb: 8192, + env: { TB_SEED: "1" }, + allowInternet: true, + }, + artifacts: ["/app/out"], + collect: [], +}; + +interface Recorded { + readonly creates: CreateSessionInput[]; + readonly execs: { + argv: readonly string[]; + env: Readonly>; + timeoutMs: number | undefined; + }[]; + readonly uploads: { localPath: string; remotePath: string }[]; + readonly downloads: { remotePath: string; localPath: string }[]; +} + +function makeFactory( + recorded: Recorded, + execHandler: (argv: readonly string[]) => ExecResult = () => OK +) { + return makeFakeSandboxLayer({ + onCreate: (input) => { + recorded.creates.push(input); + }, + onUploadFile: (localPath, remotePath) => { + recorded.uploads.push({ localPath, remotePath }); + }, + onDownloadFile: (remotePath, localPath) => { + recorded.downloads.push({ remotePath, localPath }); + }, + execHandler: (argv, env, timeoutMs) => { + recorded.execs.push({ argv, env, timeoutMs }); + return execHandler(argv); + }, + }); +} + +function emptyRecorded(): Recorded { + return { creates: [], execs: [], uploads: [], downloads: [] }; +} + +function withFactory( + layer: Layer, + body: (factory: SandboxSessionFactory) => Effect +): Promise { + return runPromise( + gen(function* () { + const factory = yield* SandboxSession; + return yield* body(factory); + }).pipe(provide(layer)) + ); +} + +describe("terminal-bench-4 sandbox creation", () => { + it("passes gpu, env and resources into the agent sandbox and forces internet on", async () => { + const recorded = emptyRecorded(); + await withFactory(makeFactory(recorded), (sessionFactory) => + createAgentSession({ + sessionFactory, + meta: META, + tasksDir: "/tasks", + imageTag: "repo/task:abc", + imageBuildSteps: ["RUN echo hi"], + }) + ); + expect(recorded.creates).toHaveLength(1); + const input = recorded.creates[0]; + expect(input?.imageTag).toBe("repo/task:abc"); + expect(input?.imageKind).toBe("modal-image-id"); + expect(input?.imageBuildSteps).toEqual(["RUN echo hi"]); + expect(input?.cpus).toBe(8); + expect(input?.memoryMb).toBe(16384); + expect(input?.gpu).toBe("H100"); + expect(input?.env).toEqual({ HF_HOME: "/cache" }); + expect(input?.allowInternet).toBe(true); + expect(input?.timeoutSec).toBe(agentSandboxTimeoutSec(META)); + expect(input?.uploads).toEqual([ + { + localPath: "/tasks/fp8-rmsnorm-gemm/instruction.md", + remotePath: "/instruction.md", + kind: "file", + }, + ]); + }); + + it("creates the verifier sandbox from the verifier resources without a gpu", async () => { + const recorded = emptyRecorded(); + await withFactory(makeFactory(recorded), (sessionFactory) => + createVerifierSession({ + sessionFactory, + meta: META, + imageTag: "repo/task-verifier:abc", + }) + ); + const input = recorded.creates[0]; + expect(input?.imageTag).toBe("repo/task-verifier:abc"); + expect(input?.imageKind).toBe("modal-image-id"); + expect(input?.cpus).toBe(4); + expect(input?.gpu).toBeUndefined(); + expect(input?.env).toEqual({ TB_SEED: "1" }); + expect(input?.timeoutSec).toBe(verifierSandboxTimeoutSec(META)); + expect(input?.uploads).toEqual([]); + }); + + it("keeps the agent sandbox alive through collect hooks and artifact transfer", () => { + const meta: TerminalBench4SampleMeta = { + ...META, + maxAgentTimeoutSec: 28_800, + collect: [ + { command: "a", timeout_sec: 120 }, + { command: "b" }, + { command: "c", timeout_sec: 0.5 }, + ], + }; + const postProcessingSec = + 120 + + DEFAULT_COLLECT_TIMEOUT_SEC + + 0.5 + + (2 * ARTIFACT_BUNDLE_TIMEOUT_MS + ARTIFACT_TRANSFER_TIMEOUT_MS) / 1000; + const lifetime = agentSandboxTimeoutSec(meta); + expect(lifetime).toBeGreaterThanOrEqual( + meta.maxAgentTimeoutSec + postProcessingSec + ); + expect(lifetime).toBe( + Math.ceil( + meta.maxAgentTimeoutSec + postProcessingSec + SANDBOX_TIMEOUT_MARGIN_SEC + ) + ); + expect(lifetime - agentSandboxTimeoutSec(META)).toBe( + Math.ceil(28_700 + 120 + DEFAULT_COLLECT_TIMEOUT_SEC + 0.5) + ); + }); + + it("keeps the verifier sandbox alive through artifact extraction and the test run", () => { + expect(verifierSandboxTimeoutSec(META)).toBeGreaterThanOrEqual( + META.maxTestTimeoutSec + + (2 * ARTIFACT_BUNDLE_TIMEOUT_MS + ARTIFACT_TRANSFER_TIMEOUT_MS) / 1000 + ); + }); + + it("records a deviation only when the task disallows internet", () => { + expect(agentNetworkDeviation(META)).toEqual({ + agentNetworkForced: true, + taskAllowInternet: false, + }); + expect( + agentNetworkDeviation({ + ...META, + agentEnv: { ...META.agentEnv, allowInternet: true }, + }) + ).toEqual({}); + }); + + it("records a deviation only when the task image declares a non-root USER", () => { + expect(agentUserDeviation(META)).toEqual({}); + expect(agentUserDeviation({ ...META, imageUser: "nobody" })).toEqual({ + agentRunsAsRoot: true, + taskImageUser: "nobody", + }); + }); +}); + +describe("terminal-bench-4 collect hooks", () => { + it("accepts hooks for the main service", async () => { + const result = await runPromise( + either( + sandboxCollectHooks([ + { command: "a" }, + { command: "b", service: "main" }, + ]) + ) + ); + assertRight(result); + expect(result.right).toHaveLength(2); + }); + + it("rejects hooks that target a compose sidecar", async () => { + const result = await runPromise( + either(sandboxCollectHooks([{ command: "a", service: "db" }])) + ); + assertLeft(result); + expect(result.left.message).toContain('"db"'); + }); + + it("runs each hook through bash with its own timeout", async () => { + const recorded = emptyRecorded(); + await withFactory(makeFactory(recorded), (sessionFactory) => + gen(function* () { + const session = yield* sessionFactory.create({ + imageTag: "x", + timeoutSec: 1, + cpus: 1, + memoryMb: 1, + allowInternet: true, + workdir: "/", + keepAliveCommand: [], + uploads: [], + }); + yield* runCollectHooks(session, [ + { command: "echo one", timeout_sec: 5 }, + { command: "echo two" }, + ]); + return session; + }) + ); + expect(recorded.execs.map((e) => e.argv)).toEqual([ + ["bash", "-c", "echo one"], + ["bash", "-c", "echo two"], + ]); + expect(recorded.execs.map((e) => e.timeoutMs)).toEqual([5000, 300_000]); + }); +}); + +describe("terminal-bench-4 artifact bundling", () => { + it("always includes /logs/artifacts and dedupes declared sources", () => { + const cmd = artifactBundleCommand([ + "/app/out", + { source: "/app/out", exclude: ["*.tmp"] }, + { source: "/data", exclude: ["*.tmp", "cache/"] }, + ]); + expect(cmd.match(/'\/logs\/artifacts'/g)).toHaveLength(2); + expect(cmd.match(/'\/app\/out'/g)).toHaveLength(2); + expect(cmd).toContain("--exclude='*.tmp' --exclude='cache/'"); + expect(cmd).toContain("tar -cf /tmp/tb4-artifacts.tar -P"); + }); + + it("produces an empty tarball when nothing exists", () => { + expect(artifactBundleCommand([])).toContain("-T /dev/null"); + }); + + it("shell-quotes single quotes in paths", () => { + expect(artifactBundleCommand(["/it's"])).toContain(String.raw`'/it'\''s'`); + }); + + it("bundles on the agent, moves the tarball, and extracts at absolute paths on the verifier", async () => { + const recorded = emptyRecorded(); + await withFactory(makeFactory(recorded), (sessionFactory) => + gen(function* () { + const create = (tag: string) => + sessionFactory.create({ + imageTag: tag, + timeoutSec: 1, + cpus: 1, + memoryMb: 1, + allowInternet: true, + workdir: "/", + keepAliveCommand: [], + uploads: [], + }); + const agent = yield* create("agent"); + const verifier = yield* create("verifier"); + yield* transferArtifacts({ agent, verifier, artifacts: ["/app/out"] }); + return agent; + }) + ); + expect(recorded.downloads).toHaveLength(1); + expect(recorded.downloads[0]?.remotePath).toBe("/tmp/tb4-artifacts.tar"); + expect(recorded.uploads).toHaveLength(1); + expect(recorded.uploads[0]?.remotePath).toBe("/tmp/tb4-artifacts.tar"); + expect(recorded.uploads[0]?.localPath).toBe( + recorded.downloads[0]?.localPath + ); + const extract = recorded.execs.at(-1); + expect(extract?.argv[2]).toBe(artifactExtractCommand(["/app/out"])); + expect(extract?.argv[2]).toContain( + "tar -xf /tmp/tb4-artifacts.tar -P -C / && rm -f /tmp/tb4-artifacts.tar" + ); + }); + + it("clears declared directory destinations so files deleted by the agent do not survive", () => { + const root = mkdtempSync(join(tmpdir(), "tb4-extract-")); + try { + const agentDir = join(root, "agent", "pkg"); + const verifierDir = join(root, "verifier", "pkg"); + mkdirSync(agentDir, { recursive: true }); + mkdirSync(verifierDir, { recursive: true }); + writeFileSync(join(agentDir, "kept.py"), "agent"); + writeFileSync(join(agentDir, "skipped.pyc"), "agent"); + writeFileSync(join(verifierDir, "kept.py"), "image"); + writeFileSync(join(verifierDir, "deleted.py"), "image"); + writeFileSync(join(verifierDir, "skipped.pyc"), "image"); + writeFileSync(join(root, "verifier", "untouched.txt"), "image"); + const bundle = join(root, "bundle.tar"); + const artifacts = [{ source: verifierDir, exclude: ["*.pyc"] }]; + const pack = spawnSync( + "bash", + [ + "-c", + `tar -cf ${bundle} -P --exclude='*.pyc' --transform 's|${agentDir}|${verifierDir}|' ${agentDir}`, + ], + { encoding: "utf8" } + ); + expect(pack.status).toBe(0); + const extract = spawnSync( + "bash", + ["-c", artifactExtractCommand(artifacts, bundle)], + { encoding: "utf8" } + ); + expect(extract.stderr).toBe(""); + expect(extract.status).toBe(0); + expect(existsSync(join(verifierDir, "deleted.py"))).toBe(false); + expect(existsSync(join(verifierDir, "skipped.pyc"))).toBe(false); + expect(Bun.file(join(verifierDir, "kept.py")).size).toBe(5); + expect(existsSync(join(root, "verifier", "untouched.txt"))).toBe(true); + expect(existsSync(bundle)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it("replaces a directory declared with a trailing slash", () => { + const root = mkdtempSync(join(tmpdir(), "tb4-extract-")); + try { + const dir = join(root, "app"); + mkdirSync(dir, { recursive: true }); + writeFileSync(join(dir, "kept.py"), "agent"); + const artifacts = [`${dir}/`]; + const bundleCmd = artifactBundleCommand(artifacts); + expect(bundleCmd).toContain(`'${dir}'`); + expect(bundleCmd).not.toContain(`'${dir}/'`); + const bundle = join(root, "bundle.tar"); + const pack = spawnSync("bash", ["-c", `tar -cf ${bundle} -P ${dir}/`], { + encoding: "utf8", + }); + expect(pack.status).toBe(0); + writeFileSync(join(dir, "deleted.py"), "image"); + const extract = spawnSync( + "bash", + ["-c", artifactExtractCommand(artifacts, bundle)], + { encoding: "utf8" } + ); + expect(extract.stderr).toBe(""); + expect(extract.status).toBe(0); + expect(existsSync(join(dir, "deleted.py"))).toBe(false); + expect(existsSync(join(dir, "kept.py"))).toBe(true); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it("leaves a destination alone when the bundle has no directory entry for it", () => { + const root = mkdtempSync(join(tmpdir(), "tb4-extract-")); + try { + const verifierDir = join(root, "verifier", "pkg"); + mkdirSync(verifierDir, { recursive: true }); + writeFileSync(join(verifierDir, "image.py"), "image"); + const bundle = join(root, "bundle.tar"); + spawnSync("bash", ["-c", `tar -cf ${bundle} -T /dev/null`]); + const extract = spawnSync( + "bash", + ["-c", artifactExtractCommand([verifierDir], bundle)], + { encoding: "utf8" } + ); + expect(extract.status).toBe(0); + expect(existsSync(join(verifierDir, "image.py"))).toBe(true); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); + +describe("terminal-bench-4 verifier", () => { + async function runWith( + execHandler: (argv: readonly string[]) => ExecResult + ): Promise<{ + recorded: Recorded; + result: { reward: number; output: string }; + }> { + const recorded = emptyRecorded(); + const result = await withFactory( + makeFactory(recorded, execHandler), + (sessionFactory) => + gen(function* () { + const verifier = yield* sessionFactory.create({ + imageTag: "v", + timeoutSec: 1, + cpus: 1, + memoryMb: 1, + allowInternet: true, + workdir: "/", + keepAliveCommand: [], + uploads: [], + }); + return yield* runVerifier(verifier, META); + }) + ); + return { recorded, result }; + } + + it("runs /tests/test.sh with the task timeout and reads reward.txt", async () => { + const { recorded, result } = await runWith((argv) => + argv[0] === "cat" + ? { exitCode: 0, stdout: "1\n", stderr: "" } + : { exitCode: 0, stdout: "PASS", stderr: "warn" } + ); + expect(recorded.execs[0]?.argv).toEqual([ + "bash", + "-c", + "mkdir -p /logs/verifier && bash /tests/test.sh", + ]); + expect(recorded.execs[0]?.timeoutMs).toBe(55_000); + expect(recorded.execs[1]?.argv).toEqual([ + "cat", + "/logs/verifier/reward.txt", + ]); + expect(result).toEqual({ reward: 1, output: "PASS\nwarn" }); + }); + + it("scores a missing or non-passing reward as zero", async () => { + const missing = await runWith((argv) => + argv[0] === "cat" ? { exitCode: 1, stdout: "", stderr: "no file" } : OK + ); + expect(missing.result.reward).toBe(0); + const partial = await runWith((argv) => + argv[0] === "cat" ? { exitCode: 0, stdout: "0.5", stderr: "" } : OK + ); + expect(partial.result.reward).toBe(0); + }); +}); diff --git a/src/benchmarks/terminal-bench-4/session.ts b/src/benchmarks/terminal-bench-4/session.ts new file mode 100644 index 0000000..d899880 --- /dev/null +++ b/src/benchmarks/terminal-bench-4/session.ts @@ -0,0 +1,271 @@ +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import type { Effect } from "effect/Effect"; +import { fail, gen, succeed } from "effect/Effect"; + +import { SolverError } from "../../harness/core"; +import { parseReward } from "../harbor/reward"; +import type { + SandboxSessionFactory, + SandboxSessionInstance, +} from "../harbor/sandbox"; +import { REMOTE_VERIFIER_SCRIPT, SANDBOX_IMAGE_KINDS } from "../harbor/sandbox"; +import type { TerminalBench4SampleMeta } from "./dataset"; +import type { Artifact, CollectHook } from "./schema"; + +export const CONTAINER_WORKDIR = "/app" as const; + +export const REMOTE_INSTRUCTION = "/instruction.md" as const; + +export const REMOTE_REWARD_PATH = "/logs/verifier/reward.txt" as const; + +export const REMOTE_ARTIFACTS_DIR = "/logs/artifacts" as const; + +export const KEEP_ALIVE_COMMAND = ["sleep", "infinity"] as const; + +const REMOTE_ARTIFACT_BUNDLE = "/tmp/tb4-artifacts.tar" as const; + +export const SANDBOX_TIMEOUT_MARGIN_SEC = 300; + +const VERIFIER_TIMEOUT_MARGIN_MS = 5000; + +const REWARD_READ_TIMEOUT_MS = 10_000; + +export const ARTIFACT_BUNDLE_TIMEOUT_MS = 600_000; + +export const ARTIFACT_TRANSFER_TIMEOUT_MS = 1_800_000; + +export const DEFAULT_COLLECT_TIMEOUT_SEC = 300; + +const ARTIFACT_PHASE_SEC = + (2 * ARTIFACT_BUNDLE_TIMEOUT_MS + ARTIFACT_TRANSFER_TIMEOUT_MS) / 1000; + +export function agentSandboxTimeoutSec(meta: TerminalBench4SampleMeta): number { + const collectSec = meta.collect.reduce( + (total, hook) => total + (hook.timeout_sec ?? DEFAULT_COLLECT_TIMEOUT_SEC), + 0 + ); + return Math.ceil( + meta.maxAgentTimeoutSec + + collectSec + + ARTIFACT_PHASE_SEC + + SANDBOX_TIMEOUT_MARGIN_SEC + ); +} + +export function verifierSandboxTimeoutSec( + meta: TerminalBench4SampleMeta +): number { + return Math.ceil( + meta.maxTestTimeoutSec + ARTIFACT_PHASE_SEC + SANDBOX_TIMEOUT_MARGIN_SEC + ); +} + +export function agentNetworkDeviation( + meta: TerminalBench4SampleMeta +): Readonly> { + return meta.agentEnv.allowInternet + ? {} + : { agentNetworkForced: true, taskAllowInternet: false }; +} + +export function agentUserDeviation( + meta: TerminalBench4SampleMeta +): Readonly> { + return meta.imageUser === undefined + ? {} + : { agentRunsAsRoot: true, taskImageUser: meta.imageUser }; +} + +export function createAgentSession(input: { + readonly sessionFactory: SandboxSessionFactory; + readonly meta: TerminalBench4SampleMeta; + readonly tasksDir: string; + readonly imageTag: string; + readonly imageBuildSteps: readonly string[]; +}): Effect { + const { sessionFactory, meta, tasksDir, imageTag, imageBuildSteps } = input; + return sessionFactory.create({ + imageTag, + imageKind: SANDBOX_IMAGE_KINDS.ModalImageId, + imageBuildSteps, + timeoutSec: agentSandboxTimeoutSec(meta), + ...meta.agentEnv, + allowInternet: true, + workdir: CONTAINER_WORKDIR, + keepAliveCommand: KEEP_ALIVE_COMMAND, + uploads: [ + { + localPath: join(tasksDir, meta.taskId, "instruction.md"), + remotePath: REMOTE_INSTRUCTION, + kind: "file", + }, + ], + }); +} + +export function createVerifierSession(input: { + readonly sessionFactory: SandboxSessionFactory; + readonly meta: TerminalBench4SampleMeta; + readonly imageTag: string; +}): Effect { + const { sessionFactory, meta, imageTag } = input; + return sessionFactory.create({ + imageTag, + imageKind: SANDBOX_IMAGE_KINDS.ModalImageId, + timeoutSec: verifierSandboxTimeoutSec(meta), + ...meta.verifierEnv, + workdir: CONTAINER_WORKDIR, + keepAliveCommand: KEEP_ALIVE_COMMAND, + uploads: [], + }); +} + +export function sandboxCollectHooks( + hooks: readonly CollectHook[] +): Effect { + const sidecar = hooks.find( + (h) => h.service !== undefined && h.service !== "main" + ); + if (sidecar !== undefined) { + return fail( + new SolverError({ + message: `collect hook targets compose service "${sidecar.service}", which is unsupported`, + }) + ); + } + return succeed(hooks); +} + +export function runCollectHooks( + session: SandboxSessionInstance, + hooks: readonly CollectHook[] +): Effect { + return gen(function* () { + for (const hook of hooks) { + yield* session.exec( + ["bash", "-c", hook.command], + {}, + Math.round((hook.timeout_sec ?? DEFAULT_COLLECT_TIMEOUT_SEC) * 1000) + ); + } + }); +} + +function artifactSource(artifact: Artifact): string { + return typeof artifact === "string" ? artifact : artifact.source; +} + +function artifactExcludes(artifact: Artifact): readonly string[] { + return typeof artifact === "string" ? [] : artifact.exclude; +} + +function shellQuote(value: string): string { + return `'${value.replaceAll("'", String.raw`'\''`)}'`; +} + +function normalizeArtifactPath(path: string): string { + const trimmed = path.replace(/\/+$/, ""); + return trimmed === "" ? "/" : trimmed; +} + +function artifactSources(artifacts: readonly Artifact[]): readonly string[] { + return [REMOTE_ARTIFACTS_DIR, ...artifacts.map(artifactSource)] + .map(normalizeArtifactPath) + .filter((s, i, all) => all.indexOf(s) === i); +} + +export function artifactBundleCommand(artifacts: readonly Artifact[]): string { + const sources = artifactSources(artifacts); + const excludes = artifacts + .flatMap(artifactExcludes) + .filter((s, i, all) => all.indexOf(s) === i) + .map((pattern) => `--exclude=${shellQuote(pattern)}`); + const existing = sources + .map( + (s) => + `if [ -e ${shellQuote(s)} ]; then printf '%s\\n' ${shellQuote(s)}; fi` + ) + .join("; "); + return [ + `rm -f ${REMOTE_ARTIFACT_BUNDLE}`, + `{ ${existing}; } > /tmp/tb4-artifact-list.txt`, + `if [ -s /tmp/tb4-artifact-list.txt ]; then tar -cf ${REMOTE_ARTIFACT_BUNDLE} -P ${excludes.join(" ")} -T /tmp/tb4-artifact-list.txt; else tar -cf ${REMOTE_ARTIFACT_BUNDLE} -T /dev/null; fi`, + ].join(" && "); +} + +export function artifactExtractCommand( + artifacts: readonly Artifact[], + bundle: string = REMOTE_ARTIFACT_BUNDLE +): string { + const clearDirs = artifactSources(artifacts).map( + (s) => + `if tar -tf ${bundle} -P | grep -qx ${shellQuote(`${s}/`)}; then rm -rf ${shellQuote(s)}; fi` + ); + return [...clearDirs, `tar -xf ${bundle} -P -C /`, `rm -f ${bundle}`].join( + " && " + ); +} + +export function transferArtifacts(input: { + readonly agent: SandboxSessionInstance; + readonly verifier: SandboxSessionInstance; + readonly artifacts: readonly Artifact[]; +}): Effect { + const { agent, verifier, artifacts } = input; + return gen(function* () { + yield* agent.exec( + ["bash", "-c", artifactBundleCommand(artifacts)], + {}, + ARTIFACT_BUNDLE_TIMEOUT_MS + ); + const staging = mkdtempSync(join(tmpdir(), "tb4-artifacts-")); + const localBundle = join(staging, "artifacts.tar"); + try { + yield* agent.downloadFile(REMOTE_ARTIFACT_BUNDLE, localBundle); + yield* verifier.uploadFile(localBundle, REMOTE_ARTIFACT_BUNDLE); + } finally { + rmSync(staging, { recursive: true, force: true }); + } + yield* verifier.exec( + ["bash", "-c", artifactExtractCommand(artifacts)], + {}, + ARTIFACT_BUNDLE_TIMEOUT_MS + ); + }); +} + +export interface TerminalBench4VerifierResult { + readonly reward: number; + readonly output: string; +} + +export function runVerifier( + verifier: SandboxSessionInstance, + meta: TerminalBench4SampleMeta +): Effect { + const verifierTimeoutMs = + Math.round(meta.maxTestTimeoutSec * 1000) + VERIFIER_TIMEOUT_MARGIN_MS; + return gen(function* () { + const run = yield* verifier.exec( + [ + "bash", + "-c", + `mkdir -p /logs/verifier && bash ${REMOTE_VERIFIER_SCRIPT}`, + ], + {}, + verifierTimeoutMs + ); + const rewardRead = yield* verifier.exec( + ["cat", REMOTE_REWARD_PATH], + {}, + REWARD_READ_TIMEOUT_MS + ); + return { + reward: parseReward(rewardRead.stdout), + output: `${run.stdout}\n${run.stderr}`.trim(), + }; + }); +} diff --git a/src/benchmarks/terminal-bench-4/solver.ts b/src/benchmarks/terminal-bench-4/solver.ts new file mode 100644 index 0000000..65b579b --- /dev/null +++ b/src/benchmarks/terminal-bench-4/solver.ts @@ -0,0 +1,136 @@ +import { gen, tryPromise } from "effect/Effect"; + +import type { ModelMessage, ModelUsage } from "../../harness/core"; +import { MessageRole, SolverError } from "../../harness/core"; +import type { SolverService } from "../../harness/solver"; +import type { OriHarnessDef } from "../agent-cli/harness"; +import type { AgentCliOpts } from "../agent-cli/runner"; +import { + agentCliMetadata, + agentImageBuildSteps, + runAgentCli, +} from "../agent-cli/runner"; +import type { SandboxSessionFactory } from "../harbor/sandbox"; +import { readTerminalBench4Meta } from "./dataset"; +import type { TerminalBench4ImageMap } from "./images"; +import { TERMINAL_BENCH_4_IMAGES, taskImages } from "./images"; +import { + agentNetworkDeviation, + agentUserDeviation, + createAgentSession, + createVerifierSession, + REMOTE_INSTRUCTION, + runCollectHooks, + runVerifier, + sandboxCollectHooks, + transferArtifacts, +} from "./session"; +import { ensureTasksCheckedOut, tasksDir } from "./tasks-source"; + +const AGENT_TIMEOUT_MARGIN_MS = 30_000; + +const ZERO_USAGE: ModelUsage = { + inputTokens: 0, + outputTokens: 0, + totalTokens: 0, + reasoningTokens: 0, + totalCost: 0, +}; + +export type TerminalBench4SolverOpts = AgentCliOpts; + +export function terminalBench4Solver( + sessionFactory: SandboxSessionFactory, + opts: TerminalBench4SolverOpts, + harness: OriHarnessDef, + imageMap: TerminalBench4ImageMap = TERMINAL_BENCH_4_IMAGES +): SolverService { + return (state) => + gen(function* () { + const meta = readTerminalBench4Meta(state.sample.metadata); + if (meta === undefined) { + return yield* new SolverError({ + message: `terminal-bench-4 solver received a sample without terminal-bench-4 metadata (id=${state.sample.id})`, + }); + } + const images = taskImages(imageMap, meta.taskId); + if (images === undefined) { + return yield* new SolverError({ + message: `terminal-bench-4 task "${meta.taskId}" has no Modal images in image-ids.json; run scripts/build-terminal-bench-4-images.py`, + }); + } + const collectHooks = yield* sandboxCollectHooks(meta.collect); + const root = yield* tryPromise({ + try: () => ensureTasksCheckedOut(), + catch: (e: unknown) => + new SolverError({ + message: `Failed to check out terminal-bench-4 tasks: ${String(e)}`, + }), + }); + const agent = yield* createAgentSession({ + sessionFactory, + meta, + tasksDir: tasksDir(root), + imageTag: images.agent, + imageBuildSteps: agentImageBuildSteps(harness, opts), + }); + try { + const run = yield* runAgentCli({ + session: agent, + harness, + opts, + instructionPath: REMOTE_INSTRUCTION, + timeoutMs: meta.maxAgentTimeoutSec * 1000 + AGENT_TIMEOUT_MARGIN_MS, + }); + yield* runCollectHooks(agent, collectHooks); + const verifier = yield* createVerifierSession({ + sessionFactory, + meta, + imageTag: images.verifier, + }); + try { + yield* transferArtifacts({ + agent, + verifier, + artifacts: meta.artifacts, + }); + yield* agent.destroy(); + const testResult = yield* runVerifier(verifier, meta); + const testOutput = run.failureDetail + ? `${run.failureDetail}\n\n${testResult.output}` + : testResult.output; + const completion = run.finalText ?? run.rawStream; + const messages: ModelMessage[] = [ + { role: MessageRole.User, content: state.sample.input }, + ...run.assistantMessages, + ]; + return { + sample: { + ...state.sample, + metadata: { + ...state.sample.metadata, + reward: testResult.reward, + testOutput, + ...agentCliMetadata(harness.id, run), + ...agentNetworkDeviation(meta), + ...agentUserDeviation(meta), + }, + }, + messages, + responseItems: run.responseItems, + output: { + completion, + message: { role: MessageRole.Assistant, content: completion }, + usage: run.usage ?? ZERO_USAGE, + generationTimeMs: run.generationTimeMs ?? 0, + }, + completed: true, + }; + } finally { + yield* verifier.destroy(); + } + } finally { + yield* agent.destroy(); + } + }); +} diff --git a/src/benchmarks/terminal-bench-4/tasks-source.ts b/src/benchmarks/terminal-bench-4/tasks-source.ts new file mode 100644 index 0000000..6450cbf --- /dev/null +++ b/src/benchmarks/terminal-bench-4/tasks-source.ts @@ -0,0 +1,31 @@ +import { join } from "node:path"; + +import { makeTasksSource } from "../harbor/tasks-source"; + +export const TERMINAL_BENCH_4_SOURCE_REPO = + "https://github.com/harbor-framework/terminal-bench.git" as const; + +export const TERMINAL_BENCH_4_SOURCE_COMMIT = + "452bf305c6daa62fc59061d22133a7cbc7c1572e" as const; + +export const TERMINAL_BENCH_4_TASKS_SUBDIR = "tasks" as const; + +const source = makeTasksSource({ + label: "terminal-bench-4", + repoUrl: TERMINAL_BENCH_4_SOURCE_REPO, + commit: TERMINAL_BENCH_4_SOURCE_COMMIT, + tasksSubdir: TERMINAL_BENCH_4_TASKS_SUBDIR, + envVar: "BENCH_TERMINAL_BENCH_4_TASKS_DIR", + tmpPrefix: "terminal-bench-4-tasks-", +}); + +export const { + ensureTasksCheckedOut, + ensureTasksCheckedOutEffect, + seedTasksRoot, + resetCheckoutCache, +} = source; + +export function tasksDir(root: string): string { + return join(root, TERMINAL_BENCH_4_TASKS_SUBDIR); +} diff --git a/src/benchmarks/terminal-bench/ori-solver.test.ts b/src/benchmarks/terminal-bench/ori-solver.test.ts index 969f237..62f5d9e 100644 --- a/src/benchmarks/terminal-bench/ori-solver.test.ts +++ b/src/benchmarks/terminal-bench/ori-solver.test.ts @@ -633,6 +633,24 @@ describe("terminal-bench ori solver", () => { expect(steps.at(-1)).toBe("RUN claude --version"); }); + it("installs system packages through apt-get, dnf or apk so non-Debian task images build", () => { + for (const harness of Object.values(ORI_HARNESSES)) { + const step = harness + .imageBuildSteps({ agentPackage: harness.defaultPackage }) + .find((s) => s.startsWith("RUN if command -v apt-get")); + expect(step).toBeDefined(); + expect(step).toContain("apt-get install -y --no-install-recommends"); + expect(step).toContain( + "elif command -v dnf >/dev/null; then dnf install -y" + ); + expect(step).toContain( + "elif command -v apk >/dev/null; then apk add --no-cache bash " + ); + expect(step).toContain("exit 1; fi"); + expect(step).not.toMatch(/^RUN apt-get/); + } + }); + it("exposes every packaged harness and runtime helper", () => { const dockerfile = ORI_HARNESSES.claude .imageBuildSteps({ diff --git a/src/cli/index.ts b/src/cli/index.ts index 922f378..e3ba0bb 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -471,6 +471,16 @@ export function buildBenchmarkConfig(opts: { reasoningEffort, }); } + case "terminal_bench_4": { + return buildSchemaValidatedConfig({ + benchmarkId: "terminal_bench_4", + model: requireModel("terminal_bench_4", model), + endpointId, + panelConfig, + costTier, + reasoningEffort, + }); + } case "draco": { const panel = parseSchema(DracoPanelConfigSchema, panelConfig); if (Either.isLeft(panel)) {