feat: add Terminal-Bench 4.0 as a separate Modal-backed benchmark

scripts/build-terminal-bench-4-images.py

@@ -0,0 +1,148 @@
+"""Build Terminal-Bench 4.0 task images into Modal's image store.
+
+Usage:
+  python3 scripts/build-terminal-bench-4-images.py --tasks-dir /path/to/terminal-bench/tasks \
+      [--task NAME ...] [--out src/benchmarks/terminal-bench-4/image-ids.json] [--dry-run]
+
+Requires MODAL_TOKEN_ID and MODAL_TOKEN_SECRET (or a Modal CLI profile). For each
+runnable (non docker-compose) task this builds environment/Dockerfile and
+tests/Dockerfile with their directories as build context, then records the
+resulting Modal image IDs keyed by task id. The output JSON is what the harness
+reads at run time, so re-run this whenever TERMINAL_BENCH_4_SOURCE_COMMIT changes.
+
+MODAL_IMAGE_BUILDER_VERSION defaults to 2025.06. Older builder versions pip-install
+Modal's client dependencies into the image, which would alter task environments.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import re
+import subprocess
+import sys
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+
+os.environ.setdefault("MODAL_IMAGE_BUILDER_VERSION", "2025.06")
+
+import modal  # noqa: E402
+
+APP_NAME = "terminal-bench-4-images"
+COMPOSE_FILE = "environment/docker-compose.yaml"
+DEFAULT_OUT = Path("src/benchmarks/terminal-bench-4/image-ids.json")
+SOURCE_RE = re.compile(r'TERMINAL_BENCH_4_SOURCE_COMMIT\s*=\s*"([0-9a-f]{40})"')
+
+
+def pinned_commit() -> str:
+    text = Path("src/benchmarks/terminal-bench-4/tasks-source.ts").read_text()
+    match = SOURCE_RE.search(text)
+    if match is None:
+        sys.exit("could not read TERMINAL_BENCH_4_SOURCE_COMMIT from tasks-source.ts")
+    return match.group(1)
+
+
+def checkout_commit(tasks_dir: Path) -> str:
+    out = subprocess.run(
+        ["git", "-C", str(tasks_dir), "rev-parse", "HEAD"],
+        check=True,
+        capture_output=True,
+        text=True,
+    )
+    return out.stdout.strip()
+
+
+def runnable_tasks(tasks_dir: Path) -> list[str]:
+    return sorted(
+        p.name
+        for p in tasks_dir.iterdir()
+        if p.is_dir()
+        and not p.name.startswith(".")
+        and (p / "task.toml").is_file()
+        and not (p / COMPOSE_FILE).is_file()
+    )
+
+
+def load_existing(out: Path, commit: str) -> dict[str, dict[str, str]]:
+    if not out.is_file():
+        return {}
+    data = json.loads(out.read_text())
+    if data.get("sourceCommit") != commit:
+        return {}
+    return dict(data.get("images", {}))
+
+
+def write_out(out: Path, commit: str, images: dict[str, dict[str, str]]) -> None:
+    payload = {
+        "sourceCommit": commit,
+        "images": {k: images[k] for k in sorted(images)},
+    }
+    out.write_text(json.dumps(payload, indent=2) + "\n")
+
+
+def build_task(app: modal.App, tasks_dir: Path, task: str) -> dict[str, str]:
+    task_dir = tasks_dir / task
+    agent = modal.Image.from_dockerfile(
+        task_dir / "environment" / "Dockerfile",
+        context_dir=task_dir / "environment",
+    )
+    verifier = modal.Image.from_dockerfile(
+        task_dir / "tests" / "Dockerfile",
+        context_dir=task_dir / "tests",
+    )
+    agent = agent.build(app)
+    verifier = verifier.build(app)
+    return {"agent": agent.object_id, "verifier": verifier.object_id}
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--tasks-dir", type=Path, required=True)
+    parser.add_argument("--task", action="append", default=[])
+    parser.add_argument("--out", type=Path, default=DEFAULT_OUT)
+    parser.add_argument("--concurrency", type=int, default=4)
+    parser.add_argument("--dry-run", action="store_true")
+    args = parser.parse_args()
+
+    tasks_dir: Path = args.tasks_dir.resolve()
+    commit = pinned_commit()
+    actual = checkout_commit(tasks_dir.parent)
+    if actual != commit:
+        sys.exit(f"tasks dir is at {actual}, expected pinned {commit}")
+
+    runnable = runnable_tasks(tasks_dir)
+    selected = args.task or runnable
+    unknown = sorted(set(selected) - set(runnable))
+    if unknown:
+        sys.exit(f"unknown or docker-compose tasks: {', '.join(unknown)}")
+
+    if args.dry_run:
+        for task in selected:
+            print(task)
+        return 0
+
+    images = load_existing(args.out, commit)
+    failures: dict[str, str] = {}
+    app = modal.App.lookup(APP_NAME, create_if_missing=True)
+
+    def run(task: str) -> None:
+        try:
+            images[task] = build_task(app, tasks_dir, task)
+            print(f"built {task}: {images[task]}", flush=True)
+        except Exception as error:  # noqa: BLE001
+            failures[task] = str(error)
+            print(f"FAILED {task}: {error}", file=sys.stderr, flush=True)
+
+    with modal.enable_output(), ThreadPoolExecutor(max_workers=args.concurrency) as pool:
+        list(pool.map(run, selected))
+
+    write_out(args.out, commit, images)
+    if failures:
+        print(f"{len(failures)} task(s) failed: {', '.join(sorted(failures))}", file=sys.stderr)
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

src/benchmarks/agent-cli/harness.ts

@@ -89,6 +89,18 @@ export interface OriHarnessDef {
   readonly parseRun: (stdout: string) => OriAgentRun;
 }
 
+export function installSystemPackagesStep(packages: readonly string[]): string {
+  const list = packages.join(" ");
+  return [
+    "RUN if command -v apt-get >/dev/null",
+    `then apt-get update && apt-get install -y --no-install-recommends ${list}`,
+    `elif command -v dnf >/dev/null; then dnf install -y --setopt=install_weak_deps=False --allowerasing ${list}`,
+    `elif command -v apk >/dev/null; then apk add --no-cache bash ${list}`,
+    'else echo "no supported package manager (apt-get, dnf, apk)" >&2 && exit 1',
+    "fi",
+  ].join("; ");
+}
+
 function buildImageSteps(opts: {
   agentPackage: string;
   binaryName: string;
@@ -100,7 +112,7 @@ function buildImageSteps(opts: {
     `npm install -g ${JSON.stringify(opts.agentPackage)}`;
   const nvmScript = "/tmp/nvm-install.sh";
   return [
-    "RUN apt-get update && apt-get install -y --no-install-recommends curl ca-certificates git",
+    installSystemPackagesStep(["curl", "ca-certificates", "git"]),
     "ENV NVM_DIR=/root/.nvm",
     `RUN ${verifiedDownload(NVM_INSTALL_URL, NVM_INSTALL_SHA256, nvmScript)} && bash ${nvmScript} && rm -f ${nvmScript}`,
     `RUN . /root/.nvm/nvm.sh && nvm install ${NODE_VERSION} && ${installCommand} && ln -sf $(which ${opts.binaryName}) /usr/local/bin/${opts.binaryName} && ln -sf $(which node) /usr/local/bin/node && ln -sf $(which npm) /usr/local/bin/npm`,
@@ -117,7 +129,7 @@ function buildAgentImageSteps(opts: {
   if (opts.agentPackage === opts.defaultPackage) {
     const archivePath = "/tmp/agent-runtime.tar.zst";
     return [
-      "RUN apt-get update && apt-get install -y --no-install-recommends curl ca-certificates git zstd",
+      installSystemPackagesStep(["curl", "ca-certificates", "git", "zstd"]),
       `RUN ${verifiedDownload(DEFAULT_AGENT_RUNTIME_URL, DEFAULT_AGENT_RUNTIME_SHA256, archivePath)} && zstd -dc ${archivePath} | tar -x -C / && rm -f ${archivePath}`,
       'ENV PATH="/root/.local/bin:$PATH"',
       "RUN ln -sf /opt/agent-runtime/app/node_modules/.bin/claude /usr/local/bin/claude && ln -sf /opt/agent-runtime/app/node_modules/.bin/pi /usr/local/bin/pi && ln -sf /opt/agent-runtime/app/node_modules/.bin/prime-agent /usr/local/bin/prime-agent && ln -sf /opt/agent-runtime/node/bin/node /usr/local/bin/node && ln -sf /opt/agent-runtime/node/bin/npm /usr/local/bin/npm && ln -sf /opt/agent-runtime/node/bin/npx /usr/local/bin/npx",
@@ -137,7 +149,7 @@ function buildOmpImageSteps(agentPackage: string): string[] {
   assertValidAgentPackage(agentPackage);
   const bunZip = "/tmp/bun.zip";
   return [
-    "RUN apt-get update && apt-get install -y --no-install-recommends curl ca-certificates git unzip",
+    installSystemPackagesStep(["curl", "ca-certificates", "git", "unzip"]),
     "ENV BUN_INSTALL=/root/.bun",
     `RUN ${verifiedDownload(BUN_RELEASE_URL, BUN_RELEASE_SHA256, bunZip)} && unzip -q ${bunZip} -d /tmp && install -m 0755 /tmp/bun-linux-x64/bun /usr/local/bin/bun && rm -rf ${bunZip} /tmp/bun-linux-x64`,
     `RUN bun install -g ${JSON.stringify(agentPackage)} && ln -sf /root/.bun/bin/omp /usr/local/bin/omp`,

src/benchmarks/benchmark-config.ts

@@ -168,6 +168,16 @@ export const TerminalBenchConfigSchema = z.object({
 
 export type TerminalBenchConfig = z.infer<typeof TerminalBenchConfigSchema>;
 
+export const TerminalBench4OptionsSchema = TerminalBenchOptionsSchema;
+
+export const TerminalBench4ConfigSchema = z.object({
+  benchmarkId: z.literal("terminal_bench_4"),
+  ...ModelBenchmarkBaseSchema.shape,
+  ...TerminalBench4OptionsSchema.shape,
+});
+
+export type TerminalBench4Config = z.infer<typeof TerminalBench4ConfigSchema>;
+
 export const DracoBenchmarkConfigSchema = z.object({
   benchmarkId: z.literal("draco"),
   panelConfig: DracoPanelConfigSchema,
@@ -330,6 +340,7 @@ export const NativeBenchmarkRunConfigSchema = z.discriminatedUnion(
     Tau3BenchBankingConfigSchema,
     MmmuProVisionBenchmarkConfigSchema,
     TerminalBenchConfigSchema,
+    TerminalBench4ConfigSchema,
     DracoBenchmarkConfigSchema,
     IfStructBenchmarkConfigSchema,
     SweAtlasQaConfigSchema,
@@ -365,6 +376,7 @@ export const BENCHMARK_OPTIONS_SCHEMAS = {
   tau3_bench_banking: Tau3BenchBankingOptionsSchema,
   mmmu_pro_vision: MmmuProVisionOptionsSchema,
   terminal_bench: TerminalBenchOptionsSchema,
+  terminal_bench_4: TerminalBench4OptionsSchema,
   ifstruct: IfStructOptionsSchema,
   swe_atlas_qa: SweAtlasOptionsSchema,
   swe_atlas_tw: SweAtlasOptionsSchema,

src/benchmarks/benchmark-meta.ts

@@ -39,6 +39,11 @@ export const TERMINAL_BENCH_META = {
   defaultEpochs: 1,
 } as const satisfies BenchmarkMeta;
 
+export const TERMINAL_BENCH_4_META = {
+  id: "terminal_bench_4",
+  defaultEpochs: 1,
+} as const satisfies BenchmarkMeta;
+
 export const DRACO_META = {
   id: "draco",
   defaultEpochs: 1,
@@ -106,6 +111,7 @@ const BENCHMARK_META: Readonly<Record<string, BenchmarkMeta>> = {
   [TAU_BENCH_AIRLINE_META.id]: TAU_BENCH_AIRLINE_META,
   [TAU3_BENCH_BANKING_META.id]: TAU3_BENCH_BANKING_META,
   [TERMINAL_BENCH_META.id]: TERMINAL_BENCH_META,
+  [TERMINAL_BENCH_4_META.id]: TERMINAL_BENCH_4_META,
   [DRACO_META.id]: DRACO_META,
   [IFSTRUCT_META.id]: IFSTRUCT_META,
   [SWE_ATLAS_QA_META.id]: SWE_ATLAS_QA_META,

src/benchmarks/harbor/modal-sandbox.ts

@@ -16,7 +16,12 @@ import type {
   SandboxSessionInstance,
   UploadSpec,
 } from "./sandbox";
-import { SandboxSession, makeSessionInstance, toSolverError } from "./sandbox";
+import {
+  SANDBOX_IMAGE_KINDS,
+  SandboxSession,
+  makeSessionInstance,
+  toSolverError,
+} from "./sandbox";
 
 export interface ModalSandboxConfig {
   readonly appName: string;
@@ -64,7 +69,17 @@ export function makeModalSandboxLayer(
         try: () => getApp(),
         catch: (e) => toSolverError("Failed to resolve Modal app", e),
       });
-      const baseImage = client.images.fromRegistry(input.imageTag);
+      const baseImage =
+        input.imageKind === SANDBOX_IMAGE_KINDS.ModalImageId
+          ? yield* tryPromise({
+              try: () => client.images.fromId(input.imageTag),
+              catch: (e: unknown) =>
+                toSolverError(
+                  `Failed to resolve Modal image ${input.imageTag}`,
+                  e
+                ),
+            })
+          : client.images.fromRegistry(input.imageTag);
       const image =
         input.imageBuildSteps !== undefined && input.imageBuildSteps.length > 0
           ? baseImage.dockerfileCommands([...input.imageBuildSteps])
@@ -76,14 +91,20 @@ export function makeModalSandboxLayer(
       });
       const sandbox = yield* tryPromise({
         try: () =>
-          client.sandboxes.create(app, builtImage, {
-            timeoutMs: input.timeoutSec * 1000,
-            cpu: input.cpus,
-            memoryMiB: input.memoryMb,
-            blockNetwork: !input.allowInternet,
-            workdir: input.workdir,
-            command: [...input.keepAliveCommand],
-          }),
+          client.sandboxes.create(
+            app,
+            builtImage,
+            definedValues({
+              timeoutMs: input.timeoutSec * 1000,
+              cpu: input.cpus,
+              memoryMiB: input.memoryMb,
+              gpu: input.gpu,
+              env: input.env === undefined ? undefined : { ...input.env },
+              blockNetwork: !input.allowInternet,
+              workdir: input.workdir,
+              command: [...input.keepAliveCommand],
+            })
+          ),
         catch: (e) => toSolverError("Failed to create Modal sandbox", e),
       });
       let handedOff = false;

src/benchmarks/harbor/sandbox.ts

@@ -46,12 +46,23 @@ export interface SandboxSessionInstance {
   readonly destroy: () => Effect<void, SolverError>;
 }
 
+export const SANDBOX_IMAGE_KINDS = {
+  Registry: "registry",
+  ModalImageId: "modal-image-id",
+} as const;
+
+export type SandboxImageKind =
+  (typeof SANDBOX_IMAGE_KINDS)[keyof typeof SANDBOX_IMAGE_KINDS];
+
 export interface CreateSessionInput {
   readonly imageTag: string;
+  readonly imageKind?: SandboxImageKind;
   readonly imageBuildSteps?: readonly string[];
   readonly timeoutSec: number;
   readonly cpus: number;
   readonly memoryMb: number;
+  readonly gpu?: string;
+  readonly env?: Readonly<Record<string, string>>;
   readonly allowInternet: boolean;
   readonly workdir: string;
   readonly keepAliveCommand: readonly string[];

src/benchmarks/registry.ts

@@ -15,6 +15,7 @@ import {
 } from "./swe-atlas/benchmark";
 import { TAU_BENCH_AIRLINE_BENCHMARK } from "./tau-bench-airline/benchmark";
 import { TAU3_BENCH_BANKING_BENCHMARK } from "./tau3-bench-banking/benchmark";
+import { TERMINAL_BENCH_4_BENCHMARK } from "./terminal-bench-4/benchmark";
 import { TERMINAL_BENCH_BENCHMARK } from "./terminal-bench/benchmark";
 import type { Benchmark } from "./types";
 import { VGI_BENCH_BENCHMARK } from "./vgi-bench/benchmark";
@@ -27,6 +28,7 @@ const BENCHMARKS: Record<string, Benchmark> = {
   [TAU3_BENCH_BANKING_BENCHMARK.id]: TAU3_BENCH_BANKING_BENCHMARK,
   [MMMU_PRO_VISION_BENCHMARK.id]: MMMU_PRO_VISION_BENCHMARK,
   [TERMINAL_BENCH_BENCHMARK.id]: TERMINAL_BENCH_BENCHMARK,
+  [TERMINAL_BENCH_4_BENCHMARK.id]: TERMINAL_BENCH_4_BENCHMARK,
   [DRACO_BENCHMARK.id]: DRACO_BENCHMARK,
   [IFSTRUCT_BENCHMARK.id]: IFSTRUCT_BENCHMARK,
   [SWE_ATLAS_QA_BENCHMARK.id]: SWE_ATLAS_QA_BENCHMARK,