From b37280606fd1d4da675ed569560d1b9b0297b00d Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 3 Sep 2026 20:07:25 +0000 Subject: [PATCH 1/6] feat: add Terminal-Bench 4.0 as a separate Modal-backed benchmark Co-Authored-By: Abhinav Pola --- scripts/build-terminal-bench-4-images.ts | 196 +++++++++++ src/benchmarks/benchmark-config.ts | 15 + src/benchmarks/benchmark-meta.ts | 6 + src/benchmarks/harbor/modal-sandbox.ts | 22 +- src/benchmarks/harbor/sandbox.ts | 2 + src/benchmarks/registry.ts | 2 + src/benchmarks/terminal-bench-4/README.md | 56 +++ .../terminal-bench-4/benchmark.test.ts | 58 ++++ src/benchmarks/terminal-bench-4/benchmark.ts | 89 +++++ .../terminal-bench-4/dataset.test.ts | 296 ++++++++++++++++ src/benchmarks/terminal-bench-4/dataset.ts | 259 ++++++++++++++ src/benchmarks/terminal-bench-4/images.ts | 20 ++ src/benchmarks/terminal-bench-4/schema.ts | 67 ++++ .../terminal-bench-4/session.test.ts | 326 ++++++++++++++++++ src/benchmarks/terminal-bench-4/session.ts | 218 ++++++++++++ src/benchmarks/terminal-bench-4/solver.ts | 129 +++++++ .../terminal-bench-4/tasks-source.ts | 31 ++ src/cli/index.ts | 10 + 18 files changed, 1794 insertions(+), 8 deletions(-) create mode 100644 scripts/build-terminal-bench-4-images.ts create mode 100644 src/benchmarks/terminal-bench-4/README.md create mode 100644 src/benchmarks/terminal-bench-4/benchmark.test.ts create mode 100644 src/benchmarks/terminal-bench-4/benchmark.ts create mode 100644 src/benchmarks/terminal-bench-4/dataset.test.ts create mode 100644 src/benchmarks/terminal-bench-4/dataset.ts create mode 100644 src/benchmarks/terminal-bench-4/images.ts create mode 100644 src/benchmarks/terminal-bench-4/schema.ts create mode 100644 src/benchmarks/terminal-bench-4/session.test.ts create mode 100644 src/benchmarks/terminal-bench-4/session.ts create mode 100644 src/benchmarks/terminal-bench-4/solver.ts create mode 100644 src/benchmarks/terminal-bench-4/tasks-source.ts diff --git a/scripts/build-terminal-bench-4-images.ts b/scripts/build-terminal-bench-4-images.ts new file mode 100644 index 0000000..d3dc04b --- /dev/null +++ b/scripts/build-terminal-bench-4-images.ts @@ -0,0 +1,196 @@ +import { existsSync } from "node:fs"; +import { join } from "node:path"; + +import { listTaskIds } from "../src/benchmarks/terminal-bench-4/dataset"; +import { + DEFAULT_TERMINAL_BENCH_4_IMAGE_REPO, + imageTags, +} from "../src/benchmarks/terminal-bench-4/images"; +import { + ensureTasksCheckedOut, + tasksDir, +} from "../src/benchmarks/terminal-bench-4/tasks-source"; + +interface BuildOptions { + readonly repo: string; + readonly tasks: readonly string[]; + readonly dryRun: boolean; + readonly concurrency: number; +} + +interface ImageBuild { + readonly tag: string; + readonly contextDir: string; + readonly dockerfile: string; +} + +function requireValue(flag: string, value: string | undefined): string { + if (value === undefined) { + throw new Error(`${flag} requires a value`); + } + return value; +} + +function parseConcurrency(raw: string): number { + const value = Number.parseInt(raw, 10); + if (!Number.isInteger(value) || value < 1) { + throw new Error(`invalid --concurrency: ${raw}`); + } + return value; +} + +function parseArgs(argv: readonly string[]): BuildOptions { + let repo: string = DEFAULT_TERMINAL_BENCH_4_IMAGE_REPO; + const tasks: string[] = []; + let dryRun = false; + let concurrency = 2; + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + const next = argv[i + 1]; + switch (arg) { + case "--repo": { + repo = requireValue(arg, next); + i++; + break; + } + case "--task": { + tasks.push(requireValue(arg, next)); + i++; + break; + } + case "--concurrency": { + concurrency = parseConcurrency(requireValue(arg, next)); + i++; + break; + } + case "--dry-run": { + dryRun = true; + break; + } + default: { + throw new Error(`unknown argument: ${arg}`); + } + } + } + return { repo, tasks, dryRun, concurrency }; +} + +export function taskImageBuilds( + repo: string, + tasksRoot: string, + taskId: string +): readonly ImageBuild[] { + const taskDir = join(tasksRoot, taskId); + const tags = imageTags(repo, taskId); + return [ + { + tag: tags.agent, + contextDir: join(taskDir, "environment"), + dockerfile: join(taskDir, "environment", "Dockerfile"), + }, + { + tag: tags.verifier, + contextDir: join(taskDir, "tests"), + dockerfile: join(taskDir, "tests", "Dockerfile"), + }, + ]; +} + +async function run(cmd: string, args: readonly string[]): Promise { + const proc = Bun.spawn([cmd, ...args], { + stdout: "inherit", + stderr: "inherit", + }); + const code = await proc.exited; + if (code !== 0) { + throw new Error(`${cmd} ${args.join(" ")} exited with ${code}`); + } +} + +async function buildAndPush(build: ImageBuild, dryRun: boolean): Promise { + if (!existsSync(build.dockerfile)) { + throw new Error(`missing Dockerfile: ${build.dockerfile}`); + } + const args = [ + "buildx", + "build", + "--platform", + "linux/amd64", + "--file", + build.dockerfile, + "--tag", + build.tag, + "--push", + build.contextDir, + ]; + if (dryRun) { + console.log(`docker ${args.join(" ")}`); + return; + } + + await run("docker", args); +} + +async function runBounded( + jobs: readonly (() => Promise)[], + concurrency: number +): Promise[]> { + const results: PromiseSettledResult[] = new Array(jobs.length); + let next = 0; + const workers = Array.from( + { length: Math.min(concurrency, jobs.length) }, + async () => { + while (next < jobs.length) { + const idx = next++; + const job = jobs[idx]; + if (job === undefined) { + return; + } + try { + await job(); + results[idx] = { status: "fulfilled", value: undefined }; + } catch (reason) { + results[idx] = { status: "rejected", reason }; + } + } + } + ); + await Promise.all(workers); + return results; +} + +async function main(): Promise { + const opts = parseArgs(process.argv.slice(2)); + const root = ensureTasksCheckedOut(); + const tasksRoot = tasksDir(root); + const taskIds = listTaskIds(tasksRoot, opts.tasks); + const missing = opts.tasks.filter((id) => !taskIds.includes(id)); + if (missing.length > 0) { + throw new Error( + `unknown or compose-only tasks: ${missing.join(", ")} (compose tasks are unsupported)` + ); + } + const builds = taskIds.flatMap((id) => + taskImageBuilds(opts.repo, tasksRoot, id) + ); + console.log( + `${opts.dryRun ? "planning" : "building"} ${builds.length} images for ${taskIds.length} tasks -> ${opts.repo}` + ); + const results = await runBounded( + builds.map((b) => () => buildAndPush(b, opts.dryRun)), + opts.concurrency + ); + const failed = results.flatMap((r, i) => + r.status === "rejected" ? [{ build: builds[i], reason: r.reason }] : [] + ); + for (const f of failed) { + console.error(`FAILED ${f.build?.tag}: ${String(f.reason)}`); + } + if (failed.length > 0) { + process.exitCode = 1; + } +} + +if (import.meta.main) { + await main(); +} diff --git a/src/benchmarks/benchmark-config.ts b/src/benchmarks/benchmark-config.ts index 8dbbe57..c767b33 100644 --- a/src/benchmarks/benchmark-config.ts +++ b/src/benchmarks/benchmark-config.ts @@ -25,6 +25,7 @@ import { DracoPanelConfigSchema } from "./draco/schemas"; import { SearchLaneConfigSchema } from "./search/core/config"; import { DEFAULT_JUDGE_MODEL, DEFAULT_STEP_LIMIT } from "./swe-atlas/schema"; import { BankingRetrievalConfigSchema } from "./tau3-bench-banking/retrieval-config"; +import { DEFAULT_TERMINAL_BENCH_4_IMAGE_REPO } from "./terminal-bench-4/images"; import { DEFAULT_TERMINAL_BENCH_AGENT, TERMINAL_BENCH_AGENTS, @@ -168,6 +169,18 @@ export const TerminalBenchConfigSchema = z.object({ export type TerminalBenchConfig = z.infer; +export const TerminalBench4OptionsSchema = TerminalBenchOptionsSchema.extend({ + imageRepo: z.string().min(1).default(DEFAULT_TERMINAL_BENCH_4_IMAGE_REPO), +}); + +export const TerminalBench4ConfigSchema = z.object({ + benchmarkId: z.literal("terminal_bench_4"), + ...ModelBenchmarkBaseSchema.shape, + ...TerminalBench4OptionsSchema.shape, +}); + +export type TerminalBench4Config = z.infer; + export const DracoBenchmarkConfigSchema = z.object({ benchmarkId: z.literal("draco"), panelConfig: DracoPanelConfigSchema, @@ -330,6 +343,7 @@ export const NativeBenchmarkRunConfigSchema = z.discriminatedUnion( Tau3BenchBankingConfigSchema, MmmuProVisionBenchmarkConfigSchema, TerminalBenchConfigSchema, + TerminalBench4ConfigSchema, DracoBenchmarkConfigSchema, IfStructBenchmarkConfigSchema, SweAtlasQaConfigSchema, @@ -365,6 +379,7 @@ export const BENCHMARK_OPTIONS_SCHEMAS = { tau3_bench_banking: Tau3BenchBankingOptionsSchema, mmmu_pro_vision: MmmuProVisionOptionsSchema, terminal_bench: TerminalBenchOptionsSchema, + terminal_bench_4: TerminalBench4OptionsSchema, ifstruct: IfStructOptionsSchema, swe_atlas_qa: SweAtlasOptionsSchema, swe_atlas_tw: SweAtlasOptionsSchema, diff --git a/src/benchmarks/benchmark-meta.ts b/src/benchmarks/benchmark-meta.ts index 87e41eb..6c84185 100644 --- a/src/benchmarks/benchmark-meta.ts +++ b/src/benchmarks/benchmark-meta.ts @@ -39,6 +39,11 @@ export const TERMINAL_BENCH_META = { defaultEpochs: 1, } as const satisfies BenchmarkMeta; +export const TERMINAL_BENCH_4_META = { + id: "terminal_bench_4", + defaultEpochs: 1, +} as const satisfies BenchmarkMeta; + export const DRACO_META = { id: "draco", defaultEpochs: 1, @@ -106,6 +111,7 @@ const BENCHMARK_META: Readonly> = { [TAU_BENCH_AIRLINE_META.id]: TAU_BENCH_AIRLINE_META, [TAU3_BENCH_BANKING_META.id]: TAU3_BENCH_BANKING_META, [TERMINAL_BENCH_META.id]: TERMINAL_BENCH_META, + [TERMINAL_BENCH_4_META.id]: TERMINAL_BENCH_4_META, [DRACO_META.id]: DRACO_META, [IFSTRUCT_META.id]: IFSTRUCT_META, [SWE_ATLAS_QA_META.id]: SWE_ATLAS_QA_META, diff --git a/src/benchmarks/harbor/modal-sandbox.ts b/src/benchmarks/harbor/modal-sandbox.ts index dd32534..38b2561 100644 --- a/src/benchmarks/harbor/modal-sandbox.ts +++ b/src/benchmarks/harbor/modal-sandbox.ts @@ -76,14 +76,20 @@ export function makeModalSandboxLayer( }); const sandbox = yield* tryPromise({ try: () => - client.sandboxes.create(app, builtImage, { - timeoutMs: input.timeoutSec * 1000, - cpu: input.cpus, - memoryMiB: input.memoryMb, - blockNetwork: !input.allowInternet, - workdir: input.workdir, - command: [...input.keepAliveCommand], - }), + client.sandboxes.create( + app, + builtImage, + definedValues({ + timeoutMs: input.timeoutSec * 1000, + cpu: input.cpus, + memoryMiB: input.memoryMb, + gpu: input.gpu, + env: input.env === undefined ? undefined : { ...input.env }, + blockNetwork: !input.allowInternet, + workdir: input.workdir, + command: [...input.keepAliveCommand], + }) + ), catch: (e) => toSolverError("Failed to create Modal sandbox", e), }); let handedOff = false; diff --git a/src/benchmarks/harbor/sandbox.ts b/src/benchmarks/harbor/sandbox.ts index 83610bc..e251b7f 100644 --- a/src/benchmarks/harbor/sandbox.ts +++ b/src/benchmarks/harbor/sandbox.ts @@ -52,6 +52,8 @@ export interface CreateSessionInput { readonly timeoutSec: number; readonly cpus: number; readonly memoryMb: number; + readonly gpu?: string; + readonly env?: Readonly>; readonly allowInternet: boolean; readonly workdir: string; readonly keepAliveCommand: readonly string[]; diff --git a/src/benchmarks/registry.ts b/src/benchmarks/registry.ts index 0a4509a..e25b0c3 100644 --- a/src/benchmarks/registry.ts +++ b/src/benchmarks/registry.ts @@ -15,6 +15,7 @@ import { } from "./swe-atlas/benchmark"; import { TAU_BENCH_AIRLINE_BENCHMARK } from "./tau-bench-airline/benchmark"; import { TAU3_BENCH_BANKING_BENCHMARK } from "./tau3-bench-banking/benchmark"; +import { TERMINAL_BENCH_4_BENCHMARK } from "./terminal-bench-4/benchmark"; import { TERMINAL_BENCH_BENCHMARK } from "./terminal-bench/benchmark"; import type { Benchmark } from "./types"; import { VGI_BENCH_BENCHMARK } from "./vgi-bench/benchmark"; @@ -27,6 +28,7 @@ const BENCHMARKS: Record = { [TAU3_BENCH_BANKING_BENCHMARK.id]: TAU3_BENCH_BANKING_BENCHMARK, [MMMU_PRO_VISION_BENCHMARK.id]: MMMU_PRO_VISION_BENCHMARK, [TERMINAL_BENCH_BENCHMARK.id]: TERMINAL_BENCH_BENCHMARK, + [TERMINAL_BENCH_4_BENCHMARK.id]: TERMINAL_BENCH_4_BENCHMARK, [DRACO_BENCHMARK.id]: DRACO_BENCHMARK, [IFSTRUCT_BENCHMARK.id]: IFSTRUCT_BENCHMARK, [SWE_ATLAS_QA_BENCHMARK.id]: SWE_ATLAS_QA_BENCHMARK, diff --git a/src/benchmarks/terminal-bench-4/README.md b/src/benchmarks/terminal-bench-4/README.md new file mode 100644 index 0000000..a9d3517 --- /dev/null +++ b/src/benchmarks/terminal-bench-4/README.md @@ -0,0 +1,56 @@ +# Terminal-Bench 4.0 + +[Terminal-Bench 4.0](https://www.tbench.ai/news/terminal-bench-4-0) is the current major release of the continuous Terminal-Bench line that began with 3.0. It is a different task set from Terminal-Bench 2.1 (`terminal_bench` in this catalog), so scores are not comparable across the two benchmarks. This benchmark is registered separately as `terminal_bench_4`. + +## Source & license + +- Repository: , pinned to the `v4.0.0` tag, commit `452bf305c6daa62fc59061d22133a7cbc7c1572e` (`TERMINAL_BENCH_4_SOURCE_COMMIT`). Override the checkout with `BENCH_TERMINAL_BENCH_4_TASKS_DIR=`. +- Harbor dataset id: `terminal-bench/terminal-bench@4.0.0` (). +- License: Apache-2.0 (repository `LICENSE`). +- The checkout contains 66 task manifests. This harness exposes **55** of them, see the exclusion below. + +## Task format + +Each task ships `task.toml`, `instruction.md`, `environment/Dockerfile` (agent image), `tests/Dockerfile` and `tests/test.sh` (verifier image). Only the manifest fields the harness reads are validated (`schema.ts`): agent and verifier timeouts, agent and verifier resources (`cpus`, `memory_mb`, `storage_mb`, `gpus`, `gpu_types`, `allow_internet`, `env`), `artifacts`, `verifier.collect`, `verifier.env` and `metadata.category`. All 66 manifests declare `verifier.environment_mode = "separate"`. + +## Execution model + +Unlike 2.1, the verifier does not run inside the agent container. + +1. Agent sandbox is created from the task's prebuilt agent image with the task's `cpus`, `memory_mb`, GPU and `environment.env`. Internet is always enabled so the agent CLI can reach OpenRouter, and tasks that declare `allow_internet = false` record `agentNetworkForced: true` in sample metadata. +2. The ori agent runs against `/instruction.md` with the task's `agent.timeout_sec`. +3. `verifier.collect` hooks run in the agent sandbox. Hooks that target a compose sidecar service fail the sample. +4. `/logs/artifacts` plus every declared artifact path is bundled into a tarball, downloaded, and extracted at the same absolute paths in a fresh verifier sandbox built from the task's verifier image with `verifier.environment` resources (falling back to the agent resources) and `verifier.env`. +5. `/tests/test.sh` runs with `verifier.timeout_sec`. Reward is read from `/logs/verifier/reward.txt` and scored 1 only when the reward is 1, otherwise 0 (`harbor/reward.ts`). + +Per-task sandbox timeouts are the task's declared timeouts plus a fixed margin, so an 8 hour task gets an 8 hour agent sandbox. + +## GPUs + +Three tasks declare one H100 each: `fp8-rmsnorm-gemm`, `jax-speedrun-gpu`, `math-eval-grader`. `toModalGpu` maps `gpus`/`gpu_types` to the Modal `gpu` string (`"H100"`, or `"H100:2"` for multiple) and refuses tasks that request GPUs without naming a type. `storage_mb` is parsed but not enforced because the Modal sandbox API used here has no disk-size parameter. + +## Images + +4.0 tasks ship Dockerfiles with local-context `COPY` steps rather than prebuilt image names, and the Modal SDK rejects those Dockerfiles. Images are therefore prebuilt and pushed once per pinned commit: + +```bash +bun scripts/build-terminal-bench-4-images.ts --repo [--task ] [--concurrency 2] [--dry-run] +``` + +Tags are `/:` and `/-verifier:` (`images.ts`). The run config `imageRepo` selects the registry prefix and defaults to `DEFAULT_TERMINAL_BENCH_4_IMAGE_REPO`. + +## Excluded tasks + +Eleven tasks use `environment/docker-compose.yaml` and need more than one container. The Modal sandbox abstraction here is single-container, so they are excluded from the dataset and reported by `listComposeTaskIds`: + +`ctr-optimization`, `cumulative-layout-shift`, `freight-dispatch-shift`, `heat-pump-warranty`, `intrastat-meldung`, `kv-live-surgery`, `legacy-utility-triage`, `live-database-cutover`, `medical-claims-processing`, `nextjs-performance`, `payments-pipeline-fix`. + +A score over the 55 runnable tasks is not an official Terminal-Bench 4.0 number. + +## Config options + +Same agent options as `terminal_bench` (`TerminalBenchOptionsSchema`) plus: + +| option | type | default | description | +| --- | --- | --- | --- | +| `imageRepo` | string | `DEFAULT_TERMINAL_BENCH_4_IMAGE_REPO` | Registry prefix holding the prebuilt agent and verifier images. | diff --git a/src/benchmarks/terminal-bench-4/benchmark.test.ts b/src/benchmarks/terminal-bench-4/benchmark.test.ts new file mode 100644 index 0000000..b750e3c --- /dev/null +++ b/src/benchmarks/terminal-bench-4/benchmark.test.ts @@ -0,0 +1,58 @@ +import { describe, expect, it } from "bun:test"; + +import { assertRight } from "../../internal/testing"; +import { parseSchema } from "../../internal/zod"; +import { BenchmarkRunConfigSchema } from "../benchmark-config"; +import { getBenchmarkMeta, TERMINAL_BENCH_4_META } from "../benchmark-meta"; +import { getBenchmark } from "../registry"; +import { TERMINAL_BENCH_BENCHMARK } from "../terminal-bench/benchmark"; +import { TERMINAL_BENCH_4_BENCHMARK } from "./benchmark"; +import { DEFAULT_TERMINAL_BENCH_4_IMAGE_REPO, imageTags } from "./images"; +import { TERMINAL_BENCH_4_SOURCE_COMMIT } from "./tasks-source"; + +describe("terminal-bench-4 registry wiring", () => { + it("registers a benchmark distinct from terminal-bench 2.1", () => { + expect(TERMINAL_BENCH_4_BENCHMARK.id).toBe("terminal_bench_4"); + expect(TERMINAL_BENCH_BENCHMARK.id).toBe("terminal_bench"); + expect(getBenchmark("terminal_bench_4")).toBe(TERMINAL_BENCH_4_BENCHMARK); + expect(getBenchmark("terminal_bench")).toBe(TERMINAL_BENCH_BENCHMARK); + }); + + it("agrees with its metadata on id, epochs and temperature", () => { + expect(TERMINAL_BENCH_4_META.id).toBe(TERMINAL_BENCH_4_BENCHMARK.id); + expect(TERMINAL_BENCH_4_BENCHMARK.defaultEpochs).toBe( + TERMINAL_BENCH_4_META.defaultEpochs + ); + expect(getBenchmarkMeta(TERMINAL_BENCH_4_META.id)).toBe( + TERMINAL_BENCH_4_META + ); + expect(TERMINAL_BENCH_4_BENCHMARK.temperature).toBe(0); + expect(TERMINAL_BENCH_4_BENCHMARK.degradeSolverErrors).toBe(true); + }); + + it("parses a run config with the default image repo and agent", () => { + const result = parseSchema(BenchmarkRunConfigSchema, { + benchmarkId: "terminal_bench_4", + model: "anthropic/claude-opus-5", + reasoningEffort: "high", + agentReasoningEffort: "high", + }); + assertRight(result); + expect(result.right.benchmarkId).toBe("terminal_bench_4"); + expect( + result.right.benchmarkId === "terminal_bench_4" && result.right.imageRepo + ).toBe(DEFAULT_TERMINAL_BENCH_4_IMAGE_REPO); + expect( + result.right.benchmarkId === "terminal_bench_4" && result.right.agent + ).toBe("pi"); + }); +}); + +describe("terminal-bench-4 image tags", () => { + it("derives agent and verifier tags from the repo, task id and pinned commit", () => { + expect(imageTags("ghcr.io/acme/tb4", "hello-world")).toEqual({ + agent: `ghcr.io/acme/tb4/hello-world:${TERMINAL_BENCH_4_SOURCE_COMMIT.slice(0, 12)}`, + verifier: `ghcr.io/acme/tb4/hello-world-verifier:${TERMINAL_BENCH_4_SOURCE_COMMIT.slice(0, 12)}`, + }); + }); +}); diff --git a/src/benchmarks/terminal-bench-4/benchmark.ts b/src/benchmarks/terminal-bench-4/benchmark.ts new file mode 100644 index 0000000..df81198 --- /dev/null +++ b/src/benchmarks/terminal-bench-4/benchmark.ts @@ -0,0 +1,89 @@ +import type { HttpClient } from "@effect/platform"; +import { gen } from "effect/Effect"; +import type { Layer } from "effect/Layer"; +import { + fail as layerFail, + effect as layerEffect, + provide as layerProvide, + mergeAll as layerMergeAll, + succeed as layerSucceed, +} from "effect/Layer"; + +import type { Dataset } from "../../harness/dataset"; +import { Scorer } from "../../harness/scorer"; +import { Solver } from "../../harness/solver"; +import { definedValues } from "../../internal/guards"; +import { getOriHarness } from "../agent-cli/harness"; +import { TERMINAL_BENCH_4_META } from "../benchmark-meta"; +import { makeModalSandboxLayer } from "../harbor/modal-sandbox"; +import { SandboxSession } from "../harbor/sandbox"; +import { terminalBenchScorer } from "../terminal-bench/scorer"; +import type { Benchmark, BenchmarkRunInput } from "../types"; +import { makeTerminalBench4DatasetLayer } from "./dataset"; +import type { TerminalBench4SolverOpts } from "./solver"; +import { terminalBench4Solver } from "./solver"; + +export const TERMINAL_BENCH_4_ID = TERMINAL_BENCH_4_META.id; + +const TERMINAL_BENCH_4_APP_NAME = "openrouter-terminal-bench-4" as const; + +function makeTerminalBench4Layer( + input: BenchmarkRunInput +): Layer { + const { benchmarkConfig } = input; + if (benchmarkConfig.benchmarkId !== "terminal_bench_4") { + return layerFail( + new Error("terminal_bench_4 received mismatched benchmarkConfig") + ); + } + const { agent } = benchmarkConfig; + const solverOpts: TerminalBench4SolverOpts = definedValues({ + model: benchmarkConfig.model, + apiKey: input.apiKey, + sessionId: input.sessionId, + endpointId: benchmarkConfig.endpointId, + agentPackage: benchmarkConfig.agentPackage, + oriInstallUrl: benchmarkConfig.oriInstallUrl, + appendSystemPrompt: benchmarkConfig.appendSystemPrompt, + systemPrompt: benchmarkConfig.systemPrompt, + agentReasoningEffort: benchmarkConfig.agentReasoningEffort, + oriChannel: benchmarkConfig.oriChannel, + allowedTools: benchmarkConfig.allowedTools, + disallowedTools: benchmarkConfig.disallowedTools, + isolateAgentConfig: benchmarkConfig.isolateAgentConfig, + imageRepo: benchmarkConfig.imageRepo, + }); + const datasetLayer = makeTerminalBench4DatasetLayer( + definedValues({ + taskSubset: benchmarkConfig.taskSubset, + maxAgentTimeoutSec: benchmarkConfig.maxAgentTimeoutSec, + }) + ); + const sandboxLayer: Layer = makeModalSandboxLayer({ + appName: TERMINAL_BENCH_4_APP_NAME, + environment: benchmarkConfig.modalEnv, + }); + const solverLayer = layerEffect(Solver)( + gen(function* () { + const sessionFactory = yield* SandboxSession; + return Solver.of( + terminalBench4Solver(sessionFactory, solverOpts, getOriHarness(agent)) + ); + }) + ); + const scorerLayer = layerSucceed(Scorer, Scorer.of(terminalBenchScorer)); + return layerMergeAll( + datasetLayer, + solverLayer.pipe(layerProvide(sandboxLayer)), + scorerLayer + ); +} + +export const TERMINAL_BENCH_4_BENCHMARK: Benchmark = { + id: TERMINAL_BENCH_4_ID, + makeDatasetLayer: () => makeTerminalBench4DatasetLayer(), + temperature: 0, + defaultEpochs: TERMINAL_BENCH_4_META.defaultEpochs, + degradeSolverErrors: true, + makeLayer: makeTerminalBench4Layer, +}; diff --git a/src/benchmarks/terminal-bench-4/dataset.test.ts b/src/benchmarks/terminal-bench-4/dataset.test.ts new file mode 100644 index 0000000..866dc15 --- /dev/null +++ b/src/benchmarks/terminal-bench-4/dataset.test.ts @@ -0,0 +1,296 @@ +import { afterAll, beforeAll, describe, expect, it } from "bun:test"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { assertLeft, assertRight } from "../../internal/testing"; +import { parseSchema } from "../../internal/zod"; +import { + TERMINAL_BENCH_4_DATASET_ID, + listComposeTaskIds, + listTaskIds, + loadTask, + readTerminalBench4Meta, + taskToSample, + toModalGpu, + toSandboxResources, +} from "./dataset"; +import { TaskTomlSchema } from "./schema"; +import { + ensureTasksCheckedOut, + resetCheckoutCache, + tasksDir as tasksDirOf, +} from "./tasks-source"; + +const NETWORK = describe.skipIf(Boolean(process.env["CI"])); + +const BASE_TOML = ` +[task] +name = "fixture" + +[metadata] +category = "software-engineering" + +[agent] +timeout_sec = 3600 + +[verifier] +timeout_sec = 600 +environment_mode = "separate" + +[environment] +cpus = 2 +memory_mb = 4096 +storage_mb = 10240 +gpus = 0 +`; + +const GPU_TOML = ` +artifacts = ["/app/out", { source = "/data", exclude = ["*.tmp"] }] + +[task] +name = "gpu-fixture" + +[metadata] +category = "machine-learning" + +[agent] +timeout_sec = 7200 + +[verifier] +timeout_sec = 900 +environment_mode = "separate" +env = { TB_SEED = "1" } + +[[verifier.collect]] +command = "cp /var/log/app.log /logs/artifacts/" + +[verifier.environment] +cpus = 16 +memory_mb = 32768 +storage_mb = 1024000 +gpus = 1 +gpu_types = ["H100"] + +[environment] +cpus = 8 +memory_mb = 16384 +storage_mb = 102400 +gpus = 1 +gpu_types = ["H100"] +allow_internet = false +env = { HF_HOME = "/cache" } +`; + +function writeTask(root: string, id: string, toml: string, compose = false) { + const dir = join(root, id); + mkdirSync(join(dir, "environment"), { recursive: true }); + writeFileSync(join(dir, "task.toml"), toml); + writeFileSync(join(dir, "instruction.md"), `Solve ${id}.\n`); + if (compose) { + writeFileSync( + join(dir, "environment", "docker-compose.yaml"), + "services: {}\n" + ); + } +} + +let fixtureRoot = ""; +beforeAll(() => { + fixtureRoot = mkdtempSync(join(tmpdir(), "tb4-fixture-")); + writeTask(fixtureRoot, "b-cpu", BASE_TOML); + writeTask(fixtureRoot, "a-gpu", GPU_TOML); + writeTask(fixtureRoot, "c-compose", BASE_TOML, true); + mkdirSync(join(fixtureRoot, ".hidden")); + writeFileSync(join(fixtureRoot, "README.md"), "not a task\n"); +}); +afterAll(() => { + rmSync(fixtureRoot, { recursive: true, force: true }); +}); + +describe("terminal-bench-4 task.toml schema", () => { + it("rejects a manifest whose verifier is not in separate mode", () => { + assertLeft( + parseSchema(TaskTomlSchema, { + task: { name: "x" }, + metadata: { category: "c" }, + agent: { timeout_sec: 10 }, + verifier: { timeout_sec: 10, environment_mode: "same" }, + environment: { cpus: 1, memory_mb: 1, storage_mb: 1 }, + }) + ); + }); + + it("defaults gpus, gpu_types, allow_internet, env, artifacts and collect", () => { + const result = parseSchema(TaskTomlSchema, { + task: { name: "x" }, + metadata: { category: "c" }, + agent: { timeout_sec: 10 }, + verifier: { timeout_sec: 10, environment_mode: "separate" }, + environment: { cpus: 1, memory_mb: 1, storage_mb: 1 }, + }); + assertRight(result); + expect(result.right.environment.gpus).toBe(0); + expect(result.right.environment.gpu_types).toEqual([]); + expect(result.right.environment.allow_internet).toBe(true); + expect(result.right.environment.env).toEqual({}); + expect(result.right.artifacts).toEqual([]); + expect(result.right.verifier.collect).toEqual([]); + expect(result.right.verifier.env).toEqual({}); + }); +}); + +describe("terminal-bench-4 task listing", () => { + it("lists non-compose task directories sorted and skips hidden dirs and files", () => { + expect(listTaskIds(fixtureRoot)).toEqual(["a-gpu", "b-cpu"]); + }); + + it("lists compose tasks separately", () => { + expect(listComposeTaskIds(fixtureRoot)).toEqual(["c-compose"]); + }); + + it("honors a task subset in the requested order and drops unknown or compose ids", () => { + expect( + listTaskIds(fixtureRoot, ["b-cpu", "nope", "c-compose", "a-gpu"]) + ).toEqual(["b-cpu", "a-gpu"]); + }); +}); + +describe("terminal-bench-4 GPU mapping", () => { + const env = { + cpus: 1, + memory_mb: 1, + storage_mb: 1, + allow_internet: true, + env: {}, + }; + + it("returns no gpu for cpu-only tasks", () => { + expect(toModalGpu({ ...env, gpus: 0, gpu_types: [] })).toBeUndefined(); + }); + + it("maps a single accelerator to its bare type", () => { + expect(toModalGpu({ ...env, gpus: 1, gpu_types: ["H100"] })).toBe("H100"); + }); + + it("maps multiple accelerators with a count suffix", () => { + expect(toModalGpu({ ...env, gpus: 2, gpu_types: ["A100"] })).toBe("A100:2"); + }); + + it("refuses to guess an accelerator when gpu_types is empty", () => { + expect(() => toModalGpu({ ...env, gpus: 1, gpu_types: [] })).toThrow( + /refusing to guess/ + ); + }); + + it("omits the gpu key entirely for cpu-only sandbox resources", () => { + const resources = toSandboxResources({ ...env, gpus: 0, gpu_types: [] }); + expect(Object.hasOwn(resources, "gpu")).toBe(false); + }); +}); + +describe("terminal-bench-4 taskToSample", () => { + it("builds a stable id, instruction input and resource metadata for a cpu task", () => { + const sample = taskToSample(loadTask("b-cpu", fixtureRoot)); + expect(sample.id).toBe(`${TERMINAL_BENCH_4_DATASET_ID}-b-cpu`); + expect(sample.input).toBe("Solve b-cpu.\n"); + expect(sample.target).toEqual({ text: "b-cpu" }); + const meta = readTerminalBench4Meta(sample.metadata); + expect(meta).toBeDefined(); + expect(meta?.maxAgentTimeoutSec).toBe(3600); + expect(meta?.maxTestTimeoutSec).toBe(600); + expect(meta?.agentEnv).toEqual({ + cpus: 2, + memoryMb: 4096, + env: {}, + allowInternet: true, + }); + expect(meta?.verifierEnv).toEqual(meta?.agentEnv); + }); + + it("propagates gpu, env vars, artifacts and collect hooks for a gpu task", () => { + const meta = readTerminalBench4Meta( + taskToSample(loadTask("a-gpu", fixtureRoot)).metadata + ); + expect(meta?.agentEnv).toEqual({ + cpus: 8, + memoryMb: 16384, + gpu: "H100", + env: { HF_HOME: "/cache" }, + allowInternet: false, + }); + expect(meta?.verifierEnv).toEqual({ + cpus: 16, + memoryMb: 32768, + gpu: "H100", + env: { TB_SEED: "1" }, + allowInternet: true, + }); + expect(meta?.artifacts).toEqual([ + "/app/out", + { source: "/data", exclude: ["*.tmp"] }, + ]); + expect(meta?.collect).toEqual([ + { command: "cp /var/log/app.log /logs/artifacts/" }, + ]); + }); + + it("applies an agent timeout override without touching the verifier timeout", () => { + const meta = readTerminalBench4Meta( + taskToSample(loadTask("b-cpu", fixtureRoot), 60).metadata + ); + expect(meta?.maxAgentTimeoutSec).toBe(60); + expect(meta?.maxTestTimeoutSec).toBe(600); + }); + + it("refuses to build a sample for a compose task", () => { + expect(() => taskToSample(loadTask("c-compose", fixtureRoot))).toThrow( + /docker-compose/ + ); + }); + + it("returns undefined for metadata that is not terminal-bench-4 metadata", () => { + expect(readTerminalBench4Meta(undefined)).toBeUndefined(); + expect(readTerminalBench4Meta({ taskId: "x" })).toBeUndefined(); + }); +}); + +NETWORK("terminal-bench-4 pinned checkout", () => { + let tasksDir = ""; + let savedCacheDisable: string | undefined; + beforeAll(async () => { + resetCheckoutCache(); + savedCacheDisable = process.env.BENCH_DATASET_CACHE_DISABLE; + process.env.BENCH_DATASET_CACHE_DISABLE ??= "0"; + tasksDir = tasksDirOf(await ensureTasksCheckedOut()); + }, 300_000); + afterAll(() => { + if (savedCacheDisable === undefined) { + delete process.env.BENCH_DATASET_CACHE_DISABLE; + } else { + process.env.BENCH_DATASET_CACHE_DISABLE = savedCacheDisable; + } + }); + + it("exposes 55 runnable tasks and 11 compose tasks", () => { + expect(listTaskIds(tasksDir)).toHaveLength(55); + expect(listComposeTaskIds(tasksDir)).toHaveLength(11); + }); + + it("parses every manifest and maps exactly three H100 tasks", () => { + const gpuTasks = listTaskIds(tasksDir) + .map((id) => taskToSample(loadTask(id, tasksDir))) + .flatMap((sample) => { + const meta = readTerminalBench4Meta(sample.metadata); + return meta?.agentEnv.gpu === undefined ? [] : [meta.taskId]; + }); + expect(gpuTasks).toEqual([ + "fp8-rmsnorm-gemm", + "jax-speedrun-gpu", + "math-eval-grader", + ]); + for (const id of listComposeTaskIds(tasksDir)) { + expect(loadTask(id, tasksDir).composeFile).toBeDefined(); + } + }); +}); diff --git a/src/benchmarks/terminal-bench-4/dataset.ts b/src/benchmarks/terminal-bench-4/dataset.ts new file mode 100644 index 0000000..c0e81c9 --- /dev/null +++ b/src/benchmarks/terminal-bench-4/dataset.ts @@ -0,0 +1,259 @@ +import { existsSync, readFileSync, readdirSync, statSync } from "node:fs"; +import { join } from "node:path"; + +import { fromIterable } from "effect/Chunk"; +import type { Effect } from "effect/Effect"; +import { fail, flatMap, mapError, succeed } from "effect/Effect"; +import type { Layer } from "effect/Layer"; +import { effect } from "effect/Layer"; +import { none, some } from "effect/Option"; +import type { Stream } from "effect/Stream"; +import { + flatMap as flatMapStream, + fromEffect, + paginateChunkEffect, +} from "effect/Stream"; +import { parse as tomlParse } from "smol-toml"; + +import type { Sample } from "../../harness/core"; +import { DatasetError } from "../../harness/core"; +import type { DatasetStreamOptions } from "../../harness/dataset"; +import { Dataset } from "../../harness/dataset"; +import { Either } from "../../internal/either"; +import { definedValues } from "../../internal/guards"; +import { parseSchema, z } from "../../internal/zod"; +import type { TaskEnvironment, TerminalBench4Task } from "./schema"; +import { ArtifactSchema, CollectHookSchema, TaskTomlSchema } from "./schema"; +import { ensureTasksCheckedOutEffect, tasksDir } from "./tasks-source"; + +export const TERMINAL_BENCH_4_DATASET_ID = "terminal_bench_4" as const; + +const COMPOSE_FILE = join("environment", "docker-compose.yaml"); + +export function loadTask( + taskId: string, + tasksDirPath: string +): TerminalBench4Task { + const taskDir = join(tasksDirPath, taskId); + const raw = readFileSync(join(taskDir, "task.toml"), "utf8"); + const tomlObj = Either.try(() => tomlParse(raw)); + if (Either.isLeft(tomlObj)) { + throw new Error( + `terminal-bench-4 task "${taskId}" task.toml failed to parse: ${String(tomlObj.left)}` + ); + } + const parsed = parseSchema(TaskTomlSchema, tomlObj.right); + if (Either.isLeft(parsed)) { + throw new Error( + `terminal-bench-4 task "${taskId}" task.toml failed validation: ${parsed.left.message}` + ); + } + const composePath = join(taskDir, COMPOSE_FILE); + return { + id: taskId, + taskToml: parsed.right, + taskDir, + instructionPath: join(taskDir, "instruction.md"), + composeFile: existsSync(composePath) ? composePath : undefined, + }; +} + +function isTaskDir(tasksDirPath: string, entry: string): boolean { + if (entry.startsWith(".")) { + return false; + } + try { + return ( + statSync(join(tasksDirPath, entry)).isDirectory() && + existsSync(join(tasksDirPath, entry, "task.toml")) + ); + } catch { + return false; + } +} + +export function listComposeTaskIds(tasksDirPath: string): readonly string[] { + return readdirSync(tasksDirPath) + .filter( + (entry) => + isTaskDir(tasksDirPath, entry) && + existsSync(join(tasksDirPath, entry, COMPOSE_FILE)) + ) + .sort(); +} + +export function listTaskIds( + tasksDirPath: string, + taskSubset?: readonly string[] +): readonly string[] { + const onDisk = readdirSync(tasksDirPath).filter( + (entry) => + isTaskDir(tasksDirPath, entry) && + !existsSync(join(tasksDirPath, entry, COMPOSE_FILE)) + ); + if (taskSubset !== undefined && taskSubset.length > 0) { + const diskSet = new Set(onDisk); + return taskSubset.filter((id) => diskSet.has(id)); + } + return [...onDisk].sort(); +} + +const SandboxResourcesSchema = z.object({ + cpus: z.number().int().positive(), + memoryMb: z.number().int().positive(), + gpu: z.string().min(1).optional(), + env: z.record(z.string(), z.string()), + allowInternet: z.boolean(), +}); + +export type SandboxResources = z.infer; + +export const TerminalBench4SampleMetaSchema = z.object({ + taskId: z.string().min(1), + maxAgentTimeoutSec: z.number().positive(), + maxTestTimeoutSec: z.number().positive(), + category: z.string(), + agentEnv: SandboxResourcesSchema, + verifierEnv: SandboxResourcesSchema, + artifacts: z.array(ArtifactSchema), + collect: z.array(CollectHookSchema), + reward: z.number().optional(), + testOutput: z.string().optional(), +}); + +export type TerminalBench4SampleMeta = z.infer< + typeof TerminalBench4SampleMetaSchema +>; + +export function toSandboxResources( + env: TaskEnvironment, + extraEnvVars: Readonly> = {} +): SandboxResources { + return definedValues({ + cpus: env.cpus, + memoryMb: env.memory_mb, + gpu: toModalGpu(env), + env: { ...env.env, ...extraEnvVars }, + allowInternet: env.allow_internet, + }); +} + +export function toModalGpu(env: TaskEnvironment): string | undefined { + if (env.gpus === 0) { + return undefined; + } + const type = env.gpu_types[0]; + if (type === undefined) { + throw new Error( + `task requests ${env.gpus} GPU(s) but declares no gpu_types; refusing to guess an accelerator` + ); + } + return env.gpus === 1 ? type : `${type}:${env.gpus}`; +} + +export function taskToSample( + task: TerminalBench4Task, + maxAgentTimeoutSecOverride?: number +): Sample { + if (task.composeFile !== undefined) { + throw new Error( + `terminal-bench-4 task "${task.id}" uses docker-compose, which this harness does not support` + ); + } + const instruction = readFileSync(task.instructionPath, "utf8"); + const { environment, verifier, agent, metadata, artifacts } = task.taskToml; + const meta: TerminalBench4SampleMeta = { + taskId: task.id, + maxAgentTimeoutSec: maxAgentTimeoutSecOverride ?? agent.timeout_sec, + maxTestTimeoutSec: verifier.timeout_sec, + category: metadata.category, + agentEnv: toSandboxResources(environment), + verifierEnv: toSandboxResources( + verifier.environment ?? environment, + verifier.env + ), + artifacts, + collect: verifier.collect, + }; + return { + id: `${TERMINAL_BENCH_4_DATASET_ID}-${task.id}`, + input: instruction, + target: { text: task.id }, + metadata: meta, + }; +} + +export function readTerminalBench4Meta( + metadata: Readonly> | undefined +): TerminalBench4SampleMeta | undefined { + if (metadata === undefined) { + return undefined; + } + const parsed = parseSchema(TerminalBench4SampleMetaSchema, metadata); + return Either.isLeft(parsed) ? undefined : parsed.right; +} + +export interface TerminalBench4DatasetConfig { + readonly taskSubset?: readonly string[]; + readonly maxAgentTimeoutSec?: number; + readonly pageSize?: number; +} + +export function makeTerminalBench4DatasetLayer( + config?: TerminalBench4DatasetConfig +): Layer { + const pageSize = config?.pageSize ?? 20; + const taskSubset = config?.taskSubset; + const maxAgentTimeoutSec = config?.maxAgentTimeoutSec; + return effect( + Dataset, + succeed(buildDatasetService({ pageSize, taskSubset, maxAgentTimeoutSec })) + ); +} + +function buildDatasetService(opts: { + readonly pageSize: number; + readonly taskSubset?: readonly string[]; + readonly maxAgentTimeoutSec?: number; +}): ReturnType { + const { pageSize, taskSubset, maxAgentTimeoutSec } = opts; + const tasksDirEffect = ensureTasksCheckedOutEffect().pipe( + mapError((e) => new DatasetError({ message: e.message })), + flatMap((root) => succeed(tasksDir(root))) + ); + const sizeEffect: Effect = tasksDirEffect.pipe( + flatMap((dir) => succeed(listTaskIds(dir, taskSubset).length)) + ); + const stream = (opts2?: DatasetStreamOptions): Stream => + fromEffect(tasksDirEffect).pipe( + flatMapStream((dir: string) => { + const allIds = listTaskIds(dir, taskSubset); + const start = opts2?.start ?? 0; + const requestedEnd = opts2?.end ?? allIds.length; + const end = Math.min(requestedEnd, allIds.length); + const pageIds = allIds.slice(start, end); + return paginateChunkEffect(0, (offset: number) => { + const pageSlice = pageIds.slice(offset, offset + pageSize); + const built = Either.try(() => + pageSlice.map((id) => + taskToSample(loadTask(id, dir), maxAgentTimeoutSec) + ) + ); + if (Either.isLeft(built)) { + return fail( + new DatasetError({ + message: `Failed to load terminal-bench-4 tasks at offset ${offset}: ${String(built.left)}`, + }) + ); + } + const nextOffset = offset + pageSize; + const hasMore = nextOffset < pageIds.length; + return succeed([ + fromIterable(built.right), + hasMore ? some(nextOffset) : none(), + ] as const); + }); + }) + ); + return Dataset.of({ stream, size: sizeEffect }); +} diff --git a/src/benchmarks/terminal-bench-4/images.ts b/src/benchmarks/terminal-bench-4/images.ts new file mode 100644 index 0000000..8123405 --- /dev/null +++ b/src/benchmarks/terminal-bench-4/images.ts @@ -0,0 +1,20 @@ +import { TERMINAL_BENCH_4_SOURCE_COMMIT } from "./tasks-source"; + +export const DEFAULT_TERMINAL_BENCH_4_IMAGE_REPO = + "ghcr.io/openrouterteam/terminal-bench-4" as const; + +export interface TerminalBench4ImageTags { + readonly agent: string; + readonly verifier: string; +} + +export function imageTags( + imageRepo: string, + taskId: string +): TerminalBench4ImageTags { + const tag = TERMINAL_BENCH_4_SOURCE_COMMIT.slice(0, 12); + return { + agent: `${imageRepo}/${taskId}:${tag}`, + verifier: `${imageRepo}/${taskId}-verifier:${tag}`, + }; +} diff --git a/src/benchmarks/terminal-bench-4/schema.ts b/src/benchmarks/terminal-bench-4/schema.ts new file mode 100644 index 0000000..d1bd891 --- /dev/null +++ b/src/benchmarks/terminal-bench-4/schema.ts @@ -0,0 +1,67 @@ +import type { ValueOf } from "../../internal/guards"; +import { z } from "../../internal/zod"; +import { ORI_AGENTS } from "../agent-cli/schema"; + +export const TERMINAL_BENCH_4_VERSION = "4.0.0" as const; + +export const TERMINAL_BENCH_4_AGENTS = ORI_AGENTS; + +export type TerminalBench4Agent = ValueOf; + +export const DEFAULT_TERMINAL_BENCH_4_AGENT: TerminalBench4Agent = "pi"; + +export const ArtifactSchema = z.union([ + z.string().min(1), + z.object({ + source: z.string().min(1), + exclude: z.array(z.string()).default([]), + service: z.string().optional(), + }), +]); + +export type Artifact = z.infer; + +const EnvironmentSchema = z.object({ + cpus: z.number().int().positive(), + memory_mb: z.number().int().positive(), + storage_mb: z.number().int().positive(), + gpus: z.number().int().nonnegative().default(0), + gpu_types: z.array(z.string().min(1)).default([]), + allow_internet: z.boolean().default(true), + env: z.record(z.string(), z.string()).default({}), +}); + +export type TaskEnvironment = z.infer; + +export const CollectHookSchema = z.object({ + command: z.string().min(1), + service: z.string().optional(), + timeout_sec: z.number().positive().optional(), +}); + +export type CollectHook = z.infer; + +export const TaskTomlSchema = z.object({ + artifacts: z.array(ArtifactSchema).default([]), + task: z.object({ name: z.string().min(1) }), + metadata: z.object({ category: z.string().min(1) }), + agent: z.object({ timeout_sec: z.number().positive() }), + verifier: z.object({ + timeout_sec: z.number().positive(), + environment_mode: z.literal("separate"), + environment: EnvironmentSchema.optional(), + collect: z.array(CollectHookSchema).default([]), + env: z.record(z.string(), z.string()).default({}), + }), + environment: EnvironmentSchema, +}); + +export type TaskToml = z.infer; + +export interface TerminalBench4Task { + readonly id: string; + readonly taskToml: TaskToml; + readonly taskDir: string; + readonly instructionPath: string; + readonly composeFile: string | undefined; +} diff --git a/src/benchmarks/terminal-bench-4/session.test.ts b/src/benchmarks/terminal-bench-4/session.test.ts new file mode 100644 index 0000000..1cb0b6c --- /dev/null +++ b/src/benchmarks/terminal-bench-4/session.test.ts @@ -0,0 +1,326 @@ +import { describe, expect, it } from "bun:test"; + +import type { Effect } from "effect/Effect"; +import { either, gen, provide, runPromise } from "effect/Effect"; +import type { Layer } from "effect/Layer"; + +import type { SolverError } from "../../harness/core"; +import { assertLeft, assertRight } from "../../internal/testing"; +import type { + CreateSessionInput, + ExecResult, + SandboxSessionFactory, +} from "../harbor/sandbox"; +import { makeFakeSandboxLayer, SandboxSession } from "../harbor/sandbox"; +import type { TerminalBench4SampleMeta } from "./dataset"; +import { + agentNetworkDeviation, + artifactBundleCommand, + createAgentSession, + createVerifierSession, + runCollectHooks, + runVerifier, + sandboxCollectHooks, + transferArtifacts, +} from "./session"; + +const OK: ExecResult = { exitCode: 0, stdout: "", stderr: "" }; + +const META: TerminalBench4SampleMeta = { + taskId: "fp8-rmsnorm-gemm", + maxAgentTimeoutSec: 100, + maxTestTimeoutSec: 50, + category: "machine-learning", + agentEnv: { + cpus: 8, + memoryMb: 16384, + gpu: "H100", + env: { HF_HOME: "/cache" }, + allowInternet: false, + }, + verifierEnv: { + cpus: 4, + memoryMb: 8192, + env: { TB_SEED: "1" }, + allowInternet: true, + }, + artifacts: ["/app/out"], + collect: [], +}; + +interface Recorded { + readonly creates: CreateSessionInput[]; + readonly execs: { + argv: readonly string[]; + env: Readonly>; + timeoutMs: number | undefined; + }[]; + readonly uploads: { localPath: string; remotePath: string }[]; + readonly downloads: { remotePath: string; localPath: string }[]; +} + +function makeFactory( + recorded: Recorded, + execHandler: (argv: readonly string[]) => ExecResult = () => OK +) { + return makeFakeSandboxLayer({ + onCreate: (input) => { + recorded.creates.push(input); + }, + onUploadFile: (localPath, remotePath) => { + recorded.uploads.push({ localPath, remotePath }); + }, + onDownloadFile: (remotePath, localPath) => { + recorded.downloads.push({ remotePath, localPath }); + }, + execHandler: (argv, env, timeoutMs) => { + recorded.execs.push({ argv, env, timeoutMs }); + return execHandler(argv); + }, + }); +} + +function emptyRecorded(): Recorded { + return { creates: [], execs: [], uploads: [], downloads: [] }; +} + +function withFactory( + layer: Layer, + body: (factory: SandboxSessionFactory) => Effect +): Promise { + return runPromise( + gen(function* () { + const factory = yield* SandboxSession; + return yield* body(factory); + }).pipe(provide(layer)) + ); +} + +describe("terminal-bench-4 sandbox creation", () => { + it("passes gpu, env and resources into the agent sandbox and forces internet on", async () => { + const recorded = emptyRecorded(); + await withFactory(makeFactory(recorded), (sessionFactory) => + createAgentSession({ + sessionFactory, + meta: META, + tasksDir: "/tasks", + imageTag: "repo/task:abc", + imageBuildSteps: ["RUN echo hi"], + }) + ); + expect(recorded.creates).toHaveLength(1); + const input = recorded.creates[0]; + expect(input?.imageTag).toBe("repo/task:abc"); + expect(input?.imageBuildSteps).toEqual(["RUN echo hi"]); + expect(input?.cpus).toBe(8); + expect(input?.memoryMb).toBe(16384); + expect(input?.gpu).toBe("H100"); + expect(input?.env).toEqual({ HF_HOME: "/cache" }); + expect(input?.allowInternet).toBe(true); + expect(input?.timeoutSec).toBe(400); + expect(input?.uploads).toEqual([ + { + localPath: "/tasks/fp8-rmsnorm-gemm/instruction.md", + remotePath: "/instruction.md", + kind: "file", + }, + ]); + }); + + it("creates the verifier sandbox from the verifier resources without a gpu", async () => { + const recorded = emptyRecorded(); + await withFactory(makeFactory(recorded), (sessionFactory) => + createVerifierSession({ + sessionFactory, + meta: META, + imageTag: "repo/task-verifier:abc", + }) + ); + const input = recorded.creates[0]; + expect(input?.imageTag).toBe("repo/task-verifier:abc"); + expect(input?.cpus).toBe(4); + expect(input?.gpu).toBeUndefined(); + expect(input?.env).toEqual({ TB_SEED: "1" }); + expect(input?.timeoutSec).toBe(350); + expect(input?.uploads).toEqual([]); + }); + + it("records a deviation only when the task disallows internet", () => { + expect(agentNetworkDeviation(META)).toEqual({ + agentNetworkForced: true, + taskAllowInternet: false, + }); + expect( + agentNetworkDeviation({ + ...META, + agentEnv: { ...META.agentEnv, allowInternet: true }, + }) + ).toEqual({}); + }); +}); + +describe("terminal-bench-4 collect hooks", () => { + it("accepts hooks for the main service", async () => { + const result = await runPromise( + either( + sandboxCollectHooks([ + { command: "a" }, + { command: "b", service: "main" }, + ]) + ) + ); + assertRight(result); + expect(result.right).toHaveLength(2); + }); + + it("rejects hooks that target a compose sidecar", async () => { + const result = await runPromise( + either(sandboxCollectHooks([{ command: "a", service: "db" }])) + ); + assertLeft(result); + expect(result.left.message).toContain('"db"'); + }); + + it("runs each hook through bash with its own timeout", async () => { + const recorded = emptyRecorded(); + await withFactory(makeFactory(recorded), (sessionFactory) => + gen(function* () { + const session = yield* sessionFactory.create({ + imageTag: "x", + timeoutSec: 1, + cpus: 1, + memoryMb: 1, + allowInternet: true, + workdir: "/", + keepAliveCommand: [], + uploads: [], + }); + yield* runCollectHooks(session, [ + { command: "echo one", timeout_sec: 5 }, + { command: "echo two" }, + ]); + return session; + }) + ); + expect(recorded.execs.map((e) => e.argv)).toEqual([ + ["bash", "-c", "echo one"], + ["bash", "-c", "echo two"], + ]); + expect(recorded.execs.map((e) => e.timeoutMs)).toEqual([5000, 300_000]); + }); +}); + +describe("terminal-bench-4 artifact bundling", () => { + it("always includes /logs/artifacts and dedupes declared sources", () => { + const cmd = artifactBundleCommand([ + "/app/out", + { source: "/app/out", exclude: ["*.tmp"] }, + { source: "/data", exclude: ["*.tmp", "cache/"] }, + ]); + expect(cmd.match(/'\/logs\/artifacts'/g)).toHaveLength(2); + expect(cmd.match(/'\/app\/out'/g)).toHaveLength(2); + expect(cmd).toContain("--exclude='*.tmp' --exclude='cache/'"); + expect(cmd).toContain("tar -cf /tmp/tb4-artifacts.tar -P"); + }); + + it("produces an empty tarball when nothing exists", () => { + expect(artifactBundleCommand([])).toContain("-T /dev/null"); + }); + + it("shell-quotes single quotes in paths", () => { + expect(artifactBundleCommand(["/it's"])).toContain(String.raw`'/it'\''s'`); + }); + + it("bundles on the agent, moves the tarball, and extracts at absolute paths on the verifier", async () => { + const recorded = emptyRecorded(); + await withFactory(makeFactory(recorded), (sessionFactory) => + gen(function* () { + const create = (tag: string) => + sessionFactory.create({ + imageTag: tag, + timeoutSec: 1, + cpus: 1, + memoryMb: 1, + allowInternet: true, + workdir: "/", + keepAliveCommand: [], + uploads: [], + }); + const agent = yield* create("agent"); + const verifier = yield* create("verifier"); + yield* transferArtifacts({ agent, verifier, artifacts: ["/app/out"] }); + return agent; + }) + ); + expect(recorded.downloads).toHaveLength(1); + expect(recorded.downloads[0]?.remotePath).toBe("/tmp/tb4-artifacts.tar"); + expect(recorded.uploads).toHaveLength(1); + expect(recorded.uploads[0]?.remotePath).toBe("/tmp/tb4-artifacts.tar"); + expect(recorded.uploads[0]?.localPath).toBe( + recorded.downloads[0]?.localPath + ); + const extract = recorded.execs.at(-1); + expect(extract?.argv[2]).toBe( + "tar -xf /tmp/tb4-artifacts.tar -P -C / && rm -f /tmp/tb4-artifacts.tar" + ); + }); +}); + +describe("terminal-bench-4 verifier", () => { + async function runWith( + execHandler: (argv: readonly string[]) => ExecResult + ): Promise<{ + recorded: Recorded; + result: { reward: number; output: string }; + }> { + const recorded = emptyRecorded(); + const result = await withFactory( + makeFactory(recorded, execHandler), + (sessionFactory) => + gen(function* () { + const verifier = yield* sessionFactory.create({ + imageTag: "v", + timeoutSec: 1, + cpus: 1, + memoryMb: 1, + allowInternet: true, + workdir: "/", + keepAliveCommand: [], + uploads: [], + }); + return yield* runVerifier(verifier, META); + }) + ); + return { recorded, result }; + } + + it("runs /tests/test.sh with the task timeout and reads reward.txt", async () => { + const { recorded, result } = await runWith((argv) => + argv[0] === "cat" + ? { exitCode: 0, stdout: "1\n", stderr: "" } + : { exitCode: 0, stdout: "PASS", stderr: "warn" } + ); + expect(recorded.execs[0]?.argv).toEqual([ + "bash", + "-c", + "mkdir -p /logs/verifier && bash /tests/test.sh", + ]); + expect(recorded.execs[0]?.timeoutMs).toBe(55_000); + expect(recorded.execs[1]?.argv).toEqual([ + "cat", + "/logs/verifier/reward.txt", + ]); + expect(result).toEqual({ reward: 1, output: "PASS\nwarn" }); + }); + + it("scores a missing or non-passing reward as zero", async () => { + const missing = await runWith((argv) => + argv[0] === "cat" ? { exitCode: 1, stdout: "", stderr: "no file" } : OK + ); + expect(missing.result.reward).toBe(0); + const partial = await runWith((argv) => + argv[0] === "cat" ? { exitCode: 0, stdout: "0.5", stderr: "" } : OK + ); + expect(partial.result.reward).toBe(0); + }); +}); diff --git a/src/benchmarks/terminal-bench-4/session.ts b/src/benchmarks/terminal-bench-4/session.ts new file mode 100644 index 0000000..72529b7 --- /dev/null +++ b/src/benchmarks/terminal-bench-4/session.ts @@ -0,0 +1,218 @@ +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import type { Effect } from "effect/Effect"; +import { fail, gen, succeed } from "effect/Effect"; + +import { SolverError } from "../../harness/core"; +import { parseReward } from "../harbor/reward"; +import type { + SandboxSessionFactory, + SandboxSessionInstance, +} from "../harbor/sandbox"; +import { REMOTE_VERIFIER_SCRIPT } from "../harbor/sandbox"; +import type { TerminalBench4SampleMeta } from "./dataset"; +import type { Artifact, CollectHook } from "./schema"; + +export const CONTAINER_WORKDIR = "/app" as const; + +export const REMOTE_INSTRUCTION = "/instruction.md" as const; + +export const REMOTE_REWARD_PATH = "/logs/verifier/reward.txt" as const; + +export const REMOTE_ARTIFACTS_DIR = "/logs/artifacts" as const; + +export const KEEP_ALIVE_COMMAND = ["sleep", "infinity"] as const; + +const REMOTE_ARTIFACT_BUNDLE = "/tmp/tb4-artifacts.tar" as const; + +const SANDBOX_TIMEOUT_MARGIN_SEC = 300; + +const VERIFIER_TIMEOUT_MARGIN_MS = 5000; + +const REWARD_READ_TIMEOUT_MS = 10_000; + +const ARTIFACT_BUNDLE_TIMEOUT_MS = 600_000; + +const DEFAULT_COLLECT_TIMEOUT_SEC = 300; + +export function agentNetworkDeviation( + meta: TerminalBench4SampleMeta +): Readonly> { + return meta.agentEnv.allowInternet + ? {} + : { agentNetworkForced: true, taskAllowInternet: false }; +} + +export function createAgentSession(input: { + readonly sessionFactory: SandboxSessionFactory; + readonly meta: TerminalBench4SampleMeta; + readonly tasksDir: string; + readonly imageTag: string; + readonly imageBuildSteps: readonly string[]; +}): Effect { + const { sessionFactory, meta, tasksDir, imageTag, imageBuildSteps } = input; + return sessionFactory.create({ + imageTag, + imageBuildSteps, + timeoutSec: meta.maxAgentTimeoutSec + SANDBOX_TIMEOUT_MARGIN_SEC, + ...meta.agentEnv, + allowInternet: true, + workdir: CONTAINER_WORKDIR, + keepAliveCommand: KEEP_ALIVE_COMMAND, + uploads: [ + { + localPath: join(tasksDir, meta.taskId, "instruction.md"), + remotePath: REMOTE_INSTRUCTION, + kind: "file", + }, + ], + }); +} + +export function createVerifierSession(input: { + readonly sessionFactory: SandboxSessionFactory; + readonly meta: TerminalBench4SampleMeta; + readonly imageTag: string; +}): Effect { + const { sessionFactory, meta, imageTag } = input; + return sessionFactory.create({ + imageTag, + timeoutSec: meta.maxTestTimeoutSec + SANDBOX_TIMEOUT_MARGIN_SEC, + ...meta.verifierEnv, + workdir: CONTAINER_WORKDIR, + keepAliveCommand: KEEP_ALIVE_COMMAND, + uploads: [], + }); +} + +export function sandboxCollectHooks( + hooks: readonly CollectHook[] +): Effect { + const sidecar = hooks.find( + (h) => h.service !== undefined && h.service !== "main" + ); + if (sidecar !== undefined) { + return fail( + new SolverError({ + message: `collect hook targets compose service "${sidecar.service}", which is unsupported`, + }) + ); + } + return succeed(hooks); +} + +export function runCollectHooks( + session: SandboxSessionInstance, + hooks: readonly CollectHook[] +): Effect { + return gen(function* () { + for (const hook of hooks) { + yield* session.exec( + ["bash", "-c", hook.command], + {}, + Math.round((hook.timeout_sec ?? DEFAULT_COLLECT_TIMEOUT_SEC) * 1000) + ); + } + }); +} + +function artifactSource(artifact: Artifact): string { + return typeof artifact === "string" ? artifact : artifact.source; +} + +function artifactExcludes(artifact: Artifact): readonly string[] { + return typeof artifact === "string" ? [] : artifact.exclude; +} + +function shellQuote(value: string): string { + return `'${value.replaceAll("'", String.raw`'\''`)}'`; +} + +export function artifactBundleCommand(artifacts: readonly Artifact[]): string { + const sources = [ + REMOTE_ARTIFACTS_DIR, + ...artifacts.map(artifactSource), + ].filter((s, i, all) => all.indexOf(s) === i); + const excludes = artifacts + .flatMap(artifactExcludes) + .filter((s, i, all) => all.indexOf(s) === i) + .map((pattern) => `--exclude=${shellQuote(pattern)}`); + const existing = sources + .map( + (s) => + `if [ -e ${shellQuote(s)} ]; then printf '%s\\n' ${shellQuote(s)}; fi` + ) + .join("; "); + return [ + `rm -f ${REMOTE_ARTIFACT_BUNDLE}`, + `{ ${existing}; } > /tmp/tb4-artifact-list.txt`, + `if [ -s /tmp/tb4-artifact-list.txt ]; then tar -cf ${REMOTE_ARTIFACT_BUNDLE} -P ${excludes.join(" ")} -T /tmp/tb4-artifact-list.txt; else tar -cf ${REMOTE_ARTIFACT_BUNDLE} -T /dev/null; fi`, + ].join(" && "); +} + +export function transferArtifacts(input: { + readonly agent: SandboxSessionInstance; + readonly verifier: SandboxSessionInstance; + readonly artifacts: readonly Artifact[]; +}): Effect { + const { agent, verifier, artifacts } = input; + return gen(function* () { + yield* agent.exec( + ["bash", "-c", artifactBundleCommand(artifacts)], + {}, + ARTIFACT_BUNDLE_TIMEOUT_MS + ); + const staging = mkdtempSync(join(tmpdir(), "tb4-artifacts-")); + const localBundle = join(staging, "artifacts.tar"); + try { + yield* agent.downloadFile(REMOTE_ARTIFACT_BUNDLE, localBundle); + yield* verifier.uploadFile(localBundle, REMOTE_ARTIFACT_BUNDLE); + } finally { + rmSync(staging, { recursive: true, force: true }); + } + yield* verifier.exec( + [ + "bash", + "-c", + `tar -xf ${REMOTE_ARTIFACT_BUNDLE} -P -C / && rm -f ${REMOTE_ARTIFACT_BUNDLE}`, + ], + {}, + ARTIFACT_BUNDLE_TIMEOUT_MS + ); + }); +} + +export interface TerminalBench4VerifierResult { + readonly reward: number; + readonly output: string; +} + +export function runVerifier( + verifier: SandboxSessionInstance, + meta: TerminalBench4SampleMeta +): Effect { + const verifierTimeoutMs = + Math.round(meta.maxTestTimeoutSec * 1000) + VERIFIER_TIMEOUT_MARGIN_MS; + return gen(function* () { + const run = yield* verifier.exec( + [ + "bash", + "-c", + `mkdir -p /logs/verifier && bash ${REMOTE_VERIFIER_SCRIPT}`, + ], + {}, + verifierTimeoutMs + ); + const rewardRead = yield* verifier.exec( + ["cat", REMOTE_REWARD_PATH], + {}, + REWARD_READ_TIMEOUT_MS + ); + return { + reward: parseReward(rewardRead.stdout), + output: `${run.stdout}\n${run.stderr}`.trim(), + }; + }); +} diff --git a/src/benchmarks/terminal-bench-4/solver.ts b/src/benchmarks/terminal-bench-4/solver.ts new file mode 100644 index 0000000..8ac7601 --- /dev/null +++ b/src/benchmarks/terminal-bench-4/solver.ts @@ -0,0 +1,129 @@ +import { gen, tryPromise } from "effect/Effect"; + +import type { ModelMessage, ModelUsage } from "../../harness/core"; +import { MessageRole, SolverError } from "../../harness/core"; +import type { SolverService } from "../../harness/solver"; +import type { OriHarnessDef } from "../agent-cli/harness"; +import type { AgentCliOpts } from "../agent-cli/runner"; +import { + agentCliMetadata, + agentImageBuildSteps, + runAgentCli, +} from "../agent-cli/runner"; +import type { SandboxSessionFactory } from "../harbor/sandbox"; +import { readTerminalBench4Meta } from "./dataset"; +import { imageTags } from "./images"; +import { + agentNetworkDeviation, + createAgentSession, + createVerifierSession, + REMOTE_INSTRUCTION, + runCollectHooks, + runVerifier, + sandboxCollectHooks, + transferArtifacts, +} from "./session"; +import { ensureTasksCheckedOut, tasksDir } from "./tasks-source"; + +const AGENT_TIMEOUT_MARGIN_MS = 30_000; + +const ZERO_USAGE: ModelUsage = { + inputTokens: 0, + outputTokens: 0, + totalTokens: 0, + reasoningTokens: 0, + totalCost: 0, +}; + +export interface TerminalBench4SolverOpts extends AgentCliOpts { + readonly imageRepo: string; +} + +export function terminalBench4Solver( + sessionFactory: SandboxSessionFactory, + opts: TerminalBench4SolverOpts, + harness: OriHarnessDef +): SolverService { + return (state) => + gen(function* () { + const meta = readTerminalBench4Meta(state.sample.metadata); + if (meta === undefined) { + return yield* new SolverError({ + message: `terminal-bench-4 solver received a sample without terminal-bench-4 metadata (id=${state.sample.id})`, + }); + } + const collectHooks = yield* sandboxCollectHooks(meta.collect); + const root = yield* tryPromise({ + try: () => ensureTasksCheckedOut(), + catch: (e: unknown) => + new SolverError({ + message: `Failed to check out terminal-bench-4 tasks: ${String(e)}`, + }), + }); + const images = imageTags(opts.imageRepo, meta.taskId); + const agent = yield* createAgentSession({ + sessionFactory, + meta, + tasksDir: tasksDir(root), + imageTag: images.agent, + imageBuildSteps: agentImageBuildSteps(harness, opts), + }); + try { + const run = yield* runAgentCli({ + session: agent, + harness, + opts, + instructionPath: REMOTE_INSTRUCTION, + timeoutMs: meta.maxAgentTimeoutSec * 1000 + AGENT_TIMEOUT_MARGIN_MS, + }); + yield* runCollectHooks(agent, collectHooks); + const verifier = yield* createVerifierSession({ + sessionFactory, + meta, + imageTag: images.verifier, + }); + try { + yield* transferArtifacts({ + agent, + verifier, + artifacts: meta.artifacts, + }); + yield* agent.destroy(); + const testResult = yield* runVerifier(verifier, meta); + const testOutput = run.failureDetail + ? `${run.failureDetail}\n\n${testResult.output}` + : testResult.output; + const completion = run.finalText ?? run.rawStream; + const messages: ModelMessage[] = [ + { role: MessageRole.User, content: state.sample.input }, + ...run.assistantMessages, + ]; + return { + sample: { + ...state.sample, + metadata: { + ...state.sample.metadata, + reward: testResult.reward, + testOutput, + ...agentCliMetadata(harness.id, run), + ...agentNetworkDeviation(meta), + }, + }, + messages, + responseItems: run.responseItems, + output: { + completion, + message: { role: MessageRole.Assistant, content: completion }, + usage: run.usage ?? ZERO_USAGE, + generationTimeMs: run.generationTimeMs ?? 0, + }, + completed: true, + }; + } finally { + yield* verifier.destroy(); + } + } finally { + yield* agent.destroy(); + } + }); +} diff --git a/src/benchmarks/terminal-bench-4/tasks-source.ts b/src/benchmarks/terminal-bench-4/tasks-source.ts new file mode 100644 index 0000000..6450cbf --- /dev/null +++ b/src/benchmarks/terminal-bench-4/tasks-source.ts @@ -0,0 +1,31 @@ +import { join } from "node:path"; + +import { makeTasksSource } from "../harbor/tasks-source"; + +export const TERMINAL_BENCH_4_SOURCE_REPO = + "https://github.com/harbor-framework/terminal-bench.git" as const; + +export const TERMINAL_BENCH_4_SOURCE_COMMIT = + "452bf305c6daa62fc59061d22133a7cbc7c1572e" as const; + +export const TERMINAL_BENCH_4_TASKS_SUBDIR = "tasks" as const; + +const source = makeTasksSource({ + label: "terminal-bench-4", + repoUrl: TERMINAL_BENCH_4_SOURCE_REPO, + commit: TERMINAL_BENCH_4_SOURCE_COMMIT, + tasksSubdir: TERMINAL_BENCH_4_TASKS_SUBDIR, + envVar: "BENCH_TERMINAL_BENCH_4_TASKS_DIR", + tmpPrefix: "terminal-bench-4-tasks-", +}); + +export const { + ensureTasksCheckedOut, + ensureTasksCheckedOutEffect, + seedTasksRoot, + resetCheckoutCache, +} = source; + +export function tasksDir(root: string): string { + return join(root, TERMINAL_BENCH_4_TASKS_SUBDIR); +} diff --git a/src/cli/index.ts b/src/cli/index.ts index 922f378..e3ba0bb 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -471,6 +471,16 @@ export function buildBenchmarkConfig(opts: { reasoningEffort, }); } + case "terminal_bench_4": { + return buildSchemaValidatedConfig({ + benchmarkId: "terminal_bench_4", + model: requireModel("terminal_bench_4", model), + endpointId, + panelConfig, + costTier, + reasoningEffort, + }); + } case "draco": { const panel = parseSchema(DracoPanelConfigSchema, panelConfig); if (Either.isLeft(panel)) { From fc97db9e24c63bfe5544dc4dc34c087c52071f82 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 3 Sep 2026 20:27:42 +0000 Subject: [PATCH 2/6] feat: resolve terminal-bench-4 images by Modal image id and harden sandbox lifetimes Co-Authored-By: Abhinav Pola --- scripts/build-terminal-bench-4-images.py | 148 +++++++++++++ scripts/build-terminal-bench-4-images.ts | 196 ------------------ src/benchmarks/benchmark-config.ts | 5 +- src/benchmarks/harbor/modal-sandbox.ts | 19 +- src/benchmarks/harbor/sandbox.ts | 9 + src/benchmarks/terminal-bench-4/README.md | 22 +- .../terminal-bench-4/benchmark.test.ts | 48 ++++- src/benchmarks/terminal-bench-4/benchmark.ts | 1 - .../terminal-bench-4/image-ids.json | 9 + src/benchmarks/terminal-bench-4/images.ts | 58 ++++-- .../terminal-bench-4/session.test.ts | 123 ++++++++++- src/benchmarks/terminal-bench-4/session.ts | 70 +++++-- src/benchmarks/terminal-bench-4/solver.ts | 17 +- 13 files changed, 467 insertions(+), 258 deletions(-) create mode 100644 scripts/build-terminal-bench-4-images.py delete mode 100644 scripts/build-terminal-bench-4-images.ts create mode 100644 src/benchmarks/terminal-bench-4/image-ids.json diff --git a/scripts/build-terminal-bench-4-images.py b/scripts/build-terminal-bench-4-images.py new file mode 100644 index 0000000..1484902 --- /dev/null +++ b/scripts/build-terminal-bench-4-images.py @@ -0,0 +1,148 @@ +"""Build Terminal-Bench 4.0 task images into Modal's image store. + +Usage: + python3 scripts/build-terminal-bench-4-images.py --tasks-dir /path/to/terminal-bench/tasks \ + [--task NAME ...] [--out src/benchmarks/terminal-bench-4/image-ids.json] [--dry-run] + +Requires MODAL_TOKEN_ID and MODAL_TOKEN_SECRET (or a Modal CLI profile). For each +runnable (non docker-compose) task this builds environment/Dockerfile and +tests/Dockerfile with their directories as build context, then records the +resulting Modal image IDs keyed by task id. The output JSON is what the harness +reads at run time, so re-run this whenever TERMINAL_BENCH_4_SOURCE_COMMIT changes. + +MODAL_IMAGE_BUILDER_VERSION defaults to 2025.06. Older builder versions pip-install +Modal's client dependencies into the image, which would alter task environments. +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import subprocess +import sys +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +os.environ.setdefault("MODAL_IMAGE_BUILDER_VERSION", "2025.06") + +import modal # noqa: E402 + +APP_NAME = "terminal-bench-4-images" +COMPOSE_FILE = "environment/docker-compose.yaml" +DEFAULT_OUT = Path("src/benchmarks/terminal-bench-4/image-ids.json") +SOURCE_RE = re.compile(r'TERMINAL_BENCH_4_SOURCE_COMMIT\s*=\s*"([0-9a-f]{40})"') + + +def pinned_commit() -> str: + text = Path("src/benchmarks/terminal-bench-4/tasks-source.ts").read_text() + match = SOURCE_RE.search(text) + if match is None: + sys.exit("could not read TERMINAL_BENCH_4_SOURCE_COMMIT from tasks-source.ts") + return match.group(1) + + +def checkout_commit(tasks_dir: Path) -> str: + out = subprocess.run( + ["git", "-C", str(tasks_dir), "rev-parse", "HEAD"], + check=True, + capture_output=True, + text=True, + ) + return out.stdout.strip() + + +def runnable_tasks(tasks_dir: Path) -> list[str]: + return sorted( + p.name + for p in tasks_dir.iterdir() + if p.is_dir() + and not p.name.startswith(".") + and (p / "task.toml").is_file() + and not (p / COMPOSE_FILE).is_file() + ) + + +def load_existing(out: Path, commit: str) -> dict[str, dict[str, str]]: + if not out.is_file(): + return {} + data = json.loads(out.read_text()) + if data.get("sourceCommit") != commit: + return {} + return dict(data.get("images", {})) + + +def write_out(out: Path, commit: str, images: dict[str, dict[str, str]]) -> None: + payload = { + "sourceCommit": commit, + "images": {k: images[k] for k in sorted(images)}, + } + out.write_text(json.dumps(payload, indent=2) + "\n") + + +def build_task(app: modal.App, tasks_dir: Path, task: str) -> dict[str, str]: + task_dir = tasks_dir / task + agent = modal.Image.from_dockerfile( + task_dir / "environment" / "Dockerfile", + context_dir=task_dir / "environment", + ) + verifier = modal.Image.from_dockerfile( + task_dir / "tests" / "Dockerfile", + context_dir=task_dir / "tests", + ) + agent = agent.build(app) + verifier = verifier.build(app) + return {"agent": agent.object_id, "verifier": verifier.object_id} + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--tasks-dir", type=Path, required=True) + parser.add_argument("--task", action="append", default=[]) + parser.add_argument("--out", type=Path, default=DEFAULT_OUT) + parser.add_argument("--concurrency", type=int, default=4) + parser.add_argument("--dry-run", action="store_true") + args = parser.parse_args() + + tasks_dir: Path = args.tasks_dir.resolve() + commit = pinned_commit() + actual = checkout_commit(tasks_dir.parent) + if actual != commit: + sys.exit(f"tasks dir is at {actual}, expected pinned {commit}") + + runnable = runnable_tasks(tasks_dir) + selected = args.task or runnable + unknown = sorted(set(selected) - set(runnable)) + if unknown: + sys.exit(f"unknown or docker-compose tasks: {', '.join(unknown)}") + + if args.dry_run: + for task in selected: + print(task) + return 0 + + images = load_existing(args.out, commit) + failures: dict[str, str] = {} + app = modal.App.lookup(APP_NAME, create_if_missing=True) + + def run(task: str) -> None: + try: + images[task] = build_task(app, tasks_dir, task) + print(f"built {task}: {images[task]}", flush=True) + except Exception as error: # noqa: BLE001 + failures[task] = str(error) + print(f"FAILED {task}: {error}", file=sys.stderr, flush=True) + + with modal.enable_output(), ThreadPoolExecutor(max_workers=args.concurrency) as pool: + list(pool.map(run, selected)) + + write_out(args.out, commit, images) + if failures: + print(f"{len(failures)} task(s) failed: {', '.join(sorted(failures))}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/build-terminal-bench-4-images.ts b/scripts/build-terminal-bench-4-images.ts deleted file mode 100644 index d3dc04b..0000000 --- a/scripts/build-terminal-bench-4-images.ts +++ /dev/null @@ -1,196 +0,0 @@ -import { existsSync } from "node:fs"; -import { join } from "node:path"; - -import { listTaskIds } from "../src/benchmarks/terminal-bench-4/dataset"; -import { - DEFAULT_TERMINAL_BENCH_4_IMAGE_REPO, - imageTags, -} from "../src/benchmarks/terminal-bench-4/images"; -import { - ensureTasksCheckedOut, - tasksDir, -} from "../src/benchmarks/terminal-bench-4/tasks-source"; - -interface BuildOptions { - readonly repo: string; - readonly tasks: readonly string[]; - readonly dryRun: boolean; - readonly concurrency: number; -} - -interface ImageBuild { - readonly tag: string; - readonly contextDir: string; - readonly dockerfile: string; -} - -function requireValue(flag: string, value: string | undefined): string { - if (value === undefined) { - throw new Error(`${flag} requires a value`); - } - return value; -} - -function parseConcurrency(raw: string): number { - const value = Number.parseInt(raw, 10); - if (!Number.isInteger(value) || value < 1) { - throw new Error(`invalid --concurrency: ${raw}`); - } - return value; -} - -function parseArgs(argv: readonly string[]): BuildOptions { - let repo: string = DEFAULT_TERMINAL_BENCH_4_IMAGE_REPO; - const tasks: string[] = []; - let dryRun = false; - let concurrency = 2; - for (let i = 0; i < argv.length; i++) { - const arg = argv[i]; - const next = argv[i + 1]; - switch (arg) { - case "--repo": { - repo = requireValue(arg, next); - i++; - break; - } - case "--task": { - tasks.push(requireValue(arg, next)); - i++; - break; - } - case "--concurrency": { - concurrency = parseConcurrency(requireValue(arg, next)); - i++; - break; - } - case "--dry-run": { - dryRun = true; - break; - } - default: { - throw new Error(`unknown argument: ${arg}`); - } - } - } - return { repo, tasks, dryRun, concurrency }; -} - -export function taskImageBuilds( - repo: string, - tasksRoot: string, - taskId: string -): readonly ImageBuild[] { - const taskDir = join(tasksRoot, taskId); - const tags = imageTags(repo, taskId); - return [ - { - tag: tags.agent, - contextDir: join(taskDir, "environment"), - dockerfile: join(taskDir, "environment", "Dockerfile"), - }, - { - tag: tags.verifier, - contextDir: join(taskDir, "tests"), - dockerfile: join(taskDir, "tests", "Dockerfile"), - }, - ]; -} - -async function run(cmd: string, args: readonly string[]): Promise { - const proc = Bun.spawn([cmd, ...args], { - stdout: "inherit", - stderr: "inherit", - }); - const code = await proc.exited; - if (code !== 0) { - throw new Error(`${cmd} ${args.join(" ")} exited with ${code}`); - } -} - -async function buildAndPush(build: ImageBuild, dryRun: boolean): Promise { - if (!existsSync(build.dockerfile)) { - throw new Error(`missing Dockerfile: ${build.dockerfile}`); - } - const args = [ - "buildx", - "build", - "--platform", - "linux/amd64", - "--file", - build.dockerfile, - "--tag", - build.tag, - "--push", - build.contextDir, - ]; - if (dryRun) { - console.log(`docker ${args.join(" ")}`); - return; - } - - await run("docker", args); -} - -async function runBounded( - jobs: readonly (() => Promise)[], - concurrency: number -): Promise[]> { - const results: PromiseSettledResult[] = new Array(jobs.length); - let next = 0; - const workers = Array.from( - { length: Math.min(concurrency, jobs.length) }, - async () => { - while (next < jobs.length) { - const idx = next++; - const job = jobs[idx]; - if (job === undefined) { - return; - } - try { - await job(); - results[idx] = { status: "fulfilled", value: undefined }; - } catch (reason) { - results[idx] = { status: "rejected", reason }; - } - } - } - ); - await Promise.all(workers); - return results; -} - -async function main(): Promise { - const opts = parseArgs(process.argv.slice(2)); - const root = ensureTasksCheckedOut(); - const tasksRoot = tasksDir(root); - const taskIds = listTaskIds(tasksRoot, opts.tasks); - const missing = opts.tasks.filter((id) => !taskIds.includes(id)); - if (missing.length > 0) { - throw new Error( - `unknown or compose-only tasks: ${missing.join(", ")} (compose tasks are unsupported)` - ); - } - const builds = taskIds.flatMap((id) => - taskImageBuilds(opts.repo, tasksRoot, id) - ); - console.log( - `${opts.dryRun ? "planning" : "building"} ${builds.length} images for ${taskIds.length} tasks -> ${opts.repo}` - ); - const results = await runBounded( - builds.map((b) => () => buildAndPush(b, opts.dryRun)), - opts.concurrency - ); - const failed = results.flatMap((r, i) => - r.status === "rejected" ? [{ build: builds[i], reason: r.reason }] : [] - ); - for (const f of failed) { - console.error(`FAILED ${f.build?.tag}: ${String(f.reason)}`); - } - if (failed.length > 0) { - process.exitCode = 1; - } -} - -if (import.meta.main) { - await main(); -} diff --git a/src/benchmarks/benchmark-config.ts b/src/benchmarks/benchmark-config.ts index c767b33..d5e4afd 100644 --- a/src/benchmarks/benchmark-config.ts +++ b/src/benchmarks/benchmark-config.ts @@ -25,7 +25,6 @@ import { DracoPanelConfigSchema } from "./draco/schemas"; import { SearchLaneConfigSchema } from "./search/core/config"; import { DEFAULT_JUDGE_MODEL, DEFAULT_STEP_LIMIT } from "./swe-atlas/schema"; import { BankingRetrievalConfigSchema } from "./tau3-bench-banking/retrieval-config"; -import { DEFAULT_TERMINAL_BENCH_4_IMAGE_REPO } from "./terminal-bench-4/images"; import { DEFAULT_TERMINAL_BENCH_AGENT, TERMINAL_BENCH_AGENTS, @@ -169,9 +168,7 @@ export const TerminalBenchConfigSchema = z.object({ export type TerminalBenchConfig = z.infer; -export const TerminalBench4OptionsSchema = TerminalBenchOptionsSchema.extend({ - imageRepo: z.string().min(1).default(DEFAULT_TERMINAL_BENCH_4_IMAGE_REPO), -}); +export const TerminalBench4OptionsSchema = TerminalBenchOptionsSchema; export const TerminalBench4ConfigSchema = z.object({ benchmarkId: z.literal("terminal_bench_4"), diff --git a/src/benchmarks/harbor/modal-sandbox.ts b/src/benchmarks/harbor/modal-sandbox.ts index 38b2561..11a03b5 100644 --- a/src/benchmarks/harbor/modal-sandbox.ts +++ b/src/benchmarks/harbor/modal-sandbox.ts @@ -16,7 +16,12 @@ import type { SandboxSessionInstance, UploadSpec, } from "./sandbox"; -import { SandboxSession, makeSessionInstance, toSolverError } from "./sandbox"; +import { + SANDBOX_IMAGE_KINDS, + SandboxSession, + makeSessionInstance, + toSolverError, +} from "./sandbox"; export interface ModalSandboxConfig { readonly appName: string; @@ -64,7 +69,17 @@ export function makeModalSandboxLayer( try: () => getApp(), catch: (e) => toSolverError("Failed to resolve Modal app", e), }); - const baseImage = client.images.fromRegistry(input.imageTag); + const baseImage = + input.imageKind === SANDBOX_IMAGE_KINDS.ModalImageId + ? yield* tryPromise({ + try: () => client.images.fromId(input.imageTag), + catch: (e: unknown) => + toSolverError( + `Failed to resolve Modal image ${input.imageTag}`, + e + ), + }) + : client.images.fromRegistry(input.imageTag); const image = input.imageBuildSteps !== undefined && input.imageBuildSteps.length > 0 ? baseImage.dockerfileCommands([...input.imageBuildSteps]) diff --git a/src/benchmarks/harbor/sandbox.ts b/src/benchmarks/harbor/sandbox.ts index e251b7f..111162d 100644 --- a/src/benchmarks/harbor/sandbox.ts +++ b/src/benchmarks/harbor/sandbox.ts @@ -46,8 +46,17 @@ export interface SandboxSessionInstance { readonly destroy: () => Effect; } +export const SANDBOX_IMAGE_KINDS = { + Registry: "registry", + ModalImageId: "modal-image-id", +} as const; + +export type SandboxImageKind = + (typeof SANDBOX_IMAGE_KINDS)[keyof typeof SANDBOX_IMAGE_KINDS]; + export interface CreateSessionInput { readonly imageTag: string; + readonly imageKind?: SandboxImageKind; readonly imageBuildSteps?: readonly string[]; readonly timeoutSec: number; readonly cpus: number; diff --git a/src/benchmarks/terminal-bench-4/README.md b/src/benchmarks/terminal-bench-4/README.md index a9d3517..78c13bf 100644 --- a/src/benchmarks/terminal-bench-4/README.md +++ b/src/benchmarks/terminal-bench-4/README.md @@ -31,13 +31,15 @@ Three tasks declare one H100 each: `fp8-rmsnorm-gemm`, `jax-speedrun-gpu`, `math ## Images -4.0 tasks ship Dockerfiles with local-context `COPY` steps rather than prebuilt image names, and the Modal SDK rejects those Dockerfiles. Images are therefore prebuilt and pushed once per pinned commit: +4.0 tasks ship Dockerfiles with local-context `COPY` steps rather than prebuilt image names, and the Modal JS SDK rejects those Dockerfiles. Images are built once per pinned commit with the Modal Python SDK (which supports a local build context) and stored in Modal's image store, so no external registry is involved: ```bash -bun scripts/build-terminal-bench-4-images.ts --repo [--task ] [--concurrency 2] [--dry-run] +pip install modal +MODAL_TOKEN_ID=... MODAL_TOKEN_SECRET=... \ + python3 scripts/build-terminal-bench-4-images.py --tasks-dir /tasks [--task ] [--concurrency 4] [--dry-run] ``` -Tags are `/:` and `/-verifier:` (`images.ts`). The run config `imageRepo` selects the registry prefix and defaults to `DEFAULT_TERMINAL_BENCH_4_IMAGE_REPO`. +The script writes `image-ids.json` (`{ sourceCommit, images: { : { agent, verifier } } }`) with Modal image IDs. `images.ts` validates it at load time and refuses a map whose `sourceCommit` differs from `TERMINAL_BENCH_4_SOURCE_COMMIT`; the solver fails a sample whose task has no entry. Sandboxes are created with `imageKind: "modal-image-id"`, which resolves the ID with `images.fromId` before layering the agent install steps. Images built into the `terminal-bench-4-images` Modal app are visible to the run-time app because Modal image IDs are workspace-scoped. ## Excluded tasks @@ -47,10 +49,14 @@ Eleven tasks use `environment/docker-compose.yaml` and need more than one contai A score over the 55 runnable tasks is not an official Terminal-Bench 4.0 number. -## Config options +## Sandbox lifetimes + +The agent sandbox outlives `maxAgentTimeoutSec` by the sum of collect-hook timeouts plus the artifact bundle, transfer and extract budgets plus a fixed margin (`agentSandboxTimeoutSec`). The verifier sandbox is created before artifact transfer, so its lifetime covers the same artifact budgets plus `maxTestTimeoutSec` (`verifierSandboxTimeoutSec`). + +## Artifact semantics -Same agent options as `terminal_bench` (`TerminalBenchOptionsSchema`) plus: +Harbor empties directory artifact targets in the verifier before uploading the agent's copy. `artifactExtractCommand` mirrors this: any declared source that appears as a directory in the bundle is removed on the verifier before extraction, so files the agent deleted do not survive. Excluded patterns are simply absent from the bundle; verifiers that need a pristine copy of an excluded file restore it themselves (see `vpp-loss-divergence`). + +## Config options -| option | type | default | description | -| --- | --- | --- | --- | -| `imageRepo` | string | `DEFAULT_TERMINAL_BENCH_4_IMAGE_REPO` | Registry prefix holding the prebuilt agent and verifier images. | +Same agent options as `terminal_bench` (`TerminalBenchOptionsSchema`). diff --git a/src/benchmarks/terminal-bench-4/benchmark.test.ts b/src/benchmarks/terminal-bench-4/benchmark.test.ts index b750e3c..2373ab1 100644 --- a/src/benchmarks/terminal-bench-4/benchmark.test.ts +++ b/src/benchmarks/terminal-bench-4/benchmark.test.ts @@ -7,7 +7,7 @@ import { getBenchmarkMeta, TERMINAL_BENCH_4_META } from "../benchmark-meta"; import { getBenchmark } from "../registry"; import { TERMINAL_BENCH_BENCHMARK } from "../terminal-bench/benchmark"; import { TERMINAL_BENCH_4_BENCHMARK } from "./benchmark"; -import { DEFAULT_TERMINAL_BENCH_4_IMAGE_REPO, imageTags } from "./images"; +import { buildImageMap, TERMINAL_BENCH_4_IMAGES, taskImages } from "./images"; import { TERMINAL_BENCH_4_SOURCE_COMMIT } from "./tasks-source"; describe("terminal-bench-4 registry wiring", () => { @@ -30,7 +30,7 @@ describe("terminal-bench-4 registry wiring", () => { expect(TERMINAL_BENCH_4_BENCHMARK.degradeSolverErrors).toBe(true); }); - it("parses a run config with the default image repo and agent", () => { + it("parses a run config with the default agent", () => { const result = parseSchema(BenchmarkRunConfigSchema, { benchmarkId: "terminal_bench_4", model: "anthropic/claude-opus-5", @@ -39,20 +39,48 @@ describe("terminal-bench-4 registry wiring", () => { }); assertRight(result); expect(result.right.benchmarkId).toBe("terminal_bench_4"); - expect( - result.right.benchmarkId === "terminal_bench_4" && result.right.imageRepo - ).toBe(DEFAULT_TERMINAL_BENCH_4_IMAGE_REPO); expect( result.right.benchmarkId === "terminal_bench_4" && result.right.agent ).toBe("pi"); }); }); -describe("terminal-bench-4 image tags", () => { - it("derives agent and verifier tags from the repo, task id and pinned commit", () => { - expect(imageTags("ghcr.io/acme/tb4", "hello-world")).toEqual({ - agent: `ghcr.io/acme/tb4/hello-world:${TERMINAL_BENCH_4_SOURCE_COMMIT.slice(0, 12)}`, - verifier: `ghcr.io/acme/tb4/hello-world-verifier:${TERMINAL_BENCH_4_SOURCE_COMMIT.slice(0, 12)}`, +describe("terminal-bench-4 image map", () => { + const raw = { + sourceCommit: TERMINAL_BENCH_4_SOURCE_COMMIT, + images: { + "hello-world": { agent: "im-abc123", verifier: "im-def456" }, + }, + }; + + it("exposes Modal image ids per task and undefined for unknown tasks", () => { + const map = buildImageMap(raw); + expect(taskImages(map, "hello-world")).toEqual({ + agent: "im-abc123", + verifier: "im-def456", }); + expect(taskImages(map, "missing")).toBeUndefined(); + }); + + it("rejects a map built from a different source commit", () => { + expect(() => + buildImageMap({ ...raw, sourceCommit: "0".repeat(40) }) + ).toThrow(/pinned to/); + }); + + it("rejects ids that are not Modal image ids", () => { + expect(() => + buildImageMap({ + ...raw, + images: { x: { agent: "ghcr.io/x:y", verifier: "im-1" } }, + }) + ).toThrow(/invalid/); + }); + + it("ships a committed map for the pinned commit", () => { + expect(TERMINAL_BENCH_4_IMAGES.sourceCommit).toBe( + TERMINAL_BENCH_4_SOURCE_COMMIT + ); + expect(TERMINAL_BENCH_4_IMAGES.images.size).toBeGreaterThan(0); }); }); diff --git a/src/benchmarks/terminal-bench-4/benchmark.ts b/src/benchmarks/terminal-bench-4/benchmark.ts index df81198..a8bdf25 100644 --- a/src/benchmarks/terminal-bench-4/benchmark.ts +++ b/src/benchmarks/terminal-bench-4/benchmark.ts @@ -51,7 +51,6 @@ function makeTerminalBench4Layer( allowedTools: benchmarkConfig.allowedTools, disallowedTools: benchmarkConfig.disallowedTools, isolateAgentConfig: benchmarkConfig.isolateAgentConfig, - imageRepo: benchmarkConfig.imageRepo, }); const datasetLayer = makeTerminalBench4DatasetLayer( definedValues({ diff --git a/src/benchmarks/terminal-bench-4/image-ids.json b/src/benchmarks/terminal-bench-4/image-ids.json new file mode 100644 index 0000000..30346fa --- /dev/null +++ b/src/benchmarks/terminal-bench-4/image-ids.json @@ -0,0 +1,9 @@ +{ + "sourceCommit": "452bf305c6daa62fc59061d22133a7cbc7c1572e", + "images": { + "music-harmony": { + "agent": "im-h4eLdJF4FdDeGLlrKSdS2D", + "verifier": "im-N1tgE2XbCSM8h5eJVr69tv" + } + } +} diff --git a/src/benchmarks/terminal-bench-4/images.ts b/src/benchmarks/terminal-bench-4/images.ts index 8123405..10e84b4 100644 --- a/src/benchmarks/terminal-bench-4/images.ts +++ b/src/benchmarks/terminal-bench-4/images.ts @@ -1,20 +1,52 @@ +import { Either } from "../../internal/either"; +import { firstZodIssueMessage, parseSchema, z } from "../../internal/zod"; +import imageIdsJson from "./image-ids.json"; import { TERMINAL_BENCH_4_SOURCE_COMMIT } from "./tasks-source"; -export const DEFAULT_TERMINAL_BENCH_4_IMAGE_REPO = - "ghcr.io/openrouterteam/terminal-bench-4" as const; +const ModalImageIdSchema = z.string().regex(/^im-[A-Za-z0-9]+$/); -export interface TerminalBench4ImageTags { - readonly agent: string; - readonly verifier: string; +const TaskImagesSchema = z.object({ + agent: ModalImageIdSchema, + verifier: ModalImageIdSchema, +}); + +export type TerminalBench4TaskImages = z.infer; + +const ImageIdsSchema = z.object({ + sourceCommit: z.string().regex(/^[0-9a-f]{40}$/), + images: z.record(z.string().min(1), TaskImagesSchema), +}); + +export interface TerminalBench4ImageMap { + readonly sourceCommit: string; + readonly images: ReadonlyMap; } -export function imageTags( - imageRepo: string, +export function buildImageMap( + raw: unknown, + expectedCommit: string = TERMINAL_BENCH_4_SOURCE_COMMIT +): TerminalBench4ImageMap { + const parsed = parseSchema(ImageIdsSchema, raw); + if (Either.isLeft(parsed)) { + throw new TypeError( + `terminal-bench-4 image-ids.json is invalid: ${firstZodIssueMessage(parsed.left)}` + ); + } + const { sourceCommit, images } = parsed.right; + if (sourceCommit !== expectedCommit) { + throw new TypeError( + `terminal-bench-4 image-ids.json was built from ${sourceCommit} but tasks are pinned to ${expectedCommit}; rerun scripts/build-terminal-bench-4-images.py` + ); + } + return { sourceCommit, images: new Map(Object.entries(images)) }; +} + +export function taskImages( + map: TerminalBench4ImageMap, taskId: string -): TerminalBench4ImageTags { - const tag = TERMINAL_BENCH_4_SOURCE_COMMIT.slice(0, 12); - return { - agent: `${imageRepo}/${taskId}:${tag}`, - verifier: `${imageRepo}/${taskId}-verifier:${tag}`, - }; +): TerminalBench4TaskImages | undefined { + return map.images.get(taskId); } + +export const TERMINAL_BENCH_4_IMAGES: TerminalBench4ImageMap = + buildImageMap(imageIdsJson); diff --git a/src/benchmarks/terminal-bench-4/session.test.ts b/src/benchmarks/terminal-bench-4/session.test.ts index 1cb0b6c..8741ae1 100644 --- a/src/benchmarks/terminal-bench-4/session.test.ts +++ b/src/benchmarks/terminal-bench-4/session.test.ts @@ -1,4 +1,14 @@ import { describe, expect, it } from "bun:test"; +import { spawnSync } from "node:child_process"; +import { + existsSync, + mkdirSync, + mkdtempSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import type { Effect } from "effect/Effect"; import { either, gen, provide, runPromise } from "effect/Effect"; @@ -15,8 +25,15 @@ import { makeFakeSandboxLayer, SandboxSession } from "../harbor/sandbox"; import type { TerminalBench4SampleMeta } from "./dataset"; import { agentNetworkDeviation, + agentSandboxTimeoutSec, + ARTIFACT_BUNDLE_TIMEOUT_MS, + ARTIFACT_TRANSFER_TIMEOUT_MS, artifactBundleCommand, + artifactExtractCommand, createAgentSession, + DEFAULT_COLLECT_TIMEOUT_SEC, + SANDBOX_TIMEOUT_MARGIN_SEC, + verifierSandboxTimeoutSec, createVerifierSession, runCollectHooks, runVerifier, @@ -111,13 +128,14 @@ describe("terminal-bench-4 sandbox creation", () => { expect(recorded.creates).toHaveLength(1); const input = recorded.creates[0]; expect(input?.imageTag).toBe("repo/task:abc"); + expect(input?.imageKind).toBe("modal-image-id"); expect(input?.imageBuildSteps).toEqual(["RUN echo hi"]); expect(input?.cpus).toBe(8); expect(input?.memoryMb).toBe(16384); expect(input?.gpu).toBe("H100"); expect(input?.env).toEqual({ HF_HOME: "/cache" }); expect(input?.allowInternet).toBe(true); - expect(input?.timeoutSec).toBe(400); + expect(input?.timeoutSec).toBe(agentSandboxTimeoutSec(META)); expect(input?.uploads).toEqual([ { localPath: "/tasks/fp8-rmsnorm-gemm/instruction.md", @@ -138,13 +156,50 @@ describe("terminal-bench-4 sandbox creation", () => { ); const input = recorded.creates[0]; expect(input?.imageTag).toBe("repo/task-verifier:abc"); + expect(input?.imageKind).toBe("modal-image-id"); expect(input?.cpus).toBe(4); expect(input?.gpu).toBeUndefined(); expect(input?.env).toEqual({ TB_SEED: "1" }); - expect(input?.timeoutSec).toBe(350); + expect(input?.timeoutSec).toBe(verifierSandboxTimeoutSec(META)); expect(input?.uploads).toEqual([]); }); + it("keeps the agent sandbox alive through collect hooks and artifact transfer", () => { + const meta: TerminalBench4SampleMeta = { + ...META, + maxAgentTimeoutSec: 28_800, + collect: [ + { command: "a", timeout_sec: 120 }, + { command: "b" }, + { command: "c", timeout_sec: 0.5 }, + ], + }; + const postProcessingSec = + 120 + + DEFAULT_COLLECT_TIMEOUT_SEC + + 0.5 + + (2 * ARTIFACT_BUNDLE_TIMEOUT_MS + ARTIFACT_TRANSFER_TIMEOUT_MS) / 1000; + const lifetime = agentSandboxTimeoutSec(meta); + expect(lifetime).toBeGreaterThanOrEqual( + meta.maxAgentTimeoutSec + postProcessingSec + ); + expect(lifetime).toBe( + Math.ceil( + meta.maxAgentTimeoutSec + postProcessingSec + SANDBOX_TIMEOUT_MARGIN_SEC + ) + ); + expect(lifetime - agentSandboxTimeoutSec(META)).toBe( + Math.ceil(28_700 + 120 + DEFAULT_COLLECT_TIMEOUT_SEC + 0.5) + ); + }); + + it("keeps the verifier sandbox alive through artifact extraction and the test run", () => { + expect(verifierSandboxTimeoutSec(META)).toBeGreaterThanOrEqual( + META.maxTestTimeoutSec + + (2 * ARTIFACT_BUNDLE_TIMEOUT_MS + ARTIFACT_TRANSFER_TIMEOUT_MS) / 1000 + ); + }); + it("records a deviation only when the task disallows internet", () => { expect(agentNetworkDeviation(META)).toEqual({ agentNetworkForced: true, @@ -260,10 +315,72 @@ describe("terminal-bench-4 artifact bundling", () => { recorded.downloads[0]?.localPath ); const extract = recorded.execs.at(-1); - expect(extract?.argv[2]).toBe( + expect(extract?.argv[2]).toBe(artifactExtractCommand(["/app/out"])); + expect(extract?.argv[2]).toContain( "tar -xf /tmp/tb4-artifacts.tar -P -C / && rm -f /tmp/tb4-artifacts.tar" ); }); + + it("clears declared directory destinations so files deleted by the agent do not survive", () => { + const root = mkdtempSync(join(tmpdir(), "tb4-extract-")); + try { + const agentDir = join(root, "agent", "pkg"); + const verifierDir = join(root, "verifier", "pkg"); + mkdirSync(agentDir, { recursive: true }); + mkdirSync(verifierDir, { recursive: true }); + writeFileSync(join(agentDir, "kept.py"), "agent"); + writeFileSync(join(agentDir, "skipped.pyc"), "agent"); + writeFileSync(join(verifierDir, "kept.py"), "image"); + writeFileSync(join(verifierDir, "deleted.py"), "image"); + writeFileSync(join(verifierDir, "skipped.pyc"), "image"); + writeFileSync(join(root, "verifier", "untouched.txt"), "image"); + const bundle = join(root, "bundle.tar"); + const artifacts = [{ source: verifierDir, exclude: ["*.pyc"] }]; + const pack = spawnSync( + "bash", + [ + "-c", + `tar -cf ${bundle} -P --exclude='*.pyc' --transform 's|${agentDir}|${verifierDir}|' ${agentDir}`, + ], + { encoding: "utf8" } + ); + expect(pack.status).toBe(0); + const extract = spawnSync( + "bash", + ["-c", artifactExtractCommand(artifacts, bundle)], + { encoding: "utf8" } + ); + expect(extract.stderr).toBe(""); + expect(extract.status).toBe(0); + expect(existsSync(join(verifierDir, "deleted.py"))).toBe(false); + expect(existsSync(join(verifierDir, "skipped.pyc"))).toBe(false); + expect(Bun.file(join(verifierDir, "kept.py")).size).toBe(5); + expect(existsSync(join(root, "verifier", "untouched.txt"))).toBe(true); + expect(existsSync(bundle)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it("leaves a destination alone when the bundle has no directory entry for it", () => { + const root = mkdtempSync(join(tmpdir(), "tb4-extract-")); + try { + const verifierDir = join(root, "verifier", "pkg"); + mkdirSync(verifierDir, { recursive: true }); + writeFileSync(join(verifierDir, "image.py"), "image"); + const bundle = join(root, "bundle.tar"); + spawnSync("bash", ["-c", `tar -cf ${bundle} -T /dev/null`]); + const extract = spawnSync( + "bash", + ["-c", artifactExtractCommand([verifierDir], bundle)], + { encoding: "utf8" } + ); + expect(extract.status).toBe(0); + expect(existsSync(join(verifierDir, "image.py"))).toBe(true); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); }); describe("terminal-bench-4 verifier", () => { diff --git a/src/benchmarks/terminal-bench-4/session.ts b/src/benchmarks/terminal-bench-4/session.ts index 72529b7..7282d1b 100644 --- a/src/benchmarks/terminal-bench-4/session.ts +++ b/src/benchmarks/terminal-bench-4/session.ts @@ -11,7 +11,7 @@ import type { SandboxSessionFactory, SandboxSessionInstance, } from "../harbor/sandbox"; -import { REMOTE_VERIFIER_SCRIPT } from "../harbor/sandbox"; +import { REMOTE_VERIFIER_SCRIPT, SANDBOX_IMAGE_KINDS } from "../harbor/sandbox"; import type { TerminalBench4SampleMeta } from "./dataset"; import type { Artifact, CollectHook } from "./schema"; @@ -27,15 +27,41 @@ export const KEEP_ALIVE_COMMAND = ["sleep", "infinity"] as const; const REMOTE_ARTIFACT_BUNDLE = "/tmp/tb4-artifacts.tar" as const; -const SANDBOX_TIMEOUT_MARGIN_SEC = 300; +export const SANDBOX_TIMEOUT_MARGIN_SEC = 300; const VERIFIER_TIMEOUT_MARGIN_MS = 5000; const REWARD_READ_TIMEOUT_MS = 10_000; -const ARTIFACT_BUNDLE_TIMEOUT_MS = 600_000; +export const ARTIFACT_BUNDLE_TIMEOUT_MS = 600_000; -const DEFAULT_COLLECT_TIMEOUT_SEC = 300; +export const ARTIFACT_TRANSFER_TIMEOUT_MS = 1_800_000; + +export const DEFAULT_COLLECT_TIMEOUT_SEC = 300; + +const ARTIFACT_PHASE_SEC = + (2 * ARTIFACT_BUNDLE_TIMEOUT_MS + ARTIFACT_TRANSFER_TIMEOUT_MS) / 1000; + +export function agentSandboxTimeoutSec(meta: TerminalBench4SampleMeta): number { + const collectSec = meta.collect.reduce( + (total, hook) => total + (hook.timeout_sec ?? DEFAULT_COLLECT_TIMEOUT_SEC), + 0 + ); + return Math.ceil( + meta.maxAgentTimeoutSec + + collectSec + + ARTIFACT_PHASE_SEC + + SANDBOX_TIMEOUT_MARGIN_SEC + ); +} + +export function verifierSandboxTimeoutSec( + meta: TerminalBench4SampleMeta +): number { + return Math.ceil( + meta.maxTestTimeoutSec + ARTIFACT_PHASE_SEC + SANDBOX_TIMEOUT_MARGIN_SEC + ); +} export function agentNetworkDeviation( meta: TerminalBench4SampleMeta @@ -55,8 +81,9 @@ export function createAgentSession(input: { const { sessionFactory, meta, tasksDir, imageTag, imageBuildSteps } = input; return sessionFactory.create({ imageTag, + imageKind: SANDBOX_IMAGE_KINDS.ModalImageId, imageBuildSteps, - timeoutSec: meta.maxAgentTimeoutSec + SANDBOX_TIMEOUT_MARGIN_SEC, + timeoutSec: agentSandboxTimeoutSec(meta), ...meta.agentEnv, allowInternet: true, workdir: CONTAINER_WORKDIR, @@ -79,7 +106,8 @@ export function createVerifierSession(input: { const { sessionFactory, meta, imageTag } = input; return sessionFactory.create({ imageTag, - timeoutSec: meta.maxTestTimeoutSec + SANDBOX_TIMEOUT_MARGIN_SEC, + imageKind: SANDBOX_IMAGE_KINDS.ModalImageId, + timeoutSec: verifierSandboxTimeoutSec(meta), ...meta.verifierEnv, workdir: CONTAINER_WORKDIR, keepAliveCommand: KEEP_ALIVE_COMMAND, @@ -130,11 +158,14 @@ function shellQuote(value: string): string { return `'${value.replaceAll("'", String.raw`'\''`)}'`; } +function artifactSources(artifacts: readonly Artifact[]): readonly string[] { + return [REMOTE_ARTIFACTS_DIR, ...artifacts.map(artifactSource)].filter( + (s, i, all) => all.indexOf(s) === i + ); +} + export function artifactBundleCommand(artifacts: readonly Artifact[]): string { - const sources = [ - REMOTE_ARTIFACTS_DIR, - ...artifacts.map(artifactSource), - ].filter((s, i, all) => all.indexOf(s) === i); + const sources = artifactSources(artifacts); const excludes = artifacts .flatMap(artifactExcludes) .filter((s, i, all) => all.indexOf(s) === i) @@ -152,6 +183,19 @@ export function artifactBundleCommand(artifacts: readonly Artifact[]): string { ].join(" && "); } +export function artifactExtractCommand( + artifacts: readonly Artifact[], + bundle: string = REMOTE_ARTIFACT_BUNDLE +): string { + const clearDirs = artifactSources(artifacts).map( + (s) => + `if tar -tf ${bundle} -P | grep -qx ${shellQuote(`${s}/`)}; then rm -rf ${shellQuote(s)}; fi` + ); + return [...clearDirs, `tar -xf ${bundle} -P -C /`, `rm -f ${bundle}`].join( + " && " + ); +} + export function transferArtifacts(input: { readonly agent: SandboxSessionInstance; readonly verifier: SandboxSessionInstance; @@ -173,11 +217,7 @@ export function transferArtifacts(input: { rmSync(staging, { recursive: true, force: true }); } yield* verifier.exec( - [ - "bash", - "-c", - `tar -xf ${REMOTE_ARTIFACT_BUNDLE} -P -C / && rm -f ${REMOTE_ARTIFACT_BUNDLE}`, - ], + ["bash", "-c", artifactExtractCommand(artifacts)], {}, ARTIFACT_BUNDLE_TIMEOUT_MS ); diff --git a/src/benchmarks/terminal-bench-4/solver.ts b/src/benchmarks/terminal-bench-4/solver.ts index 8ac7601..d50fa03 100644 --- a/src/benchmarks/terminal-bench-4/solver.ts +++ b/src/benchmarks/terminal-bench-4/solver.ts @@ -12,7 +12,8 @@ import { } from "../agent-cli/runner"; import type { SandboxSessionFactory } from "../harbor/sandbox"; import { readTerminalBench4Meta } from "./dataset"; -import { imageTags } from "./images"; +import type { TerminalBench4ImageMap } from "./images"; +import { TERMINAL_BENCH_4_IMAGES, taskImages } from "./images"; import { agentNetworkDeviation, createAgentSession, @@ -35,14 +36,13 @@ const ZERO_USAGE: ModelUsage = { totalCost: 0, }; -export interface TerminalBench4SolverOpts extends AgentCliOpts { - readonly imageRepo: string; -} +export type TerminalBench4SolverOpts = AgentCliOpts; export function terminalBench4Solver( sessionFactory: SandboxSessionFactory, opts: TerminalBench4SolverOpts, - harness: OriHarnessDef + harness: OriHarnessDef, + imageMap: TerminalBench4ImageMap = TERMINAL_BENCH_4_IMAGES ): SolverService { return (state) => gen(function* () { @@ -52,6 +52,12 @@ export function terminalBench4Solver( message: `terminal-bench-4 solver received a sample without terminal-bench-4 metadata (id=${state.sample.id})`, }); } + const images = taskImages(imageMap, meta.taskId); + if (images === undefined) { + return yield* new SolverError({ + message: `terminal-bench-4 task "${meta.taskId}" has no Modal images in image-ids.json; run scripts/build-terminal-bench-4-images.py`, + }); + } const collectHooks = yield* sandboxCollectHooks(meta.collect); const root = yield* tryPromise({ try: () => ensureTasksCheckedOut(), @@ -60,7 +66,6 @@ export function terminalBench4Solver( message: `Failed to check out terminal-bench-4 tasks: ${String(e)}`, }), }); - const images = imageTags(opts.imageRepo, meta.taskId); const agent = yield* createAgentSession({ sessionFactory, meta, From 0797bfffa6046b137a3d0f859a03ce1eba9841ff Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 3 Sep 2026 21:18:25 +0000 Subject: [PATCH 3/6] fix: add TB4 scorer, full Modal image map, artifact path normalization, and root-user deviation metadata Co-Authored-By: Abhinav Pola --- src/benchmarks/terminal-bench-4/README.md | 2 +- src/benchmarks/terminal-bench-4/benchmark.ts | 4 +- .../terminal-bench-4/dataset.test.ts | 59 ++++- src/benchmarks/terminal-bench-4/dataset.ts | 22 +- .../terminal-bench-4/image-ids.json | 216 ++++++++++++++++++ src/benchmarks/terminal-bench-4/schema.ts | 1 + .../terminal-bench-4/scorer.test.ts | 71 ++++++ src/benchmarks/terminal-bench-4/scorer.ts | 38 +++ .../terminal-bench-4/session.test.ts | 39 ++++ src/benchmarks/terminal-bench-4/session.ts | 19 +- src/benchmarks/terminal-bench-4/solver.ts | 2 + 11 files changed, 463 insertions(+), 10 deletions(-) create mode 100644 src/benchmarks/terminal-bench-4/scorer.test.ts create mode 100644 src/benchmarks/terminal-bench-4/scorer.ts diff --git a/src/benchmarks/terminal-bench-4/README.md b/src/benchmarks/terminal-bench-4/README.md index 78c13bf..142ce0b 100644 --- a/src/benchmarks/terminal-bench-4/README.md +++ b/src/benchmarks/terminal-bench-4/README.md @@ -17,7 +17,7 @@ Each task ships `task.toml`, `instruction.md`, `environment/Dockerfile` (agent i Unlike 2.1, the verifier does not run inside the agent container. -1. Agent sandbox is created from the task's prebuilt agent image with the task's `cpus`, `memory_mb`, GPU and `environment.env`. Internet is always enabled so the agent CLI can reach OpenRouter, and tasks that declare `allow_internet = false` record `agentNetworkForced: true` in sample metadata. +1. Agent sandbox is created from the task's prebuilt agent image with the task's `cpus`, `memory_mb`, GPU and `environment.env`. Internet is always enabled so the agent CLI can reach OpenRouter, and tasks that declare `allow_internet = false` record `agentNetworkForced: true` in sample metadata. Modal sandboxes ignore the image's `USER` directive and execute commands as root, which also lets the root-only agent CLI layering steps (`apt-get`, nvm under `/root`) apply to every task image. The three tasks whose agent Dockerfile ends in a non-root `USER` (`fp8-rmsnorm-gemm` and `rs-archive-clone` as `agent`, `risk-scorer-replay` as `nobody`) therefore run the agent as root, and their samples record `agentRunsAsRoot: true` with `taskImageUser` in metadata. 2. The ori agent runs against `/instruction.md` with the task's `agent.timeout_sec`. 3. `verifier.collect` hooks run in the agent sandbox. Hooks that target a compose sidecar service fail the sample. 4. `/logs/artifacts` plus every declared artifact path is bundled into a tarball, downloaded, and extracted at the same absolute paths in a fresh verifier sandbox built from the task's verifier image with `verifier.environment` resources (falling back to the agent resources) and `verifier.env`. diff --git a/src/benchmarks/terminal-bench-4/benchmark.ts b/src/benchmarks/terminal-bench-4/benchmark.ts index a8bdf25..322a2d1 100644 --- a/src/benchmarks/terminal-bench-4/benchmark.ts +++ b/src/benchmarks/terminal-bench-4/benchmark.ts @@ -17,9 +17,9 @@ import { getOriHarness } from "../agent-cli/harness"; import { TERMINAL_BENCH_4_META } from "../benchmark-meta"; import { makeModalSandboxLayer } from "../harbor/modal-sandbox"; import { SandboxSession } from "../harbor/sandbox"; -import { terminalBenchScorer } from "../terminal-bench/scorer"; import type { Benchmark, BenchmarkRunInput } from "../types"; import { makeTerminalBench4DatasetLayer } from "./dataset"; +import { terminalBench4Scorer } from "./scorer"; import type { TerminalBench4SolverOpts } from "./solver"; import { terminalBench4Solver } from "./solver"; @@ -70,7 +70,7 @@ function makeTerminalBench4Layer( ); }) ); - const scorerLayer = layerSucceed(Scorer, Scorer.of(terminalBenchScorer)); + const scorerLayer = layerSucceed(Scorer, Scorer.of(terminalBench4Scorer)); return layerMergeAll( datasetLayer, solverLayer.pipe(layerProvide(sandboxLayer)), diff --git a/src/benchmarks/terminal-bench-4/dataset.test.ts b/src/benchmarks/terminal-bench-4/dataset.test.ts index 866dc15..38ec812 100644 --- a/src/benchmarks/terminal-bench-4/dataset.test.ts +++ b/src/benchmarks/terminal-bench-4/dataset.test.ts @@ -7,6 +7,7 @@ import { assertLeft, assertRight } from "../../internal/testing"; import { parseSchema } from "../../internal/zod"; import { TERMINAL_BENCH_4_DATASET_ID, + dockerfileUser, listComposeTaskIds, listTaskIds, loadTask, @@ -15,6 +16,7 @@ import { toModalGpu, toSandboxResources, } from "./dataset"; +import { TERMINAL_BENCH_4_IMAGES } from "./images"; import { TaskTomlSchema } from "./schema"; import { ensureTasksCheckedOut, @@ -82,9 +84,16 @@ allow_internet = false env = { HF_HOME = "/cache" } `; -function writeTask(root: string, id: string, toml: string, compose = false) { +function writeTask( + root: string, + id: string, + toml: string, + compose = false, + dockerfile = "FROM ubuntu:24.04\nWORKDIR /app\n" +) { const dir = join(root, id); mkdirSync(join(dir, "environment"), { recursive: true }); + writeFileSync(join(dir, "environment", "Dockerfile"), dockerfile); writeFileSync(join(dir, "task.toml"), toml); writeFileSync(join(dir, "instruction.md"), `Solve ${id}.\n`); if (compose) { @@ -101,6 +110,13 @@ beforeAll(() => { writeTask(fixtureRoot, "b-cpu", BASE_TOML); writeTask(fixtureRoot, "a-gpu", GPU_TOML); writeTask(fixtureRoot, "c-compose", BASE_TOML, true); + writeTask( + fixtureRoot, + "d-nonroot", + BASE_TOML, + false, + "FROM ubuntu:24.04\nUSER root\nRUN useradd agent\nuser agent\n" + ); mkdirSync(join(fixtureRoot, ".hidden")); writeFileSync(join(fixtureRoot, "README.md"), "not a task\n"); }); @@ -142,7 +158,7 @@ describe("terminal-bench-4 task.toml schema", () => { describe("terminal-bench-4 task listing", () => { it("lists non-compose task directories sorted and skips hidden dirs and files", () => { - expect(listTaskIds(fixtureRoot)).toEqual(["a-gpu", "b-cpu"]); + expect(listTaskIds(fixtureRoot)).toEqual(["a-gpu", "b-cpu", "d-nonroot"]); }); it("lists compose tasks separately", () => { @@ -156,6 +172,19 @@ describe("terminal-bench-4 task listing", () => { }); }); +describe("terminal-bench-4 dockerfileUser", () => { + it("returns undefined when there is no USER or the last USER is root", () => { + expect(dockerfileUser("FROM x\nRUN true\n")).toBeUndefined(); + expect(dockerfileUser("FROM x\nUSER agent\nUSER root\n")).toBeUndefined(); + expect(dockerfileUser("FROM x\nUSER 0\n")).toBeUndefined(); + }); + + it("returns the last USER directive, ignoring case and indentation", () => { + expect(dockerfileUser("FROM x\nUSER root\n user nobody\n")).toBe("nobody"); + expect(dockerfileUser("FROM x\nUSER agent:agent\n")).toBe("agent:agent"); + }); +}); + describe("terminal-bench-4 GPU mapping", () => { const env = { cpus: 1, @@ -206,6 +235,14 @@ describe("terminal-bench-4 taskToSample", () => { allowInternet: true, }); expect(meta?.verifierEnv).toEqual(meta?.agentEnv); + expect(meta?.imageUser).toBeUndefined(); + }); + + it("records the image's final non-root USER so the run can report the root deviation", () => { + const meta = readTerminalBench4Meta( + taskToSample(loadTask("d-nonroot", fixtureRoot)).metadata + ); + expect(meta?.imageUser).toBe("agent"); }); it("propagates gpu, env vars, artifacts and collect hooks for a gpu task", () => { @@ -293,4 +330,22 @@ NETWORK("terminal-bench-4 pinned checkout", () => { expect(loadTask(id, tasksDir).composeFile).toBeDefined(); } }); + + it("ships Modal images for exactly the runnable task set", () => { + expect([...TERMINAL_BENCH_4_IMAGES.images.keys()].sort()).toEqual( + listTaskIds(tasksDir) + ); + }); + + it("identifies exactly three tasks whose agent image runs as a non-root user", () => { + const nonRoot = listTaskIds(tasksDir).flatMap((id) => { + const user = loadTask(id, tasksDir).imageUser; + return user === undefined ? [] : [`${id}:${user}`]; + }); + expect(nonRoot).toEqual([ + "fp8-rmsnorm-gemm:agent", + "risk-scorer-replay:nobody", + "rs-archive-clone:agent", + ]); + }); }); diff --git a/src/benchmarks/terminal-bench-4/dataset.ts b/src/benchmarks/terminal-bench-4/dataset.ts index c0e81c9..dd78b4e 100644 --- a/src/benchmarks/terminal-bench-4/dataset.ts +++ b/src/benchmarks/terminal-bench-4/dataset.ts @@ -29,6 +29,19 @@ import { ensureTasksCheckedOutEffect, tasksDir } from "./tasks-source"; export const TERMINAL_BENCH_4_DATASET_ID = "terminal_bench_4" as const; const COMPOSE_FILE = join("environment", "docker-compose.yaml"); +const AGENT_DOCKERFILE = join("environment", "Dockerfile"); +const USER_DIRECTIVE = /^\s*USER\s+(\S+)/i; + +export function dockerfileUser(dockerfile: string): string | undefined { + const users = dockerfile + .split("\n") + .map((line) => USER_DIRECTIVE.exec(line)?.[1]) + .filter((user): user is string => user !== undefined); + const last = users.at(-1); + return last === undefined || last === "root" || last === "0" + ? undefined + : last; +} export function loadTask( taskId: string, @@ -55,6 +68,9 @@ export function loadTask( taskDir, instructionPath: join(taskDir, "instruction.md"), composeFile: existsSync(composePath) ? composePath : undefined, + imageUser: dockerfileUser( + readFileSync(join(taskDir, AGENT_DOCKERFILE), "utf8") + ), }; } @@ -115,6 +131,7 @@ export const TerminalBench4SampleMetaSchema = z.object({ category: z.string(), agentEnv: SandboxResourcesSchema, verifierEnv: SandboxResourcesSchema, + imageUser: z.string().min(1).optional(), artifacts: z.array(ArtifactSchema), collect: z.array(CollectHookSchema), reward: z.number().optional(), @@ -162,7 +179,7 @@ export function taskToSample( } const instruction = readFileSync(task.instructionPath, "utf8"); const { environment, verifier, agent, metadata, artifacts } = task.taskToml; - const meta: TerminalBench4SampleMeta = { + const meta: TerminalBench4SampleMeta = definedValues({ taskId: task.id, maxAgentTimeoutSec: maxAgentTimeoutSecOverride ?? agent.timeout_sec, maxTestTimeoutSec: verifier.timeout_sec, @@ -172,9 +189,10 @@ export function taskToSample( verifier.environment ?? environment, verifier.env ), + imageUser: task.imageUser, artifacts, collect: verifier.collect, - }; + }); return { id: `${TERMINAL_BENCH_4_DATASET_ID}-${task.id}`, input: instruction, diff --git a/src/benchmarks/terminal-bench-4/image-ids.json b/src/benchmarks/terminal-bench-4/image-ids.json index 30346fa..afd14fb 100644 --- a/src/benchmarks/terminal-bench-4/image-ids.json +++ b/src/benchmarks/terminal-bench-4/image-ids.json @@ -1,9 +1,225 @@ { "sourceCommit": "452bf305c6daa62fc59061d22133a7cbc7c1572e", "images": { + "atrx-vep-crispr": { + "agent": "im-5mzBHPjvYN8zIeS6JeEKmR", + "verifier": "im-soU60kfnSJpxarw5z0Xsls" + }, + "batched-eval-parity": { + "agent": "im-kBn6DlBhHSrVN8JT9WgqbE", + "verifier": "im-arsDZFQI9fohCwpftm4Xmo" + }, + "biped-contact-dynamics": { + "agent": "im-nxaATwoff5WzVgqWC3ueUg", + "verifier": "im-aI2kfN2sYYh7qYYuvjf5Ki" + }, + "bun-sourcemap-leak": { + "agent": "im-VnmCRNhZKb7HmJvvSmpvSS", + "verifier": "im-Vl88ZMvXHbhJ6S02LyvIap" + }, + "cad-model": { + "agent": "im-61M5ZsDKFh5kSZ2tCZWVdB", + "verifier": "im-H0ScSMfg12YuFmxArZsWls" + }, + "cargo-flight-dispatch": { + "agent": "im-3ar3uyEVkz5JG01K03FSGL", + "verifier": "im-ewcfQ4RReECHrtBRgbIozd" + }, + "coq-block-bound": { + "agent": "im-04T13UpbEAMId2NSJUp6SF", + "verifier": "im-aFOGNEMzDXXs1v53C1EBn2" + }, + "data-anonymization": { + "agent": "im-X8mNLloN7NLDM0HER40G8k", + "verifier": "im-4GyZC9MkFZv2VmsUiZYygH" + }, + "distributed-dedup": { + "agent": "im-WnqarejRmceVgqXZG0WS4y", + "verifier": "im-uRC5MnMRmgBoMqFeqtk3nU" + }, + "embedding-drift-monitor": { + "agent": "im-PpFyIuYjqgY7lVH5zeWn8W", + "verifier": "im-GFLxtRFzTejj1HA8IuRQho" + }, + "fin-saccr-rwa": { + "agent": "im-CaLvuH98JYvektFWObLx9D", + "verifier": "im-zFbl2JjMfiRThTG467pr73" + }, + "foodstuff-beta-activity": { + "agent": "im-0D8mQR2fRuJ4CXlJ8WSU9H", + "verifier": "im-QZsMu4kmz3O4asRkrrjNMA" + }, + "formal-crypto": { + "agent": "im-oB4I8OQeHKxpydnII7FSv7", + "verifier": "im-W1MgXXhDW3Xd9IS3O2FcaJ" + }, + "fp8-rmsnorm-gemm": { + "agent": "im-FBlniwzVOyNSzbofwXX7Sf", + "verifier": "im-aRPhi7NBmOCMhMQxVOT2E6" + }, + "freecad-impeller": { + "agent": "im-bKnL2lcYxiJvy7oCmBC8kJ", + "verifier": "im-LSPLtMLQE1ZKgOLfrk2uh3" + }, + "freecad-platform-drawing": { + "agent": "im-mifytSREMlerQ7D73Vc3kp", + "verifier": "im-giErBLPDEx5peOyYKzGr2n" + }, + "freecad-spring-clip": { + "agent": "im-bKnL2lcYxiJvy7oCmBC8kJ", + "verifier": "im-D6lcNYtlx1O7yg0pLocbAK" + }, + "glycan-ms2-elucidation": { + "agent": "im-tVkTPoGRtm54VS5UxHye0p", + "verifier": "im-a6N7eouCDSEhvo7ePaG39C" + }, + "gsea-proteomics": { + "agent": "im-uzrdxyBsF9lXgsBzNsolX3", + "verifier": "im-nkIpSvrBVlb5648LWMndAK" + }, + "hof-topology-interpenetration": { + "agent": "im-EMSua2mY5hrV84ODUQio6O", + "verifier": "im-kHF4rEeFTCiLo3gYSAvs5X" + }, + "html-js-filter": { + "agent": "im-QIU15HnCYJ1n5Hrp44ovlp", + "verifier": "im-JyPSKpY5Cci0XXtr9yEDXG" + }, + "interleaved-vigenere": { + "agent": "im-u9b0sHMJDOnlaxKT6Txe5l", + "verifier": "im-zYsvvdjqBcynmj7duY40xg" + }, + "jax-speedrun-gpu": { + "agent": "im-N6iQ3hzwZC8peLicLzgL9M", + "verifier": "im-WFYkmZNiX4qKi0URADBJwt" + }, + "ks-solver-cpp": { + "agent": "im-AWBP7mR4qUflFwFvw5f2Vb", + "verifier": "im-Odr1CRDUAjeo1BhRdNVqqq" + }, + "lake-temp-glm": { + "agent": "im-i3rLQTsqFSYEOT92nur7qt", + "verifier": "im-it6BKyg4odryjD8OcPmXy9" + }, + "layout-config-recreation": { + "agent": "im-h1pPUIY5Ytp6u6fwhMjaq1", + "verifier": "im-g89G0VUFlmkOuWMDodvyQZ" + }, + "layout-config-recreation2": { + "agent": "im-KnCKkgI6t0Gpja6WXqKba4", + "verifier": "im-1ETetaATJPqnFtf6oCAm4U" + }, + "math-eval-grader": { + "agent": "im-rSssB1KmnzIxwD1Rs93NrI", + "verifier": "im-yQbArTXAtUudEH4ZjJoeab" + }, + "mp-checkpoint-consolidation": { + "agent": "im-sWyivSwRZANmAmiaPguZCK", + "verifier": "im-iPJtn3o95kUUq1O3HD8qb0" + }, "music-harmony": { "agent": "im-h4eLdJF4FdDeGLlrKSdS2D", "verifier": "im-N1tgE2XbCSM8h5eJVr69tv" + }, + "mvcc-lsm-compaction": { + "agent": "im-2gXstFD5uCfDJCcmnLtIeZ", + "verifier": "im-LfmhJztcZwKbyHI2DFujA5" + }, + "ontology-kg-querying": { + "agent": "im-2hI0nvFliMMqAFMA5llA3d", + "verifier": "im-AfLrc8nkL1xBFSob8CJ1bp" + }, + "photonic-waveguide-routing": { + "agent": "im-pCYVVyjBavAEYkuJgozWUi", + "verifier": "im-vGf57QW6Br4K2RW8l6uO00" + }, + "pretrain-shard-corruption": { + "agent": "im-Bv99gtPti1VZ961QSW4moe", + "verifier": "im-cRiRtyN0jiljrSuCMDjHS9" + }, + "production-planning": { + "agent": "im-iDZVsFKcthAsrgI0ZAnlPM", + "verifier": "im-5LTiyQOWuyHcMWoWnXl0Th" + }, + "protein-autointerp-disulfide": { + "agent": "im-si3dXWMicfkPd4VZDvVCAJ", + "verifier": "im-QUzqvygh1T8DjPDJZqoDwt" + }, + "react-lead-form": { + "agent": "im-LKft15TMrpUL9cveUjPEv8", + "verifier": "im-cgFLZ0elW4ScsRhPLNBKHP" + }, + "retro-console-soc": { + "agent": "im-PDekqRInPMIT5hA5ibSsqx", + "verifier": "im-l9VMHqhZwCChc9jReDlifb" + }, + "risk-scorer-replay": { + "agent": "im-Q9ALFBHyhVNWlslfsJ6cvq", + "verifier": "im-mVQSAczLppxJhFWJ8q22c9" + }, + "roy-polymorph-cn": { + "agent": "im-mpoKHymb4pQS0PRWMmXMJM", + "verifier": "im-LwNVFOAvkuKuYvXeMejMQG" + }, + "rs-archive-clone": { + "agent": "im-9O5meSz6MvfwQpMkRShZ4w", + "verifier": "im-dD0YhMM2uiNKvb3kM9Ts8R" + }, + "satb-audio-transcription": { + "agent": "im-ayMOwN81rBC0IYGlnCgChh", + "verifier": "im-ZHu8g2cFqoN7ZWm8CY8jWh" + }, + "session-window-debug": { + "agent": "im-PvFjlbPMy4TK8t78B85qRH", + "verifier": "im-jbGBZHefGrA7YB4yR55Jwq" + }, + "sglang-qwen-burst": { + "agent": "im-oGI1JG2N5ATgqomxqM2ZNa", + "verifier": "im-5ySAc9FIK3MD6slsX047gD" + }, + "shadow-relay": { + "agent": "im-QEpRRr0q9Cu4axYMOuDtBh", + "verifier": "im-DLOI58H0O9lewdcjaDu0wb" + }, + "sound-change-cascade": { + "agent": "im-MmUWHFroYghOfNIMBPlq0U", + "verifier": "im-TZsHjXKVzikNheZrbiXTqU" + }, + "takens-embedding-lean": { + "agent": "im-YQ7KtwWLhnbbOS7IhEudJK", + "verifier": "im-mL2UnKl0T1kTZdQ15RC9FZ" + }, + "telecom-entity-resolution": { + "agent": "im-xMkee5jCTN5uo6g2sb4a8Z", + "verifier": "im-HkAgySdmfJS99o1l12ZAwd" + }, + "uefi-bootkit": { + "agent": "im-IuXJpvadCgAxNnD1mNngYn", + "verifier": "im-YmviJTLkykmRUFzJzjLLsR" + }, + "vba-userform-port": { + "agent": "im-If0Gw9lp55OmbkJBANDhtp", + "verifier": "im-iRE4ikpfNdbayPgfalIGb9" + }, + "vf2-speedup-networkx": { + "agent": "im-4RHL1D2JzIPOaREGkR8aZu", + "verifier": "im-9hv1GCOv9a01qdfGKyrRns" + }, + "vllm-deepseek-streaming": { + "agent": "im-lZ9QozXsV4i7jNH9PNWH98", + "verifier": "im-DnbP8OnC3Mxvd8T3d2k4L6" + }, + "vpp-loss-divergence": { + "agent": "im-53aeOBx7q6eVGTRYW5NikY", + "verifier": "im-Rn4eJ2D7g0Kck62cKgtnJR" + }, + "wal-recovery-ordering": { + "agent": "im-xV4icf0Bdmezzi1PzQvqLe", + "verifier": "im-PeMankuM3Nw8YntS0AsdBA" + }, + "wdm-design": { + "agent": "im-9cQ83P2bxI3JfLSAg0duHg", + "verifier": "im-dpyIXtQ258y6uA4hjOSTD5" } } } diff --git a/src/benchmarks/terminal-bench-4/schema.ts b/src/benchmarks/terminal-bench-4/schema.ts index d1bd891..6e5ea11 100644 --- a/src/benchmarks/terminal-bench-4/schema.ts +++ b/src/benchmarks/terminal-bench-4/schema.ts @@ -64,4 +64,5 @@ export interface TerminalBench4Task { readonly taskDir: string; readonly instructionPath: string; readonly composeFile: string | undefined; + readonly imageUser: string | undefined; } diff --git a/src/benchmarks/terminal-bench-4/scorer.test.ts b/src/benchmarks/terminal-bench-4/scorer.test.ts new file mode 100644 index 0000000..141a9fc --- /dev/null +++ b/src/benchmarks/terminal-bench-4/scorer.test.ts @@ -0,0 +1,71 @@ +import { describe, expect, it } from "bun:test"; + +import { runPromise } from "effect/Effect"; + +import type { Sample } from "../../harness/core"; +import { initialTaskState, ScoreValue } from "../../harness/core"; +import type { TerminalBench4SampleMeta } from "./dataset"; +import { terminalBench4Scorer } from "./scorer"; + +const META: TerminalBench4SampleMeta = { + taskId: "music-harmony", + maxAgentTimeoutSec: 28_800, + maxTestTimeoutSec: 600, + category: "software-engineering", + agentEnv: { cpus: 2, memoryMb: 4096, env: {}, allowInternet: true }, + verifierEnv: { cpus: 2, memoryMb: 4096, env: {}, allowInternet: false }, + artifacts: [], + collect: [], +}; + +function sampleWith(reward: number | undefined, testOutput?: string): Sample { + return { + id: "terminal_bench_4-music-harmony", + input: "harmonize", + target: { text: "music-harmony" }, + metadata: { + ...META, + ...(reward !== undefined && { reward }), + ...(testOutput !== undefined && { testOutput }), + }, + }; +} + +describe("terminal-bench-4 scorer (pure)", () => { + it("scores Correct when the stashed reward is 1", async () => { + const state = initialTaskState(sampleWith(1, "1 passed")); + const score = await runPromise( + terminalBench4Scorer(state, state.sample.target) + ); + expect(score.value).toBe(ScoreValue.Correct); + expect(score.answer).toBe("music-harmony"); + expect(score.explanation).toBe("1 passed"); + expect(score.trajectory).toEqual({ kind: "verifier_log", log: "1 passed" }); + }); + it("scores Incorrect when the stashed reward is 0", async () => { + const state = initialTaskState(sampleWith(0, "1 failed")); + const score = await runPromise( + terminalBench4Scorer(state, state.sample.target) + ); + expect(score.value).toBe(ScoreValue.Incorrect); + }); + it("scores Incorrect when no reward is stashed", async () => { + const state = initialTaskState(sampleWith(undefined)); + const score = await runPromise( + terminalBench4Scorer(state, state.sample.target) + ); + expect(score.value).toBe(ScoreValue.Incorrect); + expect(score.explanation).toBe(""); + expect(score.trajectory).toBeUndefined(); + }); + it("scores Incorrect when metadata is not terminal-bench-4 metadata", async () => { + const state = initialTaskState({ + ...sampleWith(1), + metadata: { taskId: "music-harmony", reward: 1 }, + }); + const score = await runPromise( + terminalBench4Scorer(state, state.sample.target) + ); + expect(score.value).toBe(ScoreValue.Incorrect); + }); +}); diff --git a/src/benchmarks/terminal-bench-4/scorer.ts b/src/benchmarks/terminal-bench-4/scorer.ts new file mode 100644 index 0000000..e943e51 --- /dev/null +++ b/src/benchmarks/terminal-bench-4/scorer.ts @@ -0,0 +1,38 @@ +import { succeed } from "effect/Effect"; + +import type { Score, Target, TaskState } from "../../harness/core"; +import { ScoreValue } from "../../harness/core"; +import type { ScorerService } from "../../harness/scorer"; +import { definedValues } from "../../internal/guards"; +import { readTerminalBench4Meta } from "./dataset"; + +function readReward(state: TaskState): { + reward: number; + testOutput?: string; +} { + const meta = readTerminalBench4Meta(state.sample.metadata); + if (meta === undefined) { + return { reward: 0 }; + } + return definedValues({ + reward: meta.reward ?? 0, + testOutput: meta.testOutput, + }); +} + +export const terminalBench4Scorer: ScorerService = ( + state: TaskState, + target: Target +) => { + const { reward, testOutput } = readReward(state); + const score: Score = definedValues({ + value: reward >= 1 ? ScoreValue.Correct : ScoreValue.Incorrect, + answer: target.text, + explanation: testOutput ?? "", + trajectory: + testOutput !== undefined + ? ({ kind: "verifier_log", log: testOutput } as const) + : undefined, + }); + return succeed(score); +}; diff --git a/src/benchmarks/terminal-bench-4/session.test.ts b/src/benchmarks/terminal-bench-4/session.test.ts index 8741ae1..8d3a0e2 100644 --- a/src/benchmarks/terminal-bench-4/session.test.ts +++ b/src/benchmarks/terminal-bench-4/session.test.ts @@ -25,6 +25,7 @@ import { makeFakeSandboxLayer, SandboxSession } from "../harbor/sandbox"; import type { TerminalBench4SampleMeta } from "./dataset"; import { agentNetworkDeviation, + agentUserDeviation, agentSandboxTimeoutSec, ARTIFACT_BUNDLE_TIMEOUT_MS, ARTIFACT_TRANSFER_TIMEOUT_MS, @@ -212,6 +213,14 @@ describe("terminal-bench-4 sandbox creation", () => { }) ).toEqual({}); }); + + it("records a deviation only when the task image declares a non-root USER", () => { + expect(agentUserDeviation(META)).toEqual({}); + expect(agentUserDeviation({ ...META, imageUser: "nobody" })).toEqual({ + agentRunsAsRoot: true, + taskImageUser: "nobody", + }); + }); }); describe("terminal-bench-4 collect hooks", () => { @@ -362,6 +371,36 @@ describe("terminal-bench-4 artifact bundling", () => { } }); + it("replaces a directory declared with a trailing slash", () => { + const root = mkdtempSync(join(tmpdir(), "tb4-extract-")); + try { + const dir = join(root, "app"); + mkdirSync(dir, { recursive: true }); + writeFileSync(join(dir, "kept.py"), "agent"); + const artifacts = [`${dir}/`]; + const bundleCmd = artifactBundleCommand(artifacts); + expect(bundleCmd).toContain(`'${dir}'`); + expect(bundleCmd).not.toContain(`'${dir}/'`); + const bundle = join(root, "bundle.tar"); + const pack = spawnSync("bash", ["-c", `tar -cf ${bundle} -P ${dir}/`], { + encoding: "utf8", + }); + expect(pack.status).toBe(0); + writeFileSync(join(dir, "deleted.py"), "image"); + const extract = spawnSync( + "bash", + ["-c", artifactExtractCommand(artifacts, bundle)], + { encoding: "utf8" } + ); + expect(extract.stderr).toBe(""); + expect(extract.status).toBe(0); + expect(existsSync(join(dir, "deleted.py"))).toBe(false); + expect(existsSync(join(dir, "kept.py"))).toBe(true); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + it("leaves a destination alone when the bundle has no directory entry for it", () => { const root = mkdtempSync(join(tmpdir(), "tb4-extract-")); try { diff --git a/src/benchmarks/terminal-bench-4/session.ts b/src/benchmarks/terminal-bench-4/session.ts index 7282d1b..d899880 100644 --- a/src/benchmarks/terminal-bench-4/session.ts +++ b/src/benchmarks/terminal-bench-4/session.ts @@ -71,6 +71,14 @@ export function agentNetworkDeviation( : { agentNetworkForced: true, taskAllowInternet: false }; } +export function agentUserDeviation( + meta: TerminalBench4SampleMeta +): Readonly> { + return meta.imageUser === undefined + ? {} + : { agentRunsAsRoot: true, taskImageUser: meta.imageUser }; +} + export function createAgentSession(input: { readonly sessionFactory: SandboxSessionFactory; readonly meta: TerminalBench4SampleMeta; @@ -158,10 +166,15 @@ function shellQuote(value: string): string { return `'${value.replaceAll("'", String.raw`'\''`)}'`; } +function normalizeArtifactPath(path: string): string { + const trimmed = path.replace(/\/+$/, ""); + return trimmed === "" ? "/" : trimmed; +} + function artifactSources(artifacts: readonly Artifact[]): readonly string[] { - return [REMOTE_ARTIFACTS_DIR, ...artifacts.map(artifactSource)].filter( - (s, i, all) => all.indexOf(s) === i - ); + return [REMOTE_ARTIFACTS_DIR, ...artifacts.map(artifactSource)] + .map(normalizeArtifactPath) + .filter((s, i, all) => all.indexOf(s) === i); } export function artifactBundleCommand(artifacts: readonly Artifact[]): string { diff --git a/src/benchmarks/terminal-bench-4/solver.ts b/src/benchmarks/terminal-bench-4/solver.ts index d50fa03..65b579b 100644 --- a/src/benchmarks/terminal-bench-4/solver.ts +++ b/src/benchmarks/terminal-bench-4/solver.ts @@ -16,6 +16,7 @@ import type { TerminalBench4ImageMap } from "./images"; import { TERMINAL_BENCH_4_IMAGES, taskImages } from "./images"; import { agentNetworkDeviation, + agentUserDeviation, createAgentSession, createVerifierSession, REMOTE_INSTRUCTION, @@ -112,6 +113,7 @@ export function terminalBench4Solver( testOutput, ...agentCliMetadata(harness.id, run), ...agentNetworkDeviation(meta), + ...agentUserDeviation(meta), }, }, messages, From 74818611fbc34a7317999a4f74a5aa58d8e841f3 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 3 Sep 2026 21:29:10 +0000 Subject: [PATCH 4/6] fix: install agent CLI prerequisites via apt-get, dnf or apk so Fedora task images build Co-Authored-By: Abhinav Pola --- src/benchmarks/agent-cli/harness.ts | 18 +++++++++++++++--- .../terminal-bench/ori-solver.test.ts | 16 ++++++++++++++++ 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/src/benchmarks/agent-cli/harness.ts b/src/benchmarks/agent-cli/harness.ts index 173450b..1e13aff 100644 --- a/src/benchmarks/agent-cli/harness.ts +++ b/src/benchmarks/agent-cli/harness.ts @@ -89,6 +89,18 @@ export interface OriHarnessDef { readonly parseRun: (stdout: string) => OriAgentRun; } +export function installSystemPackagesStep(packages: readonly string[]): string { + const list = packages.join(" "); + return [ + "RUN if command -v apt-get >/dev/null", + `then apt-get update && apt-get install -y --no-install-recommends ${list}`, + `elif command -v dnf >/dev/null; then dnf install -y --setopt=install_weak_deps=False --allowerasing ${list}`, + `elif command -v apk >/dev/null; then apk add --no-cache ${list}`, + 'else echo "no supported package manager (apt-get, dnf, apk)" >&2 && exit 1', + "fi", + ].join("; "); +} + function buildImageSteps(opts: { agentPackage: string; binaryName: string; @@ -100,7 +112,7 @@ function buildImageSteps(opts: { `npm install -g ${JSON.stringify(opts.agentPackage)}`; const nvmScript = "/tmp/nvm-install.sh"; return [ - "RUN apt-get update && apt-get install -y --no-install-recommends curl ca-certificates git", + installSystemPackagesStep(["curl", "ca-certificates", "git"]), "ENV NVM_DIR=/root/.nvm", `RUN ${verifiedDownload(NVM_INSTALL_URL, NVM_INSTALL_SHA256, nvmScript)} && bash ${nvmScript} && rm -f ${nvmScript}`, `RUN . /root/.nvm/nvm.sh && nvm install ${NODE_VERSION} && ${installCommand} && ln -sf $(which ${opts.binaryName}) /usr/local/bin/${opts.binaryName} && ln -sf $(which node) /usr/local/bin/node && ln -sf $(which npm) /usr/local/bin/npm`, @@ -117,7 +129,7 @@ function buildAgentImageSteps(opts: { if (opts.agentPackage === opts.defaultPackage) { const archivePath = "/tmp/agent-runtime.tar.zst"; return [ - "RUN apt-get update && apt-get install -y --no-install-recommends curl ca-certificates git zstd", + installSystemPackagesStep(["curl", "ca-certificates", "git", "zstd"]), `RUN ${verifiedDownload(DEFAULT_AGENT_RUNTIME_URL, DEFAULT_AGENT_RUNTIME_SHA256, archivePath)} && zstd -dc ${archivePath} | tar -x -C / && rm -f ${archivePath}`, 'ENV PATH="/root/.local/bin:$PATH"', "RUN ln -sf /opt/agent-runtime/app/node_modules/.bin/claude /usr/local/bin/claude && ln -sf /opt/agent-runtime/app/node_modules/.bin/pi /usr/local/bin/pi && ln -sf /opt/agent-runtime/app/node_modules/.bin/prime-agent /usr/local/bin/prime-agent && ln -sf /opt/agent-runtime/node/bin/node /usr/local/bin/node && ln -sf /opt/agent-runtime/node/bin/npm /usr/local/bin/npm && ln -sf /opt/agent-runtime/node/bin/npx /usr/local/bin/npx", @@ -137,7 +149,7 @@ function buildOmpImageSteps(agentPackage: string): string[] { assertValidAgentPackage(agentPackage); const bunZip = "/tmp/bun.zip"; return [ - "RUN apt-get update && apt-get install -y --no-install-recommends curl ca-certificates git unzip", + installSystemPackagesStep(["curl", "ca-certificates", "git", "unzip"]), "ENV BUN_INSTALL=/root/.bun", `RUN ${verifiedDownload(BUN_RELEASE_URL, BUN_RELEASE_SHA256, bunZip)} && unzip -q ${bunZip} -d /tmp && install -m 0755 /tmp/bun-linux-x64/bun /usr/local/bin/bun && rm -rf ${bunZip} /tmp/bun-linux-x64`, `RUN bun install -g ${JSON.stringify(agentPackage)} && ln -sf /root/.bun/bin/omp /usr/local/bin/omp`, diff --git a/src/benchmarks/terminal-bench/ori-solver.test.ts b/src/benchmarks/terminal-bench/ori-solver.test.ts index 969f237..5fe6c22 100644 --- a/src/benchmarks/terminal-bench/ori-solver.test.ts +++ b/src/benchmarks/terminal-bench/ori-solver.test.ts @@ -633,6 +633,22 @@ describe("terminal-bench ori solver", () => { expect(steps.at(-1)).toBe("RUN claude --version"); }); + it("installs system packages through apt-get, dnf or apk so non-Debian task images build", () => { + for (const harness of Object.values(ORI_HARNESSES)) { + const step = harness + .imageBuildSteps({ agentPackage: harness.defaultPackage }) + .find((s) => s.startsWith("RUN if command -v apt-get")); + expect(step).toBeDefined(); + expect(step).toContain("apt-get install -y --no-install-recommends"); + expect(step).toContain( + "elif command -v dnf >/dev/null; then dnf install -y" + ); + expect(step).toContain("elif command -v apk >/dev/null; then apk add"); + expect(step).toContain("exit 1; fi"); + expect(step).not.toMatch(/^RUN apt-get/); + } + }); + it("exposes every packaged harness and runtime helper", () => { const dockerfile = ORI_HARNESSES.claude .imageBuildSteps({ From 2480a3dd2df1ba20db46140317798c83b7efd246 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 3 Sep 2026 21:29:35 +0000 Subject: [PATCH 5/6] docs: note portable package install for the Fedora task image Co-Authored-By: Abhinav Pola --- src/benchmarks/terminal-bench-4/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/benchmarks/terminal-bench-4/README.md b/src/benchmarks/terminal-bench-4/README.md index 142ce0b..5b7d97c 100644 --- a/src/benchmarks/terminal-bench-4/README.md +++ b/src/benchmarks/terminal-bench-4/README.md @@ -17,7 +17,7 @@ Each task ships `task.toml`, `instruction.md`, `environment/Dockerfile` (agent i Unlike 2.1, the verifier does not run inside the agent container. -1. Agent sandbox is created from the task's prebuilt agent image with the task's `cpus`, `memory_mb`, GPU and `environment.env`. Internet is always enabled so the agent CLI can reach OpenRouter, and tasks that declare `allow_internet = false` record `agentNetworkForced: true` in sample metadata. Modal sandboxes ignore the image's `USER` directive and execute commands as root, which also lets the root-only agent CLI layering steps (`apt-get`, nvm under `/root`) apply to every task image. The three tasks whose agent Dockerfile ends in a non-root `USER` (`fp8-rmsnorm-gemm` and `rs-archive-clone` as `agent`, `risk-scorer-replay` as `nobody`) therefore run the agent as root, and their samples record `agentRunsAsRoot: true` with `taskImageUser` in metadata. +1. Agent sandbox is created from the task's prebuilt agent image with the task's `cpus`, `memory_mb`, GPU and `environment.env`. Internet is always enabled so the agent CLI can reach OpenRouter, and tasks that declare `allow_internet = false` record `agentNetworkForced: true` in sample metadata. Modal sandboxes ignore the image's `USER` directive and execute commands as root, which also lets the root-only agent CLI layering steps (system packages, nvm under `/root`) apply to every task image. The package step detects `apt-get`, `dnf` or `apk`, so the Fedora-based `retro-console-soc` image builds alongside the Debian and Ubuntu ones. The three tasks whose agent Dockerfile ends in a non-root `USER` (`fp8-rmsnorm-gemm` and `rs-archive-clone` as `agent`, `risk-scorer-replay` as `nobody`) therefore run the agent as root, and their samples record `agentRunsAsRoot: true` with `taskImageUser` in metadata. 2. The ori agent runs against `/instruction.md` with the task's `agent.timeout_sec`. 3. `verifier.collect` hooks run in the agent sandbox. Hooks that target a compose sidecar service fail the sample. 4. `/logs/artifacts` plus every declared artifact path is bundled into a tarball, downloaded, and extracted at the same absolute paths in a fresh verifier sandbox built from the task's verifier image with `verifier.environment` resources (falling back to the agent resources) and `verifier.env`. From 0712971ad8685d29ba7f21eb1c9828e43fe049d0 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 3 Sep 2026 21:31:43 +0000 Subject: [PATCH 6/6] fix: install bash on apk-based agent images Co-Authored-By: Abhinav Pola --- src/benchmarks/agent-cli/harness.ts | 2 +- src/benchmarks/terminal-bench/ori-solver.test.ts | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/benchmarks/agent-cli/harness.ts b/src/benchmarks/agent-cli/harness.ts index 1e13aff..7e5490f 100644 --- a/src/benchmarks/agent-cli/harness.ts +++ b/src/benchmarks/agent-cli/harness.ts @@ -95,7 +95,7 @@ export function installSystemPackagesStep(packages: readonly string[]): string { "RUN if command -v apt-get >/dev/null", `then apt-get update && apt-get install -y --no-install-recommends ${list}`, `elif command -v dnf >/dev/null; then dnf install -y --setopt=install_weak_deps=False --allowerasing ${list}`, - `elif command -v apk >/dev/null; then apk add --no-cache ${list}`, + `elif command -v apk >/dev/null; then apk add --no-cache bash ${list}`, 'else echo "no supported package manager (apt-get, dnf, apk)" >&2 && exit 1', "fi", ].join("; "); diff --git a/src/benchmarks/terminal-bench/ori-solver.test.ts b/src/benchmarks/terminal-bench/ori-solver.test.ts index 5fe6c22..62f5d9e 100644 --- a/src/benchmarks/terminal-bench/ori-solver.test.ts +++ b/src/benchmarks/terminal-bench/ori-solver.test.ts @@ -643,7 +643,9 @@ describe("terminal-bench ori solver", () => { expect(step).toContain( "elif command -v dnf >/dev/null; then dnf install -y" ); - expect(step).toContain("elif command -v apk >/dev/null; then apk add"); + expect(step).toContain( + "elif command -v apk >/dev/null; then apk add --no-cache bash " + ); expect(step).toContain("exit 1; fi"); expect(step).not.toMatch(/^RUN apt-get/); }