diff --git a/.github/workflows/terminal-bench.yml b/.github/workflows/terminal-bench.yml
new file mode 100644
index 000000000..9cc05d23e
--- /dev/null
+++ b/.github/workflows/terminal-bench.yml
@@ -0,0 +1,205 @@
+name: Terminal Bench
+
+on:
+ schedule:
+ - cron: '0 7 * * *'
+ workflow_dispatch:
+ inputs:
+ tb2_tasks:
+ description: Comma-separated Terminal Bench 2 task names for the Linux Harbor lane
+ default: fix-git,financial-document-processor,code-from-image,chess-best-move,git-multibranch,kv-store-grpc,large-scale-text-editing
+ required: true
+ tb2_dataset:
+ description: Harbor dataset for the TB2 lane
+ default: terminal-bench@2.0
+ required: true
+ model:
+ description: Optional model passed to ug --model (blank uses ug's default)
+ default: ''
+ required: false
+ claude_version:
+ description: Exact Claude Code version
+ default: 2.1.290
+ required: true
+ codex_version:
+ description: Exact Codex version
+ default: 0.154.0
+ required: true
+
+permissions:
+ contents: read
+
+concurrency:
+ group: terminal-bench-${{ github.ref }}
+ cancel-in-progress: false
+
+env:
+ PYTHONUTF8: 1
+ TB2_TASKS: ${{ inputs.tb2_tasks || 'fix-git,financial-document-processor,code-from-image,chess-best-move,git-multibranch,kv-store-grpc,large-scale-text-editing' }}
+ TB2_DATASET: ${{ inputs.tb2_dataset || 'terminal-bench@2.0' }}
+ MODEL: ${{ inputs.model }}
+ CLAUDE_VERSION: ${{ inputs.claude_version || '2.1.290' }}
+ CODEX_VERSION: ${{ inputs.codex_version || '0.154.0' }}
+ HARBOR_VERSION: 0.23.0
+ WINDOWS_PACKAGE_INDEX: https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple
+ WINDOWS_NPM_REGISTRY: https://databricks.jfrog.io/artifactory/api/npm/db-npm/
+
+jobs:
+ # Real TB2 tasks run in Harbor's Linux containers, with ug installed inside each one.
+ harbor:
+ name: Harbor · ${{ matrix.agent == 'claude' && 'Claude' || 'Codex' }} · Linux
+ runs-on: ubuntu-latest
+ timeout-minutes: 240
+ strategy:
+ fail-fast: false
+ matrix:
+ agent: [claude, codex]
+ env:
+ UV_INDEX_URL: https://pypi.org/simple
+ # bench_auth.py mints a fresh token per task from the CUJ service principal when
+ # its secrets are set, and otherwise falls back to the pre-fetched e2e bearer.
+ UCODE_TEST_WORKSPACE: ${{ secrets.UCODE_TEST_WORKSPACE }}
+ DATABRICKS_BEARER: ${{ secrets.DATABRICKS_BEARER }}
+ UG_BENCH_SP_WORKSPACE: ${{ secrets.UG_CUJ1_WORKSPACE }}
+ UG_BENCH_CLIENT_ID: ${{ secrets.UG_CUJ_SP_CLIENT_ID }}
+ UG_BENCH_CLIENT_SECRET: ${{ secrets.UG_CUJ_SP_CLIENT_SECRET }}
+ PYTHONPATH: ${{ github.workspace }}/bench/terminal_bench
+ AGENT_CLASS: ${{ matrix.agent == 'claude' && 'UgClaude' || 'UgCodex' }}
+ steps:
+ - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+ with:
+ persist-credentials: false
+ - uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
+ - name: Build the ug wheel installed into each task container
+ run: |
+ uv build --wheel --out-dir "$RUNNER_TEMP/dist"
+ echo "UG_BENCH_WHEEL=$(ls "$RUNNER_TEMP"/dist/*.whl)" >> "$GITHUB_ENV"
+ - name: Run TB2 tasks
+ env:
+ AGENT_VERSION: ${{ matrix.agent == 'claude' && env.CLAUDE_VERSION || env.CODEX_VERSION }}
+ run: |
+ includes=()
+ IFS=, read -ra tasks <<< "$TB2_TASKS"
+ for task in "${tasks[@]}"; do includes+=(-i "$task"); done
+ uvx --python 3.12 --from "harbor==$HARBOR_VERSION" harbor run -y \
+ -d "$TB2_DATASET" "${includes[@]}" \
+ -a "ug_agent:$AGENT_CLASS" --ak "version=$AGENT_VERSION" \
+ ${MODEL:+-m "$MODEL"} -n 3 -o "$RUNNER_TEMP/harbor-jobs/tb2"
+ - name: Run ug's own tasks
+ if: ${{ !cancelled() }}
+ env:
+ AGENT_VERSION: ${{ matrix.agent == 'claude' && env.CLAUDE_VERSION || env.CODEX_VERSION }}
+ run: |
+ includes=()
+ while read -r task; do includes+=(-i "$task"); done \
+ < <(python3 bench/terminal_bench/bench_tasks.py "${{ matrix.agent }}")
+ uvx --python 3.12 --from "harbor==$HARBOR_VERSION" harbor run -y \
+ -p bench/terminal_bench/tasks "${includes[@]}" --resume-trajectory \
+ -a "ug_agent:$AGENT_CLASS" --ak "version=$AGENT_VERSION" \
+ ${MODEL:+-m "$MODEL"} -n 3 -o "$RUNNER_TEMP/harbor-jobs/ug"
+ - name: Summarize
+ if: ${{ !cancelled() }}
+ run: |
+ status=0
+ python3 bench/terminal_bench/summarize_harbor.py "$RUNNER_TEMP/harbor-jobs/tb2" "TB2 · ${{ matrix.agent }}" || status=1
+ python3 bench/terminal_bench/summarize_harbor.py "$RUNNER_TEMP/harbor-jobs/ug" "ug tasks in Harbor · ${{ matrix.agent }}" || status=1
+ exit $status
+ - name: Scrub the bearer token from job output
+ if: ${{ !cancelled() }}
+ run: python3 bench/terminal_bench/scrub_secrets.py "$RUNNER_TEMP/harbor-jobs"
+ - name: Upload Harbor jobs
+ if: ${{ !cancelled() }}
+ uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+ with:
+ name: terminal-bench-harbor-${{ matrix.agent }}
+ path: ${{ runner.temp }}/harbor-jobs/
+
+ # Windows runners can't run Linux containers, so ug's own tasks also run on the host.
+ native:
+ name: Native · ${{ matrix.agent == 'claude' && 'Claude' || 'Codex' }} · ${{ matrix.os == 'windows-server-latest' && 'Windows' || 'Linux' }}
+ runs-on: ${{ matrix.os }}
+ timeout-minutes: 90
+ permissions:
+ contents: read
+ id-token: write
+ defaults:
+ run:
+ shell: bash
+ strategy:
+ fail-fast: false
+ matrix:
+ os: [ubuntu-latest, windows-server-latest]
+ agent: [claude, codex]
+ env:
+ UV_INDEX_URL: https://pypi.org/simple
+ # bench_auth.py mints a fresh token per task from the CUJ service principal when
+ # its secrets are set, and otherwise falls back to the pre-fetched e2e bearer.
+ UCODE_TEST_WORKSPACE: ${{ secrets.UCODE_TEST_WORKSPACE }}
+ DATABRICKS_BEARER: ${{ secrets.DATABRICKS_BEARER }}
+ UG_BENCH_SP_WORKSPACE: ${{ secrets.UG_CUJ1_WORKSPACE }}
+ UG_BENCH_CLIENT_ID: ${{ secrets.UG_CUJ_SP_CLIENT_ID }}
+ UG_BENCH_CLIENT_SECRET: ${{ secrets.UG_CUJ_SP_CLIENT_SECRET }}
+ AGENT: ${{ matrix.agent }}
+ AGENT_PACKAGE: ${{ matrix.agent == 'claude' && '@anthropic-ai/claude-code' || '@openai/codex' }}
+ steps:
+ - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+ with:
+ persist-credentials: false
+ - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
+ with:
+ node-version: 22.19.0
+ - uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
+ - uses: databricks/setup-cli@bdb89f81c11a5bd647fd55b585b7c396ec68a25a # v1.0.0
+ - name: Expose the Databricks CLI on Windows
+ if: ${{ runner.os == 'Windows' }}
+ run: |
+ test -f "$RUNNER_TEMP/.bin/databricks.exe"
+ cygpath -w "$RUNNER_TEMP/.bin" >> "$GITHUB_PATH"
+ - name: Authenticate to JFrog with GitHub OIDC
+ id: jfrog
+ if: ${{ runner.os == 'Windows' }}
+ uses: jfrog/setup-jfrog-cli@279b1f629f43dd5bc658d8361ac4802a7ef8d2d5 # v4.9.1
+ env:
+ JF_URL: https://databricks.jfrog.io
+ with:
+ oidc-provider-name: github-actions
+ - name: Install the agent CLI
+ env:
+ NPM_TOKEN: ${{ steps.jfrog.outputs.oidc-token }}
+ AGENT_VERSION: ${{ matrix.agent == 'claude' && env.CLAUDE_VERSION || env.CODEX_VERSION }}
+ run: |
+ if [[ "$RUNNER_OS" == Windows ]]; then
+ npmrc="$RUNNER_TEMP/jfrog.npmrc"
+ printf 'registry=%s\n//%s:_authToken=%s\n' \
+ "$WINDOWS_NPM_REGISTRY" "${WINDOWS_NPM_REGISTRY#https://}" "$NPM_TOKEN" > "$npmrc"
+ npm install -g --userconfig "$npmrc" "$AGENT_PACKAGE@$AGENT_VERSION"
+ rm "$npmrc"
+ else
+ npm install -g "$AGENT_PACKAGE@$AGENT_VERSION"
+ fi
+ - name: Install ug
+ env:
+ UV_INDEX_DATABRICKS_PYPI_USERNAME: ${{ steps.jfrog.outputs.oidc-user }}
+ UV_INDEX_DATABRICKS_PYPI_PASSWORD: ${{ steps.jfrog.outputs.oidc-token }}
+ run: |
+ if [[ "$RUNNER_OS" == Windows ]]; then
+ unset UV_INDEX_URL
+ export UV_DEFAULT_INDEX="databricks-pypi=$WINDOWS_PACKAGE_INDEX"
+ fi
+ uv lock
+ uv tool install .
+ uv sync
+ - name: Run ug's own tasks on the host
+ run: uv run --no-sync python bench/terminal_bench/run_native.py --agent "$AGENT" --output "$RUNNER_TEMP/tb-native" ${MODEL:+--model "$MODEL"} < /dev/null
+ - name: Scrub the bearer token from task output
+ if: ${{ !cancelled() }}
+ run: uv run --no-sync python bench/terminal_bench/scrub_secrets.py "$RUNNER_TEMP/tb-native"
+ - name: Upload native results
+ if: ${{ !cancelled() }}
+ uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+ with:
+ name: terminal-bench-native-${{ matrix.agent }}-${{ matrix.os }}
+ include-hidden-files: true
+ path: |
+ ${{ runner.temp }}/tb-native/results.json
+ ${{ runner.temp }}/tb-native/logs/
diff --git a/bench/terminal_bench/README.md b/bench/terminal_bench/README.md
new file mode 100644
index 000000000..6cc4dce76
--- /dev/null
+++ b/bench/terminal_bench/README.md
@@ -0,0 +1,134 @@
+# Terminal Bench for ug
+
+```text
+ _ _ _ _ _
+| |_ ___ _ _ _ __ (_)_ _ __ _| | | |__ ___ _ _ __| |_
+| _/ -_) '_| ' \| | ' \/ _` | | | '_ \/ -_) ' \/ _| ' \
+ \__\___|_| |_|_|_|_|_||_\__,_|_| |_.__/\___|_||_\__|_||_|
+```
+
+This directory runs Terminal Bench tasks through `ug claude` and `ug codex`. It checks that an agent keeps working when ug configures and launches it. The `Terminal Bench` workflow in `.github/workflows/terminal-bench.yml` runs every night at 07:00 UTC and on manual dispatch.
+
+The workflow runs tasks in two lanes. The Harbor lane runs a [Terminal Bench 2](https://www.tbench.ai/) subset and the tasks in `tasks/` in Docker containers on Linux. The native lane runs the tasks in `tasks/` directly on Linux and Windows runners.
+
+```mermaid
+flowchart LR
+ workflow["Terminal Bench workflow"] --> harbor["Harbor lane
Linux, Docker"]
+ workflow --> native["Native lane
Linux and Windows"]
+ harbor --> tb2["TB2 subset"]
+ harbor --> own["tasks/"]
+ native --> own
+```
+
+Each lane runs its tasks for Claude and for Codex. [TASKS.md](TASKS.md) describes the task format and lists every task.
+
+## Why there are two lanes
+
+Each lane tests something the other can't.
+
+The Harbor lane exists for real TB2 tasks. Each TB2 task ships a Linux Docker image with the services, packages, and root access it needs. Harbor starts a fresh container per trial, so a task can't change the runner or another trial. `ug_agent.py` installs ug, the Databricks CLI, and the agent CLI in each container.
+
+The native lane exists for Windows. GitHub's Windows runners can't run Linux containers, and TB2 has no Windows images, so Harbor can't test ug on Windows. `run_native.py` runs the tasks in `tasks/` on the runner itself instead. Those tasks use only the Python standard library so they run on both operating systems.
+
+The native Linux job overlaps with Harbor. It adds one thing: ug running outside a container, on a machine with a normal user home.
+
+## How a task runs
+
+Both lanes run a task the same way. The agent gets the task's `instruction.md` once, at launch, and works on its own until it exits or hits the task's `timeout_sec`. Nothing sends it more instructions while it runs. That matches Terminal Bench and Harbor's built-in `claude-code` and `codex` agents, which also launch `claude --print` and `codex exec` with the instruction and wait.
+
+Claude reads the instruction on stdin in print mode (`-p`). Codex gets it as an argument to `codex exec`. The host mints the gateway token, so the agent never sees the service principal secret. The verifier runs after the agent exits and decides the result.
+
+```mermaid
+sequenceDiagram
+ participant host as Runner host
+ participant env as Task directory
+ participant agent as ug and agent CLI
+ participant gateway as AI Gateway
+ host->>env: Copy environment/app/ and run environment/setup.py
+ host->>host: bench_auth.py mints a token
+ host->>agent: ug configure, then ug claude or ug codex with instruction.md
+ agent->>gateway: Model calls with the token
+ agent->>env: Read files, run commands, write output
+ host->>env: Run tests/test_outputs.py
+ env-->>host: Exit code 0 is a pass
+```
+
+Every launch runs with smart routing on (`ENABLE_SMART_ROUTING_V2=1`). ug routes Claude's prompt before launch, because print mode (`-p`) has no terminal UI to switch models in, and it routes Claude's subagents. Codex sends ug's router header on every model call.
+
+The Harbor lane runs `ug configure` in each container. The native lane runs it once at the start of the job.
+
+A multi-step task launches the agent again for each step, with that step's instruction and a new token. The new launch resumes the previous session with `--continue` for Claude or `exec resume --last` for Codex, and the verifier runs after each step. All TB2 tasks have one step. `resume-session` is the only multi-step task here.
+
+## What the bench doesn't test
+
+The bench treats ug as a black box. It installs ug from this checkout, runs `ug configure`, and launches the agent with `ug claude` or `ug codex`. Verifiers check only what the agent left behind. Some of ug runs differently from a user's setup:
+
+- Auth goes through `DATABRICKS_BEARER` with a token the host mints. ug's `databricks auth login` and OAuth flows don't run.
+- Databricks AI Tools, the skills and plugins that `--enable-databricks-ai-tools` installs, aren't installed. That's ug's default.
+- The CUJ1 workspace's managed config lists no MCP servers, so ug registers none.
+- The harness installs a pinned Claude Code or Codex before ug starts, so ug's agent install doesn't run for that agent. It can still run for the other one. In a Claude job, ug tries to install Codex because the CUJ1 managed config enables both.
+- Claude runs with `--dangerously-skip-permissions` and Codex with `--dangerously-bypass-approvals-and-sandbox`. Permission prompts and the Codex sandbox don't run.
+
+## Run the workflow with other tasks or a model
+
+To change the TB2 tasks or pin a model, dispatch the workflow with inputs:
+
+```sh
+gh workflow run terminal-bench.yml -f tb2_tasks=fix-git,code-from-image -f model=
+```
+
+The workflow also takes `tb2_dataset`, `claude_version`, and `codex_version`. GitHub can dispatch the workflow only from a file that is already on `main`.
+
+## Read the results
+
+Each job writes a table of task results to its run summary.
+
+A Harbor job fails only when a trial hits a harness error, such as a failed agent install or an API error. A task with reward 0 doesn't fail the job, because TB2 tasks are meant to be hard. A native job fails when fewer than 75% of its tasks pass. Agents miss a different task or two from run to run, so one miss doesn't fail the job. Change the bar with `run_native.py --min-pass-rate`.
+
+Each job uploads its files as an artifact named `terminal-bench-harbor-` or `terminal-bench-native--`. `scrub_secrets.py` replaces bearer tokens in those files before upload.
+
+To tell a ug problem from an agent mistake, read the verifier output first and the agent log second:
+
+- In a Harbor artifact, read `//verifier/test-stdout.txt`, then `//agent/ug-.txt`.
+- In a native artifact, read `logs//.verifier.log`, then `logs//.log`. A single-step task names its step `agent`.
+
+## Run tasks locally
+
+To check the tasks without a gateway, run the reference solutions. Every task passes.
+
+```sh
+uv run python bench/terminal_bench/run_native.py --agent oracle --output /tmp/tb
+```
+
+Add `--task ` to run one task.
+
+To run the tasks through ug, export credentials as described in [Credentials](#credentials) and pick an agent. The runner calls `ug configure` for that agent, so it replaces your own configuration for it.
+
+```sh
+uv run python bench/terminal_bench/run_native.py --agent claude --output /tmp/tb
+```
+
+To run in Harbor, you need Docker. Build a ug wheel, point `UG_BENCH_WHEEL` at it, and start a job:
+
+```sh
+uv build --wheel --out-dir /tmp/dist
+export UG_BENCH_WHEEL=$(ls /tmp/dist/*.whl)
+PYTHONPATH=bench/terminal_bench uvx --python 3.12 --from harbor==0.23.0 harbor run -y \
+ -d terminal-bench@2.0 -i fix-git -a ug_agent:UgClaude -o /tmp/harbor-jobs
+```
+
+For Codex, use `ug_agent:UgCodex`. To run ug's own tasks, replace `-d terminal-bench@2.0` with `-p bench/terminal_bench/tasks --resume-trajectory`.
+
+## Credentials
+
+`bench_auth.py` picks the credentials for each task. If `UG_BENCH_CLIENT_ID`, `UG_BENCH_CLIENT_SECRET`, and `UG_BENCH_SP_WORKSPACE` are set, it mints a new OAuth M2M token on the host before each task. The agent gets that token and never the service principal secret, because the agent has a full shell. If those variables aren't set, it uses `DATABRICKS_BEARER` against `UCODE_TEST_WORKSPACE`.
+
+CI uses the `UG_CUJ_SP_CLIENT_ID` service principal against `UG_CUJ1_WORKSPACE`. That workspace publishes a managed coding-agent config, so its model policy applies to every CI run. The policy doesn't allow haiku, for example.
+
+## Add a task
+
+1. Copy the task in `tasks/` that is closest to yours. [TASKS.md](TASKS.md) describes each file.
+2. Write `tests/test_outputs.py` so it checks evidence that only the feature can produce, such as a hook log or an MCP call log, as well as the answer. An agent can read every file in the task, so the answer alone is easy to fake.
+3. If only one agent supports the feature, set `agents` under `[metadata]` in `task.toml`.
+4. Run `run_native.py --agent oracle --task ` and confirm the task passes.
+5. Confirm the verifier fails when the agent does nothing. Copy `environment/app/` into an empty directory, or run `environment/setup.py `. Then run `tests/test_outputs.py` with `TASK_APP_DIR` set to that directory.
diff --git a/bench/terminal_bench/TASKS.md b/bench/terminal_bench/TASKS.md
new file mode 100644
index 000000000..c16ef5e75
--- /dev/null
+++ b/bench/terminal_bench/TASKS.md
@@ -0,0 +1,54 @@
+# Task reference
+
+Each directory in `tasks/` is one task in Harbor's task format. Harbor and `run_native.py` read the same files. `bench_tasks.py ` prints the tasks that one agent runs.
+
+## Files in a task
+
+| Path | Contents |
+|------|----------|
+| `task.toml` | Timeouts, resources, and metadata. `agents` under `[metadata]` lists the agents that run the task, and defaults to Claude and Codex. Each `[[steps]]` entry adds a step. |
+| `instruction.md` | The prompt the agent gets. |
+| `environment/Dockerfile` | The Harbor image. It copies `environment/app/` to `/app` and runs `environment/setup.py /app` if the task has them. |
+| `environment/app/` | Fixture files. The runner copies them into the task directory. |
+| `environment/setup.py` | Generates fixtures. It takes the task directory as its only argument. |
+| `tests/test_outputs.py` | The verifier. Exit code 0 is a pass. |
+| `tests/test.sh` | Runs the verifier in Harbor and writes the reward to `/logs/verifier/reward.txt`. |
+| `solution/solve.py` | The reference solution that `--agent oracle` runs. |
+| `solution/solve.sh` | Runs the reference solution in Harbor. |
+
+A multi-step task has `instruction.md`, `tests/`, and `solution/` under `steps//` for each step. Each step after the first resumes the previous session. Harbor resumes with `--resume-trajectory`. `run_native.py` passes `--continue` to Claude and `exec resume --last` to Codex. A failed step stops the task.
+
+Verifiers and solutions use only the Python standard library. They run on Windows and in `python:3.12-slim`.
+
+## Environment variables
+
+Verifiers and solutions read these variables.
+
+| Variable | Value |
+|----------|-------|
+| `TASK_APP_DIR` | The task directory. Defaults to `/app`, which is the task directory in Harbor. |
+| `TASK_AGENT_LOG` | The agent's log for the current step. Only `run_native.py` sets it. In Harbor, read `/logs/agent/ug-.txt`. |
+| `TASK_ORACLE` | `1` when the oracle runs. A check that reads the agent log skips itself. |
+
+## ug's tasks
+
+| Task | Agents | Passes when |
+|------|--------|-------------|
+| `user-hooks` | Claude | `release.txt` holds the codename from the project's `UserPromptSubmit` hook, and the `PostToolUse` hook logged the write. |
+| `mcp-server` | Claude | `stock.json` matches the random values that the project's stdio MCP server logged for each lookup. |
+| `project-instructions` | Claude, Codex | The output is under `out/`, and each new Python file starts with the license header that `CLAUDE.md` and `AGENTS.md` require. |
+| `project-skill` | Claude | The release notes follow the project skill, and the skill's script logged a run. |
+| `subagent-model` | Claude | Each file has a correct audit, and the transcript shows a subagent on a haiku model. If a workspace policy disallows haiku, the transcript must show the policy notice and the subagent on the fallback model it names. |
+| `image-code` | Claude, Codex | `code.txt` holds the 8-digit code drawn in `code.png`. |
+| `pdf-extract` | Claude, Codex | `total.txt` holds the total from `invoice.pdf`, whose text stream is compressed. |
+| `web-fetch` | Claude, Codex | `answer.txt` holds a value from a file in this repo, pinned to a fixed commit. |
+| `background-shell` | Claude, Codex | `result.txt` holds the value that a 150-second job prints when it finishes. Claude's default command timeout is 2 minutes. |
+| `resume-session` | Claude, Codex | Step 1 deletes `vault.txt` without saving its passphrase to a file. Step 2 resumes the session and writes the passphrase to `answer.txt`. |
+| `log-triage` | Claude, Codex | `report.json` has the correct counts for five independent service directories. |
+| `fix-failing-tests` | Claude, Codex | The unit tests and extra hidden checks pass, and `tests/` is unchanged. |
+| `scrub-git-secret` | Claude, Codex | Every copy of the token in history is replaced with `REDACTED`, and both branches keep their commit messages in order. |
+| `background-service` | Claude, Codex | `token.txt` matches the token the server served, and nothing listens on the server's port afterward. |
+
+## Default TB2 subset
+
+The Harbor lane runs these Terminal Bench 2 tasks unless the `tb2_tasks` input overrides them: `fix-git`, `financial-document-processor`, `code-from-image`, `chess-best-move`, `git-multibranch`, `kv-store-grpc`, and `large-scale-text-editing`.
diff --git a/bench/terminal_bench/bench_auth.py b/bench/terminal_bench/bench_auth.py
new file mode 100644
index 000000000..74bb41fd4
--- /dev/null
+++ b/bench/terminal_bench/bench_auth.py
@@ -0,0 +1,60 @@
+"""Workspace auth for bench runs.
+
+With service-principal credentials set, mint a fresh bearer per task on the host so
+long runs outlive a single token and the SP secret never reaches the agent.
+Otherwise fall back to a pre-fetched DATABRICKS_BEARER.
+"""
+
+from __future__ import annotations
+
+import base64
+import json
+import os
+import urllib.parse
+import urllib.request
+
+SECRET_ENV = ("UG_BENCH_CLIENT_ID", "UG_BENCH_CLIENT_SECRET")
+
+
+def _sp_credentials() -> tuple[str, str] | None:
+ client_id, secret = (os.environ.get(name, "").strip() for name in SECRET_ENV)
+ return (client_id, secret) if client_id and secret else None
+
+
+def workspace() -> str:
+ if _sp_credentials() and (host := os.environ.get("UG_BENCH_SP_WORKSPACE", "").strip()):
+ return host
+ host = os.environ.get("UCODE_TEST_WORKSPACE", "").strip()
+ if not host:
+ raise RuntimeError("Set UCODE_TEST_WORKSPACE, or UG_BENCH_SP_WORKSPACE with SP credentials")
+ return host
+
+
+def bearer(host: str) -> str:
+ credentials = _sp_credentials()
+ if credentials is None:
+ token = os.environ.get("DATABRICKS_BEARER", "").strip()
+ if not token:
+ raise RuntimeError(
+ "Set DATABRICKS_BEARER, or UG_BENCH_CLIENT_ID/UG_BENCH_CLIENT_SECRET"
+ )
+ return token
+ basic = base64.b64encode(":".join(credentials).encode()).decode()
+ request = urllib.request.Request(
+ f"{host.rstrip('/')}/oidc/v1/token",
+ data=urllib.parse.urlencode(
+ {"grant_type": "client_credentials", "scope": "all-apis"}
+ ).encode(),
+ headers={"Authorization": f"Basic {basic}"},
+ )
+ with urllib.request.urlopen(request, timeout=30) as response:
+ return json.load(response)["access_token"]
+
+
+def agent_env(base: dict[str, str]) -> dict[str, str]:
+ """Copy of `base` with a fresh bearer and without the SP secret."""
+ env = {key: value for key, value in base.items() if key not in SECRET_ENV}
+ host = workspace()
+ env["DATABRICKS_HOST"] = host
+ env["DATABRICKS_BEARER"] = bearer(host)
+ return env
diff --git a/bench/terminal_bench/bench_tasks.py b/bench/terminal_bench/bench_tasks.py
new file mode 100644
index 000000000..8bb8cddb6
--- /dev/null
+++ b/bench/terminal_bench/bench_tasks.py
@@ -0,0 +1,24 @@
+"""List bench tasks for an agent. Tasks opt in with `[metadata] agents` in task.toml."""
+
+import sys
+import tomllib
+from pathlib import Path
+
+TASKS_DIR = Path(__file__).resolve().parent / "tasks"
+
+
+def task_agents(config: dict) -> list[str]:
+ return config.get("metadata", {}).get("agents", ["claude", "codex"])
+
+
+def names_for(agent: str) -> list[str]:
+ return [
+ task.name
+ for task in sorted(TASKS_DIR.iterdir())
+ if agent == "oracle"
+ or agent in task_agents(tomllib.loads((task / "task.toml").read_text()))
+ ]
+
+
+if __name__ == "__main__":
+ print("\n".join(names_for(sys.argv[1])))
diff --git a/bench/terminal_bench/run_native.py b/bench/terminal_bench/run_native.py
new file mode 100644
index 000000000..3ca1a9da5
--- /dev/null
+++ b/bench/terminal_bench/run_native.py
@@ -0,0 +1,230 @@
+"""Run Terminal Bench-format tasks directly on the host through ug.
+
+Windows runners can't run the Linux containers Harbor uses, so this runner
+copies each task's fixtures into a temp dir, launches the agent headlessly
+there, and runs the task's Python verifier against the result.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import shutil
+import signal
+import subprocess
+import sys
+import time
+import tomllib
+from pathlib import Path
+
+import bench_auth
+from bench_tasks import TASKS_DIR, names_for
+
+from ucode.os_compatibility.subprocess_cross_os import popen, run
+
+
+def agent_command(
+ agent: str, ug: str, model: str | None, step_dir: Path, instruction: str, resume: bool
+) -> tuple[list[str], str | None]:
+ """Return argv plus the stdin to send. Codex takes the prompt as an argument."""
+ model_args = ["--model", model] if model else []
+ if agent == "claude":
+ argv = [ug, "claude", *model_args, "--", "-p", "--verbose", "--output-format"]
+ argv += ["stream-json", "--dangerously-skip-permissions"]
+ return argv + (["--continue"] if resume else []), instruction
+ if agent == "codex":
+ argv = [ug, "codex", *model_args, "--", "exec", *(["resume", "--last"] if resume else [])]
+ argv += ["--dangerously-bypass-approvals-and-sandbox", "--skip-git-repo-check", "--json"]
+ return argv + ["--enable", "unified_exec", "--", instruction], None
+ return [sys.executable, str(step_dir / "solution" / "solve.py")], None
+
+
+def task_steps(task_dir: Path, config: dict) -> list[tuple[str, Path, float]]:
+ """(name, dir holding instruction.md/tests/solution, agent timeout) for each step."""
+ default = config.get("agent", {}).get("timeout_sec", 600)
+ if not config.get("steps"):
+ return [("agent", task_dir, default)]
+ return [
+ (
+ step["name"],
+ task_dir / "steps" / step["name"],
+ step.get("agent", {}).get("timeout_sec", default),
+ )
+ for step in config["steps"]
+ ]
+
+
+def kill_tree(proc: subprocess.Popen) -> None:
+ if os.name == "nt":
+ run(["taskkill", "/T", "/F", "/PID", str(proc.pid)], capture_output=True)
+ else:
+ try:
+ os.killpg(proc.pid, signal.SIGKILL)
+ except ProcessLookupError:
+ pass
+ proc.wait()
+
+
+def prepare(task_dir: Path, workdir: Path) -> None:
+ app = task_dir / "environment" / "app"
+ if app.is_dir():
+ shutil.copytree(app, workdir)
+ workdir.mkdir(parents=True, exist_ok=True)
+ setup = task_dir / "environment" / "setup.py"
+ if setup.exists():
+ run([sys.executable, str(setup), str(workdir)], check=True)
+
+
+def run_agent(
+ argv: list[str], stdin: str | None, workdir: Path, env: dict, log_path: Path, timeout: float
+) -> tuple[subprocess.Popen, bool]:
+ with log_path.open("w", encoding="utf-8") as log:
+ proc = popen(
+ argv,
+ cwd=workdir,
+ env=env,
+ stdin=subprocess.PIPE if stdin is not None else subprocess.DEVNULL,
+ stdout=log,
+ stderr=subprocess.STDOUT,
+ text=True,
+ start_new_session=os.name != "nt",
+ )
+ try:
+ proc.communicate(stdin, timeout=timeout)
+ except subprocess.TimeoutExpired:
+ kill_tree(proc)
+ return proc, True
+ return proc, False
+
+
+def verify(step_dir: Path, workdir: Path, env: dict, timeout: float) -> tuple[bool, str]:
+ try:
+ verifier = run(
+ [sys.executable, str(step_dir / "tests" / "test_outputs.py")],
+ cwd=workdir,
+ env=env,
+ capture_output=True,
+ text=True,
+ timeout=timeout,
+ )
+ except subprocess.TimeoutExpired:
+ return False, f"verifier timed out after {timeout}s\n"
+ return verifier.returncode == 0, verifier.stdout + verifier.stderr
+
+
+def run_task(args: argparse.Namespace, task_dir: Path, out: Path) -> dict:
+ config = tomllib.loads((task_dir / "task.toml").read_text())
+ verifier_timeout = config.get("verifier", {}).get("timeout_sec", 120)
+ workdir = out / "work" / task_dir.name
+ logs = out / "logs" / task_dir.name
+ logs.mkdir(parents=True, exist_ok=True)
+ prepare(task_dir, workdir)
+
+ env = {**os.environ, "TASK_APP_DIR": str(workdir), "IS_SANDBOX": "1"}
+ env["ENABLE_SMART_ROUTING_V2"] = "1"
+ if args.agent == "oracle":
+ # Checks that read the agent transcript have nothing to read for the oracle.
+ env["TASK_ORACLE"] = "1"
+ # Tasks say "run it with Python", so put this interpreter first on PATH.
+ env["PATH"] = os.pathsep.join([str(Path(sys.executable).parent), env.get("PATH", "")])
+ started = time.monotonic()
+ result = {"task": task_dir.name, "passed": True, "agent_exit": None, "timed_out": False}
+ for index, (name, step_dir, agent_timeout) in enumerate(task_steps(task_dir, config)):
+ if args.agent != "oracle":
+ env = bench_auth.agent_env(env)
+ log_path = logs / f"{name}.log"
+ env["TASK_AGENT_LOG"] = str(log_path)
+ instruction = (step_dir / "instruction.md").read_text()
+ argv, stdin = agent_command(
+ args.agent, args.ug, args.model, step_dir, instruction, resume=index > 0
+ )
+ proc, timed_out = run_agent(argv, stdin, workdir, env, log_path, agent_timeout)
+ # Verify before cleanup so a server the agent left running still counts against it.
+ passed, verifier_log = verify(step_dir, workdir, env, verifier_timeout)
+ (logs / f"{name}.verifier.log").write_text(verifier_log, encoding="utf-8")
+ # ug's routing wrappers log under ~/.ucode; keep them as evidence that routing ran.
+ for ucode_log in (Path.home() / ".ucode").glob("*.log"):
+ (logs / f"{name}.ucode").mkdir(exist_ok=True)
+ shutil.copy2(ucode_log, logs / f"{name}.ucode" / ucode_log.name)
+ if not timed_out and os.name != "nt":
+ kill_tree(proc)
+ result.update(agent_exit=proc.returncode, timed_out=timed_out)
+ if not passed:
+ result.update(passed=False, failed_step=name)
+ break
+ result["duration_sec"] = round(time.monotonic() - started, 1)
+ return result
+
+
+def write_summary(args: argparse.Namespace, results: list[dict], out: Path) -> None:
+ (out / "results.json").write_text(json.dumps(results, indent=2))
+ passed = sum(r["passed"] for r in results)
+ lines = [
+ f"### Native tasks · {args.agent} · {sys.platform}: {passed}/{len(results)} passed"
+ f" (needs {args.min_pass_rate:.0%})",
+ "",
+ "| Task | Result | Agent exit | Time (s) |",
+ "|------|--------|-----------|----------|",
+ ]
+ for r in results:
+ status = "pass" if r["passed"] else ("timeout" if r["timed_out"] else "fail")
+ lines.append(f"| {r['task']} | {status} | {r['agent_exit']} | {r['duration_sec']} |")
+ summary = "\n".join(lines) + "\n"
+ print(summary)
+ if step_summary := os.environ.get("GITHUB_STEP_SUMMARY"):
+ with open(step_summary, "a", encoding="utf-8") as f:
+ f.write(summary)
+
+
+def main() -> int:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--agent", choices=["claude", "codex", "oracle"], required=True)
+ parser.add_argument("--task", action="append", help="Task name; repeatable. Default: all.")
+ parser.add_argument("--output", type=Path, required=True)
+ parser.add_argument("--ug", default=shutil.which("ug"), help="Absolute path to ug.")
+ parser.add_argument("--model")
+ parser.add_argument(
+ "--min-pass-rate",
+ type=float,
+ default=0.75,
+ help="Fraction of tasks that must pass for exit code 0. Agents miss some tasks run to run.",
+ )
+ args = parser.parse_args()
+
+ names = args.task or names_for(args.agent)
+ out = args.output.resolve()
+ if (out / "work").exists():
+ shutil.rmtree(out / "work")
+ out.mkdir(parents=True, exist_ok=True)
+
+ if args.agent != "oracle":
+ if not args.ug:
+ parser.error("--ug is required when ug isn't on PATH")
+ args.ug = str(Path(args.ug).resolve())
+ env = bench_auth.agent_env(dict(os.environ))
+ run(
+ [
+ args.ug,
+ "configure",
+ "--agents",
+ args.agent,
+ "--workspace",
+ env["DATABRICKS_HOST"],
+ "--skip-validate",
+ "--skip-upgrade",
+ "--disable-databricks-ai-tools",
+ ],
+ check=True,
+ env=env,
+ stdin=subprocess.DEVNULL,
+ )
+
+ results = [run_task(args, TASKS_DIR / name, out) for name in names]
+ write_summary(args, results, out)
+ passed = sum(r["passed"] for r in results)
+ return 0 if results and passed / len(results) >= args.min_pass_rate else 1
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/bench/terminal_bench/scrub_secrets.py b/bench/terminal_bench/scrub_secrets.py
new file mode 100644
index 000000000..5991caa72
--- /dev/null
+++ b/bench/terminal_bench/scrub_secrets.py
@@ -0,0 +1,20 @@
+"""Redact Databricks bearers from every file under a directory before upload.
+
+Per-task tokens are minted on the fly, so match token shapes as well as the
+DATABRICKS_BEARER value.
+"""
+
+import os
+import re
+import sys
+from pathlib import Path
+
+TOKEN = re.compile(rb"eyJ[\w-]+\.[\w-]+\.[\w-]+|dapi[0-9a-f]{32}(?:-\d+)?")
+secret = os.environ.get("DATABRICKS_BEARER", "").strip().encode()
+
+for path in Path(sys.argv[1]).rglob("*"):
+ if path.is_file() and not path.is_symlink():
+ data = path.read_bytes()
+ scrubbed = TOKEN.sub(b"***", data.replace(secret, b"***") if secret else data)
+ if scrubbed != data:
+ path.write_bytes(scrubbed)
diff --git a/bench/terminal_bench/summarize_harbor.py b/bench/terminal_bench/summarize_harbor.py
new file mode 100644
index 000000000..93b36c249
--- /dev/null
+++ b/bench/terminal_bench/summarize_harbor.py
@@ -0,0 +1,41 @@
+"""Summarize a Harbor jobs dir as a markdown table.
+
+Exits non-zero when a trial errored for a reason other than a timeout, since
+that points at the ug harness rather than the agent's score.
+"""
+
+import json
+import os
+import sys
+from pathlib import Path
+
+
+def main(jobs_dir: Path, title: str) -> int:
+ rows = []
+ harness_errors = 0
+ for path in sorted(jobs_dir.glob("*/*/result.json")):
+ result = json.loads(path.read_text())
+ rewards = (result.get("verifier_result") or {}).get("rewards") or {}
+ error = (result.get("exception_info") or {}).get("exception_type", "")
+ if error and "Timeout" not in error:
+ harness_errors += 1
+ rows.append((result["task_name"], rewards.get("reward", "-"), error or "-"))
+
+ solved = sum(1 for _, reward, _ in rows if reward == 1)
+ lines = [
+ f"### {title}: {solved}/{len(rows)} solved",
+ "",
+ "| Task | Reward | Error |",
+ "|------|--------|-------|",
+ *(f"| {task} | {reward} | {error} |" for task, reward, error in rows),
+ ]
+ summary = "\n".join(lines) + "\n"
+ print(summary)
+ if step_summary := os.environ.get("GITHUB_STEP_SUMMARY"):
+ with open(step_summary, "a", encoding="utf-8") as f:
+ f.write(summary)
+ return 1 if harness_errors or not rows else 0
+
+
+if __name__ == "__main__":
+ sys.exit(main(Path(sys.argv[1]), sys.argv[2]))
diff --git a/bench/terminal_bench/tasks/background-service/environment/Dockerfile b/bench/terminal_bench/tasks/background-service/environment/Dockerfile
new file mode 100644
index 000000000..ed07be056
--- /dev/null
+++ b/bench/terminal_bench/tasks/background-service/environment/Dockerfile
@@ -0,0 +1,3 @@
+FROM python:3.12-slim
+WORKDIR /app
+COPY app/ /app/
diff --git a/bench/terminal_bench/tasks/background-service/environment/app/server.py b/bench/terminal_bench/tasks/background-service/environment/app/server.py
new file mode 100644
index 000000000..b5bc1c6d9
--- /dev/null
+++ b/bench/terminal_bench/tasks/background-service/environment/app/server.py
@@ -0,0 +1,39 @@
+"""Token service. Run it in the background; it stops itself after 15 minutes."""
+
+import hashlib
+import http.server
+import secrets
+import threading
+from pathlib import Path
+
+STATE = Path(__file__).resolve().parent / ".server_state"
+TOKEN = secrets.token_hex(16)
+
+
+class Handler(http.server.BaseHTTPRequestHandler):
+ def do_GET(self) -> None:
+ if self.path != "/token":
+ self.send_error(404)
+ elif self.headers.get("X-Bench-Client") != "ug":
+ self.send_error(403, "missing X-Bench-Client: ug header")
+ else:
+ body = TOKEN.encode()
+ self.send_response(200)
+ self.send_header("Content-Type", "text/plain")
+ self.send_header("Content-Length", str(len(body)))
+ self.end_headers()
+ self.wfile.write(body)
+
+
+def main() -> None:
+ server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler)
+ STATE.mkdir(exist_ok=True)
+ (STATE / "token.sha256").write_text(hashlib.sha256(TOKEN.encode()).hexdigest())
+ (STATE / "port").write_text(str(server.server_address[1]))
+ print(f"listening on 127.0.0.1:{server.server_address[1]}", flush=True)
+ threading.Timer(900, server.shutdown).start()
+ server.serve_forever()
+
+
+if __name__ == "__main__":
+ main()
diff --git a/bench/terminal_bench/tasks/background-service/instruction.md b/bench/terminal_bench/tasks/background-service/instruction.md
new file mode 100644
index 000000000..45919b376
--- /dev/null
+++ b/bench/terminal_bench/tasks/background-service/instruction.md
@@ -0,0 +1,8 @@
+`server.py` in the current directory is a small token service.
+
+1. Start it in the background with Python. Once it's ready it writes its port to `.server_state/port`.
+2. Fetch `GET http://127.0.0.1:/token`. The request needs the header `X-Bench-Client: ug`.
+3. Write the token from the response body to `token.txt`.
+4. Stop the server. Nothing should be listening on that port when you finish.
+
+Don't edit `server.py` or anything in `.server_state/`.
diff --git a/bench/terminal_bench/tasks/background-service/solution/solve.py b/bench/terminal_bench/tasks/background-service/solution/solve.py
new file mode 100644
index 000000000..ce19d1f5f
--- /dev/null
+++ b/bench/terminal_bench/tasks/background-service/solution/solve.py
@@ -0,0 +1,22 @@
+"""Reference solution for background-service."""
+
+import os
+import subprocess
+import sys
+import time
+import urllib.request
+from pathlib import Path
+
+app = Path(os.environ.get("TASK_APP_DIR", "/app"))
+proc = subprocess.Popen([sys.executable, "server.py"], cwd=app)
+port_file = app / ".server_state" / "port"
+for _ in range(100):
+ if port_file.exists() and port_file.read_text():
+ break
+ time.sleep(0.1)
+request = urllib.request.Request(
+ f"http://127.0.0.1:{port_file.read_text()}/token", headers={"X-Bench-Client": "ug"}
+)
+(app / "token.txt").write_text(urllib.request.urlopen(request).read().decode())
+proc.terminate()
+proc.wait()
diff --git a/bench/terminal_bench/tasks/background-service/solution/solve.sh b/bench/terminal_bench/tasks/background-service/solution/solve.sh
new file mode 100755
index 000000000..5dbbeddfd
--- /dev/null
+++ b/bench/terminal_bench/tasks/background-service/solution/solve.sh
@@ -0,0 +1,2 @@
+#!/bin/bash
+python3 /solution/solve.py
diff --git a/bench/terminal_bench/tasks/background-service/task.toml b/bench/terminal_bench/tasks/background-service/task.toml
new file mode 100644
index 000000000..9cad17abe
--- /dev/null
+++ b/bench/terminal_bench/tasks/background-service/task.toml
@@ -0,0 +1,18 @@
+version = "1.0"
+
+[metadata]
+author_name = "Unity Gateway"
+difficulty = "easy"
+category = "system-administration"
+tags = ["background-process", "http", "cleanup"]
+
+[verifier]
+timeout_sec = 120.0
+
+[agent]
+timeout_sec = 600.0
+
+[environment]
+build_timeout_sec = 600.0
+cpus = 1
+memory_mb = 2048
diff --git a/bench/terminal_bench/tasks/background-service/tests/test.sh b/bench/terminal_bench/tasks/background-service/tests/test.sh
new file mode 100755
index 000000000..48266bb0b
--- /dev/null
+++ b/bench/terminal_bench/tasks/background-service/tests/test.sh
@@ -0,0 +1,3 @@
+#!/bin/bash
+mkdir -p /logs/verifier
+if python3 /tests/test_outputs.py; then echo 1 > /logs/verifier/reward.txt; else echo 0 > /logs/verifier/reward.txt; fi
diff --git a/bench/terminal_bench/tasks/background-service/tests/test_outputs.py b/bench/terminal_bench/tasks/background-service/tests/test_outputs.py
new file mode 100644
index 000000000..243ee9956
--- /dev/null
+++ b/bench/terminal_bench/tasks/background-service/tests/test_outputs.py
@@ -0,0 +1,26 @@
+"""Verify the agent fetched the live token and then stopped the server."""
+
+import hashlib
+import os
+import socket
+from pathlib import Path
+
+APP = Path(os.environ.get("TASK_APP_DIR", "/app"))
+
+
+def main() -> None:
+ state = APP / ".server_state"
+ assert (state / "port").exists(), "server.py was never started"
+ token = (APP / "token.txt").read_text().strip()
+ want = (state / "token.sha256").read_text().strip()
+ assert hashlib.sha256(token.encode()).hexdigest() == want, "token.txt does not match"
+
+ port = int((state / "port").read_text())
+ with socket.socket() as sock:
+ sock.settimeout(2)
+ assert sock.connect_ex(("127.0.0.1", port)) != 0, "server is still running"
+ print("ok")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/bench/terminal_bench/tasks/background-shell/environment/Dockerfile b/bench/terminal_bench/tasks/background-shell/environment/Dockerfile
new file mode 100644
index 000000000..ed07be056
--- /dev/null
+++ b/bench/terminal_bench/tasks/background-shell/environment/Dockerfile
@@ -0,0 +1,3 @@
+FROM python:3.12-slim
+WORKDIR /app
+COPY app/ /app/
diff --git a/bench/terminal_bench/tasks/background-shell/environment/app/slow_job.py b/bench/terminal_bench/tasks/background-shell/environment/app/slow_job.py
new file mode 100644
index 000000000..54fdafbfc
--- /dev/null
+++ b/bench/terminal_bench/tasks/background-shell/environment/app/slow_job.py
@@ -0,0 +1,14 @@
+"""A slow job. It prints its result to stdout only once it finishes."""
+
+import hashlib
+import secrets
+import time
+from pathlib import Path
+
+result = secrets.token_hex(8)
+state = Path(__file__).resolve().parent / ".job_state"
+state.mkdir(exist_ok=True)
+(state / "started").write_text(str(time.time()))
+time.sleep(150)
+(state / "result.sha256").write_text(hashlib.sha256(result.encode()).hexdigest())
+print(f"RESULT={result}", flush=True)
diff --git a/bench/terminal_bench/tasks/background-shell/instruction.md b/bench/terminal_bench/tasks/background-shell/instruction.md
new file mode 100644
index 000000000..5c60f0fbb
--- /dev/null
+++ b/bench/terminal_bench/tasks/background-shell/instruction.md
@@ -0,0 +1,3 @@
+Run `python slow_job.py`. It takes about two and a half minutes and prints a line `RESULT=` only when it finishes.
+
+Write the value to `result.txt`. The result is only printed to the job's output.
diff --git a/bench/terminal_bench/tasks/background-shell/solution/solve.py b/bench/terminal_bench/tasks/background-shell/solution/solve.py
new file mode 100644
index 000000000..d744f624c
--- /dev/null
+++ b/bench/terminal_bench/tasks/background-shell/solution/solve.py
@@ -0,0 +1,12 @@
+"""Reference solution for background-shell."""
+
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+app = Path(os.environ.get("TASK_APP_DIR", "/app"))
+out = subprocess.run(
+ [sys.executable, "slow_job.py"], cwd=app, capture_output=True, text=True, check=True
+).stdout
+(app / "result.txt").write_text(out.strip().removeprefix("RESULT=") + "\n")
diff --git a/bench/terminal_bench/tasks/background-shell/solution/solve.sh b/bench/terminal_bench/tasks/background-shell/solution/solve.sh
new file mode 100755
index 000000000..5dbbeddfd
--- /dev/null
+++ b/bench/terminal_bench/tasks/background-shell/solution/solve.sh
@@ -0,0 +1,2 @@
+#!/bin/bash
+python3 /solution/solve.py
diff --git a/bench/terminal_bench/tasks/background-shell/task.toml b/bench/terminal_bench/tasks/background-shell/task.toml
new file mode 100644
index 000000000..9ed323ad6
--- /dev/null
+++ b/bench/terminal_bench/tasks/background-shell/task.toml
@@ -0,0 +1,18 @@
+version = "1.0"
+
+[metadata]
+author_name = "Unity Gateway"
+difficulty = "easy"
+category = "agent-harness"
+agents = ["claude", "codex"]
+
+[verifier]
+timeout_sec = 120.0
+
+[agent]
+timeout_sec = 600.0
+
+[environment]
+build_timeout_sec = 600.0
+cpus = 1
+memory_mb = 2048
diff --git a/bench/terminal_bench/tasks/background-shell/tests/test.sh b/bench/terminal_bench/tasks/background-shell/tests/test.sh
new file mode 100755
index 000000000..48266bb0b
--- /dev/null
+++ b/bench/terminal_bench/tasks/background-shell/tests/test.sh
@@ -0,0 +1,3 @@
+#!/bin/bash
+mkdir -p /logs/verifier
+if python3 /tests/test_outputs.py; then echo 1 > /logs/verifier/reward.txt; else echo 0 > /logs/verifier/reward.txt; fi
diff --git a/bench/terminal_bench/tasks/background-shell/tests/test_outputs.py b/bench/terminal_bench/tasks/background-shell/tests/test_outputs.py
new file mode 100644
index 000000000..845536a6a
--- /dev/null
+++ b/bench/terminal_bench/tasks/background-shell/tests/test_outputs.py
@@ -0,0 +1,19 @@
+"""Verify the agent waited for the long job and captured its output."""
+
+import hashlib
+import os
+from pathlib import Path
+
+APP = Path(os.environ.get("TASK_APP_DIR", "/app"))
+
+
+def main() -> None:
+ state = APP / ".job_state"
+ assert (state / "result.sha256").exists(), "slow_job.py never finished"
+ value = (APP / "result.txt").read_text().strip().removeprefix("RESULT=")
+ assert hashlib.sha256(value.encode()).hexdigest() == (state / "result.sha256").read_text()
+ print("ok")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/bench/terminal_bench/tasks/fix-failing-tests/environment/Dockerfile b/bench/terminal_bench/tasks/fix-failing-tests/environment/Dockerfile
new file mode 100644
index 000000000..ed07be056
--- /dev/null
+++ b/bench/terminal_bench/tasks/fix-failing-tests/environment/Dockerfile
@@ -0,0 +1,3 @@
+FROM python:3.12-slim
+WORKDIR /app
+COPY app/ /app/
diff --git a/bench/terminal_bench/tasks/fix-failing-tests/environment/app/inventory/__init__.py b/bench/terminal_bench/tasks/fix-failing-tests/environment/app/inventory/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/bench/terminal_bench/tasks/fix-failing-tests/environment/app/inventory/pricing.py b/bench/terminal_bench/tasks/fix-failing-tests/environment/app/inventory/pricing.py
new file mode 100644
index 000000000..1cccbcb30
--- /dev/null
+++ b/bench/terminal_bench/tasks/fix-failing-tests/environment/app/inventory/pricing.py
@@ -0,0 +1,34 @@
+"""Order pricing for the inventory service."""
+
+from decimal import ROUND_HALF_UP, Decimal
+
+# (minimum quantity, discount fraction), checked from the largest tier down.
+BULK_TIERS = [(100, Decimal("0.15")), (50, Decimal("0.10")), (10, Decimal("0.05"))]
+
+TAX_RATES = {"CA": Decimal("0.0725"), "NY": Decimal("0.04"), "OR": Decimal("0")}
+
+
+def bulk_discount(quantity: int) -> Decimal:
+ for minimum, discount in BULK_TIERS:
+ if quantity > minimum:
+ return discount
+ return Decimal("0")
+
+
+def round_cents(amount: Decimal) -> Decimal:
+ return amount.quantize(Decimal("0.01"))
+
+
+def line_total(unit_price: str, quantity: int) -> Decimal:
+ if quantity < 0:
+ raise ValueError("quantity must be non-negative")
+ subtotal = Decimal(unit_price) * quantity
+ return round_cents(subtotal - subtotal * bulk_discount(quantity))
+
+
+def order_total(lines: list[tuple[str, int]], state: str, coupon: str | None = None) -> Decimal:
+ subtotal = sum((line_total(price, qty) for price, qty in lines), Decimal("0"))
+ if coupon == "SAVE10":
+ subtotal -= Decimal("10")
+ tax = subtotal * TAX_RATES[state]
+ return round_cents(subtotal + tax)
diff --git a/bench/terminal_bench/tasks/fix-failing-tests/environment/app/tests/test_pricing.py b/bench/terminal_bench/tasks/fix-failing-tests/environment/app/tests/test_pricing.py
new file mode 100644
index 000000000..afc2a7982
--- /dev/null
+++ b/bench/terminal_bench/tasks/fix-failing-tests/environment/app/tests/test_pricing.py
@@ -0,0 +1,41 @@
+import unittest
+from decimal import Decimal
+
+from inventory.pricing import bulk_discount, line_total, order_total
+
+
+class BulkDiscountTest(unittest.TestCase):
+ def test_tier_boundaries_are_inclusive(self):
+ self.assertEqual(bulk_discount(9), Decimal("0"))
+ self.assertEqual(bulk_discount(10), Decimal("0.05"))
+ self.assertEqual(bulk_discount(50), Decimal("0.10"))
+ self.assertEqual(bulk_discount(100), Decimal("0.15"))
+
+
+class LineTotalTest(unittest.TestCase):
+ def test_half_cents_round_up(self):
+ # 3 x 0.125 = 0.375, which rounds to 0.38.
+ self.assertEqual(line_total("0.125", 3), Decimal("0.38"))
+
+ def test_bulk_discount_applies(self):
+ self.assertEqual(line_total("2.00", 10), Decimal("19.00"))
+
+ def test_negative_quantity_rejected(self):
+ with self.assertRaises(ValueError):
+ line_total("1.00", -1)
+
+
+class OrderTotalTest(unittest.TestCase):
+ def test_tax_is_added(self):
+ self.assertEqual(order_total([("10.00", 1)], "CA"), Decimal("10.73"))
+
+ def test_coupon_never_makes_total_negative(self):
+ self.assertEqual(order_total([("4.00", 1)], "NY", coupon="SAVE10"), Decimal("0.00"))
+
+ def test_unknown_state_is_a_value_error(self):
+ with self.assertRaises(ValueError):
+ order_total([("1.00", 1)], "ZZ")
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/bench/terminal_bench/tasks/fix-failing-tests/instruction.md b/bench/terminal_bench/tasks/fix-failing-tests/instruction.md
new file mode 100644
index 000000000..814aaa905
--- /dev/null
+++ b/bench/terminal_bench/tasks/fix-failing-tests/instruction.md
@@ -0,0 +1,9 @@
+The `inventory` package in the current directory has a failing test suite.
+
+Run it with `python -m unittest discover -s tests` and fix the code in `inventory/` until every test passes.
+
+Rules:
+- Do not modify anything under `tests/`.
+- Keep the public functions in `inventory/pricing.py` and their signatures.
+- Money values are `Decimal`s rounded to cents with half-up rounding.
+- `order_total` must raise `ValueError` for a state with no tax rate, and a coupon can never make the total negative.
diff --git a/bench/terminal_bench/tasks/fix-failing-tests/solution/solve.py b/bench/terminal_bench/tasks/fix-failing-tests/solution/solve.py
new file mode 100644
index 000000000..c727ff660
--- /dev/null
+++ b/bench/terminal_bench/tasks/fix-failing-tests/solution/solve.py
@@ -0,0 +1,23 @@
+"""Reference fix for fix-failing-tests."""
+
+import os
+from pathlib import Path
+
+path = Path(os.environ.get("TASK_APP_DIR", "/app")) / "inventory" / "pricing.py"
+source = path.read_text()
+for old, new in [
+ ("if quantity > minimum:", "if quantity >= minimum:"),
+ (
+ 'amount.quantize(Decimal("0.01"))',
+ 'amount.quantize(Decimal("0.01"), rounding=ROUND_HALF_UP)',
+ ),
+ (
+ ' subtotal -= Decimal("10")\n',
+ ' subtotal = max(subtotal - Decimal("10"), Decimal("0"))\n'
+ " if state not in TAX_RATES:\n"
+ ' raise ValueError(f"unknown state: {state}")\n',
+ ),
+]:
+ assert old in source, old
+ source = source.replace(old, new)
+path.write_text(source)
diff --git a/bench/terminal_bench/tasks/fix-failing-tests/solution/solve.sh b/bench/terminal_bench/tasks/fix-failing-tests/solution/solve.sh
new file mode 100755
index 000000000..5dbbeddfd
--- /dev/null
+++ b/bench/terminal_bench/tasks/fix-failing-tests/solution/solve.sh
@@ -0,0 +1,2 @@
+#!/bin/bash
+python3 /solution/solve.py
diff --git a/bench/terminal_bench/tasks/fix-failing-tests/task.toml b/bench/terminal_bench/tasks/fix-failing-tests/task.toml
new file mode 100644
index 000000000..6d2b95265
--- /dev/null
+++ b/bench/terminal_bench/tasks/fix-failing-tests/task.toml
@@ -0,0 +1,18 @@
+version = "1.0"
+
+[metadata]
+author_name = "Unity Gateway"
+difficulty = "easy"
+category = "debugging"
+tags = ["python", "testing", "edit-loop"]
+
+[verifier]
+timeout_sec = 120.0
+
+[agent]
+timeout_sec = 600.0
+
+[environment]
+build_timeout_sec = 600.0
+cpus = 1
+memory_mb = 2048
diff --git a/bench/terminal_bench/tasks/fix-failing-tests/tests/test.sh b/bench/terminal_bench/tasks/fix-failing-tests/tests/test.sh
new file mode 100755
index 000000000..48266bb0b
--- /dev/null
+++ b/bench/terminal_bench/tasks/fix-failing-tests/tests/test.sh
@@ -0,0 +1,3 @@
+#!/bin/bash
+mkdir -p /logs/verifier
+if python3 /tests/test_outputs.py; then echo 1 > /logs/verifier/reward.txt; else echo 0 > /logs/verifier/reward.txt; fi
diff --git a/bench/terminal_bench/tasks/fix-failing-tests/tests/test_outputs.py b/bench/terminal_bench/tasks/fix-failing-tests/tests/test_outputs.py
new file mode 100644
index 000000000..4ae6aa386
--- /dev/null
+++ b/bench/terminal_bench/tasks/fix-failing-tests/tests/test_outputs.py
@@ -0,0 +1,39 @@
+"""Verify the inventory fixes. Runs from Harbor (/app) or the native runner (TASK_APP_DIR)."""
+
+import hashlib
+import os
+import subprocess
+import sys
+from decimal import Decimal
+from pathlib import Path
+
+APP = Path(os.environ.get("TASK_APP_DIR", "/app"))
+VISIBLE_TESTS_SHA256 = "d169ead570f87b1aa2eb351c1c226f42bea4b438e5e24720a98df3a5beb96002"
+
+
+def main() -> None:
+ tests = (APP / "tests" / "test_pricing.py").read_bytes().replace(b"\r\n", b"\n")
+ assert hashlib.sha256(tests).hexdigest() == VISIBLE_TESTS_SHA256, "tests/ was modified"
+
+ result = subprocess.run(
+ [sys.executable, "-m", "unittest", "discover", "-s", "tests"],
+ cwd=APP,
+ capture_output=True,
+ text=True,
+ )
+ assert result.returncode == 0, result.stderr
+
+ sys.path.insert(0, str(APP))
+ from inventory.pricing import bulk_discount, line_total, order_total
+
+ assert bulk_discount(49) == Decimal("0.05")
+ assert bulk_discount(99) == Decimal("0.10")
+ assert line_total("0.005", 1) == Decimal("0.01")
+ assert line_total("1.00", 100) == Decimal("85.00")
+ assert order_total([("20.00", 1)], "NY", coupon="SAVE10") == Decimal("10.40")
+ assert order_total([("3.00", 2)], "OR") == Decimal("6.00")
+ print("ok")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/bench/terminal_bench/tasks/image-code/environment/Dockerfile b/bench/terminal_bench/tasks/image-code/environment/Dockerfile
new file mode 100644
index 000000000..18ba6f9bd
--- /dev/null
+++ b/bench/terminal_bench/tasks/image-code/environment/Dockerfile
@@ -0,0 +1,4 @@
+FROM python:3.12-slim
+WORKDIR /app
+COPY setup.py /tmp/setup.py
+RUN python3 /tmp/setup.py /app && rm /tmp/setup.py
diff --git a/bench/terminal_bench/tasks/image-code/environment/setup.py b/bench/terminal_bench/tasks/image-code/environment/setup.py
new file mode 100644
index 000000000..65481d7bb
--- /dev/null
+++ b/bench/terminal_bench/tasks/image-code/environment/setup.py
@@ -0,0 +1,52 @@
+"""Draw a numeric code into code.png with a 5x7 bitmap font (standard library only)."""
+
+import struct
+import sys
+import zlib
+from pathlib import Path
+
+CODE = "47193825"
+FONT = {
+ "0": ["01110", "10001", "10011", "10101", "11001", "10001", "01110"],
+ "1": ["00100", "01100", "00100", "00100", "00100", "00100", "01110"],
+ "2": ["01110", "10001", "00001", "00010", "00100", "01000", "11111"],
+ "3": ["11110", "00001", "00001", "01110", "00001", "00001", "11110"],
+ "4": ["00010", "00110", "01010", "10010", "11111", "00010", "00010"],
+ "5": ["11111", "10000", "11110", "00001", "00001", "10001", "01110"],
+ "6": ["00110", "01000", "10000", "11110", "10001", "10001", "01110"],
+ "7": ["11111", "00001", "00010", "00100", "01000", "01000", "01000"],
+ "8": ["01110", "10001", "10001", "01110", "10001", "10001", "01110"],
+ "9": ["01110", "10001", "10001", "01111", "00001", "00010", "01100"],
+}
+SCALE, GAP, MARGIN = 12, 2, 3
+
+
+def chunk(kind: bytes, data: bytes) -> bytes:
+ return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data))
+
+
+def main(app: Path) -> None:
+ cols = MARGIN * 2 + len(CODE) * (5 + GAP) - GAP
+ rows = MARGIN * 2 + 7
+ grid = [[255] * cols for _ in range(rows)]
+ for i, digit in enumerate(CODE):
+ for y, row in enumerate(FONT[digit]):
+ for x, bit in enumerate(row):
+ if bit == "1":
+ grid[MARGIN + y][MARGIN + i * (5 + GAP) + x] = 0
+ raw = b"".join(
+ b"\x00" + bytes(v for v in row for _ in range(SCALE)) for row in grid for _ in range(SCALE)
+ )
+ header = struct.pack(">IIBBBBB", cols * SCALE, rows * SCALE, 8, 0, 0, 0, 0)
+ png = (
+ b"\x89PNG\r\n\x1a\n"
+ + chunk(b"IHDR", header)
+ + chunk(b"IDAT", zlib.compress(raw))
+ + chunk(b"IEND", b"")
+ )
+ app.mkdir(parents=True, exist_ok=True)
+ (app / "code.png").write_bytes(png)
+
+
+if __name__ == "__main__":
+ main(Path(sys.argv[1]))
diff --git a/bench/terminal_bench/tasks/image-code/instruction.md b/bench/terminal_bench/tasks/image-code/instruction.md
new file mode 100644
index 000000000..ecc297105
--- /dev/null
+++ b/bench/terminal_bench/tasks/image-code/instruction.md
@@ -0,0 +1 @@
+`code.png` shows an 8-digit code. Look at the image and write the code to `code.txt`.
diff --git a/bench/terminal_bench/tasks/image-code/solution/solve.py b/bench/terminal_bench/tasks/image-code/solution/solve.py
new file mode 100644
index 000000000..5381a4a5b
--- /dev/null
+++ b/bench/terminal_bench/tasks/image-code/solution/solve.py
@@ -0,0 +1,6 @@
+"""Reference solution for image-code."""
+
+import os
+from pathlib import Path
+
+Path(os.environ.get("TASK_APP_DIR", "/app"), "code.txt").write_text("47193825\n")
diff --git a/bench/terminal_bench/tasks/image-code/solution/solve.sh b/bench/terminal_bench/tasks/image-code/solution/solve.sh
new file mode 100755
index 000000000..5dbbeddfd
--- /dev/null
+++ b/bench/terminal_bench/tasks/image-code/solution/solve.sh
@@ -0,0 +1,2 @@
+#!/bin/bash
+python3 /solution/solve.py
diff --git a/bench/terminal_bench/tasks/image-code/task.toml b/bench/terminal_bench/tasks/image-code/task.toml
new file mode 100644
index 000000000..9ed323ad6
--- /dev/null
+++ b/bench/terminal_bench/tasks/image-code/task.toml
@@ -0,0 +1,18 @@
+version = "1.0"
+
+[metadata]
+author_name = "Unity Gateway"
+difficulty = "easy"
+category = "agent-harness"
+agents = ["claude", "codex"]
+
+[verifier]
+timeout_sec = 120.0
+
+[agent]
+timeout_sec = 600.0
+
+[environment]
+build_timeout_sec = 600.0
+cpus = 1
+memory_mb = 2048
diff --git a/bench/terminal_bench/tasks/image-code/tests/test.sh b/bench/terminal_bench/tasks/image-code/tests/test.sh
new file mode 100755
index 000000000..48266bb0b
--- /dev/null
+++ b/bench/terminal_bench/tasks/image-code/tests/test.sh
@@ -0,0 +1,3 @@
+#!/bin/bash
+mkdir -p /logs/verifier
+if python3 /tests/test_outputs.py; then echo 1 > /logs/verifier/reward.txt; else echo 0 > /logs/verifier/reward.txt; fi
diff --git a/bench/terminal_bench/tasks/image-code/tests/test_outputs.py b/bench/terminal_bench/tasks/image-code/tests/test_outputs.py
new file mode 100644
index 000000000..81b22af69
--- /dev/null
+++ b/bench/terminal_bench/tasks/image-code/tests/test_outputs.py
@@ -0,0 +1,15 @@
+"""Verify the code read from the image."""
+
+import os
+from pathlib import Path
+
+APP = Path(os.environ.get("TASK_APP_DIR", "/app"))
+
+
+def main() -> None:
+ assert (APP / "code.txt").read_text().strip() == "47193825"
+ print("ok")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/bench/terminal_bench/tasks/log-triage/environment/Dockerfile b/bench/terminal_bench/tasks/log-triage/environment/Dockerfile
new file mode 100644
index 000000000..18ba6f9bd
--- /dev/null
+++ b/bench/terminal_bench/tasks/log-triage/environment/Dockerfile
@@ -0,0 +1,4 @@
+FROM python:3.12-slim
+WORKDIR /app
+COPY setup.py /tmp/setup.py
+RUN python3 /tmp/setup.py /app && rm /tmp/setup.py
diff --git a/bench/terminal_bench/tasks/log-triage/environment/setup.py b/bench/terminal_bench/tasks/log-triage/environment/setup.py
new file mode 100644
index 000000000..55140114d
--- /dev/null
+++ b/bench/terminal_bench/tasks/log-triage/environment/setup.py
@@ -0,0 +1,39 @@
+"""Generate deterministic service logs under /logs."""
+
+import random
+import sys
+from datetime import datetime, timedelta
+from pathlib import Path
+
+SERVICES = ["auth", "billing", "gateway", "search", "storage"]
+CODES = ["E101", "E202", "E303", "E404", "E505", "E606"]
+MESSAGES = {
+ "INFO": ["request served", "cache warmed", "heartbeat ok", "config reloaded"],
+ "WARN": ["slow upstream", "retrying request", "queue depth high"],
+ "ERROR": ["upstream failed", "timeout", "bad payload", "disk full"],
+}
+
+
+def main(app: Path) -> None:
+ rng = random.Random(4242)
+ start = datetime(2026, 9, 1)
+ for service in SERVICES:
+ # Each service favors a different error code so the answer isn't uniform.
+ weights = [rng.randint(1, 4) for _ in CODES]
+ weights[rng.randrange(len(CODES))] += 8
+ for day in range(4):
+ date = start + timedelta(days=day)
+ lines = []
+ for i in range(rng.randint(250, 400)):
+ ts = date + timedelta(seconds=i * 37 + rng.randint(0, 30))
+ level = rng.choices(["INFO", "WARN", "ERROR"], [80, 12, 8])[0]
+ message = rng.choice(MESSAGES[level])
+ code = f" code={rng.choices(CODES, weights)[0]}" if level == "ERROR" else ""
+ lines.append(f"{ts.isoformat()} {level:<5} [{service}] {message}{code}")
+ out = app / "logs" / service / f"{date:%Y-%m-%d}.log"
+ out.parent.mkdir(parents=True, exist_ok=True)
+ out.write_text("\n".join(lines) + "\n", newline="\n")
+
+
+if __name__ == "__main__":
+ main(Path(sys.argv[1]))
diff --git a/bench/terminal_bench/tasks/log-triage/instruction.md b/bench/terminal_bench/tasks/log-triage/instruction.md
new file mode 100644
index 000000000..2f9e2593f
--- /dev/null
+++ b/bench/terminal_bench/tasks/log-triage/instruction.md
@@ -0,0 +1,22 @@
+`logs/` holds four days of logs for five services, one directory per service.
+Each line looks like:
+
+```
+2026-09-01T00:00:12 ERROR [billing] upstream failed code=E303
+```
+
+The level is `INFO`, `WARN`, or `ERROR`. Only `ERROR` lines carry a `code=`.
+
+Write `report.json` in the current directory with one entry per service:
+
+```json
+{
+ "auth": {"errors": 0, "warnings": 0, "top_error_code": "E101"},
+ ...
+}
+```
+
+- `errors` and `warnings` count `ERROR` and `WARN` lines across all of that service's files.
+- `top_error_code` is the code seen most often in that service's `ERROR` lines. Break ties by the alphabetically smallest code.
+
+The service directories are independent of each other, so they can be processed in parallel.
diff --git a/bench/terminal_bench/tasks/log-triage/solution/solve.py b/bench/terminal_bench/tasks/log-triage/solution/solve.py
new file mode 100644
index 000000000..ad49e2135
--- /dev/null
+++ b/bench/terminal_bench/tasks/log-triage/solution/solve.py
@@ -0,0 +1,19 @@
+"""Reference solution for log-triage."""
+
+import json
+import os
+import re
+from collections import Counter
+from pathlib import Path
+
+app = Path(os.environ.get("TASK_APP_DIR", "/app"))
+report = {}
+for service_dir in sorted((app / "logs").iterdir()):
+ text = "".join(log.read_text() for log in service_dir.glob("*.log"))
+ codes = Counter(re.findall(r" ERROR .* code=(E\d+)$", text, re.MULTILINE))
+ report[service_dir.name] = {
+ "errors": len(re.findall(r"^\S+ ERROR ", text, re.MULTILINE)),
+ "warnings": len(re.findall(r"^\S+ WARN ", text, re.MULTILINE)),
+ "top_error_code": min(codes, key=lambda code: (-codes[code], code)),
+ }
+(app / "report.json").write_text(json.dumps(report, indent=2))
diff --git a/bench/terminal_bench/tasks/log-triage/solution/solve.sh b/bench/terminal_bench/tasks/log-triage/solution/solve.sh
new file mode 100755
index 000000000..5dbbeddfd
--- /dev/null
+++ b/bench/terminal_bench/tasks/log-triage/solution/solve.sh
@@ -0,0 +1,2 @@
+#!/bin/bash
+python3 /solution/solve.py
diff --git a/bench/terminal_bench/tasks/log-triage/task.toml b/bench/terminal_bench/tasks/log-triage/task.toml
new file mode 100644
index 000000000..bf1050aa1
--- /dev/null
+++ b/bench/terminal_bench/tasks/log-triage/task.toml
@@ -0,0 +1,18 @@
+version = "1.0"
+
+[metadata]
+author_name = "Unity Gateway"
+difficulty = "easy"
+category = "data-processing"
+tags = ["logs", "fan-out", "subagents"]
+
+[verifier]
+timeout_sec = 120.0
+
+[agent]
+timeout_sec = 600.0
+
+[environment]
+build_timeout_sec = 600.0
+cpus = 1
+memory_mb = 2048
diff --git a/bench/terminal_bench/tasks/log-triage/tests/test.sh b/bench/terminal_bench/tasks/log-triage/tests/test.sh
new file mode 100755
index 000000000..48266bb0b
--- /dev/null
+++ b/bench/terminal_bench/tasks/log-triage/tests/test.sh
@@ -0,0 +1,3 @@
+#!/bin/bash
+mkdir -p /logs/verifier
+if python3 /tests/test_outputs.py; then echo 1 > /logs/verifier/reward.txt; else echo 0 > /logs/verifier/reward.txt; fi
diff --git a/bench/terminal_bench/tasks/log-triage/tests/test_outputs.py b/bench/terminal_bench/tasks/log-triage/tests/test_outputs.py
new file mode 100644
index 000000000..aaebaff63
--- /dev/null
+++ b/bench/terminal_bench/tasks/log-triage/tests/test_outputs.py
@@ -0,0 +1,42 @@
+"""Verify report.json against counts recomputed from the logs."""
+
+import json
+import os
+import re
+from collections import Counter
+from pathlib import Path
+
+APP = Path(os.environ.get("TASK_APP_DIR", "/app"))
+LINE = re.compile(r"^\S+ (INFO|WARN|ERROR)\s+\[(\w+)\] .*?(?: code=(E\d+))?$")
+
+
+def expected() -> dict:
+ report = {}
+ for service_dir in sorted((APP / "logs").iterdir()):
+ levels: Counter[str] = Counter()
+ codes: Counter[str] = Counter()
+ for log in service_dir.glob("*.log"):
+ for line in log.read_text().splitlines():
+ match = LINE.match(line)
+ assert match, line
+ levels[match[1]] += 1
+ if match[3]:
+ codes[match[3]] += 1
+ top = max(sorted(codes), key=lambda code: codes[code])
+ report[service_dir.name] = {
+ "errors": levels["ERROR"],
+ "warnings": levels["WARN"],
+ "top_error_code": top,
+ }
+ return report
+
+
+def main() -> None:
+ actual = json.loads((APP / "report.json").read_text())
+ want = expected()
+ assert actual == want, {"expected": want, "actual": actual}
+ print("ok")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/bench/terminal_bench/tasks/mcp-server/environment/Dockerfile b/bench/terminal_bench/tasks/mcp-server/environment/Dockerfile
new file mode 100644
index 000000000..ed07be056
--- /dev/null
+++ b/bench/terminal_bench/tasks/mcp-server/environment/Dockerfile
@@ -0,0 +1,3 @@
+FROM python:3.12-slim
+WORKDIR /app
+COPY app/ /app/
diff --git a/bench/terminal_bench/tasks/mcp-server/environment/app/.claude/settings.json b/bench/terminal_bench/tasks/mcp-server/environment/app/.claude/settings.json
new file mode 100644
index 000000000..2fc9d2f15
--- /dev/null
+++ b/bench/terminal_bench/tasks/mcp-server/environment/app/.claude/settings.json
@@ -0,0 +1 @@
+{"enableAllProjectMcpServers": true}
diff --git a/bench/terminal_bench/tasks/mcp-server/environment/app/.mcp.json b/bench/terminal_bench/tasks/mcp-server/environment/app/.mcp.json
new file mode 100644
index 000000000..e5413902e
--- /dev/null
+++ b/bench/terminal_bench/tasks/mcp-server/environment/app/.mcp.json
@@ -0,0 +1,5 @@
+{
+ "mcpServers": {
+ "inventory": {"command": "python", "args": ["mcp_server.py"]}
+ }
+}
diff --git a/bench/terminal_bench/tasks/mcp-server/environment/app/mcp_server.py b/bench/terminal_bench/tasks/mcp-server/environment/app/mcp_server.py
new file mode 100644
index 000000000..74704dc14
--- /dev/null
+++ b/bench/terminal_bench/tasks/mcp-server/environment/app/mcp_server.py
@@ -0,0 +1,51 @@
+"""Minimal stdio MCP server with one tool. Stock levels are random per server start."""
+
+import json
+import random
+import sys
+from pathlib import Path
+
+LOG = Path(__file__).resolve().parent / ".mcp_log" / "calls.jsonl"
+STOCK: dict[str, int] = {}
+TOOL = {
+ "name": "get_stock",
+ "description": "Return the current number of units in stock for a SKU.",
+ "inputSchema": {
+ "type": "object",
+ "properties": {"sku": {"type": "string"}},
+ "required": ["sku"],
+ },
+}
+
+
+def handle(method: str, params: dict) -> dict:
+ if method == "initialize":
+ return {
+ "protocolVersion": params.get("protocolVersion", "2025-06-18"),
+ "capabilities": {"tools": {}},
+ "serverInfo": {"name": "inventory", "version": "1.0"},
+ }
+ if method == "tools/list":
+ return {"tools": [TOOL]}
+ if method == "tools/call":
+ sku = params["arguments"]["sku"].strip().upper()
+ units = STOCK.setdefault(sku, random.randint(100, 999))
+ LOG.parent.mkdir(exist_ok=True)
+ with LOG.open("a", encoding="utf-8") as f:
+ f.write(json.dumps({"sku": sku, "units": units}) + "\n")
+ return {"content": [{"type": "text", "text": f"{sku}: {units} units in stock"}]}
+ if method == "ping":
+ return {}
+ raise KeyError(method)
+
+
+for line in sys.stdin:
+ message = json.loads(line)
+ if "id" not in message:
+ continue
+ try:
+ response = {"result": handle(message["method"], message.get("params") or {})}
+ except KeyError:
+ response = {"error": {"code": -32601, "message": f"unknown method {message['method']}"}}
+ sys.stdout.write(json.dumps({"jsonrpc": "2.0", "id": message["id"], **response}) + "\n")
+ sys.stdout.flush()
diff --git a/bench/terminal_bench/tasks/mcp-server/instruction.md b/bench/terminal_bench/tasks/mcp-server/instruction.md
new file mode 100644
index 000000000..9b5642bf9
--- /dev/null
+++ b/bench/terminal_bench/tasks/mcp-server/instruction.md
@@ -0,0 +1,5 @@
+This project has an `inventory` MCP server. Use its `get_stock` tool to look up the stock for SKUs `A-100`, `B-200`, and `C-300`.
+
+Write `stock.json` in the current directory mapping each SKU to its unit count as an integer, for example `{"A-100": 0, ...}`.
+
+Stock levels are only available from the MCP tool.
diff --git a/bench/terminal_bench/tasks/mcp-server/solution/solve.py b/bench/terminal_bench/tasks/mcp-server/solution/solve.py
new file mode 100644
index 000000000..dca647d48
--- /dev/null
+++ b/bench/terminal_bench/tasks/mcp-server/solution/solve.py
@@ -0,0 +1,35 @@
+"""Reference run: speak MCP to the server over stdio."""
+
+import json
+import os
+import re
+import subprocess
+import sys
+from pathlib import Path
+
+app = Path(os.environ.get("TASK_APP_DIR", "/app"))
+requests = [{"jsonrpc": "2.0", "id": 0, "method": "initialize", "params": {}}]
+requests += [
+ {
+ "jsonrpc": "2.0",
+ "id": i,
+ "method": "tools/call",
+ "params": {"name": "get_stock", "arguments": {"sku": sku}},
+ }
+ for i, sku in enumerate(["A-100", "B-200", "C-300"], start=1)
+]
+out = subprocess.run(
+ [sys.executable, "mcp_server.py"],
+ cwd=app,
+ input="".join(json.dumps(r) + "\n" for r in requests),
+ capture_output=True,
+ text=True,
+ check=True,
+).stdout
+stock = {}
+for line in out.splitlines()[1:]:
+ sku, units = re.match(
+ r"(\S+): (\d+)", json.loads(line)["result"]["content"][0]["text"]
+ ).groups()
+ stock[sku] = int(units)
+(app / "stock.json").write_text(json.dumps(stock))
diff --git a/bench/terminal_bench/tasks/mcp-server/solution/solve.sh b/bench/terminal_bench/tasks/mcp-server/solution/solve.sh
new file mode 100755
index 000000000..5dbbeddfd
--- /dev/null
+++ b/bench/terminal_bench/tasks/mcp-server/solution/solve.sh
@@ -0,0 +1,2 @@
+#!/bin/bash
+python3 /solution/solve.py
diff --git a/bench/terminal_bench/tasks/mcp-server/task.toml b/bench/terminal_bench/tasks/mcp-server/task.toml
new file mode 100644
index 000000000..a699c38db
--- /dev/null
+++ b/bench/terminal_bench/tasks/mcp-server/task.toml
@@ -0,0 +1,18 @@
+version = "1.0"
+
+[metadata]
+author_name = "Unity Gateway"
+difficulty = "easy"
+category = "agent-harness"
+agents = ["claude"]
+
+[verifier]
+timeout_sec = 120.0
+
+[agent]
+timeout_sec = 600.0
+
+[environment]
+build_timeout_sec = 600.0
+cpus = 1
+memory_mb = 2048
diff --git a/bench/terminal_bench/tasks/mcp-server/tests/test.sh b/bench/terminal_bench/tasks/mcp-server/tests/test.sh
new file mode 100755
index 000000000..48266bb0b
--- /dev/null
+++ b/bench/terminal_bench/tasks/mcp-server/tests/test.sh
@@ -0,0 +1,3 @@
+#!/bin/bash
+mkdir -p /logs/verifier
+if python3 /tests/test_outputs.py; then echo 1 > /logs/verifier/reward.txt; else echo 0 > /logs/verifier/reward.txt; fi
diff --git a/bench/terminal_bench/tasks/mcp-server/tests/test_outputs.py b/bench/terminal_bench/tasks/mcp-server/tests/test_outputs.py
new file mode 100644
index 000000000..cc3ff235f
--- /dev/null
+++ b/bench/terminal_bench/tasks/mcp-server/tests/test_outputs.py
@@ -0,0 +1,30 @@
+"""Verify stock.json matches what the MCP server actually returned."""
+
+import json
+import os
+from pathlib import Path
+
+APP = Path(os.environ.get("TASK_APP_DIR", "/app"))
+SKUS = ["A-100", "B-200", "C-300"]
+
+
+def main() -> None:
+ log = APP / ".mcp_log" / "calls.jsonl"
+ assert log.exists(), "the MCP tool was never called"
+ served = {}
+ for line in log.read_text().splitlines():
+ call = json.loads(line)
+ served[call["sku"]] = call["units"]
+ actual = json.loads((APP / "stock.json").read_text())
+ for sku in SKUS:
+ assert sku in served, f"get_stock was never called for {sku}"
+ assert actual.get(sku) == served[sku], {
+ "sku": sku,
+ "served": served[sku],
+ "got": actual.get(sku),
+ }
+ print("ok")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/bench/terminal_bench/tasks/pdf-extract/environment/Dockerfile b/bench/terminal_bench/tasks/pdf-extract/environment/Dockerfile
new file mode 100644
index 000000000..18ba6f9bd
--- /dev/null
+++ b/bench/terminal_bench/tasks/pdf-extract/environment/Dockerfile
@@ -0,0 +1,4 @@
+FROM python:3.12-slim
+WORKDIR /app
+COPY setup.py /tmp/setup.py
+RUN python3 /tmp/setup.py /app && rm /tmp/setup.py
diff --git a/bench/terminal_bench/tasks/pdf-extract/environment/setup.py b/bench/terminal_bench/tasks/pdf-extract/environment/setup.py
new file mode 100644
index 000000000..86476b4ef
--- /dev/null
+++ b/bench/terminal_bench/tasks/pdf-extract/environment/setup.py
@@ -0,0 +1,45 @@
+"""Write invoice.pdf with a Flate-compressed text stream (standard library only)."""
+
+import sys
+import zlib
+from pathlib import Path
+
+LINES = [
+ "Northwind Supplies GmbH",
+ "Invoice INV-2026-0419",
+ "Consulting hours: 38",
+ "Subtotal: 6,169.91 EUR",
+ "VAT (19%): 1,172.28 EUR",
+ "Total due: 7,342.19 EUR",
+]
+
+
+def main(app: Path) -> None:
+ text = "BT /F1 14 Tf 72 720 Td 20 TL " + " ".join(f"({line}) Tj T*" for line in LINES) + " ET"
+ stream = zlib.compress(text.encode())
+ objects = [
+ b"<< /Type /Catalog /Pages 2 0 R >>",
+ b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>",
+ b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] "
+ b"/Resources << /Font << /F1 4 0 R >> >> /Contents 5 0 R >>",
+ b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>",
+ b"<< /Length %d /Filter /FlateDecode >>\nstream\n" % len(stream) + stream + b"\nendstream",
+ ]
+ pdf = bytearray(b"%PDF-1.4\n")
+ offsets = []
+ for number, body in enumerate(objects, start=1):
+ offsets.append(len(pdf))
+ pdf += b"%d 0 obj\n" % number + body + b"\nendobj\n"
+ xref = len(pdf)
+ pdf += b"xref\n0 %d\n0000000000 65535 f \n" % (len(objects) + 1)
+ pdf += b"".join(b"%010d 00000 n \n" % offset for offset in offsets)
+ pdf += b"trailer\n<< /Size %d /Root 1 0 R >>\nstartxref\n%d\n%%%%EOF\n" % (
+ len(objects) + 1,
+ xref,
+ )
+ app.mkdir(parents=True, exist_ok=True)
+ (app / "invoice.pdf").write_bytes(bytes(pdf))
+
+
+if __name__ == "__main__":
+ main(Path(sys.argv[1]))
diff --git a/bench/terminal_bench/tasks/pdf-extract/instruction.md b/bench/terminal_bench/tasks/pdf-extract/instruction.md
new file mode 100644
index 000000000..e931c3aea
--- /dev/null
+++ b/bench/terminal_bench/tasks/pdf-extract/instruction.md
@@ -0,0 +1 @@
+Read `invoice.pdf` and write the invoice's total due to `total.txt` exactly as printed, for example `1,234.56 EUR`.
diff --git a/bench/terminal_bench/tasks/pdf-extract/solution/solve.py b/bench/terminal_bench/tasks/pdf-extract/solution/solve.py
new file mode 100644
index 000000000..b636f8c4b
--- /dev/null
+++ b/bench/terminal_bench/tasks/pdf-extract/solution/solve.py
@@ -0,0 +1,6 @@
+"""Reference solution for pdf-extract."""
+
+import os
+from pathlib import Path
+
+Path(os.environ.get("TASK_APP_DIR", "/app"), "total.txt").write_text("7,342.19 EUR\n")
diff --git a/bench/terminal_bench/tasks/pdf-extract/solution/solve.sh b/bench/terminal_bench/tasks/pdf-extract/solution/solve.sh
new file mode 100755
index 000000000..5dbbeddfd
--- /dev/null
+++ b/bench/terminal_bench/tasks/pdf-extract/solution/solve.sh
@@ -0,0 +1,2 @@
+#!/bin/bash
+python3 /solution/solve.py
diff --git a/bench/terminal_bench/tasks/pdf-extract/task.toml b/bench/terminal_bench/tasks/pdf-extract/task.toml
new file mode 100644
index 000000000..9ed323ad6
--- /dev/null
+++ b/bench/terminal_bench/tasks/pdf-extract/task.toml
@@ -0,0 +1,18 @@
+version = "1.0"
+
+[metadata]
+author_name = "Unity Gateway"
+difficulty = "easy"
+category = "agent-harness"
+agents = ["claude", "codex"]
+
+[verifier]
+timeout_sec = 120.0
+
+[agent]
+timeout_sec = 600.0
+
+[environment]
+build_timeout_sec = 600.0
+cpus = 1
+memory_mb = 2048
diff --git a/bench/terminal_bench/tasks/pdf-extract/tests/test.sh b/bench/terminal_bench/tasks/pdf-extract/tests/test.sh
new file mode 100755
index 000000000..48266bb0b
--- /dev/null
+++ b/bench/terminal_bench/tasks/pdf-extract/tests/test.sh
@@ -0,0 +1,3 @@
+#!/bin/bash
+mkdir -p /logs/verifier
+if python3 /tests/test_outputs.py; then echo 1 > /logs/verifier/reward.txt; else echo 0 > /logs/verifier/reward.txt; fi
diff --git a/bench/terminal_bench/tasks/pdf-extract/tests/test_outputs.py b/bench/terminal_bench/tasks/pdf-extract/tests/test_outputs.py
new file mode 100644
index 000000000..8306a656d
--- /dev/null
+++ b/bench/terminal_bench/tasks/pdf-extract/tests/test_outputs.py
@@ -0,0 +1,16 @@
+"""Verify the total read from the PDF."""
+
+import os
+from pathlib import Path
+
+APP = Path(os.environ.get("TASK_APP_DIR", "/app"))
+
+
+def main() -> None:
+ total = (APP / "total.txt").read_text().strip()
+ assert total.removeprefix("Total due:").strip() == "7,342.19 EUR", total
+ print("ok")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/bench/terminal_bench/tasks/project-instructions/environment/Dockerfile b/bench/terminal_bench/tasks/project-instructions/environment/Dockerfile
new file mode 100644
index 000000000..ed07be056
--- /dev/null
+++ b/bench/terminal_bench/tasks/project-instructions/environment/Dockerfile
@@ -0,0 +1,3 @@
+FROM python:3.12-slim
+WORKDIR /app
+COPY app/ /app/
diff --git a/bench/terminal_bench/tasks/project-instructions/environment/app/AGENTS.md b/bench/terminal_bench/tasks/project-instructions/environment/app/AGENTS.md
new file mode 100644
index 000000000..22d1c0db1
--- /dev/null
+++ b/bench/terminal_bench/tasks/project-instructions/environment/app/AGENTS.md
@@ -0,0 +1,4 @@
+# Project conventions
+
+- Put every output file under `out/`. Never write output files to the project root.
+- Every Python file you create must start with the line `# SPDX-License-Identifier: Apache-2.0`.
diff --git a/bench/terminal_bench/tasks/project-instructions/environment/app/CLAUDE.md b/bench/terminal_bench/tasks/project-instructions/environment/app/CLAUDE.md
new file mode 100644
index 000000000..22d1c0db1
--- /dev/null
+++ b/bench/terminal_bench/tasks/project-instructions/environment/app/CLAUDE.md
@@ -0,0 +1,4 @@
+# Project conventions
+
+- Put every output file under `out/`. Never write output files to the project root.
+- Every Python file you create must start with the line `# SPDX-License-Identifier: Apache-2.0`.
diff --git a/bench/terminal_bench/tasks/project-instructions/environment/app/data.txt b/bench/terminal_bench/tasks/project-instructions/environment/app/data.txt
new file mode 100644
index 000000000..f6b86b2b2
--- /dev/null
+++ b/bench/terminal_bench/tasks/project-instructions/environment/app/data.txt
@@ -0,0 +1,40 @@
+model cache gateway token tool token route shell gateway tool
+gateway token cache cache token agent token tool
+gateway shell token agent shell gateway shell shell cache gateway agent
+tool model harness cache model
+token shell harness tool model token shell shell agent route token tool token
+gateway shell agent prompt tool cache route prompt shell prompt route harness agent model
+token shell harness tool prompt route prompt harness
+token token tool cache model route model prompt cache gateway token tool shell route
+route shell prompt shell prompt token token harness prompt token
+harness shell prompt harness cache
+gateway prompt route model shell token prompt gateway agent harness
+agent cache cache prompt token model prompt
+tool harness model cache tool harness cache route cache agent model
+model model agent agent gateway prompt
+model harness harness gateway model cache tool route shell shell route model tool shell
+prompt tool cache cache cache
+token prompt cache gateway agent token agent prompt model token route
+gateway token gateway shell model tool token route shell gateway token agent shell cache
+harness route shell route prompt token token
+prompt prompt prompt harness token model token route harness prompt model tool
+agent tool route model tool
+tool harness token harness tool
+model route agent tool tool tool route agent shell agent
+cache agent agent tool prompt route gateway gateway
+prompt harness agent shell route prompt route route token
+token agent prompt agent route agent prompt shell
+gateway prompt route token token cache agent prompt model cache route token cache prompt
+token model model model gateway model shell prompt model shell shell
+route model tool tool model gateway gateway token tool model cache agent
+gateway harness agent harness tool agent shell route
+tool cache model gateway route prompt shell tool cache
+model tool model tool tool gateway prompt model shell gateway model model model
+shell token tool gateway route tool tool tool prompt token tool gateway
+agent harness gateway token tool prompt tool gateway
+prompt route shell tool shell tool
+harness prompt tool tool prompt tool agent tool
+tool agent prompt model cache token cache prompt route
+agent cache token agent harness token
+route model harness model prompt agent token
+prompt model agent model cache tool cache route cache agent route
diff --git a/bench/terminal_bench/tasks/project-instructions/instruction.md b/bench/terminal_bench/tasks/project-instructions/instruction.md
new file mode 100644
index 000000000..27e157f0f
--- /dev/null
+++ b/bench/terminal_bench/tasks/project-instructions/instruction.md
@@ -0,0 +1 @@
+Write a Python script that counts the words in `data.txt`, run it, and save the count to a file named `count.txt`.
diff --git a/bench/terminal_bench/tasks/project-instructions/solution/solve.py b/bench/terminal_bench/tasks/project-instructions/solution/solve.py
new file mode 100644
index 000000000..1f9c1deab
--- /dev/null
+++ b/bench/terminal_bench/tasks/project-instructions/solution/solve.py
@@ -0,0 +1,16 @@
+"""Reference solution for project-instructions."""
+
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+app = Path(os.environ.get("TASK_APP_DIR", "/app"))
+(app / "out").mkdir(exist_ok=True)
+script = app / "out" / "count_words.py"
+script.write_text(
+ "# SPDX-License-Identifier: Apache-2.0\n"
+ "from pathlib import Path\n"
+ "Path('out/count.txt').write_text(str(len(Path('data.txt').read_text().split())))\n"
+)
+subprocess.run([sys.executable, str(script)], cwd=app, check=True)
diff --git a/bench/terminal_bench/tasks/project-instructions/solution/solve.sh b/bench/terminal_bench/tasks/project-instructions/solution/solve.sh
new file mode 100755
index 000000000..5dbbeddfd
--- /dev/null
+++ b/bench/terminal_bench/tasks/project-instructions/solution/solve.sh
@@ -0,0 +1,2 @@
+#!/bin/bash
+python3 /solution/solve.py
diff --git a/bench/terminal_bench/tasks/project-instructions/task.toml b/bench/terminal_bench/tasks/project-instructions/task.toml
new file mode 100644
index 000000000..9ed323ad6
--- /dev/null
+++ b/bench/terminal_bench/tasks/project-instructions/task.toml
@@ -0,0 +1,18 @@
+version = "1.0"
+
+[metadata]
+author_name = "Unity Gateway"
+difficulty = "easy"
+category = "agent-harness"
+agents = ["claude", "codex"]
+
+[verifier]
+timeout_sec = 120.0
+
+[agent]
+timeout_sec = 600.0
+
+[environment]
+build_timeout_sec = 600.0
+cpus = 1
+memory_mb = 2048
diff --git a/bench/terminal_bench/tasks/project-instructions/tests/test.sh b/bench/terminal_bench/tasks/project-instructions/tests/test.sh
new file mode 100755
index 000000000..48266bb0b
--- /dev/null
+++ b/bench/terminal_bench/tasks/project-instructions/tests/test.sh
@@ -0,0 +1,3 @@
+#!/bin/bash
+mkdir -p /logs/verifier
+if python3 /tests/test_outputs.py; then echo 1 > /logs/verifier/reward.txt; else echo 0 > /logs/verifier/reward.txt; fi
diff --git a/bench/terminal_bench/tasks/project-instructions/tests/test_outputs.py b/bench/terminal_bench/tasks/project-instructions/tests/test_outputs.py
new file mode 100644
index 000000000..3eb0f9c76
--- /dev/null
+++ b/bench/terminal_bench/tasks/project-instructions/tests/test_outputs.py
@@ -0,0 +1,24 @@
+"""Verify the agent followed the project's instruction file."""
+
+import os
+from pathlib import Path
+
+APP = Path(os.environ.get("TASK_APP_DIR", "/app"))
+HEADER = "# SPDX-License-Identifier: Apache-2.0"
+
+
+def main() -> None:
+ expected = len((APP / "data.txt").read_text().split())
+ assert not (APP / "count.txt").exists(), "count.txt was written to the project root"
+ assert (APP / "out" / "count.txt").read_text().strip() == str(expected)
+ scripts = [p for p in APP.rglob("*.py") if ".claude" not in p.parts]
+ assert scripts, "no Python script was written"
+ for script in scripts:
+ assert script.read_text().splitlines()[0].strip() == HEADER, (
+ f"{script.name} lacks the SPDX header"
+ )
+ print("ok")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/bench/terminal_bench/tasks/project-skill/environment/Dockerfile b/bench/terminal_bench/tasks/project-skill/environment/Dockerfile
new file mode 100644
index 000000000..ed07be056
--- /dev/null
+++ b/bench/terminal_bench/tasks/project-skill/environment/Dockerfile
@@ -0,0 +1,3 @@
+FROM python:3.12-slim
+WORKDIR /app
+COPY app/ /app/
diff --git a/bench/terminal_bench/tasks/project-skill/environment/app/.claude/skills/release-notes/SKILL.md b/bench/terminal_bench/tasks/project-skill/environment/app/.claude/skills/release-notes/SKILL.md
new file mode 100644
index 000000000..f843dca9e
--- /dev/null
+++ b/bench/terminal_bench/tasks/project-skill/environment/app/.claude/skills/release-notes/SKILL.md
@@ -0,0 +1,10 @@
+---
+name: release-notes
+description: Build release notes for this project. Use whenever someone asks for release notes or a changelog.
+---
+
+# Release notes
+
+1. Run `python .claude/skills/release-notes/build.py `. It reads `changes.txt` and prints a draft.
+2. Save the draft to `dist/release-notes-.md`.
+3. Add this as the last line of the file, exactly: `Signed-off-by: release-bot`
diff --git a/bench/terminal_bench/tasks/project-skill/environment/app/.claude/skills/release-notes/build.py b/bench/terminal_bench/tasks/project-skill/environment/app/.claude/skills/release-notes/build.py
new file mode 100644
index 000000000..6188516e1
--- /dev/null
+++ b/bench/terminal_bench/tasks/project-skill/environment/app/.claude/skills/release-notes/build.py
@@ -0,0 +1,20 @@
+"""Draft release notes from changes.txt, grouped by change type."""
+
+import json
+import sys
+from pathlib import Path
+
+root = Path(__file__).resolve().parents[3]
+version = sys.argv[1]
+groups: dict[str, list[str]] = {}
+for line in (root / "changes.txt").read_text().splitlines():
+ kind, _, text = line.partition(": ")
+ groups.setdefault(kind.strip(), []).append(text.strip())
+print(f"# Release {version}\n")
+for kind in sorted(groups):
+ print(f"## {kind.title()}\n")
+ print("\n".join(f"- {item}" for item in groups[kind]) + "\n")
+log = root / ".skill_log" / "runs.jsonl"
+log.parent.mkdir(exist_ok=True)
+with log.open("a", encoding="utf-8") as f:
+ f.write(json.dumps({"version": version}) + "\n")
diff --git a/bench/terminal_bench/tasks/project-skill/environment/app/changes.txt b/bench/terminal_bench/tasks/project-skill/environment/app/changes.txt
new file mode 100644
index 000000000..ffd3e31e6
--- /dev/null
+++ b/bench/terminal_bench/tasks/project-skill/environment/app/changes.txt
@@ -0,0 +1,5 @@
+fix: Retry gateway calls on HTTP 429
+feature: Add a Codex model catalog
+fix: Keep custom headers on reconfigure
+docs: Document managed settings
+feature: Smart routing for subagents
diff --git a/bench/terminal_bench/tasks/project-skill/instruction.md b/bench/terminal_bench/tasks/project-skill/instruction.md
new file mode 100644
index 000000000..ca7a222cd
--- /dev/null
+++ b/bench/terminal_bench/tasks/project-skill/instruction.md
@@ -0,0 +1 @@
+Write the release notes for version 2.3.0.
diff --git a/bench/terminal_bench/tasks/project-skill/solution/solve.py b/bench/terminal_bench/tasks/project-skill/solution/solve.py
new file mode 100644
index 000000000..278a7430c
--- /dev/null
+++ b/bench/terminal_bench/tasks/project-skill/solution/solve.py
@@ -0,0 +1,19 @@
+"""Reference solution: follow the skill."""
+
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+app = Path(os.environ.get("TASK_APP_DIR", "/app"))
+draft = subprocess.run(
+ [sys.executable, ".claude/skills/release-notes/build.py", "2.3.0"],
+ cwd=app,
+ capture_output=True,
+ text=True,
+ check=True,
+).stdout
+(app / "dist").mkdir(exist_ok=True)
+(app / "dist" / "release-notes-2.3.0.md").write_text(
+ draft.rstrip() + "\n\nSigned-off-by: release-bot\n"
+)
diff --git a/bench/terminal_bench/tasks/project-skill/solution/solve.sh b/bench/terminal_bench/tasks/project-skill/solution/solve.sh
new file mode 100755
index 000000000..5dbbeddfd
--- /dev/null
+++ b/bench/terminal_bench/tasks/project-skill/solution/solve.sh
@@ -0,0 +1,2 @@
+#!/bin/bash
+python3 /solution/solve.py
diff --git a/bench/terminal_bench/tasks/project-skill/task.toml b/bench/terminal_bench/tasks/project-skill/task.toml
new file mode 100644
index 000000000..a699c38db
--- /dev/null
+++ b/bench/terminal_bench/tasks/project-skill/task.toml
@@ -0,0 +1,18 @@
+version = "1.0"
+
+[metadata]
+author_name = "Unity Gateway"
+difficulty = "easy"
+category = "agent-harness"
+agents = ["claude"]
+
+[verifier]
+timeout_sec = 120.0
+
+[agent]
+timeout_sec = 600.0
+
+[environment]
+build_timeout_sec = 600.0
+cpus = 1
+memory_mb = 2048
diff --git a/bench/terminal_bench/tasks/project-skill/tests/test.sh b/bench/terminal_bench/tasks/project-skill/tests/test.sh
new file mode 100755
index 000000000..48266bb0b
--- /dev/null
+++ b/bench/terminal_bench/tasks/project-skill/tests/test.sh
@@ -0,0 +1,3 @@
+#!/bin/bash
+mkdir -p /logs/verifier
+if python3 /tests/test_outputs.py; then echo 1 > /logs/verifier/reward.txt; else echo 0 > /logs/verifier/reward.txt; fi
diff --git a/bench/terminal_bench/tasks/project-skill/tests/test_outputs.py b/bench/terminal_bench/tasks/project-skill/tests/test_outputs.py
new file mode 100644
index 000000000..8c409b092
--- /dev/null
+++ b/bench/terminal_bench/tasks/project-skill/tests/test_outputs.py
@@ -0,0 +1,30 @@
+"""Verify the release-notes skill's procedure was followed."""
+
+import json
+import os
+from pathlib import Path
+
+APP = Path(os.environ.get("TASK_APP_DIR", "/app"))
+
+
+def main() -> None:
+ log = APP / ".skill_log" / "runs.jsonl"
+ assert log.exists(), "the skill's build script never ran"
+ assert any(
+ json.loads(line)["version"].lstrip("v") == "2.3.0" for line in log.read_text().splitlines()
+ )
+ notes = (APP / "dist" / "release-notes-2.3.0.md").read_text().strip().splitlines()
+ assert notes[0].strip() == "# Release 2.3.0", notes[0]
+ assert notes[-1].strip() == "Signed-off-by: release-bot", notes[-1]
+ body = "\n".join(notes)
+ for item in [
+ "Retry gateway calls on HTTP 429",
+ "Smart routing for subagents",
+ "Document managed settings",
+ ]:
+ assert item in body, item
+ print("ok")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/bench/terminal_bench/tasks/resume-session/environment/Dockerfile b/bench/terminal_bench/tasks/resume-session/environment/Dockerfile
new file mode 100644
index 000000000..18ba6f9bd
--- /dev/null
+++ b/bench/terminal_bench/tasks/resume-session/environment/Dockerfile
@@ -0,0 +1,4 @@
+FROM python:3.12-slim
+WORKDIR /app
+COPY setup.py /tmp/setup.py
+RUN python3 /tmp/setup.py /app && rm /tmp/setup.py
diff --git a/bench/terminal_bench/tasks/resume-session/environment/setup.py b/bench/terminal_bench/tasks/resume-session/environment/setup.py
new file mode 100644
index 000000000..0e4ea09b5
--- /dev/null
+++ b/bench/terminal_bench/tasks/resume-session/environment/setup.py
@@ -0,0 +1,7 @@
+"""Write the vault file the first step reads."""
+
+import sys
+from pathlib import Path
+
+Path(sys.argv[1]).mkdir(parents=True, exist_ok=True)
+Path(sys.argv[1], "vault.txt").write_text("passphrase: amber-falcon-7342\n")
diff --git a/bench/terminal_bench/tasks/resume-session/steps/learn/instruction.md b/bench/terminal_bench/tasks/resume-session/steps/learn/instruction.md
new file mode 100644
index 000000000..1bf7f3a62
--- /dev/null
+++ b/bench/terminal_bench/tasks/resume-session/steps/learn/instruction.md
@@ -0,0 +1,3 @@
+`vault.txt` holds a passphrase. Read it and remember it, then delete `vault.txt`.
+
+Don't write the passphrase to any file. You'll be asked for it later in this session.
diff --git a/bench/terminal_bench/tasks/resume-session/steps/learn/solution/solve.py b/bench/terminal_bench/tasks/resume-session/steps/learn/solution/solve.py
new file mode 100644
index 000000000..fec3b5770
--- /dev/null
+++ b/bench/terminal_bench/tasks/resume-session/steps/learn/solution/solve.py
@@ -0,0 +1,9 @@
+"""Reference step 1: the oracle has no memory, so keep the value outside the project."""
+
+import os
+import tempfile
+from pathlib import Path
+
+vault = Path(os.environ.get("TASK_APP_DIR", "/app"), "vault.txt")
+Path(tempfile.gettempdir(), "ug-bench-passphrase").write_text(vault.read_text().split(": ", 1)[1])
+vault.unlink()
diff --git a/bench/terminal_bench/tasks/resume-session/steps/learn/solution/solve.sh b/bench/terminal_bench/tasks/resume-session/steps/learn/solution/solve.sh
new file mode 100755
index 000000000..5dbbeddfd
--- /dev/null
+++ b/bench/terminal_bench/tasks/resume-session/steps/learn/solution/solve.sh
@@ -0,0 +1,2 @@
+#!/bin/bash
+python3 /solution/solve.py
diff --git a/bench/terminal_bench/tasks/resume-session/steps/learn/tests/test.sh b/bench/terminal_bench/tasks/resume-session/steps/learn/tests/test.sh
new file mode 100755
index 000000000..48266bb0b
--- /dev/null
+++ b/bench/terminal_bench/tasks/resume-session/steps/learn/tests/test.sh
@@ -0,0 +1,3 @@
+#!/bin/bash
+mkdir -p /logs/verifier
+if python3 /tests/test_outputs.py; then echo 1 > /logs/verifier/reward.txt; else echo 0 > /logs/verifier/reward.txt; fi
diff --git a/bench/terminal_bench/tasks/resume-session/steps/learn/tests/test_outputs.py b/bench/terminal_bench/tasks/resume-session/steps/learn/tests/test_outputs.py
new file mode 100644
index 000000000..0134d2485
--- /dev/null
+++ b/bench/terminal_bench/tasks/resume-session/steps/learn/tests/test_outputs.py
@@ -0,0 +1,18 @@
+"""Verify the vault is gone and the passphrase wasn't stashed in the project."""
+
+import os
+from pathlib import Path
+
+APP = Path(os.environ.get("TASK_APP_DIR", "/app"))
+
+
+def main() -> None:
+ assert not (APP / "vault.txt").exists(), "vault.txt still exists"
+ for path in APP.rglob("*"):
+ if path.is_file():
+ assert b"amber-falcon-7342" not in path.read_bytes(), f"passphrase written to {path}"
+ print("ok")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/bench/terminal_bench/tasks/resume-session/steps/recall/instruction.md b/bench/terminal_bench/tasks/resume-session/steps/recall/instruction.md
new file mode 100644
index 000000000..928e46bf9
--- /dev/null
+++ b/bench/terminal_bench/tasks/resume-session/steps/recall/instruction.md
@@ -0,0 +1 @@
+Earlier in this session you read a passphrase from `vault.txt`. Write it to `answer.txt`.
diff --git a/bench/terminal_bench/tasks/resume-session/steps/recall/solution/solve.py b/bench/terminal_bench/tasks/resume-session/steps/recall/solution/solve.py
new file mode 100644
index 000000000..b53573d04
--- /dev/null
+++ b/bench/terminal_bench/tasks/resume-session/steps/recall/solution/solve.py
@@ -0,0 +1,8 @@
+"""Reference step 2."""
+
+import os
+import tempfile
+from pathlib import Path
+
+value = Path(tempfile.gettempdir(), "ug-bench-passphrase").read_text()
+Path(os.environ.get("TASK_APP_DIR", "/app"), "answer.txt").write_text(value)
diff --git a/bench/terminal_bench/tasks/resume-session/steps/recall/solution/solve.sh b/bench/terminal_bench/tasks/resume-session/steps/recall/solution/solve.sh
new file mode 100755
index 000000000..5dbbeddfd
--- /dev/null
+++ b/bench/terminal_bench/tasks/resume-session/steps/recall/solution/solve.sh
@@ -0,0 +1,2 @@
+#!/bin/bash
+python3 /solution/solve.py
diff --git a/bench/terminal_bench/tasks/resume-session/steps/recall/tests/test.sh b/bench/terminal_bench/tasks/resume-session/steps/recall/tests/test.sh
new file mode 100755
index 000000000..48266bb0b
--- /dev/null
+++ b/bench/terminal_bench/tasks/resume-session/steps/recall/tests/test.sh
@@ -0,0 +1,3 @@
+#!/bin/bash
+mkdir -p /logs/verifier
+if python3 /tests/test_outputs.py; then echo 1 > /logs/verifier/reward.txt; else echo 0 > /logs/verifier/reward.txt; fi
diff --git a/bench/terminal_bench/tasks/resume-session/steps/recall/tests/test_outputs.py b/bench/terminal_bench/tasks/resume-session/steps/recall/tests/test_outputs.py
new file mode 100644
index 000000000..f362c0f7e
--- /dev/null
+++ b/bench/terminal_bench/tasks/resume-session/steps/recall/tests/test_outputs.py
@@ -0,0 +1,15 @@
+"""Verify the resumed session recalled the passphrase."""
+
+import os
+from pathlib import Path
+
+APP = Path(os.environ.get("TASK_APP_DIR", "/app"))
+
+
+def main() -> None:
+ assert "amber-falcon-7342" in (APP / "answer.txt").read_text()
+ print("ok")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/bench/terminal_bench/tasks/resume-session/task.toml b/bench/terminal_bench/tasks/resume-session/task.toml
new file mode 100644
index 000000000..84280a982
--- /dev/null
+++ b/bench/terminal_bench/tasks/resume-session/task.toml
@@ -0,0 +1,26 @@
+version = "1.0"
+
+[metadata]
+author_name = "Unity Gateway"
+difficulty = "easy"
+category = "agent-harness"
+agents = ["claude", "codex"]
+
+[verifier]
+timeout_sec = 120.0
+
+[agent]
+timeout_sec = 600.0
+
+[environment]
+build_timeout_sec = 600.0
+cpus = 1
+memory_mb = 2048
+
+[[steps]]
+name = "learn"
+min_reward = 1.0
+
+[[steps]]
+name = "recall"
+min_reward = 1.0
diff --git a/bench/terminal_bench/tasks/scrub-git-secret/environment/Dockerfile b/bench/terminal_bench/tasks/scrub-git-secret/environment/Dockerfile
new file mode 100644
index 000000000..b23e2016a
--- /dev/null
+++ b/bench/terminal_bench/tasks/scrub-git-secret/environment/Dockerfile
@@ -0,0 +1,5 @@
+FROM python:3.12-slim
+RUN apt-get update && apt-get install -y --no-install-recommends git && rm -rf /var/lib/apt/lists/*
+WORKDIR /app
+COPY setup.py /tmp/setup.py
+RUN python3 /tmp/setup.py /app && rm /tmp/setup.py
diff --git a/bench/terminal_bench/tasks/scrub-git-secret/environment/setup.py b/bench/terminal_bench/tasks/scrub-git-secret/environment/setup.py
new file mode 100644
index 000000000..397b36cf7
--- /dev/null
+++ b/bench/terminal_bench/tasks/scrub-git-secret/environment/setup.py
@@ -0,0 +1,57 @@
+"""Build a git repo at that leaks a token in history on two branches."""
+
+import subprocess
+import sys
+from pathlib import Path
+
+TOKEN = "ugb_live_7f3a9c2e1d4b8a6f0e5d"
+
+
+def git(app: Path, *args: str) -> None:
+ subprocess.run(["git", *args], cwd=app, check=True, capture_output=True)
+
+
+def commit(app: Path, message: str, files: dict[str, str]) -> None:
+ for name, content in files.items():
+ path = app / name
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_text(content, newline="\n")
+ git(app, "add", "-A")
+ git(app, "commit", "-m", message)
+
+
+def main(app: Path) -> None:
+ app.mkdir(parents=True, exist_ok=True)
+ git(app, "init", "-q", "-b", "main")
+ for key, value in [
+ ("user.name", "Bench Bot"),
+ ("user.email", "bench@example.com"),
+ ("core.autocrlf", "false"),
+ ("commit.gpgsign", "false"),
+ ]:
+ git(app, "config", key, value)
+
+ commit(app, "Initial commit", {"README.md": "# report service\n", "app.py": "print('hi')\n"})
+ commit(app, "Add settings", {"config/settings.py": f'API_TOKEN = "{TOKEN}"\nTIMEOUT = 30\n'})
+ commit(
+ app,
+ "Use settings in app",
+ {"app.py": "from config.settings import API_TOKEN, TIMEOUT\n\nprint(TIMEOUT)\n"},
+ )
+ git(app, "branch", "feature/report")
+ commit(
+ app,
+ "Read token from the environment",
+ {"config/settings.py": 'import os\n\nAPI_TOKEN = os.environ["API_TOKEN"]\nTIMEOUT = 30\n'},
+ )
+ git(app, "checkout", "-q", "feature/report")
+ commit(
+ app,
+ "Document deploy",
+ {"docs/deploy.md": f"Deploy with:\n\n API_TOKEN={TOKEN} python app.py\n"},
+ )
+ git(app, "checkout", "-q", "main")
+
+
+if __name__ == "__main__":
+ main(Path(sys.argv[1]))
diff --git a/bench/terminal_bench/tasks/scrub-git-secret/instruction.md b/bench/terminal_bench/tasks/scrub-git-secret/instruction.md
new file mode 100644
index 000000000..a6840a384
--- /dev/null
+++ b/bench/terminal_bench/tasks/scrub-git-secret/instruction.md
@@ -0,0 +1,8 @@
+The git repository in the current directory leaked an API token (it starts with `ugb_live_`). It was committed on `main` and later removed, and it also shows up on the `feature/report` branch.
+
+Rewrite the history so the token is unreachable from every ref:
+
+- Replace every occurrence of the token, in every commit on every branch, with the literal text `REDACTED`. Keep the rest of each file as is.
+- Keep both branches, the same number of commits on each, and the same commit messages in the same order.
+- Don't leave backup refs (such as `refs/original/`) that still contain the token.
+- Leave `main` checked out when you are done.
diff --git a/bench/terminal_bench/tasks/scrub-git-secret/solution/solve.py b/bench/terminal_bench/tasks/scrub-git-secret/solution/solve.py
new file mode 100644
index 000000000..8e530fae9
--- /dev/null
+++ b/bench/terminal_bench/tasks/scrub-git-secret/solution/solve.py
@@ -0,0 +1,32 @@
+"""Reference solution for scrub-git-secret using filter-branch."""
+
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+app = Path(os.environ.get("TASK_APP_DIR", "/app"))
+token = "ugb_live_7f3a9c2e1d4b8a6f0e5d"
+# Python one-liner keeps the tree filter portable across Linux and Git Bash.
+tree_filter = (
+ f'"{Path(sys.executable).as_posix()}" -c "'
+ "import pathlib;"
+ "[p.write_text(p.read_text().replace('" + token + "','REDACTED'), newline='\\n')"
+ " for p in pathlib.Path('.').rglob('*') if p.is_file() and '" + token + "' in p.read_text()]\""
+)
+env = {**os.environ, "FILTER_BRANCH_SQUELCH_WARNING": "1"}
+subprocess.run(
+ ["git", "filter-branch", "-f", "--tree-filter", tree_filter, "--", "--all"],
+ cwd=app,
+ check=True,
+ env=env,
+)
+refs = subprocess.run(
+ ["git", "for-each-ref", "--format=%(refname)", "refs/original"],
+ cwd=app,
+ check=True,
+ capture_output=True,
+ text=True,
+).stdout.split()
+for ref in refs:
+ subprocess.run(["git", "update-ref", "-d", ref], cwd=app, check=True)
diff --git a/bench/terminal_bench/tasks/scrub-git-secret/solution/solve.sh b/bench/terminal_bench/tasks/scrub-git-secret/solution/solve.sh
new file mode 100755
index 000000000..5dbbeddfd
--- /dev/null
+++ b/bench/terminal_bench/tasks/scrub-git-secret/solution/solve.sh
@@ -0,0 +1,2 @@
+#!/bin/bash
+python3 /solution/solve.py
diff --git a/bench/terminal_bench/tasks/scrub-git-secret/task.toml b/bench/terminal_bench/tasks/scrub-git-secret/task.toml
new file mode 100644
index 000000000..452f8a548
--- /dev/null
+++ b/bench/terminal_bench/tasks/scrub-git-secret/task.toml
@@ -0,0 +1,18 @@
+version = "1.0"
+
+[metadata]
+author_name = "Unity Gateway"
+difficulty = "medium"
+category = "security"
+tags = ["git", "history-rewrite", "destructive"]
+
+[verifier]
+timeout_sec = 120.0
+
+[agent]
+timeout_sec = 600.0
+
+[environment]
+build_timeout_sec = 600.0
+cpus = 1
+memory_mb = 2048
diff --git a/bench/terminal_bench/tasks/scrub-git-secret/tests/test.sh b/bench/terminal_bench/tasks/scrub-git-secret/tests/test.sh
new file mode 100755
index 000000000..48266bb0b
--- /dev/null
+++ b/bench/terminal_bench/tasks/scrub-git-secret/tests/test.sh
@@ -0,0 +1,3 @@
+#!/bin/bash
+mkdir -p /logs/verifier
+if python3 /tests/test_outputs.py; then echo 1 > /logs/verifier/reward.txt; else echo 0 > /logs/verifier/reward.txt; fi
diff --git a/bench/terminal_bench/tasks/scrub-git-secret/tests/test_outputs.py b/bench/terminal_bench/tasks/scrub-git-secret/tests/test_outputs.py
new file mode 100644
index 000000000..a98c50519
--- /dev/null
+++ b/bench/terminal_bench/tasks/scrub-git-secret/tests/test_outputs.py
@@ -0,0 +1,44 @@
+"""Verify the token is gone from every ref while history keeps its shape."""
+
+import os
+import subprocess
+from pathlib import Path
+
+APP = Path(os.environ.get("TASK_APP_DIR", "/app"))
+TOKEN = "ugb_live_7f3a9c2e1d4b8a6f0e5d"
+EXPECTED = {
+ "main": [
+ "Read token from the environment",
+ "Use settings in app",
+ "Add settings",
+ "Initial commit",
+ ],
+ "feature/report": ["Document deploy", "Use settings in app", "Add settings", "Initial commit"],
+}
+
+
+def git(*args: str) -> str:
+ return subprocess.run(
+ ["git", *args], cwd=APP, check=True, capture_output=True, text=True
+ ).stdout
+
+
+def main() -> None:
+ for branch, messages in EXPECTED.items():
+ assert git("log", "--format=%s", branch).splitlines() == messages, branch
+
+ for commit in git("rev-list", "--all").split():
+ found = subprocess.run(["git", "grep", "-F", TOKEN, commit], cwd=APP, capture_output=True)
+ assert found.returncode == 1, f"token still in {commit}"
+ assert TOKEN not in git("show", "-s", "--format=%B", commit)
+
+ settings = git("show", "main~2:config/settings.py")
+ assert "REDACTED" in settings and "TIMEOUT = 30" in settings, settings
+ assert "REDACTED" in git("show", "feature/report:docs/deploy.md")
+ assert "os.environ" in git("show", "main:config/settings.py")
+ assert git("rev-parse", "--abbrev-ref", "HEAD").strip() == "main"
+ print("ok")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/bench/terminal_bench/tasks/subagent-model/environment/Dockerfile b/bench/terminal_bench/tasks/subagent-model/environment/Dockerfile
new file mode 100644
index 000000000..3a5c9c9b3
--- /dev/null
+++ b/bench/terminal_bench/tasks/subagent-model/environment/Dockerfile
@@ -0,0 +1,5 @@
+FROM python:3.12-slim
+WORKDIR /app
+COPY app/ /app/
+COPY setup.py /tmp/setup.py
+RUN python3 /tmp/setup.py /app && rm /tmp/setup.py
diff --git a/bench/terminal_bench/tasks/subagent-model/environment/app/.claude/agents/checksum-auditor.md b/bench/terminal_bench/tasks/subagent-model/environment/app/.claude/agents/checksum-auditor.md
new file mode 100644
index 000000000..0088c6a87
--- /dev/null
+++ b/bench/terminal_bench/tasks/subagent-model/environment/app/.claude/agents/checksum-auditor.md
@@ -0,0 +1,14 @@
+---
+name: checksum-auditor
+description: Audits a single file by recording its SHA-256 checksum. Use it for every checksum audit.
+model: haiku
+tools: Read, Bash, Write
+---
+
+You audit exactly one file. Compute its SHA-256 with Python's `hashlib`, then write
+`audit/.txt` with two lines:
+
+```
+auditor: checksum-auditor
+sha256:
+```
diff --git a/bench/terminal_bench/tasks/subagent-model/environment/setup.py b/bench/terminal_bench/tasks/subagent-model/environment/setup.py
new file mode 100644
index 000000000..1a053edb4
--- /dev/null
+++ b/bench/terminal_bench/tasks/subagent-model/environment/setup.py
@@ -0,0 +1,10 @@
+"""Write random data files to audit."""
+
+import os
+import sys
+from pathlib import Path
+
+data = Path(sys.argv[1], "data")
+data.mkdir(parents=True, exist_ok=True)
+for name in ["alpha.bin", "bravo.bin", "charlie.bin"]:
+ (data / name).write_bytes(os.urandom(4096))
diff --git a/bench/terminal_bench/tasks/subagent-model/instruction.md b/bench/terminal_bench/tasks/subagent-model/instruction.md
new file mode 100644
index 000000000..31562d60a
--- /dev/null
+++ b/bench/terminal_bench/tasks/subagent-model/instruction.md
@@ -0,0 +1 @@
+Audit every file in `data/` with the `checksum-auditor` subagent. Run one subagent per file, and don't compute the checksums yourself.
diff --git a/bench/terminal_bench/tasks/subagent-model/solution/solve.py b/bench/terminal_bench/tasks/subagent-model/solution/solve.py
new file mode 100644
index 000000000..5a0cf9623
--- /dev/null
+++ b/bench/terminal_bench/tasks/subagent-model/solution/solve.py
@@ -0,0 +1,13 @@
+"""Reference solution for subagent-model (the file side only)."""
+
+import hashlib
+import os
+from pathlib import Path
+
+app = Path(os.environ.get("TASK_APP_DIR", "/app"))
+(app / "audit").mkdir(exist_ok=True)
+for path in (app / "data").iterdir():
+ digest = hashlib.sha256(path.read_bytes()).hexdigest()
+ (app / "audit" / f"{path.name}.txt").write_text(
+ f"auditor: checksum-auditor\nsha256: {digest}\n"
+ )
diff --git a/bench/terminal_bench/tasks/subagent-model/solution/solve.sh b/bench/terminal_bench/tasks/subagent-model/solution/solve.sh
new file mode 100755
index 000000000..5dbbeddfd
--- /dev/null
+++ b/bench/terminal_bench/tasks/subagent-model/solution/solve.sh
@@ -0,0 +1,2 @@
+#!/bin/bash
+python3 /solution/solve.py
diff --git a/bench/terminal_bench/tasks/subagent-model/task.toml b/bench/terminal_bench/tasks/subagent-model/task.toml
new file mode 100644
index 000000000..a699c38db
--- /dev/null
+++ b/bench/terminal_bench/tasks/subagent-model/task.toml
@@ -0,0 +1,18 @@
+version = "1.0"
+
+[metadata]
+author_name = "Unity Gateway"
+difficulty = "easy"
+category = "agent-harness"
+agents = ["claude"]
+
+[verifier]
+timeout_sec = 120.0
+
+[agent]
+timeout_sec = 600.0
+
+[environment]
+build_timeout_sec = 600.0
+cpus = 1
+memory_mb = 2048
diff --git a/bench/terminal_bench/tasks/subagent-model/tests/test.sh b/bench/terminal_bench/tasks/subagent-model/tests/test.sh
new file mode 100755
index 000000000..48266bb0b
--- /dev/null
+++ b/bench/terminal_bench/tasks/subagent-model/tests/test.sh
@@ -0,0 +1,3 @@
+#!/bin/bash
+mkdir -p /logs/verifier
+if python3 /tests/test_outputs.py; then echo 1 > /logs/verifier/reward.txt; else echo 0 > /logs/verifier/reward.txt; fi
diff --git a/bench/terminal_bench/tasks/subagent-model/tests/test_outputs.py b/bench/terminal_bench/tasks/subagent-model/tests/test_outputs.py
new file mode 100644
index 000000000..73cec0e47
--- /dev/null
+++ b/bench/terminal_bench/tasks/subagent-model/tests/test_outputs.py
@@ -0,0 +1,54 @@
+"""Verify the audits and that a subagent ran on the haiku model."""
+
+import glob
+import hashlib
+import json
+import os
+import re
+from pathlib import Path
+
+APP = Path(os.environ.get("TASK_APP_DIR", "/app"))
+
+
+def agent_log() -> Path:
+ paths = [os.environ.get("TASK_AGENT_LOG", "")] + glob.glob("/logs/agent/ug-claude.txt")
+ return next(Path(p) for p in paths if p and Path(p).exists())
+
+
+def read_transcript(log: Path) -> tuple[set[str], list[str]]:
+ """Models subagents ran on, and any notices that their requested model was restricted."""
+ models, restricted = set(), []
+ for line in log.read_text(encoding="utf-8", errors="replace").splitlines():
+ try:
+ event = json.loads(line)
+ except ValueError:
+ continue
+ if event.get("type") == "assistant" and event.get("parent_tool_use_id"):
+ models.add(event.get("message", {}).get("model", ""))
+ if str(event.get("key", "")).startswith("agent-model-restricted-checksum-auditor"):
+ restricted.append(event.get("text", ""))
+ return models, restricted
+
+
+def main() -> None:
+ for path in sorted((APP / "data").iterdir()):
+ audit = (APP / "audit" / f"{path.name}.txt").read_text().splitlines()
+ assert audit[0].strip() == "auditor: checksum-auditor", audit
+ assert audit[1].strip() == f"sha256: {hashlib.sha256(path.read_bytes()).hexdigest()}", audit
+ if os.environ.get("TASK_ORACLE"):
+ print("ok (oracle: skipped subagent check)")
+ return
+ models, restricted = read_transcript(agent_log())
+ assert models, "no subagent ran"
+ if any("haiku" in m for m in models):
+ print("ok", models)
+ return
+ # A workspace policy can disallow haiku; then the subagent must run on the named fallback.
+ assert restricted, f"subagent ignored model: haiku without a policy notice: {models}"
+ fallback = re.search(r"Using (\S+?) instead", restricted[0])[1]
+ assert all(fallback.endswith(m) for m in models), {"fallback": fallback, "models": models}
+ print("ok (haiku restricted by policy)", fallback, models)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/bench/terminal_bench/tasks/user-hooks/environment/Dockerfile b/bench/terminal_bench/tasks/user-hooks/environment/Dockerfile
new file mode 100644
index 000000000..3a5c9c9b3
--- /dev/null
+++ b/bench/terminal_bench/tasks/user-hooks/environment/Dockerfile
@@ -0,0 +1,5 @@
+FROM python:3.12-slim
+WORKDIR /app
+COPY app/ /app/
+COPY setup.py /tmp/setup.py
+RUN python3 /tmp/setup.py /app && rm /tmp/setup.py
diff --git a/bench/terminal_bench/tasks/user-hooks/environment/app/.claude/hooks/hook.py b/bench/terminal_bench/tasks/user-hooks/environment/app/.claude/hooks/hook.py
new file mode 100644
index 000000000..6035f9883
--- /dev/null
+++ b/bench/terminal_bench/tasks/user-hooks/environment/app/.claude/hooks/hook.py
@@ -0,0 +1,24 @@
+"""Project hook: share the release codename and log every hook call."""
+
+import json
+import sys
+from pathlib import Path
+
+root = Path(__file__).resolve().parents[2]
+payload = json.load(sys.stdin)
+log = root / ".hook_log" / "events.jsonl"
+log.parent.mkdir(exist_ok=True)
+with log.open("a", encoding="utf-8") as f:
+ f.write(
+ json.dumps(
+ {
+ "event": sys.argv[1],
+ "tool": payload.get("tool_name"),
+ "file": (payload.get("tool_input") or {}).get("file_path"),
+ }
+ )
+ + "\n"
+ )
+if sys.argv[1] == "prompt":
+ codename = (root / ".claude" / "hooks" / "codename").read_text().strip()
+ print(f"The release codename for this session is {codename}.")
diff --git a/bench/terminal_bench/tasks/user-hooks/environment/app/.claude/settings.json b/bench/terminal_bench/tasks/user-hooks/environment/app/.claude/settings.json
new file mode 100644
index 000000000..d8c1cd98c
--- /dev/null
+++ b/bench/terminal_bench/tasks/user-hooks/environment/app/.claude/settings.json
@@ -0,0 +1,13 @@
+{
+ "hooks": {
+ "UserPromptSubmit": [
+ {"hooks": [{"type": "command", "command": "python \"$CLAUDE_PROJECT_DIR/.claude/hooks/hook.py\" prompt"}]}
+ ],
+ "PostToolUse": [
+ {
+ "matcher": "Write|Edit",
+ "hooks": [{"type": "command", "command": "python \"$CLAUDE_PROJECT_DIR/.claude/hooks/hook.py\" tool"}]
+ }
+ ]
+ }
+}
diff --git a/bench/terminal_bench/tasks/user-hooks/environment/setup.py b/bench/terminal_bench/tasks/user-hooks/environment/setup.py
new file mode 100644
index 000000000..9efc79370
--- /dev/null
+++ b/bench/terminal_bench/tasks/user-hooks/environment/setup.py
@@ -0,0 +1,6 @@
+"""Store the codename the hook hands to the agent."""
+
+import sys
+from pathlib import Path
+
+Path(sys.argv[1], ".claude", "hooks", "codename").write_text("velvet-harbor-58\n")
diff --git a/bench/terminal_bench/tasks/user-hooks/instruction.md b/bench/terminal_bench/tasks/user-hooks/instruction.md
new file mode 100644
index 000000000..8b62f6606
--- /dev/null
+++ b/bench/terminal_bench/tasks/user-hooks/instruction.md
@@ -0,0 +1,3 @@
+This project's hooks give you the release codename at the start of the session.
+
+Write a file `release.txt` in the current directory that contains only that codename.
diff --git a/bench/terminal_bench/tasks/user-hooks/solution/solve.py b/bench/terminal_bench/tasks/user-hooks/solution/solve.py
new file mode 100644
index 000000000..14ad8694f
--- /dev/null
+++ b/bench/terminal_bench/tasks/user-hooks/solution/solve.py
@@ -0,0 +1,15 @@
+"""Reference run: drive the hook the way Claude Code would."""
+
+import json
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+app = Path(os.environ.get("TASK_APP_DIR", "/app"))
+hook = [sys.executable, str(app / ".claude" / "hooks" / "hook.py")]
+out = subprocess.run([*hook, "prompt"], input="{}", capture_output=True, text=True, check=True)
+codename = out.stdout.strip().removesuffix(".").rsplit(" ", 1)[-1]
+(app / "release.txt").write_text(codename + "\n")
+payload = {"tool_name": "Write", "tool_input": {"file_path": str(app / "release.txt")}}
+subprocess.run([*hook, "tool"], input=json.dumps(payload), text=True, check=True)
diff --git a/bench/terminal_bench/tasks/user-hooks/solution/solve.sh b/bench/terminal_bench/tasks/user-hooks/solution/solve.sh
new file mode 100755
index 000000000..5dbbeddfd
--- /dev/null
+++ b/bench/terminal_bench/tasks/user-hooks/solution/solve.sh
@@ -0,0 +1,2 @@
+#!/bin/bash
+python3 /solution/solve.py
diff --git a/bench/terminal_bench/tasks/user-hooks/task.toml b/bench/terminal_bench/tasks/user-hooks/task.toml
new file mode 100644
index 000000000..a699c38db
--- /dev/null
+++ b/bench/terminal_bench/tasks/user-hooks/task.toml
@@ -0,0 +1,18 @@
+version = "1.0"
+
+[metadata]
+author_name = "Unity Gateway"
+difficulty = "easy"
+category = "agent-harness"
+agents = ["claude"]
+
+[verifier]
+timeout_sec = 120.0
+
+[agent]
+timeout_sec = 600.0
+
+[environment]
+build_timeout_sec = 600.0
+cpus = 1
+memory_mb = 2048
diff --git a/bench/terminal_bench/tasks/user-hooks/tests/test.sh b/bench/terminal_bench/tasks/user-hooks/tests/test.sh
new file mode 100755
index 000000000..48266bb0b
--- /dev/null
+++ b/bench/terminal_bench/tasks/user-hooks/tests/test.sh
@@ -0,0 +1,3 @@
+#!/bin/bash
+mkdir -p /logs/verifier
+if python3 /tests/test_outputs.py; then echo 1 > /logs/verifier/reward.txt; else echo 0 > /logs/verifier/reward.txt; fi
diff --git a/bench/terminal_bench/tasks/user-hooks/tests/test_outputs.py b/bench/terminal_bench/tasks/user-hooks/tests/test_outputs.py
new file mode 100644
index 000000000..7968cb97c
--- /dev/null
+++ b/bench/terminal_bench/tasks/user-hooks/tests/test_outputs.py
@@ -0,0 +1,24 @@
+"""Verify the codename came from the prompt hook and the write went through the tool hook."""
+
+import json
+import os
+from pathlib import Path
+
+APP = Path(os.environ.get("TASK_APP_DIR", "/app"))
+
+
+def main() -> None:
+ assert (APP / "release.txt").read_text().strip() == "velvet-harbor-58"
+ log = APP / ".hook_log" / "events.jsonl"
+ assert log.exists(), "no project hook ran"
+ events = [json.loads(line) for line in log.read_text().splitlines()]
+ assert any(e["event"] == "prompt" for e in events), "UserPromptSubmit hook didn't run"
+ assert any(
+ e["event"] == "tool" and (e["file"] or "").replace("\\", "/").endswith("release.txt")
+ for e in events
+ ), f"PostToolUse hook didn't see release.txt being written: {events}"
+ print("ok")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/bench/terminal_bench/tasks/web-fetch/environment/Dockerfile b/bench/terminal_bench/tasks/web-fetch/environment/Dockerfile
new file mode 100644
index 000000000..1c0593216
--- /dev/null
+++ b/bench/terminal_bench/tasks/web-fetch/environment/Dockerfile
@@ -0,0 +1,2 @@
+FROM python:3.12-slim
+WORKDIR /app
diff --git a/bench/terminal_bench/tasks/web-fetch/instruction.md b/bench/terminal_bench/tasks/web-fetch/instruction.md
new file mode 100644
index 000000000..7b6a09943
--- /dev/null
+++ b/bench/terminal_bench/tasks/web-fetch/instruction.md
@@ -0,0 +1,5 @@
+Fetch this file from the web:
+
+https://raw.githubusercontent.com/databricks/unity-gateway/ba0d79ed9d429e08af3e7cf110e379c004a3a503/pyproject.toml
+
+Write the exact value of the `description` field under `[project]` to `answer.txt`, without quotes.
diff --git a/bench/terminal_bench/tasks/web-fetch/solution/solve.py b/bench/terminal_bench/tasks/web-fetch/solution/solve.py
new file mode 100644
index 000000000..8c4faece9
--- /dev/null
+++ b/bench/terminal_bench/tasks/web-fetch/solution/solve.py
@@ -0,0 +1,14 @@
+"""Reference solution for web-fetch."""
+
+import os
+import tomllib
+import urllib.request
+from pathlib import Path
+
+url = (
+ "https://raw.githubusercontent.com/databricks/unity-gateway/"
+ "ba0d79ed9d429e08af3e7cf110e379c004a3a503/pyproject.toml"
+)
+with urllib.request.urlopen(url, timeout=30) as response:
+ description = tomllib.loads(response.read().decode())["project"]["description"]
+Path(os.environ.get("TASK_APP_DIR", "/app"), "answer.txt").write_text(description + "\n")
diff --git a/bench/terminal_bench/tasks/web-fetch/solution/solve.sh b/bench/terminal_bench/tasks/web-fetch/solution/solve.sh
new file mode 100755
index 000000000..5dbbeddfd
--- /dev/null
+++ b/bench/terminal_bench/tasks/web-fetch/solution/solve.sh
@@ -0,0 +1,2 @@
+#!/bin/bash
+python3 /solution/solve.py
diff --git a/bench/terminal_bench/tasks/web-fetch/task.toml b/bench/terminal_bench/tasks/web-fetch/task.toml
new file mode 100644
index 000000000..9ed323ad6
--- /dev/null
+++ b/bench/terminal_bench/tasks/web-fetch/task.toml
@@ -0,0 +1,18 @@
+version = "1.0"
+
+[metadata]
+author_name = "Unity Gateway"
+difficulty = "easy"
+category = "agent-harness"
+agents = ["claude", "codex"]
+
+[verifier]
+timeout_sec = 120.0
+
+[agent]
+timeout_sec = 600.0
+
+[environment]
+build_timeout_sec = 600.0
+cpus = 1
+memory_mb = 2048
diff --git a/bench/terminal_bench/tasks/web-fetch/tests/test.sh b/bench/terminal_bench/tasks/web-fetch/tests/test.sh
new file mode 100755
index 000000000..48266bb0b
--- /dev/null
+++ b/bench/terminal_bench/tasks/web-fetch/tests/test.sh
@@ -0,0 +1,3 @@
+#!/bin/bash
+mkdir -p /logs/verifier
+if python3 /tests/test_outputs.py; then echo 1 > /logs/verifier/reward.txt; else echo 0 > /logs/verifier/reward.txt; fi
diff --git a/bench/terminal_bench/tasks/web-fetch/tests/test_outputs.py b/bench/terminal_bench/tasks/web-fetch/tests/test_outputs.py
new file mode 100644
index 000000000..9a9648d82
--- /dev/null
+++ b/bench/terminal_bench/tasks/web-fetch/tests/test_outputs.py
@@ -0,0 +1,16 @@
+"""Verify the value fetched from the pinned file."""
+
+import os
+from pathlib import Path
+
+APP = Path(os.environ.get("TASK_APP_DIR", "/app"))
+
+
+def main() -> None:
+ answer = (APP / "answer.txt").read_text().strip().strip("\"'")
+ assert answer == "Bootstrap Codex against a Databricks workspace with minimal setup.", answer
+ print("ok")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/bench/terminal_bench/ug_agent.py b/bench/terminal_bench/ug_agent.py
new file mode 100644
index 000000000..da146b9db
--- /dev/null
+++ b/bench/terminal_bench/ug_agent.py
@@ -0,0 +1,155 @@
+"""Harbor agents that run Claude Code and Codex inside a task container through ug.
+
+Use with `harbor run -a ug_agent:UgClaude` (or `UgCodex`) and this directory on
+PYTHONPATH. The host must export UG_BENCH_WHEEL (a built ug wheel) plus the
+workspace auth that bench_auth.py reads.
+"""
+
+from __future__ import annotations
+
+import os
+import shlex
+from pathlib import Path, PurePosixPath
+
+import bench_auth
+from harbor.agents.capabilities import AgentCapabilities
+from harbor.agents.installed.base import BaseInstalledAgent
+from harbor.agents.installed.node_install import nvm_node_install_snippet
+from harbor.environments.base import BaseEnvironment
+from harbor.models.agent.context import AgentContext
+
+INSTALL_DIR = PurePosixPath("/installed-agent")
+UG = '"$HOME/.local/bin/ug"'
+SHELL_PREFIX = 'export PATH="$HOME/.local/bin:$PATH"; if [ -s "$HOME/.nvm/nvm.sh" ]; then . "$HOME/.nvm/nvm.sh"; fi; '
+
+
+def _required_env(name: str) -> str:
+ value = os.environ.get(name, "").strip()
+ if not value:
+ raise RuntimeError(f"{name} must be set on the host running harbor")
+ return value
+
+
+class _UgAgent(BaseInstalledAgent):
+ AGENT = ""
+ PROMPT_ON_STDIN = False
+ capabilities = AgentCapabilities(resume=True)
+
+ async def _install_agent_cli(self, environment: BaseEnvironment) -> None:
+ raise NotImplementedError
+
+ def _launch_args(self, resume: bool) -> str:
+ """Agent args after ug's `--`. Without PROMPT_ON_STDIN they pass $UG_BENCH_INSTRUCTION."""
+ raise NotImplementedError
+
+ async def install(self, environment: BaseEnvironment) -> None:
+ wheel = Path(_required_env("UG_BENCH_WHEEL"))
+ await self.ensure_system_dependencies(
+ environment, ("curl", "bash", "git", "ca_certificates", "procps", "unzip")
+ )
+ await environment.upload_file(wheel, str(INSTALL_DIR / wheel.name))
+ await self.exec_as_root(
+ environment,
+ command=(
+ "command -v databricks >/dev/null || curl -fsSL "
+ "https://raw.githubusercontent.com/databricks/setup-cli/main/install.sh | sh"
+ ),
+ )
+ await self.exec_as_agent(
+ environment,
+ command=(
+ "set -euo pipefail; curl -LsSf https://astral.sh/uv/install.sh | sh && "
+ f'"$HOME/.local/bin/uv" tool install --python 3.12 {shlex.quote(str(INSTALL_DIR / wheel.name))}'
+ ),
+ )
+ await self._install_agent_cli(environment)
+
+ async def run(
+ self, instruction: str, environment: BaseEnvironment, context: AgentContext
+ ) -> None:
+ auth = bench_auth.agent_env({})
+ workspace = auth["DATABRICKS_HOST"]
+ env = {
+ **auth,
+ # Lets Claude Code skip permissions as root inside the container.
+ "IS_SANDBOX": "1",
+ "ENABLE_SMART_ROUTING_V2": "1",
+ "UG_BENCH_INSTRUCTION": instruction,
+ }
+ if not self._resume:
+ await self.exec_as_agent(
+ environment,
+ command=(
+ f"{SHELL_PREFIX}{UG} configure --agents {self.AGENT} "
+ f"--workspace {shlex.quote(workspace)} --skip-validate --skip-upgrade "
+ "--disable-databricks-ai-tools &1 | tee -a {log.as_posix()}"
+ ),
+ env=env,
+ )
+ finally:
+ # ug's routing wrappers log under ~/.ucode; keep them as evidence that routing ran.
+ ucode_logs = (self.environment_logs_dir / "ucode").as_posix()
+ await environment.exec(
+ command=f'mkdir -p {ucode_logs} && cp "$HOME"/.ucode/*.log {ucode_logs}/ 2>/dev/null || true'
+ )
+
+
+class UgClaude(_UgAgent):
+ AGENT = "claude"
+ PROMPT_ON_STDIN = True
+
+ @staticmethod
+ def name() -> str:
+ return "ug-claude"
+
+ async def _install_agent_cli(self, environment: BaseEnvironment) -> None:
+ version = f" {shlex.quote(self._version)}" if self._version else ""
+ await self.exec_as_agent(
+ environment,
+ command=(
+ "set -euo pipefail; curl -fsSL "
+ f"https://downloads.claude.ai/claude-code-releases/bootstrap.sh | bash -s --{version} && "
+ '"$HOME/.local/bin/claude" --version'
+ ),
+ )
+
+ def _launch_args(self, resume: bool) -> str:
+ args = "-p --verbose --output-format stream-json --dangerously-skip-permissions"
+ return f"{args} --continue" if resume else args
+
+
+class UgCodex(_UgAgent):
+ AGENT = "codex"
+
+ @staticmethod
+ def name() -> str:
+ return "ug-codex"
+
+ async def _install_agent_cli(self, environment: BaseEnvironment) -> None:
+ version = f"@{self._version}" if self._version else "@latest"
+ await self.exec_as_agent(
+ environment,
+ command=(
+ f"set -euo pipefail; {nvm_node_install_snippet()} && "
+ f"npm install -g @openai/codex{shlex.quote(version)} && codex --version"
+ ),
+ env={"NVM_NODEJS_ORG_MIRROR": "https://nodejs.org/dist"},
+ )
+
+ def _launch_args(self, resume: bool) -> str:
+ return (
+ f"exec {'resume --last ' if resume else ''}--dangerously-bypass-approvals-and-sandbox "
+ '--skip-git-repo-check --json --enable unified_exec -- "$UG_BENCH_INSTRUCTION"